{ "apiVersion": "dashboard.grafana.app/v2", "kind": "Dashboard", "metadata": { "annotations": { "grafana.app/folder": "infra", "monitoring.materialize.cloud/min-mz-version": "v26.24.0", "monitoring.materialize.cloud/rec-mz-version": "v26.24.0", "monitoring.materialize.cloud/sql-metric-prefix": "mz_", "monitoring.materialize.cloud/target-export": "generic" }, "name": "mz-mon-infra-nodes" }, "spec": { "annotations": [ { "kind": "AnnotationQuery", "spec": { "builtIn": true, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "query": { "datasource": { "name": "-- Grafana --" }, "group": "grafana", "kind": "DataQuery", "spec": {}, "version": "v0" } } } ], "cursorSync": "Crosshair", "description": "Everything about one node a Materialize deployment runs on.\n\nWhat the machine is, how hard it is working, how much of it is already promised to pods, and what it and Kubernetes have said about it.", "editable": true, "elements": { "arp-entries": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (instance, device) (\n node_arp_entries{instance=~\"$nodeList\"}\n)\n", "legendFormat": "{{device}}", "qryType": 1, "query": "max by (instance, device) (\n node_arp_entries{instance=~\"$nodeList\"}\n)\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**ARP table entries per interface.**\n\nApproaching `gc_thresh3` makes the kernel start evicting neighbours, which presents as intermittent unreachability to pods on other nodes. Large flat clusters are how you get there.", "id": 1054, "links": [], "title": "ARP Entries", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "unit": "short" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "conntrack": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (instance) (node_nf_conntrack_entries{instance=~\"$nodeList\"})\n/\nmax by (instance) (node_nf_conntrack_entries_limit{instance=~\"$nodeList\"})\n", "legendFormat": "in use", "qryType": 1, "query": "max by (instance) (node_nf_conntrack_entries{instance=~\"$nodeList\"})\n/\nmax by (instance) (node_nf_conntrack_entries_limit{instance=~\"$nodeList\"})\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Netfilter connection-tracking table occupancy as a fraction of its limit.**\n\n**Nominal:** Below ~0.7.\n\n**Degraded:** At 1.0 the kernel drops new connections and logs `nf_conntrack: table full`. This is the classic failure behind a NAT gateway, and it presents as intermittent connection timeouts from a node that looks otherwise healthy.\n\nBoth sides are aggregated before dividing, so the ratio still resolves if the entries and the limit ever arrive under different `job` labels.", "id": 1053, "links": [], "title": "Conntrack Table", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "fixedColor": "#CCBB44", "mode": "shades" }, "max": 1, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "#CEFFFF", "value": null }, { "color": "#F94902", "value": 0.8 }, { "color": "#E40515", "value": 0.8500000000000001 }, { "color": "#E40515", "value": 0.9000000000000001 }, { "color": "#AB0003", "value": 0.9500000000000002 }, { "color": "#AB0003", "value": 1 } ] }, "unit": "percentunit" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "cpu-by-mode": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (instance) (\n avg by (instance, job) (\n rate(node_cpu_seconds_total{mode=\"system\", instance=~\"$nodeList\"}[$__rate_interval])\n )\n)\n", "legendFormat": "system", "qryType": 1, "query": "max by (instance) (\n avg by (instance, job) (\n rate(node_cpu_seconds_total{mode=\"system\", instance=~\"$nodeList\"}[$__rate_interval])\n )\n)\n" }, "version": "v0" }, "refId": "query-0" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (instance) (\n avg by (instance, job) (\n rate(node_cpu_seconds_total{mode=\"user\", instance=~\"$nodeList\"}[$__rate_interval])\n )\n)\n", "legendFormat": "user", "qryType": 1, "query": "max by (instance) (\n avg by (instance, job) (\n rate(node_cpu_seconds_total{mode=\"user\", instance=~\"$nodeList\"}[$__rate_interval])\n )\n)\n" }, "version": "v0" }, "refId": "query-1" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (instance) (\n avg by (instance, job) (\n rate(node_cpu_seconds_total{mode=\"iowait\", instance=~\"$nodeList\"}[$__rate_interval])\n )\n)\n", "legendFormat": "iowait", "qryType": 1, "query": "max by (instance) (\n avg by (instance, job) (\n rate(node_cpu_seconds_total{mode=\"iowait\", instance=~\"$nodeList\"}[$__rate_interval])\n )\n)\n" }, "version": "v0" }, "refId": "query-2" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (instance) (\n avg by (instance, job) (\n sum without (mode) (\n rate(node_cpu_seconds_total{mode=~\".*irq\", instance=~\"$nodeList\"}[$__rate_interval])\n )\n )\n)\n", "legendFormat": "irq", "qryType": 1, "query": "max by (instance) (\n avg by (instance, job) (\n sum without (mode) (\n rate(node_cpu_seconds_total{mode=~\".*irq\", instance=~\"$nodeList\"}[$__rate_interval])\n )\n )\n)\n" }, "version": "v0" }, "refId": "query-3" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (instance) (\n avg by (instance, job) (\n rate(node_cpu_seconds_total{mode=\"steal\", instance=~\"$nodeList\"}[$__rate_interval])\n )\n)\n", "legendFormat": "steal", "qryType": 1, "query": "max by (instance) (\n avg by (instance, job) (\n rate(node_cpu_seconds_total{mode=\"steal\", instance=~\"$nodeList\"}[$__rate_interval])\n )\n)\n" }, "version": "v0" }, "refId": "query-4" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**CPU time by mode — system, user, iowait, and the interrupt modes — averaged across cores.**\n\nThe first split to make when `node.cpu.utilization` is high. `iowait` points at storage rather than compute; a large `system` share on a network-heavy node often means softirq work, which `node.debug.softnet.*` quantifies.", "id": 1026, "links": [], "title": "CPU by Mode", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "unit": "percentunit" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "cpu-context-switches": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (instance) (\n rate(node_context_switches_total{instance=~\"$nodeList\"}[$__rate_interval])\n)\n", "legendFormat": "context switches", "qryType": 1, "query": "max by (instance) (\n rate(node_context_switches_total{instance=~\"$nodeList\"}[$__rate_interval])\n)\n" }, "version": "v0" }, "refId": "query-0" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (instance) (\n rate(node_intr_total{instance=~\"$nodeList\"}[$__rate_interval])\n)\n", "legendFormat": "interrupts", "qryType": 1, "query": "max by (instance) (\n rate(node_intr_total{instance=~\"$nodeList\"}[$__rate_interval])\n)\n" }, "version": "v0" }, "refId": "query-1" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Context switches and hardware interrupts per second.**\n\nA climb with no matching rise in throughput suggests thrashing between runnable threads or an interrupt storm.", "id": 1029, "links": [], "title": "Context Switches and Interrupts", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "unit": "cps" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "cpu-load": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (instance) (node_load1{instance=~\"$nodeList\"})\n/\nmax by (instance) (\n count by (instance, job) (\n count by (instance, job, cpu) (node_cpu_seconds_total{instance=~\"$nodeList\"})\n )\n)\n", "legendFormat": "load / cores", "qryType": 1, "query": "max by (instance) (node_load1{instance=~\"$nodeList\"})\n/\nmax by (instance) (\n count by (instance, job) (\n count by (instance, job, cpu) (node_cpu_seconds_total{instance=~\"$nodeList\"})\n )\n)\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**One-minute load average divided by the node's core count, so it is comparable across instance sizes.**\n\n**Nominal:** Below 1.0 — runnable work fits in the cores available.\n\n**Degraded:** Above 1.0 means more threads are runnable than there are cores to run them. On a database doing deliberate I/O this is not automatically bad; read it with `node.cpu.pressure`, which distinguishes \"busy\" from \"waiting\".", "id": 1024, "links": [], "title": "Load Average (normalized)", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "fixedColor": "#EE7733", "mode": "shades" }, "min": 0, "unit": "short" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "cpu-per-core": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "1 - max by (instance, cpu) (\n rate(node_cpu_seconds_total{mode=\"idle\", instance=~\"$nodeList\"}[$__rate_interval])\n)\n", "legendFormat": "core {{cpu}}", "qryType": 1, "query": "1 - max by (instance, cpu) (\n rate(node_cpu_seconds_total{mode=\"idle\", instance=~\"$nodeList\"}[$__rate_interval])\n)\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Non-idle CPU time per core, so a single saturated core is visible.**\n\n`node.cpu.utilization` averages across cores and therefore hides this. A workload pinned to one core, or a NIC whose interrupts all land on CPU 0, shows up here and nowhere else.", "id": 1027, "links": [], "title": "CPU per Core", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "max": 1, "min": 0, "unit": "percentunit" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "cpu-pressure": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (instance) (\n rate(node_pressure_cpu_waiting_seconds_total{instance=~\"$nodeList\"}[$__rate_interval])\n)\n", "legendFormat": "stalled", "qryType": 1, "query": "max by (instance) (\n rate(node_pressure_cpu_waiting_seconds_total{instance=~\"$nodeList\"}[$__rate_interval])\n)\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**PSI: the fraction of wall time at least one task was stalled waiting for CPU.**\n\n**Nominal:** Near zero.\n\n**Degraded:** Rising means runnable threads are queueing. For a latency-sensitive dataflow engine this is the signal that shows up as peek latency before it shows up as CPU utilization.\n\nNeeds Linux 4.20+. Where `/proc/pressure` is absent the collector reports `node_scrape_collector_success{collector=\"pressure\"} == 0` rather than returning a misleading zero.", "id": 1025, "links": [], "title": "CPU Pressure", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "fixedColor": "#EE7733", "mode": "shades" }, "max": 1, "min": 0, "unit": "percentunit" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "cpu-runqueue": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (instance, cpu) (\n rate(node_schedstat_waiting_seconds_total{instance=~\"$nodeList\"}[$__rate_interval])\n)\n", "legendFormat": "core {{cpu}}", "qryType": 1, "query": "max by (instance, cpu) (\n rate(node_schedstat_waiting_seconds_total{instance=~\"$nodeList\"}[$__rate_interval])\n)\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Time tasks spent runnable but not running, per core, from `/proc/schedstat`.**\n\nThe direct measure of scheduling contention — threads were ready and the CPU was not available. More actionable than load average for a latency-sensitive engine, because it is measured rather than smoothed.\n\nTwo kernel gates, and the second fails quietly: `CONFIG_SCHEDSTATS` must be built in, and since Linux 4.6 the counters are additionally off at runtime unless `kernel.sched_schedstats=1`. A missing file shows up in `node.collector.success`; the sysctl being off does not — the file just reads as zeros.", "id": 1028, "links": [], "title": "Run Queue Wait", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "unit": "s" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "cpu-utilization": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "1 - max by (instance) (\n avg by (instance, job) (\n rate(node_cpu_seconds_total{mode=\"idle\", instance=~\"$nodeList\"}[$__rate_interval])\n )\n)\n", "legendFormat": "busy", "qryType": 1, "query": "1 - max by (instance) (\n avg by (instance, job) (\n rate(node_cpu_seconds_total{mode=\"idle\", instance=~\"$nodeList\"}[$__rate_interval])\n )\n)\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Fraction of CPU time the node spent doing anything other than idling, averaged across its cores.**\n\n**Nominal:** Well below 1.0, with headroom for the bursty half of the workload.\n\n**Degraded:** Sustained above ~0.8 means the node has no burst capacity left; check `node.cpu.pressure` to see whether threads are actually waiting.\n\nThe average across cores, not the busiest core — a single saturated core on a large node is invisible here and shows up in `node.debug.cpu.by_mode` and `node.debug.schedstat.waiting`.", "id": 1023, "links": [], "title": "CPU Utilization", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "fixedColor": "#EE7733", "mode": "shades" }, "max": 1, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "#CEFFFF", "value": null }, { "color": "#F94902", "value": 0.8 }, { "color": "#E40515", "value": 0.8500000000000001 }, { "color": "#E40515", "value": 0.9000000000000001 }, { "color": "#AB0003", "value": 0.9500000000000002 }, { "color": "#AB0003", "value": 1 } ] }, "unit": "percentunit" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "disk-iops": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (instance, device) (\n rate(node_disk_reads_completed_total{instance=~\"$nodeList\"}[$__rate_interval])\n)\n", "legendFormat": "read {{device}}", "qryType": 1, "query": "max by (instance, device) (\n rate(node_disk_reads_completed_total{instance=~\"$nodeList\"}[$__rate_interval])\n)\n" }, "version": "v0" }, "refId": "query-0" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (instance, device) (\n rate(node_disk_writes_completed_total{instance=~\"$nodeList\"}[$__rate_interval])\n)\n", "legendFormat": "write {{device}}", "qryType": 1, "query": "max by (instance, device) (\n rate(node_disk_writes_completed_total{instance=~\"$nodeList\"}[$__rate_interval])\n)\n" }, "version": "v0" }, "refId": "query-1" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Completed reads and writes per second, per device.**\n\nPair with `node.debug.disk.throughput`: high IOPS at low throughput is a small-random-I/O pattern, which is what exhausts cloud volume IOPS budgets long before it exhausts bandwidth.", "id": 1058, "links": [], "title": "Disk IOPS", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "unit": "cps" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "disk-latency": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (instance, device) (\n (rate(node_disk_reads_completed_total{instance=~\"$nodeList\"}[$__rate_interval]) > bool 0)\n *\n (\n rate(node_disk_read_time_seconds_total{instance=~\"$nodeList\"}[$__rate_interval])\n / rate(node_disk_reads_completed_total{instance=~\"$nodeList\"}[$__rate_interval])\n )\n)\n", "legendFormat": "read {{device}}", "qryType": 1, "query": "max by (instance, device) (\n (rate(node_disk_reads_completed_total{instance=~\"$nodeList\"}[$__rate_interval]) > bool 0)\n *\n (\n rate(node_disk_read_time_seconds_total{instance=~\"$nodeList\"}[$__rate_interval])\n / rate(node_disk_reads_completed_total{instance=~\"$nodeList\"}[$__rate_interval])\n )\n)\n" }, "version": "v0" }, "refId": "query-0" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (instance, device) (\n (rate(node_disk_writes_completed_total{instance=~\"$nodeList\"}[$__rate_interval]) > bool 0)\n *\n (\n rate(node_disk_write_time_seconds_total{instance=~\"$nodeList\"}[$__rate_interval])\n / rate(node_disk_writes_completed_total{instance=~\"$nodeList\"}[$__rate_interval])\n )\n)\n", "legendFormat": "write {{device}}", "qryType": 1, "query": "max by (instance, device) (\n (rate(node_disk_writes_completed_total{instance=~\"$nodeList\"}[$__rate_interval]) > bool 0)\n *\n (\n rate(node_disk_write_time_seconds_total{instance=~\"$nodeList\"}[$__rate_interval])\n / rate(node_disk_writes_completed_total{instance=~\"$nodeList\"}[$__rate_interval])\n )\n)\n" }, "version": "v0" }, "refId": "query-1" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Average time per completed read and per completed write, per device.**\n\nTime divided by completions, guarded with `> bool 0` so an idle device reports 0 rather than dividing by zero and dropping out of the panel. This is device-level service time, not the latency any one process saw.", "id": 1060, "links": [], "title": "Disk Latency", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "unit": "s" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "disk-queue-depth": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (instance, device) (\n rate(node_disk_io_time_weighted_seconds_total{instance=~\"$nodeList\"}[$__rate_interval])\n)\n", "legendFormat": "{{device}}", "qryType": 1, "query": "max by (instance, device) (\n rate(node_disk_io_time_weighted_seconds_total{instance=~\"$nodeList\"}[$__rate_interval])\n)\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Average I/O queue depth per device.**\n\nDerived from the weighted I/O time counter. A queue that grows while `node.disk.io_utilization` sits at 1.0 confirms the device is the bottleneck rather than the workload having gone idle.", "id": 1062, "links": [], "title": "Disk Queue Depth", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "unit": "short" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "disk-throughput": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (instance, device) (\n rate(node_disk_read_bytes_total{instance=~\"$nodeList\"}[$__rate_interval])\n)\n", "legendFormat": "read {{device}}", "qryType": 1, "query": "max by (instance, device) (\n rate(node_disk_read_bytes_total{instance=~\"$nodeList\"}[$__rate_interval])\n)\n" }, "version": "v0" }, "refId": "query-0" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (instance, device) (\n rate(node_disk_written_bytes_total{instance=~\"$nodeList\"}[$__rate_interval])\n)\n", "legendFormat": "written {{device}}", "qryType": 1, "query": "max by (instance, device) (\n rate(node_disk_written_bytes_total{instance=~\"$nodeList\"}[$__rate_interval])\n)\n" }, "version": "v0" }, "refId": "query-1" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Bytes read and written per second, per device.**\n\nThe other half of the volume-limit question. Cloud block storage enforces bandwidth and IOPS as separate ceilings, and hitting either presents identically as latency.", "id": 1059, "links": [], "title": "Disk Throughput", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "unit": "Bps" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "disk-utilization": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (instance, device) (\n rate(node_disk_io_time_seconds_total{instance=~\"$nodeList\"}[$__rate_interval])\n)\n", "legendFormat": "{{device}}", "qryType": 1, "query": "max by (instance, device) (\n rate(node_disk_io_time_seconds_total{instance=~\"$nodeList\"}[$__rate_interval])\n)\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Fraction of wall time each block device had at least one I/O in flight.**\n\n**Nominal:** Below ~0.8 on the devices backing Materialize's spill and scratch.\n\n**Degraded:** Sustained near 1.0 means the device is the bottleneck. On cloud block storage this is often a burst-credit or provisioned-IOPS ceiling rather than a hardware limit.\n\nDevice-level saturation, not per-process. The `%%{interval}` rate of a seconds-per-second counter, so it is already a ratio.", "id": 1061, "links": [], "title": "Disk Utilization", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "max": 1, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "#CEFFFF", "value": null }, { "color": "#F94902", "value": 0.8 }, { "color": "#E40515", "value": 0.8500000000000001 }, { "color": "#E40515", "value": 0.9000000000000001 }, { "color": "#AB0003", "value": 0.9500000000000002 }, { "color": "#AB0003", "value": 1 } ] }, "unit": "percentunit" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "filefd": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (instance) (node_filefd_allocated{instance=~\"$nodeList\"})\n/\nmax by (instance) (node_filefd_maximum{instance=~\"$nodeList\"})\n", "legendFormat": "allocated / max", "qryType": 1, "query": "max by (instance) (node_filefd_allocated{instance=~\"$nodeList\"})\n/\nmax by (instance) (node_filefd_maximum{instance=~\"$nodeList\"})\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Allocated file descriptors as a fraction of the system-wide maximum.**\n\n**Nominal:** Well below 1.0.\n\n**Degraded:** Exhaustion fails `accept()` and `open()` across every process on the node at once, which presents as unrelated components failing together. A connection-heavy database plus a log collector tailing every container is a realistic way to get there.", "id": 1063, "links": [], "title": "File Descriptors In Use", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "fixedColor": "#BBBBBB", "mode": "shades" }, "max": 1, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "#CEFFFF", "value": null }, { "color": "#F94902", "value": 0.8 }, { "color": "#E40515", "value": 0.8500000000000001 }, { "color": "#E40515", "value": 0.9000000000000001 }, { "color": "#AB0003", "value": 0.9500000000000002 }, { "color": "#AB0003", "value": 1 } ] }, "unit": "percentunit" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "fs-available": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "min by (instance, mountpoint) (\n node_filesystem_avail_bytes{instance=~\"$nodeList\", fstype!=\"rootfs\"}\n)\n/\nmax by (instance, mountpoint) (\n node_filesystem_size_bytes{instance=~\"$nodeList\", fstype!=\"rootfs\"}\n)\n", "legendFormat": "{{mountpoint}}", "qryType": 1, "query": "min by (instance, mountpoint) (\n node_filesystem_avail_bytes{instance=~\"$nodeList\", fstype!=\"rootfs\"}\n)\n/\nmax by (instance, mountpoint) (\n node_filesystem_size_bytes{instance=~\"$nodeList\", fstype!=\"rootfs\"}\n)\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Fraction of each mounted filesystem still available, per mountpoint.**\n\n**Nominal:** Above 0.2.\n\n**Degraded:** Below ~0.1 on `/` or on the kubelet's filesystem triggers kubelet disk pressure, which evicts pods — including, without a priority class, the monitoring that would tell you about it.\n\n`min` across duplicates: a filesystem that is nearly full on one reading must not be averaged back to healthy. Per-pod kubelet mounts are excluded at the collector, so the mountpoint set here is bounded and stable.", "id": 1055, "links": [], "title": "Filesystem Available", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "max": 1, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "#AB0003", "value": null }, { "color": "#F19903", "value": 0.1 }, { "color": "#A2F49B", "value": 0.2 } ] }, "unit": "percentunit" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "fs-inodes": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "min by (instance, mountpoint) (\n node_filesystem_files_free{instance=~\"$nodeList\", fstype!=\"rootfs\"}\n)\n/\nmax by (instance, mountpoint) (\n node_filesystem_files{instance=~\"$nodeList\", fstype!=\"rootfs\"}\n)\n", "legendFormat": "{{mountpoint}}", "qryType": 1, "query": "min by (instance, mountpoint) (\n node_filesystem_files_free{instance=~\"$nodeList\", fstype!=\"rootfs\"}\n)\n/\nmax by (instance, mountpoint) (\n node_filesystem_files{instance=~\"$nodeList\", fstype!=\"rootfs\"}\n)\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Fraction of inodes still free, per mountpoint.**\n\nA filesystem can exhaust inodes with plenty of bytes left, and the failure is the same `ENOSPC` — which sends everyone looking at free space and finding nothing wrong. Many small files is the way there. `min` because low is the bad direction.", "id": 1056, "links": [], "title": "Inodes Available", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "max": 1, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "#AB0003", "value": null }, { "color": "#F19903", "value": 0.1 }, { "color": "#A2F49B", "value": 0.2 } ] }, "unit": "percentunit" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "fs-readonly": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (instance, mountpoint) (\n node_filesystem_readonly{instance=~\"$nodeList\", fstype!=\"rootfs\"}\n)\n", "legendFormat": "{{mountpoint}}", "qryType": 1, "query": "max by (instance, mountpoint) (\n node_filesystem_readonly{instance=~\"$nodeList\", fstype!=\"rootfs\"}\n)\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Whether a filesystem has been remounted read-only, per mountpoint.**\n\n**Nominal:** Zero everywhere.\n\n**Unhealthy:** A 1 means the kernel remounted the filesystem read-only, which it does after I/O errors. Every write on that node is failing; the workload usually looks hung rather than broken.", "id": 1057, "links": [], "title": "Read-only Mounts", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "decimals": 0, "max": 1, "min": 0, "noValue": "No read-only mounts — every filesystem is still writable." }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "memory-available": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "min by (instance) (\n node_memory_MemAvailable_bytes{instance=~\"$nodeList\"}\n)\n/\nmax by (instance) (\n node_memory_MemTotal_bytes{instance=~\"$nodeList\"}\n)\n", "legendFormat": "available", "qryType": 1, "query": "min by (instance) (\n node_memory_MemAvailable_bytes{instance=~\"$nodeList\"}\n)\n/\nmax by (instance) (\n node_memory_MemTotal_bytes{instance=~\"$nodeList\"}\n)\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Fraction of RAM the kernel estimates is available for new allocations without swapping, from `MemAvailable`.**\n\n**Nominal:** Comfortably above 0.2.\n\n**Degraded:** Approaching zero means the next allocation reclaims or swaps. Read with `node.memory.pressure` — pressure is what says the reclaim is hurting.\n\n`MemAvailable`, not `MemFree`: free memory is expected to be near zero on a healthy Linux box, because the page cache uses what is not otherwise claimed.", "id": 1030, "links": [], "title": "Memory Available", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "fixedColor": "#009988", "mode": "shades" }, "max": 1, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "#AB0003", "value": null }, { "color": "#F19903", "value": 0.1 }, { "color": "#A2F49B", "value": 0.2 } ] }, "unit": "percentunit" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "memory-breakdown": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (instance) (node_memory_MemTotal_bytes{instance=~\"$nodeList\"})\n", "legendFormat": "total", "qryType": 1, "query": "max by (instance) (node_memory_MemTotal_bytes{instance=~\"$nodeList\"})\n" }, "version": "v0" }, "refId": "query-0" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (instance) (\n node_memory_MemTotal_bytes{instance=~\"$nodeList\"}\n - node_memory_MemFree_bytes{instance=~\"$nodeList\"}\n - node_memory_Cached_bytes{instance=~\"$nodeList\"}\n - node_memory_Buffers_bytes{instance=~\"$nodeList\"}\n - node_memory_SReclaimable_bytes{instance=~\"$nodeList\"}\n)\n", "legendFormat": "used", "qryType": 1, "query": "max by (instance) (\n node_memory_MemTotal_bytes{instance=~\"$nodeList\"}\n - node_memory_MemFree_bytes{instance=~\"$nodeList\"}\n - node_memory_Cached_bytes{instance=~\"$nodeList\"}\n - node_memory_Buffers_bytes{instance=~\"$nodeList\"}\n - node_memory_SReclaimable_bytes{instance=~\"$nodeList\"}\n)\n" }, "version": "v0" }, "refId": "query-1" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (instance) (\n node_memory_Cached_bytes{instance=~\"$nodeList\"}\n + node_memory_Buffers_bytes{instance=~\"$nodeList\"}\n + node_memory_SReclaimable_bytes{instance=~\"$nodeList\"}\n)\n", "legendFormat": "cache and buffers", "qryType": 1, "query": "max by (instance) (\n node_memory_Cached_bytes{instance=~\"$nodeList\"}\n + node_memory_Buffers_bytes{instance=~\"$nodeList\"}\n + node_memory_SReclaimable_bytes{instance=~\"$nodeList\"}\n)\n" }, "version": "v0" }, "refId": "query-2" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "min by (instance) (node_memory_MemFree_bytes{instance=~\"$nodeList\"})\n", "legendFormat": "free", "qryType": 1, "query": "min by (instance) (node_memory_MemFree_bytes{instance=~\"$nodeList\"})\n" }, "version": "v0" }, "refId": "query-3" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Where RAM went: total, used by processes, reclaimable cache, and free.**\n\n\"Used\" here is total minus free minus cache/buffers/reclaimable-slab, which is the figure that matters — the page cache is returnable and `MemFree` being near zero is healthy.", "id": 1033, "links": [], "title": "Memory Breakdown", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "unit": "bytes" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "memory-kernel": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (instance) (node_memory_Slab_bytes{instance=~\"$nodeList\"})\n", "legendFormat": "slab", "qryType": 1, "query": "max by (instance) (node_memory_Slab_bytes{instance=~\"$nodeList\"})\n" }, "version": "v0" }, "refId": "query-0" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (instance) (node_memory_SReclaimable_bytes{instance=~\"$nodeList\"})\n", "legendFormat": "reclaimable", "qryType": 1, "query": "max by (instance) (node_memory_SReclaimable_bytes{instance=~\"$nodeList\"})\n" }, "version": "v0" }, "refId": "query-1" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (instance) (node_memory_SUnreclaim_bytes{instance=~\"$nodeList\"})\n", "legendFormat": "unreclaimable", "qryType": 1, "query": "max by (instance) (node_memory_SUnreclaim_bytes{instance=~\"$nodeList\"})\n" }, "version": "v0" }, "refId": "query-2" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (instance) (node_memory_Committed_AS_bytes{instance=~\"$nodeList\"})\n", "legendFormat": "committed", "qryType": 1, "query": "max by (instance) (node_memory_Committed_AS_bytes{instance=~\"$nodeList\"})\n" }, "version": "v0" }, "refId": "query-3" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Kernel-side memory: slab total, reclaimable and unreclaimable slab, and committed address space.**\n\nAnswers \"how much kernel memory\" without the per-cache cardinality of the `slabinfo` collector, which is off by default. Unreclaimable slab growing without bound is the shape of a kernel-side leak; a connection-heavy node with a large conntrack table also lands here.", "id": 1034, "links": [], "title": "Kernel Memory", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "unit": "bytes" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "memory-oom-kills": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (instance) (\n rate(node_vmstat_oom_kill{instance=~\"$nodeList\"}[$__rate_interval])\n)\n", "legendFormat": "kills", "qryType": 1, "query": "max by (instance) (\n rate(node_vmstat_oom_kill{instance=~\"$nodeList\"}[$__rate_interval])\n)\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Rate of OOM-killer invocations on the node.**\n\n**Nominal:** Zero.\n\n**Unhealthy:** Any non-zero value means the kernel killed a process to reclaim memory. On a node running Materialize this is very likely a replica dying; cross it with container restarts in the Kubernetes view.", "id": 1032, "links": [], "title": "OOM Kills", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "fixedColor": "#009988", "mode": "shades" }, "min": 0, "noValue": "No OOM kills — nothing has been terminated for memory.", "unit": "short" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "memory-page-faults": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (instance) (\n rate(node_vmstat_pgfault{instance=~\"$nodeList\"}[$__rate_interval])\n)\n", "legendFormat": "minor", "qryType": 1, "query": "max by (instance) (\n rate(node_vmstat_pgfault{instance=~\"$nodeList\"}[$__rate_interval])\n)\n" }, "version": "v0" }, "refId": "query-0" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (instance) (\n rate(node_vmstat_pgmajfault{instance=~\"$nodeList\"}[$__rate_interval])\n)\n", "legendFormat": "major (disk)", "qryType": 1, "query": "max by (instance) (\n rate(node_vmstat_pgmajfault{instance=~\"$nodeList\"}[$__rate_interval])\n)\n" }, "version": "v0" }, "refId": "query-1" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Total and major page faults per second.**\n\nMinor faults are ordinary. **Major** faults are the ones that touched disk, so on a node that swaps deliberately this is where the cost shows up — a major fault is a thread stopped until the page comes back.", "id": 1037, "links": [], "title": "Page Faults", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "unit": "cps" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "memory-pressure": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (instance) (\n rate(node_pressure_memory_waiting_seconds_total{instance=~\"$nodeList\"}[$__rate_interval])\n)\n", "legendFormat": "waiting", "qryType": 1, "query": "max by (instance) (\n rate(node_pressure_memory_waiting_seconds_total{instance=~\"$nodeList\"}[$__rate_interval])\n)\n" }, "version": "v0" }, "refId": "query-0" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (instance) (\n rate(node_pressure_memory_stalled_seconds_total{instance=~\"$nodeList\"}[$__rate_interval])\n)\n", "legendFormat": "stalled", "qryType": 1, "query": "max by (instance) (\n rate(node_pressure_memory_stalled_seconds_total{instance=~\"$nodeList\"}[$__rate_interval])\n)\n" }, "version": "v0" }, "refId": "query-1" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**PSI: the fraction of wall time at least one task was stalled on memory (`waiting`), and the fraction where *every* task was (`stalled`).**\n\n**Nominal:** Near zero for both.\n\n**Degraded:** The `waiting` series rising while swap activity is steady is healthy reclaim. Both series rising together is thrash — the node is spending its time moving pages rather than running the workload.\n\nThis is the leading indicator for the swap-as-a-feature workload: `node.swap.used.ratio` says swap is in use, and this says whether that is costing anything.", "id": 1031, "links": [], "title": "Memory Pressure", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "max": 1, "min": 0, "unit": "percentunit" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "memory-reclaim": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (instance) (\n rate(node_vmstat_pgscan_kswapd{instance=~\"$nodeList\"}[$__rate_interval])\n)\n", "legendFormat": "scanned (background)", "qryType": 1, "query": "max by (instance) (\n rate(node_vmstat_pgscan_kswapd{instance=~\"$nodeList\"}[$__rate_interval])\n)\n" }, "version": "v0" }, "refId": "query-0" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (instance) (\n rate(node_vmstat_pgscan_direct{instance=~\"$nodeList\"}[$__rate_interval])\n)\n", "legendFormat": "scanned (direct)", "qryType": 1, "query": "max by (instance) (\n rate(node_vmstat_pgscan_direct{instance=~\"$nodeList\"}[$__rate_interval])\n)\n" }, "version": "v0" }, "refId": "query-1" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (instance) (\n rate(node_vmstat_pgsteal_kswapd{instance=~\"$nodeList\"}[$__rate_interval])\n)\n", "legendFormat": "reclaimed (background)", "qryType": 1, "query": "max by (instance) (\n rate(node_vmstat_pgsteal_kswapd{instance=~\"$nodeList\"}[$__rate_interval])\n)\n" }, "version": "v0" }, "refId": "query-2" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (instance) (\n rate(node_vmstat_pgsteal_direct{instance=~\"$nodeList\"}[$__rate_interval])\n)\n", "legendFormat": "reclaimed (direct)", "qryType": 1, "query": "max by (instance) (\n rate(node_vmstat_pgsteal_direct{instance=~\"$nodeList\"}[$__rate_interval])\n)\n" }, "version": "v0" }, "refId": "query-3" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Pages scanned and reclaimed per second, split by who did the reclaiming: `kswapd` (background) or `direct` (an allocating thread).**\n\nThis is the distinction that separates swap working from swap hurting. Background reclaim by `kswapd` is the kernel keeping ahead of demand and costs the workload nothing. **Direct** reclaim means a thread asked for memory, none was free, and it had to go find some itself — it is stalled for the duration.\n\nThese fields are outside node_exporter's default `vmstat` filter; the chart widens `--collector.vmstat.fields` specifically to admit them.", "id": 1038, "links": [], "title": "Page Reclaim", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "unit": "cps" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "network-drops": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (instance, device) (\n rate(node_network_receive_drop_total{instance=~\"$nodeList\"}[$__rate_interval])\n)\n", "legendFormat": "rx {{device}}", "qryType": 1, "query": "max by (instance, device) (\n rate(node_network_receive_drop_total{instance=~\"$nodeList\"}[$__rate_interval])\n)\n" }, "version": "v0" }, "refId": "query-0" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (instance, device) (\n rate(node_network_transmit_drop_total{instance=~\"$nodeList\"}[$__rate_interval])\n)\n", "legendFormat": "tx {{device}}", "qryType": 1, "query": "max by (instance, device) (\n rate(node_network_transmit_drop_total{instance=~\"$nodeList\"}[$__rate_interval])\n)\n" }, "version": "v0" }, "refId": "query-1" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Receive and transmit packet drop rates per interface.**\n\n**Nominal:** Zero.\n\n**Degraded:** Receive drops usually mean the kernel could not keep up rather than the wire being faulty — check `node.debug.softnet.dropped` and `node.debug.softnet.squeezed`, which say whether the softirq budget was the limit.", "id": 1042, "links": [], "title": "Interface Drops", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "No interface drops recorded.", "unit": "cps" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "network-errors": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (instance, device) (\n rate(node_network_receive_errs_total{instance=~\"$nodeList\"}[$__rate_interval])\n)\n", "legendFormat": "rx {{device}}", "qryType": 1, "query": "max by (instance, device) (\n rate(node_network_receive_errs_total{instance=~\"$nodeList\"}[$__rate_interval])\n)\n" }, "version": "v0" }, "refId": "query-0" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (instance, device) (\n rate(node_network_transmit_errs_total{instance=~\"$nodeList\"}[$__rate_interval])\n)\n", "legendFormat": "tx {{device}}", "qryType": 1, "query": "max by (instance, device) (\n rate(node_network_transmit_errs_total{instance=~\"$nodeList\"}[$__rate_interval])\n)\n" }, "version": "v0" }, "refId": "query-1" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Receive and transmit error rates per interface.**\n\n**Nominal:** Zero.\n\n**Degraded:** Non-zero on a physical interface points at the NIC, the driver, or the link. Sustained errors on a cloud instance are more often a hypervisor or allowance problem than a cable.\n\nPer-pod veths are excluded at the collector, so `device` here is the node's real interfaces plus the stable `cilium_*` devices.", "id": 1041, "links": [], "title": "Interface Errors", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "No interface errors recorded.", "unit": "cps" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "network-operstate": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "min by (instance, device) (\n node_network_up{instance=~\"$nodeList\"}\n)\n", "legendFormat": "up {{device}}", "qryType": 1, "query": "min by (instance, device) (\n node_network_up{instance=~\"$nodeList\"}\n)\n" }, "version": "v0" }, "refId": "query-0" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "min by (instance, device) (\n node_network_carrier{instance=~\"$nodeList\"}\n)\n", "legendFormat": "carrier {{device}}", "qryType": 1, "query": "min by (instance, device) (\n node_network_carrier{instance=~\"$nodeList\"}\n)\n" }, "version": "v0" }, "refId": "query-1" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Whether each interface is operationally up, and whether it has carrier.**\n\n`min` because down is the bad direction. Useful mainly for confirming an interface disappeared rather than went quiet.\n\nThe upstream dashboard writes this as `node_network_up{operstate=\"up\"}`, which matches nothing on node_exporter 1.12: `node_network_up` is a plain 0/1 gauge per device, and `operstate` lives on `node_network_info`. The selector is dropped here rather than carried over.", "id": 1043, "links": [], "title": "Interface State", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "decimals": 0, "max": 1, "min": 0 }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "network-saturation": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (instance, device) (\n (node_network_speed_bytes{instance=~\"$nodeList\"} > bool 0)\n *\n (\n rate(node_network_receive_bytes_total{instance=~\"$nodeList\"}[$__rate_interval])\n / node_network_speed_bytes{instance=~\"$nodeList\"}\n )\n)\n", "legendFormat": "rx {{device}}", "qryType": 1, "query": "max by (instance, device) (\n (node_network_speed_bytes{instance=~\"$nodeList\"} > bool 0)\n *\n (\n rate(node_network_receive_bytes_total{instance=~\"$nodeList\"}[$__rate_interval])\n / node_network_speed_bytes{instance=~\"$nodeList\"}\n )\n)\n" }, "version": "v0" }, "refId": "query-0" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (instance, device) (\n (node_network_speed_bytes{instance=~\"$nodeList\"} > bool 0)\n *\n (\n rate(node_network_transmit_bytes_total{instance=~\"$nodeList\"}[$__rate_interval])\n / node_network_speed_bytes{instance=~\"$nodeList\"}\n )\n)\n", "legendFormat": "tx {{device}}", "qryType": 1, "query": "max by (instance, device) (\n (node_network_speed_bytes{instance=~\"$nodeList\"} > bool 0)\n *\n (\n rate(node_network_transmit_bytes_total{instance=~\"$nodeList\"}[$__rate_interval])\n / node_network_speed_bytes{instance=~\"$nodeList\"}\n )\n)\n" }, "version": "v0" }, "refId": "query-1" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Receive and transmit throughput as a fraction of the interface's reported link speed.**\n\nOnly meaningful where the driver reports a speed; the `> bool 0` guard makes interfaces that do not report one contribute 0 rather than dropping the series. On EC2 the *instance* allowance usually binds well before the reported link speed does, and that ceiling is only visible through the `ethtool` collector, which is off by default.", "id": 1040, "links": [], "title": "Link Saturation", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "max": 1, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "#CEFFFF", "value": null }, { "color": "#F94902", "value": 0.8 }, { "color": "#E40515", "value": 0.8500000000000001 }, { "color": "#E40515", "value": 0.9000000000000001 }, { "color": "#AB0003", "value": 0.9500000000000002 }, { "color": "#AB0003", "value": 1 } ] }, "unit": "percentunit" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "network-throughput": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (instance, device) (\n rate(node_network_receive_bytes_total{instance=~\"$nodeList\"}[$__rate_interval])\n)\n", "legendFormat": "rx {{device}}", "qryType": 1, "query": "max by (instance, device) (\n rate(node_network_receive_bytes_total{instance=~\"$nodeList\"}[$__rate_interval])\n)\n" }, "version": "v0" }, "refId": "query-0" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (instance, device) (\n rate(node_network_transmit_bytes_total{instance=~\"$nodeList\"}[$__rate_interval])\n)\n", "legendFormat": "tx {{device}}", "qryType": 1, "query": "max by (instance, device) (\n rate(node_network_transmit_bytes_total{instance=~\"$nodeList\"}[$__rate_interval])\n)\n" }, "version": "v0" }, "refId": "query-1" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Bytes received and transmitted per second, per interface.**\n\nNode-level only. The per-pod view comes from cAdvisor — per-pod veths are excluded at the collector because their names churn on every pod restart.", "id": 1039, "links": [], "title": "Network Throughput", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "unit": "Bps" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "node-event-feed": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${logsDatasource}" }, "group": "loki", "kind": "DataQuery", "spec": { "expr": "{job=\"loki.source.kubernetes_events\"}\n | kind=`Node`\n | name=`$node`\n |~ \"(?i)$logSearch\"\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Kubernetes events filed against this node, newest first.**\n\n**Nominal:** Quiet.\n\n**Degraded:** Lifecycle events — registration, reboot, readiness changes, eviction — arrive here first. This is the `Events:` section of `kubectl describe node`.\n\nEvents expire from Kubernetes after about an hour but persist in Loki for the retention period, so this reaches back further than `kubectl` would. The same retention means events may name a node that no longer exists.", "id": 1069, "links": [], "title": "Node Events", "vizConfig": { "group": "logs", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "noValue": "No events for this node, which is the healthy reading." }, "overrides": [] }, "options": { "dedupStrategy": "exact", "detailsMode": "inline", "displayedFields": [ "reason", "name", "msg" ], "enableInfiniteScrolling": true, "enableLogDetails": true, "fontSize": "small", "prettifyLogMessage": false, "showCommonLabels": false, "showControls": true, "showFieldSelector": true, "showLabels": false, "showLevel": true, "showLogContextToggle": false, "showTime": true, "sortOrder": "Descending", "syntaxHighlighting": true, "timestampResolution": "ms", "unwrappedColumns": true, "wrapLogMessage": false } }, "version": "" } } }, "node-event-rate": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${logsDatasource}" }, "group": "loki", "kind": "DataQuery", "spec": { "expr": "sum by (reason) (\n rate(\n {job=\"loki.source.kubernetes_events\"}\n | kind=`Node`\n | name=`$node`\n |~ \"(?i)$logSearch\"\n [$__auto]\n )\n)\n", "legendFormat": "{{reason}}" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Kubernetes events filed against this node, by reason.**\n\n**Nominal:** Quiet. A healthy node produces events when it joins and then rarely again.\n\n**Degraded:** `NodeNotReady`, `NodeHasMemoryPressure`, `NodeHasDiskPressure` and `Rebooted` are the ones that matter. Their appearance is the timestamp to give an infrastructure team.\n\n`kind=\"Node\"` selects events whose involved object *is* the machine, rather than events about pods that happen to run on it. Kubernetes aggregates repeats into one event with a rising count, so a burst under-reports.", "id": 1068, "links": [], "title": "Event Rate by Reason", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "No events for this node, which is the healthy reading.", "unit": "suffix:logs/s" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "node-journal-feed": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${logsDatasource}" }, "group": "loki", "kind": "DataQuery", "spec": { "expr": "{\n unit=~\"$logUnitList\",\n level=~\"$logLevelList\"\n}\n | node=`$node`\n |~ \"(?i)$logSearch\"\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**The systemd journal from this node, newest first.**\n\n**Nominal:** Routine chatter from `kubelet`, the container runtime and the provider's guest agent.\n\nThe whole feed, unfiltered by level, for when the warnings panel above has told you roughly where to look and you need the surrounding lines.", "id": 1072, "links": [], "title": "Node Journal", "vizConfig": { "group": "logs", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "noValue": "No journal lines from this node — is journal collection enabled?" }, "overrides": [] }, "options": { "dedupStrategy": "exact", "detailsMode": "inline", "displayedFields": [], "enableInfiniteScrolling": true, "enableLogDetails": true, "fontSize": "small", "prettifyLogMessage": false, "showCommonLabels": false, "showControls": true, "showFieldSelector": true, "showLabels": false, "showLevel": true, "showLogContextToggle": false, "showTime": true, "sortOrder": "Descending", "syntaxHighlighting": true, "timestampResolution": "ms", "unwrappedColumns": false, "wrapLogMessage": true } }, "version": "" } } }, "node-journal-rate": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${logsDatasource}" }, "group": "loki", "kind": "DataQuery", "spec": { "expr": "sum by (unit) (\n rate(\n {unit=~\"$logUnitList\"}\n | node=`$node`\n |~ \"(?i)$logSearch\"\n [$__auto]\n )\n)\n", "legendFormat": "{{unit}}" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Journal lines per second from this node, split by systemd unit.**\n\n**Nominal:** A steady low baseline, dominated by `kubelet` and the container runtime.\n\n**Degraded:** A step change in one unit is the node telling you something changed. `kubelet` and `containerd` climbing together usually means it is fighting to start or stop containers.\n\nScoped to one node by a pipeline filter rather than a stream selector, because `node` is structured metadata on journal lines — see this file's header.", "id": 1070, "links": [], "title": "Journal Rate by Unit", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "No journal lines from this node — is journal collection enabled?", "unit": "suffix:logs/s" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "node-journal-warnings": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${logsDatasource}" }, "group": "loki", "kind": "DataQuery", "spec": { "expr": "{\n unit=~\"$logUnitList\",\n level=~\"WARN|WARNING|ERROR|CRITICAL|FATAL\"\n}\n | node=`$node`\n |~ \"(?i)$logSearch\"\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Warning-and-worse journal lines from this node.**\n\n**Nominal:** Empty, or the occasional benign kernel notice.\n\n**Degraded:** Repeated warnings from `kubelet` about image pulls, volume mounts or sandbox creation are the node's own explanation for pods that will not start. Kernel messages about memory or I/O belong to the infrastructure team.\n\nDeliberately independent of the level picker: this panel answers \"is this machine complaining\", and narrowing the selection elsewhere must not silently empty it.", "id": 1071, "links": [], "title": "Journal Warnings and Errors", "vizConfig": { "group": "logs", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "noValue": "No warnings from this node's journal." }, "overrides": [] }, "options": { "dedupStrategy": "exact", "detailsMode": "inline", "displayedFields": [], "enableInfiniteScrolling": true, "enableLogDetails": true, "fontSize": "small", "prettifyLogMessage": false, "showCommonLabels": false, "showControls": true, "showFieldSelector": true, "showLabels": false, "showLevel": true, "showLogContextToggle": false, "showTime": true, "sortOrder": "Descending", "syntaxHighlighting": true, "timestampResolution": "ms", "unwrappedColumns": false, "wrapLogMessage": true } }, "version": "" } } }, "pods-budgets": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (namespace, pod) (\n sum by (namespace, pod, instance) (\n kube_pod_container_resource_requests{node=\"$node\", resource=\"cpu\"}\n )\n)\n", "format": "table", "instant": true, "legendFormat": "CPU Requested", "qryType": 2, "query": "max by (namespace, pod) (\n sum by (namespace, pod, instance) (\n kube_pod_container_resource_requests{node=\"$node\", resource=\"cpu\"}\n )\n)\n" }, "version": "v0" }, "refId": "query-0" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (namespace, pod) (\n sum by (namespace, pod, instance) (\n kube_pod_container_resource_limits{node=\"$node\", resource=\"cpu\"}\n )\n)\n", "format": "table", "instant": true, "legendFormat": "CPU Limit", "qryType": 2, "query": "max by (namespace, pod) (\n sum by (namespace, pod, instance) (\n kube_pod_container_resource_limits{node=\"$node\", resource=\"cpu\"}\n )\n)\n" }, "version": "v0" }, "refId": "query-1" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (namespace, pod) (\n sum by (namespace, pod, instance) (\n kube_pod_container_resource_requests{node=\"$node\", resource=\"memory\"}\n )\n)\n", "format": "table", "instant": true, "legendFormat": "Memory Requested", "qryType": 2, "query": "max by (namespace, pod) (\n sum by (namespace, pod, instance) (\n kube_pod_container_resource_requests{node=\"$node\", resource=\"memory\"}\n )\n)\n" }, "version": "v0" }, "refId": "query-2" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (namespace, pod) (\n sum by (namespace, pod, instance) (\n kube_pod_container_resource_limits{node=\"$node\", resource=\"memory\"}\n )\n)\n", "format": "table", "instant": true, "legendFormat": "Memory Limit", "qryType": 2, "query": "max by (namespace, pod) (\n sum by (namespace, pod, instance) (\n kube_pod_container_resource_limits{node=\"$node\", resource=\"memory\"}\n )\n)\n" }, "version": "v0" }, "refId": "query-3" } } ], "queryOptions": {}, "transformations": [ { "group": "merge", "kind": "Transformation", "spec": { "options": {} } }, { "group": "organize", "kind": "Transformation", "spec": { "options": { "excludeByName": { "Time": true }, "indexByName": { "CPU Limit": 3, "CPU Requested": 2, "Memory Limit": 5, "Memory Requested": 4, "Value #query-0": 6, "Value #query-1": 7, "Value #query-2": 8, "Value #query-3": 9, "namespace": 0, "pod": 1 }, "renameByName": { "Value #query-0": "CPU Requested", "Value #query-1": "CPU Limit", "Value #query-2": "Memory Requested", "Value #query-3": "Memory Limit" } } } } ] } }, "description": "**What each pod on this node reserved and what it is capped at — CPU and memory, requests beside limits.**\n\n**Nominal:** Requests that reflect what the workload actually uses, and limits set wherever an overrun would hurt the neighbours.\n\n**Degraded:** A pod whose request is far below its real usage is one the scheduler under-counted, which is how a node ends up oversubscribed while its gauges look calm. A pod with no memory limit can take the node down with it; a pod that crosses one is OOM-killed without warning.\n\n**Four expressions in one query so they land in one table**, joined on `namespace` and `pod`. Split across four panels an operator has to read four sorted lists and hold the pod names in their head to compare a request against its own limit — which is the only comparison that matters.\n\n**Instant.** These are current facts, not a series: evaluated over a range each pod would repeat once per scrape and the table would be unreadable. This is the same mistake the version table on `env-upgrade` guards against, from the other direction.\n\nLimits are absent for any pod that sets none, so those cells are empty rather than zero — which is the honest reading, and the difference worth seeing.", "id": 1067, "links": [], "title": "Requests and Limits by Pod", "vizConfig": { "group": "table", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "custom": { "align": "auto", "filterable": true, "inspect": false } }, "overrides": [ { "matcher": { "id": "byName", "options": "CPU Requested" }, "properties": [ { "id": "unit", "value": "cores" } ] }, { "matcher": { "id": "byName", "options": "CPU Limit" }, "properties": [ { "id": "unit", "value": "cores" } ] }, { "matcher": { "id": "byName", "options": "Memory Requested" }, "properties": [ { "id": "unit", "value": "bytes" } ] }, { "matcher": { "id": "byName", "options": "Memory Limit" }, "properties": [ { "id": "unit", "value": "bytes" } ] } ] }, "options": { "cellHeight": "sm", "footer": { "countRows": false, "reducer": [], "show": false }, "frameIndex": 0, "showHeader": true, "showTypeIcons": false } }, "version": "" } } }, "pods-by-phase": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (phase) (\n count by (phase, instance) (\n kube_pod_status_phase == 1\n and on (namespace, pod) kube_pod_info{node=\"$node\"}\n )\n)\n", "instant": true, "legendFormat": "{{phase}}", "qryType": 1, "query": "max by (phase) (\n count by (phase, instance) (\n kube_pod_status_phase == 1\n and on (namespace, pod) kube_pod_info{node=\"$node\"}\n )\n)\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Pods on this node, counted by lifecycle phase.**\n\n**Nominal:** Everything Running, or Succeeded for completed jobs.\n\n**Degraded:** `Pending` pods on a *specific* node are unusual — a pending pod normally has no node yet — and mean the kubelet accepted the pod but cannot start it, which points at image pulls or volume mounts. `Failed` and `Unknown` both warrant the node's journal.", "id": 1064, "links": [], "title": "Pods by Phase", "vizConfig": { "group": "piechart", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "noValue": "No metrics: kube-state-metrics is required", "unit": "short" }, "overrides": [] }, "options": { "displayLabels": [ "name", "value" ], "legend": { "asTable": true, "calcs": [], "displayMode": "table", "isVisible": true, "placement": "right", "showLegend": true, "values": [ "value" ] }, "orientation": "auto", "pieType": "donut", "reduceOptions": { "calcs": [], "values": false }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "pods-not-ready": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (namespace, pod) (\n kube_pod_status_ready{condition=\"true\"}\n and on (namespace, pod) kube_pod_info{node=\"$node\"}\n) == 0\nunless on (namespace, pod) (kube_pod_status_phase{phase=\"Succeeded\"} == 1)\n", "format": "table", "instant": true, "qryType": 2, "query": "max by (namespace, pod) (\n kube_pod_status_ready{condition=\"true\"}\n and on (namespace, pod) kube_pod_info{node=\"$node\"}\n) == 0\nunless on (namespace, pod) (kube_pod_status_phase{phase=\"Succeeded\"} == 1)\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [ { "group": "organize", "kind": "Transformation", "spec": { "options": { "excludeByName": { "Time": true, "Value": true, "__name__": true }, "indexByName": { "namespace": 0, "pod": 1 } } } } ] } }, "description": "**Pods on this node that are not reporting Ready.**\n\n**Nominal:** Empty.\n\n**Degraded:** A pod that is Running but not Ready is failing its readiness probe — it is up and refusing traffic. Several at once on one node, when the same workload is healthy elsewhere, is the signature of a node problem rather than an application one, and is worth escalating with the node name.\n\n**Completed Jobs are excluded.** A `Succeeded` pod is never Ready, because it has finished and exited — so without the `unless` every completed Job on the node sits here permanently, and a panel whose whole purpose is to be empty when things are well is never empty. `Failed` pods are kept: not ready is the correct reading there.", "id": 1065, "links": [], "title": "Pods Not Ready", "vizConfig": { "group": "table", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "custom": { "align": "auto", "filterable": true, "inspect": false }, "noValue": "Every pod on this node is Ready." }, "overrides": [] }, "options": { "cellHeight": "sm", "footer": { "countRows": false, "reducer": [], "show": false }, "frameIndex": 0, "showHeader": true, "showTypeIcons": false } }, "version": "" } } }, "pods-restarts": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (namespace, pod) (\n sum by (namespace, pod, instance) (\n kube_pod_container_status_restarts_total\n and on (namespace, pod) kube_pod_info{node=\"$node\"}\n )\n)\n", "legendFormat": "{{namespace}}/{{pod}}", "qryType": 1, "query": "max by (namespace, pod) (\n sum by (namespace, pod, instance) (\n kube_pod_container_status_restarts_total\n and on (namespace, pod) kube_pod_info{node=\"$node\"}\n )\n)\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Container restarts for pods on this node.**\n\n**Nominal:** Flat. A restart count that never moves is the healthy reading.\n\n**Degraded:** A climbing count is a crash loop. Restarts concentrated on one node while the same workload is stable elsewhere point at the machine — memory pressure and OOM kills are the usual cause, so read this beside the Memory tab.\n\nA cumulative counter, deliberately not rated: the absolute number is what an operator recognises, and a rate would hide a single restart.", "id": 1066, "links": [], "title": "Container Restarts", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "decimals": 0, "min": 0, "noValue": "No metrics: kube-state-metrics is required", "unit": "short" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "socket-memory": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (instance) (node_sockstat_TCP_mem_bytes{instance=~\"$nodeList\"})\n", "legendFormat": "TCP", "qryType": 1, "query": "max by (instance) (node_sockstat_TCP_mem_bytes{instance=~\"$nodeList\"})\n" }, "version": "v0" }, "refId": "query-0" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (instance) (node_sockstat_UDP_mem_bytes{instance=~\"$nodeList\"})\n", "legendFormat": "UDP", "qryType": 1, "query": "max by (instance) (node_sockstat_UDP_mem_bytes{instance=~\"$nodeList\"})\n" }, "version": "v0" }, "refId": "query-1" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Kernel socket buffer memory held by TCP and UDP.**\n\nCounts against `net.ipv4.tcp_mem`, and hitting that ceiling makes the kernel prune socket buffers — which looks like packet loss to the application while every interface counter stays clean. Worth watching on a node that is both memory-hungry and network-hungry, because the two budgets are the same RAM.", "id": 1049, "links": [], "title": "Socket Memory", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "unit": "bytes" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "softnet-dropped": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (instance, cpu) (\n rate(node_softnet_dropped_total{instance=~\"$nodeList\"}[$__rate_interval])\n)\n", "legendFormat": "core {{cpu}}", "qryType": 1, "query": "max by (instance, cpu) (\n rate(node_softnet_dropped_total{instance=~\"$nodeList\"}[$__rate_interval])\n)\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Packets dropped in the network softirq path because the backlog queue was full, per CPU.**\n\nThese are packets the kernel discarded before any socket saw them, so they are invisible to the application and to per-pod metrics. On a network-hungry workload behind an eBPF dataplane this is one of the few places the loss is attributable.", "id": 1051, "links": [], "title": "Softnet Dropped", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "No softnet drops recorded.", "unit": "cps" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "softnet-processed": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (instance, cpu) (\n rate(node_softnet_processed_total{instance=~\"$nodeList\"}[$__rate_interval])\n)\n", "legendFormat": "core {{cpu}}", "qryType": 1, "query": "max by (instance, cpu) (\n rate(node_softnet_processed_total{instance=~\"$nodeList\"}[$__rate_interval])\n)\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Packets processed by the network softirq path, per CPU.**\n\nThe denominator for the two counters below, and a way to see whether receive processing is spread across cores or piled onto one.", "id": 1050, "links": [], "title": "Softnet Processed", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "unit": "cps" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "softnet-squeezed": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (instance, cpu) (\n rate(node_softnet_times_squeezed_total{instance=~\"$nodeList\"}[$__rate_interval])\n)\n", "legendFormat": "core {{cpu}}", "qryType": 1, "query": "max by (instance, cpu) (\n rate(node_softnet_times_squeezed_total{instance=~\"$nodeList\"}[$__rate_interval])\n)\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Times the softirq handler exhausted its budget with work still queued, per CPU.**\n\nSqueezing is the precursor to dropping: the handler ran out of budget and deferred the rest. A rising rate here with drops still at zero is the warning you get before `node.debug.softnet.dropped` moves.", "id": 1052, "links": [], "title": "Softnet Squeezed", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "unit": "cps" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "summary-container-runtime": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (node, container_runtime_version) (kube_node_info{node=\"$node\"})\n", "instant": true, "legendFormat": "{{container_runtime_version}}", "qryType": 1, "query": "max by (node, container_runtime_version) (kube_node_info{node=\"$node\"})\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [ { "group": "labelsToFields", "kind": "Transformation", "spec": { "options": {} } } ] } }, "description": "**The container runtime that starts and stops containers on this node.**\n\n**Nominal:** Consistent across the pool.\n\nThe component the node journal's `containerd` messages come from, so its version is the context for anything found on the Logs & Events tab.", "id": 1003, "links": [], "title": "Container Runtime", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "fixedColor": "#0077BB", "mode": "shades" }, "noValue": "No such node — has it been removed from the cluster?" }, "overrides": [] }, "options": { "colorMode": "none", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [], "fields": "/^container_runtime_version$/", "values": false }, "showPercentChange": false, "text": { "valueSize": 16 }, "textMode": "value", "wideLayout": true } }, "version": "" } } }, "summary-cpu-allocation": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (node) (\n sum by (node, instance) (\n kube_pod_container_resource_requests{node=\"$node\", resource=\"cpu\"}\n )\n)\n/\nmax by (node) (kube_node_status_allocatable{node=\"$node\", resource=\"cpu\"})\n", "qryType": 1, "query": "max by (node) (\n sum by (node, instance) (\n kube_pod_container_resource_requests{node=\"$node\", resource=\"cpu\"}\n )\n)\n/\nmax by (node) (kube_node_status_allocatable{node=\"$node\", resource=\"cpu\"})\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Fraction of the node's schedulable CPU already promised to pods through their requests.**\n\n**Nominal:** Below ~0.8, leaving room for the scheduler to place more work.\n\n**Degraded:** Approaching 1.0 means the node is full *to the scheduler* regardless of how idle it looks. New pods will not land here, and a replica that needs to be rescheduled may find nowhere to go.\n\n**Requests, not usage.** This is the number Kubernetes schedules on, and it routinely disagrees with the CPU graphs: a node can sit at 10% actual utilization and still be unschedulable because its requests are spoken for. When \"there is plenty of CPU free\" and pods are Pending anyway, this is the panel that explains it — and the one to show an infrastructure team, because the fix is more nodes or smaller requests.", "id": 1015, "links": [], "title": "CPU Requested", "vizConfig": { "group": "gauge", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "fixedColor": "#0077BB", "mode": "shades" }, "max": 1, "min": 0, "noValue": "No metrics: kube-state-metrics is required", "thresholds": { "mode": "absolute", "steps": [ { "color": "#CEFFFF", "value": null }, { "color": "#F94902", "value": 0.8 }, { "color": "#E40515", "value": 0.8500000000000001 }, { "color": "#E40515", "value": 0.9000000000000001 }, { "color": "#AB0003", "value": 0.9500000000000002 }, { "color": "#AB0003", "value": 1 } ] }, "unit": "percentunit" }, "overrides": [] }, "options": { "minVizHeight": 75, "minVizWidth": 75, "orientation": "auto", "reduceOptions": { "calcs": [], "values": false }, "showThresholdLabels": false, "showThresholdMarkers": true, "sizing": "auto" } }, "version": "" } } }, "summary-cpu-capacity": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (node) (kube_node_status_capacity{node=\"$node\", resource=\"cpu\"})\n", "qryType": 1, "query": "max by (node) (kube_node_status_capacity{node=\"$node\", resource=\"cpu\"})\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Cores the node reports to Kubernetes.**\n\nCapacity, not allocatable: this is the whole machine, before the kubelet reserves its share. `infra.nodes.allocation.cpu` measures against the allocatable figure, which is the one that decides scheduling.", "id": 1005, "links": [], "title": "CPU Capacity", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "fixedColor": "#0077BB", "mode": "shades" }, "noValue": "No such node — has it been removed from the cluster?", "unit": "cores" }, "overrides": [] }, "options": { "colorMode": "none", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [], "values": false }, "showPercentChange": false, "textMode": "value", "wideLayout": true } }, "version": "" } } }, "summary-cpu-usage": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "1 - max by (instance) (\n avg by (instance, job) (\n rate(node_cpu_seconds_total{mode=\"idle\", instance=~\"$nodeList\"}[$__rate_interval])\n )\n)\n", "qryType": 1, "query": "1 - max by (instance) (\n avg by (instance, job) (\n rate(node_cpu_seconds_total{mode=\"idle\", instance=~\"$nodeList\"}[$__rate_interval])\n )\n)\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Fraction of CPU time the node spent doing anything other than idling, averaged across its cores.**\n\n**Nominal:** Well below 1.0, with headroom for the bursty half of the workload.\n\n**Degraded:** Sustained above ~0.8 means the node has no burst capacity left; check `node.cpu.pressure` to see whether threads are actually waiting.\n\nThe average across cores, not the busiest core — a single saturated core on a large node is invisible here and shows up in `node.debug.cpu.by_mode` and `node.debug.schedstat.waiting`.", "id": 1010, "links": [], "title": "CPU Usage", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "fixedColor": "#EE7733", "mode": "shades" }, "max": 1, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "#CEFFFF", "value": null }, { "color": "#F94902", "value": 0.8 }, { "color": "#E40515", "value": 0.8500000000000001 }, { "color": "#E40515", "value": 0.9000000000000001 }, { "color": "#AB0003", "value": 0.9500000000000002 }, { "color": "#AB0003", "value": 1 } ] }, "unit": "percentunit" }, "overrides": [] }, "options": { "colorMode": "none", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [], "values": false }, "showPercentChange": false, "textMode": "value", "wideLayout": true } }, "version": "" } } }, "summary-internal-ip": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (node, internal_ip) (kube_node_info{node=\"$node\"})\n", "instant": true, "legendFormat": "{{internal_ip}}", "qryType": 1, "query": "max by (node, internal_ip) (kube_node_info{node=\"$node\"})\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [ { "group": "labelsToFields", "kind": "Transformation", "spec": { "options": {} } } ] } }, "description": "**The address the cluster reaches this node on.**\n\n**Nominal:** Stable for the life of the node.\n\nAlso how node-exporter names this machine: its `instance` label is this address with the exporter's port. That is the join the node picker resolves through, so this cell is the value to check first when the measurement tabs are empty but the Kubernetes ones are not.", "id": 1004, "links": [], "title": "Internal IP", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "fixedColor": "#0077BB", "mode": "shades" }, "noValue": "No such node — has it been removed from the cluster?" }, "overrides": [] }, "options": { "colorMode": "none", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [], "fields": "/^internal_ip$/", "values": false }, "showPercentChange": false, "text": { "valueSize": 16 }, "textMode": "value", "wideLayout": true } }, "version": "" } } }, "summary-kernel-version": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (node, kernel_version) (kube_node_info{node=\"$node\"})\n", "instant": true, "legendFormat": "{{kernel_version}}", "qryType": 1, "query": "max by (node, kernel_version) (kube_node_info{node=\"$node\"})\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [ { "group": "labelsToFields", "kind": "Transformation", "spec": { "options": {} } } ] } }, "description": "**The node's kernel version.**\n\n**Nominal:** Whatever the OS image ships.\n\nRarely interesting on its own, and the single most useful string to quote verbatim when reporting a node-level fault — memory, I/O and network bugs are usually kernel-version-specific.", "id": 1002, "links": [], "title": "Kernel", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "fixedColor": "#0077BB", "mode": "shades" }, "noValue": "No such node — has it been removed from the cluster?" }, "overrides": [] }, "options": { "colorMode": "none", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [], "fields": "/^kernel_version$/", "values": false }, "showPercentChange": false, "text": { "valueSize": 16 }, "textMode": "value", "wideLayout": true } }, "version": "" } } }, "summary-kubelet-version": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (node, kubelet_version) (kube_node_info{node=\"$node\"})\n", "instant": true, "legendFormat": "{{kubelet_version}}", "qryType": 1, "query": "max by (node, kubelet_version) (kube_node_info{node=\"$node\"})\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [ { "group": "labelsToFields", "kind": "Transformation", "spec": { "options": {} } } ] } }, "description": "**The kubelet version this node runs, which is the version Kubernetes itself is on here.**\n\n**Nominal:** Within one minor version of the control plane, and the same across a node pool.\n\nA kubelet more than one minor behind the control plane is a node that missed an upgrade. It keeps working, so nothing alerts — but it is worth reporting, because the fix is a node-pool operation only an infrastructure team can do.", "id": 1000, "links": [], "title": "Kubernetes Version", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "fixedColor": "#0077BB", "mode": "shades" }, "noValue": "No such node — has it been removed from the cluster?" }, "overrides": [] }, "options": { "colorMode": "none", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [], "fields": "/^kubelet_version$/", "values": false }, "showPercentChange": false, "text": { "valueSize": 16 }, "textMode": "value", "wideLayout": true } }, "version": "" } } }, "summary-memory-allocation": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (node) (\n sum by (node, instance) (\n kube_pod_container_resource_requests{node=\"$node\", resource=\"memory\"}\n )\n)\n/\nmax by (node) (kube_node_status_allocatable{node=\"$node\", resource=\"memory\"})\n", "qryType": 1, "query": "max by (node) (\n sum by (node, instance) (\n kube_pod_container_resource_requests{node=\"$node\", resource=\"memory\"}\n )\n)\n/\nmax by (node) (kube_node_status_allocatable{node=\"$node\", resource=\"memory\"})\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Fraction of the node's schedulable memory already promised to pods through their requests.**\n\n**Nominal:** Below ~0.8.\n\n**Degraded:** Near 1.0 the node accepts no further work. Memory is the request that usually runs out first on a Materialize node, because a replica asks for what its dataflows need resident.\n\nRequests rather than usage, with the same caveat as `infra.nodes.allocation.cpu`: this can be full while the memory graphs look calm.", "id": 1016, "links": [], "title": "Memory Requested", "vizConfig": { "group": "gauge", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "fixedColor": "#0077BB", "mode": "shades" }, "max": 1, "min": 0, "noValue": "No metrics: kube-state-metrics is required", "thresholds": { "mode": "absolute", "steps": [ { "color": "#CEFFFF", "value": null }, { "color": "#F94902", "value": 0.8 }, { "color": "#E40515", "value": 0.8500000000000001 }, { "color": "#E40515", "value": 0.9000000000000001 }, { "color": "#AB0003", "value": 0.9500000000000002 }, { "color": "#AB0003", "value": 1 } ] }, "unit": "percentunit" }, "overrides": [] }, "options": { "minVizHeight": 75, "minVizWidth": 75, "orientation": "auto", "reduceOptions": { "calcs": [], "values": false }, "showThresholdLabels": false, "showThresholdMarkers": true, "sizing": "auto" } }, "version": "" } } }, "summary-memory-available": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "min by (instance) (\n node_memory_MemAvailable_bytes{instance=~\"$nodeList\"}\n)\n/\nmax by (instance) (\n node_memory_MemTotal_bytes{instance=~\"$nodeList\"}\n)\n", "qryType": 1, "query": "min by (instance) (\n node_memory_MemAvailable_bytes{instance=~\"$nodeList\"}\n)\n/\nmax by (instance) (\n node_memory_MemTotal_bytes{instance=~\"$nodeList\"}\n)\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Fraction of RAM the kernel estimates is available for new allocations without swapping, from `MemAvailable`.**\n\n**Nominal:** Comfortably above 0.2.\n\n**Degraded:** Approaching zero means the next allocation reclaims or swaps. Read with `node.memory.pressure` — pressure is what says the reclaim is hurting.\n\n`MemAvailable`, not `MemFree`: free memory is expected to be near zero on a healthy Linux box, because the page cache uses what is not otherwise claimed.", "id": 1011, "links": [], "title": "Memory Available", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "fixedColor": "#009988", "mode": "shades" }, "max": 1, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "#AB0003", "value": null }, { "color": "#F19903", "value": 0.1 }, { "color": "#A2F49B", "value": 0.2 } ] }, "unit": "percentunit" }, "overrides": [] }, "options": { "colorMode": "none", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [], "values": false }, "showPercentChange": false, "textMode": "value", "wideLayout": true } }, "version": "" } } }, "summary-memory-capacity": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (node) (kube_node_status_capacity{node=\"$node\", resource=\"memory\"})\n", "qryType": 1, "query": "max by (node) (kube_node_status_capacity{node=\"$node\", resource=\"memory\"})\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Bytes of RAM the node reports to Kubernetes.**\n\nThe whole machine. What is schedulable is smaller — see `infra.nodes.allocation.memory`.", "id": 1006, "links": [], "title": "Memory Capacity", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "fixedColor": "#0077BB", "mode": "shades" }, "noValue": "No such node — has it been removed from the cluster?", "unit": "bytes" }, "overrides": [] }, "options": { "colorMode": "none", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [], "values": false }, "showPercentChange": false, "textMode": "value", "wideLayout": true } }, "version": "" } } }, "summary-network-rx": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (instance) (\n max by (instance, device) (\n rate(node_network_receive_bytes_total{instance=~\"$nodeList\"}[$__rate_interval])\n )\n)\n", "qryType": 1, "query": "sum by (instance) (\n max by (instance, device) (\n rate(node_network_receive_bytes_total{instance=~\"$nodeList\"}[$__rate_interval])\n )\n)\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Bytes per second received across every interface on the node.**\n\n**Nominal:** Whatever the workload calls for, well under the link's capacity.\n\n**Degraded:** Read against `node.debug.network.saturation`, which is the same traffic as a fraction of what the link can carry.\n\n**Summed across interfaces, unlike `node.debug.network.throughput`**, which splits by device. A single-value panel cannot be hovered to find out which interface a line belongs to, so a per-device breakdown reduced to one number is worse than useless there — it silently shows one arbitrary device. Use this for stats and that one for graphs.\n\nIncludes loopback and the container-network interfaces, so on a Kubernetes node this counts pod-to-pod traffic as well as traffic that left the machine.", "id": 1013, "links": [], "title": "Network Rx", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "fixedColor": "#CCBB44", "mode": "shades" }, "min": 0, "unit": "Bps" }, "overrides": [] }, "options": { "colorMode": "none", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [], "values": false }, "showPercentChange": false, "textMode": "value", "wideLayout": true } }, "version": "" } } }, "summary-network-tx": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (instance) (\n max by (instance, device) (\n rate(node_network_transmit_bytes_total{instance=~\"$nodeList\"}[$__rate_interval])\n )\n)\n", "qryType": 1, "query": "sum by (instance) (\n max by (instance, device) (\n rate(node_network_transmit_bytes_total{instance=~\"$nodeList\"}[$__rate_interval])\n )\n)\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Bytes per second transmitted across every interface on the node.**\n\n**Nominal:** Whatever the workload calls for, well under the link's capacity.\n\n**Degraded:** Read against `node.debug.network.saturation`, which is the same traffic as a fraction of what the link can carry.\n\nThe transmit half of `node.network.rx.total`; the same summing applies, and for the same reason.", "id": 1014, "links": [], "title": "Network Tx", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "fixedColor": "#CCBB44", "mode": "shades" }, "min": 0, "unit": "Bps" }, "overrides": [] }, "options": { "colorMode": "none", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [], "values": false }, "showPercentChange": false, "textMode": "value", "wideLayout": true } }, "version": "" } } }, "summary-node-conditions": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (node, condition) (\n kube_node_status_condition{\n node=\"$node\",\n condition=~\"MemoryPressure|DiskPressure|PIDPressure|NetworkUnavailable\",\n status=\"true\"\n }\n)\n", "legendFormat": "{{condition}}", "qryType": 1, "query": "max by (node, condition) (\n kube_node_status_condition{\n node=\"$node\",\n condition=~\"MemoryPressure|DiskPressure|PIDPressure|NetworkUnavailable\",\n status=\"true\"\n }\n)\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**The node's pressure and availability conditions — memory, disk, PIDs and network — each 1 when the condition is active.**\n\n**Nominal:** All zero.\n\n**Degraded:** Any of these at 1 means the kubelet has started defending itself. `MemoryPressure` and `DiskPressure` cause eviction, `PIDPressure` blocks new processes, `NetworkUnavailable` means the node has no working pod network. All four are infrastructure problems rather than Materialize ones — report the node name and the condition.\n\nThese turn on *before* Ready turns off, which is what makes them worth watching: they are the early warning that the node is in trouble.\n\nOnly the four conditions every cluster reports are selected. A node may publish many more — cloud vendors add their own — and the extras are deliberately not shown, since their vocabulary differs by provider.", "id": 1020, "links": [], "title": "Pressure Conditions", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "decimals": 0, "max": 1, "min": 0, "noValue": "No pressure conditions active — the node is not defending itself." }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "summary-node-created": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (node) (kube_node_created{node=\"$node\"}) * 1000\n", "qryType": 1, "query": "max by (node) (kube_node_created{node=\"$node\"}) * 1000\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Wall-clock time the node joined the cluster.**\n\n**Nominal:** Old enough that the workload on it has settled.\n\n**Degraded:** A node much younger than its siblings was replaced recently — by an autoscaler, a node-pool upgrade, or a failure. If pods are unhappy and the node is minutes old, the replacement is the more likely story.\n\nDistinct from the machine's own uptime (`node.uptime`, from node-exporter): this is when Kubernetes first saw it. The two disagree when a node is rebooted without being removed from the cluster.\n\n**Scaled to milliseconds.** kube-state-metrics reports a Unix timestamp in *seconds*, and every Grafana date unit reads milliseconds — handed the raw value, a panel renders a date in January 1970 rather than an obviously wrong number, which is the kind of wrong that gets believed.", "id": 1009, "links": [], "title": "Joined Cluster", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "fixedColor": "#0077BB", "mode": "shades" }, "noValue": "No such node — has it been removed from the cluster?", "unit": "dateTimeAsIso" }, "overrides": [] }, "options": { "colorMode": "none", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [], "values": false }, "showPercentChange": false, "text": { "valueSize": 16 }, "textMode": "value", "wideLayout": true } }, "version": "" } } }, "summary-node-ready": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (node) (kube_node_status_condition{node=\"$node\", condition=\"Ready\", status=\"true\"})\n", "qryType": 1, "query": "max by (node) (kube_node_status_condition{node=\"$node\", condition=\"Ready\", status=\"true\"})\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Whether Kubernetes considers the node healthy enough to run work.**\n\n**Nominal:** 1 — the node is Ready.\n\n**Degraded:** 0 means the control plane has stopped hearing from the kubelet or the kubelet is reporting itself unhealthy. Pods on it will be evicted after the toleration window. **This is an escalation, not a tuning exercise.**\n\nReady is the single most load-bearing bit on this dashboard. It is also the one that lags: a node can be failing for minutes before Ready flips, which is why the pressure conditions sit beside it.", "id": 1018, "links": [], "title": "Ready", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "mappings": [ { "options": { "from": 1, "result": { "color": "#A2F49B", "index": 1, "text": "Healthy" }, "to": 2147483647 }, "type": "range" }, { "options": { "from": 1, "result": { "color": "#F19903", "index": 2, "text": "Degraded" }, "to": 1 }, "type": "range" }, { "options": { "from": -2147483647, "result": { "color": "#AB0003", "index": 3, "text": "Unhealthy" }, "to": 1 }, "type": "range" }, { "options": { "match": "null+nan", "result": { "color": "#888888", "index": 4, "text": "Missing Data" } }, "type": "special" } ], "noValue": "No metrics: kube-state-metrics is required", "thresholds": { "mode": "absolute", "steps": [ { "color": "#AB0003", "value": null }, { "color": "#F19903", "value": 1 }, { "color": "#A2F49B", "value": 1 } ] } }, "overrides": [] }, "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [], "values": false }, "showPercentChange": false, "textMode": "value", "wideLayout": true } }, "version": "" } } }, "summary-node-taints": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (node, key, value, effect) (kube_node_spec_taint{node=\"$node\"})\n", "format": "table", "instant": true, "qryType": 2, "query": "max by (node, key, value, effect) (kube_node_spec_taint{node=\"$node\"})\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [ { "group": "organize", "kind": "Transformation", "spec": { "options": { "excludeByName": { "Time": true, "Value": true, "__name__": true, "node": true }, "indexByName": { "effect": 2, "key": 0, "value": 1 } } } } ] } }, "description": "**The taints on this node, which restrict what may be scheduled.**\n\n**Nominal:** Whatever the node pool's design calls for — a dedicated Materialize pool is normally tainted so unrelated workloads stay off it.\n\n**Degraded:** A taint with effect `NoExecute` that appeared recently will have evicted pods. Kubernetes adds these itself when a node goes unreachable or under pressure.\n\nTaints are the other half of \"why is nothing scheduling here\", the first half being `infra.nodes.allocation.*`. A pod lands only if it tolerates every taint, so a deliberate pool taint plus a missing toleration looks exactly like a capacity problem from the workload's side.", "id": 1021, "links": [], "title": "Taints", "vizConfig": { "group": "table", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "custom": { "align": "auto", "filterable": true, "inspect": false }, "noValue": "No taints — this node accepts any workload that fits." }, "overrides": [] }, "options": { "cellHeight": "sm", "footer": { "countRows": false, "reducer": [], "show": false }, "frameIndex": 0, "showHeader": true, "showTypeIcons": false } }, "version": "" } } }, "summary-node-unschedulable": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (node) (kube_node_spec_unschedulable{node=\"$node\"})\n", "qryType": 1, "query": "max by (node) (kube_node_spec_unschedulable{node=\"$node\"})\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Whether the node has been cordoned against new work.**\n\n**Nominal:** 0 — the node accepts scheduling.\n\n**Degraded:** 1 means somebody (or something) ran the equivalent of `kubectl cordon`. Existing pods keep running; nothing new lands. During a node-pool upgrade this is normal and temporary. Outside one, it usually means an operator is working on the machine — worth asking before assuming a fault.\n\nA cordoned node is the quiet explanation for a replica that will not come back after a restart: there is nowhere to put it, and nothing is broken.", "id": 1019, "links": [], "title": "Cordoned", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "mappings": [ { "options": { "0": { "color": "#A2F49B", "index": 1, "text": "Schedulable" } }, "type": "value" }, { "options": { "1": { "color": "#F19903", "index": 2, "text": "Cordoned" } }, "type": "value" } ], "noValue": "No metrics: kube-state-metrics is required", "thresholds": { "mode": "absolute", "steps": [ { "color": "#A2F49B", "value": null }, { "color": "#F19903", "value": 1 } ] } }, "overrides": [] }, "options": { "colorMode": "background", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [], "values": false }, "showPercentChange": false, "textMode": "value", "wideLayout": true } }, "version": "" } } }, "summary-os-image": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (node, os_image) (kube_node_info{node=\"$node\"})\n", "instant": true, "legendFormat": "{{os_image}}", "qryType": 1, "query": "max by (node, os_image) (kube_node_info{node=\"$node\"})\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [ { "group": "labelsToFields", "kind": "Transformation", "spec": { "options": {} } } ] } }, "description": "**The node's operating system image.**\n\n**Nominal:** The same across every node in a pool.\n\nA node whose image differs from its siblings was replaced separately, usually by an autoscaler pulling a newer one. Harmless until it is not, and the first thing to quote when a fault appears on one node and not the others.", "id": 1001, "links": [], "title": "OS Image", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "fixedColor": "#0077BB", "mode": "shades" }, "noValue": "No such node — has it been removed from the cluster?" }, "overrides": [] }, "options": { "colorMode": "none", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [], "fields": "/^os_image$/", "values": false }, "showPercentChange": false, "text": { "valueSize": 16 }, "textMode": "value", "wideLayout": true } }, "version": "" } } }, "summary-pod-allocation": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (node) (\n count by (node, instance) (\n kube_pod_info{node=\"$node\"}\n unless on (namespace, pod) (kube_pod_status_phase{phase=~\"Succeeded|Failed\"} == 1)\n )\n)\n/\nmax by (node) (kube_node_status_allocatable{node=\"$node\", resource=\"pods\"})\n", "qryType": 1, "query": "max by (node) (\n count by (node, instance) (\n kube_pod_info{node=\"$node\"}\n unless on (namespace, pod) (kube_pod_status_phase{phase=~\"Succeeded|Failed\"} == 1)\n )\n)\n/\nmax by (node) (kube_node_status_allocatable{node=\"$node\", resource=\"pods\"})\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Fraction of the node's pod slots in use.**\n\n**Nominal:** Below ~0.8.\n\n**Degraded:** At 1.0 the node takes no more pods whatever its CPU and memory say. On clusters with many small pods this is the limit that binds first.\n\nCounted from `kube_pod_info` with terminal pods subtracted, so it tracks what the scheduler counts rather than what is merely still listed — see this file's header. A node whose completed Jobs have not been garbage collected would otherwise look fuller than it is.", "id": 1017, "links": [], "title": "Pod Slots Used", "vizConfig": { "group": "gauge", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "fixedColor": "#0077BB", "mode": "shades" }, "max": 1, "min": 0, "noValue": "No metrics: kube-state-metrics is required", "thresholds": { "mode": "absolute", "steps": [ { "color": "#CEFFFF", "value": null }, { "color": "#F94902", "value": 0.8 }, { "color": "#E40515", "value": 0.8500000000000001 }, { "color": "#E40515", "value": 0.9000000000000001 }, { "color": "#AB0003", "value": 0.9500000000000002 }, { "color": "#AB0003", "value": 1 } ] }, "unit": "percentunit" }, "overrides": [] }, "options": { "minVizHeight": 75, "minVizWidth": 75, "orientation": "auto", "reduceOptions": { "calcs": [], "values": false }, "showThresholdLabels": false, "showThresholdMarkers": true, "sizing": "auto" } }, "version": "" } } }, "summary-pod-capacity": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (node) (kube_node_status_capacity{node=\"$node\", resource=\"pods\"})\n", "qryType": 1, "query": "max by (node) (kube_node_status_capacity{node=\"$node\", resource=\"pods\"})\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**The most pods Kubernetes will place on this node.**\n\nA hard ceiling set by the kubelet, unrelated to how much CPU or memory is left. A node can be half idle and still refuse work because it is at its pod limit — which is why this sits beside the CPU and memory gauges rather than below them.", "id": 1007, "links": [], "title": "Pod Capacity", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "fixedColor": "#0077BB", "mode": "shades" }, "noValue": "No such node — has it been removed from the cluster?", "unit": "short" }, "overrides": [] }, "options": { "colorMode": "none", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [], "values": false }, "showPercentChange": false, "textMode": "value", "wideLayout": true } }, "version": "" } } }, "summary-pods-by-namespace": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (namespace) (\n count by (namespace, instance) (\n kube_pod_info{node=\"$node\"}\n unless on (namespace, pod) (kube_pod_status_phase{phase=~\"Succeeded|Failed\"} == 1)\n )\n)\n", "legendFormat": "{{namespace}}", "qryType": 1, "query": "max by (namespace) (\n count by (namespace, instance) (\n kube_pod_info{node=\"$node\"}\n unless on (namespace, pod) (kube_pod_status_phase{phase=~\"Succeeded|Failed\"} == 1)\n )\n)\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**What is actually running on this node, grouped by namespace.**\n\nThe answer to \"is my Materialize environment even on this machine\". Useful in both directions: confirming a replica landed where expected, and finding the unrelated workload that is crowding it out.\n\nTerminal pods are excluded, so this agrees with the allocation gauges beside it rather than counting completed Jobs as residents.", "id": 1022, "links": [], "title": "Pods by Namespace", "vizConfig": { "group": "piechart", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "noValue": "No metrics: kube-state-metrics is required", "unit": "short" }, "overrides": [] }, "options": { "displayLabels": [ "name", "value" ], "legend": { "asTable": true, "calcs": [], "displayMode": "table", "isVisible": true, "placement": "right", "showLegend": true, "values": [ "value" ] }, "orientation": "auto", "pieType": "donut", "reduceOptions": { "calcs": [], "values": false }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "summary-storage-capacity": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (node) (kube_node_status_capacity{node=\"$node\", resource=\"ephemeral_storage\"})\n", "qryType": 1, "query": "max by (node) (kube_node_status_capacity{node=\"$node\", resource=\"ephemeral_storage\"})\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Bytes of node-local disk available to pods for scratch space.**\n\nWhat container writable layers, `emptyDir` volumes and logs draw on. Exhausting it triggers `DiskPressure` and eviction — the node stays Ready while evicting, so this is worth reading beside `infra.nodes.conditions`.", "id": 1008, "links": [], "title": "Ephemeral Storage", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "fixedColor": "#0077BB", "mode": "shades" }, "noValue": "No such node — has it been removed from the cluster?", "unit": "bytes" }, "overrides": [] }, "options": { "colorMode": "none", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [], "values": false }, "showPercentChange": false, "textMode": "value", "wideLayout": true } }, "version": "" } } }, "summary-swap-used": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "(\n (\n max by (instance) (node_memory_SwapTotal_bytes{instance=~\"$nodeList\"})\n -\n min by (instance) (node_memory_SwapFree_bytes{instance=~\"$nodeList\"})\n )\n /\n max by (instance) (node_memory_SwapTotal_bytes{instance=~\"$nodeList\"})\n)\nand\n(max by (instance) (node_memory_SwapTotal_bytes{instance=~\"$nodeList\"}) > 0)\n", "qryType": 1, "query": "(\n (\n max by (instance) (node_memory_SwapTotal_bytes{instance=~\"$nodeList\"})\n -\n min by (instance) (node_memory_SwapFree_bytes{instance=~\"$nodeList\"})\n )\n /\n max by (instance) (node_memory_SwapTotal_bytes{instance=~\"$nodeList\"})\n)\nand\n(max by (instance) (node_memory_SwapTotal_bytes{instance=~\"$nodeList\"}) > 0)\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Fraction of configured swap in use. Zero when the node has no swap configured, rather than returning no data.**\n\n**Nominal:** Any value is nominal on its own. Materialize uses swap as a feature, so this is a capacity gauge, not an alert.\n\n**Degraded:** Approaching 1.0 leaves nowhere to spill; the next allocation pressure becomes an OOM kill instead.\n\n**A swapless node reports no series at all**, rather than 0. The earlier `> bool 0` guard did not work: on a node with no swap the ratio is `0/0`, which is NaN, and multiplying NaN by the guard's 0 is still NaN — so the panel read `NaN` instead of the intended zero. An `and` filter drops the series instead, which lets a panel say \"no swap configured\" and means a fleet view shows only the nodes that actually have any.", "id": 1012, "links": [], "title": "Swap Used", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "fixedColor": "#009988", "mode": "shades" }, "max": 1, "min": 0, "noValue": "No swap configured on this node.", "thresholds": { "mode": "absolute", "steps": [ { "color": "#CEFFFF", "value": null }, { "color": "#C6F7D6", "value": 0.1 }, { "color": "#BBE453", "value": 0.30000000000000004 }, { "color": "#E7B503", "value": 0.5 }, { "color": "#F6790B", "value": 0.7 }, { "color": "#E40515", "value": 0.8999999999999999 }, { "color": "#AB0003", "value": 1 } ] }, "unit": "percentunit" }, "overrides": [] }, "options": { "colorMode": "none", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [], "values": false }, "showPercentChange": false, "textMode": "value", "wideLayout": true } }, "version": "" } } }, "swap-activity": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (instance) (\n rate(node_vmstat_pswpin{instance=~\"$nodeList\"}[$__rate_interval])\n)\n", "legendFormat": "swapped in", "qryType": 1, "query": "max by (instance) (\n rate(node_vmstat_pswpin{instance=~\"$nodeList\"}[$__rate_interval])\n)\n" }, "version": "v0" }, "refId": "query-0" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (instance) (\n rate(node_vmstat_pswpout{instance=~\"$nodeList\"}[$__rate_interval])\n)\n", "legendFormat": "swapped out", "qryType": 1, "query": "max by (instance) (\n rate(node_vmstat_pswpout{instance=~\"$nodeList\"}[$__rate_interval])\n)\n" }, "version": "v0" }, "refId": "query-1" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Pages swapped in and out per second, from `vmstat`.**\n\n**Nominal:** Steady low-rate activity is normal here, unlike on most workloads.\n\n**Degraded:** A sustained climb in both directions at once is thrash: the same pages are going out and coming straight back. Confirm against the `stalled` series of `node.memory.pressure`.\n\nThe rate is the half of the swap story that the level cannot tell you. `node.swap.used.ratio` can sit flat and high on a perfectly healthy node.", "id": 1036, "links": [], "title": "Swap Activity", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "No swap activity — nothing is being paged to disk.", "unit": "Bps" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "swap-used": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "(\n (\n max by (instance) (node_memory_SwapTotal_bytes{instance=~\"$nodeList\"})\n -\n min by (instance) (node_memory_SwapFree_bytes{instance=~\"$nodeList\"})\n )\n /\n max by (instance) (node_memory_SwapTotal_bytes{instance=~\"$nodeList\"})\n)\nand\n(max by (instance) (node_memory_SwapTotal_bytes{instance=~\"$nodeList\"}) > 0)\n", "legendFormat": "used", "qryType": 1, "query": "(\n (\n max by (instance) (node_memory_SwapTotal_bytes{instance=~\"$nodeList\"})\n -\n min by (instance) (node_memory_SwapFree_bytes{instance=~\"$nodeList\"})\n )\n /\n max by (instance) (node_memory_SwapTotal_bytes{instance=~\"$nodeList\"})\n)\nand\n(max by (instance) (node_memory_SwapTotal_bytes{instance=~\"$nodeList\"}) > 0)\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Fraction of configured swap in use. Zero when the node has no swap configured, rather than returning no data.**\n\n**Nominal:** Any value is nominal on its own. Materialize uses swap as a feature, so this is a capacity gauge, not an alert.\n\n**Degraded:** Approaching 1.0 leaves nowhere to spill; the next allocation pressure becomes an OOM kill instead.\n\n**A swapless node reports no series at all**, rather than 0. The earlier `> bool 0` guard did not work: on a node with no swap the ratio is `0/0`, which is NaN, and multiplying NaN by the guard's 0 is still NaN — so the panel read `NaN` instead of the intended zero. An `and` filter drops the series instead, which lets a panel say \"no swap configured\" and means a fleet view shows only the nodes that actually have any.", "id": 1035, "links": [], "title": "Swap Used", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "fixedColor": "#009988", "mode": "shades" }, "max": 1, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "#CEFFFF", "value": null }, { "color": "#C6F7D6", "value": 0.1 }, { "color": "#BBE453", "value": 0.30000000000000004 }, { "color": "#E7B503", "value": 0.5 }, { "color": "#F6790B", "value": 0.7 }, { "color": "#E40515", "value": 0.8999999999999999 }, { "color": "#AB0003", "value": 1 } ] }, "unit": "percentunit" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "tcp-errors": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (instance) (\n rate(node_netstat_TcpExt_ListenOverflows{instance=~\"$nodeList\"}[$__rate_interval])\n)\n", "legendFormat": "listen overflows", "qryType": 1, "query": "max by (instance) (\n rate(node_netstat_TcpExt_ListenOverflows{instance=~\"$nodeList\"}[$__rate_interval])\n)\n" }, "version": "v0" }, "refId": "query-0" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (instance) (\n rate(node_netstat_TcpExt_ListenDrops{instance=~\"$nodeList\"}[$__rate_interval])\n)\n", "legendFormat": "listen drops", "qryType": 1, "query": "max by (instance) (\n rate(node_netstat_TcpExt_ListenDrops{instance=~\"$nodeList\"}[$__rate_interval])\n)\n" }, "version": "v0" }, "refId": "query-1" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (instance) (\n rate(node_netstat_TcpExt_TCPRcvQDrop{instance=~\"$nodeList\"}[$__rate_interval])\n)\n", "legendFormat": "receive queue drops", "qryType": 1, "query": "max by (instance) (\n rate(node_netstat_TcpExt_TCPRcvQDrop{instance=~\"$nodeList\"}[$__rate_interval])\n)\n" }, "version": "v0" }, "refId": "query-2" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (instance) (\n rate(node_netstat_TcpExt_TCPTimeouts{instance=~\"$nodeList\"}[$__rate_interval])\n)\n", "legendFormat": "timeouts", "qryType": 1, "query": "max by (instance) (\n rate(node_netstat_TcpExt_TCPTimeouts{instance=~\"$nodeList\"}[$__rate_interval])\n)\n" }, "version": "v0" }, "refId": "query-3" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**TCP listen-queue overflows, listen drops, receive-queue drops, and timeouts per second.**\n\nListen overflows and drops mean the accept queue was full — the peer completed a handshake the server never accepted, so the client sees a connection that hangs and then dies. That is a `somaxconn`/backlog problem in the application, surfaced here because nothing in the application will report it.", "id": 1046, "links": [], "title": "TCP Errors", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "No TCP errors recorded.", "unit": "cps" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "tcp-retransmits": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (instance) (\n rate(node_netstat_Tcp_RetransSegs{instance=~\"$nodeList\"}[$__rate_interval])\n)\n", "legendFormat": "retransmitted", "qryType": 1, "query": "max by (instance) (\n rate(node_netstat_Tcp_RetransSegs{instance=~\"$nodeList\"}[$__rate_interval])\n)\n" }, "version": "v0" }, "refId": "query-0" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (instance) (\n rate(node_netstat_TcpExt_TCPSynRetrans{instance=~\"$nodeList\"}[$__rate_interval])\n)\n", "legendFormat": "SYN retransmits", "qryType": 1, "query": "max by (instance) (\n rate(node_netstat_TcpExt_TCPSynRetrans{instance=~\"$nodeList\"}[$__rate_interval])\n)\n" }, "version": "v0" }, "refId": "query-1" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (instance) (\n rate(node_netstat_Tcp_OutSegs{instance=~\"$nodeList\"}[$__rate_interval])\n)\n", "legendFormat": "segments sent", "qryType": 1, "query": "max by (instance) (\n rate(node_netstat_Tcp_OutSegs{instance=~\"$nodeList\"}[$__rate_interval])\n)\n" }, "version": "v0" }, "refId": "query-2" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**TCP segment retransmits and SYN retransmits per second, against total segments out.**\n\nRead as a ratio against `OutSegs` rather than in absolute terms — a busy node retransmits constantly and that is fine. SYN retransmits rising specifically means new connections are failing to establish, which is a different problem from an established connection losing packets.", "id": 1045, "links": [], "title": "TCP Retransmits", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "unit": "cps" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "tcp-sockets": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (instance) (node_sockstat_TCP_inuse{instance=~\"$nodeList\"})\n", "legendFormat": "in use", "qryType": 1, "query": "max by (instance) (node_sockstat_TCP_inuse{instance=~\"$nodeList\"})\n" }, "version": "v0" }, "refId": "query-0" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (instance) (node_sockstat_TCP_alloc{instance=~\"$nodeList\"})\n", "legendFormat": "allocated", "qryType": 1, "query": "max by (instance) (node_sockstat_TCP_alloc{instance=~\"$nodeList\"})\n" }, "version": "v0" }, "refId": "query-1" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (instance) (node_sockstat_TCP_orphan{instance=~\"$nodeList\"})\n", "legendFormat": "orphaned", "qryType": 1, "query": "max by (instance) (node_sockstat_TCP_orphan{instance=~\"$nodeList\"})\n" }, "version": "v0" }, "refId": "query-2" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (instance) (node_sockstat_TCP_tw{instance=~\"$nodeList\"})\n", "legendFormat": "time-wait", "qryType": 1, "query": "max by (instance) (node_sockstat_TCP_tw{instance=~\"$nodeList\"})\n" }, "version": "v0" }, "refId": "query-3" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**TCP sockets by state: in use, allocated, orphaned, and TIME_WAIT.**\n\nA TIME_WAIT count climbing toward the ephemeral port range is the classic symptom of short-lived connections being opened faster than they can be retired — a connection-pooling problem, not a network one.", "id": 1044, "links": [], "title": "TCP Sockets", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "unit": "short" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "udp-errors": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (instance) (\n rate(node_netstat_Udp_InErrors{instance=~\"$nodeList\"}[$__rate_interval])\n)\n", "legendFormat": "in errors", "qryType": 1, "query": "max by (instance) (\n rate(node_netstat_Udp_InErrors{instance=~\"$nodeList\"}[$__rate_interval])\n)\n" }, "version": "v0" }, "refId": "query-0" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (instance) (\n rate(node_netstat_Udp_RcvbufErrors{instance=~\"$nodeList\"}[$__rate_interval])\n)\n", "legendFormat": "receive buffer errors", "qryType": 1, "query": "max by (instance) (\n rate(node_netstat_Udp_RcvbufErrors{instance=~\"$nodeList\"}[$__rate_interval])\n)\n" }, "version": "v0" }, "refId": "query-1" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (instance) (\n rate(node_netstat_Udp_NoPorts{instance=~\"$nodeList\"}[$__rate_interval])\n)\n", "legendFormat": "no port", "qryType": 1, "query": "max by (instance) (\n rate(node_netstat_Udp_NoPorts{instance=~\"$nodeList\"}[$__rate_interval])\n)\n" }, "version": "v0" }, "refId": "query-2" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**UDP receive errors, receive-buffer errors, and packets to no listening port, per second.**\n\nIn a Kubernetes cluster this is usually DNS. Receive-buffer errors mean datagrams arrived faster than the socket was drained, which surfaces as intermittent resolution failures rather than as a network alarm.", "id": 1047, "links": [], "title": "UDP Errors", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "No UDP errors recorded.", "unit": "cps" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "udp-queues": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (instance) (\n node_udp_queues{ip=\"v4\", queue=\"rx\", instance=~\"$nodeList\"}\n)\n", "legendFormat": "receive", "qryType": 1, "query": "max by (instance) (\n node_udp_queues{ip=\"v4\", queue=\"rx\", instance=~\"$nodeList\"}\n)\n" }, "version": "v0" }, "refId": "query-0" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (instance) (\n node_udp_queues{ip=\"v4\", queue=\"tx\", instance=~\"$nodeList\"}\n)\n", "legendFormat": "transmit", "qryType": 1, "query": "max by (instance) (\n node_udp_queues{ip=\"v4\", queue=\"tx\", instance=~\"$nodeList\"}\n)\n" }, "version": "v0" }, "refId": "query-1" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Bytes queued in UDP receive and transmit buffers.**\n\nThe level behind `node.debug.udp.errors`: a receive queue that stays deep is a socket nobody is draining fast enough, and the buffer errors follow.", "id": 1048, "links": [], "title": "UDP Queues", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "unit": "bytes" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } } }, "layout": { "kind": "TabsLayout", "spec": { "tabs": [ { "kind": "TabsLayoutTab", "spec": { "layout": { "kind": "RowsLayout", "spec": { "rows": [ { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "narrow", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "summary-kubelet-version" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "summary-os-image" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "summary-kernel-version" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "summary-container-runtime" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "summary-internal-ip" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "summary-cpu-capacity" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "summary-memory-capacity" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "summary-pod-capacity" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "summary-storage-capacity" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "summary-node-created" } } } ], "maxColumnCount": 8, "rowHeightMode": "short" } }, "title": "Node Info" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "standard", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "summary-cpu-usage" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "summary-memory-available" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "summary-swap-used" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "summary-network-rx" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "summary-network-tx" } } } ], "maxColumnCount": 5, "rowHeightMode": "short" } }, "title": "Utilization" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "standard", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "summary-cpu-allocation" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "summary-memory-allocation" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "summary-pod-allocation" } } } ], "maxColumnCount": 3, "rowHeightMode": "short" } }, "title": "Allocation" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "standard", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "summary-node-ready" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "summary-node-unschedulable" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "summary-node-conditions" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "summary-node-taints" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "summary-pods-by-namespace" } } } ], "maxColumnCount": 3, "rowHeightMode": "short" } }, "title": "Scheduling and Conditions" } } ] } }, "title": "Summary" } }, { "kind": "TabsLayoutTab", "spec": { "layout": { "kind": "RowsLayout", "spec": { "rows": [ { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "standard", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "cpu-utilization" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "cpu-load" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "cpu-pressure" } } } ], "maxColumnCount": 3, "rowHeightMode": "standard" } }, "title": "Processor Load" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "standard", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "cpu-by-mode" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "cpu-per-core" } } } ], "maxColumnCount": 2, "rowHeightMode": "standard" } }, "title": "Where the Time Went" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "standard", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "cpu-runqueue" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "cpu-context-switches" } } } ], "maxColumnCount": 2, "rowHeightMode": "standard" } }, "title": "Scheduling" } } ] } }, "title": "CPU" } }, { "kind": "TabsLayoutTab", "spec": { "layout": { "kind": "RowsLayout", "spec": { "rows": [ { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "standard", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "memory-available" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "memory-pressure" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "memory-oom-kills" } } } ], "maxColumnCount": 3, "rowHeightMode": "standard" } }, "title": "Headroom" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "standard", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "memory-breakdown" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "memory-kernel" } } } ], "maxColumnCount": 2, "rowHeightMode": "standard" } }, "title": "What Is Holding It" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "standard", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "swap-used" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "swap-activity" } } } ], "maxColumnCount": 2, "rowHeightMode": "standard" } }, "title": "Swap" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "standard", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "memory-page-faults" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "memory-reclaim" } } } ], "maxColumnCount": 2, "rowHeightMode": "standard" } }, "title": "Paging and Reclaim" } } ] } }, "title": "Memory & Swap" } }, { "kind": "TabsLayoutTab", "spec": { "layout": { "kind": "RowsLayout", "spec": { "rows": [ { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "standard", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "network-throughput" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "network-saturation" } } } ], "maxColumnCount": 2, "rowHeightMode": "standard" } }, "title": "Throughput" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "standard", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "network-errors" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "network-drops" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "network-operstate" } } } ], "maxColumnCount": 3, "rowHeightMode": "standard" } }, "title": "Errors and Drops" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "standard", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "tcp-sockets" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "tcp-retransmits" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "tcp-errors" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "udp-errors" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "udp-queues" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "socket-memory" } } } ], "maxColumnCount": 3, "rowHeightMode": "standard" } }, "title": "TCP and UDP" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": true, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "standard", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "softnet-processed" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "softnet-dropped" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "softnet-squeezed" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "conntrack" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "arp-entries" } } } ], "maxColumnCount": 3, "rowHeightMode": "standard" } }, "title": "Kernel Networking" } } ] } }, "title": "Network" } }, { "kind": "TabsLayoutTab", "spec": { "layout": { "kind": "RowsLayout", "spec": { "rows": [ { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "standard", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "fs-available" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "fs-inodes" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "fs-readonly" } } } ], "maxColumnCount": 3, "rowHeightMode": "standard" } }, "title": "Filesystems" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "standard", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "disk-iops" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "disk-throughput" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "disk-latency" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "disk-utilization" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "disk-queue-depth" } } } ], "maxColumnCount": 2, "rowHeightMode": "standard" } }, "title": "Disks" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "standard", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "filefd" } } } ], "maxColumnCount": 1, "rowHeightMode": "standard" } }, "title": "File Descriptors" } } ] } }, "title": "Storage" } }, { "kind": "TabsLayoutTab", "spec": { "layout": { "kind": "RowsLayout", "spec": { "rows": [ { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "standard", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "pods-by-phase" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "pods-not-ready" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "pods-restarts" } } } ], "maxColumnCount": 3, "rowHeightMode": "standard" } }, "title": "Pod Health" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "standard", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "pods-budgets" } } } ], "maxColumnCount": 1, "rowHeightMode": "tall" } }, "title": "Requests and Limits" } } ] } }, "title": "Pods" } }, { "kind": "TabsLayoutTab", "spec": { "layout": { "kind": "RowsLayout", "spec": { "rows": [ { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "standard", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "node-event-rate" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "node-event-feed" } } } ], "maxColumnCount": 1, "rowHeightMode": "tall" } }, "title": "Kubernetes Events" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "standard", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "node-journal-rate" } } } ], "maxColumnCount": 1, "rowHeightMode": "standard" } }, "title": "Journal Volume" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "standard", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "node-journal-warnings" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "node-journal-feed" } } } ], "maxColumnCount": 1, "rowHeightMode": "tall" } }, "title": "Node Journal" } } ] } }, "title": "Logs & Events" } } ] } }, "links": [], "liveNow": false, "preload": false, "tags": [ "infrastructure", "mzmon", "nodes" ], "timeSettings": { "autoRefresh": "", "autoRefreshIntervals": [ "5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d" ], "fiscalYearStartMonth": 0, "from": "now-6h", "hideTimepicker": false, "timezone": "browser", "to": "now" }, "title": "Infrastructure Node Detail", "variables": [ { "kind": "DatasourceVariable", "spec": { "allowCustomValue": false, "current": { "text": "", "value": "" }, "description": "Datasource for metrics queries", "hide": "dontHide", "includeAll": false, "label": "Metrics Datasource", "multi": false, "name": "metricsDatasource", "options": [], "pluginId": "prometheus", "refresh": "never", "regex": "", "skipUrlSync": false } }, { "kind": "DatasourceVariable", "spec": { "allowCustomValue": false, "current": { "text": "", "value": "" }, "description": "Datasource for log and event queries", "hide": "dontHide", "includeAll": false, "label": "Logs Datasource", "multi": false, "name": "logsDatasource", "options": [], "pluginId": "loki", "refresh": "never", "regex": "", "skipUrlSync": false } }, { "kind": "QueryVariable", "spec": { "allowCustomValue": true, "current": { "text": "", "value": "" }, "definition": "label_values(kube_node_info, node)", "description": "The node this dashboard describes", "hide": "dontHide", "includeAll": false, "label": "Node", "multi": false, "name": "node", "options": [], "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "label_values(kube_node_info, node)", "qryType": 1, "query": "label_values(kube_node_info, node)" }, "version": "v0" }, "refresh": "never", "regex": "", "regexApplyTo": "value", "skipUrlSync": false, "sort": "alphabeticalAsc" } }, { "kind": "QueryVariable", "spec": { "allowCustomValue": true, "current": { "text": "", "value": "" }, "definition": "label_values(node_uname_info{nodename=\"$node\"}, instance)", "description": "node-exporter address of the selected node (derived)", "hide": "hideVariable", "includeAll": false, "label": "Node Instance", "multi": false, "name": "nodeList", "options": [], "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "label_values(node_uname_info{nodename=\"$node\"}, instance)", "qryType": 1, "query": "label_values(node_uname_info{nodename=\"$node\"}, instance)" }, "version": "v0" }, "refresh": "never", "regex": "", "regexApplyTo": "value", "skipUrlSync": false, "sort": "alphabeticalAsc" } }, { "kind": "QueryVariable", "spec": { "allValue": ".+", "allowCustomValue": true, "current": { "text": "", "value": "" }, "definition": "label_values(unit)", "description": "The systemd unit(s) to read node journal logs from", "hide": "dontHide", "includeAll": true, "label": "Unit", "multi": true, "name": "logUnitList", "options": [], "query": { "datasource": { "name": "${logsDatasource}" }, "group": "loki", "kind": "DataQuery", "spec": { "label": "unit", "refId": "LokiVariableQueryEditor-VariableQuery", "stream": "", "type": 1 }, "version": "v0" }, "refresh": "onTimeRangeChanged", "regex": "", "regexApplyTo": "value", "skipUrlSync": false, "sort": "alphabeticalAsc" } }, { "kind": "QueryVariable", "spec": { "allValue": ".*", "allowCustomValue": true, "current": { "text": "", "value": "" }, "definition": "label_values(level)", "description": "The severity level(s) to include", "hide": "dontHide", "includeAll": true, "label": "Level", "multi": true, "name": "logLevelList", "options": [], "query": { "datasource": { "name": "${logsDatasource}" }, "group": "loki", "kind": "DataQuery", "spec": { "label": "level", "refId": "LokiVariableQueryEditor-VariableQuery", "stream": "{namespace=~\"$logNamespaceList\"}", "type": 1 }, "version": "v0" }, "refresh": "onTimeRangeChanged", "regex": "", "regexApplyTo": "value", "skipUrlSync": false, "sort": "alphabeticalAsc" } }, { "kind": "TextVariable", "spec": { "current": { "text": "", "value": "" }, "description": "Case-insensitive text to match anywhere in the log line", "hide": "dontHide", "label": "Search", "name": "logSearch", "query": "", "skipUrlSync": false } }, { "datasource": { "name": "$metricsDatasource" }, "group": "", "kind": "AdhocVariable", "spec": { "allowCustomValue": true, "baseFilters": [ { "key": "namespace", "operator": "=~", "origin": "dashboard", "value": "$mzNamespaceList" } ], "defaultKeys": [], "description": "Adhoc filters to apply to all metrics queries", "enableGroupBy": false, "filters": [], "hide": "inControlsMenu", "label": "Advanced Metric Filter", "name": "metricAdhoc", "skipUrlSync": false } }, { "datasource": { "name": "$logsDatasource" }, "group": "", "kind": "AdhocVariable", "spec": { "allowCustomValue": true, "baseFilters": [], "defaultKeys": [], "description": "Adhoc filters to apply to all logs queries", "enableGroupBy": false, "filters": [], "hide": "inControlsMenu", "label": "Advanced Log Filter", "name": "logsAdhoc", "skipUrlSync": false } } ] } }