{ "apiVersion": "dashboard.grafana.app/v2", "kind": "Dashboard", "metadata": { "annotations": { "grafana.app/folder": "materialize", "monitoring.materialize.cloud/min-mz-version": "v26.24.0", "monitoring.materialize.cloud/rec-mz-version": "v26.41.0", "monitoring.materialize.cloud/sql-metric-prefix": "mz_", "monitoring.materialize.cloud/target-export": "generic" }, "name": "mz-mon-env-persist" }, "spec": { "annotations": [ { "kind": "AnnotationQuery", "spec": { "builtIn": true, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "query": { "datasource": { "name": "-- Grafana --" }, "group": "grafana", "kind": "DataQuery", "spec": {}, "version": "v0" } } } ], "cursorSync": "Crosshair", "description": "Object storage, as Materialize experiences it.\n\nReads, writes, failures, compaction and what the stored data is for, measured by the processes that use the bucket — the same on every store. What a Ceph object store reports about itself is on the Object Store Internals tab, and what a cloud bucket's provider reports is on the Infrastructure Cloud Provider dashboard.", "editable": true, "elements": { "ceph-bucket-pool-available": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (name) (\n ceph_pool_max_avail{namespace=~\"$cephNamespace\"}\n * on (namespace, pool_id) group_left (name)\n max by (namespace, pool_id, name) (ceph_pool_metadata{namespace=~\"$cephNamespace\", name=~\".+\\\\.rgw\\\\.buckets\\\\.data\"})\n)\n", "legendFormat": "{{name}}", "qryType": 1, "query": "max by (name) (\n ceph_pool_max_avail{namespace=~\"$cephNamespace\"}\n * on (namespace, pool_id) group_left (name)\n max by (namespace, pool_id, name) (ceph_pool_metadata{namespace=~\"$cephNamespace\", name=~\".+\\\\.rgw\\\\.buckets\\\\.data\"})\n)\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Space still writable in each object store's bucket data pool, after replication.**\n\n**Nominal:** Comfortably more than a few days' growth of persist's data.\n\n**Degraded:** Little headroom left. Persist's stored data only shrinks when compaction catches up, so this is the number to plan disks against.\n\nCeph's own estimate, from the fullest OSD the pool can write to and the pool's replication. It moves when any other pool on the same disks grows.", "id": 1033, "links": [], "title": "Bucket Space Available", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "fixedColor": "#EE3377", "mode": "shades" }, "min": 0, "noValue": "No RGW bucket data pool in the selected Ceph cluster.", "unit": "bytes" }, "overrides": [] }, "options": { "colorMode": "none", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [], "values": false }, "showPercentChange": false, "textMode": "value", "wideLayout": true } }, "version": "" } } }, "ceph-capacity-pools": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (name) (\n ceph_pool_stored{namespace=~\"$cephNamespace\"}\n * on (namespace, pool_id) group_left (name)\n max by (namespace, pool_id, name) (ceph_pool_metadata{namespace=~\"$cephNamespace\"})\n) > 0\n", "legendFormat": "{{name}}", "qryType": 1, "query": "max by (name) (\n ceph_pool_stored{namespace=~\"$cephNamespace\"}\n * on (namespace, pool_id) group_left (name)\n max by (namespace, pool_id, name) (ceph_pool_metadata{namespace=~\"$cephNamespace\"})\n) > 0\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Data stored in each pool, before replication.**\n\n**Nominal:** The bucket data pool (`.rgw.buckets.data`) holding nearly all of it, close to persist's own total on the Storage tab.\n\n**Degraded:** The bucket pool well above what persist says it stores is data persist no longer refers to — objects it has not deleted yet, or uploads that never finished. Another pool growing is another user of the same disks.", "id": 1043, "links": [], "title": "Stored Data by Pool", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "Ceph has not reported this. The gateway and disk counters need rook-ceph-exporter scraped as well as the manager.", "unit": "bytes" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "ceph-capacity-raw": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(ceph_cluster_total_used_raw_bytes{namespace=~\"$cephNamespace\"})\n", "legendFormat": "used", "qryType": 1, "query": "sum(ceph_cluster_total_used_raw_bytes{namespace=~\"$cephNamespace\"})\n" }, "version": "v0" }, "refId": "query-0" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(ceph_cluster_total_bytes{namespace=~\"$cephNamespace\"})\n", "legendFormat": "total", "qryType": 1, "query": "sum(ceph_cluster_total_bytes{namespace=~\"$cephNamespace\"})\n" }, "version": "v0" }, "refId": "query-1" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Raw disk capacity used, and the cluster's total.**\n\n**Nominal:** Used well below the total, growing slowly.\n\n**Degraded:** Used approaching the total. Raw capacity counts every copy, so with three-way replication each byte persist stores uses three here.", "id": 1042, "links": [], "title": "Raw Capacity", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "Ceph has not reported this. The gateway and disk counters need rook-ceph-exporter scraped as well as the manager.", "unit": "bytes" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "ceph-health": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max(ceph_health_status{namespace=~\"$cephNamespace\"})\n", "legendFormat": "health", "qryType": 1, "query": "max(ceph_health_status{namespace=~\"$cephNamespace\"})\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Ceph's own verdict on the cluster: `HEALTH_OK`, `HEALTH_WARN` or `HEALTH_ERR`.**\n\n**Nominal:** HEALTH_OK.\n\n**Degraded:** HEALTH_WARN covers everything from a misconfiguration notice to a disk that is nearly full. _Active Health Checks_ on this tab lists which checks are raised.\n\n**Unhealthy:** HEALTH_ERR means data is unavailable or at risk: writes may be blocked, which persist reports as failed uploads.", "id": 1030, "links": [], "title": "Ceph Health", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "mappings": [ { "options": { "0": { "color": "#A2F49B", "index": 1, "text": "HEALTH_OK" } }, "type": "value" }, { "options": { "1": { "color": "#F19903", "index": 2, "text": "HEALTH_WARN" } }, "type": "value" }, { "options": { "2": { "color": "#AB0003", "index": 3, "text": "HEALTH_ERR" } }, "type": "value" } ], "noValue": "Ceph has not reported this. The gateway and disk counters need rook-ceph-exporter scraped as well as the manager.", "thresholds": { "mode": "absolute", "steps": [ { "color": "#A2F49B", "value": null }, { "color": "#F19903", "value": 1 }, { "color": "#AB0003", "value": 2 } ] } }, "overrides": [] }, "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [], "values": false }, "showPercentChange": false, "textMode": "value", "wideLayout": true } }, "version": "" } } }, "ceph-health-checks": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (name, severity) (ceph_health_detail{namespace=~\"$cephNamespace\"} == 1)\n", "format": "table", "instant": true, "qryType": 2, "query": "max by (name, severity) (ceph_health_detail{namespace=~\"$cephNamespace\"} == 1)\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [ { "group": "organize", "kind": "Transformation", "spec": { "options": { "excludeByName": { "Time": true, "Value": true }, "indexByName": { "name": 1, "severity": 0 }, "renameByName": { "name": "Check", "severity": "Severity" } } } }, { "group": "sortBy", "kind": "Transformation", "spec": { "options": { "fields": {}, "sort": [ { "desc": true, "field": "Severity" } ] } } } ] } }, "description": "**Every health check Ceph has raised and not cleared, with its severity.**\n\n**Nominal:** Empty.\n\n**Degraded:** Each row is one reason the cluster is not `HEALTH_OK`. Names are Ceph's own — `OSD_NEARFULL`, `PG_DEGRADED`, `RECENT_CRASH` — and `ceph health detail` in the Rook toolbox explains each.", "id": 1037, "links": [], "title": "Active Health Checks", "vizConfig": { "group": "table", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "custom": { "align": "auto", "filterable": true, "inspect": false }, "noValue": "No health checks raised: Ceph reports HEALTH_OK." }, "overrides": [] }, "options": { "cellHeight": "sm", "footer": { "countRows": false, "reducer": [], "show": false }, "frameIndex": 0, "showHeader": true, "showTypeIcons": false } }, "version": "" } } }, "ceph-objects-degraded": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(ceph_num_objects_degraded{namespace=~\"$cephNamespace\"})\n", "legendFormat": "degraded", "qryType": 1, "query": "sum(ceph_num_objects_degraded{namespace=~\"$cephNamespace\"})\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Object copies Ceph is missing and has not yet re-created.**\n\n**Nominal:** Zero.\n\n**Degraded:** Non-zero after a disk or node failure while Ceph recovers. Reads and writes continue, with fewer copies behind them.\n\n**Unhealthy:** A count that stops falling is recovery that cannot proceed — usually too few disks left to place another copy on.", "id": 1034, "links": [], "title": "Degraded Objects", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "decimals": 0, "min": 0, "noValue": "Ceph has not reported this. The gateway and disk counters need rook-ceph-exporter scraped as well as the manager.", "thresholds": { "mode": "absolute", "steps": [ { "color": "#A2F49B", "value": null }, { "color": "#FEDA8B", "value": 1 }, { "color": "#FDB366", "value": 250.75 }, { "color": "#F67E4B", "value": 500.5 }, { "color": "#DD3D2D", "value": 750.25 }, { "color": "#A50026", "value": 1000 } ] }, "unit": "short" }, "overrides": [] }, "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [], "values": false }, "showPercentChange": false, "textMode": "value", "wideLayout": true } }, "version": "" } } }, "ceph-osd-latency": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (ceph_daemon) (ceph_osd_commit_latency_ms{namespace=~\"$cephNamespace\"}) / 1000\n", "legendFormat": "{{ceph_daemon}}", "qryType": 1, "query": "max by (ceph_daemon) (ceph_osd_commit_latency_ms{namespace=~\"$cephNamespace\"}) / 1000\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**How long each OSD takes to commit a write to its disk.**\n\n**Nominal:** Single-digit milliseconds on SSDs, tens on spinning disks, and every OSD close to the others.\n\n**Degraded:** One OSD much slower than the rest is a failing or overloaded disk, and every write with a copy on it waits for it.\n\nCeph's own recent average, published by the manager.", "id": 1045, "links": [], "title": "OSD Commit Latency", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "Ceph has not reported this. The gateway and disk counters need rook-ceph-exporter scraped as well as the manager.", "unit": "s" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "ceph-osd-utilization": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (ceph_daemon) (\n ceph_osd_stat_bytes_used{namespace=~\"$cephNamespace\"}\n / ceph_osd_stat_bytes{namespace=~\"$cephNamespace\"}\n)\n", "legendFormat": "{{ceph_daemon}}", "qryType": 1, "query": "max by (ceph_daemon) (\n ceph_osd_stat_bytes_used{namespace=~\"$cephNamespace\"}\n / ceph_osd_stat_bytes{namespace=~\"$cephNamespace\"}\n)\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**How full each disk daemon (OSD) is.**\n\n**Nominal:** Every OSD within a few points of the others, below 75%.\n\n**Degraded:** One OSD well above the rest fills first. The balancer, or reweighting that OSD, evens them out.\n\n**Unhealthy:** Past Ceph's `nearfull` ratio (85% by default), Ceph warns, and the OSD has little room left before `full` (95%), where Ceph stops writes to its pools however empty the others are.", "id": 1044, "links": [], "title": "OSD Utilization", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "max": 1, "min": 0, "noValue": "Ceph has not reported this. The gateway and disk counters need rook-ceph-exporter scraped as well as the manager.", "thresholds": { "mode": "absolute", "steps": [ { "color": "#A2F49B", "value": null }, { "color": "#F19903", "value": 0.75 }, { "color": "#AB0003", "value": 0.85 } ] }, "unit": "percentunit" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "ceph-osds-down": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "count(ceph_osd_up{namespace=~\"$cephNamespace\"} == 0) or vector(0)\n", "legendFormat": "down", "qryType": 1, "query": "count(ceph_osd_up{namespace=~\"$cephNamespace\"} == 0) or vector(0)\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Disk daemons (OSDs) that are down.**\n\n**Nominal:** Zero.\n\n**Degraded:** Ceph keeps serving from the remaining copies, with less redundancy. If an OSD stays down, Ceph marks it out and copies its data elsewhere, which costs the remaining disks I/O.\n\n**Unhealthy:** Enough OSDs down that a placement group loses all its copies makes objects unavailable, and persist's reads and writes against them fail.", "id": 1031, "links": [], "title": "Disks (OSDs) Down", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "decimals": 0, "min": 0, "noValue": "Ceph has not reported this. The gateway and disk counters need rook-ceph-exporter scraped as well as the manager.", "thresholds": { "mode": "absolute", "steps": [ { "color": "#A2F49B", "value": null }, { "color": "#FEDA8B", "value": 1 }, { "color": "#FDB366", "value": 1.25 }, { "color": "#F67E4B", "value": 1.5 }, { "color": "#DD3D2D", "value": 1.75 }, { "color": "#A50026", "value": 2 } ] }, "unit": "short" }, "overrides": [] }, "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [], "values": false }, "showPercentChange": false, "textMode": "value", "wideLayout": true } }, "version": "" } } }, "ceph-placement-groups": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(ceph_pg_total{namespace=~\"$cephNamespace\"}) - sum(ceph_pg_active{namespace=~\"$cephNamespace\"})\n", "legendFormat": "not active", "qryType": 1, "query": "sum(ceph_pg_total{namespace=~\"$cephNamespace\"}) - sum(ceph_pg_active{namespace=~\"$cephNamespace\"})\n" }, "version": "v0" }, "refId": "query-0" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(ceph_pg_degraded{namespace=~\"$cephNamespace\"})\n", "legendFormat": "degraded", "qryType": 1, "query": "sum(ceph_pg_degraded{namespace=~\"$cephNamespace\"})\n" }, "version": "v0" }, "refId": "query-1" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(ceph_pg_undersized{namespace=~\"$cephNamespace\"})\n", "legendFormat": "undersized", "qryType": 1, "query": "sum(ceph_pg_undersized{namespace=~\"$cephNamespace\"})\n" }, "version": "v0" }, "refId": "query-2" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(ceph_pg_total{namespace=~\"$cephNamespace\"}) - sum(ceph_pg_clean{namespace=~\"$cephNamespace\"})\n", "legendFormat": "not clean", "qryType": 1, "query": "sum(ceph_pg_total{namespace=~\"$cephNamespace\"}) - sum(ceph_pg_clean{namespace=~\"$cephNamespace\"})\n" }, "version": "v0" }, "refId": "query-3" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Placement groups that are not fully healthy, by the state that is wrong.**\n\n**Nominal:** All zero. A placement group is the unit Ceph replicates and recovers; a healthy one is `active+clean`.\n\n**Degraded:** `degraded` and `undersized` after a disk failure, falling as Ceph recovers. Lines that stay up mean recovery cannot complete.\n\n**Unhealthy:** Any that are not `active` cannot serve reads or writes, and persist's calls against the objects in them fail.", "id": 1046, "links": [], "title": "Placement Groups Not Healthy", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "decimals": 0, "min": 0, "noValue": "Ceph has not reported this. The gateway and disk counters need rook-ceph-exporter scraped as well as the manager.", "unit": "short" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "ceph-raw-used": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(ceph_cluster_total_used_raw_bytes{namespace=~\"$cephNamespace\"})\n/ sum(ceph_cluster_total_bytes{namespace=~\"$cephNamespace\"})\n", "legendFormat": "used", "qryType": 1, "query": "sum(ceph_cluster_total_used_raw_bytes{namespace=~\"$cephNamespace\"})\n/ sum(ceph_cluster_total_bytes{namespace=~\"$cephNamespace\"})\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Raw disk capacity used across the cluster, replication included.**\n\n**Nominal:** Below 75%. Ceph needs room to re-replicate a failed disk's data onto the others.\n\n**Degraded:** Above 75%, losing a disk may push the others past Ceph's `nearfull` ratio (85% by default) while it recovers.\n\n**Unhealthy:** Past `nearfull`, Ceph raises a health warning, and one disk's recovery can carry the rest to the `full` ratio (95% by default). At `full` on any OSD, Ceph stops accepting writes to the pools on it, and persist cannot upload.\n\nA cluster-wide average. Ceph enforces `full` per OSD, so one uneven disk can stop writes first; _OSD Utilization_ shows each.", "id": 1032, "links": [], "title": "Raw Capacity Used", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "max": 1, "min": 0, "noValue": "Ceph has not reported this. The gateway and disk counters need rook-ceph-exporter scraped as well as the manager.", "thresholds": { "mode": "absolute", "steps": [ { "color": "#A2F49B", "value": null }, { "color": "#F19903", "value": 0.75 }, { "color": "#AB0003", "value": 0.85 } ] }, "unit": "percentunit" }, "overrides": [] }, "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [], "values": false }, "showPercentChange": false, "textMode": "value", "wideLayout": true } }, "version": "" } } }, "ceph-rgw-failure-ratio": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "(\n sum(rate(ceph_rgw_failed_req{namespace=~\"$cephNamespace\"}[$__rate_interval]))\n / sum(rate(ceph_rgw_req{namespace=~\"$cephNamespace\"}[$__rate_interval]))\n) >= 0\n", "legendFormat": "failed", "qryType": 1, "query": "(\n sum(rate(ceph_rgw_failed_req{namespace=~\"$cephNamespace\"}[$__rate_interval]))\n / sum(rate(ceph_rgw_req{namespace=~\"$cephNamespace\"}[$__rate_interval]))\n) >= 0\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**The fraction of object gateway requests that failed.**\n\n**Nominal:** A small, steady fraction. RGW counts every request that does not succeed, so a lookup of an object that does not exist counts too.\n\n**Degraded:** A rise that coincides with failures on persist's own panels is the gateway refusing work: overloaded, out of space, or unable to reach the disks behind it.", "id": 1035, "links": [], "title": "Gateway Failed Requests", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "max": 1, "min": 0, "noValue": "Ceph has not reported this. The gateway and disk counters need rook-ceph-exporter scraped as well as the manager.", "thresholds": { "mode": "absolute", "steps": [ { "color": "#A2F49B", "value": null }, { "color": "#F19903", "value": 0.05 }, { "color": "#AB0003", "value": 0.25 } ] }, "unit": "percentunit" }, "overrides": [] }, "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [], "values": false }, "showPercentChange": false, "textMode": "value", "wideLayout": true } }, "version": "" } } }, "ceph-rgw-latency": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "(sum(rate(ceph_rgw_op_put_obj_lat_sum{namespace=~\"$cephNamespace\"}[$__rate_interval]))\n / sum(rate(ceph_rgw_op_put_obj_lat_count{namespace=~\"$cephNamespace\"}[$__rate_interval]))) >= 0\n", "legendFormat": "PUT", "qryType": 1, "query": "(sum(rate(ceph_rgw_op_put_obj_lat_sum{namespace=~\"$cephNamespace\"}[$__rate_interval]))\n / sum(rate(ceph_rgw_op_put_obj_lat_count{namespace=~\"$cephNamespace\"}[$__rate_interval]))) >= 0\n" }, "version": "v0" }, "refId": "query-0" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "(sum(rate(ceph_rgw_op_get_obj_lat_sum{namespace=~\"$cephNamespace\"}[$__rate_interval]))\n / sum(rate(ceph_rgw_op_get_obj_lat_count{namespace=~\"$cephNamespace\"}[$__rate_interval]))) >= 0\n", "legendFormat": "GET", "qryType": 1, "query": "(sum(rate(ceph_rgw_op_get_obj_lat_sum{namespace=~\"$cephNamespace\"}[$__rate_interval]))\n / sum(rate(ceph_rgw_op_get_obj_lat_count{namespace=~\"$cephNamespace\"}[$__rate_interval]))) >= 0\n" }, "version": "v0" }, "refId": "query-1" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "(sum(rate(ceph_rgw_op_del_obj_lat_sum{namespace=~\"$cephNamespace\"}[$__rate_interval]))\n / sum(rate(ceph_rgw_op_del_obj_lat_count{namespace=~\"$cephNamespace\"}[$__rate_interval]))) >= 0\n", "legendFormat": "DELETE", "qryType": 1, "query": "(sum(rate(ceph_rgw_op_del_obj_lat_sum{namespace=~\"$cephNamespace\"}[$__rate_interval]))\n / sum(rate(ceph_rgw_op_del_obj_lat_count{namespace=~\"$cephNamespace\"}[$__rate_interval]))) >= 0\n" }, "version": "v0" }, "refId": "query-2" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Mean time the object gateway takes to serve each operation.**\n\n**Nominal:** Milliseconds to low tens of milliseconds, with PUTs slower than GETs: a PUT returns once every copy is on disk.\n\n**Degraded:** Rising together with _OSD Latency_ is the disks. Rising alone is the gateway itself — CPU, or a queue of requests.\n\nA mean from the gateway's own counters, so it cannot show the tail. Persist's latency on the Operations tab includes the network as well, and the difference between the two is the path in between.", "id": 1039, "links": [], "title": "Gateway Latency by Operation (Mean)", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "Ceph has not reported this. The gateway and disk counters need rook-ceph-exporter scraped as well as the manager.", "thresholds": { "mode": "absolute", "steps": [ { "color": "#A2F49B", "value": null }, { "color": "#F19903", "value": 0.1 }, { "color": "#AB0003", "value": 1 } ] }, "unit": "s" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "ceph-rgw-queue": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (instance_id) (ceph_rgw_qactive{namespace=~\"$cephNamespace\"})\n", "legendFormat": "active on {{instance_id}}", "qryType": 1, "query": "sum by (instance_id) (ceph_rgw_qactive{namespace=~\"$cephNamespace\"})\n" }, "version": "v0" }, "refId": "query-0" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (instance_id) (ceph_rgw_qlen{namespace=~\"$cephNamespace\"})\n", "legendFormat": "queued on {{instance_id}}", "qryType": 1, "query": "sum by (instance_id) (ceph_rgw_qlen{namespace=~\"$cephNamespace\"})\n" }, "version": "v0" }, "refId": "query-1" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Requests each gateway is working on, and requests queued behind them.**\n\n**Nominal:** A handful active and none queued.\n\n**Degraded:** A standing queue is a gateway at its concurrency limit: every request, persist's included, waits for a slot. Add gateway instances to the object store, or find what is holding requests open.", "id": 1041, "links": [], "title": "Gateway Requests in Progress", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "decimals": 0, "min": 0, "noValue": "Ceph has not reported this. The gateway and disk counters need rook-ceph-exporter scraped as well as the manager.", "unit": "short" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "ceph-rgw-requests": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(rate(ceph_rgw_op_put_obj_ops{namespace=~\"$cephNamespace\"}[$__rate_interval]))\n", "legendFormat": "PUT", "qryType": 1, "query": "sum(rate(ceph_rgw_op_put_obj_ops{namespace=~\"$cephNamespace\"}[$__rate_interval]))\n" }, "version": "v0" }, "refId": "query-0" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(rate(ceph_rgw_op_get_obj_ops{namespace=~\"$cephNamespace\"}[$__rate_interval]))\n", "legendFormat": "GET", "qryType": 1, "query": "sum(rate(ceph_rgw_op_get_obj_ops{namespace=~\"$cephNamespace\"}[$__rate_interval]))\n" }, "version": "v0" }, "refId": "query-1" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(rate(ceph_rgw_op_del_obj_ops{namespace=~\"$cephNamespace\"}[$__rate_interval]))\n", "legendFormat": "DELETE", "qryType": 1, "query": "sum(rate(ceph_rgw_op_del_obj_ops{namespace=~\"$cephNamespace\"}[$__rate_interval]))\n" }, "version": "v0" }, "refId": "query-2" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(rate(ceph_rgw_failed_req{namespace=~\"$cephNamespace\"}[$__rate_interval]))\n", "legendFormat": "failed", "qryType": 1, "query": "sum(rate(ceph_rgw_failed_req{namespace=~\"$cephNamespace\"}[$__rate_interval]))\n" }, "version": "v0" }, "refId": "query-3" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Object gateway requests per second, by operation, and how many failed.**\n\n**Nominal:** PUTs and DELETEs tracking persist's own uploads and deletes on the Operations tab, and GETs tracking its fetches plus its deletes.\n\n**Degraded:** Failures rising with no change in persist's traffic are the gateway's own trouble. Requests well above persist's are another client sharing the gateway.\n\nEvery client of every gateway in the selected Ceph cluster, not only Materialize. RGW counts a `HEAD` as a GET, and persist checks each object with a `HEAD` before deleting it, so GETs here run well above persist's own reads — about the sum of its reads and deletes.", "id": 1038, "links": [], "title": "Gateway Requests by Operation", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "Ceph has not reported this. The gateway and disk counters need rook-ceph-exporter scraped as well as the manager.", "unit": "reqps" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "ceph-rgw-throughput": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(rate(ceph_rgw_op_put_obj_bytes{namespace=~\"$cephNamespace\"}[$__rate_interval]))\n", "legendFormat": "written", "qryType": 1, "query": "sum(rate(ceph_rgw_op_put_obj_bytes{namespace=~\"$cephNamespace\"}[$__rate_interval]))\n" }, "version": "v0" }, "refId": "query-0" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(rate(ceph_rgw_op_get_obj_bytes{namespace=~\"$cephNamespace\"}[$__rate_interval]))\n", "legendFormat": "read", "qryType": 1, "query": "sum(rate(ceph_rgw_op_get_obj_bytes{namespace=~\"$cephNamespace\"}[$__rate_interval]))\n" }, "version": "v0" }, "refId": "query-1" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Bytes written to and read from the object gateway per second.**\n\n**Nominal:** Writes tracking persist's uploads and compaction, reads tracking hydration and queries that miss persist's cache.", "id": 1040, "links": [], "title": "Gateway Throughput", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "Ceph has not reported this. The gateway and disk counters need rook-ceph-exporter scraped as well as the manager.", "unit": "Bps" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "ceph-version": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "count by (ceph_version) (\n label_replace(\n ceph_osd_metadata{namespace=~\"$cephNamespace\"},\n \"ceph_version\", \"$1 $2\", \"ceph_version\", \"ceph version ([^ ]+) \\\\([0-9a-f]+\\\\) ([a-z]+).*\"\n )\n)\n", "instant": true, "legendFormat": "{{ceph_version}}", "qryType": 1, "query": "count by (ceph_version) (\n label_replace(\n ceph_osd_metadata{namespace=~\"$cephNamespace\"},\n \"ceph_version\", \"$1 $2\", \"ceph_version\", \"ceph version ([^ ]+) \\\\([0-9a-f]+\\\\) ([a-z]+).*\"\n )\n)\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [ { "group": "labelsToFields", "kind": "Transformation", "spec": { "options": {} } } ] } }, "description": "**The Ceph version the cluster's disk daemons run.**\n\n**Nominal:** One supported release on every daemon.\n\nMore than one version means an upgrade is rolling through. An out-of-date dependency has been the cause of an incident before, and this is the one place it is visible without a Ceph login.\n\nShortened from Ceph's own string, `ceph version 20.2.4 () tentacle (stable)`, to the release and its code name.", "id": 1036, "links": [], "title": "Ceph Version", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "fixedColor": "#EE3377", "mode": "shades" }, "noValue": "Ceph has not reported this. The gateway and disk counters need rook-ceph-exporter scraped as well as the manager." }, "overrides": [] }, "options": { "colorMode": "none", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [], "fields": "/^ceph_version$/", "values": false }, "showPercentChange": false, "text": { "valueSize": 20 }, "textMode": "value", "wideLayout": true } }, "version": "" } } }, "compaction-busy": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "(\n(\nsum by (app, cluster_environmentd_materialize_cloud_cluster_id) (\n rate(mz_persist_compaction_seconds{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval])\n)\n\n)\n* on (cluster_environmentd_materialize_cloud_cluster_id) group_left(cluster_name)\nmax by (cluster_environmentd_materialize_cloud_cluster_id, cluster_name) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"cluster_environmentd_materialize_cloud_cluster_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\nsum by (app, cluster_environmentd_materialize_cloud_cluster_id) (\n rate(mz_persist_compaction_seconds{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval])\n)\n\n)\nunless on (cluster_environmentd_materialize_cloud_cluster_id) (\ngroup by (cluster_environmentd_materialize_cloud_cluster_id) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"cluster_environmentd_materialize_cloud_cluster_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n),\n\"cluster_name\", \"$1\", \"cluster_environmentd_materialize_cloud_cluster_id\", \"(.*)\"\n)", "legendFormat": "{{app}} {{cluster_name}}", "qryType": 1, "query": "(\n(\nsum by (app, cluster_environmentd_materialize_cloud_cluster_id) (\n rate(mz_persist_compaction_seconds{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval])\n)\n\n)\n* on (cluster_environmentd_materialize_cloud_cluster_id) group_left(cluster_name)\nmax by (cluster_environmentd_materialize_cloud_cluster_id, cluster_name) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"cluster_environmentd_materialize_cloud_cluster_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\nsum by (app, cluster_environmentd_materialize_cloud_cluster_id) (\n rate(mz_persist_compaction_seconds{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval])\n)\n\n)\nunless on (cluster_environmentd_materialize_cloud_cluster_id) (\ngroup by (cluster_environmentd_materialize_cloud_cluster_id) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"cluster_environmentd_materialize_cloud_cluster_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n),\n\"cluster_name\", \"$1\", \"cluster_environmentd_materialize_cloud_cluster_id\", \"(.*)\"\n)" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Compactions running at once, on average, by process.**\n\n**Nominal:** Well under one. Compaction is background work and usually idle.\n\n**Degraded:** A process pinned at its limit is compacting continuously, which competes with its real work for CPU and object storage bandwidth.\n\nSeconds spent compacting per second, which is the average number in flight.", "id": 1022, "links": [], "title": "Compactions Running by Process", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "No metrics from this environment. Check the environment picker, and that Materialize is running.", "unit": "short" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "compaction-outcomes": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(rate(mz_persist_compaction_requested{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval]))\n", "legendFormat": "requested", "qryType": 1, "query": "sum(rate(mz_persist_compaction_requested{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval]))\n" }, "version": "v0" }, "refId": "query-0" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(rate(mz_persist_compaction_started{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval]))\n", "legendFormat": "started", "qryType": 1, "query": "sum(rate(mz_persist_compaction_started{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval]))\n" }, "version": "v0" }, "refId": "query-1" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(rate(mz_persist_compaction_applied{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval]))\n", "legendFormat": "applied", "qryType": 1, "query": "sum(rate(mz_persist_compaction_applied{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval]))\n" }, "version": "v0" }, "refId": "query-2" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(rate(mz_persist_compaction_failed{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval]))\n", "legendFormat": "failed", "qryType": 1, "query": "sum(rate(mz_persist_compaction_failed{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval]))\n" }, "version": "v0" }, "refId": "query-3" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(rate(mz_persist_compaction_timed_out{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval]))\n", "legendFormat": "timed out", "qryType": 1, "query": "sum(rate(mz_persist_compaction_timed_out{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval]))\n" }, "version": "v0" }, "refId": "query-4" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(rate(mz_persist_compaction_dropped{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval]))\n", "legendFormat": "dropped", "qryType": 1, "query": "sum(rate(mz_persist_compaction_dropped{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval]))\n" }, "version": "v0" }, "refId": "query-5" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**What happened to compaction requests, per second.**\n\n**Nominal:** Requested, started and applied on top of each other; failed, timed out and dropped at zero.\n\n**Degraded:** Dropped means the queue was full and work was abandoned — compaction is not keeping up with writes. Applied falling behind started means compactions are finishing too late to be used, usually because the data changed underneath them.\n\nCompaction merges the small data files that every write produces into larger ones, which keeps reads fast and lets the replaced files be deleted.", "id": 1020, "links": [], "title": "Compaction Outcomes", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "No metrics from this environment. Check the environment picker, and that Materialize is running.", "unit": "ops" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "compaction-queue-wait": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "(\n sum(rate(mz_persist_compaction_queued_seconds{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval]))\n /\n sum(rate(mz_persist_compaction_started{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval]))\n) >= 0\n", "legendFormat": "waiting", "qryType": 1, "query": "(\n sum(rate(mz_persist_compaction_queued_seconds{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval]))\n /\n sum(rate(mz_persist_compaction_started{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval]))\n) >= 0\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Average time a compaction request waits in the queue before it starts.**\n\n**Nominal:** Effectively zero.\n\n**Degraded:** Rising wait is compaction running out of capacity. Reads slow as uncompacted files accumulate, and dropped requests on _Compaction Outcomes_ follow if it keeps up.", "id": 1021, "links": [], "title": "Queue Wait (mean)", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "No metrics from this environment. Check the environment picker, and that Materialize is running.", "unit": "s" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "object-store-none-detected": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [], "queryOptions": {}, "transformations": [] } }, "description": "Why this tab has no object store rows on it.", "id": 1047, "links": [], "title": "No Object Store Metrics", "vizConfig": { "group": "text", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": {}, "overrides": [] }, "options": { "code": { "language": "plaintext", "showLineNumbers": false, "showMiniMap": false }, "content": "**The object store is not reporting its own metrics.**\n\nThis tab shows what an object store running on disks you own says about itself: its health, its free space, its disks, and the server side of every request persist makes. It renders for **Ceph run by Rook**, whose S3-compatible gateway (RGW) can hold the persist bucket.\n\n- **On Rook Ceph**, scrape the manager and the per-node exporter. Setting `monitoring.enabled: true` on the `CephCluster` makes Rook create a `ServiceMonitor` for each; `deploy/examples/monitoring/` in the Rook repository has both as standalone manifests, which leave out the alert rules that setting also installs.\n- **On S3, GCS or Azure Blob**, the bucket cannot fill up or lose a disk, and what the provider reports about it is on the **Infrastructure Cloud Provider** dashboard.\n\nThe other tabs here need none of this: they are persist's own measurement of the store, and work on every one.", "mode": "markdown" } }, "version": "" } } }, "ops-by-type": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (op) (\n rate(mz_persist_external_started_count{materialize_cloud_organization_name=~\"$environmentNameList\", op=~\"blob_.*\"}[$__rate_interval])\n)\n", "legendFormat": "{{op}}", "qryType": 1, "query": "sum by (op) (\n rate(mz_persist_external_started_count{materialize_cloud_organization_name=~\"$environmentNameList\", op=~\"blob_.*\"}[$__rate_interval])\n)\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Object storage calls per second, by operation.**\n\n**Nominal:** Writes and deletes roughly equal — compaction deletes about as many parts as are written — and reads depending on how much hydration and querying is going on.\n\n**Degraded:** Deletes falling well behind writes for a long time means storage is growing faster than the data is.", "id": 1012, "links": [], "title": "Operations by Type", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "No metrics from this environment. Check the environment picker, and that Materialize is running.", "unit": "ops" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "ops-cache": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "(\n sum(rate(mz_persist_blob_cache_hits_bytes{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval]))\n /\n (\n sum(rate(mz_persist_blob_cache_hits_bytes{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval]))\n + sum(rate(mz_persist_external_bytes_count{materialize_cloud_organization_name=~\"$environmentNameList\", op=\"blob_get\"}[$__rate_interval]))\n )\n) >= 0\n", "legendFormat": "from cache", "qryType": 1, "query": "(\n sum(rate(mz_persist_blob_cache_hits_bytes{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval]))\n /\n (\n sum(rate(mz_persist_blob_cache_hits_bytes{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval]))\n + sum(rate(mz_persist_external_bytes_count{materialize_cloud_organization_name=~\"$environmentNameList\", op=\"blob_get\"}[$__rate_interval]))\n )\n) >= 0\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Share of bytes read that the in-memory cache served, rather than object storage.**\n\n**Nominal:** Most of them. Recently written data is read back soon after, and the cache catches it.\n\n**Degraded:** A falling share during a steady workload means the cache is too small for the working set, so more reads go to the store — more latency and more billed requests.\n\nEmpty for a window in which nothing was read at all.", "id": 1018, "links": [], "title": "Reads Served from Cache", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "fixedColor": "#33BBEE", "mode": "shades" }, "max": 1, "min": 0, "noValue": "Nothing was read in this window, from the cache or from the store.", "unit": "percentunit" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "ops-hedging": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(rate(mz_persist_blob_hedges_fired{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval]))\n", "legendFormat": "sent", "qryType": 1, "query": "sum(rate(mz_persist_blob_hedges_fired{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval]))\n" }, "version": "v0" }, "refId": "query-0" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(rate(mz_persist_blob_hedges_won{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval]))\n", "legendFormat": "won", "qryType": 1, "query": "sum(rate(mz_persist_blob_hedges_won{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval]))\n" }, "version": "v0" }, "refId": "query-1" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Reads that were slow enough to send a second, duplicate request, and how many of those the duplicate won, per second.**\n\n**Nominal:** Zero or close to it, and only when hedged reads are enabled.\n\n**Degraded:** Hedges firing constantly means the store's tail latency is regularly past the hedge threshold. Hedges winning most of the time means the tail is individual slow requests, which hedging is working around — at the cost of extra billed requests.", "id": 1019, "links": [], "title": "Hedged Reads", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "No hedging metrics. Hedged reads were added in Materialize v26.41.", "unit": "ops" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "ops-mean-latency": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "(\n sum by (op) (\n rate(mz_persist_external_seconds{materialize_cloud_organization_name=~\"$environmentNameList\", op=~\"blob_.*\"}[$__rate_interval])\n )\n /\n sum by (op) (\n rate(mz_persist_external_started_count{materialize_cloud_organization_name=~\"$environmentNameList\", op=~\"blob_.*\"}[$__rate_interval])\n )\n) >= 0\n", "legendFormat": "{{op}}", "qryType": 1, "query": "(\n sum by (op) (\n rate(mz_persist_external_seconds{materialize_cloud_organization_name=~\"$environmentNameList\", op=~\"blob_.*\"}[$__rate_interval])\n )\n /\n sum by (op) (\n rate(mz_persist_external_started_count{materialize_cloud_organization_name=~\"$environmentNameList\", op=~\"blob_.*\"}[$__rate_interval])\n )\n) >= 0\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Average time per object storage call, by operation.**\n\n**Nominal:** Tens of milliseconds for reads, writes and deletes.\n\n**Degraded:** Deletes slowing on their own is a store under pressure from cleanup; a listing appearing at all is a process scanning the bucket, which happens at startup and is slow on large buckets.\n\nA mean hides the tail; reads and writes have real percentiles on _Read and Write Latency_.", "id": 1015, "links": [], "title": "Mean Latency by Operation", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "No metrics from this environment. Check the environment picker, and that Materialize is running.", "unit": "s" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "ops-read-latency-by-process": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "(\n(\nhistogram_quantile(0.99,\n sum by (le, app, cluster_environmentd_materialize_cloud_cluster_id) (\n rate(mz_persist_external_op_latency_bucket{materialize_cloud_organization_name=~\"$environmentNameList\", op=\"blob_get\"}[$__rate_interval])\n )\n)\n\n)\n* on (cluster_environmentd_materialize_cloud_cluster_id) group_left(cluster_name)\nmax by (cluster_environmentd_materialize_cloud_cluster_id, cluster_name) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"cluster_environmentd_materialize_cloud_cluster_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\nhistogram_quantile(0.99,\n sum by (le, app, cluster_environmentd_materialize_cloud_cluster_id) (\n rate(mz_persist_external_op_latency_bucket{materialize_cloud_organization_name=~\"$environmentNameList\", op=\"blob_get\"}[$__rate_interval])\n )\n)\n\n)\nunless on (cluster_environmentd_materialize_cloud_cluster_id) (\ngroup by (cluster_environmentd_materialize_cloud_cluster_id) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"cluster_environmentd_materialize_cloud_cluster_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n),\n\"cluster_name\", \"$1\", \"cluster_environmentd_materialize_cloud_cluster_id\", \"(.*)\"\n)", "legendFormat": "{{app}} {{cluster_name}}", "qryType": 1, "query": "(\n(\nhistogram_quantile(0.99,\n sum by (le, app, cluster_environmentd_materialize_cloud_cluster_id) (\n rate(mz_persist_external_op_latency_bucket{materialize_cloud_organization_name=~\"$environmentNameList\", op=\"blob_get\"}[$__rate_interval])\n )\n)\n\n)\n* on (cluster_environmentd_materialize_cloud_cluster_id) group_left(cluster_name)\nmax by (cluster_environmentd_materialize_cloud_cluster_id, cluster_name) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"cluster_environmentd_materialize_cloud_cluster_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\nhistogram_quantile(0.99,\n sum by (le, app, cluster_environmentd_materialize_cloud_cluster_id) (\n rate(mz_persist_external_op_latency_bucket{materialize_cloud_organization_name=~\"$environmentNameList\", op=\"blob_get\"}[$__rate_interval])\n )\n)\n\n)\nunless on (cluster_environmentd_materialize_cloud_cluster_id) (\ngroup by (cluster_environmentd_materialize_cloud_cluster_id) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"cluster_environmentd_materialize_cloud_cluster_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n),\n\"cluster_name\", \"$1\", \"cluster_environmentd_materialize_cloud_cluster_id\", \"(.*)\"\n)" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Slowest 1% of reads from object storage, by the process making them.**\n\n**Nominal:** Every process close to the others.\n\n**Degraded:** One process well above the rest is local to it — its network path or its node. All of them rising together is the store.", "id": 1016, "links": [], "title": "Read Latency by Process (p99)", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "No metrics from this environment. Check the environment picker, and that Materialize is running.", "unit": "s" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "ops-requests": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (op) (\n rate(mz_persist_s3_operations{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval])\n)\n and on () (sum(rate(mz_persist_s3_operations{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval])) > 0)\n", "legendFormat": "{{op}}", "qryType": 1, "query": "sum by (op) (\n rate(mz_persist_s3_operations{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval])\n)\n and on () (sum(rate(mz_persist_s3_operations{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval])) > 0)\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Requests sent to the object store's API per second, by call — the count a cloud provider bills.**\n\n**Nominal:** Close to _Operations by Type_: most operations are one request, and a delete is two (a check, then the delete).\n\n**Degraded:** Multipart uploads (`set_multi_*`) appear for large parts. A sustained rise without more data arriving is worth tracing to the operation driving it, since each request is billed.\n\nS3, GCS and S3-compatible stores only.", "id": 1014, "links": [], "title": "Requests to the Object Store", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "No calls through the S3 client in this window. Azure Blob does not use it; S3, GCS and S3-compatible stores do.", "unit": "reqps" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "ops-retries": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (op) (\n rate(mz_persist_retry_retries_count{materialize_cloud_organization_name=~\"$environmentNameList\", op=~\"batch::set|batch::delete|fetch_batch::get|rollup::get|rollup::set|rollup::delete|hollow_run::get|hollow_run::set|blob::open|compaction_noop::delete|storage_usage::shard_size\"}[$__rate_interval])\n)\n", "legendFormat": "{{op}}", "qryType": 1, "query": "sum by (op) (\n rate(mz_persist_retry_retries_count{materialize_cloud_organization_name=~\"$environmentNameList\", op=~\"batch::set|batch::delete|fetch_batch::get|rollup::get|rollup::set|rollup::delete|hollow_run::get|hollow_run::set|blob::open|compaction_noop::delete|storage_usage::shard_size\"}[$__rate_interval])\n)\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Object storage calls retried after a failure, per second, by the work that was retrying.**\n\n**Nominal:** Nothing drawn above zero.\n\n**Degraded:** Retries on writes (`batch::set`) slow ingestion; retries on reads (`fetch_batch::get`) slow hydration and queries; retries on deletes slow cleanup only.\n\nExcludes persist's polling loops, which are named like retries but are waiting for new data rather than recovering from an error.", "id": 1017, "links": [], "title": "Retried Operations", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "No metrics from this environment. Check the environment picker, and that Materialize is running.", "unit": "ops" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "ops-throughput": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (op) (\n rate(mz_persist_external_bytes_count{materialize_cloud_organization_name=~\"$environmentNameList\", op=~\"blob_.*\"}[$__rate_interval])\n)\n", "legendFormat": "{{op}}", "qryType": 1, "query": "sum by (op) (\n rate(mz_persist_external_bytes_count{materialize_cloud_organization_name=~\"$environmentNameList\", op=~\"blob_.*\"}[$__rate_interval])\n)\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Bytes per second read from and written to object storage, by operation.**\n\n**Nominal:** Writes track ingestion. Reads spike on hydration — a replica restarting reads back everything it maintains — and are otherwise low.\n\nReads served from the in-memory cache are not counted here; see _Reads Served from Cache_.", "id": 1013, "links": [], "title": "Throughput by Operation", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "No metrics from this environment. Check the environment picker, and that Materialize is running.", "unit": "Bps" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "overview-compaction-failures": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(rate(mz_persist_compaction_failed{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval]))\n+ sum(rate(mz_persist_compaction_timed_out{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval]))\n", "legendFormat": "failed/s", "qryType": 1, "query": "sum(rate(mz_persist_compaction_failed{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval]))\n+ sum(rate(mz_persist_compaction_timed_out{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval]))\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Compactions that failed or timed out, per second.**\n\n**Nominal:** Zero.\n\n**Degraded:** Compaction merges small data files into large ones and deletes what they replaced. When it fails, reads get slower as the files pile up, and storage grows because nothing is deleted. Failures usually follow object storage or metadata database errors, so check those first.", "id": 1004, "links": [], "title": "Compaction Failures", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "No metrics from this environment. Check the environment picker, and that Materialize is running.", "thresholds": { "mode": "absolute", "steps": [ { "color": "#A2F49B", "value": null }, { "color": "#FEDA8B", "value": 0.01 }, { "color": "#FDB366", "value": 0.2575 }, { "color": "#F67E4B", "value": 0.505 }, { "color": "#DD3D2D", "value": 0.7525 }, { "color": "#A50026", "value": 1 } ] }, "unit": "ops" }, "overrides": [] }, "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [], "values": false }, "showPercentChange": false, "textMode": "value", "wideLayout": true } }, "version": "" } } }, "overview-failed-ops": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(rate(mz_persist_external_failed_count{materialize_cloud_organization_name=~\"$environmentNameList\", op=~\"blob_.*\"}[$__rate_interval]))\n", "legendFormat": "failed/s", "qryType": 1, "query": "sum(rate(mz_persist_external_failed_count{materialize_cloud_organization_name=~\"$environmentNameList\", op=~\"blob_.*\"}[$__rate_interval]))\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Object storage calls that failed, per second, across every Materialize process in this environment.**\n\n**Nominal:** Zero. An occasional failure that a retry absorbs is harmless; a sustained rate is not.\n\n**Degraded:** Failed writes stall ingestion and materialized views; failed reads stall hydration and any query that has to fetch data. Credentials, bucket permissions and throttling are the usual causes, and _Retrying Object Storage Operations_ below has the store's own error text.", "id": 1000, "links": [], "title": "Failed Operations", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "No metrics from this environment. Check the environment picker, and that Materialize is running.", "thresholds": { "mode": "absolute", "steps": [ { "color": "#A2F49B", "value": null }, { "color": "#FEDA8B", "value": 0.01 }, { "color": "#FDB366", "value": 0.2575 }, { "color": "#F67E4B", "value": 0.505 }, { "color": "#DD3D2D", "value": 0.7525 }, { "color": "#A50026", "value": 1 } ] }, "unit": "ops" }, "overrides": [] }, "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [], "values": false }, "showPercentChange": false, "textMode": "value", "wideLayout": true } }, "version": "" } } }, "overview-failures-by-op": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (op) (\n rate(mz_persist_external_failed_count{materialize_cloud_organization_name=~\"$environmentNameList\", op=~\"blob_.*\"}[$__rate_interval])\n)\n", "legendFormat": "{{op}}", "qryType": 1, "query": "sum by (op) (\n rate(mz_persist_external_failed_count{materialize_cloud_organization_name=~\"$environmentNameList\", op=~\"blob_.*\"}[$__rate_interval])\n)\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Failed object storage calls per second, by operation.**\n\n**Nominal:** Nothing drawn above zero.\n\n**Degraded:** `blob_set` failing is writes failing. `blob_get` failing blocks hydration and queries that fetch data. `blob_delete` failing alone is cleanup falling behind, which costs storage rather than availability.", "id": 1008, "links": [], "title": "Failed Operations by Type", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "No metrics from this environment. Check the environment picker, and that Materialize is running.", "unit": "ops" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "overview-read-p99": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "histogram_quantile(0.99,\n sum by (le) (\n rate(mz_persist_external_op_latency_bucket{materialize_cloud_organization_name=~\"$environmentNameList\", op=\"blob_get\"}[$__rate_interval])\n )\n)\n", "legendFormat": "p99", "qryType": 1, "query": "histogram_quantile(0.99,\n sum by (le) (\n rate(mz_persist_external_op_latency_bucket{materialize_cloud_organization_name=~\"$environmentNameList\", op=\"blob_get\"}[$__rate_interval])\n )\n)\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**How long the slowest 1% of reads from object storage take.**\n\n**Nominal:** Around 100 ms on S3 and GCS, less on Azure Premium block blobs.\n\n**Degraded:** Reads are what hydration and cold queries wait on. A sustained rise is the store, or the network path to it; compare the bucket's own latency on the Infrastructure Cloud Provider dashboard where the provider publishes one.", "id": 1001, "links": [], "title": "Read Latency (p99)", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "No metrics from this environment. Check the environment picker, and that Materialize is running.", "thresholds": { "mode": "absolute", "steps": [ { "color": "#A2F49B", "value": null }, { "color": "#F19903", "value": 0.5 }, { "color": "#AB0003", "value": 2 } ] }, "unit": "s" }, "overrides": [] }, "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [], "values": false }, "showPercentChange": false, "textMode": "value", "wideLayout": true } }, "version": "" } } }, "overview-read-write-latency": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "histogram_quantile(0.50,\n sum by (le) (\n rate(mz_persist_external_op_latency_bucket{materialize_cloud_organization_name=~\"$environmentNameList\", op=\"blob_get\"}[$__rate_interval])\n )\n)\n", "legendFormat": "read p50", "qryType": 1, "query": "histogram_quantile(0.50,\n sum by (le) (\n rate(mz_persist_external_op_latency_bucket{materialize_cloud_organization_name=~\"$environmentNameList\", op=\"blob_get\"}[$__rate_interval])\n )\n)\n" }, "version": "v0" }, "refId": "query-0" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "histogram_quantile(0.99,\n sum by (le) (\n rate(mz_persist_external_op_latency_bucket{materialize_cloud_organization_name=~\"$environmentNameList\", op=\"blob_get\"}[$__rate_interval])\n )\n)\n", "legendFormat": "read p99", "qryType": 1, "query": "histogram_quantile(0.99,\n sum by (le) (\n rate(mz_persist_external_op_latency_bucket{materialize_cloud_organization_name=~\"$environmentNameList\", op=\"blob_get\"}[$__rate_interval])\n )\n)\n" }, "version": "v0" }, "refId": "query-1" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "histogram_quantile(0.50,\n sum by (le) (\n rate(mz_persist_external_op_latency_bucket{materialize_cloud_organization_name=~\"$environmentNameList\", op=\"blob_set\"}[$__rate_interval])\n )\n)\n", "legendFormat": "write p50", "qryType": 1, "query": "histogram_quantile(0.50,\n sum by (le) (\n rate(mz_persist_external_op_latency_bucket{materialize_cloud_organization_name=~\"$environmentNameList\", op=\"blob_set\"}[$__rate_interval])\n )\n)\n" }, "version": "v0" }, "refId": "query-2" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "histogram_quantile(0.99,\n sum by (le) (\n rate(mz_persist_external_op_latency_bucket{materialize_cloud_organization_name=~\"$environmentNameList\", op=\"blob_set\"}[$__rate_interval])\n )\n)\n", "legendFormat": "write p99", "qryType": 1, "query": "histogram_quantile(0.99,\n sum by (le) (\n rate(mz_persist_external_op_latency_bucket{materialize_cloud_organization_name=~\"$environmentNameList\", op=\"blob_set\"}[$__rate_interval])\n )\n)\n" }, "version": "v0" }, "refId": "query-3" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Read and write latency to object storage: the median and the slowest 1% of each.**\n\n**Nominal:** Flat, with medians in the tens of milliseconds.\n\n**Degraded:** Medians rising is the store, or the network to it, getting slower for everyone. The p99 rising alone is a tail — large parts, throttling, or a few slow requests that retries hide.\n\nObject storage latency depends on object size, so a workload writing larger batches reads higher here without anything being wrong.", "id": 1006, "links": [], "title": "Read and Write Latency", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "No metrics from this environment. Check the environment picker, and that Materialize is running.", "unit": "s" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "overview-retry-logs": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${logsDatasource}" }, "group": "loki", "kind": "DataQuery", "spec": { "expr": "{namespace=~\"$mzNamespaceList\"}\n |= \"external operation\"\n |~ \"external operation (batch::set|batch::delete|fetch_batch::get|rollup::get|rollup::set|rollup::delete|hollow_run::get|hollow_run::set|blob::open|compaction_noop::delete|storage_usage::shard_size) \"\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**What Materialize logged while retrying calls to object storage, with the store's own error text.**\n\n**Nominal:** Empty. A healthy store needs no retries worth logging.\n\n**Degraded:** The error text is the diagnosis: an access-denied or signature error is credentials or bucket policy, a missing bucket is the persist backend URL, throttling is request rate, and a DNS or connection error is the network path from the cluster.\n\nA retry is logged once it has failed a few times, so a single blip does not appear here. If the environment picker is empty, Materialize never started far enough to report metrics; its logs are still on Materialize Logs and Events under _Logs -> All Logs_.", "id": 1011, "links": [], "title": "Retrying Object Storage Operations", "vizConfig": { "group": "logs", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "noValue": "No retried object storage calls were logged in this time range." }, "overrides": [] }, "options": { "dedupStrategy": "exact", "detailsMode": "inline", "displayedFields": [], "enableInfiniteScrolling": true, "enableLogDetails": true, "fontSize": "small", "prettifyLogMessage": false, "showCommonLabels": false, "showControls": true, "showFieldSelector": true, "showLabels": false, "showLevel": true, "showLogContextToggle": false, "showTime": true, "sortOrder": "Descending", "syntaxHighlighting": true, "timestampResolution": "ms", "unwrappedColumns": false, "wrapLogMessage": true } }, "version": "" } } }, "overview-round-trip": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "(\n(\nmax by (app, cluster_environmentd_materialize_cloud_cluster_id) (\n mz_persist_external_rtt_latency{materialize_cloud_organization_name=~\"$environmentNameList\", external=\"blob\"}\n)\n\n)\n* on (cluster_environmentd_materialize_cloud_cluster_id) group_left(cluster_name)\nmax by (cluster_environmentd_materialize_cloud_cluster_id, cluster_name) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"cluster_environmentd_materialize_cloud_cluster_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\nmax by (app, cluster_environmentd_materialize_cloud_cluster_id) (\n mz_persist_external_rtt_latency{materialize_cloud_organization_name=~\"$environmentNameList\", external=\"blob\"}\n)\n\n)\nunless on (cluster_environmentd_materialize_cloud_cluster_id) (\ngroup by (cluster_environmentd_materialize_cloud_cluster_id) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"cluster_environmentd_materialize_cloud_cluster_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n),\n\"cluster_name\", \"$1\", \"cluster_environmentd_materialize_cloud_cluster_id\", \"(.*)\"\n)", "legendFormat": "{{app}} {{cluster_name}}", "qryType": 1, "query": "(\n(\nmax by (app, cluster_environmentd_materialize_cloud_cluster_id) (\n mz_persist_external_rtt_latency{materialize_cloud_organization_name=~\"$environmentNameList\", external=\"blob\"}\n)\n\n)\n* on (cluster_environmentd_materialize_cloud_cluster_id) group_left(cluster_name)\nmax by (cluster_environmentd_materialize_cloud_cluster_id, cluster_name) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"cluster_environmentd_materialize_cloud_cluster_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\nmax by (app, cluster_environmentd_materialize_cloud_cluster_id) (\n mz_persist_external_rtt_latency{materialize_cloud_organization_name=~\"$environmentNameList\", external=\"blob\"}\n)\n\n)\nunless on (cluster_environmentd_materialize_cloud_cluster_id) (\ngroup by (cluster_environmentd_materialize_cloud_cluster_id) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"cluster_environmentd_materialize_cloud_cluster_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n),\n\"cluster_name\", \"$1\", \"cluster_environmentd_materialize_cloud_cluster_id\", \"(.*)\"\n)" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**The round trip from each Materialize process to object storage, as that process last measured it.**\n\n**Nominal:** Tens of milliseconds, and about the same for every process.\n\n**Degraded:** One process well above the rest is that process's network path — its node, its zone, or a NAT gateway. Every process high at once is the store, or a bucket in a different region from the cluster.", "id": 1007, "links": [], "title": "Round-Trip Time by Process", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "No metrics from this environment. Check the environment picker, and that Materialize is running.", "unit": "s" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "overview-store-errors": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (op, code) (\n rate(mz_persist_s3_errors{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval])\n)\n", "legendFormat": "{{op}} {{code}}", "qryType": 1, "query": "sum by (op, code) (\n rate(mz_persist_s3_errors{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval])\n)\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Errors the object store returned, per second, by API call and error code.**\n\n**Nominal:** A trickle of `NoSuchKey` on `DeleteObject`, which is a delete of something already deleted and is harmless.\n\n**Degraded:** `AccessDenied` or `Forbidden` is credentials or bucket policy. `SlowDown` or `503` is the store throttling this bucket's request rate. `NoSuchBucket` is the persist backend URL naming the wrong bucket.\n\nS3, GCS and S3-compatible stores only. Azure Blob is reached through Azure's own client, which reports no error codes here; its failures still appear in _Failed Operations by Type_.", "id": 1009, "links": [], "title": "Object Store Errors by Code", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "The object store has returned no errors. Always empty on Azure Blob, which reports no error codes.", "unit": "ops" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "overview-stored": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(max by (shard) (mz_persist_shard_usage_current_state_batches_bytes{materialize_cloud_organization_name=~\"$environmentNameList\"}))\n+ sum(max by (shard) (mz_persist_shard_usage_current_state_rollups_bytes{materialize_cloud_organization_name=~\"$environmentNameList\"}))\n", "legendFormat": "stored", "qryType": 1, "query": "sum(max by (shard) (mz_persist_shard_usage_current_state_batches_bytes{materialize_cloud_organization_name=~\"$environmentNameList\"}))\n+ sum(max by (shard) (mz_persist_shard_usage_current_state_rollups_bytes{materialize_cloud_organization_name=~\"$environmentNameList\"}))\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Data this environment's current state refers to in object storage.**\n\n**Nominal:** Grows with the data you keep, and falls when you drop objects or compaction consolidates updates.\n\nThis is what Materialize needs, not what the bucket holds. The bucket also holds data kept only for readers, uploads in flight, orphaned data, and whatever the provider retains for versioning or soft delete — _Storage_ breaks down the first three, and the Infrastructure Cloud Provider dashboard has the bucket's own size.", "id": 1005, "links": [], "title": "Data Stored", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "fixedColor": "#0077BB", "mode": "shades" }, "min": 0, "noValue": "No metrics from this environment. Check the environment picker, and that Materialize is running.", "unit": "bytes" }, "overrides": [] }, "options": { "colorMode": "none", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [], "values": false }, "showPercentChange": false, "textMode": "value", "wideLayout": true } }, "version": "" } } }, "overview-timeouts": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(rate(mz_persist_s3_connect_timeouts{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval]))\n and on () (sum(rate(mz_persist_s3_operations{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval])) > 0)\n", "legendFormat": "connecting", "qryType": 1, "query": "sum(rate(mz_persist_s3_connect_timeouts{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval]))\n and on () (sum(rate(mz_persist_s3_operations{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval])) > 0)\n" }, "version": "v0" }, "refId": "query-0" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(rate(mz_persist_s3_read_timeouts{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval]))\n and on () (sum(rate(mz_persist_s3_operations{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval])) > 0)\n", "legendFormat": "waiting for first byte", "qryType": 1, "query": "sum(rate(mz_persist_s3_read_timeouts{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval]))\n and on () (sum(rate(mz_persist_s3_operations{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval])) > 0)\n" }, "version": "v0" }, "refId": "query-1" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(rate(mz_persist_s3_operation_attempt_timeouts{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval]))\n and on () (sum(rate(mz_persist_s3_operations{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval])) > 0)\n", "legendFormat": "one attempt", "qryType": 1, "query": "sum(rate(mz_persist_s3_operation_attempt_timeouts{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval]))\n and on () (sum(rate(mz_persist_s3_operations{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval])) > 0)\n" }, "version": "v0" }, "refId": "query-2" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(rate(mz_persist_s3_operation_timeouts{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval]))\n and on () (sum(rate(mz_persist_s3_operations{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval])) > 0)\n", "legendFormat": "whole operation", "qryType": 1, "query": "sum(rate(mz_persist_s3_operation_timeouts{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval]))\n and on () (sum(rate(mz_persist_s3_operations{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval])) > 0)\n" }, "version": "v0" }, "refId": "query-3" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Object storage requests that timed out, per second, by where they timed out.**\n\n**Nominal:** Nothing drawn above zero.\n\n**Degraded:** Connection timeouts are the network path to the store. Timeouts waiting for the first byte, or for a whole attempt, are the store being slow to answer. Operation timeouts are the worst: every retry within the operation timed out.\n\nS3, GCS and S3-compatible stores only.", "id": 1010, "links": [], "title": "Timeouts", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "No calls through the S3 client in this window. Azure Blob does not use it; S3, GCS and S3-compatible stores do.", "unit": "ops" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "overview-write-p99": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "histogram_quantile(0.99,\n sum by (le) (\n rate(mz_persist_external_op_latency_bucket{materialize_cloud_organization_name=~\"$environmentNameList\", op=\"blob_set\"}[$__rate_interval])\n )\n)\n", "legendFormat": "p99", "qryType": 1, "query": "histogram_quantile(0.99,\n sum by (le) (\n rate(mz_persist_external_op_latency_bucket{materialize_cloud_organization_name=~\"$environmentNameList\", op=\"blob_set\"}[$__rate_interval])\n )\n)\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**How long the slowest 1% of writes to object storage take.**\n\n**Nominal:** Around 100 ms on S3 and GCS, less on Azure Premium block blobs.\n\n**Degraded:** Every write — ingestion, table inserts, materialized view updates — uploads its data before it commits, so slow writes are slow freshness. A rise with _Write Stalls_ is writes queueing behind each other.", "id": 1002, "links": [], "title": "Write Latency (p99)", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "No metrics from this environment. Check the environment picker, and that Materialize is running.", "thresholds": { "mode": "absolute", "steps": [ { "color": "#A2F49B", "value": null }, { "color": "#F19903", "value": 0.5 }, { "color": "#AB0003", "value": 2 } ] }, "unit": "s" }, "overrides": [] }, "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [], "values": false }, "showPercentChange": false, "textMode": "value", "wideLayout": true } }, "version": "" } } }, "overview-write-stalls": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(rate(mz_persist_user_write_stall_count{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval]))\n+ sum(rate(mz_persist_compaction_write_stall_count{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval]))\n", "legendFormat": "stalls/s", "qryType": 1, "query": "sum(rate(mz_persist_user_write_stall_count{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval]))\n+ sum(rate(mz_persist_compaction_write_stall_count{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval]))\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Writes that had to wait because too many uploads were already in flight, per second.**\n\n**Nominal:** Zero, or close to it.\n\n**Degraded:** Uploads are not finishing as fast as data arrives, so writers are waiting on object storage. Rising write latency beside it confirms the store is the bottleneck; flat latency with stalls means more data is arriving than the configured number of concurrent uploads can carry.\n\nCounts both new data and compaction's rewrites.", "id": 1003, "links": [], "title": "Write Stalls", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "No metrics from this environment. Check the environment picker, and that Materialize is running.", "thresholds": { "mode": "absolute", "steps": [ { "color": "#A2F49B", "value": null }, { "color": "#FEDA8B", "value": 0.01 }, { "color": "#FDB366", "value": 0.2575 }, { "color": "#F67E4B", "value": 0.505 }, { "color": "#DD3D2D", "value": 0.7525 }, { "color": "#A50026", "value": 1 } ] }, "unit": "ops" }, "overrides": [] }, "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [], "values": false }, "showPercentChange": false, "textMode": "value", "wideLayout": true } }, "version": "" } } }, "storage-by-state": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(max by (shard) (mz_persist_shard_usage_current_state_batches_bytes{materialize_cloud_organization_name=~\"$environmentNameList\"}))\n", "legendFormat": "current data", "qryType": 1, "query": "sum(max by (shard) (mz_persist_shard_usage_current_state_batches_bytes{materialize_cloud_organization_name=~\"$environmentNameList\"}))\n" }, "version": "v0" }, "refId": "query-0" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(max by (shard) (mz_persist_shard_usage_current_state_rollups_bytes{materialize_cloud_organization_name=~\"$environmentNameList\"}))\n", "legendFormat": "current rollups", "qryType": 1, "query": "sum(max by (shard) (mz_persist_shard_usage_current_state_rollups_bytes{materialize_cloud_organization_name=~\"$environmentNameList\"}))\n" }, "version": "v0" }, "refId": "query-1" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(max by (shard) (mz_persist_shard_usage_referenced_not_current_state_bytes{materialize_cloud_organization_name=~\"$environmentNameList\"}))\n", "legendFormat": "held for readers", "qryType": 1, "query": "sum(max by (shard) (mz_persist_shard_usage_referenced_not_current_state_bytes{materialize_cloud_organization_name=~\"$environmentNameList\"}))\n" }, "version": "v0" }, "refId": "query-2" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(max by (shard) (mz_persist_shard_usage_not_leaked_not_referenced_bytes{materialize_cloud_organization_name=~\"$environmentNameList\"}))\n", "legendFormat": "uploads in flight", "qryType": 1, "query": "sum(max by (shard) (mz_persist_shard_usage_not_leaked_not_referenced_bytes{materialize_cloud_organization_name=~\"$environmentNameList\"}))\n" }, "version": "v0" }, "refId": "query-3" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(max by (shard) (mz_persist_shard_usage_leaked_bytes{materialize_cloud_organization_name=~\"$environmentNameList\"}))\n", "legendFormat": "orphaned", "qryType": 1, "query": "sum(max by (shard) (mz_persist_shard_usage_leaked_bytes{materialize_cloud_organization_name=~\"$environmentNameList\"}))\n" }, "version": "v0" }, "refId": "query-4" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**What this environment's data in object storage is for.**\n\n**Nominal:** Almost all of it is current data: files the latest state of each object refers to, and the rollups that summarize that state.\n\n**Degraded:** _Held for readers_ is data a reader still needs from an older state; growth there points at a stuck reader, named on _Data Held for Readers by Object_. _Uploads in flight_ is written but not yet committed. _Orphaned_ is data nothing refers to that cleanup missed, which is billed and does nothing.\n\nMaterialize's own accounting, recomputed periodically rather than on every write.", "id": 1027, "links": [], "title": "Stored Data by State", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "No metrics from this environment. Check the environment picker, and that Materialize is running.", "unit": "bytes" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "storage-current": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(max by (shard) (mz_persist_shard_usage_current_state_batches_bytes{materialize_cloud_organization_name=~\"$environmentNameList\"}))\n+ sum(max by (shard) (mz_persist_shard_usage_current_state_rollups_bytes{materialize_cloud_organization_name=~\"$environmentNameList\"}))\n", "legendFormat": "stored", "qryType": 1, "query": "sum(max by (shard) (mz_persist_shard_usage_current_state_batches_bytes{materialize_cloud_organization_name=~\"$environmentNameList\"}))\n+ sum(max by (shard) (mz_persist_shard_usage_current_state_rollups_bytes{materialize_cloud_organization_name=~\"$environmentNameList\"}))\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Data this environment's current state refers to in object storage.**\n\n**Nominal:** Grows with the data you keep, and falls when you drop objects or compaction consolidates updates.\n\nThis is what Materialize needs, not what the bucket holds. The bucket also holds data kept only for readers, uploads in flight, orphaned data, and whatever the provider retains for versioning or soft delete — _Storage_ breaks down the first three, and the Infrastructure Cloud Provider dashboard has the bucket's own size.", "id": 1025, "links": [], "title": "Data Stored", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "fixedColor": "#CCBB44", "mode": "shades" }, "min": 0, "noValue": "No metrics from this environment. Check the environment picker, and that Materialize is running.", "unit": "bytes" }, "overrides": [] }, "options": { "colorMode": "none", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [], "values": false }, "showPercentChange": false, "textMode": "value", "wideLayout": true } }, "version": "" } } }, "storage-held": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "topk(10, max by (shard) (mz_persist_shard_usage_referenced_not_current_state_bytes{materialize_cloud_organization_name=~\"$environmentNameList\"}))\n", "legendFormat": "{{shard}}", "qryType": 1, "query": "topk(10, max by (shard) (mz_persist_shard_usage_referenced_not_current_state_bytes{materialize_cloud_organization_name=~\"$environmentNameList\"}))\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**The ten objects keeping the most data only because a reader still needs an older version of it.**\n\n**Nominal:** Small and changing: readers move forward and release what they held.\n\n**Degraded:** An object whose held data only grows has a reader stuck in the past, and its storage will keep growing until that reader moves or its lease expires. The same reader holds old state in the metadata database, on Materialize Consensus under _State and Cleanup -> Versions Held by Object_.\n\nTranslate a shard id to an object with `SELECT o.name, o.type FROM mz_internal.mz_storage_shards s JOIN mz_objects o ON o.id = s.object_id WHERE s.shard_id = ''`.", "id": 1029, "links": [], "title": "Data Held for Readers by Object", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "No metrics from this environment. Check the environment picker, and that Materialize is running.", "unit": "bytes" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "storage-largest": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "topk(10,\n max by (shard) (mz_persist_shard_usage_current_state_batches_bytes{materialize_cloud_organization_name=~\"$environmentNameList\"})\n + max by (shard) (mz_persist_shard_usage_current_state_rollups_bytes{materialize_cloud_organization_name=~\"$environmentNameList\"})\n)\n", "format": "table", "instant": true, "qryType": 2, "query": "topk(10,\n max by (shard) (mz_persist_shard_usage_current_state_batches_bytes{materialize_cloud_organization_name=~\"$environmentNameList\"})\n + max by (shard) (mz_persist_shard_usage_current_state_rollups_bytes{materialize_cloud_organization_name=~\"$environmentNameList\"})\n)\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [ { "group": "organize", "kind": "Transformation", "spec": { "options": { "excludeByName": { "Time": true }, "indexByName": { "Value": 1, "shard": 0 }, "renameByName": { "Value": "stored" } } } } ] } }, "description": "**The ten objects using the most object storage, now.**\n\n**Nominal:** Your largest sources and materialized views.\n\nTranslate a shard id to an object with `SELECT o.name, o.type FROM mz_internal.mz_storage_shards s JOIN mz_objects o ON o.id = s.object_id WHERE s.shard_id = ''`, or list sizes by object with `SELECT * FROM mz_catalog.mz_recent_storage_usage ORDER BY size_bytes DESC`.", "id": 1028, "links": [], "title": "Largest Objects", "vizConfig": { "group": "table", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "custom": { "align": "auto", "filterable": true, "inspect": false }, "noValue": "No metrics from this environment. Check the environment picker, and that Materialize is running.", "unit": "bytes" }, "overrides": [] }, "options": { "cellHeight": "sm", "footer": { "countRows": false, "reducer": [], "show": false }, "frameIndex": 0, "showHeader": true, "showTypeIcons": false } }, "version": "" } } }, "storage-leaked": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(max by (shard) (mz_persist_shard_usage_leaked_bytes{materialize_cloud_organization_name=~\"$environmentNameList\"}))\n", "legendFormat": "orphaned", "qryType": 1, "query": "sum(max by (shard) (mz_persist_shard_usage_leaked_bytes{materialize_cloud_organization_name=~\"$environmentNameList\"}))\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Data in object storage that nothing refers to and that cleanup did not remove.**\n\n**Nominal:** Zero, or close to it.\n\n**Degraded:** Orphaned data is billed and never read. A little accumulates when a process is killed mid-write; a steadily growing amount is worth raising with Materialize support.", "id": 1026, "links": [], "title": "Orphaned Data", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "No metrics from this environment. Check the environment picker, and that Materialize is running.", "thresholds": { "mode": "absolute", "steps": [ { "color": "#A2F49B", "value": null }, { "color": "#FEDA8B", "value": 1000000 }, { "color": "#FDB366", "value": 250750000 }, { "color": "#F67E4B", "value": 500500000 }, { "color": "#DD3D2D", "value": 750250000 }, { "color": "#A50026", "value": 1000000000 } ] }, "unit": "bytes" }, "overrides": [] }, "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [], "values": false }, "showPercentChange": false, "textMode": "value", "wideLayout": true } }, "version": "" } } }, "writes-amplification": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(rate(mz_persist_user_bytes{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval]))\n", "legendFormat": "new data", "qryType": 1, "query": "sum(rate(mz_persist_user_bytes{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval]))\n" }, "version": "v0" }, "refId": "query-0" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(rate(mz_persist_compaction_bytes{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval]))\n", "legendFormat": "compaction rewrites", "qryType": 1, "query": "sum(rate(mz_persist_compaction_bytes{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval]))\n" }, "version": "v0" }, "refId": "query-1" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Bytes per second of new data written to object storage, beside the bytes compaction rewrites.**\n\n**Nominal:** Compaction rewriting a small multiple of what is written: every update is rewritten a few times as files are merged.\n\n**Degraded:** Compaction far above new data for a long time is a workload whose updates are being rewritten repeatedly — each one billed as storage writes.", "id": 1023, "links": [], "title": "New Data and Compaction Rewrites", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "No metrics from this environment. Check the environment picker, and that Materialize is running.", "unit": "Bps" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "writes-stalls": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "(\n(\nsum by (app, cluster_environmentd_materialize_cloud_cluster_id) (\n rate(mz_persist_user_write_stall_count{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval])\n + rate(mz_persist_compaction_write_stall_count{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval])\n)\n\n)\n* on (cluster_environmentd_materialize_cloud_cluster_id) group_left(cluster_name)\nmax by (cluster_environmentd_materialize_cloud_cluster_id, cluster_name) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"cluster_environmentd_materialize_cloud_cluster_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\nsum by (app, cluster_environmentd_materialize_cloud_cluster_id) (\n rate(mz_persist_user_write_stall_count{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval])\n + rate(mz_persist_compaction_write_stall_count{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval])\n)\n\n)\nunless on (cluster_environmentd_materialize_cloud_cluster_id) (\ngroup by (cluster_environmentd_materialize_cloud_cluster_id) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"cluster_environmentd_materialize_cloud_cluster_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n),\n\"cluster_name\", \"$1\", \"cluster_environmentd_materialize_cloud_cluster_id\", \"(.*)\"\n)", "legendFormat": "{{app}} {{cluster_name}}", "qryType": 1, "query": "(\n(\nsum by (app, cluster_environmentd_materialize_cloud_cluster_id) (\n rate(mz_persist_user_write_stall_count{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval])\n + rate(mz_persist_compaction_write_stall_count{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval])\n)\n\n)\n* on (cluster_environmentd_materialize_cloud_cluster_id) group_left(cluster_name)\nmax by (cluster_environmentd_materialize_cloud_cluster_id, cluster_name) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"cluster_environmentd_materialize_cloud_cluster_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\nsum by (app, cluster_environmentd_materialize_cloud_cluster_id) (\n rate(mz_persist_user_write_stall_count{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval])\n + rate(mz_persist_compaction_write_stall_count{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval])\n)\n\n)\nunless on (cluster_environmentd_materialize_cloud_cluster_id) (\ngroup by (cluster_environmentd_materialize_cloud_cluster_id) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"cluster_environmentd_materialize_cloud_cluster_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n),\n\"cluster_name\", \"$1\", \"cluster_environmentd_materialize_cloud_cluster_id\", \"(.*)\"\n)" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Writes that waited for in-flight uploads to finish, per second, by process.**\n\n**Nominal:** Nothing drawn above zero.\n\n**Degraded:** The process at the top is the one whose writes object storage is not keeping up with.", "id": 1024, "links": [], "title": "Write Stalls by Process", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "No metrics from this environment. Check the environment picker, and that Materialize is running.", "unit": "ops" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } } }, "layout": { "kind": "TabsLayout", "spec": { "tabs": [ { "kind": "TabsLayoutTab", "spec": { "layout": { "kind": "RowsLayout", "spec": { "rows": [ { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": true, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "narrow", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "overview-failed-ops" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "overview-read-p99" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "overview-write-p99" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "overview-write-stalls" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "overview-compaction-failures" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "overview-stored" } } } ], "maxColumnCount": 6, "rowHeightMode": "short" } }, "title": "Verdict" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "wide", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "overview-read-write-latency" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "overview-round-trip" } } } ], "maxColumnCount": 2, "rowHeightMode": "standard" } }, "title": "Latency" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "wide", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "overview-failures-by-op" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "overview-store-errors" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "overview-timeouts" } } } ], "maxColumnCount": 3, "rowHeightMode": "standard" } }, "title": "Failures" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "standard", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "overview-retry-logs" } } } ], "maxColumnCount": 1, "rowHeightMode": "tall" } }, "title": "Logs" } } ] } }, "title": "Overview" } }, { "kind": "TabsLayoutTab", "spec": { "layout": { "kind": "RowsLayout", "spec": { "rows": [ { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "wide", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "ops-by-type" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "ops-throughput" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "ops-requests" } } } ], "maxColumnCount": 3, "rowHeightMode": "standard" } }, "title": "Load" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "wide", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "ops-mean-latency" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "ops-read-latency-by-process" } } } ], "maxColumnCount": 2, "rowHeightMode": "standard" } }, "title": "Latency by Operation" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "wide", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "ops-retries" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "ops-cache" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "ops-hedging" } } } ], "maxColumnCount": 3, "rowHeightMode": "standard" } }, "title": "Retries, Cache and Hedging" } } ] } }, "title": "Operations" } }, { "kind": "TabsLayoutTab", "spec": { "layout": { "kind": "RowsLayout", "spec": { "rows": [ { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "wide", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "compaction-outcomes" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "compaction-queue-wait" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "compaction-busy" } } } ], "maxColumnCount": 3, "rowHeightMode": "standard" } }, "title": "Compaction" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "wide", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "writes-amplification" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "writes-stalls" } } } ], "maxColumnCount": 2, "rowHeightMode": "standard" } }, "title": "Writes" } } ] } }, "title": "Compaction" } }, { "kind": "TabsLayoutTab", "spec": { "layout": { "kind": "RowsLayout", "spec": { "rows": [ { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": true, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "narrow", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "storage-current" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "storage-leaked" } } } ], "maxColumnCount": 2, "rowHeightMode": "short" } }, "title": "Storage Summary" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "wide", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "storage-by-state" } } } ], "maxColumnCount": 1, "rowHeightMode": "standard" } }, "title": "Stored Data" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "wide", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "storage-largest" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "storage-held" } } } ], "maxColumnCount": 2, "rowHeightMode": "tall" } }, "title": "By Object" } } ] } }, "title": "Storage" } }, { "kind": "TabsLayoutTab", "spec": { "layout": { "kind": "RowsLayout", "spec": { "rows": [ { "kind": "RowsLayoutRow", "spec": { "collapse": false, "conditionalRendering": { "kind": "ConditionalRenderingGroup", "spec": { "condition": "and", "items": [ { "kind": "ConditionalRenderingVariable", "spec": { "operator": "matches", "value": "ceph-mgr", "variable": "cephDetected" } } ], "visibility": "show" } }, "hideHeader": false, "layout": { "kind": "RowsLayout", "spec": { "rows": [ { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": true, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "narrow", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "ceph-health" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "ceph-osds-down" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "ceph-raw-used" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "ceph-bucket-pool-available" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "ceph-objects-degraded" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "ceph-rgw-failure-ratio" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "ceph-version" } } } ], "maxColumnCount": 7, "rowHeightMode": "short" } }, "title": "Status" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "wide", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "ceph-health-checks" } } } ], "maxColumnCount": 1, "rowHeightMode": "standard" } }, "title": "Health Checks" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "wide", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "ceph-rgw-requests" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "ceph-rgw-latency" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "ceph-rgw-throughput" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "ceph-rgw-queue" } } } ], "maxColumnCount": 2, "rowHeightMode": "standard" } }, "title": "Object Gateway" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "wide", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "ceph-capacity-raw" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "ceph-capacity-pools" } } } ], "maxColumnCount": 2, "rowHeightMode": "standard" } }, "title": "Capacity" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "wide", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "ceph-osd-utilization" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "ceph-osd-latency" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "ceph-placement-groups" } } } ], "maxColumnCount": 3, "rowHeightMode": "standard" } }, "title": "Disks" } } ] } }, "title": "Ceph", "variables": [ { "kind": "QueryVariable", "spec": { "allowCustomValue": true, "current": { "text": "", "value": "" }, "definition": "label_values(ceph_health_status, namespace)", "description": "Namespace of the Rook Ceph cluster holding this environment's bucket", "hide": "dontHide", "includeAll": false, "label": "Ceph Cluster", "multi": false, "name": "cephNamespace", "options": [], "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "label_values(ceph_health_status, namespace)", "qryType": 1, "query": "label_values(ceph_health_status, namespace)" }, "version": "v0" }, "refresh": "never", "regex": "", "regexApplyTo": "value", "skipUrlSync": false, "sort": "alphabeticalAsc" } } ] } }, { "kind": "RowsLayoutRow", "spec": { "collapse": false, "conditionalRendering": { "kind": "ConditionalRenderingGroup", "spec": { "condition": "and", "items": [ { "kind": "ConditionalRenderingVariable", "spec": { "operator": "notMatches", "value": "ceph-mgr", "variable": "cephDetected" } } ], "visibility": "show" } }, "hideHeader": true, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "standard", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "object-store-none-detected" } } } ], "maxColumnCount": 1, "rowHeightMode": "tall" } }, "title": "No Object Store Metrics" } } ] } }, "title": "Object Store Internals" } } ] } }, "links": [], "liveNow": false, "preload": false, "tags": [ "materialize", "mzmon", "external-dependencies" ], "timeSettings": { "autoRefresh": "", "autoRefreshIntervals": [ "5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d" ], "fiscalYearStartMonth": 0, "from": "now-6h", "hideTimepicker": false, "timezone": "browser", "to": "now" }, "title": "Materialize Persist (Storage)", "variables": [ { "kind": "DatasourceVariable", "spec": { "allowCustomValue": false, "current": { "text": "", "value": "" }, "description": "Datasource for metrics queries", "hide": "dontHide", "includeAll": false, "label": "Metrics Datasource", "multi": false, "name": "metricsDatasource", "options": [], "pluginId": "prometheus", "refresh": "never", "regex": "", "skipUrlSync": false } }, { "kind": "DatasourceVariable", "spec": { "allowCustomValue": false, "current": { "text": "", "value": "" }, "description": "Datasource for log and event queries", "hide": "dontHide", "includeAll": false, "label": "Logs Datasource", "multi": false, "name": "logsDatasource", "options": [], "pluginId": "loki", "refresh": "never", "regex": "", "skipUrlSync": false } }, { "kind": "QueryVariable", "spec": { "allValue": ".*", "allowCustomValue": true, "current": { "text": "", "value": "" }, "definition": "label_values(mz_compute_commands_total, materialize_cloud_organization_name)", "description": "The current environment to view", "hide": "dontHide", "includeAll": false, "label": "Environment", "multi": false, "name": "environmentNameList", "options": [], "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "label_values(mz_compute_commands_total, materialize_cloud_organization_name)", "qryType": 1, "query": "label_values(mz_compute_commands_total, materialize_cloud_organization_name)" }, "version": "v0" }, "refresh": "never", "regex": "", "regexApplyTo": "value", "skipUrlSync": false, "sort": "alphabeticalAsc" } }, { "kind": "QueryVariable", "spec": { "allowCustomValue": true, "current": { "text": "", "value": "" }, "definition": "label_values(mz_compute_commands_total{ materialize_cloud_organization_name=~\"$environmentNameList\" }, materialize_cloud_organization_namespace)", "description": "The current materialize namespace where environments live", "hide": "hideVariable", "includeAll": true, "label": "Materialize Namespace", "multi": true, "name": "mzNamespaceList", "options": [], "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "label_values(mz_compute_commands_total{ materialize_cloud_organization_name=~\"$environmentNameList\" }, materialize_cloud_organization_namespace)", "qryType": 1, "query": "label_values(mz_compute_commands_total{ materialize_cloud_organization_name=~\"$environmentNameList\" }, materialize_cloud_organization_namespace)" }, "version": "v0" }, "refresh": "never", "regex": "", "regexApplyTo": "value", "skipUrlSync": true, "sort": "alphabeticalAsc" } }, { "datasource": { "name": "$metricsDatasource" }, "group": "", "kind": "AdhocVariable", "spec": { "allowCustomValue": true, "baseFilters": [ { "key": "namespace", "operator": "=~", "origin": "dashboard", "value": "$mzNamespaceList" } ], "defaultKeys": [], "description": "Adhoc filters to apply to all metrics queries", "enableGroupBy": false, "filters": [], "hide": "inControlsMenu", "label": "Advanced Metric Filter", "name": "metricAdhoc", "skipUrlSync": false } }, { "kind": "QueryVariable", "spec": { "allowCustomValue": true, "current": { "text": "", "value": "" }, "definition": "label_values(up{app=\"ceph-mgr\"}, app)", "description": "Whether a Ceph manager is being scraped; empty when none is", "hide": "hideVariable", "includeAll": false, "label": "Ceph", "multi": false, "name": "cephDetected", "options": [], "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "label_values(up{app=\"ceph-mgr\"}, app)", "qryType": 1, "query": "label_values(up{app=\"ceph-mgr\"}, app)" }, "version": "v0" }, "refresh": "never", "regex": "", "regexApplyTo": "value", "skipUrlSync": true, "sort": "alphabeticalAsc" } } ] } }