{ "apiVersion": "dashboard.grafana.app/v2", "kind": "Dashboard", "metadata": { "annotations": { "grafana.app/folder": "materialize", "monitoring.materialize.cloud/min-mz-version": "v26.24.0", "monitoring.materialize.cloud/rec-mz-version": "v26.29.0", "monitoring.materialize.cloud/sql-metric-prefix": "mz_", "monitoring.materialize.cloud/target-export": "generic" }, "name": "mz-mon-env-top" }, "spec": { "annotations": [ { "kind": "AnnotationQuery", "spec": { "builtIn": true, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "query": { "datasource": { "name": "-- Grafana --" }, "group": "grafana", "kind": "DataQuery", "spec": {}, "version": "v0" } } } ], "cursorSync": "Crosshair", "description": "Overview of a Materialize Environment.\n\nThis provides a high-level summary to catch more obvious issues\nthat may require further investigation.", "editable": true, "elements": { "active-indexes": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "(max(sum by (instance) (mz_indexes_count{materialize_cloud_organization_name=~\"$environmentNameList\"}))\n) or vector(0)", "legendFormat": "indexes", "qryType": 1, "query": "(max(sum by (instance) (mz_indexes_count{materialize_cloud_organization_name=~\"$environmentNameList\"}))\n) or vector(0)" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Indexes in the catalog. An index is an in-memory arrangement that makes reads against its relation effectively instant, in exchange for memory — so growth here is a leading indicator of cluster memory growth.**\n\nEnvironment-scoped. Hardened to read 0 on a fresh environment (the metric is absent until the first index exists).", "id": 1035, "links": [], "title": "Active Indexes", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "fixedColor": "#EE7733", "mode": "shades" }, "min": 0, "noValue": "No matches for the current filters" }, "overrides": [] }, "options": { "colorMode": "none", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [], "values": false }, "showPercentChange": false, "textMode": "value", "wideLayout": true } }, "version": "" } } }, "active-mzd-views": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max(mz_mzd_views_count{materialize_cloud_organization_name=~\"$environmentNameList\"})\n", "legendFormat": "materialized-views", "qryType": 1, "query": "max(mz_mzd_views_count{materialize_cloud_organization_name=~\"$environmentNameList\"})\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Materialized views Materialize is actively maintaining. Each one is a query whose result is kept continuously up to date, so this tracks roughly how much standing compute the environment carries.**\n\nEnvironment-scoped — the cluster/replica selectors don't narrow it. Steps on CREATE/DROP MATERIALIZED VIEW; steady otherwise.", "id": 1034, "links": [], "title": "Active Materialized Views", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "fixedColor": "#EE7733", "mode": "shades" }, "min": 0, "noValue": "No matches for the current filters" }, "overrides": [] }, "options": { "colorMode": "none", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [], "values": false }, "showPercentChange": false, "textMode": "value", "wideLayout": true } }, "version": "" } } }, "active-subscribes": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (session_type) (mz_active_subscribes{materialize_cloud_organization_name=~\"$environmentNameList\"})\n", "instant": true, "legendFormat": "{{session_type}}", "qryType": 1, "query": "sum by (session_type) (mz_active_subscribes{materialize_cloud_organization_name=~\"$environmentNameList\"})\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Live SUBSCRIBE sessions — long-running queries that stream updates to a client as data changes. A handful of `system` subscribes are Materialize's own internal probes; a persistently climbing `user` count is a classic leaked-connection signal.**\n\nEnvironment-scoped.", "id": 1037, "links": [], "title": "Active Subscribes", "vizConfig": { "group": "piechart", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "fixedColor": "#EE7733", "mode": "shades" }, "noValue": "No matches for the current filters" }, "overrides": [] }, "options": { "displayLabels": [ "name", "value" ], "legend": { "asTable": true, "calcs": [], "displayMode": "table", "isVisible": true, "placement": "right", "showLegend": true, "values": [ "value" ] }, "orientation": "auto", "pieType": "donut", "reduceOptions": { "calcs": [], "values": false }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "active-views": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max(mz_views_count{materialize_cloud_organization_name=~\"$environmentNameList\"})\n", "legendFormat": "views", "qryType": 1, "query": "max(mz_views_count{materialize_cloud_organization_name=~\"$environmentNameList\"})\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Non-materialized views — query templates evaluated on demand. They cost nothing until something reads them, so this is a catalog-shape signal rather than a load one.**\n\nEnvironment-scoped.", "id": 1036, "links": [], "title": "Active Views", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "fixedColor": "#EE7733", "mode": "shades" }, "min": 0, "noValue": "No matches for the current filters" }, "overrides": [] }, "options": { "colorMode": "none", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [], "values": false }, "showPercentChange": false, "textMode": "value", "wideLayout": true } }, "version": "" } } }, "adapter-commands-by-app": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (application_name, status) (increase(mz_adapter_commands{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__range]))\n", "instant": true, "qryType": 1, "query": "sum by (application_name, status) (increase(mz_adapter_commands{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__range]))\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [ { "group": "labelsToFields", "kind": "Transformation", "spec": { "options": { "keepLabels": [ "application_name", "status" ] } } }, { "group": "merge", "kind": "Transformation", "spec": { "options": {} } }, { "group": "groupingToMatrix", "kind": "Transformation", "spec": { "options": { "columnField": "status", "emptyValue": "zero", "rowField": "application_name", "valueField": "Value" } } }, { "group": "organize", "kind": "Transformation", "spec": { "options": { "indexByName": { "application_name\\status": 0, "error": 2, "success": 1 }, "renameByName": { "application_name\\status": "Application", "error": "Errors", "success": "Success" } } } }, { "group": "sortBy", "kind": "Transformation", "spec": { "options": { "fields": {}, "sort": [ { "desc": true, "field": "Errors" } ] } } } ] } }, "description": "**SQL control-plane command totals per client `application_name` over the window, so you can see which clients drive the adapter and which are failing. Most clients set `application_name` in their connection string; those that don't bucket as `unrecognized`/`unspecified` (normal).**\n\n**Unhealthy:** Sustained errors against a real application mean that app is consistently failing its commands — correlate with the app's own logs and the `mz_internal` activity-log views for the specific failures.\n\nThe presentation layer pivots the `status` label into Success/Errors columns. Environment-scoped.", "id": 1029, "links": [], "title": "SQL Control Plane Commands by Application", "vizConfig": { "group": "table", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "custom": { "align": "auto", "filterable": true, "inspect": false }, "noValue": "No matches for the current filters", "unit": "short" }, "overrides": [ { "matcher": { "id": "byName", "options": "Errors" }, "properties": [ { "id": "thresholds", "value": { "mode": "absolute", "steps": [ { "color": "#A2F49B", "value": null }, { "color": "#FEDA8B", "value": 1 }, { "color": "#FDB366", "value": 25.75 }, { "color": "#F67E4B", "value": 50.5 }, { "color": "#DD3D2D", "value": 75.25 }, { "color": "#A50026", "value": 100 } ] } }, { "id": "custom.cellOptions", "value": { "type": "color-background" } } ] } ] }, "options": { "cellHeight": "sm", "footer": { "countRows": false, "reducer": [], "show": false }, "frameIndex": 0, "showHeader": true, "showTypeIcons": false } }, "version": "" } } }, "arrangement-rate": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "(\n(\nsum by (cluster_environmentd_materialize_cloud_cluster_id, cluster_environmentd_materialize_cloud_replica_id) (\n max without (job) (rate(\n mz_arrangement_maintenance_seconds_total{materialize_cloud_organization_name=~\"$environmentNameList\", cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval]\n ))\n)\n\n)\n* on (cluster_environmentd_materialize_cloud_cluster_id) group_left(cluster_name)\nmax by (cluster_environmentd_materialize_cloud_cluster_id, cluster_name) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"cluster_environmentd_materialize_cloud_cluster_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\nsum by (cluster_environmentd_materialize_cloud_cluster_id, cluster_environmentd_materialize_cloud_replica_id) (\n max without (job) (rate(\n mz_arrangement_maintenance_seconds_total{materialize_cloud_organization_name=~\"$environmentNameList\", cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval]\n ))\n)\n\n)\nunless on (cluster_environmentd_materialize_cloud_cluster_id) (\ngroup by (cluster_environmentd_materialize_cloud_cluster_id) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"cluster_environmentd_materialize_cloud_cluster_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n),\n\"cluster_name\", \"$1\", \"cluster_environmentd_materialize_cloud_cluster_id\", \"(.*)\"\n)", "legendFormat": "{{cluster_name}} / {{cluster_environmentd_materialize_cloud_replica_id}}", "qryType": 1, "query": "(\n(\nsum by (cluster_environmentd_materialize_cloud_cluster_id, cluster_environmentd_materialize_cloud_replica_id) (\n max without (job) (rate(\n mz_arrangement_maintenance_seconds_total{materialize_cloud_organization_name=~\"$environmentNameList\", cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval]\n ))\n)\n\n)\n* on (cluster_environmentd_materialize_cloud_cluster_id) group_left(cluster_name)\nmax by (cluster_environmentd_materialize_cloud_cluster_id, cluster_name) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"cluster_environmentd_materialize_cloud_cluster_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\nsum by (cluster_environmentd_materialize_cloud_cluster_id, cluster_environmentd_materialize_cloud_replica_id) (\n max without (job) (rate(\n mz_arrangement_maintenance_seconds_total{materialize_cloud_organization_name=~\"$environmentNameList\", cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval]\n ))\n)\n\n)\nunless on (cluster_environmentd_materialize_cloud_cluster_id) (\ngroup by (cluster_environmentd_materialize_cloud_cluster_id) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"cluster_environmentd_materialize_cloud_cluster_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n),\n\"cluster_name\", \"$1\", \"cluster_environmentd_materialize_cloud_cluster_id\", \"(.*)\"\n)" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**CPU-cores spent maintaining arrangements — the in-memory indexed snapshots behind every index and materialized view — summed across a replica's workers, so an N-worker replica can reach N.**\n\n**Nominal:** Well below the worker count.\n\n**Degraded:** Sustained near the worker count means the replica is bottlenecked on maintenance, usually a high upstream change rate or many heavy indexes. The catalog cluster's baseline runs higher than typical user clusters.\n\nUses the long-form cluster_environmentd_* id labels. Reads high under the same multi-job scrape pitfall as the dataflow elapsed rate.", "id": 1048, "links": [], "title": "Arrangement Maintenance Rate", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "unit": "none" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "arrangement-rate-by-worker": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "(\n(\nsum by (cluster_environmentd_materialize_cloud_cluster_id, cluster_environmentd_materialize_cloud_replica_id, worker_id) (\n max without (job) (rate(\n mz_arrangement_maintenance_seconds_total{materialize_cloud_organization_name=~\"$environmentNameList\", cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval]\n ))\n)\n\n)\n* on (cluster_environmentd_materialize_cloud_cluster_id) group_left(cluster_name)\nmax by (cluster_environmentd_materialize_cloud_cluster_id, cluster_name) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"cluster_environmentd_materialize_cloud_cluster_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\nsum by (cluster_environmentd_materialize_cloud_cluster_id, cluster_environmentd_materialize_cloud_replica_id, worker_id) (\n max without (job) (rate(\n mz_arrangement_maintenance_seconds_total{materialize_cloud_organization_name=~\"$environmentNameList\", cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval]\n ))\n)\n\n)\nunless on (cluster_environmentd_materialize_cloud_cluster_id) (\ngroup by (cluster_environmentd_materialize_cloud_cluster_id) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"cluster_environmentd_materialize_cloud_cluster_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n),\n\"cluster_name\", \"$1\", \"cluster_environmentd_materialize_cloud_cluster_id\", \"(.*)\"\n)", "legendFormat": "{{cluster_name}} / {{cluster_environmentd_materialize_cloud_replica_id}} / w{{worker_id}}", "qryType": 1, "query": "(\n(\nsum by (cluster_environmentd_materialize_cloud_cluster_id, cluster_environmentd_materialize_cloud_replica_id, worker_id) (\n max without (job) (rate(\n mz_arrangement_maintenance_seconds_total{materialize_cloud_organization_name=~\"$environmentNameList\", cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval]\n ))\n)\n\n)\n* on (cluster_environmentd_materialize_cloud_cluster_id) group_left(cluster_name)\nmax by (cluster_environmentd_materialize_cloud_cluster_id, cluster_name) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"cluster_environmentd_materialize_cloud_cluster_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\nsum by (cluster_environmentd_materialize_cloud_cluster_id, cluster_environmentd_materialize_cloud_replica_id, worker_id) (\n max without (job) (rate(\n mz_arrangement_maintenance_seconds_total{materialize_cloud_organization_name=~\"$environmentNameList\", cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval]\n ))\n)\n\n)\nunless on (cluster_environmentd_materialize_cloud_cluster_id) (\ngroup by (cluster_environmentd_materialize_cloud_cluster_id) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"cluster_environmentd_materialize_cloud_cluster_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n),\n\"cluster_name\", \"$1\", \"cluster_environmentd_materialize_cloud_cluster_id\", \"(.*)\"\n)" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**The same maintenance CPU, split per worker — each worker tops out at 1.0.**\n\n**Degraded:** Watch for skew: one worker sitting near 1.0 while its siblings idle near 0 means the cluster is bottlenecked on a single worker (a hot key or hot arrangement) — the most common reason an 8-worker cluster performs like a 1-worker one. `EXPLAIN PHYSICAL PLAN` on the heavy object is the usual next step.", "id": 1049, "links": [], "title": "Arrangement Maintenance Rate (per worker)", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "unit": "percentunit" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "arrangement-records-system": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "(\n(\nmax by (collection_id) (\n mz_arrangement_record_count{materialize_cloud_organization_name=~\"$environmentNameList\", instance_id=~\"$mzClusterList\", replica_id=~\"$mzReplicaList\", collection_id=~\"s.*\"}\n)\n\n)\n* on (collection_id) group_left(name)\nmax by (collection_id, name) (\nlabel_replace(mz_object_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"collection_id\", \"$1\", \"global_id\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\nmax by (collection_id) (\n mz_arrangement_record_count{materialize_cloud_organization_name=~\"$environmentNameList\", instance_id=~\"$mzClusterList\", replica_id=~\"$mzReplicaList\", collection_id=~\"s.*\"}\n)\n\n)\nunless on (collection_id) (\ngroup by (collection_id) (\nlabel_replace(mz_object_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"collection_id\", \"$1\", \"global_id\", \"(.*)\")\n)\n),\n\"name\", \"$1\", \"collection_id\", \"(.*)\"\n)", "instant": true, "legendFormat": "{{name}}", "qryType": 1, "query": "(\n(\nmax by (collection_id) (\n mz_arrangement_record_count{materialize_cloud_organization_name=~\"$environmentNameList\", instance_id=~\"$mzClusterList\", replica_id=~\"$mzReplicaList\", collection_id=~\"s.*\"}\n)\n\n)\n* on (collection_id) group_left(name)\nmax by (collection_id, name) (\nlabel_replace(mz_object_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"collection_id\", \"$1\", \"global_id\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\nmax by (collection_id) (\n mz_arrangement_record_count{materialize_cloud_organization_name=~\"$environmentNameList\", instance_id=~\"$mzClusterList\", replica_id=~\"$mzReplicaList\", collection_id=~\"s.*\"}\n)\n\n)\nunless on (collection_id) (\ngroup by (collection_id) (\nlabel_replace(mz_object_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"collection_id\", \"$1\", \"global_id\", \"(.*)\")\n)\n),\n\"name\", \"$1\", \"collection_id\", \"(.*)\"\n)" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [ { "group": "reduce", "kind": "Transformation", "spec": { "options": { "mode": "seriesToRows", "reducers": [ "min", "max", "lastNotNull" ] } } }, { "group": "organize", "kind": "Transformation", "spec": { "options": { "renameByName": { "Field": "Collection" } } } }, { "group": "sortBy", "kind": "Transformation", "spec": { "options": { "fields": {}, "sort": [ { "desc": true, "field": "Last *" } ] } } } ] } }, "description": "**Row counts of arrangements for Materialize's internal system collections (collection id starts with `s`). These back the catalog and internal probes, not user data, so they shouldn't grow with your workload — unexpected growth here can point at a Materialize bug.**", "id": 1050, "links": [], "title": "System Collections — Record Counts", "vizConfig": { "group": "table", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "custom": { "align": "auto", "filterable": true, "inspect": false }, "noValue": "No matches for the current filters", "unit": "short" }, "overrides": [] }, "options": { "cellHeight": "sm", "footer": { "countRows": false, "reducer": [], "show": false }, "frameIndex": 0, "showHeader": true, "showTypeIcons": false } }, "version": "" } } }, "arrangement-records-transient": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (collection_id) (\n mz_arrangement_record_count{materialize_cloud_organization_name=~\"$environmentNameList\", instance_id=~\"$mzClusterList\", replica_id=~\"$mzReplicaList\", collection_id=~\"t.*|none\"}\n)\n", "instant": true, "legendFormat": "{{collection_id}}", "qryType": 1, "query": "max by (collection_id) (\n mz_arrangement_record_count{materialize_cloud_organization_name=~\"$environmentNameList\", instance_id=~\"$mzClusterList\", replica_id=~\"$mzReplicaList\", collection_id=~\"t.*|none\"}\n)\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [ { "group": "reduce", "kind": "Transformation", "spec": { "options": { "mode": "seriesToRows", "reducers": [ "min", "max", "lastNotNull" ] } } }, { "group": "organize", "kind": "Transformation", "spec": { "options": { "renameByName": { "Field": "Collection ID" } } } }, { "group": "sortBy", "kind": "Transformation", "spec": { "options": { "fields": {}, "sort": [ { "desc": true, "field": "Last *" } ] } } } ] } }, "description": "**Row counts of transient (collection id `t*`) and uncategorized (`none`) arrangements — short-lived intermediates from query optimization and dataflow execution. Normally small and ephemeral.**\n\n**Unhealthy:** Sustained non-trivial entries here can mean a stuck or leaked dataflow holding intermediate state it should have released.", "id": 1052, "links": [], "title": "Transient / Uncategorized — Record Counts", "vizConfig": { "group": "table", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "custom": { "align": "auto", "filterable": true, "inspect": false }, "noValue": "No matches for the current filters", "unit": "short" }, "overrides": [] }, "options": { "cellHeight": "sm", "footer": { "countRows": false, "reducer": [], "show": false }, "frameIndex": 0, "showHeader": true, "showTypeIcons": false } }, "version": "" } } }, "arrangement-records-user": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "(\n(\nmax by (collection_id) (\n mz_arrangement_record_count{materialize_cloud_organization_name=~\"$environmentNameList\", instance_id=~\"$mzClusterList\", replica_id=~\"$mzReplicaList\", collection_id=~\"u.*\"}\n)\n\n)\n* on (collection_id) group_left(name)\nmax by (collection_id, name) (\nlabel_replace(mz_object_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"collection_id\", \"$1\", \"global_id\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\nmax by (collection_id) (\n mz_arrangement_record_count{materialize_cloud_organization_name=~\"$environmentNameList\", instance_id=~\"$mzClusterList\", replica_id=~\"$mzReplicaList\", collection_id=~\"u.*\"}\n)\n\n)\nunless on (collection_id) (\ngroup by (collection_id) (\nlabel_replace(mz_object_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"collection_id\", \"$1\", \"global_id\", \"(.*)\")\n)\n),\n\"name\", \"$1\", \"collection_id\", \"(.*)\"\n)", "instant": true, "legendFormat": "{{name}}", "qryType": 1, "query": "(\n(\nmax by (collection_id) (\n mz_arrangement_record_count{materialize_cloud_organization_name=~\"$environmentNameList\", instance_id=~\"$mzClusterList\", replica_id=~\"$mzReplicaList\", collection_id=~\"u.*\"}\n)\n\n)\n* on (collection_id) group_left(name)\nmax by (collection_id, name) (\nlabel_replace(mz_object_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"collection_id\", \"$1\", \"global_id\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\nmax by (collection_id) (\n mz_arrangement_record_count{materialize_cloud_organization_name=~\"$environmentNameList\", instance_id=~\"$mzClusterList\", replica_id=~\"$mzReplicaList\", collection_id=~\"u.*\"}\n)\n\n)\nunless on (collection_id) (\ngroup by (collection_id) (\nlabel_replace(mz_object_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"collection_id\", \"$1\", \"global_id\", \"(.*)\")\n)\n),\n\"name\", \"$1\", \"collection_id\", \"(.*)\"\n)" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [ { "group": "reduce", "kind": "Transformation", "spec": { "options": { "mode": "seriesToRows", "reducers": [ "min", "max", "lastNotNull" ] } } }, { "group": "organize", "kind": "Transformation", "spec": { "options": { "renameByName": { "Field": "Collection" } } } }, { "group": "sortBy", "kind": "Transformation", "spec": { "options": { "fields": {}, "sort": [ { "desc": true, "field": "Last *" } ] } } } ] } }, "description": "**Row counts of arrangements for your compute objects (collection id starts with `u`) — the row count of every user index and materialized view, and the primary driver of cluster memory. Growth on a collection tracks the size of its underlying data.**", "id": 1051, "links": [], "title": "User Collections — Record Counts", "vizConfig": { "group": "table", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "custom": { "align": "auto", "filterable": true, "inspect": false }, "noValue": "No matches for the current filters", "unit": "short" }, "overrides": [] }, "options": { "cellHeight": "sm", "footer": { "countRows": false, "reducer": [], "show": false }, "frameIndex": 0, "showHeader": true, "showTypeIcons": false } }, "version": "" } } }, "availability-percent": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "avg by (materialize_cloud_organization_namespace) (\n avg_over_time(\n mz_compute_cluster_status{\n materialize_cloud_organization_name=~\"$environmentNameList\"\n }[$__range]\n ) * 100\n)\n", "qryType": 1, "query": "avg by (materialize_cloud_organization_namespace) (\n avg_over_time(\n mz_compute_cluster_status{\n materialize_cloud_organization_name=~\"$environmentNameList\"\n }[$__range]\n ) * 100\n)\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**An SLO-style snapshot: how much of the selected window the environment's clusters were ready. Sustained dips are the signal that something restarted or went down while you weren't watching.**\n\n**Nominal:** At or near full availability. Watch the decimals — 99.999% is five-nines, a very different story from 100%.\n\n**Degraded:** Availability slipped below target over the window. Correlate the dip with cluster restarts and outages; widen the range to pin down when it happened, then work backwards from there.\n\nFour-nines is about the ceiling this can measure: at a 30s scrape resolution over a week, a single missed scrape already costs a nine, so don't chase phantom sub-nine dips that are really scrape granularity.", "id": 1001, "links": [], "title": "Environment Availability (Select Time Range)", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "decimals": 4, "noValue": "No matches for the current filters", "thresholds": { "mode": "percentage", "steps": [ { "color": "#AB0003", "value": null }, { "color": "#F19903", "value": 95 }, { "color": "#A2F49B", "value": 99 } ] }, "unit": "percent" }, "overrides": [] }, "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [], "values": false }, "showPercentChange": false, "textMode": "value", "wideLayout": true } }, "version": "" } } }, "cluster-count": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "count(\n group by (compute_cluster_id) (\n mz_compute_cluster_status{materialize_cloud_organization_name=~\"$environmentNameList\", compute_cluster_id=~\"$mzClusterList\"}\n )\n)\n", "legendFormat": "Total Clusters", "qryType": 1, "query": "count(\n group by (compute_cluster_id) (\n mz_compute_cluster_status{materialize_cloud_organization_name=~\"$environmentNameList\", compute_cluster_id=~\"$mzClusterList\"}\n )\n)\n" }, "version": "v0" }, "refId": "query-0" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "count(\n group by (compute_cluster_id) (\n mz_compute_cluster_status{materialize_cloud_organization_name=~\"$environmentNameList\", compute_cluster_id=~\"$mzClusterList\", compute_cluster_id=~\"^s.*\"}\n )\n)\n", "legendFormat": "System Clusters", "qryType": 1, "query": "count(\n group by (compute_cluster_id) (\n mz_compute_cluster_status{materialize_cloud_organization_name=~\"$environmentNameList\", compute_cluster_id=~\"$mzClusterList\", compute_cluster_id=~\"^s.*\"}\n )\n)\n" }, "version": "v0" }, "refId": "query-1" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**How many clusters exist, split into the Materialize-managed system clusters (mz_catalog_server, mz_system, mz_probe, …) that every environment has and the user clusters you created. The gap between the two is your own footprint.**\n\nSteady in normal operation; steps up and down on CREATE CLUSTER / DROP CLUSTER. Sustained drift with no DDL you recognize points at automation creating clusters.", "id": 1030, "links": [], "title": "Cluster Count", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "fixedColor": "#009988", "mode": "shades" }, "min": 0, "noValue": "No matches for the current filters" }, "overrides": [] }, "options": { "colorMode": "none", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [], "values": false }, "showPercentChange": false, "textMode": "value_and_name", "wideLayout": true } }, "version": "" } } }, "cluster-table": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "mz_compute_cluster_status{materialize_cloud_organization_name=~\"$environmentNameList\", compute_cluster_id=~\"$mzClusterList\", compute_replica_id=~\"$mzReplicaList\"}\n", "instant": true, "qryType": 1, "query": "mz_compute_cluster_status{materialize_cloud_organization_name=~\"$environmentNameList\", compute_cluster_id=~\"$mzClusterList\", compute_replica_id=~\"$mzReplicaList\"}\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [ { "group": "labelsToFields", "kind": "Transformation", "spec": { "options": { "keepLabels": [ "compute_cluster_name", "compute_replica_name", "compute_cluster_id", "compute_replica_id", "mz_version", "size", "materialize_cloud_availability_zone", "topology_kubernetes_io_region", "topology_kubernetes_io_zone" ] } } }, { "group": "merge", "kind": "Transformation", "spec": { "options": {} } }, { "group": "organize", "kind": "Transformation", "spec": { "options": { "excludeByName": { "Time": true, "mz_compute_cluster_status": true, "v2_mz_compute_cluster_status": true }, "indexByName": { "compute_cluster_id": 2, "compute_cluster_name": 0, "compute_replica_id": 3, "compute_replica_name": 1, "materialize_cloud_availability_zone": 6, "mz_version": 4, "size": 5, "topology_kubernetes_io_region": 7, "topology_kubernetes_io_zone": 8 } } } }, { "group": "sortBy", "kind": "Transformation", "spec": { "options": { "fields": {}, "sort": [ { "field": "compute_cluster_name" } ] } } } ] } }, "description": "**A reference row per (cluster, replica): ids, names, size, version, and scheduling metadata. The \"what does my fleet actually look like\" lookup — most useful for grabbing a cluster or replica id to scope the rest of a dashboard to.**\n\nA catalog snapshot, not a health signal. The presentation layer projects the metric's labels (compute_cluster_name, size, mz_version, region, …) into table columns; this query just selects the underlying series.", "id": 1033, "links": [], "title": "Cluster Information", "vizConfig": { "group": "table", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "custom": { "align": "auto", "filterable": true, "inspect": false }, "noValue": "No matches for the current filters" }, "overrides": [] }, "options": { "cellHeight": "sm", "footer": { "countRows": false, "reducer": [], "show": false }, "frameIndex": 0, "showHeader": true, "showTypeIcons": false } }, "version": "" } } }, "connections-active-queries": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (session_type) (rate(mz_query_total{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval]))\n", "legendFormat": "{{session_type}}", "qryType": 1, "query": "sum by (session_type) (rate(mz_query_total{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval]))\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Queries per second by session type — `user` tracks your client traffic, `system` is the steady single-digit baseline of internal health checks. Bursty is normal.**\n\nEnvironment-scoped.", "id": 1022, "links": [], "title": "Active Queries", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "fixedColor": "#33BBEE", "mode": "shades" }, "min": 0, "noValue": "No matches for the current filters", "unit": "cps" }, "overrides": [] }, "options": { "colorMode": "none", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [], "values": false }, "showPercentChange": false, "textMode": "value_and_name", "wideLayout": true } }, "version": "" } } }, "connections-active-sessions": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (session_type) (mz_active_sessions{materialize_cloud_organization_name=~\"$environmentNameList\"})\n", "legendFormat": "{{session_type}}", "qryType": 1, "query": "sum by (session_type) (mz_active_sessions{materialize_cloud_organization_name=~\"$environmentNameList\"})\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Open SQL sessions, split into `system` (Materialize's internal probing — a few are always present) and `user` (client connections).**\n\n**Degraded:** A `user` count that climbs and never settles is the classic leaked-connection signal; cross-check whether query activity rose to match, or whether connections are just accumulating idle.\n\nEnvironment-scoped — the cluster/replica selectors don't narrow it.", "id": 1021, "links": [], "title": "Active Sessions", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "fixedColor": "#33BBEE", "mode": "shades" }, "min": 0, "noValue": "No matches for the current filters" }, "overrides": [] }, "options": { "colorMode": "none", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [], "values": false }, "showPercentChange": false, "textMode": "value_and_name", "wideLayout": true } }, "version": "" } } }, "connections-adapter-command-rate": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(rate(mz_adapter_commands{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval]))\n", "legendFormat": "commands", "qryType": 1, "query": "sum(rate(mz_adapter_commands{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval]))\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Commands per second through the adapter — the SQL protocol layer (parse, execute, prepare, fetch). Normally runs higher than the query rate, since one query is several commands.**\n\n**Degraded:** A flat-line on an environment that's usually busy is unusual and can indicate the adapter is in trouble; check per-application command activity and its error column.\n\nEnvironment-scoped.", "id": 1023, "links": [], "title": "SQL Control Plane Command Rate", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "fixedColor": "#33BBEE", "mode": "shades" }, "min": 0, "noValue": "No matches for the current filters", "unit": "cps" }, "overrides": [] }, "options": { "colorMode": "none", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [], "values": false }, "showPercentChange": false, "textMode": "value", "wideLayout": true } }, "version": "" } } }, "cpu-usage-current": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (namespace, container) (\n rate(container_cpu_usage_seconds_total{namespace=~\"$mzNamespaceList\",container!=\"\",container!=\"POD\"}[5m])\n) / sum by (namespace, container) (\n kube_pod_container_resource_limits{resource=\"cpu\", namespace=~\"$mzNamespaceList\"}\n)\n", "instant": true, "legendFormat": "{{container}}", "qryType": 1, "query": "sum by (namespace, container) (\n rate(container_cpu_usage_seconds_total{namespace=~\"$mzNamespaceList\",container!=\"\",container!=\"POD\"}[5m])\n) / sum by (namespace, container) (\n kube_pod_container_resource_limits{resource=\"cpu\", namespace=~\"$mzNamespaceList\"}\n)\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Current CPU usage per container type as a fraction of its limit, averaged over the last 5 minutes — shows the worst-loaded container types.**\n\n**Nominal:** Well below 1.0.\n\n**Degraded:** Sustained near 1.0 means a container type is CPU-bound.\n\nNeeds CPU limits from kube-state-metrics; use the `.absolute` sibling where limits aren't exposed. Requires cAdvisor + kube-state-metrics.", "id": 1005, "links": [], "title": "Current CPU Usage (5 min)", "vizConfig": { "group": "gauge", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "noValue": "No metrics: cadvisor metrics are required (via kubelet)", "thresholds": { "mode": "absolute", "steps": [ { "color": "#CEFFFF", "value": null }, { "color": "#CEFFFF", "value": 0 }, { "color": "#C6F7D6", "value": 0.1 }, { "color": "#A2F49B", "value": 0.2 }, { "color": "#BBE453", "value": 0.30000000000000004 }, { "color": "#D5CE04", "value": 0.4 }, { "color": "#E7B503", "value": 0.5 }, { "color": "#F19903", "value": 0.6000000000000001 }, { "color": "#F6790B", "value": 0.7000000000000001 }, { "color": "#F94902", "value": 0.8 }, { "color": "#E40515", "value": 0.9 }, { "color": "#AB0003", "value": 1 } ] }, "unit": "percentunit" }, "overrides": [] }, "options": { "minVizHeight": 75, "minVizWidth": 75, "orientation": "auto", "reduceOptions": { "calcs": [], "values": false }, "showThresholdLabels": false, "showThresholdMarkers": true, "sizing": "auto" } }, "version": "" } } }, "dataflow-count": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "(\n(\nmax by (cluster_environmentd_materialize_cloud_cluster_id, cluster_environmentd_materialize_cloud_replica_id) (\n mz_compute_replica_history_dataflow_count{materialize_cloud_organization_name=~\"$environmentNameList\", cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}\n)\n\n)\n* on (cluster_environmentd_materialize_cloud_cluster_id) group_left(cluster_name)\nmax by (cluster_environmentd_materialize_cloud_cluster_id, cluster_name) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"cluster_environmentd_materialize_cloud_cluster_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\nmax by (cluster_environmentd_materialize_cloud_cluster_id, cluster_environmentd_materialize_cloud_replica_id) (\n mz_compute_replica_history_dataflow_count{materialize_cloud_organization_name=~\"$environmentNameList\", cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}\n)\n\n)\nunless on (cluster_environmentd_materialize_cloud_cluster_id) (\ngroup by (cluster_environmentd_materialize_cloud_cluster_id) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"cluster_environmentd_materialize_cloud_cluster_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n),\n\"cluster_name\", \"$1\", \"cluster_environmentd_materialize_cloud_cluster_id\", \"(.*)\"\n)", "legendFormat": "{{cluster_name}} / {{cluster_environmentd_materialize_cloud_replica_id}}", "qryType": 1, "query": "(\n(\nmax by (cluster_environmentd_materialize_cloud_cluster_id, cluster_environmentd_materialize_cloud_replica_id) (\n mz_compute_replica_history_dataflow_count{materialize_cloud_organization_name=~\"$environmentNameList\", cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}\n)\n\n)\n* on (cluster_environmentd_materialize_cloud_cluster_id) group_left(cluster_name)\nmax by (cluster_environmentd_materialize_cloud_cluster_id, cluster_name) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"cluster_environmentd_materialize_cloud_cluster_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\nmax by (cluster_environmentd_materialize_cloud_cluster_id, cluster_environmentd_materialize_cloud_replica_id) (\n mz_compute_replica_history_dataflow_count{materialize_cloud_organization_name=~\"$environmentNameList\", cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}\n)\n\n)\nunless on (cluster_environmentd_materialize_cloud_cluster_id) (\ngroup by (cluster_environmentd_materialize_cloud_cluster_id) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"cluster_environmentd_materialize_cloud_cluster_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n),\n\"cluster_name\", \"$1\", \"cluster_environmentd_materialize_cloud_cluster_id\", \"(.*)\"\n)" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Active dataflows on each replica. Every index, materialized view, and live SUBSCRIBE runs as one or more dataflows, so this count rises with DDL and subscribe activity.**\n\n**Nominal:** Stable for a steady workload.\n\n**Degraded:** A sharp drop with no matching DDL usually means a replica restarted; a sharp rise usually means bulk object creation.", "id": 1045, "links": [], "title": "Dataflow Count", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "unit": "short" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "dataflow-count-by-worker": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "(\n(\nmax by (cluster_environmentd_materialize_cloud_cluster_id, cluster_environmentd_materialize_cloud_replica_id, worker_id) (\n mz_compute_replica_history_dataflow_count{materialize_cloud_organization_name=~\"$environmentNameList\", cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}\n)\n\n)\n* on (cluster_environmentd_materialize_cloud_cluster_id) group_left(cluster_name)\nmax by (cluster_environmentd_materialize_cloud_cluster_id, cluster_name) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"cluster_environmentd_materialize_cloud_cluster_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\nmax by (cluster_environmentd_materialize_cloud_cluster_id, cluster_environmentd_materialize_cloud_replica_id, worker_id) (\n mz_compute_replica_history_dataflow_count{materialize_cloud_organization_name=~\"$environmentNameList\", cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}\n)\n\n)\nunless on (cluster_environmentd_materialize_cloud_cluster_id) (\ngroup by (cluster_environmentd_materialize_cloud_cluster_id) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"cluster_environmentd_materialize_cloud_cluster_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n),\n\"cluster_name\", \"$1\", \"cluster_environmentd_materialize_cloud_cluster_id\", \"(.*)\"\n)", "legendFormat": "{{cluster_name}} / {{cluster_environmentd_materialize_cloud_replica_id}} / w{{worker_id}}", "qryType": 1, "query": "(\n(\nmax by (cluster_environmentd_materialize_cloud_cluster_id, cluster_environmentd_materialize_cloud_replica_id, worker_id) (\n mz_compute_replica_history_dataflow_count{materialize_cloud_organization_name=~\"$environmentNameList\", cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}\n)\n\n)\n* on (cluster_environmentd_materialize_cloud_cluster_id) group_left(cluster_name)\nmax by (cluster_environmentd_materialize_cloud_cluster_id, cluster_name) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"cluster_environmentd_materialize_cloud_cluster_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\nmax by (cluster_environmentd_materialize_cloud_cluster_id, cluster_environmentd_materialize_cloud_replica_id, worker_id) (\n mz_compute_replica_history_dataflow_count{materialize_cloud_organization_name=~\"$environmentNameList\", cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}\n)\n\n)\nunless on (cluster_environmentd_materialize_cloud_cluster_id) (\ngroup by (cluster_environmentd_materialize_cloud_cluster_id) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"cluster_environmentd_materialize_cloud_cluster_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n),\n\"cluster_name\", \"$1\", \"cluster_environmentd_materialize_cloud_cluster_id\", \"(.*)\"\n)" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**The dataflow count broken out per worker. Workers in a replica run in lockstep and should see exactly the same dataflows, so their series should overlap perfectly.**\n\n**Unhealthy:** Visible divergence between worker series within one (cluster, replica) is a bug-class signal — the workers disagree on what they're running — and worth filing.", "id": 1046, "links": [], "title": "Dataflow Count (per worker)", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "unit": "short" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "dataflow-elapsed-rate": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "(\n(\nsum by (instance_id) (\n max without (job) (rate(\n mz_dataflow_elapsed_seconds_total{materialize_cloud_organization_name=~\"$environmentNameList\", instance_id=~\"$mzClusterList\", replica_id=~\"$mzReplicaList\"}[$__rate_interval]\n ))\n)\n\n)\n* on (instance_id) group_left(cluster_name)\nmax by (instance_id, cluster_name) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"instance_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\nsum by (instance_id) (\n max without (job) (rate(\n mz_dataflow_elapsed_seconds_total{materialize_cloud_organization_name=~\"$environmentNameList\", instance_id=~\"$mzClusterList\", replica_id=~\"$mzReplicaList\"}[$__rate_interval]\n ))\n)\n\n)\nunless on (instance_id) (\ngroup by (instance_id) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"instance_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n),\n\"cluster_name\", \"$1\", \"instance_id\", \"(.*)\"\n)", "legendFormat": "{{cluster_name}}", "qryType": 1, "query": "(\n(\nsum by (instance_id) (\n max without (job) (rate(\n mz_dataflow_elapsed_seconds_total{materialize_cloud_organization_name=~\"$environmentNameList\", instance_id=~\"$mzClusterList\", replica_id=~\"$mzReplicaList\"}[$__rate_interval]\n ))\n)\n\n)\n* on (instance_id) group_left(cluster_name)\nmax by (instance_id, cluster_name) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"instance_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\nsum by (instance_id) (\n max without (job) (rate(\n mz_dataflow_elapsed_seconds_total{materialize_cloud_organization_name=~\"$environmentNameList\", instance_id=~\"$mzClusterList\", replica_id=~\"$mzReplicaList\"}[$__rate_interval]\n ))\n)\n\n)\nunless on (instance_id) (\ngroup by (instance_id) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"instance_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n),\n\"cluster_name\", \"$1\", \"instance_id\", \"(.*)\"\n)" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**CPU-cores busy inside dataflows, per cluster — the whole of dataflow work: arrangement maintenance, query evaluation, and hydration. Capped by cluster size (a 400cc cluster can't exceed 400 cores).**\n\n**Nominal:** Well below the cluster's size.\n\n**Degraded:** Sustained near the size cap means the cluster is CPU-bound and a candidate for upsizing. The catalog cluster (s2) is typically the busiest in any environment.\n\nReads high if the same clusterd endpoint is scraped by more than one Prometheus job (a known multi-job scrape-config pitfall); fix the scrape config, or collapse duplicate `job` series, at the deployment.", "id": 1047, "links": [], "title": "Dataflow Elapsed Rate", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "custom": { "scaleDistribution": { "log": 10, "type": "log" } }, "unit": "none" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "freshness-lag-by-cluster": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "(\n(\nmax by (instance_id) (\n mz_dataflow_wallclock_lag_seconds{materialize_cloud_organization_name=~\"$environmentNameList\", instance_id=~\"$mzClusterList\", instance_id!=\"\", quantile=\"1\"} < 1e9\n)\n\n)\n* on (instance_id) group_left(cluster_name)\nmax by (instance_id, cluster_name) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"instance_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\nmax by (instance_id) (\n mz_dataflow_wallclock_lag_seconds{materialize_cloud_organization_name=~\"$environmentNameList\", instance_id=~\"$mzClusterList\", instance_id!=\"\", quantile=\"1\"} < 1e9\n)\n\n)\nunless on (instance_id) (\ngroup by (instance_id) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"instance_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n),\n\"cluster_name\", \"$1\", \"instance_id\", \"(.*)\"\n)", "legendFormat": "{{cluster_name}}", "qryType": 1, "query": "(\n(\nmax by (instance_id) (\n mz_dataflow_wallclock_lag_seconds{materialize_cloud_organization_name=~\"$environmentNameList\", instance_id=~\"$mzClusterList\", instance_id!=\"\", quantile=\"1\"} < 1e9\n)\n\n)\n* on (instance_id) group_left(cluster_name)\nmax by (instance_id, cluster_name) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"instance_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\nmax by (instance_id) (\n mz_dataflow_wallclock_lag_seconds{materialize_cloud_organization_name=~\"$environmentNameList\", instance_id=~\"$mzClusterList\", instance_id!=\"\", quantile=\"1\"} < 1e9\n)\n\n)\nunless on (instance_id) (\ngroup by (instance_id) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"instance_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n),\n\"cluster_name\", \"$1\", \"instance_id\", \"(.*)\"\n)" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**How far behind real time each cluster's most-lagged collection is — the worst-case freshness across every index, materialized view, and source on the cluster.**\n\n**Nominal:** Low and flat, sub-second to a few seconds, for a cluster keeping up.\n\n**Degraded:** A sustained climb means a collection can't keep pace with its input — an under-provisioned cluster or an expensive dataflow. A sharp spike that decays on its own is normal re-hydration after a restart or DDL, not a problem.\n\nThe catalog cluster (s2) sits a couple seconds back as its baseline. Collections that have not produced results yet report a sentinel and are excluded here. For which collection is responsible, see materialize.compute.freshness.top_collections.", "id": 1039, "links": [], "title": "Freshness Lag by Cluster", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "custom": { "scaleDistribution": { "log": 10, "type": "log" } }, "min": 0, "noValue": "No matches for the current filters", "unit": "s" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "freshness-lag-total-by-cluster": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "(\n(\nsum by (instance_id) (\n max by (instance_id, collection_id) (\n mz_dataflow_wallclock_lag_seconds{materialize_cloud_organization_name=~\"$environmentNameList\", instance_id=~\"$mzClusterList\", instance_id!=\"\", quantile=\"1\"} < 1e9\n )\n)\n\n)\n* on (instance_id) group_left(cluster_name)\nmax by (instance_id, cluster_name) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"instance_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\nsum by (instance_id) (\n max by (instance_id, collection_id) (\n mz_dataflow_wallclock_lag_seconds{materialize_cloud_organization_name=~\"$environmentNameList\", instance_id=~\"$mzClusterList\", instance_id!=\"\", quantile=\"1\"} < 1e9\n )\n)\n\n)\nunless on (instance_id) (\ngroup by (instance_id) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"instance_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n),\n\"cluster_name\", \"$1\", \"instance_id\", \"(.*)\"\n)", "legendFormat": "{{cluster_name}}", "qryType": 1, "query": "(\n(\nsum by (instance_id) (\n max by (instance_id, collection_id) (\n mz_dataflow_wallclock_lag_seconds{materialize_cloud_organization_name=~\"$environmentNameList\", instance_id=~\"$mzClusterList\", instance_id!=\"\", quantile=\"1\"} < 1e9\n )\n)\n\n)\n* on (instance_id) group_left(cluster_name)\nmax by (instance_id, cluster_name) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"instance_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\nsum by (instance_id) (\n max by (instance_id, collection_id) (\n mz_dataflow_wallclock_lag_seconds{materialize_cloud_organization_name=~\"$environmentNameList\", instance_id=~\"$mzClusterList\", instance_id!=\"\", quantile=\"1\"} < 1e9\n )\n)\n\n)\nunless on (instance_id) (\ngroup by (instance_id) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"instance_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n),\n\"cluster_name\", \"$1\", \"instance_id\", \"(.*)\"\n)" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**The lag of every collection on each cluster, added together — one number for how far behind the cluster is in total.**\n\n**Nominal:** Low and flat. The absolute value is not comparable between clusters, since a cluster with more collections has more to add up; the *shape* is what this panel is for.\n\n**Degraded:** A climb that does not come back down. A rise that decays steadily is a cluster catching up after a restart, a rollout, or DDL, and the decay is the thing worth watching.\n\n**Read this beside the worst-case panel rather than instead of it.** The max says whether anything is behind; this says how much is behind in total, and the two answer different questions. When a hundred collections rehydrate and fifty of them catch up, the max barely moves — it is still reporting whichever one is furthest behind — while this halves. That descent is the clearest picture of hydration progress the metrics can give, and it is invisible in the max.\n\n**Deduplicated across replicas.** A collection served by several replicas reports once per replica, so the inner `max` picks the worst reading per collection before the outer `sum` adds collections together. Without it a two-replica cluster would report double, which is exactly the shape a blue/green rollout has.\n\nCollections that have not produced results yet report the `u64::MAX` sentinel and are excluded (`< 1e9`) — including them would swamp the total by nineteen orders of magnitude. They are counted by the hydration panels instead, so the two are complementary: this totals the collections that are *producing results*, those count the ones that are not. Datadog cannot express that value filter, so its query includes not-yet-hydrated collections and can overstate the total badly.", "id": 1040, "links": [], "title": "Total Lag by Cluster", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "No matches for the current filters", "unit": "s" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "freshness-top-collections": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "(\n(\n(\n(\ntopk(15,\n max by (instance_id, collection_id) (\n mz_dataflow_wallclock_lag_seconds{materialize_cloud_organization_name=~\"$environmentNameList\", instance_id=~\"$mzClusterList\", instance_id!=\"\", replica_id=~\"$mzReplicaList\", quantile=\"1\"} < 1e9\n )\n)\n\n)\n* on (collection_id) group_left(name)\nmax by (collection_id, name) (\nlabel_replace(mz_object_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"collection_id\", \"$1\", \"global_id\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\ntopk(15,\n max by (instance_id, collection_id) (\n mz_dataflow_wallclock_lag_seconds{materialize_cloud_organization_name=~\"$environmentNameList\", instance_id=~\"$mzClusterList\", instance_id!=\"\", replica_id=~\"$mzReplicaList\", quantile=\"1\"} < 1e9\n )\n)\n\n)\nunless on (collection_id) (\ngroup by (collection_id) (\nlabel_replace(mz_object_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"collection_id\", \"$1\", \"global_id\", \"(.*)\")\n)\n),\n\"name\", \"$1\", \"collection_id\", \"(.*)\"\n)\n)\n* on (instance_id) group_left(cluster_name)\nmax by (instance_id, cluster_name) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"instance_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\n(\n(\ntopk(15,\n max by (instance_id, collection_id) (\n mz_dataflow_wallclock_lag_seconds{materialize_cloud_organization_name=~\"$environmentNameList\", instance_id=~\"$mzClusterList\", instance_id!=\"\", replica_id=~\"$mzReplicaList\", quantile=\"1\"} < 1e9\n )\n)\n\n)\n* on (collection_id) group_left(name)\nmax by (collection_id, name) (\nlabel_replace(mz_object_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"collection_id\", \"$1\", \"global_id\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\ntopk(15,\n max by (instance_id, collection_id) (\n mz_dataflow_wallclock_lag_seconds{materialize_cloud_organization_name=~\"$environmentNameList\", instance_id=~\"$mzClusterList\", instance_id!=\"\", replica_id=~\"$mzReplicaList\", quantile=\"1\"} < 1e9\n )\n)\n\n)\nunless on (collection_id) (\ngroup by (collection_id) (\nlabel_replace(mz_object_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"collection_id\", \"$1\", \"global_id\", \"(.*)\")\n)\n),\n\"name\", \"$1\", \"collection_id\", \"(.*)\"\n)\n)\nunless on (instance_id) (\ngroup by (instance_id) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"instance_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n),\n\"cluster_name\", \"$1\", \"instance_id\", \"(.*)\"\n)", "instant": true, "legendFormat": "{{cluster_name}} / {{name}}", "qryType": 1, "query": "(\n(\n(\n(\ntopk(15,\n max by (instance_id, collection_id) (\n mz_dataflow_wallclock_lag_seconds{materialize_cloud_organization_name=~\"$environmentNameList\", instance_id=~\"$mzClusterList\", instance_id!=\"\", replica_id=~\"$mzReplicaList\", quantile=\"1\"} < 1e9\n )\n)\n\n)\n* on (collection_id) group_left(name)\nmax by (collection_id, name) (\nlabel_replace(mz_object_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"collection_id\", \"$1\", \"global_id\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\ntopk(15,\n max by (instance_id, collection_id) (\n mz_dataflow_wallclock_lag_seconds{materialize_cloud_organization_name=~\"$environmentNameList\", instance_id=~\"$mzClusterList\", instance_id!=\"\", replica_id=~\"$mzReplicaList\", quantile=\"1\"} < 1e9\n )\n)\n\n)\nunless on (collection_id) (\ngroup by (collection_id) (\nlabel_replace(mz_object_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"collection_id\", \"$1\", \"global_id\", \"(.*)\")\n)\n),\n\"name\", \"$1\", \"collection_id\", \"(.*)\"\n)\n)\n* on (instance_id) group_left(cluster_name)\nmax by (instance_id, cluster_name) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"instance_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\n(\n(\ntopk(15,\n max by (instance_id, collection_id) (\n mz_dataflow_wallclock_lag_seconds{materialize_cloud_organization_name=~\"$environmentNameList\", instance_id=~\"$mzClusterList\", instance_id!=\"\", replica_id=~\"$mzReplicaList\", quantile=\"1\"} < 1e9\n )\n)\n\n)\n* on (collection_id) group_left(name)\nmax by (collection_id, name) (\nlabel_replace(mz_object_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"collection_id\", \"$1\", \"global_id\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\ntopk(15,\n max by (instance_id, collection_id) (\n mz_dataflow_wallclock_lag_seconds{materialize_cloud_organization_name=~\"$environmentNameList\", instance_id=~\"$mzClusterList\", instance_id!=\"\", replica_id=~\"$mzReplicaList\", quantile=\"1\"} < 1e9\n )\n)\n\n)\nunless on (collection_id) (\ngroup by (collection_id) (\nlabel_replace(mz_object_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"collection_id\", \"$1\", \"global_id\", \"(.*)\")\n)\n),\n\"name\", \"$1\", \"collection_id\", \"(.*)\"\n)\n)\nunless on (instance_id) (\ngroup by (instance_id) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"instance_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n),\n\"cluster_name\", \"$1\", \"instance_id\", \"(.*)\"\n)" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**The 15 collections whose results are furthest behind real time — the per-collection breakdown behind the per-cluster freshness lag, labeled by object name.**\n\nIdle or still-hydrating collections report a sentinel and are filtered out; check hydration state directly in SQL if a collection you expect is missing. Catalog (s2) collections commonly appear with a small, expected baseline lag.", "id": 1041, "links": [], "title": "Most-Lagged Collections", "vizConfig": { "group": "barchart", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "fixedColor": "#EE7733", "mode": "shades" }, "custom": { "fillOpacity": 80, "gradientMode": "none", "lineWidth": 1, "scaleDistribution": { "log": 10, "type": "log" } }, "noValue": "No matches for the current filters", "unit": "s" }, "overrides": [] }, "options": { "barRadius": 0, "barWidth": 0.8, "fullHighlight": false, "groupWidth": 0.95, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": false }, "orientation": "horizontal", "showValue": "auto", "stacking": "none", "tooltip": { "mode": "single", "sort": "asc" }, "xTickLabelMaxLength": 0, "xTickLabelRotation": 0, "xTickLabelSpacing": 100 } }, "version": "" } } }, "hydration-queue-size": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "(\n(\nsum by (instance_id, replica_id) (\n mz_compute_controller_hydration_queue_size{materialize_cloud_organization_name=~\"$environmentNameList\", instance_id=~\"$mzClusterList\", replica_id=~\"$mzReplicaList\"}\n) > 0\n\n)\n* on (instance_id) group_left(cluster_name)\nmax by (instance_id, cluster_name) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"instance_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\nsum by (instance_id, replica_id) (\n mz_compute_controller_hydration_queue_size{materialize_cloud_organization_name=~\"$environmentNameList\", instance_id=~\"$mzClusterList\", replica_id=~\"$mzReplicaList\"}\n) > 0\n\n)\nunless on (instance_id) (\ngroup by (instance_id) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"instance_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n),\n\"cluster_name\", \"$1\", \"instance_id\", \"(.*)\"\n)", "legendFormat": "{{cluster_name}} / r{{replica_id}}", "qryType": 1, "query": "(\n(\nsum by (instance_id, replica_id) (\n mz_compute_controller_hydration_queue_size{materialize_cloud_organization_name=~\"$environmentNameList\", instance_id=~\"$mzClusterList\", replica_id=~\"$mzReplicaList\"}\n) > 0\n\n)\n* on (instance_id) group_left(cluster_name)\nmax by (instance_id, cluster_name) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"instance_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\nsum by (instance_id, replica_id) (\n mz_compute_controller_hydration_queue_size{materialize_cloud_organization_name=~\"$environmentNameList\", instance_id=~\"$mzClusterList\", replica_id=~\"$mzReplicaList\"}\n) > 0\n\n)\nunless on (instance_id) (\ngroup by (instance_id) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"instance_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n),\n\"cluster_name\", \"$1\", \"instance_id\", \"(.*)\"\n)" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Collections waiting in each replica's hydration queue. environmentd schedules hydration in batches; a backlog means work is arriving faster than the replica can rebuild it.**\n\n**Nominal:** 0 — nothing is waiting to hydrate.\n\n**Unhealthy:** Sustained non-zero means the replica is undersized for its workload, or one slow-hydrating collection is holding up the line behind it.", "id": 1043, "links": [], "title": "Hydration Queue Size", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "Hydration Queue is empty", "unit": "short" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "hydration-slowest-collections": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "(\n(\n(\n(\ntopk(15,\n mz_compute_hydration_time_seconds{materialize_cloud_organization_name=~\"$environmentNameList\", instance_id=~\"$mzClusterList\", replica_id=~\"$mzReplicaList\", hydrated=\"1\"}\n)\n\n)\n* on (collection_id) group_left(name)\nmax by (collection_id, name) (\nlabel_replace(mz_object_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"collection_id\", \"$1\", \"global_id\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\ntopk(15,\n mz_compute_hydration_time_seconds{materialize_cloud_organization_name=~\"$environmentNameList\", instance_id=~\"$mzClusterList\", replica_id=~\"$mzReplicaList\", hydrated=\"1\"}\n)\n\n)\nunless on (collection_id) (\ngroup by (collection_id) (\nlabel_replace(mz_object_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"collection_id\", \"$1\", \"global_id\", \"(.*)\")\n)\n),\n\"name\", \"$1\", \"collection_id\", \"(.*)\"\n)\n)\n* on (instance_id) group_left(cluster_name)\nmax by (instance_id, cluster_name) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"instance_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\n(\n(\ntopk(15,\n mz_compute_hydration_time_seconds{materialize_cloud_organization_name=~\"$environmentNameList\", instance_id=~\"$mzClusterList\", replica_id=~\"$mzReplicaList\", hydrated=\"1\"}\n)\n\n)\n* on (collection_id) group_left(name)\nmax by (collection_id, name) (\nlabel_replace(mz_object_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"collection_id\", \"$1\", \"global_id\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\ntopk(15,\n mz_compute_hydration_time_seconds{materialize_cloud_organization_name=~\"$environmentNameList\", instance_id=~\"$mzClusterList\", replica_id=~\"$mzReplicaList\", hydrated=\"1\"}\n)\n\n)\nunless on (collection_id) (\ngroup by (collection_id) (\nlabel_replace(mz_object_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"collection_id\", \"$1\", \"global_id\", \"(.*)\")\n)\n),\n\"name\", \"$1\", \"collection_id\", \"(.*)\"\n)\n)\nunless on (instance_id) (\ngroup by (instance_id) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"instance_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n),\n\"cluster_name\", \"$1\", \"instance_id\", \"(.*)\"\n)", "instant": true, "legendFormat": "{{cluster_name}} / r{{replica_id}} / {{name}}", "qryType": 1, "query": "(\n(\n(\n(\ntopk(15,\n mz_compute_hydration_time_seconds{materialize_cloud_organization_name=~\"$environmentNameList\", instance_id=~\"$mzClusterList\", replica_id=~\"$mzReplicaList\", hydrated=\"1\"}\n)\n\n)\n* on (collection_id) group_left(name)\nmax by (collection_id, name) (\nlabel_replace(mz_object_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"collection_id\", \"$1\", \"global_id\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\ntopk(15,\n mz_compute_hydration_time_seconds{materialize_cloud_organization_name=~\"$environmentNameList\", instance_id=~\"$mzClusterList\", replica_id=~\"$mzReplicaList\", hydrated=\"1\"}\n)\n\n)\nunless on (collection_id) (\ngroup by (collection_id) (\nlabel_replace(mz_object_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"collection_id\", \"$1\", \"global_id\", \"(.*)\")\n)\n),\n\"name\", \"$1\", \"collection_id\", \"(.*)\"\n)\n)\n* on (instance_id) group_left(cluster_name)\nmax by (instance_id, cluster_name) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"instance_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\n(\n(\ntopk(15,\n mz_compute_hydration_time_seconds{materialize_cloud_organization_name=~\"$environmentNameList\", instance_id=~\"$mzClusterList\", replica_id=~\"$mzReplicaList\", hydrated=\"1\"}\n)\n\n)\n* on (collection_id) group_left(name)\nmax by (collection_id, name) (\nlabel_replace(mz_object_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"collection_id\", \"$1\", \"global_id\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\ntopk(15,\n mz_compute_hydration_time_seconds{materialize_cloud_organization_name=~\"$environmentNameList\", instance_id=~\"$mzClusterList\", replica_id=~\"$mzReplicaList\", hydrated=\"1\"}\n)\n\n)\nunless on (collection_id) (\ngroup by (collection_id) (\nlabel_replace(mz_object_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"collection_id\", \"$1\", \"global_id\", \"(.*)\")\n)\n),\n\"name\", \"$1\", \"collection_id\", \"(.*)\"\n)\n)\nunless on (instance_id) (\ngroup by (instance_id) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"instance_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n),\n\"cluster_name\", \"$1\", \"instance_id\", \"(.*)\"\n)" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**The 15 collections that took longest to finish hydrating. Hydration time scales with the size of the state being rebuilt, so large materialized views and indexes naturally top the list.**\n\nThe per-collection hydration-time metric is cloud-only, so this is blank on self-managed. Get it from SQL instead: `SELECT object_id, time_ns/1e9 AS seconds FROM mz_internal.mz_compute_hydration_times ORDER BY time_ns DESC`. The live metric-side proxy for what's behind is materialize.compute.freshness.top_collections.", "id": 1044, "links": [], "title": "Slowest Hydrating Collections", "vizConfig": { "group": "barchart", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "fixedColor": "#EE7733", "mode": "shades" }, "custom": { "fillOpacity": 80, "gradientMode": "none", "lineWidth": 1, "scaleDistribution": { "log": 10, "type": "log" }, "thresholdsStyle": { "mode": "area" } }, "noValue": "No matches for the current filters", "thresholds": { "mode": "absolute", "steps": [ { "color": "#CEFFFF", "value": null }, { "color": "#CEFFFF", "value": 2 }, { "color": "#C6F7D6", "value": 6 }, { "color": "#A2F49B", "value": 15 }, { "color": "#BBE453", "value": 37 }, { "color": "#D5CE04", "value": 93 }, { "color": "#E7B503", "value": 231 }, { "color": "#F19903", "value": 573 }, { "color": "#F6790B", "value": 1420 }, { "color": "#F94902", "value": 3518 }, { "color": "#E40515", "value": 8717 }, { "color": "#AB0003", "value": 21599 } ] }, "unit": "s" }, "overrides": [] }, "options": { "barRadius": 0, "barWidth": 0.8, "fullHighlight": false, "groupWidth": 0.95, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": false }, "orientation": "horizontal", "showValue": "auto", "stacking": "none", "tooltip": { "mode": "single", "sort": "asc" }, "xTickLabelMaxLength": 0, "xTickLabelRotation": 0, "xTickLabelSpacing": 100 } }, "version": "" } } }, "hydration-unhydrated-count": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "(count(\n max by (instance_id, collection_id) (\n mz_dataflow_wallclock_lag_seconds{materialize_cloud_organization_name=~\"$environmentNameList\", instance_id=~\"$mzClusterList\", instance_id!=\"\", quantile=\"1\"} > 1e15\n )\n)\n) or vector(0)", "legendFormat": "hydrating", "qryType": 1, "query": "(count(\n max by (instance_id, collection_id) (\n mz_dataflow_wallclock_lag_seconds{materialize_cloud_organization_name=~\"$environmentNameList\", instance_id=~\"$mzClusterList\", instance_id!=\"\", quantile=\"1\"} > 1e15\n )\n)\n) or vector(0)" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Collections still rebuilding their in-memory state — a live hydration-queue proxy. After a restart, replica creation, or some DDL, a dataflow has to rebuild from persisted storage before it can serve, and until it does it produces no results.**\n\n**Nominal:** 0, with brief spikes right after a replica restart that drain back to 0 as dataflows catch up. Those spikes are healthy re-hydration, not a problem.\n\n**Unhealthy:** A count that stays elevated means something can't finish hydrating — a source whose CREATE never completed, or a wedged dataflow. Confirm what with `SELECT * FROM mz_internal.mz_hydration_statuses WHERE NOT hydrated`.\n\nStopgap, not a real signal: self-managed has no hydration-state metric, so this counts collections parked at the wallclock-lag u64::MAX sentinel (> 1e15) as a proxy. Deprecate this query once an actual hydration-state metric exists.", "id": 1042, "links": [], "title": "Currently Hydrating", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "fixedColor": "#EE7733", "mode": "shades" }, "min": 0, "noValue": "No matches for the current filters" }, "overrides": [] }, "options": { "colorMode": "none", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [], "values": false }, "showPercentChange": false, "textMode": "value", "wideLayout": true } }, "version": "" } } }, "index-types": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (relation_type) (mz_indexes_count{materialize_cloud_organization_name=~\"$environmentNameList\"})\n", "instant": true, "legendFormat": "{{relation_type}}", "qryType": 1, "query": "sum by (relation_type) (mz_indexes_count{materialize_cloud_organization_name=~\"$environmentNameList\"})\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Indexes split by the kind of relation they sit on. Workloads normally lean heavily on indexes over views (the standard \"keep a query's result hot\" pattern); a large share of indexes on base tables is unusual and usually worth a second look.**\n\nEnvironment-scoped.", "id": 1038, "links": [], "title": "Index Relationship Types", "vizConfig": { "group": "piechart", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "fixedColor": "#EE7733", "mode": "shades" }, "noValue": "No matches for the current filters" }, "overrides": [] }, "options": { "displayLabels": [ "name", "value" ], "legend": { "asTable": true, "calcs": [], "displayMode": "table", "isVisible": true, "placement": "right", "showLegend": true, "values": [ "value" ] }, "orientation": "auto", "pieType": "donut", "reduceOptions": { "calcs": [], "values": false }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "is-healthy": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "count(\n mz_compute_cluster_status{\n materialize_cloud_organization_name=~\"$environmentNameList\"\n } == 1\n) / count(\n mz_compute_cluster_status{\n materialize_cloud_organization_name=~\"$environmentNameList\"\n }\n) * 100\n", "qryType": 1, "query": "count(\n mz_compute_cluster_status{\n materialize_cloud_organization_name=~\"$environmentNameList\"\n } == 1\n) / count(\n mz_compute_cluster_status{\n materialize_cloud_organization_name=~\"$environmentNameList\"\n }\n) * 100\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**The share of the environment's clusters currently reporting ready — the at-a-glance health headline for the whole environment.**\n\n**Nominal:** Every cluster is ready.\n\n**Degraded:** Some clusters aren't ready, but most are. Usually a rollout in flight — replicas cycling during an upgrade or resize — rather than a real outage. Find which clusters are affected before reacting; a brief dip during a known change is expected.\n\n**Unhealthy:** A large fraction of clusters are down and not recovering on their own. Check pod restart history and narrow to the specific clusters and replicas involved; this is an outage, not a rollout.\n\nThe healthy/degraded/unhealthy bands (100% / 80-100% / <80%) are set by the consuming dashboard, not by this query.", "id": 1000, "links": [], "title": "Environment Status", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "mappings": [ { "options": { "from": 100, "result": { "color": "#A2F49B", "index": 1, "text": "Healthy" }, "to": 2147483647 }, "type": "range" }, { "options": { "from": 80, "result": { "color": "#F19903", "index": 2, "text": "Degraded" }, "to": 100 }, "type": "range" }, { "options": { "from": -2147483647, "result": { "color": "#AB0003", "index": 3, "text": "Unhealthy" }, "to": 80 }, "type": "range" }, { "options": { "match": "null+nan", "result": { "color": "#888888", "index": 4, "text": "Missing Data" } }, "type": "special" } ], "noValue": "No matches for the current filters" }, "overrides": [] }, "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [], "values": false }, "showPercentChange": false, "textMode": "value", "wideLayout": true } }, "version": "" } } }, "k8s-res-cpu-total": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (container) (\n container_spec_cpu_quota{namespace=~\"$mzNamespaceList\",container!=\"\",container!=\"POD\"}\n / container_spec_cpu_period{namespace=~\"$mzNamespaceList\",container!=\"\",container!=\"POD\"}\n)\n", "instant": true, "legendFormat": "CPUs ({{container}})", "qryType": 1, "query": "sum by (container) (\n container_spec_cpu_quota{namespace=~\"$mzNamespaceList\",container!=\"\",container!=\"POD\"}\n / container_spec_cpu_period{namespace=~\"$mzNamespaceList\",container!=\"\",container!=\"POD\"}\n)\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Total CPU cores configured across every container in the environment, including the monitoring exporter — the Kubernetes view of what is provisioned rather than what is available to the workload.**\n\nDiffers from `materialize.kubernetes.cpu.capacity` only by counting the monitoring exporter. Use that one to read workload capacity; use this one when reconciling against what Kubernetes reports. Requires cAdvisor.", "id": 1010, "links": [], "title": "Total CPU Capacity", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "fixedColor": "#0077BB", "mode": "shades" }, "noValue": "No metrics: cadvisor metrics are required (via kubelet)", "unit": "cores" }, "overrides": [] }, "options": { "colorMode": "none", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [], "values": false }, "showPercentChange": false, "textMode": "value_and_name", "wideLayout": true } }, "version": "" } } }, "k8s-res-memory-total": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (container) (\n container_spec_memory_limit_bytes{namespace=~\"$mzNamespaceList\",container!=\"\",container!=\"POD\"}\n)\n", "instant": true, "legendFormat": "Memory ({{container}})", "qryType": 1, "query": "sum by (container) (\n container_spec_memory_limit_bytes{namespace=~\"$mzNamespaceList\",container!=\"\",container!=\"POD\"}\n)\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Total memory configured across every container in the environment, including the monitoring exporter — the Kubernetes view of what is provisioned rather than what is available to the workload.**\n\nDiffers from `materialize.kubernetes.memory.capacity` only by counting the monitoring exporter. Memory is the dominant constraint on Materialize: in-memory arrangements live inside this total. Requires cAdvisor.", "id": 1011, "links": [], "title": "Total Memory", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "fixedColor": "#0077BB", "mode": "shades" }, "noValue": "No metrics: cadvisor metrics are required (via kubelet)", "unit": "bytes" }, "overrides": [] }, "options": { "colorMode": "none", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [], "values": false }, "showPercentChange": false, "textMode": "value_and_name", "wideLayout": true } }, "version": "" } } }, "last-restart": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "time()\n- topk(1,\n container_start_time_seconds{namespace=~\"$mzNamespaceList\",container!=\"\",container!=\"POD\", container!=\"new-promsql-exporter\"}\n )\n", "instant": true, "legendFormat": "{{pod}}", "qryType": 1, "query": "time()\n- topk(1,\n container_start_time_seconds{namespace=~\"$mzNamespaceList\",container!=\"\",container!=\"POD\", container!=\"new-promsql-exporter\"}\n )\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Seconds since the most recent container restart in the environment.**\n\n**Nominal:** Hours to days ago.\n\n**Degraded:** A restart seconds/minutes ago is worth correlating with availability and pod-health — it may be an active incident (crash loop, OOM, eviction).\n\nRequires cAdvisor.", "id": 1002, "links": [], "title": "Last Restart Time", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "noValue": "No metrics: cadvisor metrics are required (via kubelet)", "thresholds": { "mode": "absolute", "steps": [ { "color": "#AB0003", "value": null }, { "color": "#AB0003", "value": 2 }, { "color": "#E40515", "value": 8 }, { "color": "#F94902", "value": 26 }, { "color": "#F6790B", "value": 80 }, { "color": "#F19903", "value": 240 }, { "color": "#E7B503", "value": 719 }, { "color": "#D5CE04", "value": 2152 }, { "color": "#BBE453", "value": 6443 }, { "color": "#A2F49B", "value": 19286 }, { "color": "#C6F7D6", "value": 57730 }, { "color": "#CEFFFF", "value": 172800 } ] }, "unit": "s" }, "overrides": [] }, "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "center", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [], "values": false }, "showPercentChange": false, "text": { "valueSize": 25 }, "textMode": "value_and_name", "wideLayout": true } }, "version": "" } } }, "materialize-version": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "group by (mz_version) (\n mz_compute_cluster_status{materialize_cloud_organization_name=~\"$environmentNameList\"}\n)\n", "instant": true, "legendFormat": "{{mz_version}}", "qryType": 1, "query": "group by (mz_version) (\n mz_compute_cluster_status{materialize_cloud_organization_name=~\"$environmentNameList\"}\n)\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [ { "group": "labelsToFields", "kind": "Transformation", "spec": { "options": {} } }, { "group": "extractFields", "kind": "Transformation", "spec": { "options": { "format": "regexp", "regExp": "/.+\\((?[a-fA-F0-9]+)\\)/", "replace": false, "source": "mz_version" } } } ] } }, "description": "**The version of Materialize running in the environment. A single version is the steady state; multiple values appear briefly during a rolling upgrade.**\n\nThe presentation layer extracts the commit hash from the `mz_version` label (e.g. to link the commit). If the version isn't what you expect, check the environment's manifest in the deployment pipeline.", "id": 1007, "links": [], "title": "Materialize Version", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "links": [ { "targetBlank": true, "title": "View Materialize at Commit", "url": "https://github.com/MaterializeInc/materialize/commit/${__data.fields.commit}" } ] }, "overrides": [] }, "options": { "colorMode": "none", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [], "fields": "/^mz_version$/", "values": false }, "showPercentChange": false, "text": { "valueSize": 20 }, "textMode": "value", "wideLayout": true } }, "version": "" } } }, "memory-usage-current": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (namespace, container) (\n avg by (namespace, pod, container) (\n container_memory_working_set_bytes{namespace=~\"$mzNamespaceList\",container!=\"\",container!=\"POD\", container!=\"new-promsql-exporter\"}\n )\n) / sum by (namespace, container) (\n avg by (namespace, pod, container) (\n container_spec_memory_limit_bytes{namespace=~\"$mzNamespaceList\",container!=\"\",container!=\"POD\", container!=\"new-promsql-exporter\"}\n )\n)\n", "instant": true, "legendFormat": "{{container}}", "qryType": 1, "query": "sum by (namespace, container) (\n avg by (namespace, pod, container) (\n container_memory_working_set_bytes{namespace=~\"$mzNamespaceList\",container!=\"\",container!=\"POD\", container!=\"new-promsql-exporter\"}\n )\n) / sum by (namespace, container) (\n avg by (namespace, pod, container) (\n container_spec_memory_limit_bytes{namespace=~\"$mzNamespaceList\",container!=\"\",container!=\"POD\", container!=\"new-promsql-exporter\"}\n )\n)\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Current memory usage per container type as a fraction of its limit — shows the worst-loaded container types.**\n\n**Unhealthy:** Sustained near 1.0 is dangerous: hitting the limit triggers an OOM-kill, which on a compute replica forces a hydration cycle (in-memory state rebuilt from persisted storage, minutes-to-hours). The offending workload usually shows in the arrangement record counts.\n\nNeeds memory limits from kube-state-metrics; use the `.absolute` sibling where limits aren't exposed. Requires cAdvisor + kube-state-metrics.", "id": 1006, "links": [], "title": "Current Memory Usage", "vizConfig": { "group": "gauge", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "noValue": "No metrics: cadvisor metrics are required (via kubelet)", "thresholds": { "mode": "absolute", "steps": [ { "color": "#CEFFFF", "value": null }, { "color": "#CEFFFF", "value": 0 }, { "color": "#C6F7D6", "value": 0.1 }, { "color": "#A2F49B", "value": 0.2 }, { "color": "#BBE453", "value": 0.30000000000000004 }, { "color": "#D5CE04", "value": 0.4 }, { "color": "#E7B503", "value": 0.5 }, { "color": "#F19903", "value": 0.6000000000000001 }, { "color": "#F6790B", "value": 0.7000000000000001 }, { "color": "#F94902", "value": 0.8 }, { "color": "#E40515", "value": 0.9 }, { "color": "#AB0003", "value": 1 } ] }, "unit": "percentunit" }, "overrides": [] }, "options": { "minVizHeight": 75, "minVizWidth": 75, "orientation": "auto", "reduceOptions": { "calcs": [], "values": false }, "showThresholdLabels": false, "showThresholdMarkers": true, "sizing": "auto" } }, "version": "" } } }, "pod-cpu-percent": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (namespace, pod, container) (\n rate(container_cpu_usage_seconds_total{namespace=~\"$mzNamespaceList\",container!=\"\",container!=\"POD\", pod=~\".*-cluster-${mzClusterList:regex}-replica-${mzReplicaList:regex}-.*\"}[$__rate_interval])\n) / sum by (namespace, pod, container) (\n kube_pod_container_resource_limits{resource=\"cpu\", namespace=~\"$mzNamespaceList\", pod=~\".*-cluster-${mzClusterList:regex}-replica-${mzReplicaList:regex}-.*\"}\n)\n", "legendFormat": "{{pod}} / {{container}}", "qryType": 1, "query": "sum by (namespace, pod, container) (\n rate(container_cpu_usage_seconds_total{namespace=~\"$mzNamespaceList\",container!=\"\",container!=\"POD\", pod=~\".*-cluster-${mzClusterList:regex}-replica-${mzReplicaList:regex}-.*\"}[$__rate_interval])\n) / sum by (namespace, pod, container) (\n kube_pod_container_resource_limits{resource=\"cpu\", namespace=~\"$mzNamespaceList\", pod=~\".*-cluster-${mzClusterList:regex}-replica-${mzReplicaList:regex}-.*\"}\n)\n" }, "version": "v0" }, "refId": "query-0" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (namespace, pod, container) (\n rate(container_cpu_usage_seconds_total{namespace=~\"$mzNamespaceList\",container!=\"\",container!=\"POD\", pod!~\".*-cluster-.*-replica-.*\"}[$__rate_interval])\n) / sum by (namespace, pod, container) (\n kube_pod_container_resource_limits{resource=\"cpu\", namespace=~\"$mzNamespaceList\", pod!~\".*-cluster-.*-replica-.*\"}\n)\n", "legendFormat": "{{pod}} / {{container}}", "qryType": 1, "query": "sum by (namespace, pod, container) (\n rate(container_cpu_usage_seconds_total{namespace=~\"$mzNamespaceList\",container!=\"\",container!=\"POD\", pod!~\".*-cluster-.*-replica-.*\"}[$__rate_interval])\n) / sum by (namespace, pod, container) (\n kube_pod_container_resource_limits{resource=\"cpu\", namespace=~\"$mzNamespaceList\", pod!~\".*-cluster-.*-replica-.*\"}\n)\n" }, "version": "v0" }, "refId": "query-1" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**CPU utilization per pod as a fraction of the pod's limit. Split so the cluster/replica selectors filter the cluster pods while envd/balancer/ exporter stay visible.**\n\n**Degraded:** Sustained near 1.0 for a pod means it's CPU-bound; for the Materialize-level cause see the dataflow elapsed / arrangement rates.\n\nRequires cAdvisor + kube-state-metrics.", "id": 1015, "links": [], "title": "Pod CPU Usage", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "noValue": "No metrics: cadvisor and kube-state-metrics are required", "unit": "percentunit" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "pod-memory-percent": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "avg by (namespace, pod, container) (\n container_memory_working_set_bytes{namespace=~\"$mzNamespaceList\",container!=\"\",container!=\"POD\", container!=\"new-promsql-exporter\", pod=~\".*-cluster-${mzClusterList:regex}-replica-${mzReplicaList:regex}-.*\"}\n) / avg by (namespace, pod, container) (\n container_spec_memory_limit_bytes{namespace=~\"$mzNamespaceList\",container!=\"\",container!=\"POD\", container!=\"new-promsql-exporter\", pod=~\".*-cluster-${mzClusterList:regex}-replica-${mzReplicaList:regex}-.*\"}\n)\n", "legendFormat": "{{pod}} / {{container}}", "qryType": 1, "query": "avg by (namespace, pod, container) (\n container_memory_working_set_bytes{namespace=~\"$mzNamespaceList\",container!=\"\",container!=\"POD\", container!=\"new-promsql-exporter\", pod=~\".*-cluster-${mzClusterList:regex}-replica-${mzReplicaList:regex}-.*\"}\n) / avg by (namespace, pod, container) (\n container_spec_memory_limit_bytes{namespace=~\"$mzNamespaceList\",container!=\"\",container!=\"POD\", container!=\"new-promsql-exporter\", pod=~\".*-cluster-${mzClusterList:regex}-replica-${mzReplicaList:regex}-.*\"}\n)\n" }, "version": "v0" }, "refId": "query-0" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "avg by (namespace, pod, container) (\n container_memory_working_set_bytes{namespace=~\"$mzNamespaceList\",container!=\"\",container!=\"POD\", container!=\"new-promsql-exporter\", pod!~\".*-cluster-.*-replica-.*\"}\n) / avg by (namespace, pod, container) (\n container_spec_memory_limit_bytes{namespace=~\"$mzNamespaceList\",container!=\"\",container!=\"POD\", container!=\"new-promsql-exporter\", pod!~\".*-cluster-.*-replica-.*\"}\n)\n", "legendFormat": "{{pod}} / {{container}}", "qryType": 1, "query": "avg by (namespace, pod, container) (\n container_memory_working_set_bytes{namespace=~\"$mzNamespaceList\",container!=\"\",container!=\"POD\", container!=\"new-promsql-exporter\", pod!~\".*-cluster-.*-replica-.*\"}\n) / avg by (namespace, pod, container) (\n container_spec_memory_limit_bytes{namespace=~\"$mzNamespaceList\",container!=\"\",container!=\"POD\", container!=\"new-promsql-exporter\", pod!~\".*-cluster-.*-replica-.*\"}\n)\n" }, "version": "v0" }, "refId": "query-1" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Memory usage per pod as a fraction of the pod's limit (working-set basis), same cluster/non-cluster split as pod CPU.**\n\n**Unhealthy:** A sustained climb toward 1.0 is dangerous — the pod gets OOM-killed, and on a compute replica that triggers a hydration cycle. If a cluster pod is the offender, the arrangement record counts show which arrangements hold the memory.\n\nRequires cAdvisor.", "id": 1016, "links": [], "title": "Pod Memory Usage", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "noValue": "No metrics: cadvisor metrics are required (via kubelet)", "unit": "percentunit" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "pod-network-drops": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (namespace, pod) (\n rate(container_network_receive_packets_dropped_total{namespace=~\"$mzNamespaceList\", pod=~\".*-cluster-${mzClusterList:regex}-replica-${mzReplicaList:regex}-.*\"}[$__rate_interval])\n)\n", "legendFormat": "{{pod}} rx", "qryType": 1, "query": "sum by (namespace, pod) (\n rate(container_network_receive_packets_dropped_total{namespace=~\"$mzNamespaceList\", pod=~\".*-cluster-${mzClusterList:regex}-replica-${mzReplicaList:regex}-.*\"}[$__rate_interval])\n)\n" }, "version": "v0" }, "refId": "query-0" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (namespace, pod) (\n rate(container_network_receive_packets_dropped_total{namespace=~\"$mzNamespaceList\", pod!~\".*-cluster-.*-replica-.*\"}[$__rate_interval])\n)\n", "legendFormat": "{{pod}} rx", "qryType": 1, "query": "sum by (namespace, pod) (\n rate(container_network_receive_packets_dropped_total{namespace=~\"$mzNamespaceList\", pod!~\".*-cluster-.*-replica-.*\"}[$__rate_interval])\n)\n" }, "version": "v0" }, "refId": "query-1" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (namespace, pod) (\n rate(container_network_transmit_packets_dropped_total{namespace=~\"$mzNamespaceList\", pod=~\".*-cluster-${mzClusterList:regex}-replica-${mzReplicaList:regex}-.*\"}[$__rate_interval])\n)\n", "legendFormat": "{{pod}} tx", "qryType": 1, "query": "sum by (namespace, pod) (\n rate(container_network_transmit_packets_dropped_total{namespace=~\"$mzNamespaceList\", pod=~\".*-cluster-${mzClusterList:regex}-replica-${mzReplicaList:regex}-.*\"}[$__rate_interval])\n)\n" }, "version": "v0" }, "refId": "query-2" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (namespace, pod) (\n rate(container_network_transmit_packets_dropped_total{namespace=~\"$mzNamespaceList\", pod!~\".*-cluster-.*-replica-.*\"}[$__rate_interval])\n)\n", "legendFormat": "{{pod}} tx", "qryType": 1, "query": "sum by (namespace, pod) (\n rate(container_network_transmit_packets_dropped_total{namespace=~\"$mzNamespaceList\", pod!~\".*-cluster-.*-replica-.*\"}[$__rate_interval])\n)\n" }, "version": "v0" }, "refId": "query-3" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Network packets dropped (rx + tx) per pod per second — when kernel buffers fill faster than the app reads (rx) or egress rate-limiting kicks in (tx).**\n\n**Nominal:** 0 (single-digit pps drops are usually harmless background noise).\n\n**Degraded:** Sustained higher rates mean the pod is overwhelmed at the network layer — often paired with elevated pod CPU.\n\nRequires cAdvisor.", "id": 1020, "links": [], "title": "Pod Network Packet Drops", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "noValue": "No metrics: cadvisor metrics are required (via kubelet)", "unit": "pps" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "pod-network-errors": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (namespace, pod) (\n rate(container_network_receive_errors_total{namespace=~\"$mzNamespaceList\", pod=~\".*-cluster-${mzClusterList:regex}-replica-${mzReplicaList:regex}-.*\"}[$__rate_interval])\n)\n", "legendFormat": "{{pod}} rx", "qryType": 1, "query": "sum by (namespace, pod) (\n rate(container_network_receive_errors_total{namespace=~\"$mzNamespaceList\", pod=~\".*-cluster-${mzClusterList:regex}-replica-${mzReplicaList:regex}-.*\"}[$__rate_interval])\n)\n" }, "version": "v0" }, "refId": "query-0" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (namespace, pod) (\n rate(container_network_receive_errors_total{namespace=~\"$mzNamespaceList\", pod!~\".*-cluster-.*-replica-.*\"}[$__rate_interval])\n)\n", "legendFormat": "{{pod}} rx", "qryType": 1, "query": "sum by (namespace, pod) (\n rate(container_network_receive_errors_total{namespace=~\"$mzNamespaceList\", pod!~\".*-cluster-.*-replica-.*\"}[$__rate_interval])\n)\n" }, "version": "v0" }, "refId": "query-1" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (namespace, pod) (\n rate(container_network_transmit_errors_total{namespace=~\"$mzNamespaceList\", pod=~\".*-cluster-${mzClusterList:regex}-replica-${mzReplicaList:regex}-.*\"}[$__rate_interval])\n)\n", "legendFormat": "{{pod}} tx", "qryType": 1, "query": "sum by (namespace, pod) (\n rate(container_network_transmit_errors_total{namespace=~\"$mzNamespaceList\", pod=~\".*-cluster-${mzClusterList:regex}-replica-${mzReplicaList:regex}-.*\"}[$__rate_interval])\n)\n" }, "version": "v0" }, "refId": "query-2" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (namespace, pod) (\n rate(container_network_transmit_errors_total{namespace=~\"$mzNamespaceList\", pod!~\".*-cluster-.*-replica-.*\"}[$__rate_interval])\n)\n", "legendFormat": "{{pod}} tx", "qryType": 1, "query": "sum by (namespace, pod) (\n rate(container_network_transmit_errors_total{namespace=~\"$mzNamespaceList\", pod!~\".*-cluster-.*-replica-.*\"}[$__rate_interval])\n)\n" }, "version": "v0" }, "refId": "query-3" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Network rx + tx errors per pod per second (counted at the NIC/kernel level).**\n\n**Nominal:** 0 — no NIC/kernel-level errors.\n\n**Unhealthy:** Non-zero is unusual and points at infrastructure (faulty NIC, kernel network stack, container runtime) — not a Materialize-level problem; file an infra ticket.\n\nRequires cAdvisor.", "id": 1019, "links": [], "title": "Pod Network Errors", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "noValue": "No metrics: cadvisor metrics are required (via kubelet)", "unit": "cps" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "pod-network-rx": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (namespace, pod) (\n rate(container_network_receive_bytes_total{namespace=~\"$mzNamespaceList\", pod=~\".*-cluster-${mzClusterList:regex}-replica-${mzReplicaList:regex}-.*\"}[$__rate_interval])\n)\n", "legendFormat": "{{pod}}", "qryType": 1, "query": "sum by (namespace, pod) (\n rate(container_network_receive_bytes_total{namespace=~\"$mzNamespaceList\", pod=~\".*-cluster-${mzClusterList:regex}-replica-${mzReplicaList:regex}-.*\"}[$__rate_interval])\n)\n" }, "version": "v0" }, "refId": "query-0" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (namespace, pod) (\n rate(container_network_receive_bytes_total{namespace=~\"$mzNamespaceList\", pod!~\".*-cluster-.*-replica-.*\"}[$__rate_interval])\n)\n", "legendFormat": "{{pod}}", "qryType": 1, "query": "sum by (namespace, pod) (\n rate(container_network_receive_bytes_total{namespace=~\"$mzNamespaceList\", pod!~\".*-cluster-.*-replica-.*\"}[$__rate_interval])\n)\n" }, "version": "v0" }, "refId": "query-1" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Network bytes/sec received per pod. For cluster pods, Rx tracks ingest from upstream and inter-pod replication; for envd/balancer it's client SQL traffic. Surges alongside hydration are normal catchup.**\n\nRequires cAdvisor.", "id": 1017, "links": [], "title": "Pod Network Rx", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "noValue": "No metrics: cadvisor metrics are required (via kubelet)", "unit": "Bps" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "pod-network-tx": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (namespace, pod) (\n rate(container_network_transmit_bytes_total{namespace=~\"$mzNamespaceList\", pod=~\".*-cluster-${mzClusterList:regex}-replica-${mzReplicaList:regex}-.*\"}[$__rate_interval])\n)\n", "legendFormat": "{{pod}}", "qryType": 1, "query": "sum by (namespace, pod) (\n rate(container_network_transmit_bytes_total{namespace=~\"$mzNamespaceList\", pod=~\".*-cluster-${mzClusterList:regex}-replica-${mzReplicaList:regex}-.*\"}[$__rate_interval])\n)\n" }, "version": "v0" }, "refId": "query-0" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (namespace, pod) (\n rate(container_network_transmit_bytes_total{namespace=~\"$mzNamespaceList\", pod!~\".*-cluster-.*-replica-.*\"}[$__rate_interval])\n)\n", "legendFormat": "{{pod}}", "qryType": 1, "query": "sum by (namespace, pod) (\n rate(container_network_transmit_bytes_total{namespace=~\"$mzNamespaceList\", pod!~\".*-cluster-.*-replica-.*\"}[$__rate_interval])\n)\n" }, "version": "v0" }, "refId": "query-1" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Network bytes/sec transmitted per pod. For cluster pods Tx covers sink output, inter-pod replication, and query results returning to envd; for envd it's client query responses.**\n\nRequires cAdvisor.", "id": 1018, "links": [], "title": "Pod Network Tx", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "noValue": "No metrics: cadvisor metrics are required (via kubelet)", "unit": "Bps" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "queries-distribution": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (statement_type) (increase(mz_query_total{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__range])) > 0\n", "instant": true, "legendFormat": "{{statement_type}}", "qryType": 1, "query": "sum by (statement_type) (increase(mz_query_total{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__range])) > 0\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**The mix of query kinds over the selected window — a workload-shape signal, not a rate. Heavy `set_variable`/`fetch` traffic is normal (that's how Postgres clients manage session state); heavy `insert`/`update`/`delete` on something you think of as read-mostly is worth a look.**\n\nIdle statement types are filtered out. Environment-scoped.", "id": 1024, "links": [], "title": "Query Distribution (by statement_type)", "vizConfig": { "group": "piechart", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "noValue": "No matches for the current filters" }, "overrides": [] }, "options": { "displayLabels": [ "name", "value" ], "legend": { "asTable": true, "calcs": [], "displayMode": "table", "isVisible": true, "placement": "right", "showLegend": true, "values": [ "value" ] }, "orientation": "auto", "pieType": "donut", "reduceOptions": { "calcs": [], "values": false }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "queries-peek-latency-p50": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "(\n(\nhistogram_quantile(0.50,\n sum by (le, instance_id) (\n rate(mz_compute_peek_duration_seconds_bucket{materialize_cloud_organization_name=~\"$environmentNameList\", instance_id=~\"$mzClusterList\"}[$__rate_interval])\n )\n)\n\n)\n* on (instance_id) group_left(cluster_name)\nmax by (instance_id, cluster_name) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"instance_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\nhistogram_quantile(0.50,\n sum by (le, instance_id) (\n rate(mz_compute_peek_duration_seconds_bucket{materialize_cloud_organization_name=~\"$environmentNameList\", instance_id=~\"$mzClusterList\"}[$__rate_interval])\n )\n)\n\n)\nunless on (instance_id) (\ngroup by (instance_id) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"instance_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n),\n\"cluster_name\", \"$1\", \"instance_id\", \"(.*)\"\n)", "legendFormat": "{{cluster_name}}", "qryType": 1, "query": "(\n(\nhistogram_quantile(0.50,\n sum by (le, instance_id) (\n rate(mz_compute_peek_duration_seconds_bucket{materialize_cloud_organization_name=~\"$environmentNameList\", instance_id=~\"$mzClusterList\"}[$__rate_interval])\n )\n)\n\n)\n* on (instance_id) group_left(cluster_name)\nmax by (instance_id, cluster_name) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"instance_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\nhistogram_quantile(0.50,\n sum by (le, instance_id) (\n rate(mz_compute_peek_duration_seconds_bucket{materialize_cloud_organization_name=~\"$environmentNameList\", instance_id=~\"$mzClusterList\"}[$__rate_interval])\n )\n)\n\n)\nunless on (instance_id) (\ngroup by (instance_id) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"instance_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n),\n\"cluster_name\", \"$1\", \"instance_id\", \"(.*)\"\n)" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Median read-query latency — the typical time to look up the current state of an arrangement, which is the operation behind every `SELECT` against an index. Your \"what does a normal query feel like\" number.**\n\n**Nominal:** A few milliseconds on a healthy cluster.\n\n**Degraded:** Sustained multi-second p50 means the cluster is overwhelmed — it can't keep up with the read load on it.\n\nPer-cluster: the self-managed metric carries the cluster (instance_id) but not the replica, so the replica selector doesn't split it further.", "id": 1026, "links": [], "title": "Peek Latency (p50)", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "custom": { "scaleDistribution": { "log": 10, "type": "log" } }, "unit": "s" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "queries-peek-latency-p90": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "(\n(\nhistogram_quantile(0.90,\n sum by (le, instance_id) (\n rate(mz_compute_peek_duration_seconds_bucket{materialize_cloud_organization_name=~\"$environmentNameList\", instance_id=~\"$mzClusterList\"}[$__rate_interval])\n )\n)\n\n)\n* on (instance_id) group_left(cluster_name)\nmax by (instance_id, cluster_name) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"instance_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\nhistogram_quantile(0.90,\n sum by (le, instance_id) (\n rate(mz_compute_peek_duration_seconds_bucket{materialize_cloud_organization_name=~\"$environmentNameList\", instance_id=~\"$mzClusterList\"}[$__rate_interval])\n )\n)\n\n)\nunless on (instance_id) (\ngroup by (instance_id) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"instance_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n),\n\"cluster_name\", \"$1\", \"instance_id\", \"(.*)\"\n)", "legendFormat": "{{cluster_name}}", "qryType": 1, "query": "(\n(\nhistogram_quantile(0.90,\n sum by (le, instance_id) (\n rate(mz_compute_peek_duration_seconds_bucket{materialize_cloud_organization_name=~\"$environmentNameList\", instance_id=~\"$mzClusterList\"}[$__rate_interval])\n )\n)\n\n)\n* on (instance_id) group_left(cluster_name)\nmax by (instance_id, cluster_name) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"instance_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\nhistogram_quantile(0.90,\n sum by (le, instance_id) (\n rate(mz_compute_peek_duration_seconds_bucket{materialize_cloud_organization_name=~\"$environmentNameList\", instance_id=~\"$mzClusterList\"}[$__rate_interval])\n )\n)\n\n)\nunless on (instance_id) (\ngroup by (instance_id) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"instance_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n),\n\"cluster_name\", \"$1\", \"instance_id\", \"(.*)\"\n)" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**90th-percentile read-query latency — how slow the slowest 10% of queries feel. Catches the contention bursts and cold paths that the median hides.**\n\n**Nominal:** A small multiple of p50, roughly 2–5×.\n\n**Degraded:** p90 running 10–100× p50 means a bimodal distribution — usually cold-cache effects on infrequently-queried indexes, or contention.", "id": 1027, "links": [], "title": "Peek Latency (p90)", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "custom": { "scaleDistribution": { "log": 10, "type": "log" } }, "unit": "s" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "queries-peek-latency-p99": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "(\n(\nhistogram_quantile(0.99,\n sum by (le, instance_id) (\n rate(mz_compute_peek_duration_seconds_bucket{materialize_cloud_organization_name=~\"$environmentNameList\", instance_id=~\"$mzClusterList\"}[$__rate_interval])\n )\n)\n\n)\n* on (instance_id) group_left(cluster_name)\nmax by (instance_id, cluster_name) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"instance_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\nhistogram_quantile(0.99,\n sum by (le, instance_id) (\n rate(mz_compute_peek_duration_seconds_bucket{materialize_cloud_organization_name=~\"$environmentNameList\", instance_id=~\"$mzClusterList\"}[$__rate_interval])\n )\n)\n\n)\nunless on (instance_id) (\ngroup by (instance_id) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"instance_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n),\n\"cluster_name\", \"$1\", \"instance_id\", \"(.*)\"\n)", "legendFormat": "{{cluster_name}}", "qryType": 1, "query": "(\n(\nhistogram_quantile(0.99,\n sum by (le, instance_id) (\n rate(mz_compute_peek_duration_seconds_bucket{materialize_cloud_organization_name=~\"$environmentNameList\", instance_id=~\"$mzClusterList\"}[$__rate_interval])\n )\n)\n\n)\n* on (instance_id) group_left(cluster_name)\nmax by (instance_id, cluster_name) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"instance_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\nhistogram_quantile(0.99,\n sum by (le, instance_id) (\n rate(mz_compute_peek_duration_seconds_bucket{materialize_cloud_organization_name=~\"$environmentNameList\", instance_id=~\"$mzClusterList\"}[$__rate_interval])\n )\n)\n\n)\nunless on (instance_id) (\ngroup by (instance_id) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"instance_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n),\n\"cluster_name\", \"$1\", \"instance_id\", \"(.*)\"\n)" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Tail read-query latency — the slowest 1% of queries, the ones users complain about.**\n\n**Nominal:** A small multiple of p50 (2–10×), with occasional spikes during query plan recompilation or hydration.\n\n**Degraded:** Sustained p99 in the seconds range — especially when p50/p90 are NOT elevated — points at a single bad query or a genuinely tail-sensitive use case. Confirm the latency is on real traffic (against the query rate), not idle scrapes.", "id": 1028, "links": [], "title": "Peek Latency (p99)", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "custom": { "scaleDistribution": { "log": 10, "type": "log" } }, "unit": "s" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "queries-rate-by-statement": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (statement_type, session_type) (rate(mz_query_total{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval])) > 0\n", "legendFormat": "{{statement_type}} / {{session_type}}", "qryType": 1, "query": "sum by (statement_type, session_type) (rate(mz_query_total{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval])) > 0\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Query rate broken down by statement type and session type, fully time-resolved — the moving picture behind the distribution donut. A spike in `select`/`user` is the thing to line up against peek latency to confirm the system kept pace.**\n\nIdle (statement, session) tuples are filtered out. Environment-scoped.", "id": 1025, "links": [], "title": "Query Rate (by statement_type / session_type)", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "unit": "cps" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "replica-count": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "count(\n group by (compute_cluster_id, compute_replica_id) (\n mz_compute_cluster_status{materialize_cloud_organization_name=~\"$environmentNameList\", compute_cluster_id=~\"$mzClusterList\", compute_replica_id=~\"$mzReplicaList\"}\n )\n)\n", "legendFormat": "Total Replicas", "qryType": 1, "query": "count(\n group by (compute_cluster_id, compute_replica_id) (\n mz_compute_cluster_status{materialize_cloud_organization_name=~\"$environmentNameList\", compute_cluster_id=~\"$mzClusterList\", compute_replica_id=~\"$mzReplicaList\"}\n )\n)\n" }, "version": "v0" }, "refId": "query-0" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "(count(\n group by (compute_cluster_id, compute_replica_id) (\n mz_compute_cluster_status{materialize_cloud_organization_name=~\"$environmentNameList\", compute_cluster_id=~\"$mzClusterList\", compute_replica_id=~\"$mzReplicaList\", compute_replica_name!=\"r1\"}\n )\n)\n) or vector(0)", "legendFormat": "Redundant Replicas", "qryType": 1, "query": "(count(\n group by (compute_cluster_id, compute_replica_id) (\n mz_compute_cluster_status{materialize_cloud_organization_name=~\"$environmentNameList\", compute_cluster_id=~\"$mzClusterList\", compute_replica_id=~\"$mzReplicaList\", compute_replica_name!=\"r1\"}\n )\n)\n) or vector(0)" }, "version": "v0" }, "refId": "query-1" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**How many replicas back the selected clusters, and how many of those are redundancy beyond the first. Every cluster needs one replica to run; anything above that is capacity or availability headroom you've opted into, so a non-zero \"additional\" count is the quick check that HA is actually configured where you expect it.**\n\nSteps on CREATE/DROP CLUSTER REPLICA. The \"additional\" line is hardened to read 0 rather than \"no data\" on clusters with a single replica.", "id": 1031, "links": [], "title": "Replica Count", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "fixedColor": "#009988", "mode": "shades" }, "min": 0, "noValue": "No matches for the current filters" }, "overrides": [] }, "options": { "colorMode": "none", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [], "values": false }, "showPercentChange": false, "textMode": "value_and_name", "wideLayout": true } }, "version": "" } } }, "replica-sizes": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "count by (size) (\n mz_compute_cluster_status{materialize_cloud_organization_name=~\"$environmentNameList\", compute_cluster_id=~\"$mzClusterList\", compute_replica_id=~\"$mzReplicaList\"}\n)\n", "instant": true, "legendFormat": "{{size}}", "qryType": 1, "query": "count by (size) (\n mz_compute_cluster_status{materialize_cloud_organization_name=~\"$environmentNameList\", compute_cluster_id=~\"$mzClusterList\", compute_replica_id=~\"$mzReplicaList\"}\n)\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**The replica fleet grouped by configured size. Most deployments settle on a handful of sizes; a long tail of one-off sizes usually means an experiment or a half-finished migration. The total agrees with the replica count.**", "id": 1032, "links": [], "title": "Replica Sizes", "vizConfig": { "group": "piechart", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "fixedColor": "#009988", "mode": "shades" }, "noValue": "No matches for the current filters" }, "overrides": [] }, "options": { "displayLabels": [ "name", "value" ], "legend": { "asTable": true, "calcs": [], "displayMode": "table", "isVisible": true, "placement": "right", "showLegend": true, "values": [ "value" ] }, "orientation": "auto", "pieType": "pie", "reduceOptions": { "calcs": [], "values": false }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "resource-deployment-status": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (namespace) (\n sum by (namespace, instance) (\n kube_deployment_status_replicas_ready{namespace=~\"$mzNamespaceList\"}\n )\n)\n", "instant": true, "legendFormat": "Ready", "qryType": 1, "query": "max by (namespace) (\n sum by (namespace, instance) (\n kube_deployment_status_replicas_ready{namespace=~\"$mzNamespaceList\"}\n )\n)\n" }, "version": "v0" }, "refId": "query-0" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (namespace) (\n sum by (namespace, instance) (\n kube_deployment_status_replicas_unavailable{namespace=~\"$mzNamespaceList\"}\n )\n)\n", "instant": true, "legendFormat": "Unavailable", "qryType": 1, "query": "max by (namespace) (\n sum by (namespace, instance) (\n kube_deployment_status_replicas_unavailable{namespace=~\"$mzNamespaceList\"}\n )\n)\n" }, "version": "v0" }, "refId": "query-1" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Deployment replica health — Ready vs Unavailable. Deployments back stateless services (e.g. the promsql exporter).**\n\n**Nominal:** All replicas Ready, zero Unavailable.\n\n**Degraded:** Unavailable counts indicate failed rollouts or crashing pods.\n\nRequires kube-state-metrics.\n\nCanonical because the `kube_*` families it reads are asserted against a running stack by `kube_state::labels_are_honored` in the e2e suite. That check exists for one specific failure: when the scrape does not set `honorLabels`, kube-state-metrics' own `namespace` and `pod` labels are renamed `exported_*` and replaced by the exporter pod's own identity, so this query matches nothing while `up` and `scrape_samples_scraped` stay healthy. It shipped that way once and was found by eye.", "id": 1014, "links": [], "title": "Deployment Readiness", "vizConfig": { "group": "piechart", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "fixedColor": "#0077BB", "mode": "shades" }, "noValue": "No metrics: kube-state-metrics is required" }, "overrides": [] }, "options": { "displayLabels": [ "name", "value" ], "legend": { "asTable": true, "calcs": [], "displayMode": "table", "isVisible": true, "placement": "right", "showLegend": true, "values": [ "value" ] }, "orientation": "auto", "pieType": "donut", "reduceOptions": { "calcs": [], "values": false }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "resource-pod-status": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (phase, namespace) (\n sum by (phase, namespace, instance) (\n kube_pod_status_phase{namespace=~\"$mzNamespaceList\"}\n )\n)\n", "instant": true, "legendFormat": "{{phase}}", "qryType": 1, "query": "max by (phase, namespace) (\n sum by (phase, namespace, instance) (\n kube_pod_status_phase{namespace=~\"$mzNamespaceList\"}\n )\n)\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Pods in the Materialize namespace grouped by phase (Running, Pending, Failed, …).**\n\n**Nominal:** Nearly all Running.\n\n**Degraded:** Pending usually means Kubernetes can't schedule them (capacity, taints, AZ constraints); Failed means a container exited and won't be restarted.\n\nRequires kube-state-metrics.\n\nCanonical because the `kube_*` families it reads are asserted against a running stack by `kube_state::labels_are_honored` in the e2e suite. That check exists for one specific failure: when the scrape does not set `honorLabels`, kube-state-metrics' own `namespace` and `pod` labels are renamed `exported_*` and replaced by the exporter pod's own identity, so this query matches nothing while `up` and `scrape_samples_scraped` stay healthy. It shipped that way once and was found by eye.", "id": 1012, "links": [], "title": "Pod Readiness", "vizConfig": { "group": "piechart", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "fixedColor": "#0077BB", "mode": "shades" }, "noValue": "No metrics: kube-state-metrics is required" }, "overrides": [] }, "options": { "displayLabels": [ "name", "value" ], "legend": { "asTable": true, "calcs": [], "displayMode": "table", "isVisible": true, "placement": "right", "showLegend": true, "values": [ "value" ] }, "orientation": "auto", "pieType": "donut", "reduceOptions": { "calcs": [], "values": false }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "resource-statefulset-status": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (namespace) (\n sum by (namespace, instance) (\n kube_statefulset_status_replicas_ready{namespace=~\"$mzNamespaceList\"}\n )\n)\n", "instant": true, "legendFormat": "Ready", "qryType": 1, "query": "max by (namespace) (\n sum by (namespace, instance) (\n kube_statefulset_status_replicas_ready{namespace=~\"$mzNamespaceList\"}\n )\n)\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**StatefulSet replicas reporting Ready. environmentd and the cluster pods are StatefulSets.**\n\n**Nominal:** Matches the configured replica count.\n\n**Degraded:** A drop indicates a pod stuck in initialization or hydration.\n\nRequires kube-state-metrics.\n\nCanonical because the `kube_*` families it reads are asserted against a running stack by `kube_state::labels_are_honored` in the e2e suite. That check exists for one specific failure: when the scrape does not set `honorLabels`, kube-state-metrics' own `namespace` and `pod` labels are renamed `exported_*` and replaced by the exporter pod's own identity, so this query matches nothing while `up` and `scrape_samples_scraped` stay healthy. It shipped that way once and was found by eye.", "id": 1013, "links": [], "title": "StatefulSet Readiness", "vizConfig": { "group": "piechart", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "fixedColor": "#0077BB", "mode": "shades" }, "noValue": "No metrics: kube-state-metrics is required" }, "overrides": [] }, "options": { "displayLabels": [ "name", "value" ], "legend": { "asTable": true, "calcs": [], "displayMode": "table", "isVisible": true, "placement": "right", "showLegend": true, "values": [ "value" ] }, "orientation": "auto", "pieType": "donut", "reduceOptions": { "calcs": [], "values": false }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "sinks-iceberg-commit-latency": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "histogram_quantile(0.50, sum by (le) (max without (job) (rate(mz_sink_iceberg_commit_duration_seconds_bucket{materialize_cloud_organization_name=~\"$environmentNameList\", cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval]))))\n", "legendFormat": "p50", "qryType": 1, "query": "histogram_quantile(0.50, sum by (le) (max without (job) (rate(mz_sink_iceberg_commit_duration_seconds_bucket{materialize_cloud_organization_name=~\"$environmentNameList\", cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval]))))\n" }, "version": "v0" }, "refId": "query-0" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "histogram_quantile(0.90, sum by (le) (max without (job) (rate(mz_sink_iceberg_commit_duration_seconds_bucket{materialize_cloud_organization_name=~\"$environmentNameList\", cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval]))))\n", "legendFormat": "p90", "qryType": 1, "query": "histogram_quantile(0.90, sum by (le) (max without (job) (rate(mz_sink_iceberg_commit_duration_seconds_bucket{materialize_cloud_organization_name=~\"$environmentNameList\", cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval]))))\n" }, "version": "v0" }, "refId": "query-1" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "histogram_quantile(0.99, sum by (le) (max without (job) (rate(mz_sink_iceberg_commit_duration_seconds_bucket{materialize_cloud_organization_name=~\"$environmentNameList\", cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval]))))\n", "legendFormat": "p99", "qryType": 1, "query": "histogram_quantile(0.99, sum by (le) (max without (job) (rate(mz_sink_iceberg_commit_duration_seconds_bucket{materialize_cloud_organization_name=~\"$environmentNameList\", cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval]))))\n" }, "version": "v0" }, "refId": "query-2" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Iceberg commit-duration percentiles (p50/p90/p99) — how long each `COMMIT` against the Iceberg catalog takes (write a snapshot manifest, ask the catalog to atomically swap it in).**\n\n**Nominal:** p50 sub-second to low seconds; p99 a few seconds even when healthy.\n\n**Degraded:** Sustained p99 in the tens of seconds points at a slow Iceberg catalog (REST catalog under load, Glue API throttling); sink lag will be growing at the same time.\n\nScoped to the selected clusters/replicas.", "id": 1064, "links": [], "title": "Iceberg Commit Latency (p50 / p90 / p99)", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "custom": { "scaleDistribution": { "log": 10, "type": "log" } }, "unit": "s" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "sinks-iceberg-failures": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "(\n(\nsum by (sink_id) (max without (job) (rate(mz_sink_iceberg_commit_failures{materialize_cloud_organization_name=~\"$environmentNameList\", cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval])))\n\n)\n* on (sink_id) group_left(name)\nmax by (sink_id, name) (\nlabel_replace(mz_object_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"sink_id\", \"$1\", \"global_id\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\nsum by (sink_id) (max without (job) (rate(mz_sink_iceberg_commit_failures{materialize_cloud_organization_name=~\"$environmentNameList\", cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval])))\n\n)\nunless on (sink_id) (\ngroup by (sink_id) (\nlabel_replace(mz_object_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"sink_id\", \"$1\", \"global_id\", \"(.*)\")\n)\n),\n\"name\", \"$1\", \"sink_id\", \"(.*)\"\n)", "legendFormat": "{{name}} failures", "qryType": 1, "query": "(\n(\nsum by (sink_id) (max without (job) (rate(mz_sink_iceberg_commit_failures{materialize_cloud_organization_name=~\"$environmentNameList\", cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval])))\n\n)\n* on (sink_id) group_left(name)\nmax by (sink_id, name) (\nlabel_replace(mz_object_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"sink_id\", \"$1\", \"global_id\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\nsum by (sink_id) (max without (job) (rate(mz_sink_iceberg_commit_failures{materialize_cloud_organization_name=~\"$environmentNameList\", cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval])))\n\n)\nunless on (sink_id) (\ngroup by (sink_id) (\nlabel_replace(mz_object_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"sink_id\", \"$1\", \"global_id\", \"(.*)\")\n)\n),\n\"name\", \"$1\", \"sink_id\", \"(.*)\"\n)" }, "version": "v0" }, "refId": "query-0" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "(\n(\nsum by (sink_id) (max without (job) (rate(mz_sink_iceberg_commit_conflicts{materialize_cloud_organization_name=~\"$environmentNameList\", cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval])))\n\n)\n* on (sink_id) group_left(name)\nmax by (sink_id, name) (\nlabel_replace(mz_object_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"sink_id\", \"$1\", \"global_id\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\nsum by (sink_id) (max without (job) (rate(mz_sink_iceberg_commit_conflicts{materialize_cloud_organization_name=~\"$environmentNameList\", cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval])))\n\n)\nunless on (sink_id) (\ngroup by (sink_id) (\nlabel_replace(mz_object_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"sink_id\", \"$1\", \"global_id\", \"(.*)\")\n)\n),\n\"name\", \"$1\", \"sink_id\", \"(.*)\"\n)", "legendFormat": "{{name}} conflicts", "qryType": 1, "query": "(\n(\nsum by (sink_id) (max without (job) (rate(mz_sink_iceberg_commit_conflicts{materialize_cloud_organization_name=~\"$environmentNameList\", cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval])))\n\n)\n* on (sink_id) group_left(name)\nmax by (sink_id, name) (\nlabel_replace(mz_object_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"sink_id\", \"$1\", \"global_id\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\nsum by (sink_id) (max without (job) (rate(mz_sink_iceberg_commit_conflicts{materialize_cloud_organization_name=~\"$environmentNameList\", cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval])))\n\n)\nunless on (sink_id) (\ngroup by (sink_id) (\nlabel_replace(mz_object_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"sink_id\", \"$1\", \"global_id\", \"(.*)\")\n)\n),\n\"name\", \"$1\", \"sink_id\", \"(.*)\"\n)" }, "version": "v0" }, "refId": "query-1" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Per-sink rate of failed and conflicting Iceberg commits. Conflicts (concurrent-writer races on the snapshot pointer) are recoverable — Materialize retries — but a high rate means something else is writing the same Iceberg table; failures are commit-side errors (network, auth, schema).**\n\n**Unhealthy:** Non-zero in either dimension is worth investigating; climbing failures drag sink lag up with them.\n\nScoped to the selected clusters/replicas.", "id": 1065, "links": [], "title": "Iceberg Commit Failures & Conflicts", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "#A2F49B", "value": null }, { "color": "#FEDA8B", "value": 1 }, { "color": "#FDB366", "value": 3.25 }, { "color": "#F67E4B", "value": 5.5 }, { "color": "#DD3D2D", "value": 7.75 }, { "color": "#A50026", "value": 10 } ] }, "unit": "cps" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "sinks-iceberg-files": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "(\n(\nsum by (sink_id) (max without (job) (rate(mz_sink_iceberg_data_files_written{materialize_cloud_organization_name=~\"$environmentNameList\", cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval])))\n\n)\n* on (sink_id) group_left(name)\nmax by (sink_id, name) (\nlabel_replace(mz_object_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"sink_id\", \"$1\", \"global_id\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\nsum by (sink_id) (max without (job) (rate(mz_sink_iceberg_data_files_written{materialize_cloud_organization_name=~\"$environmentNameList\", cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval])))\n\n)\nunless on (sink_id) (\ngroup by (sink_id) (\nlabel_replace(mz_object_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"sink_id\", \"$1\", \"global_id\", \"(.*)\")\n)\n),\n\"name\", \"$1\", \"sink_id\", \"(.*)\"\n)", "legendFormat": "{{name}} data", "qryType": 1, "query": "(\n(\nsum by (sink_id) (max without (job) (rate(mz_sink_iceberg_data_files_written{materialize_cloud_organization_name=~\"$environmentNameList\", cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval])))\n\n)\n* on (sink_id) group_left(name)\nmax by (sink_id, name) (\nlabel_replace(mz_object_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"sink_id\", \"$1\", \"global_id\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\nsum by (sink_id) (max without (job) (rate(mz_sink_iceberg_data_files_written{materialize_cloud_organization_name=~\"$environmentNameList\", cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval])))\n\n)\nunless on (sink_id) (\ngroup by (sink_id) (\nlabel_replace(mz_object_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"sink_id\", \"$1\", \"global_id\", \"(.*)\")\n)\n),\n\"name\", \"$1\", \"sink_id\", \"(.*)\"\n)" }, "version": "v0" }, "refId": "query-0" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "(\n(\nsum by (sink_id) (max without (job) (rate(mz_sink_iceberg_delete_files_written{materialize_cloud_organization_name=~\"$environmentNameList\", cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval])))\n\n)\n* on (sink_id) group_left(name)\nmax by (sink_id, name) (\nlabel_replace(mz_object_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"sink_id\", \"$1\", \"global_id\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\nsum by (sink_id) (max without (job) (rate(mz_sink_iceberg_delete_files_written{materialize_cloud_organization_name=~\"$environmentNameList\", cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval])))\n\n)\nunless on (sink_id) (\ngroup by (sink_id) (\nlabel_replace(mz_object_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"sink_id\", \"$1\", \"global_id\", \"(.*)\")\n)\n),\n\"name\", \"$1\", \"sink_id\", \"(.*)\"\n)", "legendFormat": "{{name}} deletes", "qryType": 1, "query": "(\n(\nsum by (sink_id) (max without (job) (rate(mz_sink_iceberg_delete_files_written{materialize_cloud_organization_name=~\"$environmentNameList\", cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval])))\n\n)\n* on (sink_id) group_left(name)\nmax by (sink_id, name) (\nlabel_replace(mz_object_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"sink_id\", \"$1\", \"global_id\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\nsum by (sink_id) (max without (job) (rate(mz_sink_iceberg_delete_files_written{materialize_cloud_organization_name=~\"$environmentNameList\", cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval])))\n\n)\nunless on (sink_id) (\ngroup by (sink_id) (\nlabel_replace(mz_object_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"sink_id\", \"$1\", \"global_id\", \"(.*)\")\n)\n),\n\"name\", \"$1\", \"sink_id\", \"(.*)\"\n)" }, "version": "v0" }, "refId": "query-1" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "(\n(\nsum by (sink_id) (max without (job) (rate(mz_sink_iceberg_snapshots_committed{materialize_cloud_organization_name=~\"$environmentNameList\", cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval])))\n\n)\n* on (sink_id) group_left(name)\nmax by (sink_id, name) (\nlabel_replace(mz_object_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"sink_id\", \"$1\", \"global_id\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\nsum by (sink_id) (max without (job) (rate(mz_sink_iceberg_snapshots_committed{materialize_cloud_organization_name=~\"$environmentNameList\", cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval])))\n\n)\nunless on (sink_id) (\ngroup by (sink_id) (\nlabel_replace(mz_object_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"sink_id\", \"$1\", \"global_id\", \"(.*)\")\n)\n),\n\"name\", \"$1\", \"sink_id\", \"(.*)\"\n)", "legendFormat": "{{name}} snapshots", "qryType": 1, "query": "(\n(\nsum by (sink_id) (max without (job) (rate(mz_sink_iceberg_snapshots_committed{materialize_cloud_organization_name=~\"$environmentNameList\", cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval])))\n\n)\n* on (sink_id) group_left(name)\nmax by (sink_id, name) (\nlabel_replace(mz_object_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"sink_id\", \"$1\", \"global_id\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\nsum by (sink_id) (max without (job) (rate(mz_sink_iceberg_snapshots_committed{materialize_cloud_organization_name=~\"$environmentNameList\", cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval])))\n\n)\nunless on (sink_id) (\ngroup by (sink_id) (\nlabel_replace(mz_object_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"sink_id\", \"$1\", \"global_id\", \"(.*)\")\n)\n),\n\"name\", \"$1\", \"sink_id\", \"(.*)\"\n)" }, "version": "v0" }, "refId": "query-2" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Per-sink rate of files and snapshots written to Iceberg. Each commit produces one snapshot with data files (new rows) and delete files (tombstones for upserts). The data:delete ratio reflects your workload — pure-insert sinks produce ~0 deletes; upsert-heavy ones roughly 1:1.**\n\nSustained delete-file rate with no data files means the sink is mostly deleting (data evaporating upstream). Scoped to the selected clusters/replicas.", "id": 1066, "links": [], "title": "Iceberg File & Snapshot Rate", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "unit": "cps" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "sinks-kafka-connects": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "(\n(\nsum by (sink_id) (max without (job) (rate(mz_sink_rdkafka_connects{materialize_cloud_organization_name=~\"$environmentNameList\", cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval])))\n\n)\n* on (sink_id) group_left(name)\nmax by (sink_id, name) (\nlabel_replace(mz_object_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"sink_id\", \"$1\", \"global_id\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\nsum by (sink_id) (max without (job) (rate(mz_sink_rdkafka_connects{materialize_cloud_organization_name=~\"$environmentNameList\", cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval])))\n\n)\nunless on (sink_id) (\ngroup by (sink_id) (\nlabel_replace(mz_object_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"sink_id\", \"$1\", \"global_id\", \"(.*)\")\n)\n),\n\"name\", \"$1\", \"sink_id\", \"(.*)\"\n)", "legendFormat": "{{name}} connects", "qryType": 1, "query": "(\n(\nsum by (sink_id) (max without (job) (rate(mz_sink_rdkafka_connects{materialize_cloud_organization_name=~\"$environmentNameList\", cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval])))\n\n)\n* on (sink_id) group_left(name)\nmax by (sink_id, name) (\nlabel_replace(mz_object_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"sink_id\", \"$1\", \"global_id\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\nsum by (sink_id) (max without (job) (rate(mz_sink_rdkafka_connects{materialize_cloud_organization_name=~\"$environmentNameList\", cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval])))\n\n)\nunless on (sink_id) (\ngroup by (sink_id) (\nlabel_replace(mz_object_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"sink_id\", \"$1\", \"global_id\", \"(.*)\")\n)\n),\n\"name\", \"$1\", \"sink_id\", \"(.*)\"\n)" }, "version": "v0" }, "refId": "query-0" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "(\n(\nsum by (sink_id) (max without (job) (rate(mz_sink_rdkafka_disconnects{materialize_cloud_organization_name=~\"$environmentNameList\", cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval])))\n\n)\n* on (sink_id) group_left(name)\nmax by (sink_id, name) (\nlabel_replace(mz_object_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"sink_id\", \"$1\", \"global_id\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\nsum by (sink_id) (max without (job) (rate(mz_sink_rdkafka_disconnects{materialize_cloud_organization_name=~\"$environmentNameList\", cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval])))\n\n)\nunless on (sink_id) (\ngroup by (sink_id) (\nlabel_replace(mz_object_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"sink_id\", \"$1\", \"global_id\", \"(.*)\")\n)\n),\n\"name\", \"$1\", \"sink_id\", \"(.*)\"\n)", "legendFormat": "{{name}} disconnects", "qryType": 1, "query": "(\n(\nsum by (sink_id) (max without (job) (rate(mz_sink_rdkafka_disconnects{materialize_cloud_organization_name=~\"$environmentNameList\", cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval])))\n\n)\n* on (sink_id) group_left(name)\nmax by (sink_id, name) (\nlabel_replace(mz_object_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"sink_id\", \"$1\", \"global_id\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\nsum by (sink_id) (max without (job) (rate(mz_sink_rdkafka_disconnects{materialize_cloud_organization_name=~\"$environmentNameList\", cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval])))\n\n)\nunless on (sink_id) (\ngroup by (sink_id) (\nlabel_replace(mz_object_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"sink_id\", \"$1\", \"global_id\", \"(.*)\")\n)\n),\n\"name\", \"$1\", \"sink_id\", \"(.*)\"\n)" }, "version": "v0" }, "refId": "query-1" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Connect and disconnect events per sink against the Kafka broker. Healthy connections are persistent — a couple of connects at startup and zero disconnects afterward.**\n\n**Unhealthy:** A sustained non-zero disconnect rate signals unhealthy connectivity (network flakiness, broker restarting, auth tokens expiring); pairs with TX errors when the issue is broker-side rather than purely network.\n\nScoped to the selected clusters/replicas.", "id": 1069, "links": [], "title": "Kafka Connect / Disconnect Rate", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "unit": "cps" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "sinks-kafka-outbuf": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "(\n(\nsum by (sink_id) (max without (job) (mz_sink_rdkafka_outbuf_msg_cnt{materialize_cloud_organization_name=~\"$environmentNameList\", cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}))\n\n)\n* on (sink_id) group_left(name)\nmax by (sink_id, name) (\nlabel_replace(mz_object_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"sink_id\", \"$1\", \"global_id\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\nsum by (sink_id) (max without (job) (mz_sink_rdkafka_outbuf_msg_cnt{materialize_cloud_organization_name=~\"$environmentNameList\", cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}))\n\n)\nunless on (sink_id) (\ngroup by (sink_id) (\nlabel_replace(mz_object_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"sink_id\", \"$1\", \"global_id\", \"(.*)\")\n)\n),\n\"name\", \"$1\", \"sink_id\", \"(.*)\"\n)", "legendFormat": "{{name}}", "qryType": 1, "query": "(\n(\nsum by (sink_id) (max without (job) (mz_sink_rdkafka_outbuf_msg_cnt{materialize_cloud_organization_name=~\"$environmentNameList\", cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}))\n\n)\n* on (sink_id) group_left(name)\nmax by (sink_id, name) (\nlabel_replace(mz_object_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"sink_id\", \"$1\", \"global_id\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\nsum by (sink_id) (max without (job) (mz_sink_rdkafka_outbuf_msg_cnt{materialize_cloud_organization_name=~\"$environmentNameList\", cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}))\n\n)\nunless on (sink_id) (\ngroup by (sink_id) (\nlabel_replace(mz_object_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"sink_id\", \"$1\", \"global_id\", \"(.*)\")\n)\n),\n\"name\", \"$1\", \"sink_id\", \"(.*)\"\n)" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Messages sitting in the librdkafka output buffer, waiting to be sent to the broker. Normal buffer fluctuates briefly as messages flow through.**\n\n**Unhealthy:** Sustained high values mean Materialize is producing faster than the broker accepts (often paired with TX errors); if the buffer hits its bound the sink stalls and lag climbs.\n\nScoped to the selected clusters/replicas.", "id": 1068, "links": [], "title": "Kafka Output Buffer (messages)", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "unit": "short" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "sinks-kafka-tx-errors": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "(\n(\nsum by (sink_id) (max without (job) (rate(mz_sink_rdkafka_txerrs{materialize_cloud_organization_name=~\"$environmentNameList\", cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval])))\n\n)\n* on (sink_id) group_left(name)\nmax by (sink_id, name) (\nlabel_replace(mz_object_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"sink_id\", \"$1\", \"global_id\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\nsum by (sink_id) (max without (job) (rate(mz_sink_rdkafka_txerrs{materialize_cloud_organization_name=~\"$environmentNameList\", cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval])))\n\n)\nunless on (sink_id) (\ngroup by (sink_id) (\nlabel_replace(mz_object_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"sink_id\", \"$1\", \"global_id\", \"(.*)\")\n)\n),\n\"name\", \"$1\", \"sink_id\", \"(.*)\"\n)", "legendFormat": "{{name}}", "qryType": 1, "query": "(\n(\nsum by (sink_id) (max without (job) (rate(mz_sink_rdkafka_txerrs{materialize_cloud_organization_name=~\"$environmentNameList\", cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval])))\n\n)\n* on (sink_id) group_left(name)\nmax by (sink_id, name) (\nlabel_replace(mz_object_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"sink_id\", \"$1\", \"global_id\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\nsum by (sink_id) (max without (job) (rate(mz_sink_rdkafka_txerrs{materialize_cloud_organization_name=~\"$environmentNameList\", cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval])))\n\n)\nunless on (sink_id) (\ngroup by (sink_id) (\nlabel_replace(mz_object_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"sink_id\", \"$1\", \"global_id\", \"(.*)\")\n)\n),\n\"name\", \"$1\", \"sink_id\", \"(.*)\"\n)" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Per-sink rate of TX errors from the librdkafka client — each is one failed produce-request against the broker.**\n\n**Unhealthy:** Non-zero is a problem: broker outages, ACL changes, topic deletion/recreation, or partition rebalancing. Sustained errors grow sink lag and can fill the output buffer.\n\nScoped to the selected clusters/replicas.", "id": 1067, "links": [], "title": "Kafka TX Error Rate", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "#A2F49B", "value": null }, { "color": "#FEDA8B", "value": 1 }, { "color": "#FDB366", "value": 3.25 }, { "color": "#F67E4B", "value": 5.5 }, { "color": "#DD3D2D", "value": 7.75 }, { "color": "#A50026", "value": 10 } ] }, "unit": "cps" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "sinks-lag": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "(\n(\nclamp_min(\n sum by (sink_id) (max without (job) (mz_sink_bytes_staged{materialize_cloud_organization_name=~\"$environmentNameList\", cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}))\n - sum by (sink_id) (max without (job) (mz_sink_bytes_committed{materialize_cloud_organization_name=~\"$environmentNameList\", cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"})),\n 0\n)\n\n)\n* on (sink_id) group_left(name)\nmax by (sink_id, name) (\nlabel_replace(mz_object_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"sink_id\", \"$1\", \"global_id\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\nclamp_min(\n sum by (sink_id) (max without (job) (mz_sink_bytes_staged{materialize_cloud_organization_name=~\"$environmentNameList\", cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}))\n - sum by (sink_id) (max without (job) (mz_sink_bytes_committed{materialize_cloud_organization_name=~\"$environmentNameList\", cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"})),\n 0\n)\n\n)\nunless on (sink_id) (\ngroup by (sink_id) (\nlabel_replace(mz_object_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"sink_id\", \"$1\", \"global_id\", \"(.*)\")\n)\n),\n\"name\", \"$1\", \"sink_id\", \"(.*)\"\n)", "legendFormat": "{{name}}", "qryType": 1, "query": "(\n(\nclamp_min(\n sum by (sink_id) (max without (job) (mz_sink_bytes_staged{materialize_cloud_organization_name=~\"$environmentNameList\", cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}))\n - sum by (sink_id) (max without (job) (mz_sink_bytes_committed{materialize_cloud_organization_name=~\"$environmentNameList\", cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"})),\n 0\n)\n\n)\n* on (sink_id) group_left(name)\nmax by (sink_id, name) (\nlabel_replace(mz_object_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"sink_id\", \"$1\", \"global_id\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\nclamp_min(\n sum by (sink_id) (max without (job) (mz_sink_bytes_staged{materialize_cloud_organization_name=~\"$environmentNameList\", cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}))\n - sum by (sink_id) (max without (job) (mz_sink_bytes_committed{materialize_cloud_organization_name=~\"$environmentNameList\", cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"})),\n 0\n)\n\n)\nunless on (sink_id) (\ngroup by (sink_id) (\nlabel_replace(mz_object_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"sink_id\", \"$1\", \"global_id\", \"(.*)\")\n)\n),\n\"name\", \"$1\", \"sink_id\", \"(.*)\"\n)" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Bytes staged for a sink but not yet committed downstream — an in-flight queue depth in bytes. Oscillates around a small value in normal operation as commits happen periodically.**\n\n**Unhealthy:** Sustained growth means the sink can't keep up — usually downstream back-pressure (broker overloaded, Iceberg catalog slow) or repeated commit failures (see the Iceberg/Kafka sink error panels).\n\nClamped at 0 (brief negatives come from scrape skew between the staged and committed counters). Scoped to the selected clusters/replicas.", "id": 1063, "links": [], "title": "Sink Lag (staged minus committed)", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "unit": "bytes" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "sinks-throughput": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "(\n(\nsum by (sink_id) (\n max without (job) (rate(mz_sink_bytes_committed{materialize_cloud_organization_name=~\"$environmentNameList\", cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval]))\n) > 0\n\n)\n* on (sink_id) group_left(name)\nmax by (sink_id, name) (\nlabel_replace(mz_object_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"sink_id\", \"$1\", \"global_id\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\nsum by (sink_id) (\n max without (job) (rate(mz_sink_bytes_committed{materialize_cloud_organization_name=~\"$environmentNameList\", cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval]))\n) > 0\n\n)\nunless on (sink_id) (\ngroup by (sink_id) (\nlabel_replace(mz_object_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"sink_id\", \"$1\", \"global_id\", \"(.*)\")\n)\n),\n\"name\", \"$1\", \"sink_id\", \"(.*)\"\n)", "legendFormat": "{{name}}", "qryType": 1, "query": "(\n(\nsum by (sink_id) (\n max without (job) (rate(mz_sink_bytes_committed{materialize_cloud_organization_name=~\"$environmentNameList\", cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval]))\n) > 0\n\n)\n* on (sink_id) group_left(name)\nmax by (sink_id, name) (\nlabel_replace(mz_object_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"sink_id\", \"$1\", \"global_id\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\nsum by (sink_id) (\n max without (job) (rate(mz_sink_bytes_committed{materialize_cloud_organization_name=~\"$environmentNameList\", cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval]))\n) > 0\n\n)\nunless on (sink_id) (\ngroup by (sink_id) (\nlabel_replace(mz_object_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"sink_id\", \"$1\", \"global_id\", \"(.*)\")\n)\n),\n\"name\", \"$1\", \"sink_id\", \"(.*)\"\n)" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Outbound throughput per sink — bytes/second successfully committed to the downstream system (Kafka broker, Iceberg catalog, …).**\n\nIdle sinks are filtered out. Scoped to the selected clusters/replicas.", "id": 1062, "links": [], "title": "Sink Throughput (committed)", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "custom": { "scaleDistribution": { "log": 10, "type": "log" } }, "unit": "Bps" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "sinks-types": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "count by (object_type, envelope_type) (\n group by (id, object_type, envelope_type) (\n mz_storage_objects{materialize_cloud_organization_name=~\"$environmentNameList\", type=\"sink\"}\n )\n) > 0\n", "instant": true, "legendFormat": "{{object_type}} / {{envelope_type}}", "qryType": 1, "query": "count by (object_type, envelope_type) (\n group by (id, object_type, envelope_type) (\n mz_storage_objects{materialize_cloud_organization_name=~\"$environmentNameList\", type=\"sink\"}\n )\n) > 0\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Sinks by (type, envelope) — e.g. `kafka / upsert`, `kafka / debezium`, `iceberg / upsert`. The envelope is how Materialize encodes changes: `upsert` writes the latest value per key, `debezium` writes change events with old+new values.**\n\nEnvironment-scoped.", "id": 1061, "links": [], "title": "Sink Types", "vizConfig": { "group": "piechart", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "fixedColor": "#CCBB44", "mode": "shades" }, "noValue": "No matches for the current filters" }, "overrides": [] }, "options": { "displayLabels": [ "name", "value" ], "legend": { "asTable": true, "calcs": [], "displayMode": "table", "isVisible": true, "placement": "right", "showLegend": true, "values": [ "value" ] }, "orientation": "auto", "pieType": "donut", "reduceOptions": { "calcs": [], "values": false }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "sources-bytes-received-rate": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "(\n(\nsum by (parent_source_id) (\n max without (job) (rate(mz_source_bytes_received{materialize_cloud_organization_name=~\"$environmentNameList\", cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval]))\n) > 0\n\n)\n* on (parent_source_id) group_left(name)\nmax by (parent_source_id, name) (\nlabel_replace(mz_object_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"parent_source_id\", \"$1\", \"global_id\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\nsum by (parent_source_id) (\n max without (job) (rate(mz_source_bytes_received{materialize_cloud_organization_name=~\"$environmentNameList\", cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval]))\n) > 0\n\n)\nunless on (parent_source_id) (\ngroup by (parent_source_id) (\nlabel_replace(mz_object_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"parent_source_id\", \"$1\", \"global_id\", \"(.*)\")\n)\n),\n\"name\", \"$1\", \"parent_source_id\", \"(.*)\"\n)", "legendFormat": "{{name}}", "qryType": 1, "query": "(\n(\nsum by (parent_source_id) (\n max without (job) (rate(mz_source_bytes_received{materialize_cloud_organization_name=~\"$environmentNameList\", cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval]))\n) > 0\n\n)\n* on (parent_source_id) group_left(name)\nmax by (parent_source_id, name) (\nlabel_replace(mz_object_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"parent_source_id\", \"$1\", \"global_id\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\nsum by (parent_source_id) (\n max without (job) (rate(mz_source_bytes_received{materialize_cloud_organization_name=~\"$environmentNameList\", cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval]))\n) > 0\n\n)\nunless on (parent_source_id) (\ngroup by (parent_source_id) (\nlabel_replace(mz_object_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"parent_source_id\", \"$1\", \"global_id\", \"(.*)\")\n)\n),\n\"name\", \"$1\", \"parent_source_id\", \"(.*)\"\n)" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Inbound throughput per primary source — bytes/second pulled from upstream. Subsources (e.g. per-table Postgres replication) roll up to their primary, so each line is one logical source.**\n\nIdle sources are filtered out. Scoped to the selected clusters/replicas.", "id": 1058, "links": [], "title": "Source Bytes Received (rate)", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "custom": { "scaleDistribution": { "log": 10, "type": "log" } }, "unit": "Bps" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "sources-errors": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (source_id) (\n max without (job) (rate(mz_source_offset_commit_failures{materialize_cloud_organization_name=~\"$environmentNameList\", cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval]))\n) > 0\n", "legendFormat": "{{source_id}} commit failures", "qryType": 1, "query": "sum by (source_id) (\n max without (job) (rate(mz_source_offset_commit_failures{materialize_cloud_organization_name=~\"$environmentNameList\", cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval]))\n) > 0\n" }, "version": "v0" }, "refId": "query-0" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "(\n max by (source_id) (mz_source_offset_committed{materialize_cloud_organization_name=~\"$environmentNameList\", cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"})\n > bool max by (source_id) (mz_source_offset_known{materialize_cloud_organization_name=~\"$environmentNameList\", cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"})\n) > 0\n", "legendFormat": "{{source_id}} disconnected", "qryType": 1, "query": "(\n max by (source_id) (mz_source_offset_committed{materialize_cloud_organization_name=~\"$environmentNameList\", cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"})\n > bool max by (source_id) (mz_source_offset_known{materialize_cloud_organization_name=~\"$environmentNameList\", cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"})\n) > 0\n" }, "version": "v0" }, "refId": "query-1" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Per-source upstream health, with two complementary signals — both nominal at 0, so an empty panel is healthy and any series means a source needs attention.**\n\n**Unhealthy:** Commit failures fire when the upstream is reachable but rejects the offset/replication-slot commit (auth/ACL, broker rejecting). The disconnected indicator flips to 1 when the source has lost sight of its upstream (`offset_known` fell below `offset_committed`) — the broker/DB-unreachable case (BrokerTransportFailure, severed security group, DNS) that commit-failures can't catch, because the source never reaches the commit step. When disconnected is 1, throughput flat-lines and lag climbs. Read the exact error via `SELECT name, status, error FROM mz_internal.mz_source_statuses WHERE status != 'running'`.\n\nData-decode errors are separate (`mz_source_error_inserts`). Scoped to the selected clusters/replicas.", "id": 1060, "links": [], "title": "Source Upstream Errors", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "No matches for the current filters", "thresholds": { "mode": "absolute", "steps": [ { "color": "#A2F49B", "value": null }, { "color": "#FEDA8B", "value": 1 }, { "color": "#FDB366", "value": 1 }, { "color": "#F67E4B", "value": 1 }, { "color": "#DD3D2D", "value": 1 }, { "color": "#A50026", "value": 1 } ] }, "unit": "short" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "sources-ingestion-by-replica": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "(\n(\nsum by (parent_source_id, cluster_environmentd_materialize_cloud_replica_id) (\n max without (job) (rate(mz_source_messages_received{materialize_cloud_organization_name=~\"$environmentNameList\", cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval]))\n)\n\n)\n* on (parent_source_id) group_left(name)\nmax by (parent_source_id, name) (\nlabel_replace(mz_object_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"parent_source_id\", \"$1\", \"global_id\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\nsum by (parent_source_id, cluster_environmentd_materialize_cloud_replica_id) (\n max without (job) (rate(mz_source_messages_received{materialize_cloud_organization_name=~\"$environmentNameList\", cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval]))\n)\n\n)\nunless on (parent_source_id) (\ngroup by (parent_source_id) (\nlabel_replace(mz_object_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"parent_source_id\", \"$1\", \"global_id\", \"(.*)\")\n)\n),\n\"name\", \"$1\", \"parent_source_id\", \"(.*)\"\n)", "legendFormat": "{{name}} / r{{cluster_environmentd_materialize_cloud_replica_id}}", "qryType": 1, "query": "(\n(\nsum by (parent_source_id, cluster_environmentd_materialize_cloud_replica_id) (\n max without (job) (rate(mz_source_messages_received{materialize_cloud_organization_name=~\"$environmentNameList\", cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval]))\n)\n\n)\n* on (parent_source_id) group_left(name)\nmax by (parent_source_id, name) (\nlabel_replace(mz_object_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"parent_source_id\", \"$1\", \"global_id\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\nsum by (parent_source_id, cluster_environmentd_materialize_cloud_replica_id) (\n max without (job) (rate(mz_source_messages_received{materialize_cloud_organization_name=~\"$environmentNameList\", cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval]))\n)\n\n)\nunless on (parent_source_id) (\ngroup by (parent_source_id) (\nlabel_replace(mz_object_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"parent_source_id\", \"$1\", \"global_id\", \"(.*)\")\n)\n),\n\"name\", \"$1\", \"parent_source_id\", \"(.*)\"\n)" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Messages ingested per second, split per source AND replica. Replicas read their upstream independently and should track together.**\n\n**Unhealthy:** A replica flat at 0 while its siblings keep ingesting has lost its upstream connection (e.g. restarted and couldn't resume pulling from Kafka). The source still shows `Running` overall and the aggregate throughput hides it, so this per-replica split is where it surfaces — lag will be climbing too; restarting that replica usually clears the stale connection.\n\nScoped to the selected clusters/replicas.", "id": 1059, "links": [], "title": "Source Ingestion by Replica", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "No matches for the current filters", "unit": "cps" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "sources-status-table": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "(\n(\ngroup by (id, object_type, connection_type, envelope_type, cluster_id) (\n mz_storage_objects{materialize_cloud_organization_name=~\"$environmentNameList\", type=\"source\"}\n)\n\n)\n* on (id) group_left(name)\nmax by (id, name) (\nlabel_replace(mz_object_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"id\", \"$1\", \"global_id\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\ngroup by (id, object_type, connection_type, envelope_type, cluster_id) (\n mz_storage_objects{materialize_cloud_organization_name=~\"$environmentNameList\", type=\"source\"}\n)\n\n)\nunless on (id) (\ngroup by (id) (\nlabel_replace(mz_object_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"id\", \"$1\", \"global_id\", \"(.*)\")\n)\n),\n\"name\", \"$1\", \"id\", \"(.*)\"\n)", "instant": true, "qryType": 1, "query": "(\n(\ngroup by (id, object_type, connection_type, envelope_type, cluster_id) (\n mz_storage_objects{materialize_cloud_organization_name=~\"$environmentNameList\", type=\"source\"}\n)\n\n)\n* on (id) group_left(name)\nmax by (id, name) (\nlabel_replace(mz_object_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"id\", \"$1\", \"global_id\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\ngroup by (id, object_type, connection_type, envelope_type, cluster_id) (\n mz_storage_objects{materialize_cloud_organization_name=~\"$environmentNameList\", type=\"source\"}\n)\n\n)\nunless on (id) (\ngroup by (id) (\nlabel_replace(mz_object_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"id\", \"$1\", \"global_id\", \"(.*)\")\n)\n),\n\"name\", \"$1\", \"id\", \"(.*)\"\n)" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [ { "group": "labelsToFields", "kind": "Transformation", "spec": { "options": { "keepLabels": [ "name", "id", "object_type", "connection_type", "envelope_type", "cluster_id" ] } } }, { "group": "merge", "kind": "Transformation", "spec": { "options": {} } }, { "group": "organize", "kind": "Transformation", "spec": { "options": { "excludeByName": { "Time": true, "Value": true }, "indexByName": { "cluster_id": 5, "connection_type": 3, "envelope_type": 4, "id": 1, "name": 0, "object_type": 2 }, "renameByName": { "cluster_id": "Cluster", "connection_type": "Connection", "envelope_type": "Envelope", "id": "Source ID", "name": "Name", "object_type": "Type" } } } }, { "group": "sortBy", "kind": "Transformation", "spec": { "options": { "fields": {}, "sort": [ { "field": "Name" } ] } } } ] } }, "description": "**A catalog of sources — one row per source (by name) with its connector type, envelope, and the cluster it ingests on. The metric-side \"what sources do I have\" reference.**\n\nSelf-managed exposes no source *status* metric, so running/stalled isn't here — check `SELECT name, type, status FROM mz_internal.mz_source_statuses`, and use source throughput to confirm a source is actively ingesting. `_progress` subsources are excluded, so the row count matches the source count. Environment-scoped.", "id": 1057, "links": [], "title": "Sources", "vizConfig": { "group": "table", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "custom": { "align": "auto", "filterable": true, "inspect": false }, "noValue": "No matches for the current filters" }, "overrides": [] }, "options": { "cellHeight": "sm", "footer": { "countRows": false, "reducer": [], "show": false }, "frameIndex": 0, "showHeader": true, "showTypeIcons": false } }, "version": "" } } }, "sources-types": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "count by (object_type) (\n group by (id, object_type) (\n mz_storage_objects{materialize_cloud_organization_name=~\"$environmentNameList\", type=\"source\"}\n )\n) > 0\n", "instant": true, "legendFormat": "{{object_type}}", "qryType": 1, "query": "count by (object_type) (\n group by (id, object_type) (\n mz_storage_objects{materialize_cloud_organization_name=~\"$environmentNameList\", type=\"source\"}\n )\n) > 0\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Sources by connector type (kafka / postgres / mysql / …) — what flavors of upstream feed make up the ingest workload. Most environments concentrate on one or two.**\n\nEnvironment-scoped.", "id": 1056, "links": [], "title": "Source Types", "vizConfig": { "group": "piechart", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "fixedColor": "#CCBB44", "mode": "shades" }, "noValue": "No matches for the current filters" }, "overrides": [] }, "options": { "displayLabels": [ "name", "value" ], "legend": { "asTable": true, "calcs": [], "displayMode": "table", "isVisible": true, "placement": "right", "showLegend": true, "values": [ "value" ] }, "orientation": "auto", "pieType": "donut", "reduceOptions": { "calcs": [], "values": false }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "storage-active-sinks": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "(count(group by (id) (mz_storage_objects{materialize_cloud_organization_name=~\"$environmentNameList\", type=\"sink\"}))\n) or vector(0)", "legendFormat": "sinks", "qryType": 1, "query": "(count(group by (id) (mz_storage_objects{materialize_cloud_organization_name=~\"$environmentNameList\", type=\"sink\"}))\n) or vector(0)" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Active sinks in the catalog — each emits the results of a materialized view or query to an external system (Kafka, Iceberg, …). Counts distinct sink objects (excluding `_progress` subsources), matching `mz_sinks`.**\n\nEnvironment-scoped.", "id": 1054, "links": [], "title": "Active Sinks", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "fixedColor": "#CCBB44", "mode": "shades" }, "min": 0, "noValue": "No matches for the current filters" }, "overrides": [] }, "options": { "colorMode": "none", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [], "values": false }, "showPercentChange": false, "textMode": "value", "wideLayout": true } }, "version": "" } } }, "storage-active-sources": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "(count(group by (id) (mz_storage_objects{materialize_cloud_organization_name=~\"$environmentNameList\", type=\"source\"}))\n) or vector(0)", "legendFormat": "sources", "qryType": 1, "query": "(count(group by (id) (mz_storage_objects{materialize_cloud_organization_name=~\"$environmentNameList\", type=\"source\"}))\n) or vector(0)" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Active sources in the catalog — each is a continuous ingestion connection from an external system (Kafka, Postgres, MySQL, S3, …), so this is roughly how many upstream feeds the environment maintains. Counts distinct source objects (the hidden per-source `_progress` subsources are excluded), matching `mz_sources`.**\n\nEnvironment-scoped — the cluster/replica selectors don't narrow it.", "id": 1053, "links": [], "title": "Active Sources", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "fixedColor": "#CCBB44", "mode": "shades" }, "min": 0, "noValue": "No matches for the current filters" }, "overrides": [] }, "options": { "colorMode": "none", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [], "values": false }, "showPercentChange": false, "textMode": "value", "wideLayout": true } }, "version": "" } } }, "storage-active-tables": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "(max(sum by (instance) (mz_tables_count{materialize_cloud_organization_name=~\"$environmentNameList\"}))\n) or vector(0)", "legendFormat": "mz_tables_count", "qryType": 1, "query": "(max(sum by (instance) (mz_tables_count{materialize_cloud_organization_name=~\"$environmentNameList\"}))\n) or vector(0)" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**User-created tables in the catalog. Tables are write-once-read-many; `INSERT`s feed dataflows downstream. Mostly a catalog-shape signal — for actual ingest activity look at source throughput.**\n\nEnvironment-scoped.", "id": 1055, "links": [], "title": "Active Tables", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "fixedColor": "#CCBB44", "mode": "shades" }, "min": 0, "noValue": "No matches for the current filters" }, "overrides": [] }, "options": { "colorMode": "none", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [], "values": false }, "showPercentChange": false, "textMode": "value", "wideLayout": true } }, "version": "" } } }, "summary-cpu-total": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (container) (\n container_spec_cpu_quota{namespace=~\"$mzNamespaceList\",container!=\"\",container!=\"POD\", container!=\"new-promsql-exporter\"}\n / container_spec_cpu_period{namespace=~\"$mzNamespaceList\",container!=\"\",container!=\"POD\", container!=\"new-promsql-exporter\"}\n)\n", "legendFormat": "CPUs ({{container}})", "qryType": 1, "query": "sum by (container) (\n container_spec_cpu_quota{namespace=~\"$mzNamespaceList\",container!=\"\",container!=\"POD\", container!=\"new-promsql-exporter\"}\n / container_spec_cpu_period{namespace=~\"$mzNamespaceList\",container!=\"\",container!=\"POD\", container!=\"new-promsql-exporter\"}\n)\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Total CPU cores configured across the environment's containers (sum of cAdvisor CPU limits), excluding the monitoring exporter — i.e. the CPU available to the actual workload.**\n\nSteps track `ALTER CLUSTER REPLICA SIZE`, replica create/drop, and pod restarts. Requires cAdvisor. For the same total *including* the monitoring exporter see `.all_containers`.", "id": 1008, "links": [], "title": "Total CPU Capacity", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "fixedColor": "#0077BB", "mode": "shades" }, "noValue": "No metrics: cadvisor metrics are required (via kubelet)", "unit": "cores" }, "overrides": [] }, "options": { "colorMode": "none", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [], "values": false }, "showPercentChange": false, "textMode": "value_and_name", "wideLayout": true } }, "version": "" } } }, "summary-currently-hydrating": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "(count(\n max by (instance_id, collection_id) (\n mz_dataflow_wallclock_lag_seconds{materialize_cloud_organization_name=~\"$environmentNameList\", instance_id=~\"$mzClusterList\", instance_id!=\"\", quantile=\"1\"} > 1e15\n )\n)\n) or vector(0)", "legendFormat": "hydrating", "qryType": 1, "query": "(count(\n max by (instance_id, collection_id) (\n mz_dataflow_wallclock_lag_seconds{materialize_cloud_organization_name=~\"$environmentNameList\", instance_id=~\"$mzClusterList\", instance_id!=\"\", quantile=\"1\"} > 1e15\n )\n)\n) or vector(0)" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Collections still rebuilding their in-memory state — a live hydration-queue proxy. After a restart, replica creation, or some DDL, a dataflow has to rebuild from persisted storage before it can serve, and until it does it produces no results.**\n\n**Nominal:** 0, with brief spikes right after a replica restart that drain back to 0 as dataflows catch up. Those spikes are healthy re-hydration, not a problem.\n\n**Unhealthy:** A count that stays elevated means something can't finish hydrating — a source whose CREATE never completed, or a wedged dataflow. Confirm what with `SELECT * FROM mz_internal.mz_hydration_statuses WHERE NOT hydrated`.\n\nStopgap, not a real signal: self-managed has no hydration-state metric, so this counts collections parked at the wallclock-lag u64::MAX sentinel (> 1e15) as a proxy. Deprecate this query once an actual hydration-state metric exists.", "id": 1003, "links": [], "title": "Currently Hydrating", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "fixedColor": "#EE7733", "mode": "shades" }, "min": 0, "noValue": "No matches for the current filters" }, "overrides": [] }, "options": { "colorMode": "none", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [], "values": false }, "showPercentChange": false, "textMode": "value", "wideLayout": true } }, "version": "" } } }, "summary-max-lag": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max(\n max_over_time(\n (\n mz_dataflow_wallclock_lag_seconds{materialize_cloud_organization_name=~\"$environmentNameList\", instance_id!=\"\", quantile=\"1\"} < 1e9\n )[$__range:1m]\n )\n)\n", "instant": true, "legendFormat": "max lag", "qryType": 1, "query": "max(\n max_over_time(\n (\n mz_dataflow_wallclock_lag_seconds{materialize_cloud_organization_name=~\"$environmentNameList\", instance_id!=\"\", quantile=\"1\"} < 1e9\n )[$__range:1m]\n )\n)\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**The worst lag seen anywhere in the environment over the selected window — how far the most-behind collection's output trailed real time. A top-level freshness pointer.**\n\n**Degraded:** Climbing toward an hour means some collection is falling behind; drill into per-cluster and per-collection freshness to find which, and into hydration / source health for why.\n\nNot-yet-hydrated collections are excluded (they report the sentinel and surface as \"currently hydrating\"). This is the peak over the window, so it reflects the worst moment rather than the current value.", "id": 1004, "links": [], "title": "Max Lag (Select Time Range)", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "noValue": "No matches for the current filters", "thresholds": { "mode": "absolute", "steps": [ { "color": "#CEFFFF", "value": null }, { "color": "#CEFFFF", "value": 2 }, { "color": "#C6F7D6", "value": 4 }, { "color": "#A2F49B", "value": 9 }, { "color": "#BBE453", "value": 19 }, { "color": "#D5CE04", "value": 41 }, { "color": "#E7B503", "value": 87 }, { "color": "#F19903", "value": 183 }, { "color": "#F6790B", "value": 385 }, { "color": "#F94902", "value": 812 }, { "color": "#E40515", "value": 1710 }, { "color": "#AB0003", "value": 3600 } ] }, "unit": "s" }, "overrides": [] }, "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [], "values": false }, "showPercentChange": false, "textMode": "value", "wideLayout": true } }, "version": "" } } }, "summary-memory-total": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (container) (\n container_spec_memory_limit_bytes{namespace=~\"$mzNamespaceList\",container!=\"\",container!=\"POD\", container!=\"new-promsql-exporter\"}\n)\n", "legendFormat": "{{container}}", "qryType": 1, "query": "sum by (container) (\n container_spec_memory_limit_bytes{namespace=~\"$mzNamespaceList\",container!=\"\",container!=\"POD\", container!=\"new-promsql-exporter\"}\n)\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Total memory configured across the environment's containers (sum of cAdvisor memory limits), excluding the monitoring exporter. Memory is Materialize's dominant constraint — in-memory arrangements live in here.**\n\nSteps track replica sizing and pod restarts. Requires cAdvisor.", "id": 1009, "links": [], "title": "Total Memory", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "fixedColor": "#0077BB", "mode": "shades" }, "noValue": "No metrics: cadvisor metrics are required (via kubelet)", "unit": "bytes" }, "overrides": [] }, "options": { "colorMode": "none", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [], "values": false }, "showPercentChange": false, "textMode": "value_and_name", "wideLayout": true } }, "version": "" } } } }, "layout": { "kind": "TabsLayout", "spec": { "tabs": [ { "kind": "TabsLayoutTab", "spec": { "layout": { "kind": "RowsLayout", "spec": { "rows": [ { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "standard", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "is-healthy" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "availability-percent" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "last-restart" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "summary-currently-hydrating" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "summary-max-lag" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "cpu-usage-current" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "memory-usage-current" } } } ], "maxColumnCount": 3, "rowHeightMode": "short" } }, "title": "Environment Health" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "standard", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "materialize-version" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "summary-cpu-total" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "summary-memory-total" } } } ], "maxColumnCount": 3, "rowHeightMode": "short" } }, "title": "Environment Info" } } ] } }, "title": "Summary" } }, { "kind": "TabsLayoutTab", "spec": { "layout": { "kind": "RowsLayout", "spec": { "rows": [ { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": true, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "standard", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "k8s-res-cpu-total" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "k8s-res-memory-total" } } } ], "maxColumnCount": 3, "rowHeightMode": "short" } }, "title": "Resources Summary" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": true, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "standard", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "resource-pod-status" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "resource-statefulset-status" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "resource-deployment-status" } } } ], "maxColumnCount": 5, "rowHeightMode": "short" } }, "title": "Workload Readiness" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "standard", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "pod-cpu-percent" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "pod-memory-percent" } } } ], "maxColumnCount": 3, "rowHeightMode": "standard" } }, "title": "Pod Metrics" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "standard", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "pod-network-rx" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "pod-network-tx" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "pod-network-errors" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "pod-network-drops" } } } ], "maxColumnCount": 2, "rowHeightMode": "standard" } }, "title": "Pod Networking" } } ] } }, "title": "Kubernetes Workloads" } }, { "kind": "TabsLayoutTab", "spec": { "layout": { "kind": "RowsLayout", "spec": { "rows": [ { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": true, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "standard", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "connections-active-sessions" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "connections-active-queries" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "connections-adapter-command-rate" } } } ], "maxColumnCount": 3, "rowHeightMode": "short" } }, "title": "Connection Summary" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "standard", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "queries-distribution" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "queries-rate-by-statement" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "queries-peek-latency-p50" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "queries-peek-latency-p90" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "queries-peek-latency-p99" } } } ], "maxColumnCount": 3, "rowHeightMode": "standard" } }, "title": "Queries" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "standard", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "adapter-commands-by-app" } } } ], "maxColumnCount": 1, "rowHeightMode": "standard" } }, "title": "SQL Control Plane Commands" } } ] } }, "title": "Connections / Activity" } }, { "kind": "TabsLayoutTab", "spec": { "layout": { "kind": "RowsLayout", "spec": { "rows": [ { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": true, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "standard", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "cluster-count" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "replica-count" } } } ], "maxColumnCount": 3, "rowHeightMode": "short" } }, "title": "Cluster Summary" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "standard", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "replica-sizes" } } } ], "maxColumnCount": 3, "rowHeightMode": "standard" } }, "title": "Replication / Availability" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "standard", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "cluster-table" } } } ], "maxColumnCount": 3, "rowHeightMode": "standard" } }, "title": "Cluster Information" } } ] } }, "title": "Cluster Objects / Replicas" } }, { "kind": "TabsLayoutTab", "spec": { "layout": { "kind": "RowsLayout", "spec": { "rows": [ { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": true, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "narrow", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "active-mzd-views" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "active-indexes" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "active-views" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "active-subscribes" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "index-types" } } } ], "maxColumnCount": 5, "rowHeightMode": "short" } }, "title": "Compute Objects Summary" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "wide", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "freshness-lag-by-cluster" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "freshness-lag-total-by-cluster" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "freshness-top-collections" } } } ], "maxColumnCount": 3, "rowHeightMode": "standard" } }, "title": "Freshness" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "standard", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "hydration-unhydrated-count" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "hydration-queue-size" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "hydration-slowest-collections" } } } ], "maxColumnCount": 3, "rowHeightMode": "standard" } }, "title": "Hydration" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "standard", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "dataflow-count" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "dataflow-count-by-worker" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "dataflow-elapsed-rate" } } } ], "maxColumnCount": 3, "rowHeightMode": "standard" } }, "title": "Dataflows" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "wide", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "arrangement-rate" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "arrangement-rate-by-worker" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "arrangement-records-system" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "arrangement-records-user" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "arrangement-records-transient" } } } ], "maxColumnCount": 3, "rowHeightMode": "standard" } }, "title": "Arrangements" } } ] } }, "title": "Compute Objects" } }, { "kind": "TabsLayoutTab", "spec": { "layout": { "kind": "RowsLayout", "spec": { "rows": [ { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": true, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "narrow", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "storage-active-sources" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "storage-active-sinks" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "storage-active-tables" } } } ], "maxColumnCount": 3, "rowHeightMode": "short" } }, "title": "Storage Objects Summary" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "wide", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "sources-types" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "sources-status-table" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "sources-bytes-received-rate" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "sources-ingestion-by-replica" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "sources-errors" } } } ], "maxColumnCount": 3, "rowHeightMode": "standard" } }, "title": "Sources" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "wide", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "sinks-types" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "sinks-throughput" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "sinks-lag" } } } ], "maxColumnCount": 3, "rowHeightMode": "standard" } }, "title": "Sinks" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": true, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "wide", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "sinks-iceberg-commit-latency" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "sinks-iceberg-failures" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "sinks-iceberg-files" } } } ], "maxColumnCount": 3, "rowHeightMode": "standard" } }, "title": "Iceberg Sinks" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": true, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "wide", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "sinks-kafka-tx-errors" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "sinks-kafka-outbuf" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "sinks-kafka-connects" } } } ], "maxColumnCount": 3, "rowHeightMode": "standard" } }, "title": "Kafka Sinks" } } ] } }, "title": "Sources and Sinks" } } ] } }, "links": [], "liveNow": false, "preload": false, "tags": [ "materialize", "mzmon" ], "timeSettings": { "autoRefresh": "", "autoRefreshIntervals": [ "5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d" ], "fiscalYearStartMonth": 0, "from": "now-6h", "hideTimepicker": false, "timezone": "browser", "to": "now" }, "title": "Materialize Environment Overview", "variables": [ { "kind": "DatasourceVariable", "spec": { "allowCustomValue": false, "current": { "text": "", "value": "" }, "description": "Datasource for metrics queries", "hide": "dontHide", "includeAll": false, "label": "Metrics Datasource", "multi": false, "name": "metricsDatasource", "options": [], "pluginId": "prometheus", "refresh": "never", "regex": "", "skipUrlSync": false } }, { "kind": "QueryVariable", "spec": { "allValue": ".*", "allowCustomValue": true, "current": { "text": "", "value": "" }, "definition": "label_values(mz_compute_commands_total, materialize_cloud_organization_name)", "description": "The current environment to view", "hide": "dontHide", "includeAll": false, "label": "Environment", "multi": false, "name": "environmentNameList", "options": [], "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "label_values(mz_compute_commands_total, materialize_cloud_organization_name)", "qryType": 1, "query": "label_values(mz_compute_commands_total, materialize_cloud_organization_name)" }, "version": "v0" }, "refresh": "never", "regex": "", "regexApplyTo": "value", "skipUrlSync": false, "sort": "alphabeticalAsc" } }, { "kind": "QueryVariable", "spec": { "allowCustomValue": true, "current": { "text": "", "value": "" }, "definition": "label_values(mz_compute_commands_total{ materialize_cloud_organization_name=~\"$environmentNameList\" }, materialize_cloud_organization_namespace)", "description": "The current materialize namespace where environments live", "hide": "hideVariable", "includeAll": true, "label": "Materialize Namespace", "multi": true, "name": "mzNamespaceList", "options": [], "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "label_values(mz_compute_commands_total{ materialize_cloud_organization_name=~\"$environmentNameList\" }, materialize_cloud_organization_namespace)", "qryType": 1, "query": "label_values(mz_compute_commands_total{ materialize_cloud_organization_name=~\"$environmentNameList\" }, materialize_cloud_organization_namespace)" }, "version": "v0" }, "refresh": "never", "regex": "", "regexApplyTo": "value", "skipUrlSync": true, "sort": "alphabeticalAsc" } }, { "kind": "SwitchVariable", "spec": { "current": ".*", "description": "Whether to include materialize system clusters in the cluster list.", "disabledValue": "^[^s].*", "enabledValue": ".*", "hide": "inControlsMenu", "label": "Include System Clusters", "name": "includeSystemClusters", "skipUrlSync": true } }, { "kind": "QueryVariable", "spec": { "allowCustomValue": true, "current": { "text": "", "value": "" }, "definition": "query_result(mz_compute_cluster_status{materialize_cloud_organization_name=~\"$environmentNameList\", compute_cluster_id=~\"$includeSystemClusters\" })", "description": "The cluster within the current environment to filter to", "hide": "inControlsMenu", "includeAll": true, "label": "Cluster", "multi": true, "name": "mzClusterList", "options": [], "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "query_result(mz_compute_cluster_status{materialize_cloud_organization_name=~\"$environmentNameList\", compute_cluster_id=~\"$includeSystemClusters\" })", "qryType": 1, "query": "query_result(mz_compute_cluster_status{materialize_cloud_organization_name=~\"$environmentNameList\", compute_cluster_id=~\"$includeSystemClusters\" })" }, "version": "v0" }, "refresh": "never", "regex": ".*compute_cluster_id=\\\"(?[^\\\"]+)\\\",.*compute_cluster_name=\\\"(?[^\\\"]+)\\\",.*", "regexApplyTo": "value", "skipUrlSync": false, "sort": "naturalAsc" } }, { "kind": "QueryVariable", "spec": { "allValue": ".*", "allowCustomValue": true, "current": { "text": "", "value": "" }, "definition": "label_values(mz_compute_commands_total{materialize_cloud_organization_name=~\"$environmentNameList\", instance_id=~\"$mzClusterList\"}, replica_id)", "description": "The replica within the current cluster to filter to", "hide": "inControlsMenu", "includeAll": true, "label": "Replica", "multi": true, "name": "mzReplicaList", "options": [], "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "label_values(mz_compute_commands_total{materialize_cloud_organization_name=~\"$environmentNameList\", instance_id=~\"$mzClusterList\"}, replica_id)", "qryType": 1, "query": "label_values(mz_compute_commands_total{materialize_cloud_organization_name=~\"$environmentNameList\", instance_id=~\"$mzClusterList\"}, replica_id)" }, "version": "v0" }, "refresh": "never", "regex": "", "regexApplyTo": "value", "skipUrlSync": false, "sort": "naturalAsc" } }, { "datasource": { "name": "$metricsDatasource" }, "group": "", "kind": "AdhocVariable", "spec": { "allowCustomValue": true, "baseFilters": [ { "key": "namespace", "operator": "=~", "origin": "dashboard", "value": "$mzNamespaceList" } ], "defaultKeys": [], "description": "Adhoc filters to apply to all metrics queries", "enableGroupBy": false, "filters": [], "hide": "inControlsMenu", "label": "Advanced Metric Filter", "name": "metricAdhoc", "skipUrlSync": false } } ] } }