{ "kind": "Dashboard", "apiVersion": "dashboard.grafana.app/v2", "metadata": { "name": "mz-mon-env-top" }, "spec": { "annotations": [], "cursorSync": "Off", "elements": { "is-healthy": { "kind": "Panel", "spec": { "id": 1000, "title": "Environment Status", "description": "**High-level environment health based on the fraction of clusters reporting healthy.** Aggregates `mz_compute_cluster_status` across the env; the result is mapped to text via thresholds: Healthy (100%), Degraded (80-100%), Unhealthy (<80%). When this turns Degraded or Unhealthy, check _Kubernetes Workloads_ for pod restart history and _Cluster Objects / Replicas_ to see which cluster(s) are affected.", "links": [], "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "\ncount(\n ${sqlMetricPrefix}compute_cluster_status{$environmentFilter} == 1\n) / count(\n ${sqlMetricPrefix}compute_cluster_status{$environmentFilter}\n) * 100\n", "query": "\ncount(\n ${sqlMetricPrefix}compute_cluster_status{$environmentFilter} == 1\n) / count(\n ${sqlMetricPrefix}compute_cluster_status{$environmentFilter}\n) * 100\n", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-0", "hidden": false } } ], "transformations": [], "queryOptions": {} } }, "vizConfig": { "kind": "VizConfig", "group": "stat", "version": "", "spec": { "options": { "graphMode": "area", "colorMode": "background", "justifyMode": "auto", "textMode": "value", "wideLayout": true, "showPercentChange": false, "reduceOptions": { "calcs": [] }, "percentChangeColorMode": "standard", "orientation": "auto" }, "fieldConfig": { "defaults": { "mappings": [ { "type": "range", "options": { "from": 100, "to": 2147483647.0, "result": { "text": "Healthy", "color": "#A2F49B", "index": 1 } } }, { "type": "range", "options": { "from": 80, "to": 100, "result": { "text": "Degraded", "color": "#F19903", "index": 2 } } }, { "type": "range", "options": { "from": -2147483647.0, "to": 80, "result": { "text": "Unhealthy", "color": "#AB0003", "index": 3 } } }, { "type": "special", "options": { "match": "null+nan", "result": { "text": "Missing Data", "color": "#888888", "index": 4 } } } ], "noValue": "No matches for the current filters" }, "overrides": [] } } } } }, "availability-percent": { "kind": "Panel", "spec": { "id": 1001, "title": "Environment Availability (Select Time Range)", "description": "**Fraction of time the environment was healthy over the dashboard's selected time range** \u2014 computed from `${sqlMetricPrefix}compute_cluster_status{$environmentFilter}` averaged over `$__range`. Effectively an SLO snapshot. Nominal: 100.0000% (note the four decimals \u2014 five-nines = 99.999%). Sustained dips correlate with cluster restarts or outages; widen the time range to find when they happened, then check _Last Restart Time_ and _Kubernetes Workloads_ for pod restart context.", "links": [], "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "\navg by (materialize_cloud_organization_namespace) (\n avg_over_time(\n ${sqlMetricPrefix}compute_cluster_status{$environmentFilter}[$__range]\n ) * 100\n)\n", "query": "\navg by (materialize_cloud_organization_namespace) (\n avg_over_time(\n ${sqlMetricPrefix}compute_cluster_status{$environmentFilter}[$__range]\n ) * 100\n)\n", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-0", "hidden": false } } ], "transformations": [], "queryOptions": {} } }, "vizConfig": { "kind": "VizConfig", "group": "stat", "version": "", "spec": { "options": { "graphMode": "area", "colorMode": "background", "justifyMode": "auto", "textMode": "value", "wideLayout": true, "showPercentChange": false, "reduceOptions": { "calcs": [] }, "percentChangeColorMode": "standard", "orientation": "auto" }, "fieldConfig": { "defaults": { "unit": "percent", "decimals": 4, "thresholds": { "mode": "percentage", "steps": [ { "value": -2147483647.0, "color": "#AB0003" }, { "value": 95.0, "color": "#F19903" }, { "value": 99.0, "color": "#A2F49B" } ] }, "noValue": "No matches for the current filters" }, "overrides": [] } } } } }, "last-restart": { "kind": "Panel", "spec": { "id": 1002, "title": "Last Restart Time", "description": "**Seconds since the most recent container restart in the environment.** Threshold-colored from red (seconds ago \u2014 likely an active incident) through to gray (days ago, fine). Nominal: hours-to-days. Recent restarts (red/orange) are worth correlating with _Environment Availability_ and the _Kubernetes Workloads_ tab's pod-health panels.", "links": [], "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "\ntime()\n- topk(1,\n container_start_time_seconds{$containerFilter, container!=\"new-promsql-exporter\"}\n)\n", "instant": true, "range": false, "legendFormat": "{{pod}}", "query": "\ntime()\n- topk(1,\n container_start_time_seconds{$containerFilter, container!=\"new-promsql-exporter\"}\n)\n", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-0", "hidden": false } } ], "transformations": [], "queryOptions": {} } }, "vizConfig": { "kind": "VizConfig", "group": "stat", "version": "", "spec": { "options": { "graphMode": "area", "colorMode": "background", "justifyMode": "center", "textMode": "value_and_name", "wideLayout": true, "showPercentChange": false, "reduceOptions": { "calcs": [] }, "percentChangeColorMode": "standard", "orientation": "auto", "text": { "valueSize": 25 } }, "fieldConfig": { "defaults": { "unit": "s", "thresholds": { "mode": "absolute", "steps": [ { "value": 2, "color": "#AB0003" }, { "value": 8, "color": "#E40515" }, { "value": 26, "color": "#F94902" }, { "value": 80, "color": "#F6790B" }, { "value": 240, "color": "#F19903" }, { "value": 719, "color": "#E7B503" }, { "value": 2152, "color": "#D5CE04" }, { "value": 6443, "color": "#BBE453" }, { "value": 19286, "color": "#A2F49B" }, { "value": 57730, "color": "#C6F7D6" }, { "value": 172800, "color": "#CEFFFF" } ] }, "noValue": "No metrics: cadvisor metrics are required (via kubelet)" }, "overrides": [] } } } } }, "summary-currently-hydrating": { "kind": "Panel", "spec": { "id": 1003, "title": "Currently Hydrating", "description": "**Collections still (re)building their in-memory state \u2014 a live hydration-queue proxy.** Hydration rebuilds a dataflow's state from persisted storage after a cluster/replica restart, replica creation, or some DDL; until it finishes, the collection has no output frontier, which this counts (via the `mz_dataflow_wallclock_lag_seconds` sentinel). **Nominal: 0, with brief spikes right after a replica restart that drain back to 0 as dataflows catch up \u2014 that's healthy.** A count that stays elevated means something can't finish hydrating (e.g. a source whose `CREATE` didn't complete, or a wedged dataflow). Confirm what's stuck with `SELECT * FROM mz_internal.mz_hydration_statuses WHERE NOT hydrated`; watch _Compute -> Freshness_ for the lag those collections accrue.", "links": [], "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "\ncount(\n max by (instance_id, collection_id) (\n mz_dataflow_wallclock_lag_seconds{\n $environmentFilter,\n instance_id=~\"$mzClusterList\",\n instance_id!=\"\",\n quantile=\"1\"\n } > 1e15\n )\n) or vector(0)\n", "legendFormat": "hydrating", "query": "\ncount(\n max by (instance_id, collection_id) (\n mz_dataflow_wallclock_lag_seconds{\n $environmentFilter,\n instance_id=~\"$mzClusterList\",\n instance_id!=\"\",\n quantile=\"1\"\n } > 1e15\n )\n) or vector(0)\n", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-0", "hidden": false } } ], "transformations": [], "queryOptions": {} } }, "vizConfig": { "kind": "VizConfig", "group": "stat", "version": "", "spec": { "options": { "graphMode": "area", "colorMode": "none", "justifyMode": "auto", "textMode": "value", "wideLayout": true, "showPercentChange": false, "reduceOptions": { "calcs": [] }, "percentChangeColorMode": "standard", "orientation": "auto" }, "fieldConfig": { "defaults": { "min": 0, "color": { "mode": "shades", "fixedColor": "#EE7733" }, "noValue": "No matches for the current filters" }, "overrides": [] } } } } }, "summary-max-lag": { "kind": "Panel", "spec": { "id": 1004, "title": "Max Lag (Select Time Range)", "description": "**Worst frontier lag seen anywhere in the environment over the dashboard's selected time range** \u2014 how far the most behind collection's output trailed real time. A top-level freshness pointer: low (seconds) is fine and stays gray; if it climbs toward an hour it turns red, meaning some collection is falling behind \u2014 open _Compute Objects -> Freshness_ to see which one (and _Currently Hydrating_ / _Storage -> Sources_ for why). Not-yet-hydrated collections are excluded here (they show in _Currently Hydrating_).", "links": [], "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "\nmax(\n max_over_time(\n (\n mz_dataflow_wallclock_lag_seconds{\n $environmentFilter, instance_id!=\"\", quantile=\"1\"\n } < 1e9\n )[$__range:1m]\n )\n)\n", "instant": true, "range": false, "legendFormat": "max lag", "query": "\nmax(\n max_over_time(\n (\n mz_dataflow_wallclock_lag_seconds{\n $environmentFilter, instance_id!=\"\", quantile=\"1\"\n } < 1e9\n )[$__range:1m]\n )\n)\n", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-0", "hidden": false } } ], "transformations": [], "queryOptions": {} } }, "vizConfig": { "kind": "VizConfig", "group": "stat", "version": "", "spec": { "options": { "graphMode": "area", "colorMode": "background", "justifyMode": "auto", "textMode": "value", "wideLayout": true, "showPercentChange": false, "reduceOptions": { "calcs": [] }, "percentChangeColorMode": "standard", "orientation": "auto" }, "fieldConfig": { "defaults": { "unit": "s", "thresholds": { "mode": "absolute", "steps": [ { "value": 2, "color": "#CEFFFF" }, { "value": 4, "color": "#C6F7D6" }, { "value": 9, "color": "#A2F49B" }, { "value": 19, "color": "#BBE453" }, { "value": 41, "color": "#D5CE04" }, { "value": 87, "color": "#E7B503" }, { "value": 183, "color": "#F19903" }, { "value": 385, "color": "#F6790B" }, { "value": 812, "color": "#F94902" }, { "value": 1710, "color": "#E40515" }, { "value": 3600, "color": "#AB0003" } ] }, "noValue": "No matches for the current filters" }, "overrides": [] } } } } }, "cpu-usage-current": { "kind": "Panel", "spec": { "id": 1005, "title": "Current CPU Usage (5 min)", "description": "**Current CPU usage as a fraction of each container's limit, averaged over the last 5 minutes.** Per-container gauge \u2014 shows the worst-loaded container types in the env. Nominal: well below 1.0; sustained near 1.0 means a container type is CPU-bound. For time-resolved per-pod view see _Kubernetes Workloads -> Pod CPU Usage_; for the Materialize workload causing it see _Compute Objects -> Dataflow Elapsed Rate_.", "links": [], "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "\nsum by (namespace, container) (\n rate(\n container_cpu_usage_seconds_total{$containerFilter}[5m]\n )\n) / sum by (namespace, container) (\n kube_pod_container_resource_limits{resource=\"cpu\", namespace=~\"$mzNamespaceList\"}\n)\n", "instant": true, "range": false, "legendFormat": "{{container}}", "query": "\nsum by (namespace, container) (\n rate(\n container_cpu_usage_seconds_total{$containerFilter}[5m]\n )\n) / sum by (namespace, container) (\n kube_pod_container_resource_limits{resource=\"cpu\", namespace=~\"$mzNamespaceList\"}\n)\n", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-0", "hidden": false } } ], "transformations": [], "queryOptions": {} } }, "vizConfig": { "kind": "VizConfig", "group": "gauge", "version": "", "spec": { "options": { "showThresholdLabels": false, "showThresholdMarkers": true, "sizing": "auto", "minVizWidth": 75, "reduceOptions": { "calcs": [] }, "minVizHeight": 75, "orientation": "auto" }, "fieldConfig": { "defaults": { "unit": "percentunit", "thresholds": { "mode": "absolute", "steps": [ { "value": 0.0, "color": "#CEFFFF" }, { "value": 0.09090909090909091, "color": "#C6F7D6" }, { "value": 0.18181818181818182, "color": "#A2F49B" }, { "value": 0.2727272727272727, "color": "#BBE453" }, { "value": 0.36363636363636365, "color": "#D5CE04" }, { "value": 0.4545454545454546, "color": "#E7B503" }, { "value": 0.5454545454545454, "color": "#F19903" }, { "value": 0.6363636363636364, "color": "#F6790B" }, { "value": 0.7272727272727273, "color": "#F94902" }, { "value": 0.8181818181818182, "color": "#E40515" }, { "value": 0.9090909090909092, "color": "#AB0003" } ] }, "noValue": "No metrics: cadvisor metrics are required (via kubelet)" }, "overrides": [] } } } } }, "memory-usage-current": { "kind": "Panel", "spec": { "id": 1006, "title": "Current Memory Usage", "description": "**Current memory usage as a fraction of each container's limit.** Per-container gauge \u2014 shows the worst-loaded container types. **Sustained near 1.0 is dangerous** \u2014 OOM-kill triggers a hydration cycle (in-memory state has to be rebuilt from persisted storage, taking minutes-to-hours depending on data size). For time-resolved view see _Kubernetes Workloads -> Pod Memory Usage_; the offending workload usually shows in _Compute Objects -> Arrangements_.", "links": [], "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "\nsum by (namespace, container) (\n avg by (namespace, pod, container) (\n container_memory_working_set_bytes{$containerFilter, container!=\"new-promsql-exporter\"}\n )\n) / sum by (namespace, container) (\n avg by (namespace, pod, container) (\n container_spec_memory_limit_bytes{$containerFilter, container!=\"new-promsql-exporter\"}\n )\n)\n", "instant": true, "range": false, "legendFormat": "{{container}}", "query": "\nsum by (namespace, container) (\n avg by (namespace, pod, container) (\n container_memory_working_set_bytes{$containerFilter, container!=\"new-promsql-exporter\"}\n )\n) / sum by (namespace, container) (\n avg by (namespace, pod, container) (\n container_spec_memory_limit_bytes{$containerFilter, container!=\"new-promsql-exporter\"}\n )\n)\n", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-0", "hidden": false } } ], "transformations": [], "queryOptions": {} } }, "vizConfig": { "kind": "VizConfig", "group": "gauge", "version": "", "spec": { "options": { "showThresholdLabels": false, "showThresholdMarkers": true, "sizing": "auto", "minVizWidth": 75, "reduceOptions": { "calcs": [] }, "minVizHeight": 75, "orientation": "auto" }, "fieldConfig": { "defaults": { "unit": "percentunit", "thresholds": { "mode": "absolute", "steps": [ { "value": 0.0, "color": "#CEFFFF" }, { "value": 0.09090909090909091, "color": "#C6F7D6" }, { "value": 0.18181818181818182, "color": "#A2F49B" }, { "value": 0.2727272727272727, "color": "#BBE453" }, { "value": 0.36363636363636365, "color": "#D5CE04" }, { "value": 0.4545454545454546, "color": "#E7B503" }, { "value": 0.5454545454545454, "color": "#F19903" }, { "value": 0.6363636363636364, "color": "#F6790B" }, { "value": 0.7272727272727273, "color": "#F94902" }, { "value": 0.8181818181818182, "color": "#E40515" }, { "value": 0.9090909090909092, "color": "#AB0003" } ] }, "noValue": "No metrics: cadvisor metrics are required (via kubelet)" }, "overrides": [] } } } } }, "materialize-version": { "kind": "Panel", "spec": { "id": 1007, "title": "Materialize Version", "description": "**The version of Materialize currently running in the environment.** A single version is the steady state; multiple distinct values typically appear briefly during a rolling upgrade. Click the value to open the corresponding commit on GitHub. If the version doesn't match what you expect, check the env's manifest in the deployment pipeline.", "links": [], "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "\ngroup by (mz_version) (\n ${sqlMetricPrefix}compute_cluster_status{$environmentFilter}\n)\n", "instant": true, "range": false, "legendFormat": "{{mz_version}}", "query": "\ngroup by (mz_version) (\n ${sqlMetricPrefix}compute_cluster_status{$environmentFilter}\n)\n", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-0", "hidden": false } } ], "transformations": [ { "kind": "Transformation", "spec": { "id": "labelsToFields", "options": {} }, "group": "labelsToFields" }, { "kind": "Transformation", "spec": { "id": "extractFields", "options": { "format": "regexp", "regExp": "/.+\\((?[a-fA-F0-9]+)\\)/", "source": "mz_version", "replace": false } }, "group": "extractFields" } ], "queryOptions": {} } }, "vizConfig": { "kind": "VizConfig", "group": "stat", "version": "", "spec": { "options": { "graphMode": "none", "colorMode": "none", "justifyMode": "auto", "textMode": "value", "wideLayout": true, "showPercentChange": false, "reduceOptions": { "calcs": [], "fields": "/^mz_version$/" }, "percentChangeColorMode": "standard", "orientation": "auto", "text": { "valueSize": 20 } }, "fieldConfig": { "defaults": { "links": [ { "title": "View Materialize at Commit", "url": "https://github.com/MaterializeInc/materialize/commit/${__data.fields.commit}", "targetBlank": true } ] }, "overrides": [] } } } } }, "summary-cpu-total": { "kind": "Panel", "spec": { "id": 1008, "title": "Total CPU Capacity", "description": "**Total CPU cores configured across containers in the selected scope** (sum of CPU limits from cAdvisor). Steps correlate with `ALTER CLUSTER REPLICA SIZE`, `CREATE`/`DROP CLUSTER REPLICA`, or pod restarts. On the Summary tab the monitoring exporter is excluded (so this reflects user-workload capacity); on the Kubernetes Workloads tab it's included.", "links": [], "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "\nsum by (container) (\n container_spec_cpu_quota{ $containerFilter, container!=\"new-promsql-exporter\" }\n / container_spec_cpu_period{ $containerFilter, container!=\"new-promsql-exporter\" }\n)\n", "legendFormat": "CPUs ({{container}})", "query": "\nsum by (container) (\n container_spec_cpu_quota{ $containerFilter, container!=\"new-promsql-exporter\" }\n / container_spec_cpu_period{ $containerFilter, container!=\"new-promsql-exporter\" }\n)\n", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-0", "hidden": false } } ], "transformations": [], "queryOptions": {} } }, "vizConfig": { "kind": "VizConfig", "group": "stat", "version": "", "spec": { "options": { "graphMode": "area", "colorMode": "none", "justifyMode": "auto", "textMode": "value_and_name", "wideLayout": true, "showPercentChange": false, "reduceOptions": { "calcs": [] }, "percentChangeColorMode": "standard", "orientation": "auto" }, "fieldConfig": { "defaults": { "unit": "cores", "color": { "mode": "shades", "fixedColor": "#0077BB" }, "noValue": "No metrics: cadvisor metrics are required (via kubelet)" }, "overrides": [] } } } } }, "summary-memory-total": { "kind": "Panel", "spec": { "id": 1009, "title": "Total Memory", "description": "**Total memory configured across containers in the selected scope** (sum of memory limits from cAdvisor). Memory is the dominant constraint on Materialize: in-memory arrangements (see _Compute Objects -> Arrangements_) live in here. Steps correlate with `ALTER CLUSTER REPLICA SIZE` or pod restarts.", "links": [], "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "\nsum by (container) (\n container_spec_memory_limit_bytes{ $containerFilter, container!=\"new-promsql-exporter\" }\n)\n", "legendFormat": "Memory ({{container}})", "query": "\nsum by (container) (\n container_spec_memory_limit_bytes{ $containerFilter, container!=\"new-promsql-exporter\" }\n)\n", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-0", "hidden": false } } ], "transformations": [], "queryOptions": {} } }, "vizConfig": { "kind": "VizConfig", "group": "stat", "version": "", "spec": { "options": { "graphMode": "area", "colorMode": "none", "justifyMode": "auto", "textMode": "value_and_name", "wideLayout": true, "showPercentChange": false, "reduceOptions": { "calcs": [] }, "percentChangeColorMode": "standard", "orientation": "auto" }, "fieldConfig": { "defaults": { "unit": "bytes", "color": { "mode": "shades", "fixedColor": "#0077BB" }, "noValue": "No metrics: cadvisor metrics are required (via kubelet)" }, "overrides": [] } } } } }, "k8s-res-cpu-total": { "kind": "Panel", "spec": { "id": 1010, "title": "Total CPU Capacity", "description": "**Total CPU cores configured across containers in the selected scope** (sum of CPU limits from cAdvisor). Steps correlate with `ALTER CLUSTER REPLICA SIZE`, `CREATE`/`DROP CLUSTER REPLICA`, or pod restarts. On the Summary tab the monitoring exporter is excluded (so this reflects user-workload capacity); on the Kubernetes Workloads tab it's included.", "links": [], "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "\nsum by (container) (\n container_spec_cpu_quota{ $containerFilter }\n / container_spec_cpu_period{ $containerFilter }\n)\n", "legendFormat": "CPUs ({{container}})", "query": "\nsum by (container) (\n container_spec_cpu_quota{ $containerFilter }\n / container_spec_cpu_period{ $containerFilter }\n)\n", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-0", "hidden": false } } ], "transformations": [], "queryOptions": {} } }, "vizConfig": { "kind": "VizConfig", "group": "stat", "version": "", "spec": { "options": { "graphMode": "area", "colorMode": "none", "justifyMode": "auto", "textMode": "value_and_name", "wideLayout": true, "showPercentChange": false, "reduceOptions": { "calcs": [] }, "percentChangeColorMode": "standard", "orientation": "auto" }, "fieldConfig": { "defaults": { "unit": "cores", "color": { "mode": "shades", "fixedColor": "#0077BB" }, "noValue": "No metrics: cadvisor metrics are required (via kubelet)" }, "overrides": [] } } } } }, "k8s-res-memory-total": { "kind": "Panel", "spec": { "id": 1011, "title": "Total Memory", "description": "**Total memory configured across containers in the selected scope** (sum of memory limits from cAdvisor). Memory is the dominant constraint on Materialize: in-memory arrangements (see _Compute Objects -> Arrangements_) live in here. Steps correlate with `ALTER CLUSTER REPLICA SIZE` or pod restarts.", "links": [], "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "\nsum by (container) (\n container_spec_memory_limit_bytes{ $containerFilter }\n)\n", "legendFormat": "Memory ({{container}})", "query": "\nsum by (container) (\n container_spec_memory_limit_bytes{ $containerFilter }\n)\n", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-0", "hidden": false } } ], "transformations": [], "queryOptions": {} } }, "vizConfig": { "kind": "VizConfig", "group": "stat", "version": "", "spec": { "options": { "graphMode": "area", "colorMode": "none", "justifyMode": "auto", "textMode": "value_and_name", "wideLayout": true, "showPercentChange": false, "reduceOptions": { "calcs": [] }, "percentChangeColorMode": "standard", "orientation": "auto" }, "fieldConfig": { "defaults": { "unit": "bytes", "color": { "mode": "shades", "fixedColor": "#0077BB" }, "noValue": "No metrics: cadvisor metrics are required (via kubelet)" }, "overrides": [] } } } } }, "resource-pod-status": { "kind": "Panel", "spec": { "id": 1012, "title": "Pod Readiness", "description": "**Pods in the Materialize namespace grouped by phase** (Running, Pending, Failed, etc.). Nominal: nearly all `Running`. Pods stuck in `Pending` usually mean Kubernetes can't schedule them (capacity, taints, AZ constraints); `Failed` means a container exited and won't be restarted. Pairs with _Last Restart Time_ on the Summary tab. Requires kube-state-metrics.", "links": [], "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "\nmax by (phase, namespace) (\n sum by (phase, namespace, instance) (\n kube_pod_status_phase{namespace=~\"$mzNamespaceList\"}\n )\n)\n", "instant": true, "range": false, "legendFormat": "{{phase}}", "query": "\nmax by (phase, namespace) (\n sum by (phase, namespace, instance) (\n kube_pod_status_phase{namespace=~\"$mzNamespaceList\"}\n )\n)\n", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-0", "hidden": false } } ], "transformations": [], "queryOptions": {} } }, "vizConfig": { "kind": "VizConfig", "group": "piechart", "version": "", "spec": { "options": { "pieType": "donut", "tooltip": { "mode": "single", "sort": "asc" }, "reduceOptions": { "calcs": [] }, "legend": { "values": [ "value" ], "displayMode": "table", "placement": "right", "showLegend": true, "calcs": [], "asTable": true, "isVisible": true }, "orientation": "auto", "displayLabels": [ "name", "value" ] }, "fieldConfig": { "defaults": { "color": { "mode": "shades", "fixedColor": "#0077BB" }, "noValue": "No metrics: kube-state-metrics is required" }, "overrides": [] } } } } }, "resource-statefulset-status": { "kind": "Panel", "spec": { "id": 1013, "title": "StatefulSet Readiness", "description": "**Number of StatefulSet replicas reporting Ready.** environmentd and Materialize's cluster pods are StatefulSets; this panel counts the replicas that have reached the Ready state. Nominal: matches the configured replica count. A drop indicates a pod stuck in initialization or hydration. Requires kube-state-metrics.", "links": [], "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "\nmax by (namespace) (\n sum by (namespace, instance) (\n kube_statefulset_status_replicas_ready{namespace=~\"$mzNamespaceList\"}\n )\n)\n", "instant": true, "range": false, "legendFormat": "Ready", "query": "\nmax by (namespace) (\n sum by (namespace, instance) (\n kube_statefulset_status_replicas_ready{namespace=~\"$mzNamespaceList\"}\n )\n)\n", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-0", "hidden": false } } ], "transformations": [], "queryOptions": {} } }, "vizConfig": { "kind": "VizConfig", "group": "piechart", "version": "", "spec": { "options": { "pieType": "donut", "tooltip": { "mode": "single", "sort": "asc" }, "reduceOptions": { "calcs": [] }, "legend": { "values": [ "value" ], "displayMode": "table", "placement": "right", "showLegend": true, "calcs": [], "asTable": true, "isVisible": true }, "orientation": "auto", "displayLabels": [ "name", "value" ] }, "fieldConfig": { "defaults": { "color": { "mode": "shades", "fixedColor": "#0077BB" }, "noValue": "No metrics: kube-state-metrics is required" }, "overrides": [] } } } } }, "resource-deployment-status": { "kind": "Panel", "spec": { "id": 1014, "title": "Deployment Readiness", "description": "**Deployment replica health \u2014 Ready vs Unavailable.** Deployments back stateless services (e.g., the promsql exporter). Nominal: all replicas Ready, zero Unavailable. Unavailable counts indicate failed rollouts or crashing pods. Requires kube-state-metrics.", "links": [], "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "\nmax by (namespace) (\n sum by (namespace, instance) (\n kube_deployment_status_replicas_ready{namespace=~\"$mzNamespaceList\"}\n )\n)\n", "instant": true, "range": false, "legendFormat": "Ready", "query": "\nmax by (namespace) (\n sum by (namespace, instance) (\n kube_deployment_status_replicas_ready{namespace=~\"$mzNamespaceList\"}\n )\n)\n", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-0", "hidden": false } }, { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "\nmax by (namespace) (\n sum by (namespace, instance) (\n kube_deployment_status_replicas_unavailable{namespace=~\"$mzNamespaceList\"}\n )\n)\n", "instant": true, "range": false, "legendFormat": "Unavailable", "query": "\nmax by (namespace) (\n sum by (namespace, instance) (\n kube_deployment_status_replicas_unavailable{namespace=~\"$mzNamespaceList\"}\n )\n)\n", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-1", "hidden": false } } ], "transformations": [], "queryOptions": {} } }, "vizConfig": { "kind": "VizConfig", "group": "piechart", "version": "", "spec": { "options": { "pieType": "donut", "tooltip": { "mode": "single", "sort": "asc" }, "reduceOptions": { "calcs": [] }, "legend": { "values": [ "value" ], "displayMode": "table", "placement": "right", "showLegend": true, "calcs": [], "asTable": true, "isVisible": true }, "orientation": "auto", "displayLabels": [ "name", "value" ] }, "fieldConfig": { "defaults": { "color": { "mode": "shades", "fixedColor": "#0077BB" }, "noValue": "No metrics: kube-state-metrics is required" }, "overrides": [] } } } } }, "pod-cpu-percent": { "kind": "Panel", "spec": { "id": 1015, "title": "Pod CPU Usage", "description": "**CPU utilization per pod, as a fraction of the pod's CPU limit.** Two-query split: one for cluster replica pods (filtered by the dashboard's cluster/replica selectors), one for everything else (envd, balancer, exporter, etc.). Sustained near 1.0 for a pod means it's CPU-bound. For the Materialize-level cause see _Compute Objects -> Dataflow Elapsed Rate_ or _Arrangements_.", "links": [], "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "\nsum by (namespace, pod, container) (\n rate(\n container_cpu_usage_seconds_total{$containerFilter, pod=~\".*-cluster-${mzClusterList:regex}-replica-${mzReplicaList:regex}-.*\"}[$__rate_interval]\n )\n) / sum by (namespace, pod, container) (\n kube_pod_container_resource_limits{resource=\"cpu\", namespace=~\"$mzNamespaceList\", pod=~\".*-cluster-${mzClusterList:regex}-replica-${mzReplicaList:regex}-.*\"}\n)\n", "legendFormat": "{{pod}} / {{container}}", "query": "\nsum by (namespace, pod, container) (\n rate(\n container_cpu_usage_seconds_total{$containerFilter, pod=~\".*-cluster-${mzClusterList:regex}-replica-${mzReplicaList:regex}-.*\"}[$__rate_interval]\n )\n) / sum by (namespace, pod, container) (\n kube_pod_container_resource_limits{resource=\"cpu\", namespace=~\"$mzNamespaceList\", pod=~\".*-cluster-${mzClusterList:regex}-replica-${mzReplicaList:regex}-.*\"}\n)\n", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-0", "hidden": false } }, { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "\nsum by (namespace, pod, container) (\n rate(\n container_cpu_usage_seconds_total{$containerFilter, pod!~\".*-cluster-.*-replica-.*\"}[$__rate_interval]\n )\n) / sum by (namespace, pod, container) (\n kube_pod_container_resource_limits{resource=\"cpu\", namespace=~\"$mzNamespaceList\", pod!~\".*-cluster-.*-replica-.*\"}\n)\n", "legendFormat": "{{pod}} / {{container}}", "query": "\nsum by (namespace, pod, container) (\n rate(\n container_cpu_usage_seconds_total{$containerFilter, pod!~\".*-cluster-.*-replica-.*\"}[$__rate_interval]\n )\n) / sum by (namespace, pod, container) (\n kube_pod_container_resource_limits{resource=\"cpu\", namespace=~\"$mzNamespaceList\", pod!~\".*-cluster-.*-replica-.*\"}\n)\n", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-1", "hidden": false } } ], "transformations": [], "queryOptions": {} } }, "vizConfig": { "kind": "VizConfig", "group": "timeseries", "version": "", "spec": { "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": [ "max", "mean", "lastNotNull" ] }, "tooltip": { "mode": "single", "sort": "asc" } }, "fieldConfig": { "defaults": { "unit": "percentunit", "noValue": "No metrics: cadvisor and kube-state-metrics are required" }, "overrides": [] } } } } }, "pod-memory-percent": { "kind": "Panel", "spec": { "id": 1016, "title": "Pod Memory Usage", "description": "**Memory usage per pod, as a fraction of the pod's memory limit (working-set basis).** Same two-query split as Pod CPU Usage. **Sustained climb toward 1.0 is dangerous** \u2014 a pod hitting its memory limit gets OOM-killed, which on a compute replica triggers a hydration cycle (in-memory state rebuilt from persistence, often minutes). If a Materialize cluster pod is the offender, _Compute Objects -> Arrangements_ shows which arrangements consume the memory.", "links": [], "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "\navg by (namespace, pod, container) (\n container_memory_working_set_bytes{$containerFilter, container!=\"new-promsql-exporter\", pod=~\".*-cluster-${mzClusterList:regex}-replica-${mzReplicaList:regex}-.*\"}\n) / avg by (namespace, pod, container) (\n container_spec_memory_limit_bytes{$containerFilter, container!=\"new-promsql-exporter\", pod=~\".*-cluster-${mzClusterList:regex}-replica-${mzReplicaList:regex}-.*\"}\n)\n", "legendFormat": "{{pod}} / {{container}}", "query": "\navg by (namespace, pod, container) (\n container_memory_working_set_bytes{$containerFilter, container!=\"new-promsql-exporter\", pod=~\".*-cluster-${mzClusterList:regex}-replica-${mzReplicaList:regex}-.*\"}\n) / avg by (namespace, pod, container) (\n container_spec_memory_limit_bytes{$containerFilter, container!=\"new-promsql-exporter\", pod=~\".*-cluster-${mzClusterList:regex}-replica-${mzReplicaList:regex}-.*\"}\n)\n", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-0", "hidden": false } }, { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "\navg by (namespace, pod, container) (\n container_memory_working_set_bytes{$containerFilter, container!=\"new-promsql-exporter\", pod!~\".*-cluster-.*-replica-.*\"}\n) / avg by (namespace, pod, container) (\n container_spec_memory_limit_bytes{$containerFilter, container!=\"new-promsql-exporter\", pod!~\".*-cluster-.*-replica-.*\"}\n)\n", "legendFormat": "{{pod}} / {{container}}", "query": "\navg by (namespace, pod, container) (\n container_memory_working_set_bytes{$containerFilter, container!=\"new-promsql-exporter\", pod!~\".*-cluster-.*-replica-.*\"}\n) / avg by (namespace, pod, container) (\n container_spec_memory_limit_bytes{$containerFilter, container!=\"new-promsql-exporter\", pod!~\".*-cluster-.*-replica-.*\"}\n)\n", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-1", "hidden": false } } ], "transformations": [], "queryOptions": {} } }, "vizConfig": { "kind": "VizConfig", "group": "timeseries", "version": "", "spec": { "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": [ "max", "mean", "lastNotNull" ] }, "tooltip": { "mode": "single", "sort": "asc" } }, "fieldConfig": { "defaults": { "unit": "percentunit", "noValue": "No metrics: cadvisor metrics are required (via kubelet)" }, "overrides": [] } } } } }, "pod-network-rx": { "kind": "Panel", "spec": { "id": 1017, "title": "Pod Network Rx", "description": "**Network bytes/sec received per pod**, aggregated across all network interfaces. Same cluster/non-cluster split as the pod CPU/memory panels. For cluster pods, Rx tracks ingest from upstream (Kafka, Postgres, etc.) and inter-pod replication; for envd and the balancer it reflects client SQL traffic. Surges that coincide with _Compute Objects -> Hydration_ activity are normal (catchup); surges otherwise can mean a runaway client or source.", "links": [], "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "\nsum by (namespace, pod) (\n rate(\n container_network_receive_bytes_total{namespace=~\"$mzNamespaceList\", pod=~\".*-cluster-${mzClusterList:regex}-replica-${mzReplicaList:regex}-.*\"}[$__rate_interval]\n )\n)\n", "legendFormat": "{{pod}}", "query": "\nsum by (namespace, pod) (\n rate(\n container_network_receive_bytes_total{namespace=~\"$mzNamespaceList\", pod=~\".*-cluster-${mzClusterList:regex}-replica-${mzReplicaList:regex}-.*\"}[$__rate_interval]\n )\n)\n", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-0", "hidden": false } }, { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "\nsum by (namespace, pod) (\n rate(\n container_network_receive_bytes_total{namespace=~\"$mzNamespaceList\", pod!~\".*-cluster-.*-replica-.*\"}[$__rate_interval]\n )\n)\n", "legendFormat": "{{pod}}", "query": "\nsum by (namespace, pod) (\n rate(\n container_network_receive_bytes_total{namespace=~\"$mzNamespaceList\", pod!~\".*-cluster-.*-replica-.*\"}[$__rate_interval]\n )\n)\n", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-1", "hidden": false } } ], "transformations": [], "queryOptions": {} } }, "vizConfig": { "kind": "VizConfig", "group": "timeseries", "version": "", "spec": { "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": [ "max", "mean", "lastNotNull" ] }, "tooltip": { "mode": "single", "sort": "asc" } }, "fieldConfig": { "defaults": { "unit": "Bps", "noValue": "No metrics: cadvisor metrics are required (via kubelet)" }, "overrides": [] } } } } }, "pod-network-tx": { "kind": "Panel", "spec": { "id": 1018, "title": "Pod Network Tx", "description": "**Network bytes/sec transmitted per pod**, aggregated across interfaces. For cluster pods Tx covers sink output, inter-pod replication, and query results returning to envd; for envd it's client query responses. Pairs with _Storage Objects -> Sink Throughput_ when investigating sink-side bandwidth.", "links": [], "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "\nsum by (namespace, pod) (\n rate(\n container_network_transmit_bytes_total{namespace=~\"$mzNamespaceList\", pod=~\".*-cluster-${mzClusterList:regex}-replica-${mzReplicaList:regex}-.*\"}[$__rate_interval]\n )\n)\n", "legendFormat": "{{pod}}", "query": "\nsum by (namespace, pod) (\n rate(\n container_network_transmit_bytes_total{namespace=~\"$mzNamespaceList\", pod=~\".*-cluster-${mzClusterList:regex}-replica-${mzReplicaList:regex}-.*\"}[$__rate_interval]\n )\n)\n", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-0", "hidden": false } }, { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "\nsum by (namespace, pod) (\n rate(\n container_network_transmit_bytes_total{namespace=~\"$mzNamespaceList\", pod!~\".*-cluster-.*-replica-.*\"}[$__rate_interval]\n )\n)\n", "legendFormat": "{{pod}}", "query": "\nsum by (namespace, pod) (\n rate(\n container_network_transmit_bytes_total{namespace=~\"$mzNamespaceList\", pod!~\".*-cluster-.*-replica-.*\"}[$__rate_interval]\n )\n)\n", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-1", "hidden": false } } ], "transformations": [], "queryOptions": {} } }, "vizConfig": { "kind": "VizConfig", "group": "timeseries", "version": "", "spec": { "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": [ "max", "mean", "lastNotNull" ] }, "tooltip": { "mode": "single", "sort": "asc" } }, "fieldConfig": { "defaults": { "unit": "Bps", "noValue": "No metrics: cadvisor metrics are required (via kubelet)" }, "overrides": [] } } } } }, "pod-network-errors": { "kind": "Panel", "spec": { "id": 1019, "title": "Pod Network Errors", "description": "**Network rx + tx errors per pod per second** (errors counted at the NIC/kernel level). Nominal: 0. Non-zero is unusual and points at infrastructure problems (faulty NIC, kernel network stack issues, container runtime bugs) \u2014 not Materialize-level. If you see persistent non-zero, file an infra ticket; this isn't fixable from within Materialize.", "links": [], "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "\nsum by (namespace, pod) (\n rate(\n container_network_receive_errors_total{namespace=~\"$mzNamespaceList\", pod=~\".*-cluster-${mzClusterList:regex}-replica-${mzReplicaList:regex}-.*\"}[$__rate_interval]\n )\n)\n", "legendFormat": "{{pod}} rx", "query": "\nsum by (namespace, pod) (\n rate(\n container_network_receive_errors_total{namespace=~\"$mzNamespaceList\", pod=~\".*-cluster-${mzClusterList:regex}-replica-${mzReplicaList:regex}-.*\"}[$__rate_interval]\n )\n)\n", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-0", "hidden": false } }, { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "\nsum by (namespace, pod) (\n rate(\n container_network_receive_errors_total{namespace=~\"$mzNamespaceList\", pod!~\".*-cluster-.*-replica-.*\"}[$__rate_interval]\n )\n)\n", "legendFormat": "{{pod}} rx", "query": "\nsum by (namespace, pod) (\n rate(\n container_network_receive_errors_total{namespace=~\"$mzNamespaceList\", pod!~\".*-cluster-.*-replica-.*\"}[$__rate_interval]\n )\n)\n", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-1", "hidden": false } }, { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "\nsum by (namespace, pod) (\n rate(\n container_network_transmit_errors_total{namespace=~\"$mzNamespaceList\", pod=~\".*-cluster-${mzClusterList:regex}-replica-${mzReplicaList:regex}-.*\"}[$__rate_interval]\n )\n)\n", "legendFormat": "{{pod}} tx", "query": "\nsum by (namespace, pod) (\n rate(\n container_network_transmit_errors_total{namespace=~\"$mzNamespaceList\", pod=~\".*-cluster-${mzClusterList:regex}-replica-${mzReplicaList:regex}-.*\"}[$__rate_interval]\n )\n)\n", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-2", "hidden": false } }, { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "\nsum by (namespace, pod) (\n rate(\n container_network_transmit_errors_total{namespace=~\"$mzNamespaceList\", pod!~\".*-cluster-.*-replica-.*\"}[$__rate_interval]\n )\n)\n", "legendFormat": "{{pod}} tx", "query": "\nsum by (namespace, pod) (\n rate(\n container_network_transmit_errors_total{namespace=~\"$mzNamespaceList\", pod!~\".*-cluster-.*-replica-.*\"}[$__rate_interval]\n )\n)\n", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-3", "hidden": false } } ], "transformations": [], "queryOptions": {} } }, "vizConfig": { "kind": "VizConfig", "group": "timeseries", "version": "", "spec": { "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": [ "max", "mean", "lastNotNull" ] }, "tooltip": { "mode": "single", "sort": "asc" } }, "fieldConfig": { "defaults": { "unit": "cps", "noValue": "No metrics: cadvisor metrics are required (via kubelet)" }, "overrides": [] } } } } }, "pod-network-drops": { "kind": "Panel", "spec": { "id": 1020, "title": "Pod Network Packet Drops", "description": "**Network packets dropped (rx + tx) per pod per second.** Drops happen when the kernel's network buffers fill up faster than the application can read from them (rx) or when egress rate-limiting kicks in (tx). Nominal: 0. Low-level non-zero drops (single-digit pps) are usually harmless background noise; sustained higher rates indicate the pod is overwhelmed at the network layer \u2014 often paired with elevated _Pod CPU Usage_.", "links": [], "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "\nsum by (namespace, pod) (\n rate(\n container_network_receive_packets_dropped_total{namespace=~\"$mzNamespaceList\", pod=~\".*-cluster-${mzClusterList:regex}-replica-${mzReplicaList:regex}-.*\"}[$__rate_interval]\n )\n)\n", "legendFormat": "{{pod}} rx", "query": "\nsum by (namespace, pod) (\n rate(\n container_network_receive_packets_dropped_total{namespace=~\"$mzNamespaceList\", pod=~\".*-cluster-${mzClusterList:regex}-replica-${mzReplicaList:regex}-.*\"}[$__rate_interval]\n )\n)\n", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-0", "hidden": false } }, { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "\nsum by (namespace, pod) (\n rate(\n container_network_receive_packets_dropped_total{namespace=~\"$mzNamespaceList\", pod!~\".*-cluster-.*-replica-.*\"}[$__rate_interval]\n )\n)\n", "legendFormat": "{{pod}} rx", "query": "\nsum by (namespace, pod) (\n rate(\n container_network_receive_packets_dropped_total{namespace=~\"$mzNamespaceList\", pod!~\".*-cluster-.*-replica-.*\"}[$__rate_interval]\n )\n)\n", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-1", "hidden": false } }, { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "\nsum by (namespace, pod) (\n rate(\n container_network_transmit_packets_dropped_total{namespace=~\"$mzNamespaceList\", pod=~\".*-cluster-${mzClusterList:regex}-replica-${mzReplicaList:regex}-.*\"}[$__rate_interval]\n )\n)\n", "legendFormat": "{{pod}} tx", "query": "\nsum by (namespace, pod) (\n rate(\n container_network_transmit_packets_dropped_total{namespace=~\"$mzNamespaceList\", pod=~\".*-cluster-${mzClusterList:regex}-replica-${mzReplicaList:regex}-.*\"}[$__rate_interval]\n )\n)\n", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-2", "hidden": false } }, { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "\nsum by (namespace, pod) (\n rate(\n container_network_transmit_packets_dropped_total{namespace=~\"$mzNamespaceList\", pod!~\".*-cluster-.*-replica-.*\"}[$__rate_interval]\n )\n)\n", "legendFormat": "{{pod}} tx", "query": "\nsum by (namespace, pod) (\n rate(\n container_network_transmit_packets_dropped_total{namespace=~\"$mzNamespaceList\", pod!~\".*-cluster-.*-replica-.*\"}[$__rate_interval]\n )\n)\n", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-3", "hidden": false } } ], "transformations": [], "queryOptions": {} } }, "vizConfig": { "kind": "VizConfig", "group": "timeseries", "version": "", "spec": { "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": [ "max", "mean", "lastNotNull" ] }, "tooltip": { "mode": "single", "sort": "asc" } }, "fieldConfig": { "defaults": { "unit": "pps", "noValue": "No metrics: cadvisor metrics are required (via kubelet)" }, "overrides": [] } } } } }, "connections-active-sessions": { "kind": "Panel", "spec": { "id": 1021, "title": "Active Sessions", "description": "**Currently-open SQL sessions, broken down by session type (`system` vs `user`).** `system` sessions come from Materialize's internal probing (a few are always present); `user` sessions come from client connections. Nominal: a small steady `system` count and a variable `user` count tracking your client activity. Sustained high `user` count is often a leaked-connection signal \u2014 sanity-check by seeing whether _Active Queries_ shows commensurate activity. Environment-scoped.", "links": [], "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "\nsum by (session_type) (\n mz_active_sessions{$environmentFilter}\n)\n", "legendFormat": "{{session_type}}", "query": "\nsum by (session_type) (\n mz_active_sessions{$environmentFilter}\n)\n", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-0", "hidden": false } } ], "transformations": [], "queryOptions": {} } }, "vizConfig": { "kind": "VizConfig", "group": "stat", "version": "", "spec": { "options": { "graphMode": "area", "colorMode": "none", "justifyMode": "auto", "textMode": "value_and_name", "wideLayout": true, "showPercentChange": false, "reduceOptions": { "calcs": [] }, "percentChangeColorMode": "standard", "orientation": "auto" }, "fieldConfig": { "defaults": { "min": 0, "color": { "mode": "shades", "fixedColor": "#33BBEE" }, "noValue": "No matches for the current filters" }, "overrides": [] } } } } }, "connections-active-queries": { "kind": "Panel", "spec": { "id": 1022, "title": "Active Queries", "description": "**Queries per second by session type, rated from the `mz_query_total` counter.** Bursty in normal operation \u2014 `user` tracks your client traffic shape, `system` reflects internal health-checks (typically a steady single-digit baseline). Use _Query Distribution_ to see *what kinds* of queries make up the rate, and _Peek Latency_ to confirm the queries are running fast enough. Environment-scoped.", "links": [], "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "\nsum by (session_type) (\n rate(mz_query_total{$environmentFilter}[$__rate_interval])\n)\n", "legendFormat": "{{session_type}}", "query": "\nsum by (session_type) (\n rate(mz_query_total{$environmentFilter}[$__rate_interval])\n)\n", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-0", "hidden": false } } ], "transformations": [], "queryOptions": {} } }, "vizConfig": { "kind": "VizConfig", "group": "stat", "version": "", "spec": { "options": { "graphMode": "area", "colorMode": "none", "justifyMode": "auto", "textMode": "value_and_name", "wideLayout": true, "showPercentChange": false, "reduceOptions": { "calcs": [] }, "percentChangeColorMode": "standard", "orientation": "auto" }, "fieldConfig": { "defaults": { "unit": "cps", "min": 0, "color": { "mode": "shades", "fixedColor": "#33BBEE" }, "noValue": "No matches for the current filters" }, "overrides": [] } } } } }, "connections-adapter-command-rate": { "kind": "Panel", "spec": { "id": 1023, "title": "Adapter Command Rate", "description": "**Commands per second across the adapter** \u2014 the SQL protocol layer that handles parse, execute, prepare, fetch, etc. Usually higher than the query rate because each query produces several commands. Sudden flat-line on a usually-busy env is unusual (could indicate adapter trouble). Use _Adapter Commands by Application_ below to see which clients dominate, and watch its Errors column for failed commands. Environment-scoped.", "links": [], "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "\nsum(\n rate(mz_adapter_commands{$environmentFilter}[$__rate_interval])\n)\n", "legendFormat": "commands", "query": "\nsum(\n rate(mz_adapter_commands{$environmentFilter}[$__rate_interval])\n)\n", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-0", "hidden": false } } ], "transformations": [], "queryOptions": {} } }, "vizConfig": { "kind": "VizConfig", "group": "stat", "version": "", "spec": { "options": { "graphMode": "area", "colorMode": "none", "justifyMode": "auto", "textMode": "value", "wideLayout": true, "showPercentChange": false, "reduceOptions": { "calcs": [] }, "percentChangeColorMode": "standard", "orientation": "auto" }, "fieldConfig": { "defaults": { "unit": "cps", "min": 0, "color": { "mode": "shades", "fixedColor": "#33BBEE" }, "noValue": "No matches for the current filters" }, "overrides": [] } } } } }, "queries-distribution": { "kind": "Panel", "spec": { "id": 1024, "title": "Query Distribution (by statement_type)", "description": "**Share of queries by statement type over the dashboard's time range** \u2014 uses `increase()`, so the slice sizes are total counts over the time selector, not per-second rates. Workload-shape signal. Heavy `set_variable` / `reset_variable` / `fetch` traffic is normal \u2014 that's how PostgreSQL clients manage session state. Heavy `insert` / `update` / `delete` on a service you think of as read-mostly is worth investigating. Idle statement types are filtered out (`> 0`). Environment-scoped.", "links": [], "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "\nsum by (statement_type) (\n increase(mz_query_total{$environmentFilter}[$__range])\n) > 0\n", "instant": true, "range": false, "legendFormat": "{{statement_type}}", "query": "\nsum by (statement_type) (\n increase(mz_query_total{$environmentFilter}[$__range])\n) > 0\n", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-0", "hidden": false } } ], "transformations": [], "queryOptions": {} } }, "vizConfig": { "kind": "VizConfig", "group": "piechart", "version": "", "spec": { "options": { "pieType": "donut", "tooltip": { "mode": "single", "sort": "asc" }, "reduceOptions": { "calcs": [] }, "legend": { "values": [ "value" ], "displayMode": "table", "placement": "right", "showLegend": true, "calcs": [], "asTable": true, "isVisible": true }, "orientation": "auto", "displayLabels": [ "name", "value" ] }, "fieldConfig": { "defaults": { "noValue": "No matches for the current filters" }, "overrides": [] } } } } }, "queries-rate-by-statement": { "kind": "Panel", "spec": { "id": 1025, "title": "Query Rate (by statement_type / session_type)", "description": "**Queries per second broken down by statement type AND session type, fully time-resolved.** Pairs with the _Query Distribution_ donut (which shows the time-range total): this panel shows *how those slices move over time*. Watch for sudden spikes in `select / user` \u2014 pair with _Peek Latency (p99)_ to see if the system kept up. Idle (statement, session) tuples are filtered out. Environment-scoped.", "links": [], "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "\nsum by (statement_type, session_type) (\n rate(mz_query_total{$environmentFilter}[$__rate_interval])\n) > 0\n", "legendFormat": "{{statement_type}} / {{session_type}}", "query": "\nsum by (statement_type, session_type) (\n rate(mz_query_total{$environmentFilter}[$__rate_interval])\n) > 0\n", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-0", "hidden": false } } ], "transformations": [], "queryOptions": {} } }, "vizConfig": { "kind": "VizConfig", "group": "timeseries", "version": "", "spec": { "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": [ "max", "mean", "lastNotNull" ] }, "tooltip": { "mode": "single", "sort": "asc" } }, "fieldConfig": { "defaults": { "unit": "cps", "min": 0 }, "overrides": [] } } } } }, "queries-peek-latency-p50": { "kind": "Panel", "spec": { "id": 1026, "title": "Peek Latency (p50)", "description": "**Median read-query latency** \u2014 the typical time it takes to look up the current state of an arrangement (the operation behind every `SELECT \u2026 FROM ` against an index). p50 is your \"what does a normal query feel like\" number. Nominal: typically a few milliseconds on a healthy cluster. Sustained multi-second p50 means the cluster is overwhelmed. One line per cluster \u2014 narrow the cluster selector to focus. Log Y-axis. See also: _Dataflow Elapsed Rate_ and _Arrangement Maintenance Rate_ on the _Compute Objects_ tab.", "links": [], "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "(\n\nhistogram_quantile(0.5,\n sum by (le, instance_id) (\n rate(\n mz_compute_peek_duration_seconds_bucket{$environmentFilter, instance_id=~\"$mzClusterList\"}[$__rate_interval]\n )\n )\n)\n\n)\n* on (instance_id) group_left(cluster_name)\nlabel_replace(label_replace(mz_cluster_info{$environmentFilter}, \"instance_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")", "legendFormat": "{{cluster_name}}", "query": "(\n\nhistogram_quantile(0.5,\n sum by (le, instance_id) (\n rate(\n mz_compute_peek_duration_seconds_bucket{$environmentFilter, instance_id=~\"$mzClusterList\"}[$__rate_interval]\n )\n )\n)\n\n)\n* on (instance_id) group_left(cluster_name)\nlabel_replace(label_replace(mz_cluster_info{$environmentFilter}, \"instance_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-0", "hidden": false } } ], "transformations": [], "queryOptions": {} } }, "vizConfig": { "kind": "VizConfig", "group": "timeseries", "version": "", "spec": { "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": [ "max", "mean", "lastNotNull" ] }, "tooltip": { "mode": "single", "sort": "asc" } }, "fieldConfig": { "defaults": { "unit": "s", "custom": { "scaleDistribution": { "type": "log", "log": 10 } } }, "overrides": [] } } } } }, "queries-peek-latency-p90": { "kind": "Panel", "spec": { "id": 1027, "title": "Peek Latency (p90)", "description": "**90th-percentile read-query latency** \u2014 \"how slow do my slowest 10% of queries feel?\" Catches contention bursts and rarely-hit cold paths that p50 hides. Nominal: usually a small multiple of p50 (2-5x). If p90 is 10-100x p50, your latency distribution is bimodal \u2014 typically cold-cache effects on infrequently-queried indexes or contention. Same per-cluster split as p50.", "links": [], "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "(\n\nhistogram_quantile(0.9,\n sum by (le, instance_id) (\n rate(\n mz_compute_peek_duration_seconds_bucket{$environmentFilter, instance_id=~\"$mzClusterList\"}[$__rate_interval]\n )\n )\n)\n\n)\n* on (instance_id) group_left(cluster_name)\nlabel_replace(label_replace(mz_cluster_info{$environmentFilter}, \"instance_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")", "legendFormat": "{{cluster_name}}", "query": "(\n\nhistogram_quantile(0.9,\n sum by (le, instance_id) (\n rate(\n mz_compute_peek_duration_seconds_bucket{$environmentFilter, instance_id=~\"$mzClusterList\"}[$__rate_interval]\n )\n )\n)\n\n)\n* on (instance_id) group_left(cluster_name)\nlabel_replace(label_replace(mz_cluster_info{$environmentFilter}, \"instance_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-0", "hidden": false } } ], "transformations": [], "queryOptions": {} } }, "vizConfig": { "kind": "VizConfig", "group": "timeseries", "version": "", "spec": { "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": [ "max", "mean", "lastNotNull" ] }, "tooltip": { "mode": "single", "sort": "asc" } }, "fieldConfig": { "defaults": { "unit": "s", "custom": { "scaleDistribution": { "type": "log", "log": 10 } } }, "overrides": [] } } } } }, "queries-peek-latency-p99": { "kind": "Panel", "spec": { "id": 1028, "title": "Peek Latency (p99)", "description": "**Tail read-query latency (99th percentile)** \u2014 the slowest 1% of queries, the ones users complain about. Nominal: a small multiple of p50 (typically 2-10x), with occasional spikes during query plan recompilation or hydration. Sustained p99 in the seconds range \u2014 especially when *not* paired with elevated p50/p90 \u2014 points at a single bad query or a tail-latency-sensitive use case worth investigating directly. Pair with _Query Rate_ above to confirm the latency is happening on actual traffic, not just idle scrapes.", "links": [], "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "(\n\nhistogram_quantile(0.99,\n sum by (le, instance_id) (\n rate(\n mz_compute_peek_duration_seconds_bucket{$environmentFilter, instance_id=~\"$mzClusterList\"}[$__rate_interval]\n )\n )\n)\n\n)\n* on (instance_id) group_left(cluster_name)\nlabel_replace(label_replace(mz_cluster_info{$environmentFilter}, \"instance_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")", "legendFormat": "{{cluster_name}}", "query": "(\n\nhistogram_quantile(0.99,\n sum by (le, instance_id) (\n rate(\n mz_compute_peek_duration_seconds_bucket{$environmentFilter, instance_id=~\"$mzClusterList\"}[$__rate_interval]\n )\n )\n)\n\n)\n* on (instance_id) group_left(cluster_name)\nlabel_replace(label_replace(mz_cluster_info{$environmentFilter}, \"instance_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-0", "hidden": false } } ], "transformations": [], "queryOptions": {} } }, "vizConfig": { "kind": "VizConfig", "group": "timeseries", "version": "", "spec": { "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": [ "max", "mean", "lastNotNull" ] }, "tooltip": { "mode": "single", "sort": "asc" } }, "fieldConfig": { "defaults": { "unit": "s", "custom": { "scaleDistribution": { "type": "log", "log": 10 } } }, "overrides": [] } } } } }, "adapter-commands-by-app": { "kind": "Panel", "spec": { "id": 1029, "title": "Adapter Commands by Application", "description": "**Adapter command totals per `application_name` over the dashboard time range, split into Success and Errors columns.** Rows sorted by Errors (descending) so anything bad floats to the top. The Errors column is threshold-colored \u2014 non-zero jumps out visually. Most clients set `application_name` via the PostgreSQL connection string; clients that don't are bucketed as `unrecognized` or `unspecified` (normal). Sustained non-zero Errors on a real application means that app is consistently failing \u2014 investigate by correlating with that application's own logs, and inspect recent failures via Materialize's `mz_internal` activity-log views.", "links": [], "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "\nsum by (application_name, status) (\n increase(mz_adapter_commands{$environmentFilter}[$__range])\n)\n", "instant": true, "range": false, "query": "\nsum by (application_name, status) (\n increase(mz_adapter_commands{$environmentFilter}[$__range])\n)\n", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-0", "hidden": false } } ], "transformations": [ { "kind": "Transformation", "spec": { "id": "labelsToFields", "options": { "keepLabels": [ "application_name", "status" ] } }, "group": "labelsToFields" }, { "kind": "Transformation", "spec": { "id": "merge", "options": {} }, "group": "merge" }, { "kind": "Transformation", "spec": { "id": "groupingToMatrix", "options": { "rowField": "application_name", "columnField": "status", "valueField": "Value", "emptyValue": "zero" } }, "group": "groupingToMatrix" }, { "kind": "Transformation", "spec": { "id": "organize", "options": { "renameByName": { "application_name\\status": "Application", "success": "Success", "error": "Errors" }, "indexByName": { "application_name\\status": 0, "success": 1, "error": 2 } } }, "group": "organize" }, { "kind": "Transformation", "spec": { "id": "sortBy", "options": { "fields": {}, "sort": [ { "field": "Errors", "desc": true } ] } }, "group": "sortBy" } ], "queryOptions": {} } }, "vizConfig": { "kind": "VizConfig", "group": "table", "version": "", "spec": { "options": { "frameIndex": 0, "showHeader": true, "showTypeIcons": false, "footer": { "show": false, "reducer": [], "countRows": false }, "cellHeight": "sm" }, "fieldConfig": { "defaults": { "unit": "short", "noValue": "No matches for the current filters", "custom": { "align": "auto", "inspect": false, "filterable": true } }, "overrides": [ { "matcher": { "id": "byName", "options": "Errors" }, "properties": [ { "id": "thresholds", "value": { "mode": "absolute", "steps": [ { "value": 1.0, "color": "#FEDA8B" }, { "value": 20.8, "color": "#FDB366" }, { "value": 40.6, "color": "#F67E4B" }, { "value": 60.400000000000006, "color": "#DD3D2D" }, { "value": 80.2, "color": "#A50026" } ] } }, { "id": "custom.cellOptions", "value": { "type": "color-background" } } ] } ] } } } } }, "cluster-count": { "kind": "Panel", "spec": { "id": 1030, "title": "Cluster Count", "description": "**Number of clusters in the environment, split into \"Total\" and \"System\".** System clusters are Materialize-managed (e.g., `mz_catalog_server`, `mz_system`, `mz_probe`) and exist in every env; the difference between Total and System is the user clusters you've created. Stable in steady state; expected to step on `CREATE CLUSTER` / `DROP CLUSTER`. Scoped to the selected clusters.", "links": [], "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "\ncount(\n group by (compute_cluster_id) (\n ${sqlMetricPrefix}compute_cluster_status{$environmentFilter, compute_cluster_id=~\"$mzClusterList\"}\n )\n)\n", "legendFormat": "Total Clusters", "query": "\ncount(\n group by (compute_cluster_id) (\n ${sqlMetricPrefix}compute_cluster_status{$environmentFilter, compute_cluster_id=~\"$mzClusterList\"}\n )\n)\n", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-0", "hidden": false } }, { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "\ncount(\n group by (compute_cluster_id) (\n ${sqlMetricPrefix}compute_cluster_status{$environmentFilter, compute_cluster_id=~\"$mzClusterList\", compute_cluster_id=~\"^s.*\"}\n )\n)\n", "legendFormat": "System Clusters", "query": "\ncount(\n group by (compute_cluster_id) (\n ${sqlMetricPrefix}compute_cluster_status{$environmentFilter, compute_cluster_id=~\"$mzClusterList\", compute_cluster_id=~\"^s.*\"}\n )\n)\n", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-1", "hidden": false } } ], "transformations": [], "queryOptions": {} } }, "vizConfig": { "kind": "VizConfig", "group": "stat", "version": "", "spec": { "options": { "graphMode": "area", "colorMode": "none", "justifyMode": "auto", "textMode": "value_and_name", "wideLayout": true, "showPercentChange": false, "reduceOptions": { "calcs": [] }, "percentChangeColorMode": "standard", "orientation": "auto" }, "fieldConfig": { "defaults": { "min": 0, "color": { "mode": "shades", "fixedColor": "#009988" }, "noValue": "No matches for the current filters" }, "overrides": [] } } } } }, "replica-count": { "kind": "Panel", "spec": { "id": 1031, "title": "Replica Count", "description": "**Number of replicas across the selected clusters, with \"Additional Replicas\" calling out those beyond the first.** Every cluster needs at least one replica to run; \"Additional\" counts the redundancy on top of that \u2014 non-zero means at least one cluster has been configured for higher availability or extra capacity. Expected to step on `CREATE CLUSTER REPLICA` / `DROP CLUSTER REPLICA`. Scoped to the selected clusters.", "links": [], "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "\ncount(\n group by (compute_cluster_id, compute_replica_id) (\n ${sqlMetricPrefix}compute_cluster_status{$environmentFilter, compute_cluster_id=~\"$mzClusterList\", compute_replica_id=~\"$mzReplicaList\"}\n )\n)\n", "legendFormat": "Total Replicas", "query": "\ncount(\n group by (compute_cluster_id, compute_replica_id) (\n ${sqlMetricPrefix}compute_cluster_status{$environmentFilter, compute_cluster_id=~\"$mzClusterList\", compute_replica_id=~\"$mzReplicaList\"}\n )\n)\n", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-0", "hidden": false } }, { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "\ncount(\n group by (compute_cluster_id, compute_replica_id) (\n ${sqlMetricPrefix}compute_cluster_status{$environmentFilter, compute_cluster_id=~\"$mzClusterList\", compute_replica_id=~\"$mzReplicaList\", compute_replica_name!=\"r1\"}\n )\n) or vector(0)\n", "legendFormat": "Additional Replicas", "query": "\ncount(\n group by (compute_cluster_id, compute_replica_id) (\n ${sqlMetricPrefix}compute_cluster_status{$environmentFilter, compute_cluster_id=~\"$mzClusterList\", compute_replica_id=~\"$mzReplicaList\", compute_replica_name!=\"r1\"}\n )\n) or vector(0)\n", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-1", "hidden": false } } ], "transformations": [], "queryOptions": {} } }, "vizConfig": { "kind": "VizConfig", "group": "stat", "version": "", "spec": { "options": { "graphMode": "area", "colorMode": "none", "justifyMode": "auto", "textMode": "value_and_name", "wideLayout": true, "showPercentChange": false, "reduceOptions": { "calcs": [] }, "percentChangeColorMode": "standard", "orientation": "auto" }, "fieldConfig": { "defaults": { "min": 0, "color": { "mode": "shades", "fixedColor": "#009988" }, "noValue": "No matches for the current filters" }, "overrides": [] } } } } }, "replica-sizes": { "kind": "Panel", "spec": { "id": 1032, "title": "Replica Sizes", "description": "**Replicas grouped by their configured size.** Most workloads cluster around a small number of sizes; a long tail of one-off sizes usually means experimentation or migration in progress. The total here matches the Replica Count panel. Scoped to the selected clusters.", "links": [], "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "\ncount by (size) (\n ${sqlMetricPrefix}compute_cluster_status{$environmentFilter, compute_cluster_id=~\"$mzClusterList\", compute_replica_id=~\"$mzReplicaList\"}\n)\n", "instant": true, "range": false, "legendFormat": "{{size}}", "query": "\ncount by (size) (\n ${sqlMetricPrefix}compute_cluster_status{$environmentFilter, compute_cluster_id=~\"$mzClusterList\", compute_replica_id=~\"$mzReplicaList\"}\n)\n", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-0", "hidden": false } } ], "transformations": [], "queryOptions": {} } }, "vizConfig": { "kind": "VizConfig", "group": "piechart", "version": "", "spec": { "options": { "pieType": "pie", "tooltip": { "mode": "single", "sort": "asc" }, "reduceOptions": { "calcs": [] }, "legend": { "values": [ "value" ], "displayMode": "table", "placement": "right", "showLegend": true, "calcs": [], "asTable": true, "isVisible": true }, "orientation": "auto", "displayLabels": [ "name", "value" ] }, "fieldConfig": { "defaults": { "color": { "mode": "shades", "fixedColor": "#009988" }, "noValue": "No matches for the current filters" }, "overrides": [] } } } } }, "cluster-table": { "kind": "Panel", "spec": { "id": 1033, "title": "Cluster Information", "description": "**A row per (cluster, replica) tuple, with cluster_id / cluster_name / replica metadata and size / AZ / region info.** Operator's \"what does my fleet look like\" reference. The column-header filters let you narrow without changing the dashboard's cluster/replica selectors. Useful for copying a `cluster_id` or `replica_id` into the dashboard selectors to scope the rest of the dashboard.", "links": [], "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "\n${sqlMetricPrefix}compute_cluster_status{$environmentFilter, compute_cluster_id=~\"$mzClusterList\", compute_replica_id=~\"$mzReplicaList\"}\n", "instant": true, "range": false, "query": "\n${sqlMetricPrefix}compute_cluster_status{$environmentFilter, compute_cluster_id=~\"$mzClusterList\", compute_replica_id=~\"$mzReplicaList\"}\n", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-0", "hidden": false } } ], "transformations": [ { "kind": "Transformation", "spec": { "id": "labelsToFields", "options": { "keepLabels": [ "compute_cluster_name", "compute_replica_name", "compute_cluster_id", "compute_replica_id", "mz_version", "size", "materialize_cloud_availability_zone", "topology_kubernetes_io_region", "topology_kubernetes_io_zone" ] } }, "group": "labelsToFields" }, { "kind": "Transformation", "spec": { "id": "merge", "options": {} }, "group": "merge" }, { "kind": "Transformation", "spec": { "id": "organize", "options": { "excludeByName": { "Time": true, "mz_compute_cluster_status": true, "v2_mz_compute_cluster_status": true }, "indexByName": { "compute_cluster_name": 0, "compute_replica_name": 1, "compute_cluster_id": 2, "compute_replica_id": 3, "mz_version": 4, "size": 5, "materialize_cloud_availability_zone": 6, "topology_kubernetes_io_region": 7, "topology_kubernetes_io_zone": 8 } } }, "group": "organize" }, { "kind": "Transformation", "spec": { "id": "sortBy", "options": { "fields": {}, "sort": [ { "field": "compute_cluster_name" } ] } }, "group": "sortBy" } ], "queryOptions": {} } }, "vizConfig": { "kind": "VizConfig", "group": "table", "version": "", "spec": { "options": { "frameIndex": 0, "showHeader": true, "showTypeIcons": false, "footer": { "show": false, "reducer": [], "countRows": false }, "cellHeight": "sm" }, "fieldConfig": { "defaults": { "noValue": "No matches for the current filters", "custom": { "align": "auto", "inspect": false, "filterable": true } }, "overrides": [] } } } } }, "active-mzd-views": { "kind": "Panel", "spec": { "id": 1034, "title": "Active Materialized Views", "description": "**Number of materialized views actively maintained by Materialize.** Each materialized view is a persistent compute object that incrementally keeps a query's result up to date \u2014 so this count is roughly proportional to how much work the cluster is doing. Nominal: stable in steady state; expected to step on `CREATE MATERIALIZED VIEW` / `DROP MATERIALIZED VIEW`. Sustained drift suggests automated DDL activity. Environment-scoped \u2014 not affected by the cluster/replica filters.", "links": [], "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "\nmax(${sqlMetricPrefix}mzd_views_count{$environmentFilter})\n", "legendFormat": "materialized-views", "query": "\nmax(${sqlMetricPrefix}mzd_views_count{$environmentFilter})\n", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-0", "hidden": false } } ], "transformations": [], "queryOptions": {} } }, "vizConfig": { "kind": "VizConfig", "group": "stat", "version": "", "spec": { "options": { "graphMode": "area", "colorMode": "none", "justifyMode": "auto", "textMode": "value", "wideLayout": true, "showPercentChange": false, "reduceOptions": { "calcs": [] }, "percentChangeColorMode": "standard", "orientation": "auto" }, "fieldConfig": { "defaults": { "min": 0, "color": { "mode": "shades", "fixedColor": "#EE7733" }, "noValue": "No matches for the current filters" }, "overrides": [] } } } } }, "active-indexes": { "kind": "Panel", "spec": { "id": 1035, "title": "Active Indexes", "description": "**Number of indexes in the catalog.** An index is an in-memory arrangement that makes `SELECT`s against its underlying relation effectively instant \u2014 at the cost of memory. Rapid growth here typically pairs with growing memory usage on the cluster's pods (see _Kubernetes Workloads -> Pod Memory Usage_). For the table/view/materialized-view split see _Index Types_. Environment-scoped \u2014 not affected by the cluster/replica filters.", "links": [], "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "\nmax(sum by (instance) (${sqlMetricPrefix}indexes_count{$environmentFilter})) or vector(0)\n", "legendFormat": "indexes", "query": "\nmax(sum by (instance) (${sqlMetricPrefix}indexes_count{$environmentFilter})) or vector(0)\n", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-0", "hidden": false } } ], "transformations": [], "queryOptions": {} } }, "vizConfig": { "kind": "VizConfig", "group": "stat", "version": "", "spec": { "options": { "graphMode": "area", "colorMode": "none", "justifyMode": "auto", "textMode": "value", "wideLayout": true, "showPercentChange": false, "reduceOptions": { "calcs": [] }, "percentChangeColorMode": "standard", "orientation": "auto" }, "fieldConfig": { "defaults": { "min": 0, "color": { "mode": "shades", "fixedColor": "#EE7733" }, "noValue": "No matches for the current filters" }, "overrides": [] } } } } }, "active-views": { "kind": "Panel", "spec": { "id": 1036, "title": "Active Views", "description": "**Views (non-materialized) in the catalog.** Views are query templates evaluated on demand \u2014 they don't consume compute or memory until something queries them. Mostly a catalog-shape signal; for read-side activity see _Connections / Activity -> Query Rate_. Environment-scoped \u2014 not affected by the cluster/replica filters.", "links": [], "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "\nmax(${sqlMetricPrefix}views_count{$environmentFilter})\n", "legendFormat": "views", "query": "\nmax(${sqlMetricPrefix}views_count{$environmentFilter})\n", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-0", "hidden": false } } ], "transformations": [], "queryOptions": {} } }, "vizConfig": { "kind": "VizConfig", "group": "stat", "version": "", "spec": { "options": { "graphMode": "area", "colorMode": "none", "justifyMode": "auto", "textMode": "value", "wideLayout": true, "showPercentChange": false, "reduceOptions": { "calcs": [] }, "percentChangeColorMode": "standard", "orientation": "auto" }, "fieldConfig": { "defaults": { "min": 0, "color": { "mode": "shades", "fixedColor": "#EE7733" }, "noValue": "No matches for the current filters" }, "overrides": [] } } } } }, "active-subscribes": { "kind": "Panel", "spec": { "id": 1037, "title": "Active Subscribes", "description": "**Live SUBSCRIBE sessions \u2014 long-running queries that push updates to a client as the underlying data changes.** `system` subscribes are Materialize's internal probing / health checks (always a few); `user` subscribes come from client connections. A persistently high `user` count is often a leaked-connection signal. Environment-scoped \u2014 not affected by the cluster/replica filters.", "links": [], "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "\nsum by (session_type) (\n mz_active_subscribes{$environmentFilter}\n)\n", "instant": true, "range": false, "legendFormat": "{{session_type}}", "query": "\nsum by (session_type) (\n mz_active_subscribes{$environmentFilter}\n)\n", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-0", "hidden": false } } ], "transformations": [], "queryOptions": {} } }, "vizConfig": { "kind": "VizConfig", "group": "piechart", "version": "", "spec": { "options": { "pieType": "donut", "tooltip": { "mode": "single", "sort": "asc" }, "reduceOptions": { "calcs": [] }, "legend": { "values": [ "value" ], "displayMode": "table", "placement": "right", "showLegend": true, "calcs": [], "asTable": true, "isVisible": true }, "orientation": "auto", "displayLabels": [ "name", "value" ] }, "fieldConfig": { "defaults": { "color": { "mode": "shades", "fixedColor": "#EE7733" }, "noValue": "No matches for the current filters" }, "overrides": [] } } } } }, "index-types": { "kind": "Panel", "spec": { "id": 1038, "title": "Index Types", "description": "**Indexes by the underlying relation type** (view / table / materialized-view). Most workloads heavily favor indexes on views \u2014 that's the canonical 'maintain a query's result' pattern. A high ratio of indexes on tables is unusual and worth understanding (often a misunderstanding of materialization). Environment-scoped \u2014 not affected by the cluster/replica filters.", "links": [], "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "\nsum by (relation_type) (\n ${sqlMetricPrefix}indexes_count{$environmentFilter}\n)\n", "instant": true, "range": false, "legendFormat": "{{relation_type}}", "query": "\nsum by (relation_type) (\n ${sqlMetricPrefix}indexes_count{$environmentFilter}\n)\n", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-0", "hidden": false } } ], "transformations": [], "queryOptions": {} } }, "vizConfig": { "kind": "VizConfig", "group": "piechart", "version": "", "spec": { "options": { "pieType": "donut", "tooltip": { "mode": "single", "sort": "asc" }, "reduceOptions": { "calcs": [] }, "legend": { "values": [ "value" ], "displayMode": "table", "placement": "right", "showLegend": true, "calcs": [], "asTable": true, "isVisible": true }, "orientation": "auto", "displayLabels": [ "name", "value" ] }, "fieldConfig": { "defaults": { "color": { "mode": "shades", "fixedColor": "#EE7733" }, "noValue": "No matches for the current filters" }, "overrides": [] } } } } }, "freshness-lag-by-cluster": { "kind": "Panel", "spec": { "id": 1039, "title": "Frontier Lag by Cluster", "description": "**How far behind real time each cluster's most-lagged collection is** \u2014 the worst-case freshness across all indexes, materialized views, and sources on the cluster. Nominal: low and flat (sub-second to a few seconds) for a keeping-up cluster. A sustained climb means a collection can't keep pace with its input rate (under-provisioned cluster or an expensive dataflow); a sharp spike that decays back down is normal right after a restart or DDL while dataflows re-hydrate. The catalog cluster (`s2`) sits a couple seconds back as its baseline. For which collection is responsible see _Most-Lagged Collections_; for hydration specifically see the _Hydration_ row.", "links": [], "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "(\n\nmax by (instance_id) (\n mz_dataflow_wallclock_lag_seconds{\n $environmentFilter,\n instance_id=~\"$mzClusterList\",\n instance_id!=\"\",\n quantile=\"1\"\n } < 1e9\n)\n\n)\n* on (instance_id) group_left(cluster_name)\nlabel_replace(label_replace(mz_cluster_info{$environmentFilter}, \"instance_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")", "legendFormat": "{{cluster_name}}", "query": "(\n\nmax by (instance_id) (\n mz_dataflow_wallclock_lag_seconds{\n $environmentFilter,\n instance_id=~\"$mzClusterList\",\n instance_id!=\"\",\n quantile=\"1\"\n } < 1e9\n)\n\n)\n* on (instance_id) group_left(cluster_name)\nlabel_replace(label_replace(mz_cluster_info{$environmentFilter}, \"instance_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-0", "hidden": false } } ], "transformations": [], "queryOptions": {} } }, "vizConfig": { "kind": "VizConfig", "group": "timeseries", "version": "", "spec": { "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": [ "max", "mean", "lastNotNull" ] }, "tooltip": { "mode": "single", "sort": "asc" } }, "fieldConfig": { "defaults": { "unit": "s", "min": 0, "noValue": "No matches for the current filters", "custom": { "scaleDistribution": { "type": "log", "log": 10 } } }, "overrides": [] } } } } }, "freshness-top-collections": { "kind": "Panel", "spec": { "id": 1040, "title": "Most-Lagged Collections", "description": "**The 15 collections whose output frontier is furthest behind real time.** This is the per-collection breakdown behind _Frontier Lag by Cluster_, labeled by object name (resolved via `mz_object_info`). Collections with no frontier yet (idle or still hydrating) report a sentinel and are filtered out here \u2014 check hydration state directly with `SELECT * FROM mz_internal.mz_hydration_statuses WHERE NOT hydrated`. `s2` (catalog) collections commonly appear with a small baseline lag; that's expected.", "links": [], "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "topk(15,\n(\n(\n\nmax by (instance_id, collection_id) (\n mz_dataflow_wallclock_lag_seconds{\n $environmentFilter,\n instance_id=~\"$mzClusterList\",\n instance_id!=\"\",\n replica_id=~\"$mzReplicaList\",\n quantile=\"1\"\n } < 1e9\n)\n\n)\n* on (collection_id) group_left(name)\nlabel_replace(mz_object_info{$environmentFilter}, \"collection_id\", \"$1\", \"global_id\", \"(.*)\")\n)\n* on (instance_id) group_left(cluster_name)\nlabel_replace(label_replace(mz_cluster_info{$environmentFilter}, \"instance_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)", "instant": true, "range": false, "legendFormat": "{{cluster_name}} / {{name}}", "query": "topk(15,\n(\n(\n\nmax by (instance_id, collection_id) (\n mz_dataflow_wallclock_lag_seconds{\n $environmentFilter,\n instance_id=~\"$mzClusterList\",\n instance_id!=\"\",\n replica_id=~\"$mzReplicaList\",\n quantile=\"1\"\n } < 1e9\n)\n\n)\n* on (collection_id) group_left(name)\nlabel_replace(mz_object_info{$environmentFilter}, \"collection_id\", \"$1\", \"global_id\", \"(.*)\")\n)\n* on (instance_id) group_left(cluster_name)\nlabel_replace(label_replace(mz_cluster_info{$environmentFilter}, \"instance_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-0", "hidden": false } } ], "transformations": [], "queryOptions": {} } }, "vizConfig": { "kind": "VizConfig", "group": "barchart", "version": "", "spec": { "options": { "orientation": "horizontal", "xTickLabelRotation": 0, "xTickLabelMaxLength": 0, "stacking": "none", "showValue": "auto", "barWidth": 0.8, "groupWidth": 0.95, "legend": { "displayMode": "list", "placement": "bottom", "showLegend": false, "calcs": [] }, "tooltip": { "mode": "single", "sort": "asc" }, "fullHighlight": false, "barRadius": 0, "xTickLabelSpacing": 100 }, "fieldConfig": { "defaults": { "unit": "s", "color": { "mode": "shades", "fixedColor": "#EE7733" }, "noValue": "No matches for the current filters", "custom": { "lineWidth": 1, "fillOpacity": 80, "gradientMode": "none", "scaleDistribution": { "type": "log", "log": 10 } } }, "overrides": [] } } } } }, "hydration-unhydrated-count": { "kind": "Panel", "spec": { "id": 1041, "title": "Currently Hydrating", "description": "**Collections still (re)building their in-memory state \u2014 a live hydration-queue proxy.** Hydration rebuilds a dataflow's state from persisted storage after a cluster/replica restart, replica creation, or some DDL; until it finishes, the collection has no output frontier, which this counts (via the `mz_dataflow_wallclock_lag_seconds` sentinel). **Nominal: 0, with brief spikes right after a replica restart that drain back to 0 as dataflows catch up \u2014 that's healthy.** A count that stays elevated means something can't finish hydrating (e.g. a source whose `CREATE` didn't complete, or a wedged dataflow). Confirm what's stuck with `SELECT * FROM mz_internal.mz_hydration_statuses WHERE NOT hydrated`; watch _Compute -> Freshness_ for the lag those collections accrue.", "links": [], "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "\ncount(\n max by (instance_id, collection_id) (\n mz_dataflow_wallclock_lag_seconds{\n $environmentFilter,\n instance_id=~\"$mzClusterList\",\n instance_id!=\"\",\n quantile=\"1\"\n } > 1e15\n )\n) or vector(0)\n", "legendFormat": "hydrating", "query": "\ncount(\n max by (instance_id, collection_id) (\n mz_dataflow_wallclock_lag_seconds{\n $environmentFilter,\n instance_id=~\"$mzClusterList\",\n instance_id!=\"\",\n quantile=\"1\"\n } > 1e15\n )\n) or vector(0)\n", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-0", "hidden": false } } ], "transformations": [], "queryOptions": {} } }, "vizConfig": { "kind": "VizConfig", "group": "stat", "version": "", "spec": { "options": { "graphMode": "area", "colorMode": "none", "justifyMode": "auto", "textMode": "value", "wideLayout": true, "showPercentChange": false, "reduceOptions": { "calcs": [] }, "percentChangeColorMode": "standard", "orientation": "auto" }, "fieldConfig": { "defaults": { "min": 0, "color": { "mode": "shades", "fixedColor": "#EE7733" }, "noValue": "No matches for the current filters" }, "overrides": [] } } } } }, "hydration-queue-size": { "kind": "Panel", "spec": { "id": 1042, "title": "Hydration Queue Size", "description": "**Collections waiting in the compute controller's hydration queue, per replica.** environmentd schedules hydration work in batches; backlog here means it's queueing faster than replicas can complete it \u2014 typical during mass cluster restarts, atypical otherwise. Nominal: 0. Sustained non-zero means the replica is undersized or one slow-hydrating collection is blocking the line.", "links": [], "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "(\n\nsum by (instance_id, replica_id) (\n mz_compute_controller_hydration_queue_size{\n $environmentFilter,\n instance_id=~\"$mzClusterList\",\n replica_id=~\"$mzReplicaList\"\n }\n) > 0\n\n)\n* on (instance_id) group_left(cluster_name)\nlabel_replace(label_replace(mz_cluster_info{$environmentFilter}, \"instance_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")", "legendFormat": "{{cluster_name}} / r{{replica_id}}", "query": "(\n\nsum by (instance_id, replica_id) (\n mz_compute_controller_hydration_queue_size{\n $environmentFilter,\n instance_id=~\"$mzClusterList\",\n replica_id=~\"$mzReplicaList\"\n }\n) > 0\n\n)\n* on (instance_id) group_left(cluster_name)\nlabel_replace(label_replace(mz_cluster_info{$environmentFilter}, \"instance_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-0", "hidden": false } } ], "transformations": [], "queryOptions": {} } }, "vizConfig": { "kind": "VizConfig", "group": "timeseries", "version": "", "spec": { "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": [ "max", "mean", "lastNotNull" ] }, "tooltip": { "mode": "single", "sort": "asc" } }, "fieldConfig": { "defaults": { "unit": "short", "min": 0, "noValue": "Hydration Queue is empty" }, "overrides": [] } } } } }, "hydration-slowest-collections": { "kind": "Panel", "spec": { "id": 1043, "title": "Slowest Hydrating Collections", "description": "**The 15 collections that took the longest to finish hydrating** in the current time range. Hydration time scales roughly with the size of the maintained state, so large materialized views and indexes naturally lead the list. **Note (self-managed): per-collection hydration time is not exposed as a metric here** (the cloud-only `v2_mz_compute_hydration_time_seconds` has no equivalent), so this is blank. Get it from SQL: `SELECT object_id, time_ns/1e9 AS seconds FROM mz_internal.mz_compute_hydration_times ORDER BY time_ns DESC`. The live metric-side proxy for what's behind is _Freshness -> Most-Lagged Collections_.", "links": [], "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "topk(15,\n(\n(\n\n${sqlMetricPrefix}compute_hydration_time_seconds{\n $environmentFilter,\n instance_id=~\"$mzClusterList\",\n replica_id=~\"$mzReplicaList\",\n hydrated=\"1\"\n}\n\n)\n* on (collection_id) group_left(name)\nlabel_replace(mz_object_info{$environmentFilter}, \"collection_id\", \"$1\", \"global_id\", \"(.*)\")\n)\n* on (instance_id) group_left(cluster_name)\nlabel_replace(label_replace(mz_cluster_info{$environmentFilter}, \"instance_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)", "instant": true, "range": false, "legendFormat": "{{cluster_name}} / r{{replica_id}} / {{name}}", "query": "topk(15,\n(\n(\n\n${sqlMetricPrefix}compute_hydration_time_seconds{\n $environmentFilter,\n instance_id=~\"$mzClusterList\",\n replica_id=~\"$mzReplicaList\",\n hydrated=\"1\"\n}\n\n)\n* on (collection_id) group_left(name)\nlabel_replace(mz_object_info{$environmentFilter}, \"collection_id\", \"$1\", \"global_id\", \"(.*)\")\n)\n* on (instance_id) group_left(cluster_name)\nlabel_replace(label_replace(mz_cluster_info{$environmentFilter}, \"instance_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-0", "hidden": false } } ], "transformations": [], "queryOptions": {} } }, "vizConfig": { "kind": "VizConfig", "group": "barchart", "version": "", "spec": { "options": { "orientation": "horizontal", "xTickLabelRotation": 0, "xTickLabelMaxLength": 0, "stacking": "none", "showValue": "auto", "barWidth": 0.8, "groupWidth": 0.95, "legend": { "displayMode": "list", "placement": "bottom", "showLegend": false, "calcs": [] }, "tooltip": { "mode": "single", "sort": "asc" }, "fullHighlight": false, "barRadius": 0, "xTickLabelSpacing": 100 }, "fieldConfig": { "defaults": { "unit": "s", "thresholds": { "mode": "absolute", "steps": [ { "value": 2, "color": "#CEFFFF" }, { "value": 6, "color": "#C6F7D6" }, { "value": 15, "color": "#A2F49B" }, { "value": 37, "color": "#BBE453" }, { "value": 93, "color": "#D5CE04" }, { "value": 231, "color": "#E7B503" }, { "value": 573, "color": "#F19903" }, { "value": 1420, "color": "#F6790B" }, { "value": 3518, "color": "#F94902" }, { "value": 8717, "color": "#E40515" }, { "value": 21599, "color": "#AB0003" } ] }, "color": { "mode": "shades", "fixedColor": "#EE7733" }, "noValue": "No matches for the current filters", "custom": { "lineWidth": 1, "fillOpacity": 80, "gradientMode": "none", "scaleDistribution": { "type": "log", "log": 10 }, "thresholdsStyle": { "mode": "area" } } }, "overrides": [] } } } } }, "dataflow-count": { "kind": "Panel", "spec": { "id": 1044, "title": "Dataflow Count", "description": "**Number of active dataflows running on each replica.** Every index, materialized view, and live `SUBSCRIBE` manifests as one or more dataflows on a replica \u2014 so this count rises with DDL and SUBSCRIBE activity. Nominal: stable for steady workloads. A sharp drop without correlating DDL usually means a replica restart (cross-check _Kubernetes Workloads_); a sharp rise typically means bulk object creation.", "links": [], "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "(\n\nmax by (\n cluster_environmentd_materialize_cloud_cluster_id,\n cluster_environmentd_materialize_cloud_replica_id\n) (\n mz_compute_replica_history_dataflow_count{$environmentFilter, cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}\n)\n\n)\n* on (cluster_environmentd_materialize_cloud_cluster_id) group_left(cluster_name)\nlabel_replace(label_replace(mz_cluster_info{$environmentFilter}, \"cluster_environmentd_materialize_cloud_cluster_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")", "legendFormat": "{{cluster_name}} / {{cluster_environmentd_materialize_cloud_replica_id}}", "query": "(\n\nmax by (\n cluster_environmentd_materialize_cloud_cluster_id,\n cluster_environmentd_materialize_cloud_replica_id\n) (\n mz_compute_replica_history_dataflow_count{$environmentFilter, cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}\n)\n\n)\n* on (cluster_environmentd_materialize_cloud_cluster_id) group_left(cluster_name)\nlabel_replace(label_replace(mz_cluster_info{$environmentFilter}, \"cluster_environmentd_materialize_cloud_cluster_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-0", "hidden": false } } ], "transformations": [], "queryOptions": {} } }, "vizConfig": { "kind": "VizConfig", "group": "timeseries", "version": "", "spec": { "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": [ "max", "mean", "lastNotNull" ] }, "tooltip": { "mode": "single", "sort": "asc" } }, "fieldConfig": { "defaults": { "unit": "short", "min": 0 }, "overrides": [] } } } } }, "dataflow-count-by-worker": { "kind": "Panel", "spec": { "id": 1045, "title": "Dataflow Count (per worker)", "description": "**Per-worker view of the dataflow count.** Workers within the same replica run in lockstep and should always see the same dataflows \u2014 so the worker series for a given replica should overlap exactly. Visible divergence between worker series within a single (cluster, replica) tuple is a bug-class signal and worth filing.", "links": [], "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "(\n\nmax by (\n cluster_environmentd_materialize_cloud_cluster_id,\n cluster_environmentd_materialize_cloud_replica_id,\n worker_id\n) (\n mz_compute_replica_history_dataflow_count{$environmentFilter, cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}\n)\n\n)\n* on (cluster_environmentd_materialize_cloud_cluster_id) group_left(cluster_name)\nlabel_replace(label_replace(mz_cluster_info{$environmentFilter}, \"cluster_environmentd_materialize_cloud_cluster_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")", "legendFormat": "{{cluster_name}} / {{cluster_environmentd_materialize_cloud_replica_id}} / w{{worker_id}}", "query": "(\n\nmax by (\n cluster_environmentd_materialize_cloud_cluster_id,\n cluster_environmentd_materialize_cloud_replica_id,\n worker_id\n) (\n mz_compute_replica_history_dataflow_count{$environmentFilter, cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}\n)\n\n)\n* on (cluster_environmentd_materialize_cloud_cluster_id) group_left(cluster_name)\nlabel_replace(label_replace(mz_cluster_info{$environmentFilter}, \"cluster_environmentd_materialize_cloud_cluster_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-0", "hidden": false } } ], "transformations": [], "queryOptions": {} } }, "vizConfig": { "kind": "VizConfig", "group": "timeseries", "version": "", "spec": { "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": [ "max", "mean", "lastNotNull" ] }, "tooltip": { "mode": "single", "sort": "asc" } }, "fieldConfig": { "defaults": { "unit": "short", "min": 0 }, "overrides": [] } } } } }, "dataflow-elapsed-rate": { "kind": "Panel", "spec": { "id": 1046, "title": "Dataflow Elapsed Rate", "description": "**CPU-cores busy inside dataflows, per cluster.** Covers all dataflow work \u2014 arrangement maintenance, evaluation, and hydration. Capped by cluster size: a `400cc` cluster can't exceed 400 cores. Nominal: well below cluster size; sustained near-max means the cluster is CPU-bound and a candidate for upsizing. The catalog cluster (`s2`) is typically the busiest by far in any environment. Log Y-axis so idle and busy clusters share the chart. For maintenance-only breakdown see the _Arrangements_ row below.", "links": [], "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "(\n\nsum by (instance_id) (\n max without (job) (\n rate(\n ${sqlMetricPrefix}dataflow_elapsed_seconds_total{\n $environmentFilter,\n instance_id=~\"$mzClusterList\",\n replica_id=~\"$mzReplicaList\"\n }[$__rate_interval]\n )\n )\n)\n\n)\n* on (instance_id) group_left(cluster_name)\nlabel_replace(label_replace(mz_cluster_info{$environmentFilter}, \"instance_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")", "legendFormat": "{{cluster_name}}", "query": "(\n\nsum by (instance_id) (\n max without (job) (\n rate(\n ${sqlMetricPrefix}dataflow_elapsed_seconds_total{\n $environmentFilter,\n instance_id=~\"$mzClusterList\",\n replica_id=~\"$mzReplicaList\"\n }[$__rate_interval]\n )\n )\n)\n\n)\n* on (instance_id) group_left(cluster_name)\nlabel_replace(label_replace(mz_cluster_info{$environmentFilter}, \"instance_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-0", "hidden": false } } ], "transformations": [], "queryOptions": {} } }, "vizConfig": { "kind": "VizConfig", "group": "timeseries", "version": "", "spec": { "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": [ "max", "mean", "lastNotNull" ] }, "tooltip": { "mode": "single", "sort": "asc" } }, "fieldConfig": { "defaults": { "unit": "none", "custom": { "scaleDistribution": { "type": "log", "log": 10 } } }, "overrides": [] } } } } }, "arrangement-rate": { "kind": "Panel", "spec": { "id": 1047, "title": "Arrangement Maintenance Rate", "description": "**CPU-cores spent maintaining arrangements** \u2014 the in-memory indexed snapshots that back every index and materialized view. Summed across workers in each replica, so an N-worker replica can hit N. Nominal: well below worker count. Sustained near-max indicates the replica is bottlenecked on maintenance work \u2014 usually high upstream change rate or many heavy indexes. The catalog cluster's normal baseline is higher than typical user clusters.", "links": [], "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "(\n\nsum by (\n cluster_environmentd_materialize_cloud_cluster_id,\n cluster_environmentd_materialize_cloud_replica_id\n) (\n max without (job) (\n rate(\n mz_arrangement_maintenance_seconds_total{$environmentFilter, cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval]\n )\n )\n)\n\n)\n* on (cluster_environmentd_materialize_cloud_cluster_id) group_left(cluster_name)\nlabel_replace(label_replace(mz_cluster_info{$environmentFilter}, \"cluster_environmentd_materialize_cloud_cluster_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")", "legendFormat": "{{cluster_name}} / {{cluster_environmentd_materialize_cloud_replica_id}}", "query": "(\n\nsum by (\n cluster_environmentd_materialize_cloud_cluster_id,\n cluster_environmentd_materialize_cloud_replica_id\n) (\n max without (job) (\n rate(\n mz_arrangement_maintenance_seconds_total{$environmentFilter, cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval]\n )\n )\n)\n\n)\n* on (cluster_environmentd_materialize_cloud_cluster_id) group_left(cluster_name)\nlabel_replace(label_replace(mz_cluster_info{$environmentFilter}, \"cluster_environmentd_materialize_cloud_cluster_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-0", "hidden": false } } ], "transformations": [], "queryOptions": {} } }, "vizConfig": { "kind": "VizConfig", "group": "timeseries", "version": "", "spec": { "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": [ "max", "mean", "lastNotNull" ] }, "tooltip": { "mode": "single", "sort": "asc" } }, "fieldConfig": { "defaults": { "unit": "none" }, "overrides": [] } } } } }, "arrangement-rate-by-worker": { "kind": "Panel", "spec": { "id": 1048, "title": "Arrangement Maintenance Rate (per worker)", "description": "**Same metric as the aggregate, but split per worker** \u2014 each worker tops out at 1.0. Watch for skew: if one worker series sits near 1.0 while siblings sit near 0, the cluster is bottlenecked on that single worker (a 'hot key' or hot arrangement). Skew is the most common reason an 8-worker cluster behaves like a 1-worker one. If you see it, the next step is usually `EXPLAIN PHYSICAL PLAN` on the offending object to find the heavy operator.", "links": [], "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "(\n\nsum by (\n cluster_environmentd_materialize_cloud_cluster_id,\n cluster_environmentd_materialize_cloud_replica_id,\n worker_id\n) (\n max without (job) (\n rate(\n mz_arrangement_maintenance_seconds_total{$environmentFilter, cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval]\n )\n )\n)\n\n)\n* on (cluster_environmentd_materialize_cloud_cluster_id) group_left(cluster_name)\nlabel_replace(label_replace(mz_cluster_info{$environmentFilter}, \"cluster_environmentd_materialize_cloud_cluster_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")", "legendFormat": "{{cluster_name}} / {{cluster_environmentd_materialize_cloud_replica_id}} / w{{worker_id}}", "query": "(\n\nsum by (\n cluster_environmentd_materialize_cloud_cluster_id,\n cluster_environmentd_materialize_cloud_replica_id,\n worker_id\n) (\n max without (job) (\n rate(\n mz_arrangement_maintenance_seconds_total{$environmentFilter, cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval]\n )\n )\n)\n\n)\n* on (cluster_environmentd_materialize_cloud_cluster_id) group_left(cluster_name)\nlabel_replace(label_replace(mz_cluster_info{$environmentFilter}, \"cluster_environmentd_materialize_cloud_cluster_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-0", "hidden": false } } ], "transformations": [], "queryOptions": {} } }, "vizConfig": { "kind": "VizConfig", "group": "timeseries", "version": "", "spec": { "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": [ "max", "mean", "lastNotNull" ] }, "tooltip": { "mode": "single", "sort": "asc" } }, "fieldConfig": { "defaults": { "unit": "percentunit" }, "overrides": [] } } } } }, "arrangement-records-system": { "kind": "Panel", "spec": { "id": 1049, "title": "System Collections \u2014 Record Counts", "description": "**Row counts of arrangements maintained for Materialize's internal system collections** (`collection_id` starts with `s`). These back catalog tables, internal probes, and other infrastructure \u2014 they're not user data and shouldn't grow with workload. Columns are Min / Max / Last over the selected time range, sorted by Last (largest current arrangements first). Useful for spotting unexpected growth in system collections, which can indicate a Materialize bug.", "links": [], "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "(\n\nmax by (collection_id) (\n ${sqlMetricPrefix}arrangement_record_count{\n $environmentFilter,\n instance_id=~\"$mzClusterList\",\n replica_id=~\"$mzReplicaList\",\n collection_id=~\"s.*\"\n }\n)\n\n)\n* on (collection_id) group_left(name)\nlabel_replace(mz_object_info{$environmentFilter}, \"collection_id\", \"$1\", \"global_id\", \"(.*)\")", "legendFormat": "{{name}}", "query": "(\n\nmax by (collection_id) (\n ${sqlMetricPrefix}arrangement_record_count{\n $environmentFilter,\n instance_id=~\"$mzClusterList\",\n replica_id=~\"$mzReplicaList\",\n collection_id=~\"s.*\"\n }\n)\n\n)\n* on (collection_id) group_left(name)\nlabel_replace(mz_object_info{$environmentFilter}, \"collection_id\", \"$1\", \"global_id\", \"(.*)\")", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-0", "hidden": false } } ], "transformations": [ { "kind": "Transformation", "spec": { "id": "reduce", "options": { "reducers": [ "min", "max", "lastNotNull" ], "mode": "seriesToRows" } }, "group": "reduce" }, { "kind": "Transformation", "spec": { "id": "organize", "options": { "renameByName": { "Field": "Collection" } } }, "group": "organize" }, { "kind": "Transformation", "spec": { "id": "sortBy", "options": { "fields": {}, "sort": [ { "field": "Last *", "desc": true } ] } }, "group": "sortBy" } ], "queryOptions": {} } }, "vizConfig": { "kind": "VizConfig", "group": "table", "version": "", "spec": { "options": { "frameIndex": 0, "showHeader": true, "showTypeIcons": false, "footer": { "show": false, "reducer": [], "countRows": false }, "cellHeight": "sm" }, "fieldConfig": { "defaults": { "unit": "short", "noValue": "No matches for the current filters", "custom": { "align": "auto", "inspect": false, "filterable": true } }, "overrides": [] } } } } }, "arrangement-records-user": { "kind": "Panel", "spec": { "id": 1050, "title": "User Collections \u2014 Record Counts", "description": "**Row counts of arrangements maintained for user-created compute objects** (`collection_id` starts with `u`). This is the row count of every index and materialized view on the selected clusters \u2014 the primary driver of memory consumption. Sudden growth on a specific collection tracks the size of its underlying data. Columns are Min / Max / Last over the time range; sorted by Last desc so the largest current arrangements are at the top. Rows are labeled by object name (resolved via `mz_object_info`).", "links": [], "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "(\n\nmax by (collection_id) (\n ${sqlMetricPrefix}arrangement_record_count{\n $environmentFilter,\n instance_id=~\"$mzClusterList\",\n replica_id=~\"$mzReplicaList\",\n collection_id=~\"u.*\"\n }\n)\n\n)\n* on (collection_id) group_left(name)\nlabel_replace(mz_object_info{$environmentFilter}, \"collection_id\", \"$1\", \"global_id\", \"(.*)\")", "legendFormat": "{{name}}", "query": "(\n\nmax by (collection_id) (\n ${sqlMetricPrefix}arrangement_record_count{\n $environmentFilter,\n instance_id=~\"$mzClusterList\",\n replica_id=~\"$mzReplicaList\",\n collection_id=~\"u.*\"\n }\n)\n\n)\n* on (collection_id) group_left(name)\nlabel_replace(mz_object_info{$environmentFilter}, \"collection_id\", \"$1\", \"global_id\", \"(.*)\")", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-0", "hidden": false } } ], "transformations": [ { "kind": "Transformation", "spec": { "id": "reduce", "options": { "reducers": [ "min", "max", "lastNotNull" ], "mode": "seriesToRows" } }, "group": "reduce" }, { "kind": "Transformation", "spec": { "id": "organize", "options": { "renameByName": { "Field": "Collection" } } }, "group": "organize" }, { "kind": "Transformation", "spec": { "id": "sortBy", "options": { "fields": {}, "sort": [ { "field": "Last *", "desc": true } ] } }, "group": "sortBy" } ], "queryOptions": {} } }, "vizConfig": { "kind": "VizConfig", "group": "table", "version": "", "spec": { "options": { "frameIndex": 0, "showHeader": true, "showTypeIcons": false, "footer": { "show": false, "reducer": [], "countRows": false }, "cellHeight": "sm" }, "fieldConfig": { "defaults": { "unit": "short", "noValue": "No matches for the current filters", "custom": { "align": "auto", "inspect": false, "filterable": true } }, "overrides": [] } } } } }, "arrangement-records-transient": { "kind": "Panel", "spec": { "id": 1051, "title": "Transient / Uncategorized \u2014 Record Counts", "description": "**Row counts of arrangements with `collection_id` starting with `t` (transient) or labeled `none`.** Transient arrangements are short-lived intermediates created during query optimization and dataflow execution; the `none` sentinel is for arrangements whose collection is unidentified. Both are usually small and ephemeral. Sustained non-trivial entries here are worth investigating \u2014 they may indicate stuck or leaked dataflows.", "links": [], "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "\nmax by (collection_id) (\n ${sqlMetricPrefix}arrangement_record_count{\n $environmentFilter,\n instance_id=~\"$mzClusterList\",\n replica_id=~\"$mzReplicaList\",\n collection_id=~\"t.*|none\"\n }\n)\n", "legendFormat": "{{collection_id}}", "query": "\nmax by (collection_id) (\n ${sqlMetricPrefix}arrangement_record_count{\n $environmentFilter,\n instance_id=~\"$mzClusterList\",\n replica_id=~\"$mzReplicaList\",\n collection_id=~\"t.*|none\"\n }\n)\n", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-0", "hidden": false } } ], "transformations": [ { "kind": "Transformation", "spec": { "id": "reduce", "options": { "reducers": [ "min", "max", "lastNotNull" ], "mode": "seriesToRows" } }, "group": "reduce" }, { "kind": "Transformation", "spec": { "id": "organize", "options": { "renameByName": { "Field": "Collection ID" } } }, "group": "organize" }, { "kind": "Transformation", "spec": { "id": "sortBy", "options": { "fields": {}, "sort": [ { "field": "Last *", "desc": true } ] } }, "group": "sortBy" } ], "queryOptions": {} } }, "vizConfig": { "kind": "VizConfig", "group": "table", "version": "", "spec": { "options": { "frameIndex": 0, "showHeader": true, "showTypeIcons": false, "footer": { "show": false, "reducer": [], "countRows": false }, "cellHeight": "sm" }, "fieldConfig": { "defaults": { "unit": "short", "noValue": "No matches for the current filters", "custom": { "align": "auto", "inspect": false, "filterable": true } }, "overrides": [] } } } } }, "storage-active-sources": { "kind": "Panel", "spec": { "id": 1052, "title": "Active Sources", "description": "**Number of active sources in the catalog.** Each source is a continuous ingestion connection from an external system (Kafka, Postgres, MySQL, S3, etc.) \u2014 so this count is roughly the number of upstream feeds the environment is maintaining. Counts distinct source objects (the hidden per-source `_progress` subsources are excluded), so it matches what you'd see in `mz_sources`. See _Sources_ row below for type breakdown and per-source throughput. Environment-scoped \u2014 not affected by the cluster/replica filters.", "links": [], "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "\ncount(\n group by (id) (\n ${sqlMetricPrefix}storage_objects{$environmentFilter, type=\"source\"}\n )\n) or vector(0)\n", "legendFormat": "sources", "query": "\ncount(\n group by (id) (\n ${sqlMetricPrefix}storage_objects{$environmentFilter, type=\"source\"}\n )\n) or vector(0)\n", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-0", "hidden": false } } ], "transformations": [], "queryOptions": {} } }, "vizConfig": { "kind": "VizConfig", "group": "stat", "version": "", "spec": { "options": { "graphMode": "area", "colorMode": "none", "justifyMode": "auto", "textMode": "value", "wideLayout": true, "showPercentChange": false, "reduceOptions": { "calcs": [] }, "percentChangeColorMode": "standard", "orientation": "auto" }, "fieldConfig": { "defaults": { "min": 0, "color": { "mode": "shades", "fixedColor": "#CCBB44" }, "noValue": "No matches for the current filters" }, "overrides": [] } } } } }, "storage-active-sinks": { "kind": "Panel", "spec": { "id": 1053, "title": "Active Sinks", "description": "**Number of active sinks in the catalog.** Each sink is an outbound feed (Kafka, Iceberg, etc.) that emits the results of a materialized view or query to an external system. Counts distinct sink objects (excluding hidden `_progress` subsources), matching `mz_sinks`. See _Sinks_ row below for per-sink throughput and lag. Environment-scoped \u2014 not affected by the cluster/replica filters.", "links": [], "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "\ncount(\n group by (id) (\n ${sqlMetricPrefix}storage_objects{$environmentFilter, type=\"sink\"}\n )\n) or vector(0)\n", "legendFormat": "sinks", "query": "\ncount(\n group by (id) (\n ${sqlMetricPrefix}storage_objects{$environmentFilter, type=\"sink\"}\n )\n) or vector(0)\n", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-0", "hidden": false } } ], "transformations": [], "queryOptions": {} } }, "vizConfig": { "kind": "VizConfig", "group": "stat", "version": "", "spec": { "options": { "graphMode": "area", "colorMode": "none", "justifyMode": "auto", "textMode": "value", "wideLayout": true, "showPercentChange": false, "reduceOptions": { "calcs": [] }, "percentChangeColorMode": "standard", "orientation": "auto" }, "fieldConfig": { "defaults": { "min": 0, "color": { "mode": "shades", "fixedColor": "#CCBB44" }, "noValue": "No matches for the current filters" }, "overrides": [] } } } } }, "storage-active-tables": { "kind": "Panel", "spec": { "id": 1054, "title": "Active Tables", "description": "**Number of user-created tables in the catalog.** Tables in Materialize are write-once-read-many; `INSERT`s feed dataflows downstream. Mostly a catalog-shape signal \u2014 for actual ingest activity see _Sources -> Source Bytes Received_. Environment-scoped \u2014 not affected by the cluster/replica filters.", "links": [], "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "\nmax(\n sum by (instance) (${sqlMetricPrefix}tables_count{$environmentFilter})\n) or vector(0)\n", "legendFormat": "${sqlMetricPrefix}tables_count", "query": "\nmax(\n sum by (instance) (${sqlMetricPrefix}tables_count{$environmentFilter})\n) or vector(0)\n", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-0", "hidden": false } } ], "transformations": [], "queryOptions": {} } }, "vizConfig": { "kind": "VizConfig", "group": "stat", "version": "", "spec": { "options": { "graphMode": "area", "colorMode": "none", "justifyMode": "auto", "textMode": "value", "wideLayout": true, "showPercentChange": false, "reduceOptions": { "calcs": [] }, "percentChangeColorMode": "standard", "orientation": "auto" }, "fieldConfig": { "defaults": { "min": 0, "color": { "mode": "shades", "fixedColor": "#CCBB44" }, "noValue": "No matches for the current filters" }, "overrides": [] } } } } }, "sources-types": { "kind": "Panel", "spec": { "id": 1055, "title": "Source Types", "description": "**Sources broken down by source type** (kafka / postgres / mysql / etc.). Tells you what flavors of upstream feed make up your ingest workload. Most environments concentrate on one or two types. Environment-scoped \u2014 not affected by the cluster/replica filters.", "links": [], "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "\ncount by (object_type) (\n group by (id, object_type) (\n ${sqlMetricPrefix}storage_objects{$environmentFilter, type=\"source\"}\n )\n) > 0\n", "instant": true, "range": false, "legendFormat": "{{object_type}}", "query": "\ncount by (object_type) (\n group by (id, object_type) (\n ${sqlMetricPrefix}storage_objects{$environmentFilter, type=\"source\"}\n )\n) > 0\n", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-0", "hidden": false } } ], "transformations": [], "queryOptions": {} } }, "vizConfig": { "kind": "VizConfig", "group": "piechart", "version": "", "spec": { "options": { "pieType": "donut", "tooltip": { "mode": "single", "sort": "asc" }, "reduceOptions": { "calcs": [] }, "legend": { "values": [ "value" ], "displayMode": "table", "placement": "right", "showLegend": true, "calcs": [], "asTable": true, "isVisible": true }, "orientation": "auto", "displayLabels": [ "name", "value" ] }, "fieldConfig": { "defaults": { "color": { "mode": "shades", "fixedColor": "#CCBB44" }, "noValue": "No matches for the current filters" }, "overrides": [] } } } } }, "sources-status-table": { "kind": "Panel", "spec": { "id": 1056, "title": "Sources", "description": "**Catalog of sources running in the environment \u2014 one row per source (by name) with its connector type, envelope, and the cluster it ingests on.** Names are resolved via `mz_object_info`. Self-managed Materialize exposes no source *status* metric, so running/stalled/errored isn't shown here \u2014 check live status with `SELECT name, type, status FROM mz_internal.mz_source_statuses;`, and use _Source Bytes Received (rate)_ to confirm a source is actively ingesting. The hidden `_progress` subsources are excluded, so the row count matches _Active Sources_.", "links": [], "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "(\n\ngroup by (id, object_type, connection_type, envelope_type, cluster_id) (\n ${sqlMetricPrefix}storage_objects{$environmentFilter, type=\"source\"}\n)\n\n)\n* on (id) group_left(name)\nlabel_replace(mz_object_info{$environmentFilter}, \"id\", \"$1\", \"global_id\", \"(.*)\")", "instant": true, "range": false, "query": "(\n\ngroup by (id, object_type, connection_type, envelope_type, cluster_id) (\n ${sqlMetricPrefix}storage_objects{$environmentFilter, type=\"source\"}\n)\n\n)\n* on (id) group_left(name)\nlabel_replace(mz_object_info{$environmentFilter}, \"id\", \"$1\", \"global_id\", \"(.*)\")", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-0", "hidden": false } } ], "transformations": [ { "kind": "Transformation", "spec": { "id": "labelsToFields", "options": { "keepLabels": [ "name", "id", "object_type", "connection_type", "envelope_type", "cluster_id" ] } }, "group": "labelsToFields" }, { "kind": "Transformation", "spec": { "id": "merge", "options": {} }, "group": "merge" }, { "kind": "Transformation", "spec": { "id": "organize", "options": { "excludeByName": { "Time": true, "Value": true }, "renameByName": { "name": "Name", "id": "Source ID", "object_type": "Type", "connection_type": "Connection", "envelope_type": "Envelope", "cluster_id": "Cluster" }, "indexByName": { "name": 0, "id": 1, "object_type": 2, "connection_type": 3, "envelope_type": 4, "cluster_id": 5 } } }, "group": "organize" }, { "kind": "Transformation", "spec": { "id": "sortBy", "options": { "fields": {}, "sort": [ { "field": "Name" } ] } }, "group": "sortBy" } ], "queryOptions": {} } }, "vizConfig": { "kind": "VizConfig", "group": "table", "version": "", "spec": { "options": { "frameIndex": 0, "showHeader": true, "showTypeIcons": false, "footer": { "show": false, "reducer": [], "countRows": false }, "cellHeight": "sm" }, "fieldConfig": { "defaults": { "noValue": "No matches for the current filters", "custom": { "align": "auto", "inspect": false, "filterable": true } }, "overrides": [] } } } } }, "sources-bytes-received-rate": { "kind": "Panel", "spec": { "id": 1057, "title": "Source Bytes Received (rate)", "description": "**Inbound throughput per primary source \u2014 bytes per second pulled from upstream.** Subsources (e.g., per-table Postgres replication subsources) are aggregated up to their primary, so each line represents one logical source, labeled by source name (resolved via `mz_object_info`). Idle sources are filtered out (`> 0`). Log Y-axis so kB/s and tens-of-MB/s sources share the chart. Scoped to the selected clusters/replicas.", "links": [], "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "(\n\nsum by (parent_source_id) (\n max without (job) (\n rate(mz_source_bytes_received{$environmentFilter, cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval])\n )\n) > 0\n\n)\n* on (parent_source_id) group_left(name)\nlabel_replace(mz_object_info{$environmentFilter}, \"parent_source_id\", \"$1\", \"global_id\", \"(.*)\")", "legendFormat": "{{name}}", "query": "(\n\nsum by (parent_source_id) (\n max without (job) (\n rate(mz_source_bytes_received{$environmentFilter, cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval])\n )\n) > 0\n\n)\n* on (parent_source_id) group_left(name)\nlabel_replace(mz_object_info{$environmentFilter}, \"parent_source_id\", \"$1\", \"global_id\", \"(.*)\")", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-0", "hidden": false } } ], "transformations": [], "queryOptions": {} } }, "vizConfig": { "kind": "VizConfig", "group": "timeseries", "version": "", "spec": { "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": [ "max", "mean", "lastNotNull" ] }, "tooltip": { "mode": "single", "sort": "asc" } }, "fieldConfig": { "defaults": { "unit": "Bps", "custom": { "scaleDistribution": { "type": "log", "log": 10 } } }, "overrides": [] } } } } }, "sources-ingestion-by-replica": { "kind": "Panel", "spec": { "id": 1058, "title": "Source Ingestion by Replica", "description": "**Messages ingested per second, split per source and replica.** Replicas read their upstream independently and should track together. **A replica flat at 0 while its siblings keep ingesting has lost its upstream connection** (e.g. it was restarted and couldn't resume pulling from Kafka) \u2014 the source still shows `Running` overall and the aggregate _Source Bytes Received_ hides it, so this split is where it surfaces. Legends are ids; map with `SELECT id, name FROM mz_sources` and the replica via `SELECT id, name FROM mz_cluster_replicas`. When you see a replica drop out, _Compute -> Freshness_ frontier lag will be climbing too; restarting that replica usually clears the stale connection.", "links": [], "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "(\n\nsum by (parent_source_id, cluster_environmentd_materialize_cloud_replica_id) (\n max without (job) (\n rate(mz_source_messages_received{$environmentFilter, cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval])\n )\n)\n\n)\n* on (parent_source_id) group_left(name)\nlabel_replace(mz_object_info{$environmentFilter}, \"parent_source_id\", \"$1\", \"global_id\", \"(.*)\")", "legendFormat": "{{name}} / r{{cluster_environmentd_materialize_cloud_replica_id}}", "query": "(\n\nsum by (parent_source_id, cluster_environmentd_materialize_cloud_replica_id) (\n max without (job) (\n rate(mz_source_messages_received{$environmentFilter, cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval])\n )\n)\n\n)\n* on (parent_source_id) group_left(name)\nlabel_replace(mz_object_info{$environmentFilter}, \"parent_source_id\", \"$1\", \"global_id\", \"(.*)\")", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-0", "hidden": false } } ], "transformations": [], "queryOptions": {} } }, "vizConfig": { "kind": "VizConfig", "group": "timeseries", "version": "", "spec": { "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": [ "max", "mean", "lastNotNull" ] }, "tooltip": { "mode": "single", "sort": "asc" } }, "fieldConfig": { "defaults": { "unit": "cps", "min": 0, "noValue": "No matches for the current filters" }, "overrides": [] } } } } }, "sources-errors": { "kind": "Panel", "spec": { "id": 1059, "title": "Source Upstream Errors", "description": "**Per-source upstream health \u2014 healthy sources are filtered out (`> 0`), so this panel is empty when all is well and any series at all means a source needs attention.** Two signals: **commit failures** (`mz_source_offset_commit_failures` rate) fire when the upstream is reachable but rejects the offset / replication-slot commit (auth/ACL, broker rejecting); the **disconnected** indicator flips to **1** when the source has lost sight of its upstream (`offset_known` fell below `offset_committed`) \u2014 the broker/DB-unreachable case (`BrokerTransportFailure`, severed security group, DNS) that the commit-failure counter can't catch because the source never reaches the commit step. When `disconnected` is 1, _Source Bytes Received_ flat-lines and _Compute -> Freshness_ frontier lag climbs. Legend is `source_id` \u2014 name it with `SELECT id, name FROM mz_sources` and read the exact error via `SELECT name, status, error FROM mz_internal.mz_source_statuses WHERE status != 'running'`. (Data-decode errors are separate: `mz_source_error_inserts`.)", "links": [], "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "\nsum by (source_id) (\n max without (job) (\n rate(mz_source_offset_commit_failures{$environmentFilter, cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval])\n )\n) > 0\n", "legendFormat": "{{source_id}} commit failures", "query": "\nsum by (source_id) (\n max without (job) (\n rate(mz_source_offset_commit_failures{$environmentFilter, cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval])\n )\n) > 0\n", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-0", "hidden": false } }, { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "\n(\n max by (source_id) (\n max without (job) (mz_source_offset_committed{$environmentFilter, cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"})\n ) > bool max by (source_id) (\n max without (job) (mz_source_offset_known{$environmentFilter, cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"})\n )\n) > 0\n", "legendFormat": "{{source_id}} disconnected", "query": "\n(\n max by (source_id) (\n max without (job) (mz_source_offset_committed{$environmentFilter, cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"})\n ) > bool max by (source_id) (\n max without (job) (mz_source_offset_known{$environmentFilter, cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"})\n )\n) > 0\n", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-1", "hidden": false } } ], "transformations": [], "queryOptions": {} } }, "vizConfig": { "kind": "VizConfig", "group": "timeseries", "version": "", "spec": { "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": [ "max", "mean", "lastNotNull" ] }, "tooltip": { "mode": "single", "sort": "asc" } }, "fieldConfig": { "defaults": { "unit": "short", "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "value": 1.0, "color": "#FEDA8B" }, { "value": 1.0, "color": "#FDB366" }, { "value": 1.0, "color": "#F67E4B" }, { "value": 1.0, "color": "#DD3D2D" }, { "value": 1.0, "color": "#A50026" } ] }, "noValue": "No matches for the current filters" }, "overrides": [] } } } } }, "sinks-types": { "kind": "Panel", "spec": { "id": 1060, "title": "Sink Types", "description": "**Sinks broken down by (type, envelope_type)** \u2014 e.g., `kafka / upsert`, `kafka / debezium`, `iceberg / upsert`. The envelope determines how Materialize encodes changes: `upsert` writes the latest value per key, `debezium` writes change events with old+new values. Most envs concentrate on one combination. Environment-scoped \u2014 not affected by the cluster/replica filters.", "links": [], "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "\ncount by (object_type, envelope_type) (\n group by (id, object_type, envelope_type) (\n ${sqlMetricPrefix}storage_objects{$environmentFilter, type=\"sink\"}\n )\n) > 0\n", "instant": true, "range": false, "legendFormat": "{{object_type}} / {{envelope_type}}", "query": "\ncount by (object_type, envelope_type) (\n group by (id, object_type, envelope_type) (\n ${sqlMetricPrefix}storage_objects{$environmentFilter, type=\"sink\"}\n )\n) > 0\n", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-0", "hidden": false } } ], "transformations": [], "queryOptions": {} } }, "vizConfig": { "kind": "VizConfig", "group": "piechart", "version": "", "spec": { "options": { "pieType": "donut", "tooltip": { "mode": "single", "sort": "asc" }, "reduceOptions": { "calcs": [] }, "legend": { "values": [ "value" ], "displayMode": "table", "placement": "right", "showLegend": true, "calcs": [], "asTable": true, "isVisible": true }, "orientation": "auto", "displayLabels": [ "name", "value" ] }, "fieldConfig": { "defaults": { "color": { "mode": "shades", "fixedColor": "#CCBB44" }, "noValue": "No matches for the current filters" }, "overrides": [] } } } } }, "sinks-throughput": { "kind": "Panel", "spec": { "id": 1061, "title": "Sink Throughput (committed)", "description": "**Outbound throughput per sink \u2014 bytes per second successfully committed to the downstream system** (Kafka broker, Iceberg catalog, etc.). Log Y-axis so low- and high-volume sinks share the chart. Labeled by sink name (resolved via `mz_object_info`). Idle sinks are filtered out (`> 0`). Scoped to the selected clusters/replicas.", "links": [], "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "(\n\nsum by (sink_id) (\n max without (job) (\n rate(mz_sink_bytes_committed{$environmentFilter, cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval])\n )\n) > 0\n\n)\n* on (sink_id) group_left(name)\nlabel_replace(mz_object_info{$environmentFilter}, \"sink_id\", \"$1\", \"global_id\", \"(.*)\")", "legendFormat": "{{name}}", "query": "(\n\nsum by (sink_id) (\n max without (job) (\n rate(mz_sink_bytes_committed{$environmentFilter, cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval])\n )\n) > 0\n\n)\n* on (sink_id) group_left(name)\nlabel_replace(mz_object_info{$environmentFilter}, \"sink_id\", \"$1\", \"global_id\", \"(.*)\")", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-0", "hidden": false } } ], "transformations": [], "queryOptions": {} } }, "vizConfig": { "kind": "VizConfig", "group": "timeseries", "version": "", "spec": { "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": [ "max", "mean", "lastNotNull" ] }, "tooltip": { "mode": "single", "sort": "asc" } }, "fieldConfig": { "defaults": { "unit": "Bps", "custom": { "scaleDistribution": { "type": "log", "log": 10 } } }, "overrides": [] } } } } }, "sinks-lag": { "kind": "Panel", "spec": { "id": 1062, "title": "Sink Lag (staged minus committed)", "description": "**Bytes staged for a sink but not yet committed downstream \u2014 a queue depth in bytes.** Both metrics are counters; their difference at any moment is the in-flight write that's been prepared but not yet acknowledged by the downstream system. Nominal: oscillates around a small value as commits happen periodically. **Sustained growth means the sink can't keep up** \u2014 usually downstream back-pressure (broker overloaded, Iceberg catalog slow) or repeated commit failures (see _Iceberg Commit Failures & Conflicts_ or _Kafka TX Error Rate_ in the collapsed rows below). Scoped to the selected clusters/replicas.", "links": [], "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "(\n\nclamp_min(\n sum by (sink_id) (max without (job) (mz_sink_bytes_staged{$environmentFilter, cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}))\n - sum by (sink_id) (max without (job) (mz_sink_bytes_committed{$environmentFilter, cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"})),\n 0\n)\n\n)\n* on (sink_id) group_left(name)\nlabel_replace(mz_object_info{$environmentFilter}, \"sink_id\", \"$1\", \"global_id\", \"(.*)\")", "legendFormat": "{{name}}", "query": "(\n\nclamp_min(\n sum by (sink_id) (max without (job) (mz_sink_bytes_staged{$environmentFilter, cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}))\n - sum by (sink_id) (max without (job) (mz_sink_bytes_committed{$environmentFilter, cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"})),\n 0\n)\n\n)\n* on (sink_id) group_left(name)\nlabel_replace(mz_object_info{$environmentFilter}, \"sink_id\", \"$1\", \"global_id\", \"(.*)\")", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-0", "hidden": false } } ], "transformations": [], "queryOptions": {} } }, "vizConfig": { "kind": "VizConfig", "group": "timeseries", "version": "", "spec": { "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": [ "max", "mean", "lastNotNull" ] }, "tooltip": { "mode": "single", "sort": "asc" } }, "fieldConfig": { "defaults": { "unit": "bytes", "min": 0 }, "overrides": [] } } } } }, "sinks-iceberg-commit-latency": { "kind": "Panel", "spec": { "id": 1063, "title": "Iceberg Commit Latency (p50 / p90 / p99)", "description": "**Iceberg commit duration percentiles** \u2014 how long each `COMMIT` against the Iceberg catalog takes. Iceberg writes are batched and committed periodically; the commit involves writing a snapshot manifest and asking the catalog to atomically swap it in. Nominal: p50 sub-second to low seconds; p99 a few seconds even on healthy systems. Sustained p99 in tens of seconds points at a slow Iceberg catalog (REST catalog under load, Glue API throttling) \u2014 _Sink Lag_ will be growing at the same time. Log Y-axis. Scoped to the selected clusters/replicas.", "links": [], "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "\nhistogram_quantile(0.5,\n sum by (le) (\n max without (job) (\n rate(mz_sink_iceberg_commit_duration_seconds_bucket{$environmentFilter, cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval])\n )\n )\n)\n", "legendFormat": "p50", "query": "\nhistogram_quantile(0.5,\n sum by (le) (\n max without (job) (\n rate(mz_sink_iceberg_commit_duration_seconds_bucket{$environmentFilter, cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval])\n )\n )\n)\n", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-0", "hidden": false } }, { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "\nhistogram_quantile(0.9,\n sum by (le) (\n max without (job) (\n rate(mz_sink_iceberg_commit_duration_seconds_bucket{$environmentFilter, cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval])\n )\n )\n)\n", "legendFormat": "p90", "query": "\nhistogram_quantile(0.9,\n sum by (le) (\n max without (job) (\n rate(mz_sink_iceberg_commit_duration_seconds_bucket{$environmentFilter, cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval])\n )\n )\n)\n", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-1", "hidden": false } }, { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "\nhistogram_quantile(0.99,\n sum by (le) (\n max without (job) (\n rate(mz_sink_iceberg_commit_duration_seconds_bucket{$environmentFilter, cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval])\n )\n )\n)\n", "legendFormat": "p99", "query": "\nhistogram_quantile(0.99,\n sum by (le) (\n max without (job) (\n rate(mz_sink_iceberg_commit_duration_seconds_bucket{$environmentFilter, cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval])\n )\n )\n)\n", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-2", "hidden": false } } ], "transformations": [], "queryOptions": {} } }, "vizConfig": { "kind": "VizConfig", "group": "timeseries", "version": "", "spec": { "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": [ "max", "mean", "lastNotNull" ] }, "tooltip": { "mode": "single", "sort": "asc" } }, "fieldConfig": { "defaults": { "unit": "s", "custom": { "scaleDistribution": { "type": "log", "log": 10 } } }, "overrides": [] } } } } }, "sinks-iceberg-failures": { "kind": "Panel", "spec": { "id": 1064, "title": "Iceberg Commit Failures & Conflicts", "description": "**Per-sink rate of failed and conflicting Iceberg commits.** Conflicts (concurrent-writer races on the Iceberg snapshot pointer) are recoverable \u2014 Materialize retries \u2014 but a high rate signals that something else is writing to the same Iceberg table. Failures are commit-side errors (network, auth, schema). **Non-zero in either dimension is worth investigating.** If failures are climbing, _Sink Lag_ will follow. The Errors threshold-coloring is calibrated for \"any non-zero is interesting\".", "links": [], "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "(\n\nsum by (sink_id) (\n max without (job) (\n rate(mz_sink_iceberg_commit_failures{$environmentFilter, cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval])\n )\n)\n\n)\n* on (sink_id) group_left(name)\nlabel_replace(mz_object_info{$environmentFilter}, \"sink_id\", \"$1\", \"global_id\", \"(.*)\")", "legendFormat": "{{name}} failures", "query": "(\n\nsum by (sink_id) (\n max without (job) (\n rate(mz_sink_iceberg_commit_failures{$environmentFilter, cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval])\n )\n)\n\n)\n* on (sink_id) group_left(name)\nlabel_replace(mz_object_info{$environmentFilter}, \"sink_id\", \"$1\", \"global_id\", \"(.*)\")", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-0", "hidden": false } }, { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "(\n\nsum by (sink_id) (\n max without (job) (\n rate(mz_sink_iceberg_commit_conflicts{$environmentFilter, cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval])\n )\n)\n\n)\n* on (sink_id) group_left(name)\nlabel_replace(mz_object_info{$environmentFilter}, \"sink_id\", \"$1\", \"global_id\", \"(.*)\")", "legendFormat": "{{name}} conflicts", "query": "(\n\nsum by (sink_id) (\n max without (job) (\n rate(mz_sink_iceberg_commit_conflicts{$environmentFilter, cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval])\n )\n)\n\n)\n* on (sink_id) group_left(name)\nlabel_replace(mz_object_info{$environmentFilter}, \"sink_id\", \"$1\", \"global_id\", \"(.*)\")", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-1", "hidden": false } } ], "transformations": [], "queryOptions": {} } }, "vizConfig": { "kind": "VizConfig", "group": "timeseries", "version": "", "spec": { "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": [ "max", "mean", "lastNotNull" ] }, "tooltip": { "mode": "single", "sort": "asc" } }, "fieldConfig": { "defaults": { "unit": "cps", "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "value": 1.0, "color": "#FEDA8B" }, { "value": 2.8, "color": "#FDB366" }, { "value": 4.6, "color": "#F67E4B" }, { "value": 6.4, "color": "#DD3D2D" }, { "value": 8.2, "color": "#A50026" } ] } }, "overrides": [] } } } } }, "sinks-iceberg-files": { "kind": "Panel", "spec": { "id": 1065, "title": "Iceberg File & Snapshot Rate", "description": "**Per-sink rate of files and snapshots written to Iceberg.** Each commit produces one snapshot containing some data files (new rows) and delete files (tombstones for upserts). The data:delete file ratio tells you about your workload: pure-insert sinks produce ~0 deletes; upsert-heavy workloads produce roughly 1:1. Sustained delete-file rate without data files means the sink is mostly deleting (data evaporating upstream). Scoped to the selected clusters/replicas.", "links": [], "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "(\n\nsum by (sink_id) (\n max without (job) (\n rate(mz_sink_iceberg_data_files_written{$environmentFilter, cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval])\n )\n)\n\n)\n* on (sink_id) group_left(name)\nlabel_replace(mz_object_info{$environmentFilter}, \"sink_id\", \"$1\", \"global_id\", \"(.*)\")", "legendFormat": "{{name}} data", "query": "(\n\nsum by (sink_id) (\n max without (job) (\n rate(mz_sink_iceberg_data_files_written{$environmentFilter, cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval])\n )\n)\n\n)\n* on (sink_id) group_left(name)\nlabel_replace(mz_object_info{$environmentFilter}, \"sink_id\", \"$1\", \"global_id\", \"(.*)\")", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-0", "hidden": false } }, { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "(\n\nsum by (sink_id) (\n max without (job) (\n rate(mz_sink_iceberg_delete_files_written{$environmentFilter, cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval])\n )\n)\n\n)\n* on (sink_id) group_left(name)\nlabel_replace(mz_object_info{$environmentFilter}, \"sink_id\", \"$1\", \"global_id\", \"(.*)\")", "legendFormat": "{{name}} deletes", "query": "(\n\nsum by (sink_id) (\n max without (job) (\n rate(mz_sink_iceberg_delete_files_written{$environmentFilter, cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval])\n )\n)\n\n)\n* on (sink_id) group_left(name)\nlabel_replace(mz_object_info{$environmentFilter}, \"sink_id\", \"$1\", \"global_id\", \"(.*)\")", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-1", "hidden": false } }, { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "(\n\nsum by (sink_id) (\n max without (job) (\n rate(mz_sink_iceberg_snapshots_committed{$environmentFilter, cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval])\n )\n)\n\n)\n* on (sink_id) group_left(name)\nlabel_replace(mz_object_info{$environmentFilter}, \"sink_id\", \"$1\", \"global_id\", \"(.*)\")", "legendFormat": "{{name}} snapshots", "query": "(\n\nsum by (sink_id) (\n max without (job) (\n rate(mz_sink_iceberg_snapshots_committed{$environmentFilter, cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval])\n )\n)\n\n)\n* on (sink_id) group_left(name)\nlabel_replace(mz_object_info{$environmentFilter}, \"sink_id\", \"$1\", \"global_id\", \"(.*)\")", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-2", "hidden": false } } ], "transformations": [], "queryOptions": {} } }, "vizConfig": { "kind": "VizConfig", "group": "timeseries", "version": "", "spec": { "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": [ "max", "mean", "lastNotNull" ] }, "tooltip": { "mode": "single", "sort": "asc" } }, "fieldConfig": { "defaults": { "unit": "cps", "min": 0 }, "overrides": [] } } } } }, "sinks-kafka-tx-errors": { "kind": "Panel", "spec": { "id": 1066, "title": "Kafka TX Error Rate", "description": "**Per-sink rate of TX errors from the librdkafka client.** Each TX error is one failed produce-request against the Kafka broker. **Non-zero is a problem** \u2014 likely causes are broker outages, ACL changes, topic deletion/recreation, or partition rebalancing. If errors are sustained, _Sink Lag_ will grow and _Kafka Output Buffer_ may fill. Errors threshold-coloring is calibrated for \"any non-zero is interesting\".", "links": [], "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "(\n\nsum by (sink_id) (\n max without (job) (\n rate(mz_sink_rdkafka_txerrs{$environmentFilter, cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval])\n )\n)\n\n)\n* on (sink_id) group_left(name)\nlabel_replace(mz_object_info{$environmentFilter}, \"sink_id\", \"$1\", \"global_id\", \"(.*)\")", "legendFormat": "{{name}}", "query": "(\n\nsum by (sink_id) (\n max without (job) (\n rate(mz_sink_rdkafka_txerrs{$environmentFilter, cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval])\n )\n)\n\n)\n* on (sink_id) group_left(name)\nlabel_replace(mz_object_info{$environmentFilter}, \"sink_id\", \"$1\", \"global_id\", \"(.*)\")", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-0", "hidden": false } } ], "transformations": [], "queryOptions": {} } }, "vizConfig": { "kind": "VizConfig", "group": "timeseries", "version": "", "spec": { "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": [ "max", "mean", "lastNotNull" ] }, "tooltip": { "mode": "single", "sort": "asc" } }, "fieldConfig": { "defaults": { "unit": "cps", "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "value": 1.0, "color": "#FEDA8B" }, { "value": 2.8, "color": "#FDB366" }, { "value": 4.6, "color": "#F67E4B" }, { "value": 6.4, "color": "#DD3D2D" }, { "value": 8.2, "color": "#A50026" } ] } }, "overrides": [] } } } } }, "sinks-kafka-outbuf": { "kind": "Panel", "spec": { "id": 1067, "title": "Kafka Output Buffer (messages)", "description": "**Messages currently sitting in the librdkafka output buffer, waiting to be sent to the broker.** Normal buffer fluctuates briefly as messages flow through; sustained high values mean Materialize is producing faster than the broker is accepting. Often paired with a non-zero _Kafka TX Error Rate_. If the buffer hits its bound, the sink stalls and _Sink Lag_ starts climbing.", "links": [], "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "(\n\nsum by (sink_id) (\n max without (job) (\n mz_sink_rdkafka_outbuf_msg_cnt{$environmentFilter, cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}\n )\n)\n\n)\n* on (sink_id) group_left(name)\nlabel_replace(mz_object_info{$environmentFilter}, \"sink_id\", \"$1\", \"global_id\", \"(.*)\")", "legendFormat": "{{name}}", "query": "(\n\nsum by (sink_id) (\n max without (job) (\n mz_sink_rdkafka_outbuf_msg_cnt{$environmentFilter, cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}\n )\n)\n\n)\n* on (sink_id) group_left(name)\nlabel_replace(mz_object_info{$environmentFilter}, \"sink_id\", \"$1\", \"global_id\", \"(.*)\")", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-0", "hidden": false } } ], "transformations": [], "queryOptions": {} } }, "vizConfig": { "kind": "VizConfig", "group": "timeseries", "version": "", "spec": { "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": [ "max", "mean", "lastNotNull" ] }, "tooltip": { "mode": "single", "sort": "asc" } }, "fieldConfig": { "defaults": { "unit": "short", "min": 0 }, "overrides": [] } } } } }, "sinks-kafka-connects": { "kind": "Panel", "spec": { "id": 1068, "title": "Kafka Connect / Disconnect Rate", "description": "**Connect and disconnect events per sink against the Kafka broker.** Healthy connections are persistent \u2014 a couple of connects at sink startup and zero disconnects afterward. **Sustained non-zero disconnect rate is a sign of unhealthy connectivity** (network flakiness, broker restarting, auth tokens expiring). Pairs with _Kafka TX Error Rate_ when the issue is broker-side rather than purely network.", "links": [], "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "(\n\nsum by (sink_id) (\n max without (job) (\n rate(mz_sink_rdkafka_connects{$environmentFilter, cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval])\n )\n)\n\n)\n* on (sink_id) group_left(name)\nlabel_replace(mz_object_info{$environmentFilter}, \"sink_id\", \"$1\", \"global_id\", \"(.*)\")", "legendFormat": "{{name}} connects", "query": "(\n\nsum by (sink_id) (\n max without (job) (\n rate(mz_sink_rdkafka_connects{$environmentFilter, cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval])\n )\n)\n\n)\n* on (sink_id) group_left(name)\nlabel_replace(mz_object_info{$environmentFilter}, \"sink_id\", \"$1\", \"global_id\", \"(.*)\")", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-0", "hidden": false } }, { "kind": "PanelQuery", "spec": { "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "(\n\nsum by (sink_id) (\n max without (job) (\n rate(mz_sink_rdkafka_disconnects{$environmentFilter, cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval])\n )\n)\n\n)\n* on (sink_id) group_left(name)\nlabel_replace(mz_object_info{$environmentFilter}, \"sink_id\", \"$1\", \"global_id\", \"(.*)\")", "legendFormat": "{{name}} disconnects", "query": "(\n\nsum by (sink_id) (\n max without (job) (\n rate(mz_sink_rdkafka_disconnects{$environmentFilter, cluster_environmentd_materialize_cloud_cluster_id=~\"$mzClusterList\", cluster_environmentd_materialize_cloud_replica_id=~\"$mzReplicaList\"}[$__rate_interval])\n )\n)\n\n)\n* on (sink_id) group_left(name)\nlabel_replace(mz_object_info{$environmentFilter}, \"sink_id\", \"$1\", \"global_id\", \"(.*)\")", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "refId": "query-1", "hidden": false } } ], "transformations": [], "queryOptions": {} } }, "vizConfig": { "kind": "VizConfig", "group": "timeseries", "version": "", "spec": { "options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true, "calcs": [ "max", "mean", "lastNotNull" ] }, "tooltip": { "mode": "single", "sort": "asc" } }, "fieldConfig": { "defaults": { "unit": "cps", "min": 0 }, "overrides": [] } } } } } }, "layout": { "kind": "TabsLayout", "spec": { "tabs": [ { "kind": "TabsLayoutTab", "spec": { "layout": { "kind": "RowsLayout", "spec": { "rows": [ { "kind": "RowsLayoutRow", "spec": { "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "standard", "rowHeightMode": "short", "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "is-healthy" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "availability-percent" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "last-restart" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "summary-currently-hydrating" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "summary-max-lag" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "cpu-usage-current" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "memory-usage-current" } } } ], "maxColumnCount": 3, "fillScreen": false } }, "title": "Environment Health", "hideHeader": false } }, { "kind": "RowsLayoutRow", "spec": { "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "standard", "rowHeightMode": "short", "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "materialize-version" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "summary-cpu-total" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "summary-memory-total" } } } ], "maxColumnCount": 3, "fillScreen": false } }, "title": "Environment Info", "hideHeader": false } } ] } }, "title": "Summary" } }, { "kind": "TabsLayoutTab", "spec": { "layout": { "kind": "RowsLayout", "spec": { "rows": [ { "kind": "RowsLayoutRow", "spec": { "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "standard", "rowHeightMode": "short", "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "k8s-res-cpu-total" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "k8s-res-memory-total" } } } ], "maxColumnCount": 3, "fillScreen": false } }, "title": "Resources Summary", "hideHeader": true } }, { "kind": "RowsLayoutRow", "spec": { "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "standard", "rowHeightMode": "short", "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "resource-pod-status" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "resource-statefulset-status" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "resource-deployment-status" } } } ], "maxColumnCount": 5, "fillScreen": false } }, "title": "Workload Readiness", "hideHeader": true } }, { "kind": "RowsLayoutRow", "spec": { "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "standard", "rowHeightMode": "standard", "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "pod-cpu-percent" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "pod-memory-percent" } } } ], "maxColumnCount": 3, "fillScreen": false } }, "title": "Pod Metrics", "hideHeader": false } }, { "kind": "RowsLayoutRow", "spec": { "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "standard", "rowHeightMode": "standard", "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "pod-network-rx" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "pod-network-tx" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "pod-network-errors" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "pod-network-drops" } } } ], "maxColumnCount": 2, "fillScreen": false } }, "title": "Pod Networking", "hideHeader": false } } ] } }, "title": "Kubernetes Workloads" } }, { "kind": "TabsLayoutTab", "spec": { "layout": { "kind": "RowsLayout", "spec": { "rows": [ { "kind": "RowsLayoutRow", "spec": { "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "standard", "rowHeightMode": "short", "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "connections-active-sessions" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "connections-active-queries" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "connections-adapter-command-rate" } } } ], "maxColumnCount": 3, "fillScreen": false } }, "title": "Connection Summary", "hideHeader": true } }, { "kind": "RowsLayoutRow", "spec": { "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "standard", "rowHeightMode": "standard", "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "queries-distribution" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "queries-rate-by-statement" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "queries-peek-latency-p50" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "queries-peek-latency-p90" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "queries-peek-latency-p99" } } } ], "maxColumnCount": 3, "fillScreen": false } }, "title": "Queries", "hideHeader": false } }, { "kind": "RowsLayoutRow", "spec": { "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "standard", "rowHeightMode": "standard", "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "adapter-commands-by-app" } } } ], "maxColumnCount": 1, "fillScreen": false } }, "title": "Adapter Commands", "hideHeader": false } } ] } }, "title": "Connections / Activity" } }, { "kind": "TabsLayoutTab", "spec": { "layout": { "kind": "RowsLayout", "spec": { "rows": [ { "kind": "RowsLayoutRow", "spec": { "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "standard", "rowHeightMode": "short", "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "cluster-count" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "replica-count" } } } ], "maxColumnCount": 3, "fillScreen": false } }, "title": "Cluster Summary", "hideHeader": true } }, { "kind": "RowsLayoutRow", "spec": { "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "standard", "rowHeightMode": "standard", "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "replica-sizes" } } } ], "maxColumnCount": 3, "fillScreen": false } }, "title": "Replication / Availability" } }, { "kind": "RowsLayoutRow", "spec": { "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "standard", "rowHeightMode": "standard", "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "cluster-table" } } } ], "maxColumnCount": 3, "fillScreen": false } }, "title": "Cluster Information" } } ] } }, "title": "Cluster Objects / Replicas" } }, { "kind": "TabsLayoutTab", "spec": { "layout": { "kind": "RowsLayout", "spec": { "rows": [ { "kind": "RowsLayoutRow", "spec": { "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "narrow", "rowHeightMode": "short", "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "active-mzd-views" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "active-indexes" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "active-views" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "active-subscribes" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "index-types" } } } ], "maxColumnCount": 5, "fillScreen": false } }, "title": "Compute Objects Summary", "hideHeader": true } }, { "kind": "RowsLayoutRow", "spec": { "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "wide", "rowHeightMode": "standard", "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "freshness-lag-by-cluster" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "freshness-top-collections" } } } ], "maxColumnCount": 3, "fillScreen": false } }, "title": "Freshness", "hideHeader": false } }, { "kind": "RowsLayoutRow", "spec": { "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "standard", "rowHeightMode": "standard", "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "hydration-unhydrated-count" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "hydration-queue-size" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "hydration-slowest-collections" } } } ], "maxColumnCount": 3, "fillScreen": false } }, "title": "Hydration", "hideHeader": false } }, { "kind": "RowsLayoutRow", "spec": { "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "standard", "rowHeightMode": "standard", "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "dataflow-count" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "dataflow-count-by-worker" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "dataflow-elapsed-rate" } } } ], "maxColumnCount": 3, "fillScreen": false } }, "title": "Dataflows", "hideHeader": false } }, { "kind": "RowsLayoutRow", "spec": { "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "wide", "rowHeightMode": "standard", "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "arrangement-rate" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "arrangement-rate-by-worker" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "arrangement-records-system" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "arrangement-records-user" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "arrangement-records-transient" } } } ], "maxColumnCount": 3, "fillScreen": false } }, "title": "Arrangements", "hideHeader": false } } ] } }, "title": "Compute Objects" } }, { "kind": "TabsLayoutTab", "spec": { "layout": { "kind": "RowsLayout", "spec": { "rows": [ { "kind": "RowsLayoutRow", "spec": { "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "narrow", "rowHeightMode": "short", "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "storage-active-sources" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "storage-active-sinks" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "storage-active-tables" } } } ], "maxColumnCount": 3, "fillScreen": false } }, "title": "Storage Objects Summary", "hideHeader": true } }, { "kind": "RowsLayoutRow", "spec": { "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "wide", "rowHeightMode": "standard", "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "sources-types" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "sources-status-table" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "sources-bytes-received-rate" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "sources-ingestion-by-replica" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "sources-errors" } } } ], "maxColumnCount": 3, "fillScreen": false } }, "title": "Sources", "hideHeader": false } }, { "kind": "RowsLayoutRow", "spec": { "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "wide", "rowHeightMode": "standard", "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "sinks-types" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "sinks-throughput" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "sinks-lag" } } } ], "maxColumnCount": 3, "fillScreen": false } }, "title": "Sinks", "hideHeader": false } }, { "kind": "RowsLayoutRow", "spec": { "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "wide", "rowHeightMode": "standard", "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "sinks-iceberg-commit-latency" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "sinks-iceberg-failures" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "sinks-iceberg-files" } } } ], "maxColumnCount": 3, "fillScreen": false } }, "title": "Iceberg Sinks", "collapse": true, "hideHeader": false } }, { "kind": "RowsLayoutRow", "spec": { "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "wide", "rowHeightMode": "standard", "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "sinks-kafka-tx-errors" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "sinks-kafka-outbuf" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "sinks-kafka-connects" } } } ], "maxColumnCount": 3, "fillScreen": false } }, "title": "Kafka Sinks", "collapse": true, "hideHeader": false } } ] } }, "title": "Sources and Sinks" } } ] } }, "links": [], "preload": false, "tags": [ "materialize", "monitoring" ], "timeSettings": { "from": "now-6h", "to": "now", "autoRefresh": "", "autoRefreshIntervals": [ "5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d" ], "hideTimepicker": false, "fiscalYearStartMonth": 0, "timezone": "browser" }, "title": "Materialize Environment Overview", "variables": [ { "kind": "DatasourceVariable", "spec": { "name": "metricsDatasource", "pluginId": "prometheus", "refresh": "never", "regex": "", "current": { "text": "", "value": "" }, "options": [], "multi": false, "includeAll": false, "hide": "dontHide", "skipUrlSync": false, "allowCustomValue": false, "label": "Metrics Datasource", "description": "Datasource for metrics queries" } }, { "kind": "AdhocVariable", "group": "", "spec": { "name": "metricAdhoc", "baseFilters": [ { "key": "namespace", "operator": "=~", "value": "$mzNamespaceList", "origin": "dashboard" } ], "filters": [], "defaultKeys": [], "hide": "inControlsMenu", "skipUrlSync": false, "allowCustomValue": true, "label": "Advanced Metric Filter", "description": "Adhoc filters to apply to all metrics queries" }, "datasource": { "name": "$metricsDatasource" } }, { "kind": "QueryVariable", "spec": { "name": "sqlMetricPrefix", "current": { "text": "", "value": "" }, "hide": "hideVariable", "refresh": "never", "skipUrlSync": true, "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "query_result(group by (__name__) ({__name__=~\".*compute_cluster_status\"}))", "query": "query_result(group by (__name__) ({__name__=~\".*compute_cluster_status\"}))", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "regex": "/(?(?.*))compute_cluster_status.*/", "sort": "disabled", "options": [], "multi": false, "includeAll": false, "allowCustomValue": true, "label": "SQL Metric Prefix", "description": "The prefix for all SQL-derived metrics. In self-managed, this is `mz_`; in cloud, this uses `v2_mz_`.", "definition": "query_result(group by (__name__) ({__name__=~\".*compute_cluster_status\"}))" } }, { "kind": "QueryVariable", "spec": { "name": "mzNamespaceList", "current": { "text": "", "value": "" }, "hide": "hideVariable", "refresh": "never", "skipUrlSync": true, "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "label_values(mz_compute_commands_total{ materialize_cloud_organization_name=~\"$environmentIdList\" }, materialize_cloud_organization_namespace)", "query": "label_values(mz_compute_commands_total{ materialize_cloud_organization_name=~\"$environmentIdList\" }, materialize_cloud_organization_namespace)", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "regex": "", "sort": "alphabeticalAsc", "options": [], "multi": true, "includeAll": true, "allowCustomValue": true, "label": "Materialize Namespace", "description": "The current materialize namespace where environments live", "definition": "label_values(mz_compute_commands_total{ materialize_cloud_organization_name=~\"$environmentIdList\" }, materialize_cloud_organization_namespace)" } }, { "kind": "QueryVariable", "spec": { "name": "environmentIdList", "current": { "text": "", "value": "" }, "hide": "dontHide", "refresh": "never", "skipUrlSync": false, "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "label_values(mz_compute_commands_total, materialize_cloud_organization_name)", "query": "label_values(mz_compute_commands_total, materialize_cloud_organization_name)", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "regex": "", "sort": "alphabeticalAsc", "options": [], "multi": false, "includeAll": false, "allowCustomValue": true, "label": "Environment", "description": "The current environment to view", "definition": "label_values(mz_compute_commands_total, materialize_cloud_organization_name)", "allValue": ".*" } }, { "kind": "ConstantVariable", "spec": { "name": "containerFilter", "query": "namespace=~\"$mzNamespaceList\",container!=\"\",container!=\"POD\"", "current": { "text": "", "value": "" }, "hide": "hideVariable", "skipUrlSync": true, "label": "Container Filter", "description": "A filter to apply to cAdvisor queries to remove irrelevant series" } }, { "kind": "SwitchVariable", "spec": { "name": "includeSystemClusters", "current": ".*", "enabledValue": ".*", "disabledValue": "^[^s].*", "hide": "inControlsMenu", "skipUrlSync": true, "label": "Include System Clusters", "description": "Whether to include materialize system clusters in the cluster list." } }, { "kind": "QueryVariable", "spec": { "name": "mzClusterList", "current": { "text": "", "value": "" }, "hide": "inControlsMenu", "refresh": "never", "skipUrlSync": false, "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "query_result(${sqlMetricPrefix}compute_cluster_status{materialize_cloud_organization_name=~\"$environmentIdList\", compute_cluster_id=~\"$includeSystemClusters\" })", "query": "query_result(${sqlMetricPrefix}compute_cluster_status{materialize_cloud_organization_name=~\"$environmentIdList\", compute_cluster_id=~\"$includeSystemClusters\" })", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "regex": ".*compute_cluster_id=\\\"(?[^\\\"]+)\\\",.*compute_cluster_name=\\\"(?[^\\\"]+)\\\",.*", "sort": "naturalAsc", "options": [], "multi": true, "includeAll": true, "allowCustomValue": true, "label": "Cluster", "description": "The cluster within the current environment to filter to", "definition": "query_result(${sqlMetricPrefix}compute_cluster_status{materialize_cloud_organization_name=~\"$environmentIdList\", compute_cluster_id=~\"$includeSystemClusters\" })" } }, { "kind": "QueryVariable", "spec": { "name": "mzReplicaList", "current": { "text": "", "value": "" }, "hide": "inControlsMenu", "refresh": "never", "skipUrlSync": false, "query": { "kind": "DataQuery", "group": "prometheus", "version": "v0", "spec": { "expr": "label_values(mz_compute_commands_total{materialize_cloud_organization_name=~\"$environmentIdList\", instance_id=~\"$mzClusterList\"}, replica_id)", "query": "label_values(mz_compute_commands_total{materialize_cloud_organization_name=~\"$environmentIdList\", instance_id=~\"$mzClusterList\"}, replica_id)", "qryType": 1 }, "datasource": { "name": "${metricsDatasource}" } }, "regex": "", "sort": "naturalAsc", "options": [], "multi": true, "includeAll": true, "allowCustomValue": true, "label": "Replica", "description": "The replica within the current cluster to filter to", "definition": "label_values(mz_compute_commands_total{materialize_cloud_organization_name=~\"$environmentIdList\", instance_id=~\"$mzClusterList\"}, replica_id)", "allValue": ".*" } }, { "kind": "ConstantVariable", "spec": { "name": "environmentFilter", "query": "materialize_cloud_organization_name=~\"$environmentIdList\"", "current": { "text": "", "value": "" }, "hide": "hideVariable", "skipUrlSync": true, "label": "Environment Filter", "description": "A filter to apply to queries to filter to the current environment" } }, { "kind": "ConstantVariable", "spec": { "name": "clusterFilter", "query": "materialize_cloud_organization_name=~\"$environmentIdList\", instance_id=~\"$mzClusterList\"", "current": { "text": "", "value": "" }, "hide": "hideVariable", "skipUrlSync": true, "label": "Cluster Filter", "description": "A filter to apply to queries to filter to the current cluster" } }, { "kind": "ConstantVariable", "spec": { "name": "replicaFilter", "query": "materialize_cloud_organization_name=~\"$environmentIdList\", instance_id=~\"$mzClusterList\", replica_id=~\"$mzReplicaList\"", "current": { "text": "", "value": "" }, "hide": "hideVariable", "skipUrlSync": true, "label": "Replica Filter", "description": "A filter to apply to queries to filter to the current replica" } } ], "description": "Overview of a Materialize Environment.\n\nThis provides a high-level summary to catch more obvious issues\nthat may require further investigation.", "editable": true } }