{ "apiVersion": "dashboard.grafana.app/v2", "kind": "Dashboard", "metadata": { "annotations": { "grafana.app/folder": "materialize", "monitoring.materialize.cloud/min-mz-version": "v26.24.0", "monitoring.materialize.cloud/rec-mz-version": "v26.24.0", "monitoring.materialize.cloud/sql-metric-prefix": "mz_", "monitoring.materialize.cloud/target-export": "generic" }, "name": "mz-mon-env-consensus" }, "spec": { "annotations": [ { "kind": "AnnotationQuery", "spec": { "builtIn": true, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "query": { "datasource": { "name": "-- Grafana --" }, "group": "grafana", "kind": "DataQuery", "spec": {}, "version": "v0" } } } ], "cursorSync": "Crosshair", "description": "The metadata database, as Materialize experiences it.\n\nCommits, failures, latency and connection pools, measured by the processes that use the database — the same on every database flavor. What the database reports about itself, through the postgres-exporter or CloudNativePG, is on the Database Internals tab, and what a managed database's cloud provider reports is on the Infrastructure Cloud Provider dashboard.", "editable": true, "elements": { "cnpg-archive-failures": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (pod) (rate(cnpg_pg_stat_archiver_failed_count{pod=~\"($cnpgClusterList)-[0-9]+\"}[$__rate_interval]))\n", "legendFormat": "{{pod}}", "qryType": 1, "query": "sum by (pod) (rate(cnpg_pg_stat_archiver_failed_count{pod=~\"($cnpgClusterList)-[0-9]+\"}[$__rate_interval]))\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**WAL segments that failed to archive, per second.**\n\n**Nominal:** Zero, or no data where WAL archiving is not configured.\n\n**Degraded:** Failed archiving makes point-in-time recovery impossible past the last good segment, and keeps WAL on disk until it succeeds.", "id": 1048, "links": [], "title": "WAL Archiving Failures", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "The selected CNPG cluster has not reported this. Check the CNPG Cluster picker.", "unit": "ops" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "cnpg-backup-age": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "time() - max(cnpg_collector_last_available_backup_timestamp{pod=~\"($cnpgClusterList)-[0-9]+\"} > 0)\n", "legendFormat": "age", "qryType": 1, "query": "time() - max(cnpg_collector_last_available_backup_timestamp{pod=~\"($cnpgClusterList)-[0-9]+\"} > 0)\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Time since the most recent backup CNPG can restore from.**\n\n**Nominal:** Under the backup schedule's interval — typically under a day.\n\n**Degraded:** Older than the schedule means backups are failing. The metadata database cannot be rebuilt from object storage, so it is the backup that makes a lost database recoverable.\n\nAbsent when the cluster has no completed backup, which is what a cluster with no backup configured reports.", "id": 1036, "links": [], "title": "Last Backup", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "fixedColor": "#009988", "mode": "shades" }, "min": 0, "noValue": "No completed backup. CNPG reports none when backups are not configured.", "unit": "s" }, "overrides": [] }, "options": { "colorMode": "none", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [], "values": false }, "showPercentChange": false, "textMode": "value", "wideLayout": true } }, "version": "" } } }, "cnpg-cache-hit": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "(\n sum by (datname) (rate(cnpg_pg_stat_database_blks_hit{pod=~\"($cnpgClusterList)-[0-9]+\", datname!=\"\"}[$__rate_interval]))\n / (\n sum by (datname) (rate(cnpg_pg_stat_database_blks_hit{pod=~\"($cnpgClusterList)-[0-9]+\", datname!=\"\"}[$__rate_interval]))\n + sum by (datname) (rate(cnpg_pg_stat_database_blks_read{pod=~\"($cnpgClusterList)-[0-9]+\", datname!=\"\"}[$__rate_interval]))\n )\n) >= 0\n", "legendFormat": "{{datname}}", "qryType": 1, "query": "(\n sum by (datname) (rate(cnpg_pg_stat_database_blks_hit{pod=~\"($cnpgClusterList)-[0-9]+\", datname!=\"\"}[$__rate_interval]))\n / (\n sum by (datname) (rate(cnpg_pg_stat_database_blks_hit{pod=~\"($cnpgClusterList)-[0-9]+\", datname!=\"\"}[$__rate_interval]))\n + sum by (datname) (rate(cnpg_pg_stat_database_blks_read{pod=~\"($cnpgClusterList)-[0-9]+\", datname!=\"\"}[$__rate_interval]))\n )\n) >= 0\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**The fraction of block reads served from PostgreSQL's shared buffers rather than from disk, by database.**\n\n**Nominal:** Close to 100%. The consensus table is small and hot, and should live in memory.\n\n**Degraded:** Well below 99% means reads are going to disk, so commits wait on I/O. The instance needs more memory, or its `shared_buffers` setting is smaller than the working set.", "id": 1043, "links": [], "title": "Buffer Cache Hit Ratio", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "max": 1, "noValue": "The selected CNPG cluster has not reported this. Check the CNPG Cluster picker.", "unit": "percentunit" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "cnpg-connections-by-database": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (datname, usename) (\n cnpg_backends_total{pod=~\"($cnpgClusterList)-[0-9]+\"}\n and on (pod) (cnpg_pg_replication_in_recovery{pod=~\"($cnpgClusterList)-[0-9]+\"} == 0)\n)\n", "legendFormat": "{{datname}} ({{usename}})", "qryType": 1, "query": "sum by (datname, usename) (\n cnpg_backends_total{pod=~\"($cnpgClusterList)-[0-9]+\"}\n and on (pod) (cnpg_pg_replication_in_recovery{pod=~\"($cnpgClusterList)-[0-9]+\"} == 0)\n)\n" }, "version": "v0" }, "refId": "query-0" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max(\n (\n max by (pod) (cnpg_pg_settings_setting{pod=~\"($cnpgClusterList)-[0-9]+\", name=\"max_connections\"})\n - on (pod) sum by (pod) (cnpg_pg_settings_setting{pod=~\"($cnpgClusterList)-[0-9]+\", name=~\"superuser_reserved_connections|reserved_connections\"})\n )\n and on (pod) (cnpg_pg_replication_in_recovery{pod=~\"($cnpgClusterList)-[0-9]+\"} == 0)\n)\n", "legendFormat": "ceiling", "qryType": 1, "query": "max(\n (\n max by (pod) (cnpg_pg_settings_setting{pod=~\"($cnpgClusterList)-[0-9]+\", name=\"max_connections\"})\n - on (pod) sum by (pod) (cnpg_pg_settings_setting{pod=~\"($cnpgClusterList)-[0-9]+\", name=~\"superuser_reserved_connections|reserved_connections\"})\n )\n and on (pod) (cnpg_pg_replication_in_recovery{pod=~\"($cnpgClusterList)-[0-9]+\"} == 0)\n)\n" }, "version": "v0" }, "refId": "query-1" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Client connections open on the primary, by database and role, beside the ceiling an ordinary role is refused at.**\n\n**Nominal:** Materialize's role holding nearly all of them, well under the ceiling.\n\n**Degraded:** Connections near the ceiling. If they are Materialize's, its pools have grown with the number of replicas; if they are another database's, something else is sharing the instance and Materialize is competing with it.\n\nThe primary only, because the ceiling is per instance and Materialize connects only to the primary. Materialize's own count of the connections it holds is on the Connections tab, and should match its line here.", "id": 1038, "links": [], "title": "Primary Connections by Database and Role", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "decimals": 0, "min": 0, "noValue": "The selected CNPG cluster has not reported this. Check the CNPG Cluster picker.", "unit": "short" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "cnpg-connections-by-state": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (state) (cnpg_backends_total{pod=~\"($cnpgClusterList)-[0-9]+\"})\n", "legendFormat": "{{state}}", "qryType": 1, "query": "sum by (state) (cnpg_backends_total{pod=~\"($cnpgClusterList)-[0-9]+\"})\n" }, "version": "v0" }, "refId": "query-0" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(cnpg_backends_waiting_total{pod=~\"($cnpgClusterList)-[0-9]+\"})\n", "legendFormat": "waiting on a lock", "qryType": 1, "query": "sum(cnpg_backends_waiting_total{pod=~\"($cnpgClusterList)-[0-9]+\"})\n" }, "version": "v0" }, "refId": "query-1" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Client connections by what they are doing — running a statement (`active`), waiting for the next one (`idle`), or holding a transaction open between statements (`idle in transaction`) — and how many are waiting on a lock.**\n\n**Nominal:** Mostly `idle`, and nothing waiting on a lock. A connection pool keeps connections open between uses.\n\n**Degraded:** `idle in transaction` that does not drain is a client holding a transaction open, which blocks vacuum and can hold locks. It is a different fault from too many connections, and the fix is in the client that opened it. Sessions waiting on locks are contention: persist's commits are conditional updates to a handful of rows, so a client locking those rows stalls Materialize's writes.\n\nEvery instance, replicas included.", "id": 1039, "links": [], "title": "Connections by State", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "decimals": 0, "min": 0, "noValue": "The selected CNPG cluster has not reported this. Check the CNPG Cluster picker.", "unit": "short" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "cnpg-connections-used": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max(\n sum by (pod) (cnpg_backends_total{pod=~\"($cnpgClusterList)-[0-9]+\"})\n / on (pod) (\n max by (pod) (cnpg_pg_settings_setting{pod=~\"($cnpgClusterList)-[0-9]+\", name=\"max_connections\"})\n - on (pod) sum by (pod) (cnpg_pg_settings_setting{pod=~\"($cnpgClusterList)-[0-9]+\", name=~\"superuser_reserved_connections|reserved_connections\"})\n )\n)\n", "legendFormat": "used", "qryType": 1, "query": "max(\n sum by (pod) (cnpg_backends_total{pod=~\"($cnpgClusterList)-[0-9]+\"})\n / on (pod) (\n max by (pod) (cnpg_pg_settings_setting{pod=~\"($cnpgClusterList)-[0-9]+\", name=\"max_connections\"})\n - on (pod) sum by (pod) (cnpg_pg_settings_setting{pod=~\"($cnpgClusterList)-[0-9]+\", name=~\"superuser_reserved_connections|reserved_connections\"})\n )\n)\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Client connections open on the primary, as a fraction of the connections an ordinary role may open.**\n\n**Nominal:** Well below 1. Materialize holds a pool of connections per process, so the count rises with every cluster replica added.\n\n**Degraded:** Above 80%, the next cluster scale-up or rollout — which briefly runs two generations of every process — can exhaust it.\n\n**Unhealthy:** At 1, PostgreSQL refuses new connections with `remaining connection slots are reserved`, and every Materialize process reports connection errors at once.\n\nThe ceiling is `max_connections` less `superuser_reserved_connections` and `reserved_connections`, which is where an ordinary role is refused. Connections from every database on the instance count against it, not only Materialize's.", "id": 1033, "links": [], "title": "Connections Used", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "max": 1, "min": 0, "noValue": "The selected CNPG cluster has not reported this. Check the CNPG Cluster picker.", "thresholds": { "mode": "absolute", "steps": [ { "color": "#A2F49B", "value": null }, { "color": "#F19903", "value": 0.8 }, { "color": "#AB0003", "value": 0.95 } ] }, "unit": "percentunit" }, "overrides": [] }, "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [], "values": false }, "showPercentChange": false, "textMode": "value", "wideLayout": true } }, "version": "" } } }, "cnpg-database-size": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (datname) (cnpg_pg_database_size_bytes{pod=~\"($cnpgClusterList)-[0-9]+\"})\n", "legendFormat": "{{datname}}", "qryType": 1, "query": "max by (datname) (cnpg_pg_database_size_bytes{pod=~\"($cnpgClusterList)-[0-9]+\"})\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**The size of each database on the instance.**\n\n**Nominal:** Materialize's database small and roughly flat: persist keeps only current state there, and the data itself is in object storage.\n\n**Degraded:** Materialize's database growing without bound means truncation has stopped. Another database growing is a neighbour consuming the volume — a full disk looks the same either way from outside the database, and this is what tells them apart.", "id": 1044, "links": [], "title": "Size by Database", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "The selected CNPG cluster has not reported this. Check the CNPG Cluster picker.", "unit": "bytes" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "cnpg-not-serving": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "(count(cnpg_collector_up{pod=~\"($cnpgClusterList)-[0-9]+\"} == 0) or vector(0))\n+ (count(up{pod=~\"($cnpgClusterList)-[0-9]+\", container=\"postgres\"} == 0) or vector(0))\n", "legendFormat": "instances", "qryType": 1, "query": "(count(cnpg_collector_up{pod=~\"($cnpgClusterList)-[0-9]+\"} == 0) or vector(0))\n+ (count(up{pod=~\"($cnpgClusterList)-[0-9]+\", container=\"postgres\"} == 0) or vector(0))\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**PostgreSQL instances in the selected CNPG clusters that are not serving: the exporter cannot reach PostgreSQL, or cannot be scraped.**\n\n**Nominal:** Zero.\n\n**Unhealthy:** One or more instances is down. If it is the primary, Materialize's commits fail until CNPG promotes a replica; with a single instance, until the pod comes back.\n\nCounts both ways an instance can go dark: CNPG's own `cnpg_collector_up` reading 0, and the pod's metrics endpoint not answering at all.", "id": 1031, "links": [], "title": "Instances Not Serving", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "decimals": 0, "min": 0, "noValue": "The selected CNPG cluster has not reported this. Check the CNPG Cluster picker.", "thresholds": { "mode": "absolute", "steps": [ { "color": "#A2F49B", "value": null }, { "color": "#FEDA8B", "value": 1 }, { "color": "#FDB366", "value": 1 }, { "color": "#F67E4B", "value": 1 }, { "color": "#DD3D2D", "value": 1 }, { "color": "#A50026", "value": 1 } ] }, "unit": "short" }, "overrides": [] }, "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [], "values": false }, "showPercentChange": false, "textMode": "value", "wideLayout": true } }, "version": "" } } }, "cnpg-oldest-transaction": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max(cnpg_backends_max_tx_duration_seconds{pod=~\"($cnpgClusterList)-[0-9]+\"})\n", "legendFormat": "oldest", "qryType": 1, "query": "max(cnpg_backends_max_tx_duration_seconds{pod=~\"($cnpgClusterList)-[0-9]+\"})\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**How long the oldest open transaction on any instance has been running.**\n\n**Nominal:** Under a second. Persist's commits are single, short transactions.\n\n**Degraded:** Minutes. A long-running transaction holds back vacuum on every table, so the consensus table grows and every commit slows. It is usually something other than Materialize: a migration, a backup tool, or a session left `idle in transaction`.\n\n_Oldest Transaction by Database_ on this tab names the database and state it is in.", "id": 1034, "links": [], "title": "Oldest Open Transaction", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "The selected CNPG cluster has not reported this. Check the CNPG Cluster picker.", "thresholds": { "mode": "absolute", "steps": [ { "color": "#A2F49B", "value": null }, { "color": "#F19903", "value": 60 }, { "color": "#AB0003", "value": 900 } ] }, "unit": "s" }, "overrides": [] }, "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [], "values": false }, "showPercentChange": false, "textMode": "value", "wideLayout": true } }, "version": "" } } }, "cnpg-primary": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (pod) (cnpg_pg_replication_in_recovery{pod=~\"($cnpgClusterList)-[0-9]+\"} == 0)\n", "instant": true, "legendFormat": "{{pod}}", "qryType": 1, "query": "max by (pod) (cnpg_pg_replication_in_recovery{pod=~\"($cnpgClusterList)-[0-9]+\"} == 0)\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [ { "group": "labelsToFields", "kind": "Transformation", "spec": { "options": {} } } ] } }, "description": "**The instance currently serving as primary — the only one that accepts Materialize's writes.**\n\n**Nominal:** One instance, which stays the same between maintenance windows.\n\n**Degraded:** A change of name is a switchover or a failover. Materialize reconnects on its own; expect a burst of connection errors on the other tabs at the same moment.\n\n**Unhealthy:** No instance at all means there is no primary, and nothing can commit.", "id": 1032, "links": [], "title": "Primary", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "fixedColor": "#009988", "mode": "shades" }, "noValue": "No instance reports itself as primary." }, "overrides": [] }, "options": { "colorMode": "none", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [], "fields": "/^pod$/", "values": false }, "showPercentChange": false, "text": { "valueSize": 20 }, "textMode": "value", "wideLayout": true } }, "version": "" } } }, "cnpg-replication-lag": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (pod) (cnpg_pg_replication_lag{pod=~\"($cnpgClusterList)-[0-9]+\"})\n", "legendFormat": "{{pod}}", "qryType": 1, "query": "max by (pod) (cnpg_pg_replication_lag{pod=~\"($cnpgClusterList)-[0-9]+\"})\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**How far each replica is behind the primary, in seconds.**\n\n**Nominal:** Under a second. A cluster with one instance has no replicas and reads zero.\n\n**Degraded:** A replica that falls behind is a replica that would lose recent commits if it were promoted in a failover, or take longer to promote.\n\nMaterialize reads and writes only the primary, so replica lag never slows it down directly.", "id": 1047, "links": [], "title": "Replication Lag", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "The selected CNPG cluster has not reported this. Check the CNPG Cluster picker.", "unit": "s" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "cnpg-rows": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (datname) (rate(cnpg_pg_stat_database_tup_inserted{pod=~\"($cnpgClusterList)-[0-9]+\", datname!=\"\"}[$__rate_interval])) > 0\n", "legendFormat": "{{datname}} inserted", "qryType": 1, "query": "sum by (datname) (rate(cnpg_pg_stat_database_tup_inserted{pod=~\"($cnpgClusterList)-[0-9]+\", datname!=\"\"}[$__rate_interval])) > 0\n" }, "version": "v0" }, "refId": "query-0" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (datname) (rate(cnpg_pg_stat_database_tup_updated{pod=~\"($cnpgClusterList)-[0-9]+\", datname!=\"\"}[$__rate_interval])) > 0\n", "legendFormat": "{{datname}} updated", "qryType": 1, "query": "sum by (datname) (rate(cnpg_pg_stat_database_tup_updated{pod=~\"($cnpgClusterList)-[0-9]+\", datname!=\"\"}[$__rate_interval])) > 0\n" }, "version": "v0" }, "refId": "query-1" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (datname) (rate(cnpg_pg_stat_database_tup_deleted{pod=~\"($cnpgClusterList)-[0-9]+\", datname!=\"\"}[$__rate_interval])) > 0\n", "legendFormat": "{{datname}} deleted", "qryType": 1, "query": "sum by (datname) (rate(cnpg_pg_stat_database_tup_deleted{pod=~\"($cnpgClusterList)-[0-9]+\", datname!=\"\"}[$__rate_interval])) > 0\n" }, "version": "v0" }, "refId": "query-2" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Rows inserted, updated and deleted per second, by database.**\n\n**Nominal:** A steady churn on Materialize's database. Every commit rewrites a small state record and truncation deletes the versions behind it, so deletes track inserts.\n\n**Degraded:** Inserts with no matching deletes means truncation has stopped and the consensus table is growing; _State and Cleanup_ shows it from Materialize's side. Heavy churn on another database is a neighbour competing for the same disk.", "id": 1042, "links": [], "title": "Rows Written by Database", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "The selected CNPG cluster has not reported this. Check the CNPG Cluster picker.", "unit": "rowsps" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "cnpg-transactions": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (datname) (rate(cnpg_pg_stat_database_xact_commit{pod=~\"($cnpgClusterList)-[0-9]+\", datname!=\"\"}[$__rate_interval])) > 0\n", "legendFormat": "{{datname}} committed", "qryType": 1, "query": "sum by (datname) (rate(cnpg_pg_stat_database_xact_commit{pod=~\"($cnpgClusterList)-[0-9]+\", datname!=\"\"}[$__rate_interval])) > 0\n" }, "version": "v0" }, "refId": "query-0" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (datname) (rate(cnpg_pg_stat_database_xact_rollback{pod=~\"($cnpgClusterList)-[0-9]+\", datname!=\"\"}[$__rate_interval])) > 0\n", "legendFormat": "{{datname}} rolled back", "qryType": 1, "query": "sum by (datname) (rate(cnpg_pg_stat_database_xact_rollback{pod=~\"($cnpgClusterList)-[0-9]+\", datname!=\"\"}[$__rate_interval])) > 0\n" }, "version": "v0" }, "refId": "query-1" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Transactions committed and rolled back per second, by database.**\n\n**Nominal:** Materialize's database committing steadily — tens per second on an idle environment — and few rollbacks.\n\n**Degraded:** A commit rate well below Materialize's own count of commits means its calls are not arriving. Rising rollbacks on Materialize's database are usually commits that lost a race, which persist expects and retries.\n\nCompare with _Commits_ on the Overview tab, which is the same work counted by Materialize.", "id": 1041, "links": [], "title": "Transactions by Database", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "The selected CNPG cluster has not reported this. Check the CNPG Cluster picker.", "unit": "ops" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "cnpg-transactions-held": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (datname, state) (cnpg_backends_max_tx_duration_seconds{pod=~\"($cnpgClusterList)-[0-9]+\"})\n", "legendFormat": "{{datname}} ({{state}})", "qryType": 1, "query": "max by (datname, state) (cnpg_backends_max_tx_duration_seconds{pod=~\"($cnpgClusterList)-[0-9]+\"})\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**The oldest open transaction, by database and connection state.**\n\n**Nominal:** Under a second.\n\n**Degraded:** A transaction that keeps aging holds back vacuum on every table, so the consensus table grows and every commit slows.", "id": 1040, "links": [], "title": "Oldest Transaction by Database", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "The selected CNPG cluster has not reported this. Check the CNPG Cluster picker.", "unit": "s" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "cnpg-version": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (full) (cnpg_collector_postgres_version{pod=~\"($cnpgClusterList)-[0-9]+\"})\n", "instant": true, "legendFormat": "{{full}}", "qryType": 1, "query": "max by (full) (cnpg_collector_postgres_version{pod=~\"($cnpgClusterList)-[0-9]+\"})\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [ { "group": "labelsToFields", "kind": "Transformation", "spec": { "options": {} } } ] } }, "description": "**The PostgreSQL version each selected CNPG cluster runs.**\n\n**Nominal:** A supported major version, the same on every instance.\n\nAn out-of-date dependency has been the cause of an incident before, and this is the one place it is visible without a database login. Two versions at once means a rolling upgrade is under way.", "id": 1037, "links": [], "title": "PostgreSQL Version", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "fixedColor": "#009988", "mode": "shades" }, "noValue": "The selected CNPG cluster has not reported this. Check the CNPG Cluster picker." }, "overrides": [] }, "options": { "colorMode": "none", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [], "fields": "/^full$/", "values": false }, "showPercentChange": false, "text": { "valueSize": 20 }, "textMode": "value", "wideLayout": true } }, "version": "" } } }, "cnpg-wal": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (pod) (cnpg_collector_pg_wal{pod=~\"($cnpgClusterList)-[0-9]+\", value=\"size\"})\n", "legendFormat": "{{pod}} on disk", "qryType": 1, "query": "max by (pod) (cnpg_collector_pg_wal{pod=~\"($cnpgClusterList)-[0-9]+\", value=\"size\"})\n" }, "version": "v0" }, "refId": "query-0" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (pod) (cnpg_collector_pg_wal_archive_status{pod=~\"($cnpgClusterList)-[0-9]+\", value=\"ready\"})\n* on (pod) (\n max by (pod) (cnpg_collector_pg_wal{pod=~\"($cnpgClusterList)-[0-9]+\", value=\"size\"})\n / on (pod) max by (pod) (cnpg_collector_pg_wal{pod=~\"($cnpgClusterList)-[0-9]+\", value=\"count\"})\n)\n", "legendFormat": "{{pod}} waiting to archive", "qryType": 1, "query": "max by (pod) (cnpg_collector_pg_wal_archive_status{pod=~\"($cnpgClusterList)-[0-9]+\", value=\"ready\"})\n* on (pod) (\n max by (pod) (cnpg_collector_pg_wal{pod=~\"($cnpgClusterList)-[0-9]+\", value=\"size\"})\n / on (pod) max by (pod) (cnpg_collector_pg_wal{pod=~\"($cnpgClusterList)-[0-9]+\", value=\"count\"})\n)\n" }, "version": "v0" }, "refId": "query-1" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Write-ahead log kept on each instance's disk, and the part of it still waiting to be archived.**\n\n**Nominal:** WAL size steady, around `min_wal_size` to `max_wal_size`, and nothing waiting to be archived for long.\n\n**Degraded:** WAL that grows without falling back is being retained: by an archive command that keeps failing, by a replica that has stopped consuming it, or by an abandoned replication slot. It fills the volume, and a full volume stops PostgreSQL.\n\nSegments waiting to be archived exist only where backups or WAL archiving are configured.", "id": 1045, "links": [], "title": "Write-Ahead Log", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "The selected CNPG cluster has not reported this. Check the CNPG Cluster picker.", "unit": "bytes" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "cnpg-xid-age": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (datname) (cnpg_pg_database_xid_age{pod=~\"($cnpgClusterList)-[0-9]+\"}) / 2147483648\n", "legendFormat": "{{datname}}", "qryType": 1, "query": "max by (datname) (cnpg_pg_database_xid_age{pod=~\"($cnpgClusterList)-[0-9]+\"}) / 2147483648\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Transaction-ID space consumed by each database since it was last fully frozen, as a fraction of the wraparound limit.**\n\n**Nominal:** Below 10%, and saw-toothed: each autovacuum freeze pass brings it down.\n\n**Degraded:** A line that only rises means vacuum cannot freeze that database — usually a long-running transaction or an abandoned replication slot holding it back.", "id": 1046, "links": [], "title": "Transaction IDs Used by Database", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "The selected CNPG cluster has not reported this. Check the CNPG Cluster picker.", "thresholds": { "mode": "absolute", "steps": [ { "color": "#A2F49B", "value": null }, { "color": "#F19903", "value": 0.25 }, { "color": "#AB0003", "value": 0.5 } ] }, "unit": "percentunit" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "cnpg-xid-used": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max(cnpg_pg_database_xid_age{pod=~\"($cnpgClusterList)-[0-9]+\"}) / 2147483648\n", "legendFormat": "used", "qryType": 1, "query": "max(cnpg_pg_database_xid_age{pod=~\"($cnpgClusterList)-[0-9]+\"}) / 2147483648\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**How much of the transaction-ID space the most at-risk database has consumed since it was last fully frozen by vacuum.**\n\n**Nominal:** Below 10%. Autovacuum freezes old rows long before this matters.\n\n**Degraded:** Above 25% means freezing is falling behind. Persist commits constantly, so a small instance whose autovacuum is blocked reaches this faster than most workloads.\n\n**Unhealthy:** Past 50%, treat it as urgent: the remaining headroom is what a stalled vacuum has left before wraparound. At 100%, PostgreSQL stops accepting writes to protect itself, and Materialize stops with it; its own warnings start only once fewer than 40 million IDs remain, at about 98%.\n\n`age(datfrozenxid)` against the 2^31 wraparound limit, the same measure Cloud SQL publishes as transaction-ID utilization.", "id": 1035, "links": [], "title": "Transaction IDs Used", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "max": 1, "min": 0, "noValue": "The selected CNPG cluster has not reported this. Check the CNPG Cluster picker.", "thresholds": { "mode": "absolute", "steps": [ { "color": "#A2F49B", "value": null }, { "color": "#F19903", "value": 0.25 }, { "color": "#AB0003", "value": 0.5 } ] }, "unit": "percentunit" }, "overrides": [] }, "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [], "values": false }, "showPercentChange": false, "textMode": "value", "wideLayout": true } }, "version": "" } } }, "database-none-detected": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [], "queryOptions": {}, "transformations": [] } }, "description": "Why this tab has no database rows on it.", "id": 1067, "links": [], "title": "No Database Metrics", "vizConfig": { "group": "text", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": {}, "overrides": [] }, "options": { "code": { "language": "plaintext", "showLineNumbers": false, "showMiniMap": false }, "content": "**The metadata database is not reporting its own metrics.**\n\nThis tab shows what the database says about itself: connections against its ceiling, transactions, the size of each database on the instance, transaction-ID age and vacuum. It renders for **any PostgreSQL** the chart's postgres-exporter watches, and for **CloudNativePG (CNPG)**, which publishes these from every instance with nothing extra to deploy.\n\n- **On any PostgreSQL**, managed or not, set `tags.postgres-exporter: true` and list the database under `postgres-exporter.targets`, with a `postgres-exporter` Secret holding its credentials. The external dependencies guide has the grant. If the exporter runs and this note remains, check `up{job=\"postgres-exporter\"}`.\n- **On CNPG**, create a `PodMonitor` for the cluster, selecting `cnpg.io/cluster: ` and the `metrics` port. CNPG's documentation has the manifest; its `spec.monitoring.enablePodMonitor` is deprecated. If one exists and this note remains, check `up{container=\"postgres\"}` — a scrape that fails outright, such as one blocked by a NetworkPolicy, reports nothing else.\n- **On a managed database**, what the cloud provider sees of the machine under it — CPU, memory, disk and IOPS — is on the **Infrastructure Cloud Provider** dashboard, beside the exporter's view here.\n\nThe other tabs here need none of this: they are Materialize's own measurement of the database, and work on every flavor.", "mode": "markdown" } }, "version": "" } } }, "ops-by-type": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (op) (\n rate(mz_persist_external_started_count{materialize_cloud_organization_name=~\"$environmentNameList\", op=~\"consensus_.*\"}[$__rate_interval])\n)\n", "legendFormat": "{{op}}", "qryType": 1, "query": "sum by (op) (\n rate(mz_persist_external_started_count{materialize_cloud_organization_name=~\"$environmentNameList\", op=~\"consensus_.*\"}[$__rate_interval])\n)\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Metadata database calls per second, by operation.**\n\n**Nominal:** `consensus_cas` — commits — is nearly all of it. `consensus_scan` reads an object's recent history and `consensus_truncate` deletes history nothing needs any more, both at a few per second.\n\n**Degraded:** A step change in commits without a matching change in the number of objects usually means something is being rewritten in a loop — an object that cannot settle.", "id": 1012, "links": [], "title": "Operations by Type", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "No metrics from this environment. Check the environment picker, and that Materialize is running.", "unit": "ops" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "ops-bytes": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (op) (\n rate(mz_persist_external_bytes_count{materialize_cloud_organization_name=~\"$environmentNameList\", op=~\"consensus_.*\"}[$__rate_interval])\n)\n", "legendFormat": "{{op}}", "qryType": 1, "query": "sum by (op) (\n rate(mz_persist_external_bytes_count{materialize_cloud_organization_name=~\"$environmentNameList\", op=~\"consensus_.*\"}[$__rate_interval])\n)\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Bytes per second exchanged with the metadata database, by operation.**\n\n**Nominal:** Small — the state record of an object is kilobytes, not data. A commit writes one; a scan reads several.\n\n**Degraded:** Rising commit bytes with a steady commit rate means the state records themselves are growing, which the _State and Cleanup_ tab explains.", "id": 1014, "links": [], "title": "Data Exchanged by Operation", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "No metrics from this environment. Check the environment picker, and that Materialize is running.", "unit": "Bps" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "ops-commit-latency-by-process": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "(\n(\nhistogram_quantile(0.99,\n sum by (le, app, cluster_environmentd_materialize_cloud_cluster_id) (\n rate(mz_persist_external_op_latency_bucket{materialize_cloud_organization_name=~\"$environmentNameList\", op=\"consensus_cas\"}[$__rate_interval])\n )\n)\n\n)\n* on (cluster_environmentd_materialize_cloud_cluster_id) group_left(cluster_name)\nmax by (cluster_environmentd_materialize_cloud_cluster_id, cluster_name) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"cluster_environmentd_materialize_cloud_cluster_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\nhistogram_quantile(0.99,\n sum by (le, app, cluster_environmentd_materialize_cloud_cluster_id) (\n rate(mz_persist_external_op_latency_bucket{materialize_cloud_organization_name=~\"$environmentNameList\", op=\"consensus_cas\"}[$__rate_interval])\n )\n)\n\n)\nunless on (cluster_environmentd_materialize_cloud_cluster_id) (\ngroup by (cluster_environmentd_materialize_cloud_cluster_id) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"cluster_environmentd_materialize_cloud_cluster_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n),\n\"cluster_name\", \"$1\", \"cluster_environmentd_materialize_cloud_cluster_id\", \"(.*)\"\n)", "legendFormat": "{{app}} {{cluster_name}}", "qryType": 1, "query": "(\n(\nhistogram_quantile(0.99,\n sum by (le, app, cluster_environmentd_materialize_cloud_cluster_id) (\n rate(mz_persist_external_op_latency_bucket{materialize_cloud_organization_name=~\"$environmentNameList\", op=\"consensus_cas\"}[$__rate_interval])\n )\n)\n\n)\n* on (cluster_environmentd_materialize_cloud_cluster_id) group_left(cluster_name)\nmax by (cluster_environmentd_materialize_cloud_cluster_id, cluster_name) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"cluster_environmentd_materialize_cloud_cluster_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\nhistogram_quantile(0.99,\n sum by (le, app, cluster_environmentd_materialize_cloud_cluster_id) (\n rate(mz_persist_external_op_latency_bucket{materialize_cloud_organization_name=~\"$environmentNameList\", op=\"consensus_cas\"}[$__rate_interval])\n )\n)\n\n)\nunless on (cluster_environmentd_materialize_cloud_cluster_id) (\ngroup by (cluster_environmentd_materialize_cloud_cluster_id) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"cluster_environmentd_materialize_cloud_cluster_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n),\n\"cluster_name\", \"$1\", \"cluster_environmentd_materialize_cloud_cluster_id\", \"(.*)\"\n)" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Slowest 1% of commits, by the process making them.**\n\n**Nominal:** Every process close to the others.\n\n**Degraded:** One process well above the rest is local to it — its network path, or its own pool running out of connections (compare _Connections -> Waiting for a Connection by Process_). All of them rising together is the database.", "id": 1016, "links": [], "title": "Commit Latency by Process (p99)", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "No metrics from this environment. Check the environment picker, and that Materialize is running.", "unit": "s" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "ops-commits-by-process": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "(\n(\nsum by (app, cluster_environmentd_materialize_cloud_cluster_id) (\n rate(mz_persist_external_started_count{materialize_cloud_organization_name=~\"$environmentNameList\", op=\"consensus_cas\"}[$__rate_interval])\n)\n\n)\n* on (cluster_environmentd_materialize_cloud_cluster_id) group_left(cluster_name)\nmax by (cluster_environmentd_materialize_cloud_cluster_id, cluster_name) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"cluster_environmentd_materialize_cloud_cluster_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\nsum by (app, cluster_environmentd_materialize_cloud_cluster_id) (\n rate(mz_persist_external_started_count{materialize_cloud_organization_name=~\"$environmentNameList\", op=\"consensus_cas\"}[$__rate_interval])\n)\n\n)\nunless on (cluster_environmentd_materialize_cloud_cluster_id) (\ngroup by (cluster_environmentd_materialize_cloud_cluster_id) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"cluster_environmentd_materialize_cloud_cluster_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n),\n\"cluster_name\", \"$1\", \"cluster_environmentd_materialize_cloud_cluster_id\", \"(.*)\"\n)", "legendFormat": "{{app}} {{cluster_name}}", "qryType": 1, "query": "(\n(\nsum by (app, cluster_environmentd_materialize_cloud_cluster_id) (\n rate(mz_persist_external_started_count{materialize_cloud_organization_name=~\"$environmentNameList\", op=\"consensus_cas\"}[$__rate_interval])\n)\n\n)\n* on (cluster_environmentd_materialize_cloud_cluster_id) group_left(cluster_name)\nmax by (cluster_environmentd_materialize_cloud_cluster_id, cluster_name) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"cluster_environmentd_materialize_cloud_cluster_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\nsum by (app, cluster_environmentd_materialize_cloud_cluster_id) (\n rate(mz_persist_external_started_count{materialize_cloud_organization_name=~\"$environmentNameList\", op=\"consensus_cas\"}[$__rate_interval])\n)\n\n)\nunless on (cluster_environmentd_materialize_cloud_cluster_id) (\ngroup by (cluster_environmentd_materialize_cloud_cluster_id) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"cluster_environmentd_materialize_cloud_cluster_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n),\n\"cluster_name\", \"$1\", \"cluster_environmentd_materialize_cloud_cluster_id\", \"(.*)\"\n)" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Commits per second, by the process making them.**\n\n**Nominal:** environmentd and the system cluster (`mz_catalog_server`) commit continuously even with no user workload. User clusters add commits in proportion to the sources, tables and materialized views they maintain.\n\n**Degraded:** The process at the top is the one to look at when the database is busier than expected.", "id": 1013, "links": [], "title": "Commits by Process", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "No metrics from this environment. Check the environment picker, and that Materialize is running.", "unit": "ops" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "ops-conflicts": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "(\n(\nsum by (app, cluster_environmentd_materialize_cloud_cluster_id) (\n rate(mz_persist_cmd_cas_mismatch_count{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval])\n)\n\n)\n* on (cluster_environmentd_materialize_cloud_cluster_id) group_left(cluster_name)\nmax by (cluster_environmentd_materialize_cloud_cluster_id, cluster_name) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"cluster_environmentd_materialize_cloud_cluster_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\nsum by (app, cluster_environmentd_materialize_cloud_cluster_id) (\n rate(mz_persist_cmd_cas_mismatch_count{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval])\n)\n\n)\nunless on (cluster_environmentd_materialize_cloud_cluster_id) (\ngroup by (cluster_environmentd_materialize_cloud_cluster_id) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"cluster_environmentd_materialize_cloud_cluster_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n),\n\"cluster_name\", \"$1\", \"cluster_environmentd_materialize_cloud_cluster_id\", \"(.*)\"\n)", "legendFormat": "{{app}} {{cluster_name}}", "qryType": 1, "query": "(\n(\nsum by (app, cluster_environmentd_materialize_cloud_cluster_id) (\n rate(mz_persist_cmd_cas_mismatch_count{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval])\n)\n\n)\n* on (cluster_environmentd_materialize_cloud_cluster_id) group_left(cluster_name)\nmax by (cluster_environmentd_materialize_cloud_cluster_id, cluster_name) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"cluster_environmentd_materialize_cloud_cluster_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\nsum by (app, cluster_environmentd_materialize_cloud_cluster_id) (\n rate(mz_persist_cmd_cas_mismatch_count{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval])\n)\n\n)\nunless on (cluster_environmentd_materialize_cloud_cluster_id) (\ngroup by (cluster_environmentd_materialize_cloud_cluster_id) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"cluster_environmentd_materialize_cloud_cluster_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n),\n\"cluster_name\", \"$1\", \"cluster_environmentd_materialize_cloud_cluster_id\", \"(.*)\"\n)" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Commits that lost a race with another writer and were retried, per second, by process.**\n\n**Nominal:** A trickle. Several processes write the same objects, so the odd conflict is expected and costs one retry.\n\n**Degraded:** A burst during an upgrade is the two generations overlapping, and stops when the old one is retired. A sustained high rate outside a rollout means two processes are fighting over the same objects — usually a replica that should have been shut down and was not.", "id": 1017, "links": [], "title": "Write Conflicts by Process", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "No metrics from this environment. Check the environment picker, and that Materialize is running.", "unit": "ops" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "ops-mean-latency": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "(\n sum by (op) (\n rate(mz_persist_external_seconds{materialize_cloud_organization_name=~\"$environmentNameList\", op=~\"consensus_.*\"}[$__rate_interval])\n )\n /\n sum by (op) (\n rate(mz_persist_external_started_count{materialize_cloud_organization_name=~\"$environmentNameList\", op=~\"consensus_.*\"}[$__rate_interval])\n )\n) >= 0\n", "legendFormat": "{{op}}", "qryType": 1, "query": "(\n sum by (op) (\n rate(mz_persist_external_seconds{materialize_cloud_organization_name=~\"$environmentNameList\", op=~\"consensus_.*\"}[$__rate_interval])\n )\n /\n sum by (op) (\n rate(mz_persist_external_started_count{materialize_cloud_organization_name=~\"$environmentNameList\", op=~\"consensus_.*\"}[$__rate_interval])\n )\n) >= 0\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Average time per metadata database call, by operation.**\n\n**Nominal:** Milliseconds, and similar across operations.\n\n**Degraded:** Truncate alone slowing down is the database struggling to delete — on PostgreSQL, usually a sign that autovacuum is behind on the table persist writes to.\n\nA mean hides the tail. The commit has a real percentile on _Commit Latency by Process (p99)_.", "id": 1015, "links": [], "title": "Mean Latency by Operation", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "No metrics from this environment. Check the environment picker, and that Materialize is running.", "unit": "s" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "ops-oracle-latency": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "(\n sum by (op) (\n rate(mz_ts_oracle_seconds{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval])\n )\n /\n sum by (op) (\n rate(mz_ts_oracle_started_count{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval])\n )\n) >= 0\n", "legendFormat": "{{op}}", "qryType": 1, "query": "(\n sum by (op) (\n rate(mz_ts_oracle_seconds{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval])\n )\n /\n sum by (op) (\n rate(mz_ts_oracle_started_count{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval])\n )\n) >= 0\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Average time per timestamp oracle call, by operation.**\n\n**Nominal:** A millisecond or two.\n\n**Degraded:** Every `SELECT` waits for `read_ts`, so this adds directly to query latency. A rise here beside a rise in commit latency is the database; a rise here alone points at the oracle's own single connection pool, on _Connections_.", "id": 1020, "links": [], "title": "Timestamp Oracle Latency (mean)", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "No timestamp oracle metrics. The oracle runs in environmentd, so check that it is up.", "unit": "s" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "ops-oracle-ops": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (op) (\n rate(mz_ts_oracle_started_count{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval])\n)\n", "legendFormat": "{{op}}", "qryType": 1, "query": "sum by (op) (\n rate(mz_ts_oracle_started_count{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval])\n)\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Timestamp oracle calls per second, by operation.**\n\n**Nominal:** A few per second. `read_ts` is asked for by queries, `write_ts` and `apply_write` by writes.\n\nThe timestamp oracle keeps the timestamps every query and write is assigned, in the metadata database. It runs in environmentd only, so a failure here stops queries and writes environment-wide rather than on one cluster.", "id": 1019, "links": [], "title": "Timestamp Oracle Operations", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "No timestamp oracle metrics. The oracle runs in environmentd, so check that it is up.", "unit": "ops" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "ops-retries": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (op) (\n rate(mz_persist_retry_retries_count{materialize_cloud_organization_name=~\"$environmentNameList\", op=~\"fetch_state::scan|apply_unbatched_cmd::cas|gc::truncate|maybe_init::cas|consensus::open\"}[$__rate_interval])\n)\n", "legendFormat": "{{op}}", "qryType": 1, "query": "sum by (op) (\n rate(mz_persist_retry_retries_count{materialize_cloud_organization_name=~\"$environmentNameList\", op=~\"fetch_state::scan|apply_unbatched_cmd::cas|gc::truncate|maybe_init::cas|consensus::open\"}[$__rate_interval])\n)\n" }, "version": "v0" }, "refId": "query-0" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (op) (\n rate(mz_ts_oracle_retry_retries_count{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval])\n)\n", "legendFormat": "timestamp oracle {{op}}", "qryType": 1, "query": "sum by (op) (\n rate(mz_ts_oracle_retry_retries_count{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval])\n)\n" }, "version": "v0" }, "refId": "query-1" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Metadata database calls retried after a failure, per second, by the work that was retrying.**\n\n**Nominal:** Nothing drawn above zero.\n\n**Degraded:** A retry is a failure that has not been given up on yet. Retries across every operation at once mean the database is unreachable or refusing work; retries on truncation alone are cleanup falling behind, which is not urgent until _State and Cleanup_ shows the backlog growing.\n\nExcludes persist's polling loops, which are named like retries but are waiting for new data rather than recovering from an error.", "id": 1018, "links": [], "title": "Retried Operations", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "No metrics from this environment. Check the environment picker, and that Materialize is running.", "unit": "ops" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "overview-collections": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "count(group by (shard) (mz_persist_shard_usage_current_state_batches_bytes{materialize_cloud_organization_name=~\"$environmentNameList\"}))\n", "legendFormat": "objects", "qryType": 1, "query": "count(group by (shard) (mz_persist_shard_usage_current_state_batches_bytes{materialize_cloud_organization_name=~\"$environmentNameList\"}))\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Durable objects in this environment — each one a stream of commits to the metadata database.**\n\n**Nominal:** Grows with what you create. The system catalog alone is around a hundred.\n\nMetadata database sizing guidance is stated in continuously updated objects: roughly 4,500 for the smallest recommended instance and 18,000 for the default. This count is an upper bound on that, since it also includes objects that rarely change.\n\nList them with `SELECT s.shard_id, o.name, o.type FROM mz_internal.mz_storage_shards s JOIN mz_objects o ON o.id = s.object_id`.", "id": 1006, "links": [], "title": "Durable Objects", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "fixedColor": "#0077BB", "mode": "shades" }, "decimals": 0, "min": 0, "noValue": "No metrics from this environment. Check the environment picker, and that Materialize is running.", "unit": "short" }, "overrides": [] }, "options": { "colorMode": "none", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [], "values": false }, "showPercentChange": false, "textMode": "value", "wideLayout": true } }, "version": "" } } }, "overview-commit-latency": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "histogram_quantile(0.50,\n sum by (le) (\n rate(mz_persist_external_op_latency_bucket{materialize_cloud_organization_name=~\"$environmentNameList\", op=\"consensus_cas\"}[$__rate_interval])\n )\n)\n", "legendFormat": "p50", "qryType": 1, "query": "histogram_quantile(0.50,\n sum by (le) (\n rate(mz_persist_external_op_latency_bucket{materialize_cloud_organization_name=~\"$environmentNameList\", op=\"consensus_cas\"}[$__rate_interval])\n )\n)\n" }, "version": "v0" }, "refId": "query-0" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "histogram_quantile(0.99,\n sum by (le) (\n rate(mz_persist_external_op_latency_bucket{materialize_cloud_organization_name=~\"$environmentNameList\", op=\"consensus_cas\"}[$__rate_interval])\n )\n)\n", "legendFormat": "p99", "qryType": 1, "query": "histogram_quantile(0.99,\n sum by (le) (\n rate(mz_persist_external_op_latency_bucket{materialize_cloud_organization_name=~\"$environmentNameList\", op=\"consensus_cas\"}[$__rate_interval])\n )\n)\n" }, "version": "v0" }, "refId": "query-1" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Commit latency to the metadata database: the median and the slowest 1%.**\n\n**Nominal:** Flat, with the p99 a small multiple of the median.\n\n**Degraded:** Both rising together is the database or the network getting slower for everyone. The p99 rising alone is contention — some commits are waiting on others, or on the pool.", "id": 1007, "links": [], "title": "Commit Latency", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "No metrics from this environment. Check the environment picker, and that Materialize is running.", "unit": "s" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "overview-commit-p99": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "histogram_quantile(0.99,\n sum by (le) (\n rate(mz_persist_external_op_latency_bucket{materialize_cloud_organization_name=~\"$environmentNameList\", op=\"consensus_cas\"}[$__rate_interval])\n )\n)\n", "legendFormat": "p99", "qryType": 1, "query": "histogram_quantile(0.99,\n sum by (le) (\n rate(mz_persist_external_op_latency_bucket{materialize_cloud_organization_name=~\"$environmentNameList\", op=\"consensus_cas\"}[$__rate_interval])\n )\n)\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**How long the slowest 1% of commits to the metadata database take.**\n\n**Nominal:** Single-digit to low tens of milliseconds when the database is in the same region as the cluster.\n\n**Degraded:** Every write in Materialize waits on a commit, so this is a floor under how fresh anything can be. A rise that tracks the database's CPU is the database saturating; one that does not is usually the network, or a process waiting for its own connection pool.\n\nMeasured by the client, so it includes the round trip and any wait for a pooled connection. Compare with _Round-Trip Time by Process_ to separate the two.", "id": 1002, "links": [], "title": "Commit Latency (p99)", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "No metrics from this environment. Check the environment picker, and that Materialize is running.", "thresholds": { "mode": "absolute", "steps": [ { "color": "#A2F49B", "value": null }, { "color": "#F19903", "value": 0.1 }, { "color": "#AB0003", "value": 1 } ] }, "unit": "s" }, "overrides": [] }, "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [], "values": false }, "showPercentChange": false, "textMode": "value", "wideLayout": true } }, "version": "" } } }, "overview-commits": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(rate(mz_persist_external_started_count{materialize_cloud_organization_name=~\"$environmentNameList\", op=\"consensus_cas\"}[$__rate_interval]))\n", "legendFormat": "commits/s", "qryType": 1, "query": "sum(rate(mz_persist_external_started_count{materialize_cloud_organization_name=~\"$environmentNameList\", op=\"consensus_cas\"}[$__rate_interval]))\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Commits to the metadata database per second — the load this environment puts on it.**\n\n**Nominal:** Steady. It scales with the number of durable objects that are being updated, not with data volume or query rate, and an idle environment already commits dozens of times a second.\n\nThis is the number the metadata database is sized against. See _Operations -> Load_ for where it comes from.", "id": 1005, "links": [], "title": "Commits", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "fixedColor": "#0077BB", "mode": "shades" }, "min": 0, "noValue": "No metrics from this environment. Check the environment picker, and that Materialize is running.", "unit": "ops" }, "overrides": [] }, "options": { "colorMode": "none", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [], "values": false }, "showPercentChange": false, "textMode": "value", "wideLayout": true } }, "version": "" } } }, "overview-connection-errors": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(rate(mz_persist_postgres_connpool_connection_errors{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval]))\n+ (sum(rate(mz_ts_oracle_postgres_connpool_connection_errors{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval])) or vector(0))\n", "legendFormat": "errors/s", "qryType": 1, "query": "sum(rate(mz_persist_postgres_connpool_connection_errors{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval]))\n+ (sum(rate(mz_ts_oracle_postgres_connpool_connection_errors{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval])) or vector(0))\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Failed attempts to open a connection to the metadata database, per second.**\n\n**Nominal:** Zero.\n\n**Degraded:** Materialize cannot connect. On a new install this is almost always configuration: the host, port, database name or credentials in the metadata backend URL, TLS settings, or a network path the cluster does not have. On a running install it is the database refusing connections — restarting, failing over, or at its `max_connections` limit.\n\nCounts both clients: persist's connection pools and the timestamp oracle's.", "id": 1001, "links": [], "title": "Connection Errors", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "No metrics from this environment. Check the environment picker, and that Materialize is running.", "thresholds": { "mode": "absolute", "steps": [ { "color": "#A2F49B", "value": null }, { "color": "#FEDA8B", "value": 0.01 }, { "color": "#FDB366", "value": 0.2575 }, { "color": "#F67E4B", "value": 0.505 }, { "color": "#DD3D2D", "value": 0.7525 }, { "color": "#A50026", "value": 1 } ] }, "unit": "cps" }, "overrides": [] }, "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [], "values": false }, "showPercentChange": false, "textMode": "value", "wideLayout": true } }, "version": "" } } }, "overview-connection-errors-by-process": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "(\n(\nsum by (app, cluster_environmentd_materialize_cloud_cluster_id) (\n rate(mz_persist_postgres_connpool_connection_errors{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval])\n)\n\n)\n* on (cluster_environmentd_materialize_cloud_cluster_id) group_left(cluster_name)\nmax by (cluster_environmentd_materialize_cloud_cluster_id, cluster_name) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"cluster_environmentd_materialize_cloud_cluster_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\nsum by (app, cluster_environmentd_materialize_cloud_cluster_id) (\n rate(mz_persist_postgres_connpool_connection_errors{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval])\n)\n\n)\nunless on (cluster_environmentd_materialize_cloud_cluster_id) (\ngroup by (cluster_environmentd_materialize_cloud_cluster_id) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"cluster_environmentd_materialize_cloud_cluster_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n),\n\"cluster_name\", \"$1\", \"cluster_environmentd_materialize_cloud_cluster_id\", \"(.*)\"\n)", "legendFormat": "{{app}} {{cluster_name}}", "qryType": 1, "query": "(\n(\nsum by (app, cluster_environmentd_materialize_cloud_cluster_id) (\n rate(mz_persist_postgres_connpool_connection_errors{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval])\n)\n\n)\n* on (cluster_environmentd_materialize_cloud_cluster_id) group_left(cluster_name)\nmax by (cluster_environmentd_materialize_cloud_cluster_id, cluster_name) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"cluster_environmentd_materialize_cloud_cluster_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\nsum by (app, cluster_environmentd_materialize_cloud_cluster_id) (\n rate(mz_persist_postgres_connpool_connection_errors{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval])\n)\n\n)\nunless on (cluster_environmentd_materialize_cloud_cluster_id) (\ngroup by (cluster_environmentd_materialize_cloud_cluster_id) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"cluster_environmentd_materialize_cloud_cluster_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n),\n\"cluster_name\", \"$1\", \"cluster_environmentd_materialize_cloud_cluster_id\", \"(.*)\"\n)" }, "version": "v0" }, "refId": "query-0" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(rate(mz_ts_oracle_postgres_connpool_connection_errors{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval]))\n", "legendFormat": "timestamp oracle", "qryType": 1, "query": "sum(rate(mz_ts_oracle_postgres_connpool_connection_errors{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval]))\n" }, "version": "v0" }, "refId": "query-1" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Failed connection attempts per second, by the process that made them.**\n\n**Nominal:** Nothing drawn above zero.\n\n**Degraded:** Every process at once is the database, its address, or its credentials. One process alone is that process's network path.", "id": 1010, "links": [], "title": "Connection Errors by Process", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "No metrics from this environment. Check the environment picker, and that Materialize is running.", "unit": "cps" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "overview-connections-held": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(mz_persist_postgres_connpool_size{materialize_cloud_organization_name=~\"$environmentNameList\"})\n+ (sum(mz_ts_oracle_postgres_connpool_size{materialize_cloud_organization_name=~\"$environmentNameList\"}) or vector(0))\n", "legendFormat": "connections", "qryType": 1, "query": "sum(mz_persist_postgres_connpool_size{materialize_cloud_organization_name=~\"$environmentNameList\"})\n+ (sum(mz_ts_oracle_postgres_connpool_size{materialize_cloud_organization_name=~\"$environmentNameList\"}) or vector(0))\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Connections to the metadata database this environment currently holds open.**\n\n**Nominal:** A few per process — each cluster replica keeps its own pool, and so does environmentd. Around 50 is typical for a small environment.\n\n**Degraded:** This is what the database counts against its `max_connections`. Adding clusters and replicas adds connections, and a rollout briefly runs two generations of every process at once, so the peak during an upgrade is roughly double the steady state. If the database is shared with anything else, that headroom is shared too.\n\nThe database's own count, and its limit, are on the Infrastructure Cloud Provider dashboard under _Metadata Database -> Headroom_.", "id": 1004, "links": [], "title": "Connections Held", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "fixedColor": "#0077BB", "mode": "shades" }, "decimals": 0, "min": 0, "noValue": "No metrics from this environment. Check the environment picker, and that Materialize is running.", "unit": "short" }, "overrides": [] }, "options": { "colorMode": "none", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [], "values": false }, "showPercentChange": false, "textMode": "value", "wideLayout": true } }, "version": "" } } }, "overview-failed-ops": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(rate(mz_persist_external_failed_count{materialize_cloud_organization_name=~\"$environmentNameList\", op=~\"consensus_.*\"}[$__rate_interval]))\n", "legendFormat": "failed/s", "qryType": 1, "query": "sum(rate(mz_persist_external_failed_count{materialize_cloud_organization_name=~\"$environmentNameList\", op=~\"consensus_.*\"}[$__rate_interval]))\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Metadata database calls that failed, per second, across every Materialize process in this environment.**\n\n**Nominal:** Zero. A short burst while a process restarts or the database fails over is retried and harmless.\n\n**Degraded:** Any sustained rate means Materialize is retrying its writes and cannot make progress while it does: freshness stalls everywhere at once. The cause is almost always on the database side — unreachable, out of connection slots, or restarting.\n\n_Retrying Metadata Database Operations_ below has the database's own error text, which is what tells those causes apart.", "id": 1000, "links": [], "title": "Failed Operations", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "No metrics from this environment. Check the environment picker, and that Materialize is running.", "thresholds": { "mode": "absolute", "steps": [ { "color": "#A2F49B", "value": null }, { "color": "#FEDA8B", "value": 0.01 }, { "color": "#FDB366", "value": 0.2575 }, { "color": "#F67E4B", "value": 0.505 }, { "color": "#DD3D2D", "value": 0.7525 }, { "color": "#A50026", "value": 1 } ] }, "unit": "ops" }, "overrides": [] }, "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [], "values": false }, "showPercentChange": false, "textMode": "value", "wideLayout": true } }, "version": "" } } }, "overview-failures-by-op": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (op) (\n rate(mz_persist_external_failed_count{materialize_cloud_organization_name=~\"$environmentNameList\", op=~\"consensus_.*\"}[$__rate_interval])\n)\n", "legendFormat": "{{op}}", "qryType": 1, "query": "sum by (op) (\n rate(mz_persist_external_failed_count{materialize_cloud_organization_name=~\"$environmentNameList\", op=~\"consensus_.*\"}[$__rate_interval])\n)\n" }, "version": "v0" }, "refId": "query-0" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (op) (\n rate(mz_ts_oracle_failed_count{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval])\n)\n", "legendFormat": "timestamp oracle {{op}}", "qryType": 1, "query": "sum by (op) (\n rate(mz_ts_oracle_failed_count{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval])\n)\n" }, "version": "v0" }, "refId": "query-1" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Failed metadata database calls per second, by operation.**\n\n**Nominal:** Nothing drawn above zero.\n\n**Degraded:** `consensus_cas` failing is writes failing. `consensus_scan` and `consensus_head` failing are reads of an object's state, which every process does when it opens one — failures there on their own often mean a process that is starting up cannot reach the database. Timestamp oracle failures stop every query from being assigned a timestamp.", "id": 1009, "links": [], "title": "Failed Operations by Type", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "No metrics from this environment. Check the environment picker, and that Materialize is running.", "unit": "ops" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "overview-retry-logs": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${logsDatasource}" }, "group": "loki", "kind": "DataQuery", "spec": { "expr": "{namespace=~\"$mzNamespaceList\"}\n |= \"external operation\"\n |~ \"external operation (fetch_state::scan|apply_unbatched_cmd::cas|gc::truncate|maybe_init::cas|consensus::open|read_ts|write_ts|peek_write_ts|apply_write|get_all_timelines|open) \"\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**What Materialize logged while retrying calls to the metadata database, with the database's own error text.**\n\n**Nominal:** Empty. A healthy database needs no retries worth logging.\n\n**Degraded:** The error text is the diagnosis. The common ones:\n\n- `Connection refused`, or a timeout — the database is down, restarting, or unreachable from the cluster. - `remaining connection slots are reserved` or `too many clients` — the database is at `max_connections`. - `password authentication failed`, or a TLS error — the credentials or `sslmode` in the metadata backend URL. - `could not serialize access` — contention between writers, which retries resolve on their own.\n\nA retry is logged once it has failed a few times, so a single blip does not appear here. If the environment picker is empty, Materialize never started far enough to report metrics; its logs are still on Materialize Logs and Events under _Logs -> All Logs_.", "id": 1011, "links": [], "title": "Retrying Metadata Database Operations", "vizConfig": { "group": "logs", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "noValue": "No retried metadata database calls were logged in this time range." }, "overrides": [] }, "options": { "dedupStrategy": "exact", "detailsMode": "inline", "displayedFields": [], "enableInfiniteScrolling": true, "enableLogDetails": true, "fontSize": "small", "prettifyLogMessage": false, "showCommonLabels": false, "showControls": true, "showFieldSelector": true, "showLabels": false, "showLevel": true, "showLogContextToggle": false, "showTime": true, "sortOrder": "Descending", "syntaxHighlighting": true, "timestampResolution": "ms", "unwrappedColumns": false, "wrapLogMessage": true } }, "version": "" } } }, "overview-round-trip": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "(\n(\nmax by (app, cluster_environmentd_materialize_cloud_cluster_id) (\n mz_persist_external_rtt_latency{materialize_cloud_organization_name=~\"$environmentNameList\", external=\"consensus\"}\n)\n\n)\n* on (cluster_environmentd_materialize_cloud_cluster_id) group_left(cluster_name)\nmax by (cluster_environmentd_materialize_cloud_cluster_id, cluster_name) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"cluster_environmentd_materialize_cloud_cluster_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\nmax by (app, cluster_environmentd_materialize_cloud_cluster_id) (\n mz_persist_external_rtt_latency{materialize_cloud_organization_name=~\"$environmentNameList\", external=\"consensus\"}\n)\n\n)\nunless on (cluster_environmentd_materialize_cloud_cluster_id) (\ngroup by (cluster_environmentd_materialize_cloud_cluster_id) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"cluster_environmentd_materialize_cloud_cluster_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n),\n\"cluster_name\", \"$1\", \"cluster_environmentd_materialize_cloud_cluster_id\", \"(.*)\"\n)", "legendFormat": "{{app}} {{cluster_name}}", "qryType": 1, "query": "(\n(\nmax by (app, cluster_environmentd_materialize_cloud_cluster_id) (\n mz_persist_external_rtt_latency{materialize_cloud_organization_name=~\"$environmentNameList\", external=\"consensus\"}\n)\n\n)\n* on (cluster_environmentd_materialize_cloud_cluster_id) group_left(cluster_name)\nmax by (cluster_environmentd_materialize_cloud_cluster_id, cluster_name) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"cluster_environmentd_materialize_cloud_cluster_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\nmax by (app, cluster_environmentd_materialize_cloud_cluster_id) (\n mz_persist_external_rtt_latency{materialize_cloud_organization_name=~\"$environmentNameList\", external=\"consensus\"}\n)\n\n)\nunless on (cluster_environmentd_materialize_cloud_cluster_id) (\ngroup by (cluster_environmentd_materialize_cloud_cluster_id) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"cluster_environmentd_materialize_cloud_cluster_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n),\n\"cluster_name\", \"$1\", \"cluster_environmentd_materialize_cloud_cluster_id\", \"(.*)\"\n)" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**The network round trip from each Materialize process to the metadata database, as that process last measured it.**\n\n**Nominal:** About a millisecond within one region, and roughly the same for every process.\n\n**Degraded:** One process far above the rest is that process's network path — its node, its zone, or a proxy in the way. Every process high at once is the database, or a database that is not in the cluster's region.\n\nA gauge the client refreshes on its own schedule, so it can trail the latency panels by a few seconds.", "id": 1008, "links": [], "title": "Round-Trip Time by Process", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "No metrics from this environment. Check the environment picker, and that Materialize is running.", "unit": "s" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "overview-waiting": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(clamp_min(-mz_persist_postgres_connpool_available{materialize_cloud_organization_name=~\"$environmentNameList\"}, 0))\n+ (sum(clamp_min(-mz_ts_oracle_postgres_connpool_available{materialize_cloud_organization_name=~\"$environmentNameList\"}, 0)) or vector(0))\n", "legendFormat": "waiting", "qryType": 1, "query": "sum(clamp_min(-mz_persist_postgres_connpool_available{materialize_cloud_organization_name=~\"$environmentNameList\"}, 0))\n+ (sum(clamp_min(-mz_ts_oracle_postgres_connpool_available{materialize_cloud_organization_name=~\"$environmentNameList\"}, 0)) or vector(0))\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Calls currently queued for a free database connection, across every process.**\n\n**Nominal:** Zero. Each pool normally has spare connections.\n\n**Degraded:** Calls are waiting on the pool rather than on the database. Brief bursts are fine; a queue that persists means the pools are too small for the load or connections are being held too long — which a slow database causes.\n\nA snapshot taken at each connection request, so an idle process reports its last reading.", "id": 1003, "links": [], "title": "Waiting for a Connection", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "decimals": 0, "min": 0, "noValue": "No metrics from this environment. Check the environment picker, and that Materialize is running.", "thresholds": { "mode": "absolute", "steps": [ { "color": "#A2F49B", "value": null }, { "color": "#FEDA8B", "value": 1 }, { "color": "#FDB366", "value": 3.25 }, { "color": "#F67E4B", "value": 5.5 }, { "color": "#DD3D2D", "value": 7.75 }, { "color": "#A50026", "value": 10 } ] }, "unit": "short" }, "overrides": [] }, "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [], "values": false }, "showPercentChange": false, "textMode": "value", "wideLayout": true } }, "version": "" } } }, "pg-autovacuum-running": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "time() - max by (relname) (pg_stat_activity_autovacuum_timestamp_seconds{job=\"postgres-exporter\", target=\"$postgresTarget\"})\n", "legendFormat": "{{relname}}", "qryType": 1, "query": "time() - max by (relname) (pg_stat_activity_autovacuum_timestamp_seconds{job=\"postgres-exporter\", target=\"$postgresTarget\"})\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**How long each autovacuum worker running now has been at it, by table.**\n\n**Nominal:** Short, frequent passes. No data most of the time.\n\n**Degraded:** One pass on the consensus table that runs for many minutes, or restarts without finishing, means vacuum cannot keep up — often because it is throttled by `autovacuum_vacuum_cost_limit`, or cancelled by a lock it conflicts with.", "id": 1066, "links": [], "title": "Autovacuum Running", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "No autovacuum worker is running now, which is usual between passes.", "unit": "s" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "pg-cache-hit": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "(\n sum by (datname) (max without (pod) (rate(pg_stat_database_blks_hit{job=\"postgres-exporter\", target=\"$postgresTarget\", datname!=\"\"}[$__rate_interval])))\n / (\n sum by (datname) (max without (pod) (rate(pg_stat_database_blks_hit{job=\"postgres-exporter\", target=\"$postgresTarget\", datname!=\"\"}[$__rate_interval])))\n + sum by (datname) (max without (pod) (rate(pg_stat_database_blks_read{job=\"postgres-exporter\", target=\"$postgresTarget\", datname!=\"\"}[$__rate_interval])))\n )\n) >= 0\n", "legendFormat": "{{datname}}", "qryType": 1, "query": "(\n sum by (datname) (max without (pod) (rate(pg_stat_database_blks_hit{job=\"postgres-exporter\", target=\"$postgresTarget\", datname!=\"\"}[$__rate_interval])))\n / (\n sum by (datname) (max without (pod) (rate(pg_stat_database_blks_hit{job=\"postgres-exporter\", target=\"$postgresTarget\", datname!=\"\"}[$__rate_interval])))\n + sum by (datname) (max without (pod) (rate(pg_stat_database_blks_read{job=\"postgres-exporter\", target=\"$postgresTarget\", datname!=\"\"}[$__rate_interval])))\n )\n) >= 0\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**The fraction of block reads served from PostgreSQL's shared buffers rather than from disk, by database.**\n\n**Nominal:** Close to 100%. The consensus table is small and hot, and should live in memory.\n\n**Degraded:** Well below 99% means reads are going to disk, so commits wait on I/O. The instance needs more memory, or its `shared_buffers` setting is smaller than the working set.", "id": 1061, "links": [], "title": "Buffer Cache Hit Ratio", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "max": 1, "noValue": "The selected database has not reported this. Check the Database picker, and that the exporter can connect.", "unit": "percentunit" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "pg-connections-by-database": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (datname, usename) (max without (pod) (pg_stat_activity_count{job=\"postgres-exporter\", target=\"$postgresTarget\", backend_type=\"client backend\"})) > 0\n", "legendFormat": "{{datname}} ({{usename}})", "qryType": 1, "query": "sum by (datname, usename) (max without (pod) (pg_stat_activity_count{job=\"postgres-exporter\", target=\"$postgresTarget\", backend_type=\"client backend\"})) > 0\n" }, "version": "v0" }, "refId": "query-0" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max(pg_settings_max_connections{job=\"postgres-exporter\", target=\"$postgresTarget\"})\n- max(pg_settings_superuser_reserved_connections{job=\"postgres-exporter\", target=\"$postgresTarget\"})\n- (\n max(pg_settings_reserved_connections{job=\"postgres-exporter\", target=\"$postgresTarget\"})\n or vector(0)\n )\n", "legendFormat": "ceiling", "qryType": 1, "query": "max(pg_settings_max_connections{job=\"postgres-exporter\", target=\"$postgresTarget\"})\n- max(pg_settings_superuser_reserved_connections{job=\"postgres-exporter\", target=\"$postgresTarget\"})\n- (\n max(pg_settings_reserved_connections{job=\"postgres-exporter\", target=\"$postgresTarget\"})\n or vector(0)\n )\n" }, "version": "v0" }, "refId": "query-1" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Client connections open on the instance, by database and role, beside the ceiling an ordinary role is refused at.**\n\n**Nominal:** Materialize's role holding nearly all of them, well under the ceiling.\n\n**Degraded:** Connections near the ceiling. If they are Materialize's, its pools have grown with the number of replicas; if they are another database's, something else is sharing the instance and Materialize is competing with it.\n\nMaterialize's own count of the connections it holds is on the Connections tab, and should match its line here.", "id": 1055, "links": [], "title": "Connections by Database and Role", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "decimals": 0, "min": 0, "noValue": "The selected database has not reported this. Check the Database picker, and that the exporter can connect.", "unit": "short" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "pg-connections-by-state": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (state) (max without (pod) (pg_stat_activity_count{job=\"postgres-exporter\", target=\"$postgresTarget\", backend_type=\"client backend\"}))\n", "legendFormat": "{{state}}", "qryType": 1, "query": "sum by (state) (max without (pod) (pg_stat_activity_count{job=\"postgres-exporter\", target=\"$postgresTarget\", backend_type=\"client backend\"}))\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Client connections by what they are doing: running a statement (`active`), waiting for the next one (`idle`), or holding a transaction open between statements (`idle in transaction`).**\n\n**Nominal:** Mostly `idle`. A connection pool keeps connections open between uses.\n\n**Degraded:** `idle in transaction` that does not drain is a client holding a transaction open, which blocks vacuum and can hold locks. It is a different fault from too many connections, and the fix is in the client that opened it.", "id": 1056, "links": [], "title": "Connections by State", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "decimals": 0, "min": 0, "noValue": "The selected database has not reported this. Check the Database picker, and that the exporter can connect.", "unit": "short" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "pg-connections-used": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum (max without (pod) (pg_stat_activity_count{job=\"postgres-exporter\", target=\"$postgresTarget\", backend_type=\"client backend\"}))\n/ (\n max(pg_settings_max_connections{job=\"postgres-exporter\", target=\"$postgresTarget\"})\n - max(pg_settings_superuser_reserved_connections{job=\"postgres-exporter\", target=\"$postgresTarget\"})\n - (\n max(pg_settings_reserved_connections{job=\"postgres-exporter\", target=\"$postgresTarget\"})\n or vector(0)\n )\n)\n", "legendFormat": "used", "qryType": 1, "query": "sum (max without (pod) (pg_stat_activity_count{job=\"postgres-exporter\", target=\"$postgresTarget\", backend_type=\"client backend\"}))\n/ (\n max(pg_settings_max_connections{job=\"postgres-exporter\", target=\"$postgresTarget\"})\n - max(pg_settings_superuser_reserved_connections{job=\"postgres-exporter\", target=\"$postgresTarget\"})\n - (\n max(pg_settings_reserved_connections{job=\"postgres-exporter\", target=\"$postgresTarget\"})\n or vector(0)\n )\n)\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Client connections open on the selected database's instance, as a fraction of the connections an ordinary role may open.**\n\n**Nominal:** Well below 1. Materialize holds a pool of connections per process, so the count rises with every cluster replica added.\n\n**Degraded:** Above 80%, the next cluster scale-up or rollout — which briefly runs two generations of every process — can exhaust it.\n\n**Unhealthy:** At 1, PostgreSQL refuses new connections with `remaining connection slots are reserved`, and every Materialize process reports connection errors at once.\n\nThe ceiling is `max_connections` less `superuser_reserved_connections` and, from PostgreSQL 16, `reserved_connections`, which is where an ordinary role is refused. Connections to every database on the instance count against it, not only Materialize's. A managed service's own maintenance connections count too.", "id": 1050, "links": [], "title": "Connections Used", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "max": 1, "min": 0, "noValue": "The selected database has not reported this. Check the Database picker, and that the exporter can connect.", "thresholds": { "mode": "absolute", "steps": [ { "color": "#A2F49B", "value": null }, { "color": "#F19903", "value": 0.8 }, { "color": "#AB0003", "value": 0.95 } ] }, "unit": "percentunit" }, "overrides": [] }, "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [], "values": false }, "showPercentChange": false, "textMode": "value", "wideLayout": true } }, "version": "" } } }, "pg-database-size": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (datname) (pg_database_size_bytes{job=\"postgres-exporter\", target=\"$postgresTarget\"})\n", "legendFormat": "{{datname}}", "qryType": 1, "query": "max by (datname) (pg_database_size_bytes{job=\"postgres-exporter\", target=\"$postgresTarget\"})\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**The size of each database on the instance.**\n\n**Nominal:** Materialize's database small and roughly flat: persist keeps only current state there, and the data itself is in object storage.\n\n**Degraded:** Materialize's database growing without bound means truncation has stopped. Another database growing is a neighbour consuming the volume — a full disk looks the same either way from outside the database, and this is what tells them apart.", "id": 1062, "links": [], "title": "Size by Database", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "The selected database has not reported this. Check the Database picker, and that the exporter can connect.", "unit": "bytes" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "pg-dead-tuples": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max(\n pg_stat_user_tables_n_dead_tup{job=\"postgres-exporter\", target=\"$postgresTarget\"}\n / (\n pg_stat_user_tables_n_live_tup{job=\"postgres-exporter\", target=\"$postgresTarget\"}\n + pg_stat_user_tables_n_dead_tup{job=\"postgres-exporter\", target=\"$postgresTarget\"}\n > 1000\n )\n)\n", "legendFormat": "dead", "qryType": 1, "query": "max(\n pg_stat_user_tables_n_dead_tup{job=\"postgres-exporter\", target=\"$postgresTarget\"}\n / (\n pg_stat_user_tables_n_live_tup{job=\"postgres-exporter\", target=\"$postgresTarget\"}\n + pg_stat_user_tables_n_dead_tup{job=\"postgres-exporter\", target=\"$postgresTarget\"}\n > 1000\n )\n)\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**The dead-row fraction of the table with the most dead rows, in the database the exporter connects to.**\n\n**Nominal:** Below 20%, and falling back after each autovacuum. Persist rewrites its consensus rows constantly, so dead rows accumulate quickly and autovacuum clears them just as quickly.\n\n**Degraded:** A fraction that keeps climbing means autovacuum is not keeping up with the table, or cannot run: a long-running transaction, an abandoned replication slot, or `autovacuum_vacuum_cost_limit` throttling it. Every commit then scans more dead rows, and the database's own metrics stay flat while Materialize's commit latency climbs.\n\nIgnores tables with fewer than 1,000 rows, whose ratio swings on a handful of updates. _Dead Rows by Table_ names the table.", "id": 1053, "links": [], "title": "Worst Dead-Row Ratio", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "max": 1, "min": 0, "noValue": "No table statistics. They come from the database the target connects to, and it has no tables with rows to report.", "thresholds": { "mode": "absolute", "steps": [ { "color": "#A2F49B", "value": null }, { "color": "#F19903", "value": 0.2 }, { "color": "#AB0003", "value": 0.5 } ] }, "unit": "percentunit" }, "overrides": [] }, "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [], "values": false }, "showPercentChange": false, "textMode": "value", "wideLayout": true } }, "version": "" } } }, "pg-dead-tuples-by-table": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "topk(10, max by (datname, relname) (pg_stat_user_tables_n_dead_tup{job=\"postgres-exporter\", target=\"$postgresTarget\"}))\n", "legendFormat": "{{datname}}.{{relname}}", "qryType": 1, "query": "topk(10, max by (datname, relname) (pg_stat_user_tables_n_dead_tup{job=\"postgres-exporter\", target=\"$postgresTarget\"}))\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Dead rows waiting for vacuum, for the ten tables with the most, in the database the exporter connects to.**\n\n**Nominal:** A saw-tooth on persist's `consensus` table: dead rows build up between autovacuum passes and drop at each one.\n\n**Degraded:** A line that only rises is a table autovacuum is not reaching. On the consensus table, that is the workload-specific failure: every commit scans the dead rows, so commits slow while nothing else changes.", "id": 1064, "links": [], "title": "Dead Rows by Table", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "No table statistics. They come from the database the target connects to, and it has no tables with rows to report.", "unit": "short" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "pg-locks": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (datname, mode) (max without (pod) (pg_locks_count{job=\"postgres-exporter\", target=\"$postgresTarget\"})) > 0\n", "legendFormat": "{{datname}} {{mode}}", "qryType": 1, "query": "sum by (datname, mode) (max without (pod) (pg_locks_count{job=\"postgres-exporter\", target=\"$postgresTarget\"})) > 0\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Locks held, by database and lock mode.**\n\n**Nominal:** A steady handful of `accesssharelock` and `rowexclusivelock` from ordinary reads and writes.\n\n**Degraded:** A sudden rise in `exclusivelock` or `accessexclusivelock` on Materialize's database is something taking the consensus table for itself, such as a `VACUUM FULL`, a migration, or a manual `LOCK`, and Materialize's commits wait behind it.\n\nLocks held, not locks waited on: PostgreSQL's lock table does not say which holder is blocking which waiter. The exporter reports each mode in lowercase. Compare with commit latency on the Operations tab.", "id": 1058, "links": [], "title": "Locks Held by Database and Mode", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "decimals": 0, "min": 0, "noValue": "The selected database has not reported this. Check the Database picker, and that the exporter can connect.", "unit": "short" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "pg-not-reachable": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "(1 - max(pg_up{job=\"postgres-exporter\", target=\"$postgresTarget\"}))\nor (1 - max(up{job=\"postgres-exporter\", target=\"$postgresTarget\"}))\n", "legendFormat": "unreachable", "qryType": 1, "query": "(1 - max(pg_up{job=\"postgres-exporter\", target=\"$postgresTarget\"}))\nor (1 - max(up{job=\"postgres-exporter\", target=\"$postgresTarget\"}))\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Whether the exporter can reach the selected database: 1 when it cannot connect, or cannot itself be scraped.**\n\n**Nominal:** Zero.\n\n**Unhealthy:** The exporter cannot connect. If Materialize's own commits on the other tabs are failing at the same time, the database is down or unreachable from the cluster. If they are not, the exporter's credentials or grant are wrong: check the pod's log and the `postgres-exporter` Secret.\n\nCounts both ways a target can go dark: `pg_up` reading 0, and the exporter's `/probe` not answering at all.", "id": 1049, "links": [], "title": "Not Reachable", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "decimals": 0, "min": 0, "noValue": "The selected database has not reported this. Check the Database picker, and that the exporter can connect.", "thresholds": { "mode": "absolute", "steps": [ { "color": "#A2F49B", "value": null }, { "color": "#FEDA8B", "value": 1 }, { "color": "#FDB366", "value": 1 }, { "color": "#F67E4B", "value": 1 }, { "color": "#DD3D2D", "value": 1 }, { "color": "#A50026", "value": 1 } ] }, "unit": "short" }, "overrides": [] }, "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [], "values": false }, "showPercentChange": false, "textMode": "value", "wideLayout": true } }, "version": "" } } }, "pg-oldest-transaction": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max(pg_stat_activity_max_tx_duration{job=\"postgres-exporter\", target=\"$postgresTarget\"})\n", "legendFormat": "oldest", "qryType": 1, "query": "max(pg_stat_activity_max_tx_duration{job=\"postgres-exporter\", target=\"$postgresTarget\"})\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**How long the oldest open transaction on the instance has been running.**\n\n**Nominal:** Under a second. Persist's commits are single, short transactions.\n\n**Degraded:** Minutes. A long-running transaction holds back vacuum on every table, so the consensus table grows and every commit slows. It is usually something other than Materialize: a migration, a backup tool, or a session left `idle in transaction`.\n\n_Oldest Transaction by Database_ on this tab names the database and state it is in.", "id": 1051, "links": [], "title": "Oldest Open Transaction", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "The selected database has not reported this. Check the Database picker, and that the exporter can connect.", "thresholds": { "mode": "absolute", "steps": [ { "color": "#A2F49B", "value": null }, { "color": "#F19903", "value": 60 }, { "color": "#AB0003", "value": 900 } ] }, "unit": "s" }, "overrides": [] }, "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [], "values": false }, "showPercentChange": false, "textMode": "value", "wideLayout": true } }, "version": "" } } }, "pg-rows": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (datname) (max without (pod) (rate(pg_stat_database_tup_inserted{job=\"postgres-exporter\", target=\"$postgresTarget\", datname!=\"\"}[$__rate_interval]))) > 0\n", "legendFormat": "{{datname}} inserted", "qryType": 1, "query": "sum by (datname) (max without (pod) (rate(pg_stat_database_tup_inserted{job=\"postgres-exporter\", target=\"$postgresTarget\", datname!=\"\"}[$__rate_interval]))) > 0\n" }, "version": "v0" }, "refId": "query-0" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (datname) (max without (pod) (rate(pg_stat_database_tup_updated{job=\"postgres-exporter\", target=\"$postgresTarget\", datname!=\"\"}[$__rate_interval]))) > 0\n", "legendFormat": "{{datname}} updated", "qryType": 1, "query": "sum by (datname) (max without (pod) (rate(pg_stat_database_tup_updated{job=\"postgres-exporter\", target=\"$postgresTarget\", datname!=\"\"}[$__rate_interval]))) > 0\n" }, "version": "v0" }, "refId": "query-1" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (datname) (max without (pod) (rate(pg_stat_database_tup_deleted{job=\"postgres-exporter\", target=\"$postgresTarget\", datname!=\"\"}[$__rate_interval]))) > 0\n", "legendFormat": "{{datname}} deleted", "qryType": 1, "query": "sum by (datname) (max without (pod) (rate(pg_stat_database_tup_deleted{job=\"postgres-exporter\", target=\"$postgresTarget\", datname!=\"\"}[$__rate_interval]))) > 0\n" }, "version": "v0" }, "refId": "query-2" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Rows inserted, updated and deleted per second, by database.**\n\n**Nominal:** A steady churn on Materialize's database. Every commit rewrites a small state record and truncation deletes the versions behind it, so deletes track inserts.\n\n**Degraded:** Inserts with no matching deletes means truncation has stopped and the consensus table is growing; _State and Cleanup_ shows it from Materialize's side. Heavy churn on another database is a neighbour competing for the same disk.", "id": 1060, "links": [], "title": "Rows Written by Database", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "The selected database has not reported this. Check the Database picker, and that the exporter can connect.", "unit": "rowsps" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "pg-since-autovacuum": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "time() - (\n max by (datname, relname) (pg_stat_user_tables_last_autovacuum{job=\"postgres-exporter\", target=\"$postgresTarget\"} > 0)\n and on (datname, relname)\n topk(10, max by (datname, relname) (pg_stat_user_tables_n_dead_tup{job=\"postgres-exporter\", target=\"$postgresTarget\"}))\n)\n", "legendFormat": "{{datname}}.{{relname}}", "qryType": 1, "query": "time() - (\n max by (datname, relname) (pg_stat_user_tables_last_autovacuum{job=\"postgres-exporter\", target=\"$postgresTarget\"} > 0)\n and on (datname, relname)\n topk(10, max by (datname, relname) (pg_stat_user_tables_n_dead_tup{job=\"postgres-exporter\", target=\"$postgresTarget\"}))\n)\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Time since autovacuum last finished on each of the ten tables with the most dead rows, in the database the exporter connects to.**\n\n**Nominal:** Minutes on the consensus table, which churns fast enough to be vacuumed constantly. Hours or days on tables that rarely change.\n\n**Degraded:** A table with many dead rows and a growing time since its last autovacuum is the leading indicator of the slowdown _Dead Rows by Table_ shows.\n\nAbsent for a table autovacuum has never processed.", "id": 1065, "links": [], "title": "Time Since Autovacuum", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "No table here has been autovacuumed since statistics were last reset, or the database the target connects to has no tables.", "unit": "s" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "pg-transactions": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (datname) (max without (pod) (rate(pg_stat_database_xact_commit{job=\"postgres-exporter\", target=\"$postgresTarget\", datname!=\"\"}[$__rate_interval]))) > 0\n", "legendFormat": "{{datname}} committed", "qryType": 1, "query": "sum by (datname) (max without (pod) (rate(pg_stat_database_xact_commit{job=\"postgres-exporter\", target=\"$postgresTarget\", datname!=\"\"}[$__rate_interval]))) > 0\n" }, "version": "v0" }, "refId": "query-0" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (datname) (max without (pod) (rate(pg_stat_database_xact_rollback{job=\"postgres-exporter\", target=\"$postgresTarget\", datname!=\"\"}[$__rate_interval]))) > 0\n", "legendFormat": "{{datname}} rolled back", "qryType": 1, "query": "sum by (datname) (max without (pod) (rate(pg_stat_database_xact_rollback{job=\"postgres-exporter\", target=\"$postgresTarget\", datname!=\"\"}[$__rate_interval]))) > 0\n" }, "version": "v0" }, "refId": "query-1" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Transactions committed and rolled back per second, by database.**\n\n**Nominal:** Materialize's database committing steadily — tens per second on an idle environment — and few rollbacks.\n\n**Degraded:** A commit rate well below Materialize's own count of commits means its calls are not arriving. Rising rollbacks on Materialize's database are usually commits that lost a race, which persist expects and retries.\n\nCompare with _Commits_ on the Overview tab, which is the same work counted by Materialize.", "id": 1059, "links": [], "title": "Transactions by Database", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "The selected database has not reported this. Check the Database picker, and that the exporter can connect.", "unit": "ops" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "pg-transactions-held": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (datname, state) (pg_stat_activity_max_tx_duration{job=\"postgres-exporter\", target=\"$postgresTarget\"}) > 0\n", "legendFormat": "{{datname}} ({{state}})", "qryType": 1, "query": "max by (datname, state) (pg_stat_activity_max_tx_duration{job=\"postgres-exporter\", target=\"$postgresTarget\"}) > 0\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**The oldest open transaction, by database and connection state.**\n\n**Nominal:** Under a second.\n\n**Degraded:** A transaction that keeps aging holds back vacuum on every table, so the consensus table grows and every commit slows.", "id": 1057, "links": [], "title": "Oldest Transaction by Database", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "No transaction is open. Persist's commits finish between scrapes.", "unit": "s" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "pg-version": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (short_version) (pg_static{job=\"postgres-exporter\", target=\"$postgresTarget\"})\n", "instant": true, "legendFormat": "{{short_version}}", "qryType": 1, "query": "max by (short_version) (pg_static{job=\"postgres-exporter\", target=\"$postgresTarget\"})\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [ { "group": "labelsToFields", "kind": "Transformation", "spec": { "options": {} } } ] } }, "description": "**The PostgreSQL version the selected database runs.**\n\n**Nominal:** A supported major version.\n\nAn out-of-date dependency has been the cause of an incident before, and this is the one place it is visible without a database login.", "id": 1054, "links": [], "title": "PostgreSQL Version", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "fixedColor": "#009988", "mode": "shades" }, "noValue": "The selected database has not reported this. Check the Database picker, and that the exporter can connect." }, "overrides": [] }, "options": { "colorMode": "none", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [], "fields": "/^short_version$/", "values": false }, "showPercentChange": false, "text": { "valueSize": 20 }, "textMode": "value", "wideLayout": true } }, "version": "" } } }, "pg-xid-age": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (datname) (pg_database_wraparound_age_datfrozenxid_seconds{job=\"postgres-exporter\", target=\"$postgresTarget\"}) / 2147483648\n", "legendFormat": "{{datname}}", "qryType": 1, "query": "max by (datname) (pg_database_wraparound_age_datfrozenxid_seconds{job=\"postgres-exporter\", target=\"$postgresTarget\"}) / 2147483648\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Transaction-ID space consumed by each database since vacuum last froze it, as a fraction of the wraparound limit.**\n\n**Nominal:** Below 10%, and saw-toothed: each autovacuum freeze pass brings it down.\n\n**Degraded:** A line that only rises means vacuum cannot freeze that database — usually a long-running transaction or an abandoned replication slot holding it back.", "id": 1063, "links": [], "title": "Transaction IDs Used by Database", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "Not reported. The exporter needs --collector.database_wraparound, which the chart sets.", "thresholds": { "mode": "absolute", "steps": [ { "color": "#A2F49B", "value": null }, { "color": "#F19903", "value": 0.25 }, { "color": "#AB0003", "value": 0.5 } ] }, "unit": "percentunit" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "pg-xid-used": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max(pg_database_wraparound_age_datfrozenxid_seconds{job=\"postgres-exporter\", target=\"$postgresTarget\"}) / 2147483648\n", "legendFormat": "used", "qryType": 1, "query": "max(pg_database_wraparound_age_datfrozenxid_seconds{job=\"postgres-exporter\", target=\"$postgresTarget\"}) / 2147483648\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**How much of the transaction-ID space the most at-risk database on the instance has consumed since vacuum last froze it.**\n\n**Nominal:** Below 10%. Autovacuum freezes old rows long before this matters.\n\n**Degraded:** Above 25% means freezing is falling behind. Persist commits constantly, so a small instance whose autovacuum is blocked reaches this faster than most workloads.\n\n**Unhealthy:** Past 50%, treat it as urgent: the remaining headroom is what a stalled vacuum has left before wraparound. At 100%, PostgreSQL stops accepting writes to protect itself, and Materialize stops with it; its own warnings start only once fewer than 40 million IDs remain, at about 98%.\n\n`age(datfrozenxid)` against the 2^31 wraparound limit, the same measure Cloud SQL publishes as transaction-ID utilization. The exporter names it `…_seconds`, but it counts transactions.", "id": 1052, "links": [], "title": "Transaction IDs Used", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "max": 1, "min": 0, "noValue": "The selected database has not reported this. Check the Database picker, and that the exporter can connect.", "thresholds": { "mode": "absolute", "steps": [ { "color": "#A2F49B", "value": null }, { "color": "#F19903", "value": 0.25 }, { "color": "#AB0003", "value": 0.5 } ] }, "unit": "percentunit" }, "overrides": [] }, "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [], "values": false }, "showPercentChange": false, "textMode": "value", "wideLayout": true } }, "version": "" } } }, "pool-acquire-wait": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "(\n(\n(\n sum by (app, cluster_environmentd_materialize_cloud_cluster_id) (\n rate(mz_persist_postgres_connpool_acquire_seconds{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval])\n )\n /\n sum by (app, cluster_environmentd_materialize_cloud_cluster_id) (\n rate(mz_persist_postgres_connpool_acquires{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval])\n )\n) >= 0\n\n)\n* on (cluster_environmentd_materialize_cloud_cluster_id) group_left(cluster_name)\nmax by (cluster_environmentd_materialize_cloud_cluster_id, cluster_name) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"cluster_environmentd_materialize_cloud_cluster_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\n(\n sum by (app, cluster_environmentd_materialize_cloud_cluster_id) (\n rate(mz_persist_postgres_connpool_acquire_seconds{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval])\n )\n /\n sum by (app, cluster_environmentd_materialize_cloud_cluster_id) (\n rate(mz_persist_postgres_connpool_acquires{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval])\n )\n) >= 0\n\n)\nunless on (cluster_environmentd_materialize_cloud_cluster_id) (\ngroup by (cluster_environmentd_materialize_cloud_cluster_id) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"cluster_environmentd_materialize_cloud_cluster_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n),\n\"cluster_name\", \"$1\", \"cluster_environmentd_materialize_cloud_cluster_id\", \"(.*)\"\n)", "legendFormat": "{{app}} {{cluster_name}}", "qryType": 1, "query": "(\n(\n(\n sum by (app, cluster_environmentd_materialize_cloud_cluster_id) (\n rate(mz_persist_postgres_connpool_acquire_seconds{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval])\n )\n /\n sum by (app, cluster_environmentd_materialize_cloud_cluster_id) (\n rate(mz_persist_postgres_connpool_acquires{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval])\n )\n) >= 0\n\n)\n* on (cluster_environmentd_materialize_cloud_cluster_id) group_left(cluster_name)\nmax by (cluster_environmentd_materialize_cloud_cluster_id, cluster_name) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"cluster_environmentd_materialize_cloud_cluster_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\n(\n sum by (app, cluster_environmentd_materialize_cloud_cluster_id) (\n rate(mz_persist_postgres_connpool_acquire_seconds{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval])\n )\n /\n sum by (app, cluster_environmentd_materialize_cloud_cluster_id) (\n rate(mz_persist_postgres_connpool_acquires{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval])\n )\n) >= 0\n\n)\nunless on (cluster_environmentd_materialize_cloud_cluster_id) (\ngroup by (cluster_environmentd_materialize_cloud_cluster_id) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"cluster_environmentd_materialize_cloud_cluster_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n),\n\"cluster_name\", \"$1\", \"cluster_environmentd_materialize_cloud_cluster_id\", \"(.*)\"\n)" }, "version": "v0" }, "refId": "query-0" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "(\n sum(rate(mz_ts_oracle_postgres_connpool_acquire_seconds{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval]))\n /\n sum(rate(mz_ts_oracle_postgres_connpool_acquires{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval]))\n) >= 0\n", "legendFormat": "timestamp oracle", "qryType": 1, "query": "(\n sum(rate(mz_ts_oracle_postgres_connpool_acquire_seconds{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval]))\n /\n sum(rate(mz_ts_oracle_postgres_connpool_acquires{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval]))\n) >= 0\n" }, "version": "v0" }, "refId": "query-1" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Average time a call waits to be handed a connection, by process.**\n\n**Nominal:** Microseconds — a free connection is almost always ready.\n\n**Degraded:** Milliseconds or more is time added to every commit before it reaches the database, and is invisible from the database's side.", "id": 1024, "links": [], "title": "Connection Wait (mean)", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "No metrics from this environment. Check the environment picker, and that Materialize is running.", "unit": "s" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "pool-churn": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(rate(mz_persist_postgres_connpool_connections_created{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval]))\n+ (sum(rate(mz_ts_oracle_postgres_connpool_connections_created{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval])) or vector(0))\n", "legendFormat": "opened", "qryType": 1, "query": "sum(rate(mz_persist_postgres_connpool_connections_created{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval]))\n+ (sum(rate(mz_ts_oracle_postgres_connpool_connections_created{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval])) or vector(0))\n" }, "version": "v0" }, "refId": "query-0" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(rate(mz_persist_postgres_connpool_ttl_reconnections{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval]))\n+ (sum(rate(mz_ts_oracle_postgres_connpool_ttl_reconnections{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval])) or vector(0))\n", "legendFormat": "replaced at age limit", "qryType": 1, "query": "sum(rate(mz_persist_postgres_connpool_ttl_reconnections{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval]))\n+ (sum(rate(mz_ts_oracle_postgres_connpool_ttl_reconnections{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval])) or vector(0))\n" }, "version": "v0" }, "refId": "query-1" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**New connections opened per second, and how many of those replaced a connection that had reached its age limit.**\n\n**Nominal:** The two lines close together and low: connections are recycled on a timer so that a database failover is picked up, and that recycling is most of what opens new ones.\n\n**Degraded:** New connections well above the recycling rate means connections are being lost and reopened — a database dropping them, or a proxy with a short idle timeout in the path. Each new connection costs the database a TLS handshake and a backend process.", "id": 1025, "links": [], "title": "New Connections", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "No metrics from this environment. Check the environment picker, and that Materialize is running.", "unit": "cps" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "pool-held": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "(\n(\nsum by (app, cluster_environmentd_materialize_cloud_cluster_id) (\n mz_persist_postgres_connpool_size{materialize_cloud_organization_name=~\"$environmentNameList\"}\n)\n\n)\n* on (cluster_environmentd_materialize_cloud_cluster_id) group_left(cluster_name)\nmax by (cluster_environmentd_materialize_cloud_cluster_id, cluster_name) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"cluster_environmentd_materialize_cloud_cluster_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\nsum by (app, cluster_environmentd_materialize_cloud_cluster_id) (\n mz_persist_postgres_connpool_size{materialize_cloud_organization_name=~\"$environmentNameList\"}\n)\n\n)\nunless on (cluster_environmentd_materialize_cloud_cluster_id) (\ngroup by (cluster_environmentd_materialize_cloud_cluster_id) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"cluster_environmentd_materialize_cloud_cluster_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n),\n\"cluster_name\", \"$1\", \"cluster_environmentd_materialize_cloud_cluster_id\", \"(.*)\"\n)", "legendFormat": "{{app}} {{cluster_name}}", "qryType": 1, "query": "(\n(\nsum by (app, cluster_environmentd_materialize_cloud_cluster_id) (\n mz_persist_postgres_connpool_size{materialize_cloud_organization_name=~\"$environmentNameList\"}\n)\n\n)\n* on (cluster_environmentd_materialize_cloud_cluster_id) group_left(cluster_name)\nmax by (cluster_environmentd_materialize_cloud_cluster_id, cluster_name) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"cluster_environmentd_materialize_cloud_cluster_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\nsum by (app, cluster_environmentd_materialize_cloud_cluster_id) (\n mz_persist_postgres_connpool_size{materialize_cloud_organization_name=~\"$environmentNameList\"}\n)\n\n)\nunless on (cluster_environmentd_materialize_cloud_cluster_id) (\ngroup by (cluster_environmentd_materialize_cloud_cluster_id) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"cluster_environmentd_materialize_cloud_cluster_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n),\n\"cluster_name\", \"$1\", \"cluster_environmentd_materialize_cloud_cluster_id\", \"(.*)\"\n)" }, "version": "v0" }, "refId": "query-0" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(mz_ts_oracle_postgres_connpool_size{materialize_cloud_organization_name=~\"$environmentNameList\"})\n", "legendFormat": "timestamp oracle", "qryType": 1, "query": "sum(mz_ts_oracle_postgres_connpool_size{materialize_cloud_organization_name=~\"$environmentNameList\"})\n" }, "version": "v0" }, "refId": "query-1" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Connections to the metadata database held open, by process.**\n\n**Nominal:** A handful per process, stable. Pools shrink as idle connections expire and grow back on demand.\n\n**Degraded:** The sum across processes is what the database counts against `max_connections`. A process whose pool keeps growing is waiting on a slow database and opening more connections to compensate.", "id": 1021, "links": [], "title": "Connections Held by Process", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "decimals": 0, "min": 0, "noValue": "No metrics from this environment. Check the environment picker, and that Materialize is running.", "unit": "short" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "pool-in-use": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "(\n(\nsum by (app, cluster_environmentd_materialize_cloud_cluster_id) (\n mz_persist_postgres_connpool_size{materialize_cloud_organization_name=~\"$environmentNameList\"}\n - clamp_min(mz_persist_postgres_connpool_available{materialize_cloud_organization_name=~\"$environmentNameList\"}, 0)\n)\n\n)\n* on (cluster_environmentd_materialize_cloud_cluster_id) group_left(cluster_name)\nmax by (cluster_environmentd_materialize_cloud_cluster_id, cluster_name) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"cluster_environmentd_materialize_cloud_cluster_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\nsum by (app, cluster_environmentd_materialize_cloud_cluster_id) (\n mz_persist_postgres_connpool_size{materialize_cloud_organization_name=~\"$environmentNameList\"}\n - clamp_min(mz_persist_postgres_connpool_available{materialize_cloud_organization_name=~\"$environmentNameList\"}, 0)\n)\n\n)\nunless on (cluster_environmentd_materialize_cloud_cluster_id) (\ngroup by (cluster_environmentd_materialize_cloud_cluster_id) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"cluster_environmentd_materialize_cloud_cluster_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n),\n\"cluster_name\", \"$1\", \"cluster_environmentd_materialize_cloud_cluster_id\", \"(.*)\"\n)", "legendFormat": "{{app}} {{cluster_name}}", "qryType": 1, "query": "(\n(\nsum by (app, cluster_environmentd_materialize_cloud_cluster_id) (\n mz_persist_postgres_connpool_size{materialize_cloud_organization_name=~\"$environmentNameList\"}\n - clamp_min(mz_persist_postgres_connpool_available{materialize_cloud_organization_name=~\"$environmentNameList\"}, 0)\n)\n\n)\n* on (cluster_environmentd_materialize_cloud_cluster_id) group_left(cluster_name)\nmax by (cluster_environmentd_materialize_cloud_cluster_id, cluster_name) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"cluster_environmentd_materialize_cloud_cluster_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\nsum by (app, cluster_environmentd_materialize_cloud_cluster_id) (\n mz_persist_postgres_connpool_size{materialize_cloud_organization_name=~\"$environmentNameList\"}\n - clamp_min(mz_persist_postgres_connpool_available{materialize_cloud_organization_name=~\"$environmentNameList\"}, 0)\n)\n\n)\nunless on (cluster_environmentd_materialize_cloud_cluster_id) (\ngroup by (cluster_environmentd_materialize_cloud_cluster_id) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"cluster_environmentd_materialize_cloud_cluster_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n),\n\"cluster_name\", \"$1\", \"cluster_environmentd_materialize_cloud_cluster_id\", \"(.*)\"\n)" }, "version": "v0" }, "refId": "query-0" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(\n mz_ts_oracle_postgres_connpool_size{materialize_cloud_organization_name=~\"$environmentNameList\"}\n - clamp_min(mz_ts_oracle_postgres_connpool_available{materialize_cloud_organization_name=~\"$environmentNameList\"}, 0)\n)\n", "legendFormat": "timestamp oracle", "qryType": 1, "query": "sum(\n mz_ts_oracle_postgres_connpool_size{materialize_cloud_organization_name=~\"$environmentNameList\"}\n - clamp_min(mz_ts_oracle_postgres_connpool_available{materialize_cloud_organization_name=~\"$environmentNameList\"}, 0)\n)\n" }, "version": "v0" }, "refId": "query-1" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Connections actively lent out to a call, by process.**\n\n**Nominal:** Well under the pool's size.\n\n**Degraded:** In use equal to held, with callers queued in _Waiting for a Connection by Process_, is a pool that is too small or a database that is holding each connection too long.", "id": 1022, "links": [], "title": "Connections in Use by Process", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "decimals": 0, "min": 0, "noValue": "No metrics from this environment. Check the environment picker, and that Materialize is running.", "unit": "short" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "pool-waiting": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "(\n(\nsum by (app, cluster_environmentd_materialize_cloud_cluster_id) (\n clamp_min(-mz_persist_postgres_connpool_available{materialize_cloud_organization_name=~\"$environmentNameList\"}, 0)\n)\n\n)\n* on (cluster_environmentd_materialize_cloud_cluster_id) group_left(cluster_name)\nmax by (cluster_environmentd_materialize_cloud_cluster_id, cluster_name) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"cluster_environmentd_materialize_cloud_cluster_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\nsum by (app, cluster_environmentd_materialize_cloud_cluster_id) (\n clamp_min(-mz_persist_postgres_connpool_available{materialize_cloud_organization_name=~\"$environmentNameList\"}, 0)\n)\n\n)\nunless on (cluster_environmentd_materialize_cloud_cluster_id) (\ngroup by (cluster_environmentd_materialize_cloud_cluster_id) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"cluster_environmentd_materialize_cloud_cluster_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n),\n\"cluster_name\", \"$1\", \"cluster_environmentd_materialize_cloud_cluster_id\", \"(.*)\"\n)", "legendFormat": "{{app}} {{cluster_name}}", "qryType": 1, "query": "(\n(\nsum by (app, cluster_environmentd_materialize_cloud_cluster_id) (\n clamp_min(-mz_persist_postgres_connpool_available{materialize_cloud_organization_name=~\"$environmentNameList\"}, 0)\n)\n\n)\n* on (cluster_environmentd_materialize_cloud_cluster_id) group_left(cluster_name)\nmax by (cluster_environmentd_materialize_cloud_cluster_id, cluster_name) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"cluster_environmentd_materialize_cloud_cluster_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n)\nor\nlabel_replace(\n(\nsum by (app, cluster_environmentd_materialize_cloud_cluster_id) (\n clamp_min(-mz_persist_postgres_connpool_available{materialize_cloud_organization_name=~\"$environmentNameList\"}, 0)\n)\n\n)\nunless on (cluster_environmentd_materialize_cloud_cluster_id) (\ngroup by (cluster_environmentd_materialize_cloud_cluster_id) (\nlabel_replace(label_replace(mz_cluster_info{materialize_cloud_organization_name=~\"$environmentNameList\"}, \"cluster_environmentd_materialize_cloud_cluster_id\", \"$1\", \"cluster_id\", \"(.*)\"), \"cluster_name\", \"$1\", \"name\", \"(.*)\")\n)\n),\n\"cluster_name\", \"$1\", \"cluster_environmentd_materialize_cloud_cluster_id\", \"(.*)\"\n)" }, "version": "v0" }, "refId": "query-0" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(clamp_min(-mz_ts_oracle_postgres_connpool_available{materialize_cloud_organization_name=~\"$environmentNameList\"}, 0))\n", "legendFormat": "timestamp oracle", "qryType": 1, "query": "sum(clamp_min(-mz_ts_oracle_postgres_connpool_available{materialize_cloud_organization_name=~\"$environmentNameList\"}, 0))\n" }, "version": "v0" }, "refId": "query-1" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Calls queued for a connection, by process.**\n\n**Nominal:** Zero.\n\n**Degraded:** Queueing in one process is that process's pool. Queueing in every process at once, with commit latency rising, is a slow database holding each connection longer.\n\nA snapshot at each connection request rather than a continuous measure.", "id": 1023, "links": [], "title": "Waiting for a Connection by Process", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "decimals": 0, "min": 0, "noValue": "No metrics from this environment. Check the environment picker, and that Materialize is running.", "unit": "short" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "state-gc-runs": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(rate(mz_persist_gc_started{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval]))\n", "legendFormat": "started", "qryType": 1, "query": "sum(rate(mz_persist_gc_started{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval]))\n" }, "version": "v0" }, "refId": "query-0" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(rate(mz_persist_gc_finished{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval]))\n", "legendFormat": "finished", "qryType": 1, "query": "sum(rate(mz_persist_gc_finished{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval]))\n" }, "version": "v0" }, "refId": "query-1" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(rate(mz_persist_gc_noop{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval]))\n", "legendFormat": "skipped", "qryType": 1, "query": "sum(rate(mz_persist_gc_noop{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval]))\n" }, "version": "v0" }, "refId": "query-2" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Cleanup passes per second: started, finished, and skipped because another process had already done the work.**\n\n**Nominal:** Started and finished on top of each other.\n\n**Degraded:** Started above finished for any length of time is cleanup failing or stalling, usually on the same database trouble the _Overview_ tab shows. It is what makes _Versions Written and Deleted_ diverge.", "id": 1029, "links": [], "title": "Cleanup Runs", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "No metrics from this environment. Check the environment picker, and that Materialize is running.", "unit": "ops" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "state-held": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "topk(10, max by (shard) (mz_persist_shard_seqnos_held{materialize_cloud_organization_name=~\"$environmentNameList\"}))\n", "legendFormat": "{{shard}}", "qryType": 1, "query": "topk(10, max by (shard) (mz_persist_shard_seqnos_held{materialize_cloud_organization_name=~\"$environmentNameList\"}))\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**The ten objects holding the most old state versions, which cleanup cannot delete until whatever is reading them moves on.**\n\n**Nominal:** Hundreds at most, and moving.\n\n**Degraded:** An object whose count only ever grows has a reader stuck in the past: a subscription or a replica that is not keeping up, or one that went away without releasing its hold. Its hold is released when the reader's lease expires.\n\nTranslate a shard id to an object with `SELECT o.name, o.type FROM mz_internal.mz_storage_shards s JOIN mz_objects o ON o.id = s.object_id WHERE s.shard_id = ''`.", "id": 1028, "links": [], "title": "Versions Held by Object", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "decimals": 0, "min": 0, "noValue": "No metrics from this environment. Check the environment picker, and that Materialize is running.", "unit": "short" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "state-lease-timeouts": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(rate(mz_persist_lease_timeout_read{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval]))\n", "legendFormat": "expired/s", "qryType": 1, "query": "sum(rate(mz_persist_lease_timeout_read{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval]))\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Readers whose lease on an object expired, per second.**\n\n**Nominal:** Occasional, around restarts: a process that goes away without saying so has its leases expire instead.\n\n**Degraded:** Steady expirations with no restarts mean readers are being starved — a process too busy to renew its leases. Each expiry releases the old state versions that reader was holding, so this is also what un-sticks _Versions Held by Object_.", "id": 1030, "links": [], "title": "Expired Reader Leases", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "No metrics from this environment. Check the environment picker, and that Materialize is running.", "unit": "ops" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "state-live": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(max by (shard) (mz_persist_shard_gc_live_diffs{materialize_cloud_organization_name=~\"$environmentNameList\"}))\n", "legendFormat": "versions", "qryType": 1, "query": "sum(max by (shard) (mz_persist_shard_gc_live_diffs{materialize_cloud_organization_name=~\"$environmentNameList\"}))\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**State versions currently stored in the metadata database, summed across every object — roughly the number of rows persist keeps there.**\n\n**Nominal:** Flat, at a few hundred per object.\n\n**Degraded:** Steady growth is the backlog that _Versions Written and Deleted_ shows as a gap. It is not urgent on its own, but it is the database's storage and vacuum load growing without anything new being stored.\n\nMeasured at each object's last cleanup, so it moves in steps.", "id": 1027, "links": [], "title": "Versions Stored", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "fixedColor": "#EE3377", "mode": "shades" }, "decimals": 0, "min": 0, "noValue": "No metrics from this environment. Check the environment picker, and that Materialize is running.", "unit": "short" }, "overrides": [] }, "options": { "colorMode": "none", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [], "values": false }, "showPercentChange": false, "textMode": "value", "wideLayout": true } }, "version": "" } } }, "state-written-deleted": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(rate(mz_persist_external_succeeded_count{materialize_cloud_organization_name=~\"$environmentNameList\", op=\"consensus_cas\"}[$__rate_interval]))\n", "legendFormat": "written", "qryType": 1, "query": "sum(rate(mz_persist_external_succeeded_count{materialize_cloud_organization_name=~\"$environmentNameList\", op=\"consensus_cas\"}[$__rate_interval]))\n" }, "version": "v0" }, "refId": "query-0" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(rate(mz_persist_external_consensus_truncated_count{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval]))\n", "legendFormat": "deleted", "qryType": 1, "query": "sum(rate(mz_persist_external_consensus_truncated_count{materialize_cloud_organization_name=~\"$environmentNameList\"}[$__rate_interval]))\n" }, "version": "v0" }, "refId": "query-1" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**State versions written to the metadata database per second, beside versions deleted.**\n\n**Nominal:** The two lines on top of each other: every commit adds a version, and cleanup deletes the ones nothing needs any more.\n\n**Degraded:** Written staying above deleted for hours means the table persist writes to is growing. The usual cause is a reader holding old versions — _Versions Held by Object_ names it. On PostgreSQL a growing table is also more work for autovacuum, which is how this eventually shows up as commit latency.", "id": 1026, "links": [], "title": "Versions Written and Deleted", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "No metrics from this environment. Check the environment picker, and that Materialize is running.", "unit": "suffix:versions/s" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } } }, "layout": { "kind": "TabsLayout", "spec": { "tabs": [ { "kind": "TabsLayoutTab", "spec": { "layout": { "kind": "RowsLayout", "spec": { "rows": [ { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": true, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "narrow", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "overview-failed-ops" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "overview-connection-errors" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "overview-commit-p99" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "overview-waiting" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "overview-connections-held" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "overview-commits" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "overview-collections" } } } ], "maxColumnCount": 7, "rowHeightMode": "short" } }, "title": "Verdict" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "wide", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "overview-commit-latency" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "overview-round-trip" } } } ], "maxColumnCount": 2, "rowHeightMode": "standard" } }, "title": "Latency" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "wide", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "overview-failures-by-op" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "overview-connection-errors-by-process" } } } ], "maxColumnCount": 2, "rowHeightMode": "standard" } }, "title": "Failures" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "standard", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "overview-retry-logs" } } } ], "maxColumnCount": 1, "rowHeightMode": "tall" } }, "title": "Logs" } } ] } }, "title": "Overview" } }, { "kind": "TabsLayoutTab", "spec": { "layout": { "kind": "RowsLayout", "spec": { "rows": [ { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "wide", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "ops-by-type" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "ops-commits-by-process" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "ops-bytes" } } } ], "maxColumnCount": 2, "rowHeightMode": "standard" } }, "title": "Load" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "wide", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "ops-mean-latency" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "ops-commit-latency-by-process" } } } ], "maxColumnCount": 2, "rowHeightMode": "standard" } }, "title": "Latency by Operation" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "wide", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "ops-conflicts" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "ops-retries" } } } ], "maxColumnCount": 2, "rowHeightMode": "standard" } }, "title": "Contention and Retries" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "wide", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "ops-oracle-ops" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "ops-oracle-latency" } } } ], "maxColumnCount": 2, "rowHeightMode": "standard" } }, "title": "Timestamp Oracle" } } ] } }, "title": "Operations" } }, { "kind": "TabsLayoutTab", "spec": { "layout": { "kind": "RowsLayout", "spec": { "rows": [ { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "wide", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "pool-held" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "pool-in-use" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "pool-waiting" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "pool-acquire-wait" } } } ], "maxColumnCount": 2, "rowHeightMode": "standard" } }, "title": "Connection Pools" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "wide", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "pool-churn" } } } ], "maxColumnCount": 1, "rowHeightMode": "standard" } }, "title": "Connection Churn" } } ] } }, "title": "Connections" } }, { "kind": "TabsLayoutTab", "spec": { "layout": { "kind": "RowsLayout", "spec": { "rows": [ { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "wide", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "state-written-deleted" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "state-live" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "state-held" } } } ], "maxColumnCount": 3, "rowHeightMode": "standard" } }, "title": "State Versions" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "wide", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "state-gc-runs" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "state-lease-timeouts" } } } ], "maxColumnCount": 2, "rowHeightMode": "standard" } }, "title": "Cleanup" } } ] } }, "title": "State and Cleanup" } }, { "kind": "TabsLayoutTab", "spec": { "layout": { "kind": "RowsLayout", "spec": { "rows": [ { "kind": "RowsLayoutRow", "spec": { "collapse": false, "conditionalRendering": { "kind": "ConditionalRenderingGroup", "spec": { "condition": "and", "items": [ { "kind": "ConditionalRenderingVariable", "spec": { "operator": "matches", "value": "cnpg_collector_up", "variable": "cnpgDetected" } } ], "visibility": "show" } }, "hideHeader": false, "layout": { "kind": "RowsLayout", "spec": { "rows": [ { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": true, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "narrow", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "cnpg-not-serving" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "cnpg-primary" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "cnpg-connections-used" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "cnpg-oldest-transaction" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "cnpg-xid-used" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "cnpg-backup-age" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "cnpg-version" } } } ], "maxColumnCount": 7, "rowHeightMode": "short" } }, "title": "Status" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "wide", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "cnpg-connections-by-database" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "cnpg-connections-by-state" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "cnpg-transactions-held" } } } ], "maxColumnCount": 3, "rowHeightMode": "standard" } }, "title": "Connections" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "wide", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "cnpg-transactions" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "cnpg-rows" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "cnpg-cache-hit" } } } ], "maxColumnCount": 3, "rowHeightMode": "standard" } }, "title": "Activity" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "wide", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "cnpg-database-size" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "cnpg-wal" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "cnpg-xid-age" } } } ], "maxColumnCount": 3, "rowHeightMode": "standard" } }, "title": "Storage and Vacuum" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "wide", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "cnpg-replication-lag" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "cnpg-archive-failures" } } } ], "maxColumnCount": 2, "rowHeightMode": "standard" } }, "title": "Replication and Archiving" } } ] } }, "title": "CloudNativePG", "variables": [ { "kind": "QueryVariable", "spec": { "allowCustomValue": true, "current": { "text": "", "value": "" }, "definition": "label_values(cnpg_collector_up, pod)", "description": "The CloudNativePG cluster serving as this environment's metadata database", "hide": "dontHide", "includeAll": false, "label": "CNPG Cluster", "multi": false, "name": "cnpgClusterList", "options": [], "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "label_values(cnpg_collector_up, pod)", "qryType": 1, "query": "label_values(cnpg_collector_up, pod)" }, "version": "v0" }, "refresh": "never", "regex": "/^(?.+)-[0-9]+$/", "regexApplyTo": "value", "skipUrlSync": false, "sort": "alphabeticalAsc" } } ] } }, { "kind": "RowsLayoutRow", "spec": { "collapse": false, "conditionalRendering": { "kind": "ConditionalRenderingGroup", "spec": { "condition": "and", "items": [ { "kind": "ConditionalRenderingVariable", "spec": { "operator": "matches", "value": "postgres-exporter", "variable": "postgresDetected" } } ], "visibility": "show" } }, "hideHeader": false, "layout": { "kind": "RowsLayout", "spec": { "rows": [ { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": true, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "narrow", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "pg-not-reachable" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "pg-connections-used" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "pg-oldest-transaction" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "pg-xid-used" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "pg-dead-tuples" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "pg-version" } } } ], "maxColumnCount": 6, "rowHeightMode": "short" } }, "title": "Status" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "wide", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "pg-connections-by-database" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "pg-connections-by-state" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "pg-transactions-held" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "pg-locks" } } } ], "maxColumnCount": 2, "rowHeightMode": "standard" } }, "title": "Connections" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "wide", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "pg-transactions" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "pg-rows" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "pg-cache-hit" } } } ], "maxColumnCount": 3, "rowHeightMode": "standard" } }, "title": "Activity" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "wide", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "pg-database-size" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "pg-xid-age" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "pg-dead-tuples-by-table" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "pg-since-autovacuum" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "pg-autovacuum-running" } } } ], "maxColumnCount": 3, "rowHeightMode": "standard" } }, "title": "Storage and Vacuum" } } ] } }, "title": "PostgreSQL", "variables": [ { "kind": "QueryVariable", "spec": { "allowCustomValue": true, "current": { "text": "", "value": "" }, "definition": "label_values(up{job=\"postgres-exporter\"}, target)", "description": "The postgres-exporter target serving as this environment's metadata database", "hide": "dontHide", "includeAll": false, "label": "Database", "multi": false, "name": "postgresTarget", "options": [], "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "label_values(up{job=\"postgres-exporter\"}, target)", "qryType": 1, "query": "label_values(up{job=\"postgres-exporter\"}, target)" }, "version": "v0" }, "refresh": "never", "regex": "", "regexApplyTo": "value", "skipUrlSync": false, "sort": "alphabeticalAsc" } } ] } }, { "kind": "RowsLayoutRow", "spec": { "collapse": false, "conditionalRendering": { "kind": "ConditionalRenderingGroup", "spec": { "condition": "and", "items": [ { "kind": "ConditionalRenderingVariable", "spec": { "operator": "notMatches", "value": "cnpg_collector_up", "variable": "cnpgDetected" } }, { "kind": "ConditionalRenderingVariable", "spec": { "operator": "notMatches", "value": "postgres-exporter", "variable": "postgresDetected" } } ], "visibility": "show" } }, "hideHeader": true, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "standard", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "database-none-detected" } } } ], "maxColumnCount": 1, "rowHeightMode": "tall" } }, "title": "No Database Metrics" } } ] } }, "title": "Database Internals" } } ] } }, "links": [], "liveNow": false, "preload": false, "tags": [ "materialize", "mzmon", "external-dependencies" ], "timeSettings": { "autoRefresh": "", "autoRefreshIntervals": [ "5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d" ], "fiscalYearStartMonth": 0, "from": "now-6h", "hideTimepicker": false, "timezone": "browser", "to": "now" }, "title": "Materialize Consensus (Metadata)", "variables": [ { "kind": "DatasourceVariable", "spec": { "allowCustomValue": false, "current": { "text": "", "value": "" }, "description": "Datasource for metrics queries", "hide": "dontHide", "includeAll": false, "label": "Metrics Datasource", "multi": false, "name": "metricsDatasource", "options": [], "pluginId": "prometheus", "refresh": "never", "regex": "", "skipUrlSync": false } }, { "kind": "DatasourceVariable", "spec": { "allowCustomValue": false, "current": { "text": "", "value": "" }, "description": "Datasource for log and event queries", "hide": "dontHide", "includeAll": false, "label": "Logs Datasource", "multi": false, "name": "logsDatasource", "options": [], "pluginId": "loki", "refresh": "never", "regex": "", "skipUrlSync": false } }, { "kind": "QueryVariable", "spec": { "allValue": ".*", "allowCustomValue": true, "current": { "text": "", "value": "" }, "definition": "label_values(mz_compute_commands_total, materialize_cloud_organization_name)", "description": "The current environment to view", "hide": "dontHide", "includeAll": false, "label": "Environment", "multi": false, "name": "environmentNameList", "options": [], "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "label_values(mz_compute_commands_total, materialize_cloud_organization_name)", "qryType": 1, "query": "label_values(mz_compute_commands_total, materialize_cloud_organization_name)" }, "version": "v0" }, "refresh": "never", "regex": "", "regexApplyTo": "value", "skipUrlSync": false, "sort": "alphabeticalAsc" } }, { "kind": "QueryVariable", "spec": { "allowCustomValue": true, "current": { "text": "", "value": "" }, "definition": "label_values(mz_compute_commands_total{ materialize_cloud_organization_name=~\"$environmentNameList\" }, materialize_cloud_organization_namespace)", "description": "The current materialize namespace where environments live", "hide": "hideVariable", "includeAll": true, "label": "Materialize Namespace", "multi": true, "name": "mzNamespaceList", "options": [], "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "label_values(mz_compute_commands_total{ materialize_cloud_organization_name=~\"$environmentNameList\" }, materialize_cloud_organization_namespace)", "qryType": 1, "query": "label_values(mz_compute_commands_total{ materialize_cloud_organization_name=~\"$environmentNameList\" }, materialize_cloud_organization_namespace)" }, "version": "v0" }, "refresh": "never", "regex": "", "regexApplyTo": "value", "skipUrlSync": true, "sort": "alphabeticalAsc" } }, { "datasource": { "name": "$metricsDatasource" }, "group": "", "kind": "AdhocVariable", "spec": { "allowCustomValue": true, "baseFilters": [ { "key": "namespace", "operator": "=~", "origin": "dashboard", "value": "$mzNamespaceList" } ], "defaultKeys": [], "description": "Adhoc filters to apply to all metrics queries", "enableGroupBy": false, "filters": [], "hide": "inControlsMenu", "label": "Advanced Metric Filter", "name": "metricAdhoc", "skipUrlSync": false } }, { "kind": "QueryVariable", "spec": { "allowCustomValue": true, "current": { "text": "", "value": "" }, "definition": "label_values(cnpg_collector_up, __name__)", "description": "Whether a CloudNativePG instance is being scraped; empty when none is", "hide": "hideVariable", "includeAll": false, "label": "CNPG", "multi": false, "name": "cnpgDetected", "options": [], "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "label_values(cnpg_collector_up, __name__)", "qryType": 1, "query": "label_values(cnpg_collector_up, __name__)" }, "version": "v0" }, "refresh": "never", "regex": "", "regexApplyTo": "value", "skipUrlSync": true, "sort": "alphabeticalAsc" } }, { "kind": "QueryVariable", "spec": { "allowCustomValue": true, "current": { "text": "", "value": "" }, "definition": "label_values(up{job=\"postgres-exporter\"}, job)", "description": "Whether the postgres-exporter is scraping a database; empty when it is not", "hide": "hideVariable", "includeAll": false, "label": "PostgreSQL Exporter", "multi": false, "name": "postgresDetected", "options": [], "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "label_values(up{job=\"postgres-exporter\"}, job)", "qryType": 1, "query": "label_values(up{job=\"postgres-exporter\"}, job)" }, "version": "v0" }, "refresh": "never", "regex": "", "regexApplyTo": "value", "skipUrlSync": true, "sort": "alphabeticalAsc" } } ] } }