{ "apiVersion": "dashboard.grafana.app/v2", "kind": "Dashboard", "metadata": { "annotations": { "grafana.app/folder": "infra", "monitoring.materialize.cloud/min-mz-version": "v26.24.0", "monitoring.materialize.cloud/rec-mz-version": "v26.24.0", "monitoring.materialize.cloud/sql-metric-prefix": "mz_", "monitoring.materialize.cloud/target-export": "generic" }, "name": "mz-mon-infra-autoscaling" }, "spec": { "annotations": [ { "kind": "AnnotationQuery", "spec": { "builtIn": true, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "query": { "datasource": { "name": "-- Grafana --" }, "group": "grafana", "kind": "DataQuery", "spec": {}, "version": "v0" } } } ], "cursorSync": "Crosshair", "description": "Whether the cluster's nodes are keeping up with its pods, on any cloud.\n\nPods waiting for a node, how full each node pool is, what the autoscaler did about it, and whether the cloud could supply the node. Karpenter's own view on EKS is the Karpenter dashboard.", "editable": true, "elements": { "aks-state": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "min by (resourceName) (\n last_over_time(azure_microsoft_containerservice_managedclusters_cluster_autoscaler_cluster_safe_to_autoscale_minimum_count[15m])\n)\n", "legendFormat": "{{resourceName}} safe to scale", "qryType": 1, "query": "min by (resourceName) (\n last_over_time(azure_microsoft_containerservice_managedclusters_cluster_autoscaler_cluster_safe_to_autoscale_minimum_count[15m])\n)\n" }, "version": "v0" }, "refId": "query-0" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (resourceName) (\n last_over_time(azure_microsoft_containerservice_managedclusters_cluster_autoscaler_scale_down_in_cooldown_maximum_count[15m])\n)\n", "legendFormat": "{{resourceName}} scale-down cooling down", "qryType": 1, "query": "max by (resourceName) (\n last_over_time(azure_microsoft_containerservice_managedclusters_cluster_autoscaler_scale_down_in_cooldown_maximum_count[15m])\n)\n" }, "version": "v0" }, "refId": "query-1" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (resourceName) (\n last_over_time(azure_microsoft_containerservice_managedclusters_cluster_autoscaler_unneeded_nodes_count_maximum_count[15m])\n)\n", "legendFormat": "{{resourceName}} unneeded nodes", "qryType": 1, "query": "max by (resourceName) (\n last_over_time(azure_microsoft_containerservice_managedclusters_cluster_autoscaler_unneeded_nodes_count_maximum_count[15m])\n)\n" }, "version": "v0" }, "refId": "query-2" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Whether AKS's cluster autoscaler considers the cluster safe to scale, whether scale-down is paused, and nodes it would remove.**\n\n**Nominal:** Safe to scale at 1, cooldown at 0 apart from the minutes after a scale-up, and a few unneeded nodes at most.\n\n**Unhealthy:** Safe to scale at 0: too many nodes are not ready, and the autoscaler has stopped adding or removing any until they recover.", "id": 1038, "links": [], "title": "Cluster Autoscaler State", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "decimals": 0, "min": 0, "noValue": "Azure Monitor is pulled, but not for this cluster. Add it under pipeline.metrics.provider.azure.aks.clusters.", "unit": "short" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "aks-unschedulable": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (resourceName) (\n last_over_time(azure_microsoft_containerservice_managedclusters_cluster_autoscaler_unschedulable_pods_count_maximum_count[15m])\n)\n", "legendFormat": "{{resourceName}}", "qryType": 1, "query": "max by (resourceName) (\n last_over_time(azure_microsoft_containerservice_managedclusters_cluster_autoscaler_unschedulable_pods_count_maximum_count[15m])\n)\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Pods AKS's cluster autoscaler cannot place on any existing node, the worst of each five minutes.**\n\n**Nominal:** Zero, or briefly non-zero while a node is added.\n\n**Degraded:** Non-zero while the autoscaler is allowed to scale means it is not adding a node: the pool is at its maximum count, or Azure refused the VM for quota or capacity. The events on the Pending Pods tab carry its reason.\n\nAKS runs the cluster autoscaler in its control plane, where nothing in the cluster can scrape it, so Azure Monitor is the only source of its metrics.", "id": 1037, "links": [], "title": "Pods the Autoscaler Cannot Place", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "decimals": 0, "min": 0, "noValue": "Azure Monitor is pulled, but not for this cluster. Add it under pipeline.metrics.provider.azure.aks.clusters.", "unit": "short" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "aks-vms-down": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (resourceName) (\n 1 - min by (resourceName, dimensionVmname) (\n last_over_time(azure_microsoft_compute_virtualmachinescalesets_vmavailabilitymetric_minimum_count[15m])\n )\n)\n", "legendFormat": "{{resourceName}}", "qryType": 1, "query": "sum by (resourceName) (\n 1 - min by (resourceName, dimensionVmname) (\n last_over_time(azure_microsoft_compute_virtualmachinescalesets_vmavailabilitymetric_minimum_count[15m])\n )\n)\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Node VMs Azure reports as unavailable, by scale set, at any point in each five minutes.**\n\n**Nominal:** Zero.\n\n**Unhealthy:** A VM down while Kubernetes still lists its node. Azure took it down for maintenance or a host fault, or its owner stopped it.\n\nAzure splits a VM's series by `Context`, which says whether the platform or its owner took it down, so one VM can carry two series inside the lookback. Each VM is taken at its worst before counting.", "id": 1039, "links": [], "title": "Node VMs Down", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "decimals": 0, "min": 0, "noValue": "Azure Monitor is pulled, but not for this cluster. Add it under pipeline.metrics.provider.azure.aks.clusters.", "thresholds": { "mode": "absolute", "steps": [ { "color": "#A2F49B", "value": null }, { "color": "#FEDA8B", "value": 1 }, { "color": "#FDB366", "value": 1.25 }, { "color": "#F67E4B", "value": 1.5 }, { "color": "#DD3D2D", "value": 1.75 }, { "color": "#A50026", "value": 2 } ] }, "unit": "short" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "capacity-none-detected": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [], "queryOptions": {}, "transformations": [] } }, "description": "Why this tab has no provider rows on it.", "id": 1040, "links": [], "title": "No Provider Metrics", "vizConfig": { "group": "text", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": {}, "overrides": [] }, "options": { "code": { "language": "plaintext", "showLineNumbers": false, "showMiniMap": false }, "content": "**No cloud provider metrics are being collected.**\n\nThis is the default, not a fault. Pulling from CloudWatch, Cloud Monitoring or Azure Monitor is opt-in, because it needs a cloud identity with read access and every pull is billed.\n\n- **EKS:** list the cluster under `pipeline.metrics.provider.cloudwatch.eks.clusters` for EC2 status checks, managed node group sizes and On-Demand vCPU usage. The gateway's role needs `cloudwatch:GetMetricData`, `cloudwatch:ListMetrics`, `tag:GetResources` and `autoscaling:DescribeAutoScalingGroups`.\n- **GKE:** list the region under `pipeline.metrics.provider.gcp.compute.regions` for CPU and local SSD quota per machine family. `roles/monitoring.viewer` covers it.\n- **AKS:** list the cluster under `pipeline.metrics.provider.azure.aks.clusters` for the managed cluster autoscaler's own gauges, which nothing else can read, and node VM availability. The identity needs Monitoring Reader on the cluster and its node resource group.\n\nWithout them, the other tabs still answer whether pods are waiting and what the autoscaler did; this one says whether the cloud was the reason.", "mode": "markdown" } }, "version": "" } } }, "ec2-node-groups": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (tag_eks_nodegroup_name) (last_over_time(aws_autoscaling_group_desired_capacity_average[15m]))\n", "legendFormat": "{{tag_eks_nodegroup_name}} desired", "qryType": 1, "query": "max by (tag_eks_nodegroup_name) (last_over_time(aws_autoscaling_group_desired_capacity_average[15m]))\n" }, "version": "v0" }, "refId": "query-0" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (tag_eks_nodegroup_name) (last_over_time(aws_autoscaling_group_in_service_instances_average[15m]))\n", "legendFormat": "{{tag_eks_nodegroup_name}} in service", "qryType": 1, "query": "max by (tag_eks_nodegroup_name) (last_over_time(aws_autoscaling_group_in_service_instances_average[15m]))\n" }, "version": "v0" }, "refId": "query-1" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (tag_eks_nodegroup_name) (last_over_time(aws_autoscaling_group_max_size_average[15m]))\n", "legendFormat": "{{tag_eks_nodegroup_name}} maximum", "qryType": 1, "query": "max by (tag_eks_nodegroup_name) (last_over_time(aws_autoscaling_group_max_size_average[15m]))\n" }, "version": "v0" }, "refId": "query-2" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Each EKS managed node group's desired and in-service instances, against its maximum.**\n\n**Nominal:** In-service equal to desired.\n\n**Degraded:** Desired above in-service for longer than an instance takes to start: EC2 is not supplying the instances, for capacity or quota. Desired at the maximum is a group that cannot grow.\n\nKarpenter's pools have no Auto Scaling group; the Karpenter dashboard covers them.", "id": 1032, "links": [], "title": "Managed Node Groups", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "decimals": 0, "min": 0, "noValue": "No managed node groups reported. Karpenter's nodes have none; if the cluster has some, add it under pipeline.metrics.provider.cloudwatch.eks.clusters.", "unit": "short" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "ec2-status-checks": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(last_over_time(aws_ec2_status_check_failed_system_maximum[15m]))\n", "legendFormat": "System (host)", "qryType": 1, "query": "sum(last_over_time(aws_ec2_status_check_failed_system_maximum[15m]))\n" }, "version": "v0" }, "refId": "query-0" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(last_over_time(aws_ec2_status_check_failed_instance_maximum[15m]))\n", "legendFormat": "Instance", "qryType": 1, "query": "sum(last_over_time(aws_ec2_status_check_failed_instance_maximum[15m]))\n" }, "version": "v0" }, "refId": "query-1" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(last_over_time(aws_ec2_status_check_failed_attached_ebs_maximum[15m]))\n", "legendFormat": "Attached EBS", "qryType": 1, "query": "sum(last_over_time(aws_ec2_status_check_failed_attached_ebs_maximum[15m]))\n" }, "version": "v0" }, "refId": "query-2" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Cluster nodes failing an EC2 status check, by check.**\n\n**Nominal:** Zero.\n\n**Unhealthy:** **System** is the host under the node, which only AWS can repair; stop and start the instance to move it to another host, or let the autoscaler replace it. **Instance** is the node's own operating system, usually out of memory or disk. **Attached EBS** is a volume the node cannot reach.\n\nFrom `pipeline.metrics.provider.cloudwatch.eks.clusters`, which finds the nodes by the cluster tag EKS and Karpenter set.", "id": 1031, "links": [], "title": "Nodes Failing a Status Check", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "decimals": 0, "min": 0, "noValue": "CloudWatch is pulled, but not for this cluster's nodes. Add the cluster under pipeline.metrics.provider.cloudwatch.eks.clusters.", "thresholds": { "mode": "absolute", "steps": [ { "color": "#A2F49B", "value": null }, { "color": "#FEDA8B", "value": 1 }, { "color": "#FDB366", "value": 1.25 }, { "color": "#F67E4B", "value": 1.5 }, { "color": "#DD3D2D", "value": 1.75 }, { "color": "#A50026", "value": 2 } ] }, "unit": "short" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "ec2-vcpus": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max(last_over_time(aws_usage_resource_count_maximum{dimension_Resource=\"vCPU\"}[15m]))\n", "legendFormat": "vCPUs in use", "qryType": 1, "query": "max(last_over_time(aws_usage_resource_count_maximum{dimension_Resource=\"vCPU\"}[15m]))\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**On-Demand vCPUs running in the region, for the whole AWS account.**\n\n**Degraded:** Close to the account's quota for Running On-Demand Standard instances (Service Quotas code `L-1216C47A`), which CloudWatch does not publish. At the quota, every launch in these families fails, whatever the cluster's own limits allow.\n\nAccount-wide rather than the cluster's, because the quota is. Covers the standard families (A, C, D, H, I, M, R, T, Z), which include every instance type the Terraform examples use.", "id": 1033, "links": [], "title": "On-Demand vCPUs in the Account", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "decimals": 0, "min": 0, "noValue": "CloudWatch is pulled, but not for this cluster's nodes. Add the cluster under pipeline.metrics.provider.cloudwatch.eks.clusters.", "unit": "short" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "events-all": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${logsDatasource}" }, "group": "loki", "kind": "DataQuery", "spec": { "expr": "{job=\"loki.source.kubernetes_events\"}\n | reportingcontroller=~\"karpenter|cluster-autoscaler|node-controller|cloud-node-lifecycle-controller\" or reason=~\"FailedScheduling|Preempted|TriggeredScaleUp|NotTriggerScaleUp|FailedScaleUp|ScaledUpGroup|ScaleDown\"\n | reason!~\"DisruptionBlocked|Unconsolidatable\"\n | line_format \"{{.reason}} {{.kind}}/{{.name}} — {{.msg}}\"\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Every node and autoscaler event, newest first.**\n\nWithout Karpenter's `DisruptionBlocked` and `Unconsolidatable`, for the reason _Events by Reason_ gives.", "id": 1043, "links": [], "title": "All Autoscaling Events", "vizConfig": { "group": "logs", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "noValue": "No autoscaling events in this time range" }, "overrides": [] }, "options": { "dedupStrategy": "signature", "detailsMode": "inline", "displayedFields": [], "enableInfiniteScrolling": true, "enableLogDetails": true, "fontSize": "small", "prettifyLogMessage": false, "showCommonLabels": false, "showControls": true, "showFieldSelector": true, "showLabels": false, "showLevel": true, "showLogContextToggle": false, "showTime": true, "sortOrder": "Descending", "syntaxHighlighting": true, "timestampResolution": "ms", "unwrappedColumns": false, "wrapLogMessage": true } }, "version": "" } } }, "events-by-reason": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${logsDatasource}" }, "group": "loki", "kind": "DataQuery", "spec": { "expr": "sum by (reason) (\n count_over_time(\n {job=\"loki.source.kubernetes_events\"}\n | reportingcontroller=~\"karpenter|cluster-autoscaler|node-controller|cloud-node-lifecycle-controller\" or reason=~\"FailedScheduling|Preempted|TriggeredScaleUp|NotTriggerScaleUp|FailedScaleUp|ScaledUpGroup|ScaleDown\"\n | reason!~\"DisruptionBlocked|Unconsolidatable\"\n [1m])\n)\n", "legendFormat": "{{reason}}" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Node and autoscaler events per minute, by reason.**\n\nFrom Karpenter, the cluster autoscaler, the node lifecycle and the scheduler's placement failures. Karpenter's `DisruptionBlocked` and `Unconsolidatable` are left out: they repeat every few minutes for every node holding a pod that must not be moved, which on a Materialize cluster is most of them, and the Karpenter dashboard breaks them down.", "id": 1041, "links": [], "title": "Autoscaling Events by Reason", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "No autoscaling events in this time range", "unit": "short" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "events-warnings": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${logsDatasource}" }, "group": "loki", "kind": "DataQuery", "spec": { "expr": "{job=\"loki.source.kubernetes_events\", level=\"WARN\"}\n | reportingcontroller=~\"karpenter|cluster-autoscaler|node-controller|cloud-node-lifecycle-controller\" or reason=~\"FailedScheduling|Preempted|NotTriggerScaleUp|FailedScaleUp\"\n | line_format \"{{.reason}} {{.kind}}/{{.name}} — {{.msg}}\"\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Warning events from the autoscalers and the node lifecycle.**\n\n**Nominal:** Occasional scheduling failures that clear once a node is added.\n\n**Degraded:** The same warning repeating: a pod that cannot be placed, a node that cannot be drained, or a scale-up the cloud refused.", "id": 1042, "links": [], "title": "Warning Events", "vizConfig": { "group": "logs", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "noValue": "No warning events in this time range" }, "overrides": [] }, "options": { "dedupStrategy": "signature", "detailsMode": "inline", "displayedFields": [], "enableInfiniteScrolling": true, "enableLogDetails": true, "fontSize": "small", "prettifyLogMessage": false, "showCommonLabels": false, "showControls": true, "showFieldSelector": true, "showLabels": false, "showLevel": true, "showLogContextToggle": false, "showTime": true, "sortOrder": "Descending", "syntaxHighlighting": true, "timestampResolution": "ms", "unwrappedColumns": false, "wrapLogMessage": true } }, "version": "" } } }, "gce-cpu-quota": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (vm_family, location) (\n last_over_time(stackdriver_compute_googleapis_com_location_compute_googleapis_com_quota_cpus_per_vm_family_usage{limit_name=~\".*per-project-region\"}[15m])\n)\n/\nmax by (vm_family, location) (\n last_over_time(stackdriver_compute_googleapis_com_location_compute_googleapis_com_quota_cpus_per_vm_family_limit{limit_name=~\".*per-project-region\"}[15m])\n)\n", "legendFormat": "{{vm_family}} {{location}}", "qryType": 1, "query": "max by (vm_family, location) (\n last_over_time(stackdriver_compute_googleapis_com_location_compute_googleapis_com_quota_cpus_per_vm_family_usage{limit_name=~\".*per-project-region\"}[15m])\n)\n/\nmax by (vm_family, location) (\n last_over_time(stackdriver_compute_googleapis_com_location_compute_googleapis_com_quota_cpus_per_vm_family_limit{limit_name=~\".*per-project-region\"}[15m])\n)\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**CPUs in use against the regional quota, for each machine family.**\n\n**Nominal:** Well below 1.0.\n\n**Unhealthy:** At 1.0, every node that family's pools try to add is refused, and the cluster autoscaler reports `FailedScaleUp`. Request more quota for the family in the region.\n\nPer family because C4, C4A and the other current families each have their own quota; the older regional `CPUS` quota does not cover them. Google publishes quota a few times a day, so this lags a change by hours.", "id": 1034, "links": [], "title": "CPU Quota Used by Family", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "max": 1, "min": 0, "noValue": "Cloud Monitoring is pulled, but not for Compute Engine quota. Add the region under pipeline.metrics.provider.gcp.compute.regions.", "thresholds": { "mode": "absolute", "steps": [ { "color": "#A2F49B", "value": null }, { "color": "#F19903", "value": 0.8 }, { "color": "#AB0003", "value": 0.95 } ] }, "unit": "percentunit" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "gce-refusals": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (vm_family, location) (\n increase({__name__=~\"stackdriver_compute_googleapis_com_location_compute_googleapis_com_quota_cpus_per_vm_family_exceeded(_total)?\"}[1h])\n)\n", "legendFormat": "CPU {{vm_family}} {{location}}", "qryType": 1, "query": "sum by (vm_family, location) (\n increase({__name__=~\"stackdriver_compute_googleapis_com_location_compute_googleapis_com_quota_cpus_per_vm_family_exceeded(_total)?\"}[1h])\n)\n" }, "version": "v0" }, "refId": "query-0" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (vm_family, location) (\n increase({__name__=~\"stackdriver_compute_googleapis_com_location_compute_googleapis_com_quota_local_ssd_total_storage_per_vm_family_exceeded(_total)?\"}[1h])\n)\n", "legendFormat": "Local SSD {{vm_family}} {{location}}", "qryType": 1, "query": "sum by (vm_family, location) (\n increase({__name__=~\"stackdriver_compute_googleapis_com_location_compute_googleapis_com_quota_local_ssd_total_storage_per_vm_family_exceeded(_total)?\"}[1h])\n)\n" }, "version": "v0" }, "refId": "query-1" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Requests Compute Engine refused for CPU or local SSD quota, per hour.**\n\n**Nominal:** Zero.\n\n**Unhealthy:** Any refusal is a node that could not be created. The quota, family and location say which limit to raise.\n\nWritten only when a request is refused, so there is usually no series at all. One expression per quota: `increase` drops the metric name, so a single expression over both would merge CPU and local SSD refusals for the same family.", "id": 1036, "links": [], "title": "Quota Refusals per Hour", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "No refusals. Compute Engine records one only when a quota turns a request away.", "unit": "short" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "gce-ssd-quota": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (vm_family, location) (\n last_over_time(stackdriver_compute_googleapis_com_location_compute_googleapis_com_quota_local_ssd_total_storage_per_vm_family_usage{limit_name=~\".*per-project-region\"}[15m])\n)\n/\nmax by (vm_family, location) (\n last_over_time(stackdriver_compute_googleapis_com_location_compute_googleapis_com_quota_local_ssd_total_storage_per_vm_family_limit{limit_name=~\".*per-project-region\"}[15m])\n)\n", "legendFormat": "{{vm_family}} {{location}}", "qryType": 1, "query": "max by (vm_family, location) (\n last_over_time(stackdriver_compute_googleapis_com_location_compute_googleapis_com_quota_local_ssd_total_storage_per_vm_family_usage{limit_name=~\".*per-project-region\"}[15m])\n)\n/\nmax by (vm_family, location) (\n last_over_time(stackdriver_compute_googleapis_com_location_compute_googleapis_com_quota_local_ssd_total_storage_per_vm_family_limit{limit_name=~\".*per-project-region\"}[15m])\n)\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Local SSD in use against the regional quota, for each machine family.**\n\n**Nominal:** Well below 1.0.\n\n**Unhealthy:** At 1.0, nodes with local SSD, which Materialize's pools use for spill, cannot be added.", "id": 1035, "links": [], "title": "Local SSD Quota Used by Family", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "max": 1, "min": 0, "noValue": "Cloud Monitoring is pulled, but not for Compute Engine quota. Add the region under pipeline.metrics.provider.gcp.compute.regions.", "thresholds": { "mode": "absolute", "steps": [ { "color": "#A2F49B", "value": null }, { "color": "#F19903", "value": 0.8 }, { "color": "#AB0003", "value": 0.95 } ] }, "unit": "percentunit" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "hpa-events": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${logsDatasource}" }, "group": "loki", "kind": "DataQuery", "spec": { "expr": "{job=\"loki.source.kubernetes_events\"}\n | reportingcontroller=\"horizontal-pod-autoscaler\"\n | line_format \"{{.reason}} {{.name}} — {{.msg}}\"\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**HorizontalPodAutoscaler events, newest first.**", "id": 1030, "links": [], "title": "Autoscaler Events", "vizConfig": { "group": "logs", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "noValue": "No autoscaler events in this time range" }, "overrides": [] }, "options": { "dedupStrategy": "signature", "detailsMode": "inline", "displayedFields": [], "enableInfiniteScrolling": true, "enableLogDetails": true, "fontSize": "small", "prettifyLogMessage": false, "showCommonLabels": false, "showControls": true, "showFieldSelector": true, "showLabels": false, "showLevel": true, "showLogContextToggle": false, "showTime": true, "sortOrder": "Descending", "syntaxHighlighting": true, "timestampResolution": "ms", "unwrappedColumns": false, "wrapLogMessage": true } }, "version": "" } } }, "hpa-events-by-reason": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${logsDatasource}" }, "group": "loki", "kind": "DataQuery", "spec": { "expr": "sum by (reason) (\n count_over_time(\n {job=\"loki.source.kubernetes_events\"} | reportingcontroller=\"horizontal-pod-autoscaler\"\n [1m])\n)\n", "legendFormat": "{{reason}}" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**HorizontalPodAutoscaler events per minute, by reason.**\n\n`SuccessfulRescale` is a change of size. `FailedGetResourceMetric` and `FailedComputeMetricsReplicas` are an autoscaler that could not read its metric; a few are normal while a new pod starts, a steady stream is not.", "id": 1029, "links": [], "title": "Autoscaler Events by Reason", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "No autoscaler events in this time range", "unit": "short" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "hpa-share-of-max": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (namespace, horizontalpodautoscaler) (kube_horizontalpodautoscaler_status_current_replicas)\n/\nmax by (namespace, horizontalpodautoscaler) (kube_horizontalpodautoscaler_spec_max_replicas)\n", "legendFormat": "{{namespace}}/{{horizontalpodautoscaler}}", "qryType": 1, "query": "max by (namespace, horizontalpodautoscaler) (kube_horizontalpodautoscaler_status_current_replicas)\n/\nmax by (namespace, horizontalpodautoscaler) (kube_horizontalpodautoscaler_spec_max_replicas)\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Each HorizontalPodAutoscaler's replicas as a share of its maximum.**\n\n**Nominal:** Below 1.0.\n\n**Degraded:** A workload pinned at 1.0 during its busy hours has no room left to scale out.", "id": 1027, "links": [], "title": "Replicas as a Share of Maximum", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "max": 1, "min": 0, "noValue": "No HorizontalPodAutoscalers", "unit": "percentunit" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "hpa-table": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (namespace, horizontalpodautoscaler) (kube_horizontalpodautoscaler_status_current_replicas)\n", "format": "table", "instant": true, "legendFormat": "Current", "qryType": 2, "query": "max by (namespace, horizontalpodautoscaler) (kube_horizontalpodautoscaler_status_current_replicas)\n" }, "version": "v0" }, "refId": "query-0" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (namespace, horizontalpodautoscaler) (kube_horizontalpodautoscaler_status_desired_replicas)\n", "format": "table", "instant": true, "legendFormat": "Desired", "qryType": 2, "query": "max by (namespace, horizontalpodautoscaler) (kube_horizontalpodautoscaler_status_desired_replicas)\n" }, "version": "v0" }, "refId": "query-1" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (namespace, horizontalpodautoscaler) (kube_horizontalpodautoscaler_spec_min_replicas)\n", "format": "table", "instant": true, "legendFormat": "Min", "qryType": 2, "query": "max by (namespace, horizontalpodautoscaler) (kube_horizontalpodautoscaler_spec_min_replicas)\n" }, "version": "v0" }, "refId": "query-2" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "max by (namespace, horizontalpodautoscaler) (kube_horizontalpodautoscaler_spec_max_replicas)\n", "format": "table", "instant": true, "legendFormat": "Max", "qryType": 2, "query": "max by (namespace, horizontalpodautoscaler) (kube_horizontalpodautoscaler_spec_max_replicas)\n" }, "version": "v0" }, "refId": "query-3" } } ], "queryOptions": {}, "transformations": [ { "group": "merge", "kind": "Transformation", "spec": { "options": {} } }, { "group": "organize", "kind": "Transformation", "spec": { "options": { "excludeByName": { "Time": true }, "indexByName": { "Current": 2, "Desired": 3, "Max": 5, "Min": 4, "Value #query-0": 6, "Value #query-1": 7, "Value #query-2": 8, "Value #query-3": 9, "horizontalpodautoscaler": 1, "namespace": 0 }, "renameByName": { "Value #query-0": "Current", "Value #query-1": "Desired", "Value #query-2": "Min", "Value #query-3": "Max", "horizontalpodautoscaler": "Autoscaler", "namespace": "Namespace" } } } } ] } }, "description": "**Every HorizontalPodAutoscaler with its current and desired replicas and the bounds it scales between.**\n\nDesired above current is a scale-up under way, or one blocked because the new pods cannot be scheduled. Current equal to the maximum is a workload that cannot grow any further by adding pods.", "id": 1026, "links": [], "title": "HorizontalPodAutoscalers", "vizConfig": { "group": "table", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "custom": { "align": "auto", "filterable": true, "inspect": false }, "decimals": 0, "noValue": "No HorizontalPodAutoscalers", "unit": "short" }, "overrides": [] }, "options": { "cellHeight": "sm", "footer": { "countRows": false, "reducer": [], "show": false }, "frameIndex": 0, "showHeader": true, "showTypeIcons": false } }, "version": "" } } }, "hpa-unable": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (condition) (\n max by (namespace, horizontalpodautoscaler, condition) (\n kube_horizontalpodautoscaler_status_condition{condition=~\"AbleToScale|ScalingActive\", status=\"false\"}\n )\n)\n", "legendFormat": "{{condition}} false", "qryType": 1, "query": "sum by (condition) (\n max by (namespace, horizontalpodautoscaler, condition) (\n kube_horizontalpodautoscaler_status_condition{condition=~\"AbleToScale|ScalingActive\", status=\"false\"}\n )\n)\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**HorizontalPodAutoscalers that cannot act, by the condition that stopped them.**\n\n**Nominal:** Zero.\n\n**Unhealthy:** `ScalingActive` false means the autoscaler cannot read the metric it scales on, usually because metrics-server is down or the pods set no resource request, and it holds the workload at its current size. `AbleToScale` false means it cannot change the workload's replica count at all.", "id": 1028, "links": [], "title": "Autoscalers Unable to Scale", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "decimals": 0, "min": 0, "noValue": "No HorizontalPodAutoscalers", "unit": "short" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "overview-added": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "count(\n max by (node) (kube_node_info)\n unless max by (node) (kube_node_info offset $__range)\n)\nor 0 * count(max by (node) (kube_node_info))\n", "instant": true, "legendFormat": "added", "qryType": 1, "query": "count(\n max by (node) (kube_node_info)\n unless max by (node) (kube_node_info offset $__range)\n)\nor 0 * count(max by (node) (kube_node_info))\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Nodes in the cluster now that were not at the start of the selected time range.**\n\nNot read from the `RegisteredNode` event, which the node controller repeats for every node whenever the managed control plane restarts. A node that joined and left within the range is not counted; _Nodes Removed_ counts it leaving.", "id": 1004, "links": [], "title": "Nodes Added (Select Time Range)", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "fixedColor": "#0077BB", "mode": "shades" }, "decimals": 0, "min": 0, "noValue": "No metrics: kube-state-metrics is required", "unit": "short" }, "overrides": [] }, "options": { "colorMode": "none", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [], "values": false }, "showPercentChange": false, "textMode": "value", "wideLayout": true } }, "version": "" } } }, "overview-events-by-reason": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${logsDatasource}" }, "group": "loki", "kind": "DataQuery", "spec": { "expr": "sum by (reason) (\n count_over_time(\n {job=\"loki.source.kubernetes_events\"}\n | reportingcontroller=~\"karpenter|cluster-autoscaler|node-controller|cloud-node-lifecycle-controller\" or reason=~\"FailedScheduling|Preempted|TriggeredScaleUp|NotTriggerScaleUp|FailedScaleUp|ScaledUpGroup|ScaleDown\"\n | reason!~\"DisruptionBlocked|Unconsolidatable\"\n [1m])\n)\n", "legendFormat": "{{reason}}" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Node and autoscaler events per minute, by reason.**\n\nFrom Karpenter, the cluster autoscaler, the node lifecycle and the scheduler's placement failures. Karpenter's `DisruptionBlocked` and `Unconsolidatable` are left out: they repeat every few minutes for every node holding a pod that must not be moved, which on a Materialize cluster is most of them, and the Karpenter dashboard breaks them down.", "id": 1010, "links": [], "title": "Autoscaling Events by Reason", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "No autoscaling events in this time range", "unit": "short" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "overview-hpa-at-max": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "count(\n max by (namespace, horizontalpodautoscaler) (kube_horizontalpodautoscaler_status_current_replicas)\n >= max by (namespace, horizontalpodautoscaler) (kube_horizontalpodautoscaler_spec_max_replicas)\n)\nor 0 * count(max by (namespace, horizontalpodautoscaler) (kube_horizontalpodautoscaler_spec_max_replicas))\n", "legendFormat": "at maximum", "qryType": 1, "query": "count(\n max by (namespace, horizontalpodautoscaler) (kube_horizontalpodautoscaler_status_current_replicas)\n >= max by (namespace, horizontalpodautoscaler) (kube_horizontalpodautoscaler_spec_max_replicas)\n)\nor 0 * count(max by (namespace, horizontalpodautoscaler) (kube_horizontalpodautoscaler_spec_max_replicas))\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Workloads whose HorizontalPodAutoscaler is running as many replicas as it is allowed to.**\n\n**Nominal:** Zero, or a workload that sits at its maximum by design.\n\n**Degraded:** A workload at its maximum cannot absorb more load by adding pods. If its CPU or memory is high, raise the maximum or give each pod more.", "id": 1006, "links": [], "title": "Workloads at Max Replicas", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "decimals": 0, "min": 0, "noValue": "No HorizontalPodAutoscalers", "thresholds": { "mode": "absolute", "steps": [ { "color": "#A2F49B", "value": null }, { "color": "#FEDA8B", "value": 1 }, { "color": "#FDB366", "value": 1.5 }, { "color": "#F67E4B", "value": 2 }, { "color": "#DD3D2D", "value": 2.5 }, { "color": "#A50026", "value": 3 } ] }, "unit": "short" }, "overrides": [] }, "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [], "values": false }, "showPercentChange": false, "textMode": "value", "wideLayout": true } }, "version": "" } } }, "overview-nodes": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "count(max by (node) (kube_node_info))\n", "legendFormat": "nodes", "qryType": 1, "query": "count(max by (node) (kube_node_info))\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Nodes in the cluster.**\n\nEvery node, whatever provisioned it, including nodes that are not yet Ready or are being drained.", "id": 1002, "links": [], "title": "Nodes", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "fixedColor": "#0077BB", "mode": "shades" }, "decimals": 0, "min": 0, "noValue": "No metrics: kube-state-metrics is required", "unit": "short" }, "overrides": [] }, "options": { "colorMode": "none", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [], "values": false }, "showPercentChange": false, "textMode": "value", "wideLayout": true } }, "version": "" } } }, "overview-nodes-by-pool": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "count by (pool) (max by (node, pool, instance_type, zone) (label_replace(label_replace(label_replace(label_replace(label_replace(label_replace(label_replace(kube_node_labels, \"pool\", \"$1\", \"label_karpenter_sh_nodepool\", \"(.+)\"), \"pool\", \"$1\", \"label_eks_amazonaws_com_nodegroup\", \"(.+)\"), \"pool\", \"$1\", \"label_cloud_google_com_gke_nodepool\", \"(.+)\"), \"pool\", \"$1\", \"label_kubernetes_azure_com_agentpool\", \"(.+)\"), \"pool\", \"unpooled\", \"pool\", \"\"), \"instance_type\", \"$1\", \"label_node_kubernetes_io_instance_type\", \"(.+)\"), \"zone\", \"$1\", \"label_topology_kubernetes_io_zone\", \"(.+)\")))\n", "legendFormat": "{{pool}}", "qryType": 1, "query": "count by (pool) (max by (node, pool, instance_type, zone) (label_replace(label_replace(label_replace(label_replace(label_replace(label_replace(label_replace(kube_node_labels, \"pool\", \"$1\", \"label_karpenter_sh_nodepool\", \"(.+)\"), \"pool\", \"$1\", \"label_eks_amazonaws_com_nodegroup\", \"(.+)\"), \"pool\", \"$1\", \"label_cloud_google_com_gke_nodepool\", \"(.+)\"), \"pool\", \"$1\", \"label_kubernetes_azure_com_agentpool\", \"(.+)\"), \"pool\", \"unpooled\", \"pool\", \"\"), \"instance_type\", \"$1\", \"label_node_kubernetes_io_instance_type\", \"(.+)\"), \"zone\", \"$1\", \"label_topology_kubernetes_io_zone\", \"(.+)\")))\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Nodes in each node pool over time.**\n\nA step up is an autoscaler adding a node; a step down is one removing a node it judged empty or replaceable. The pool is the Karpenter NodePool, EKS node group, GKE node pool or AKS agent pool that owns the node, and `unpooled` a node with none of those labels. Every node reads `unpooled` until kube-state-metrics publishes node labels, which the chart configures.", "id": 1008, "links": [], "title": "Nodes by Pool", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "decimals": 0, "min": 0, "noValue": "No node labels. kube-state-metrics publishes them only with the chart's metricLabelsAllowlist.", "unit": "short" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "overview-not-ready": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(1 - max by (node) (kube_node_status_condition{condition=\"Ready\", status=\"true\"}))\n", "legendFormat": "not ready", "qryType": 1, "query": "sum(1 - max by (node) (kube_node_status_condition{condition=\"Ready\", status=\"true\"}))\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Nodes whose kubelet is not reporting Ready.**\n\n**Nominal:** Zero. A new node reads not-ready for its first minute or so while its networking starts.\n\n**Unhealthy:** A node that stays not-ready. Its pods are evicted after five minutes, and an autoscaler counts it as capacity it cannot use. Infrastructure Node Detail has the node's own view.", "id": 1003, "links": [], "title": "Nodes Not Ready", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "decimals": 0, "min": 0, "noValue": "No metrics: kube-state-metrics is required", "thresholds": { "mode": "absolute", "steps": [ { "color": "#A2F49B", "value": null }, { "color": "#FEDA8B", "value": 1 }, { "color": "#FDB366", "value": 1.25 }, { "color": "#F67E4B", "value": 1.5 }, { "color": "#DD3D2D", "value": 1.75 }, { "color": "#A50026", "value": 2 } ] }, "unit": "short" }, "overrides": [] }, "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [], "values": false }, "showPercentChange": false, "textMode": "value", "wideLayout": true } }, "version": "" } } }, "overview-pending": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(max by (namespace, pod) (kube_pod_status_phase{phase=\"Pending\"}))\n", "legendFormat": "pending", "qryType": 1, "query": "sum(max by (namespace, pod) (kube_pod_status_phase{phase=\"Pending\"}))\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Pods in the Pending phase right now, across the cluster.**\n\n**Nominal:** Zero most of the time. A pod is Pending from creation until every container has started, so brief non-zero readings during a rollout are normal.\n\n**Degraded:** Pods that stay Pending for minutes. Compare with _Pods Waiting for a Node_: if those are the same pods, the cluster has no room for them; if not, they have a node and are stuck starting, usually pulling an image or mounting a volume.", "id": 1000, "links": [], "title": "Pods Pending", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "decimals": 0, "min": 0, "noValue": "No metrics: kube-state-metrics is required", "thresholds": { "mode": "absolute", "steps": [ { "color": "#A2F49B", "value": null }, { "color": "#FEDA8B", "value": 1 }, { "color": "#FDB366", "value": 3.25 }, { "color": "#F67E4B", "value": 5.5 }, { "color": "#DD3D2D", "value": 7.75 }, { "color": "#A50026", "value": 10 } ] }, "unit": "short" }, "overrides": [] }, "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [], "values": false }, "showPercentChange": false, "textMode": "value", "wideLayout": true } }, "version": "" } } }, "overview-removed": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${logsDatasource}" }, "group": "loki", "kind": "DataQuery", "spec": { "expr": "count(\n sum by (name) (\n count_over_time(\n {job=\"loki.source.kubernetes_events\"} | reason=~\"RemovingNode|DeletingNode\"\n [$__range])\n )\n)\n", "instant": true, "legendFormat": "removed" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Nodes that left the cluster in the selected time range.**\n\nCounted once per node from its `RemovingNode` or `DeletingNode` event, the node controller's and the cloud's record of the same removal.", "id": 1005, "links": [], "title": "Nodes Removed (Select Time Range)", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "fixedColor": "#0077BB", "mode": "shades" }, "decimals": 0, "min": 0, "noValue": "None", "unit": "short" }, "overrides": [] }, "options": { "colorMode": "none", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [], "values": false }, "showPercentChange": false, "textMode": "value", "wideLayout": true } }, "version": "" } } }, "overview-requested": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(\n max by (namespace, pod, container, node) (kube_pod_container_resource_requests{resource=\"cpu\", node!=\"\"})\n unless on (namespace, pod) (max by (namespace, pod) (kube_pod_status_phase{phase=~\"Succeeded|Failed\"}) == 1)\n)\n/\nsum(max by (node) (kube_node_status_allocatable{resource=\"cpu\"}))\n", "legendFormat": "CPU", "qryType": 1, "query": "sum(\n max by (namespace, pod, container, node) (kube_pod_container_resource_requests{resource=\"cpu\", node!=\"\"})\n unless on (namespace, pod) (max by (namespace, pod) (kube_pod_status_phase{phase=~\"Succeeded|Failed\"}) == 1)\n)\n/\nsum(max by (node) (kube_node_status_allocatable{resource=\"cpu\"}))\n" }, "version": "v0" }, "refId": "query-0" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(\n max by (namespace, pod, container, node) (kube_pod_container_resource_requests{resource=\"memory\", node!=\"\"})\n unless on (namespace, pod) (max by (namespace, pod) (kube_pod_status_phase{phase=~\"Succeeded|Failed\"}) == 1)\n)\n/\nsum(max by (node) (kube_node_status_allocatable{resource=\"memory\"}))\n", "legendFormat": "Memory", "qryType": 1, "query": "sum(\n max by (namespace, pod, container, node) (kube_pod_container_resource_requests{resource=\"memory\", node!=\"\"})\n unless on (namespace, pod) (max by (namespace, pod) (kube_pod_status_phase{phase=~\"Succeeded|Failed\"}) == 1)\n)\n/\nsum(max by (node) (kube_node_status_allocatable{resource=\"memory\"}))\n" }, "version": "v0" }, "refId": "query-1" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**How much of the cluster's allocatable CPU and memory is promised to running pods through their requests.**\n\n**Nominal:** Below about 0.85. An autoscaler keeps this high on purpose, removing nodes whose pods fit elsewhere, so a high reading with no pods waiting is efficiency rather than a problem.\n\n**Degraded:** Near 1.0 with pods waiting for a node: the cluster is full and not growing. The Pending Pods and Cloud Capacity tabs say why.\n\n**Requests, not usage**, the number the scheduler places pods by. Completed pods are excluded; pods still waiting for a node are too, since they are not on one yet.", "id": 1009, "links": [], "title": "Requested Share of Allocatable", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "max": 1, "min": 0, "noValue": "No metrics: kube-state-metrics is required", "unit": "percentunit" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "overview-unscheduled": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(max by (namespace, pod) (kube_pod_status_scheduled{condition=\"false\"}))\n", "legendFormat": "waiting", "qryType": 1, "query": "sum(max by (namespace, pod) (kube_pod_status_scheduled{condition=\"false\"}))\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Pods the scheduler has not placed on any node, because no node has room for them or satisfies their constraints.**\n\n**Nominal:** Zero, or briefly non-zero while an autoscaler adds a node. These are exactly the pods an autoscaler acts on.\n\n**Degraded:** Non-zero for longer than a node takes to start, a few minutes. Either the autoscaler is not adding a node, or the node it adds does not fit the pod. The Pending Pods tab says which.\n\nRead from the pod's `PodScheduled` condition, which is false exactly while the scheduler has not found it a node.", "id": 1001, "links": [], "title": "Pods Waiting for a Node", "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "decimals": 0, "min": 0, "noValue": "No metrics: kube-state-metrics is required", "thresholds": { "mode": "absolute", "steps": [ { "color": "#A2F49B", "value": null }, { "color": "#FEDA8B", "value": 1 }, { "color": "#FDB366", "value": 2 }, { "color": "#F67E4B", "value": 3 }, { "color": "#DD3D2D", "value": 4 }, { "color": "#A50026", "value": 5 } ] }, "unit": "short" }, "overrides": [] }, "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [], "values": false }, "showPercentChange": false, "textMode": "value", "wideLayout": true } }, "version": "" } } }, "overview-unscheduled-by-namespace": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (namespace) (max by (namespace, pod) (kube_pod_status_scheduled{condition=\"false\"})) > 0\n", "legendFormat": "{{namespace}}", "qryType": 1, "query": "sum by (namespace) (max by (namespace, pod) (kube_pod_status_scheduled{condition=\"false\"})) > 0\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Pods waiting for a node, by namespace.**\n\n**Nominal:** Empty, or a brief spike while a node is added.\n\n**Degraded:** A namespace whose pods keep waiting. In a Materialize namespace, that is a cluster replica that cannot start; the Pending Pods tab lists each pod and the scheduler's reason.", "id": 1007, "links": [], "title": "Pods Waiting for a Node by Namespace", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "decimals": 0, "min": 0, "noValue": "No pods waiting for a node", "unit": "short" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "overview-warnings": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${logsDatasource}" }, "group": "loki", "kind": "DataQuery", "spec": { "expr": "{job=\"loki.source.kubernetes_events\", level=\"WARN\"}\n | reportingcontroller=~\"karpenter|cluster-autoscaler|node-controller|cloud-node-lifecycle-controller\" or reason=~\"FailedScheduling|Preempted|NotTriggerScaleUp|FailedScaleUp\"\n | line_format \"{{.reason}} {{.kind}}/{{.name}} — {{.msg}}\"\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Warning events from the autoscalers and the node lifecycle.**\n\n**Nominal:** Occasional scheduling failures that clear once a node is added.\n\n**Degraded:** The same warning repeating: a pod that cannot be placed, a node that cannot be drained, or a scale-up the cloud refused.", "id": 1011, "links": [], "title": "Warning Events", "vizConfig": { "group": "logs", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "noValue": "No warning events in this time range" }, "overrides": [] }, "options": { "dedupStrategy": "signature", "detailsMode": "inline", "displayedFields": [], "enableInfiniteScrolling": true, "enableLogDetails": true, "fontSize": "small", "prettifyLogMessage": false, "showCommonLabels": false, "showControls": true, "showFieldSelector": true, "showLabels": false, "showLevel": true, "showLogContextToggle": false, "showTime": true, "sortOrder": "Descending", "syntaxHighlighting": true, "timestampResolution": "ms", "unwrappedColumns": false, "wrapLogMessage": true } }, "version": "" } } }, "pending-failure-rate": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${logsDatasource}" }, "group": "loki", "kind": "DataQuery", "spec": { "expr": "sum by (namespace) (\n count_over_time(\n {job=\"loki.source.kubernetes_events\"} | reason=\"FailedScheduling\"\n [1m])\n)\n", "legendFormat": "{{namespace}}" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Scheduling failures per minute, by namespace.**\n\nThe scheduler retries a pod it cannot place and records each attempt, so this tracks how long pods have been waiting as much as how many.", "id": 1022, "links": [], "title": "Scheduling Failures per Minute", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "min": 0, "noValue": "No scheduling failures in this time range", "unit": "short" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "pending-list": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "(time() - max by (namespace, pod) (kube_pod_created))\nand on (namespace, pod) (max by (namespace, pod) (kube_pod_status_scheduled{condition=\"false\"}) == 1)\n", "format": "table", "instant": true, "qryType": 2, "query": "(time() - max by (namespace, pod) (kube_pod_created))\nand on (namespace, pod) (max by (namespace, pod) (kube_pod_status_scheduled{condition=\"false\"}) == 1)\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [ { "group": "organize", "kind": "Transformation", "spec": { "options": { "excludeByName": { "Time": true, "__name__": true }, "indexByName": { "Value": 2, "namespace": 0, "pod": 1 }, "renameByName": { "Value": "Age", "namespace": "Namespace", "pod": "Pod" } } } }, { "group": "sortBy", "kind": "Transformation", "spec": { "options": { "fields": {}, "sort": [ { "desc": true, "field": "Age" } ] } } } ] } }, "description": "**Each pod waiting for a node, and how long since it was created.**\n\n**Nominal:** Empty.\n\n**Degraded:** Pods that have waited longer than a node takes to start. The scheduler's reason for each is in the feed beside this, and the autoscaler's response below it.", "id": 1023, "links": [], "title": "Pods Waiting for a Node", "vizConfig": { "group": "table", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "custom": { "align": "auto", "filterable": true, "inspect": false }, "noValue": "Every pod has a node.", "unit": "s" }, "overrides": [] }, "options": { "cellHeight": "sm", "footer": { "countRows": false, "reducer": [], "show": false }, "frameIndex": 0, "showHeader": true, "showTypeIcons": false } }, "version": "" } } }, "pending-pending": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (namespace) (max by (namespace, pod) (kube_pod_status_phase{phase=\"Pending\"})) > 0\n", "legendFormat": "{{namespace}}", "qryType": 1, "query": "sum by (namespace) (max by (namespace, pod) (kube_pod_status_phase{phase=\"Pending\"})) > 0\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Pods in the Pending phase, by namespace.**\n\nIncludes pods that have a node and are still starting. The gap between this and _Pods Waiting for a Node_ is pods pulling images or mounting volumes.", "id": 1021, "links": [], "title": "Pods Pending by Namespace", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "decimals": 0, "min": 0, "noValue": "No pods pending", "unit": "short" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "pending-scale-up-decisions": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${logsDatasource}" }, "group": "loki", "kind": "DataQuery", "spec": { "expr": "{job=\"loki.source.kubernetes_events\"}\n | reason=~\"Nominated|TriggeredScaleUp|NotTriggerScaleUp|FailedScaleUp|ScaledUpGroup\"\n | line_format \"{{.reason}} {{.kind}}/{{.name}} — {{.msg}}\"\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**What each autoscaler did about pods waiting for a node, newest first.**\n\nKarpenter records `Nominated` when it has chosen a node for a pod, new or existing. The cluster autoscaler on GKE and AKS records `TriggeredScaleUp` when it adds a node for a pod, and `NotTriggerScaleUp` with the reason when no node group could take it: a group at its maximum size, or none whose nodes would fit the pod. `FailedScaleUp` is a scale-up the cloud refused, usually for quota or capacity; the Cloud Capacity tab says which.", "id": 1025, "links": [], "title": "Autoscaler Decisions", "vizConfig": { "group": "logs", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "noValue": "No autoscaler decisions in this time range" }, "overrides": [] }, "options": { "dedupStrategy": "signature", "detailsMode": "inline", "displayedFields": [], "enableInfiniteScrolling": true, "enableLogDetails": true, "fontSize": "small", "prettifyLogMessage": false, "showCommonLabels": false, "showControls": true, "showFieldSelector": true, "showLabels": false, "showLevel": true, "showLogContextToggle": false, "showTime": true, "sortOrder": "Descending", "syntaxHighlighting": true, "timestampResolution": "ms", "unwrappedColumns": false, "wrapLogMessage": true } }, "version": "" } } }, "pending-scheduling-failures": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${logsDatasource}" }, "group": "loki", "kind": "DataQuery", "spec": { "expr": "{job=\"loki.source.kubernetes_events\"}\n | reason=\"FailedScheduling\"\n | line_format \"{{.name}} — {{.msg}}\"\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**The scheduler's explanation for each pod it could not place, newest first.**\n\nThe message counts the nodes and the reason each was ruled out: `Insufficient memory` means no node has room, while `didn't match Pod's node affinity/selector` or `had untolerated taint` means no node of the right kind exists yet. An autoscaler adds a node for the first kind as long as a pool can hold the pod, and for the second only if a pool offers the kind asked for.", "id": 1024, "links": [], "title": "Scheduling Failures", "vizConfig": { "group": "logs", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "noValue": "No scheduling failures in this time range" }, "overrides": [] }, "options": { "dedupStrategy": "signature", "detailsMode": "inline", "displayedFields": [], "enableInfiniteScrolling": true, "enableLogDetails": true, "fontSize": "small", "prettifyLogMessage": false, "showCommonLabels": false, "showControls": true, "showFieldSelector": true, "showLabels": false, "showLevel": true, "showLogContextToggle": false, "showTime": true, "sortOrder": "Descending", "syntaxHighlighting": true, "timestampResolution": "ms", "unwrappedColumns": false, "wrapLogMessage": true } }, "version": "" } } }, "pending-unscheduled": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (namespace) (max by (namespace, pod) (kube_pod_status_scheduled{condition=\"false\"})) > 0\n", "legendFormat": "{{namespace}}", "qryType": 1, "query": "sum by (namespace) (max by (namespace, pod) (kube_pod_status_scheduled{condition=\"false\"})) > 0\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Pods waiting for a node, by namespace.**\n\n**Nominal:** Empty, or a brief spike while a node is added.\n\n**Degraded:** A namespace whose pods keep waiting. In a Materialize namespace, that is a cluster replica that cannot start; the Pending Pods tab lists each pod and the scheduler's reason.", "id": 1020, "links": [], "title": "Pods Waiting for a Node by Namespace", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "decimals": 0, "min": 0, "noValue": "No pods waiting for a node", "unit": "short" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "pools-cordoned": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (pool) (\n max by (node) (kube_node_spec_unschedulable)\n * on (node) group_left (pool) max by (node, pool, instance_type, zone) (label_replace(label_replace(label_replace(label_replace(label_replace(label_replace(label_replace(kube_node_labels, \"pool\", \"$1\", \"label_karpenter_sh_nodepool\", \"(.+)\"), \"pool\", \"$1\", \"label_eks_amazonaws_com_nodegroup\", \"(.+)\"), \"pool\", \"$1\", \"label_cloud_google_com_gke_nodepool\", \"(.+)\"), \"pool\", \"$1\", \"label_kubernetes_azure_com_agentpool\", \"(.+)\"), \"pool\", \"unpooled\", \"pool\", \"\"), \"instance_type\", \"$1\", \"label_node_kubernetes_io_instance_type\", \"(.+)\"), \"zone\", \"$1\", \"label_topology_kubernetes_io_zone\", \"(.+)\"))\n)\n", "legendFormat": "{{pool}}", "qryType": 1, "query": "sum by (pool) (\n max by (node) (kube_node_spec_unschedulable)\n * on (node) group_left (pool) max by (node, pool, instance_type, zone) (label_replace(label_replace(label_replace(label_replace(label_replace(label_replace(label_replace(kube_node_labels, \"pool\", \"$1\", \"label_karpenter_sh_nodepool\", \"(.+)\"), \"pool\", \"$1\", \"label_eks_amazonaws_com_nodegroup\", \"(.+)\"), \"pool\", \"$1\", \"label_cloud_google_com_gke_nodepool\", \"(.+)\"), \"pool\", \"$1\", \"label_kubernetes_azure_com_agentpool\", \"(.+)\"), \"pool\", \"unpooled\", \"pool\", \"\"), \"instance_type\", \"$1\", \"label_node_kubernetes_io_instance_type\", \"(.+)\"), \"zone\", \"$1\", \"label_topology_kubernetes_io_zone\", \"(.+)\"))\n)\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Nodes marked unschedulable, by node pool.**\n\n**Nominal:** Zero, or briefly one while an autoscaler or an upgrade drains a node before removing it.\n\n**Degraded:** A node that stays cordoned. Something started draining it and did not finish, usually because a pod cannot be evicted: a PodDisruptionBudget at its limit, or a Materialize pod marked not to be disrupted.", "id": 1018, "links": [], "title": "Cordoned Nodes by Pool", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "decimals": 0, "min": 0, "noValue": "No node labels. kube-state-metrics publishes them only with the chart's metricLabelsAllowlist.", "unit": "short" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "pools-cpu-requested": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (pool) (\n sum by (node) (\n max by (namespace, pod, container, node) (kube_pod_container_resource_requests{resource=\"cpu\", node!=\"\"})\n unless on (namespace, pod) (max by (namespace, pod) (kube_pod_status_phase{phase=~\"Succeeded|Failed\"}) == 1)\n )\n * on (node) group_left (pool) max by (node, pool, instance_type, zone) (label_replace(label_replace(label_replace(label_replace(label_replace(label_replace(label_replace(kube_node_labels, \"pool\", \"$1\", \"label_karpenter_sh_nodepool\", \"(.+)\"), \"pool\", \"$1\", \"label_eks_amazonaws_com_nodegroup\", \"(.+)\"), \"pool\", \"$1\", \"label_cloud_google_com_gke_nodepool\", \"(.+)\"), \"pool\", \"$1\", \"label_kubernetes_azure_com_agentpool\", \"(.+)\"), \"pool\", \"unpooled\", \"pool\", \"\"), \"instance_type\", \"$1\", \"label_node_kubernetes_io_instance_type\", \"(.+)\"), \"zone\", \"$1\", \"label_topology_kubernetes_io_zone\", \"(.+)\"))\n)\n/\nsum by (pool) (\n max by (node) (kube_node_status_allocatable{resource=\"cpu\"})\n * on (node) group_left (pool) max by (node, pool, instance_type, zone) (label_replace(label_replace(label_replace(label_replace(label_replace(label_replace(label_replace(kube_node_labels, \"pool\", \"$1\", \"label_karpenter_sh_nodepool\", \"(.+)\"), \"pool\", \"$1\", \"label_eks_amazonaws_com_nodegroup\", \"(.+)\"), \"pool\", \"$1\", \"label_cloud_google_com_gke_nodepool\", \"(.+)\"), \"pool\", \"$1\", \"label_kubernetes_azure_com_agentpool\", \"(.+)\"), \"pool\", \"unpooled\", \"pool\", \"\"), \"instance_type\", \"$1\", \"label_node_kubernetes_io_instance_type\", \"(.+)\"), \"zone\", \"$1\", \"label_topology_kubernetes_io_zone\", \"(.+)\"))\n)\n", "legendFormat": "{{pool}}", "qryType": 1, "query": "sum by (pool) (\n sum by (node) (\n max by (namespace, pod, container, node) (kube_pod_container_resource_requests{resource=\"cpu\", node!=\"\"})\n unless on (namespace, pod) (max by (namespace, pod) (kube_pod_status_phase{phase=~\"Succeeded|Failed\"}) == 1)\n )\n * on (node) group_left (pool) max by (node, pool, instance_type, zone) (label_replace(label_replace(label_replace(label_replace(label_replace(label_replace(label_replace(kube_node_labels, \"pool\", \"$1\", \"label_karpenter_sh_nodepool\", \"(.+)\"), \"pool\", \"$1\", \"label_eks_amazonaws_com_nodegroup\", \"(.+)\"), \"pool\", \"$1\", \"label_cloud_google_com_gke_nodepool\", \"(.+)\"), \"pool\", \"$1\", \"label_kubernetes_azure_com_agentpool\", \"(.+)\"), \"pool\", \"unpooled\", \"pool\", \"\"), \"instance_type\", \"$1\", \"label_node_kubernetes_io_instance_type\", \"(.+)\"), \"zone\", \"$1\", \"label_topology_kubernetes_io_zone\", \"(.+)\"))\n)\n/\nsum by (pool) (\n max by (node) (kube_node_status_allocatable{resource=\"cpu\"})\n * on (node) group_left (pool) max by (node, pool, instance_type, zone) (label_replace(label_replace(label_replace(label_replace(label_replace(label_replace(label_replace(kube_node_labels, \"pool\", \"$1\", \"label_karpenter_sh_nodepool\", \"(.+)\"), \"pool\", \"$1\", \"label_eks_amazonaws_com_nodegroup\", \"(.+)\"), \"pool\", \"$1\", \"label_cloud_google_com_gke_nodepool\", \"(.+)\"), \"pool\", \"$1\", \"label_kubernetes_azure_com_agentpool\", \"(.+)\"), \"pool\", \"unpooled\", \"pool\", \"\"), \"instance_type\", \"$1\", \"label_node_kubernetes_io_instance_type\", \"(.+)\"), \"zone\", \"$1\", \"label_topology_kubernetes_io_zone\", \"(.+)\"))\n)\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**The share of each node pool's allocatable CPU promised to running pods.**\n\n**Nominal:** Anywhere below 1.0. Pools an autoscaler manages sit high, because it removes nodes whose pods would fit elsewhere.\n\n**Degraded:** A pool near 1.0 with pods waiting for it. A pool that stays low is either sized for a workload that has gone, or held up by pods that block consolidation; on EKS the Karpenter dashboard says which.", "id": 1015, "links": [], "title": "CPU Requested by Pool", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "max": 1, "min": 0, "noValue": "No node labels. kube-state-metrics publishes them only with the chart's metricLabelsAllowlist.", "unit": "percentunit" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "pools-memory-requested": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (pool) (\n sum by (node) (\n max by (namespace, pod, container, node) (kube_pod_container_resource_requests{resource=\"memory\", node!=\"\"})\n unless on (namespace, pod) (max by (namespace, pod) (kube_pod_status_phase{phase=~\"Succeeded|Failed\"}) == 1)\n )\n * on (node) group_left (pool) max by (node, pool, instance_type, zone) (label_replace(label_replace(label_replace(label_replace(label_replace(label_replace(label_replace(kube_node_labels, \"pool\", \"$1\", \"label_karpenter_sh_nodepool\", \"(.+)\"), \"pool\", \"$1\", \"label_eks_amazonaws_com_nodegroup\", \"(.+)\"), \"pool\", \"$1\", \"label_cloud_google_com_gke_nodepool\", \"(.+)\"), \"pool\", \"$1\", \"label_kubernetes_azure_com_agentpool\", \"(.+)\"), \"pool\", \"unpooled\", \"pool\", \"\"), \"instance_type\", \"$1\", \"label_node_kubernetes_io_instance_type\", \"(.+)\"), \"zone\", \"$1\", \"label_topology_kubernetes_io_zone\", \"(.+)\"))\n)\n/\nsum by (pool) (\n max by (node) (kube_node_status_allocatable{resource=\"memory\"})\n * on (node) group_left (pool) max by (node, pool, instance_type, zone) (label_replace(label_replace(label_replace(label_replace(label_replace(label_replace(label_replace(kube_node_labels, \"pool\", \"$1\", \"label_karpenter_sh_nodepool\", \"(.+)\"), \"pool\", \"$1\", \"label_eks_amazonaws_com_nodegroup\", \"(.+)\"), \"pool\", \"$1\", \"label_cloud_google_com_gke_nodepool\", \"(.+)\"), \"pool\", \"$1\", \"label_kubernetes_azure_com_agentpool\", \"(.+)\"), \"pool\", \"unpooled\", \"pool\", \"\"), \"instance_type\", \"$1\", \"label_node_kubernetes_io_instance_type\", \"(.+)\"), \"zone\", \"$1\", \"label_topology_kubernetes_io_zone\", \"(.+)\"))\n)\n", "legendFormat": "{{pool}}", "qryType": 1, "query": "sum by (pool) (\n sum by (node) (\n max by (namespace, pod, container, node) (kube_pod_container_resource_requests{resource=\"memory\", node!=\"\"})\n unless on (namespace, pod) (max by (namespace, pod) (kube_pod_status_phase{phase=~\"Succeeded|Failed\"}) == 1)\n )\n * on (node) group_left (pool) max by (node, pool, instance_type, zone) (label_replace(label_replace(label_replace(label_replace(label_replace(label_replace(label_replace(kube_node_labels, \"pool\", \"$1\", \"label_karpenter_sh_nodepool\", \"(.+)\"), \"pool\", \"$1\", \"label_eks_amazonaws_com_nodegroup\", \"(.+)\"), \"pool\", \"$1\", \"label_cloud_google_com_gke_nodepool\", \"(.+)\"), \"pool\", \"$1\", \"label_kubernetes_azure_com_agentpool\", \"(.+)\"), \"pool\", \"unpooled\", \"pool\", \"\"), \"instance_type\", \"$1\", \"label_node_kubernetes_io_instance_type\", \"(.+)\"), \"zone\", \"$1\", \"label_topology_kubernetes_io_zone\", \"(.+)\"))\n)\n/\nsum by (pool) (\n max by (node) (kube_node_status_allocatable{resource=\"memory\"})\n * on (node) group_left (pool) max by (node, pool, instance_type, zone) (label_replace(label_replace(label_replace(label_replace(label_replace(label_replace(label_replace(kube_node_labels, \"pool\", \"$1\", \"label_karpenter_sh_nodepool\", \"(.+)\"), \"pool\", \"$1\", \"label_eks_amazonaws_com_nodegroup\", \"(.+)\"), \"pool\", \"$1\", \"label_cloud_google_com_gke_nodepool\", \"(.+)\"), \"pool\", \"$1\", \"label_kubernetes_azure_com_agentpool\", \"(.+)\"), \"pool\", \"unpooled\", \"pool\", \"\"), \"instance_type\", \"$1\", \"label_node_kubernetes_io_instance_type\", \"(.+)\"), \"zone\", \"$1\", \"label_topology_kubernetes_io_zone\", \"(.+)\"))\n)\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**The share of each node pool's allocatable memory promised to running pods.**\n\n**Nominal:** Anywhere below 1.0.\n\n**Degraded:** A Materialize pool near 1.0: the next cluster replica, or a larger size, needs a new node. Memory is usually what fills first on a Materialize node.", "id": 1016, "links": [], "title": "Memory Requested by Pool", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "max": 1, "min": 0, "noValue": "No node labels. kube-state-metrics publishes them only with the chart's metricLabelsAllowlist.", "unit": "percentunit" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "pools-node-age": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "(time() - max by (node) (kube_node_created))\n* on (node) group_left (pool, instance_type, zone) max by (node, pool, instance_type, zone) (label_replace(label_replace(label_replace(label_replace(label_replace(label_replace(label_replace(kube_node_labels, \"pool\", \"$1\", \"label_karpenter_sh_nodepool\", \"(.+)\"), \"pool\", \"$1\", \"label_eks_amazonaws_com_nodegroup\", \"(.+)\"), \"pool\", \"$1\", \"label_cloud_google_com_gke_nodepool\", \"(.+)\"), \"pool\", \"$1\", \"label_kubernetes_azure_com_agentpool\", \"(.+)\"), \"pool\", \"unpooled\", \"pool\", \"\"), \"instance_type\", \"$1\", \"label_node_kubernetes_io_instance_type\", \"(.+)\"), \"zone\", \"$1\", \"label_topology_kubernetes_io_zone\", \"(.+)\"))\n", "format": "table", "instant": true, "qryType": 2, "query": "(time() - max by (node) (kube_node_created))\n* on (node) group_left (pool, instance_type, zone) max by (node, pool, instance_type, zone) (label_replace(label_replace(label_replace(label_replace(label_replace(label_replace(label_replace(kube_node_labels, \"pool\", \"$1\", \"label_karpenter_sh_nodepool\", \"(.+)\"), \"pool\", \"$1\", \"label_eks_amazonaws_com_nodegroup\", \"(.+)\"), \"pool\", \"$1\", \"label_cloud_google_com_gke_nodepool\", \"(.+)\"), \"pool\", \"$1\", \"label_kubernetes_azure_com_agentpool\", \"(.+)\"), \"pool\", \"unpooled\", \"pool\", \"\"), \"instance_type\", \"$1\", \"label_node_kubernetes_io_instance_type\", \"(.+)\"), \"zone\", \"$1\", \"label_topology_kubernetes_io_zone\", \"(.+)\"))\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [ { "group": "organize", "kind": "Transformation", "spec": { "options": { "excludeByName": { "Time": true, "__name__": true }, "indexByName": { "Value": 4, "instance_type": 2, "node": 0, "pool": 1, "zone": 3 }, "renameByName": { "Value": "Age", "instance_type": "Instance Type", "node": "Node", "pool": "Pool", "zone": "Zone" } } } }, { "group": "sortBy", "kind": "Transformation", "spec": { "options": { "fields": {}, "sort": [ { "field": "Age" } ] } } } ] } }, "description": "**Every node with its pool, instance type, zone, and how long ago it joined.**\n\nAutoscaled nodes come and go, so a fleet of young nodes is normal. Nodes that are all minutes old and keep being replaced are not: an autoscaler that cannot settle, or nodes failing soon after they join.", "id": 1019, "links": [], "title": "Nodes", "vizConfig": { "group": "table", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "custom": { "align": "auto", "filterable": true, "inspect": false }, "noValue": "No node labels. kube-state-metrics publishes them only with the chart's metricLabelsAllowlist.", "unit": "s" }, "overrides": [] }, "options": { "cellHeight": "sm", "footer": { "countRows": false, "reducer": [], "show": false }, "frameIndex": 0, "showHeader": true, "showTypeIcons": false } }, "version": "" } } }, "pools-nodes-by-pool": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "count by (pool) (max by (node, pool, instance_type, zone) (label_replace(label_replace(label_replace(label_replace(label_replace(label_replace(label_replace(kube_node_labels, \"pool\", \"$1\", \"label_karpenter_sh_nodepool\", \"(.+)\"), \"pool\", \"$1\", \"label_eks_amazonaws_com_nodegroup\", \"(.+)\"), \"pool\", \"$1\", \"label_cloud_google_com_gke_nodepool\", \"(.+)\"), \"pool\", \"$1\", \"label_kubernetes_azure_com_agentpool\", \"(.+)\"), \"pool\", \"unpooled\", \"pool\", \"\"), \"instance_type\", \"$1\", \"label_node_kubernetes_io_instance_type\", \"(.+)\"), \"zone\", \"$1\", \"label_topology_kubernetes_io_zone\", \"(.+)\")))\n", "legendFormat": "{{pool}}", "qryType": 1, "query": "count by (pool) (max by (node, pool, instance_type, zone) (label_replace(label_replace(label_replace(label_replace(label_replace(label_replace(label_replace(kube_node_labels, \"pool\", \"$1\", \"label_karpenter_sh_nodepool\", \"(.+)\"), \"pool\", \"$1\", \"label_eks_amazonaws_com_nodegroup\", \"(.+)\"), \"pool\", \"$1\", \"label_cloud_google_com_gke_nodepool\", \"(.+)\"), \"pool\", \"$1\", \"label_kubernetes_azure_com_agentpool\", \"(.+)\"), \"pool\", \"unpooled\", \"pool\", \"\"), \"instance_type\", \"$1\", \"label_node_kubernetes_io_instance_type\", \"(.+)\"), \"zone\", \"$1\", \"label_topology_kubernetes_io_zone\", \"(.+)\")))\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Nodes in each node pool over time.**\n\nA step up is an autoscaler adding a node; a step down is one removing a node it judged empty or replaceable. The pool is the Karpenter NodePool, EKS node group, GKE node pool or AKS agent pool that owns the node, and `unpooled` a node with none of those labels. Every node reads `unpooled` until kube-state-metrics publishes node labels, which the chart configures.", "id": 1012, "links": [], "title": "Nodes by Pool", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "decimals": 0, "min": 0, "noValue": "No node labels. kube-state-metrics publishes them only with the chart's metricLabelsAllowlist.", "unit": "short" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "pools-nodes-by-type": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "count by (instance_type) (max by (node, pool, instance_type, zone) (label_replace(label_replace(label_replace(label_replace(label_replace(label_replace(label_replace(kube_node_labels, \"pool\", \"$1\", \"label_karpenter_sh_nodepool\", \"(.+)\"), \"pool\", \"$1\", \"label_eks_amazonaws_com_nodegroup\", \"(.+)\"), \"pool\", \"$1\", \"label_cloud_google_com_gke_nodepool\", \"(.+)\"), \"pool\", \"$1\", \"label_kubernetes_azure_com_agentpool\", \"(.+)\"), \"pool\", \"unpooled\", \"pool\", \"\"), \"instance_type\", \"$1\", \"label_node_kubernetes_io_instance_type\", \"(.+)\"), \"zone\", \"$1\", \"label_topology_kubernetes_io_zone\", \"(.+)\")))\n", "legendFormat": "{{instance_type}}", "qryType": 1, "query": "count by (instance_type) (max by (node, pool, instance_type, zone) (label_replace(label_replace(label_replace(label_replace(label_replace(label_replace(label_replace(kube_node_labels, \"pool\", \"$1\", \"label_karpenter_sh_nodepool\", \"(.+)\"), \"pool\", \"$1\", \"label_eks_amazonaws_com_nodegroup\", \"(.+)\"), \"pool\", \"$1\", \"label_cloud_google_com_gke_nodepool\", \"(.+)\"), \"pool\", \"$1\", \"label_kubernetes_azure_com_agentpool\", \"(.+)\"), \"pool\", \"unpooled\", \"pool\", \"\"), \"instance_type\", \"$1\", \"label_node_kubernetes_io_instance_type\", \"(.+)\"), \"zone\", \"$1\", \"label_topology_kubernetes_io_zone\", \"(.+)\")))\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Nodes of each instance type over time.**", "id": 1013, "links": [], "title": "Nodes by Instance Type", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "decimals": 0, "min": 0, "noValue": "No node labels. kube-state-metrics publishes them only with the chart's metricLabelsAllowlist.", "unit": "short" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "pools-nodes-by-zone": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "count by (zone) (max by (node, pool, instance_type, zone) (label_replace(label_replace(label_replace(label_replace(label_replace(label_replace(label_replace(kube_node_labels, \"pool\", \"$1\", \"label_karpenter_sh_nodepool\", \"(.+)\"), \"pool\", \"$1\", \"label_eks_amazonaws_com_nodegroup\", \"(.+)\"), \"pool\", \"$1\", \"label_cloud_google_com_gke_nodepool\", \"(.+)\"), \"pool\", \"$1\", \"label_kubernetes_azure_com_agentpool\", \"(.+)\"), \"pool\", \"unpooled\", \"pool\", \"\"), \"instance_type\", \"$1\", \"label_node_kubernetes_io_instance_type\", \"(.+)\"), \"zone\", \"$1\", \"label_topology_kubernetes_io_zone\", \"(.+)\")))\n", "legendFormat": "{{zone}}", "qryType": 1, "query": "count by (zone) (max by (node, pool, instance_type, zone) (label_replace(label_replace(label_replace(label_replace(label_replace(label_replace(label_replace(kube_node_labels, \"pool\", \"$1\", \"label_karpenter_sh_nodepool\", \"(.+)\"), \"pool\", \"$1\", \"label_eks_amazonaws_com_nodegroup\", \"(.+)\"), \"pool\", \"$1\", \"label_cloud_google_com_gke_nodepool\", \"(.+)\"), \"pool\", \"$1\", \"label_kubernetes_azure_com_agentpool\", \"(.+)\"), \"pool\", \"unpooled\", \"pool\", \"\"), \"instance_type\", \"$1\", \"label_node_kubernetes_io_instance_type\", \"(.+)\"), \"zone\", \"$1\", \"label_topology_kubernetes_io_zone\", \"(.+)\")))\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Nodes in each availability zone over time.**\n\nA pool that grows in one zone only can mean the others have no capacity for its instance type, or no subnet for it.", "id": 1014, "links": [], "title": "Nodes by Zone", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "decimals": 0, "min": 0, "noValue": "No node labels. kube-state-metrics publishes them only with the chart's metricLabelsAllowlist.", "unit": "short" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } }, "pools-not-ready": { "kind": "Panel", "spec": { "data": { "kind": "QueryGroup", "spec": { "queries": [ { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (pool) (\n (1 - max by (node) (kube_node_status_condition{condition=\"Ready\", status=\"true\"}))\n * on (node) group_left (pool) max by (node, pool, instance_type, zone) (label_replace(label_replace(label_replace(label_replace(label_replace(label_replace(label_replace(kube_node_labels, \"pool\", \"$1\", \"label_karpenter_sh_nodepool\", \"(.+)\"), \"pool\", \"$1\", \"label_eks_amazonaws_com_nodegroup\", \"(.+)\"), \"pool\", \"$1\", \"label_cloud_google_com_gke_nodepool\", \"(.+)\"), \"pool\", \"$1\", \"label_kubernetes_azure_com_agentpool\", \"(.+)\"), \"pool\", \"unpooled\", \"pool\", \"\"), \"instance_type\", \"$1\", \"label_node_kubernetes_io_instance_type\", \"(.+)\"), \"zone\", \"$1\", \"label_topology_kubernetes_io_zone\", \"(.+)\"))\n)\n", "legendFormat": "{{pool}}", "qryType": 1, "query": "sum by (pool) (\n (1 - max by (node) (kube_node_status_condition{condition=\"Ready\", status=\"true\"}))\n * on (node) group_left (pool) max by (node, pool, instance_type, zone) (label_replace(label_replace(label_replace(label_replace(label_replace(label_replace(label_replace(kube_node_labels, \"pool\", \"$1\", \"label_karpenter_sh_nodepool\", \"(.+)\"), \"pool\", \"$1\", \"label_eks_amazonaws_com_nodegroup\", \"(.+)\"), \"pool\", \"$1\", \"label_cloud_google_com_gke_nodepool\", \"(.+)\"), \"pool\", \"$1\", \"label_kubernetes_azure_com_agentpool\", \"(.+)\"), \"pool\", \"unpooled\", \"pool\", \"\"), \"instance_type\", \"$1\", \"label_node_kubernetes_io_instance_type\", \"(.+)\"), \"zone\", \"$1\", \"label_topology_kubernetes_io_zone\", \"(.+)\"))\n)\n" }, "version": "v0" }, "refId": "query-0" } } ], "queryOptions": {}, "transformations": [] } }, "description": "**Nodes not reporting Ready, by node pool.**\n\n**Nominal:** Zero, apart from a new node's first minute.\n\n**Unhealthy:** A pool with nodes that stay not-ready. An autoscaler may keep adding nodes to replace capacity it cannot use.", "id": 1017, "links": [], "title": "Nodes Not Ready by Pool", "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "decimals": 0, "min": 0, "noValue": "No node labels. kube-state-metrics publishes them only with the chart's metricLabelsAllowlist.", "unit": "short" }, "overrides": [] }, "options": { "legend": { "calcs": [ "max", "mean", "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "asc" } } }, "version": "" } } } }, "layout": { "kind": "TabsLayout", "spec": { "tabs": [ { "kind": "TabsLayoutTab", "spec": { "layout": { "kind": "RowsLayout", "spec": { "rows": [ { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": true, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "narrow", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "overview-pending" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "overview-unscheduled" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "overview-nodes" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "overview-not-ready" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "overview-added" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "overview-removed" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "overview-hpa-at-max" } } } ], "maxColumnCount": 7, "rowHeightMode": "short" } }, "title": "Verdict" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "wide", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "overview-unscheduled-by-namespace" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "overview-nodes-by-pool" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "overview-requested" } } } ], "maxColumnCount": 3, "rowHeightMode": "standard" } }, "title": "Demand and Capacity" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "standard", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "overview-events-by-reason" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "overview-warnings" } } } ], "maxColumnCount": 2, "rowHeightMode": "tall" } }, "title": "Activity" } } ] } }, "title": "Overview" } }, { "kind": "TabsLayoutTab", "spec": { "layout": { "kind": "RowsLayout", "spec": { "rows": [ { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "wide", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "pools-nodes-by-pool" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "pools-nodes-by-type" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "pools-nodes-by-zone" } } } ], "maxColumnCount": 3, "rowHeightMode": "standard" } }, "title": "Nodes" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "wide", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "pools-cpu-requested" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "pools-memory-requested" } } } ], "maxColumnCount": 2, "rowHeightMode": "standard" } }, "title": "Requests Against Allocatable" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "wide", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "pools-not-ready" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "pools-cordoned" } } } ], "maxColumnCount": 2, "rowHeightMode": "standard" } }, "title": "Node Health" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "standard", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "pools-node-age" } } } ], "maxColumnCount": 1, "rowHeightMode": "tall" } }, "title": "Nodes by Age" } } ] } }, "title": "Node Pools" } }, { "kind": "TabsLayoutTab", "spec": { "layout": { "kind": "RowsLayout", "spec": { "rows": [ { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "wide", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "pending-unscheduled" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "pending-pending" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "pending-failure-rate" } } } ], "maxColumnCount": 3, "rowHeightMode": "standard" } }, "title": "Waiting" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "standard", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "pending-list" } } } ], "maxColumnCount": 1, "rowHeightMode": "standard" } }, "title": "Pods Waiting Now" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "standard", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "pending-scheduling-failures" } } } ], "maxColumnCount": 1, "rowHeightMode": "tall" } }, "title": "Why They Wait" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "standard", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "pending-scale-up-decisions" } } } ], "maxColumnCount": 1, "rowHeightMode": "tall" } }, "title": "What the Autoscaler Did" } } ] } }, "title": "Pending Pods" } }, { "kind": "TabsLayoutTab", "spec": { "layout": { "kind": "RowsLayout", "spec": { "rows": [ { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "standard", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "hpa-table" } } } ], "maxColumnCount": 1, "rowHeightMode": "standard" } }, "title": "Replicas" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "wide", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "hpa-share-of-max" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "hpa-unable" } } } ], "maxColumnCount": 2, "rowHeightMode": "standard" } }, "title": "Room to Scale" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "standard", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "hpa-events-by-reason" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "hpa-events" } } } ], "maxColumnCount": 2, "rowHeightMode": "tall" } }, "title": "Events" } } ] } }, "title": "Workload Autoscaling" } }, { "kind": "TabsLayoutTab", "spec": { "layout": { "kind": "RowsLayout", "spec": { "rows": [ { "kind": "RowsLayoutRow", "spec": { "collapse": false, "conditionalRendering": { "kind": "ConditionalRenderingGroup", "spec": { "condition": "and", "items": [ { "kind": "ConditionalRenderingVariable", "spec": { "operator": "matches", "value": "cloudwatch", "variable": "cloudProviderList" } } ], "visibility": "show" } }, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "wide", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "ec2-status-checks" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "ec2-node-groups" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "ec2-vcpus" } } } ], "maxColumnCount": 3, "rowHeightMode": "standard" } }, "title": "EC2" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": false, "conditionalRendering": { "kind": "ConditionalRenderingGroup", "spec": { "condition": "and", "items": [ { "kind": "ConditionalRenderingVariable", "spec": { "operator": "matches", "value": "gcp", "variable": "cloudProviderList" } } ], "visibility": "show" } }, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "wide", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "gce-cpu-quota" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "gce-ssd-quota" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "gce-refusals" } } } ], "maxColumnCount": 3, "rowHeightMode": "standard" } }, "title": "Compute Engine Quota" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": false, "conditionalRendering": { "kind": "ConditionalRenderingGroup", "spec": { "condition": "and", "items": [ { "kind": "ConditionalRenderingVariable", "spec": { "operator": "matches", "value": "azure", "variable": "cloudProviderList" } } ], "visibility": "show" } }, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "wide", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "aks-unschedulable" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "aks-state" } } }, { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "aks-vms-down" } } } ], "maxColumnCount": 3, "rowHeightMode": "standard" } }, "title": "AKS" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": false, "conditionalRendering": { "kind": "ConditionalRenderingGroup", "spec": { "condition": "and", "items": [ { "kind": "ConditionalRenderingVariable", "spec": { "operator": "notMatches", "value": "cloudwatch|gcp|azure", "variable": "cloudProviderList" } } ], "visibility": "show" } }, "hideHeader": true, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "standard", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "capacity-none-detected" } } } ], "maxColumnCount": 1, "rowHeightMode": "tall" } }, "title": "No Provider Metrics" } } ] } }, "title": "Cloud Capacity" } }, { "kind": "TabsLayoutTab", "spec": { "layout": { "kind": "RowsLayout", "spec": { "rows": [ { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "standard", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "events-by-reason" } } } ], "maxColumnCount": 1, "rowHeightMode": "standard" } }, "title": "Activity" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "standard", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "events-warnings" } } } ], "maxColumnCount": 1, "rowHeightMode": "tall" } }, "title": "Warnings" } }, { "kind": "RowsLayoutRow", "spec": { "collapse": false, "hideHeader": false, "layout": { "kind": "AutoGridLayout", "spec": { "columnWidthMode": "standard", "fillScreen": false, "items": [ { "kind": "AutoGridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "events-all" } } } ], "maxColumnCount": 1, "rowHeightMode": "tall" } }, "title": "All Autoscaling Events" } } ] } }, "title": "Events" } } ] } }, "links": [], "liveNow": false, "preload": false, "tags": [ "infrastructure", "mzmon" ], "timeSettings": { "autoRefresh": "", "autoRefreshIntervals": [ "5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d" ], "fiscalYearStartMonth": 0, "from": "now-6h", "hideTimepicker": false, "timezone": "browser", "to": "now" }, "title": "Infrastructure Autoscaling", "variables": [ { "kind": "DatasourceVariable", "spec": { "allowCustomValue": false, "current": { "text": "", "value": "" }, "description": "Datasource for metrics queries", "hide": "dontHide", "includeAll": false, "label": "Metrics Datasource", "multi": false, "name": "metricsDatasource", "options": [], "pluginId": "prometheus", "refresh": "never", "regex": "", "skipUrlSync": false } }, { "kind": "DatasourceVariable", "spec": { "allowCustomValue": false, "current": { "text": "", "value": "" }, "description": "Datasource for log and event queries", "hide": "dontHide", "includeAll": false, "label": "Logs Datasource", "multi": false, "name": "logsDatasource", "options": [], "pluginId": "loki", "refresh": "never", "regex": "", "skipUrlSync": false } }, { "kind": "QueryVariable", "spec": { "allowCustomValue": true, "current": { "text": "", "value": "" }, "definition": "label_values(up{job=~\"integrations/(cloudwatch|gcp|azure)\"}, job)", "description": "Cloud providers the gateway was found to be pulling metrics from", "hide": "dontHide", "includeAll": true, "label": "Cloud Provider", "multi": true, "name": "cloudProviderList", "options": [], "query": { "datasource": { "name": "${metricsDatasource}" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "label_values(up{job=~\"integrations/(cloudwatch|gcp|azure)\"}, job)", "qryType": 1, "query": "label_values(up{job=~\"integrations/(cloudwatch|gcp|azure)\"}, job)" }, "version": "v0" }, "refresh": "never", "regex": "", "regexApplyTo": "value", "skipUrlSync": false, "sort": "alphabeticalAsc" } } ] } }