From bd8a632334d96ed670f1132a7effd4aa11f5ec74 Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Fri, 17 Jul 2026 00:40:54 +0200 Subject: [PATCH] =?UTF-8?q?fix(node-metrics):=20repair=20dead=20dashboard?= =?UTF-8?q?=20=E2=80=94=20nonexistent=20'host'=20label=20broke=20every=20p?= =?UTF-8?q?anel?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The template variable queried label_values(node_uname_info, host), but no node-exporter metric in this Prometheus carries a 'host' label (that label only exists on blackbox-probe metrics). The variable resolved to empty and all five panels rendered blank. - Replace $host with a visible $node variable (nodename: k8s-1/2/3) plus a hidden chained $instance variable (nodename -> instance via node_uname_info) - Filter Cilium pod interfaces (lxc*, cilium_*) and always-zero kernel pseudo-devices (ip6tnl0, sit0, teql0, tunl0) out of Network I/O - Filter Ceph RBD-mapped volumes (rbd0..11) out of Disk I/O so the panel shows physical disks (nvme0n1, vda) only - Exclude the rootfs bind mount (/etc/extensions.yaml) from Filesystem Usage, leaving the meaningful /var xfs mount All queries verified against live Prometheus before commit. --- .../monitoring/dashboards/node-metrics.json | 43 +++++++++++++------ 1 file changed, 30 insertions(+), 13 deletions(-) diff --git a/kubernetes/apps/monitoring/dashboards/node-metrics.json b/kubernetes/apps/monitoring/dashboards/node-metrics.json index 59a9a009..e8b6049f 100644 --- a/kubernetes/apps/monitoring/dashboards/node-metrics.json +++ b/kubernetes/apps/monitoring/dashboards/node-metrics.json @@ -23,7 +23,7 @@ "id": 1, "options": {"legend": {"calcs": ["mean", "max"], "displayMode": "list", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}}, "targets": [ - {"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "100 - (avg by(cpu) (rate(node_cpu_seconds_total{mode=\"idle\",host=~\"$host\"}[5m])) * 100)", "legendFormat": "CPU {{cpu}}", "refId": "A"} + {"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "100 - (avg by(cpu) (rate(node_cpu_seconds_total{mode=\"idle\",instance=\"$instance\"}[5m])) * 100)", "legendFormat": "CPU {{cpu}}", "refId": "A"} ], "title": "CPU Usage per Core", "type": "timeseries" @@ -44,9 +44,9 @@ "id": 2, "options": {"legend": {"calcs": ["mean", "max"], "displayMode": "list", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}}, "targets": [ - {"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "node_memory_MemTotal_bytes{host=~\"$host\"} - node_memory_MemAvailable_bytes{host=~\"$host\"}", "legendFormat": "Used", "refId": "A"}, - {"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "node_memory_Cached_bytes{host=~\"$host\"}", "legendFormat": "Cached", "refId": "B"}, - {"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "node_memory_Buffers_bytes{host=~\"$host\"}", "legendFormat": "Buffers", "refId": "C"} + {"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "node_memory_MemTotal_bytes{instance=\"$instance\"} - node_memory_MemAvailable_bytes{instance=\"$instance\"}", "legendFormat": "Used", "refId": "A"}, + {"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "node_memory_Cached_bytes{instance=\"$instance\"}", "legendFormat": "Cached", "refId": "B"}, + {"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "node_memory_Buffers_bytes{instance=\"$instance\"}", "legendFormat": "Buffers", "refId": "C"} ], "title": "Memory Usage", "type": "timeseries" @@ -67,8 +67,8 @@ "id": 3, "options": {"legend": {"calcs": ["mean", "max"], "displayMode": "list", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}}, "targets": [ - {"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "rate(node_network_receive_bytes_total{host=~\"$host\",device!~\"lo|veth.*|br.*|docker.*\"}[5m])", "legendFormat": "{{device}} RX", "refId": "A"}, - {"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "-rate(node_network_transmit_bytes_total{host=~\"$host\",device!~\"lo|veth.*|br.*|docker.*\"}[5m])", "legendFormat": "{{device}} TX", "refId": "B"} + {"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "rate(node_network_receive_bytes_total{instance=\"$instance\",device!~\"lo|veth.*|br.*|docker.*|lxc.*|cilium.*|ip6tnl.*|sit.*|teql.*|tunl.*\"}[5m])", "legendFormat": "{{device}} RX", "refId": "A"}, + {"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "-rate(node_network_transmit_bytes_total{instance=\"$instance\",device!~\"lo|veth.*|br.*|docker.*|lxc.*|cilium.*|ip6tnl.*|sit.*|teql.*|tunl.*\"}[5m])", "legendFormat": "{{device}} TX", "refId": "B"} ], "title": "Network I/O", "type": "timeseries" @@ -89,8 +89,8 @@ "id": 4, "options": {"legend": {"calcs": ["mean", "max"], "displayMode": "list", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}}, "targets": [ - {"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "rate(node_disk_read_bytes_total{host=~\"$host\"}[5m])", "legendFormat": "{{device}} Read", "refId": "A"}, - {"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "-rate(node_disk_written_bytes_total{host=~\"$host\"}[5m])", "legendFormat": "{{device}} Write", "refId": "B"} + {"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "rate(node_disk_read_bytes_total{instance=\"$instance\",device!~\"rbd.*|dm-.*|loop.*\"}[5m])", "legendFormat": "{{device}} Read", "refId": "A"}, + {"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "-rate(node_disk_written_bytes_total{instance=\"$instance\",device!~\"rbd.*|dm-.*|loop.*\"}[5m])", "legendFormat": "{{device}} Write", "refId": "B"} ], "title": "Disk I/O", "type": "timeseries" @@ -112,7 +112,7 @@ "id": 5, "options": {"displayMode": "lcd", "minVizHeight": 10, "minVizWidth": 0, "orientation": "horizontal", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "showUnfilled": true}, "targets": [ - {"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "(1 - (node_filesystem_avail_bytes{host=~\"$host\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{host=~\"$host\",fstype!~\"tmpfs|overlay\"})) * 100", "legendFormat": "{{mountpoint}}", "refId": "A"} + {"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "(1 - (node_filesystem_avail_bytes{instance=\"$instance\",fstype!~\"tmpfs|overlay|rootfs\"} / node_filesystem_size_bytes{instance=\"$instance\",fstype!~\"tmpfs|overlay|rootfs\"})) * 100", "legendFormat": "{{mountpoint}}", "refId": "A"} ], "title": "Filesystem Usage", "type": "bargauge" @@ -127,14 +127,31 @@ { "current": {}, "datasource": {"type": "prometheus", "uid": "prometheus"}, - "definition": "label_values(node_uname_info, host)", + "definition": "label_values(node_uname_info{job=\"node-exporter\"}, nodename)", "hide": 0, "includeAll": false, - "label": "Host", + "label": "Node", "multi": false, - "name": "host", + "name": "node", "options": [], - "query": {"query": "label_values(node_uname_info, host)", "refId": "PrometheusVariableQueryEditor-VariableQuery"}, + "query": {"query": "label_values(node_uname_info{job=\"node-exporter\"}, nodename)", "refId": "PrometheusVariableQueryEditor-VariableQuery"}, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": {}, + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "definition": "label_values(node_uname_info{job=\"node-exporter\", nodename=\"$node\"}, instance)", + "hide": 2, + "includeAll": false, + "label": "Instance", + "multi": false, + "name": "instance", + "options": [], + "query": {"query": "label_values(node_uname_info{job=\"node-exporter\", nodename=\"$node\"}, instance)", "refId": "PrometheusVariableQueryEditor-VariableQuery"}, "refresh": 2, "regex": "", "skipUrlSync": false, -- 2.54.0