konairius/homelab
10
0
Fork 0
Code Issues 9 Pull Requests 1 Packages Projects Releases 1 Wiki Activity
Files
efd5191c6ddc004956e4e41ec92591c805ea4021
homelab/kubernetes/apps/monitoring/dashboards/node-overview.json
T
claude-botandClaude Opus 4.8 ac623c1e9c fix(node-overview): repair 4 dead panels, exclude rbd/lxc devices, nodename legends 
- Nodes Online/Down used job="node" but the scrape job is "node-exporter":
  both stats showed No data. Nodes Down additionally gets OR on() vector(0)
  so it renders 0 instead of No data when all nodes are up, and green-at-0/
  red-at-1 thresholds (was permanently red).
- Avg Disk Usage + Disk Usage by Node filtered mountpoint="/" which does not
  exist on Talos (only /var is a real xfs filesystem): both panels were empty.
- Disk I/O summed all block devices including the ~12 Rook-Ceph rbd* mappings
  per node (~438 kB/s vs 8-129 kB/s physical): filter to nvme.*|vd.*.
- Network I/O filter was written for a Docker host (veth/docker/br-) and let
  ~40 Cilium lxc* per-pod interfaces plus kernel pseudo-devices through,
  double-counting pod traffic: filter to eth.* (bond0 is down, 0 slaves).
- Legends showed instance IP:port; join node_uname_info via
  on(instance) group_left(nodename) so panels label k8s-1/2/3. Both join
  sides are wrapped in max by (...) so historical duplicate series from
  pod restarts / the pre-2026-07-16 duplicate-scrape residue cannot break
  the join or rainbow-duplicate long-window views.
- Stat panels reduce lastNotNull instead of mean: show current value, not
  a time-average over the selected window.

All queries verified against live Prometheus before applying.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-07-17 01:08:35 +02:00

231 lines
14 KiB
JSON
Raw Blame History

{
"annotations": {"list": []},
"description": "System metrics from all nodes",
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 1,
"id": null,
"links": [],
"panels": [
{
"datasource": {"type": "prometheus", "uid": "prometheus"},
"fieldConfig": {
"defaults": {"color": {"mode": "thresholds"}, "mappings": [], "thresholds": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "green", "value": 3}]}},
"overrides": []
},
"gridPos": {"h": 4, "w": 4, "x": 0, "y": 0},
"id": 1,
"options": {"colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto"},
"targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "count(up{job=\"node-exporter\"} == 1)", "refId": "A"}],
"title": "Nodes Online",
"type": "stat"
},
{
"datasource": {"type": "prometheus", "uid": "prometheus"},
"fieldConfig": {
"defaults": {"color": {"mode": "thresholds"}, "mappings": [], "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "yellow", "value": 60}, {"color": "red", "value": 80}]}, "unit": "percent"},
"overrides": []
},
"gridPos": {"h": 4, "w": 5, "x": 4, "y": 0},
"id": 2,
"options": {"colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto"},
"targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "avg(100 - (avg by(instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m])) * 100))", "refId": "A"}],
"title": "Avg CPU Usage",
"type": "stat"
},
{
"datasource": {"type": "prometheus", "uid": "prometheus"},
"fieldConfig": {
"defaults": {"color": {"mode": "thresholds"}, "mappings": [], "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "yellow", "value": 70}, {"color": "red", "value": 85}]}, "unit": "percent"},
"overrides": []
},
"gridPos": {"h": 4, "w": 5, "x": 9, "y": 0},
"id": 3,
"options": {"colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto"},
"targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "avg((1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100)", "refId": "A"}],
"title": "Avg Memory Usage",
"type": "stat"
},
{
"datasource": {"type": "prometheus", "uid": "prometheus"},
"fieldConfig": {
"defaults": {"color": {"mode": "thresholds"}, "mappings": [], "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "yellow", "value": 70}, {"color": "red", "value": 85}]}, "unit": "percent"},
"overrides": []
},
"gridPos": {"h": 4, "w": 5, "x": 14, "y": 0},
"id": 4,
"options": {"colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto"},
"targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "avg((1 - (node_filesystem_avail_bytes{mountpoint=\"/var\"} / node_filesystem_size_bytes{mountpoint=\"/var\"})) * 100)", "refId": "A"}],
"title": "Avg Disk Usage",
"type": "stat"
},
{
"datasource": {"type": "prometheus", "uid": "prometheus"},
"fieldConfig": {
"defaults": {"color": {"mode": "thresholds"}, "mappings": [], "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "red", "value": 1}]}},
"overrides": []
},
"gridPos": {"h": 4, "w": 5, "x": 19, "y": 0},
"id": 5,
"options": {"colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto"},
"targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "count(up{job=\"node-exporter\"} == 0) OR on() vector(0)", "refId": "A"}],
"title": "Nodes Down",
"type": "stat"
},
{
"datasource": {"type": "prometheus", "uid": "prometheus"},
"fieldConfig": {
"defaults": {
"color": {"mode": "thresholds"},
"mappings": [],
"max": 100,
"min": 0,
"thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "yellow", "value": 60}, {"color": "red", "value": 80}]},
"unit": "percent"
},
"overrides": []
},
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 4},
"id": 6,
"options": {"displayMode": "lcd", "minVizHeight": 10, "minVizWidth": 0, "orientation": "horizontal", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "showUnfilled": true},
"targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "(100 - (avg by(instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m])) * 100)) * on(instance) group_left(nodename) max by (instance, nodename) (node_uname_info)", "legendFormat": "{{nodename}}", "refId": "A"}],
"title": "CPU Usage by Node",
"type": "bargauge"
},
{
"datasource": {"type": "prometheus", "uid": "prometheus"},
"fieldConfig": {
"defaults": {
"color": {"mode": "thresholds"},
"mappings": [],
"max": 100,
"min": 0,
"thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "yellow", "value": 70}, {"color": "red", "value": 85}]},
"unit": "percent"
},
"overrides": []
},
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 4},
"id": 7,
"options": {"displayMode": "lcd", "minVizHeight": 10, "minVizWidth": 0, "orientation": "horizontal", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "showUnfilled": true},
"targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "max by (instance) ((1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100) * on(instance) group_left(nodename) max by (instance, nodename) (node_uname_info)", "legendFormat": "{{nodename}}", "refId": "A"}],
"title": "Memory Usage by Node",
"type": "bargauge"
},
{
"datasource": {"type": "prometheus", "uid": "prometheus"},
"fieldConfig": {
"defaults": {
"color": {"mode": "palette-classic"},
"custom": {"axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", "hideFrom": {"legend": false, "tooltip": false, "viz": false}, "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "scaleDistribution": {"type": "linear"}, "showPoints": "never", "spanNulls": false, "stacking": {"group": "A", "mode": "none"}, "thresholdsStyle": {"mode": "off"}},
"mappings": [],
"thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}]},
"unit": "percent"
},
"overrides": []
},
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 12},
"id": 8,
"options": {"legend": {"calcs": ["mean", "max"], "displayMode": "table", "placement": "right", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
"targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "(100 - (avg by(instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m])) * 100)) * on(instance) group_left(nodename) max by (instance, nodename) (node_uname_info)", "legendFormat": "{{nodename}}", "refId": "A"}],
"title": "CPU Usage Over Time",
"type": "timeseries"
},
{
"datasource": {"type": "prometheus", "uid": "prometheus"},
"fieldConfig": {
"defaults": {
"color": {"mode": "palette-classic"},
"custom": {"axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", "hideFrom": {"legend": false, "tooltip": false, "viz": false}, "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "scaleDistribution": {"type": "linear"}, "showPoints": "never", "spanNulls": false, "stacking": {"group": "A", "mode": "none"}, "thresholdsStyle": {"mode": "off"}},
"mappings": [],
"thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}]},
"unit": "bytes"
},
"overrides": []
},
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 12},
"id": 9,
"options": {"legend": {"calcs": ["mean", "max"], "displayMode": "table", "placement": "right", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
"targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "max by (instance) (node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) * on(instance) group_left(nodename) max by (instance, nodename) (node_uname_info)", "legendFormat": "{{nodename}}", "refId": "A"}],
"title": "Memory Usage Over Time",
"type": "timeseries"
},
{
"datasource": {"type": "prometheus", "uid": "prometheus"},
"fieldConfig": {
"defaults": {
"color": {"mode": "palette-classic"},
"custom": {"axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", "hideFrom": {"legend": false, "tooltip": false, "viz": false}, "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "scaleDistribution": {"type": "linear"}, "showPoints": "never", "spanNulls": false, "stacking": {"group": "A", "mode": "none"}, "thresholdsStyle": {"mode": "off"}},
"mappings": [],
"thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}]},
"unit": "Bps"
},
"overrides": []
},
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 20},
"id": 10,
"options": {"legend": {"calcs": ["mean", "max"], "displayMode": "table", "placement": "right", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
"targets": [
{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "sum by (instance) (max by (instance, device) (rate(node_network_receive_bytes_total{device=~\"eth.*\"}[5m]))) * on(instance) group_left(nodename) max by (instance, nodename) (node_uname_info)", "legendFormat": "{{nodename}} rx", "refId": "A"},
{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "-(sum by (instance) (max by (instance, device) (rate(node_network_transmit_bytes_total{device=~\"eth.*\"}[5m]))) * on(instance) group_left(nodename) max by (instance, nodename) (node_uname_info))", "legendFormat": "{{nodename}} tx", "refId": "B"}
],
"title": "Network I/O",
"type": "timeseries"
},
{
"datasource": {"type": "prometheus", "uid": "prometheus"},
"fieldConfig": {
"defaults": {
"color": {"mode": "palette-classic"},
"custom": {"axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", "hideFrom": {"legend": false, "tooltip": false, "viz": false}, "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "scaleDistribution": {"type": "linear"}, "showPoints": "never", "spanNulls": false, "stacking": {"group": "A", "mode": "none"}, "thresholdsStyle": {"mode": "off"}},
"mappings": [],
"thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}]},
"unit": "Bps"
},
"overrides": []
},
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 20},
"id": 11,
"options": {"legend": {"calcs": ["mean", "max"], "displayMode": "table", "placement": "right", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
"targets": [
{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "sum by (instance) (max by (instance, device) (rate(node_disk_read_bytes_total{device=~\"nvme.*|vd.*\"}[5m]))) * on(instance) group_left(nodename) max by (instance, nodename) (node_uname_info)", "legendFormat": "{{nodename}} read", "refId": "A"},
{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "-(sum by (instance) (max by (instance, device) (rate(node_disk_written_bytes_total{device=~\"nvme.*|vd.*\"}[5m]))) * on(instance) group_left(nodename) max by (instance, nodename) (node_uname_info))", "legendFormat": "{{nodename}} write", "refId": "B"}
],
"title": "Disk I/O",
"type": "timeseries"
},
{
"datasource": {"type": "prometheus", "uid": "prometheus"},
"fieldConfig": {
"defaults": {
"color": {"mode": "thresholds"},
"mappings": [],
"max": 100,
"min": 0,
"thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "yellow", "value": 70}, {"color": "red", "value": 85}]},
"unit": "percent"
},
"overrides": []
},
"gridPos": {"h": 8, "w": 24, "x": 0, "y": 28},
"id": 12,
"options": {"displayMode": "lcd", "minVizHeight": 10, "minVizWidth": 0, "orientation": "horizontal", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "showUnfilled": true},
"targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "max by (instance) ((1 - (node_filesystem_avail_bytes{mountpoint=\"/var\"} / node_filesystem_size_bytes{mountpoint=\"/var\"})) * 100) * on(instance) group_left(nodename) max by (instance, nodename) (node_uname_info)", "legendFormat": "{{nodename}}", "refId": "A"}],
"title": "Disk Usage by Node",
"type": "bargauge"
}
],
"refresh": "30s",
"schemaVersion": 38,
"style": "dark",
"tags": ["homelab", "nodes", "infrastructure"],
"templating": {"list": []},
"time": {"from": "now-1h", "to": "now"},
"timepicker": {},
"timezone": "",
"title": "Node Overview",
"uid": "node-overview",
"version": 1,
"weekStart": ""
}
Reference in New Issue View Git Blame Copy Permalink