From d04b09310f9e9b1b1f12a039345e7fffc2899f9c Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Thu, 27 Aug 2026 14:02:11 +0200 Subject: [PATCH] feat(monitoring): add a disk health dashboard MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Throughput, IOPS, mean operation latency, utilisation, queue depth and controller temperature for all nine NVMe drives, plus Ceph's own OSD latency for comparison against the block layer. Every panel joins node_disk_* to node_nvme_info and node_uname_info rather than plotting the device name. That is not decoration: nvme0 is the Kingston system disk on k8s-1 and k8s-3 and the Crucial on k8s-2, so a legend keyed on the device compares different hardware across nodes and does so silently. The inventory table resolves the OSD column through ceph_disk_occupation for the same reason — it follows Rook instead of restating a device list that goes stale the next time a drive is swapped. Every expression was run against Prometheus before being written into the JSON, which is also how the caveats in the panel descriptions were found. Latency is accumulated-time over completed-operations, so it is a mean with no tail and it vanishes while a device is idle: absence of traffic, not absence of a problem. Utilisation near 100% on NVMe does not mean saturation the way it does on a spindle, which is why queue depth shares that panel. Temperature is the only SMART-adjacent value present, because it is the only one the kernel publishes through sysfs. Wear, written-bytes totals, power-on hours and media errors need an NVMe admin ioctl and no exporter here produces them (#549). The dashboard says so in a panel rather than letting a green screen imply a healthy drive. That gap is worse than #549 describes, and the note records why. The Crucial P3 drives were retired from Ceph because they were worn, and now carry the EPHEMERAL volume — so the drives with the least endurance left are the ones holding container state, and they are the ones nothing watches. The measured write latency already separates them from the Samsung OSDs by more than an order of magnitude. Also fixes the add-a-dashboard instructions above the generator, whose third step told the reader to tick a row off ../DASHBOARD-MIGRATION.md. That file does not exist anywhere in the repo, so the step could not be followed. Refs #549 Co-Authored-By: Claude Opus 5 (1M context) --- .../monitoring/dashboards/disk-health.json | 270 ++++++++++++++++++ .../monitoring/dashboards/kustomization.yaml | 10 +- 2 files changed, 278 insertions(+), 2 deletions(-) create mode 100644 kubernetes/apps/monitoring/dashboards/disk-health.json diff --git a/kubernetes/apps/monitoring/dashboards/disk-health.json b/kubernetes/apps/monitoring/dashboards/disk-health.json new file mode 100644 index 00000000..6bc35ea9 --- /dev/null +++ b/kubernetes/apps/monitoring/dashboards/disk-health.json @@ -0,0 +1,270 @@ +{ + "annotations": {"list": []}, + "description": "NVMe throughput, latency and temperature per node. Every panel joins node_disk_* to node_nvme_info and node_uname_info on purpose: device names are NOT stable across nodes (nvme0 is the Kingston system disk on k8s-1 and k8s-3, but the Crucial on k8s-2), so a legend keyed on the device alone compares different hardware. Endurance/wear is deliberately absent — see the note at the bottom and issue #549.", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "panels": [ + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "description": "Which physical drive is which, resolved live rather than written down. The OSD column comes from ceph_disk_occupation, so it follows Rook instead of restating a device list that would go stale the next time a drive is swapped.", + "fieldConfig": { + "defaults": { + "color": {"mode": "thresholds"}, + "custom": {"align": "auto", "cellOptions": {"type": "auto"}, "inspect": false}, + "mappings": [], + "thresholds": {"mode": "absolute", "steps": [{"color": "text", "value": null}]} + }, + "overrides": [] + }, + "gridPos": {"h": 9, "w": 24, "x": 0, "y": 0}, + "id": 1, + "options": {"cellHeight": "sm", "footer": {"countRows": false, "fields": "", "reducer": ["sum"], "show": false}, "showHeader": true, "sortBy": [{"desc": false, "displayName": "node"}]}, + "targets": [ + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "expr": "node_nvme_info * on (instance) group_left(nodename) node_uname_info", + "format": "table", + "instant": true, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Drive inventory", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": {"Time": true, "Value": true, "__name__": true, "cntlid": true, "container": true, "endpoint": true, "instance": true, "job": true, "namespace": true, "pod": true, "service": true, "domainname": true, "machine": true, "release": true, "sysname": true, "version": true, "state": true}, + "indexByName": {}, + "renameByName": {"nodename": "node", "device": "device", "model": "model", "serial": "serial", "firmware_revision": "firmware"} + } + } + ], + "type": "table" + }, + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "description": "Read and write throughput per drive.", + "fieldConfig": { + "defaults": { + "color": {"mode": "palette-classic"}, + "custom": {"axisCenteredZero": false, "axisColorMode": "text", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": {"legend": false, "tooltip": false, "viz": false}, "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "scaleDistribution": {"type": "linear"}, "showPoints": "never", "spanNulls": false, "stacking": {"group": "A", "mode": "none"}, "thresholdsStyle": {"mode": "off"}}, + "mappings": [], + "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}]}, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": {"h": 9, "w": 12, "x": 0, "y": 9}, + "id": 2, + "options": {"legend": {"calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}}, + "targets": [ + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "expr": "label_replace(rate(node_disk_read_bytes_total{device=~\"nvme[0-9]+n[0-9]+\"}[$__rate_interval]), \"nvmedev\", \"$1\", \"device\", \"(nvme[0-9]+)n[0-9]+\") * on (instance, nvmedev) group_left(model) label_replace(node_nvme_info, \"nvmedev\", \"$1\", \"device\", \"(nvme[0-9]+)\") * on (instance) group_left(nodename) node_uname_info", + "legendFormat": "read {{nodename}} {{model}}", + "refId": "A" + }, + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "expr": "label_replace(rate(node_disk_written_bytes_total{device=~\"nvme[0-9]+n[0-9]+\"}[$__rate_interval]), \"nvmedev\", \"$1\", \"device\", \"(nvme[0-9]+)n[0-9]+\") * on (instance, nvmedev) group_left(model) label_replace(node_nvme_info, \"nvmedev\", \"$1\", \"device\", \"(nvme[0-9]+)\") * on (instance) group_left(nodename) node_uname_info", + "legendFormat": "write {{nodename}} {{model}}", + "refId": "B" + } + ], + "title": "Throughput", + "type": "timeseries" + }, + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "description": "Completed operations per second. Read together with throughput this separates many small operations from few large ones.", + "fieldConfig": { + "defaults": { + "color": {"mode": "palette-classic"}, + "custom": {"axisCenteredZero": false, "axisColorMode": "text", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": {"legend": false, "tooltip": false, "viz": false}, "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "scaleDistribution": {"type": "linear"}, "showPoints": "never", "spanNulls": false, "stacking": {"group": "A", "mode": "none"}, "thresholdsStyle": {"mode": "off"}}, + "mappings": [], + "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}]}, + "unit": "iops" + }, + "overrides": [] + }, + "gridPos": {"h": 9, "w": 12, "x": 12, "y": 9}, + "id": 3, + "options": {"legend": {"calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}}, + "targets": [ + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "expr": "label_replace(rate(node_disk_reads_completed_total{device=~\"nvme[0-9]+n[0-9]+\"}[$__rate_interval]), \"nvmedev\", \"$1\", \"device\", \"(nvme[0-9]+)n[0-9]+\") * on (instance, nvmedev) group_left(model) label_replace(node_nvme_info, \"nvmedev\", \"$1\", \"device\", \"(nvme[0-9]+)\") * on (instance) group_left(nodename) node_uname_info", + "legendFormat": "read {{nodename}} {{model}}", + "refId": "A" + }, + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "expr": "label_replace(rate(node_disk_writes_completed_total{device=~\"nvme[0-9]+n[0-9]+\"}[$__rate_interval]), \"nvmedev\", \"$1\", \"device\", \"(nvme[0-9]+)n[0-9]+\") * on (instance, nvmedev) group_left(model) label_replace(node_nvme_info, \"nvmedev\", \"$1\", \"device\", \"(nvme[0-9]+)\") * on (instance) group_left(nodename) node_uname_info", + "legendFormat": "write {{nodename}} {{model}}", + "refId": "B" + } + ], + "title": "IOPS", + "type": "timeseries" + }, + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "description": "Mean service time per operation, derived as accumulated time divided by completed operations. This is an average and will not show a tail — a drive can look healthy here while a small fraction of operations stall. The line disappears while a device is idle, because the denominator is zero; that is absence of traffic, not absence of a problem.", + "fieldConfig": { + "defaults": { + "color": {"mode": "palette-classic"}, + "custom": {"axisCenteredZero": false, "axisColorMode": "text", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": {"legend": false, "tooltip": false, "viz": false}, "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "scaleDistribution": {"type": "linear"}, "showPoints": "never", "spanNulls": false, "stacking": {"group": "A", "mode": "none"}, "thresholdsStyle": {"mode": "off"}}, + "mappings": [], + "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}]}, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": {"h": 9, "w": 12, "x": 0, "y": 18}, + "id": 4, + "options": {"legend": {"calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}}, + "targets": [ + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "expr": "label_replace(rate(node_disk_read_time_seconds_total{device=~\"nvme[0-9]+n[0-9]+\"}[$__rate_interval]) / rate(node_disk_reads_completed_total{device=~\"nvme[0-9]+n[0-9]+\"}[$__rate_interval]), \"nvmedev\", \"$1\", \"device\", \"(nvme[0-9]+)n[0-9]+\") * on (instance, nvmedev) group_left(model) label_replace(node_nvme_info, \"nvmedev\", \"$1\", \"device\", \"(nvme[0-9]+)\") * on (instance) group_left(nodename) node_uname_info", + "legendFormat": "read {{nodename}} {{model}}", + "refId": "A" + }, + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "expr": "label_replace(rate(node_disk_write_time_seconds_total{device=~\"nvme[0-9]+n[0-9]+\"}[$__rate_interval]) / rate(node_disk_writes_completed_total{device=~\"nvme[0-9]+n[0-9]+\"}[$__rate_interval]), \"nvmedev\", \"$1\", \"device\", \"(nvme[0-9]+)n[0-9]+\") * on (instance, nvmedev) group_left(model) label_replace(node_nvme_info, \"nvmedev\", \"$1\", \"device\", \"(nvme[0-9]+)\") * on (instance) group_left(nodename) node_uname_info", + "legendFormat": "write {{nodename}} {{model}}", + "refId": "B" + } + ], + "title": "Mean operation latency", + "type": "timeseries" + }, + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "description": "Fraction of wall-clock time the device had at least one operation in flight, and the number of operations queued. Utilisation near 100% on an NVMe drive does not mean saturation the way it does on a spindle — these devices serve many operations in parallel — so read it alongside the queue depth.", + "fieldConfig": { + "defaults": { + "color": {"mode": "palette-classic"}, + "custom": {"axisCenteredZero": false, "axisColorMode": "text", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": {"legend": false, "tooltip": false, "viz": false}, "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "scaleDistribution": {"type": "linear"}, "showPoints": "never", "spanNulls": false, "stacking": {"group": "A", "mode": "none"}, "thresholdsStyle": {"mode": "off"}}, + "mappings": [], + "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}]}, + "unit": "percentunit" + }, + "overrides": [ + { + "matcher": {"id": "byRegexp", "options": "queue .*"}, + "properties": [{"id": "unit", "value": "short"}, {"id": "custom.axisPlacement", "value": "right"}] + } + ] + }, + "gridPos": {"h": 9, "w": 12, "x": 12, "y": 18}, + "id": 5, + "options": {"legend": {"calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}}, + "targets": [ + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "expr": "label_replace(rate(node_disk_io_time_seconds_total{device=~\"nvme[0-9]+n[0-9]+\"}[$__rate_interval]), \"nvmedev\", \"$1\", \"device\", \"(nvme[0-9]+)n[0-9]+\") * on (instance, nvmedev) group_left(model) label_replace(node_nvme_info, \"nvmedev\", \"$1\", \"device\", \"(nvme[0-9]+)\") * on (instance) group_left(nodename) node_uname_info", + "legendFormat": "busy {{nodename}} {{model}}", + "refId": "A" + }, + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "expr": "label_replace(node_disk_io_now{device=~\"nvme[0-9]+n[0-9]+\"}, \"nvmedev\", \"$1\", \"device\", \"(nvme[0-9]+)n[0-9]+\") * on (instance, nvmedev) group_left(model) label_replace(node_nvme_info, \"nvmedev\", \"$1\", \"device\", \"(nvme[0-9]+)\") * on (instance) group_left(nodename) node_uname_info", + "legendFormat": "queue {{nodename}} {{model}}", + "refId": "B" + } + ], + "title": "Utilisation and queue depth", + "type": "timeseries" + }, + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "description": "Composite temperature reported by each controller, read from hwmon in sysfs. This is the one health value available without an admin ioctl, which is why it is the only SMART-adjacent series on this dashboard.", + "fieldConfig": { + "defaults": { + "color": {"mode": "palette-classic"}, + "custom": {"axisCenteredZero": false, "axisColorMode": "text", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": {"legend": false, "tooltip": false, "viz": false}, "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "scaleDistribution": {"type": "linear"}, "showPoints": "never", "spanNulls": false, "stacking": {"group": "A", "mode": "none"}, "thresholdsStyle": {"mode": "dashed"}}, + "mappings": [], + "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "orange", "value": 70}, {"color": "red", "value": 80}]}, + "unit": "celsius" + }, + "overrides": [] + }, + "gridPos": {"h": 9, "w": 12, "x": 0, "y": 27}, + "id": 6, + "options": {"legend": {"calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}}, + "targets": [ + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "expr": "label_replace(node_hwmon_temp_celsius{chip=~\"nvme_nvme[0-9]+\", sensor=\"temp1\"}, \"nvmedev\", \"$1\", \"chip\", \"nvme_(nvme[0-9]+)\") * on (instance, nvmedev) group_left(model) label_replace(node_nvme_info, \"nvmedev\", \"$1\", \"device\", \"(nvme[0-9]+)\") * on (instance) group_left(nodename) node_uname_info", + "legendFormat": "{{nodename}} {{model}}", + "refId": "A" + } + ], + "title": "Controller temperature", + "type": "timeseries" + }, + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "description": "Ceph's own view of the OSDs, for comparison with the block-layer numbers above. A divergence between these and the device latency is the interesting case: it points at Ceph rather than at the hardware.", + "fieldConfig": { + "defaults": { + "color": {"mode": "palette-classic"}, + "custom": {"axisCenteredZero": false, "axisColorMode": "text", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": {"legend": false, "tooltip": false, "viz": false}, "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "scaleDistribution": {"type": "linear"}, "showPoints": "never", "spanNulls": false, "stacking": {"group": "A", "mode": "none"}, "thresholdsStyle": {"mode": "off"}}, + "mappings": [], + "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}]}, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": {"h": 9, "w": 12, "x": 12, "y": 27}, + "id": 7, + "options": {"legend": {"calcs": ["mean", "max"], "displayMode": "table", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}}, + "targets": [ + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "expr": "ceph_osd_apply_latency_ms", + "legendFormat": "apply {{ceph_daemon}}", + "refId": "A" + }, + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "expr": "ceph_osd_commit_latency_ms", + "legendFormat": "commit {{ceph_daemon}}", + "refId": "B" + } + ], + "title": "Ceph OSD latency", + "type": "timeseries" + }, + { + "datasource": {"type": "prometheus", "uid": "prometheus"}, + "gridPos": {"h": 8, "w": 24, "x": 0, "y": 36}, + "id": 8, + "options": { + "code": {"language": "plaintext", "showLineNumbers": false, "showMiniMap": false}, + "content": "### What this dashboard cannot tell you\n\n**Endurance is missing.** There is no wear, no written-bytes total, no power-on hours and no media-error count here, because no exporter in this cluster produces them. NVMe reports those over an admin ioctl that needs elevated privileges, and temperature is the only health value the kernel publishes through sysfs. Tracked as **#549** — do not read a healthy dashboard as a healthy drive.\n\n**The three roles are not interchangeable.** Each node carries a Kingston OM8PGP41024Q as the Talos system disk, a Samsung PM983 that Rook claims as the Ceph OSD, and a Crucial P3 that was retired from Ceph and now carries the EPHEMERAL volume. The Crucial drives were replaced *because* they were worn, so the drive with the least endurance left is the one holding container state — and it is the one nothing is watching.\n\n**Device names are not identity.** `nvme0` is the Kingston on k8s-1 and k8s-3 and the Crucial on k8s-2. Every panel here resolves the model through `node_nvme_info` for that reason; a query written against a bare device name compares different hardware and will keep doing so silently.", + "mode": "markdown" + }, + "title": "Gaps and caveats", + "type": "text" + } + ], + "preload": false, + "refresh": "1m", + "schemaVersion": 39, + "tags": ["storage", "nvme", "ceph"], + "templating": {"list": []}, + "time": {"from": "now-6h", "to": "now"}, + "timepicker": {}, + "timezone": "", + "title": "Disk Health", + "uid": "disk-health", + "version": 1, + "weekStart": "" +} diff --git a/kubernetes/apps/monitoring/dashboards/kustomization.yaml b/kubernetes/apps/monitoring/dashboards/kustomization.yaml index 063a10f3..52629e33 100644 --- a/kubernetes/apps/monitoring/dashboards/kustomization.yaml +++ b/kubernetes/apps/monitoring/dashboards/kustomization.yaml @@ -7,9 +7,12 @@ namespace: monitoring # the kube-prometheus-stack Grafana sidecar watches for. # # To add a dashboard: -# 1. Copy the ported JSON to this directory as .json +# 1. Copy the JSON to this directory as .json # 2. Add a configMapGenerator entry below referencing the file -# 3. Tick off the row in ../DASHBOARD-MIGRATION.md +# +# There used to be a third step, ticking a row off ../DASHBOARD-MIGRATION.md. +# That file no longer exists anywhere in the repo — the port it tracked is +# finished — so following the instruction was impossible. # # Dashboard datasource UIDs must match K8s Grafana datasources: # - prometheus (kube-prom-stack default, in-cluster) @@ -35,6 +38,9 @@ configMapGenerator: - name: dashboard-blackbox-overview files: - blackbox-overview.json + - name: dashboard-disk-health + files: + - disk-health.json - name: dashboard-gitea-overview files: - gitea-overview.json -- 2.54.0