From e8b217e8a1aee78708b9007e17f2f25b180df6ef Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Thu, 27 Aug 2026 14:42:18 +0200 Subject: [PATCH] feat(monitoring): expose NVMe endurance, with the privilege exception it needs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Drive wear has been invisible in every signal this cluster has. `ceph -s` reads active+clean at any wear level, no Ceph health check fires until a device actually fails, and the block-layer metrics show throughput and latency but never endurance. That blind spot is how a set of OSD drives reached 81-87% before anyone looked (#549, #434). This closes it for every drive, and it costs a deliberate exception to the no-privileged-containers rule. The owner made that call explicitly after being shown the alternative. The alternative was real and is worth recording. Ceph's devicehealth module already archives complete smartctl output daily — percentage_used, power_on_hours, media_errors, critical_warning, available_spare — collected without any elevated privileges, because the OSD pods hold the device access already. Exporting that needs no exception at all. It also covers OSDs and nothing else: three of nine drives here, and the six it misses include the ones retired from Ceph for wear that now carry EPHEMERAL. Monitoring the healthy drives while leaving the worn ones dark was not worth a new service. The requirement comes from the kernel, not from the tool. NVMe health is read through an admin ioctl on the character device, which is gated on CAP_SYS_ADMIN; sysfs publishes the controller temperature and nothing more. Swapping smartctl_exporter for another exporter would not change that, so the header says so — the thing to re-check one day is whether the kernel grew a sysfs interface, not whether some other tool asks for less. The exception is kept to the smallest form that works: no `privileged: true`, no host namespaces, no hostPID. Root plus CAP_SYS_ADMIN and CAP_SYS_RAWIO, all other capabilities dropped, a read-only root filesystem, and /dev as the only host mount — required because Kubernetes cannot hand individual character devices to a pod without a device plugin. The admission policy that forbids privileged pods is bound to the woodpecker-ci namespace only, so nothing here circumvents it; monitoring already enforces pod-security `privileged`. Two details that would otherwise have made the metrics useless: The ServiceMonitor relabels the pod's node onto every series. Without it each sample carries only the DaemonSet pod's IP, which says nothing about which machine the drive is in and changes on every reschedule — an unattributable wear figure is most of the value gone. No --smartctl.device list. The exporter scans and rescans on its own, and a written-down device list would quietly stop covering a drive the next time one is replaced. Device names are not stable across these machines in the first place. The alert rules deliberately separate wear from damage: a wear percentage is a budget being spent and warns at 80, while spare exhaustion, a critical-warning flag or new media errors mean the drive is failing now. Media errors alert on the increase rather than the total, so a drive that recorded errors long ago does not alert forever while one that starts failing today does. The completeness check compares the number of reporting drives against node-exporter's own NVMe inventory rather than a written-down fleet size, so adding or removing a drive moves both sides together. A rule that alerts on nothing looks exactly like a healthy cluster, which is the failure mode worth guarding. Dashboard panels follow separately, once the metrics are live and the label names can be read off the real series rather than guessed from a README. Closes #549 Co-Authored-By: Claude Bot Co-Authored-By: Claude Opus 5 (1M context) --- .../apps/monitoring-rules/kustomization.yaml | 2 + .../prometheusrule-disk-wear.yaml | 121 +++++++++++++++++ .../smartctl-exporter-servicemonitor.yaml | 29 +++++ kubernetes/apps/monitoring/kustomization.yaml | 1 + .../apps/monitoring/smartctl-exporter.yaml | 123 ++++++++++++++++++ 5 files changed, 276 insertions(+) create mode 100644 kubernetes/apps/monitoring-rules/prometheusrule-disk-wear.yaml create mode 100644 kubernetes/apps/monitoring-rules/smartctl-exporter-servicemonitor.yaml create mode 100644 kubernetes/apps/monitoring/smartctl-exporter.yaml diff --git a/kubernetes/apps/monitoring-rules/kustomization.yaml b/kubernetes/apps/monitoring-rules/kustomization.yaml index e01dac6e..748cc9d2 100644 --- a/kubernetes/apps/monitoring-rules/kustomization.yaml +++ b/kubernetes/apps/monitoring-rules/kustomization.yaml @@ -6,6 +6,7 @@ kind: Kustomization resources: - prometheusrule-ceph.yaml - prometheusrule-cilium.yaml + - prometheusrule-disk-wear.yaml - prometheusrule-hetzner-backup.yaml - prometheusrule-loki.yaml - prometheusrule-mariadb-backup.yaml @@ -24,4 +25,5 @@ resources: - home-assistant-servicemonitor.yaml - immich-servicemonitor.yaml - loki-servicemonitor.yaml + - smartctl-exporter-servicemonitor.yaml - probes-blackbox.yaml diff --git a/kubernetes/apps/monitoring-rules/prometheusrule-disk-wear.yaml b/kubernetes/apps/monitoring-rules/prometheusrule-disk-wear.yaml new file mode 100644 index 00000000..1bb7f23a --- /dev/null +++ b/kubernetes/apps/monitoring-rules/prometheusrule-disk-wear.yaml @@ -0,0 +1,121 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: disk-wear + namespace: monitoring + labels: + # kube-prometheus-stack's Prometheus resource selects on this label; + # without it the rule is created but never loaded into Prometheus. + release: kube-prometheus-stack +spec: + groups: + - name: disk-wear.rules + rules: + # THE GAP THESE RULES CLOSE + # + # Drive wear is invisible in every other signal this cluster has. + # `ceph -s` reads active+clean at any wear level, no Ceph health check + # fires until a device actually fails, and the block-layer metrics show + # throughput and latency but never endurance. That blind spot is how a + # set of OSD drives reached 81-87% before anyone looked (#549, #434). + # + # The numbers below are wear percentages reported by the drive itself: + # 100 means the manufacturer's rated endurance is used up. Drives do + # not stop at 100 — they typically keep working and start consuming + # spare blocks — so treat these as planning thresholds, not failure + # predictions. + - alert: NvmeDriveWearHigh + # 80% with months of runway left is the point where ordering a + # replacement is still a calm decision rather than an outage. + # `for` is long because wear is monotonic and slow: it cannot + # flap, and a short window would only make the alert noisier + # after a restart. + expr: smartctl_device_percentage_used >= 80 + for: 1h + labels: + severity: warning + annotations: + summary: "NVMe {{ $labels.device }} on {{ $labels.instance }} is at {{ $value }}% rated wear" + description: >- + The drive has used {{ $value }}% of its rated write endurance. + Nothing is failing and no data is at risk yet. Check what the + drive is carrying before deciding urgency: an OSD is replaceable + with a rebalance, whereas the EPHEMERAL volume has no such + escape — a node simply stops when it goes. The Disk Health + dashboard resolves which drive holds which role. + - alert: NvmeDriveWearCritical + expr: smartctl_device_percentage_used >= 95 + for: 1h + labels: + severity: critical + annotations: + summary: "NVMe {{ $labels.device }} on {{ $labels.instance }} is at {{ $value }}% rated wear" + description: >- + The drive is at or past the end of its rated endurance. It will + likely keep working for a while, but it is now operating outside + what the manufacturer stands behind, and replacement should be + scheduled rather than considered. + # A drive that has consumed its spare area is no longer masking media + # defects, which is a materially different state from being worn: + # wear is a budget, spare exhaustion is damage. + - alert: NvmeDriveSpareExhausted + expr: smartctl_device_available_spare <= smartctl_device_available_spare_threshold + for: 15m + labels: + severity: critical + annotations: + summary: "NVMe {{ $labels.device }} on {{ $labels.instance }} has exhausted its spare area" + description: >- + Available spare has fallen to the manufacturer's threshold. The + drive has run out of reserve blocks to remap failures into. + Replace it. + - alert: NvmeDriveCriticalWarning + # The NVMe critical warning byte is a bitfield: spare below + # threshold, temperature out of range, degraded reliability, + # read-only mode, volatile-memory backup failed. Any non-zero value + # is the drive itself asking for attention. + expr: smartctl_device_critical_warning > 0 + for: 5m + labels: + severity: critical + annotations: + summary: "NVMe {{ $labels.device }} on {{ $labels.instance }} reports critical warning {{ $value }}" + description: >- + The drive has raised its own critical warning flag. Read the + detail with `smartctl -a` on the node; the value is a bitfield, + not a count. + - alert: NvmeDriveMediaErrors + # Unlike wear, this counter should never move. Alert on the increase + # rather than the absolute value, so a drive that recorded errors + # long ago does not alert forever while a drive that starts failing + # today does. + expr: increase(smartctl_device_media_errors[6h]) > 0 + for: 5m + labels: + severity: warning + annotations: + summary: "NVMe {{ $labels.device }} on {{ $labels.instance }} recorded {{ $value }} new media errors" + description: >- + The drive reported unrecoverable data-integrity errors in the + last six hours. A healthy drive never increments this. Check + whether Ceph has logged scrub errors on the same node. + # Without this the whole group is silently useless: no series means no + # alert fires, and a dashboard of empty panels reads much like a + # healthy one. This is the check that the checks are running. + - alert: SmartctlExporterIncomplete + # Compared against node-exporter's own NVMe inventory rather than a + # written-down fleet size. Adding or removing a drive then moves both + # sides together, so this cannot rot into a permanent false alarm the + # way a hardcoded count would. + expr: count(smartctl_device_percentage_used) < count(node_nvme_info) or absent(smartctl_device_percentage_used) + for: 30m + labels: + severity: warning + annotations: + summary: "Fewer NVMe drives report SMART than node-exporter can see" + description: >- + Some drives visible to node-exporter are not reporting SMART, or + none are. Either a DaemonSet pod is down or a drive stopped + answering — both mean the wear alerts above have stopped watching + part of the fleet, which looks identical to everything being + fine. diff --git a/kubernetes/apps/monitoring-rules/smartctl-exporter-servicemonitor.yaml b/kubernetes/apps/monitoring-rules/smartctl-exporter-servicemonitor.yaml new file mode 100644 index 00000000..7b2617af --- /dev/null +++ b/kubernetes/apps/monitoring-rules/smartctl-exporter-servicemonitor.yaml @@ -0,0 +1,29 @@ +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: smartctl-exporter + namespace: monitoring + labels: + app.kubernetes.io/name: smartctl-exporter + # kube-prometheus-stack's Prometheus resource selects ServiceMonitors + # by this label; without it the resource exists but is never + # discovered. + release: kube-prometheus-stack +spec: + selector: + matchLabels: + app.kubernetes.io/name: smartctl-exporter + endpoints: + # smartctl itself polls the drives every 60s, so scraping faster than that + # returns the same numbers with a new timestamp. Wear moves on a scale of + # weeks; 60s is already far finer than the signal. + - port: http-metrics + path: /metrics + interval: 60s + relabelings: + # Without this every series is labelled with the exporter pod's IP and + # nothing else. A DaemonSet pod's IP says nothing about which machine + # the drive is in, and it changes on every reschedule — so the wear + # figure would be unattributable, which is most of its value gone. + - sourceLabels: [__meta_kubernetes_pod_node_name] + targetLabel: node diff --git a/kubernetes/apps/monitoring/kustomization.yaml b/kubernetes/apps/monitoring/kustomization.yaml index db3c7a97..c1811724 100644 --- a/kubernetes/apps/monitoring/kustomization.yaml +++ b/kubernetes/apps/monitoring/kustomization.yaml @@ -15,6 +15,7 @@ resources: - prometheus-gate-service.yaml - prometheus-gate-ingress.yaml - ci-diagnostics-rbac.yaml + - smartctl-exporter.yaml - dashboards/ # prometheusrule-garage.yaml lives in ./monitoring-rules/ and is applied by a # separate Flux Kustomization that dependsOn `monitoring` — otherwise the diff --git a/kubernetes/apps/monitoring/smartctl-exporter.yaml b/kubernetes/apps/monitoring/smartctl-exporter.yaml new file mode 100644 index 00000000..d937dcb4 --- /dev/null +++ b/kubernetes/apps/monitoring/smartctl-exporter.yaml @@ -0,0 +1,123 @@ +# SMART/endurance metrics for every NVMe drive in the fleet (#549). +# +# WHY THIS HOLDS ELEVATED CAPABILITIES, and why that is a deliberate exception +# rather than an oversight: +# +# NVMe reports health over an admin ioctl on the character device +# (NVME_IOCTL_ADMIN_CMD), which the kernel gates on CAP_SYS_ADMIN. There is no +# sysfs path for wear, media errors or written-bytes totals — hwmon publishes +# the controller temperature and nothing else. Upstream smartctl_exporter +# documents `privileged: true` and `user: root` for exactly this reason. +# +# The alternative that needs no privileges — reading Ceph's devicehealth +# archive, which does hold full smartctl output — covers OSD drives only. In +# this cluster that is three of nine, and the six it misses include the drives +# that were retired from Ceph for wear and now carry EPHEMERAL. Covering the +# healthy drives and not the worn ones was judged not worth a new service; the +# owner chose fleet-wide coverage with the privilege exception instead. +# +# The exception is kept as small as the kernel allows: no `privileged: true`, +# no host namespaces, no hostPID. Only CAP_SYS_ADMIN and CAP_SYS_RAWIO are +# added, everything else is dropped, and the only host mount is /dev — which +# is required because the exporter must open the NVMe character devices and +# Kubernetes has no way to pass individual character devices to a pod without +# a device plugin. +# +# If this ever needs revisiting: the thing to check is whether the kernel has +# grown a sysfs interface for NVMe health, not whether some other exporter +# needs fewer privileges. The requirement comes from the ioctl gate, not from +# the tool. +# +# Upstream had no release between 2025-04-22 (v0.14.0) and this deployment. +# The repository is not archived; the pin is simply its newest tag. +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: smartctl-exporter + namespace: monitoring + labels: + app.kubernetes.io/name: smartctl-exporter +spec: + selector: + matchLabels: + app.kubernetes.io/name: smartctl-exporter + template: + metadata: + labels: + app.kubernetes.io/name: smartctl-exporter + spec: + # Control planes are also the storage nodes here, so the exporter has to + # tolerate the control-plane taint or it would cover nothing at all. + tolerations: + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + containers: + - name: smartctl-exporter + image: quay.io/prometheuscommunity/smartctl-exporter:v0.14.0 + # No --smartctl.device on purpose: the exporter scans for devices + # itself and rescans every 10m by default. Device names are not + # stable across these machines, so a hardcoded list would quietly + # stop covering a drive the next time one is replaced. + args: + - --web.listen-address=:9633 + ports: + - name: http-metrics + containerPort: 9633 + protocol: TCP + securityContext: + # Root plus two capabilities, NOT privileged. See the header. + runAsUser: 0 + runAsNonRoot: false + privileged: false + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + add: ["SYS_ADMIN", "SYS_RAWIO"] + volumeMounts: + - name: dev + mountPath: /dev + readOnly: true + # smartctl is not known to need scratch space, but the root + # filesystem is read-only and a tool that turns out to want /tmp + # would crashloop instead of degrading. Cheaper to provide it than + # to find out from a restart counter. + - name: tmp + mountPath: /tmp + resources: + requests: + cpu: 10m + memory: 32Mi + limits: + memory: 128Mi + livenessProbe: + httpGet: + path: /metrics + port: http-metrics + initialDelaySeconds: 15 + periodSeconds: 30 + volumes: + - name: dev + hostPath: + path: /dev + type: Directory + - name: tmp + emptyDir: {} +--- +apiVersion: v1 +kind: Service +metadata: + name: smartctl-exporter + namespace: monitoring + labels: + app.kubernetes.io/name: smartctl-exporter +spec: + clusterIP: None + selector: + app.kubernetes.io/name: smartctl-exporter + ports: + - name: http-metrics + port: 9633 + targetPort: http-metrics + protocol: TCP -- 2.54.0