diff --git a/kubernetes/apps/monitoring-rules/kustomization.yaml b/kubernetes/apps/monitoring-rules/kustomization.yaml index e01dac6e..748cc9d2 100644 --- a/kubernetes/apps/monitoring-rules/kustomization.yaml +++ b/kubernetes/apps/monitoring-rules/kustomization.yaml @@ -6,6 +6,7 @@ kind: Kustomization resources: - prometheusrule-ceph.yaml - prometheusrule-cilium.yaml + - prometheusrule-disk-wear.yaml - prometheusrule-hetzner-backup.yaml - prometheusrule-loki.yaml - prometheusrule-mariadb-backup.yaml @@ -24,4 +25,5 @@ resources: - home-assistant-servicemonitor.yaml - immich-servicemonitor.yaml - loki-servicemonitor.yaml + - smartctl-exporter-servicemonitor.yaml - probes-blackbox.yaml diff --git a/kubernetes/apps/monitoring-rules/prometheusrule-disk-wear.yaml b/kubernetes/apps/monitoring-rules/prometheusrule-disk-wear.yaml new file mode 100644 index 00000000..1bb7f23a --- /dev/null +++ b/kubernetes/apps/monitoring-rules/prometheusrule-disk-wear.yaml @@ -0,0 +1,121 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: disk-wear + namespace: monitoring + labels: + # kube-prometheus-stack's Prometheus resource selects on this label; + # without it the rule is created but never loaded into Prometheus. + release: kube-prometheus-stack +spec: + groups: + - name: disk-wear.rules + rules: + # THE GAP THESE RULES CLOSE + # + # Drive wear is invisible in every other signal this cluster has. + # `ceph -s` reads active+clean at any wear level, no Ceph health check + # fires until a device actually fails, and the block-layer metrics show + # throughput and latency but never endurance. That blind spot is how a + # set of OSD drives reached 81-87% before anyone looked (#549, #434). + # + # The numbers below are wear percentages reported by the drive itself: + # 100 means the manufacturer's rated endurance is used up. Drives do + # not stop at 100 — they typically keep working and start consuming + # spare blocks — so treat these as planning thresholds, not failure + # predictions. + - alert: NvmeDriveWearHigh + # 80% with months of runway left is the point where ordering a + # replacement is still a calm decision rather than an outage. + # `for` is long because wear is monotonic and slow: it cannot + # flap, and a short window would only make the alert noisier + # after a restart. + expr: smartctl_device_percentage_used >= 80 + for: 1h + labels: + severity: warning + annotations: + summary: "NVMe {{ $labels.device }} on {{ $labels.instance }} is at {{ $value }}% rated wear" + description: >- + The drive has used {{ $value }}% of its rated write endurance. + Nothing is failing and no data is at risk yet. Check what the + drive is carrying before deciding urgency: an OSD is replaceable + with a rebalance, whereas the EPHEMERAL volume has no such + escape — a node simply stops when it goes. The Disk Health + dashboard resolves which drive holds which role. + - alert: NvmeDriveWearCritical + expr: smartctl_device_percentage_used >= 95 + for: 1h + labels: + severity: critical + annotations: + summary: "NVMe {{ $labels.device }} on {{ $labels.instance }} is at {{ $value }}% rated wear" + description: >- + The drive is at or past the end of its rated endurance. It will + likely keep working for a while, but it is now operating outside + what the manufacturer stands behind, and replacement should be + scheduled rather than considered. + # A drive that has consumed its spare area is no longer masking media + # defects, which is a materially different state from being worn: + # wear is a budget, spare exhaustion is damage. + - alert: NvmeDriveSpareExhausted + expr: smartctl_device_available_spare <= smartctl_device_available_spare_threshold + for: 15m + labels: + severity: critical + annotations: + summary: "NVMe {{ $labels.device }} on {{ $labels.instance }} has exhausted its spare area" + description: >- + Available spare has fallen to the manufacturer's threshold. The + drive has run out of reserve blocks to remap failures into. + Replace it. + - alert: NvmeDriveCriticalWarning + # The NVMe critical warning byte is a bitfield: spare below + # threshold, temperature out of range, degraded reliability, + # read-only mode, volatile-memory backup failed. Any non-zero value + # is the drive itself asking for attention. + expr: smartctl_device_critical_warning > 0 + for: 5m + labels: + severity: critical + annotations: + summary: "NVMe {{ $labels.device }} on {{ $labels.instance }} reports critical warning {{ $value }}" + description: >- + The drive has raised its own critical warning flag. Read the + detail with `smartctl -a` on the node; the value is a bitfield, + not a count. + - alert: NvmeDriveMediaErrors + # Unlike wear, this counter should never move. Alert on the increase + # rather than the absolute value, so a drive that recorded errors + # long ago does not alert forever while a drive that starts failing + # today does. + expr: increase(smartctl_device_media_errors[6h]) > 0 + for: 5m + labels: + severity: warning + annotations: + summary: "NVMe {{ $labels.device }} on {{ $labels.instance }} recorded {{ $value }} new media errors" + description: >- + The drive reported unrecoverable data-integrity errors in the + last six hours. A healthy drive never increments this. Check + whether Ceph has logged scrub errors on the same node. + # Without this the whole group is silently useless: no series means no + # alert fires, and a dashboard of empty panels reads much like a + # healthy one. This is the check that the checks are running. + - alert: SmartctlExporterIncomplete + # Compared against node-exporter's own NVMe inventory rather than a + # written-down fleet size. Adding or removing a drive then moves both + # sides together, so this cannot rot into a permanent false alarm the + # way a hardcoded count would. + expr: count(smartctl_device_percentage_used) < count(node_nvme_info) or absent(smartctl_device_percentage_used) + for: 30m + labels: + severity: warning + annotations: + summary: "Fewer NVMe drives report SMART than node-exporter can see" + description: >- + Some drives visible to node-exporter are not reporting SMART, or + none are. Either a DaemonSet pod is down or a drive stopped + answering — both mean the wear alerts above have stopped watching + part of the fleet, which looks identical to everything being + fine. diff --git a/kubernetes/apps/monitoring-rules/smartctl-exporter-servicemonitor.yaml b/kubernetes/apps/monitoring-rules/smartctl-exporter-servicemonitor.yaml new file mode 100644 index 00000000..7b2617af --- /dev/null +++ b/kubernetes/apps/monitoring-rules/smartctl-exporter-servicemonitor.yaml @@ -0,0 +1,29 @@ +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: smartctl-exporter + namespace: monitoring + labels: + app.kubernetes.io/name: smartctl-exporter + # kube-prometheus-stack's Prometheus resource selects ServiceMonitors + # by this label; without it the resource exists but is never + # discovered. + release: kube-prometheus-stack +spec: + selector: + matchLabels: + app.kubernetes.io/name: smartctl-exporter + endpoints: + # smartctl itself polls the drives every 60s, so scraping faster than that + # returns the same numbers with a new timestamp. Wear moves on a scale of + # weeks; 60s is already far finer than the signal. + - port: http-metrics + path: /metrics + interval: 60s + relabelings: + # Without this every series is labelled with the exporter pod's IP and + # nothing else. A DaemonSet pod's IP says nothing about which machine + # the drive is in, and it changes on every reschedule — so the wear + # figure would be unattributable, which is most of its value gone. + - sourceLabels: [__meta_kubernetes_pod_node_name] + targetLabel: node diff --git a/kubernetes/apps/monitoring/kustomization.yaml b/kubernetes/apps/monitoring/kustomization.yaml index db3c7a97..c1811724 100644 --- a/kubernetes/apps/monitoring/kustomization.yaml +++ b/kubernetes/apps/monitoring/kustomization.yaml @@ -15,6 +15,7 @@ resources: - prometheus-gate-service.yaml - prometheus-gate-ingress.yaml - ci-diagnostics-rbac.yaml + - smartctl-exporter.yaml - dashboards/ # prometheusrule-garage.yaml lives in ./monitoring-rules/ and is applied by a # separate Flux Kustomization that dependsOn `monitoring` — otherwise the diff --git a/kubernetes/apps/monitoring/smartctl-exporter.yaml b/kubernetes/apps/monitoring/smartctl-exporter.yaml new file mode 100644 index 00000000..d937dcb4 --- /dev/null +++ b/kubernetes/apps/monitoring/smartctl-exporter.yaml @@ -0,0 +1,123 @@ +# SMART/endurance metrics for every NVMe drive in the fleet (#549). +# +# WHY THIS HOLDS ELEVATED CAPABILITIES, and why that is a deliberate exception +# rather than an oversight: +# +# NVMe reports health over an admin ioctl on the character device +# (NVME_IOCTL_ADMIN_CMD), which the kernel gates on CAP_SYS_ADMIN. There is no +# sysfs path for wear, media errors or written-bytes totals — hwmon publishes +# the controller temperature and nothing else. Upstream smartctl_exporter +# documents `privileged: true` and `user: root` for exactly this reason. +# +# The alternative that needs no privileges — reading Ceph's devicehealth +# archive, which does hold full smartctl output — covers OSD drives only. In +# this cluster that is three of nine, and the six it misses include the drives +# that were retired from Ceph for wear and now carry EPHEMERAL. Covering the +# healthy drives and not the worn ones was judged not worth a new service; the +# owner chose fleet-wide coverage with the privilege exception instead. +# +# The exception is kept as small as the kernel allows: no `privileged: true`, +# no host namespaces, no hostPID. Only CAP_SYS_ADMIN and CAP_SYS_RAWIO are +# added, everything else is dropped, and the only host mount is /dev — which +# is required because the exporter must open the NVMe character devices and +# Kubernetes has no way to pass individual character devices to a pod without +# a device plugin. +# +# If this ever needs revisiting: the thing to check is whether the kernel has +# grown a sysfs interface for NVMe health, not whether some other exporter +# needs fewer privileges. The requirement comes from the ioctl gate, not from +# the tool. +# +# Upstream had no release between 2025-04-22 (v0.14.0) and this deployment. +# The repository is not archived; the pin is simply its newest tag. +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: smartctl-exporter + namespace: monitoring + labels: + app.kubernetes.io/name: smartctl-exporter +spec: + selector: + matchLabels: + app.kubernetes.io/name: smartctl-exporter + template: + metadata: + labels: + app.kubernetes.io/name: smartctl-exporter + spec: + # Control planes are also the storage nodes here, so the exporter has to + # tolerate the control-plane taint or it would cover nothing at all. + tolerations: + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + containers: + - name: smartctl-exporter + image: quay.io/prometheuscommunity/smartctl-exporter:v0.14.0 + # No --smartctl.device on purpose: the exporter scans for devices + # itself and rescans every 10m by default. Device names are not + # stable across these machines, so a hardcoded list would quietly + # stop covering a drive the next time one is replaced. + args: + - --web.listen-address=:9633 + ports: + - name: http-metrics + containerPort: 9633 + protocol: TCP + securityContext: + # Root plus two capabilities, NOT privileged. See the header. + runAsUser: 0 + runAsNonRoot: false + privileged: false + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + add: ["SYS_ADMIN", "SYS_RAWIO"] + volumeMounts: + - name: dev + mountPath: /dev + readOnly: true + # smartctl is not known to need scratch space, but the root + # filesystem is read-only and a tool that turns out to want /tmp + # would crashloop instead of degrading. Cheaper to provide it than + # to find out from a restart counter. + - name: tmp + mountPath: /tmp + resources: + requests: + cpu: 10m + memory: 32Mi + limits: + memory: 128Mi + livenessProbe: + httpGet: + path: /metrics + port: http-metrics + initialDelaySeconds: 15 + periodSeconds: 30 + volumes: + - name: dev + hostPath: + path: /dev + type: Directory + - name: tmp + emptyDir: {} +--- +apiVersion: v1 +kind: Service +metadata: + name: smartctl-exporter + namespace: monitoring + labels: + app.kubernetes.io/name: smartctl-exporter +spec: + clusterIP: None + selector: + app.kubernetes.io/name: smartctl-exporter + ports: + - name: http-metrics + port: 9633 + targetPort: http-metrics + protocol: TCP