kubernetes/apps/monitoring-rules/prometheusrule-disk-wear.yaml

@@ -11,6 +11,31 @@ spec:
   groups:
     - name: disk-wear.rules
       rules:
+        # The drives carrying Talos's EPHEMERAL volume (/var), as (node, device)
+        # in smartctl-exporter's labels. EPHEMERAL sits on retired Ceph drives
+        # whose contents rebuild, and the owner's call is that they run to
+        # failure rather than be replaced on wear (REN-419, REN-420). So the two
+        # wear alerts below skip them. The failure alerts — spare, critical
+        # warning, media errors — still cover them: run to failure still means
+        # knowing when it failed.
+        #
+        # Resolved from node-exporter's /var mount, not a list of serials, so it
+        # follows a drive swap without an edit. If the join stops matching (a
+        # relabel, an encrypted /var on a dm device) the series is empty, and
+        # the wear alerts return for EPHEMERAL drives: it fails loud, not
+        # silent.
+        - record: node_device:nvme_ephemeral_volume:info
+          expr: >-
+            count by (node, device) (
+              label_replace(
+                label_replace(
+                  node_filesystem_size_bytes{mountpoint="/var"}
+                    * on (instance) group_left (nodename) node_uname_info,
+                  "node", "$1", "nodename", "(.+)"
+                ),
+                "device", "$1", "device", "/dev/(nvme[0-9]+)n[0-9]+(p[0-9]+)?"
+              )
+            )
         # Wear percentages are self-reported: 100 means the rated endurance is
         # spent, not that the drive stops.
         #
@@ -45,6 +70,7 @@ spec:
               )
             ) < 180
             and smartctl_device_percentage_used >= 80
+            unless on (node, device) node_device:nvme_ephemeral_volume:info
           # Long, because a projection off a 7d rate should not react to a day.
           for: 6h
           labels:
@@ -55,13 +81,17 @@ spec:
               The drive is past 80% of its rated endurance and the write rate
               of the last week would spend the rest within six months. This is
               a lead-time warning, not a fault: decide whether to replace the
-              drive or move what is writing to it. The Disk Health dashboard
+              drive or move what is writing to it. Drives carrying EPHEMERAL
+              are excluded; they run to failure. The Disk Health dashboard
               shows which drives carry which role.
         # Kept as a flat threshold on purpose: past rated endurance the drive
         # runs outside what the manufacturer stands behind regardless of how
-        # fast it got there, and the alert clears by replacing it.
+        # fast it got there, and the alert clears by replacing it. EPHEMERAL
+        # drives are excluded for the same reason as above.
         - alert: NvmeDriveWearCritical
-          expr: smartctl_device_percentage_used >= 95
+          expr: >-
+            smartctl_device_percentage_used >= 95
+            unless on (node, device) node_device:nvme_ephemeral_volume:info
           for: 1h
           labels:
             severity: critical

kubernetes/apps/monitoring-rules/tests/README.md

@@ -30,6 +30,7 @@ retired the depth rule in favour of one persistence rule (`> 5` for `20m`).
 | `external-secrets_test.yaml` | Hand-written suite for `prometheusrule-external-secrets.yaml` (REN-103). |
 | `home-assistant_test.yaml` | Hand-written suite for `prometheusrule-home-assistant.yaml` (REN-245). |
 | `ceph_test.yaml` | Hand-written suite for the `CephPoolLostRedundancyBlind` canary in `prometheusrule-ceph.yaml` (REN-93). |
+| `disk-wear_test.yaml` | Hand-written suite for the EPHEMERAL exclusion in `prometheusrule-disk-wear.yaml` (REN-419). |
 | `mutations.txt` | Negative controls. Each must make the suite fail. |
 
 ## Adding a suite

kubernetes/apps/monitoring-rules/tests/disk-wear_test.yaml

@@ -0,0 +1,124 @@
+# promtool test rules for the EPHEMERAL exclusion in prometheusrule-disk-wear.yaml
+# (REN-419).
+#
+# Run by the `promtool` step of .woodpecker/ci-manifests-test.yml through
+# scripts/ci/promtool-rules.sh, which strips `spec:` from the manifest into a
+# mirrored tree — hence `../` below, resolved from this file's directory. See
+# README.md here. The negative controls for this suite are the `disk-wear-*`
+# entries in mutations.txt.
+#
+# WHAT IT IS FOR. The two wear alerts skip the drive that carries /var, found
+# by joining node-exporter's /var mount onto smartctl-exporter's (node, device).
+# Each case puts an EPHEMERAL drive and a non-EPHEMERAL drive side by side with
+# identical SMART data, so the only thing that can separate them is the join.
+#
+# Fixture shape: smartctl-exporter labels drives `nvme0`, `nvme1` and carries
+# `node` from the ServiceMonitor relabel; node-exporter carries `instance` and
+# reports /var on the partition, `/dev/nvme1n1p1`.
+rule_files:
+  - ../prometheusrule-disk-wear.yaml
+evaluation_interval: 1m
+tests:
+  # A. Projection. k8s-1 nvme0 and nvme1 wear identically: 89%, 1.5e9 bytes
+  #    per 5m (4.32e11 a day), 9e14 lifetime bytes at 192h. That is 0.04272 % a
+  #    day, 6 % to go, 140 days — under 180. Only nvme0 may fire; nvme1
+  #    carries /var. Neither is at 95%, so the critical alert stays quiet.
+  #
+  #    Sampled at 5m, not 1h: the instant selectors look back 5m, so hourly
+  #    samples leave the series absent for most of each hour and the 6h `for`
+  #    never completes. Not 1m either: a 7d rate over 1m samples exceeds
+  #    promtool's query sample limit.
+  - interval: 5m
+    input_series:
+      - series: 'smartctl_device_percentage_used{node="k8s-1",device="nvme0",instance="10.0.0.1:9633"}'
+        values: '89+0x2320'
+      - series: 'smartctl_device_bytes_written{node="k8s-1",device="nvme0",instance="10.0.0.1:9633"}'
+        values: '896544000000000+1500000000x2320'
+      - series: 'smartctl_device_percentage_used{node="k8s-1",device="nvme1",instance="10.0.0.1:9633"}'
+        values: '89+0x2320'
+      - series: 'smartctl_device_bytes_written{node="k8s-1",device="nvme1",instance="10.0.0.1:9633"}'
+        values: '896544000000000+1500000000x2320'
+      - series: 'node_filesystem_size_bytes{instance="10.0.0.1:9100",mountpoint="/var",device="/dev/nvme1n1p1",fstype="xfs"}'
+        values: '1000000000000+0x2320'
+      - series: 'node_uname_info{instance="10.0.0.1:9100",nodename="k8s-1"}'
+        values: '1+0x2320'
+    alert_rule_test:
+      - eval_time: 192h
+        alertname: NvmeDriveWearProjectedExhaustion
+        exp_alerts:
+          - exp_labels:
+              severity: warning
+              node: k8s-1
+              device: nvme0
+              instance: 10.0.0.1:9633
+            exp_annotations:
+              summary: "NVMe nvme0 on 10.0.0.1:9633 projects to 95% wear in 140 days"
+              description: "The drive is past 80% of its rated endurance and the write rate of the last week would spend the rest within six months. This is a lead-time warning, not a fault: decide whether to replace the drive or move what is writing to it. Drives carrying EPHEMERAL are excluded; they run to failure. The Disk Health dashboard shows which drives carry which role."
+      - eval_time: 192h
+        alertname: NvmeDriveWearCritical
+        exp_alerts: []
+
+  # B. Critical, two nodes. Every drive is at 96%. On k8s-1 /var is nvme1; on
+  #    k8s-2 it is nvme2, which reports no SMART here. So k8s-1 nvme1 is the only
+  #    one excluded, and k8s-2 nvme1 must still fire — the exclusion is per
+  #    node, not per device name.
+  - interval: 1m
+    input_series:
+      - series: 'smartctl_device_percentage_used{node="k8s-1",device="nvme0",instance="10.0.0.1:9633"}'
+        values: '96+0x200'
+      - series: 'smartctl_device_percentage_used{node="k8s-1",device="nvme1",instance="10.0.0.1:9633"}'
+        values: '96+0x200'
+      - series: 'smartctl_device_percentage_used{node="k8s-2",device="nvme1",instance="10.0.0.2:9633"}'
+        values: '96+0x200'
+      - series: 'node_filesystem_size_bytes{instance="10.0.0.1:9100",mountpoint="/var",device="/dev/nvme1n1p1",fstype="xfs"}'
+        values: '1000000000000+0x200'
+      - series: 'node_uname_info{instance="10.0.0.1:9100",nodename="k8s-1"}'
+        values: '1+0x200'
+      - series: 'node_filesystem_size_bytes{instance="10.0.0.2:9100",mountpoint="/var",device="/dev/nvme2n1p1",fstype="xfs"}'
+        values: '1000000000000+0x200'
+      - series: 'node_uname_info{instance="10.0.0.2:9100",nodename="k8s-2"}'
+        values: '1+0x200'
+    alert_rule_test:
+      - eval_time: 120m
+        alertname: NvmeDriveWearCritical
+        exp_alerts:
+          - exp_labels:
+              severity: critical
+              node: k8s-1
+              device: nvme0
+              instance: 10.0.0.1:9633
+            exp_annotations:
+              summary: "NVMe nvme0 on 10.0.0.1:9633 is at 96% rated wear"
+              description: "The drive is at or past the end of its rated endurance. It will likely keep working for a while, but it is now operating outside what the manufacturer stands behind, and replacement should be scheduled rather than considered."
+          - exp_labels:
+              severity: critical
+              node: k8s-2
+              device: nvme1
+              instance: 10.0.0.2:9633
+            exp_annotations:
+              summary: "NVMe nvme1 on 10.0.0.2:9633 is at 96% rated wear"
+              description: "The drive is at or past the end of its rated endurance. It will likely keep working for a while, but it is now operating outside what the manufacturer stands behind, and replacement should be scheduled rather than considered."
+
+  # C. Fails loud. /var is on a device the join cannot map to an NVMe name (an
+  #    encrypted volume on dm-0). The exclusion then matches nothing and the
+  #    EPHEMERAL drive alerts again, which is the intended direction to fail.
+  - interval: 1m
+    input_series:
+      - series: 'smartctl_device_percentage_used{node="k8s-1",device="nvme1",instance="10.0.0.1:9633"}'
+        values: '96+0x200'
+      - series: 'node_filesystem_size_bytes{instance="10.0.0.1:9100",mountpoint="/var",device="/dev/dm-0",fstype="xfs"}'
+        values: '1000000000000+0x200'
+      - series: 'node_uname_info{instance="10.0.0.1:9100",nodename="k8s-1"}'
+        values: '1+0x200'
+    alert_rule_test:
+      - eval_time: 120m
+        alertname: NvmeDriveWearCritical
+        exp_alerts:
+          - exp_labels:
+              severity: critical
+              node: k8s-1
+              device: nvme1
+              instance: 10.0.0.1:9633
+            exp_annotations:
+              summary: "NVMe nvme1 on 10.0.0.1:9633 is at 96% rated wear"
+              description: "The drive is at or past the end of its rated endurance. It will likely keep working for a while, but it is now operating outside what the manufacturer stands behind, and replacement should be scheduled rather than considered."

kubernetes/apps/monitoring-rules/tests/mutations.txt

@@ -149,3 +149,21 @@ retention-no-size-clause|kubernetes/apps/monitoring-rules/prometheusrule-prometh
 retention-ratio-0.9|kubernetes/apps/monitoring-rules/prometheusrule-prometheus-retention.yaml|s/^\([[:space:]]*\)> 0.8$/\1> 0.9/
 retention-lookback-1d|kubernetes/apps/monitoring-rules/prometheusrule-prometheus-retention.yaml|s/)\[7d:1h\]$/)[1d:1h]/
 retention-cap-at-peak-time|kubernetes/apps/monitoring-rules/prometheusrule-prometheus-retention.yaml|s/\/ max(prometheus_tsdb_retention_limit_bytes{job="kube-prometheus-stack-prometheus"})$/\/ min_over_time(max(prometheus_tsdb_retention_limit_bytes{job="kube-prometheus-stack-prometheus"})[7d:1h])/
+
+# prometheusrule-disk-wear.yaml (REN-419): the EPHEMERAL exclusion on the two
+# wear alerts. Detected by the cases in disk-wear_test.yaml.
+#
+# Gate inverted on the projection: the EPHEMERAL drive alone fires.
+# Detected by: case A.
+disk-wear-projection-gate-inverted|kubernetes/apps/monitoring-rules/prometheusrule-disk-wear.yaml|/alert: NvmeDriveWearProjectedExhaustion/,/for: 6h/s/unless on (node, device)/and on (node, device)/
+# Gate inverted on the critical rule.
+# Detected by: case B.
+disk-wear-critical-gate-inverted|kubernetes/apps/monitoring-rules/prometheusrule-disk-wear.yaml|/alert: NvmeDriveWearCritical/,/for: 1h/s/unless on (node, device)/and on (node, device)/
+# Matched on device name only: k8s-1's /var on nvme1 then silences nvme1 on
+# every node.
+# Detected by: case B, k8s-2 nvme1.
+disk-wear-critical-device-only|kubernetes/apps/monitoring-rules/prometheusrule-disk-wear.yaml|/alert: NvmeDriveWearCritical/,/for: 1h/s/unless on (node, device)/unless on (device)/
+# Partition suffix not stripped: /dev/nvme1n1p1 no longer maps to nvme1 and the
+# exclusion matches nothing.
+# Detected by: cases A and B.
+disk-wear-ephemeral-keeps-partition|kubernetes/apps/monitoring-rules/prometheusrule-disk-wear.yaml|s/n\[0-9\]+(p\[0-9\]+)?"/n[0-9]+"/