From 18629fd496a1c9340d20e1589887acbf26aec0ec Mon Sep 17 00:00:00 2001 From: atlas-bot Date: Thu, 1 Oct 2026 05:40:32 +0000 Subject: [PATCH 1/2] monitoring: stop wear alerts on EPHEMERAL NVMe drives (REN-419) The owner decided EPHEMERAL (/var) drives run to failure (REN-420), so NvmeDriveWearProjectedExhaustion and NvmeDriveWearCritical now skip the drive carrying /var. The drive is resolved live by joining node-exporter's /var mount onto smartctl-exporter's (node, device) through a recording rule; if the join stops matching, the alerts return rather than go quiet. Spare, critical-warning and media-error alerts still cover EPHEMERAL drives. Adds a promtool suite and four negative controls for the exclusion. Co-Authored-By: Claude Opus 5.5 --- .../prometheusrule-disk-wear.yaml | 36 +++++- .../apps/monitoring-rules/tests/README.md | 1 + .../tests/disk-wear_test.yaml | 119 ++++++++++++++++++ .../apps/monitoring-rules/tests/mutations.txt | 18 +++ 4 files changed, 171 insertions(+), 3 deletions(-) create mode 100644 kubernetes/apps/monitoring-rules/tests/disk-wear_test.yaml diff --git a/kubernetes/apps/monitoring-rules/prometheusrule-disk-wear.yaml b/kubernetes/apps/monitoring-rules/prometheusrule-disk-wear.yaml index 13cc1cf9..921e9073 100644 --- a/kubernetes/apps/monitoring-rules/prometheusrule-disk-wear.yaml +++ b/kubernetes/apps/monitoring-rules/prometheusrule-disk-wear.yaml @@ -11,6 +11,31 @@ spec: groups: - name: disk-wear.rules rules: + # The drives carrying Talos's EPHEMERAL volume (/var), as (node, device) + # in smartctl-exporter's labels. EPHEMERAL sits on retired Ceph drives + # whose contents rebuild, and the owner's call is that they run to + # failure rather than be replaced on wear (REN-419, REN-420). So the two + # wear alerts below skip them. The failure alerts — spare, critical + # warning, media errors — still cover them: run to failure still means + # knowing when it failed. + # + # Resolved from node-exporter's /var mount, not a list of serials, so it + # follows a drive swap without an edit. If the join stops matching (a + # relabel, an encrypted /var on a dm device) the series is empty, and + # the wear alerts return for EPHEMERAL drives: it fails loud, not + # silent. + - record: node_device:nvme_ephemeral_volume:info + expr: >- + count by (node, device) ( + label_replace( + label_replace( + node_filesystem_size_bytes{mountpoint="/var"} + * on (instance) group_left (nodename) node_uname_info, + "node", "$1", "nodename", "(.+)" + ), + "device", "$1", "device", "/dev/(nvme[0-9]+)n[0-9]+(p[0-9]+)?" + ) + ) # Wear percentages are self-reported: 100 means the rated endurance is # spent, not that the drive stops. # @@ -45,6 +70,7 @@ spec: ) ) < 180 and smartctl_device_percentage_used >= 80 + unless on (node, device) node_device:nvme_ephemeral_volume:info # Long, because a projection off a 7d rate should not react to a day. for: 6h labels: @@ -55,13 +81,17 @@ spec: The drive is past 80% of its rated endurance and the write rate of the last week would spend the rest within six months. This is a lead-time warning, not a fault: decide whether to replace the - drive or move what is writing to it. The Disk Health dashboard + drive or move what is writing to it. Drives carrying EPHEMERAL + are excluded; they run to failure. The Disk Health dashboard shows which drives carry which role. # Kept as a flat threshold on purpose: past rated endurance the drive # runs outside what the manufacturer stands behind regardless of how - # fast it got there, and the alert clears by replacing it. + # fast it got there, and the alert clears by replacing it. EPHEMERAL + # drives are excluded for the same reason as above. - alert: NvmeDriveWearCritical - expr: smartctl_device_percentage_used >= 95 + expr: >- + smartctl_device_percentage_used >= 95 + unless on (node, device) node_device:nvme_ephemeral_volume:info for: 1h labels: severity: critical diff --git a/kubernetes/apps/monitoring-rules/tests/README.md b/kubernetes/apps/monitoring-rules/tests/README.md index d5c14acc..7bff6713 100644 --- a/kubernetes/apps/monitoring-rules/tests/README.md +++ b/kubernetes/apps/monitoring-rules/tests/README.md @@ -30,6 +30,7 @@ retired the depth rule in favour of one persistence rule (`> 5` for `20m`). | `external-secrets_test.yaml` | Hand-written suite for `prometheusrule-external-secrets.yaml` (REN-103). | | `home-assistant_test.yaml` | Hand-written suite for `prometheusrule-home-assistant.yaml` (REN-245). | | `ceph_test.yaml` | Hand-written suite for the `CephPoolLostRedundancyBlind` canary in `prometheusrule-ceph.yaml` (REN-93). | +| `disk-wear_test.yaml` | Hand-written suite for the EPHEMERAL exclusion in `prometheusrule-disk-wear.yaml` (REN-419). | | `mutations.txt` | Negative controls. Each must make the suite fail. | ## Adding a suite diff --git a/kubernetes/apps/monitoring-rules/tests/disk-wear_test.yaml b/kubernetes/apps/monitoring-rules/tests/disk-wear_test.yaml new file mode 100644 index 00000000..b989a677 --- /dev/null +++ b/kubernetes/apps/monitoring-rules/tests/disk-wear_test.yaml @@ -0,0 +1,119 @@ +# promtool test rules for the EPHEMERAL exclusion in prometheusrule-disk-wear.yaml +# (REN-419). +# +# Run by the `promtool` step of .woodpecker/ci-manifests-test.yml through +# scripts/ci/promtool-rules.sh, which strips `spec:` from the manifest into a +# mirrored tree — hence `../` below, resolved from this file's directory. See +# README.md here. The negative controls for this suite are the `disk-wear-*` +# entries in mutations.txt. +# +# WHAT IT IS FOR. The two wear alerts skip the drive that carries /var, found +# by joining node-exporter's /var mount onto smartctl-exporter's (node, device). +# Each case puts an EPHEMERAL drive and a non-EPHEMERAL drive side by side with +# identical SMART data, so the only thing that can separate them is the join. +# +# Fixture shape: smartctl-exporter labels drives `nvme0`, `nvme1` and carries +# `node` from the ServiceMonitor relabel; node-exporter carries `instance` and +# reports /var on the partition, `/dev/nvme1n1p1`. +rule_files: + - ../prometheusrule-disk-wear.yaml +evaluation_interval: 1m +tests: + # A. Projection. k8s-1 nvme0 and nvme1 wear identically: 89%, 2e10 bytes an + # hour, 1e15 lifetime bytes at 192h. That is 0.04272 % a day, 6 % to go, + # 140 days — under 180. Only nvme0 may fire; nvme1 carries /var. + # Neither is at 95%, so the critical alert stays quiet. + - interval: 1h + input_series: + - series: 'smartctl_device_percentage_used{node="k8s-1",device="nvme0",instance="10.0.0.1:9633"}' + values: '89+0x200' + - series: 'smartctl_device_bytes_written{node="k8s-1",device="nvme0",instance="10.0.0.1:9633"}' + values: '996160000000000+20000000000x200' + - series: 'smartctl_device_percentage_used{node="k8s-1",device="nvme1",instance="10.0.0.1:9633"}' + values: '89+0x200' + - series: 'smartctl_device_bytes_written{node="k8s-1",device="nvme1",instance="10.0.0.1:9633"}' + values: '996160000000000+20000000000x200' + - series: 'node_filesystem_size_bytes{instance="10.0.0.1:9100",mountpoint="/var",device="/dev/nvme1n1p1",fstype="xfs"}' + values: '1000000000000+0x200' + - series: 'node_uname_info{instance="10.0.0.1:9100",nodename="k8s-1"}' + values: '1+0x200' + alert_rule_test: + - eval_time: 192h + alertname: NvmeDriveWearProjectedExhaustion + exp_alerts: + - exp_labels: + severity: warning + node: k8s-1 + device: nvme0 + instance: 10.0.0.1:9633 + exp_annotations: + summary: "NVMe nvme0 on 10.0.0.1:9633 projects to 95% wear in 140 days" + description: "The drive is past 80% of its rated endurance and the write rate of the last week would spend the rest within six months. This is a lead-time warning, not a fault: decide whether to replace the drive or move what is writing to it. Drives carrying EPHEMERAL are excluded; they run to failure. The Disk Health dashboard shows which drives carry which role." + - eval_time: 192h + alertname: NvmeDriveWearCritical + exp_alerts: [] + + # B. Critical, two nodes. Every drive is at 96%. On k8s-1 /var is nvme1; on + # k8s-2 it is nvme2, which reports no SMART here. So k8s-1 nvme1 is the only + # one excluded, and k8s-2 nvme1 must still fire — the exclusion is per + # node, not per device name. + - interval: 1m + input_series: + - series: 'smartctl_device_percentage_used{node="k8s-1",device="nvme0",instance="10.0.0.1:9633"}' + values: '96+0x200' + - series: 'smartctl_device_percentage_used{node="k8s-1",device="nvme1",instance="10.0.0.1:9633"}' + values: '96+0x200' + - series: 'smartctl_device_percentage_used{node="k8s-2",device="nvme1",instance="10.0.0.2:9633"}' + values: '96+0x200' + - series: 'node_filesystem_size_bytes{instance="10.0.0.1:9100",mountpoint="/var",device="/dev/nvme1n1p1",fstype="xfs"}' + values: '1000000000000+0x200' + - series: 'node_uname_info{instance="10.0.0.1:9100",nodename="k8s-1"}' + values: '1+0x200' + - series: 'node_filesystem_size_bytes{instance="10.0.0.2:9100",mountpoint="/var",device="/dev/nvme2n1p1",fstype="xfs"}' + values: '1000000000000+0x200' + - series: 'node_uname_info{instance="10.0.0.2:9100",nodename="k8s-2"}' + values: '1+0x200' + alert_rule_test: + - eval_time: 120m + alertname: NvmeDriveWearCritical + exp_alerts: + - exp_labels: + severity: critical + node: k8s-1 + device: nvme0 + instance: 10.0.0.1:9633 + exp_annotations: + summary: "NVMe nvme0 on 10.0.0.1:9633 is at 96% rated wear" + description: "The drive is at or past the end of its rated endurance. It will likely keep working for a while, but it is now operating outside what the manufacturer stands behind, and replacement should be scheduled rather than considered." + - exp_labels: + severity: critical + node: k8s-2 + device: nvme1 + instance: 10.0.0.2:9633 + exp_annotations: + summary: "NVMe nvme1 on 10.0.0.2:9633 is at 96% rated wear" + description: "The drive is at or past the end of its rated endurance. It will likely keep working for a while, but it is now operating outside what the manufacturer stands behind, and replacement should be scheduled rather than considered." + + # C. Fails loud. /var is on a device the join cannot map to an NVMe name (an + # encrypted volume on dm-0). The exclusion then matches nothing and the + # EPHEMERAL drive alerts again, which is the intended direction to fail. + - interval: 1m + input_series: + - series: 'smartctl_device_percentage_used{node="k8s-1",device="nvme1",instance="10.0.0.1:9633"}' + values: '96+0x200' + - series: 'node_filesystem_size_bytes{instance="10.0.0.1:9100",mountpoint="/var",device="/dev/dm-0",fstype="xfs"}' + values: '1000000000000+0x200' + - series: 'node_uname_info{instance="10.0.0.1:9100",nodename="k8s-1"}' + values: '1+0x200' + alert_rule_test: + - eval_time: 120m + alertname: NvmeDriveWearCritical + exp_alerts: + - exp_labels: + severity: critical + node: k8s-1 + device: nvme1 + instance: 10.0.0.1:9633 + exp_annotations: + summary: "NVMe nvme1 on 10.0.0.1:9633 is at 96% rated wear" + description: "The drive is at or past the end of its rated endurance. It will likely keep working for a while, but it is now operating outside what the manufacturer stands behind, and replacement should be scheduled rather than considered." diff --git a/kubernetes/apps/monitoring-rules/tests/mutations.txt b/kubernetes/apps/monitoring-rules/tests/mutations.txt index e00fb6d8..ad2eaaca 100644 --- a/kubernetes/apps/monitoring-rules/tests/mutations.txt +++ b/kubernetes/apps/monitoring-rules/tests/mutations.txt @@ -149,3 +149,21 @@ retention-no-size-clause|kubernetes/apps/monitoring-rules/prometheusrule-prometh retention-ratio-0.9|kubernetes/apps/monitoring-rules/prometheusrule-prometheus-retention.yaml|s/^\([[:space:]]*\)> 0.8$/\1> 0.9/ retention-lookback-1d|kubernetes/apps/monitoring-rules/prometheusrule-prometheus-retention.yaml|s/)\[7d:1h\]$/)[1d:1h]/ retention-cap-at-peak-time|kubernetes/apps/monitoring-rules/prometheusrule-prometheus-retention.yaml|s/\/ max(prometheus_tsdb_retention_limit_bytes{job="kube-prometheus-stack-prometheus"})$/\/ min_over_time(max(prometheus_tsdb_retention_limit_bytes{job="kube-prometheus-stack-prometheus"})[7d:1h])/ + +# prometheusrule-disk-wear.yaml (REN-419): the EPHEMERAL exclusion on the two +# wear alerts. Detected by the cases in disk-wear_test.yaml. +# +# Gate inverted on the projection: the EPHEMERAL drive alone fires. +# Detected by: case A. +disk-wear-projection-gate-inverted|kubernetes/apps/monitoring-rules/prometheusrule-disk-wear.yaml|/alert: NvmeDriveWearProjectedExhaustion/,/for: 6h/s/unless on (node, device)/and on (node, device)/ +# Gate inverted on the critical rule. +# Detected by: case B. +disk-wear-critical-gate-inverted|kubernetes/apps/monitoring-rules/prometheusrule-disk-wear.yaml|/alert: NvmeDriveWearCritical/,/for: 1h/s/unless on (node, device)/and on (node, device)/ +# Matched on device name only: k8s-1's /var on nvme1 then silences nvme1 on +# every node. +# Detected by: case B, k8s-2 nvme1. +disk-wear-critical-device-only|kubernetes/apps/monitoring-rules/prometheusrule-disk-wear.yaml|/alert: NvmeDriveWearCritical/,/for: 1h/s/unless on (node, device)/unless on (device)/ +# Partition suffix not stripped: /dev/nvme1n1p1 no longer maps to nvme1 and the +# exclusion matches nothing. +# Detected by: cases A and B. +disk-wear-ephemeral-keeps-partition|kubernetes/apps/monitoring-rules/prometheusrule-disk-wear.yaml|s/n\[0-9\]+(p\[0-9\]+)?"/n[0-9]+"/ -- 2.54.0 From 42e67219e0fd504e778c346837ec2a86113e6a46 Mon Sep 17 00:00:00 2001 From: atlas-bot Date: Thu, 1 Oct 2026 09:07:58 +0000 Subject: [PATCH 2/2] monitoring: feed disk-wear case A at 5m so the 6h for completes (REN-438) Case A sampled at 1h; with the 5m lookback the series were absent for most of each hour and the pending alert reset, so the projection never fired. Now 5m samples (1m exceeds promtool's sample limit on the 7d rate), same 0.04272 %/day and 140-day summary. Co-Authored-By: Claude Opus 5.5 --- .../tests/disk-wear_test.yaml | 27 +++++++++++-------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/kubernetes/apps/monitoring-rules/tests/disk-wear_test.yaml b/kubernetes/apps/monitoring-rules/tests/disk-wear_test.yaml index b989a677..94ab2893 100644 --- a/kubernetes/apps/monitoring-rules/tests/disk-wear_test.yaml +++ b/kubernetes/apps/monitoring-rules/tests/disk-wear_test.yaml @@ -19,24 +19,29 @@ rule_files: - ../prometheusrule-disk-wear.yaml evaluation_interval: 1m tests: - # A. Projection. k8s-1 nvme0 and nvme1 wear identically: 89%, 2e10 bytes an - # hour, 1e15 lifetime bytes at 192h. That is 0.04272 % a day, 6 % to go, - # 140 days — under 180. Only nvme0 may fire; nvme1 carries /var. - # Neither is at 95%, so the critical alert stays quiet. - - interval: 1h + # A. Projection. k8s-1 nvme0 and nvme1 wear identically: 89%, 1.5e9 bytes + # per 5m (4.32e11 a day), 9e14 lifetime bytes at 192h. That is 0.04272 % a + # day, 6 % to go, 140 days — under 180. Only nvme0 may fire; nvme1 + # carries /var. Neither is at 95%, so the critical alert stays quiet. + # + # Sampled at 5m, not 1h: the instant selectors look back 5m, so hourly + # samples leave the series absent for most of each hour and the 6h `for` + # never completes. Not 1m either: a 7d rate over 1m samples exceeds + # promtool's query sample limit. + - interval: 5m input_series: - series: 'smartctl_device_percentage_used{node="k8s-1",device="nvme0",instance="10.0.0.1:9633"}' - values: '89+0x200' + values: '89+0x2320' - series: 'smartctl_device_bytes_written{node="k8s-1",device="nvme0",instance="10.0.0.1:9633"}' - values: '996160000000000+20000000000x200' + values: '896544000000000+1500000000x2320' - series: 'smartctl_device_percentage_used{node="k8s-1",device="nvme1",instance="10.0.0.1:9633"}' - values: '89+0x200' + values: '89+0x2320' - series: 'smartctl_device_bytes_written{node="k8s-1",device="nvme1",instance="10.0.0.1:9633"}' - values: '996160000000000+20000000000x200' + values: '896544000000000+1500000000x2320' - series: 'node_filesystem_size_bytes{instance="10.0.0.1:9100",mountpoint="/var",device="/dev/nvme1n1p1",fstype="xfs"}' - values: '1000000000000+0x200' + values: '1000000000000+0x2320' - series: 'node_uname_info{instance="10.0.0.1:9100",nodename="k8s-1"}' - values: '1+0x200' + values: '1+0x2320' alert_rule_test: - eval_time: 192h alertname: NvmeDriveWearProjectedExhaustion -- 2.54.0