diff --git a/kubernetes/apps/monitoring-rules/prometheusrule-disk-wear.yaml b/kubernetes/apps/monitoring-rules/prometheusrule-disk-wear.yaml index 13cc1cf9..921e9073 100644 --- a/kubernetes/apps/monitoring-rules/prometheusrule-disk-wear.yaml +++ b/kubernetes/apps/monitoring-rules/prometheusrule-disk-wear.yaml @@ -11,6 +11,31 @@ spec: groups: - name: disk-wear.rules rules: + # The drives carrying Talos's EPHEMERAL volume (/var), as (node, device) + # in smartctl-exporter's labels. EPHEMERAL sits on retired Ceph drives + # whose contents rebuild, and the owner's call is that they run to + # failure rather than be replaced on wear (REN-419, REN-420). So the two + # wear alerts below skip them. The failure alerts — spare, critical + # warning, media errors — still cover them: run to failure still means + # knowing when it failed. + # + # Resolved from node-exporter's /var mount, not a list of serials, so it + # follows a drive swap without an edit. If the join stops matching (a + # relabel, an encrypted /var on a dm device) the series is empty, and + # the wear alerts return for EPHEMERAL drives: it fails loud, not + # silent. + - record: node_device:nvme_ephemeral_volume:info + expr: >- + count by (node, device) ( + label_replace( + label_replace( + node_filesystem_size_bytes{mountpoint="/var"} + * on (instance) group_left (nodename) node_uname_info, + "node", "$1", "nodename", "(.+)" + ), + "device", "$1", "device", "/dev/(nvme[0-9]+)n[0-9]+(p[0-9]+)?" + ) + ) # Wear percentages are self-reported: 100 means the rated endurance is # spent, not that the drive stops. # @@ -45,6 +70,7 @@ spec: ) ) < 180 and smartctl_device_percentage_used >= 80 + unless on (node, device) node_device:nvme_ephemeral_volume:info # Long, because a projection off a 7d rate should not react to a day. for: 6h labels: @@ -55,13 +81,17 @@ spec: The drive is past 80% of its rated endurance and the write rate of the last week would spend the rest within six months. This is a lead-time warning, not a fault: decide whether to replace the - drive or move what is writing to it. The Disk Health dashboard + drive or move what is writing to it. Drives carrying EPHEMERAL + are excluded; they run to failure. The Disk Health dashboard shows which drives carry which role. # Kept as a flat threshold on purpose: past rated endurance the drive # runs outside what the manufacturer stands behind regardless of how - # fast it got there, and the alert clears by replacing it. + # fast it got there, and the alert clears by replacing it. EPHEMERAL + # drives are excluded for the same reason as above. - alert: NvmeDriveWearCritical - expr: smartctl_device_percentage_used >= 95 + expr: >- + smartctl_device_percentage_used >= 95 + unless on (node, device) node_device:nvme_ephemeral_volume:info for: 1h labels: severity: critical diff --git a/kubernetes/apps/monitoring-rules/tests/README.md b/kubernetes/apps/monitoring-rules/tests/README.md index d5c14acc..7bff6713 100644 --- a/kubernetes/apps/monitoring-rules/tests/README.md +++ b/kubernetes/apps/monitoring-rules/tests/README.md @@ -30,6 +30,7 @@ retired the depth rule in favour of one persistence rule (`> 5` for `20m`). | `external-secrets_test.yaml` | Hand-written suite for `prometheusrule-external-secrets.yaml` (REN-103). | | `home-assistant_test.yaml` | Hand-written suite for `prometheusrule-home-assistant.yaml` (REN-245). | | `ceph_test.yaml` | Hand-written suite for the `CephPoolLostRedundancyBlind` canary in `prometheusrule-ceph.yaml` (REN-93). | +| `disk-wear_test.yaml` | Hand-written suite for the EPHEMERAL exclusion in `prometheusrule-disk-wear.yaml` (REN-419). | | `mutations.txt` | Negative controls. Each must make the suite fail. | ## Adding a suite diff --git a/kubernetes/apps/monitoring-rules/tests/disk-wear_test.yaml b/kubernetes/apps/monitoring-rules/tests/disk-wear_test.yaml new file mode 100644 index 00000000..94ab2893 --- /dev/null +++ b/kubernetes/apps/monitoring-rules/tests/disk-wear_test.yaml @@ -0,0 +1,124 @@ +# promtool test rules for the EPHEMERAL exclusion in prometheusrule-disk-wear.yaml +# (REN-419). +# +# Run by the `promtool` step of .woodpecker/ci-manifests-test.yml through +# scripts/ci/promtool-rules.sh, which strips `spec:` from the manifest into a +# mirrored tree — hence `../` below, resolved from this file's directory. See +# README.md here. The negative controls for this suite are the `disk-wear-*` +# entries in mutations.txt. +# +# WHAT IT IS FOR. The two wear alerts skip the drive that carries /var, found +# by joining node-exporter's /var mount onto smartctl-exporter's (node, device). +# Each case puts an EPHEMERAL drive and a non-EPHEMERAL drive side by side with +# identical SMART data, so the only thing that can separate them is the join. +# +# Fixture shape: smartctl-exporter labels drives `nvme0`, `nvme1` and carries +# `node` from the ServiceMonitor relabel; node-exporter carries `instance` and +# reports /var on the partition, `/dev/nvme1n1p1`. +rule_files: + - ../prometheusrule-disk-wear.yaml +evaluation_interval: 1m +tests: + # A. Projection. k8s-1 nvme0 and nvme1 wear identically: 89%, 1.5e9 bytes + # per 5m (4.32e11 a day), 9e14 lifetime bytes at 192h. That is 0.04272 % a + # day, 6 % to go, 140 days — under 180. Only nvme0 may fire; nvme1 + # carries /var. Neither is at 95%, so the critical alert stays quiet. + # + # Sampled at 5m, not 1h: the instant selectors look back 5m, so hourly + # samples leave the series absent for most of each hour and the 6h `for` + # never completes. Not 1m either: a 7d rate over 1m samples exceeds + # promtool's query sample limit. + - interval: 5m + input_series: + - series: 'smartctl_device_percentage_used{node="k8s-1",device="nvme0",instance="10.0.0.1:9633"}' + values: '89+0x2320' + - series: 'smartctl_device_bytes_written{node="k8s-1",device="nvme0",instance="10.0.0.1:9633"}' + values: '896544000000000+1500000000x2320' + - series: 'smartctl_device_percentage_used{node="k8s-1",device="nvme1",instance="10.0.0.1:9633"}' + values: '89+0x2320' + - series: 'smartctl_device_bytes_written{node="k8s-1",device="nvme1",instance="10.0.0.1:9633"}' + values: '896544000000000+1500000000x2320' + - series: 'node_filesystem_size_bytes{instance="10.0.0.1:9100",mountpoint="/var",device="/dev/nvme1n1p1",fstype="xfs"}' + values: '1000000000000+0x2320' + - series: 'node_uname_info{instance="10.0.0.1:9100",nodename="k8s-1"}' + values: '1+0x2320' + alert_rule_test: + - eval_time: 192h + alertname: NvmeDriveWearProjectedExhaustion + exp_alerts: + - exp_labels: + severity: warning + node: k8s-1 + device: nvme0 + instance: 10.0.0.1:9633 + exp_annotations: + summary: "NVMe nvme0 on 10.0.0.1:9633 projects to 95% wear in 140 days" + description: "The drive is past 80% of its rated endurance and the write rate of the last week would spend the rest within six months. This is a lead-time warning, not a fault: decide whether to replace the drive or move what is writing to it. Drives carrying EPHEMERAL are excluded; they run to failure. The Disk Health dashboard shows which drives carry which role." + - eval_time: 192h + alertname: NvmeDriveWearCritical + exp_alerts: [] + + # B. Critical, two nodes. Every drive is at 96%. On k8s-1 /var is nvme1; on + # k8s-2 it is nvme2, which reports no SMART here. So k8s-1 nvme1 is the only + # one excluded, and k8s-2 nvme1 must still fire — the exclusion is per + # node, not per device name. + - interval: 1m + input_series: + - series: 'smartctl_device_percentage_used{node="k8s-1",device="nvme0",instance="10.0.0.1:9633"}' + values: '96+0x200' + - series: 'smartctl_device_percentage_used{node="k8s-1",device="nvme1",instance="10.0.0.1:9633"}' + values: '96+0x200' + - series: 'smartctl_device_percentage_used{node="k8s-2",device="nvme1",instance="10.0.0.2:9633"}' + values: '96+0x200' + - series: 'node_filesystem_size_bytes{instance="10.0.0.1:9100",mountpoint="/var",device="/dev/nvme1n1p1",fstype="xfs"}' + values: '1000000000000+0x200' + - series: 'node_uname_info{instance="10.0.0.1:9100",nodename="k8s-1"}' + values: '1+0x200' + - series: 'node_filesystem_size_bytes{instance="10.0.0.2:9100",mountpoint="/var",device="/dev/nvme2n1p1",fstype="xfs"}' + values: '1000000000000+0x200' + - series: 'node_uname_info{instance="10.0.0.2:9100",nodename="k8s-2"}' + values: '1+0x200' + alert_rule_test: + - eval_time: 120m + alertname: NvmeDriveWearCritical + exp_alerts: + - exp_labels: + severity: critical + node: k8s-1 + device: nvme0 + instance: 10.0.0.1:9633 + exp_annotations: + summary: "NVMe nvme0 on 10.0.0.1:9633 is at 96% rated wear" + description: "The drive is at or past the end of its rated endurance. It will likely keep working for a while, but it is now operating outside what the manufacturer stands behind, and replacement should be scheduled rather than considered." + - exp_labels: + severity: critical + node: k8s-2 + device: nvme1 + instance: 10.0.0.2:9633 + exp_annotations: + summary: "NVMe nvme1 on 10.0.0.2:9633 is at 96% rated wear" + description: "The drive is at or past the end of its rated endurance. It will likely keep working for a while, but it is now operating outside what the manufacturer stands behind, and replacement should be scheduled rather than considered." + + # C. Fails loud. /var is on a device the join cannot map to an NVMe name (an + # encrypted volume on dm-0). The exclusion then matches nothing and the + # EPHEMERAL drive alerts again, which is the intended direction to fail. + - interval: 1m + input_series: + - series: 'smartctl_device_percentage_used{node="k8s-1",device="nvme1",instance="10.0.0.1:9633"}' + values: '96+0x200' + - series: 'node_filesystem_size_bytes{instance="10.0.0.1:9100",mountpoint="/var",device="/dev/dm-0",fstype="xfs"}' + values: '1000000000000+0x200' + - series: 'node_uname_info{instance="10.0.0.1:9100",nodename="k8s-1"}' + values: '1+0x200' + alert_rule_test: + - eval_time: 120m + alertname: NvmeDriveWearCritical + exp_alerts: + - exp_labels: + severity: critical + node: k8s-1 + device: nvme1 + instance: 10.0.0.1:9633 + exp_annotations: + summary: "NVMe nvme1 on 10.0.0.1:9633 is at 96% rated wear" + description: "The drive is at or past the end of its rated endurance. It will likely keep working for a while, but it is now operating outside what the manufacturer stands behind, and replacement should be scheduled rather than considered." diff --git a/kubernetes/apps/monitoring-rules/tests/mutations.txt b/kubernetes/apps/monitoring-rules/tests/mutations.txt index e00fb6d8..ad2eaaca 100644 --- a/kubernetes/apps/monitoring-rules/tests/mutations.txt +++ b/kubernetes/apps/monitoring-rules/tests/mutations.txt @@ -149,3 +149,21 @@ retention-no-size-clause|kubernetes/apps/monitoring-rules/prometheusrule-prometh retention-ratio-0.9|kubernetes/apps/monitoring-rules/prometheusrule-prometheus-retention.yaml|s/^\([[:space:]]*\)> 0.8$/\1> 0.9/ retention-lookback-1d|kubernetes/apps/monitoring-rules/prometheusrule-prometheus-retention.yaml|s/)\[7d:1h\]$/)[1d:1h]/ retention-cap-at-peak-time|kubernetes/apps/monitoring-rules/prometheusrule-prometheus-retention.yaml|s/\/ max(prometheus_tsdb_retention_limit_bytes{job="kube-prometheus-stack-prometheus"})$/\/ min_over_time(max(prometheus_tsdb_retention_limit_bytes{job="kube-prometheus-stack-prometheus"})[7d:1h])/ + +# prometheusrule-disk-wear.yaml (REN-419): the EPHEMERAL exclusion on the two +# wear alerts. Detected by the cases in disk-wear_test.yaml. +# +# Gate inverted on the projection: the EPHEMERAL drive alone fires. +# Detected by: case A. +disk-wear-projection-gate-inverted|kubernetes/apps/monitoring-rules/prometheusrule-disk-wear.yaml|/alert: NvmeDriveWearProjectedExhaustion/,/for: 6h/s/unless on (node, device)/and on (node, device)/ +# Gate inverted on the critical rule. +# Detected by: case B. +disk-wear-critical-gate-inverted|kubernetes/apps/monitoring-rules/prometheusrule-disk-wear.yaml|/alert: NvmeDriveWearCritical/,/for: 1h/s/unless on (node, device)/and on (node, device)/ +# Matched on device name only: k8s-1's /var on nvme1 then silences nvme1 on +# every node. +# Detected by: case B, k8s-2 nvme1. +disk-wear-critical-device-only|kubernetes/apps/monitoring-rules/prometheusrule-disk-wear.yaml|/alert: NvmeDriveWearCritical/,/for: 1h/s/unless on (node, device)/unless on (device)/ +# Partition suffix not stripped: /dev/nvme1n1p1 no longer maps to nvme1 and the +# exclusion matches nothing. +# Detected by: cases A and B. +disk-wear-ephemeral-keeps-partition|kubernetes/apps/monitoring-rules/prometheusrule-disk-wear.yaml|s/n\[0-9\]+(p\[0-9\]+)?"/n[0-9]+"/