From bdffe1804b27839fc02adae8c9c58e82066ebe4e Mon Sep 17 00:00:00 2001 From: Alert Fix Agent Date: Thu, 27 Aug 2026 19:45:14 +0000 Subject: [PATCH] fix: [alert] NvmeDriveWearHigh (monitoring) (#675) --- kubernetes/talos/patches/k8s-1.yaml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/kubernetes/talos/patches/k8s-1.yaml b/kubernetes/talos/patches/k8s-1.yaml index b26f3700..0e9af181 100644 --- a/kubernetes/talos/patches/k8s-1.yaml +++ b/kubernetes/talos/patches/k8s-1.yaml @@ -69,6 +69,25 @@ cluster: # which is the constraint that makes k8s-2 and k8s-3 trip DiskPressure whenever # they absorb another node's pods. # +# One caveat this node does NOT share with k8s-2 and k8s-3: their P3s report +# zero media errors, this one reports 18 (smartctl_device_media_errors, +# verified 2026-08-27 — the only non-zero counter in the fleet). Everything +# else on the drive is clean, available spare 100% and critical-warning byte +# 0, and NvmeDriveMediaErrors alerts on the increase rather than the total, so +# a static count never surfaces there. +# +# Whether those 18 are old damage from its OSD career or ongoing is NOT +# established. Prometheus only has a few hours of history for this drive, flat +# at 18, because the exporter first scraped this node at its conversion — the +# series being young is not evidence the count is stable. Settle it on the +# node with `smartctl -a /dev/nvme1n1` and compare the error-log entries +# against power-on hours. +# +# That question matters more here than the wear number does, because +# /var/lib/etcd lives on EPHEMERAL: if the count is climbing, this node's etcd +# member data is sitting on failing media and EPHEMERAL belongs on the system +# disk instead — which means a wipe and install media, per the note below. +# # THIS ONLY TAKES EFFECT ON FIRST PROVISION. Talos applies volume configuration # solely when the volume does not yet exist; a config applied after EPHEMERAL # has been created elsewhere is silently ignored, and relocating it then -- 2.54.0