kubernetes/apps/monitoring-rules/prometheusrule-ceph.yaml

@@ -25,13 +25,27 @@ spec:
           #
           # 6h: Ceph warns for self-clearing conditions. Only a warning that
           # does not clear is worth paging on.
+          #
+          # BLUESTORE_SLOW_OP_ALERT breaks that premise and is still NOT
+          # excluded (#557). Ceph holds the check for
+          # bluestore_slow_ops_warn_lifetime (24h) after the LAST slow op, so
+          # one multi-second stall outlives the 6h `for` and always pages,
+          # with latency back to normal long before anyone looks. Measured:
+          # both episodes in the 30d to 2026-08-23 ran ~26h and cleared
+          # untouched, the first ~24h after the last stall.
+          #
+          # Both began inside the weekly Velero restore-test window (Sun
+          # 08:00-13:00Z) — a 5h sustained restore that outruns the SLC cache
+          # on these QLC NVMe. Expected for the hardware, but not excludable:
+          # this is also the earliest signal of a genuinely failing drive, and
+          # all three are at 80-87% endurance used (see cephcluster.yaml).
           expr: ceph_health_detail{name!~"AUTH_INSECURE_.+"} == 1
           for: 6h
           labels:
             severity: warning
           annotations:
             summary: "Ceph health check {{ $labels.name }} active for 6h"
-            description: "Ceph health check {{ $labels.name }} has been active for six hours, so it is not a self-clearing blip. Only AUTH_INSECURE_* is excluded (kernel-blocked, see #527), so this is a new condition. Detail: `kubectl -n rook-ceph exec deploy/rook-ceph-tools -- ceph health detail`."
+            description: "Ceph health check {{ $labels.name }} has been active for six hours. Only AUTH_INSECURE_* is excluded (kernel-blocked, see #527). BLUESTORE_SLOW_OP_ALERT is sticky for 24h after the last stall and clears itself: check whether the stall fell inside the Sunday 08:00-13:00Z Velero restore-test window, which is expected on these QLC drives — one outside that window, or a repeat on the same OSD, means check that drive (#557). Detail: `kubectl -n rook-ceph exec deploy/rook-ceph-tools -- ceph health detail`."
 
         - alert: CephPoolLostRedundancy
           # Names the pool, which POOL_NO_REDUNDANCY does not.