diff --git a/kubernetes/apps/monitoring-rules/prometheusrule-ceph.yaml b/kubernetes/apps/monitoring-rules/prometheusrule-ceph.yaml index 6821ca00..a429be85 100644 --- a/kubernetes/apps/monitoring-rules/prometheusrule-ceph.yaml +++ b/kubernetes/apps/monitoring-rules/prometheusrule-ceph.yaml @@ -25,13 +25,27 @@ spec: # # 6h: Ceph warns for self-clearing conditions. Only a warning that # does not clear is worth paging on. + # + # BLUESTORE_SLOW_OP_ALERT breaks that premise and is still NOT + # excluded (#557). Ceph holds the check for + # bluestore_slow_ops_warn_lifetime (24h) after the LAST slow op, so + # one multi-second stall outlives the 6h `for` and always pages, + # with latency back to normal long before anyone looks. Measured: + # both episodes in the 30d to 2026-08-23 ran ~26h and cleared + # untouched, the first ~24h after the last stall. + # + # Both began inside the weekly Velero restore-test window (Sun + # 08:00-13:00Z) — a 5h sustained restore that outruns the SLC cache + # on these QLC NVMe. Expected for the hardware, but not excludable: + # this is also the earliest signal of a genuinely failing drive, and + # all three are at 80-87% endurance used (see cephcluster.yaml). expr: ceph_health_detail{name!~"AUTH_INSECURE_.+"} == 1 for: 6h labels: severity: warning annotations: summary: "Ceph health check {{ $labels.name }} active for 6h" - description: "Ceph health check {{ $labels.name }} has been active for six hours, so it is not a self-clearing blip. Only AUTH_INSECURE_* is excluded (kernel-blocked, see #527), so this is a new condition. Detail: `kubectl -n rook-ceph exec deploy/rook-ceph-tools -- ceph health detail`." + description: "Ceph health check {{ $labels.name }} has been active for six hours. Only AUTH_INSECURE_* is excluded (kernel-blocked, see #527). BLUESTORE_SLOW_OP_ALERT is sticky for 24h after the last stall and clears itself: check whether the stall fell inside the Sunday 08:00-13:00Z Velero restore-test window, which is expected on these QLC drives — one outside that window, or a repeat on the same OSD, means check that drive (#557). Detail: `kubectl -n rook-ceph exec deploy/rook-ceph-tools -- ceph health detail`." - alert: CephPoolLostRedundancy # Names the pool, which POOL_NO_REDUNDANCY does not.