fix: [alert] CephHealthWarn (rook-ceph) (#557 )

2026-08-23 18:46:50 +00:00
2 changed files with 16 additions and 10 deletions
@@ -25,13 +25,27 @@ spec:
          #
          # 6h: Ceph warns for self-clearing conditions. Only a warning that
          # does not clear is worth paging on.
+          #
+          # BLUESTORE_SLOW_OP_ALERT breaks that premise and is still NOT
+          # excluded (#557). Ceph holds the check for
+          # bluestore_slow_ops_warn_lifetime (24h) after the LAST slow op, so
+          # one multi-second stall outlives the 6h `for` and always pages,
+          # with latency back to normal long before anyone looks. Measured:
+          # both episodes in the 30d to 2026-08-23 ran ~26h and cleared
+          # untouched, the first ~24h after the last stall.
+          #
+          # Both began inside the weekly Velero restore-test window (Sun
+          # 08:00-13:00Z) — a 5h sustained restore that outruns the SLC cache
+          # on these QLC NVMe. Expected for the hardware, but not excludable:
+          # this is also the earliest signal of a genuinely failing drive, and
+          # all three are at 80-87% endurance used (see cephcluster.yaml).
          expr: ceph_health_detail{name!~"AUTH_INSECURE_.+"} == 1
          for: 6h
          labels:
            severity: warning
          annotations:
            summary: "Ceph health check {{ $labels.name }} active for 6h"
-            description: "Ceph health check {{ $labels.name }} has been active for six hours, so it is not a self-clearing blip. Only AUTH_INSECURE_* is excluded (kernel-blocked, see #527), so this is a new condition. Detail: `kubectl -n rook-ceph exec deploy/rook-ceph-tools -- ceph health detail`."
+            description: "Ceph health check {{ $labels.name }} has been active for six hours. Only AUTH_INSECURE_* is excluded (kernel-blocked, see #527). BLUESTORE_SLOW_OP_ALERT is sticky for 24h after the last stall and clears itself: check whether the stall fell inside the Sunday 08:00-13:00Z Velero restore-test window, which is expected on these QLC drives — one outside that window, or a repeat on the same OSD, means check that drive (#557). Detail: `kubectl -n rook-ceph exec deploy/rook-ceph-tools -- ceph health detail`."

        - alert: CephPoolLostRedundancy
          # Names the pool, which POOL_NO_REDUNDANCY does not.
@@ -51,15 +51,7 @@ spec:
  startingDeadlineSeconds: 3600
  jobTemplate:
    spec:
-      # 4 (not the sibling rotators' 1-2) because this is the only DAILY
-      # rotator — every other CronJob here is quarterly, so it takes ~90x as
-      # many runs and is correspondingly more exposed to per-run transients.
-      # backoffLimit: 1 gave two attempts ~11 s apart, and on 2026-08-19 a
-      # sub-minute resolver transient took both out (Job
-      # claude-oauth-secret-sync-29785100, BackoffLimitExceeded), costing a
-      # day's sync. 4 spreads the attempts over ~150 s (10/20/40/80 s backoff),
-      # still well inside activeDeadlineSeconds.
-      backoffLimit: 4
+      backoffLimit: 1
      activeDeadlineSeconds: 300
      ttlSecondsAfterFinished: 259200 # 3 d
      template: