diff --git a/kubernetes/apps/monitoring-rules/prometheusrule-ceph.yaml b/kubernetes/apps/monitoring-rules/prometheusrule-ceph.yaml index a429be85..ab14490c 100644 --- a/kubernetes/apps/monitoring-rules/prometheusrule-ceph.yaml +++ b/kubernetes/apps/monitoring-rules/prometheusrule-ceph.yaml @@ -31,21 +31,30 @@ spec: # bluestore_slow_ops_warn_lifetime (24h) after the LAST slow op, so # one multi-second stall outlives the 6h `for` and always pages, # with latency back to normal long before anyone looks. Measured: - # both episodes in the 30d to 2026-08-23 ran ~26h and cleared - # untouched, the first ~24h after the last stall. + # three episodes in the 30d to 2026-08-24 (Sun Jul 26, Aug 2 and + # Aug 23), each ran ~24-26h and cleared untouched. # - # Both began inside the weekly Velero restore-test window (Sun - # 08:00-13:00Z) — a 5h sustained restore that outruns the SLC cache - # on these QLC NVMe. Expected for the hardware, but not excludable: - # this is also the earliest signal of a genuinely failing drive, and - # all three are at 80-87% endurance used (see cephcluster.yaml). + # All began inside the weekly Velero restore-test window (Sun 08:00Z + # plus up to 6h, see velero/restore-test-cronjob.yaml) — a sustained + # restore that outruns the SLC cache on these QLC NVMe. Expected for + # the hardware, but not excludable: this is also the earliest signal + # of a genuinely failing drive, and all three are at 80-87% + # endurance used (see cephcluster.yaml). + # + # No metric names the stalling OSD: ceph_health_detail carries no osd + # label, and ceph_daemon_health_metrics{type="SLOW_OPS"} reads 0 at + # every scrape (a stall is shorter than the scrape interval). Only + # the OSD's own log has it, hence the Loki query in the description. + # Jul 26 and Aug 2 hit osd.0+1+2 together, which is load-driven; + # Aug 23 hit osd.1 alone in one burst. If osd.1 alone repeats, + # suspect that drive rather than the restore load. expr: ceph_health_detail{name!~"AUTH_INSECURE_.+"} == 1 for: 6h labels: severity: warning annotations: summary: "Ceph health check {{ $labels.name }} active for 6h" - description: "Ceph health check {{ $labels.name }} has been active for six hours. Only AUTH_INSECURE_* is excluded (kernel-blocked, see #527). BLUESTORE_SLOW_OP_ALERT is sticky for 24h after the last stall and clears itself: check whether the stall fell inside the Sunday 08:00-13:00Z Velero restore-test window, which is expected on these QLC drives — one outside that window, or a repeat on the same OSD, means check that drive (#557). Detail: `kubectl -n rook-ceph exec deploy/rook-ceph-tools -- ceph health detail`." + description: "Ceph health check {{ $labels.name }} has been active for six hours. Only AUTH_INSECURE_* is excluded (kernel-blocked, see #527). BLUESTORE_SLOW_OP_ALERT is sticky for 24h after the last stall and clears itself: check whether the stall fell inside the Sunday 08:00-14:00Z Velero restore-test window (08:00 start, 6h activeDeadlineSeconds cap), which is expected on these QLC drives — one outside that window, or a repeat on the same OSD, means check that drive (#557). No metric names the OSD — get it from Loki, which also works after the check clears: `{namespace=\"rook-ceph\",pod=~\"rook-ceph-osd-.*\"} |= \"slow operation observed\"`. While still active, `kubectl -n rook-ceph exec deploy/rook-ceph-tools -- ceph health detail` adds the op detail." - alert: CephPoolLostRedundancy # Names the pool, which POOL_NO_REDUNDANCY does not.