fix: [alert] CephHealthError (rook-ceph) (#525 )

2026-08-19 16:46:25 +00:00
1 changed files with 42 additions and 2 deletions
@@ -26,8 +26,32 @@ spec:
            description: "The Ceph cluster has held HEALTH_WARN for six hours, so it is not a self-clearing blip. Get the detail with `kubectl -n rook-ceph exec deploy/rook-ceph-tools -- ceph health detail`."

        - alert: CephHealthError
-          # HEALTH_ERR means data availability or redundancy is at risk.
-          expr: ceph_health_status == 2
+          # HEALTH_ERR used to mean "data availability or redundancy is at
+          # risk". Ceph 19.2.6 broke that assumption: it added the
+          # AUTH_INSECURE_* cephx checks, two of them ERR-severity, which flag
+          # the legacy `aes` key type every keyring in this cluster still uses.
+          # That is a security-posture finding, not a durability one — see
+          # CephAuthInsecureKeys below, which is what actually tracks it.
+          #
+          # The problem with leaving `ceph_health_status == 2` alone is not
+          # noise, it is blindness: while the auth checks hold the cluster at
+          # HEALTH_ERR, a genuine PG_AVAILABILITY or OSD_FULL is
+          # indistinguishable from them, because the aggregate is already
+          # saturated. So page on the ERR checks that are *not* auth ones.
+          #
+          # Written as "unless every active ERR check is an AUTH_INSECURE_ one"
+          # rather than a direct match so that it fails open: if
+          # ceph_health_detail goes missing or Ceph renames a check, both counts
+          # come back empty, the `unless` removes nothing, and this still pages.
+          # The `> 0` matters — the mgr exports all 31 known checks, the
+          # inactive ones at 0, so an unfiltered count would compare constants.
+          expr: |
+            ceph_health_status == 2
+              unless on (cluster) (
+                count by (cluster) (ceph_health_detail{severity="HEALTH_ERR"} > 0)
+                  ==
+                count by (cluster) (ceph_health_detail{severity="HEALTH_ERR", name=~"AUTH_INSECURE_.*"} > 0)
+              )
          for: 5m
          labels:
            severity: critical
@@ -35,6 +59,22 @@ spec:
            summary: "Ceph is HEALTH_ERR"
            description: "The Ceph cluster is in HEALTH_ERR — availability or redundancy is at risk. Inspect immediately with `kubectl -n rook-ceph exec deploy/rook-ceph-tools -- ceph health detail` and `ceph osd df tree`."

+        - alert: CephAuthInsecureKeys
+          # Carries the condition CephHealthError deliberately stops paging for,
+          # so excluding it there narrows the alert rather than silencing it.
+          # Warning, not critical: nothing is at risk of being lost, but this
+          # does not self-clear either — it needs a planned cephx key rotation
+          # across client.admin, the CSI provisioner/node keys and the RGW key.
+          # Rotating those carelessly breaks RBD/CephFS provisioning
+          # cluster-wide, so this stays a human decision, not a 3am one.
+          expr: count(ceph_health_detail{name=~"AUTH_INSECURE_.*"} > 0) > 0
+          for: 1h
+          labels:
+            severity: warning
+          annotations:
+            summary: "Ceph cephx keys use insecure key types"
+            description: "Ceph reports {{ $value }} active AUTH_INSECURE_* health checks — cephx entities still using the legacy `aes` key type, flagged since the 19.2.6 upgrade. No data is at risk. Enumerate with `kubectl -n rook-ceph exec deploy/rook-ceph-tools -- ceph health detail` and plan rotation against the 19.2.6 release notes; durable settings belong in `spec.cephConfig` in kubernetes/infrastructure/rook-ceph-cluster/cephcluster.yaml."
+
        - alert: CephMonDiskLow
          # Named explicitly rather than left to CephHealthWarn, because the
          # mons sit on the node's /var alongside container storage: this