kubernetes/apps/monitoring-rules/prometheusrule-ceph.yaml

@@ -12,35 +12,19 @@ spec:
     - name: ceph.rules
       rules:
         - alert: CephHealthWarn
-          # Was `ceph_health_status == 1`, which had been FIRING CONTINUOUSLY
-          # since at least 2026-08-21T00:26Z and was therefore telling us
-          # nothing: four AUTH_INSECURE_* checks are permanently active
-          # (CVE-2025-30156, issue #527) and cannot clear until Rook >=1.20.5.
-          # A permanently-firing alert masks every subsequent Ceph warning,
-          # which is the opposite of what this rule is for.
+          # Per-check rather than ceph_health_status, so one permanently-active
+          # check cannot mask every other Ceph warning.
           #
-          # ceph_health_detail exposes each check individually, so alert on any
-          # check that is NOT a known-accepted one. Verified against live data:
-          # 5 checks active, 0 unexpected — so this is quiet now and fires the
-          # moment a genuinely new condition appears.
+          # AUTH_INSECURE_* is excluded: it cannot clear until the nodes run a
+          # kernel supporting aes256k over the CSI mount path (#527).
           #
-          # Excluded:
-          #   AUTH_INSECURE_*   CVE-2025-30156, tracked in #527. NOT "accepted"
-          #                     in the sense of a decision that the risk is
-          #                     fine — the client-key checks cannot clear until
-          #                     the nodes run a kernel supporting aes256k over
-          #                     the CSI mount path, so alerting on them would
-          #                     just re-mask everything else. The daemon-key
-          #                     half is remediated; the rest is kernel-blocked.
+          # POOL_NO_REDUNDANCY is in scope. An unreplicated pool makes
+          # `ceph osd ok-to-stop` refuse for every OSD, so it reports blocked
+          # maintenance and not just data risk — do not exclude it for a pool
+          # that is unreplicated deliberately.
           #
-          # POOL_NO_REDUNDANCY is deliberately NOT excluded. It was, while
-          # cephfs-ci ran at size 1; no pool is unreplicated now, so excluding
-          # it would hide a real pool losing its replicas.
-          #
-          # 6h is kept: Ceph warns for self-clearing conditions — a slow
-          # BlueStore op during the Sunday restore sweep, a mon briefly below
-          # its disk threshold — and paging on those trains people to ignore
-          # the alert. What matters is a warning that does not clear.
+          # 6h: Ceph warns for self-clearing conditions. Only a warning that
+          # does not clear is worth paging on.
           expr: ceph_health_detail{name!~"AUTH_INSECURE_.+"} == 1
           for: 6h
           labels:
@@ -50,21 +34,13 @@ spec:
             description: "Ceph health check {{ $labels.name }} has been active for six hours, so it is not a self-clearing blip. Only AUTH_INSECURE_* is excluded (kernel-blocked, see #527), so this is a new condition. Detail: `kubectl -n rook-ceph exec deploy/rook-ceph-tools -- ceph health detail`."
 
         - alert: CephPoolLostRedundancy
-          # Names the pool, which POOL_NO_REDUNDANCY does not — that check
-          # fires cluster-wide without saying which pool lost its replicas.
+          # Names the pool, which POOL_NO_REDUNDANCY does not.
           #
           # There is no pool-size metric, but bytes_used/stored IS the
-          # replication factor.
+          # replication factor. The >1e6 floor drops pools too small to measure
+          # meaningfully, empty ones included.
           #
-          # The >1e6 floor excludes trivially small pools. It is not needed to
-          # suppress the near-empty rgw pools — their ratios are 9-36 because
-          # per-object overhead dominates, so they never match `< 1.5` anyway.
-          # Note it also puts cephfs-ci-data0 out of scope while it is empty;
-          # cephfs-ci-metadata carries enough bytes to be covered.
-          #
-          # No pool is exempt. cephfs-ci was exempt while it ran at size 1;
-          # every pool is replicated now, so an exemption would only hide a
-          # regression.
+          # No pool is exempt.
           expr: |
             ceph_pool_bytes_used / ceph_pool_stored < 1.5 and ceph_pool_stored > 1e6
           for: 30m

kubernetes/infrastructure/rook-ceph-cluster/cephcluster.yaml

@@ -17,28 +17,12 @@ spec:
     modules:
       - name: pg_autoscaler
         enabled: true
-  # CVE-2025-30156: CephX AES-128-CBC service tickets carry no HMAC and use a
-  # hard-coded IV, so a holder of any low-privilege key can bit-flip a ticket
-  # into cluster-admin. The fix is rotating keys to the aes256k cipher —
-  # upgrading the binaries alone does not close it. Rotation is opt-in and
-  # defaults to Disabled, so it must be requested here.
+  # Rotates daemon keys to aes256k for CVE-2025-30156 (#527). Opt-in; disabled
+  # by default. Increment keyGeneration to rotate again.
   #
-  # Daemon keys only. CSI keys stay on aes: aes256k over the CSI mount path
-  # needs a host kernel of 7.0 or newer and these nodes run 6.18. `aes`
-  # therefore has to remain in auth_allowed_ciphers for CSI to authenticate,
-  # which leaves AUTH_INSECURE_CLIENT_KEY_TYPE, AUTH_INSECURE_KEYS_ALLOWED and
-  # AUTH_INSECURE_KEYS_CREATABLE raised. Revisit once the nodes run a kernel
-  # that supports it.
-  #
-  # Rotating the daemon keys clears AUTH_INSECURE_SERVICE_KEY_TYPE, the only
-  # one of the six checks at ERR severity. That matters beyond the CVE itself:
-  # Rook skips CephFS and RGW reconciliation entirely while health is
-  # HEALTH_ERR, and that check is currently held down only by a mon-side mute
-  # with a finite TTL.
-  #
-  # keyType is deliberately unset — Rook picks the best type for daemon keys.
-  # To rotate again later, increment keyGeneration; Rook restarts the daemons
-  # to pick up the new keys.
+  # Daemon keys only — aes256k over the CSI mount path needs kernel 7.0+ and
+  # these nodes run 6.18, so CSI stays on aes and the client-key checks stay
+  # raised. keyType is unset so Rook picks it.
   security:
     cephx:
       daemon:

kubernetes/infrastructure/rook-ceph-cluster/cephfs-ci.yaml

@@ -1,30 +1,11 @@
 ---
-# A SEPARATE CephFS whose only purpose is Woodpecker CI workspaces.
+# A SEPARATE CephFS whose only purpose is Woodpecker CI workspaces. Each
+# pipeline gets a PVC that is created, filled and deleted within ~2 minutes;
+# keeping that churn off `cephfs` spares its size-3 metadataPool.
 #
-# WHY THIS EXISTS
-# Woodpecker's Kubernetes backend creates a PVC per pipeline, fills it with a
-# checkout plus build output, and deletes it when the pipeline ends. Measured
-# 2026-08-21 by polling both pools at 15s (a 5-minute interval misses a whole
-# pipeline — its PVC is created, used and deleted inside ~2 minutes):
-#
-#   one pipeline, a single 15s interval:  data0 +71.6 MB   metadata +8.2 MB
-#   every interval without a CI pod:      data0   ~0        metadata   ~0
-#
-# CI drove essentially all of it. Keeping that churn off `cephfs`, whose
-# metadataPool is size 3, is where most of the saving comes from — that part
-# does not depend on the replica count chosen here.
-#
-# WHY size 2 AND NOT size 1
-# Size 1 puts every PG on exactly one OSD, so stopping that OSD takes the PGs
-# offline and `ceph osd ok-to-stop` refuses. Rook gates OSD restarts on that
-# check, so a size-1 pool anywhere on a 3-OSD cluster blocks ALL OSD
-# maintenance — CephX key rotation, Ceph upgrades, node drains and reboots,
-# drive replacement. The cost is not confined to this filesystem's own data;
-# it freezes the whole cluster, and `ceph -s` still reads active+clean while
-# it does, because nothing is unhealthy — the restarts are simply refused.
-#
-# size 1 becomes viable once these pools sit on OSDs of their own, where the
-# permanently un-stoppable OSD holds nothing but throwaway data.
+# size 2, not 1: an unreplicated pool makes `ceph osd ok-to-stop` refuse for
+# every OSD, blocking all OSD maintenance cluster-wide. size 1 is viable only
+# once these pools have OSDs of their own.
 apiVersion: ceph.rook.io/v1
 kind: CephFilesystem
 metadata: