From 9a43330701a68ed207a665a679e2d91706048f0e Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Fri, 21 Aug 2026 22:27:10 +0200 Subject: [PATCH] docs(rook-ceph): cut the narrative out of the ceph manifest comments Comments should say what the setting does and what constrains it. The blocks added in #542, #544, #545 and #547 had grown into incident write-ups: measurement runs, prior values, how a problem was found, what a check used to be. That belongs in commit messages and the wiki, both of which already carry it. Trimmed, no behaviour change: cephcluster.yaml 22 lines -> 6 (cephx rotation) cephfs-ci.yaml 26 lines -> 7 (why size 2) prometheusrule-ceph.yaml (CephHealthWarn, 43 -> 13; CephPoolLostRedundancy, 15 -> 7) The operative facts are kept: which checks are excluded and why they cannot clear, that POOL_NO_REDUNDANCY reports blocked maintenance and not only data risk, and that size 1 needs dedicated OSDs. --- .../monitoring-rules/prometheusrule-ceph.yaml | 52 +++++-------------- .../rook-ceph-cluster/cephcluster.yaml | 26 ++-------- .../rook-ceph-cluster/cephfs-ci.yaml | 31 +++-------- 3 files changed, 25 insertions(+), 84 deletions(-) diff --git a/kubernetes/apps/monitoring-rules/prometheusrule-ceph.yaml b/kubernetes/apps/monitoring-rules/prometheusrule-ceph.yaml index 977c2234..6821ca00 100644 --- a/kubernetes/apps/monitoring-rules/prometheusrule-ceph.yaml +++ b/kubernetes/apps/monitoring-rules/prometheusrule-ceph.yaml @@ -12,35 +12,19 @@ spec: - name: ceph.rules rules: - alert: CephHealthWarn - # Was `ceph_health_status == 1`, which had been FIRING CONTINUOUSLY - # since at least 2026-08-21T00:26Z and was therefore telling us - # nothing: four AUTH_INSECURE_* checks are permanently active - # (CVE-2025-30156, issue #527) and cannot clear until Rook >=1.20.5. - # A permanently-firing alert masks every subsequent Ceph warning, - # which is the opposite of what this rule is for. + # Per-check rather than ceph_health_status, so one permanently-active + # check cannot mask every other Ceph warning. # - # ceph_health_detail exposes each check individually, so alert on any - # check that is NOT a known-accepted one. Verified against live data: - # 5 checks active, 0 unexpected — so this is quiet now and fires the - # moment a genuinely new condition appears. + # AUTH_INSECURE_* is excluded: it cannot clear until the nodes run a + # kernel supporting aes256k over the CSI mount path (#527). # - # Excluded: - # AUTH_INSECURE_* CVE-2025-30156, tracked in #527. NOT "accepted" - # in the sense of a decision that the risk is - # fine — the client-key checks cannot clear until - # the nodes run a kernel supporting aes256k over - # the CSI mount path, so alerting on them would - # just re-mask everything else. The daemon-key - # half is remediated; the rest is kernel-blocked. + # POOL_NO_REDUNDANCY is in scope. An unreplicated pool makes + # `ceph osd ok-to-stop` refuse for every OSD, so it reports blocked + # maintenance and not just data risk — do not exclude it for a pool + # that is unreplicated deliberately. # - # POOL_NO_REDUNDANCY is deliberately NOT excluded. It was, while - # cephfs-ci ran at size 1; no pool is unreplicated now, so excluding - # it would hide a real pool losing its replicas. - # - # 6h is kept: Ceph warns for self-clearing conditions — a slow - # BlueStore op during the Sunday restore sweep, a mon briefly below - # its disk threshold — and paging on those trains people to ignore - # the alert. What matters is a warning that does not clear. + # 6h: Ceph warns for self-clearing conditions. Only a warning that + # does not clear is worth paging on. expr: ceph_health_detail{name!~"AUTH_INSECURE_.+"} == 1 for: 6h labels: @@ -50,21 +34,13 @@ spec: description: "Ceph health check {{ $labels.name }} has been active for six hours, so it is not a self-clearing blip. Only AUTH_INSECURE_* is excluded (kernel-blocked, see #527), so this is a new condition. Detail: `kubectl -n rook-ceph exec deploy/rook-ceph-tools -- ceph health detail`." - alert: CephPoolLostRedundancy - # Names the pool, which POOL_NO_REDUNDANCY does not — that check - # fires cluster-wide without saying which pool lost its replicas. + # Names the pool, which POOL_NO_REDUNDANCY does not. # # There is no pool-size metric, but bytes_used/stored IS the - # replication factor. + # replication factor. The >1e6 floor drops pools too small to measure + # meaningfully, empty ones included. # - # The >1e6 floor excludes trivially small pools. It is not needed to - # suppress the near-empty rgw pools — their ratios are 9-36 because - # per-object overhead dominates, so they never match `< 1.5` anyway. - # Note it also puts cephfs-ci-data0 out of scope while it is empty; - # cephfs-ci-metadata carries enough bytes to be covered. - # - # No pool is exempt. cephfs-ci was exempt while it ran at size 1; - # every pool is replicated now, so an exemption would only hide a - # regression. + # No pool is exempt. expr: | ceph_pool_bytes_used / ceph_pool_stored < 1.5 and ceph_pool_stored > 1e6 for: 30m diff --git a/kubernetes/infrastructure/rook-ceph-cluster/cephcluster.yaml b/kubernetes/infrastructure/rook-ceph-cluster/cephcluster.yaml index d242c531..895d5da3 100644 --- a/kubernetes/infrastructure/rook-ceph-cluster/cephcluster.yaml +++ b/kubernetes/infrastructure/rook-ceph-cluster/cephcluster.yaml @@ -17,28 +17,12 @@ spec: modules: - name: pg_autoscaler enabled: true - # CVE-2025-30156: CephX AES-128-CBC service tickets carry no HMAC and use a - # hard-coded IV, so a holder of any low-privilege key can bit-flip a ticket - # into cluster-admin. The fix is rotating keys to the aes256k cipher — - # upgrading the binaries alone does not close it. Rotation is opt-in and - # defaults to Disabled, so it must be requested here. + # Rotates daemon keys to aes256k for CVE-2025-30156 (#527). Opt-in; disabled + # by default. Increment keyGeneration to rotate again. # - # Daemon keys only. CSI keys stay on aes: aes256k over the CSI mount path - # needs a host kernel of 7.0 or newer and these nodes run 6.18. `aes` - # therefore has to remain in auth_allowed_ciphers for CSI to authenticate, - # which leaves AUTH_INSECURE_CLIENT_KEY_TYPE, AUTH_INSECURE_KEYS_ALLOWED and - # AUTH_INSECURE_KEYS_CREATABLE raised. Revisit once the nodes run a kernel - # that supports it. - # - # Rotating the daemon keys clears AUTH_INSECURE_SERVICE_KEY_TYPE, the only - # one of the six checks at ERR severity. That matters beyond the CVE itself: - # Rook skips CephFS and RGW reconciliation entirely while health is - # HEALTH_ERR, and that check is currently held down only by a mon-side mute - # with a finite TTL. - # - # keyType is deliberately unset — Rook picks the best type for daemon keys. - # To rotate again later, increment keyGeneration; Rook restarts the daemons - # to pick up the new keys. + # Daemon keys only — aes256k over the CSI mount path needs kernel 7.0+ and + # these nodes run 6.18, so CSI stays on aes and the client-key checks stay + # raised. keyType is unset so Rook picks it. security: cephx: daemon: diff --git a/kubernetes/infrastructure/rook-ceph-cluster/cephfs-ci.yaml b/kubernetes/infrastructure/rook-ceph-cluster/cephfs-ci.yaml index ca0922ca..34959239 100644 --- a/kubernetes/infrastructure/rook-ceph-cluster/cephfs-ci.yaml +++ b/kubernetes/infrastructure/rook-ceph-cluster/cephfs-ci.yaml @@ -1,30 +1,11 @@ --- -# A SEPARATE CephFS whose only purpose is Woodpecker CI workspaces. +# A SEPARATE CephFS whose only purpose is Woodpecker CI workspaces. Each +# pipeline gets a PVC that is created, filled and deleted within ~2 minutes; +# keeping that churn off `cephfs` spares its size-3 metadataPool. # -# WHY THIS EXISTS -# Woodpecker's Kubernetes backend creates a PVC per pipeline, fills it with a -# checkout plus build output, and deletes it when the pipeline ends. Measured -# 2026-08-21 by polling both pools at 15s (a 5-minute interval misses a whole -# pipeline — its PVC is created, used and deleted inside ~2 minutes): -# -# one pipeline, a single 15s interval: data0 +71.6 MB metadata +8.2 MB -# every interval without a CI pod: data0 ~0 metadata ~0 -# -# CI drove essentially all of it. Keeping that churn off `cephfs`, whose -# metadataPool is size 3, is where most of the saving comes from — that part -# does not depend on the replica count chosen here. -# -# WHY size 2 AND NOT size 1 -# Size 1 puts every PG on exactly one OSD, so stopping that OSD takes the PGs -# offline and `ceph osd ok-to-stop` refuses. Rook gates OSD restarts on that -# check, so a size-1 pool anywhere on a 3-OSD cluster blocks ALL OSD -# maintenance — CephX key rotation, Ceph upgrades, node drains and reboots, -# drive replacement. The cost is not confined to this filesystem's own data; -# it freezes the whole cluster, and `ceph -s` still reads active+clean while -# it does, because nothing is unhealthy — the restarts are simply refused. -# -# size 1 becomes viable once these pools sit on OSDs of their own, where the -# permanently un-stoppable OSD holds nothing but throwaway data. +# size 2, not 1: an unreplicated pool makes `ceph osd ok-to-stop` refuse for +# every OSD, blocking all OSD maintenance cluster-wide. size 1 is viable only +# once these pools have OSDs of their own. apiVersion: ceph.rook.io/v1 kind: CephFilesystem metadata: -- 2.54.0