diff --git a/kubernetes/infrastructure/rook-ceph-cluster/cephcluster.yaml b/kubernetes/infrastructure/rook-ceph-cluster/cephcluster.yaml index 17828b25..49895bec 100644 --- a/kubernetes/infrastructure/rook-ceph-cluster/cephcluster.yaml +++ b/kubernetes/infrastructure/rook-ceph-cluster/cephcluster.yaml @@ -45,6 +45,49 @@ spec: # would be circular, and Rook only applies mon.volumeClaimTemplate to # newly created mons, never migrating existing ones. mon_data_avail_warn: "20" + # Stop compacting the whole RocksDB prefix on every paxos trim. + # + # Upstream default is true — "Compact a certain prefix (including paxos) + # when we trim its old states" — which is fine at low paxos rates and + # pathological here. Measured 2026-08-19: + # + # paxos transactions ~99,000/day (1.15/s, sampled over 60 s) + # paxos logical data 811 MB/day (21.9 GB over 27 d uptime) + # ACTUAL mon disk writes 80 GB/day per mon, 240 GB/day for three + # (cgroup io.stat: 2.15 TB/27 d) + # => ~98x write amplification, for a ~106 MB database + # + # That was ~62% of all OS-disk writes on these nodes and roughly a + # quarter of everything the cluster writes — more than every client + # workload combined. With paxos_trim_min at 250, ~99k transactions/day + # means hundreds of forced range compactions per day, each rewriting the + # store. + # + # The churn itself is 99% cluster log and mgr stats, not real cluster + # activity: logm holds 6.7M paxos versions and mgrstat 4.4M, against + # osdmap's 9.6k. Every Rook-operator poll (mgr stat / versions / status / + # report / osd dump — 23 audit entries per minute per mon) becomes a + # cluster-log entry, hence a paxos transaction, replicated to all three + # mons. Reducing that polling is the complementary fix and is tracked + # separately; this setting removes the amplification regardless of rate. + # + # This does NOT disable compaction. RocksDB's own levelled compaction + # still runs and still reclaims trimmed keys; what stops is the extra + # forced compact_range on every trim. Store growth is therefore bounded + # by normal compaction, and there are two backstops if that assumption is + # wrong: mon_data_size_warn (15 GB default) raises HEALTH_WARN, which + # CephHealthWarn in monitoring-rules already alerts on, and + # mon_compact_on_start below compacts fully on every mon restart. + # + # Watch after applying: `du -sh /var/lib/rook/mon-*/store.db` on a node, + # or the mon store size in `ceph status`. If it grows past ~1 GB and does + # not come back down, revisit rather than leaving it. + mon_compact_on_trim: "false" + # Compensating control for the above: a full compaction on mon start. + # Upstream default is false. Mons restart on upgrades, node reboots and + # reschedules, so this gives a periodic full reclaim without paying for + # one on every trim. Cheap here — the store is ~106 MB. + mon_compact_on_start: "true" mgr: mgr/volumes/max_concurrent_clones: "16" # Discard/TRIM for the QLC NVMe per host (CT1000P3PSSD8, Crucial P3 Plus —