kubernetes/infrastructure/rook-ceph-cluster/cephcluster.yaml

@@ -45,6 +45,49 @@ spec:
       # would be circular, and Rook only applies mon.volumeClaimTemplate to
       # newly created mons, never migrating existing ones.
       mon_data_avail_warn: "20"
+      # Stop compacting the whole RocksDB prefix on every paxos trim.
+      #
+      # Upstream default is true — "Compact a certain prefix (including paxos)
+      # when we trim its old states" — which is fine at low paxos rates and
+      # pathological here. Measured 2026-08-19:
+      #
+      #   paxos transactions      ~99,000/day  (1.15/s, sampled over 60 s)
+      #   paxos logical data       811 MB/day  (21.9 GB over 27 d uptime)
+      #   ACTUAL mon disk writes    80 GB/day  per mon, 240 GB/day for three
+      #                                        (cgroup io.stat: 2.15 TB/27 d)
+      #   => ~98x write amplification, for a ~106 MB database
+      #
+      # That was ~62% of all OS-disk writes on these nodes and roughly a
+      # quarter of everything the cluster writes — more than every client
+      # workload combined. With paxos_trim_min at 250, ~99k transactions/day
+      # means hundreds of forced range compactions per day, each rewriting the
+      # store.
+      #
+      # The churn itself is 99% cluster log and mgr stats, not real cluster
+      # activity: logm holds 6.7M paxos versions and mgrstat 4.4M, against
+      # osdmap's 9.6k. Every Rook-operator poll (mgr stat / versions / status /
+      # report / osd dump — 23 audit entries per minute per mon) becomes a
+      # cluster-log entry, hence a paxos transaction, replicated to all three
+      # mons. Reducing that polling is the complementary fix and is tracked
+      # separately; this setting removes the amplification regardless of rate.
+      #
+      # This does NOT disable compaction. RocksDB's own levelled compaction
+      # still runs and still reclaims trimmed keys; what stops is the extra
+      # forced compact_range on every trim. Store growth is therefore bounded
+      # by normal compaction, and there are two backstops if that assumption is
+      # wrong: mon_data_size_warn (15 GB default) raises HEALTH_WARN, which
+      # CephHealthWarn in monitoring-rules already alerts on, and
+      # mon_compact_on_start below compacts fully on every mon restart.
+      #
+      # Watch after applying: `du -sh /var/lib/rook/mon-*/store.db` on a node,
+      # or the mon store size in `ceph status`. If it grows past ~1 GB and does
+      # not come back down, revisit rather than leaving it.
+      mon_compact_on_trim: "false"
+      # Compensating control for the above: a full compaction on mon start.
+      # Upstream default is false. Mons restart on upgrades, node reboots and
+      # reschedules, so this gives a periodic full reclaim without paying for
+      # one on every trim. Cheap here — the store is ~106 MB.
+      mon_compact_on_start: "true"
     mgr:
       mgr/volumes/max_concurrent_clones: "16"
     # Discard/TRIM for the QLC NVMe per host (CT1000P3PSSD8, Crucial P3 Plus —