From 8ae1cc71875b80244be15d63aae133546ead890b Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Fri, 28 Aug 2026 06:39:44 +0200 Subject: [PATCH] docs(infra): cut the narration out of cephcluster and the velero schedules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Comments only — filtering the diff for non-comment lines returns empty, and both files still parse and build. cephcluster.yaml goes from 224 comment lines to 76. Kept: that `ceph config` retains values which merely disappear from this file, so deleting a line does not restore a default and a live provider change needs `ceph config rm` first; that osdMaxUpdatesInParallel and upgradeOSDRequiresHealthyPGs are the only things keeping a rolling OSD change off all three replicas at once; that `addressRanges` drags the mons onto host networking and ties them to a DHCP lease; that devices are named by-id because kernel names move between boots; that the bluestore deferral settings must not be raised again. Cut from it: the paxos write-amplification measurement down to the byte, the history of why mon_data_avail_warn was once 20, and a table of every drive serial in the fleet sitting three lines above the by-id paths that already name them. velero-schedules.yaml loses a per-PVC inventory of the home-assistant namespace — eleven names, which had already been wrong once by two — in favour of saying the schedule sweeps the namespace. Two stale claims went with it. Both files said PVC contents are uploaded to Garage; the BackupStorageLocation reads bucket renner-uno-k8s-velero at fsn1.your-objectstorage.com, and Garage was retired to nas-1. And the ha-pg exclusion still explained itself by quoting the comment it had replaced. --- .../post-crds/velero-schedules.yaml | 61 ++-- .../rook-ceph-cluster/cephcluster.yaml | 270 ++++-------------- 2 files changed, 78 insertions(+), 253 deletions(-) diff --git a/kubernetes/infrastructure/post-crds/velero-schedules.yaml b/kubernetes/infrastructure/post-crds/velero-schedules.yaml index 50bac00e..3f0cdf07 100644 --- a/kubernetes/infrastructure/post-crds/velero-schedules.yaml +++ b/kubernetes/infrastructure/post-crds/velero-schedules.yaml @@ -1,27 +1,16 @@ --- # Velero Schedules for stateful app namespaces. # -# Scope is restricted to PersistentVolumeClaims via includedResources. -# Flux owns every other K8s object (Deployment, Service, Ingress, -# ConfigMap, Secret, …) — backing those up shadows current git state -# on a rebuild: Velero restores stale manifests from yesterday's -# snapshot, then the app Kustomization clobbers them with whatever git -# says — except when the Kustomization is blocked on the restore chain, -# leaving the cluster temporarily serving the stale spec (e.g. the -# paperless Ingress missing prod hostnames after the 2026-04-22 -# rebuild). PVC-only is the canonical "back up data, not state" scope -# (see ~/Notes/wiki/k8s-backup-scope.md and the project memory rule). +# Scope is PersistentVolumeClaims only. Flux owns every other object, and +# backing those up shadows git on a rebuild: Velero restores yesterday's +# manifests and the cluster serves them until the Kustomization catches up. +# Back up data, not state. # -# snapshotMoveData=true keeps doing the heavy lifting via Kopia + -# node-agent: PVC contents are uploaded to Garage and restored +# snapshotMoveData uploads PVC contents through Kopia and the node-agent, # independently of the K8s API surface. # -# Naming convention: -daily for the app's Schedule. Restore is a -# manual, operator-driven procedure — see kubernetes/RUNBOOK-cold-rebuild.md. -# (There is no automatic bootstrap-restore Job; it was removed as dead code, -# issue #200.) -# -# Retention: 30 days, matching the live CNPG Barman ObjectStore policy. +# Naming: -daily. Restore is manual — see RUNBOOK-cold-rebuild.md. +# Retention matches the CNPG Barman policy. apiVersion: velero.io/v1 kind: Schedule @@ -203,22 +192,13 @@ metadata: labels: app.kubernetes.io/part-of: homelab-backup spec: - # Daily at 06:45 local (next free stagger slot). The app PVCs live in - # the home-assistant namespace regardless of which app/ dir owns - # their manifest (matter-server, otbr, whisper, speech-to-phrase, - # zone-configurator all set `namespace: home-assistant` via their - # Kustomization, not per-resource): home-assistant-config (.storage - # entity registry, recorder DB, secrets.yaml), esphome-data, - # shelly-manager-data, matter-server-data, otbr-data, whisper-data, - # speech-to-phrase-models, speech-to-phrase-train, zigbee2mqtt-data, - # zwavejs-store, zone-configurator-config. On restore, - # home-assistant-config has its - # own path (the HA pod's `seed` initContainer pulling the native HAOS - # backup from Garage, see the home-assistant Kustomization comment in - # kubernetes/clusters/homelab/home-assistant.yaml); layering a Velero - # PVC restore on top of that flow for the same PVC needs its own review. - # Restore of these schedules is manual — see - # kubernetes/RUNBOOK-cold-rebuild.md. + # Daily at 06:45 local. Sweeps every PVC in the home-assistant namespace — + # several apps under apps/ set `namespace: home-assistant` in their + # Kustomization, so their PVCs land here too. + # + # home-assistant-config has its own restore path: the HA pod's `seed` + # initContainer pulls the native HAOS backup. Layering a Velero PVC restore + # on top of that for the same PVC needs its own review. schedule: "45 6 * * *" useOwnerReferencesInBackup: false template: @@ -229,16 +209,9 @@ spec: storageLocation: default ttl: 720h0m0s snapshotMoveData: true - # The ha-pg CNPG cluster now runs in this namespace (it did not when - # this schedule was written — the old comment here still claimed - # "no CNPG cluster in this namespace"). Its PVC ha-pg-1 carries the - # cnpg.io/cluster label and was being swept in: a crash-consistent - # snapshot of a running Postgres data directory is not a restorable - # backup, the same trap the wordpress-daily comment below describes. - # ha-pg is covered by Barman — daily base backup at 02:50 UTC - # (kubernetes/apps/home-assistant/ha-pg-scheduled-backup.yaml) plus - # continuous WAL archiving — so this exclusion removes a redundant - # 20 GiB upload, matching paperless/authentik/immich/trmnl above. + # Excludes the CNPG cluster's PVC. A crash-consistent snapshot of a + # running Postgres data directory is not a restorable backup, and Barman + # covers that database anyway — see ha-pg-scheduled-backup.yaml. labelSelector: matchExpressions: - key: cnpg.io/cluster diff --git a/kubernetes/infrastructure/rook-ceph-cluster/cephcluster.yaml b/kubernetes/infrastructure/rook-ceph-cluster/cephcluster.yaml index 11a9ba3d..c0f7f146 100644 --- a/kubernetes/infrastructure/rook-ceph-cluster/cephcluster.yaml +++ b/kubernetes/infrastructure/rook-ceph-cluster/cephcluster.yaml @@ -8,16 +8,9 @@ spec: image: quay.io/ceph/ceph:v19.2.6 # {"$imagepolicy": "flux-system:ceph"} allowUnsupported: false dataDirHostPath: /var/lib/rook - # Do not start restarting OSDs while PGs are still recovering. The default - # is false, so Rook would proceed regardless — and paired with the old - # osdMaxUpdatesInParallel default that means it can take the next OSD down - # while the previous one's data is still being caught up. - # - # With three OSDs and size 3 there is no spare copy to fall back on: the - # gate between one OSD returning and the next going down is the only thing - # that keeps a rolling change from becoming an outage. See - # storage.osdMaxUpdatesInParallel below, which bounds the same operation - # from the other side. + # Defaults to false. With three OSDs and size 3 there is no spare copy, so + # this gate is the only thing keeping a rolling OSD change from taking the + # next one down mid-recovery. Paired with storage.osdMaxUpdatesInParallel. upgradeOSDRequiresHealthyPGs: true mon: count: 3 @@ -39,113 +32,40 @@ spec: daemon: keyRotationPolicy: KeyGeneration keyGeneration: 2 - # Bump CephFS subvolume cloner pool from 4 → 16. Velero CSI Snapshot - # Data Movement provisions a backupPVC per source PVC by cloning the - # VolumeSnapshot, and a 4-thread default stalls migrate-immich's - # multi-PVC Backup with `rados: "all cloner threads are busy"`. 16 - # gives headroom for parallel restores during cluster rebuild too - # (paperless + audiobookshelf + immich + ... all clone at once). + # CephFS cloner pool raised from the default 4. Velero clones a + # VolumeSnapshot per PVC, and 4 threads stall a multi-PVC backup with + # `rados: "all cloner threads are busy"`. cephConfig: mon: - # The Ceph default, stated rather than left out. Removing the line would - # not restore it: `ceph config` keeps a value that only disappears from - # this file, so the old 20 would sit in the mon config database - # indefinitely — the same trap the bluestore settings below describe, - # and the one that crash-looped an OSD during the network provider - # change when public_network survived being deleted here. - # - # It was 20 because the mons share the node's /var with containerd, and - # on the VMs that was ~60 GB: 30% free measured image and snapshot churn - # rather than anything about Ceph, so k8s-2 tripped MON_DISK_LOW whenever - # the image cache filled and GC could not reclaim it (#434). - # - # Bare metal removed the premise. /var is now ~931 GB per node at 7-8% - # used, so 30% free is a real reserve again and the ~106 MB mon database - # is irrelevant against it. Reaching 70% would be a genuine signal about - # the node rather than noise about the image cache. - # - # mon_data_avail_crit stays at the 5% default, and CephMonDiskLow in - # monitoring-rules still watches /var directly for the real risk, which - # is the node filling rather than the mon growing. + # The Ceph default, stated rather than omitted. Deleting a line here does + # NOT restore the default: `ceph config` keeps whatever was last applied, + # so the old value would sit in the mon config database indefinitely. + # Change values in place; to actually drop one, `ceph config rm` it. mon_data_avail_warn: "30" - # Stop compacting the whole RocksDB prefix on every paxos trim. + # Off, against the upstream default. The forced compact_range on every + # paxos trim rewrote the whole store hundreds of times a day and was the + # single largest source of disk writes on these nodes. # - # Upstream default is true — "Compact a certain prefix (including paxos) - # when we trim its old states" — which is fine at low paxos rates and - # pathological here. Measured 2026-08-19: - # - # paxos transactions ~99,000/day (1.15/s, sampled over 60 s) - # paxos logical data 811 MB/day (21.9 GB over 27 d uptime) - # ACTUAL mon disk writes 80 GB/day per mon, 240 GB/day for three - # (cgroup io.stat: 2.15 TB/27 d) - # => ~98x write amplification, for a ~106 MB database - # - # That was ~62% of all OS-disk writes on these nodes and roughly a - # quarter of everything the cluster writes — more than every client - # workload combined. With paxos_trim_min at 250, ~99k transactions/day - # means hundreds of forced range compactions per day, each rewriting the - # store. - # - # The churn itself is 99% cluster log and mgr stats, not real cluster - # activity: logm holds 6.7M paxos versions and mgrstat 4.4M, against - # osdmap's 9.6k. Every Rook-operator poll (mgr stat / versions / status / - # report / osd dump — 23 audit entries per minute per mon) becomes a - # cluster-log entry, hence a paxos transaction, replicated to all three - # mons. Reducing that polling is the complementary fix and is tracked - # separately; this setting removes the amplification regardless of rate. - # - # This does NOT disable compaction. RocksDB's own levelled compaction - # still runs and still reclaims trimmed keys; what stops is the extra - # forced compact_range on every trim. Store growth is therefore bounded - # by normal compaction, and there are two backstops if that assumption is - # wrong: mon_data_size_warn (15 GB default) raises HEALTH_WARN, which - # CephHealthWarn in monitoring-rules already alerts on, and - # mon_compact_on_start below compacts fully on every mon restart. - # - # Watch after applying: `du -sh /var/lib/rook/mon-*/store.db` on a node, - # or the mon store size in `ceph status`. If it grows past ~1 GB and does - # not come back down, revisit rather than leaving it. + # This does NOT disable compaction — RocksDB's levelled compaction still + # reclaims trimmed keys. Backstops if that proves wrong: + # mon_data_size_warn raises HEALTH_WARN, and mon_compact_on_start below + # compacts fully on every mon restart. mon_compact_on_trim: "false" - # Compensating control for the above: a full compaction on mon start. - # Upstream default is false. Mons restart on upgrades, node reboots and - # reschedules, so this gives a periodic full reclaim without paying for - # one on every trim. Cheap here — the store is ~106 MB. + # Compensating control, against the upstream default: mons restart often + # enough for this to give periodic full reclaim without paying for one + # on every trim. mon_compact_on_start: "true" mgr: mgr/volumes/max_concurrent_clones: "16" - # Discard/TRIM for the QLC NVMe per host (CT1000P3PSSD8, Crucial P3 Plus — - # earlier comments here said "P2", which is a different drive). These need - # an OSD restart to take effect (init-time). Baseline 2026-05-05: - # 71.6 MB/s avg / 0.89 s avg latency on a 5-min replicated 4 MiB write - # bench (post-SLC-cache). See ceph-speed-baseline-2026-05-05 in the wiki. + # Discard/TRIM. Init-time settings — an OSD restart is needed to apply. # - # bluestore_prefer_deferred_size_ssd / bluestore_deferred_batch_ops_ssd are - # DELIBERATELY LEFT AT THE CEPH DEFAULTS (0 and 16) and must not be raised - # again without re-reading this. - # - # They were previously set to the HDD-style 65536 / 64 as a throughput tune. - # prefer_deferred_size defers every write SMALLER than the threshold: the - # write goes first into RocksDB's WAL and is applied to the object store - # afterwards. That is a double write, and at 65536 it caught essentially - # every write this cluster makes — SQLite pages are 4 KiB, Postgres pages - # 8 KiB. Ceph disables deferral on SSD (default 0) precisely because solid - # state does not need the random-write batching that helps spinning rust. - # - # Two reasons it was the wrong trade here: - # - The benchmark that justified it cannot have exercised it. It used - # 4 MiB writes, 64x ABOVE the 64 KiB cutoff, so not one of those writes - # was ever deferred. The setting was validated by a workload it does - # not affect. - # - It doubles writes onto QLC, which has the lowest endurance of any - # NAND type. Measured 2026-08-19: all three OSD drives at 80/82/87% - # endurance used, advancing ~0.13 points/day, i.e. rated life exhausted - # between Nov 2026 and Jan 2027 — and simultaneously, since size=3 on - # 3 OSDs means every drive absorbs 100% of cluster writes (see #434). - # - # This cluster is write-light (~117 GB/day at the RBD client layer), so - # throughput was never the binding constraint. Endurance is. If small-write - # latency ever becomes the real problem, fix it with more OSDs rather than - # by paying a 2x write tax on every 4 KiB page. + # bluestore_prefer_deferred_size_ssd and bluestore_deferred_batch_ops_ssd + # are DELIBERATELY LEFT AT THE CEPH DEFAULTS and must not be raised again. + # Deferral double-writes everything below the threshold, and an HDD-style + # value catches every write this cluster makes (4 KiB SQLite, 8 KiB + # Postgres pages). Ceph disables it on SSD for that reason. Throughput has + # never been the binding constraint here; endurance is. If small-write + # latency becomes a real problem, add OSDs instead. osd: bdev_enable_discard: "true" bdev_async_discard: "true" @@ -159,106 +79,44 @@ spec: ssl: false monitoring: enabled: true - # Dedicated Ceph cluster network on VLAN 49, carried by the second X710 - # SFP+ port (enp2s0f1np1) on each node, attached to the OSDs through Multus. + # Dedicated Ceph cluster network on VLAN 49, attached to the OSDs through + # Multus. The attachment is san-network-attachment.yaml. # - # WHY NOT provider: host, WHICH ALSO REACHES THAT LINK + # Only `cluster` is selected. Do not add `public`, and do not reach for + # `addressRanges` — that is honoured only under provider: host, which puts + # every daemon including the mons on host networking. A mon's address then + # becomes the node's DHCP lease, and when a lease moves the mon crash-loops + # on "Cannot assign requested address" with no node free to fail over to, + # because all three already hold the mon host ports. # - # Because it reaches it for every daemon, not just the ones that need it. - # The cluster network is an OSD concern: per the Ceph documentation it - # carries OSD heartbeat, replication and recovery, and "Ceph Monitors always - # operate on the public network". `addressRanges` is only honoured under - # provider: host, so using it put the mons on host networking as a side - # effect — making a mon's address the node's DHCP lease. + # With `public` unset, client traffic stays on the pod network and VLAN 49 + # carries Ceph-internal traffic only — which is also why Rook's Multus + # prerequisites about host routing do not apply here. # - # That is not theoretical. When a lease moved, the mon could no longer bind - # ("Cannot assign requested address") and crash-looped. Rook detected it and - # started a failover, but with three mons on three nodes each holding the - # mon host ports, the replacement could not be scheduled anywhere — the - # canary sat Pending on "didn't have free ports". Two of three nodes hit - # this during the Talos v1.12.11 upgrade and both needed hand intervention. - # - # Only `cluster` is selected here. With `public` left unset Rook puts client - # traffic on the pod network, so mons, MDS, MGR and the CSI pods keep - # Rook-managed identities and hold no host ports — which is what removes - # that failure mode rather than working around it. - # - # ISOLATION IS PRESERVED - # - # Rook's Multus prerequisites about hosts routing to the Multus network - # apply to a Multus *public* network. Selecting only `cluster` means they do - # not apply, so VLAN 49 stays what it is: Ceph-internal traffic and nothing - # else, no router, no other participants. After this settles, the node - # addresses on that segment (10.99.0.1-.3) have no remaining purpose and - # should come out of the Talos configs — at which point the attachment's - # pool can take the whole /29. - # - # Reached from provider: "" rather than directly from host, which Rook's CRD - # rejects. See the phase A commit. - # - # ONE-TIME STEP THAT THIS FILE CANNOT EXPRESS, recorded because it is not - # reproducible from the manifests alone. Removing addressRanges here does - # not clear public_network and cluster_network from Ceph's own config - # database — the same trap the bluestore comment above describes: `ceph - # config` retains a value that is only deleted from this file. During the - # phase A rollout the first OSD came up on the pod network, still had - # public_network 10.2.40.0/24 in the config database, found no interface in - # it and crash-looped on "Failed to pick public address". Both keys were - # removed with `ceph config rm global public_network` and the same for - # cluster_network; Rook then wrote cluster_network afresh from the - # attachment below. Anyone rebuilding this cluster from scratch will not hit - # it — the config database starts empty — but anyone repeating a provider - # change on a live cluster will. - # - # Bounded by upgradeOSDRequiresHealthyPGs and storage.osdMaxUpdatesInParallel - # above: one OSD restarts at a time and only once PGs are clean again. - # - # The attachment itself is san-network-attachment.yaml. + # Changing the provider on a LIVE cluster needs a step this file cannot + # express: `ceph config` retains public_network and cluster_network after + # they disappear from here, and an OSD that comes up on the pod network with + # a stale public_network crash-loops on "Failed to pick public address". + # Clear them with `ceph config rm global public_network` (and cluster_network) + # before the rollout. A cluster built from scratch starts with an empty + # config database and never sees this. network: provider: multus selectors: cluster: rook-ceph/ceph-san - # Nodes are listed explicitly rather than useAllNodes, so that a node can be - # taken out of the storage fleet by removing it here. Under useAllNodes the - # operator re-provisions an OSD on any node whose device still carries a - # BlueStore signature, which makes purging an OSD ahead of pulling its drive - # impossible: the OSD is recreated within a reconcile and starts backfilling - # onto the drive being replaced. + # Nodes listed explicitly, not useAllNodes: under useAllNodes the operator + # re-provisions an OSD on any node whose device still carries a BlueStore + # signature, so purging an OSD before pulling its drive is impossible — it + # comes back within a reconcile and backfills onto the drive being replaced. # - # All three nodes are bare metal and see three NVMe drives each, so they name - # their OSDs by /dev/disk/by-id path. A bare name would be actively wrong: - # Talos enumerates these machines' drives in a different order than Proxmox - # did, and the kernel name lands on the Kingston SYSTEM disk or on EPHEMERAL - # depending on the boot. Rook would probably decline the system disk for - # carrying partitions, but that is luck rather than intent. The by-id path is - # derived from model and serial and does not depend on enumeration at all. - # - # k8s-3 demonstrated why: it enumerated the Kingston as nvme0n1 on one boot - # and the Crucial as nvme0n1 on the very next one. - # - # k8s-1 (jupiter) - # 50026B7382B48B86 Kingston — Talos system disk - # 2332E86B749A Crucial — Talos EPHEMERAL (/var) - # S437NX0M400426 PM983 — this OSD - # - # k8s-2 (saturn) - # 50026B7382DA5F6E Kingston — Talos system disk - # 2332E86B74BC Crucial — Talos EPHEMERAL (/var) - # S437NX0M403161 PM983 — this OSD - # - # k8s-3 (neptune) - # 50026B7382B48B90 Kingston — Talos system disk - # 2326E6E8E67D Crucial — Talos EPHEMERAL (/var) - # S437NF0M501176 PM983 — this OSD + # Devices are named by /dev/disk/by-id path. Kernel names are not stable + # across boots on these machines — k8s-3 enumerated a different drive as + # nvme0n1 on two consecutive boots — and a bare name can land on the system + # disk or on EPHEMERAL. storage: - # One OSD restarted at a time. The default is 20, which on a three-OSD - # cluster means all of them at once — every PG inactive, every pool - # unreachable, for as long as the restart takes. The default suits a - # cluster large enough that 20 OSDs is a small fraction of it; here it is - # the whole thing. - # - # This binds any change that rewrites the OSD deployment spec, not just - # Ceph version upgrades — a network provider change is one such. + # One at a time. The default of 20 would restart every OSD in this cluster + # simultaneously — every PG inactive until they return. Binds any change + # that rewrites the OSD deployment spec, not just version upgrades. osdMaxUpdatesInParallel: 1 useAllNodes: false useAllDevices: false @@ -274,18 +132,12 @@ spec: - fullpath: /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NF0M501176 config: osdsPerDevice: "1" - # Storage layer must never be evicted by kubelet under node pressure - # and must never be picked first by the Linux OOM killer. system-node- - # critical (PriorityValue 2000001000) is the highest built-in class: - # kubelet preserves these pods last, and kernel sets oom_score_adj to - # -997 so anything else dies before a Ceph daemon does. + # Highest built-in class: kubelet evicts these last and the kernel gives + # them the lowest oom_score_adj. priorityClassNames: all: system-node-critical - # memory requests == limits => Guaranteed QoS. Combined with the - # priority class above, the pod's oom_score_adj is the lowest possible. - # CPU limits are deliberately NOT set — Ceph daemons benefit from - # bursting under recovery/backfill, and a CPU limit would throttle - # them; CPU pressure on this cluster is not a real concern (~15%). + # requests == limits gives Guaranteed QoS. CPU limits are deliberately NOT + # set — Ceph daemons burst during recovery and a limit would throttle them. resources: mon: requests: -- 2.54.0