From 68527e824411238d6544cb542e11a5f2ce9a30e6 Mon Sep 17 00:00:00 2001 From: atlas-bot Date: Sat, 26 Sep 2026 15:06:01 +0000 Subject: [PATCH 1/2] ha-pg: add dedicated walStorage 18Gi, switch primary on update (REN-176) Owner-approved option B on REN-176. Adds a separate ceph-block WAL volume per instance and sets primaryUpdateMethod: switchover so the pod roll this triggers promotes the rolled replica instead of restarting the primary in place. Permanent: the CNPG 1.30.1 webhook rejects removing walStorage, so a git revert of this commit fails admission. Merge only in an owner-named window. Co-Authored-By: Claude Opus 5.5 --- .../apps/home-assistant/ha-pg-cluster.yaml | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/kubernetes/apps/home-assistant/ha-pg-cluster.yaml b/kubernetes/apps/home-assistant/ha-pg-cluster.yaml index 7b2ee685..1167036f 100644 --- a/kubernetes/apps/home-assistant/ha-pg-cluster.yaml +++ b/kubernetes/apps/home-assistant/ha-pg-cluster.yaml @@ -31,6 +31,11 @@ metadata: namespace: home-assistant spec: instances: 2 + # switchover, not CNPG's default `restart`: any change that rolls the pods + # (walStorage below did) then promotes the already-rolled replica instead of + # restarting the primary in place, so the recorder loses seconds of writes + # rather than stopDelay plus a pg_wal move plus startup (REN-176). + primaryUpdateMethod: switchover smartShutdownTimeout: 10 stopDelay: 60 imageName: ghcr.io/cloudnative-pg/postgresql:16 @@ -62,6 +67,30 @@ spec: # about four. Approved by the owner on REN-135. # ceph-block has allowVolumeExpansion: true, so this grows in place. size: 32Gi + # WAL on its own volume (ha-pg-N-wal), approved by the owner on REN-176. + # + # PERMANENT. The CNPG webhook rejects removing walStorage once set ("walStorage + # cannot be disabled once configured"), so reverting this block fails + # admission and leaves the home-assistant Kustomization stuck. Undoing it + # means a new Cluster recovered from ha-pg-store. Adding it on 1.30.1 needs + # no re-clone: the operator creates the PVC, rolls each pod, and the instance + # manager moves pg_wal onto it before Postgres starts. + # + # This does NOT keep the recorder up when WAL fills: a full WAL disk still + # stops Postgres. What it does is keep WAL writable if PGDATA fills (a purge + # that stops running), and make WAL-volume usage a direct archiver-backlog + # signal. + # + # Sized so stall survival is no worse than the shared 32Gi gave (~4 days): + # 1.86 GiB normal pg_wal footprint (max_wal_size 2GB) + # + 1.86 GiB worst nightly burst queued behind the archiver (119 x 16 MiB, REN-136) + # + 13.98 GiB three days of full archiver outage at 4.66 GiB/day + # = 17.7 GiB -> 18Gi. Smaller cuts survival to hours, which is worse than + # today. RBD is thin, so the ceiling costs nothing until a stall fills it. + # The webhook requires min_wal_size < max_wal_size < this size. + walStorage: + storageClass: ceph-block + size: 18Gi plugins: - name: barman-cloud.cloudnative-pg.io isWALArchiver: true -- 2.54.0 From b9a85474c023326ee6e18c5a6e48344bc722da37 Mon Sep 17 00:00:00 2001 From: atlas-bot Date: Sat, 26 Sep 2026 15:09:54 +0000 Subject: [PATCH 2/2] ha-pg: walStorage 21Gi so stall survival matches today; note WAL left PGDATA (REN-176) Review #1114: 18Gi gave ~3.5 days of archiver outage, below the ~4.0 days the shared 32Gi gives today. 21Gi gives ~4.1. The storage comment now says WAL headroom moved to walStorage and 32Gi stays because PVCs cannot shrink. Co-Authored-By: Claude Opus 5.5 --- .../apps/home-assistant/ha-pg-cluster.yaml | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/kubernetes/apps/home-assistant/ha-pg-cluster.yaml b/kubernetes/apps/home-assistant/ha-pg-cluster.yaml index 1167036f..c0a05589 100644 --- a/kubernetes/apps/home-assistant/ha-pg-cluster.yaml +++ b/kubernetes/apps/home-assistant/ha-pg-cluster.yaml @@ -66,6 +66,9 @@ spec: # one stalled night survives and two fill the volume; ~18.6 GiB carries # about four. Approved by the owner on REN-135. # ceph-block has allowVolumeExpansion: true, so this grows in place. + # + # Since REN-176 WAL lives on walStorage below, so the stall headroom above + # now belongs to that volume. 32Gi stays because a PVC cannot shrink. size: 32Gi # WAL on its own volume (ha-pg-N-wal), approved by the owner on REN-176. # @@ -81,16 +84,17 @@ spec: # that stops running), and make WAL-volume usage a direct archiver-backlog # signal. # - # Sized so stall survival is no worse than the shared 32Gi gave (~4 days): + # Sized so stall survival is no worse than the shared 32Gi gives today. + # Today: ~18.6 GiB free on PGDATA / 4.66 GiB/day of WAL = ~4.0 days. # 1.86 GiB normal pg_wal footprint (max_wal_size 2GB) - # + 1.86 GiB worst nightly burst queued behind the archiver (119 x 16 MiB, REN-136) - # + 13.98 GiB three days of full archiver outage at 4.66 GiB/day - # = 17.7 GiB -> 18Gi. Smaller cuts survival to hours, which is worse than - # today. RBD is thin, so the ceiling costs nothing until a stall fills it. - # The webhook requires min_wal_size < max_wal_size < this size. + # + 18.64 GiB four days of full archiver outage at 4.66 GiB/day + # (the nightly 56-119 segment burst, REN-136, is inside that rate) + # = 20.5 GiB -> 21Gi: (21 - 1.86) / 4.66 = ~4.1 days. 18Gi would give ~3.5, + # worse than today. RBD is thin, so the ceiling costs nothing until a stall + # fills it. The webhook requires min_wal_size < max_wal_size < this size. walStorage: storageClass: ceph-block - size: 18Gi + size: 21Gi plugins: - name: barman-cloud.cloudnative-pg.io isWALArchiver: true -- 2.54.0