From 3632640b927c058d32630ec3d0a460f61b180bf6 Mon Sep 17 00:00:00 2001 From: Konstantin Renner Date: Mon, 15 Jun 2026 10:37:41 +0200 Subject: [PATCH] cnpg: cap stopDelay (+smartShutdownTimeout) to bound failover wedge MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Roll out shutdown tuning across the barman-backed CNPG clusters: - stopDelay: 60 (all three) — caps terminationGracePeriodSeconds. With the barman-cloud plugin the demoted/restarting pod's instance-manager deadlocks on shutdown: a plugin runnable misses the 30s controller-runtime grace, the process never exits, and the native sidecar (which only stops after the main container) keeps running -> pod hangs Terminating (upstream cloudnative-pg#9770). Default 1800s leaves 2-instance clusters at 1/2 (and single-instance im-pg fully down) for ~30min after each failover/restart; 60s caps that to ~1min. Postgres itself is cleanly down by ~12s, so the trimmed time is only the wedged instance-manager shell. - smartShutdownTimeout: 10 (pprl-pg, im-pg) — already on ak-pg (#16); caps the smart-shutdown drain so -rw isn't backendless for 180s. Validated on ak-pg: app downtime 162s -> ~10s. Note: changing stopDelay alters terminationGracePeriodSeconds, so CNPG rolls each cluster's pods once to apply it. Co-Authored-By: Claude Opus 4.8 (1M context) --- kubernetes/seed/authentik/cluster.yaml | 7 +++++++ kubernetes/seed/immich/cluster.yaml | 6 ++++++ kubernetes/seed/paperless/cluster.yaml | 5 +++++ 3 files changed, 18 insertions(+) diff --git a/kubernetes/seed/authentik/cluster.yaml b/kubernetes/seed/authentik/cluster.yaml index 7820cf28..c5056c9e 100644 --- a/kubernetes/seed/authentik/cluster.yaml +++ b/kubernetes/seed/authentik/cluster.yaml @@ -12,6 +12,13 @@ spec: # -rw backendless that whole time. Capping at 10s bounds failover downtime to # ~10-15s (fast shutdown + promotion). smartShutdownTimeout: 10 + # Bound the post-failover wedge: with the barman-cloud plugin the demoted + # primary's instance-manager deadlocks on shutdown (the native sidecar never + # terminates — upstream cloudnative-pg#9770), so the pod hangs Terminating + # until SIGKILL at terminationGracePeriodSeconds (= stopDelay). The default + # 1800s leaves the cluster at 1/2 for ~30min after every failover; 60s caps + # that to ~1min. Postgres itself is already cleanly down by ~12s. + stopDelay: 60 imageName: ghcr.io/cloudnative-pg/postgresql:16 enableSuperuserAccess: true storage: diff --git a/kubernetes/seed/immich/cluster.yaml b/kubernetes/seed/immich/cluster.yaml index 143efa84..e58a3bd3 100644 --- a/kubernetes/seed/immich/cluster.yaml +++ b/kubernetes/seed/immich/cluster.yaml @@ -9,6 +9,12 @@ spec: # paths, blocking replica join. Bump back to 2 once Immich's in-app # vchord migration drops the vectors extension. instances: 1 + # See ak-pg cluster.yaml for rationale. Single-instance, so no failover, but + # the barman-plugin shutdown wedge (cloudnative-pg#9770) takes the whole DB + # down until SIGKILL — bounding terminationGracePeriod (stopDelay) matters + # even without a standby. smartShutdownTimeout keeps pod restarts quick. + smartShutdownTimeout: 10 + stopDelay: 60 imageName: ghcr.io/corentingiraud/cnpg-pgvector-vectorchord:16-migration enableSuperuserAccess: true storage: diff --git a/kubernetes/seed/paperless/cluster.yaml b/kubernetes/seed/paperless/cluster.yaml index cb253ed5..5ca955bc 100644 --- a/kubernetes/seed/paperless/cluster.yaml +++ b/kubernetes/seed/paperless/cluster.yaml @@ -5,6 +5,11 @@ metadata: namespace: paperless spec: instances: 2 + # See ak-pg cluster.yaml for rationale: smartShutdownTimeout bounds failover + # downtime (the -rw backendless window); stopDelay bounds the barman-plugin + # shutdown-wedge window (cloudnative-pg#9770) from ~30min to ~1min. + smartShutdownTimeout: 10 + stopDelay: 60 imageName: ghcr.io/cloudnative-pg/postgresql:16 enableSuperuserAccess: true storage: -- 2.54.0