From bfc885762ccc3d4e53d00d2337a16514f9f0804b Mon Sep 17 00:00:00 2001 From: Konstantin Renner Date: Wed, 6 May 2026 07:46:44 +0200 Subject: [PATCH] flux(seed): split monolithic seed KS into per-app seed- KSes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each `` KS now dependsOn its own `seed-` instead of the shared `seed` KS, so a slow sub-tree (immich-library-seed at ~26m) no longer blocks unrelated apps. Audiobookshelf / paperless / music-assistant / Authentik unblock at their own seed-Job completion (T+~13–17 vs T+~38–39 in pipeline #884). Old `seed` KS is suspended + prune:false during the migration window so SSA-driven ownership transfer to the new KSes doesn't risk pruning Cluster CRs (which are not prune:disabled-annotated). A follow-up commit deletes seed.yaml + seed/kustomization.yaml. bootstrap.sh wait-apps drops the monolithic `wait_kustomization seed`; per-app waits absorb the seed runtime (immich 1800s, audiobookshelf 900s, paperless/authentik 600s, music-assistant 300s) and a `seed-influxdb` wait (600s) covers the InfluxDB HR + token Jobs since InfluxDB has no separate top-level KS. Co-Authored-By: Claude Opus 4.7 (1M context) --- kubernetes/bootstrap.sh | 28 +++++++-------- .../clusters/homelab/audiobookshelf.yaml | 2 +- kubernetes/clusters/homelab/authentik.yaml | 2 +- kubernetes/clusters/homelab/immich.yaml | 2 +- .../clusters/homelab/music-assistant.yaml | 2 +- kubernetes/clusters/homelab/paperless.yaml | 2 +- .../clusters/homelab/seed-audiobookshelf.yaml | 30 ++++++++++++++++ .../clusters/homelab/seed-authentik.yaml | 33 +++++++++++++++++ kubernetes/clusters/homelab/seed-immich.yaml | 35 +++++++++++++++++++ .../clusters/homelab/seed-influxdb.yaml | 28 +++++++++++++++ .../homelab/seed-music-assistant.yaml | 28 +++++++++++++++ .../clusters/homelab/seed-paperless.yaml | 33 +++++++++++++++++ kubernetes/clusters/homelab/seed.yaml | 27 ++++++-------- 13 files changed, 216 insertions(+), 36 deletions(-) create mode 100644 kubernetes/clusters/homelab/seed-audiobookshelf.yaml create mode 100644 kubernetes/clusters/homelab/seed-authentik.yaml create mode 100644 kubernetes/clusters/homelab/seed-immich.yaml create mode 100644 kubernetes/clusters/homelab/seed-influxdb.yaml create mode 100644 kubernetes/clusters/homelab/seed-music-assistant.yaml create mode 100644 kubernetes/clusters/homelab/seed-paperless.yaml diff --git a/kubernetes/bootstrap.sh b/kubernetes/bootstrap.sh index 748f6108..6c135893 100755 --- a/kubernetes/bootstrap.sh +++ b/kubernetes/bootstrap.sh @@ -286,22 +286,22 @@ if phase_runs wait-apps; then return 1 } - # The seed phase populates every CNPG database and every migrating - # PVC from legacy hosts before any app Kustomization reconciles. - # Wall-clock dominated by immich-library-seed (~250 GiB rsync over - # LAN, ~30-45 min). Every app KS dependsOn `seed`, so the per-app - # waits below should be near-instant once seed lands Ready. - wait_kustomization seed "Seed phase (DB + PVC populate from legacy)" 1440 + # Per-app seed KSes (`seed-`) populate each CNPG database and + # each migrating PVC from legacy hosts. Each downstream `` KS + # gates only on its own `seed-`, so a slow seed (immich-library + # rsync at ~26 min) no longer blocks the unrelated apps. The per-app + # waits below absorb both the seed- runtime and the app rollout. # apps waits on infrastructure-ceph-cluster (Ceph mon/mgr/OSD bring-up # is genuinely 5–10 min on a fresh OSD-wiped cluster). - wait_kustomization apps "apps" 90 - wait_kustomization authentik "Authentik" 60 - wait_kustomization monitoring "kube-prometheus-stack (Loki + Prom + AM + Grafana)" 180 - wait_kustomization music-assistant "Music Assistant" 60 - wait_kustomization paperless "Paperless" 60 - wait_kustomization home-assistant "Home Assistant (HAOS backup → native restore)" 180 - wait_kustomization audiobookshelf "Audiobookshelf" 60 - wait_kustomization immich "Immich" 60 + wait_kustomization apps "apps" 90 + wait_kustomization seed-influxdb "InfluxDB seed (HR install + token Jobs)" 600 + wait_kustomization authentik "Authentik (incl. ak-pg seed)" 600 + wait_kustomization monitoring "kube-prometheus-stack (Loki + Prom + AM + Grafana)" 180 + wait_kustomization music-assistant "Music Assistant (incl. data seed)" 300 + wait_kustomization paperless "Paperless (incl. pprl-pg + data seed)" 600 + wait_kustomization home-assistant "Home Assistant (HAOS backup → native restore)" 180 + wait_kustomization audiobookshelf "Audiobookshelf (incl. ~70 GiB library seed)" 900 + wait_kustomization immich "Immich (incl. ~26 min library-seed rsync)" 1800 # Wait for cert-manager to finish issuing every Certificate. Kustomization # Ready only means the Ingress + Certificate manifests are applied; cert- diff --git a/kubernetes/clusters/homelab/audiobookshelf.yaml b/kubernetes/clusters/homelab/audiobookshelf.yaml index 69dfb331..71c82f2b 100644 --- a/kubernetes/clusters/homelab/audiobookshelf.yaml +++ b/kubernetes/clusters/homelab/audiobookshelf.yaml @@ -23,4 +23,4 @@ spec: - name: infrastructure - name: infrastructure-crds - name: infrastructure-ceph-cluster - - name: seed + - name: seed-audiobookshelf diff --git a/kubernetes/clusters/homelab/authentik.yaml b/kubernetes/clusters/homelab/authentik.yaml index 695d61e4..a0020fe5 100644 --- a/kubernetes/clusters/homelab/authentik.yaml +++ b/kubernetes/clusters/homelab/authentik.yaml @@ -21,4 +21,4 @@ spec: - name: infrastructure - name: infrastructure-crds - name: infrastructure-ceph-cluster - - name: seed + - name: seed-authentik diff --git a/kubernetes/clusters/homelab/immich.yaml b/kubernetes/clusters/homelab/immich.yaml index 8b343db7..5807c578 100644 --- a/kubernetes/clusters/homelab/immich.yaml +++ b/kubernetes/clusters/homelab/immich.yaml @@ -23,4 +23,4 @@ spec: - name: infrastructure - name: infrastructure-crds - name: infrastructure-ceph-cluster - - name: seed + - name: seed-immich diff --git a/kubernetes/clusters/homelab/music-assistant.yaml b/kubernetes/clusters/homelab/music-assistant.yaml index 917b70e5..88bb7aa5 100644 --- a/kubernetes/clusters/homelab/music-assistant.yaml +++ b/kubernetes/clusters/homelab/music-assistant.yaml @@ -23,4 +23,4 @@ spec: - name: infrastructure - name: infrastructure-crds - name: infrastructure-ceph-cluster - - name: seed + - name: seed-music-assistant diff --git a/kubernetes/clusters/homelab/paperless.yaml b/kubernetes/clusters/homelab/paperless.yaml index 1f85c3f4..a7efcec5 100644 --- a/kubernetes/clusters/homelab/paperless.yaml +++ b/kubernetes/clusters/homelab/paperless.yaml @@ -23,4 +23,4 @@ spec: - name: infrastructure - name: infrastructure-crds - name: infrastructure-ceph-cluster - - name: seed + - name: seed-paperless diff --git a/kubernetes/clusters/homelab/seed-audiobookshelf.yaml b/kubernetes/clusters/homelab/seed-audiobookshelf.yaml new file mode 100644 index 00000000..22e09fbd --- /dev/null +++ b/kubernetes/clusters/homelab/seed-audiobookshelf.yaml @@ -0,0 +1,30 @@ +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: seed-audiobookshelf + namespace: flux-system +spec: + # Per-app slice of the former monolithic `seed` KS — owns the + # audiobookshelf library/metadata PVCs and the rsync seed Job from the + # legacy audiobookshelf VM. The downstream `audiobookshelf` KS depends + # only on this, so it's no longer blocked by the slowest sibling + # (immich-library-seed at ~26m). + interval: 10m + retryInterval: 2m + # ~5m observed for 70.5 GiB rsync; buffer for cold cache / network. + timeout: 1h + sourceRef: + kind: GitRepository + name: flux-system + path: ./kubernetes/seed/audiobookshelf + prune: true + wait: true + decryption: + provider: sops + secretRef: + name: sops-age + dependsOn: + - name: infrastructure + - name: infrastructure-crds + - name: infrastructure-ceph-cluster + - name: infrastructure-cnpg-barman diff --git a/kubernetes/clusters/homelab/seed-authentik.yaml b/kubernetes/clusters/homelab/seed-authentik.yaml new file mode 100644 index 00000000..66bad9ec --- /dev/null +++ b/kubernetes/clusters/homelab/seed-authentik.yaml @@ -0,0 +1,33 @@ +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: seed-authentik + namespace: flux-system +spec: + # Per-app slice of the former monolithic `seed` KS — owns the ak-pg + # CNPG Cluster, ObjectStore, ScheduledBackup, and the pg_dump→pg_restore + # seed Job from authentik-db-1.srv.renner.uno. The downstream `authentik` + # KS depends only on this. + interval: 10m + retryInterval: 2m + timeout: 30m + sourceRef: + kind: GitRepository + name: flux-system + path: ./kubernetes/seed/authentik + prune: true + wait: true + decryption: + provider: sops + secretRef: + name: sops-age + dependsOn: + - name: infrastructure + - name: infrastructure-crds + - name: infrastructure-ceph-cluster + - name: infrastructure-cnpg-barman + healthCheckExprs: + - apiVersion: postgresql.cnpg.io/v1 + kind: Cluster + failed: "has(status.phase) && status.phase.startsWith('Failed')" + current: "has(status.phase) && (status.phase == 'Cluster in healthy state' || status.readyInstances > 0)" diff --git a/kubernetes/clusters/homelab/seed-immich.yaml b/kubernetes/clusters/homelab/seed-immich.yaml new file mode 100644 index 00000000..e7ea36dc --- /dev/null +++ b/kubernetes/clusters/homelab/seed-immich.yaml @@ -0,0 +1,35 @@ +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: seed-immich + namespace: flux-system +spec: + # Per-app slice of the former monolithic `seed` KS — owns the im-pg + # CNPG Cluster, ObjectStore, ScheduledBackup, the pg_restore seed Job, + # and the long library-seed rsync (~250 GiB, the wall-time pole of + # the rebuild). Only the downstream `immich` KS gates on this; other + # apps are no longer blocked by it. + interval: 10m + retryInterval: 2m + # Long enough for the worst-case immich-library rsync (~30–45 min). + timeout: 4h + sourceRef: + kind: GitRepository + name: flux-system + path: ./kubernetes/seed/immich + prune: true + wait: true + decryption: + provider: sops + secretRef: + name: sops-age + dependsOn: + - name: infrastructure + - name: infrastructure-crds + - name: infrastructure-ceph-cluster + - name: infrastructure-cnpg-barman + healthCheckExprs: + - apiVersion: postgresql.cnpg.io/v1 + kind: Cluster + failed: "has(status.phase) && status.phase.startsWith('Failed')" + current: "has(status.phase) && (status.phase == 'Cluster in healthy state' || status.readyInstances > 0)" diff --git a/kubernetes/clusters/homelab/seed-influxdb.yaml b/kubernetes/clusters/homelab/seed-influxdb.yaml new file mode 100644 index 00000000..76d0add0 --- /dev/null +++ b/kubernetes/clusters/homelab/seed-influxdb.yaml @@ -0,0 +1,28 @@ +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: seed-influxdb + namespace: flux-system +spec: + # Per-app slice of the former monolithic `seed` KS — owns the InfluxDB + # HelmRelease itself plus the legacy-token + ha-token bootstrap Jobs. + # InfluxDB has no separate top-level app KS; this is its full lifecycle + # owner during the migration window. + interval: 10m + retryInterval: 2m + timeout: 30m + sourceRef: + kind: GitRepository + name: flux-system + path: ./kubernetes/seed/influxdb + prune: true + wait: true + decryption: + provider: sops + secretRef: + name: sops-age + dependsOn: + - name: infrastructure + - name: infrastructure-crds + - name: infrastructure-ceph-cluster + - name: infrastructure-cnpg-barman diff --git a/kubernetes/clusters/homelab/seed-music-assistant.yaml b/kubernetes/clusters/homelab/seed-music-assistant.yaml new file mode 100644 index 00000000..0b240e89 --- /dev/null +++ b/kubernetes/clusters/homelab/seed-music-assistant.yaml @@ -0,0 +1,28 @@ +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: seed-music-assistant + namespace: flux-system +spec: + # Per-app slice of the former monolithic `seed` KS — owns the + # music-assistant data PVC and the ssh+tar seed Job from the legacy + # music-assistant LXC. The downstream `music-assistant` KS depends + # only on this. + interval: 10m + retryInterval: 2m + timeout: 30m + sourceRef: + kind: GitRepository + name: flux-system + path: ./kubernetes/seed/music-assistant + prune: true + wait: true + decryption: + provider: sops + secretRef: + name: sops-age + dependsOn: + - name: infrastructure + - name: infrastructure-crds + - name: infrastructure-ceph-cluster + - name: infrastructure-cnpg-barman diff --git a/kubernetes/clusters/homelab/seed-paperless.yaml b/kubernetes/clusters/homelab/seed-paperless.yaml new file mode 100644 index 00000000..ca6b73d3 --- /dev/null +++ b/kubernetes/clusters/homelab/seed-paperless.yaml @@ -0,0 +1,33 @@ +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: seed-paperless + namespace: flux-system +spec: + # Per-app slice of the former monolithic `seed` KS — owns the pprl-pg + # CNPG Cluster, ObjectStore, ScheduledBackup, the pg_restore seed Job, + # and the data/media rsync seed Job from paperless.srv.renner.uno. + # The downstream `paperless` KS depends only on this. + interval: 10m + retryInterval: 2m + timeout: 30m + sourceRef: + kind: GitRepository + name: flux-system + path: ./kubernetes/seed/paperless + prune: true + wait: true + decryption: + provider: sops + secretRef: + name: sops-age + dependsOn: + - name: infrastructure + - name: infrastructure-crds + - name: infrastructure-ceph-cluster + - name: infrastructure-cnpg-barman + healthCheckExprs: + - apiVersion: postgresql.cnpg.io/v1 + kind: Cluster + failed: "has(status.phase) && status.phase.startsWith('Failed')" + current: "has(status.phase) && (status.phase == 'Cluster in healthy state' || status.readyInstances > 0)" diff --git a/kubernetes/clusters/homelab/seed.yaml b/kubernetes/clusters/homelab/seed.yaml index 8a56448b..e4507ce2 100644 --- a/kubernetes/clusters/homelab/seed.yaml +++ b/kubernetes/clusters/homelab/seed.yaml @@ -4,21 +4,22 @@ metadata: name: seed namespace: flux-system spec: - # Migration phase: pulls data from legacy hosts into in-cluster CNPG - # databases and PVCs so app Kustomizations can reconcile against - # populated state. Lives between infrastructure and apps. Removed - # entirely once all services have cut over from legacy. + # **Deprecated.** Split into per-app `seed-` KSes so a slow + # sub-tree (immich-library-seed at ~26m) no longer blocks unrelated + # apps. This object is kept only so its inventory can be pruned + # cleanly by a follow-up commit; until then it's suspended and + # prune-disabled so resources whose ownership has migrated to the + # new KSes aren't torn down. + suspend: true interval: 10m retryInterval: 2m - # Long enough to cover the slowest seed (immich library: ~250 GiB - # rsync over LAN ~ 30-45 min) plus all the smaller seeds in parallel. - timeout: 4h + timeout: 5m sourceRef: kind: GitRepository name: flux-system path: ./kubernetes/seed - prune: true - wait: true + prune: false + wait: false decryption: provider: sops secretRef: @@ -28,11 +29,3 @@ spec: - name: infrastructure-crds - name: infrastructure-ceph-cluster - name: infrastructure-cnpg-barman - # kstatus already handles Job (succeeded == 1) by default. CNPG Cluster - # readiness needs an explicit expression because kstatus doesn't know - # the Cluster CRD's status semantics. - healthCheckExprs: - - apiVersion: postgresql.cnpg.io/v1 - kind: Cluster - failed: "has(status.phase) && status.phase.startsWith('Failed')" - current: "has(status.phase) && (status.phase == 'Cluster in healthy state' || status.readyInstances > 0)"