From 5927e4bc0e92682e0fc5c912ee584931e19bc560 Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Thu, 15 Oct 2026 11:29:33 +0200 Subject: [PATCH 01/42] fix(talos): IPv4 first in nodeIP.validSubnets, as the live nodes run MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Talos keeps the validSubnets order (siderolabs/net FilterIPs) and the first entry becomes the primary node IP. Git had fd02:46::/64 first since be643533 while k8s-1 runs with 10.2.46.0/24 first. A cluster built from git gets an IPv6 primary and kube-apiserver exits with 'service IP family "10.96.0.0/12" must match public address family "fd02:46::11"' — reproduced on a Hetzner rebuild rehearsal 2026-10-15. Co-Authored-By: Claude Opus 5.5 --- kubernetes/talos/patches/site-homelab.yaml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/kubernetes/talos/patches/site-homelab.yaml b/kubernetes/talos/patches/site-homelab.yaml index 4f790a2d..1338ae01 100644 --- a/kubernetes/talos/patches/site-homelab.yaml +++ b/kubernetes/talos/patches/site-homelab.yaml @@ -10,12 +10,15 @@ machine: # The node IP lives on node0, VLAN 46: the closed node↔node network (#576). # Both families are listed so the Node object carries an IPv6 InternalIP at # all; without one Cilium detects the node's IPv6 itself and picks a SLAAC - # address from the dynamic provider prefix (#789). Talos picks one address - # per family, IPv4 first, whatever the order here. + # address from the dynamic provider prefix (#789). + # + # Order is load-bearing: Talos keeps this order, the first entry is the + # primary node IP, and kube-apiserver requires its family to match the + # first service subnet (IPv4, cluster.yaml). nodeIP: validSubnets: - - fd02:46::/64 - 10.2.46.0/24 + - fd02:46::/64 # The UDM on the ingress segment. Published here rather than as a route # because bird owns table 47 and originates the default from it; this is the # one place the address is stated. -- 2.54.0 From fb7c12a92aff1a18e08773d5fd07ff8d6680f95a Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Thu, 15 Oct 2026 20:03:20 +0200 Subject: [PATCH 02/42] fix(gitea): no pinned volumeName on data-gitea-0 The claim named its production PV, which does not exist on a rebuilt cluster: the claim stayed Pending there, and a restored claim bound to a new volume made the gitea Kustomization fail on the immutable field. volumeName leaves git and the gitea Kustomization ignores /spec/volumeName on this claim. kustomize-controller is the field's only manager on the live claim, so its ignore rule adopts the live value instead of clearing it; size growth still applies. To verify after merge: the gitea Kustomization reconciles Ready and the live claim still names pvc-e7c56db2-481d-40cb-b94a-b264b7b4b5be. Co-Authored-By: Claude Opus 5.5 --- kubernetes/apps/gitea/pvc.yaml | 7 ++----- kubernetes/clusters/homelab/gitea.yaml | 6 ++++++ 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/kubernetes/apps/gitea/pvc.yaml b/kubernetes/apps/gitea/pvc.yaml index bbc49a1e..cab84949 100644 --- a/kubernetes/apps/gitea/pvc.yaml +++ b/kubernetes/apps/gitea/pvc.yaml @@ -10,10 +10,8 @@ # - Name, namespace, accessModes, storageClassName and volumeMode must match # the live claim exactly: they are immutable, a mismatch fails the whole # gitea Kustomization (no change to the live claim). -# - volumeName must stay: kustomize-controller has owned spec.volumeName -# since the namespace move pre-created this claim (eb23d3ac, #878). -# Without it, server-side apply tries to clear the field and the API server -# rejects the whole claim as an immutable-spec change. +# - No volumeName: the gitea Kustomization ignores /spec/volumeName on this +# claim, so the bound volume is kept and a fresh cluster binds a new one. # - Grow only. A smaller value is rejected by the API server. # - prune: disabled, so removing this file from git never deletes the claim # and its data. To hand ownership back, delete the file; the claim stays. @@ -33,7 +31,6 @@ spec: accessModes: ["ReadWriteOnce"] storageClassName: ceph-block volumeMode: Filesystem - volumeName: pvc-e7c56db2-481d-40cb-b94a-b264b7b4b5be resources: requests: storage: 60Gi diff --git a/kubernetes/clusters/homelab/gitea.yaml b/kubernetes/clusters/homelab/gitea.yaml index ff9bff09..82a8df8f 100644 --- a/kubernetes/clusters/homelab/gitea.yaml +++ b/kubernetes/clusters/homelab/gitea.yaml @@ -14,6 +14,12 @@ spec: path: ./kubernetes/apps/gitea prune: true wait: true + # The claim's bound volume differs per cluster (apps/gitea/pvc.yaml). + ignore: + - paths: ["/spec/volumeName"] + target: + kind: PersistentVolumeClaim + name: data-gitea-0 dependsOn: - name: infrastructure - name: infrastructure-crds -- 2.54.0 From dcc034be924aceb3ce6b13c3c3a2ad5f87829f00 Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Thu, 15 Oct 2026 20:10:14 +0200 Subject: [PATCH 03/42] feat(velero): the cluster restores its app volumes itself One Velero Restore per Schedule (kubernetes/restore/restores.yaml), applied by Flux, restoring the schedule's newest completed backup. Every app Kustomization with backed-up volumes depends on the new `restore` Kustomization, so on a fresh cluster no app starts on an empty volume. Restores the capability 952e6150 removed without an owner decision. Ordering and failure handling, from Velero release-1.17 source: - A Restore whose schedule has no synced backup yet goes FailedValidation for good (restore_controller.go). `restore` therefore depends on `velero-backups-synced`, which is Ready once the BackupStorageLocation has completed a sync (status.lastSyncedTime). - After that sync, FailedValidation with "No backups found for schedule" means there is nothing to restore and counts as done; any other FailedValidation, Failed or PartiallyFailed fails the Kustomization. - A claim that already exists is skipped (csi pvc_action: "PVC already exists"), and the Restore completes with a warning. On the running cluster the Restores touch nothing. - Velero deletes a Restore with the backup it came from (30-day TTL); Flux recreates it, and it again skips the existing claims. - All StorageClasses bind Immediate, so the data mover does not wait for a consuming pod. Co-Authored-By: Claude Opus 5.5 --- kubernetes/bootstrap.sh | 4 +- kubernetes/clusters/homelab/apps.yaml | 1 + .../clusters/homelab/audiobookshelf.yaml | 1 + kubernetes/clusters/homelab/authentik.yaml | 1 + kubernetes/clusters/homelab/gitea.yaml | 1 + .../clusters/homelab/home-assistant.yaml | 4 + kubernetes/clusters/homelab/immich.yaml | 1 + kubernetes/clusters/homelab/influxdb.yaml | 1 + kubernetes/clusters/homelab/matrix.yaml | 1 + .../clusters/homelab/music-assistant.yaml | 1 + kubernetes/clusters/homelab/navidrome.yaml | 1 + kubernetes/clusters/homelab/paperless.yaml | 1 + kubernetes/clusters/homelab/restore.yaml | 56 +++++++++ kubernetes/clusters/homelab/trmnl.yaml | 1 + kubernetes/clusters/homelab/wordpress.yaml | 1 + .../post-crds/velero-schedules.yaml | 4 +- kubernetes/restore/gate/kustomization.yaml | 5 + kubernetes/restore/kustomization.yaml | 4 + kubernetes/restore/restores.yaml | 106 ++++++++++++++++++ 19 files changed, 191 insertions(+), 4 deletions(-) create mode 100644 kubernetes/clusters/homelab/restore.yaml create mode 100644 kubernetes/restore/gate/kustomization.yaml create mode 100644 kubernetes/restore/kustomization.yaml create mode 100644 kubernetes/restore/restores.yaml diff --git a/kubernetes/bootstrap.sh b/kubernetes/bootstrap.sh index f2872850..fbda05ef 100755 --- a/kubernetes/bootstrap.sh +++ b/kubernetes/bootstrap.sh @@ -1940,8 +1940,8 @@ if phase_runs wait-apps; then # Each app KS now owns its CNPG database (+ any data PVCs) directly — the # per-app `seed-` KSes that used to bootstrap them were folded into the # app KSes when the seed layer was retired. Each wait below gates on the app - # KS, whose readiness includes its CNPG cluster coming up (initdb on a fresh - # cluster; a cold-rebuild data restore is the separate RECOVERY.md runbook). + # KS, whose readiness includes its CNPG cluster coming up (bootstrap.recovery) + # and, through the restore KS it depends on, its volumes restored by Velero. # Timeouts stay generous to cover first-time cluster bootstrap on fresh Ceph. # apps waits on infrastructure-ceph-cluster (Ceph mon/mgr/OSD bring-up # is genuinely 5–10 min on a fresh OSD-wiped cluster). diff --git a/kubernetes/clusters/homelab/apps.yaml b/kubernetes/clusters/homelab/apps.yaml index e5965cfe..8b83fd76 100644 --- a/kubernetes/clusters/homelab/apps.yaml +++ b/kubernetes/clusters/homelab/apps.yaml @@ -17,6 +17,7 @@ spec: # woodpecker migrated) — secrets come from OpenBao via ESO # (MIGRATION-sops-openbao.md). dependsOn: + - name: restore - name: infrastructure - name: infrastructure-crds - name: infrastructure-ceph-cluster diff --git a/kubernetes/clusters/homelab/audiobookshelf.yaml b/kubernetes/clusters/homelab/audiobookshelf.yaml index 6e936ae4..b81b5668 100644 --- a/kubernetes/clusters/homelab/audiobookshelf.yaml +++ b/kubernetes/clusters/homelab/audiobookshelf.yaml @@ -22,6 +22,7 @@ spec: secretRef: name: sops-age dependsOn: + - name: restore - name: infrastructure - name: infrastructure-crds - name: infrastructure-ceph-cluster diff --git a/kubernetes/clusters/homelab/authentik.yaml b/kubernetes/clusters/homelab/authentik.yaml index 6eb5e39e..fc43a022 100644 --- a/kubernetes/clusters/homelab/authentik.yaml +++ b/kubernetes/clusters/homelab/authentik.yaml @@ -20,6 +20,7 @@ spec: # from OpenBao via ESO (MIGRATION-sops-openbao.md). This was the LAST # sops consumer; phase 4 (age-key retirement) may proceed after soak. dependsOn: + - name: restore - name: infrastructure - name: infrastructure-crds - name: infrastructure-ceph-cluster diff --git a/kubernetes/clusters/homelab/gitea.yaml b/kubernetes/clusters/homelab/gitea.yaml index 82a8df8f..1f6d7203 100644 --- a/kubernetes/clusters/homelab/gitea.yaml +++ b/kubernetes/clusters/homelab/gitea.yaml @@ -21,6 +21,7 @@ spec: kind: PersistentVolumeClaim name: data-gitea-0 dependsOn: + - name: restore - name: infrastructure - name: infrastructure-crds - name: infrastructure-ceph-cluster diff --git a/kubernetes/clusters/homelab/home-assistant.yaml b/kubernetes/clusters/homelab/home-assistant.yaml index 4bdfff33..521e35ef 100644 --- a/kubernetes/clusters/homelab/home-assistant.yaml +++ b/kubernetes/clusters/homelab/home-assistant.yaml @@ -25,6 +25,7 @@ spec: # decryption block dropped 2026-07-13: path is sops-free — secrets come # from OpenBao via ESO (MIGRATION-sops-openbao.md). dependsOn: + - name: restore - name: infrastructure - name: infrastructure-crds - name: infrastructure-ceph-cluster @@ -58,6 +59,7 @@ spec: prune: true wait: true dependsOn: + - name: restore - name: infrastructure - name: infrastructure-crds - name: infrastructure-ceph-cluster @@ -80,6 +82,7 @@ spec: # decryption block dropped 2026-07-13: path is sops-free — secrets come # from OpenBao via ESO (MIGRATION-sops-openbao.md). dependsOn: + - name: restore - name: infrastructure - name: infrastructure-crds --- @@ -120,6 +123,7 @@ spec: # First boot downloads the Whisper model (a few hundred MB) — give it room. wait: true dependsOn: + - name: restore - name: infrastructure - name: infrastructure-crds - name: infrastructure-ceph-cluster diff --git a/kubernetes/clusters/homelab/immich.yaml b/kubernetes/clusters/homelab/immich.yaml index 01d0f5c5..ae8a66be 100644 --- a/kubernetes/clusters/homelab/immich.yaml +++ b/kubernetes/clusters/homelab/immich.yaml @@ -20,6 +20,7 @@ spec: # decryption block dropped 2026-07-13: path is sops-free — secrets come # from OpenBao via ESO (MIGRATION-sops-openbao.md). dependsOn: + - name: restore - name: infrastructure - name: infrastructure-crds - name: infrastructure-ceph-cluster diff --git a/kubernetes/clusters/homelab/influxdb.yaml b/kubernetes/clusters/homelab/influxdb.yaml index f9947bde..56b66521 100644 --- a/kubernetes/clusters/homelab/influxdb.yaml +++ b/kubernetes/clusters/homelab/influxdb.yaml @@ -25,6 +25,7 @@ spec: # boilerplate from the monolithic seed KS — InfluxDB has no CNPG cluster, only # a chart-provisioned ceph-block PVC, so ceph is the only storage dep needed. dependsOn: + - name: restore - name: infrastructure - name: infrastructure-crds - name: infrastructure-ceph-cluster diff --git a/kubernetes/clusters/homelab/matrix.yaml b/kubernetes/clusters/homelab/matrix.yaml index 1ab1a467..2ad996ff 100644 --- a/kubernetes/clusters/homelab/matrix.yaml +++ b/kubernetes/clusters/homelab/matrix.yaml @@ -17,6 +17,7 @@ spec: prune: true wait: true dependsOn: + - name: restore - name: infrastructure - name: infrastructure-crds - name: infrastructure-ceph-cluster diff --git a/kubernetes/clusters/homelab/music-assistant.yaml b/kubernetes/clusters/homelab/music-assistant.yaml index b1697caa..8393523d 100644 --- a/kubernetes/clusters/homelab/music-assistant.yaml +++ b/kubernetes/clusters/homelab/music-assistant.yaml @@ -22,6 +22,7 @@ spec: secretRef: name: sops-age dependsOn: + - name: restore - name: infrastructure - name: infrastructure-crds - name: infrastructure-ceph-cluster diff --git a/kubernetes/clusters/homelab/navidrome.yaml b/kubernetes/clusters/homelab/navidrome.yaml index ee88da95..68beb828 100644 --- a/kubernetes/clusters/homelab/navidrome.yaml +++ b/kubernetes/clusters/homelab/navidrome.yaml @@ -14,6 +14,7 @@ spec: prune: true wait: true dependsOn: + - name: restore - name: infrastructure - name: infrastructure-crds - name: infrastructure-ceph-cluster diff --git a/kubernetes/clusters/homelab/paperless.yaml b/kubernetes/clusters/homelab/paperless.yaml index 7171042d..a6796067 100644 --- a/kubernetes/clusters/homelab/paperless.yaml +++ b/kubernetes/clusters/homelab/paperless.yaml @@ -20,6 +20,7 @@ spec: # decryption block dropped 2026-07-13: path is sops-free — secrets come # from OpenBao via ESO (MIGRATION-sops-openbao.md). dependsOn: + - name: restore - name: infrastructure - name: infrastructure-crds - name: infrastructure-ceph-cluster diff --git a/kubernetes/clusters/homelab/restore.yaml b/kubernetes/clusters/homelab/restore.yaml new file mode 100644 index 00000000..8192ce73 --- /dev/null +++ b/kubernetes/clusters/homelab/restore.yaml @@ -0,0 +1,56 @@ +# Ready once Velero has completed a sync of the backups in the bucket. A +# Restore created before that finds no backup and fails for good. +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: velero-backups-synced + namespace: flux-system +spec: + interval: 10m + timeout: 30m + sourceRef: + kind: GitRepository + name: flux-system + path: ./kubernetes/restore/gate + prune: true + dependsOn: + - name: infrastructure-crds + healthChecks: + - apiVersion: velero.io/v1 + kind: BackupStorageLocation + name: default + namespace: velero + healthCheckExprs: + - apiVersion: velero.io/v1 + kind: BackupStorageLocation + current: has(status.lastSyncedTime) +--- +# App data: every app Kustomization with backed-up volumes depends on this +# one, so no app starts on an empty volume. +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: restore + namespace: flux-system +spec: + interval: 10m + timeout: 6h + sourceRef: + kind: GitRepository + name: flux-system + path: ./kubernetes/restore + prune: true + wait: true + dependsOn: + - name: velero-backups-synced + - name: infrastructure-ceph-cluster + healthCheckExprs: + # A schedule without any backup has nothing to restore. + - apiVersion: velero.io/v1 + kind: Restore + failed: >- + has(status.phase) && (status.phase in ['Failed', 'PartiallyFailed'] || + (status.phase == 'FailedValidation' && !(has(status.validationErrors) && + status.validationErrors.exists(e, e.startsWith('No backups found for schedule'))))) + current: >- + has(status.phase) && (status.phase == 'Completed' || status.phase == 'FailedValidation') diff --git a/kubernetes/clusters/homelab/trmnl.yaml b/kubernetes/clusters/homelab/trmnl.yaml index 328ab405..b6667b6e 100644 --- a/kubernetes/clusters/homelab/trmnl.yaml +++ b/kubernetes/clusters/homelab/trmnl.yaml @@ -17,6 +17,7 @@ spec: prune: true wait: true dependsOn: + - name: restore - name: infrastructure - name: infrastructure-crds - name: infrastructure-ceph-cluster diff --git a/kubernetes/clusters/homelab/wordpress.yaml b/kubernetes/clusters/homelab/wordpress.yaml index 054b7345..3ebc4593 100644 --- a/kubernetes/clusters/homelab/wordpress.yaml +++ b/kubernetes/clusters/homelab/wordpress.yaml @@ -13,6 +13,7 @@ spec: path: ./kubernetes/apps/wordpress prune: true dependsOn: + - name: restore # infrastructure gates on the mariadb-operator HelmRelease being Ready, # so the MariaDB CRD exists and has a controller before this applies. - name: infrastructure diff --git a/kubernetes/infrastructure/post-crds/velero-schedules.yaml b/kubernetes/infrastructure/post-crds/velero-schedules.yaml index 7a25784d..104ac235 100644 --- a/kubernetes/infrastructure/post-crds/velero-schedules.yaml +++ b/kubernetes/infrastructure/post-crds/velero-schedules.yaml @@ -9,7 +9,8 @@ # snapshotMoveData uploads PVC contents through Kopia and the node-agent, # independently of the K8s API surface. # -# Naming: -daily. Restore is manual — see RUNBOOK-cold-rebuild.md. +# Naming: -daily. Each has a Restore in kubernetes/restore/, which a fresh +# cluster runs before the app starts. # Retention matches the CNPG Barman policy. apiVersion: velero.io/v1 @@ -199,7 +200,6 @@ spec: # This is the ONLY restore path for home-assistant-config (.storage, # secrets.yaml, the config git checkout). The HAOS-backup `seed` # initContainer that used to repopulate it was removed in 21a0a43f. - # Restore procedure and test: REN-249 runbook section C.2. schedule: "45 6 * * *" useOwnerReferencesInBackup: false template: diff --git a/kubernetes/restore/gate/kustomization.yaml b/kubernetes/restore/gate/kustomization.yaml new file mode 100644 index 00000000..5191cdb5 --- /dev/null +++ b/kubernetes/restore/gate/kustomization.yaml @@ -0,0 +1,5 @@ +# Applies nothing: its Kustomization exists for its health check on the +# BackupStorageLocation (clusters/homelab/restore.yaml). +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: [] diff --git a/kubernetes/restore/kustomization.yaml b/kubernetes/restore/kustomization.yaml new file mode 100644 index 00000000..ba89ae7b --- /dev/null +++ b/kubernetes/restore/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - restores.yaml diff --git a/kubernetes/restore/restores.yaml b/kubernetes/restore/restores.yaml new file mode 100644 index 00000000..ead1d2d2 --- /dev/null +++ b/kubernetes/restore/restores.yaml @@ -0,0 +1,106 @@ +# One Restore per Velero Schedule (infrastructure/post-crds/velero-schedules.yaml), +# from its newest completed backup. A claim that already exists is skipped. +# openbao-daily has none: bootstrap.sh restores OpenBao from its raft snapshot. +apiVersion: velero.io/v1 +kind: Restore +metadata: + name: paperless + namespace: velero +spec: + scheduleName: paperless-daily +--- +apiVersion: velero.io/v1 +kind: Restore +metadata: + name: audiobookshelf + namespace: velero +spec: + scheduleName: audiobookshelf-daily +--- +apiVersion: velero.io/v1 +kind: Restore +metadata: + name: authentik + namespace: velero +spec: + scheduleName: authentik-daily +--- +apiVersion: velero.io/v1 +kind: Restore +metadata: + name: music-assistant + namespace: velero +spec: + scheduleName: music-assistant-daily +--- +apiVersion: velero.io/v1 +kind: Restore +metadata: + name: immich + namespace: velero +spec: + scheduleName: immich-daily +--- +apiVersion: velero.io/v1 +kind: Restore +metadata: + name: influxdb + namespace: velero +spec: + scheduleName: influxdb-daily +--- +apiVersion: velero.io/v1 +kind: Restore +metadata: + name: home-assistant + namespace: velero +spec: + scheduleName: home-assistant-daily +--- +apiVersion: velero.io/v1 +kind: Restore +metadata: + name: apps + namespace: velero +spec: + scheduleName: apps-daily +--- +apiVersion: velero.io/v1 +kind: Restore +metadata: + name: gitea + namespace: velero +spec: + scheduleName: gitea-daily +--- +apiVersion: velero.io/v1 +kind: Restore +metadata: + name: trmnl + namespace: velero +spec: + scheduleName: trmnl-daily +--- +apiVersion: velero.io/v1 +kind: Restore +metadata: + name: wordpress + namespace: velero +spec: + scheduleName: wordpress-daily +--- +apiVersion: velero.io/v1 +kind: Restore +metadata: + name: navidrome + namespace: velero +spec: + scheduleName: navidrome-daily +--- +apiVersion: velero.io/v1 +kind: Restore +metadata: + name: matrix + namespace: velero +spec: + scheduleName: matrix-daily -- 2.54.0 From 99af3d6efc9df8a4e7752b0ad639cb3eb0af11ad Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Thu, 15 Oct 2026 20:57:42 +0200 Subject: [PATCH 04/42] fix(rook): install the ceph-csi-drivers chart Rook requires Since Rook 1.20 the CSI Driver and OperatorConfig resources come from the ceph-csi-drivers chart (rook Documentation/Helm-Charts/csi-drivers-chart.md: "Rook requires the installation of the Ceph-CSI Driver chart"). The chart was never installed here: the live cluster's Drivers are the ones the operator migrated once during the 1.19->1.20 upgrade (2026-05-09), and PR #12 only rendered the ServiceAccounts and RBAC. A fresh cluster therefore has no CSI driver and provisions no volume: found by the Hetzner restore test, where every PVC stayed Pending ("Waiting for a volume to be created either by the external provisioner 'rook-ceph.rbd.csi.ceph.com'"). Values: Rook's recommended values for 1.21.0 plus the live shape: 2 controller replicas with Recreate, the workload-class=pinned toleration on the node plugins, a snapshotter for RBD (Velero's CSI snapshots of ceph-block) and group snapshots for CephFS. The chart's ServiceAccounts are named per driver (rook-ceph-rbd-csi-ceph-com-ctrlplugin-sa, ...), so they do not collide with the static ones; the static ServiceAccounts and RBAC stay until the live CSI pods run on the chart's, and go in a second change. Co-Authored-By: Claude Opus 5.5 --- .../ceph-csi-drivers/helmrelease.yaml | 79 +++++++++++++++++++ .../ceph-csi-drivers/kustomization.yaml | 1 + 2 files changed, 80 insertions(+) create mode 100644 kubernetes/infrastructure/ceph-csi-drivers/helmrelease.yaml diff --git a/kubernetes/infrastructure/ceph-csi-drivers/helmrelease.yaml b/kubernetes/infrastructure/ceph-csi-drivers/helmrelease.yaml new file mode 100644 index 00000000..8c548ce3 --- /dev/null +++ b/kubernetes/infrastructure/ceph-csi-drivers/helmrelease.yaml @@ -0,0 +1,79 @@ +apiVersion: source.toolkit.fluxcd.io/v1 +kind: HelmRepository +metadata: + name: ceph-csi-operator + namespace: rook-ceph +spec: + interval: 24h + url: https://ceph.github.io/ceph-csi-operator +--- +# The CSI Driver and OperatorConfig resources Rook requires +# (rook/Documentation/Helm-Charts/csi-drivers-chart.md). Values are Rook's +# recommended ones (rook/deploy/charts/ceph-csi-drivers/values.yaml) plus the +# shape this cluster runs with. +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: ceph-csi-drivers + namespace: rook-ceph +spec: + interval: 30m + dependsOn: + - name: rook-ceph-operator + chart: + spec: + chart: ceph-csi-drivers + sourceRef: + kind: HelmRepository + name: ceph-csi-operator + namespace: rook-ceph + install: + remediation: + retries: 3 + upgrade: + remediation: + retries: 3 + values: + operatorConfig: + namespace: rook-ceph + driverSpecDefaults: + imageSet: + name: rook-csi-operator-image-set-configmap + nodePlugin: + priorityClassName: system-node-critical + controllerPlugin: + priorityClassName: system-cluster-critical + drivers: + rbd: + enabled: true + name: rook-ceph.rbd.csi.ceph.com + # Velero snapshots ceph-block claims through CSI. + snapshotPolicy: volumeSnapshot + nodePlugin: + tolerations: + - key: workload-class + operator: Equal + value: pinned + effect: NoSchedule + controllerPlugin: + replicas: 2 + deploymentStrategy: + type: Recreate + cephfs: + enabled: true + name: rook-ceph.cephfs.csi.ceph.com + snapshotPolicy: volumeGroupSnapshot + nodePlugin: + tolerations: + - key: workload-class + operator: Equal + value: pinned + effect: NoSchedule + controllerPlugin: + replicas: 2 + deploymentStrategy: + type: Recreate + nfs: + enabled: false + nvmeof: + enabled: false diff --git a/kubernetes/infrastructure/ceph-csi-drivers/kustomization.yaml b/kubernetes/infrastructure/ceph-csi-drivers/kustomization.yaml index df7a9f93..a53f6ad0 100644 --- a/kubernetes/infrastructure/ceph-csi-drivers/kustomization.yaml +++ b/kubernetes/infrastructure/ceph-csi-drivers/kustomization.yaml @@ -1,5 +1,6 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: + - helmrelease.yaml - serviceaccounts.yaml - rbac.yaml -- 2.54.0 From 33077cd1bcf30633af2d916649c4804d9c288e88 Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Thu, 15 Oct 2026 21:42:33 +0200 Subject: [PATCH 05/42] fix(velero): restore Gitea first, the rest after it Gitea is the registry for the in-house images. With one shared restore gate it waited for every volume, Immich's included, while everything pulling from it looped on image pull errors. restore-gitea restores only Gitea and the gitea Kustomization depends on it alone; the other Restores depend on it, so Velero moves Gitea's data first. Co-Authored-By: Claude Opus 5.5 --- kubernetes/clusters/homelab/gitea.yaml | 2 +- kubernetes/clusters/homelab/restore.yaml | 41 ++++++++++++++++++--- kubernetes/restore/gitea/kustomization.yaml | 4 ++ kubernetes/restore/gitea/restore.yaml | 8 ++++ kubernetes/restore/restores.yaml | 9 +---- 5 files changed, 50 insertions(+), 14 deletions(-) create mode 100644 kubernetes/restore/gitea/kustomization.yaml create mode 100644 kubernetes/restore/gitea/restore.yaml diff --git a/kubernetes/clusters/homelab/gitea.yaml b/kubernetes/clusters/homelab/gitea.yaml index 1f6d7203..ba6fc93a 100644 --- a/kubernetes/clusters/homelab/gitea.yaml +++ b/kubernetes/clusters/homelab/gitea.yaml @@ -21,7 +21,7 @@ spec: kind: PersistentVolumeClaim name: data-gitea-0 dependsOn: - - name: restore + - name: restore-gitea - name: infrastructure - name: infrastructure-crds - name: infrastructure-ceph-cluster diff --git a/kubernetes/clusters/homelab/restore.yaml b/kubernetes/clusters/homelab/restore.yaml index 8192ce73..e892b8ed 100644 --- a/kubernetes/clusters/homelab/restore.yaml +++ b/kubernetes/clusters/homelab/restore.yaml @@ -25,20 +25,20 @@ spec: kind: BackupStorageLocation current: has(status.lastSyncedTime) --- -# App data: every app Kustomization with backed-up volumes depends on this -# one, so no app starts on an empty volume. +# Gitea's data first: it is the registry for the in-house images the rest of +# the cluster pulls. The gitea Kustomization depends on this one only. apiVersion: kustomize.toolkit.fluxcd.io/v1 kind: Kustomization metadata: - name: restore + name: restore-gitea namespace: flux-system spec: interval: 10m - timeout: 6h + timeout: 1h sourceRef: kind: GitRepository name: flux-system - path: ./kubernetes/restore + path: ./kubernetes/restore/gitea prune: true wait: true dependsOn: @@ -54,3 +54,34 @@ spec: status.validationErrors.exists(e, e.startsWith('No backups found for schedule'))))) current: >- has(status.phase) && (status.phase == 'Completed' || status.phase == 'FailedValidation') +--- +# App data: every other app Kustomization with backed-up volumes depends on +# this one, so no app starts on an empty volume. +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: restore + namespace: flux-system +spec: + interval: 10m + timeout: 6h + sourceRef: + kind: GitRepository + name: flux-system + path: ./kubernetes/restore + prune: true + wait: true + dependsOn: + - name: restore-gitea + - name: velero-backups-synced + - name: infrastructure-ceph-cluster + healthCheckExprs: + # A schedule without any backup has nothing to restore. + - apiVersion: velero.io/v1 + kind: Restore + failed: >- + has(status.phase) && (status.phase in ['Failed', 'PartiallyFailed'] || + (status.phase == 'FailedValidation' && !(has(status.validationErrors) && + status.validationErrors.exists(e, e.startsWith('No backups found for schedule'))))) + current: >- + has(status.phase) && (status.phase == 'Completed' || status.phase == 'FailedValidation') diff --git a/kubernetes/restore/gitea/kustomization.yaml b/kubernetes/restore/gitea/kustomization.yaml new file mode 100644 index 00000000..bc08b2d5 --- /dev/null +++ b/kubernetes/restore/gitea/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - restore.yaml diff --git a/kubernetes/restore/gitea/restore.yaml b/kubernetes/restore/gitea/restore.yaml new file mode 100644 index 00000000..f68caa0e --- /dev/null +++ b/kubernetes/restore/gitea/restore.yaml @@ -0,0 +1,8 @@ +# Gitea first: it is the registry for the in-house images. +apiVersion: velero.io/v1 +kind: Restore +metadata: + name: gitea + namespace: velero +spec: + scheduleName: gitea-daily diff --git a/kubernetes/restore/restores.yaml b/kubernetes/restore/restores.yaml index ead1d2d2..5337352e 100644 --- a/kubernetes/restore/restores.yaml +++ b/kubernetes/restore/restores.yaml @@ -1,6 +1,7 @@ # One Restore per Velero Schedule (infrastructure/post-crds/velero-schedules.yaml), # from its newest completed backup. A claim that already exists is skipped. # openbao-daily has none: bootstrap.sh restores OpenBao from its raft snapshot. +# gitea-daily is restored first, by restore-gitea (gitea/restore.yaml). apiVersion: velero.io/v1 kind: Restore metadata: @@ -67,14 +68,6 @@ spec: --- apiVersion: velero.io/v1 kind: Restore -metadata: - name: gitea - namespace: velero -spec: - scheduleName: gitea-daily ---- -apiVersion: velero.io/v1 -kind: Restore metadata: name: trmnl namespace: velero -- 2.54.0 From af7c48e29f8526670eb3bca33b7ca65f5a53fcb0 Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Fri, 16 Oct 2026 08:58:41 +0200 Subject: [PATCH 06/42] fix(talos): etcd subnet in the site patches; hcloud OSD on the system disk cluster.yaml named VLAN 46 as etcd's advertised subnet, a fact of the home network. It moves unchanged to site-homelab.yaml (the rendered production config is byte-identical); site-hcloud.yaml advertises etcd on the node segment bootstrap.sh provisions, 10.90.0.0/16. cpx52 carries 480 GB local disk and the project's volume quota cannot hold three OSD volumes: EPHEMERAL is capped at 60GiB and osd-data takes at least 345GiB of the system disk (ported from 92c7ee73, 8d42d89c). Co-Authored-By: Claude Opus 5.5 --- kubernetes/talos/patches/cluster.yaml | 7 +------ kubernetes/talos/patches/hcloud-1.yaml | 18 ++++++++++++------ kubernetes/talos/patches/hcloud-2.yaml | 12 ++++++++++-- kubernetes/talos/patches/hcloud-3.yaml | 12 ++++++++++-- kubernetes/talos/patches/site-hcloud.yaml | 5 +++++ kubernetes/talos/patches/site-homelab.yaml | 7 +++++++ 6 files changed, 45 insertions(+), 16 deletions(-) diff --git a/kubernetes/talos/patches/cluster.yaml b/kubernetes/talos/patches/cluster.yaml index 6541a69b..e0e5d41e 100644 --- a/kubernetes/talos/patches/cluster.yaml +++ b/kubernetes/talos/patches/cluster.yaml @@ -58,12 +58,7 @@ machine: imageGCHighThresholdPercent: 70 imageGCLowThresholdPercent: 60 cluster: - # etcd peers on node0, VLAN 46 (#576). Pinned: Talos never updates an - # existing member's peer URLs, so changing this subnet means leave + rejoin - # per member (talosctl etcd leave, reboot). - etcd: - advertisedSubnets: - - 10.2.46.0/24 + # etcd.advertisedSubnets is in the site patch: it names the site's node segment. # controlPlane.endpoint is NOT here: `talosctl gen config` takes it as its # second argument, and bootstrap.sh already passes https://k8s.renner.uno:6443 # there. Setting it again in this patch restated the same value in a second diff --git a/kubernetes/talos/patches/hcloud-1.yaml b/kubernetes/talos/patches/hcloud-1.yaml index 298a0251..13a7f6d6 100644 --- a/kubernetes/talos/patches/hcloud-1.yaml +++ b/kubernetes/talos/patches/hcloud-1.yaml @@ -55,15 +55,21 @@ kind: LinkConfig name: ceph-san up: true --- +# Capped so the OSD partition fits beside it on cpx52's 480 GB local disk. +apiVersion: v1alpha1 +kind: VolumeConfig +name: EPHEMERAL +provisioning: + maxSize: 60GiB + grow: false +--- +# The OSD is a partition on the local system disk; bootstrap.sh creates no +# Hetzner volume for a raw volume matching system_disk. apiVersion: v1alpha1 kind: RawVolumeConfig name: osd-data provisioning: diskSelector: - # Two disks: the system disk and the attached volume. Matching on transport - # as the qemu patches do selects nothing here — a Hetzner volume arrives - # over virtio-scsi, and a selector matching zero disks is skipped silently. - match: '!system_disk' - # bootstrap.sh reads this size and creates the volume to match. - minSize: 50GiB + match: system_disk + minSize: 345GiB grow: true diff --git a/kubernetes/talos/patches/hcloud-2.yaml b/kubernetes/talos/patches/hcloud-2.yaml index a32341e7..32d3b3d0 100644 --- a/kubernetes/talos/patches/hcloud-2.yaml +++ b/kubernetes/talos/patches/hcloud-2.yaml @@ -40,11 +40,19 @@ kind: LinkConfig name: ceph-san up: true --- +# See hcloud-1.yaml. +apiVersion: v1alpha1 +kind: VolumeConfig +name: EPHEMERAL +provisioning: + maxSize: 60GiB + grow: false +--- apiVersion: v1alpha1 kind: RawVolumeConfig name: osd-data provisioning: diskSelector: - match: '!system_disk' - minSize: 50GiB + match: system_disk + minSize: 345GiB grow: true diff --git a/kubernetes/talos/patches/hcloud-3.yaml b/kubernetes/talos/patches/hcloud-3.yaml index 05a424b3..cdcceb03 100644 --- a/kubernetes/talos/patches/hcloud-3.yaml +++ b/kubernetes/talos/patches/hcloud-3.yaml @@ -40,11 +40,19 @@ kind: LinkConfig name: ceph-san up: true --- +# See hcloud-1.yaml. +apiVersion: v1alpha1 +kind: VolumeConfig +name: EPHEMERAL +provisioning: + maxSize: 60GiB + grow: false +--- apiVersion: v1alpha1 kind: RawVolumeConfig name: osd-data provisioning: diskSelector: - match: '!system_disk' - minSize: 50GiB + match: system_disk + minSize: 345GiB grow: true diff --git a/kubernetes/talos/patches/site-hcloud.yaml b/kubernetes/talos/patches/site-hcloud.yaml index e2ce3856..d279887b 100644 --- a/kubernetes/talos/patches/site-hcloud.yaml +++ b/kubernetes/talos/patches/site-hcloud.yaml @@ -19,6 +19,11 @@ machine: validSubnets: - 10.90.0.0/16 cluster: + # etcd peers on the node segment. Hetzner firewalls do not filter private + # networks; the public addresses admit only this host and the nodes. + etcd: + advertisedSubnets: + - 10.90.0.0/16 # Three control planes that also carry workloads, which is the production # shape. The qemu site cannot mirror it: that provisioner gives a second disk # to workers only, so a Ceph OSD forces 1 control plane and 3 workers there. diff --git a/kubernetes/talos/patches/site-homelab.yaml b/kubernetes/talos/patches/site-homelab.yaml index 1338ae01..323b58e8 100644 --- a/kubernetes/talos/patches/site-homelab.yaml +++ b/kubernetes/talos/patches/site-homelab.yaml @@ -25,6 +25,13 @@ machine: nodeAnnotations: net.renner.uno/egress-gw-v6: fd02:47::1 net.renner.uno/egress-gw-v4: 10.2.47.1 +cluster: + # etcd peers on node0, VLAN 46 (#576). Pinned: Talos never updates an + # existing member's peer URLs, so changing this subnet means leave + rejoin + # per member (talosctl etcd leave, reboot). + etcd: + advertisedSubnets: + - 10.2.46.0/24 --- # All pod traffic goes to table 47, and the table decides — longest prefix # match, no exceptions to carve out. bird fills it with the node's attached -- 2.54.0 From bb6874f59c3c271fe91b3d9f4a18b68a6c88e012 Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Fri, 16 Oct 2026 09:05:13 +0200 Subject: [PATCH 07/42] feat(bootstrap): hcloudtest boots production's config, contained by egress firewall hcloudtest runs SITE=hcloud on cpx52 with the OSD on the system disk, and nothing stands in for the home network. Ported from worktree-hcloud-wg-site: the Talos snapshot builder (removes its server on failure), this host's inbound firewall replacing earlier runs', the Hetzner token from OpenBao, the FLUX_SOURCE=restore-test phase (flux install, per-run read-only deploy key on the GitHub mirror, restore-test-git Secret, flux-root.yaml on FLUX_BRANCH), its safety prechecks, access files under ~/.local/state/homelab-hcloudtest instead of access-staging, hcloudclean deleting ssh keys and restore-test deploy keys, and the 30-minute OpenBao status wait. Containment: an outbound firewall on the nodes, applied before any config, admits udp and icmp anywhere and tcp to the complement of proxy.renner.uno's addresses, resolved over DoH at run time (cidr_complement, in bash; fails above Hetzner's 500 effective rules). The egress check pod runs at the end of hcloudtest and again before the restore test writes any secret. Co-Authored-By: Claude Opus 5.5 --- kubernetes/bootstrap.sh | 517 +++++++++++++++++++++++++++++++++------- 1 file changed, 429 insertions(+), 88 deletions(-) diff --git a/kubernetes/bootstrap.sh b/kubernetes/bootstrap.sh index fbda05ef..1dc820a1 100755 --- a/kubernetes/bootstrap.sh +++ b/kubernetes/bootstrap.sh @@ -15,10 +15,17 @@ set -euo pipefail # # the only place the registry fallback can be # # tested: gitea.renner.uno has no external # # ingress, so a node there genuinely cannot -# # reach it. BILLS BY THE HOUR — teardown runs -# # from the EXIT trap and is by label. -# ./bootstrap.sh hcloudclean # that teardown on its own, for a run that was -# # killed before its trap could fire. +# # reach it. BILLS BY THE HOUR — the machines +# # stay up until hcloudclean. Access files land in +# # ~/.local/state/homelab-hcloudtest/. +# ./bootstrap.sh hcloudclean # deletes everything hcloudtest created, by +# # label, and the restore test's deploy keys. +# +# Restore test on the hcloudtest cluster (production tree, read-only source, +# kubernetes/restore-test/flux-root.yaml): +# KUBECONFIG=~/.local/state/homelab-hcloudtest/kubeconfig \ +# TALOSCONFIG=~/.local/state/homelab-hcloudtest/talosconfig \ +# FLUX_SOURCE=restore-test ./bootstrap.sh flux # # Flux git source (break-glass): the in-cluster Gitea that Flux normally pulls # from is itself Flux-managed, so a cold rebuild cannot pull from it until the @@ -176,6 +183,10 @@ resolve_node_ip() { phase_runs() { [[ "$PHASE" == "all" || "$PHASE" == "$1" ]] } +# The restore test targets the cluster bootstrap.sh hcloudtest builds. +if [[ "$FLUX_SOURCE" == "restore-test" && "$PHASE" != "flux" && "$PHASE" != "wait-apps" ]]; then + fail "FLUX_SOURCE=restore-test runs the flux and wait-apps phases only" +fi OPENBAO_NS="openbao" # Where the secrets come from. On a workstation, the operator's pass store. In @@ -217,7 +228,7 @@ OPENBAO_SECRETS_DIR="${OPENBAO_SECRETS_DIR:-}" # that cannot be told apart from a fault. Its non-zero exit propagates, which is # what makes require_secrets print the reason instead. git_token_for_source() { - if [[ "$FLUX_SOURCE" == "github" ]]; then + if [[ "$FLUX_SOURCE" == "github" || "$FLUX_SOURCE" == "restore-test" ]]; then if [[ -n "${GITHUB_TOKEN:-}" ]]; then printf '%s' "$GITHUB_TOKEN" else @@ -255,13 +266,13 @@ OPENBAO_REQUIRED_SECRETS=( # rest, because a rebuild that discovers it missing has already wiped the # disks. "ghcr-pull|GHCR pull token for the image mirror|pass show homelab/github/k8s-package-pull" - # Only the hcloudtest and hcloudclean phases ask for this one, by name, so a - # rebuild of the real cluster never demands it. - # - # Note the path: homelab/hetzner/, spelled correctly, while the two restore - # keys above live under homelab/hetztner/. Two folders, one typo, both real - # — do not "fix" one of these paths without moving the entry first. - "hcloud-token|Hetzner Cloud API token for the throwaway test cluster|pass show homelab/hetzner/cloud-api-secret" +) +# Asked for by name only, never by a bare require_secrets: a rebuild of the +# real cluster cannot depend on the OpenBao it is rebuilding. Scoped to the +# Hetzner project homelab-backup-verify, which holds nothing but throwaway +# machines. +HCLOUD_REQUIRED_SECRETS=( + "hcloud-token|Hetzner Cloud API token for the throwaway test cluster|bao kv get -mount=homelab -field=token infrastructure/hcloud-backup-verify" ) # Collects all of them up front, into one directory, so everything after this @@ -288,7 +299,7 @@ require_secrets() { local entry name what how local -a wanted=() if [[ $# -gt 0 ]]; then - for entry in "${OPENBAO_REQUIRED_SECRETS[@]}"; do + for entry in "${OPENBAO_REQUIRED_SECRETS[@]}" "${HCLOUD_REQUIRED_SECRETS[@]}"; do for name in "$@"; do [[ "${entry%%|*}" == "$name" ]] && wanted+=("$entry") done @@ -405,15 +416,18 @@ read_secret() { # which does nothing but the sweep. Anything this phase creates carries # HCLOUD_LABEL so that sweep can find it without being told. # -# WHAT IT DOES NOT COVER. +# WHAT IT IS. # -# Everything downstream of Multus. The provider is not the limit — a cloud -# server takes up to 16 volumes and 3 private networks — but ceph-san binds to -# the kernel link name enp2s0f1np1, and that name cannot be manufactured here -# without assuming a Hetzner private network passes VLAN tags. See -# talos/patches/site-hcloud.yaml. Nor Flux and everything after it: that phase -# needs the OpenBao seal key, which deliberately never reaches CI, and this -# test is meant to run unattended. +# The production config on new hardware: cluster.yaml, site-hcloud.yaml (what +# the Hetzner site is), containerd-layer-retention.yaml and hcloud-N.yaml (each +# machine). Nothing stands in for the home network. Flux is a separate run, +# FLUX_SOURCE=restore-test bootstrap.sh flux, because it needs the OpenBao seal +# key. Ceph replicates over the pod network there: Hetzner networks are layer 3 +# (restore-test/flux-root.yaml). +# +# Containment: an outbound Hetzner firewall on every node admits udp and icmp +# anywhere and tcp anywhere except the addresses of proxy.renner.uno, the public +# edge in front of production. # Talos version and Image Factory schematic, read out of the installer image in # cluster.yaml. Both throwaway-cluster phases need them and neither may state # them: a second copy is a copy that goes stale, and the day it did the test @@ -433,7 +447,7 @@ HCLOUD_LABEL="managed-by=bootstrap-hcloudtest" # The disks a machine has are stated in its Talos patch and read from there, so # the provisioner carries no second copy. Emits one size in whole GB per raw -# volume the patch declares, in order. +# volume the patch declares off the system disk, in order. hcloud_volumes_for_node() { local patch="$TALOS_DIR/patches/$1.yaml" size gib [[ -f "$patch" ]] || fail "$patch not found — the machine config is the list" @@ -444,22 +458,256 @@ hcloud_volumes_for_node() { gib=${size%GiB} [[ "$gib" =~ ^[0-9]+$ ]] || fail "$patch: cannot read volume size '$size' — state it as GiB" echo $(( (gib * 1074 + 999) / 1000 )) - done < <(yq -r 'select(.kind == "RawVolumeConfig") | .provisioning.minSize' "$patch") + done < <(yq -r 'select(.kind == "RawVolumeConfig" and .provisioning.diskSelector.match != "system_disk") | .provisioning.minSize' "$patch") } # Deletes everything this phase created, and NOTHING else. # -# That distinction is not theoretical: the project this runs in also holds -# netbird.renner.uno, which has been up for months and is the VPN way in. The -# label is the only thing separating a throwaway server from it — and -# `hcloud server list -l ""` lists EVERY server, so an unset HCLOUD_LABEL would -# turn this function into a project wipe. Hence the guard: the label is checked -# before it is used, and every candidate must also carry the name prefix this -# phase assigns. Two independent conditions, because one of them is a variable. -# -# A separate Hetzner project, with a token scoped to it, would make this -# structural rather than defensive. Worth doing before this runs unattended. +# The project (homelab-backup-verify) also runs the backup-verify CronJob's +# machines. `hcloud server list -l ""` lists EVERY server, so an unset +# HCLOUD_LABEL would turn this function into a project wipe: the label is +# checked before it is used, and every candidate must also carry the name +# prefix this phase assigns. HCLOUD_NAME_PREFIX="hcloud" +# Outlives the run: the test cluster's talosconfig and kubeconfig. +HCLOUD_STATE_DIR="${XDG_STATE_HOME:-$HOME/.local/state}/homelab-hcloudtest" +# The public edge in front of production. The test cluster must not reach it. +HCLOUD_PROXY_HOST="proxy.renner.uno" +# Hetzner's limit on effective rules per firewall and direction. +HCLOUD_FW_RULE_LIMIT=500 + +# The Talos snapshot this cluster installs from, into HCLOUD_IMAGE. Built when +# missing: a throwaway server in rescue mode writes the Image Factory hcloud +# image to its disk, which is then snapshotted. Matched by version and +# schematic. Called directly, not in $(...), so that fail stops the run. +hcloud_talos_snapshot() { + local labels="os=talos,talos-version=$VM_VERSION,talos-schematic=${VM_SCHEMATIC:0:16}" + local builder="$HCLOUD_NAME_PREFIX-imgbuild" key_dir addr ids listing + listing=$(hcloud image list --type snapshot -l "$labels" -o noheader -o columns=id) \ + || fail "could not list the Talos snapshots" + mapfile -t ids <<<"$listing" + if [[ -n "${ids[0]:-}" ]]; then + HCLOUD_IMAGE=${ids[0]} + return 0 + fi + echo " no snapshot for $VM_VERSION/${VM_SCHEMATIC:0:16}, building one" + key_dir=$(mktemp -d) + ssh-keygen -q -t ed25519 -N '' -C "$builder" -f "$key_dir/key" + if hcloud ssh-key list -l "$HCLOUD_LABEL" -o noheader -o columns=name | grep -qx "$builder"; then + hcloud ssh-key delete "$builder" >/dev/null || fail "could not replace the stale image builder key" + fi + hcloud ssh-key create --name "$builder" --label "$HCLOUD_LABEL" \ + --public-key-from-file "$key_dir/key.pub" >/dev/null || fail "could not upload the image builder key" + hcloud server create --name "$builder" --type cx23 --image debian-12 \ + --location "${HCLOUD_LOCATION:-fsn1}" --ssh-key "$builder" --label "$HCLOUD_LABEL" >/dev/null \ + || fail "could not create the image builder" + # From here a failure leaves a billed server: remove it before failing. + builder_fail() { + hcloud server delete "$builder" >/dev/null || echo " WARNING: $builder not deleted" >&2 + hcloud ssh-key delete "$builder" >/dev/null || echo " WARNING: ssh key $builder not deleted" >&2 + rm -rf "$key_dir" + fail "$1" + } + hcloud server enable-rescue "$builder" --ssh-key "$builder" >/dev/null || builder_fail "could not enable rescue on $builder" + hcloud server reset "$builder" >/dev/null || builder_fail "could not reset $builder into rescue" + addr=$(hcloud server ip "$builder") || builder_fail "could not read the address of $builder" + for i in $(seq 1 30); do + ssh -i "$key_dir/key" -o BatchMode=yes -o ConnectTimeout=10 \ + -o UserKnownHostsFile="$key_dir/known_hosts" -o StrictHostKeyChecking=accept-new \ + "root@$addr" 'grep -q rescue /etc/hostname || hostname | grep -q rescue' && break + sleep 10 + [[ $i -eq 30 ]] && builder_fail "$builder never came up in rescue" + done + ssh -i "$key_dir/key" -o BatchMode=yes -o UserKnownHostsFile="$key_dir/known_hosts" "root@$addr" \ + "set -eo pipefail; curl -fsSL https://factory.talos.dev/image/$VM_SCHEMATIC/$VM_VERSION/hcloud-amd64.raw.xz | xz -d | dd of=/dev/sda bs=4M conv=fsync" \ + || builder_fail "writing the Talos image on $builder failed" + hcloud server poweroff "$builder" >/dev/null || builder_fail "could not power off $builder" + hcloud server create-image "$builder" --type snapshot \ + --description "talos $VM_VERSION ${VM_SCHEMATIC:0:16}" --label "$labels" \ + || builder_fail "could not snapshot $builder" + listing=$(hcloud image list --type snapshot -l "$labels" -o noheader -o columns=id) \ + || builder_fail "could not list the Talos snapshots" + mapfile -t ids <<<"$listing" + [[ -n "${ids[0]:-}" ]] || builder_fail "the snapshot of $builder is not listed under $labels" + HCLOUD_IMAGE=${ids[0]} + hcloud server delete "$builder" >/dev/null || echo " WARNING: $builder not deleted" >&2 + hcloud ssh-key delete "$builder" >/dev/null || echo " WARNING: ssh key $builder not deleted" >&2 + rm -rf "$key_dir" +} + +# One address as a string of bits: 32 for IPv4, 128 for IPv6. Returns 1, with +# the reason on stderr, for anything else. +ip_to_bits() { + local addr=$1 bits="" part v i head tail + local -a parts=() hparts=() tparts=() + if [[ "$addr" == *:* ]]; then + if [[ "$addr" == *::* ]]; then + head=${addr%%::*}; tail=${addr#*::} + [[ -n "$head" ]] && IFS=: read -r -a hparts <<<"$head" + [[ -n "$tail" ]] && IFS=: read -r -a tparts <<<"$tail" + parts=("${hparts[@]}") + for (( i = ${#hparts[@]} + ${#tparts[@]}; i < 8; i++ )); do parts+=(0); done + parts+=("${tparts[@]}") + else + IFS=: read -r -a parts <<<"$addr" + fi + [[ ${#parts[@]} -eq 8 ]] || { echo "cannot read IPv6 address '$addr'" >&2; return 1; } + for part in "${parts[@]}"; do + [[ "$part" =~ ^[0-9a-fA-F]{1,4}$ ]] || { echo "cannot read IPv6 address '$addr'" >&2; return 1; } + v=$(( 16#$part )) + for (( i = 15; i >= 0; i-- )); do bits+=$(( (v >> i) & 1 )); done + done + else + IFS=. read -r -a parts <<<"$addr" + [[ ${#parts[@]} -eq 4 ]] || { echo "cannot read IPv4 address '$addr'" >&2; return 1; } + for part in "${parts[@]}"; do + [[ "$part" =~ ^[0-9]{1,3}$ ]] || { echo "cannot read IPv4 address '$addr'" >&2; return 1; } + v=$(( 10#$part )) + (( v <= 255 )) || { echo "cannot read IPv4 address '$addr'" >&2; return 1; } + for (( i = 7; i >= 0; i-- )); do bits+=$(( (v >> i) & 1 )); done + done + fi + printf '%s' "$bits" +} + +# A prefix given as bits, in CIDR notation. $2 is the address length (32|128). +bits_to_cidr() { + local p=$1 width=$2 full out="" i + full=$p + while (( ${#full} < width )); do full+="0"; done + if (( width == 32 )); then + for (( i = 0; i < 32; i += 8 )); do out+="${out:+.}$(( 2#${full:i:8} ))"; done + else + for (( i = 0; i < 128; i += 16 )); do out+="${out:+:}$(printf '%x' $(( 2#${full:i:16} )))"; done + fi + printf '%s/%s\n' "$out" "${#p}" +} + +# cidr_complement
...: the prefixes that, with the given addresses +# of one family, cover 0.0.0.0/0 or ::/0 exactly once. Returns 1, with the +# reason on stderr, on a malformed or mixed-family list. +cidr_complement() { + local width=32 a bits + local -a excl=() + [[ "$1" == *:* ]] && width=128 + for a in "$@"; do + [[ ( "$a" == *:* && $width -eq 128 ) || ( "$a" != *:* && $width -eq 32 ) ]] \ + || { echo "cidr_complement: '$a' is not of the same family as '$1'" >&2; return 1; } + bits=$(ip_to_bits "$a") || return 1 + excl+=("$bits") + done + _cidr_walk() { + local p=$1 x hit=0 + for x in "${excl[@]}"; do + [[ "$x" == "$p"* ]] && { hit=1; break; } + done + if (( hit == 0 )); then + bits_to_cidr "$p" "$width" + return 0 + fi + (( ${#p} == width )) && return 0 + _cidr_walk "${p}0" + _cidr_walk "${p}1" + } + _cidr_walk "" +} + +# Addresses of HCLOUD_PROXY_HOST for one record type (A|AAAA), over DoH: the +# operator's LAN resolver intercepts plain DNS. Returns 1, with the reason on +# stderr, when the lookup fails. +hcloud_proxy_addrs() { + local type=$1 num answer + case "$type" in + A) num=1 ;; + AAAA) num=28 ;; + *) echo "hcloud_proxy_addrs: unknown type $type" >&2; return 1 ;; + esac + answer=$(curl -sS --max-time 10 -H 'accept: application/dns-json' \ + "https://cloudflare-dns.com/dns-query?name=$HCLOUD_PROXY_HOST&type=$type") \ + || { echo "could not resolve $HCLOUD_PROXY_HOST $type over DoH" >&2; return 1; } + jq -e '.Status == 0' >/dev/null <<<"$answer" \ + || { echo "DoH lookup of $HCLOUD_PROXY_HOST $type failed: $answer" >&2; return 1; } + jq -r --argjson t "$num" '.Answer[]? | select(.type == $t) | .data' <<<"$answer" | sort -u +} + +# Outbound firewall for the nodes: udp and icmp anywhere, tcp anywhere except +# HCLOUD_PROXY_HOST. Attached before an earlier run's egress firewall is +# removed, so no node is ever without one. +hcloud_apply_egress() { + local fw="hcloudtest-node-egress-$$" v4 v6 c4 c6 n count fw_id fw_name srv_id + local -a a4=() a6=() tcp_dst=() + v4=$(hcloud_proxy_addrs A) || fail "could not resolve $HCLOUD_PROXY_HOST (A)" + v6=$(hcloud_proxy_addrs AAAA) || fail "could not resolve $HCLOUD_PROXY_HOST (AAAA)" + [[ -n "$v4" ]] && mapfile -t a4 <<<"$v4" + [[ -n "$v6" ]] && mapfile -t a6 <<<"$v6" + (( ${#a4[@]} + ${#a6[@]} > 0 )) || fail "$HCLOUD_PROXY_HOST resolved to no address — refusing to build the egress firewall" + echo " blocking tcp to $HCLOUD_PROXY_HOST: ${a4[*]} ${a6[*]}" + if (( ${#a4[@]} > 0 )); then + c4=$(cidr_complement "${a4[@]}") || fail "could not compute the IPv4 complement of ${a4[*]}" + else + c4=0.0.0.0/0 + fi + if (( ${#a6[@]} > 0 )); then + c6=$(cidr_complement "${a6[@]}") || fail "could not compute the IPv6 complement of ${a6[*]}" + else + c6=::/0 + fi + mapfile -t tcp_dst <<<"$c4"$'\n'"$c6" + # Effective rules are counted per destination prefix; 0.0.0.0/0 and ::/0 + # carry udp and icmp. + count=$(printf '%s\n' 0.0.0.0/0 ::/0 "${tcp_dst[@]}" | sort -u | grep -c .) + (( count < HCLOUD_FW_RULE_LIMIT )) \ + || fail "the egress firewall needs $count effective rules, Hetzner allows $HCLOUD_FW_RULE_LIMIT per firewall" + hcloud firewall create --name "$fw" --label "$HCLOUD_LABEL" >/dev/null || fail "could not create $fw" + hcloud firewall add-rule "$fw" --direction out --protocol udp --port any \ + --destination-ips 0.0.0.0/0,::/0 --description "udp" >/dev/null || fail "could not add udp to $fw" + hcloud firewall add-rule "$fw" --direction out --protocol icmp \ + --destination-ips 0.0.0.0/0,::/0 --description "icmp" >/dev/null || fail "could not add icmp to $fw" + hcloud firewall add-rule "$fw" --direction out --protocol tcp --port any \ + --destination-ips "$(IFS=,; printf '%s' "${tcp_dst[*]}")" \ + --description "tcp except $HCLOUD_PROXY_HOST" >/dev/null || fail "could not add tcp to $fw" + for n in "${HCLOUD_NODES[@]}"; do + hcloud firewall apply-to-resource "$fw" --type server --server "$n" >/dev/null \ + || fail "could not attach $fw to $n" + done + while read -r fw_id fw_name; do + [[ -n "$fw_id" && "$fw_name" == hcloudtest-node-egress-* && "$fw_name" != "$fw" ]] || continue + while read -r srv_id; do + [[ -n "$srv_id" ]] || continue + hcloud firewall remove-from-resource "$fw_id" --type server --server "$srv_id" >/dev/null \ + || fail "could not detach $fw_name from server $srv_id" + done < <(hcloud firewall describe "$fw_id" -o json | jq -r '.applied_to[] | select(.type == "server") | .server.id') + hcloud firewall delete "$fw_id" >/dev/null || fail "could not delete the previous firewall $fw_name" + echo " removed $fw_name" + done < <(hcloud firewall list -l "$HCLOUD_LABEL" -o noheader -o columns=id,name) + echo " $fw: $count effective rules" +} + +# Runs restore-test/egress-check.yaml in the cluster KUBECONFIG names and fails +# unless every expectation holds. +hcloud_egress_check() { + local addrs phase="" + step 0 "Checking egress containment from a pod..." + addrs=$( { hcloud_proxy_addrs A; hcloud_proxy_addrs AAAA; } | paste -sd' ' -) \ + || fail "could not resolve $HCLOUD_PROXY_HOST" + [[ -n "$addrs" ]] || fail "$HCLOUD_PROXY_HOST resolved to no address" + kubectl delete pod egress-check --ignore-not-found --wait=true >/dev/null + sed "s|PROXY_ADDRS_PLACEHOLDER|$addrs|" "$REPO_ROOT/kubernetes/restore-test/egress-check.yaml" \ + | kubectl apply -f - >/dev/null || fail "could not start the egress check" + for i in $(seq 1 48); do + phase=$(kubectl get pod egress-check -o jsonpath='{.status.phase}') || phase="" + [[ "$phase" == Succeeded || "$phase" == Failed ]] && break + sleep 5 + done + if [[ "$phase" != Succeeded && "$phase" != Failed ]]; then + kubectl describe pod egress-check | sed 's/^/ /' || true + fail "the egress check never finished (phase '${phase:-none}') — do not start Flux" + fi + kubectl logs egress-check | sed 's/^/ /' || true + [[ "$phase" == "Succeeded" ]] || fail "egress containment does not hold — do not start Flux" + kubectl delete pod egress-check --wait=false >/dev/null + ok +} cleanup_hcloudtest() { command -v hcloud >/dev/null || return 0 @@ -490,6 +738,11 @@ cleanup_hcloudtest() { hcloud volume delete "$id" >/dev/null \ || echo " WARNING: volume $id not deleted — check the console" >&2 done + for id in $(hcloud ssh-key list -l "$HCLOUD_LABEL" -o noheader -o columns=id); do + echo " deleting ssh key $id" + hcloud ssh-key delete "$id" >/dev/null \ + || echo " WARNING: ssh key $id not deleted — check the console" >&2 + done # Last: a network with a server still attached cannot be deleted, and a # leftover one takes its name with it, so the next run cannot recreate it. for id in $(hcloud network list -l "$HCLOUD_LABEL" -o noheader -o columns=id); do @@ -499,6 +752,17 @@ cleanup_hcloudtest() { done } +# The restore test's source, as restore-test/flux-root.yaml names it: sets +# restore_url and restore_repo (/). +restore_test_repo() { + command -v yq >/dev/null || fail "yq not found (reads restore-test/flux-root.yaml)" + restore_url=$(yq -r 'select(.kind == "GitRepository" and .metadata.name == "restore-test") | .spec.url' \ + "$REPO_ROOT/kubernetes/restore-test/flux-root.yaml") + [[ "$restore_url" =~ ^ssh://git@github\.com/([^/]+)/([^/]+)\.git$ ]] \ + || fail "restore-test/flux-root.yaml names '$restore_url', not ssh://git@github.com//.git" + restore_repo="${BASH_REMATCH[1]}/${BASH_REMATCH[2]}" +} + # Sweep only. For the case where a previous run was killed hard enough that its # own trap never ran, and for a cron job to call before it starts. if [[ "$PHASE" == "hcloudclean" ]]; then @@ -507,6 +771,18 @@ if [[ "$PHASE" == "hcloudclean" ]]; then require_secrets hcloud-token HCLOUD_TOKEN=$(read_secret hcloud-token); export HCLOUD_TOKEN cleanup_hcloudtest + # The restore test's read-only deploy keys on the GitHub mirror. + if command -v gh >/dev/null; then + restore_test_repo + for key_id in $(gh api "repos/$restore_repo/keys" \ + --jq '.[] | select(.title | startswith("restore-test ")) | .id'); do + echo " deleting deploy key $key_id" + gh api -X DELETE "repos/$restore_repo/keys/$key_id" \ + || echo " WARNING: deploy key $key_id not deleted — check the repo settings" >&2 + done + else + echo " WARNING: gh not found — restore-test deploy keys on the GitHub mirror not checked" >&2 + fi ok exit 0 fi @@ -518,6 +794,8 @@ if [[ "$PHASE" == "hcloudtest" ]]; then command -v yq >/dev/null || fail "yq not found" command -v jq >/dev/null || fail "jq not found (needed to read the SAN pool and server attachments)" command -v curl >/dev/null || fail "curl not found (needed to learn this host's public address)" + command -v bao >/dev/null || fail "bao not found (the Hetzner token is read from OpenBao)" + command -v kubectl >/dev/null || fail "kubectl not found (runs the egress check)" # Three control planes, all carrying workloads — the production shape. One # machine is a node, not a cluster: etcd has no quorum to lose and nothing @@ -536,18 +814,11 @@ if [[ "$PHASE" == "hcloudtest" ]]; then # this one, adopted below. The names are fixed, so duplicates cannot # accumulate. `hcloudclean` is what deletes. - # The Talos snapshot is a prerequisite, not something this builds: making it - # means booting a server into rescue and writing the raw image, which is a - # one-off and needs a different set of permissions. Fail with the recipe - # rather than silently booting Debian and timing out on the Talos API. step 0 "Looking up the Talos snapshot..." talos_image_coords - HCLOUD_IMAGE=$(hcloud image list --type snapshot -l os=talos -o noheader -o columns=id | head -1) - [[ -n "$HCLOUD_IMAGE" ]] || fail "no snapshot labelled os=talos. Create one once — boot any server into rescue, then: - wget -O /tmp/talos.raw.xz https://factory.talos.dev/image/$VM_SCHEMATIC/$VM_VERSION/hcloud-amd64.raw.xz - xz -d -c /tmp/talos.raw.xz | dd of=/dev/sda && sync && shutdown -h now - then snapshot that server in the console and label the snapshot os=talos. - The schematic and version above are this cluster's, read from cluster.yaml." + HCLOUD_IMAGE="" + hcloud_talos_snapshot + [[ -n "$HCLOUD_IMAGE" ]] || fail "no Talos snapshot" echo " snapshot $HCLOUD_IMAGE" # The Talos API is unauthenticated in maintenance mode, which is how @@ -590,26 +861,30 @@ if [[ "$PHASE" == "hcloudtest" ]]; then --source-ips "$fw_src" --description "Kubernetes API, this host only" >/dev/null \ || fail "could not add the Kubernetes API rule" echo " open to $fw_src only" + # Servers held from an earlier run get this firewall first; only then are + # the earlier runs' inbound firewalls detached and deleted, so no server is + # ever without one. Egress firewalls are replaced by hcloud_apply_egress. + while read -r srv; do + [[ -n "$srv" ]] || continue + hcloud firewall apply-to-resource "hcloudtest-$$" --type server --server "$srv" >/dev/null \ + || fail "could not attach this run's firewall to $srv" + done < <(hcloud server list -l "$HCLOUD_LABEL" -o noheader -o columns=name) + while read -r fw_id fw_name; do + [[ -n "$fw_id" && "$fw_name" != "hcloudtest-$$" && "$fw_name" != hcloudtest-node-egress-* ]] || continue + while read -r srv_id; do + [[ -n "$srv_id" ]] || continue + hcloud firewall remove-from-resource "$fw_id" --type server --server "$srv_id" >/dev/null \ + || fail "could not detach $fw_name from server $srv_id" + done < <(hcloud firewall describe "$fw_id" -o json | jq -r '.applied_to[] | select(.type == "server") | .server.id') + hcloud firewall delete "$fw_id" >/dev/null || fail "could not delete the previous firewall $fw_name" + echo " removed $fw_name" + done < <(hcloud firewall list -l "$HCLOUD_LABEL" -o noheader -o columns=id,name) ok - # cx43 — 16 GB, 8 vCPU. Sized from what the workload needs, not from the - # Talos guide, which names cx22 as a minimal example (and that type no - # longer exists; `server create` rejects it outright). + # Default type cpx52 (12 vCPU, 24 GB, 480 GB local disk): the hcloud-N + # patches put a 345GiB OSD partition beside a 60GiB EPHEMERAL on the system + # disk. cax* are ARM and do not match the amd64 schematic. # - # cx23 was tried first, on the reasoning that it is cx22's successor. It has - # 4 GB, 3.2 GB of it allocatable, and the three nodes sat at 59-78% of that - # with the infrastructure skeleton alone — no Ceph OSDs, no OpenBao running, - # not one app. Limits were already at 134-207% of allocatable and both - # kube-controller-manager and kube-scheduler had restarted twice. - # - # The local rehearsal is the reference: it runs a 16GiB control plane and - # three 8GiB workers, 40GiB in total, and needed every step of the way up - # from 6GiB to get there. Three nodes here carry the same workload plus a - # control plane each, so 48GiB total is the comparable figure. - # - # Constraints on the choice: cpx11/cpx21 are US-only and cax* are ARM, which - # would not match the amd64 schematic this cluster installs. That leaves the - # cx*/cpx*2 families in the German locations. # Two segments only, unlike the homelab's split in # NETWORK-traffic-classes.md: here the node address carries management, # cluster-internal and external alike, and Ceph replication has its own. @@ -691,12 +966,7 @@ if [[ "$PHASE" == "hcloudtest" ]]; then echo " $n: reusing the server we already hold" hcloud server rebuild "$n" --image "$HCLOUD_IMAGE" >/dev/null \ || fail "could not rebuild $n back into maintenance mode" - # The firewall is named per run (hcloudtest-$$), so an adopted - # server is still attached to the previous run's. Attach this run's - # too — without it the Talos API is unreachable once the old one is - # cleaned up, and the addresses in it are last run's anyway. - hcloud firewall apply-to-resource "hcloudtest-$$" --type server --server "$n" >/dev/null \ - || fail "could not attach this run's firewall to $n" + # This run's firewall was attached above, before the old ones went. else # Retried, because "error during placement (resource_unavailable)" # is the provider being momentarily out of that type in that @@ -708,13 +978,13 @@ if [[ "$PHASE" == "hcloudtest" ]]; then create_err="" for attempt in $(seq 1 30); do create_err=$(hcloud server create --name "$n" --image "$HCLOUD_IMAGE" \ - --type "${HCLOUD_TYPE:-cx43}" --location "${HCLOUD_LOCATION:-fsn1}" \ + --type "${HCLOUD_TYPE:-cpx52}" --location "${HCLOUD_LOCATION:-fsn1}" \ --firewall "hcloudtest-$$" --label "$HCLOUD_LABEL" 2>&1) && break grep -qi 'resource_unavailable\|placement' <<<"$create_err" \ || fail "could not create $n: ${create_err//$'\n'/ | }" [[ $attempt -eq 1 ]] && echo " $n: no capacity yet, retrying" sleep 20 - [[ $attempt -eq 30 ]] && fail "no capacity for ${HCLOUD_TYPE:-cx43} in ${HCLOUD_LOCATION:-fsn1} after 10 minutes" + [[ $attempt -eq 30 ]] && fail "no capacity for ${HCLOUD_TYPE:-cpx52} in ${HCLOUD_LOCATION:-fsn1} after 10 minutes" done fi addr=$(hcloud server ip "$n") || fail "could not read the address of $n" @@ -757,13 +1027,18 @@ if [[ "$PHASE" == "hcloudtest" ]]; then hcloud volume delete "$n-osd" >/dev/null \ || fail "could not delete the previous OSD volume for $n" fi + yq -e 'select(.kind == "RawVolumeConfig" and .name == "osd-data")' "$TALOS_DIR/patches/$n.yaml" >/dev/null \ + || fail "$n declares no osd-data RawVolumeConfig — a node with no OSD device is not this cluster's shape" + # A raw volume on the system disk needs no Hetzner volume. vol_size=$(hcloud_volumes_for_node "$n") - [[ -n "$vol_size" ]] || fail "$n declares no RawVolumeConfig — a node with no OSD device is not this cluster's shape" - hcloud volume create --name "$n-osd" --size "$vol_size" --server "$n" \ - --label "$HCLOUD_LABEL" >/dev/null \ - || fail "could not create the OSD volume for $n" - - echo " $n at $addr, with a ${vol_size}GB OSD volume" + if [[ -n "$vol_size" ]]; then + hcloud volume create --name "$n-osd" --size "$vol_size" --server "$n" \ + --label "$HCLOUD_LABEL" >/dev/null \ + || fail "could not create the OSD volume for $n" + echo " $n at $addr, with a ${vol_size}GB OSD volume" + else + echo " $n at $addr, OSD on the system disk" + fi NODE_ADDRS+=("$addr") done ok @@ -792,20 +1067,26 @@ if [[ "$PHASE" == "hcloudtest" ]]; then echo " $node_src" ok - # Handover, and this list is the whole of it — the same four variables the - # vmtest phase sets. If it grows, the test is drifting away from production. + # Before any node is configured. + step 0 "Closing tcp egress to $HCLOUD_PROXY_HOST (Hetzner firewall)..." + hcloud_apply_egress + ok + + # Handover: the variables the vmtest phase sets, and HCLOUDTEST. SITE=hcloud NODES=("${HCLOUD_NODES[@]}") WORKERS=() ALL_NODES=("${NODES[@]}") K8S_API_HOST="${NODE_ADDRS[0]}" + HCLOUDTEST=1 # Isolation from the production cluster. Both halves are required: the # _PATH variables are derived before any phase runs, and the talos phase # writes to those. - HCLOUD_DIR=$(mktemp -d) - export TALOSCONFIG="$HCLOUD_DIR/talosconfig" - export KUBECONFIG="$HCLOUD_DIR/kubeconfig" + mkdir -p "$HCLOUD_STATE_DIR" + chmod 700 "$HCLOUD_STATE_DIR" + export TALOSCONFIG="$HCLOUD_STATE_DIR/talosconfig" + export KUBECONFIG="$HCLOUD_STATE_DIR/kubeconfig" TALOSCONFIG_PATH="$TALOSCONFIG" KUBECONFIG_PATH="$KUBECONFIG" @@ -1369,9 +1650,13 @@ EOF done # Stash configs for subsequent phases / export-access step. Skipped under - # vmtest: those are a throwaway cluster's credentials, and writing them here - # would leave the flux phase pointed at VMs that no longer exist. - if [[ "${VMTEST:-0}" == "1" ]]; then + # vmtest and hcloudtest: those are a throwaway cluster's credentials, and a + # later production `bootstrap.sh flux` copies access-staging/ over the + # operator's kubeconfig and talosconfig. + if [[ "${HCLOUDTEST:-0}" == "1" ]]; then + hcloud_egress_check + echo -e "${GREEN}Talos phase complete.${NC} Access files in $HCLOUD_STATE_DIR/ (not staged)" + elif [[ "${VMTEST:-0}" == "1" ]]; then # Nodes being Ready says nothing about storage. The CephCluster picks # OSD nodes by a label out of the machine config and their device by the # partition label RawVolumeConfig creates, so assert both here — running @@ -1620,7 +1905,9 @@ bring_up_openbao() { # entered by mistake. Wait for the container to be running first. echo -n " waiting for ${pods[0]} to answer" local status="" status_err="" - for i in $(seq 1 90); do + # Bounded at 30 minutes: on a fresh cluster the pod waits for Ceph to + # provision its volume. + for i in $(seq 1 180); do if status=$(bao_exec "${pods[0]}" bao status -format=json 2>"$OPENBAO_SECRETS_DIR/.baoerr"); then echo; break fi @@ -1638,7 +1925,7 @@ bring_up_openbao() { if jq -e 'has("initialized")' >/dev/null 2>&1 <<<"$status"; then echo; break; fi printf '.' sleep 10 - if [[ $i -eq 90 ]]; then + if [[ $i -eq 180 ]]; then echo echo " last error: $status_err" >&2 fail "${pods[0]} never answered bao status — refusing to guess whether it is sealed or uninitialised" @@ -1747,9 +2034,33 @@ if phase_runs flux; then case "$FLUX_SOURCE" in gitea) GITEA_TOKEN=$(read_secret git-token); export GITEA_TOKEN ;; github) GITHUB_TOKEN=$(read_secret git-token); export GITHUB_TOKEN ;; - *) fail "Unknown FLUX_SOURCE '$FLUX_SOURCE' (want gitea|github)" ;; + restore-test) + GITHUB_TOKEN=$(read_secret git-token); export GITHUB_TOKEN + command -v gh >/dev/null || fail "gh not found (registers the read-only deploy key)" + command -v jq >/dev/null || fail "jq not found (reads the DoH answer for the egress check)" + # The throwaway cluster carries production secrets, and this phase + # copies access files over KUBECONFIG_PATH and TALOSCONFIG_PATH: + # neither may be the operator's production file. + [[ "$KUBECONFIG_PATH" != "$HOME/.kube/config" && "$TALOSCONFIG_PATH" != "$HOME/.talos/config" ]] \ + || fail "FLUX_SOURCE=restore-test needs KUBECONFIG and TALOSCONFIG pointed at the test cluster ($HCLOUD_STATE_DIR)" + ;; + *) fail "Unknown FLUX_SOURCE '$FLUX_SOURCE' (want gitea|github|restore-test)" ;; esac - [[ "$PHASE" == "flux" ]] && load_staged_access + # The restore test reads its access from KUBECONFIG/TALOSCONFIG, which + # hcloudtest writes; access-staging/ may hold an older run's files. + [[ "$PHASE" == "flux" && "$FLUX_SOURCE" != "restore-test" ]] && load_staged_access + if [[ "$FLUX_SOURCE" == "restore-test" ]]; then + # Whatever the paths say, the cluster itself must be the test cluster, + # before anything below writes a secret to it. + node_names=$(kubectl get nodes -o jsonpath='{.items[*].metadata.name}') \ + || fail "cannot list the nodes of the target cluster" + [[ -n "$node_names" ]] || fail "the target cluster has no nodes" + for n in $node_names; do + [[ "$n" == "$HCLOUD_NAME_PREFIX"-* ]] \ + || fail "node '$n' is not a $HCLOUD_NAME_PREFIX-* test node — refusing the restore test against this cluster" + done + hcloud_egress_check + fi echo -n "Creating SOPS age secret... " kubectl create namespace flux-system --dry-run=client -o yaml | kubectl apply -f - @@ -1789,7 +2100,36 @@ if phase_runs flux; then fi unset seal_key existing - if [[ "$FLUX_SOURCE" == "github" ]]; then + if [[ "$FLUX_SOURCE" == "restore-test" ]]; then + # The production tree through kubernetes/restore-test/flux-root.yaml, + # whose patches are the test's whole deviation from production. Nothing + # is committed anywhere: the source is the GitHub mirror, read through a + # deploy key GitHub itself holds to read-only, so the test cluster + # cannot push (image automation) even with production's manifests. + step 0 "Bootstrapping FluxCD for the restore test (restore-test/flux-root.yaml)..." + flux install --components-extra=image-reflector-controller,image-automation-controller \ + || fail "flux install failed" + restore_test_repo + deploy_key="$OPENBAO_SECRETS_DIR/restore-test-git" + rm -f "$deploy_key" "$deploy_key.pub" + ssh-keygen -q -t ed25519 -N '' -C "restore-test" -f "$deploy_key" + gh api -X POST "repos/$restore_repo/keys" \ + -f title="restore-test $(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + -f key="$(cat "$deploy_key.pub")" -F read_only=true >/dev/null \ + || fail "could not register the read-only deploy key on $restore_repo" + flux create secret git restore-test-git \ + --url="$restore_url" \ + --private-key-file="$deploy_key" >/dev/null \ + || fail "could not create the restore-test-git Secret" + # On FLUX_BRANCH, so a branch can be tried here before it is merged. + # shellcheck disable=SC2016 # $b is a jq variable + yq -y --arg b "$FLUX_BRANCH" \ + 'if .kind == "GitRepository" and .metadata.name == "restore-test" then .spec.ref.branch = $b else . end' \ + "$REPO_ROOT/kubernetes/restore-test/flux-root.yaml" | kubectl apply -f - >/dev/null \ + || fail "could not apply restore-test/flux-root.yaml" + echo " source: $restore_url, branch $FLUX_BRANCH" + ok + elif [[ "$FLUX_SOURCE" == "github" ]]; then step 0 "Bootstrapping FluxCD from the GitHub break-glass mirror..." echo -e "${YELLOW}BREAK-GLASS: Flux source is github.com/${GITHUB_OWNER}/${GITHUB_REPO}, NOT in-cluster Gitea.${NC}" echo -e "${YELLOW}Switch the source back to Gitea once it is serving again (see RUNBOOK-cold-rebuild.md).${NC}" @@ -1893,7 +2233,8 @@ if phase_runs wait-apps; then echo -e "${YELLOW}=== Phase: Wait for apps ===${NC}" command -v kubectl >/dev/null || fail "kubectl not found" command -v flux >/dev/null || fail "flux not found" - [[ "$PHASE" == "wait-apps" ]] && load_staged_access + # The restore test reads its access from KUBECONFIG/TALOSCONFIG. + [[ "$PHASE" == "wait-apps" && "$FLUX_SOURCE" != "restore-test" ]] && load_staged_access # Wait for a Flux Kustomization to become Ready=True. Prints a status line # whenever the Ready/Message pair changes, plus a heartbeat every 60 s so a -- 2.54.0 From b550e84fcc6661b195e1ef51394094e4bdca9e28 Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Fri, 16 Oct 2026 09:05:13 +0200 Subject: [PATCH 08/42] feat(restore-test): Flux root with the test's deviations, and the egress check flux-root.yaml applies the production tree from the GitHub mirror; its patches are the source rewrite, no Ceph cluster network (Hetzner networks are layer 3), and the safety set: Velero ReadOnly, no CNPG WAL archiving, retention or base backups, no MariaDB dumps, rotators suspended, Gitea without world egress. egress-check.yaml asserts from a pod: public DNS, github.com https and ssh, ghcr.io and object storage reachable; every proxy.renner.uno address closed on 443. stderr is reported, not discarded. Co-Authored-By: Claude Opus 5.5 --- kubernetes/restore-test/egress-check.yaml | 36 ++++++ kubernetes/restore-test/flux-root.yaml | 146 ++++++++++++++++++++++ 2 files changed, 182 insertions(+) create mode 100644 kubernetes/restore-test/egress-check.yaml create mode 100644 kubernetes/restore-test/flux-root.yaml diff --git a/kubernetes/restore-test/egress-check.yaml b/kubernetes/restore-test/egress-check.yaml new file mode 100644 index 00000000..d49be521 --- /dev/null +++ b/kubernetes/restore-test/egress-check.yaml @@ -0,0 +1,36 @@ +# Asserts the restore test's containment from a pod, before Flux restores +# production's secrets. Applied by bootstrap.sh (hcloudtest, and +# FLUX_SOURCE=restore-test flux), which substitutes PROXY_ADDRS; exits non-zero +# if any expectation does not hold. +apiVersion: v1 +kind: Pod +metadata: + name: egress-check + namespace: default +spec: + restartPolicy: Never + containers: + - name: check + image: docker.io/nicolaka/netshoot:v0.13 + command: + - /bin/bash + - -c + - | + rc=0 + expect() { # + if [[ "$3" == $2 ]]; then echo "PASS $1: $3"; else echo "FAIL $1: got '$3', want '$2'"; rc=1; fi + } + code() { curl -sS -m 8 -o /dev/null -w '%{http_code} via %{remote_ip}' "$1" 2>&1 || true; } + port() { local out; if out=$(nc -z -w 5 "$1" "$2" 2>&1); then echo open; else echo "closed ${out}"; fi; } + expect "public DNS resolves" "[0-9]*.[0-9]*.[0-9]*.[0-9]*" "$(dig +short example.com A 2>&1 | head -1)" + expect "github.com https reachable" "[23]?? via *" "$(code https://github.com)" + expect "github.com ssh reachable" "open" "$(port github.com 22)" + expect "ghcr.io reachable" "401 via *" "$(code https://ghcr.io/v2/)" + expect "object storage reachable" "[1-4]?? via *" "$(code https://fsn1.your-objectstorage.com)" + for a in $PROXY_ADDRS; do + expect "proxy $a:443 blocked" "closed*" "$(port "$a" 443)" + done + exit $rc + env: + - name: PROXY_ADDRS + value: PROXY_ADDRS_PLACEHOLDER diff --git a/kubernetes/restore-test/flux-root.yaml b/kubernetes/restore-test/flux-root.yaml new file mode 100644 index 00000000..9a0c2d44 --- /dev/null +++ b/kubernetes/restore-test/flux-root.yaml @@ -0,0 +1,146 @@ +# Flux entry point of the off-site restore test (bootstrap.sh hcloudtest, then +# FLUX_SOURCE=restore-test bootstrap.sh flux). Not reconciled by production. +# +# Applies the production tree unchanged. spec.patches below is the complete +# list of differences between the test cluster and production; each is marked +# source, hardware or safety. +apiVersion: source.toolkit.fluxcd.io/v1 +kind: GitRepository +metadata: + name: restore-test + namespace: flux-system +spec: + interval: 10m + url: ssh://git@github.com/konairius/Homelab.git + ref: + branch: master + # Read-only deploy key, created per run by bootstrap.sh. + secretRef: + name: restore-test-git +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: restore-test + namespace: flux-system +spec: + interval: 10m + path: ./kubernetes/clusters/homelab + prune: true + sourceRef: + kind: GitRepository + name: restore-test + patches: + # 1. Source: the GitHub mirror, read-only. In-cluster Gitea is restored by + # this test, not available to it. + - target: + kind: GitRepository + name: flux-system + namespace: flux-system + patch: | + - op: replace + path: /spec/url + value: ssh://git@github.com/konairius/Homelab.git + - op: replace + path: /spec/secretRef/name + value: restore-test-git + # 2. Source: the tree's own root sync is suspended; this Kustomization + # replaces it. + - target: + kind: Kustomization + name: flux-system + namespace: flux-system + patch: | + - op: add + path: /spec/suspend + value: true + # 3. Source: every Kustomization of the tree reads restore-test (the branch + # bootstrap.sh sets) and carries the patches below; a target a + # Kustomization does not contain matches nothing. + - target: + kind: Kustomization + patch: | + - op: replace + path: /spec/sourceRef/name + value: restore-test + - op: add + path: /spec/patches + value: + # 3a. Hardware: no Ceph cluster network. Hetzner networks are + # layer 3 and a macvlan child does not cross servers; Ceph + # replicates over the public (pod) network. + - target: + kind: CephCluster + name: rook-ceph + patch: | + - op: remove + path: /spec/network + # 3b. Safety: Velero ReadOnly blocks backups, deletion, GC and + # repository maintenance; restores and backup sync still work. + - target: + kind: HelmRelease + name: velero + patch: | + - op: add + path: /spec/values/configuration/backupStorageLocation/0/accessMode + value: ReadOnly + # 3c. Safety: CNPG archives no WAL. The plugin stays loaded for + # recovery. + - target: + group: postgresql.cnpg.io + kind: Cluster + patch: | + - op: add + path: /spec/plugins/0/isWALArchiver + value: false + # 3d. Safety: no CNPG retention. The plugin sidecar deletes by + # retentionPolicy whether or not it archives. + - target: + group: barmancloud.cnpg.io + kind: ObjectStore + patch: | + apiVersion: barmancloud.cnpg.io/v1 + kind: ObjectStore + metadata: + name: _ + spec: + retentionPolicy: null + # 3e. Safety: no CNPG base backups. + - target: + group: postgresql.cnpg.io + kind: ScheduledBackup + patch: | + - op: add + path: /spec/suspend + value: true + # 3f. Safety: no MariaDB logical dumps. + - target: + group: k8s.mariadb.com + kind: Backup + patch: | + - op: replace + path: /spec/schedule/suspend + value: true + # 3g. Safety: credential rotators would roll production's tokens + # at their issuers. + - target: + group: batch + kind: CronJob + labelSelector: app.kubernetes.io/component=secret-rotator + patch: | + - op: add + path: /spec/suspend + value: true + # 3h. Safety: Gitea gets no egress beyond the cluster. Its restored + # push mirror force-pushes (--mirror) over the GitHub mirror. + - target: + kind: CiliumNetworkPolicy + name: default-deny + namespace: gitea + patch: | + - op: test + path: /spec/egress/5/toEntities + value: [host, remote-node, world] + - op: replace + path: /spec/egress/5/toEntities + value: [host, remote-node] -- 2.54.0 From f58e06fc76dc1a3aaa97cc8a64e769b3e3840125 Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Fri, 16 Oct 2026 09:07:00 +0200 Subject: [PATCH 09/42] fix(bootstrap): create Cilium's CNI subdirectory and install Multus on a cold cluster Cilium's cni.confPath is /etc/cni/net.d/multus, a directory Multus creates (#832). On an empty cluster neither exists: /etc/cni is read-only to the kubelet on Talos, so the agent's DirectoryOrCreate mount cannot make it, and the install waits forever. Without Multus the kubelet reports "cni plugin not initialized" and nothing schedules, Flux included. A hostNetwork pod per node creates the directory (there is no pod network before Cilium), and Multus is applied from the manifests Flux owns right after Cilium. Found in the Hetzner restore rehearsal, where both steps ran successfully in runs 22 and 23; this is the production bootstrap path. Co-Authored-By: Claude Opus 5.5 --- kubernetes/bootstrap.sh | 68 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/kubernetes/bootstrap.sh b/kubernetes/bootstrap.sh index 1dc820a1..371506d0 100755 --- a/kubernetes/bootstrap.sh +++ b/kubernetes/bootstrap.sh @@ -1602,6 +1602,64 @@ EOF grep -q 'policyMapMax' "$CILIUM_VALUES" \ || fail "extracted Cilium values look wrong (no policyMapMax) — check $CILIUM_HR" + # Cilium's cni.confPath is a subdirectory of /etc/cni/net.d that Multus + # creates (#832). /etc/cni is read-only to the kubelet on Talos, so the + # agent's DirectoryOrCreate mount cannot make it, and on an empty cluster + # Multus is not there yet. A hostNetwork pod per node creates it; there is + # no pod network before Cilium. + cni_conf_dir=$(yq -r '.cni.confPath' "$CILIUM_VALUES") + if [[ "$cni_conf_dir" == /etc/cni/net.d/* ]]; then + step 0 "Creating $cni_conf_dir on every node..." + # kube-controller-manager creates kube-system/default shortly after the API answers. + for i in $(seq 1 30); do + kubectl -n kube-system get serviceaccount default >/dev/null && break + sleep 5 + [[ $i -eq 30 ]] && fail "kube-system/default ServiceAccount never appeared" + done + # A pod bound to a node that has not registered yet is garbage collected. + for node in "${ALL_NODES[@]}"; do + for i in $(seq 1 60); do + kubectl get node "$node" >/dev/null && break + sleep 5 + [[ $i -eq 60 ]] && fail "$node never registered with the API server" + done + done + for node in "${ALL_NODES[@]}"; do + kubectl apply -f - >/dev/null </dev/null \ + || fail "creating $cni_conf_dir on $node did not succeed (kubectl -n kube-system describe pod cni-confdir-$node)" + kubectl -n kube-system delete pod "cni-confdir-$node" >/dev/null + done + ok + fi + helm repo add cilium "$(yq -r 'select(.kind == "HelmRepository") | .spec.url' "$CILIUM_HR")" >/dev/null helm repo update cilium >/dev/null helm upgrade --install cilium cilium/cilium \ @@ -1613,6 +1671,16 @@ EOF rm -f "$CILIUM_VALUES" ok + # Multus is the only writer of /etc/cni/net.d (#832); without it the kubelet + # reports "cni plugin not initialized" and nothing schedules, Flux included. + # Applied from the manifests Flux owns, which adopts them. + step 0 "Installing Multus (the CNI entry point)..." + kubectl apply -k "$REPO_ROOT/kubernetes/infrastructure/multus" >/dev/null \ + || fail "could not apply kubernetes/infrastructure/multus" + kubectl -n kube-system rollout status daemonset/kube-multus-ds-subdir --timeout=10m >/dev/null \ + || fail "Multus never became ready (kubectl -n kube-system get pods -l app=multus)" + ok + step 0 "Waiting for K8s nodes..." expected=${#ALL_NODES[@]} # Before the loop, because 0 == 0 would satisfy its exit condition on the -- 2.54.0 From 4922d8777e9c80cf373227b5bc9a53a9095beebd Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Fri, 16 Oct 2026 09:19:52 +0200 Subject: [PATCH 10/42] fix(bootstrap): split the hcloudtest tcp egress rule at 100 prefixes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Hetzner rejects a firewall rule with more than 100 destination prefixes ("rules[2].destination_ips: value required to be smaller"; the limit is not in the API spec or the FAQ — measured on an unattached firewall: 100 accepted, 101 rejected). Run 24 failed on the single rule carrying all 245 complement prefixes. The tcp allow-list is now one rule per 100. Co-Authored-By: Claude Opus 5.5 --- kubernetes/bootstrap.sh | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/kubernetes/bootstrap.sh b/kubernetes/bootstrap.sh index 371506d0..109314ab 100755 --- a/kubernetes/bootstrap.sh +++ b/kubernetes/bootstrap.sh @@ -475,6 +475,8 @@ HCLOUD_STATE_DIR="${XDG_STATE_HOME:-$HOME/.local/state}/homelab-hcloudtest" HCLOUD_PROXY_HOST="proxy.renner.uno" # Hetzner's limit on effective rules per firewall and direction. HCLOUD_FW_RULE_LIMIT=500 +# Hetzner's limit on prefixes in one rule (not documented; 101 is rejected). +HCLOUD_FW_RULE_PREFIXES=100 # The Talos snapshot this cluster installs from, into HCLOUD_IMAGE. Built when # missing: a throwaway server in rescue mode writes the Image Factory hcloud @@ -663,9 +665,11 @@ hcloud_apply_egress() { --destination-ips 0.0.0.0/0,::/0 --description "udp" >/dev/null || fail "could not add udp to $fw" hcloud firewall add-rule "$fw" --direction out --protocol icmp \ --destination-ips 0.0.0.0/0,::/0 --description "icmp" >/dev/null || fail "could not add icmp to $fw" - hcloud firewall add-rule "$fw" --direction out --protocol tcp --port any \ - --destination-ips "$(IFS=,; printf '%s' "${tcp_dst[*]}")" \ - --description "tcp except $HCLOUD_PROXY_HOST" >/dev/null || fail "could not add tcp to $fw" + for (( n = 0; n < ${#tcp_dst[@]}; n += HCLOUD_FW_RULE_PREFIXES )); do + hcloud firewall add-rule "$fw" --direction out --protocol tcp --port any \ + --destination-ips "$(IFS=,; printf '%s' "${tcp_dst[*]:n:HCLOUD_FW_RULE_PREFIXES}")" \ + --description "tcp except $HCLOUD_PROXY_HOST" >/dev/null || fail "could not add tcp to $fw" + done for n in "${HCLOUD_NODES[@]}"; do hcloud firewall apply-to-resource "$fw" --type server --server "$n" >/dev/null \ || fail "could not attach $fw to $n" -- 2.54.0 From 16b6aee04d0f65e8a6ac8daf8994f7ff781bec01 Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Fri, 16 Oct 2026 09:59:21 +0200 Subject: [PATCH 11/42] feat(cilium): pod routing is a site fact, from the site patch routingMode, autoDirectNodeRoutes and nodePort.directRoutingDevice describe the site's network, not the cluster: at home pod traffic crosses nodes natively on pod0 (VLAN 48, one L2 segment). On Hetzner Cloud that cannot work: networks are layer 3, route only server addresses, and carry IPv4 only (docs.hetzner.com networks FAQ), so cross-node pod traffic had no path (run 25: every DNS query to a CoreDNS pod on another node timed out). Each Talos site patch now carries a cilium-site ConfigMap as an inline manifest with its part of Cilium's values; the HelmRelease reads it via valuesFrom and bootstrap.sh merges it the same way (site values first, HelmRelease values on top, as Flux merges them): - homelab: native, autoDirectNodeRoutes, directRoutingDevice pod0 - hcloud: tunnel (VXLAN) over the node segment - qemu: native on the provisioner's single L2 bridge Production renders unchanged: helm template of cilium 1.20.3 with master's values vs. homelab site values + the new values gives identical manifests (random TLS secrets excluded). talosctl validate passes for k8s-1 (metal) and hcloud-1 (cloud). Rollout order for production: the ConfigMap must exist before this HelmRelease reconciles (valuesFrom is not optional). Talos creates inline manifests on a config change but never updates them; changes go through talosctl upgrade-k8s. Co-Authored-By: Claude Opus 5.5 --- kubernetes/bootstrap.sh | 11 ++++++++- .../infrastructure/cilium/helmrelease.yaml | 23 ++++++++----------- kubernetes/talos/patches/site-hcloud.yaml | 18 +++++++++++++++ kubernetes/talos/patches/site-homelab.yaml | 22 ++++++++++++++++++ kubernetes/talos/patches/site-qemu.yaml | 17 ++++++++++++++ 5 files changed, 76 insertions(+), 15 deletions(-) diff --git a/kubernetes/bootstrap.sh b/kubernetes/bootstrap.sh index 109314ab..95828e08 100755 --- a/kubernetes/bootstrap.sh +++ b/kubernetes/bootstrap.sh @@ -1605,6 +1605,14 @@ EOF yq 'select(.kind == "HelmRelease") | .spec.values' "$CILIUM_HR" > "$CILIUM_VALUES" grep -q 'policyMapMax' "$CILIUM_VALUES" \ || fail "extracted Cilium values look wrong (no policyMapMax) — check $CILIUM_HR" + # The site's part, from the cilium-site ConfigMap in the site patch: the + # HelmRelease reads it through valuesFrom, under its own values. + CILIUM_SITE_VALUES=$(mktemp) + yq -r 'select(.cluster) | .cluster.inlineManifests[] | select(.name == "cilium-site") | .contents' \ + "$TALOS_DIR/patches/site-${SITE}.yaml" \ + | yq -r '.data["values.yaml"]' > "$CILIUM_SITE_VALUES" + grep -q 'routingMode' "$CILIUM_SITE_VALUES" \ + || fail "no cilium-site values (routingMode) in talos/patches/site-${SITE}.yaml" # Cilium's cni.confPath is a subdirectory of /etc/cni/net.d that Multus # creates (#832). /etc/cni is read-only to the kubelet on Talos, so the @@ -1669,10 +1677,11 @@ EOF helm upgrade --install cilium cilium/cilium \ --version "$(yq -r 'select(.kind == "HelmRelease") | .spec.chart.spec.version' "$CILIUM_HR")" \ --namespace kube-system \ + -f "$CILIUM_SITE_VALUES" \ -f "$CILIUM_VALUES" \ --set prometheus.serviceMonitor.enabled=false \ --wait --timeout 10m - rm -f "$CILIUM_VALUES" + rm -f "$CILIUM_VALUES" "$CILIUM_SITE_VALUES" ok # Multus is the only writer of /etc/cni/net.d (#832); without it the kubelet diff --git a/kubernetes/infrastructure/cilium/helmrelease.yaml b/kubernetes/infrastructure/cilium/helmrelease.yaml index cae0738c..2fa38f83 100644 --- a/kubernetes/infrastructure/cilium/helmrelease.yaml +++ b/kubernetes/infrastructure/cilium/helmrelease.yaml @@ -193,17 +193,12 @@ spec: # the supernets here cover the LAN as well, which is the point. ipv4NativeRoutingCIDR: 10.0.0.0/8 ipv6NativeRoutingCIDR: fd00::/8 - # Pod east-west leaves the node on pod0 (VLAN 48) rather than the - # management VLAN. Native routing hands forwarding to the kernel; - # autoDirectNodeRoutes installs the remote PodCIDR routes, which is - # allowed because all three nodes share pod0's L2 segment. - # - # directRoutingDevice is explicit on purpose: with several addressed - # devices Cilium falls back to the one carrying the Kubernetes InternalIP, - # which is mgmt0. - # - # Encapsulation is gone, so the datapath MTU goes 1450 -> 1500. - routingMode: native - autoDirectNodeRoutes: true - nodePort: - directRoutingDevice: pod0 + # How pod traffic crosses between nodes depends on the site's network: + # routingMode, autoDirectNodeRoutes and the direct-routing device come from + # the cilium-site ConfigMap that each Talos site patch carries + # (talos/patches/site-*.yaml). They must not be set in values above, which + # would override it. + valuesFrom: + - kind: ConfigMap + name: cilium-site + valuesKey: values.yaml diff --git a/kubernetes/talos/patches/site-hcloud.yaml b/kubernetes/talos/patches/site-hcloud.yaml index d279887b..86d7f27a 100644 --- a/kubernetes/talos/patches/site-hcloud.yaml +++ b/kubernetes/talos/patches/site-hcloud.yaml @@ -28,3 +28,21 @@ cluster: # shape. The qemu site cannot mirror it: that provisioner gives a second disk # to workers only, so a Ceph OSD forces 1 control plane and 3 workers there. allowSchedulingOnControlPlanes: true + # The site's part of Cilium's values (see site-homelab.yaml). Hetzner + # networks route only server addresses and only IPv4, so pod traffic between + # nodes is tunnelled (VXLAN) over the node segment. + inlineManifests: + - name: cilium-site + contents: | + apiVersion: v1 + kind: ConfigMap + metadata: + name: cilium-site + namespace: kube-system + labels: + reconcile.fluxcd.io/watch: Enabled + data: + values.yaml: | + routingMode: tunnel + tunnelProtocol: vxlan + autoDirectNodeRoutes: false diff --git a/kubernetes/talos/patches/site-homelab.yaml b/kubernetes/talos/patches/site-homelab.yaml index 323b58e8..d9b43eeb 100644 --- a/kubernetes/talos/patches/site-homelab.yaml +++ b/kubernetes/talos/patches/site-homelab.yaml @@ -32,6 +32,28 @@ cluster: etcd: advertisedSubnets: - 10.2.46.0/24 + # The site's part of Cilium's values, read by the cilium HelmRelease + # (valuesFrom) and by bootstrap.sh. Pod east-west leaves on pod0 (VLAN 48): + # native routing, and autoDirectNodeRoutes installs the remote PodCIDR routes + # because all nodes share pod0's L2 segment. directRoutingDevice is explicit: + # otherwise Cilium takes the InternalIP device, mgmt0. Talos only creates + # this ConfigMap; a changed one is synced with talosctl upgrade-k8s. + inlineManifests: + - name: cilium-site + contents: | + apiVersion: v1 + kind: ConfigMap + metadata: + name: cilium-site + namespace: kube-system + labels: + reconcile.fluxcd.io/watch: Enabled + data: + values.yaml: | + routingMode: native + autoDirectNodeRoutes: true + nodePort: + directRoutingDevice: pod0 --- # All pod traffic goes to table 47, and the table decides — longest prefix # match, no exceptions to carve out. bird fills it with the node's attached diff --git a/kubernetes/talos/patches/site-qemu.yaml b/kubernetes/talos/patches/site-qemu.yaml index fd22df2f..0fa1687f 100644 --- a/kubernetes/talos/patches/site-qemu.yaml +++ b/kubernetes/talos/patches/site-qemu.yaml @@ -13,3 +13,20 @@ machine: nodeIP: validSubnets: - 10.5.0.0/24 +cluster: + # The site's part of Cilium's values (see site-homelab.yaml). The nodes share + # the provisioner's bridge, one L2 segment; Cilium picks its only NIC. + inlineManifests: + - name: cilium-site + contents: | + apiVersion: v1 + kind: ConfigMap + metadata: + name: cilium-site + namespace: kube-system + labels: + reconcile.fluxcd.io/watch: Enabled + data: + values.yaml: | + routingMode: native + autoDirectNodeRoutes: true -- 2.54.0 From ed8241384de0455c0a25135912684194c26bed53 Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Fri, 16 Oct 2026 10:14:15 +0200 Subject: [PATCH 12/42] feat(cilium): native-routing CIDRs are a site fact too ipv4/ipv6NativeRoutingCIDR exempt traffic into those ranges from masquerading (Cilium: handed to the kernel "without applying any SNAT"). At home that is the point: 10.0.0.0/8 and fd00::/8 let LAN services see real pod IPs, routed back by the UDM via BGP. On Hetzner the same range covers the node segment 10.90.0.0/16, whose gateway drops packets with a source not assigned to the sending server (uRPF, Hetzner networks FAQ). Run 26: a pod on hcloud-2 reached only its own node's kube-apiserver (10.90.0.3:6443 -> 401); 10.90.0.2 and 10.90.0.4 timed out, the pod's SYNs went to-stack with the pod address. Flux controllers on hcloud-2 crash- looped and flux install failed. The CIDRs move into each site's cilium-site values: homelab and qemu keep 10.0.0.0/8 and fd00::/8, hcloud exempts only the pod CIDRs. Production still renders identically (helm template, TLS secrets excluded); talosctl validate passes for k8s-1 and hcloud-1. Co-Authored-By: Claude Opus 5.5 --- .../infrastructure/cilium/helmrelease.yaml | 20 ++++++------------- kubernetes/talos/patches/site-hcloud.yaml | 6 +++++- kubernetes/talos/patches/site-homelab.yaml | 9 +++++++-- kubernetes/talos/patches/site-qemu.yaml | 2 ++ 4 files changed, 20 insertions(+), 17 deletions(-) diff --git a/kubernetes/infrastructure/cilium/helmrelease.yaml b/kubernetes/infrastructure/cilium/helmrelease.yaml index 2fa38f83..ec50484e 100644 --- a/kubernetes/infrastructure/cilium/helmrelease.yaml +++ b/kubernetes/infrastructure/cilium/helmrelease.yaml @@ -184,20 +184,12 @@ spec: # Cilium's, the sessions re-establish. bgpControlPlane: enabled: true - # Masquerading exception, not a routing statement: pod traffic to these - # ranges leaves unmasqueraded, so LAN services see real pod IPs. That only - # works because BGP advertises the pod CIDRs, which is what gives the UDM a - # route back. The two settings are one decision and have to move together. - # - # Pod CIDRs are 10.244.0.0/16 and fd10:244::/48 (talos/patches/cluster.yaml); - # the supernets here cover the LAN as well, which is the point. - ipv4NativeRoutingCIDR: 10.0.0.0/8 - ipv6NativeRoutingCIDR: fd00::/8 - # How pod traffic crosses between nodes depends on the site's network: - # routingMode, autoDirectNodeRoutes and the direct-routing device come from - # the cilium-site ConfigMap that each Talos site patch carries - # (talos/patches/site-*.yaml). They must not be set in values above, which - # would override it. + # What the site's network can carry decides how pod traffic crosses between + # nodes and which destinations see real pod IPs: routingMode, + # autoDirectNodeRoutes, the direct-routing device and the native-routing + # (masquerade-exempt) CIDRs come from the cilium-site ConfigMap that each + # Talos site patch carries (talos/patches/site-*.yaml). They must not be set + # in values above, which would override it. valuesFrom: - kind: ConfigMap name: cilium-site diff --git a/kubernetes/talos/patches/site-hcloud.yaml b/kubernetes/talos/patches/site-hcloud.yaml index 86d7f27a..fb8debf7 100644 --- a/kubernetes/talos/patches/site-hcloud.yaml +++ b/kubernetes/talos/patches/site-hcloud.yaml @@ -30,7 +30,9 @@ cluster: allowSchedulingOnControlPlanes: true # The site's part of Cilium's values (see site-homelab.yaml). Hetzner # networks route only server addresses and only IPv4, so pod traffic between - # nodes is tunnelled (VXLAN) over the node segment. + # nodes is tunnelled (VXLAN) over the node segment. Their gateway drops any + # source address not assigned to the sending server (uRPF), so only the pod + # CIDRs are exempt from masquerading. inlineManifests: - name: cilium-site contents: | @@ -46,3 +48,5 @@ cluster: routingMode: tunnel tunnelProtocol: vxlan autoDirectNodeRoutes: false + ipv4NativeRoutingCIDR: 10.244.0.0/16 + ipv6NativeRoutingCIDR: fd10:244::/48 diff --git a/kubernetes/talos/patches/site-homelab.yaml b/kubernetes/talos/patches/site-homelab.yaml index d9b43eeb..8a88895e 100644 --- a/kubernetes/talos/patches/site-homelab.yaml +++ b/kubernetes/talos/patches/site-homelab.yaml @@ -36,8 +36,11 @@ cluster: # (valuesFrom) and by bootstrap.sh. Pod east-west leaves on pod0 (VLAN 48): # native routing, and autoDirectNodeRoutes installs the remote PodCIDR routes # because all nodes share pod0's L2 segment. directRoutingDevice is explicit: - # otherwise Cilium takes the InternalIP device, mgmt0. Talos only creates - # this ConfigMap; a changed one is synced with talosctl upgrade-k8s. + # otherwise Cilium takes the InternalIP device, mgmt0. The native-routing + # CIDRs are a masquerading exception: pod traffic to the LAN keeps its pod + # IP, which works because BGP advertises the pod CIDRs to the UDM — the two + # settings move together. Talos only creates this ConfigMap; a changed one + # is synced with talosctl upgrade-k8s. inlineManifests: - name: cilium-site contents: | @@ -54,6 +57,8 @@ cluster: autoDirectNodeRoutes: true nodePort: directRoutingDevice: pod0 + ipv4NativeRoutingCIDR: 10.0.0.0/8 + ipv6NativeRoutingCIDR: fd00::/8 --- # All pod traffic goes to table 47, and the table decides — longest prefix # match, no exceptions to carve out. bird fills it with the node's attached diff --git a/kubernetes/talos/patches/site-qemu.yaml b/kubernetes/talos/patches/site-qemu.yaml index 0fa1687f..7f74f446 100644 --- a/kubernetes/talos/patches/site-qemu.yaml +++ b/kubernetes/talos/patches/site-qemu.yaml @@ -30,3 +30,5 @@ cluster: values.yaml: | routingMode: native autoDirectNodeRoutes: true + ipv4NativeRoutingCIDR: 10.0.0.0/8 + ipv6NativeRoutingCIDR: fd00::/8 -- 2.54.0 From a5c5d0e79d4025ef8a2e638259150e0b45b3e3c3 Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Fri, 16 Oct 2026 10:34:42 +0200 Subject: [PATCH 13/42] fix(flux): infrastructure is Ready only once OpenBao is unsealed The infrastructure Kustomization health-checked the openbao HelmRelease, which is Ready as soon as the chart is installed, sealed or not. On a cold cluster infrastructure-crds then applied the openbao ClusterSecretStore while OpenBao was unreachable and then sealed ("503 Vault is sealed"). External Secrets backs the store off exponentially, so every ExternalSecret stayed "ClusterSecretStore not ready" for minutes after OpenBao was restored and unsealed; Velero's pod could not mount cloud-credentials-hetzner meanwhile (restore rehearsal run 27; ~15 min in run 22). The openbao StatefulSet is now a health check too. Its readiness probe is `bao status`, which fails while sealed, and podManagementPolicy is OrderedReady, so 3/3 ready means every replica is unsealed. No cycle: OpenBao's PVC comes from infrastructure-ceph-cluster, which does not depend on infrastructure. Co-Authored-By: Claude Opus 5.5 --- kubernetes/clusters/homelab/infrastructure.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/kubernetes/clusters/homelab/infrastructure.yaml b/kubernetes/clusters/homelab/infrastructure.yaml index 5f5a4a82..1892cf91 100644 --- a/kubernetes/clusters/homelab/infrastructure.yaml +++ b/kubernetes/clusters/homelab/infrastructure.yaml @@ -40,6 +40,13 @@ spec: kind: HelmRelease name: openbao # the ESO backend ExternalSecrets resolve against namespace: openbao + # Ready only when every replica is unsealed (readiness probe: bao status). + # The HelmRelease is Ready as soon as the chart is installed; the + # ClusterSecretStore in infrastructure-crds must not come up before. + - apiVersion: apps/v1 + kind: StatefulSet + name: openbao + namespace: openbao - apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease name: rook-ceph-operator # storage — apps mount PVCs -- 2.54.0 From 4a4390ef464d5f8936b3b187b8879c3b5c3af490 Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Fri, 16 Oct 2026 11:04:05 +0200 Subject: [PATCH 14/42] feat(dns): the nodes resolve through k8s-gateway at every site, from the first pod on Node-level image pulls of gitea.renner.uno/* go through the node's resolver, not the cluster's. Only site-homelab.yaml pointed it at k8s-gateway (PR #1292); on Hetzner the nodes asked Hetzner's resolvers, got the public proxy (blocked by the test firewall) and fell through to ghcr, which only mirrors konairius/* - bird, netbird-service-sync and review-bot stayed in ImagePullBackOff (restore rehearsal run 27) while the restored Gitea had them. PR #1292 also left a cold-start gap (run 23): until k8s-gateway exists the node's first resolver does not answer, every lookup waits out the 2 s read timeout, CoreDNS gives up first and pods resolve nothing - Flux included. - Each site patch states its node resolver list (ResolverConfig: k8s-gateway first, then the site's resolver: UDM at home, Hetzner's on hcloud) and k8s-gateway's upstream as a k8s-gateway-site ConfigMap (deployment dnsPolicy/dnsConfig), read by the HelmRelease via valuesFrom. The Corefile forwards to the pod's resolv.conf, never to the node's resolver (loop). - bootstrap.sh installs k8s-gateway right after the nodes are Ready, from the HelmRelease under the site values, as Flux merges them; Flux adopts the release. k8s-gateway answers before anything else needs DNS. helm template (chart 4.0.1) renders forward . /etc/resolv.conf with the site's nameservers in dnsConfig for homelab and hcloud. talosctl validate passes for k8s-1 and hcloud-1. Rollout order for production: the k8s-gateway-site ConfigMap must exist before this HelmRelease reconciles (valuesFrom is not optional). Co-Authored-By: Claude Opus 5.5 --- kubernetes/bootstrap.sh | 27 +++++++++++++++++ .../k8s-gateway/helmrelease.yaml | 20 ++++++++----- kubernetes/talos/patches/site-hcloud.yaml | 30 +++++++++++++++++++ kubernetes/talos/patches/site-homelab.yaml | 24 +++++++++++++-- kubernetes/talos/patches/site-qemu.yaml | 15 ++++++++++ 5 files changed, 105 insertions(+), 11 deletions(-) diff --git a/kubernetes/bootstrap.sh b/kubernetes/bootstrap.sh index 95828e08..c3074c54 100755 --- a/kubernetes/bootstrap.sh +++ b/kubernetes/bootstrap.sh @@ -46,6 +46,7 @@ REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)" # needs is read out of this file at run time so there is no second copy of a # version or a value to keep in step. See PROPOSAL-cilium-single-owner.md. CILIUM_HR="$REPO_ROOT/kubernetes/infrastructure/cilium/helmrelease.yaml" +K8SGW_HR="$REPO_ROOT/kubernetes/infrastructure/k8s-gateway/helmrelease.yaml" ACCESS_STAGING="$REPO_ROOT/kubernetes/access-staging" NODES=(k8s-1 k8s-2 k8s-3) # No dedicated workers since the k8s-edge VM was retired (2026-07-02) — @@ -1730,6 +1731,32 @@ EOF fi done + # The nodes' resolver asks k8s-gateway first (ResolverConfig in the site + # patch), and CoreDNS forwards external names to the node. Until k8s-gateway + # answers, every lookup waits out the node's 2 s read timeout and CoreDNS + # gives up first: pods resolve nothing, Flux included. So it is installed + # here, from the values Flux will reconcile (the HelmRelease, under the + # site's k8s-gateway-site values, as valuesFrom merges them), and Flux + # adopts the release. + step 0 "Installing k8s-gateway (the nodes' first resolver)..." + k8sgw_values=$(mktemp) + k8sgw_site_values=$(mktemp) + yq -y 'select(.kind == "HelmRelease") | .spec.values' "$K8SGW_HR" > "$k8sgw_values" + yq -r 'select(.cluster) | .cluster.inlineManifests[] | select(.name == "k8s-gateway-site") | .contents' \ + "$TALOS_DIR/patches/site-${SITE}.yaml" \ + | yq -r '.data["values.yaml"]' > "$k8sgw_site_values" + grep -q 'dnsPolicy' "$k8sgw_site_values" \ + || fail "no k8s-gateway-site values (dnsPolicy) in talos/patches/site-${SITE}.yaml" + helm upgrade --install k8s-gateway \ + "$(yq -r 'select(.kind == "OCIRepository") | .spec.url' "$K8SGW_HR")" \ + --namespace kube-system \ + -f "$k8sgw_site_values" \ + -f "$k8sgw_values" \ + --wait --timeout 5m >/dev/null \ + || fail "k8s-gateway did not come up (kubectl -n kube-system get pods -l app.kubernetes.io/name=k8s-gateway)" + rm -f "$k8sgw_values" "$k8sgw_site_values" + ok + # Stash configs for subsequent phases / export-access step. Skipped under # vmtest and hcloudtest: those are a throwaway cluster's credentials, and a # later production `bootstrap.sh flux` copies access-staging/ over the diff --git a/kubernetes/infrastructure/k8s-gateway/helmrelease.yaml b/kubernetes/infrastructure/k8s-gateway/helmrelease.yaml index 20c99f49..903853ee 100644 --- a/kubernetes/infrastructure/k8s-gateway/helmrelease.yaml +++ b/kubernetes/infrastructure/k8s-gateway/helmrelease.yaml @@ -19,13 +19,10 @@ spec: # `auth.renner.uno`, `audiobooks.renner.uno`, etc. stay inside the # cluster instead of hairpinning through the legacy Caddy VIP. # -# Names not backed by an in-cluster Ingress fall through to the UDM -# (10.2.40.1 / fd00::1), which serves the split-horizon renner.uno zone — -# preserving resolution of LXC/VM-hosted services (gitea, immich, …) and -# physical devices (udm.renner.uno, …) without manual enumeration. -# NOTE: the old Proxmox DNS HA VIP (10.2.40.53) is DECOMMISSIONED; pointing -# here dead-ended non-Ingress resolution and broke ~1374 HA entities on a -# pod restart (2026-07-14). Do not reintroduce it. +# Names not backed by an in-cluster Ingress fall through to the site's +# resolver: the pod's own resolv.conf, which the site patch sets through the +# k8s-gateway-site ConfigMap (deployment.dnsPolicy/dnsConfig). Never the node's +# resolver: the nodes ask k8s-gateway first, which would loop. apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: @@ -95,7 +92,7 @@ spec: - name: loop - name: reload - name: forward - parameters: ". 10.2.40.1 fd00::1" + parameters: ". /etc/resolv.conf" - name: prometheus parameters: "0.0.0.0:9153" serviceType: ClusterIP @@ -106,3 +103,10 @@ spec: # populated end of the service CIDR (kube-dns is 10.96.0.10). clusterIP: 10.96.0.11 ipFamilyPolicy: PreferDualStack + # The site's upstream resolvers (deployment.dnsPolicy/dnsConfig), from the + # k8s-gateway-site ConfigMap each Talos site patch carries. Not set in values + # above, which would override it. + valuesFrom: + - kind: ConfigMap + name: k8s-gateway-site + valuesKey: values.yaml diff --git a/kubernetes/talos/patches/site-hcloud.yaml b/kubernetes/talos/patches/site-hcloud.yaml index fb8debf7..fad5a8f0 100644 --- a/kubernetes/talos/patches/site-hcloud.yaml +++ b/kubernetes/talos/patches/site-hcloud.yaml @@ -50,3 +50,33 @@ cluster: autoDirectNodeRoutes: false ipv4NativeRoutingCIDR: 10.244.0.0/16 ipv6NativeRoutingCIDR: fd10:244::/48 + # The site's upstream for k8s-gateway (its pod resolv.conf): the + # resolvers Hetzner's DHCP hands out. + - name: k8s-gateway-site + contents: | + apiVersion: v1 + kind: ConfigMap + metadata: + name: k8s-gateway-site + namespace: kube-system + labels: + reconcile.fluxcd.io/watch: Enabled + data: + values.yaml: | + deployment: + dnsPolicy: None + dnsConfig: + nameservers: + - 185.12.64.1 + - 185.12.64.2 +--- +# Host DNS asks k8s-gateway first, so containerd resolves in-cluster names +# (gitea.renner.uno) in the cluster; bootstrap.sh installs k8s-gateway right +# after the CNI. Hetzner's resolvers after it. This list replaces the +# nameservers from DHCP. +apiVersion: v1alpha1 +kind: ResolverConfig +nameservers: + - address: 10.96.0.11 + - address: 185.12.64.1 + - address: 185.12.64.2 diff --git a/kubernetes/talos/patches/site-homelab.yaml b/kubernetes/talos/patches/site-homelab.yaml index 8a88895e..b92f7654 100644 --- a/kubernetes/talos/patches/site-homelab.yaml +++ b/kubernetes/talos/patches/site-homelab.yaml @@ -59,6 +59,24 @@ cluster: directRoutingDevice: pod0 ipv4NativeRoutingCIDR: 10.0.0.0/8 ipv6NativeRoutingCIDR: fd00::/8 + # The site's upstream for k8s-gateway (its pod resolv.conf): the UDM. + - name: k8s-gateway-site + contents: | + apiVersion: v1 + kind: ConfigMap + metadata: + name: k8s-gateway-site + namespace: kube-system + labels: + reconcile.fluxcd.io/watch: Enabled + data: + values.yaml: | + deployment: + dnsPolicy: None + dnsConfig: + nameservers: + - 10.2.40.1 + - fd00::1 --- # All pod traffic goes to table 47, and the table decides — longest prefix # match, no exceptions to carve out. bird fills it with the node's attached @@ -90,9 +108,9 @@ src: 10.244.0.0/16 table: "47" --- # Host DNS asks k8s-gateway first, so containerd resolves in-cluster names -# (gitea.renner.uno) in the cluster. The UDM answers while k8s-gateway is not -# reachable yet; Talos moves on after a 2 s read timeout. This list replaces -# the nameservers from DHCP. +# (gitea.renner.uno) in the cluster; bootstrap.sh installs k8s-gateway right +# after the CNI, before anything else needs DNS. The UDM is the site's +# resolver after it. This list replaces the nameservers from DHCP. apiVersion: v1alpha1 kind: ResolverConfig nameservers: diff --git a/kubernetes/talos/patches/site-qemu.yaml b/kubernetes/talos/patches/site-qemu.yaml index 7f74f446..077d9954 100644 --- a/kubernetes/talos/patches/site-qemu.yaml +++ b/kubernetes/talos/patches/site-qemu.yaml @@ -32,3 +32,18 @@ cluster: autoDirectNodeRoutes: true ipv4NativeRoutingCIDR: 10.0.0.0/8 ipv6NativeRoutingCIDR: fd00::/8 + # The nodes keep the provisioner's resolver (no ResolverConfig here), so + # k8s-gateway can use the node's. + - name: k8s-gateway-site + contents: | + apiVersion: v1 + kind: ConfigMap + metadata: + name: k8s-gateway-site + namespace: kube-system + labels: + reconcile.fluxcd.io/watch: Enabled + data: + values.yaml: | + deployment: + dnsPolicy: Default -- 2.54.0 From d25366a80cad6b022a5ebb5b13a01bf974c1f0f1 Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Fri, 16 Oct 2026 11:12:18 +0200 Subject: [PATCH 15/42] fix(bootstrap): read the hcloud node range from the machine document only site-hcloud.yaml now holds a ResolverConfig document too; the unselected query printed "10.90.0.0/16" and "null" and hcloud network create rejected the range (run 28). Co-Authored-By: Claude Opus 5.5 --- kubernetes/bootstrap.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes/bootstrap.sh b/kubernetes/bootstrap.sh index c3074c54..4992571f 100755 --- a/kubernetes/bootstrap.sh +++ b/kubernetes/bootstrap.sh @@ -909,7 +909,7 @@ if [[ "$PHASE" == "hcloudtest" ]]; then seg_range() { case "$1" in cluster) - yq -r '.machine.kubelet.nodeIP.validSubnets[0]' "$TALOS_DIR/patches/site-hcloud.yaml" ;; + yq -r 'select(.machine) | .machine.kubelet.nodeIP.validSubnets[0]' "$TALOS_DIR/patches/site-hcloud.yaml" ;; storage) local pool pool=$(yq -r '.spec.config' "$REPO_ROOT/kubernetes/infrastructure/rook-ceph-cluster/san-network-attachment.yaml" | jq -r '.ipam.range') -- 2.54.0 From abc2a1b0a50669a988b585503e320a78e250b2f9 Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Fri, 16 Oct 2026 11:40:25 +0200 Subject: [PATCH 16/42] fix(flux): apps with in-house images depend on gitea Their images are pulled from Gitea's registry. On a cold cluster they started before the restored Gitea served it and sat in ImagePullBackOff, retried on the kubelet's back-off (restore rehearsal runs 27, 28). They now wait for the gitea Kustomization: apps (ical-proxy), authentik (server, LDAP outpost, gate-sync), home-assistant (shelly-manager, mqtt-account-controller), paperless (ftp-scanner), review-bot, trmnl (renderer), wiki. No cycle: gitea depends only on restore-gitea and the infrastructure layers. The infrastructure Kustomization itself also runs in-house images (bird, netbird-service-sync, secret-rotator, backup-verify, cni-plugins); it cannot depend on gitea, which depends on it, and is left for a separate change. Co-Authored-By: Claude Opus 5.5 --- kubernetes/clusters/homelab/apps.yaml | 1 + kubernetes/clusters/homelab/authentik.yaml | 1 + kubernetes/clusters/homelab/home-assistant.yaml | 1 + kubernetes/clusters/homelab/paperless.yaml | 1 + kubernetes/clusters/homelab/review-bot.yaml | 1 + kubernetes/clusters/homelab/trmnl.yaml | 1 + kubernetes/clusters/homelab/wiki.yaml | 1 + 7 files changed, 7 insertions(+) diff --git a/kubernetes/clusters/homelab/apps.yaml b/kubernetes/clusters/homelab/apps.yaml index 8b83fd76..2d81df20 100644 --- a/kubernetes/clusters/homelab/apps.yaml +++ b/kubernetes/clusters/homelab/apps.yaml @@ -21,3 +21,4 @@ spec: - name: infrastructure - name: infrastructure-crds - name: infrastructure-ceph-cluster + - name: gitea # ical-proxy image is in Gitea's registry diff --git a/kubernetes/clusters/homelab/authentik.yaml b/kubernetes/clusters/homelab/authentik.yaml index fc43a022..c006cafc 100644 --- a/kubernetes/clusters/homelab/authentik.yaml +++ b/kubernetes/clusters/homelab/authentik.yaml @@ -25,6 +25,7 @@ spec: - name: infrastructure-crds - name: infrastructure-ceph-cluster - name: infrastructure-cnpg-barman + - name: gitea # server, LDAP outpost and gate-sync images are in Gitea's registry healthCheckExprs: - apiVersion: postgresql.cnpg.io/v1 kind: Cluster diff --git a/kubernetes/clusters/homelab/home-assistant.yaml b/kubernetes/clusters/homelab/home-assistant.yaml index 521e35ef..2ec4db57 100644 --- a/kubernetes/clusters/homelab/home-assistant.yaml +++ b/kubernetes/clusters/homelab/home-assistant.yaml @@ -42,6 +42,7 @@ spec: # it took speech-to-phrase, zone-configurator and matter-server down with # it, since all three wait on home-assistant. - name: infrastructure-cnpg-barman + - name: gitea # shelly-manager, mqtt-account-controller images are in Gitea's registry --- apiVersion: kustomize.toolkit.fluxcd.io/v1 kind: Kustomization diff --git a/kubernetes/clusters/homelab/paperless.yaml b/kubernetes/clusters/homelab/paperless.yaml index a6796067..f826d07d 100644 --- a/kubernetes/clusters/homelab/paperless.yaml +++ b/kubernetes/clusters/homelab/paperless.yaml @@ -25,6 +25,7 @@ spec: - name: infrastructure-crds - name: infrastructure-ceph-cluster - name: infrastructure-cnpg-barman + - name: gitea # ftp-scanner (vsftpd) image is in Gitea's registry healthCheckExprs: - apiVersion: postgresql.cnpg.io/v1 kind: Cluster diff --git a/kubernetes/clusters/homelab/review-bot.yaml b/kubernetes/clusters/homelab/review-bot.yaml index d5f7264d..083ead44 100644 --- a/kubernetes/clusters/homelab/review-bot.yaml +++ b/kubernetes/clusters/homelab/review-bot.yaml @@ -16,3 +16,4 @@ spec: dependsOn: - name: infrastructure - name: infrastructure-crds + - name: gitea # its image is in Gitea's registry diff --git a/kubernetes/clusters/homelab/trmnl.yaml b/kubernetes/clusters/homelab/trmnl.yaml index b6667b6e..5386779b 100644 --- a/kubernetes/clusters/homelab/trmnl.yaml +++ b/kubernetes/clusters/homelab/trmnl.yaml @@ -22,6 +22,7 @@ spec: - name: infrastructure-crds - name: infrastructure-ceph-cluster - name: infrastructure-cnpg-barman + - name: gitea # renderer image is in Gitea's registry healthCheckExprs: - apiVersion: postgresql.cnpg.io/v1 kind: Cluster diff --git a/kubernetes/clusters/homelab/wiki.yaml b/kubernetes/clusters/homelab/wiki.yaml index 55303de8..2e6c52cd 100644 --- a/kubernetes/clusters/homelab/wiki.yaml +++ b/kubernetes/clusters/homelab/wiki.yaml @@ -20,3 +20,4 @@ spec: - name: infrastructure-ceph-cluster # wiki-embed (the embedding backend) lives in the llama-server app. - name: llama-server + - name: gitea # its image is in Gitea's registry -- 2.54.0 From 36b42677d6959b637b7c9c680b8193c91dd9150a Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Fri, 16 Oct 2026 11:43:34 +0200 Subject: [PATCH 17/42] fix(flux): in-house-image infrastructure waits for Gitea; fallback images don't bird and netbird-service-sync run images from Gitea's registry with no copy on ghcr.io. In the infrastructure Kustomization they started long before the restored Gitea served them and sat in ImagePullBackOff (runs 27, 28). infrastructure cannot depend on gitea, which depends on it, so both move to a new Kustomization, infrastructure-inhouse, that depends on infrastructure, infrastructure-crds and gitea. Their namespace stays in infrastructure. Images published to ghcr.io too have a fallback in the nodes' registry mirror and need no ordering: apps (ical-proxy: GitHub CI pushes ghcr.io/konairius/ical-proxy main-) and trmnl (alpine-jq) lose the gitea edge from the previous commit; multus cni-plugins stays in infrastructure. wiki keeps it: no ghcr copy of konairius/wiki was found. The secret-rotator and backup-verify CronJobs also use Gitea images but only run on schedule; they are left in place. Production rollout must be two-phase (moving objects between pruned Kustomizations): first add infrastructure-inhouse with both paths while infrastructure still lists them, then remove them from infrastructure. Applied as one change, infrastructure prunes bird before the new Kustomization re-creates it. Co-Authored-By: Claude Opus 5.5 --- kubernetes/clusters/homelab/apps.yaml | 3 +-- .../homelab/infrastructure-inhouse.yaml | 19 +++++++++++++++++++ kubernetes/clusters/homelab/trmnl.yaml | 1 - .../bird/bird.conf | 0 .../bird/daemonset.yaml | 0 .../bird/defines.tmpl | 0 .../bird/kustomization.yaml | 0 .../bird/peers.tmpl | 0 .../infrastructure-inhouse/kustomization.yaml | 7 +++++++ .../netbird-service-sync/deployment.yaml | 0 .../netbird-service-sync/kustomization.yaml | 0 kubernetes/infrastructure/kustomization.yaml | 4 ++-- 12 files changed, 29 insertions(+), 5 deletions(-) create mode 100644 kubernetes/clusters/homelab/infrastructure-inhouse.yaml rename kubernetes/{infrastructure => infrastructure-inhouse}/bird/bird.conf (100%) rename kubernetes/{infrastructure => infrastructure-inhouse}/bird/daemonset.yaml (100%) rename kubernetes/{infrastructure => infrastructure-inhouse}/bird/defines.tmpl (100%) rename kubernetes/{infrastructure => infrastructure-inhouse}/bird/kustomization.yaml (100%) rename kubernetes/{infrastructure => infrastructure-inhouse}/bird/peers.tmpl (100%) create mode 100644 kubernetes/infrastructure-inhouse/kustomization.yaml rename kubernetes/{infrastructure => infrastructure-inhouse}/netbird-service-sync/deployment.yaml (100%) rename kubernetes/{infrastructure => infrastructure-inhouse}/netbird-service-sync/kustomization.yaml (100%) diff --git a/kubernetes/clusters/homelab/apps.yaml b/kubernetes/clusters/homelab/apps.yaml index 2d81df20..1ad38e42 100644 --- a/kubernetes/clusters/homelab/apps.yaml +++ b/kubernetes/clusters/homelab/apps.yaml @@ -20,5 +20,4 @@ spec: - name: restore - name: infrastructure - name: infrastructure-crds - - name: infrastructure-ceph-cluster - - name: gitea # ical-proxy image is in Gitea's registry + - name: infrastructure-ceph-cluster \ No newline at end of file diff --git a/kubernetes/clusters/homelab/infrastructure-inhouse.yaml b/kubernetes/clusters/homelab/infrastructure-inhouse.yaml new file mode 100644 index 00000000..a2a10814 --- /dev/null +++ b/kubernetes/clusters/homelab/infrastructure-inhouse.yaml @@ -0,0 +1,19 @@ +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: infrastructure-inhouse + namespace: flux-system +spec: + interval: 10m + retryInterval: 2m + timeout: 10m + sourceRef: + kind: GitRepository + name: flux-system + path: ./kubernetes/infrastructure-inhouse + prune: true + wait: true + dependsOn: + - name: infrastructure + - name: infrastructure-crds + - name: gitea # its images are in Gitea's registry diff --git a/kubernetes/clusters/homelab/trmnl.yaml b/kubernetes/clusters/homelab/trmnl.yaml index 5386779b..b6667b6e 100644 --- a/kubernetes/clusters/homelab/trmnl.yaml +++ b/kubernetes/clusters/homelab/trmnl.yaml @@ -22,7 +22,6 @@ spec: - name: infrastructure-crds - name: infrastructure-ceph-cluster - name: infrastructure-cnpg-barman - - name: gitea # renderer image is in Gitea's registry healthCheckExprs: - apiVersion: postgresql.cnpg.io/v1 kind: Cluster diff --git a/kubernetes/infrastructure/bird/bird.conf b/kubernetes/infrastructure-inhouse/bird/bird.conf similarity index 100% rename from kubernetes/infrastructure/bird/bird.conf rename to kubernetes/infrastructure-inhouse/bird/bird.conf diff --git a/kubernetes/infrastructure/bird/daemonset.yaml b/kubernetes/infrastructure-inhouse/bird/daemonset.yaml similarity index 100% rename from kubernetes/infrastructure/bird/daemonset.yaml rename to kubernetes/infrastructure-inhouse/bird/daemonset.yaml diff --git a/kubernetes/infrastructure/bird/defines.tmpl b/kubernetes/infrastructure-inhouse/bird/defines.tmpl similarity index 100% rename from kubernetes/infrastructure/bird/defines.tmpl rename to kubernetes/infrastructure-inhouse/bird/defines.tmpl diff --git a/kubernetes/infrastructure/bird/kustomization.yaml b/kubernetes/infrastructure-inhouse/bird/kustomization.yaml similarity index 100% rename from kubernetes/infrastructure/bird/kustomization.yaml rename to kubernetes/infrastructure-inhouse/bird/kustomization.yaml diff --git a/kubernetes/infrastructure/bird/peers.tmpl b/kubernetes/infrastructure-inhouse/bird/peers.tmpl similarity index 100% rename from kubernetes/infrastructure/bird/peers.tmpl rename to kubernetes/infrastructure-inhouse/bird/peers.tmpl diff --git a/kubernetes/infrastructure-inhouse/kustomization.yaml b/kubernetes/infrastructure-inhouse/kustomization.yaml new file mode 100644 index 00000000..30d1a476 --- /dev/null +++ b/kubernetes/infrastructure-inhouse/kustomization.yaml @@ -0,0 +1,7 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +# Infrastructure that runs images from Gitea's registry with no copy on +# ghcr.io, so it cannot start before Gitea serves them. +resources: + - bird/ + - netbird-service-sync/ diff --git a/kubernetes/infrastructure/netbird-service-sync/deployment.yaml b/kubernetes/infrastructure-inhouse/netbird-service-sync/deployment.yaml similarity index 100% rename from kubernetes/infrastructure/netbird-service-sync/deployment.yaml rename to kubernetes/infrastructure-inhouse/netbird-service-sync/deployment.yaml diff --git a/kubernetes/infrastructure/netbird-service-sync/kustomization.yaml b/kubernetes/infrastructure-inhouse/netbird-service-sync/kustomization.yaml similarity index 100% rename from kubernetes/infrastructure/netbird-service-sync/kustomization.yaml rename to kubernetes/infrastructure-inhouse/netbird-service-sync/kustomization.yaml diff --git a/kubernetes/infrastructure/kustomization.yaml b/kubernetes/infrastructure/kustomization.yaml index 577755c8..c39814c2 100644 --- a/kubernetes/infrastructure/kustomization.yaml +++ b/kubernetes/infrastructure/kustomization.yaml @@ -7,7 +7,6 @@ resources: - cluster-settings/ - cilium/ - multus/ - - bird/ - cert-manager/ - snapshot-controller/ - metrics-server/ @@ -19,7 +18,6 @@ resources: - cloudnative-pg/ - mariadb-operator/ - netbird-ingress/ - - netbird-service-sync/ - coredns-config/ - k8s-gateway/ - external-dns-unifi/ @@ -39,3 +37,5 @@ resources: # Deployment mounts. Including it here causes the Deployment to block # on missing Secrets while infrastructure itself cannot finish without # the Deployment becoming Ready — circular. +# bird/ and netbird-service-sync/ run images from Gitea's registry and live in +# infrastructure-inhouse/, which waits for Gitea. -- 2.54.0 From 37352808c6eb8eb912549388708b9b3d48d6ddf0 Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Fri, 16 Oct 2026 11:48:16 +0200 Subject: [PATCH 18/42] fix(flux): monitoring depends on influxdb Grafana mounts the grafana-influxdb-token Secret, which a Job in the influxdb app writes (apps/influxdb/grafana-token-job.yaml). On a cold cluster monitoring started first; Grafana sat in CreateContainerConfigError and the monitoring Kustomization held a kustomize-controller worker on its health check (restore rehearsal run 28). No cycle: influxdb does not use monitoring's CRDs. Co-Authored-By: Claude Opus 5.5 --- kubernetes/clusters/homelab/monitoring.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/kubernetes/clusters/homelab/monitoring.yaml b/kubernetes/clusters/homelab/monitoring.yaml index 30547bd4..38ea4190 100644 --- a/kubernetes/clusters/homelab/monitoring.yaml +++ b/kubernetes/clusters/homelab/monitoring.yaml @@ -46,6 +46,7 @@ spec: # from OpenBao via ESO (MIGRATION-sops-openbao.md). dependsOn: - name: grafana-db + - name: influxdb # its Job writes the grafana-influxdb-token Secret Grafana mounts --- apiVersion: kustomize.toolkit.fluxcd.io/v1 kind: Kustomization -- 2.54.0 From 4ebc00e57aeb1f6cf149b0352d9b06b79e3a8a6e Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Fri, 16 Oct 2026 11:57:25 +0200 Subject: [PATCH 19/42] fix(bird): runs only where the site routes pods into table 47 bird fills routing table 47, which exists only where the site patch routes pod traffic into it (site-homelab.yaml RoutingRuleConfig), and renders its config from node annotations that only that site and the k8s-N patches set (pod0, egress gateway). On Hetzner the template rendered and the init container crash-looped ("syntax error, unexpected '<'", run 28). site-homelab.yaml labels its nodes net.renner.uno/pod-table=47 and the DaemonSet selects on it; on a site without table 47 it has no pods. Production rollout order: the label must be on the nodes (talosctl patch, no reboot) before the DaemonSet change reconciles, or bird is removed from every node and pod traffic falls through to the main table. Co-Authored-By: Claude Opus 5.5 --- kubernetes/infrastructure-inhouse/bird/daemonset.yaml | 4 ++++ kubernetes/talos/patches/site-homelab.yaml | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/kubernetes/infrastructure-inhouse/bird/daemonset.yaml b/kubernetes/infrastructure-inhouse/bird/daemonset.yaml index 0a5da93e..c5aea481 100644 --- a/kubernetes/infrastructure-inhouse/bird/daemonset.yaml +++ b/kubernetes/infrastructure-inhouse/bird/daemonset.yaml @@ -51,6 +51,10 @@ spec: # Owns table 47, where all pod traffic is routed. Without it the table # stays empty and the rules fall through to main, unseparated. priorityClassName: system-node-critical + # Table 47 exists only where the site patch routes pod traffic into it + # (RoutingRuleConfig); the same patch labels those nodes. + nodeSelector: + net.renner.uno/pod-table: "47" # Writes host routes and peers on host addresses. hostNetwork: true serviceAccountName: bird diff --git a/kubernetes/talos/patches/site-homelab.yaml b/kubernetes/talos/patches/site-homelab.yaml index b92f7654..32df7d7d 100644 --- a/kubernetes/talos/patches/site-homelab.yaml +++ b/kubernetes/talos/patches/site-homelab.yaml @@ -25,6 +25,10 @@ machine: nodeAnnotations: net.renner.uno/egress-gw-v6: fd02:47::1 net.renner.uno/egress-gw-v4: 10.2.47.1 + # Pod traffic is routed into table 47 here (RoutingRuleConfig below); bird + # fills that table and runs only on nodes with this label. + nodeLabels: + net.renner.uno/pod-table: "47" cluster: # etcd peers on node0, VLAN 46 (#576). Pinned: Talos never updates an # existing member's peer URLs, so changing this subnet means leave + rejoin -- 2.54.0 From 0ac74b7b5be3032bca0753554d76a32e1220d10c Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Fri, 16 Oct 2026 11:57:46 +0200 Subject: [PATCH 20/42] fix(flux): evcc depends on home-assistant evcc reads its MQTT client certificate from mqtt-evcc-client-tls (apps/evcc/deployment.yaml:48-59), which the mqtt-account-controller in the home-assistant Kustomization delivers into the evcc namespace (apps/home-assistant/mqtt-accounts.yaml:40-43), and connects to mosquitto.home-assistant. On a cold cluster evcc started first and sat in CreateContainerConfigError (restore rehearsal runs 27, 28). No cycle: home-assistant does not reach evcc. Co-Authored-By: Claude Opus 5.5 --- kubernetes/clusters/homelab/evcc.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/kubernetes/clusters/homelab/evcc.yaml b/kubernetes/clusters/homelab/evcc.yaml index 1240d82f..cb3eb3b0 100644 --- a/kubernetes/clusters/homelab/evcc.yaml +++ b/kubernetes/clusters/homelab/evcc.yaml @@ -19,3 +19,5 @@ spec: - name: infrastructure - name: infrastructure-crds - name: infrastructure-ceph-cluster + # mqtt-account-controller there delivers mqtt-evcc-client-tls; mosquitto is the broker. + - name: home-assistant -- 2.54.0 From a02affc8725ffeb790f8eba3c15817579c1f568c Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Fri, 16 Oct 2026 12:35:24 +0200 Subject: [PATCH 21/42] fix(velero): one restore Kustomization per schedule; each app waits for its own Every app with backed-up volumes depended on one restore Kustomization that was Ready only after all twelve Velero restores finished. authentik waited for immich's photo library, home-assistant for paperless; in restore rehearsal run 28 that was 45 minutes, during which the restore Kustomization also held one of kustomize-controller's four workers. kubernetes/restore// now holds one Restore each, applied by a restore- Kustomization (clusters/homelab/restore.yaml) with the same health check, after restore-gitea so Gitea's data still goes first. Each app depends on its own: apps, audiobookshelf (libation through it), authentik, home-assistant (matter-server, otbr, whisper too, same namespace and schedule), immich, influxdb, matrix, music-assistant, navidrome, paperless, trmnl, wordpress. Timeouts follow the data: immich 6h, audiobookshelf 3h, the rest 1h. Not on production yet: the restore Kustomizations exist only on this branch, so no Restore objects move between pruned Kustomizations there. Co-Authored-By: Claude Opus 5.5 --- kubernetes/clusters/homelab/apps.yaml | 2 +- .../clusters/homelab/audiobookshelf.yaml | 2 +- kubernetes/clusters/homelab/authentik.yaml | 2 +- .../clusters/homelab/home-assistant.yaml | 8 +- kubernetes/clusters/homelab/immich.yaml | 2 +- kubernetes/clusters/homelab/influxdb.yaml | 2 +- kubernetes/clusters/homelab/matrix.yaml | 2 +- .../clusters/homelab/music-assistant.yaml | 2 +- kubernetes/clusters/homelab/navidrome.yaml | 2 +- kubernetes/clusters/homelab/paperless.yaml | 2 +- kubernetes/clusters/homelab/restore.yaml | 327 +++++++++++++++++- kubernetes/clusters/homelab/trmnl.yaml | 2 +- kubernetes/clusters/homelab/wordpress.yaml | 2 +- .../post-crds/velero-schedules.yaml | 3 +- .../restore/{ => apps}/kustomization.yaml | 2 +- kubernetes/restore/apps/restore.yaml | 7 + .../restore/audiobookshelf/kustomization.yaml | 4 + .../restore/audiobookshelf/restore.yaml | 7 + .../restore/authentik/kustomization.yaml | 4 + kubernetes/restore/authentik/restore.yaml | 7 + .../restore/home-assistant/kustomization.yaml | 4 + .../restore/home-assistant/restore.yaml | 7 + kubernetes/restore/immich/kustomization.yaml | 4 + kubernetes/restore/immich/restore.yaml | 7 + .../restore/influxdb/kustomization.yaml | 4 + kubernetes/restore/influxdb/restore.yaml | 7 + kubernetes/restore/matrix/kustomization.yaml | 4 + kubernetes/restore/matrix/restore.yaml | 7 + .../music-assistant/kustomization.yaml | 4 + .../restore/music-assistant/restore.yaml | 7 + .../restore/navidrome/kustomization.yaml | 4 + kubernetes/restore/navidrome/restore.yaml | 7 + .../restore/paperless/kustomization.yaml | 4 + kubernetes/restore/paperless/restore.yaml | 7 + kubernetes/restore/restores.yaml | 99 ------ kubernetes/restore/trmnl/kustomization.yaml | 4 + kubernetes/restore/trmnl/restore.yaml | 7 + .../restore/wordpress/kustomization.yaml | 4 + kubernetes/restore/wordpress/restore.yaml | 7 + 39 files changed, 464 insertions(+), 125 deletions(-) rename kubernetes/restore/{ => apps}/kustomization.yaml (80%) create mode 100644 kubernetes/restore/apps/restore.yaml create mode 100644 kubernetes/restore/audiobookshelf/kustomization.yaml create mode 100644 kubernetes/restore/audiobookshelf/restore.yaml create mode 100644 kubernetes/restore/authentik/kustomization.yaml create mode 100644 kubernetes/restore/authentik/restore.yaml create mode 100644 kubernetes/restore/home-assistant/kustomization.yaml create mode 100644 kubernetes/restore/home-assistant/restore.yaml create mode 100644 kubernetes/restore/immich/kustomization.yaml create mode 100644 kubernetes/restore/immich/restore.yaml create mode 100644 kubernetes/restore/influxdb/kustomization.yaml create mode 100644 kubernetes/restore/influxdb/restore.yaml create mode 100644 kubernetes/restore/matrix/kustomization.yaml create mode 100644 kubernetes/restore/matrix/restore.yaml create mode 100644 kubernetes/restore/music-assistant/kustomization.yaml create mode 100644 kubernetes/restore/music-assistant/restore.yaml create mode 100644 kubernetes/restore/navidrome/kustomization.yaml create mode 100644 kubernetes/restore/navidrome/restore.yaml create mode 100644 kubernetes/restore/paperless/kustomization.yaml create mode 100644 kubernetes/restore/paperless/restore.yaml delete mode 100644 kubernetes/restore/restores.yaml create mode 100644 kubernetes/restore/trmnl/kustomization.yaml create mode 100644 kubernetes/restore/trmnl/restore.yaml create mode 100644 kubernetes/restore/wordpress/kustomization.yaml create mode 100644 kubernetes/restore/wordpress/restore.yaml diff --git a/kubernetes/clusters/homelab/apps.yaml b/kubernetes/clusters/homelab/apps.yaml index 1ad38e42..88ff04e4 100644 --- a/kubernetes/clusters/homelab/apps.yaml +++ b/kubernetes/clusters/homelab/apps.yaml @@ -17,7 +17,7 @@ spec: # woodpecker migrated) — secrets come from OpenBao via ESO # (MIGRATION-sops-openbao.md). dependsOn: - - name: restore + - name: restore-apps - name: infrastructure - name: infrastructure-crds - name: infrastructure-ceph-cluster \ No newline at end of file diff --git a/kubernetes/clusters/homelab/audiobookshelf.yaml b/kubernetes/clusters/homelab/audiobookshelf.yaml index b81b5668..764dde33 100644 --- a/kubernetes/clusters/homelab/audiobookshelf.yaml +++ b/kubernetes/clusters/homelab/audiobookshelf.yaml @@ -22,7 +22,7 @@ spec: secretRef: name: sops-age dependsOn: - - name: restore + - name: restore-audiobookshelf - name: infrastructure - name: infrastructure-crds - name: infrastructure-ceph-cluster diff --git a/kubernetes/clusters/homelab/authentik.yaml b/kubernetes/clusters/homelab/authentik.yaml index c006cafc..b3650e34 100644 --- a/kubernetes/clusters/homelab/authentik.yaml +++ b/kubernetes/clusters/homelab/authentik.yaml @@ -20,7 +20,7 @@ spec: # from OpenBao via ESO (MIGRATION-sops-openbao.md). This was the LAST # sops consumer; phase 4 (age-key retirement) may proceed after soak. dependsOn: - - name: restore + - name: restore-authentik - name: infrastructure - name: infrastructure-crds - name: infrastructure-ceph-cluster diff --git a/kubernetes/clusters/homelab/home-assistant.yaml b/kubernetes/clusters/homelab/home-assistant.yaml index 2ec4db57..be82d7dd 100644 --- a/kubernetes/clusters/homelab/home-assistant.yaml +++ b/kubernetes/clusters/homelab/home-assistant.yaml @@ -25,7 +25,7 @@ spec: # decryption block dropped 2026-07-13: path is sops-free — secrets come # from OpenBao via ESO (MIGRATION-sops-openbao.md). dependsOn: - - name: restore + - name: restore-home-assistant - name: infrastructure - name: infrastructure-crds - name: infrastructure-ceph-cluster @@ -60,7 +60,7 @@ spec: prune: true wait: true dependsOn: - - name: restore + - name: restore-home-assistant - name: infrastructure - name: infrastructure-crds - name: infrastructure-ceph-cluster @@ -83,7 +83,7 @@ spec: # decryption block dropped 2026-07-13: path is sops-free — secrets come # from OpenBao via ESO (MIGRATION-sops-openbao.md). dependsOn: - - name: restore + - name: restore-home-assistant - name: infrastructure - name: infrastructure-crds --- @@ -124,7 +124,7 @@ spec: # First boot downloads the Whisper model (a few hundred MB) — give it room. wait: true dependsOn: - - name: restore + - name: restore-home-assistant - name: infrastructure - name: infrastructure-crds - name: infrastructure-ceph-cluster diff --git a/kubernetes/clusters/homelab/immich.yaml b/kubernetes/clusters/homelab/immich.yaml index ae8a66be..f3f4d5d4 100644 --- a/kubernetes/clusters/homelab/immich.yaml +++ b/kubernetes/clusters/homelab/immich.yaml @@ -20,7 +20,7 @@ spec: # decryption block dropped 2026-07-13: path is sops-free — secrets come # from OpenBao via ESO (MIGRATION-sops-openbao.md). dependsOn: - - name: restore + - name: restore-immich - name: infrastructure - name: infrastructure-crds - name: infrastructure-ceph-cluster diff --git a/kubernetes/clusters/homelab/influxdb.yaml b/kubernetes/clusters/homelab/influxdb.yaml index 56b66521..e4bc17a7 100644 --- a/kubernetes/clusters/homelab/influxdb.yaml +++ b/kubernetes/clusters/homelab/influxdb.yaml @@ -25,7 +25,7 @@ spec: # boilerplate from the monolithic seed KS — InfluxDB has no CNPG cluster, only # a chart-provisioned ceph-block PVC, so ceph is the only storage dep needed. dependsOn: - - name: restore + - name: restore-influxdb - name: infrastructure - name: infrastructure-crds - name: infrastructure-ceph-cluster diff --git a/kubernetes/clusters/homelab/matrix.yaml b/kubernetes/clusters/homelab/matrix.yaml index 2ad996ff..e4499e01 100644 --- a/kubernetes/clusters/homelab/matrix.yaml +++ b/kubernetes/clusters/homelab/matrix.yaml @@ -17,7 +17,7 @@ spec: prune: true wait: true dependsOn: - - name: restore + - name: restore-matrix - name: infrastructure - name: infrastructure-crds - name: infrastructure-ceph-cluster diff --git a/kubernetes/clusters/homelab/music-assistant.yaml b/kubernetes/clusters/homelab/music-assistant.yaml index 8393523d..dbd995c7 100644 --- a/kubernetes/clusters/homelab/music-assistant.yaml +++ b/kubernetes/clusters/homelab/music-assistant.yaml @@ -22,7 +22,7 @@ spec: secretRef: name: sops-age dependsOn: - - name: restore + - name: restore-music-assistant - name: infrastructure - name: infrastructure-crds - name: infrastructure-ceph-cluster diff --git a/kubernetes/clusters/homelab/navidrome.yaml b/kubernetes/clusters/homelab/navidrome.yaml index 68beb828..782fd751 100644 --- a/kubernetes/clusters/homelab/navidrome.yaml +++ b/kubernetes/clusters/homelab/navidrome.yaml @@ -14,7 +14,7 @@ spec: prune: true wait: true dependsOn: - - name: restore + - name: restore-navidrome - name: infrastructure - name: infrastructure-crds - name: infrastructure-ceph-cluster diff --git a/kubernetes/clusters/homelab/paperless.yaml b/kubernetes/clusters/homelab/paperless.yaml index f826d07d..b346c56e 100644 --- a/kubernetes/clusters/homelab/paperless.yaml +++ b/kubernetes/clusters/homelab/paperless.yaml @@ -20,7 +20,7 @@ spec: # decryption block dropped 2026-07-13: path is sops-free — secrets come # from OpenBao via ESO (MIGRATION-sops-openbao.md). dependsOn: - - name: restore + - name: restore-paperless - name: infrastructure - name: infrastructure-crds - name: infrastructure-ceph-cluster diff --git a/kubernetes/clusters/homelab/restore.yaml b/kubernetes/clusters/homelab/restore.yaml index e892b8ed..e0c09369 100644 --- a/kubernetes/clusters/homelab/restore.yaml +++ b/kubernetes/clusters/homelab/restore.yaml @@ -25,8 +25,13 @@ spec: kind: BackupStorageLocation current: has(status.lastSyncedTime) --- -# Gitea's data first: it is the registry for the in-house images the rest of -# the cluster pulls. The gitea Kustomization depends on this one only. +# One Kustomization per Velero Schedule (infrastructure/post-crds/velero-schedules.yaml), +# each restoring that schedule's newest backup. An app depends on its own, so +# it waits for its data and for nothing else. Gitea's data goes first: it is +# the registry for the in-house images; the others depend on restore-gitea. +# openbao-daily has none: bootstrap.sh restores OpenBao from its raft snapshot. +# Every one carries the same health check: a schedule without any backup has +# nothing to restore (FailedValidation "No backups found for schedule"). apiVersion: kustomize.toolkit.fluxcd.io/v1 kind: Kustomization metadata: @@ -45,7 +50,6 @@ spec: - name: velero-backups-synced - name: infrastructure-ceph-cluster healthCheckExprs: - # A schedule without any backup has nothing to restore. - apiVersion: velero.io/v1 kind: Restore failed: >- @@ -55,20 +59,326 @@ spec: current: >- has(status.phase) && (status.phase == 'Completed' || status.phase == 'FailedValidation') --- -# App data: every other app Kustomization with backed-up volumes depends on -# this one, so no app starts on an empty volume. apiVersion: kustomize.toolkit.fluxcd.io/v1 kind: Kustomization metadata: - name: restore + name: restore-apps namespace: flux-system spec: interval: 10m - timeout: 6h + timeout: 1h sourceRef: kind: GitRepository name: flux-system - path: ./kubernetes/restore + path: ./kubernetes/restore/apps + prune: true + wait: true + dependsOn: + - name: restore-gitea + - name: velero-backups-synced + - name: infrastructure-ceph-cluster + healthCheckExprs: + - apiVersion: velero.io/v1 + kind: Restore + failed: >- + has(status.phase) && (status.phase in ['Failed', 'PartiallyFailed'] || + (status.phase == 'FailedValidation' && !(has(status.validationErrors) && + status.validationErrors.exists(e, e.startsWith('No backups found for schedule'))))) + current: >- + has(status.phase) && (status.phase == 'Completed' || status.phase == 'FailedValidation') +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: restore-audiobookshelf + namespace: flux-system +spec: + interval: 10m + timeout: 3h + sourceRef: + kind: GitRepository + name: flux-system + path: ./kubernetes/restore/audiobookshelf + prune: true + wait: true + dependsOn: + - name: restore-gitea + - name: velero-backups-synced + - name: infrastructure-ceph-cluster + healthCheckExprs: + - apiVersion: velero.io/v1 + kind: Restore + failed: >- + has(status.phase) && (status.phase in ['Failed', 'PartiallyFailed'] || + (status.phase == 'FailedValidation' && !(has(status.validationErrors) && + status.validationErrors.exists(e, e.startsWith('No backups found for schedule'))))) + current: >- + has(status.phase) && (status.phase == 'Completed' || status.phase == 'FailedValidation') +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: restore-authentik + namespace: flux-system +spec: + interval: 10m + timeout: 1h + sourceRef: + kind: GitRepository + name: flux-system + path: ./kubernetes/restore/authentik + prune: true + wait: true + dependsOn: + - name: restore-gitea + - name: velero-backups-synced + - name: infrastructure-ceph-cluster + healthCheckExprs: + - apiVersion: velero.io/v1 + kind: Restore + failed: >- + has(status.phase) && (status.phase in ['Failed', 'PartiallyFailed'] || + (status.phase == 'FailedValidation' && !(has(status.validationErrors) && + status.validationErrors.exists(e, e.startsWith('No backups found for schedule'))))) + current: >- + has(status.phase) && (status.phase == 'Completed' || status.phase == 'FailedValidation') +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: restore-home-assistant + namespace: flux-system +spec: + interval: 10m + timeout: 1h + sourceRef: + kind: GitRepository + name: flux-system + path: ./kubernetes/restore/home-assistant + prune: true + wait: true + dependsOn: + - name: restore-gitea + - name: velero-backups-synced + - name: infrastructure-ceph-cluster + healthCheckExprs: + - apiVersion: velero.io/v1 + kind: Restore + failed: >- + has(status.phase) && (status.phase in ['Failed', 'PartiallyFailed'] || + (status.phase == 'FailedValidation' && !(has(status.validationErrors) && + status.validationErrors.exists(e, e.startsWith('No backups found for schedule'))))) + current: >- + has(status.phase) && (status.phase == 'Completed' || status.phase == 'FailedValidation') +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: restore-immich + namespace: flux-system +spec: + interval: 10m + timeout: 6h + sourceRef: + kind: GitRepository + name: flux-system + path: ./kubernetes/restore/immich + prune: true + wait: true + dependsOn: + - name: restore-gitea + - name: velero-backups-synced + - name: infrastructure-ceph-cluster + healthCheckExprs: + - apiVersion: velero.io/v1 + kind: Restore + failed: >- + has(status.phase) && (status.phase in ['Failed', 'PartiallyFailed'] || + (status.phase == 'FailedValidation' && !(has(status.validationErrors) && + status.validationErrors.exists(e, e.startsWith('No backups found for schedule'))))) + current: >- + has(status.phase) && (status.phase == 'Completed' || status.phase == 'FailedValidation') +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: restore-influxdb + namespace: flux-system +spec: + interval: 10m + timeout: 1h + sourceRef: + kind: GitRepository + name: flux-system + path: ./kubernetes/restore/influxdb + prune: true + wait: true + dependsOn: + - name: restore-gitea + - name: velero-backups-synced + - name: infrastructure-ceph-cluster + healthCheckExprs: + - apiVersion: velero.io/v1 + kind: Restore + failed: >- + has(status.phase) && (status.phase in ['Failed', 'PartiallyFailed'] || + (status.phase == 'FailedValidation' && !(has(status.validationErrors) && + status.validationErrors.exists(e, e.startsWith('No backups found for schedule'))))) + current: >- + has(status.phase) && (status.phase == 'Completed' || status.phase == 'FailedValidation') +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: restore-matrix + namespace: flux-system +spec: + interval: 10m + timeout: 1h + sourceRef: + kind: GitRepository + name: flux-system + path: ./kubernetes/restore/matrix + prune: true + wait: true + dependsOn: + - name: restore-gitea + - name: velero-backups-synced + - name: infrastructure-ceph-cluster + healthCheckExprs: + - apiVersion: velero.io/v1 + kind: Restore + failed: >- + has(status.phase) && (status.phase in ['Failed', 'PartiallyFailed'] || + (status.phase == 'FailedValidation' && !(has(status.validationErrors) && + status.validationErrors.exists(e, e.startsWith('No backups found for schedule'))))) + current: >- + has(status.phase) && (status.phase == 'Completed' || status.phase == 'FailedValidation') +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: restore-music-assistant + namespace: flux-system +spec: + interval: 10m + timeout: 1h + sourceRef: + kind: GitRepository + name: flux-system + path: ./kubernetes/restore/music-assistant + prune: true + wait: true + dependsOn: + - name: restore-gitea + - name: velero-backups-synced + - name: infrastructure-ceph-cluster + healthCheckExprs: + - apiVersion: velero.io/v1 + kind: Restore + failed: >- + has(status.phase) && (status.phase in ['Failed', 'PartiallyFailed'] || + (status.phase == 'FailedValidation' && !(has(status.validationErrors) && + status.validationErrors.exists(e, e.startsWith('No backups found for schedule'))))) + current: >- + has(status.phase) && (status.phase == 'Completed' || status.phase == 'FailedValidation') +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: restore-navidrome + namespace: flux-system +spec: + interval: 10m + timeout: 1h + sourceRef: + kind: GitRepository + name: flux-system + path: ./kubernetes/restore/navidrome + prune: true + wait: true + dependsOn: + - name: restore-gitea + - name: velero-backups-synced + - name: infrastructure-ceph-cluster + healthCheckExprs: + - apiVersion: velero.io/v1 + kind: Restore + failed: >- + has(status.phase) && (status.phase in ['Failed', 'PartiallyFailed'] || + (status.phase == 'FailedValidation' && !(has(status.validationErrors) && + status.validationErrors.exists(e, e.startsWith('No backups found for schedule'))))) + current: >- + has(status.phase) && (status.phase == 'Completed' || status.phase == 'FailedValidation') +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: restore-paperless + namespace: flux-system +spec: + interval: 10m + timeout: 1h + sourceRef: + kind: GitRepository + name: flux-system + path: ./kubernetes/restore/paperless + prune: true + wait: true + dependsOn: + - name: restore-gitea + - name: velero-backups-synced + - name: infrastructure-ceph-cluster + healthCheckExprs: + - apiVersion: velero.io/v1 + kind: Restore + failed: >- + has(status.phase) && (status.phase in ['Failed', 'PartiallyFailed'] || + (status.phase == 'FailedValidation' && !(has(status.validationErrors) && + status.validationErrors.exists(e, e.startsWith('No backups found for schedule'))))) + current: >- + has(status.phase) && (status.phase == 'Completed' || status.phase == 'FailedValidation') +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: restore-trmnl + namespace: flux-system +spec: + interval: 10m + timeout: 1h + sourceRef: + kind: GitRepository + name: flux-system + path: ./kubernetes/restore/trmnl + prune: true + wait: true + dependsOn: + - name: restore-gitea + - name: velero-backups-synced + - name: infrastructure-ceph-cluster + healthCheckExprs: + - apiVersion: velero.io/v1 + kind: Restore + failed: >- + has(status.phase) && (status.phase in ['Failed', 'PartiallyFailed'] || + (status.phase == 'FailedValidation' && !(has(status.validationErrors) && + status.validationErrors.exists(e, e.startsWith('No backups found for schedule'))))) + current: >- + has(status.phase) && (status.phase == 'Completed' || status.phase == 'FailedValidation') +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: restore-wordpress + namespace: flux-system +spec: + interval: 10m + timeout: 1h + sourceRef: + kind: GitRepository + name: flux-system + path: ./kubernetes/restore/wordpress prune: true wait: true dependsOn: @@ -76,7 +386,6 @@ spec: - name: velero-backups-synced - name: infrastructure-ceph-cluster healthCheckExprs: - # A schedule without any backup has nothing to restore. - apiVersion: velero.io/v1 kind: Restore failed: >- diff --git a/kubernetes/clusters/homelab/trmnl.yaml b/kubernetes/clusters/homelab/trmnl.yaml index b6667b6e..80efb8fb 100644 --- a/kubernetes/clusters/homelab/trmnl.yaml +++ b/kubernetes/clusters/homelab/trmnl.yaml @@ -17,7 +17,7 @@ spec: prune: true wait: true dependsOn: - - name: restore + - name: restore-trmnl - name: infrastructure - name: infrastructure-crds - name: infrastructure-ceph-cluster diff --git a/kubernetes/clusters/homelab/wordpress.yaml b/kubernetes/clusters/homelab/wordpress.yaml index 3ebc4593..7ea5c93d 100644 --- a/kubernetes/clusters/homelab/wordpress.yaml +++ b/kubernetes/clusters/homelab/wordpress.yaml @@ -13,7 +13,7 @@ spec: path: ./kubernetes/apps/wordpress prune: true dependsOn: - - name: restore + - name: restore-wordpress # infrastructure gates on the mariadb-operator HelmRelease being Ready, # so the MariaDB CRD exists and has a controller before this applies. - name: infrastructure diff --git a/kubernetes/infrastructure/post-crds/velero-schedules.yaml b/kubernetes/infrastructure/post-crds/velero-schedules.yaml index 104ac235..ccea8bc6 100644 --- a/kubernetes/infrastructure/post-crds/velero-schedules.yaml +++ b/kubernetes/infrastructure/post-crds/velero-schedules.yaml @@ -9,7 +9,8 @@ # snapshotMoveData uploads PVC contents through Kopia and the node-agent, # independently of the K8s API surface. # -# Naming: -daily. Each has a Restore in kubernetes/restore/, which a fresh +# Naming: -daily. Each has a Restore in kubernetes/restore// and a +# restore- Kustomization (clusters/homelab/restore.yaml), which a fresh # cluster runs before the app starts. # Retention matches the CNPG Barman policy. diff --git a/kubernetes/restore/kustomization.yaml b/kubernetes/restore/apps/kustomization.yaml similarity index 80% rename from kubernetes/restore/kustomization.yaml rename to kubernetes/restore/apps/kustomization.yaml index ba89ae7b..bc08b2d5 100644 --- a/kubernetes/restore/kustomization.yaml +++ b/kubernetes/restore/apps/kustomization.yaml @@ -1,4 +1,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - - restores.yaml + - restore.yaml diff --git a/kubernetes/restore/apps/restore.yaml b/kubernetes/restore/apps/restore.yaml new file mode 100644 index 00000000..9bb5a23b --- /dev/null +++ b/kubernetes/restore/apps/restore.yaml @@ -0,0 +1,7 @@ +apiVersion: velero.io/v1 +kind: Restore +metadata: + name: apps + namespace: velero +spec: + scheduleName: apps-daily diff --git a/kubernetes/restore/audiobookshelf/kustomization.yaml b/kubernetes/restore/audiobookshelf/kustomization.yaml new file mode 100644 index 00000000..bc08b2d5 --- /dev/null +++ b/kubernetes/restore/audiobookshelf/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - restore.yaml diff --git a/kubernetes/restore/audiobookshelf/restore.yaml b/kubernetes/restore/audiobookshelf/restore.yaml new file mode 100644 index 00000000..15b6b741 --- /dev/null +++ b/kubernetes/restore/audiobookshelf/restore.yaml @@ -0,0 +1,7 @@ +apiVersion: velero.io/v1 +kind: Restore +metadata: + name: audiobookshelf + namespace: velero +spec: + scheduleName: audiobookshelf-daily diff --git a/kubernetes/restore/authentik/kustomization.yaml b/kubernetes/restore/authentik/kustomization.yaml new file mode 100644 index 00000000..bc08b2d5 --- /dev/null +++ b/kubernetes/restore/authentik/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - restore.yaml diff --git a/kubernetes/restore/authentik/restore.yaml b/kubernetes/restore/authentik/restore.yaml new file mode 100644 index 00000000..88bb44d6 --- /dev/null +++ b/kubernetes/restore/authentik/restore.yaml @@ -0,0 +1,7 @@ +apiVersion: velero.io/v1 +kind: Restore +metadata: + name: authentik + namespace: velero +spec: + scheduleName: authentik-daily diff --git a/kubernetes/restore/home-assistant/kustomization.yaml b/kubernetes/restore/home-assistant/kustomization.yaml new file mode 100644 index 00000000..bc08b2d5 --- /dev/null +++ b/kubernetes/restore/home-assistant/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - restore.yaml diff --git a/kubernetes/restore/home-assistant/restore.yaml b/kubernetes/restore/home-assistant/restore.yaml new file mode 100644 index 00000000..72f4e6b0 --- /dev/null +++ b/kubernetes/restore/home-assistant/restore.yaml @@ -0,0 +1,7 @@ +apiVersion: velero.io/v1 +kind: Restore +metadata: + name: home-assistant + namespace: velero +spec: + scheduleName: home-assistant-daily diff --git a/kubernetes/restore/immich/kustomization.yaml b/kubernetes/restore/immich/kustomization.yaml new file mode 100644 index 00000000..bc08b2d5 --- /dev/null +++ b/kubernetes/restore/immich/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - restore.yaml diff --git a/kubernetes/restore/immich/restore.yaml b/kubernetes/restore/immich/restore.yaml new file mode 100644 index 00000000..1c31c184 --- /dev/null +++ b/kubernetes/restore/immich/restore.yaml @@ -0,0 +1,7 @@ +apiVersion: velero.io/v1 +kind: Restore +metadata: + name: immich + namespace: velero +spec: + scheduleName: immich-daily diff --git a/kubernetes/restore/influxdb/kustomization.yaml b/kubernetes/restore/influxdb/kustomization.yaml new file mode 100644 index 00000000..bc08b2d5 --- /dev/null +++ b/kubernetes/restore/influxdb/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - restore.yaml diff --git a/kubernetes/restore/influxdb/restore.yaml b/kubernetes/restore/influxdb/restore.yaml new file mode 100644 index 00000000..9a9b2d0b --- /dev/null +++ b/kubernetes/restore/influxdb/restore.yaml @@ -0,0 +1,7 @@ +apiVersion: velero.io/v1 +kind: Restore +metadata: + name: influxdb + namespace: velero +spec: + scheduleName: influxdb-daily diff --git a/kubernetes/restore/matrix/kustomization.yaml b/kubernetes/restore/matrix/kustomization.yaml new file mode 100644 index 00000000..bc08b2d5 --- /dev/null +++ b/kubernetes/restore/matrix/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - restore.yaml diff --git a/kubernetes/restore/matrix/restore.yaml b/kubernetes/restore/matrix/restore.yaml new file mode 100644 index 00000000..83b5b34f --- /dev/null +++ b/kubernetes/restore/matrix/restore.yaml @@ -0,0 +1,7 @@ +apiVersion: velero.io/v1 +kind: Restore +metadata: + name: matrix + namespace: velero +spec: + scheduleName: matrix-daily diff --git a/kubernetes/restore/music-assistant/kustomization.yaml b/kubernetes/restore/music-assistant/kustomization.yaml new file mode 100644 index 00000000..bc08b2d5 --- /dev/null +++ b/kubernetes/restore/music-assistant/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - restore.yaml diff --git a/kubernetes/restore/music-assistant/restore.yaml b/kubernetes/restore/music-assistant/restore.yaml new file mode 100644 index 00000000..4c5b2643 --- /dev/null +++ b/kubernetes/restore/music-assistant/restore.yaml @@ -0,0 +1,7 @@ +apiVersion: velero.io/v1 +kind: Restore +metadata: + name: music-assistant + namespace: velero +spec: + scheduleName: music-assistant-daily diff --git a/kubernetes/restore/navidrome/kustomization.yaml b/kubernetes/restore/navidrome/kustomization.yaml new file mode 100644 index 00000000..bc08b2d5 --- /dev/null +++ b/kubernetes/restore/navidrome/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - restore.yaml diff --git a/kubernetes/restore/navidrome/restore.yaml b/kubernetes/restore/navidrome/restore.yaml new file mode 100644 index 00000000..4b8a9e1f --- /dev/null +++ b/kubernetes/restore/navidrome/restore.yaml @@ -0,0 +1,7 @@ +apiVersion: velero.io/v1 +kind: Restore +metadata: + name: navidrome + namespace: velero +spec: + scheduleName: navidrome-daily diff --git a/kubernetes/restore/paperless/kustomization.yaml b/kubernetes/restore/paperless/kustomization.yaml new file mode 100644 index 00000000..bc08b2d5 --- /dev/null +++ b/kubernetes/restore/paperless/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - restore.yaml diff --git a/kubernetes/restore/paperless/restore.yaml b/kubernetes/restore/paperless/restore.yaml new file mode 100644 index 00000000..9daaaf0e --- /dev/null +++ b/kubernetes/restore/paperless/restore.yaml @@ -0,0 +1,7 @@ +apiVersion: velero.io/v1 +kind: Restore +metadata: + name: paperless + namespace: velero +spec: + scheduleName: paperless-daily diff --git a/kubernetes/restore/restores.yaml b/kubernetes/restore/restores.yaml deleted file mode 100644 index 5337352e..00000000 --- a/kubernetes/restore/restores.yaml +++ /dev/null @@ -1,99 +0,0 @@ -# One Restore per Velero Schedule (infrastructure/post-crds/velero-schedules.yaml), -# from its newest completed backup. A claim that already exists is skipped. -# openbao-daily has none: bootstrap.sh restores OpenBao from its raft snapshot. -# gitea-daily is restored first, by restore-gitea (gitea/restore.yaml). -apiVersion: velero.io/v1 -kind: Restore -metadata: - name: paperless - namespace: velero -spec: - scheduleName: paperless-daily ---- -apiVersion: velero.io/v1 -kind: Restore -metadata: - name: audiobookshelf - namespace: velero -spec: - scheduleName: audiobookshelf-daily ---- -apiVersion: velero.io/v1 -kind: Restore -metadata: - name: authentik - namespace: velero -spec: - scheduleName: authentik-daily ---- -apiVersion: velero.io/v1 -kind: Restore -metadata: - name: music-assistant - namespace: velero -spec: - scheduleName: music-assistant-daily ---- -apiVersion: velero.io/v1 -kind: Restore -metadata: - name: immich - namespace: velero -spec: - scheduleName: immich-daily ---- -apiVersion: velero.io/v1 -kind: Restore -metadata: - name: influxdb - namespace: velero -spec: - scheduleName: influxdb-daily ---- -apiVersion: velero.io/v1 -kind: Restore -metadata: - name: home-assistant - namespace: velero -spec: - scheduleName: home-assistant-daily ---- -apiVersion: velero.io/v1 -kind: Restore -metadata: - name: apps - namespace: velero -spec: - scheduleName: apps-daily ---- -apiVersion: velero.io/v1 -kind: Restore -metadata: - name: trmnl - namespace: velero -spec: - scheduleName: trmnl-daily ---- -apiVersion: velero.io/v1 -kind: Restore -metadata: - name: wordpress - namespace: velero -spec: - scheduleName: wordpress-daily ---- -apiVersion: velero.io/v1 -kind: Restore -metadata: - name: navidrome - namespace: velero -spec: - scheduleName: navidrome-daily ---- -apiVersion: velero.io/v1 -kind: Restore -metadata: - name: matrix - namespace: velero -spec: - scheduleName: matrix-daily diff --git a/kubernetes/restore/trmnl/kustomization.yaml b/kubernetes/restore/trmnl/kustomization.yaml new file mode 100644 index 00000000..bc08b2d5 --- /dev/null +++ b/kubernetes/restore/trmnl/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - restore.yaml diff --git a/kubernetes/restore/trmnl/restore.yaml b/kubernetes/restore/trmnl/restore.yaml new file mode 100644 index 00000000..dda59430 --- /dev/null +++ b/kubernetes/restore/trmnl/restore.yaml @@ -0,0 +1,7 @@ +apiVersion: velero.io/v1 +kind: Restore +metadata: + name: trmnl + namespace: velero +spec: + scheduleName: trmnl-daily diff --git a/kubernetes/restore/wordpress/kustomization.yaml b/kubernetes/restore/wordpress/kustomization.yaml new file mode 100644 index 00000000..bc08b2d5 --- /dev/null +++ b/kubernetes/restore/wordpress/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - restore.yaml diff --git a/kubernetes/restore/wordpress/restore.yaml b/kubernetes/restore/wordpress/restore.yaml new file mode 100644 index 00000000..b1725f6e --- /dev/null +++ b/kubernetes/restore/wordpress/restore.yaml @@ -0,0 +1,7 @@ +apiVersion: velero.io/v1 +kind: Restore +metadata: + name: wordpress + namespace: velero +spec: + scheduleName: wordpress-daily -- 2.54.0 From f3ba845e52c7bbadbb5b8ffca28dacb116c0bee8 Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Fri, 16 Oct 2026 12:43:19 +0200 Subject: [PATCH 22/42] fix(restore-test): no writes to production NetBird from the test cluster netbird-service-sync in the test cluster reconciled production NetBird (netbird.renner.uno) with the PAT from the restored OpenBao: it rewrote the public proxy services to the test cluster's Traefik ClusterIP, and production's own controller re-created 14 and updated 5 of them every minute from 09:53 to 10:39 UTC during run 28. The netbird-ingress routing peers likewise joined the production mesh, announcing the same service and LoadBalancer ranges as production. Two safety patches in flux-root.yaml: netbird-service-sync runs with zero replicas (3i), and netbird-ingress gets a nodeSelector no node carries (3j). Co-Authored-By: Claude Opus 5.5 --- kubernetes/restore-test/flux-root.yaml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/kubernetes/restore-test/flux-root.yaml b/kubernetes/restore-test/flux-root.yaml index 9a0c2d44..9380e155 100644 --- a/kubernetes/restore-test/flux-root.yaml +++ b/kubernetes/restore-test/flux-root.yaml @@ -144,3 +144,26 @@ spec: - op: replace path: /spec/egress/5/toEntities value: [host, remote-node] + # 3i. Safety: no writes to production NetBird. netbird-service-sync + # rewrites the public proxy services at netbird.renner.uno to + # this cluster's Traefik, fighting production's own controller. + - target: + kind: Deployment + name: netbird-service-sync + namespace: netbird-service-sync + patch: | + - op: replace + path: /spec/replicas + value: 0 + # 3j. Safety: no routing peers in the production mesh. They would + # announce this cluster's service and LoadBalancer ranges, the + # same ranges production announces. + - target: + kind: DaemonSet + name: netbird-ingress + namespace: ingress + patch: | + - op: add + path: /spec/template/spec/nodeSelector + value: + restore-test.renner.uno/never: "true" -- 2.54.0 From e9ca0c4e90df0b36df23c38772d5e78d3ff83544 Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Fri, 16 Oct 2026 12:44:26 +0200 Subject: [PATCH 23/42] fix(mqtt-account-controller): start without the HA client certificate The controller mounted mqtt-home-assistant-tls as a required volume, and is the only creator of that Secret (it applies the cert-manager Certificates of the register's cert accounts). On a new cluster - Velero restores volumes, not Secrets - its pod could never start, and mosquitto, Home Assistant, Zigbee2MQTT and evcc stayed in ContainerCreating behind the Secrets it would have created. Optional now. Verified in konairius/mqtt-account-controller main: the cert dir is read only by the HA cert sync (02:00-05:00 window, errors logged); the broker is dialled lazily; Secret.Read treats a missing Secret as empty; Reconcile runs reconcileCerts even when the broker step fails. So the controller starts, creates the Certificates, the consumers start, and the next loop reconciles the broker. Co-Authored-By: Claude Opus 5.5 --- kubernetes/apps/home-assistant/mqtt-account-controller.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/kubernetes/apps/home-assistant/mqtt-account-controller.yaml b/kubernetes/apps/home-assistant/mqtt-account-controller.yaml index 145dc29e..a1be2ac8 100644 --- a/kubernetes/apps/home-assistant/mqtt-account-controller.yaml +++ b/kubernetes/apps/home-assistant/mqtt-account-controller.yaml @@ -305,10 +305,13 @@ spec: - key: api-key path: api-key # Client certificate HA gets (cert-manager renews it in place; not a - # subPath mount, so the next loop sees the new file). + # subPath mount, so the next loop sees the new file). Optional: this + # controller creates its Certificate, so on a new cluster the Secret + # appears only after the controller has started. - name: ha-mqtt-tls secret: secretName: mqtt-home-assistant-tls + optional: true # HA long-lived token of the service user `MQTT Cert Sync` # (system-admin; config flows need admin). REN-1268. - name: ha-token -- 2.54.0 From 7186e4d0359fc0675e236ef648e9f57572cd5c42 Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Fri, 16 Oct 2026 12:51:49 +0200 Subject: [PATCH 24/42] feat(gpu): iGPU workloads run only where a node declares the iGPU llama-server, llama-embed, wiki-embed (gpu.intel.com/i915) and the wiki behind them are non-essential and bound to the Intel iGPU. On a site without one (Hetzner, restore rehearsal runs 27/28) their pods stayed Pending, llama-server failed its health check and the Kustomizations held kustomize-controller workers; the GPU plugin and exporter DaemonSets sat on every node with nothing to serve. The node patches k8s-1..3 label their machines hardware.renner.uno/intel-gpu=true. The GPU plugin, the exporter, the three llama Deployments and the wiki select on it, and llama-server and wiki are no longer waited on. Elsewhere the DaemonSets have no pods and the Deployments stay unscheduled without blocking anything. Production rollout order: the label must be on the nodes (talosctl patch, no reboot) before these manifests reconcile, or the GPU plugin and the llama/wiki pods are removed or left Pending. Co-Authored-By: Claude Opus 5.5 --- kubernetes/apps/llama-server/deployment.yaml | 3 +++ kubernetes/apps/llama-server/embed.yaml | 3 +++ kubernetes/apps/llama-server/wiki-embed.yaml | 3 +++ kubernetes/apps/monitoring/intel-gpu-exporter.yaml | 2 ++ kubernetes/apps/wiki/deployment.yaml | 4 ++++ kubernetes/clusters/homelab/llama-server.yaml | 5 ++++- kubernetes/clusters/homelab/wiki.yaml | 3 ++- kubernetes/infrastructure/intel-gpu-plugin/daemonset.yaml | 1 + kubernetes/talos/patches/k8s-1.yaml | 3 +++ kubernetes/talos/patches/k8s-2.yaml | 2 ++ kubernetes/talos/patches/k8s-3.yaml | 2 ++ 11 files changed, 29 insertions(+), 2 deletions(-) diff --git a/kubernetes/apps/llama-server/deployment.yaml b/kubernetes/apps/llama-server/deployment.yaml index 57460f58..051729c0 100644 --- a/kubernetes/apps/llama-server/deployment.yaml +++ b/kubernetes/apps/llama-server/deployment.yaml @@ -44,6 +44,9 @@ spec: labels: app.kubernetes.io/name: llama-server spec: + # Runs only where the node patch declares the Intel iGPU. + nodeSelector: + hardware.renner.uno/intel-gpu: "true" securityContext: runAsNonRoot: true runAsUser: 1000 diff --git a/kubernetes/apps/llama-server/embed.yaml b/kubernetes/apps/llama-server/embed.yaml index 5382f32d..9dd13a49 100644 --- a/kubernetes/apps/llama-server/embed.yaml +++ b/kubernetes/apps/llama-server/embed.yaml @@ -34,6 +34,9 @@ spec: labels: app.kubernetes.io/name: llama-embed spec: + # Runs only where the node patch declares the Intel iGPU. + nodeSelector: + hardware.renner.uno/intel-gpu: "true" securityContext: runAsNonRoot: true runAsUser: 1000 diff --git a/kubernetes/apps/llama-server/wiki-embed.yaml b/kubernetes/apps/llama-server/wiki-embed.yaml index ba80bf11..405eb1cc 100644 --- a/kubernetes/apps/llama-server/wiki-embed.yaml +++ b/kubernetes/apps/llama-server/wiki-embed.yaml @@ -23,6 +23,9 @@ spec: labels: app.kubernetes.io/name: wiki-embed spec: + # Runs only where the node patch declares the Intel iGPU. + nodeSelector: + hardware.renner.uno/intel-gpu: "true" securityContext: runAsNonRoot: true runAsUser: 1000 diff --git a/kubernetes/apps/monitoring/intel-gpu-exporter.yaml b/kubernetes/apps/monitoring/intel-gpu-exporter.yaml index 2ab049e1..5df5d89c 100644 --- a/kubernetes/apps/monitoring/intel-gpu-exporter.yaml +++ b/kubernetes/apps/monitoring/intel-gpu-exporter.yaml @@ -25,6 +25,8 @@ spec: - key: node-role.kubernetes.io/control-plane operator: Exists effect: NoSchedule + nodeSelector: + hardware.renner.uno/intel-gpu: "true" securityContext: seccompProfile: type: RuntimeDefault diff --git a/kubernetes/apps/wiki/deployment.yaml b/kubernetes/apps/wiki/deployment.yaml index 5c12bf59..970c51a2 100644 --- a/kubernetes/apps/wiki/deployment.yaml +++ b/kubernetes/apps/wiki/deployment.yaml @@ -39,6 +39,10 @@ spec: labels: app.kubernetes.io/name: wiki spec: + # Its embedding backend (wiki-embed) needs the Intel iGPU; without it + # the wiki has nothing to serve, so it runs where the iGPU is declared. + nodeSelector: + hardware.renner.uno/intel-gpu: "true" automountServiceAccountToken: false securityContext: runAsNonRoot: true diff --git a/kubernetes/clusters/homelab/llama-server.yaml b/kubernetes/clusters/homelab/llama-server.yaml index 4165331c..754cb1fc 100644 --- a/kubernetes/clusters/homelab/llama-server.yaml +++ b/kubernetes/clusters/homelab/llama-server.yaml @@ -15,7 +15,10 @@ spec: name: flux-system path: ./kubernetes/apps/llama-server prune: true - wait: true + # Non-essential, and its pods run only on nodes with the Intel iGPU + # (hardware.renner.uno/intel-gpu); on a site without one they stay + # Pending, which must not hold a reconcile worker or any dependent. + wait: false dependsOn: - name: infrastructure - name: infrastructure-crds diff --git a/kubernetes/clusters/homelab/wiki.yaml b/kubernetes/clusters/homelab/wiki.yaml index 2e6c52cd..4b89f96f 100644 --- a/kubernetes/clusters/homelab/wiki.yaml +++ b/kubernetes/clusters/homelab/wiki.yaml @@ -13,7 +13,8 @@ spec: name: flux-system path: ./kubernetes/apps/wiki prune: true - wait: true + # Non-essential and iGPU-bound like llama-server: not waited on. + wait: false dependsOn: - name: infrastructure - name: infrastructure-crds diff --git a/kubernetes/infrastructure/intel-gpu-plugin/daemonset.yaml b/kubernetes/infrastructure/intel-gpu-plugin/daemonset.yaml index 556dfe76..fb9cde14 100644 --- a/kubernetes/infrastructure/intel-gpu-plugin/daemonset.yaml +++ b/kubernetes/infrastructure/intel-gpu-plugin/daemonset.yaml @@ -119,3 +119,4 @@ spec: type: DirectoryOrCreate nodeSelector: kubernetes.io/arch: amd64 + hardware.renner.uno/intel-gpu: "true" diff --git a/kubernetes/talos/patches/k8s-1.yaml b/kubernetes/talos/patches/k8s-1.yaml index 1c3fead3..0a12f186 100644 --- a/kubernetes/talos/patches/k8s-1.yaml +++ b/kubernetes/talos/patches/k8s-1.yaml @@ -54,6 +54,9 @@ machine: # this label instead of listing hostnames, so which boxes carry storage is # stated once, here, by the machine that knows. storage.renner.uno/ceph-osd: "true" + # This machine has the Intel iGPU. The GPU plugin and everything that needs + # gpu.intel.com run only on nodes with this label. + hardware.renner.uno/intel-gpu: "true" # Publishes this node's pod0 addresses, declared below in its VLANConfig, so # the bird DaemonSet reads its peers from the node list rather than from a # copy. Annotations, not labels: a label value cannot contain a colon. diff --git a/kubernetes/talos/patches/k8s-2.yaml b/kubernetes/talos/patches/k8s-2.yaml index a5ceabf1..fdc3484e 100644 --- a/kubernetes/talos/patches/k8s-2.yaml +++ b/kubernetes/talos/patches/k8s-2.yaml @@ -34,6 +34,8 @@ machine: topology.kubernetes.io/zone: saturn # Holds a Ceph OSD drive — see k8s-1.yaml. storage.renner.uno/ceph-osd: "true" + # Has the Intel iGPU — see k8s-1.yaml. + hardware.renner.uno/intel-gpu: "true" # pod0 addresses for the bird peer list — see k8s-1.yaml. nodeAnnotations: net.renner.uno/pod0-v6: fd02:48::12 diff --git a/kubernetes/talos/patches/k8s-3.yaml b/kubernetes/talos/patches/k8s-3.yaml index a26b57e5..050297d4 100644 --- a/kubernetes/talos/patches/k8s-3.yaml +++ b/kubernetes/talos/patches/k8s-3.yaml @@ -30,6 +30,8 @@ machine: topology.kubernetes.io/zone: neptune # Holds a Ceph OSD drive — see k8s-1.yaml. storage.renner.uno/ceph-osd: "true" + # Has the Intel iGPU — see k8s-1.yaml. + hardware.renner.uno/intel-gpu: "true" # pod0 addresses for the bird peer list — see k8s-1.yaml. nodeAnnotations: net.renner.uno/pod0-v6: fd02:48::13 -- 2.54.0 From ec8f496d95b3fe206e48e7765596afdaa15214c1 Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Fri, 16 Oct 2026 20:25:21 +0200 Subject: [PATCH 25/42] feat(talos): hcloud nodes carry ens19 as a dummy link The v30iot NetworkAttachmentDefinitions of matter-server, music-assistant, otbr, shelly-manager and the mdns relay name master: ens19. On the hcloud site no such link existed, so macvlan creation failed outright. Each hcloud node now declares ens19 as a DummyLinkConfig: no address, nothing behind it. The NADs lease by DHCP, and nothing answers on a dummy link, so the pods get their macvlan but the dhcp IPAM step still times out. Validated: talosctl gen config (cluster + site-hcloud + hcloud-1) then talosctl validate --mode cloud --strict. Co-Authored-By: Claude Opus 5.5 --- kubernetes/talos/patches/hcloud-1.yaml | 7 +++++++ kubernetes/talos/patches/hcloud-2.yaml | 6 ++++++ kubernetes/talos/patches/hcloud-3.yaml | 6 ++++++ 3 files changed, 19 insertions(+) diff --git a/kubernetes/talos/patches/hcloud-1.yaml b/kubernetes/talos/patches/hcloud-1.yaml index 13a7f6d6..2dca05cd 100644 --- a/kubernetes/talos/patches/hcloud-1.yaml +++ b/kubernetes/talos/patches/hcloud-1.yaml @@ -55,6 +55,13 @@ kind: LinkConfig name: ceph-san up: true --- +# The v30iot macvlan parent the NADs name (master: ens19). Hetzner has no IoT +# VLAN, so it is a dummy link: no address, nothing behind it. +apiVersion: v1alpha1 +kind: DummyLinkConfig +name: ens19 +up: true +--- # Capped so the OSD partition fits beside it on cpx52's 480 GB local disk. apiVersion: v1alpha1 kind: VolumeConfig diff --git a/kubernetes/talos/patches/hcloud-2.yaml b/kubernetes/talos/patches/hcloud-2.yaml index 32d3b3d0..a0356e53 100644 --- a/kubernetes/talos/patches/hcloud-2.yaml +++ b/kubernetes/talos/patches/hcloud-2.yaml @@ -42,6 +42,12 @@ up: true --- # See hcloud-1.yaml. apiVersion: v1alpha1 +kind: DummyLinkConfig +name: ens19 +up: true +--- +# See hcloud-1.yaml. +apiVersion: v1alpha1 kind: VolumeConfig name: EPHEMERAL provisioning: diff --git a/kubernetes/talos/patches/hcloud-3.yaml b/kubernetes/talos/patches/hcloud-3.yaml index cdcceb03..3fc6a6a8 100644 --- a/kubernetes/talos/patches/hcloud-3.yaml +++ b/kubernetes/talos/patches/hcloud-3.yaml @@ -42,6 +42,12 @@ up: true --- # See hcloud-1.yaml. apiVersion: v1alpha1 +kind: DummyLinkConfig +name: ens19 +up: true +--- +# See hcloud-1.yaml. +apiVersion: v1alpha1 kind: VolumeConfig name: EPHEMERAL provisioning: -- 2.54.0 From 26976414e447015b402448c5ecbc153cafa16362 Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Fri, 16 Oct 2026 20:26:28 +0200 Subject: [PATCH 26/42] fix(authentik): its ServiceMonitor lives in monitoring-rules The authentik chart renders server/servicemonitor.yaml whenever server.metrics.serviceMonitor.enabled is set, without checking .Capabilities for monitoring.coreos.com (verified in chart 2026.8.3). The authentik Kustomization does not depend on monitoring, so on an empty cluster the Helm install fails until kube-prometheus-stack has installed the CRDs. The chart's ServiceMonitor is switched off and declared standalone in monitoring-rules, as cilium and velero already do. The metrics Service is gated only on server.metrics.enabled and stays. The selector matches the live Service authentik-server-metrics (port metrics, 9300). The new name, authentik-server-metrics, differs from the chart's authentik-server, so the two never contend for one object while production rolls over. Swept every manifest for other chart-rendered monitoring CRs outside the monitoring Kustomization: cilium and velero already off, blackbox-exporter renders none (no targets, selfMonitor off). Co-Authored-By: Claude Opus 5.5 --- kubernetes/apps/authentik/helmrelease.yaml | 7 +++--- .../authentik-servicemonitor.yaml | 24 +++++++++++++++++++ .../apps/monitoring-rules/kustomization.yaml | 1 + 3 files changed, 29 insertions(+), 3 deletions(-) create mode 100644 kubernetes/apps/monitoring-rules/authentik-servicemonitor.yaml diff --git a/kubernetes/apps/authentik/helmrelease.yaml b/kubernetes/apps/authentik/helmrelease.yaml index 1b0eca2a..fa4daf99 100644 --- a/kubernetes/apps/authentik/helmrelease.yaml +++ b/kubernetes/apps/authentik/helmrelease.yaml @@ -152,10 +152,11 @@ spec: targetCPUUtilizationPercentage: 70 metrics: enabled: true + # The chart renders it without checking for monitoring.coreos.com, so + # the install would wait on monitoring. Declared in + # monitoring-rules/authentik-servicemonitor.yaml instead. serviceMonitor: - enabled: true - labels: - release: kube-prometheus-stack + enabled: false ingress: enabled: true ingressClassName: traefik diff --git a/kubernetes/apps/monitoring-rules/authentik-servicemonitor.yaml b/kubernetes/apps/monitoring-rules/authentik-servicemonitor.yaml new file mode 100644 index 00000000..2f49d266 --- /dev/null +++ b/kubernetes/apps/monitoring-rules/authentik-servicemonitor.yaml @@ -0,0 +1,24 @@ +--- +# authentik server metrics. The chart's metrics Service stays; only its +# ServiceMonitor is declared here (see apps/authentik/helmrelease.yaml). +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: authentik-server-metrics + namespace: authentik + labels: + release: kube-prometheus-stack +spec: + namespaceSelector: + matchNames: + - authentik + selector: + matchLabels: + app.kubernetes.io/name: authentik + app.kubernetes.io/instance: authentik + app.kubernetes.io/component: server-metrics + endpoints: + - port: metrics + path: /metrics + interval: 30s + scrapeTimeout: 3s diff --git a/kubernetes/apps/monitoring-rules/kustomization.yaml b/kubernetes/apps/monitoring-rules/kustomization.yaml index 43e3ed03..e6cbc503 100644 --- a/kubernetes/apps/monitoring-rules/kustomization.yaml +++ b/kubernetes/apps/monitoring-rules/kustomization.yaml @@ -50,6 +50,7 @@ resources: - velero-podmonitor.yaml - velero-servicemonitor.yaml - authentik-outpost-servicemonitors.yaml + - authentik-servicemonitor.yaml # Rook's exporter/mgr monitors: the operator aborts its node-daemon reconcile # when it cannot create these, which held the CephCluster in Progressing and # deadlocked the whole apps layer. See the file for the cycle. -- 2.54.0 From 09bf50731d30a2f62432d54932a38ae42d0b4df8 Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Fri, 16 Oct 2026 20:35:12 +0200 Subject: [PATCH 27/42] feat(scripts): flux-graph.sh draws the Kustomization dependency graph Reads every kustomize.toolkit.fluxcd.io/v1 Kustomization in kubernetes/clusters/homelab and emits Graphviz DOT, one edge per dependsOn entry, pointing from the dependency to its dependent. Each node carries its wait mode and timeout. Infrastructure is green, the Velero restore chain blue, wait: false dashed. With no argument or a .dot path it writes the full graph; with a .svg or .png path it renders the transitive reduction (tred), which drops edges implied by a longer path. Generated rather than checked in, so it cannot drift from the manifests. Works with either yq (mikefarah's, as in the CI images, or the Python wrapper): yq only converts YAML to JSON, jq builds the DOT. Verified: the DOT matches the earlier hand-run yq output byte for byte; .png and .svg render; shellcheck clean. Co-Authored-By: Claude Opus 5.5 --- scripts/flux-graph.sh | 78 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100755 scripts/flux-graph.sh diff --git a/scripts/flux-graph.sh b/scripts/flux-graph.sh new file mode 100755 index 00000000..e5a2f3fb --- /dev/null +++ b/scripts/flux-graph.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Draws the Flux Kustomization dependency graph from the dependsOn fields in +# kubernetes/clusters/homelab. Edges point from a dependency to its dependent. +# +# flux-graph.sh DOT on stdout, every edge +# flux-graph.sh out.dot the same, written to out.dot +# flux-graph.sh out.svg rendered; out.png likewise +# +# Rendered output is the transitive reduction (tred): an edge implied by a +# longer path is left out, the order is the same. +# +# Green: infrastructure. Blue: the Velero restore chain. Dashed: wait: false. +# +# Needs jq, either yq (mikefarah's or the Python wrapper) and, to render, +# graphviz. + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +CLUSTER_DIR="${REPO_ROOT}/kubernetes/clusters/homelab" +OUT="${1:-}" + +command -v jq >/dev/null || { echo "jq not found" >&2; exit 1; } +command -v yq >/dev/null || { echo "yq not found" >&2; exit 1; } + +# One JSON document per YAML document. +yaml_to_json() { + if [[ "$(yq --version)" == *mikefarah* ]]; then + yq eval-all -o=json -I=0 '.' "$@" + else + yq -c '.' "$@" + fi +} + +to_dot() { + yaml_to_json "${CLUSTER_DIR}"/*.yaml | jq -r -s ' + [ .[] + | select(.kind == "Kustomization" + and .apiVersion == "kustomize.toolkit.fluxcd.io/v1") ] as $ks + | "digraph flux {", + " rankdir=LR;", + " node [shape=box, style=\"rounded\", fontname=\"Helvetica\", fontsize=10];", + " edge [color=\"#888888\"];", + ( $ks[] + | .metadata.name as $n + | (if .spec.wait == false then "wait: false" + elif .spec.wait == true then "wait" + else "health checks" end) as $wait + | (if .spec.wait == false then ", style=\"rounded,dashed\"" else "" end) as $dash + | (if ($n | startswith("restore-")) or $n == "velero-backups-synced" + then ", color=\"#1f77b4\", fontcolor=\"#1f77b4\"" + elif ($n | startswith("infrastructure")) + then ", color=\"#2ca02c\", fontcolor=\"#2ca02c\"" + else "" end) as $colour + | " \"\($n)\" [label=\"\($n)\\n\($wait) · \(.spec.timeout // "-")\"\($dash)\($colour)];" ), + ( $ks[] + | .metadata.name as $n + | (.spec.dependsOn // [])[] + | " \"\(.name)\" -> \"\($n)\";" ), + "}"' +} + +case "${OUT}" in + "") + to_dot + ;; + *.dot) + to_dot > "${OUT}" + ;; + *.svg|*.png) + command -v dot >/dev/null || { echo "graphviz (dot) not found" >&2; exit 1; } + to_dot | tred | dot -T"${OUT##*.}" -Gdpi=110 -o "${OUT}" + ;; + *) + echo "usage: $0 [out.dot|out.svg|out.png]" >&2 + exit 1 + ;; +esac -- 2.54.0 From 813822462ac655b7a9c75c1518993384cb7aefcb Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Fri, 16 Oct 2026 20:40:44 +0200 Subject: [PATCH 28/42] fix(flux): every app Kustomization waits for its workloads; otbr suspended With wait: false and no healthChecks, Ready means only that the manifests were applied: dependents are released before the app runs, and a broken app reports green. kustomize-controller's checkHealth returns before any check in that case (len(Spec.HealthChecks) == 0 && !Spec.Wait), so healthCheckExprs are not evaluated either. - home-assistant: wait: true. The reason given (the USB meter, the canary) no longer holds; HA reads the meter over MQTT. evcc, zone-configurator and speech-to-phrase depend on it, and evcc needs mqtt-evcc-client-tls from it. - llama-server, wiki: wait: true, reverting the wait: false of 7186e4d0. On a site without the iGPU they fail rather than report Ready. - wordpress: wait: true. It set no wait, so its MariaDB healthCheckExprs were never evaluated. - otbr: suspended. It is not in use. infrastructure keeps wait: false with its explicit healthChecks, which is a health gate on the operators, not the absence of one. Co-Authored-By: Claude Opus 5.5 --- kubernetes/clusters/homelab/home-assistant.yaml | 8 ++++---- kubernetes/clusters/homelab/llama-server.yaml | 5 +---- kubernetes/clusters/homelab/wiki.yaml | 3 +-- kubernetes/clusters/homelab/wordpress.yaml | 1 + 4 files changed, 7 insertions(+), 10 deletions(-) diff --git a/kubernetes/clusters/homelab/home-assistant.yaml b/kubernetes/clusters/homelab/home-assistant.yaml index be82d7dd..0efe7858 100644 --- a/kubernetes/clusters/homelab/home-assistant.yaml +++ b/kubernetes/clusters/homelab/home-assistant.yaml @@ -19,9 +19,7 @@ spec: name: flux-system path: ./kubernetes/apps/home-assistant prune: true - # Same TBD-USB caveat for the stromzaehler hostPath. Manifests apply; - # HA pod may CrashLoop until the USB is present — acceptable during canary. - wait: false + wait: true # decryption block dropped 2026-07-13: path is sops-free — secrets come # from OpenBao via ESO (MIGRATION-sops-openbao.md). dependsOn: @@ -79,7 +77,9 @@ spec: name: flux-system path: ./kubernetes/apps/otbr prune: true - wait: false + # Not in use. + suspend: true + wait: true # decryption block dropped 2026-07-13: path is sops-free — secrets come # from OpenBao via ESO (MIGRATION-sops-openbao.md). dependsOn: diff --git a/kubernetes/clusters/homelab/llama-server.yaml b/kubernetes/clusters/homelab/llama-server.yaml index 754cb1fc..4165331c 100644 --- a/kubernetes/clusters/homelab/llama-server.yaml +++ b/kubernetes/clusters/homelab/llama-server.yaml @@ -15,10 +15,7 @@ spec: name: flux-system path: ./kubernetes/apps/llama-server prune: true - # Non-essential, and its pods run only on nodes with the Intel iGPU - # (hardware.renner.uno/intel-gpu); on a site without one they stay - # Pending, which must not hold a reconcile worker or any dependent. - wait: false + wait: true dependsOn: - name: infrastructure - name: infrastructure-crds diff --git a/kubernetes/clusters/homelab/wiki.yaml b/kubernetes/clusters/homelab/wiki.yaml index 4b89f96f..2e6c52cd 100644 --- a/kubernetes/clusters/homelab/wiki.yaml +++ b/kubernetes/clusters/homelab/wiki.yaml @@ -13,8 +13,7 @@ spec: name: flux-system path: ./kubernetes/apps/wiki prune: true - # Non-essential and iGPU-bound like llama-server: not waited on. - wait: false + wait: true dependsOn: - name: infrastructure - name: infrastructure-crds diff --git a/kubernetes/clusters/homelab/wordpress.yaml b/kubernetes/clusters/homelab/wordpress.yaml index 7ea5c93d..00f0607c 100644 --- a/kubernetes/clusters/homelab/wordpress.yaml +++ b/kubernetes/clusters/homelab/wordpress.yaml @@ -12,6 +12,7 @@ spec: name: flux-system path: ./kubernetes/apps/wordpress prune: true + wait: true dependsOn: - name: restore-wordpress # infrastructure gates on the mariadb-operator HelmRelease being Ready, -- 2.54.0 From abf549bb9aac882d6b722b4c00e9ec303dadc606 Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Fri, 16 Oct 2026 20:40:44 +0200 Subject: [PATCH 29/42] fix(scripts): flux-graph.sh draws what Flux actually checks A Kustomization with healthChecks and no wait is health-checked (as infrastructure is), and one with neither has no check at all, whatever its healthCheckExprs say. Suspended Kustomizations are filled grey and labelled. Co-Authored-By: Claude Opus 5.5 --- scripts/flux-graph.sh | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/scripts/flux-graph.sh b/scripts/flux-graph.sh index e5a2f3fb..a5d86243 100755 --- a/scripts/flux-graph.sh +++ b/scripts/flux-graph.sh @@ -11,7 +11,8 @@ set -euo pipefail # Rendered output is the transitive reduction (tred): an edge implied by a # longer path is left out, the order is the same. # -# Green: infrastructure. Blue: the Velero restore chain. Dashed: wait: false. +# Green: infrastructure. Blue: the Velero restore chain. Dashed: nothing is +# health-checked. Grey fill: suspended. # # Needs jq, either yq (mikefarah's or the Python wrapper) and, to render, # graphviz. @@ -43,16 +44,21 @@ to_dot() { " edge [color=\"#888888\"];", ( $ks[] | .metadata.name as $n - | (if .spec.wait == false then "wait: false" - elif .spec.wait == true then "wait" - else "health checks" end) as $wait - | (if .spec.wait == false then ", style=\"rounded,dashed\"" else "" end) as $dash + # healthChecks are ignored under wait: true; without either, nothing + # is checked, healthCheckExprs included. + | (if .spec.wait == true then "wait" + elif ((.spec.healthChecks // []) | length) > 0 then "health checks" + else "wait: false" end) as $wait + | (if .spec.suspend == true then ", style=\"rounded,filled\", fillcolor=\"#dddddd\"" + elif $wait == "wait: false" then ", style=\"rounded,dashed\"" + else "" end) as $dash + | (if .spec.suspend == true then "\\nsuspended" else "" end) as $suspended | (if ($n | startswith("restore-")) or $n == "velero-backups-synced" then ", color=\"#1f77b4\", fontcolor=\"#1f77b4\"" elif ($n | startswith("infrastructure")) then ", color=\"#2ca02c\", fontcolor=\"#2ca02c\"" else "" end) as $colour - | " \"\($n)\" [label=\"\($n)\\n\($wait) · \(.spec.timeout // "-")\"\($dash)\($colour)];" ), + | " \"\($n)\" [label=\"\($n)\\n\($wait) · \(.spec.timeout // "-")\($suspended)\"\($dash)\($colour)];" ), ( $ks[] | .metadata.name as $n | (.spec.dependsOn // [])[] -- 2.54.0 From 0486ea2d9942a04b155bacfd3ad3769d623d741e Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Fri, 16 Oct 2026 20:49:34 +0200 Subject: [PATCH 30/42] fix(flux): gitea-mcp depends on gitea gitea-mcp is a front end to Gitea's API (-H https://gitea.renner.uno) and does nothing without it. Its image is on docker.gitea.com, which is why the in-house-image sweep did not add this edge. /healthz reports the process only, so without the edge the Kustomization went Ready with no Gitea behind it. Co-Authored-By: Claude Opus 5.5 --- kubernetes/clusters/homelab/gitea-mcp.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/kubernetes/clusters/homelab/gitea-mcp.yaml b/kubernetes/clusters/homelab/gitea-mcp.yaml index 6c70ff97..34fb1e08 100644 --- a/kubernetes/clusters/homelab/gitea-mcp.yaml +++ b/kubernetes/clusters/homelab/gitea-mcp.yaml @@ -16,3 +16,4 @@ spec: dependsOn: - name: infrastructure - name: infrastructure-crds + - name: gitea # serves Gitea's API -- 2.54.0 From 7d9dadccbaa191e3a48cef472539f8560874343e Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Fri, 16 Oct 2026 20:51:28 +0200 Subject: [PATCH 31/42] fix(flux): woodpecker depends on gitea Gitea is woodpecker's forge and its OAuth login provider (WOODPECKER_GITEA_URL https://gitea.renner.uno); the server has no function without it. Co-Authored-By: Claude Opus 5.5 --- kubernetes/clusters/homelab/woodpecker.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/kubernetes/clusters/homelab/woodpecker.yaml b/kubernetes/clusters/homelab/woodpecker.yaml index 5eb61493..10662621 100644 --- a/kubernetes/clusters/homelab/woodpecker.yaml +++ b/kubernetes/clusters/homelab/woodpecker.yaml @@ -25,6 +25,7 @@ spec: - name: infrastructure-crds - name: infrastructure-ceph-cluster - name: infrastructure-cnpg-barman + - name: gitea # its forge and login provider healthCheckExprs: - apiVersion: postgresql.cnpg.io/v1 kind: Cluster -- 2.54.0 From cae4d83f1b0e7ac57dffce2545d9118cc86b09ce Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Fri, 16 Oct 2026 20:54:21 +0200 Subject: [PATCH 32/42] fix(flux): trmnl depends on home-assistant, home-assistant on influxdb - trmnl: the renderer reads HA's API (http://home-assistant-bgp.home-assistant.svc.cluster.local:8123). - home-assistant: its InfluxDB integration writes to influxdb (influxdb_token in HA's secrets.yaml). No cycle: influxdb depends only on its restore and infrastructure. Co-Authored-By: Claude Opus 5.5 --- kubernetes/clusters/homelab/home-assistant.yaml | 1 + kubernetes/clusters/homelab/trmnl.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/kubernetes/clusters/homelab/home-assistant.yaml b/kubernetes/clusters/homelab/home-assistant.yaml index 0efe7858..e963fdce 100644 --- a/kubernetes/clusters/homelab/home-assistant.yaml +++ b/kubernetes/clusters/homelab/home-assistant.yaml @@ -41,6 +41,7 @@ spec: # it, since all three wait on home-assistant. - name: infrastructure-cnpg-barman - name: gitea # shelly-manager, mqtt-account-controller images are in Gitea's registry + - name: influxdb # HA's InfluxDB integration writes there --- apiVersion: kustomize.toolkit.fluxcd.io/v1 kind: Kustomization diff --git a/kubernetes/clusters/homelab/trmnl.yaml b/kubernetes/clusters/homelab/trmnl.yaml index 80efb8fb..3d61d32c 100644 --- a/kubernetes/clusters/homelab/trmnl.yaml +++ b/kubernetes/clusters/homelab/trmnl.yaml @@ -22,6 +22,7 @@ spec: - name: infrastructure-crds - name: infrastructure-ceph-cluster - name: infrastructure-cnpg-barman + - name: home-assistant # the renderer reads HA's API healthCheckExprs: - apiVersion: postgresql.cnpg.io/v1 kind: Cluster -- 2.54.0 From 43dfa6158c9f6c63d0f8b986814b0ab49a9f0525 Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Fri, 16 Oct 2026 21:13:36 +0200 Subject: [PATCH 33/42] fix(woodpecker): recover wp-pg from the prefix it archives to wp-pg-store-source, the recovery source of wp-pg's bootstrap.recovery, pointed at s3://renner-uno-k8s-cnpg/wp-pg: the prefix of the retired apps/wp-pg. Since the namespace move wp-pg archives to wp-pg-woodpecker (wp-pg-store). A cold rebuild would therefore have restored woodpecker as of 2026-08-18 and reported success. Bucket, listed read-only 2026-10-16: - wp-pg/wp-pg/base: newest 20260818T023502 - wp-pg-woodpecker/wp-pg/base: 20260916T024000 .. 20261016T024000 (daily), wals/ through timeline 6 (00000006.history from 2026-09-23) The source now reads wp-pg-woodpecker; serverName stays wp-pg, the name the live cluster archives under. bootstrap.recovery is honoured only when PGDATA is absent, so the running cluster is unaffected. The old prefix wp-pg is no longer referenced and keeps its objects; nothing applies retention to it. Co-Authored-By: Claude Opus 5.5 --- kubernetes/apps/woodpecker/objectstore.yaml | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/kubernetes/apps/woodpecker/objectstore.yaml b/kubernetes/apps/woodpecker/objectstore.yaml index e00eb14a..ef132eb7 100644 --- a/kubernetes/apps/woodpecker/objectstore.yaml +++ b/kubernetes/apps/woodpecker/objectstore.yaml @@ -1,10 +1,5 @@ --- -# RECOVERY SOURCE — read-only. The prefix the retired `apps/wp-pg` archived to. -# -# Kept after the migration on purpose. Nothing writes to it, and the backups -# under it age out on their own retention, but while they last this is a second, -# independent restore point that predates the move — worth having until the new -# prefix has a meaningful backup history of its own. +# RECOVERY SOURCE — read-only. The prefix wp-pg archives to (wp-pg-store below). # # serverName on the consuming externalCluster must be `wp-pg`, the name of the # cluster that produced these backups. @@ -15,7 +10,7 @@ metadata: namespace: woodpecker spec: configuration: - destinationPath: s3://renner-uno-k8s-cnpg/wp-pg + destinationPath: s3://renner-uno-k8s-cnpg/wp-pg-woodpecker endpointURL: https://fsn1.your-objectstorage.com s3Credentials: accessKeyId: @@ -31,15 +26,13 @@ spec: compression: gzip data: compression: gzip - # No retentionPolicy: this store is a read source during the migration, and - # the old cluster in `apps` owns the lifecycle of the data underneath it. + # No retentionPolicy: wp-pg-store owns the lifecycle of this prefix. --- -# ARCHIVING TARGET — a DIFFERENT prefix from the source, deliberately. +# ARCHIVING TARGET. # -# Two clusters archiving WAL to one destinationPath interleave their streams and -# corrupt both. That mattered during the parallel-run window; it still matters -# now, because the source prefix above is retained and must stay frozen for the -# independent restore point it provides. +# Not s3://renner-uno-k8s-cnpg/wp-pg: the retired `apps/wp-pg` archived there, +# and two clusters archiving WAL to one destinationPath interleave their streams +# and corrupt both. apiVersion: barmancloud.cnpg.io/v1 kind: ObjectStore metadata: -- 2.54.0 From cffb7e027ad11340892afe5d079f40fa794f868c Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Fri, 16 Oct 2026 21:27:21 +0200 Subject: [PATCH 34/42] feat(cnpg): every cluster incarnation archives to a new prefix; recovery reads the newest A CNPG cluster restored from barman archived back into the location it recovered from. plugin-barman-cloud's restore job checks the archive target (plugins[] ObjectStore + serverName) and stops with "Expected empty archive" when it holds WAL, which after a disaster it always does. It has no exception for target == recovery source (internal/cnpgi/restore/restore.go, v0.15). Run 28 hit it on wp-pg; the other six never got that far. The check stays. Instead, archive and source become two ObjectStores whose paths are chosen per incarnation: - kubernetes/cnpg-object-stores/: all 14 ObjectStores (-store archives, -store-source is read by bootstrap.recovery), in a Kustomization of their own with postBuild.substituteFrom: cnpg-paths. Nothing else lives there: substitution blanks every ${...}, which is why it is not on the app Kustomizations (home-assistant's scripts, the Grafana dashboards). The ConfigMap is not optional; without it the paths would render empty. - bootstrap.sh writes cnpg-paths before Flux starts, once per incarnation (an existing one is kept): CNPG__SOURCE the prefix among , -g<14 digits> and the store's cnpg.renner.uno/source-prefixes whose /base/ holds the newest base backup CNPG__ARCHIVE -g, new and so empty Every ScheduledBackup has immediate: true, so a new incarnation has a base backup in its own prefix minutes after it starts, and a later recovery picks it. S3 access is curl --aws-sigv4 with the restore key (already in bootstrap.sh's secret list), so the CI image needs no new tool. - The restore test archives to renner-uno-k8s-cnpg-rtest, created by the flux phase and emptied and deleted by hcloudclean. A test incarnation with a base backup in the production bucket would be its newest, and the next real recovery would read the test's data. - The six clusters that recovered from their archive store now name -store-source in externalClusters. serverName stays the cluster name everywhere. The app Kustomizations depend on cnpg-object-stores, which depends on infrastructure-cnpg-barman. Verified against the bucket, read-only, 2026-10-16: - the restore key lists renner-uno-k8s-cnpg and can create and delete a bucket (renner-uno-k8s-cnpg-rtest created and deleted again); - the listing parser on real responses, top level and an encoded nested prefix; for wp-pg it picks wp-pg-woodpecker (newest 20261016T024000) over wp-pg (20260818T023502); - leftover test serverNames (ak-pg-rtest, im-pg-rtest, ha-pg-restore-test, pprleval-pg) hold no base backups and match no candidate pattern. shellcheck clean; all seven apps and cnpg-object-stores build; the render holds exactly the 14 expected variables. Production rollout, one time: 1. Create flux-system/cnpg-paths with the current prefixes as both source and archive (ak-pg, ha-pg, im-pg, mx-pg, pprl-pg, tr-pg; wp-pg-woodpecker), so the running clusters keep archiving where they do. 2. Move the ObjectStores in two phases (add in cnpg-object-stores, then drop from the apps). All 14 now carry prune: disabled; before this only ak, im and pprl did. 3. Whether changing externalClusters on a live Cluster restarts its pods is not verified; check on one cluster first. Older generations are not cleaned up: retention applies only to the current archive prefix. A cleanup script is planned separately. RECOVERY.md: the table describes the new layout; the manual restore drill reads -store-source. That drill wrote 3 WAL files to ak-pg/ak-pg-rtest on 2026-07-25 despite isWALArchiver: false, so its "only ever reads" claim does not hold; not changed here. Co-Authored-By: Claude Opus 5.5 --- kubernetes/RECOVERY.md | 30 ++-- kubernetes/apps/authentik/cluster.yaml | 2 +- kubernetes/apps/authentik/kustomization.yaml | 1 - kubernetes/apps/authentik/objectstore.yaml | 33 ---- .../apps/home-assistant/ha-pg-cluster.yaml | 2 +- .../home-assistant/ha-pg-objectstore.yaml | 35 ---- .../apps/home-assistant/kustomization.yaml | 1 - kubernetes/apps/immich/cluster.yaml | 2 +- kubernetes/apps/immich/kustomization.yaml | 1 - kubernetes/apps/immich/objectstore.yaml | 33 ---- kubernetes/apps/matrix/cluster.yaml | 2 +- kubernetes/apps/matrix/kustomization.yaml | 1 - kubernetes/apps/matrix/objectstore.yaml | 30 ---- kubernetes/apps/paperless/cluster.yaml | 2 +- kubernetes/apps/paperless/kustomization.yaml | 1 - kubernetes/apps/paperless/objectstore.yaml | 35 ---- kubernetes/apps/trmnl/cluster.yaml | 2 +- kubernetes/apps/trmnl/kustomization.yaml | 1 - kubernetes/apps/trmnl/objectstore.yaml | 30 ---- kubernetes/apps/woodpecker/kustomization.yaml | 1 - kubernetes/bootstrap.sh | 164 +++++++++++++++++- kubernetes/clusters/homelab/authentik.yaml | 2 +- .../clusters/homelab/cnpg-object-stores.yaml | 27 +++ .../clusters/homelab/home-assistant.yaml | 14 +- kubernetes/clusters/homelab/immich.yaml | 2 +- kubernetes/clusters/homelab/matrix.yaml | 2 +- kubernetes/clusters/homelab/paperless.yaml | 2 +- kubernetes/clusters/homelab/trmnl.yaml | 2 +- kubernetes/clusters/homelab/woodpecker.yaml | 2 +- kubernetes/cnpg-object-stores/ak-pg.yaml | 54 ++++++ kubernetes/cnpg-object-stores/ha-pg.yaml | 54 ++++++ kubernetes/cnpg-object-stores/im-pg.yaml | 54 ++++++ .../cnpg-object-stores/kustomization.yaml | 24 +++ kubernetes/cnpg-object-stores/mx-pg.yaml | 54 ++++++ kubernetes/cnpg-object-stores/pprl-pg.yaml | 54 ++++++ kubernetes/cnpg-object-stores/tr-pg.yaml | 54 ++++++ .../wp-pg.yaml} | 67 ++++--- 37 files changed, 600 insertions(+), 277 deletions(-) delete mode 100644 kubernetes/apps/authentik/objectstore.yaml delete mode 100644 kubernetes/apps/home-assistant/ha-pg-objectstore.yaml delete mode 100644 kubernetes/apps/immich/objectstore.yaml delete mode 100644 kubernetes/apps/matrix/objectstore.yaml delete mode 100644 kubernetes/apps/paperless/objectstore.yaml delete mode 100644 kubernetes/apps/trmnl/objectstore.yaml create mode 100644 kubernetes/clusters/homelab/cnpg-object-stores.yaml create mode 100644 kubernetes/cnpg-object-stores/ak-pg.yaml create mode 100644 kubernetes/cnpg-object-stores/ha-pg.yaml create mode 100644 kubernetes/cnpg-object-stores/im-pg.yaml create mode 100644 kubernetes/cnpg-object-stores/kustomization.yaml create mode 100644 kubernetes/cnpg-object-stores/mx-pg.yaml create mode 100644 kubernetes/cnpg-object-stores/pprl-pg.yaml create mode 100644 kubernetes/cnpg-object-stores/tr-pg.yaml rename kubernetes/{apps/woodpecker/objectstore.yaml => cnpg-object-stores/wp-pg.yaml} (63%) diff --git a/kubernetes/RECOVERY.md b/kubernetes/RECOVERY.md index 5cdd0240..7bfa57ef 100644 --- a/kubernetes/RECOVERY.md +++ b/kubernetes/RECOVERY.md @@ -6,18 +6,24 @@ below (like `wp-pg`) keeps `bootstrap.recovery` in git: a cold rebuild restores each from its backup with no manual step. This document covers the precondition, watching and verifying that restore, and the reasoning behind it. -| Cluster | Namespace | ObjectStore | barman path (`destinationPath`) | app DB / owner | -|-----------|----------------|-----------------|--------------------------------------|---------------------------------| -| `ak-pg` | authentik | `ak-pg-store` | `s3://renner-uno-k8s-cnpg/ak-pg` | `authentik`/`authentik` | -| `im-pg` | immich | `im-pg-store` | `s3://renner-uno-k8s-cnpg/im-pg` | `immich`/`immich` | -| `pprl-pg` | paperless | `pprl-pg-store` | `s3://renner-uno-k8s-cnpg/pprl-pg` | `paperless`/`paperless` | -| `ha-pg` | home-assistant | `ha-pg-store` | `s3://renner-uno-k8s-cnpg/ha-pg` | `homeassistant`/`homeassistant` | -| `mx-pg` | matrix | `mx-pg-store` | `s3://renner-uno-k8s-cnpg/mx-pg` | `synapse`/`synapse` | -| `tr-pg` | trmnl | `tr-pg-store` | `s3://renner-uno-k8s-cnpg/tr-pg` | `terminus`/`terminus` | +| Cluster | Namespace | app DB / owner | +|-----------|----------------|---------------------------------| +| `ak-pg` | authentik | `authentik`/`authentik` | +| `im-pg` | immich | `immich`/`immich` | +| `pprl-pg` | paperless | `paperless`/`paperless` | +| `ha-pg` | home-assistant | `homeassistant`/`homeassistant` | +| `mx-pg` | matrix | `synapse`/`synapse` | +| `tr-pg` | trmnl | `terminus`/`terminus` | +| `wp-pg` | woodpecker | `woodpecker`/`woodpecker` | -The barman **`serverName`** is the last path segment of `destinationPath`, i.e. -it equals the cluster name. `gf-pg` (monitoring) has no ObjectStore and stays -on `initdb`. +Each cluster `` has two ObjectStores in `kubernetes/cnpg-object-stores/`: +`-store`, where it archives WAL and takes base backups, and +`-store-source`, which `bootstrap.recovery` reads. Their paths come from the +`flux-system/cnpg-paths` ConfigMap, which `bootstrap.sh` writes once per +cluster incarnation: the source is the prefix in `s3://renner-uno-k8s-cnpg` +whose `/base/` holds the newest base backup, the archive a new prefix +`-g`. The barman **`serverName`** is the cluster name in both. +`gf-pg` (monitoring) has no ObjectStore and stays on `initdb`. --- @@ -129,7 +135,7 @@ cluster in the *same* namespace (so it can reference the existing namespaced restore, but never archiving, so it only ever *reads* the backup path. ```sh -NS=authentik ; SRC=ak-pg ; STORE=ak-pg-store ; DB=authentik +NS=authentik ; SRC=ak-pg ; STORE=ak-pg-store-source ; DB=authentik kubectl apply -f - <authentik-KS handoff (backup target). - kustomize.toolkit.fluxcd.io/prune: disabled -spec: - configuration: - # Offsite Hetzner Object Storage (fsn1) — replaces Garage on nas-1. - # No SSE: Hetzner supports only SSE-C, unusable by barman-cloud - # (plugin #646), so at-rest encryption is unavailable; transit is TLS. - destinationPath: s3://renner-uno-k8s-cnpg/ak-pg - endpointURL: https://fsn1.your-objectstorage.com - s3Credentials: - accessKeyId: - name: hetzner-s3-creds - key: ACCESS_KEY_ID - secretAccessKey: - name: hetzner-s3-creds - key: ACCESS_SECRET_KEY - # Region is required (REGION=fsn1) — without it boto3's HeadBucket - # (barman-cloud-check-wal-archive during bootstrap.recovery) 400s. - region: - name: hetzner-s3-creds - key: REGION - wal: - compression: gzip - data: - compression: gzip - retentionPolicy: "30d" diff --git a/kubernetes/apps/home-assistant/ha-pg-cluster.yaml b/kubernetes/apps/home-assistant/ha-pg-cluster.yaml index f1190645..8a0f57f2 100644 --- a/kubernetes/apps/home-assistant/ha-pg-cluster.yaml +++ b/kubernetes/apps/home-assistant/ha-pg-cluster.yaml @@ -116,7 +116,7 @@ spec: plugin: name: barman-cloud.cloudnative-pg.io parameters: - barmanObjectName: ha-pg-store + barmanObjectName: ha-pg-store-source serverName: ha-pg postgresql: parameters: diff --git a/kubernetes/apps/home-assistant/ha-pg-objectstore.yaml b/kubernetes/apps/home-assistant/ha-pg-objectstore.yaml deleted file mode 100644 index 444ec01a..00000000 --- a/kubernetes/apps/home-assistant/ha-pg-objectstore.yaml +++ /dev/null @@ -1,35 +0,0 @@ ---- -# Barman object store for ha-pg: base backups + continuous WAL archiving to -# Hetzner Object Storage, same target bucket as the other CNPG clusters. -# -# Its own prefix. Two clusters archiving WAL to one destinationPath interleave -# their streams and corrupt both — see the note in -# kubernetes/apps/woodpecker/objectstore.yaml, which learned that the hard way. -apiVersion: barmancloud.cnpg.io/v1 -kind: ObjectStore -metadata: - name: ha-pg-store - namespace: home-assistant -spec: - configuration: - destinationPath: s3://renner-uno-k8s-cnpg/ha-pg - endpointURL: https://fsn1.your-objectstorage.com - s3Credentials: - accessKeyId: - name: hetzner-s3-creds - key: ACCESS_KEY_ID - secretAccessKey: - name: hetzner-s3-creds - key: ACCESS_SECRET_KEY - region: - name: hetzner-s3-creds - key: REGION - wal: - compression: gzip - data: - compression: gzip - # Matches the 30d retention the other CNPG clusters use and the - # renner-uno-k8s-cnpg bucket's noncurrent-version expiry in - # opentofu/object_storage.tf, so old S3 versions don't outlive the backups - # that superseded them. - retentionPolicy: "30d" diff --git a/kubernetes/apps/home-assistant/kustomization.yaml b/kubernetes/apps/home-assistant/kustomization.yaml index 6ffb8cd1..76f00839 100644 --- a/kubernetes/apps/home-assistant/kustomization.yaml +++ b/kubernetes/apps/home-assistant/kustomization.yaml @@ -57,7 +57,6 @@ resources: # Live: recorder.db_url in the homeassistant repo points here, and the # long-term statistics carry their full history from 2024. - ha-pg-hetzner-creds-externalsecret.yaml - - ha-pg-objectstore.yaml - ha-pg-cluster.yaml - ha-pg-scheduled-backup.yaml # MQTT credential management (REN-802): the account controller manages the diff --git a/kubernetes/apps/immich/cluster.yaml b/kubernetes/apps/immich/cluster.yaml index f16049ee..8cd16413 100644 --- a/kubernetes/apps/immich/cluster.yaml +++ b/kubernetes/apps/immich/cluster.yaml @@ -43,7 +43,7 @@ spec: plugin: name: barman-cloud.cloudnative-pg.io parameters: - barmanObjectName: im-pg-store + barmanObjectName: im-pg-store-source serverName: im-pg postgresql: # vchord only. The image also ships pgvecto.rs (vectors.so), but its diff --git a/kubernetes/apps/immich/kustomization.yaml b/kubernetes/apps/immich/kustomization.yaml index 258d32c7..c4010e65 100644 --- a/kubernetes/apps/immich/kustomization.yaml +++ b/kubernetes/apps/immich/kustomization.yaml @@ -11,7 +11,6 @@ resources: # a teardown). This KS gains the infrastructure-cnpg-barman dep + CNPG Cluster # healthCheckExprs (see clusters/homelab/immich.yaml). - cluster.yaml - - objectstore.yaml - scheduled-backup.yaml - hetzner-creds-externalsecret.yaml - pvcs.yaml diff --git a/kubernetes/apps/immich/objectstore.yaml b/kubernetes/apps/immich/objectstore.yaml deleted file mode 100644 index 1e66103a..00000000 --- a/kubernetes/apps/immich/objectstore.yaml +++ /dev/null @@ -1,33 +0,0 @@ ---- -apiVersion: barmancloud.cnpg.io/v1 -kind: ObjectStore -metadata: - name: im-pg-store - namespace: immich - annotations: - # Prune-disabled — a standing guard kept from the seed->immich-KS handoff (backup target). - kustomize.toolkit.fluxcd.io/prune: disabled -spec: - configuration: - # Offsite Hetzner Object Storage (fsn1) — replaces Garage on nas-1. - # No SSE: Hetzner supports only SSE-C, unusable by barman-cloud - # (plugin #646), so at-rest encryption is unavailable; transit is TLS. - destinationPath: s3://renner-uno-k8s-cnpg/im-pg - endpointURL: https://fsn1.your-objectstorage.com - s3Credentials: - accessKeyId: - name: hetzner-s3-creds - key: ACCESS_KEY_ID - secretAccessKey: - name: hetzner-s3-creds - key: ACCESS_SECRET_KEY - # Region is required (REGION=fsn1) — without it boto3's HeadBucket - # (barman-cloud-check-wal-archive during bootstrap.recovery) 400s. - region: - name: hetzner-s3-creds - key: REGION - wal: - compression: gzip - data: - compression: gzip - retentionPolicy: "30d" diff --git a/kubernetes/apps/matrix/cluster.yaml b/kubernetes/apps/matrix/cluster.yaml index eae9dc88..b7d4a4f0 100644 --- a/kubernetes/apps/matrix/cluster.yaml +++ b/kubernetes/apps/matrix/cluster.yaml @@ -31,7 +31,7 @@ spec: plugin: name: barman-cloud.cloudnative-pg.io parameters: - barmanObjectName: mx-pg-store + barmanObjectName: mx-pg-store-source serverName: mx-pg postgresql: parameters: diff --git a/kubernetes/apps/matrix/kustomization.yaml b/kubernetes/apps/matrix/kustomization.yaml index 94a5282b..7453c856 100644 --- a/kubernetes/apps/matrix/kustomization.yaml +++ b/kubernetes/apps/matrix/kustomization.yaml @@ -9,6 +9,5 @@ resources: # mx-pg CNPG Postgres + offsite Barman backup. The media store and signing # key are on synapse-data, backed up by Velero (matrix-daily). - cluster.yaml - - objectstore.yaml - scheduled-backup.yaml - hetzner-creds-externalsecret.yaml diff --git a/kubernetes/apps/matrix/objectstore.yaml b/kubernetes/apps/matrix/objectstore.yaml deleted file mode 100644 index 808a04c1..00000000 --- a/kubernetes/apps/matrix/objectstore.yaml +++ /dev/null @@ -1,30 +0,0 @@ ---- -apiVersion: barmancloud.cnpg.io/v1 -kind: ObjectStore -metadata: - name: mx-pg-store - namespace: matrix -spec: - configuration: - # Offsite Hetzner Object Storage (fsn1), same bucket as the other CNPG - # clusters. No SSE: Hetzner supports only SSE-C, unusable by barman-cloud - # (plugin #646), so at-rest encryption is unavailable; transit is TLS. - destinationPath: s3://renner-uno-k8s-cnpg/mx-pg - endpointURL: https://fsn1.your-objectstorage.com - s3Credentials: - accessKeyId: - name: hetzner-s3-creds - key: ACCESS_KEY_ID - secretAccessKey: - name: hetzner-s3-creds - key: ACCESS_SECRET_KEY - # Region is required (REGION=fsn1) — without it boto3's HeadBucket - # (barman-cloud-check-wal-archive during bootstrap.recovery) 400s. - region: - name: hetzner-s3-creds - key: REGION - wal: - compression: gzip - data: - compression: gzip - retentionPolicy: "30d" diff --git a/kubernetes/apps/paperless/cluster.yaml b/kubernetes/apps/paperless/cluster.yaml index e50fbb8d..0cc852cf 100644 --- a/kubernetes/apps/paperless/cluster.yaml +++ b/kubernetes/apps/paperless/cluster.yaml @@ -39,7 +39,7 @@ spec: plugin: name: barman-cloud.cloudnative-pg.io parameters: - barmanObjectName: pprl-pg-store + barmanObjectName: pprl-pg-store-source serverName: pprl-pg postgresql: parameters: diff --git a/kubernetes/apps/paperless/kustomization.yaml b/kubernetes/apps/paperless/kustomization.yaml index 48b5e086..20ee427f 100644 --- a/kubernetes/apps/paperless/kustomization.yaml +++ b/kubernetes/apps/paperless/kustomization.yaml @@ -17,7 +17,6 @@ resources: # transferred without a teardown). This KS gains the infrastructure-cnpg-barman # dep + CNPG Cluster healthCheckExprs (see clusters/homelab/paperless.yaml). - cluster.yaml - - objectstore.yaml - scheduled-backup.yaml - hetzner-creds-externalsecret.yaml - pvcs.yaml diff --git a/kubernetes/apps/paperless/objectstore.yaml b/kubernetes/apps/paperless/objectstore.yaml deleted file mode 100644 index 03d006bf..00000000 --- a/kubernetes/apps/paperless/objectstore.yaml +++ /dev/null @@ -1,35 +0,0 @@ ---- -apiVersion: barmancloud.cnpg.io/v1 -kind: ObjectStore -metadata: - name: pprl-pg-store - namespace: paperless - annotations: - # Prune-disabled — a standing guard kept from the seed->paperless-KS handoff (backup target). - kustomize.toolkit.fluxcd.io/prune: disabled -spec: - configuration: - # Offsite Hetzner Object Storage (fsn1) — replaces local Garage on - # nas-1 so paperless backups survive nas-1 being powered off. barman - # uses boto3 default (auto) addressing; Hetzner (Ceph RGW) accepts it. - destinationPath: s3://renner-uno-k8s-cnpg/pprl-pg - endpointURL: https://fsn1.your-objectstorage.com - s3Credentials: - accessKeyId: - name: hetzner-s3-creds - key: ACCESS_KEY_ID - secretAccessKey: - name: hetzner-s3-creds - key: ACCESS_SECRET_KEY - region: - name: hetzner-s3-creds - key: REGION - # No `encryption: AES256` (SSE-S3): Hetzner Object Storage supports - # only SSE-C, which the barman-cloud plugin can't use (cnpg - # plugin-barman-cloud #646). At-rest encryption is therefore not - # available here; transit is TLS and the bucket is private. - wal: - compression: gzip - data: - compression: gzip - retentionPolicy: "30d" diff --git a/kubernetes/apps/trmnl/cluster.yaml b/kubernetes/apps/trmnl/cluster.yaml index 187c2050..e3b5554c 100644 --- a/kubernetes/apps/trmnl/cluster.yaml +++ b/kubernetes/apps/trmnl/cluster.yaml @@ -36,7 +36,7 @@ spec: plugin: name: barman-cloud.cloudnative-pg.io parameters: - barmanObjectName: tr-pg-store + barmanObjectName: tr-pg-store-source serverName: tr-pg postgresql: parameters: diff --git a/kubernetes/apps/trmnl/kustomization.yaml b/kubernetes/apps/trmnl/kustomization.yaml index 20b26fb7..011ba273 100644 --- a/kubernetes/apps/trmnl/kustomization.yaml +++ b/kubernetes/apps/trmnl/kustomization.yaml @@ -18,6 +18,5 @@ resources: # tr-pg CNPG Postgres + offsite Barman backup. Note this cluster runs # PostgreSQL 18, not the repo-standard 16 — see the comment in cluster.yaml. - cluster.yaml - - objectstore.yaml - scheduled-backup.yaml - hetzner-creds-externalsecret.yaml diff --git a/kubernetes/apps/trmnl/objectstore.yaml b/kubernetes/apps/trmnl/objectstore.yaml deleted file mode 100644 index 61b0bc3b..00000000 --- a/kubernetes/apps/trmnl/objectstore.yaml +++ /dev/null @@ -1,30 +0,0 @@ ---- -apiVersion: barmancloud.cnpg.io/v1 -kind: ObjectStore -metadata: - name: tr-pg-store - namespace: trmnl -spec: - configuration: - # Offsite Hetzner Object Storage (fsn1), same bucket as the other CNPG - # clusters. No SSE: Hetzner supports only SSE-C, unusable by barman-cloud - # (plugin #646), so at-rest encryption is unavailable; transit is TLS. - destinationPath: s3://renner-uno-k8s-cnpg/tr-pg - endpointURL: https://fsn1.your-objectstorage.com - s3Credentials: - accessKeyId: - name: hetzner-s3-creds - key: ACCESS_KEY_ID - secretAccessKey: - name: hetzner-s3-creds - key: ACCESS_SECRET_KEY - # Region is required (REGION=fsn1) — without it boto3's HeadBucket - # (barman-cloud-check-wal-archive during bootstrap.recovery) 400s. - region: - name: hetzner-s3-creds - key: REGION - wal: - compression: gzip - data: - compression: gzip - retentionPolicy: "30d" diff --git a/kubernetes/apps/woodpecker/kustomization.yaml b/kubernetes/apps/woodpecker/kustomization.yaml index b820f6b3..16709f57 100644 --- a/kubernetes/apps/woodpecker/kustomization.yaml +++ b/kubernetes/apps/woodpecker/kustomization.yaml @@ -34,6 +34,5 @@ resources: # This KS depends on infrastructure-cnpg-barman for the barman plugin behind # the ObjectStore and ScheduledBackup. - hetzner-creds-externalsecret.yaml - - objectstore.yaml - cluster.yaml - scheduled-backup.yaml diff --git a/kubernetes/bootstrap.sh b/kubernetes/bootstrap.sh index 4992571f..386e08ce 100755 --- a/kubernetes/bootstrap.sh +++ b/kubernetes/bootstrap.sh @@ -394,6 +394,155 @@ read_secret() { cat "$OPENBAO_SECRETS_DIR/$name" } +# Hetzner Object Storage, where the offsite backups live. Read with the restore +# key (s3-access, s3-secret). +S3_ENDPOINT="fsn1.your-objectstorage.com" +S3_REGION="fsn1" + +# One request, signed by curl with the restore key. The body goes to stdout; +# anything but a 2xx fails, with S3's error document on stderr. +s3_request() { + local method=$1 path=$2 body code + body=$(mktemp) + if ! code=$(curl -sS -o "$body" -w '%{http_code}' -X "$method" \ + --aws-sigv4 "aws:amz:$S3_REGION:s3" \ + --user "$(read_secret s3-access):$(read_secret s3-secret)" \ + "https://$S3_ENDPOINT/$path"); then + rm -f "$body" + return 1 + fi + if [[ "$code" != 2?? ]]; then + echo "$method $path: HTTP $code" >&2 + cat "$body" >&2 + echo >&2 + rm -f "$body" + return 1 + fi + cat "$body" + rm -f "$body" +} + +# The "directories" directly under PREFIX in BUCKET, one per line, without +# PREFIX and the trailing slash. +s3_dirs() { + local bucket=$1 prefix=$2 xml + xml=$(s3_request GET "$bucket?list-type=2&delimiter=%2F&prefix=${prefix//\//%2F}") || return 1 + [[ "$xml" == *"false"* ]] \ + || { echo "the listing of $bucket/$prefix is truncated or not a listing" >&2; return 1; } + # awk, not grep: no match is not an error. + awk -v p="$prefix" '{ + while (match($0, /[^<]*<\/Prefix>/)) { + d = substr($0, RSTART + 24, RLENGTH - 33) + $0 = substr($0, RSTART + RLENGTH) + sub(/\/$/, "", d) + print substr(d, length(p) + 1) + } + }' <<<"$xml" +} + +# The bucket every CNPG cluster recovers from (cnpg-object-stores/). The +# restore test archives to CNPG_TEST_BUCKET instead: a test incarnation with a +# base backup in the production bucket would be the newest there, and the next +# real recovery would read the test's data. +CNPG_BUCKET="renner-uno-k8s-cnpg" +CNPG_TEST_BUCKET="renner-uno-k8s-cnpg-rtest" +CNPG_STORES_DIR="$REPO_ROOT/kubernetes/cnpg-object-stores" + +# cnpg-paths, the ConfigMap cnpg-object-stores takes its paths from. Written +# once per cluster incarnation: an existing one belongs to clusters already +# archiving under its paths and is kept. +# +# For each source ObjectStore in cnpg-object-stores/, cluster : +# CNPG__SOURCE the candidate prefix whose /base/ holds the newest +# base backup. Candidates: , -g<14 digits> and the +# store's cnpg.renner.uno/source-prefixes. +# CNPG__ARCHIVE -g in ARCHIVE_BUCKET: new, so empty, so the +# plugin's empty-archive check passes rather than being +# skipped. +write_cnpg_paths() { + local archive_bucket=$1 + local gen dirs name extra var p listing newest best best_at + local -a literals=() + echo "Writing the CNPG backup paths (cnpg-paths)..." + if kubectl -n flux-system get configmap cnpg-paths -o name >/dev/null 2>"$OPENBAO_SECRETS_DIR/.err"; then + echo " already present, kept" + return 0 + fi + grep -q NotFound "$OPENBAO_SECRETS_DIR/.err" \ + || { cat "$OPENBAO_SECRETS_DIR/.err" >&2; fail "could not read the cnpg-paths ConfigMap"; } + + gen=g$(date -u +%Y%m%d%H%M%S) + dirs=$(s3_dirs "$CNPG_BUCKET" "") || fail "could not list $CNPG_BUCKET" + while IFS=$'\t' read -r name extra; do + best='' + best_at='' + for p in $dirs; do + [[ "$p" == "$name" || "$p" =~ ^${name}-g[0-9]{14}$ || " $extra " == *" $p "* ]] || continue + listing=$(s3_dirs "$CNPG_BUCKET" "$p/$name/base/") \ + || fail "could not list $CNPG_BUCKET/$p/$name/base/" + newest=$(sort <<<"$listing" | tail -1) + if [[ -n "$newest" && "$newest" > "$best_at" ]]; then + best=$p + best_at=$newest + fi + done + [[ -n "$best" ]] \ + || fail "no base backup of $name in $CNPG_BUCKET (looked under $name, $name-g${extra:+, $extra})" + var=CNPG_$(tr 'a-z-' 'A-Z_' <<<"$name") + literals+=("--from-literal=${var}_SOURCE=s3://$CNPG_BUCKET/$best" + "--from-literal=${var}_ARCHIVE=s3://$archive_bucket/$name-$gen") + echo " $name: recovers from $CNPG_BUCKET/$best (base backup $best_at), archives to $archive_bucket/$name-$gen" + done < <(yq -r 'select(.kind == "ObjectStore" and .metadata.annotations["cnpg.renner.uno/server-name"] != null) + | [.metadata.annotations["cnpg.renner.uno/server-name"], + (.metadata.annotations["cnpg.renner.uno/source-prefixes"] // "")] | @tsv' \ + "$CNPG_STORES_DIR"/*.yaml) + [[ ${#literals[@]} -gt 0 ]] \ + || fail "no source ObjectStore (cnpg.renner.uno/server-name) in $CNPG_STORES_DIR" + kubectl -n flux-system create configmap cnpg-paths "${literals[@]}" >/dev/null \ + || fail "could not create the cnpg-paths ConfigMap" + ok +} + +# The restore test's archive bucket. Created when missing; on a rerun it +# already exists, which S3 reports as an error. +create_cnpg_test_bucket() { + local out + out=$(s3_request PUT "$CNPG_TEST_BUCKET" 2>&1) && return 0 + [[ "$out" == *BucketAlreadyOwnedByYou* ]] && return 0 + echo "$out" >&2 + fail "could not create $CNPG_TEST_BUCKET" +} + +# Empties and deletes the restore test's archive bucket. Each pass deletes the +# first page of keys and lists again, so no continuation token is needed. +delete_cnpg_test_bucket() { + local xml key + while :; do + if ! xml=$(s3_request GET "$CNPG_TEST_BUCKET?list-type=2" 2>&1); then + [[ "$xml" == *NoSuchBucket* ]] && return 0 + echo "$xml" >&2 + echo " WARNING: $CNPG_TEST_BUCKET not emptied — check the console" >&2 + return 0 + fi + while read -r key; do + [[ -n "$key" ]] || continue + s3_request DELETE "$CNPG_TEST_BUCKET/$key" >/dev/null \ + || { echo " WARNING: $CNPG_TEST_BUCKET/$key not deleted — check the console" >&2; return 0; } + done < <(awk '{ + while (match($0, /[^<]*<\/Key>/)) { + print substr($0, RSTART + 5, RLENGTH - 11) + $0 = substr($0, RSTART + RLENGTH) + } + }' <<<"$xml") + [[ "$xml" == *"true"* ]] || break + done + if s3_request DELETE "$CNPG_TEST_BUCKET" >/dev/null; then + echo " deleted bucket $CNPG_TEST_BUCKET" + else + echo " WARNING: bucket $CNPG_TEST_BUCKET not deleted — check the console" >&2 + fi +} + # Provision a throwaway cluster on Hetzner Cloud, then run the Talos phase # against it unchanged — same shape as the vmtest phase below, and for the same # reason: a rehearsal that runs different code proves less than one that does @@ -776,6 +925,8 @@ if [[ "$PHASE" == "hcloudclean" ]]; then require_secrets hcloud-token HCLOUD_TOKEN=$(read_secret hcloud-token); export HCLOUD_TOKEN cleanup_hcloudtest + require_secrets s3-access s3-secret + delete_cnpg_test_bucket # The restore test's read-only deploy keys on the GitHub mirror. if command -v gh >/dev/null; then restore_test_repo @@ -1834,8 +1985,8 @@ bao_exec() { # OpenBao — so Velero cannot be the thing that restores OpenBao. OPENBAO_KOPIA_BUCKET="renner-uno-k8s-velero" OPENBAO_KOPIA_PREFIX="kopia/openbao/" -OPENBAO_S3_ENDPOINT="fsn1.your-objectstorage.com" -OPENBAO_S3_REGION="fsn1" +OPENBAO_S3_ENDPOINT="$S3_ENDPOINT" +OPENBAO_S3_REGION="$S3_REGION" # Velero's hardcoded backup repository password (pkg/repository/keys/keys.go). OPENBAO_KOPIA_PASSWORD="static-passw0rd" @@ -2208,6 +2359,15 @@ if phase_runs flux; then fi unset seal_key existing + # Before Flux: cnpg-object-stores fails without it rather than render empty + # paths, and the paths must exist before any Cluster bootstraps. + if [[ "$FLUX_SOURCE" == "restore-test" ]]; then + create_cnpg_test_bucket + write_cnpg_paths "$CNPG_TEST_BUCKET" + else + write_cnpg_paths "$CNPG_BUCKET" + fi + if [[ "$FLUX_SOURCE" == "restore-test" ]]; then # The production tree through kubernetes/restore-test/flux-root.yaml, # whose patches are the test's whole deviation from production. Nothing diff --git a/kubernetes/clusters/homelab/authentik.yaml b/kubernetes/clusters/homelab/authentik.yaml index b3650e34..571c8a6b 100644 --- a/kubernetes/clusters/homelab/authentik.yaml +++ b/kubernetes/clusters/homelab/authentik.yaml @@ -24,7 +24,7 @@ spec: - name: infrastructure - name: infrastructure-crds - name: infrastructure-ceph-cluster - - name: infrastructure-cnpg-barman + - name: cnpg-object-stores # the ObjectStores its Cluster names - name: gitea # server, LDAP outpost and gate-sync images are in Gitea's registry healthCheckExprs: - apiVersion: postgresql.cnpg.io/v1 diff --git a/kubernetes/clusters/homelab/cnpg-object-stores.yaml b/kubernetes/clusters/homelab/cnpg-object-stores.yaml new file mode 100644 index 00000000..29c6df13 --- /dev/null +++ b/kubernetes/clusters/homelab/cnpg-object-stores.yaml @@ -0,0 +1,27 @@ +# The CNPG ObjectStores. Their paths come from the cnpg-paths ConfigMap, which +# bootstrap.sh writes once per cluster incarnation and never replaces: +# CNPG__SOURCE the bucket prefix holding 's newest base backup +# CNPG__ARCHIVE a new, empty prefix -g +# Not optional: without it every path renders empty. +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: cnpg-object-stores + namespace: flux-system +spec: + interval: 10m + retryInterval: 1m + timeout: 5m + sourceRef: + kind: GitRepository + name: flux-system + path: ./kubernetes/cnpg-object-stores + prune: true + wait: true + dependsOn: + - name: infrastructure + - name: infrastructure-cnpg-barman + postBuild: + substituteFrom: + - kind: ConfigMap + name: cnpg-paths diff --git a/kubernetes/clusters/homelab/home-assistant.yaml b/kubernetes/clusters/homelab/home-assistant.yaml index e963fdce..8750bcc5 100644 --- a/kubernetes/clusters/homelab/home-assistant.yaml +++ b/kubernetes/clusters/homelab/home-assistant.yaml @@ -27,19 +27,7 @@ spec: - name: infrastructure - name: infrastructure-crds - name: infrastructure-ceph-cluster - # ha-pg-objectstore.yaml declares an ObjectStore, whose CRD comes from the - # CNPG Barman plugin in infrastructure-cnpg-barman. Without this edge the - # kind is unknown when this path is first applied, and a server-side - # dry-run rejects the ENTIRE set over the one object it cannot recognise: - # - # ObjectStore/home-assistant/ha-pg-store dry-run failed: - # no matches for kind "ObjectStore" in version ... - # - # Every other ObjectStore user — authentik, immich, paperless, trmnl, - # woodpecker — already declares it. This was the only one that did not, and - # it took speech-to-phrase, zone-configurator and matter-server down with - # it, since all three wait on home-assistant. - - name: infrastructure-cnpg-barman + - name: cnpg-object-stores # the ObjectStores its Cluster names - name: gitea # shelly-manager, mqtt-account-controller images are in Gitea's registry - name: influxdb # HA's InfluxDB integration writes there --- diff --git a/kubernetes/clusters/homelab/immich.yaml b/kubernetes/clusters/homelab/immich.yaml index f3f4d5d4..4afdbcf2 100644 --- a/kubernetes/clusters/homelab/immich.yaml +++ b/kubernetes/clusters/homelab/immich.yaml @@ -24,7 +24,7 @@ spec: - name: infrastructure - name: infrastructure-crds - name: infrastructure-ceph-cluster - - name: infrastructure-cnpg-barman + - name: cnpg-object-stores # the ObjectStores its Cluster names healthCheckExprs: - apiVersion: postgresql.cnpg.io/v1 kind: Cluster diff --git a/kubernetes/clusters/homelab/matrix.yaml b/kubernetes/clusters/homelab/matrix.yaml index e4499e01..ff54b15d 100644 --- a/kubernetes/clusters/homelab/matrix.yaml +++ b/kubernetes/clusters/homelab/matrix.yaml @@ -21,7 +21,7 @@ spec: - name: infrastructure - name: infrastructure-crds - name: infrastructure-ceph-cluster - - name: infrastructure-cnpg-barman + - name: cnpg-object-stores # the ObjectStores its Cluster names healthCheckExprs: - apiVersion: postgresql.cnpg.io/v1 kind: Cluster diff --git a/kubernetes/clusters/homelab/paperless.yaml b/kubernetes/clusters/homelab/paperless.yaml index b346c56e..c0ad9b7e 100644 --- a/kubernetes/clusters/homelab/paperless.yaml +++ b/kubernetes/clusters/homelab/paperless.yaml @@ -24,7 +24,7 @@ spec: - name: infrastructure - name: infrastructure-crds - name: infrastructure-ceph-cluster - - name: infrastructure-cnpg-barman + - name: cnpg-object-stores # the ObjectStores its Cluster names - name: gitea # ftp-scanner (vsftpd) image is in Gitea's registry healthCheckExprs: - apiVersion: postgresql.cnpg.io/v1 diff --git a/kubernetes/clusters/homelab/trmnl.yaml b/kubernetes/clusters/homelab/trmnl.yaml index 3d61d32c..eeb2d3bb 100644 --- a/kubernetes/clusters/homelab/trmnl.yaml +++ b/kubernetes/clusters/homelab/trmnl.yaml @@ -21,7 +21,7 @@ spec: - name: infrastructure - name: infrastructure-crds - name: infrastructure-ceph-cluster - - name: infrastructure-cnpg-barman + - name: cnpg-object-stores # the ObjectStores its Cluster names - name: home-assistant # the renderer reads HA's API healthCheckExprs: - apiVersion: postgresql.cnpg.io/v1 diff --git a/kubernetes/clusters/homelab/woodpecker.yaml b/kubernetes/clusters/homelab/woodpecker.yaml index 10662621..4c68137b 100644 --- a/kubernetes/clusters/homelab/woodpecker.yaml +++ b/kubernetes/clusters/homelab/woodpecker.yaml @@ -24,7 +24,7 @@ spec: - name: infrastructure - name: infrastructure-crds - name: infrastructure-ceph-cluster - - name: infrastructure-cnpg-barman + - name: cnpg-object-stores # the ObjectStores its Cluster names - name: gitea # its forge and login provider healthCheckExprs: - apiVersion: postgresql.cnpg.io/v1 diff --git a/kubernetes/cnpg-object-stores/ak-pg.yaml b/kubernetes/cnpg-object-stores/ak-pg.yaml new file mode 100644 index 00000000..b466a9b7 --- /dev/null +++ b/kubernetes/cnpg-object-stores/ak-pg.yaml @@ -0,0 +1,54 @@ +--- +apiVersion: barmancloud.cnpg.io/v1 +kind: ObjectStore +metadata: + name: ak-pg-store + namespace: authentik + annotations: + kustomize.toolkit.fluxcd.io/prune: disabled +spec: + configuration: + destinationPath: ${CNPG_AK_PG_ARCHIVE} + endpointURL: https://fsn1.your-objectstorage.com + s3Credentials: + accessKeyId: + name: hetzner-s3-creds + key: ACCESS_KEY_ID + secretAccessKey: + name: hetzner-s3-creds + key: ACCESS_SECRET_KEY + region: + name: hetzner-s3-creds + key: REGION + wal: + compression: gzip + data: + compression: gzip + retentionPolicy: "30d" +--- +apiVersion: barmancloud.cnpg.io/v1 +kind: ObjectStore +metadata: + name: ak-pg-store-source + namespace: authentik + annotations: + kustomize.toolkit.fluxcd.io/prune: disabled + cnpg.renner.uno/server-name: ak-pg +spec: + configuration: + destinationPath: ${CNPG_AK_PG_SOURCE} + endpointURL: https://fsn1.your-objectstorage.com + s3Credentials: + accessKeyId: + name: hetzner-s3-creds + key: ACCESS_KEY_ID + secretAccessKey: + name: hetzner-s3-creds + key: ACCESS_SECRET_KEY + region: + name: hetzner-s3-creds + key: REGION + wal: + compression: gzip + data: + compression: gzip diff --git a/kubernetes/cnpg-object-stores/ha-pg.yaml b/kubernetes/cnpg-object-stores/ha-pg.yaml new file mode 100644 index 00000000..c32f4cdc --- /dev/null +++ b/kubernetes/cnpg-object-stores/ha-pg.yaml @@ -0,0 +1,54 @@ +--- +apiVersion: barmancloud.cnpg.io/v1 +kind: ObjectStore +metadata: + name: ha-pg-store + namespace: home-assistant + annotations: + kustomize.toolkit.fluxcd.io/prune: disabled +spec: + configuration: + destinationPath: ${CNPG_HA_PG_ARCHIVE} + endpointURL: https://fsn1.your-objectstorage.com + s3Credentials: + accessKeyId: + name: hetzner-s3-creds + key: ACCESS_KEY_ID + secretAccessKey: + name: hetzner-s3-creds + key: ACCESS_SECRET_KEY + region: + name: hetzner-s3-creds + key: REGION + wal: + compression: gzip + data: + compression: gzip + retentionPolicy: "30d" +--- +apiVersion: barmancloud.cnpg.io/v1 +kind: ObjectStore +metadata: + name: ha-pg-store-source + namespace: home-assistant + annotations: + kustomize.toolkit.fluxcd.io/prune: disabled + cnpg.renner.uno/server-name: ha-pg +spec: + configuration: + destinationPath: ${CNPG_HA_PG_SOURCE} + endpointURL: https://fsn1.your-objectstorage.com + s3Credentials: + accessKeyId: + name: hetzner-s3-creds + key: ACCESS_KEY_ID + secretAccessKey: + name: hetzner-s3-creds + key: ACCESS_SECRET_KEY + region: + name: hetzner-s3-creds + key: REGION + wal: + compression: gzip + data: + compression: gzip diff --git a/kubernetes/cnpg-object-stores/im-pg.yaml b/kubernetes/cnpg-object-stores/im-pg.yaml new file mode 100644 index 00000000..a4e4025e --- /dev/null +++ b/kubernetes/cnpg-object-stores/im-pg.yaml @@ -0,0 +1,54 @@ +--- +apiVersion: barmancloud.cnpg.io/v1 +kind: ObjectStore +metadata: + name: im-pg-store + namespace: immich + annotations: + kustomize.toolkit.fluxcd.io/prune: disabled +spec: + configuration: + destinationPath: ${CNPG_IM_PG_ARCHIVE} + endpointURL: https://fsn1.your-objectstorage.com + s3Credentials: + accessKeyId: + name: hetzner-s3-creds + key: ACCESS_KEY_ID + secretAccessKey: + name: hetzner-s3-creds + key: ACCESS_SECRET_KEY + region: + name: hetzner-s3-creds + key: REGION + wal: + compression: gzip + data: + compression: gzip + retentionPolicy: "30d" +--- +apiVersion: barmancloud.cnpg.io/v1 +kind: ObjectStore +metadata: + name: im-pg-store-source + namespace: immich + annotations: + kustomize.toolkit.fluxcd.io/prune: disabled + cnpg.renner.uno/server-name: im-pg +spec: + configuration: + destinationPath: ${CNPG_IM_PG_SOURCE} + endpointURL: https://fsn1.your-objectstorage.com + s3Credentials: + accessKeyId: + name: hetzner-s3-creds + key: ACCESS_KEY_ID + secretAccessKey: + name: hetzner-s3-creds + key: ACCESS_SECRET_KEY + region: + name: hetzner-s3-creds + key: REGION + wal: + compression: gzip + data: + compression: gzip diff --git a/kubernetes/cnpg-object-stores/kustomization.yaml b/kubernetes/cnpg-object-stores/kustomization.yaml new file mode 100644 index 00000000..585621c7 --- /dev/null +++ b/kubernetes/cnpg-object-stores/kustomization.yaml @@ -0,0 +1,24 @@ +# Every CNPG cluster's two ObjectStores: -store, where it archives WAL and +# takes base backups, and -store-source, which its bootstrap.recovery reads. +# +# Their destinationPaths are Flux variables from the cnpg-paths ConfigMap +# (clusters/homelab/cnpg-object-stores.yaml). Nothing else may live in this +# directory: postBuild substitution blanks every ${...} it renders. +# +# The source store carries cnpg.renner.uno/server-name, the cluster whose +# backups bootstrap.sh looks for, and optionally +# cnpg.renner.uno/source-prefixes, bucket prefixes outside the and +# -g pattern that may still hold them. +# +# No SSE: Hetzner supports only SSE-C, unusable by barman-cloud (plugin #646), +# so at-rest encryption is unavailable; transit is TLS. +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ak-pg.yaml + - ha-pg.yaml + - im-pg.yaml + - mx-pg.yaml + - pprl-pg.yaml + - tr-pg.yaml + - wp-pg.yaml diff --git a/kubernetes/cnpg-object-stores/mx-pg.yaml b/kubernetes/cnpg-object-stores/mx-pg.yaml new file mode 100644 index 00000000..21b16b73 --- /dev/null +++ b/kubernetes/cnpg-object-stores/mx-pg.yaml @@ -0,0 +1,54 @@ +--- +apiVersion: barmancloud.cnpg.io/v1 +kind: ObjectStore +metadata: + name: mx-pg-store + namespace: matrix + annotations: + kustomize.toolkit.fluxcd.io/prune: disabled +spec: + configuration: + destinationPath: ${CNPG_MX_PG_ARCHIVE} + endpointURL: https://fsn1.your-objectstorage.com + s3Credentials: + accessKeyId: + name: hetzner-s3-creds + key: ACCESS_KEY_ID + secretAccessKey: + name: hetzner-s3-creds + key: ACCESS_SECRET_KEY + region: + name: hetzner-s3-creds + key: REGION + wal: + compression: gzip + data: + compression: gzip + retentionPolicy: "30d" +--- +apiVersion: barmancloud.cnpg.io/v1 +kind: ObjectStore +metadata: + name: mx-pg-store-source + namespace: matrix + annotations: + kustomize.toolkit.fluxcd.io/prune: disabled + cnpg.renner.uno/server-name: mx-pg +spec: + configuration: + destinationPath: ${CNPG_MX_PG_SOURCE} + endpointURL: https://fsn1.your-objectstorage.com + s3Credentials: + accessKeyId: + name: hetzner-s3-creds + key: ACCESS_KEY_ID + secretAccessKey: + name: hetzner-s3-creds + key: ACCESS_SECRET_KEY + region: + name: hetzner-s3-creds + key: REGION + wal: + compression: gzip + data: + compression: gzip diff --git a/kubernetes/cnpg-object-stores/pprl-pg.yaml b/kubernetes/cnpg-object-stores/pprl-pg.yaml new file mode 100644 index 00000000..aae6cb6f --- /dev/null +++ b/kubernetes/cnpg-object-stores/pprl-pg.yaml @@ -0,0 +1,54 @@ +--- +apiVersion: barmancloud.cnpg.io/v1 +kind: ObjectStore +metadata: + name: pprl-pg-store + namespace: paperless + annotations: + kustomize.toolkit.fluxcd.io/prune: disabled +spec: + configuration: + destinationPath: ${CNPG_PPRL_PG_ARCHIVE} + endpointURL: https://fsn1.your-objectstorage.com + s3Credentials: + accessKeyId: + name: hetzner-s3-creds + key: ACCESS_KEY_ID + secretAccessKey: + name: hetzner-s3-creds + key: ACCESS_SECRET_KEY + region: + name: hetzner-s3-creds + key: REGION + wal: + compression: gzip + data: + compression: gzip + retentionPolicy: "30d" +--- +apiVersion: barmancloud.cnpg.io/v1 +kind: ObjectStore +metadata: + name: pprl-pg-store-source + namespace: paperless + annotations: + kustomize.toolkit.fluxcd.io/prune: disabled + cnpg.renner.uno/server-name: pprl-pg +spec: + configuration: + destinationPath: ${CNPG_PPRL_PG_SOURCE} + endpointURL: https://fsn1.your-objectstorage.com + s3Credentials: + accessKeyId: + name: hetzner-s3-creds + key: ACCESS_KEY_ID + secretAccessKey: + name: hetzner-s3-creds + key: ACCESS_SECRET_KEY + region: + name: hetzner-s3-creds + key: REGION + wal: + compression: gzip + data: + compression: gzip diff --git a/kubernetes/cnpg-object-stores/tr-pg.yaml b/kubernetes/cnpg-object-stores/tr-pg.yaml new file mode 100644 index 00000000..d8b93654 --- /dev/null +++ b/kubernetes/cnpg-object-stores/tr-pg.yaml @@ -0,0 +1,54 @@ +--- +apiVersion: barmancloud.cnpg.io/v1 +kind: ObjectStore +metadata: + name: tr-pg-store + namespace: trmnl + annotations: + kustomize.toolkit.fluxcd.io/prune: disabled +spec: + configuration: + destinationPath: ${CNPG_TR_PG_ARCHIVE} + endpointURL: https://fsn1.your-objectstorage.com + s3Credentials: + accessKeyId: + name: hetzner-s3-creds + key: ACCESS_KEY_ID + secretAccessKey: + name: hetzner-s3-creds + key: ACCESS_SECRET_KEY + region: + name: hetzner-s3-creds + key: REGION + wal: + compression: gzip + data: + compression: gzip + retentionPolicy: "30d" +--- +apiVersion: barmancloud.cnpg.io/v1 +kind: ObjectStore +metadata: + name: tr-pg-store-source + namespace: trmnl + annotations: + kustomize.toolkit.fluxcd.io/prune: disabled + cnpg.renner.uno/server-name: tr-pg +spec: + configuration: + destinationPath: ${CNPG_TR_PG_SOURCE} + endpointURL: https://fsn1.your-objectstorage.com + s3Credentials: + accessKeyId: + name: hetzner-s3-creds + key: ACCESS_KEY_ID + secretAccessKey: + name: hetzner-s3-creds + key: ACCESS_SECRET_KEY + region: + name: hetzner-s3-creds + key: REGION + wal: + compression: gzip + data: + compression: gzip diff --git a/kubernetes/apps/woodpecker/objectstore.yaml b/kubernetes/cnpg-object-stores/wp-pg.yaml similarity index 63% rename from kubernetes/apps/woodpecker/objectstore.yaml rename to kubernetes/cnpg-object-stores/wp-pg.yaml index ef132eb7..e4f7f6e6 100644 --- a/kubernetes/apps/woodpecker/objectstore.yaml +++ b/kubernetes/cnpg-object-stores/wp-pg.yaml @@ -1,46 +1,14 @@ --- -# RECOVERY SOURCE — read-only. The prefix wp-pg archives to (wp-pg-store below). -# -# serverName on the consuming externalCluster must be `wp-pg`, the name of the -# cluster that produced these backups. -apiVersion: barmancloud.cnpg.io/v1 -kind: ObjectStore -metadata: - name: wp-pg-store-source - namespace: woodpecker -spec: - configuration: - destinationPath: s3://renner-uno-k8s-cnpg/wp-pg-woodpecker - endpointURL: https://fsn1.your-objectstorage.com - s3Credentials: - accessKeyId: - name: hetzner-s3-creds - key: ACCESS_KEY_ID - secretAccessKey: - name: hetzner-s3-creds - key: ACCESS_SECRET_KEY - region: - name: hetzner-s3-creds - key: REGION - wal: - compression: gzip - data: - compression: gzip - # No retentionPolicy: wp-pg-store owns the lifecycle of this prefix. ---- -# ARCHIVING TARGET. -# -# Not s3://renner-uno-k8s-cnpg/wp-pg: the retired `apps/wp-pg` archived there, -# and two clusters archiving WAL to one destinationPath interleave their streams -# and corrupt both. apiVersion: barmancloud.cnpg.io/v1 kind: ObjectStore metadata: name: wp-pg-store namespace: woodpecker + annotations: + kustomize.toolkit.fluxcd.io/prune: disabled spec: configuration: - destinationPath: s3://renner-uno-k8s-cnpg/wp-pg-woodpecker + destinationPath: ${CNPG_WP_PG_ARCHIVE} endpointURL: https://fsn1.your-objectstorage.com s3Credentials: accessKeyId: @@ -57,3 +25,32 @@ spec: data: compression: gzip retentionPolicy: "30d" +--- +apiVersion: barmancloud.cnpg.io/v1 +kind: ObjectStore +metadata: + name: wp-pg-store-source + namespace: woodpecker + annotations: + kustomize.toolkit.fluxcd.io/prune: disabled + cnpg.renner.uno/server-name: wp-pg + # wp-pg's archive since its move out of `apps`. + cnpg.renner.uno/source-prefixes: wp-pg-woodpecker +spec: + configuration: + destinationPath: ${CNPG_WP_PG_SOURCE} + endpointURL: https://fsn1.your-objectstorage.com + s3Credentials: + accessKeyId: + name: hetzner-s3-creds + key: ACCESS_KEY_ID + secretAccessKey: + name: hetzner-s3-creds + key: ACCESS_SECRET_KEY + region: + name: hetzner-s3-creds + key: REGION + wal: + compression: gzip + data: + compression: gzip -- 2.54.0 From 420bd35e7fca6dce1f4481e2c5df8c568208c326 Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Fri, 16 Oct 2026 21:36:01 +0200 Subject: [PATCH 35/42] feat(iot): v30iot workloads run only where a node declares the IoT VLAN; HA no longer waits on them Five workloads attach a macvlan on ens19 (v30iot) with DHCP IPAM: shelly-manager and the mDNS relay (inside the home-assistant Kustomization), matter-server, music-assistant and otbr. On a node without that VLAN the CNI add fails and the pod never starts. - k8s-1..3 carry net.renner.uno/iot-vlan: "true" beside the GPU label; each of them declares ens19 in its VLANConfig. The five Deployments select on it. The comments they replace described the retired k8s-edge taint. - shelly-manager and the mDNS relay move to apps/home-assistant-iot, with a Kustomization of their own (depends on restore-home-assistant, gitea, home-assistant and the infrastructure Kustomizations). home-assistant's wait: true no longer covers them, so HA and everything waiting on it (evcc, trmnl, zone-configurator, speech-to-phrase) are Ready without the VLAN, and a broken IoT bridge no longer holds them back in production either. On a site without the label the IoT Kustomizations fail at their timeout, naming the Pending pods; nothing else depends on them. Verified: home-assistant, home-assistant-iot, matter-server, music-assistant, otbr, authentik and monitoring-rules build; the home-assistant render has no IoT attachment left; every IoT workload carries the selector; k8s-1 with site-homelab passes talosctl validate --strict. Production rollout: 1. Label the nodes first (talosctl patch the nodeLabels), or the five pods go Pending. 2. The move between Kustomizations is two-phase (add under home-assistant-iot, then drop from home-assistant). shelly-manager's PVC already carries prune: disabled. Co-Authored-By: Claude Opus 5.5 --- .../apps/authentik/blueprints-configmap.yaml | 2 +- .../home-assistant-iot/kustomization.yaml | 34 +++++++++++++++++++ .../mdns-relay-deployment.yaml | 3 ++ .../mdns-relay-network-attachment.yaml | 0 .../mdns-relay-service.yaml | 0 .../shelly-ble-proxy-key-externalsecret.yaml | 0 .../shelly-manager-api-proxy.conf | 0 .../shelly-manager-auth-externalsecret.yaml | 0 .../shelly-manager-deployment.yaml | 8 ++--- .../shelly-manager-firmware-rbac.yaml | 0 .../shelly-manager-ingress.yaml | 0 .../shelly-manager-llat-externalsecret.yaml | 0 .../shelly-manager-network-attachment.yaml | 0 .../shelly-manager-pvc.yaml | 0 .../shelly-manager-service.yaml | 0 .../apps/home-assistant/kustomization.yaml | 25 +------------- kubernetes/apps/matter-server/deployment.yaml | 8 ++--- .../monitoring-rules/probes-blackbox.yaml | 4 +-- .../apps/music-assistant/deployment.yaml | 9 ++--- kubernetes/apps/otbr/deployment.yaml | 9 +++-- .../clusters/homelab/home-assistant.yaml | 26 +++++++++++++- kubernetes/talos/patches/k8s-1.yaml | 3 ++ kubernetes/talos/patches/k8s-2.yaml | 2 ++ kubernetes/talos/patches/k8s-3.yaml | 2 ++ 24 files changed, 87 insertions(+), 48 deletions(-) create mode 100644 kubernetes/apps/home-assistant-iot/kustomization.yaml rename kubernetes/apps/{home-assistant => home-assistant-iot}/mdns-relay-deployment.yaml (98%) rename kubernetes/apps/{home-assistant => home-assistant-iot}/mdns-relay-network-attachment.yaml (100%) rename kubernetes/apps/{home-assistant => home-assistant-iot}/mdns-relay-service.yaml (100%) rename kubernetes/apps/{home-assistant => home-assistant-iot}/shelly-ble-proxy-key-externalsecret.yaml (100%) rename kubernetes/apps/{home-assistant => home-assistant-iot}/shelly-manager-api-proxy.conf (100%) rename kubernetes/apps/{home-assistant => home-assistant-iot}/shelly-manager-auth-externalsecret.yaml (100%) rename kubernetes/apps/{home-assistant => home-assistant-iot}/shelly-manager-deployment.yaml (97%) rename kubernetes/apps/{home-assistant => home-assistant-iot}/shelly-manager-firmware-rbac.yaml (100%) rename kubernetes/apps/{home-assistant => home-assistant-iot}/shelly-manager-ingress.yaml (100%) rename kubernetes/apps/{home-assistant => home-assistant-iot}/shelly-manager-llat-externalsecret.yaml (100%) rename kubernetes/apps/{home-assistant => home-assistant-iot}/shelly-manager-network-attachment.yaml (100%) rename kubernetes/apps/{home-assistant => home-assistant-iot}/shelly-manager-pvc.yaml (100%) rename kubernetes/apps/{home-assistant => home-assistant-iot}/shelly-manager-service.yaml (100%) diff --git a/kubernetes/apps/authentik/blueprints-configmap.yaml b/kubernetes/apps/authentik/blueprints-configmap.yaml index 221f5b7b..775cd638 100644 --- a/kubernetes/apps/authentik/blueprints-configmap.yaml +++ b/kubernetes/apps/authentik/blueprints-configmap.yaml @@ -131,7 +131,7 @@ data: state: absent # shelly-manager.home.renner.uno is gated by authentik-gate-sync - # (kubernetes/apps/home-assistant/shelly-manager-ingress.yaml), which + # (kubernetes/apps/home-assistant-iot/shelly-manager-ingress.yaml), which # admits this group. Members are managed in Authentik. - model: authentik_core.group identifiers: diff --git a/kubernetes/apps/home-assistant-iot/kustomization.yaml b/kubernetes/apps/home-assistant-iot/kustomization.yaml new file mode 100644 index 00000000..9b1694ba --- /dev/null +++ b/kubernetes/apps/home-assistant-iot/kustomization.yaml @@ -0,0 +1,34 @@ +# The home-assistant workloads attached to the IoT VLAN (v30iot). Separate from +# ../home-assistant so that HA and what depends on it do not wait on a VLAN +# the node may not have; the pods run only where it does +# (net.renner.uno/iot-vlan). +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - shelly-manager-pvc.yaml + - shelly-manager-network-attachment.yaml + - shelly-manager-firmware-rbac.yaml + - shelly-ble-proxy-key-externalsecret.yaml + - shelly-manager-auth-externalsecret.yaml + - shelly-manager-llat-externalsecret.yaml + - shelly-manager-deployment.yaml + - shelly-manager-service.yaml + - shelly-manager-ingress.yaml + # mDNS relay — carries IoT-VLAN announcements into HA's zeroconf listener as + # unicast, which is the only direction the CNI will move them. Without it HA + # never learns that a device's DHCP lease moved. See the header comment in + # mdns-relay-deployment.yaml. + - mdns-relay-network-attachment.yaml + - mdns-relay-service.yaml + - mdns-relay-deployment.yaml +configMapGenerator: + # shelly-manager api-proxy sidecar config (REN-935). Hash suffix disabled + # (name stays stable), so editing shelly-manager-api-proxy.conf does not + # auto-roll the pod — restart manually with + # `kubectl rollout restart deploy/shelly-manager -n home-assistant`. + - name: shelly-manager-api-proxy + namespace: home-assistant + files: + - api.conf.template=shelly-manager-api-proxy.conf +generatorOptions: + disableNameSuffixHash: true diff --git a/kubernetes/apps/home-assistant/mdns-relay-deployment.yaml b/kubernetes/apps/home-assistant-iot/mdns-relay-deployment.yaml similarity index 98% rename from kubernetes/apps/home-assistant/mdns-relay-deployment.yaml rename to kubernetes/apps/home-assistant-iot/mdns-relay-deployment.yaml index cceabccb..a981372a 100644 --- a/kubernetes/apps/home-assistant/mdns-relay-deployment.yaml +++ b/kubernetes/apps/home-assistant-iot/mdns-relay-deployment.yaml @@ -47,6 +47,9 @@ spec: homelab.renner.uno/restarted-for: "2026-10-06 cni-plugins v1.9.1 attach check" k8s.v1.cni.cncf.io/networks: mdns-relay-iot spec: + # Runs where the node patch declares the IoT VLAN (ens19). + nodeSelector: + net.renner.uno/iot-vlan: "true" enableServiceLinks: false securityContext: runAsNonRoot: true diff --git a/kubernetes/apps/home-assistant/mdns-relay-network-attachment.yaml b/kubernetes/apps/home-assistant-iot/mdns-relay-network-attachment.yaml similarity index 100% rename from kubernetes/apps/home-assistant/mdns-relay-network-attachment.yaml rename to kubernetes/apps/home-assistant-iot/mdns-relay-network-attachment.yaml diff --git a/kubernetes/apps/home-assistant/mdns-relay-service.yaml b/kubernetes/apps/home-assistant-iot/mdns-relay-service.yaml similarity index 100% rename from kubernetes/apps/home-assistant/mdns-relay-service.yaml rename to kubernetes/apps/home-assistant-iot/mdns-relay-service.yaml diff --git a/kubernetes/apps/home-assistant/shelly-ble-proxy-key-externalsecret.yaml b/kubernetes/apps/home-assistant-iot/shelly-ble-proxy-key-externalsecret.yaml similarity index 100% rename from kubernetes/apps/home-assistant/shelly-ble-proxy-key-externalsecret.yaml rename to kubernetes/apps/home-assistant-iot/shelly-ble-proxy-key-externalsecret.yaml diff --git a/kubernetes/apps/home-assistant/shelly-manager-api-proxy.conf b/kubernetes/apps/home-assistant-iot/shelly-manager-api-proxy.conf similarity index 100% rename from kubernetes/apps/home-assistant/shelly-manager-api-proxy.conf rename to kubernetes/apps/home-assistant-iot/shelly-manager-api-proxy.conf diff --git a/kubernetes/apps/home-assistant/shelly-manager-auth-externalsecret.yaml b/kubernetes/apps/home-assistant-iot/shelly-manager-auth-externalsecret.yaml similarity index 100% rename from kubernetes/apps/home-assistant/shelly-manager-auth-externalsecret.yaml rename to kubernetes/apps/home-assistant-iot/shelly-manager-auth-externalsecret.yaml diff --git a/kubernetes/apps/home-assistant/shelly-manager-deployment.yaml b/kubernetes/apps/home-assistant-iot/shelly-manager-deployment.yaml similarity index 97% rename from kubernetes/apps/home-assistant/shelly-manager-deployment.yaml rename to kubernetes/apps/home-assistant-iot/shelly-manager-deployment.yaml index 1ef51599..a3335472 100644 --- a/kubernetes/apps/home-assistant/shelly-manager-deployment.yaml +++ b/kubernetes/apps/home-assistant-iot/shelly-manager-deployment.yaml @@ -47,10 +47,10 @@ spec: spec: # eth0 (Cilium-managed pod network) is still the primary — it # carries the ingress, mosquitto LB return path, etc. The macvlan - # above is additive. No explicit edge exclusion is needed: the - # `workload-class=pinned:NoSchedule` taint on k8s-edge already - # keeps this pod off there (we never tolerate it), and the NAD's - # `master: ens19` (stable predictable name for the v30iot NIC) is present on every k8s-{1,2,3} worker. + # above is additive. + # Runs where the node patch declares the IoT VLAN (ens19). + nodeSelector: + net.renner.uno/iot-vlan: "true" enableServiceLinks: false # Grants shelly-manager the in-cluster identity used by the firmware # pipeline to create esphome flash Jobs and stream their logs. RBAC is diff --git a/kubernetes/apps/home-assistant/shelly-manager-firmware-rbac.yaml b/kubernetes/apps/home-assistant-iot/shelly-manager-firmware-rbac.yaml similarity index 100% rename from kubernetes/apps/home-assistant/shelly-manager-firmware-rbac.yaml rename to kubernetes/apps/home-assistant-iot/shelly-manager-firmware-rbac.yaml diff --git a/kubernetes/apps/home-assistant/shelly-manager-ingress.yaml b/kubernetes/apps/home-assistant-iot/shelly-manager-ingress.yaml similarity index 100% rename from kubernetes/apps/home-assistant/shelly-manager-ingress.yaml rename to kubernetes/apps/home-assistant-iot/shelly-manager-ingress.yaml diff --git a/kubernetes/apps/home-assistant/shelly-manager-llat-externalsecret.yaml b/kubernetes/apps/home-assistant-iot/shelly-manager-llat-externalsecret.yaml similarity index 100% rename from kubernetes/apps/home-assistant/shelly-manager-llat-externalsecret.yaml rename to kubernetes/apps/home-assistant-iot/shelly-manager-llat-externalsecret.yaml diff --git a/kubernetes/apps/home-assistant/shelly-manager-network-attachment.yaml b/kubernetes/apps/home-assistant-iot/shelly-manager-network-attachment.yaml similarity index 100% rename from kubernetes/apps/home-assistant/shelly-manager-network-attachment.yaml rename to kubernetes/apps/home-assistant-iot/shelly-manager-network-attachment.yaml diff --git a/kubernetes/apps/home-assistant/shelly-manager-pvc.yaml b/kubernetes/apps/home-assistant-iot/shelly-manager-pvc.yaml similarity index 100% rename from kubernetes/apps/home-assistant/shelly-manager-pvc.yaml rename to kubernetes/apps/home-assistant-iot/shelly-manager-pvc.yaml diff --git a/kubernetes/apps/home-assistant/shelly-manager-service.yaml b/kubernetes/apps/home-assistant-iot/shelly-manager-service.yaml similarity index 100% rename from kubernetes/apps/home-assistant/shelly-manager-service.yaml rename to kubernetes/apps/home-assistant-iot/shelly-manager-service.yaml diff --git a/kubernetes/apps/home-assistant/kustomization.yaml b/kubernetes/apps/home-assistant/kustomization.yaml index 76f00839..f2267a6d 100644 --- a/kubernetes/apps/home-assistant/kustomization.yaml +++ b/kubernetes/apps/home-assistant/kustomization.yaml @@ -21,26 +21,11 @@ resources: - mosquitto-iot-service.yaml - mosquitto-deployment.yaml - mosquitto-evcc-credentials-externalsecret.yaml - - shelly-manager-pvc.yaml - - shelly-manager-network-attachment.yaml - - shelly-manager-firmware-rbac.yaml - - shelly-ble-proxy-key-externalsecret.yaml - - shelly-manager-auth-externalsecret.yaml - - shelly-manager-llat-externalsecret.yaml - - shelly-manager-deployment.yaml - - shelly-manager-service.yaml - - shelly-manager-ingress.yaml + # shelly-manager and the mDNS relay sit on the IoT VLAN: ../home-assistant-iot. - esphome-pvc.yaml - esphome-deployment.yaml - esphome-service.yaml - esphome-ingress.yaml - # mDNS relay — carries IoT-VLAN announcements into HA's zeroconf listener as - # unicast, which is the only direction the CNI will move them. Without it HA - # never learns that a device's DHCP lease moved. See the header comment in - # mdns-relay-deployment.yaml. - - mdns-relay-network-attachment.yaml - - mdns-relay-service.yaml - - mdns-relay-deployment.yaml # Zigbee2MQTT — ZHA replacement bridge. Ships scaled to 0; the cutover # runbook (zigbee2mqtt-migration-runbook.md) scales it up. The MQTT # `zigbee2mqtt` broker user is provisioned by mosquitto-bootstrap.yaml @@ -115,13 +100,5 @@ configMapGenerator: namespace: home-assistant files: - mqtt-accounts.yaml - # shelly-manager api-proxy sidecar config (REN-935). Hash suffix disabled - # (name stays stable), so editing shelly-manager-api-proxy.conf does not - # auto-roll the pod — restart manually with - # `kubectl rollout restart deploy/shelly-manager -n home-assistant`. - - name: shelly-manager-api-proxy - namespace: home-assistant - files: - - api.conf.template=shelly-manager-api-proxy.conf generatorOptions: disableNameSuffixHash: true diff --git a/kubernetes/apps/matter-server/deployment.yaml b/kubernetes/apps/matter-server/deployment.yaml index 9621177d..2587aec9 100644 --- a/kubernetes/apps/matter-server/deployment.yaml +++ b/kubernetes/apps/matter-server/deployment.yaml @@ -32,11 +32,9 @@ spec: # OTBR via OTBR's ClusterIP Service). The macvlan attachment # above is additive, not a replacement. # - # No nodeSelector/toleration: this pod only needs v30iot (present - # on all nodes), not the edge USB devices. Dropping the - # workload-class=pinned toleration lets the edge taint keep it off - # the (tiny) edge box, reserving that node for the USB-bound - # home-assistant + zwave-js pods. + # Runs where the node patch declares the IoT VLAN (ens19). + nodeSelector: + net.renner.uno/iot-vlan: "true" containers: - name: matter-server # Use the HAOS-published image from `ghcr.io/matter-js/...`, diff --git a/kubernetes/apps/monitoring-rules/probes-blackbox.yaml b/kubernetes/apps/monitoring-rules/probes-blackbox.yaml index f2009161..83df9a47 100644 --- a/kubernetes/apps/monitoring-rules/probes-blackbox.yaml +++ b/kubernetes/apps/monitoring-rules/probes-blackbox.yaml @@ -269,9 +269,9 @@ spec: # per the app's own test (shelly-manager repo, # AuthenticationTests.Health_IsUnauthenticated); it is also the pod's # kubelet liveness/readiness path - # (../home-assistant/shelly-manager-deployment.yaml). AllowedHosts is + # (../home-assistant-iot/shelly-manager-deployment.yaml). AllowedHosts is # "*", so the cluster-local Host header is accepted. Service :80 -> - # container :8080 (../home-assistant/shelly-manager-service.yaml). + # container :8080 (../home-assistant-iot/shelly-manager-service.yaml). # # NO CiliumNetworkPolicy change: the home-assistant default-deny # admits the whole monitoring namespace on every port diff --git a/kubernetes/apps/music-assistant/deployment.yaml b/kubernetes/apps/music-assistant/deployment.yaml index 4dcecdb3..828ddd4e 100644 --- a/kubernetes/apps/music-assistant/deployment.yaml +++ b/kubernetes/apps/music-assistant/deployment.yaml @@ -33,12 +33,9 @@ spec: k8s.v1.cni.cncf.io/networks: music-assistant-iot spec: enableServiceLinks: false - # No nodeSelector/toleration: MA only needs a node on v30iot (all - # of them) for audio-sync L2, not the edge USB devices. It's the - # heaviest of the formerly edge-pinned pods (2Gi limit), so - # dropping the workload-class=pinned toleration — which lets the - # edge taint keep it off the tiny edge box — is the main capacity - # win for this step. + # Runs where the node patch declares the IoT VLAN (ens19). + nodeSelector: + net.renner.uno/iot-vlan: "true" initContainers: # MA advertises a stored publish_ip to players (Sendspin / Snapcast / # SlimProto) and binds its stream + Sendspin servers to a stored diff --git a/kubernetes/apps/otbr/deployment.yaml b/kubernetes/apps/otbr/deployment.yaml index d260664f..25381a25 100644 --- a/kubernetes/apps/otbr/deployment.yaml +++ b/kubernetes/apps/otbr/deployment.yaml @@ -30,11 +30,10 @@ spec: # the IoT VLAN for mDNS/SRP advertisements and RA listening. k8s.v1.cni.cncf.io/networks: otbr-iot spec: - # No nodeSelector/toleration: OTBR only needs v30iot L2 presence - # (macvlan master: ens19, present on every node), not the edge USB - # devices, and its Thread RCP is reached over the network - # (slzb-thread.renner.uno). Dropping the workload-class=pinned - # toleration lets the edge taint keep it off the tiny edge box. + # Runs where the node patch declares the IoT VLAN (ens19). Its Thread + # RCP is reached over the network (slzb-thread.renner.uno). + nodeSelector: + net.renner.uno/iot-vlan: "true" containers: # The Thread RCP is a network-attached radio (Serial-over-IP), so the # spinel transport needs a TCP-to-PTY bridge. Two obvious approaches diff --git a/kubernetes/clusters/homelab/home-assistant.yaml b/kubernetes/clusters/homelab/home-assistant.yaml index 8750bcc5..c1aab30c 100644 --- a/kubernetes/clusters/homelab/home-assistant.yaml +++ b/kubernetes/clusters/homelab/home-assistant.yaml @@ -28,7 +28,7 @@ spec: - name: infrastructure-crds - name: infrastructure-ceph-cluster - name: cnpg-object-stores # the ObjectStores its Cluster names - - name: gitea # shelly-manager, mqtt-account-controller images are in Gitea's registry + - name: gitea # mqtt-account-controller's image is in Gitea's registry - name: influxdb # HA's InfluxDB integration writes there --- apiVersion: kustomize.toolkit.fluxcd.io/v1 @@ -52,6 +52,30 @@ spec: - name: infrastructure-crds - name: infrastructure-ceph-cluster --- +# shelly-manager and the mDNS relay: home-assistant's IoT-VLAN workloads. +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: home-assistant-iot + namespace: flux-system +spec: + interval: 10m + retryInterval: 1m + timeout: 10m + sourceRef: + kind: GitRepository + name: flux-system + path: ./kubernetes/apps/home-assistant-iot + prune: true + wait: true + dependsOn: + - name: restore-home-assistant # shelly-manager's PVC + - name: infrastructure + - name: infrastructure-crds + - name: infrastructure-ceph-cluster + - name: gitea # shelly-manager's image is in Gitea's registry + - name: home-assistant # both serve HA: its API, its zeroconf listener +--- apiVersion: kustomize.toolkit.fluxcd.io/v1 kind: Kustomization metadata: diff --git a/kubernetes/talos/patches/k8s-1.yaml b/kubernetes/talos/patches/k8s-1.yaml index 0a12f186..3f7aa0de 100644 --- a/kubernetes/talos/patches/k8s-1.yaml +++ b/kubernetes/talos/patches/k8s-1.yaml @@ -57,6 +57,9 @@ machine: # This machine has the Intel iGPU. The GPU plugin and everything that needs # gpu.intel.com run only on nodes with this label. hardware.renner.uno/intel-gpu: "true" + # This machine is on the IoT VLAN: ens19, declared below in its VLANConfig. + # The macvlan workloads on v30iot run only on nodes with this label. + net.renner.uno/iot-vlan: "true" # Publishes this node's pod0 addresses, declared below in its VLANConfig, so # the bird DaemonSet reads its peers from the node list rather than from a # copy. Annotations, not labels: a label value cannot contain a colon. diff --git a/kubernetes/talos/patches/k8s-2.yaml b/kubernetes/talos/patches/k8s-2.yaml index fdc3484e..32b16c79 100644 --- a/kubernetes/talos/patches/k8s-2.yaml +++ b/kubernetes/talos/patches/k8s-2.yaml @@ -36,6 +36,8 @@ machine: storage.renner.uno/ceph-osd: "true" # Has the Intel iGPU — see k8s-1.yaml. hardware.renner.uno/intel-gpu: "true" + # See k8s-1.yaml. + net.renner.uno/iot-vlan: "true" # pod0 addresses for the bird peer list — see k8s-1.yaml. nodeAnnotations: net.renner.uno/pod0-v6: fd02:48::12 diff --git a/kubernetes/talos/patches/k8s-3.yaml b/kubernetes/talos/patches/k8s-3.yaml index 050297d4..20012676 100644 --- a/kubernetes/talos/patches/k8s-3.yaml +++ b/kubernetes/talos/patches/k8s-3.yaml @@ -32,6 +32,8 @@ machine: storage.renner.uno/ceph-osd: "true" # Has the Intel iGPU — see k8s-1.yaml. hardware.renner.uno/intel-gpu: "true" + # See k8s-1.yaml. + net.renner.uno/iot-vlan: "true" # pod0 addresses for the bird peer list — see k8s-1.yaml. nodeAnnotations: net.renner.uno/pod0-v6: fd02:48::13 -- 2.54.0 From 4befc7c492724a5b5e737a9bd083af3c13fc11c0 Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Fri, 16 Oct 2026 21:36:04 +0200 Subject: [PATCH 36/42] Revert "feat(talos): hcloud nodes carry ens19 as a dummy link" This reverts commit ec8f496d. The v30iot workloads now select net.renner.uno/iot-vlan, which no hcloud node carries, so nothing on that site attaches to ens19 any more. Co-Authored-By: Claude Opus 5.5 --- kubernetes/talos/patches/hcloud-1.yaml | 7 ------- kubernetes/talos/patches/hcloud-2.yaml | 6 ------ kubernetes/talos/patches/hcloud-3.yaml | 6 ------ 3 files changed, 19 deletions(-) diff --git a/kubernetes/talos/patches/hcloud-1.yaml b/kubernetes/talos/patches/hcloud-1.yaml index 2dca05cd..13a7f6d6 100644 --- a/kubernetes/talos/patches/hcloud-1.yaml +++ b/kubernetes/talos/patches/hcloud-1.yaml @@ -55,13 +55,6 @@ kind: LinkConfig name: ceph-san up: true --- -# The v30iot macvlan parent the NADs name (master: ens19). Hetzner has no IoT -# VLAN, so it is a dummy link: no address, nothing behind it. -apiVersion: v1alpha1 -kind: DummyLinkConfig -name: ens19 -up: true ---- # Capped so the OSD partition fits beside it on cpx52's 480 GB local disk. apiVersion: v1alpha1 kind: VolumeConfig diff --git a/kubernetes/talos/patches/hcloud-2.yaml b/kubernetes/talos/patches/hcloud-2.yaml index a0356e53..32d3b3d0 100644 --- a/kubernetes/talos/patches/hcloud-2.yaml +++ b/kubernetes/talos/patches/hcloud-2.yaml @@ -42,12 +42,6 @@ up: true --- # See hcloud-1.yaml. apiVersion: v1alpha1 -kind: DummyLinkConfig -name: ens19 -up: true ---- -# See hcloud-1.yaml. -apiVersion: v1alpha1 kind: VolumeConfig name: EPHEMERAL provisioning: diff --git a/kubernetes/talos/patches/hcloud-3.yaml b/kubernetes/talos/patches/hcloud-3.yaml index 3fc6a6a8..cdcceb03 100644 --- a/kubernetes/talos/patches/hcloud-3.yaml +++ b/kubernetes/talos/patches/hcloud-3.yaml @@ -42,12 +42,6 @@ up: true --- # See hcloud-1.yaml. apiVersion: v1alpha1 -kind: DummyLinkConfig -name: ens19 -up: true ---- -# See hcloud-1.yaml. -apiVersion: v1alpha1 kind: VolumeConfig name: EPHEMERAL provisioning: -- 2.54.0 From 9dc3f5b4a0fcee4297999f796a5b6abbf09604e4 Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Fri, 16 Oct 2026 21:38:07 +0200 Subject: [PATCH 37/42] fix(home-assistant): 30m timeout, not 4h The 4h was sized for a seed initContainer that waited up to 3h for an HAOS backup to appear in Garage, then downloaded and staged it. That step no longer exists: Velero restores the config PVC in restore-home-assistant before this Kustomization runs, and Garage is retired. The HA container's startupProbe allows 10 minutes. The slowest part left is ha-pg recovering from barman: its newest base backup (ha-pg/ha-pg/base/20261016T025000) is 1.7 GiB compressed, then WAL replay and the second instance's clone. 30m, as the other CNPG apps have; the next restore test measures it. With wait: true, the 4h also held one of kustomize-controller's workers that long on a failed cold start, and with it evcc, trmnl, zone-configurator and speech-to-phrase. Co-Authored-By: Claude Opus 5.5 --- kubernetes/clusters/homelab/home-assistant.yaml | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/kubernetes/clusters/homelab/home-assistant.yaml b/kubernetes/clusters/homelab/home-assistant.yaml index c1aab30c..62def0d2 100644 --- a/kubernetes/clusters/homelab/home-assistant.yaml +++ b/kubernetes/clusters/homelab/home-assistant.yaml @@ -4,16 +4,12 @@ metadata: name: home-assistant namespace: flux-system spec: - # Seed Job collapsed into the HA pod's `seed` initContainer so the - # backup tarball + .HA_RESTORE manifest staging happens in the same - # pod as HA Core (no Multi-Attach race against the RWO Ceph-RBD - # PVC if production grows to multiple edge nodes). HA Core still - # does the actual restore from the staged manifest on its boot. - # Timeout has to absorb the worst-case 3h wait for the HAOS backup - # to appear in Garage plus the in-pod download + HA's native restore. + # The slowest part on a cold start is ha-pg recovering from barman (base + # backup ~1.7 GiB compressed, then WAL replay); the config PVC is restored + # beforehand by restore-home-assistant. interval: 10m retryInterval: 1m - timeout: 4h + timeout: 30m sourceRef: kind: GitRepository name: flux-system -- 2.54.0 From 12a0f8c000fcf300eee42fb60582a1adc1e01501 Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Fri, 16 Oct 2026 21:39:07 +0200 Subject: [PATCH 38/42] fix(flux): renovate depends on gitea Renovate's platform is Gitea: endpoint https://git.renner.uno/api/v1 in apps/renovate/configmap.yaml, the same instance as gitea.renner.uno. It has no function without it. Co-Authored-By: Claude Opus 5.5 --- kubernetes/clusters/homelab/renovate.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/kubernetes/clusters/homelab/renovate.yaml b/kubernetes/clusters/homelab/renovate.yaml index 6e0173fe..a3c5e283 100644 --- a/kubernetes/clusters/homelab/renovate.yaml +++ b/kubernetes/clusters/homelab/renovate.yaml @@ -35,3 +35,4 @@ spec: # The `openbao` ClusterSecretStore, including the `renovate` entry in its # conditions.namespaces allowlist. - name: infrastructure-crds + - name: gitea # the platform it runs against (git.renner.uno) -- 2.54.0 From 9b488b037e91e47e423b14d0f1741b633f7f54f2 Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Fri, 16 Oct 2026 22:01:55 +0200 Subject: [PATCH 39/42] fix(flux): services that need OpenBao secrets apply after the secret store crowdsec, external-dns-unifi, netbird-ingress and velero were in the infrastructure Kustomization, while the ExternalSecrets that create their Secrets were in infrastructure-crds, which depends on infrastructure. On a cold start each was therefore installed before its Secret could exist: the pods sat in CreateContainerConfigError until OpenBao was unsealed and ESO synced. For the HelmReleases (crowdsec, external-dns-unifi, velero) Helm's wait timed out meanwhile; with remediation.retries: 3 they give up after four attempts and stay failed even once the Secret appears. Velero failing that way stops every restore. | workload | Secrets | |---|---| | crowdsec | crowdsec-bouncer-key (crowdsec, ingress), crowdsec-edge-bouncer-key | | external-dns-unifi | external-dns-unifi-secret | | netbird-ingress | netbird-ingress-setup-key | | velero (+ backup-verify CronJob) | cloud-credentials, cloud-credentials-hetzner, hcloud-backup-verify | Each now has a Kustomization of its own (clusters/homelab/ infrastructure-services.yaml), depending on infrastructure and infrastructure-crds, with its ExternalSecrets beside it. The openbao ClusterSecretStore in infrastructure-crds turns Ready only once OpenBao is unsealed (a5c5d0e7), so the Secret exists before the consumer installs. One Kustomization each so that a broken service holds back only itself. Velero's Schedules need the velero.io CRDs from its HelmRelease, so they move from post-crds to infrastructure/velero-schedules with a Kustomization after velero. velero-backups-synced depends on velero (the BackupStorageLocation and its credentials) instead of infrastructure-crds. post-crds keeps cloudflare-api-token (consumed by its own ClusterIssuer) and flux-git. The directories themselves stay where they are, so paths such as the CI test infrastructure/velero/backup-verify/tests/run.sh are unchanged. Verified: infrastructure, post-crds and the six service paths build; infrastructure renders no ExternalSecret. Production rollout: moving objects out of the pruned infrastructure and infrastructure-crds Kustomizations is two-phase (add the new Kustomizations first, then drop the old entries). Pruning the velero HelmRelease would uninstall Velero. Co-Authored-By: Claude Opus 5.5 --- .../homelab/infrastructure-services.yaml | 98 +++++++++++++++++++ kubernetes/clusters/homelab/restore.yaml | 4 +- .../crowdsec-bouncer-key-externalsecret.yaml | 0 ...wdsec-edge-bouncer-key-externalsecret.yaml | 0 .../crowdsec/kustomization.yaml | 2 + .../externalsecret.yaml} | 0 .../external-dns-unifi/kustomization.yaml | 1 + kubernetes/infrastructure/kustomization.yaml | 8 +- .../externalsecret.yaml} | 0 .../netbird-ingress/kustomization.yaml | 1 + .../post-crds/kustomization.yaml | 12 +-- .../velero-schedules/kustomization.yaml | 6 ++ .../schedules.yaml} | 0 .../hcloud-backup-verify-externalsecret.yaml | 0 .../infrastructure/velero/kustomization.yaml | 8 +- .../velero-externalsecret.yaml | 0 .../velero-hetzner-creds-externalsecret.yaml | 0 17 files changed, 123 insertions(+), 17 deletions(-) create mode 100644 kubernetes/clusters/homelab/infrastructure-services.yaml rename kubernetes/infrastructure/{post-crds => crowdsec}/crowdsec-bouncer-key-externalsecret.yaml (100%) rename kubernetes/infrastructure/{post-crds => crowdsec}/crowdsec-edge-bouncer-key-externalsecret.yaml (100%) rename kubernetes/infrastructure/{post-crds/external-dns-unifi-externalsecret.yaml => external-dns-unifi/externalsecret.yaml} (100%) rename kubernetes/infrastructure/{post-crds/netbird-ingress-externalsecret.yaml => netbird-ingress/externalsecret.yaml} (100%) create mode 100644 kubernetes/infrastructure/velero-schedules/kustomization.yaml rename kubernetes/infrastructure/{post-crds/velero-schedules.yaml => velero-schedules/schedules.yaml} (100%) rename kubernetes/infrastructure/{post-crds => velero}/hcloud-backup-verify-externalsecret.yaml (100%) rename kubernetes/infrastructure/{post-crds => velero}/velero-externalsecret.yaml (100%) rename kubernetes/infrastructure/{post-crds => velero}/velero-hetzner-creds-externalsecret.yaml (100%) diff --git a/kubernetes/clusters/homelab/infrastructure-services.yaml b/kubernetes/clusters/homelab/infrastructure-services.yaml new file mode 100644 index 00000000..a4035bd8 --- /dev/null +++ b/kubernetes/clusters/homelab/infrastructure-services.yaml @@ -0,0 +1,98 @@ +# Infrastructure services that consume Secrets from OpenBao. Each carries its +# ExternalSecrets and depends on infrastructure-crds, where the openbao +# ClusterSecretStore turns Ready only once OpenBao is unsealed, so the +# HelmRelease never installs before its Secret exists. One Kustomization each: +# a broken service holds back nothing but itself (velero: the restore chain). +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: crowdsec + namespace: flux-system +spec: + interval: 10m + retryInterval: 1m + timeout: 10m + sourceRef: + kind: GitRepository + name: flux-system + path: ./kubernetes/infrastructure/crowdsec + prune: true + wait: true + dependsOn: + - name: infrastructure + - name: infrastructure-crds +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: external-dns-unifi + namespace: flux-system +spec: + interval: 10m + retryInterval: 1m + timeout: 5m + sourceRef: + kind: GitRepository + name: flux-system + path: ./kubernetes/infrastructure/external-dns-unifi + prune: true + wait: true + dependsOn: + - name: infrastructure + - name: infrastructure-crds +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: netbird-ingress + namespace: flux-system +spec: + interval: 10m + retryInterval: 1m + timeout: 5m + sourceRef: + kind: GitRepository + name: flux-system + path: ./kubernetes/infrastructure/netbird-ingress + prune: true + wait: true + dependsOn: + - name: infrastructure + - name: infrastructure-crds +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: velero + namespace: flux-system +spec: + interval: 10m + retryInterval: 1m + timeout: 15m + sourceRef: + kind: GitRepository + name: flux-system + path: ./kubernetes/infrastructure/velero + prune: true + wait: true + dependsOn: + - name: infrastructure + - name: infrastructure-crds +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: velero-schedules + namespace: flux-system +spec: + interval: 10m + retryInterval: 1m + timeout: 5m + sourceRef: + kind: GitRepository + name: flux-system + path: ./kubernetes/infrastructure/velero-schedules + prune: true + wait: true + dependsOn: + - name: velero # the velero.io CRDs diff --git a/kubernetes/clusters/homelab/restore.yaml b/kubernetes/clusters/homelab/restore.yaml index e0c09369..33f6d1ee 100644 --- a/kubernetes/clusters/homelab/restore.yaml +++ b/kubernetes/clusters/homelab/restore.yaml @@ -14,7 +14,7 @@ spec: path: ./kubernetes/restore/gate prune: true dependsOn: - - name: infrastructure-crds + - name: velero # the BackupStorageLocation and its credentials healthChecks: - apiVersion: velero.io/v1 kind: BackupStorageLocation @@ -25,7 +25,7 @@ spec: kind: BackupStorageLocation current: has(status.lastSyncedTime) --- -# One Kustomization per Velero Schedule (infrastructure/post-crds/velero-schedules.yaml), +# One Kustomization per Velero Schedule (infrastructure/velero-schedules/schedules.yaml), # each restoring that schedule's newest backup. An app depends on its own, so # it waits for its data and for nothing else. Gitea's data goes first: it is # the registry for the in-house images; the others depend on restore-gitea. diff --git a/kubernetes/infrastructure/post-crds/crowdsec-bouncer-key-externalsecret.yaml b/kubernetes/infrastructure/crowdsec/crowdsec-bouncer-key-externalsecret.yaml similarity index 100% rename from kubernetes/infrastructure/post-crds/crowdsec-bouncer-key-externalsecret.yaml rename to kubernetes/infrastructure/crowdsec/crowdsec-bouncer-key-externalsecret.yaml diff --git a/kubernetes/infrastructure/post-crds/crowdsec-edge-bouncer-key-externalsecret.yaml b/kubernetes/infrastructure/crowdsec/crowdsec-edge-bouncer-key-externalsecret.yaml similarity index 100% rename from kubernetes/infrastructure/post-crds/crowdsec-edge-bouncer-key-externalsecret.yaml rename to kubernetes/infrastructure/crowdsec/crowdsec-edge-bouncer-key-externalsecret.yaml diff --git a/kubernetes/infrastructure/crowdsec/kustomization.yaml b/kubernetes/infrastructure/crowdsec/kustomization.yaml index 23d63af4..6e4ac540 100644 --- a/kubernetes/infrastructure/crowdsec/kustomization.yaml +++ b/kubernetes/infrastructure/crowdsec/kustomization.yaml @@ -3,3 +3,5 @@ kind: Kustomization resources: - lapi-ingress.yaml - helmrelease.yaml + - crowdsec-bouncer-key-externalsecret.yaml + - crowdsec-edge-bouncer-key-externalsecret.yaml diff --git a/kubernetes/infrastructure/post-crds/external-dns-unifi-externalsecret.yaml b/kubernetes/infrastructure/external-dns-unifi/externalsecret.yaml similarity index 100% rename from kubernetes/infrastructure/post-crds/external-dns-unifi-externalsecret.yaml rename to kubernetes/infrastructure/external-dns-unifi/externalsecret.yaml diff --git a/kubernetes/infrastructure/external-dns-unifi/kustomization.yaml b/kubernetes/infrastructure/external-dns-unifi/kustomization.yaml index 4fd939d4..bc610851 100644 --- a/kubernetes/infrastructure/external-dns-unifi/kustomization.yaml +++ b/kubernetes/infrastructure/external-dns-unifi/kustomization.yaml @@ -2,3 +2,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - helmrelease.yaml + - externalsecret.yaml diff --git a/kubernetes/infrastructure/kustomization.yaml b/kubernetes/infrastructure/kustomization.yaml index c39814c2..79656783 100644 --- a/kubernetes/infrastructure/kustomization.yaml +++ b/kubernetes/infrastructure/kustomization.yaml @@ -11,17 +11,13 @@ resources: - snapshot-controller/ - metrics-server/ - intel-gpu-plugin/ - - crowdsec/ - traefik/ - rook-ceph/ - ceph-csi-drivers/ - cloudnative-pg/ - mariadb-operator/ - - netbird-ingress/ - coredns-config/ - k8s-gateway/ - - external-dns-unifi/ - - velero/ - image-automation/ - flux-notifications/ - spegel/ @@ -39,3 +35,7 @@ resources: # the Deployment becoming Ready — circular. # bird/ and netbird-service-sync/ run images from Gitea's registry and live in # infrastructure-inhouse/, which waits for Gitea. +# crowdsec/, external-dns-unifi/, netbird-ingress/ and velero/ consume Secrets +# from OpenBao and have Kustomizations of their own after infrastructure-crds +# (clusters/homelab/infrastructure-services.yaml); velero-schedules/ follows +# velero. diff --git a/kubernetes/infrastructure/post-crds/netbird-ingress-externalsecret.yaml b/kubernetes/infrastructure/netbird-ingress/externalsecret.yaml similarity index 100% rename from kubernetes/infrastructure/post-crds/netbird-ingress-externalsecret.yaml rename to kubernetes/infrastructure/netbird-ingress/externalsecret.yaml diff --git a/kubernetes/infrastructure/netbird-ingress/kustomization.yaml b/kubernetes/infrastructure/netbird-ingress/kustomization.yaml index 48f1d3d7..43bb9c6c 100644 --- a/kubernetes/infrastructure/netbird-ingress/kustomization.yaml +++ b/kubernetes/infrastructure/netbird-ingress/kustomization.yaml @@ -2,3 +2,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - daemonset.yaml + - externalsecret.yaml diff --git a/kubernetes/infrastructure/post-crds/kustomization.yaml b/kubernetes/infrastructure/post-crds/kustomization.yaml index ddc88e40..de063631 100644 --- a/kubernetes/infrastructure/post-crds/kustomization.yaml +++ b/kubernetes/infrastructure/post-crds/kustomization.yaml @@ -7,7 +7,6 @@ resources: - cnpg-barman-certs.yaml # Private CA for MQTT client/broker certificates (REN-802). - mqtt-ca.yaml - - velero-schedules.yaml # Only safe here because OpenBao is already initialized + k8s-auth is # configured (2026-07-13): a store that can't turn Ready would block this # KS's wait gate and freeze the dependsOn chain. @@ -18,14 +17,11 @@ resources: # the ESO HelmRelease that would have installed the CRDs. They sat next to # their consumers until a cold-rebuild test hit that deadlock (2026-09-01); # nothing had ever applied this layer against an empty API server before. - - crowdsec-bouncer-key-externalsecret.yaml - - crowdsec-edge-bouncer-key-externalsecret.yaml - - external-dns-unifi-externalsecret.yaml + # ExternalSecrets whose consumer is a workload live beside it, in that + # workload's own Kustomization after this one (crowdsec, external-dns-unifi, + # netbird-ingress, velero): applied here, the consumer would start before + # its Secret exists. - flux-git-externalsecret.yaml - - netbird-ingress-externalsecret.yaml - - velero-externalsecret.yaml - - velero-hetzner-creds-externalsecret.yaml - - hcloud-backup-verify-externalsecret.yaml # traefik.io CRDs, same reasoning — installed by the Traefik HelmRelease in # infrastructure. Traefik is deliberately absent from that Kustomization's # healthChecks, so this KS may dry-run before the chart lands; it retries on diff --git a/kubernetes/infrastructure/velero-schedules/kustomization.yaml b/kubernetes/infrastructure/velero-schedules/kustomization.yaml new file mode 100644 index 00000000..bcaf1271 --- /dev/null +++ b/kubernetes/infrastructure/velero-schedules/kustomization.yaml @@ -0,0 +1,6 @@ +# Velero's Schedules: their velero.io CRDs come from the Velero HelmRelease +# (../velero), so they apply after it. +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - schedules.yaml diff --git a/kubernetes/infrastructure/post-crds/velero-schedules.yaml b/kubernetes/infrastructure/velero-schedules/schedules.yaml similarity index 100% rename from kubernetes/infrastructure/post-crds/velero-schedules.yaml rename to kubernetes/infrastructure/velero-schedules/schedules.yaml diff --git a/kubernetes/infrastructure/post-crds/hcloud-backup-verify-externalsecret.yaml b/kubernetes/infrastructure/velero/hcloud-backup-verify-externalsecret.yaml similarity index 100% rename from kubernetes/infrastructure/post-crds/hcloud-backup-verify-externalsecret.yaml rename to kubernetes/infrastructure/velero/hcloud-backup-verify-externalsecret.yaml diff --git a/kubernetes/infrastructure/velero/kustomization.yaml b/kubernetes/infrastructure/velero/kustomization.yaml index 83984f0b..f929d884 100644 --- a/kubernetes/infrastructure/velero/kustomization.yaml +++ b/kubernetes/infrastructure/velero/kustomization.yaml @@ -6,9 +6,11 @@ resources: - helmrelease.yaml - backup-verify-rbac.yaml - backup-verify-cronjob.yaml -# Velero Schedule CRs live in kubernetes/infrastructure/post-crds/ -# (velero-schedules.yaml) so they reconcile after the Velero HelmRelease -# finishes installing velero.io CRDs. + - velero-externalsecret.yaml + - velero-hetzner-creds-externalsecret.yaml + - hcloud-backup-verify-externalsecret.yaml +# Velero Schedule CRs live in ../velero-schedules so they reconcile after the +# Velero HelmRelease installs the velero.io CRDs. # Bake the backup-verify scripts into a ConfigMap mounted by the CronJob. # Hash suffix is disabled because Kustomize's NameReference transformer diff --git a/kubernetes/infrastructure/post-crds/velero-externalsecret.yaml b/kubernetes/infrastructure/velero/velero-externalsecret.yaml similarity index 100% rename from kubernetes/infrastructure/post-crds/velero-externalsecret.yaml rename to kubernetes/infrastructure/velero/velero-externalsecret.yaml diff --git a/kubernetes/infrastructure/post-crds/velero-hetzner-creds-externalsecret.yaml b/kubernetes/infrastructure/velero/velero-hetzner-creds-externalsecret.yaml similarity index 100% rename from kubernetes/infrastructure/post-crds/velero-hetzner-creds-externalsecret.yaml rename to kubernetes/infrastructure/velero/velero-hetzner-creds-externalsecret.yaml -- 2.54.0 From 0933ccdf7b28f8364572135dcde31a3bb2f5dfb0 Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Fri, 16 Oct 2026 22:19:54 +0200 Subject: [PATCH 40/42] fix(restore-test): CNPG archives, backs up and prunes as in production Patches 3c (isWALArchiver: false), 3d (no retentionPolicy) and 3e (ScheduledBackups suspended) kept the test cluster from writing into the production CNPG bucket. Since cffb7e02 it cannot: bootstrap.sh writes cnpg-paths with every archive path in renner-uno-k8s-cnpg-rtest, the cnpg-object-stores Kustomization fails rather than render without it, and the source stores, the only ones in the production bucket, carry no retentionPolicy. The patches now only stopped the test from exercising what a real recovery does next: archiving WAL into a new prefix and taking the immediate base backup there. Removing them leaves three fewer deviations from production. Co-Authored-By: Claude Opus 5.5 --- kubernetes/restore-test/flux-root.yaml | 33 ++++---------------------- 1 file changed, 4 insertions(+), 29 deletions(-) diff --git a/kubernetes/restore-test/flux-root.yaml b/kubernetes/restore-test/flux-root.yaml index 9380e155..7aacc935 100644 --- a/kubernetes/restore-test/flux-root.yaml +++ b/kubernetes/restore-test/flux-root.yaml @@ -84,35 +84,10 @@ spec: - op: add path: /spec/values/configuration/backupStorageLocation/0/accessMode value: ReadOnly - # 3c. Safety: CNPG archives no WAL. The plugin stays loaded for - # recovery. - - target: - group: postgresql.cnpg.io - kind: Cluster - patch: | - - op: add - path: /spec/plugins/0/isWALArchiver - value: false - # 3d. Safety: no CNPG retention. The plugin sidecar deletes by - # retentionPolicy whether or not it archives. - - target: - group: barmancloud.cnpg.io - kind: ObjectStore - patch: | - apiVersion: barmancloud.cnpg.io/v1 - kind: ObjectStore - metadata: - name: _ - spec: - retentionPolicy: null - # 3e. Safety: no CNPG base backups. - - target: - group: postgresql.cnpg.io - kind: ScheduledBackup - patch: | - - op: add - path: /spec/suspend - value: true + # CNPG needs no patch: bootstrap.sh points every cluster's archive + # at the test bucket (cnpg-paths), and the source stores carry no + # retentionPolicy, so WAL, base backups and retention all act on + # the test bucket only. # 3f. Safety: no MariaDB logical dumps. - target: group: k8s.mariadb.com -- 2.54.0 From afdeecc161c7da792447b2ca8fb81f6be04a0d74 Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Fri, 16 Oct 2026 22:51:46 +0200 Subject: [PATCH 41/42] fix(monitoring-rules): drop the radius metrics Service; authentik's operator creates it authentik-outpost-servicemonitors.yaml hand-wrote a headless Service ak-outpost-local-radius-metrics (9300/TCP) and a ServiceMonitor for it, because the radius outpost controller used not to expose its metrics port. authentik 2026.8.3 does: the operator creates that same Service (http-metrics 9300, managed-by goauthentik.io, service-type metrics) and its own ServiceMonitor ak-outpost-local-radius selecting exactly those labels. Restore test run 29 (2026-10-16): on a cold cluster the operator created the Service first, with a ClusterIP. clusterIP is immutable, so the dry-run of the hand-written clusterIP: None copy failed and took the whole monitoring-rules Kustomization with it: Service/authentik/ak-outpost-local-radius-metrics dry-run failed (Invalid): spec.clusterIPs[0]: Invalid value: ["None"]: may not change once set In production Flux had created it first; the operator adopted it, and Flux's label managed-by: homelab now keeps the operator's ServiceMonitor from matching. Removing the file lets the operator own both; when Flux prunes the Service, the operator recreates it with its own labels. No alert rule refers to the radius job. Co-Authored-By: Claude Opus 5.5 --- .../authentik-outpost-servicemonitors.yaml | 63 ------------------- .../apps/monitoring-rules/kustomization.yaml | 1 - 2 files changed, 64 deletions(-) delete mode 100644 kubernetes/apps/monitoring-rules/authentik-outpost-servicemonitors.yaml diff --git a/kubernetes/apps/monitoring-rules/authentik-outpost-servicemonitors.yaml b/kubernetes/apps/monitoring-rules/authentik-outpost-servicemonitors.yaml deleted file mode 100644 index 92371c68..00000000 --- a/kubernetes/apps/monitoring-rules/authentik-outpost-servicemonitors.yaml +++ /dev/null @@ -1,63 +0,0 @@ ---- -# Only radius needs a hand-written ServiceMonitor. The goauthentik.io operator -# creates one per outpost and Prometheus discovers those directly, because -# serviceMonitorSelector here is empty (see monitoring/helmrelease-prometheus.yaml). -# Standalone copies for ldap and paperless-gate used to live here as well; they -# duplicated the operator's and were removed. -# -# Radius requires extra plumbing. The radius binary listens on :9300 just -# like ldap/proxy (hardcoded in authentik/lib/default.yml), but authentik's -# Python operator (`providers/radius/controllers/kubernetes.py`) omits the -# `DeploymentPort(9300, "http-metrics", "tcp", 9300)` entry from its -# `deployment_ports` list — that's an upstream bug; the ldap controller has -# it, radius does not. Result: the operator-managed `ak-outpost-local-radius` -# Service exposes only UDP 1812. Add a sibling Service here that selects the -# same pod labels and exposes 9300/TCP, then a ServiceMonitor for it. -# -# The operator's own `ak-outpost-local-radius` ServiceMonitor cannot cover -# this: it selects on `app.kubernetes.io/managed-by: goauthentik.io` together -# with `goauthentik.io/service-type: metrics`, and no Service carries both — -# the operator's has the first, the one below has the second. -apiVersion: v1 -kind: Service -metadata: - name: ak-outpost-local-radius-metrics - namespace: authentik - labels: - app.kubernetes.io/managed-by: homelab - goauthentik.io/outpost-name: local-radius - goauthentik.io/outpost-type: radius -spec: - type: ClusterIP - clusterIP: None - selector: - goauthentik.io/outpost-name: local-radius - goauthentik.io/outpost-type: radius - ports: - - name: http-metrics - port: 9300 - targetPort: 9300 - protocol: TCP ---- -apiVersion: monitoring.coreos.com/v1 -kind: ServiceMonitor -metadata: - name: authentik-outpost-radius - namespace: authentik - labels: - release: kube-prometheus-stack -spec: - namespaceSelector: - matchNames: - - authentik - selector: - # Pinned to the Service above. Without managed-by this would also match the - # operator's Service and scrape the same endpoint twice. - matchLabels: - app.kubernetes.io/managed-by: homelab - goauthentik.io/outpost-name: local-radius - goauthentik.io/outpost-type: radius - endpoints: - - port: http-metrics - path: /metrics - interval: 30s diff --git a/kubernetes/apps/monitoring-rules/kustomization.yaml b/kubernetes/apps/monitoring-rules/kustomization.yaml index e6cbc503..97f14961 100644 --- a/kubernetes/apps/monitoring-rules/kustomization.yaml +++ b/kubernetes/apps/monitoring-rules/kustomization.yaml @@ -49,7 +49,6 @@ resources: # cannot live beside the Velero HelmRelease. - velero-podmonitor.yaml - velero-servicemonitor.yaml - - authentik-outpost-servicemonitors.yaml - authentik-servicemonitor.yaml # Rook's exporter/mgr monitors: the operator aborts its node-daemon reconcile # when it cannot create these, which held the CephCluster in Progressing and -- 2.54.0 From 20b0621e2eb6738e6d8717d1825643fc391ecee5 Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Fri, 16 Oct 2026 23:04:15 +0200 Subject: [PATCH 42/42] fix(home-assistant): shelly-manager-auth stays with mqtt-account-controller 420bd35e moved shelly-manager-auth-externalsecret.yaml to home-assistant-iot with the rest of shelly-manager. mqtt-account-controller, which stays in home-assistant, mounts that Secret as well (volume shelly-manager-key, item api-key, not optional). That made a cycle: - mqtt-account-controller does not start without shelly-manager-auth; - it creates the MQTT client certificates HA, mosquitto and zigbee2mqtt mount (mqtt-home-assistant-tls, mqtt-mosquitto-probe-tls, mqtt-zigbee2mqtt-bridge-tls), so they do not start either; - home-assistant therefore never turns Ready, and home-assistant-iot, which would create shelly-manager-auth, depends on it. Restore test run 29 (2026-10-16) sat in it: every other Secret in the namespace existed, mqtt-account-controller reported MountVolume.SetUp failed for volume "shelly-manager-key" : secret "shelly-manager-auth" not found and evcc, trmnl, zone-configurator, speech-to-phrase and home-assistant-iot waited behind it. The ExternalSecret goes back to home-assistant; shelly-manager in home-assistant-iot runs after it and finds the Secret. Every Secret the home-assistant render mounts now has its producer in the same Kustomization (mqtt-dynsec-seed is optional). Co-Authored-By: Claude Opus 5.5 --- kubernetes/apps/home-assistant-iot/kustomization.yaml | 3 ++- kubernetes/apps/home-assistant/kustomization.yaml | 2 ++ .../shelly-manager-auth-externalsecret.yaml | 0 3 files changed, 4 insertions(+), 1 deletion(-) rename kubernetes/apps/{home-assistant-iot => home-assistant}/shelly-manager-auth-externalsecret.yaml (100%) diff --git a/kubernetes/apps/home-assistant-iot/kustomization.yaml b/kubernetes/apps/home-assistant-iot/kustomization.yaml index 9b1694ba..ea19f14c 100644 --- a/kubernetes/apps/home-assistant-iot/kustomization.yaml +++ b/kubernetes/apps/home-assistant-iot/kustomization.yaml @@ -9,7 +9,8 @@ resources: - shelly-manager-network-attachment.yaml - shelly-manager-firmware-rbac.yaml - shelly-ble-proxy-key-externalsecret.yaml - - shelly-manager-auth-externalsecret.yaml + # shelly-manager-auth comes from ../home-assistant: mqtt-account-controller + # mounts it too, and that Kustomization comes first. - shelly-manager-llat-externalsecret.yaml - shelly-manager-deployment.yaml - shelly-manager-service.yaml diff --git a/kubernetes/apps/home-assistant/kustomization.yaml b/kubernetes/apps/home-assistant/kustomization.yaml index f2267a6d..bb727887 100644 --- a/kubernetes/apps/home-assistant/kustomization.yaml +++ b/kubernetes/apps/home-assistant/kustomization.yaml @@ -50,6 +50,8 @@ resources: # Register: mqtt-accounts.yaml (configMapGenerator). - mosquitto-dynsec-test.yaml - mqtt-account-controller.yaml + # Also mounted by shelly-manager (../home-assistant-iot), which runs after this. + - shelly-manager-auth-externalsecret.yaml configMapGenerator: # nginx config for the reverse-proxy sidecar. Hash suffix disabled so # the cm name stays stable; editing proxy.conf does not auto-roll the diff --git a/kubernetes/apps/home-assistant-iot/shelly-manager-auth-externalsecret.yaml b/kubernetes/apps/home-assistant/shelly-manager-auth-externalsecret.yaml similarity index 100% rename from kubernetes/apps/home-assistant-iot/shelly-manager-auth-externalsecret.yaml rename to kubernetes/apps/home-assistant/shelly-manager-auth-externalsecret.yaml -- 2.54.0