Restore test 7/9: the cluster restores its app volumes itself #1303

Closed
claude-bot wants to merge 1 commits from split/07-velero-self-restore into split/06-hardware-gating
43 changed files with 568 additions and 5 deletions
+2 -2
View File
@@ -2044,8 +2044,8 @@ if phase_runs wait-apps; then
# Each app KS now owns its CNPG database (+ any data PVCs) directly — the
# per-app `seed-<app>` KSes that used to bootstrap them were folded into the
# app KSes when the seed layer was retired. Each wait below gates on the app
# KS, whose readiness includes its CNPG cluster coming up (initdb on a fresh
# cluster; a cold-rebuild data restore is the separate RECOVERY.md runbook).
# KS, whose readiness includes its CNPG cluster coming up (bootstrap.recovery)
# and, through the restore KS it depends on, its volumes restored by Velero.
# Timeouts stay generous to cover first-time cluster bootstrap on fresh Ceph.
# apps waits on infrastructure-ceph-cluster (Ceph mon/mgr/OSD bring-up
# is genuinely 5–10 min on a fresh OSD-wiped cluster).
+2 -1
View File
@@ -17,6 +17,7 @@ spec:
# woodpecker migrated) — secrets come from OpenBao via ESO
# (MIGRATION-sops-openbao.md).
dependsOn:
- name: restore-apps
- name: infrastructure
- name: infrastructure-crds
- name: infrastructure-ceph-cluster
- name: infrastructure-ceph-cluster
@@ -22,6 +22,7 @@ spec:
secretRef:
name: sops-age
dependsOn:
- name: restore-audiobookshelf
- name: infrastructure
- name: infrastructure-crds
- name: infrastructure-ceph-cluster
@@ -20,6 +20,7 @@ spec:
# from OpenBao via ESO (MIGRATION-sops-openbao.md). This was the LAST
# sops consumer; phase 4 (age-key retirement) may proceed after soak.
dependsOn:
- name: restore-authentik
- name: infrastructure
- name: infrastructure-crds
- name: infrastructure-ceph-cluster
+1
View File
@@ -21,6 +21,7 @@ spec:
kind: PersistentVolumeClaim
name: data-gitea-0
dependsOn:
- name: restore-gitea
- name: infrastructure
- name: infrastructure-crds
- name: infrastructure-ceph-cluster
@@ -19,6 +19,7 @@ spec:
# decryption block dropped 2026-07-13: path is sops-free — secrets come
# from OpenBao via ESO (MIGRATION-sops-openbao.md).
dependsOn:
- name: restore-home-assistant
- name: infrastructure
- name: infrastructure-crds
- name: infrastructure-ceph-cluster
@@ -54,6 +55,7 @@ spec:
prune: true
wait: true
dependsOn:
- name: restore-home-assistant
- name: infrastructure
- name: infrastructure-crds
- name: infrastructure-ceph-cluster
@@ -75,6 +77,7 @@ spec:
prune: true
wait: true
dependsOn:
- name: restore-home-assistant # shelly-manager's PVC
- name: infrastructure
- name: infrastructure-crds
- name: infrastructure-ceph-cluster
@@ -101,6 +104,7 @@ spec:
# decryption block dropped 2026-07-13: path is sops-free — secrets come
# from OpenBao via ESO (MIGRATION-sops-openbao.md).
dependsOn:
- name: restore-home-assistant
- name: infrastructure
- name: infrastructure-crds
---
@@ -141,6 +145,7 @@ spec:
# First boot downloads the Whisper model (a few hundred MB) — give it room.
wait: true
dependsOn:
- name: restore-home-assistant
- name: infrastructure
- name: infrastructure-crds
- name: infrastructure-ceph-cluster
+1
View File
@@ -20,6 +20,7 @@ spec:
# decryption block dropped 2026-07-13: path is sops-free — secrets come
# from OpenBao via ESO (MIGRATION-sops-openbao.md).
dependsOn:
- name: restore-immich
- name: infrastructure
- name: infrastructure-crds
- name: infrastructure-ceph-cluster
@@ -25,6 +25,7 @@ spec:
# boilerplate from the monolithic seed KS — InfluxDB has no CNPG cluster, only
# a chart-provisioned ceph-block PVC, so ceph is the only storage dep needed.
dependsOn:
- name: restore-influxdb
- name: infrastructure
- name: infrastructure-crds
- name: infrastructure-ceph-cluster
+1
View File
@@ -17,6 +17,7 @@ spec:
prune: true
wait: true
dependsOn:
- name: restore-matrix
- name: infrastructure
- name: infrastructure-crds
- name: infrastructure-ceph-cluster
@@ -22,6 +22,7 @@ spec:
secretRef:
name: sops-age
dependsOn:
- name: restore-music-assistant
- name: infrastructure
- name: infrastructure-crds
- name: infrastructure-ceph-cluster
@@ -14,6 +14,7 @@ spec:
prune: true
wait: true
dependsOn:
- name: restore-navidrome
- name: infrastructure
- name: infrastructure-crds
- name: infrastructure-ceph-cluster
@@ -20,6 +20,7 @@ spec:
# decryption block dropped 2026-07-13: path is sops-free — secrets come
# from OpenBao via ESO (MIGRATION-sops-openbao.md).
dependsOn:
- name: restore-paperless
- name: infrastructure
- name: infrastructure-crds
- name: infrastructure-ceph-cluster
+396
View File
@@ -0,0 +1,396 @@
# Ready once Velero has completed a sync of the backups in the bucket. A
# Restore created before that finds no backup and fails for good.
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: velero-backups-synced
namespace: flux-system
spec:
interval: 10m
timeout: 30m
sourceRef:
kind: GitRepository
name: flux-system
path: ./kubernetes/restore/gate
prune: true
dependsOn:
- name: velero # the BackupStorageLocation and its credentials
healthChecks:
- apiVersion: velero.io/v1
kind: BackupStorageLocation
name: default
namespace: velero
healthCheckExprs:
- apiVersion: velero.io/v1
kind: BackupStorageLocation
current: has(status.lastSyncedTime)
---
# One Kustomization per Velero Schedule (infrastructure/velero-schedules/schedules.yaml),
# each restoring that schedule's newest backup. An app depends on its own, so
# it waits for its data and for nothing else. Gitea's data goes first: it is
# the registry for the in-house images; the others depend on restore-gitea.
# openbao-daily has none: bootstrap.sh restores OpenBao from its raft snapshot.
# Every one carries the same health check: a schedule without any backup has
# nothing to restore (FailedValidation "No backups found for schedule").
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: restore-gitea
namespace: flux-system
spec:
interval: 10m
timeout: 1h
sourceRef:
kind: GitRepository
name: flux-system
path: ./kubernetes/restore/gitea
prune: true
wait: true
dependsOn:
- name: velero-backups-synced
- name: infrastructure-ceph-cluster
healthCheckExprs:
- apiVersion: velero.io/v1
kind: Restore
failed: >-
has(status.phase) && (status.phase in ['Failed', 'PartiallyFailed'] ||
(status.phase == 'FailedValidation' && !(has(status.validationErrors) &&
status.validationErrors.exists(e, e.startsWith('No backups found for schedule')))))
current: >-
has(status.phase) && (status.phase == 'Completed' || status.phase == 'FailedValidation')
---
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: restore-apps
namespace: flux-system
spec:
interval: 10m
timeout: 1h
sourceRef:
kind: GitRepository
name: flux-system
path: ./kubernetes/restore/apps
prune: true
wait: true
dependsOn:
- name: restore-gitea
- name: velero-backups-synced
- name: infrastructure-ceph-cluster
healthCheckExprs:
- apiVersion: velero.io/v1
kind: Restore
failed: >-
has(status.phase) && (status.phase in ['Failed', 'PartiallyFailed'] ||
(status.phase == 'FailedValidation' && !(has(status.validationErrors) &&
status.validationErrors.exists(e, e.startsWith('No backups found for schedule')))))
current: >-
has(status.phase) && (status.phase == 'Completed' || status.phase == 'FailedValidation')
---
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: restore-audiobookshelf
namespace: flux-system
spec:
interval: 10m
timeout: 3h
sourceRef:
kind: GitRepository
name: flux-system
path: ./kubernetes/restore/audiobookshelf
prune: true
wait: true
dependsOn:
- name: restore-gitea
- name: velero-backups-synced
- name: infrastructure-ceph-cluster
healthCheckExprs:
- apiVersion: velero.io/v1
kind: Restore
failed: >-
has(status.phase) && (status.phase in ['Failed', 'PartiallyFailed'] ||
(status.phase == 'FailedValidation' && !(has(status.validationErrors) &&
status.validationErrors.exists(e, e.startsWith('No backups found for schedule')))))
current: >-
has(status.phase) && (status.phase == 'Completed' || status.phase == 'FailedValidation')
---
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: restore-authentik
namespace: flux-system
spec:
interval: 10m
timeout: 1h
sourceRef:
kind: GitRepository
name: flux-system
path: ./kubernetes/restore/authentik
prune: true
wait: true
dependsOn:
- name: restore-gitea
- name: velero-backups-synced
- name: infrastructure-ceph-cluster
healthCheckExprs:
- apiVersion: velero.io/v1
kind: Restore
failed: >-
has(status.phase) && (status.phase in ['Failed', 'PartiallyFailed'] ||
(status.phase == 'FailedValidation' && !(has(status.validationErrors) &&
status.validationErrors.exists(e, e.startsWith('No backups found for schedule')))))
current: >-
has(status.phase) && (status.phase == 'Completed' || status.phase == 'FailedValidation')
---
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: restore-home-assistant
namespace: flux-system
spec:
interval: 10m
timeout: 1h
sourceRef:
kind: GitRepository
name: flux-system
path: ./kubernetes/restore/home-assistant
prune: true
wait: true
dependsOn:
- name: restore-gitea
- name: velero-backups-synced
- name: infrastructure-ceph-cluster
healthCheckExprs:
- apiVersion: velero.io/v1
kind: Restore
failed: >-
has(status.phase) && (status.phase in ['Failed', 'PartiallyFailed'] ||
(status.phase == 'FailedValidation' && !(has(status.validationErrors) &&
status.validationErrors.exists(e, e.startsWith('No backups found for schedule')))))
current: >-
has(status.phase) && (status.phase == 'Completed' || status.phase == 'FailedValidation')
---
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: restore-immich
namespace: flux-system
spec:
interval: 10m
timeout: 6h
sourceRef:
kind: GitRepository
name: flux-system
path: ./kubernetes/restore/immich
prune: true
wait: true
dependsOn:
- name: restore-gitea
- name: velero-backups-synced
- name: infrastructure-ceph-cluster
healthCheckExprs:
- apiVersion: velero.io/v1
kind: Restore
failed: >-
has(status.phase) && (status.phase in ['Failed', 'PartiallyFailed'] ||
(status.phase == 'FailedValidation' && !(has(status.validationErrors) &&
status.validationErrors.exists(e, e.startsWith('No backups found for schedule')))))
current: >-
has(status.phase) && (status.phase == 'Completed' || status.phase == 'FailedValidation')
---
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: restore-influxdb
namespace: flux-system
spec:
interval: 10m
timeout: 1h
sourceRef:
kind: GitRepository
name: flux-system
path: ./kubernetes/restore/influxdb
prune: true
wait: true
dependsOn:
- name: restore-gitea
- name: velero-backups-synced
- name: infrastructure-ceph-cluster
healthCheckExprs:
- apiVersion: velero.io/v1
kind: Restore
failed: >-
has(status.phase) && (status.phase in ['Failed', 'PartiallyFailed'] ||
(status.phase == 'FailedValidation' && !(has(status.validationErrors) &&
status.validationErrors.exists(e, e.startsWith('No backups found for schedule')))))
current: >-
has(status.phase) && (status.phase == 'Completed' || status.phase == 'FailedValidation')
---
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: restore-matrix
namespace: flux-system
spec:
interval: 10m
timeout: 1h
sourceRef:
kind: GitRepository
name: flux-system
path: ./kubernetes/restore/matrix
prune: true
wait: true
dependsOn:
- name: restore-gitea
- name: velero-backups-synced
- name: infrastructure-ceph-cluster
healthCheckExprs:
- apiVersion: velero.io/v1
kind: Restore
failed: >-
has(status.phase) && (status.phase in ['Failed', 'PartiallyFailed'] ||
(status.phase == 'FailedValidation' && !(has(status.validationErrors) &&
status.validationErrors.exists(e, e.startsWith('No backups found for schedule')))))
current: >-
has(status.phase) && (status.phase == 'Completed' || status.phase == 'FailedValidation')
---
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: restore-music-assistant
namespace: flux-system
spec:
interval: 10m
timeout: 1h
sourceRef:
kind: GitRepository
name: flux-system
path: ./kubernetes/restore/music-assistant
prune: true
wait: true
dependsOn:
- name: restore-gitea
- name: velero-backups-synced
- name: infrastructure-ceph-cluster
healthCheckExprs:
- apiVersion: velero.io/v1
kind: Restore
failed: >-
has(status.phase) && (status.phase in ['Failed', 'PartiallyFailed'] ||
(status.phase == 'FailedValidation' && !(has(status.validationErrors) &&
status.validationErrors.exists(e, e.startsWith('No backups found for schedule')))))
current: >-
has(status.phase) && (status.phase == 'Completed' || status.phase == 'FailedValidation')
---
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: restore-navidrome
namespace: flux-system
spec:
interval: 10m
timeout: 1h
sourceRef:
kind: GitRepository
name: flux-system
path: ./kubernetes/restore/navidrome
prune: true
wait: true
dependsOn:
- name: restore-gitea
- name: velero-backups-synced
- name: infrastructure-ceph-cluster
healthCheckExprs:
- apiVersion: velero.io/v1
kind: Restore
failed: >-
has(status.phase) && (status.phase in ['Failed', 'PartiallyFailed'] ||
(status.phase == 'FailedValidation' && !(has(status.validationErrors) &&
status.validationErrors.exists(e, e.startsWith('No backups found for schedule')))))
current: >-
has(status.phase) && (status.phase == 'Completed' || status.phase == 'FailedValidation')
---
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: restore-paperless
namespace: flux-system
spec:
interval: 10m
timeout: 1h
sourceRef:
kind: GitRepository
name: flux-system
path: ./kubernetes/restore/paperless
prune: true
wait: true
dependsOn:
- name: restore-gitea
- name: velero-backups-synced
- name: infrastructure-ceph-cluster
healthCheckExprs:
- apiVersion: velero.io/v1
kind: Restore
failed: >-
has(status.phase) && (status.phase in ['Failed', 'PartiallyFailed'] ||
(status.phase == 'FailedValidation' && !(has(status.validationErrors) &&
status.validationErrors.exists(e, e.startsWith('No backups found for schedule')))))
current: >-
has(status.phase) && (status.phase == 'Completed' || status.phase == 'FailedValidation')
---
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: restore-trmnl
namespace: flux-system
spec:
interval: 10m
timeout: 1h
sourceRef:
kind: GitRepository
name: flux-system
path: ./kubernetes/restore/trmnl
prune: true
wait: true
dependsOn:
- name: restore-gitea
- name: velero-backups-synced
- name: infrastructure-ceph-cluster
healthCheckExprs:
- apiVersion: velero.io/v1
kind: Restore
failed: >-
has(status.phase) && (status.phase in ['Failed', 'PartiallyFailed'] ||
(status.phase == 'FailedValidation' && !(has(status.validationErrors) &&
status.validationErrors.exists(e, e.startsWith('No backups found for schedule')))))
current: >-
has(status.phase) && (status.phase == 'Completed' || status.phase == 'FailedValidation')
---
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: restore-wordpress
namespace: flux-system
spec:
interval: 10m
timeout: 1h
sourceRef:
kind: GitRepository
name: flux-system
path: ./kubernetes/restore/wordpress
prune: true
wait: true
dependsOn:
- name: restore-gitea
- name: velero-backups-synced
- name: infrastructure-ceph-cluster
healthCheckExprs:
- apiVersion: velero.io/v1
kind: Restore
failed: >-
has(status.phase) && (status.phase in ['Failed', 'PartiallyFailed'] ||
(status.phase == 'FailedValidation' && !(has(status.validationErrors) &&
status.validationErrors.exists(e, e.startsWith('No backups found for schedule')))))
current: >-
has(status.phase) && (status.phase == 'Completed' || status.phase == 'FailedValidation')
+1
View File
@@ -17,6 +17,7 @@ spec:
prune: true
wait: true
dependsOn:
- name: restore-trmnl
- name: infrastructure
- name: infrastructure-crds
- name: infrastructure-ceph-cluster
@@ -14,6 +14,7 @@ spec:
prune: true
wait: true
dependsOn:
- name: restore-wordpress
# infrastructure gates on the mariadb-operator HelmRelease being Ready,
# so the MariaDB CRD exists and has a controller before this applies.
- name: infrastructure
@@ -9,7 +9,9 @@
# snapshotMoveData uploads PVC contents through Kopia and the node-agent,
# independently of the K8s API surface.
#
# Naming: <app>-daily. Restore is manual — see RUNBOOK-cold-rebuild.md.
# Naming: <app>-daily. Each has a Restore in kubernetes/restore/<app>/ and a
# restore-<app> Kustomization (clusters/homelab/restore.yaml), which a fresh
# cluster runs before the app starts.
# Retention matches the CNPG Barman policy.
apiVersion: velero.io/v1
@@ -199,7 +201,6 @@ spec:
# This is the ONLY restore path for home-assistant-config (.storage,
# secrets.yaml, the config git checkout). The HAOS-backup `seed`
# initContainer that used to repopulate it was removed in 21a0a43f.
# Restore procedure and test: REN-249 runbook section C.2.
schedule: "45 6 * * *"
useOwnerReferencesInBackup: false
template:
@@ -0,0 +1,4 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- restore.yaml
+7
View File
@@ -0,0 +1,7 @@
apiVersion: velero.io/v1
kind: Restore
metadata:
name: apps
namespace: velero
spec:
scheduleName: apps-daily
@@ -0,0 +1,4 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- restore.yaml
@@ -0,0 +1,7 @@
apiVersion: velero.io/v1
kind: Restore
metadata:
name: audiobookshelf
namespace: velero
spec:
scheduleName: audiobookshelf-daily
@@ -0,0 +1,4 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- restore.yaml
@@ -0,0 +1,7 @@
apiVersion: velero.io/v1
kind: Restore
metadata:
name: authentik
namespace: velero
spec:
scheduleName: authentik-daily
@@ -0,0 +1,5 @@
# Applies nothing: its Kustomization exists for its health check on the
# BackupStorageLocation (clusters/homelab/restore.yaml).
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources: []
@@ -0,0 +1,4 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- restore.yaml
+8
View File
@@ -0,0 +1,8 @@
# Gitea first: it is the registry for the in-house images.
apiVersion: velero.io/v1
kind: Restore
metadata:
name: gitea
namespace: velero
spec:
scheduleName: gitea-daily
@@ -0,0 +1,4 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- restore.yaml
@@ -0,0 +1,7 @@
apiVersion: velero.io/v1
kind: Restore
metadata:
name: home-assistant
namespace: velero
spec:
scheduleName: home-assistant-daily
@@ -0,0 +1,4 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- restore.yaml
+7
View File
@@ -0,0 +1,7 @@
apiVersion: velero.io/v1
kind: Restore
metadata:
name: immich
namespace: velero
spec:
scheduleName: immich-daily
@@ -0,0 +1,4 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- restore.yaml
+7
View File
@@ -0,0 +1,7 @@
apiVersion: velero.io/v1
kind: Restore
metadata:
name: influxdb
namespace: velero
spec:
scheduleName: influxdb-daily
@@ -0,0 +1,4 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- restore.yaml
+7
View File
@@ -0,0 +1,7 @@
apiVersion: velero.io/v1
kind: Restore
metadata:
name: matrix
namespace: velero
spec:
scheduleName: matrix-daily
@@ -0,0 +1,4 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- restore.yaml
@@ -0,0 +1,7 @@
apiVersion: velero.io/v1
kind: Restore
metadata:
name: music-assistant
namespace: velero
spec:
scheduleName: music-assistant-daily
@@ -0,0 +1,4 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- restore.yaml
@@ -0,0 +1,7 @@
apiVersion: velero.io/v1
kind: Restore
metadata:
name: navidrome
namespace: velero
spec:
scheduleName: navidrome-daily
@@ -0,0 +1,4 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- restore.yaml
@@ -0,0 +1,7 @@
apiVersion: velero.io/v1
kind: Restore
metadata:
name: paperless
namespace: velero
spec:
scheduleName: paperless-daily
@@ -0,0 +1,4 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- restore.yaml
+7
View File
@@ -0,0 +1,7 @@
apiVersion: velero.io/v1
kind: Restore
metadata:
name: trmnl
namespace: velero
spec:
scheduleName: trmnl-daily
@@ -0,0 +1,4 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- restore.yaml
@@ -0,0 +1,7 @@
apiVersion: velero.io/v1
kind: Restore
metadata:
name: wordpress
namespace: velero
spec:
scheduleName: wordpress-daily