From e82f55bbc70dc543878d161b900913d536e13bbb Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Sat, 17 Oct 2026 11:46:32 +0200 Subject: [PATCH] feat(velero): the cluster restores its app volumes itself, Gitea first One Velero Restore per Schedule (kubernetes/restore//), applied by Flux through a restore- Kustomization (clusters/homelab/restore.yaml), restoring the schedule's newest completed backup. Every app Kustomization with backed-up volumes depends on its own restore Kustomization, so on a fresh cluster no app starts on an empty volume, and no app waits for another app's data. Restores the capability 952e6150 removed without an owner decision. Gitea goes first: it is the registry for the in-house images. The gitea Kustomization depends on restore-gitea alone; every other restore Kustomization depends on restore-gitea, so Velero moves Gitea's data first. Each app depends on its own: apps, audiobookshelf (libation through it), authentik, home-assistant (matter-server, home-assistant-iot, otbr, whisper too, same namespace and schedule), immich, influxdb, matrix, music-assistant, navidrome, paperless, trmnl, wordpress. Timeouts follow the data: immich 6h, audiobookshelf 3h, the rest 1h. One shared gate made authentik wait for immich's photo library (45 minutes in restore rehearsal run 28) and held a kustomize-controller worker all that time. Ordering and failure handling, from Velero release-1.17 source: - A Restore whose schedule has no synced backup yet goes FailedValidation for good (restore_controller.go). The restore Kustomizations therefore depend on velero-backups-synced, which is Ready once the BackupStorageLocation has completed a sync (status.lastSyncedTime), and which depends on the velero Kustomization (the BackupStorageLocation and its credentials). - After that sync, FailedValidation with "No backups found for schedule" means there is nothing to restore and counts as done; any other FailedValidation, Failed or PartiallyFailed fails the Kustomization. - A claim that already exists is skipped (csi pvc_action: "PVC already exists"), and the Restore completes with a warning. On the running cluster the Restores touch nothing. - Velero deletes a Restore with the backup it came from (30-day TTL); Flux recreates it, and it again skips the existing claims. - All StorageClasses bind Immediate, so the data mover does not wait for a consuming pod. The velero-schedules header points at the restores instead of the manual runbook, and bootstrap.sh's wait-apps comment says the volumes come from Velero. Co-Authored-By: Claude Opus 5.5 --- kubernetes/bootstrap.sh | 4 +- kubernetes/clusters/homelab/apps.yaml | 3 +- .../clusters/homelab/audiobookshelf.yaml | 1 + kubernetes/clusters/homelab/authentik.yaml | 1 + kubernetes/clusters/homelab/gitea.yaml | 1 + .../clusters/homelab/home-assistant.yaml | 5 + kubernetes/clusters/homelab/immich.yaml | 1 + kubernetes/clusters/homelab/influxdb.yaml | 1 + kubernetes/clusters/homelab/matrix.yaml | 1 + .../clusters/homelab/music-assistant.yaml | 1 + kubernetes/clusters/homelab/navidrome.yaml | 1 + kubernetes/clusters/homelab/paperless.yaml | 1 + kubernetes/clusters/homelab/restore.yaml | 396 ++++++++++++++++++ kubernetes/clusters/homelab/trmnl.yaml | 1 + kubernetes/clusters/homelab/wordpress.yaml | 1 + .../velero-schedules/schedules.yaml | 5 +- kubernetes/restore/apps/kustomization.yaml | 4 + kubernetes/restore/apps/restore.yaml | 7 + .../restore/audiobookshelf/kustomization.yaml | 4 + .../restore/audiobookshelf/restore.yaml | 7 + .../restore/authentik/kustomization.yaml | 4 + kubernetes/restore/authentik/restore.yaml | 7 + kubernetes/restore/gate/kustomization.yaml | 5 + kubernetes/restore/gitea/kustomization.yaml | 4 + kubernetes/restore/gitea/restore.yaml | 8 + .../restore/home-assistant/kustomization.yaml | 4 + .../restore/home-assistant/restore.yaml | 7 + kubernetes/restore/immich/kustomization.yaml | 4 + kubernetes/restore/immich/restore.yaml | 7 + .../restore/influxdb/kustomization.yaml | 4 + kubernetes/restore/influxdb/restore.yaml | 7 + kubernetes/restore/matrix/kustomization.yaml | 4 + kubernetes/restore/matrix/restore.yaml | 7 + .../music-assistant/kustomization.yaml | 4 + .../restore/music-assistant/restore.yaml | 7 + .../restore/navidrome/kustomization.yaml | 4 + kubernetes/restore/navidrome/restore.yaml | 7 + .../restore/paperless/kustomization.yaml | 4 + kubernetes/restore/paperless/restore.yaml | 7 + kubernetes/restore/trmnl/kustomization.yaml | 4 + kubernetes/restore/trmnl/restore.yaml | 7 + .../restore/wordpress/kustomization.yaml | 4 + kubernetes/restore/wordpress/restore.yaml | 7 + 43 files changed, 568 insertions(+), 5 deletions(-) create mode 100644 kubernetes/clusters/homelab/restore.yaml create mode 100644 kubernetes/restore/apps/kustomization.yaml create mode 100644 kubernetes/restore/apps/restore.yaml create mode 100644 kubernetes/restore/audiobookshelf/kustomization.yaml create mode 100644 kubernetes/restore/audiobookshelf/restore.yaml create mode 100644 kubernetes/restore/authentik/kustomization.yaml create mode 100644 kubernetes/restore/authentik/restore.yaml create mode 100644 kubernetes/restore/gate/kustomization.yaml create mode 100644 kubernetes/restore/gitea/kustomization.yaml create mode 100644 kubernetes/restore/gitea/restore.yaml create mode 100644 kubernetes/restore/home-assistant/kustomization.yaml create mode 100644 kubernetes/restore/home-assistant/restore.yaml create mode 100644 kubernetes/restore/immich/kustomization.yaml create mode 100644 kubernetes/restore/immich/restore.yaml create mode 100644 kubernetes/restore/influxdb/kustomization.yaml create mode 100644 kubernetes/restore/influxdb/restore.yaml create mode 100644 kubernetes/restore/matrix/kustomization.yaml create mode 100644 kubernetes/restore/matrix/restore.yaml create mode 100644 kubernetes/restore/music-assistant/kustomization.yaml create mode 100644 kubernetes/restore/music-assistant/restore.yaml create mode 100644 kubernetes/restore/navidrome/kustomization.yaml create mode 100644 kubernetes/restore/navidrome/restore.yaml create mode 100644 kubernetes/restore/paperless/kustomization.yaml create mode 100644 kubernetes/restore/paperless/restore.yaml create mode 100644 kubernetes/restore/trmnl/kustomization.yaml create mode 100644 kubernetes/restore/trmnl/restore.yaml create mode 100644 kubernetes/restore/wordpress/kustomization.yaml create mode 100644 kubernetes/restore/wordpress/restore.yaml diff --git a/kubernetes/bootstrap.sh b/kubernetes/bootstrap.sh index 3970cba5..5bbacffb 100755 --- a/kubernetes/bootstrap.sh +++ b/kubernetes/bootstrap.sh @@ -2044,8 +2044,8 @@ if phase_runs wait-apps; then # Each app KS now owns its CNPG database (+ any data PVCs) directly — the # per-app `seed-` KSes that used to bootstrap them were folded into the # app KSes when the seed layer was retired. Each wait below gates on the app - # KS, whose readiness includes its CNPG cluster coming up (initdb on a fresh - # cluster; a cold-rebuild data restore is the separate RECOVERY.md runbook). + # KS, whose readiness includes its CNPG cluster coming up (bootstrap.recovery) + # and, through the restore KS it depends on, its volumes restored by Velero. # Timeouts stay generous to cover first-time cluster bootstrap on fresh Ceph. # apps waits on infrastructure-ceph-cluster (Ceph mon/mgr/OSD bring-up # is genuinely 5–10 min on a fresh OSD-wiped cluster). diff --git a/kubernetes/clusters/homelab/apps.yaml b/kubernetes/clusters/homelab/apps.yaml index e5965cfe..88ff04e4 100644 --- a/kubernetes/clusters/homelab/apps.yaml +++ b/kubernetes/clusters/homelab/apps.yaml @@ -17,6 +17,7 @@ spec: # woodpecker migrated) — secrets come from OpenBao via ESO # (MIGRATION-sops-openbao.md). dependsOn: + - name: restore-apps - name: infrastructure - name: infrastructure-crds - - name: infrastructure-ceph-cluster + - name: infrastructure-ceph-cluster \ No newline at end of file diff --git a/kubernetes/clusters/homelab/audiobookshelf.yaml b/kubernetes/clusters/homelab/audiobookshelf.yaml index 6e936ae4..764dde33 100644 --- a/kubernetes/clusters/homelab/audiobookshelf.yaml +++ b/kubernetes/clusters/homelab/audiobookshelf.yaml @@ -22,6 +22,7 @@ spec: secretRef: name: sops-age dependsOn: + - name: restore-audiobookshelf - name: infrastructure - name: infrastructure-crds - name: infrastructure-ceph-cluster diff --git a/kubernetes/clusters/homelab/authentik.yaml b/kubernetes/clusters/homelab/authentik.yaml index 37a405e2..b3650e34 100644 --- a/kubernetes/clusters/homelab/authentik.yaml +++ b/kubernetes/clusters/homelab/authentik.yaml @@ -20,6 +20,7 @@ spec: # from OpenBao via ESO (MIGRATION-sops-openbao.md). This was the LAST # sops consumer; phase 4 (age-key retirement) may proceed after soak. dependsOn: + - name: restore-authentik - name: infrastructure - name: infrastructure-crds - name: infrastructure-ceph-cluster diff --git a/kubernetes/clusters/homelab/gitea.yaml b/kubernetes/clusters/homelab/gitea.yaml index 82a8df8f..ba6fc93a 100644 --- a/kubernetes/clusters/homelab/gitea.yaml +++ b/kubernetes/clusters/homelab/gitea.yaml @@ -21,6 +21,7 @@ spec: kind: PersistentVolumeClaim name: data-gitea-0 dependsOn: + - name: restore-gitea - name: infrastructure - name: infrastructure-crds - name: infrastructure-ceph-cluster diff --git a/kubernetes/clusters/homelab/home-assistant.yaml b/kubernetes/clusters/homelab/home-assistant.yaml index 9c092d1c..24c02418 100644 --- a/kubernetes/clusters/homelab/home-assistant.yaml +++ b/kubernetes/clusters/homelab/home-assistant.yaml @@ -19,6 +19,7 @@ spec: # decryption block dropped 2026-07-13: path is sops-free — secrets come # from OpenBao via ESO (MIGRATION-sops-openbao.md). dependsOn: + - name: restore-home-assistant - name: infrastructure - name: infrastructure-crds - name: infrastructure-ceph-cluster @@ -54,6 +55,7 @@ spec: prune: true wait: true dependsOn: + - name: restore-home-assistant - name: infrastructure - name: infrastructure-crds - name: infrastructure-ceph-cluster @@ -75,6 +77,7 @@ spec: prune: true wait: true dependsOn: + - name: restore-home-assistant # shelly-manager's PVC - name: infrastructure - name: infrastructure-crds - name: infrastructure-ceph-cluster @@ -101,6 +104,7 @@ spec: # decryption block dropped 2026-07-13: path is sops-free — secrets come # from OpenBao via ESO (MIGRATION-sops-openbao.md). dependsOn: + - name: restore-home-assistant - name: infrastructure - name: infrastructure-crds --- @@ -141,6 +145,7 @@ spec: # First boot downloads the Whisper model (a few hundred MB) — give it room. wait: true dependsOn: + - name: restore-home-assistant - name: infrastructure - name: infrastructure-crds - name: infrastructure-ceph-cluster diff --git a/kubernetes/clusters/homelab/immich.yaml b/kubernetes/clusters/homelab/immich.yaml index 01d0f5c5..f3f4d5d4 100644 --- a/kubernetes/clusters/homelab/immich.yaml +++ b/kubernetes/clusters/homelab/immich.yaml @@ -20,6 +20,7 @@ spec: # decryption block dropped 2026-07-13: path is sops-free — secrets come # from OpenBao via ESO (MIGRATION-sops-openbao.md). dependsOn: + - name: restore-immich - name: infrastructure - name: infrastructure-crds - name: infrastructure-ceph-cluster diff --git a/kubernetes/clusters/homelab/influxdb.yaml b/kubernetes/clusters/homelab/influxdb.yaml index f9947bde..e4bc17a7 100644 --- a/kubernetes/clusters/homelab/influxdb.yaml +++ b/kubernetes/clusters/homelab/influxdb.yaml @@ -25,6 +25,7 @@ spec: # boilerplate from the monolithic seed KS — InfluxDB has no CNPG cluster, only # a chart-provisioned ceph-block PVC, so ceph is the only storage dep needed. dependsOn: + - name: restore-influxdb - name: infrastructure - name: infrastructure-crds - name: infrastructure-ceph-cluster diff --git a/kubernetes/clusters/homelab/matrix.yaml b/kubernetes/clusters/homelab/matrix.yaml index 1ab1a467..e4499e01 100644 --- a/kubernetes/clusters/homelab/matrix.yaml +++ b/kubernetes/clusters/homelab/matrix.yaml @@ -17,6 +17,7 @@ spec: prune: true wait: true dependsOn: + - name: restore-matrix - name: infrastructure - name: infrastructure-crds - name: infrastructure-ceph-cluster diff --git a/kubernetes/clusters/homelab/music-assistant.yaml b/kubernetes/clusters/homelab/music-assistant.yaml index b1697caa..dbd995c7 100644 --- a/kubernetes/clusters/homelab/music-assistant.yaml +++ b/kubernetes/clusters/homelab/music-assistant.yaml @@ -22,6 +22,7 @@ spec: secretRef: name: sops-age dependsOn: + - name: restore-music-assistant - name: infrastructure - name: infrastructure-crds - name: infrastructure-ceph-cluster diff --git a/kubernetes/clusters/homelab/navidrome.yaml b/kubernetes/clusters/homelab/navidrome.yaml index ee88da95..782fd751 100644 --- a/kubernetes/clusters/homelab/navidrome.yaml +++ b/kubernetes/clusters/homelab/navidrome.yaml @@ -14,6 +14,7 @@ spec: prune: true wait: true dependsOn: + - name: restore-navidrome - name: infrastructure - name: infrastructure-crds - name: infrastructure-ceph-cluster diff --git a/kubernetes/clusters/homelab/paperless.yaml b/kubernetes/clusters/homelab/paperless.yaml index ef9df784..b346c56e 100644 --- a/kubernetes/clusters/homelab/paperless.yaml +++ b/kubernetes/clusters/homelab/paperless.yaml @@ -20,6 +20,7 @@ spec: # decryption block dropped 2026-07-13: path is sops-free — secrets come # from OpenBao via ESO (MIGRATION-sops-openbao.md). dependsOn: + - name: restore-paperless - name: infrastructure - name: infrastructure-crds - name: infrastructure-ceph-cluster diff --git a/kubernetes/clusters/homelab/restore.yaml b/kubernetes/clusters/homelab/restore.yaml new file mode 100644 index 00000000..33f6d1ee --- /dev/null +++ b/kubernetes/clusters/homelab/restore.yaml @@ -0,0 +1,396 @@ +# Ready once Velero has completed a sync of the backups in the bucket. A +# Restore created before that finds no backup and fails for good. +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: velero-backups-synced + namespace: flux-system +spec: + interval: 10m + timeout: 30m + sourceRef: + kind: GitRepository + name: flux-system + path: ./kubernetes/restore/gate + prune: true + dependsOn: + - name: velero # the BackupStorageLocation and its credentials + healthChecks: + - apiVersion: velero.io/v1 + kind: BackupStorageLocation + name: default + namespace: velero + healthCheckExprs: + - apiVersion: velero.io/v1 + kind: BackupStorageLocation + current: has(status.lastSyncedTime) +--- +# One Kustomization per Velero Schedule (infrastructure/velero-schedules/schedules.yaml), +# each restoring that schedule's newest backup. An app depends on its own, so +# it waits for its data and for nothing else. Gitea's data goes first: it is +# the registry for the in-house images; the others depend on restore-gitea. +# openbao-daily has none: bootstrap.sh restores OpenBao from its raft snapshot. +# Every one carries the same health check: a schedule without any backup has +# nothing to restore (FailedValidation "No backups found for schedule"). +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: restore-gitea + namespace: flux-system +spec: + interval: 10m + timeout: 1h + sourceRef: + kind: GitRepository + name: flux-system + path: ./kubernetes/restore/gitea + prune: true + wait: true + dependsOn: + - name: velero-backups-synced + - name: infrastructure-ceph-cluster + healthCheckExprs: + - apiVersion: velero.io/v1 + kind: Restore + failed: >- + has(status.phase) && (status.phase in ['Failed', 'PartiallyFailed'] || + (status.phase == 'FailedValidation' && !(has(status.validationErrors) && + status.validationErrors.exists(e, e.startsWith('No backups found for schedule'))))) + current: >- + has(status.phase) && (status.phase == 'Completed' || status.phase == 'FailedValidation') +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: restore-apps + namespace: flux-system +spec: + interval: 10m + timeout: 1h + sourceRef: + kind: GitRepository + name: flux-system + path: ./kubernetes/restore/apps + prune: true + wait: true + dependsOn: + - name: restore-gitea + - name: velero-backups-synced + - name: infrastructure-ceph-cluster + healthCheckExprs: + - apiVersion: velero.io/v1 + kind: Restore + failed: >- + has(status.phase) && (status.phase in ['Failed', 'PartiallyFailed'] || + (status.phase == 'FailedValidation' && !(has(status.validationErrors) && + status.validationErrors.exists(e, e.startsWith('No backups found for schedule'))))) + current: >- + has(status.phase) && (status.phase == 'Completed' || status.phase == 'FailedValidation') +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: restore-audiobookshelf + namespace: flux-system +spec: + interval: 10m + timeout: 3h + sourceRef: + kind: GitRepository + name: flux-system + path: ./kubernetes/restore/audiobookshelf + prune: true + wait: true + dependsOn: + - name: restore-gitea + - name: velero-backups-synced + - name: infrastructure-ceph-cluster + healthCheckExprs: + - apiVersion: velero.io/v1 + kind: Restore + failed: >- + has(status.phase) && (status.phase in ['Failed', 'PartiallyFailed'] || + (status.phase == 'FailedValidation' && !(has(status.validationErrors) && + status.validationErrors.exists(e, e.startsWith('No backups found for schedule'))))) + current: >- + has(status.phase) && (status.phase == 'Completed' || status.phase == 'FailedValidation') +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: restore-authentik + namespace: flux-system +spec: + interval: 10m + timeout: 1h + sourceRef: + kind: GitRepository + name: flux-system + path: ./kubernetes/restore/authentik + prune: true + wait: true + dependsOn: + - name: restore-gitea + - name: velero-backups-synced + - name: infrastructure-ceph-cluster + healthCheckExprs: + - apiVersion: velero.io/v1 + kind: Restore + failed: >- + has(status.phase) && (status.phase in ['Failed', 'PartiallyFailed'] || + (status.phase == 'FailedValidation' && !(has(status.validationErrors) && + status.validationErrors.exists(e, e.startsWith('No backups found for schedule'))))) + current: >- + has(status.phase) && (status.phase == 'Completed' || status.phase == 'FailedValidation') +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: restore-home-assistant + namespace: flux-system +spec: + interval: 10m + timeout: 1h + sourceRef: + kind: GitRepository + name: flux-system + path: ./kubernetes/restore/home-assistant + prune: true + wait: true + dependsOn: + - name: restore-gitea + - name: velero-backups-synced + - name: infrastructure-ceph-cluster + healthCheckExprs: + - apiVersion: velero.io/v1 + kind: Restore + failed: >- + has(status.phase) && (status.phase in ['Failed', 'PartiallyFailed'] || + (status.phase == 'FailedValidation' && !(has(status.validationErrors) && + status.validationErrors.exists(e, e.startsWith('No backups found for schedule'))))) + current: >- + has(status.phase) && (status.phase == 'Completed' || status.phase == 'FailedValidation') +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: restore-immich + namespace: flux-system +spec: + interval: 10m + timeout: 6h + sourceRef: + kind: GitRepository + name: flux-system + path: ./kubernetes/restore/immich + prune: true + wait: true + dependsOn: + - name: restore-gitea + - name: velero-backups-synced + - name: infrastructure-ceph-cluster + healthCheckExprs: + - apiVersion: velero.io/v1 + kind: Restore + failed: >- + has(status.phase) && (status.phase in ['Failed', 'PartiallyFailed'] || + (status.phase == 'FailedValidation' && !(has(status.validationErrors) && + status.validationErrors.exists(e, e.startsWith('No backups found for schedule'))))) + current: >- + has(status.phase) && (status.phase == 'Completed' || status.phase == 'FailedValidation') +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: restore-influxdb + namespace: flux-system +spec: + interval: 10m + timeout: 1h + sourceRef: + kind: GitRepository + name: flux-system + path: ./kubernetes/restore/influxdb + prune: true + wait: true + dependsOn: + - name: restore-gitea + - name: velero-backups-synced + - name: infrastructure-ceph-cluster + healthCheckExprs: + - apiVersion: velero.io/v1 + kind: Restore + failed: >- + has(status.phase) && (status.phase in ['Failed', 'PartiallyFailed'] || + (status.phase == 'FailedValidation' && !(has(status.validationErrors) && + status.validationErrors.exists(e, e.startsWith('No backups found for schedule'))))) + current: >- + has(status.phase) && (status.phase == 'Completed' || status.phase == 'FailedValidation') +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: restore-matrix + namespace: flux-system +spec: + interval: 10m + timeout: 1h + sourceRef: + kind: GitRepository + name: flux-system + path: ./kubernetes/restore/matrix + prune: true + wait: true + dependsOn: + - name: restore-gitea + - name: velero-backups-synced + - name: infrastructure-ceph-cluster + healthCheckExprs: + - apiVersion: velero.io/v1 + kind: Restore + failed: >- + has(status.phase) && (status.phase in ['Failed', 'PartiallyFailed'] || + (status.phase == 'FailedValidation' && !(has(status.validationErrors) && + status.validationErrors.exists(e, e.startsWith('No backups found for schedule'))))) + current: >- + has(status.phase) && (status.phase == 'Completed' || status.phase == 'FailedValidation') +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: restore-music-assistant + namespace: flux-system +spec: + interval: 10m + timeout: 1h + sourceRef: + kind: GitRepository + name: flux-system + path: ./kubernetes/restore/music-assistant + prune: true + wait: true + dependsOn: + - name: restore-gitea + - name: velero-backups-synced + - name: infrastructure-ceph-cluster + healthCheckExprs: + - apiVersion: velero.io/v1 + kind: Restore + failed: >- + has(status.phase) && (status.phase in ['Failed', 'PartiallyFailed'] || + (status.phase == 'FailedValidation' && !(has(status.validationErrors) && + status.validationErrors.exists(e, e.startsWith('No backups found for schedule'))))) + current: >- + has(status.phase) && (status.phase == 'Completed' || status.phase == 'FailedValidation') +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: restore-navidrome + namespace: flux-system +spec: + interval: 10m + timeout: 1h + sourceRef: + kind: GitRepository + name: flux-system + path: ./kubernetes/restore/navidrome + prune: true + wait: true + dependsOn: + - name: restore-gitea + - name: velero-backups-synced + - name: infrastructure-ceph-cluster + healthCheckExprs: + - apiVersion: velero.io/v1 + kind: Restore + failed: >- + has(status.phase) && (status.phase in ['Failed', 'PartiallyFailed'] || + (status.phase == 'FailedValidation' && !(has(status.validationErrors) && + status.validationErrors.exists(e, e.startsWith('No backups found for schedule'))))) + current: >- + has(status.phase) && (status.phase == 'Completed' || status.phase == 'FailedValidation') +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: restore-paperless + namespace: flux-system +spec: + interval: 10m + timeout: 1h + sourceRef: + kind: GitRepository + name: flux-system + path: ./kubernetes/restore/paperless + prune: true + wait: true + dependsOn: + - name: restore-gitea + - name: velero-backups-synced + - name: infrastructure-ceph-cluster + healthCheckExprs: + - apiVersion: velero.io/v1 + kind: Restore + failed: >- + has(status.phase) && (status.phase in ['Failed', 'PartiallyFailed'] || + (status.phase == 'FailedValidation' && !(has(status.validationErrors) && + status.validationErrors.exists(e, e.startsWith('No backups found for schedule'))))) + current: >- + has(status.phase) && (status.phase == 'Completed' || status.phase == 'FailedValidation') +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: restore-trmnl + namespace: flux-system +spec: + interval: 10m + timeout: 1h + sourceRef: + kind: GitRepository + name: flux-system + path: ./kubernetes/restore/trmnl + prune: true + wait: true + dependsOn: + - name: restore-gitea + - name: velero-backups-synced + - name: infrastructure-ceph-cluster + healthCheckExprs: + - apiVersion: velero.io/v1 + kind: Restore + failed: >- + has(status.phase) && (status.phase in ['Failed', 'PartiallyFailed'] || + (status.phase == 'FailedValidation' && !(has(status.validationErrors) && + status.validationErrors.exists(e, e.startsWith('No backups found for schedule'))))) + current: >- + has(status.phase) && (status.phase == 'Completed' || status.phase == 'FailedValidation') +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: restore-wordpress + namespace: flux-system +spec: + interval: 10m + timeout: 1h + sourceRef: + kind: GitRepository + name: flux-system + path: ./kubernetes/restore/wordpress + prune: true + wait: true + dependsOn: + - name: restore-gitea + - name: velero-backups-synced + - name: infrastructure-ceph-cluster + healthCheckExprs: + - apiVersion: velero.io/v1 + kind: Restore + failed: >- + has(status.phase) && (status.phase in ['Failed', 'PartiallyFailed'] || + (status.phase == 'FailedValidation' && !(has(status.validationErrors) && + status.validationErrors.exists(e, e.startsWith('No backups found for schedule'))))) + current: >- + has(status.phase) && (status.phase == 'Completed' || status.phase == 'FailedValidation') diff --git a/kubernetes/clusters/homelab/trmnl.yaml b/kubernetes/clusters/homelab/trmnl.yaml index 639dde93..3d61d32c 100644 --- a/kubernetes/clusters/homelab/trmnl.yaml +++ b/kubernetes/clusters/homelab/trmnl.yaml @@ -17,6 +17,7 @@ spec: prune: true wait: true dependsOn: + - name: restore-trmnl - name: infrastructure - name: infrastructure-crds - name: infrastructure-ceph-cluster diff --git a/kubernetes/clusters/homelab/wordpress.yaml b/kubernetes/clusters/homelab/wordpress.yaml index d13d1be6..00f0607c 100644 --- a/kubernetes/clusters/homelab/wordpress.yaml +++ b/kubernetes/clusters/homelab/wordpress.yaml @@ -14,6 +14,7 @@ spec: prune: true wait: true dependsOn: + - name: restore-wordpress # infrastructure gates on the mariadb-operator HelmRelease being Ready, # so the MariaDB CRD exists and has a controller before this applies. - name: infrastructure diff --git a/kubernetes/infrastructure/velero-schedules/schedules.yaml b/kubernetes/infrastructure/velero-schedules/schedules.yaml index 7a25784d..ccea8bc6 100644 --- a/kubernetes/infrastructure/velero-schedules/schedules.yaml +++ b/kubernetes/infrastructure/velero-schedules/schedules.yaml @@ -9,7 +9,9 @@ # snapshotMoveData uploads PVC contents through Kopia and the node-agent, # independently of the K8s API surface. # -# Naming: -daily. Restore is manual — see RUNBOOK-cold-rebuild.md. +# Naming: -daily. Each has a Restore in kubernetes/restore// and a +# restore- Kustomization (clusters/homelab/restore.yaml), which a fresh +# cluster runs before the app starts. # Retention matches the CNPG Barman policy. apiVersion: velero.io/v1 @@ -199,7 +201,6 @@ spec: # This is the ONLY restore path for home-assistant-config (.storage, # secrets.yaml, the config git checkout). The HAOS-backup `seed` # initContainer that used to repopulate it was removed in 21a0a43f. - # Restore procedure and test: REN-249 runbook section C.2. schedule: "45 6 * * *" useOwnerReferencesInBackup: false template: diff --git a/kubernetes/restore/apps/kustomization.yaml b/kubernetes/restore/apps/kustomization.yaml new file mode 100644 index 00000000..bc08b2d5 --- /dev/null +++ b/kubernetes/restore/apps/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - restore.yaml diff --git a/kubernetes/restore/apps/restore.yaml b/kubernetes/restore/apps/restore.yaml new file mode 100644 index 00000000..9bb5a23b --- /dev/null +++ b/kubernetes/restore/apps/restore.yaml @@ -0,0 +1,7 @@ +apiVersion: velero.io/v1 +kind: Restore +metadata: + name: apps + namespace: velero +spec: + scheduleName: apps-daily diff --git a/kubernetes/restore/audiobookshelf/kustomization.yaml b/kubernetes/restore/audiobookshelf/kustomization.yaml new file mode 100644 index 00000000..bc08b2d5 --- /dev/null +++ b/kubernetes/restore/audiobookshelf/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - restore.yaml diff --git a/kubernetes/restore/audiobookshelf/restore.yaml b/kubernetes/restore/audiobookshelf/restore.yaml new file mode 100644 index 00000000..15b6b741 --- /dev/null +++ b/kubernetes/restore/audiobookshelf/restore.yaml @@ -0,0 +1,7 @@ +apiVersion: velero.io/v1 +kind: Restore +metadata: + name: audiobookshelf + namespace: velero +spec: + scheduleName: audiobookshelf-daily diff --git a/kubernetes/restore/authentik/kustomization.yaml b/kubernetes/restore/authentik/kustomization.yaml new file mode 100644 index 00000000..bc08b2d5 --- /dev/null +++ b/kubernetes/restore/authentik/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - restore.yaml diff --git a/kubernetes/restore/authentik/restore.yaml b/kubernetes/restore/authentik/restore.yaml new file mode 100644 index 00000000..88bb44d6 --- /dev/null +++ b/kubernetes/restore/authentik/restore.yaml @@ -0,0 +1,7 @@ +apiVersion: velero.io/v1 +kind: Restore +metadata: + name: authentik + namespace: velero +spec: + scheduleName: authentik-daily diff --git a/kubernetes/restore/gate/kustomization.yaml b/kubernetes/restore/gate/kustomization.yaml new file mode 100644 index 00000000..5191cdb5 --- /dev/null +++ b/kubernetes/restore/gate/kustomization.yaml @@ -0,0 +1,5 @@ +# Applies nothing: its Kustomization exists for its health check on the +# BackupStorageLocation (clusters/homelab/restore.yaml). +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: [] diff --git a/kubernetes/restore/gitea/kustomization.yaml b/kubernetes/restore/gitea/kustomization.yaml new file mode 100644 index 00000000..bc08b2d5 --- /dev/null +++ b/kubernetes/restore/gitea/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - restore.yaml diff --git a/kubernetes/restore/gitea/restore.yaml b/kubernetes/restore/gitea/restore.yaml new file mode 100644 index 00000000..f68caa0e --- /dev/null +++ b/kubernetes/restore/gitea/restore.yaml @@ -0,0 +1,8 @@ +# Gitea first: it is the registry for the in-house images. +apiVersion: velero.io/v1 +kind: Restore +metadata: + name: gitea + namespace: velero +spec: + scheduleName: gitea-daily diff --git a/kubernetes/restore/home-assistant/kustomization.yaml b/kubernetes/restore/home-assistant/kustomization.yaml new file mode 100644 index 00000000..bc08b2d5 --- /dev/null +++ b/kubernetes/restore/home-assistant/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - restore.yaml diff --git a/kubernetes/restore/home-assistant/restore.yaml b/kubernetes/restore/home-assistant/restore.yaml new file mode 100644 index 00000000..72f4e6b0 --- /dev/null +++ b/kubernetes/restore/home-assistant/restore.yaml @@ -0,0 +1,7 @@ +apiVersion: velero.io/v1 +kind: Restore +metadata: + name: home-assistant + namespace: velero +spec: + scheduleName: home-assistant-daily diff --git a/kubernetes/restore/immich/kustomization.yaml b/kubernetes/restore/immich/kustomization.yaml new file mode 100644 index 00000000..bc08b2d5 --- /dev/null +++ b/kubernetes/restore/immich/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - restore.yaml diff --git a/kubernetes/restore/immich/restore.yaml b/kubernetes/restore/immich/restore.yaml new file mode 100644 index 00000000..1c31c184 --- /dev/null +++ b/kubernetes/restore/immich/restore.yaml @@ -0,0 +1,7 @@ +apiVersion: velero.io/v1 +kind: Restore +metadata: + name: immich + namespace: velero +spec: + scheduleName: immich-daily diff --git a/kubernetes/restore/influxdb/kustomization.yaml b/kubernetes/restore/influxdb/kustomization.yaml new file mode 100644 index 00000000..bc08b2d5 --- /dev/null +++ b/kubernetes/restore/influxdb/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - restore.yaml diff --git a/kubernetes/restore/influxdb/restore.yaml b/kubernetes/restore/influxdb/restore.yaml new file mode 100644 index 00000000..9a9b2d0b --- /dev/null +++ b/kubernetes/restore/influxdb/restore.yaml @@ -0,0 +1,7 @@ +apiVersion: velero.io/v1 +kind: Restore +metadata: + name: influxdb + namespace: velero +spec: + scheduleName: influxdb-daily diff --git a/kubernetes/restore/matrix/kustomization.yaml b/kubernetes/restore/matrix/kustomization.yaml new file mode 100644 index 00000000..bc08b2d5 --- /dev/null +++ b/kubernetes/restore/matrix/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - restore.yaml diff --git a/kubernetes/restore/matrix/restore.yaml b/kubernetes/restore/matrix/restore.yaml new file mode 100644 index 00000000..83b5b34f --- /dev/null +++ b/kubernetes/restore/matrix/restore.yaml @@ -0,0 +1,7 @@ +apiVersion: velero.io/v1 +kind: Restore +metadata: + name: matrix + namespace: velero +spec: + scheduleName: matrix-daily diff --git a/kubernetes/restore/music-assistant/kustomization.yaml b/kubernetes/restore/music-assistant/kustomization.yaml new file mode 100644 index 00000000..bc08b2d5 --- /dev/null +++ b/kubernetes/restore/music-assistant/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - restore.yaml diff --git a/kubernetes/restore/music-assistant/restore.yaml b/kubernetes/restore/music-assistant/restore.yaml new file mode 100644 index 00000000..4c5b2643 --- /dev/null +++ b/kubernetes/restore/music-assistant/restore.yaml @@ -0,0 +1,7 @@ +apiVersion: velero.io/v1 +kind: Restore +metadata: + name: music-assistant + namespace: velero +spec: + scheduleName: music-assistant-daily diff --git a/kubernetes/restore/navidrome/kustomization.yaml b/kubernetes/restore/navidrome/kustomization.yaml new file mode 100644 index 00000000..bc08b2d5 --- /dev/null +++ b/kubernetes/restore/navidrome/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - restore.yaml diff --git a/kubernetes/restore/navidrome/restore.yaml b/kubernetes/restore/navidrome/restore.yaml new file mode 100644 index 00000000..4b8a9e1f --- /dev/null +++ b/kubernetes/restore/navidrome/restore.yaml @@ -0,0 +1,7 @@ +apiVersion: velero.io/v1 +kind: Restore +metadata: + name: navidrome + namespace: velero +spec: + scheduleName: navidrome-daily diff --git a/kubernetes/restore/paperless/kustomization.yaml b/kubernetes/restore/paperless/kustomization.yaml new file mode 100644 index 00000000..bc08b2d5 --- /dev/null +++ b/kubernetes/restore/paperless/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - restore.yaml diff --git a/kubernetes/restore/paperless/restore.yaml b/kubernetes/restore/paperless/restore.yaml new file mode 100644 index 00000000..9daaaf0e --- /dev/null +++ b/kubernetes/restore/paperless/restore.yaml @@ -0,0 +1,7 @@ +apiVersion: velero.io/v1 +kind: Restore +metadata: + name: paperless + namespace: velero +spec: + scheduleName: paperless-daily diff --git a/kubernetes/restore/trmnl/kustomization.yaml b/kubernetes/restore/trmnl/kustomization.yaml new file mode 100644 index 00000000..bc08b2d5 --- /dev/null +++ b/kubernetes/restore/trmnl/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - restore.yaml diff --git a/kubernetes/restore/trmnl/restore.yaml b/kubernetes/restore/trmnl/restore.yaml new file mode 100644 index 00000000..dda59430 --- /dev/null +++ b/kubernetes/restore/trmnl/restore.yaml @@ -0,0 +1,7 @@ +apiVersion: velero.io/v1 +kind: Restore +metadata: + name: trmnl + namespace: velero +spec: + scheduleName: trmnl-daily diff --git a/kubernetes/restore/wordpress/kustomization.yaml b/kubernetes/restore/wordpress/kustomization.yaml new file mode 100644 index 00000000..bc08b2d5 --- /dev/null +++ b/kubernetes/restore/wordpress/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - restore.yaml diff --git a/kubernetes/restore/wordpress/restore.yaml b/kubernetes/restore/wordpress/restore.yaml new file mode 100644 index 00000000..b1725f6e --- /dev/null +++ b/kubernetes/restore/wordpress/restore.yaml @@ -0,0 +1,7 @@ +apiVersion: velero.io/v1 +kind: Restore +metadata: + name: wordpress + namespace: velero +spec: + scheduleName: wordpress-daily -- 2.54.0