fix(velero): remove dead bootstrap-restore Job (Fixes #200)
The velero-bootstrap-restore Job (restore-job.yaml + its RBAC) was never wired into any Flux Kustomization — no `path` field in kubernetes/clusters/homelab/ referenced it, and the parent infrastructure/kustomization.yaml only aggregates `velero/`, not `velero-bootstrap-restore/`. It never ran, creating false confidence that app-data auto-restore-on-bootstrap existed. Per owner decision on #200: delete it and rely on the documented manual restore procedure in kubernetes/RUNBOOK-cold-rebuild.md. The real seed path is the seed-* Kustomizations (rsync from legacy host) plus the per-app restore steps in the runbook. Also updates two now-dangling doc comments in post-crds/velero-schedules.yaml that referenced the deleted Job's SCHEDULES mechanism, pointing them at the runbook instead. kubectl kustomize on infrastructure/ and infrastructure/post-crds/ both build clean after removal. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 4.8
@@ -16,9 +16,10 @@
|
||||
# node-agent: PVC contents are uploaded to Garage and restored
|
||||
# independently of the K8s API surface.
|
||||
#
|
||||
# Naming convention: <app>-daily for the app's Schedule. The bootstrap
|
||||
# Restore flow (see velero-bootstrap-restore/restore-job.yaml) keys off
|
||||
# these names.
|
||||
# Naming convention: <app>-daily for the app's Schedule. Restore is a
|
||||
# manual, operator-driven procedure — see kubernetes/RUNBOOK-cold-rebuild.md.
|
||||
# (There is no automatic bootstrap-restore Job; it was removed as dead code,
|
||||
# issue #200.)
|
||||
#
|
||||
# Retention: 30 days, matching the live CNPG Barman ObjectStore policy.
|
||||
|
||||
@@ -211,14 +212,13 @@ spec:
|
||||
# shelly-manager-data, matter-server-data, otbr-data, whisper-data,
|
||||
# speech-to-phrase-models, speech-to-phrase-train,
|
||||
# zone-configurator-config. No CNPG cluster in this namespace, so no
|
||||
# label exclusion is needed. Not added to velero-bootstrap-restore's
|
||||
# SCHEDULES: home-assistant-config already has its own bootstrap
|
||||
# restore path (the HA pod's `seed` initContainer pulling the native
|
||||
# HAOS backup from Garage, see the home-assistant Kustomization
|
||||
# comment in kubernetes/clusters/homelab/home-assistant.yaml) and
|
||||
# layering an automatic Velero PVC restore on top of that flow for
|
||||
# the same PVC needs its own review — same reasoning as leaving
|
||||
# openbao-daily out of SCHEDULES.
|
||||
# label exclusion is needed. On restore, home-assistant-config has its
|
||||
# own path (the HA pod's `seed` initContainer pulling the native HAOS
|
||||
# backup from Garage, see the home-assistant Kustomization comment in
|
||||
# kubernetes/clusters/homelab/home-assistant.yaml); layering a Velero
|
||||
# PVC restore on top of that flow for the same PVC needs its own review.
|
||||
# Restore of these schedules is manual — see
|
||||
# kubernetes/RUNBOOK-cold-rebuild.md.
|
||||
schedule: "45 6 * * *"
|
||||
useOwnerReferencesInBackup: false
|
||||
template:
|
||||
|
||||
@@ -1,5 +0,0 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
resources:
|
||||
- rbac.yaml
|
||||
- restore-job.yaml
|
||||
@@ -1,30 +0,0 @@
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: bootstrap-restore
|
||||
namespace: velero
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: Role
|
||||
metadata:
|
||||
name: bootstrap-restore
|
||||
namespace: velero
|
||||
rules:
|
||||
- apiGroups: ["velero.io"]
|
||||
resources: ["restores", "backups", "schedules"]
|
||||
verbs: ["create", "get", "list", "watch", "delete"]
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: RoleBinding
|
||||
metadata:
|
||||
name: bootstrap-restore
|
||||
namespace: velero
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: bootstrap-restore
|
||||
namespace: velero
|
||||
roleRef:
|
||||
kind: Role
|
||||
name: bootstrap-restore
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
@@ -1,121 +0,0 @@
|
||||
---
|
||||
# Bootstrap-time Velero Restore driver.
|
||||
#
|
||||
# For each Schedule listed in SCHEDULES, this Job submits a Restore CR
|
||||
# referencing the latest Backup that Schedule produced. Intended behaviour:
|
||||
#
|
||||
# - First-ever bootstrap (fresh cluster, no Velero Backups yet)
|
||||
# → every Schedule has zero Backups, the Job skips each one with a
|
||||
# log line, exits 0. Migration seed Jobs (Phase 3 *-seed/) are
|
||||
# expected to populate PVCs from legacy tarballs instead.
|
||||
#
|
||||
# - Normal bootstrap (Velero has Backups in Garage)
|
||||
# → Job submits one Restore per Schedule named `<schedule>-bootstrap`.
|
||||
# Existing completed Restores with that name are left alone
|
||||
# (idempotent); other phases are deleted and re-created.
|
||||
#
|
||||
# Ordering: the enclosing Flux Kustomization dependsOn `infrastructure`
|
||||
# (Velero server) and `infrastructure-ceph-cluster` (so the restored PVCs
|
||||
# can actually be provisioned). App Kustomizations dependsOn this one, so
|
||||
# no app pod starts against an empty PVC.
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: velero-bootstrap-restore
|
||||
namespace: velero
|
||||
spec:
|
||||
backoffLimit: 3
|
||||
ttlSecondsAfterFinished: 86400
|
||||
template:
|
||||
spec:
|
||||
restartPolicy: OnFailure
|
||||
serviceAccountName: bootstrap-restore
|
||||
containers:
|
||||
- name: restore
|
||||
image: alpine:3.21
|
||||
env:
|
||||
- name: VELERO_VERSION
|
||||
value: v1.16.0
|
||||
- name: SCHEDULES
|
||||
value: "paperless-daily audiobookshelf-daily authentik-daily music-assistant-daily immich-daily influxdb-daily apps-daily"
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
set -eu
|
||||
apk add --no-cache --quiet curl ca-certificates >/dev/null
|
||||
curl -sfL "https://github.com/vmware-tanzu/velero/releases/download/${VELERO_VERSION}/velero-${VELERO_VERSION}-linux-amd64.tar.gz" \
|
||||
| tar -xz -C /tmp
|
||||
mv "/tmp/velero-${VELERO_VERSION}-linux-amd64/velero" /usr/local/bin/velero
|
||||
|
||||
count_backups() {
|
||||
velero backup get -l "velero.io/schedule-name=$1" 2>/dev/null \
|
||||
| tail -n +2 | wc -l | tr -d ' '
|
||||
}
|
||||
|
||||
# Wait for the BackupStorageLocation to validate at least once
|
||||
# so the BSL controller has had a chance to sync Backup CRs
|
||||
# from Garage into the cluster API. Without this, on a fresh
|
||||
# cluster boot we race ahead of sync and falsely conclude
|
||||
# certain schedules have "no Backups produced yet".
|
||||
echo "Waiting for BSL default to validate ..."
|
||||
for i in $(seq 1 60); do
|
||||
phase=$(kubectl -n velero get backupstoragelocation default \
|
||||
-o jsonpath='{.status.phase}' 2>/dev/null || true)
|
||||
if [ "$phase" = "Available" ]; then
|
||||
echo "BSL default Available."
|
||||
break
|
||||
fi
|
||||
echo " BSL phase=${phase:-?} ($i/60) — sleeping 5s"; sleep 5
|
||||
done
|
||||
|
||||
# Belt-and-braces: even after BSL is Available, the per-schedule
|
||||
# Backup CR sync can lag. Sleep an extra grace period before
|
||||
# iterating schedules.
|
||||
sleep 30
|
||||
|
||||
for sched in $SCHEDULES; do
|
||||
stable="${sched}-bootstrap"
|
||||
|
||||
# Idempotency: a completed (or partially-failed) Restore
|
||||
# with the stable name means work for this schedule is done.
|
||||
phase=$(velero restore get "$stable" -n velero -o json 2>/dev/null \
|
||||
| grep -o '"phase":[[:space:]]*"[^"]*"' | head -1 \
|
||||
| sed 's/.*"\([^"]*\)"$/\1/' || true)
|
||||
case "$phase" in
|
||||
Completed|PartiallyFailed)
|
||||
echo "[$sched] existing Restore $stable phase=$phase — skipping"
|
||||
continue
|
||||
;;
|
||||
esac
|
||||
|
||||
# Per-schedule sync-retry: if count is 0 on first read, give
|
||||
# the BSL controller a few more chances before declaring
|
||||
# "no Backups produced yet". Ten attempts × 10 s = up to
|
||||
# ~100 s additional wait per schedule, only when needed.
|
||||
count=0
|
||||
for j in $(seq 1 10); do
|
||||
count=$(count_backups "$sched")
|
||||
if [ "$count" -gt 0 ]; then break; fi
|
||||
echo "[$sched] no Backups visible yet ($j/10) — sleeping 10s"
|
||||
sleep 10
|
||||
done
|
||||
if [ "$count" -eq 0 ]; then
|
||||
echo "[$sched] no Backups produced yet — skipping (expected on first bootstrap)"
|
||||
continue
|
||||
fi
|
||||
|
||||
# Clear any stale Restore (InProgress, Failed, New) so we
|
||||
# can re-create with the stable name.
|
||||
if [ -n "$phase" ]; then
|
||||
echo "[$sched] removing stale Restore $stable (phase=$phase)"
|
||||
velero restore delete "$stable" --confirm 2>/dev/null || true
|
||||
fi
|
||||
|
||||
echo "[$sched] creating Restore $stable from latest scheduled Backup"
|
||||
velero restore create "$stable" \
|
||||
--from-schedule "$sched" \
|
||||
--existing-resource-policy update \
|
||||
--wait
|
||||
done
|
||||
echo "velero-bootstrap-restore: done"
|
||||
Reference in New Issue
Block a user