fix(velero,k8s-cp): wait for BSL sync + 48 GB CP nodes
bootstrap-restore Job:
- Wait for BackupStorageLocation default to reach phase=Available
before iterating schedules so the BSL controller has had a chance
to sync Backup CRs from Garage.
- Per-schedule retry: if the visible Backup count is 0 on first read,
sleep 10 s × 10 retries before declaring "no Backups produced yet".
This unblocks the BSL-sync-race regression where rebuild 741 skipped
authentik-daily, music-assistant-daily, and immich-daily.
- Use wc -l | tr -d ' ' for backup count instead of grep -c which can
double-emit on empty input (matches the migrate-* pipeline pattern).
opentofu k8s control plane:
- Bump VM memory 24 GB → 48 GB on k8s-1/k8s-2/k8s-3. Rebuild 741
showed sustained API-server timeouts under Velero data-mover
coordination load (DataDownload list calls timing out with
"the server was unable to return a response in the time allotted"),
causing audiobookshelf-books restore to PartiallyFail and
migrate-immich seed Backup to also PartiallyFail. Hosts have
plenty of free RAM, so increasing CP headroom is the cheap fix.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -48,6 +48,32 @@ spec:
|
||||
| tar -xz -C /tmp
|
||||
mv "/tmp/velero-${VELERO_VERSION}-linux-amd64/velero" /usr/local/bin/velero
|
||||
|
||||
count_backups() {
|
||||
velero backup get -l "velero.io/schedule-name=$1" 2>/dev/null \
|
||||
| tail -n +2 | wc -l | tr -d ' '
|
||||
}
|
||||
|
||||
# Wait for the BackupStorageLocation to validate at least once
|
||||
# so the BSL controller has had a chance to sync Backup CRs
|
||||
# from Garage into the cluster API. Without this, on a fresh
|
||||
# cluster boot we race ahead of sync and falsely conclude
|
||||
# certain schedules have "no Backups produced yet".
|
||||
echo "Waiting for BSL default to validate ..."
|
||||
for i in $(seq 1 60); do
|
||||
phase=$(kubectl -n velero get backupstoragelocation default \
|
||||
-o jsonpath='{.status.phase}' 2>/dev/null || true)
|
||||
if [ "$phase" = "Available" ]; then
|
||||
echo "BSL default Available."
|
||||
break
|
||||
fi
|
||||
echo " BSL phase=${phase:-?} ($i/60) — sleeping 5s"; sleep 5
|
||||
done
|
||||
|
||||
# Belt-and-braces: even after BSL is Available, the per-schedule
|
||||
# Backup CR sync can lag. Sleep an extra grace period before
|
||||
# iterating schedules.
|
||||
sleep 30
|
||||
|
||||
for sched in $SCHEDULES; do
|
||||
stable="${sched}-bootstrap"
|
||||
|
||||
@@ -63,11 +89,18 @@ spec:
|
||||
;;
|
||||
esac
|
||||
|
||||
# First-bootstrap guard: no Backups = nothing to restore.
|
||||
# Count non-header rows from `velero backup get`.
|
||||
count=$(velero backup get -l "velero.io/schedule-name=$sched" 2>/dev/null \
|
||||
| tail -n +2 | grep -c . || true)
|
||||
if [ "${count:-0}" -eq 0 ]; then
|
||||
# Per-schedule sync-retry: if count is 0 on first read, give
|
||||
# the BSL controller a few more chances before declaring
|
||||
# "no Backups produced yet". Ten attempts × 10 s = up to
|
||||
# ~100 s additional wait per schedule, only when needed.
|
||||
count=0
|
||||
for j in $(seq 1 10); do
|
||||
count=$(count_backups "$sched")
|
||||
if [ "$count" -gt 0 ]; then break; fi
|
||||
echo "[$sched] no Backups visible yet ($j/10) — sleeping 10s"
|
||||
sleep 10
|
||||
done
|
||||
if [ "$count" -eq 0 ]; then
|
||||
echo "[$sched] no Backups produced yet — skipping (expected on first bootstrap)"
|
||||
continue
|
||||
fi
|
||||
|
||||
@@ -4,7 +4,7 @@ module "k8s_1" {
|
||||
name = "k8s-1.srv.renner.uno"
|
||||
target_node = "jupiter"
|
||||
vmid = 150
|
||||
memory = 24576
|
||||
memory = 49152
|
||||
cores = 8
|
||||
|
||||
template_name = null
|
||||
|
||||
@@ -4,7 +4,7 @@ module "k8s_2" {
|
||||
name = "k8s-2.srv.renner.uno"
|
||||
target_node = "saturn"
|
||||
vmid = 151
|
||||
memory = 24576
|
||||
memory = 49152
|
||||
cores = 8
|
||||
|
||||
template_name = null
|
||||
|
||||
@@ -4,7 +4,7 @@ module "k8s_3" {
|
||||
name = "k8s-3.srv.renner.uno"
|
||||
target_node = "neptune"
|
||||
vmid = 152
|
||||
memory = 24576
|
||||
memory = 49152
|
||||
cores = 8
|
||||
|
||||
template_name = null
|
||||
|
||||
Reference in New Issue
Block a user