fix: [alert] KubeJobFailed (velero) (#1095 )

2026-10-01 15:44:33 +00:00
2 changed files with 28 additions and 2 deletions
@@ -78,6 +78,11 @@ spec:
              env:
                - name: RESTORE_PHASE_TIMEOUT_S
                  value: "14400"
+                # Whole-sweep budget; caps each restore's timeout. Keep
+                # ~30 min below activeDeadlineSeconds so the summary and
+                # cleanup run before Kubernetes kills the pod.
+                - name: SWEEP_BUDGET_S
+                  value: "19800"
                # Skip a restore projected to take the fullest OSD past
                # this (SKIP, not FAIL). See restore-test.sh.
                - name: OSD_MAX_PCT
@@ -42,6 +42,14 @@ FAILURES=()
 SKIPS=()
 RUN_ID=$(date -u +%s)
 RESTORE_PHASE_TIMEOUT_S=${RESTORE_PHASE_TIMEOUT_S:-14400}   # 4 h — immich library (~180 GiB, millions of small files) does not finish in 2 h
+# Whole-sweep budget, kept below the Job's activeDeadlineSeconds. The
+# 2026-10-01 run started immich 2 h 17 m in, so its 4 h phase timeout could
+# never fire before the 6 h deadline killed the pod: no summary, no
+# cleanup. Each restore's timeout is capped by what is left of this budget,
+# and services that start after it is used up are SKIPped. The slack to the
+# deadline covers the last service's parity probes, cleanup and summary.
+SWEEP_BUDGET_S=${SWEEP_BUDGET_S:-19800}   # 5 h 30 m
+SWEEP_DEADLINE=$(( $(date +%s) + SWEEP_BUDGET_S ))

 log() { printf '[%s] %s\n' "$(date -u +%H:%M:%S)" "$*"; }

@@ -53,6 +61,8 @@ check() {
        PASS=$((PASS + 1))
    else
        echo "FAIL"
+        # Also printed now: a pod killed before the summary loses FAILURES.
+        tail -4 /tmp/check.out | sed 's/^/      | /'
        FAILURES+=("$name -- $(tail -4 /tmp/check.out | tr '\n' ' ')")
        FAIL=$((FAIL + 1))
    fi
@@ -497,6 +507,11 @@ for schedule in "${SCHEDULES[@]}"; do
    echo
    echo "=== service: ${src_ns} (schedule=${schedule}) ==="

+    if (( $(date +%s) >= SWEEP_DEADLINE )); then
+        skip "${src_ns}: restore (time budget)" "sweep budget of ${SWEEP_BUDGET_S}s used up"
+        continue
+    fi
+
    # Newest terminal Backup wins. PartiallyFailed is intentionally
    # included: Velero only restores DataUploads that completed, so
    # the partial source still gives a valid restore for the PVCs that
@@ -578,8 +593,10 @@ spec:
    - persistentvolumes
 EOF

+    phase_timeout=$(( SWEEP_DEADLINE - $(date +%s) ))
+    (( phase_timeout > RESTORE_PHASE_TIMEOUT_S )) && phase_timeout=$RESTORE_PHASE_TIMEOUT_S
    check "${src_ns}: restore reaches phase=Completed" bash -c "
-        deadline=\$(( \$(date +%s) + ${RESTORE_PHASE_TIMEOUT_S} ))
+        deadline=\$(( \$(date +%s) + ${phase_timeout} ))
        last=''
        while [ \$(date +%s) -lt \$deadline ]; do
            p=\$(kubectl -n velero get restore.velero.io ${restore_name} \
@@ -595,7 +612,7 @@ EOF
            fi
            sleep 15
        done
-        echo 'TIMEOUT after ${RESTORE_PHASE_TIMEOUT_S}s'
+        echo 'TIMEOUT after ${phase_timeout}s'
        kubectl -n velero get restore.velero.io ${restore_name} -o yaml | tail -60
        exit 1
    "
@@ -609,6 +626,10 @@ EOF
        check "${src_ns}: at least one PVC was restored" false
    else
        for pvc in "${PVCS[@]}"; do
+            if (( $(date +%s) >= SWEEP_DEADLINE )); then
+                skip "${src_ns}: remaining PVC parity probes" "sweep time budget used up"
+                break
+            fi
            check "${src_ns}: PVC ${pvc} file count parity (±10% / ±${ABS_PARITY_SLACK})" \
                verify_pvc_file_count_parity "$src_ns" "$dst_ns" "$pvc"
        done