fix: [alert] KubeJobFailed (velero) (#726)

Reviewed-on: k.renner/homelab#732

kubernetes/infrastructure/velero/restore-test.sh

@@ -5,8 +5,9 @@
 #   1. Velero Restore reaches phase=Completed against yesterday's
 #      scheduled Backup (real artifact, not a synthetic one).
 #   2. Each restored PVC is mountable and its file count is within ±10%
-#      of the live source volume, ignoring the transient subtrees listed
-#      in volatile_paths_for().
+#      (or ±ABS_PARITY_SLACK files, whichever is wider) of the live
+#      source volume, ignoring the transient subtrees listed in
+#      volatile_paths_for().
 #
 # What this does NOT exercise:
 #   - CNPG database restore (barman-cloud, separate code path).
@@ -317,11 +318,37 @@ EOF
     count_files_in_pvc "$src_ns" "$tmp_pvc" "src-${pvc:0:40}-${RUN_ID}" "$excludes"
 }
 
-# Restored count must be within ±10% of the source's current file count.
-# That tolerance absorbs day-of drift since the backup ran without
-# letting "lost half the data" pass silently. Volatile subtrees are pruned
-# from BOTH sides — keyed off the source identity, since the PVC name is
-# preserved across the restore but the destination namespace is not.
+# Restored count must be within max(±10%, ±ABS_PARITY_SLACK files) of the
+# source's current file count. That tolerance absorbs day-of drift since
+# the backup ran without letting "lost half the data" pass silently.
+# Volatile subtrees are pruned from BOTH sides — keyed off the source
+# identity, since the PVC name is preserved across the restore but the
+# destination namespace is not.
+#
+# Why the absolute floor exists. A purely relative window is structurally
+# unable to pass a small volume: ±10% of 9 files is ±0.9, i.e. a budget of
+# ZERO files, and ±10% of 2 files is ±0.2. The 2026-09-01 run failed on
+# exactly that — audiobookshelf/libation-config source=9 restored=8 and
+# trmnl/trmnl-renderer-data source=2 restored=3, both off by one file and
+# in opposite directions, with all 12 restores at phase=Completed and the
+# other 46 checks green. Parity compares a backup taken up to ~24 h ago
+# against the live volume *now*, so a single config write on a tiny volume
+# between those two moments is expected, not a defect in the backup.
+#
+# The floor only governs volumes below ~10*ABS_PARITY_SLACK files; above
+# that the 10% window still binds, so real data loss on the volumes that
+# carry data keeps failing loudly. Two files of slack rather than one
+# because a single logical write on a config volume routinely moves more
+# than one file (an app writing a settings file plus a transient sidecar),
+# and a floor that buys exactly one month is not worth the change.
+#
+# What the floor must NOT swallow, and does not: a non-empty source that
+# restored to an *empty* volume fails unconditionally below, before any
+# tolerance is applied. The old relative-only check caught that case as a
+# side effect (delta = 1.0 > 0.10); it is now explicit so the floor cannot
+# erode it on a volume of ABS_PARITY_SLACK files or fewer.
+ABS_PARITY_SLACK=${ABS_PARITY_SLACK:-2}
+
 verify_pvc_file_count_parity() {
     local src_ns="$1" dst_ns="$2" pvc="$3"
     local restored source excludes
@@ -339,11 +366,15 @@ verify_pvc_file_count_parity() {
     if [[ "$source" -eq 0 && "$restored" -eq 0 ]]; then
         return 0
     fi
-    # Use awk to compute a 10% window without bc.
-    awk -v s="$source" -v r="$restored" 'BEGIN {
-        if (s == 0) { exit !(r == 0) }
-        delta = (r - s) / s; if (delta < 0) delta = -delta
-        exit !(delta <= 0.10)
+    # Empty on either side while the other has data: never tolerable.
+    if [[ "$source" -eq 0 || "$restored" -eq 0 ]]; then
+        return 1
+    fi
+    # Use awk to compute the window without bc.
+    awk -v s="$source" -v r="$restored" -v slack="$ABS_PARITY_SLACK" 'BEGIN {
+        delta = r - s; if (delta < 0) delta = -delta
+        tol = 0.10 * s; if (tol < slack) tol = slack
+        exit !(delta <= tol)
     }'
 }
 
@@ -492,7 +523,7 @@ EOF
         check "${src_ns}: at least one PVC was restored" false
     else
         for pvc in "${PVCS[@]}"; do
-            check "${src_ns}: PVC ${pvc} file count parity (±10%)" \
+            check "${src_ns}: PVC ${pvc} file count parity (±10% / ±${ABS_PARITY_SLACK})" \
                 verify_pvc_file_count_parity "$src_ns" "$dst_ns" "$pvc"
         done
     fi