fix: [alert] VeleroBackupFailed (velero) (#657)

Reviewed-on: k.renner/homelab#658

kubernetes/RUNBOOK-cold-rebuild.md

@@ -87,7 +87,7 @@ all three nodes identically from there, since `apply-config --insecure` does
 not care whether the target is a VM or bare metal.
 
 ```
-https://factory.talos.dev/image/0f6fb8bd8bb53013e9743b7c7b099d96fb9c32fd2464ced32693c2dc0ea5feb9/v1.12.11/metal-amd64.iso
+https://factory.talos.dev/image/0f6fb8bd8bb53013e9743b7c7b099d96fb9c32fd2464ced32693c2dc0ea5feb9/v1.13.9/metal-amd64.iso
 ```
 
 That schematic is deliberately not the one `.woodpecker/deploy-infrastructure.yml`

kubernetes/apps/monitoring-rules/prometheusrule-velero.yaml

@@ -40,17 +40,31 @@ spec:
             description: "A Velero backup run for {{ $labels.schedule }} completed with Failed or PartiallyFailed status in the last hour. Inspect with `kubectl -n velero get backup.velero.io -l velero.io/schedule-name={{ $labels.schedule }}` and the velero pod logs."
 
         - alert: VeleroBackupStale
-          # Daily schedule with no successful backup in over 30h: covers
-          # the case where multiple consecutive runs failed (Garage flapping
-          # over the backup window) without the per-event alerts being
-          # acted on.
+          # Daily schedule with no successful backup in over 60h: covers the
+          # case where consecutive runs failed without the per-event alerts
+          # being acted on.
+          #
+          # 60h, not 30h. At 30h a single missed cycle pages, and a single
+          # missed cycle is the normal cost of touching the nodes: Velero moves
+          # data through pods, so draining a node kills whatever upload is in
+          # flight and the schedule is simply lost for that day — it is not
+          # retried. Rolling node work between 03:15 and 08:45 UTC therefore
+          # produced a guaranteed critical alert for every schedule it
+          # overlapped, which is noise about planned work rather than a signal
+          # about backups.
+          #
+          # 60h still catches what this is for. Two consecutive failures is no
+          # longer explicable by one maintenance window, and the per-event
+          # VeleroBackupFailed alert fires on the first failure regardless — so
+          # nothing is hidden, only the escalation to critical is deferred
+          # until a second cycle confirms it.
           expr: |
-            time() - velero_backup_last_successful_timestamp{schedule!=""} > 30 * 3600
+            time() - velero_backup_last_successful_timestamp{schedule!=""} > 60 * 3600
             and
             velero_backup_last_successful_timestamp{schedule!=""} > 0
           for: 10m
           labels:
             severity: critical
           annotations:
-            summary: "Velero {{ $labels.schedule }} has no successful backup in 30h+"
-            description: "Last successful backup for {{ $labels.schedule }} was over 30 hours ago. With a daily schedule that means at least one full cycle missed. RPO is degrading."
+            summary: "Velero {{ $labels.schedule }} has no successful backup in 60h+"
+            description: "Last successful backup for {{ $labels.schedule }} was over 60 hours ago. With a daily schedule that is at least two consecutive cycles missed, which one maintenance window does not explain. RPO is degrading."

kubernetes/infrastructure/post-crds/velero-schedules.yaml

@@ -203,16 +203,16 @@ metadata:
   labels:
     app.kubernetes.io/part-of: homelab-backup
 spec:
-  # Daily at 06:45 local (next free stagger slot). All nine PVCs live
-  # in the home-assistant namespace regardless of which app/ dir owns
+  # Daily at 06:45 local (next free stagger slot). The app PVCs live in
+  # the home-assistant namespace regardless of which app/ dir owns
   # their manifest (matter-server, otbr, whisper, speech-to-phrase,
   # zone-configurator all set `namespace: home-assistant` via their
   # Kustomization, not per-resource): home-assistant-config (.storage
   # entity registry, recorder DB, secrets.yaml), esphome-data,
   # shelly-manager-data, matter-server-data, otbr-data, whisper-data,
-  # speech-to-phrase-models, speech-to-phrase-train,
-  # zone-configurator-config. No CNPG cluster in this namespace, so no
-  # label exclusion is needed. On restore, home-assistant-config has its
+  # speech-to-phrase-models, speech-to-phrase-train, zigbee2mqtt-data,
+  # zwavejs-store, zone-configurator-config. On restore,
+  # home-assistant-config has its
   # own path (the HA pod's `seed` initContainer pulling the native HAOS
   # backup from Garage, see the home-assistant Kustomization comment in
   # kubernetes/clusters/homelab/home-assistant.yaml); layering a Velero
@@ -229,6 +229,20 @@ spec:
     storageLocation: default
     ttl: 720h0m0s
     snapshotMoveData: true
+    # The ha-pg CNPG cluster now runs in this namespace (it did not when
+    # this schedule was written — the old comment here still claimed
+    # "no CNPG cluster in this namespace"). Its PVC ha-pg-1 carries the
+    # cnpg.io/cluster label and was being swept in: a crash-consistent
+    # snapshot of a running Postgres data directory is not a restorable
+    # backup, the same trap the wordpress-daily comment below describes.
+    # ha-pg is covered by Barman — daily base backup at 02:50 UTC
+    # (kubernetes/apps/home-assistant/ha-pg-scheduled-backup.yaml) plus
+    # continuous WAL archiving — so this exclusion removes a redundant
+    # 20 GiB upload, matching paperless/authentik/immich/trmnl above.
+    labelSelector:
+      matchExpressions:
+        - key: cnpg.io/cluster
+          operator: DoesNotExist
 ---
 apiVersion: velero.io/v1
 kind: Schedule

kubernetes/talos/patches/cluster.yaml

@@ -42,8 +42,17 @@ machine:
   # Only read at install and upgrade time. Changing it here does nothing to a
   # running node until that node is upgraded. Bump the tag with the Talos
   # version — it must match the running version, or the upgrade is a downgrade.
+  #
+  # Upgrades follow the documented path: migration is only tested between
+  # adjacent minor releases, so go to the latest patch of each intermediate
+  # minor rather than skipping. 1.12.11 -> 1.13.9 satisfies that — one minor
+  # step from the last 1.12 patch, no intermediate needed.
+  #
+  # Kubernetes is a separate decision and is not forced by this. Talos 1.13
+  # supports Kubernetes 1.36, 1.35, 1.34, 1.33, 1.32 and 1.31, so the cluster
+  # stays on 1.35.2 until `talosctl upgrade-k8s` moves it deliberately.
   install:
-    image: factory.talos.dev/metal-installer/0f6fb8bd8bb53013e9743b7c7b099d96fb9c32fd2464ced32693c2dc0ea5feb9:v1.12.11
+    image: factory.talos.dev/metal-installer/0f6fb8bd8bb53013e9743b7c7b099d96fb9c32fd2464ced32693c2dc0ea5feb9:v1.13.9
   features:
     rbac: true
     stableHostname: true