diff --git a/kubernetes/apps/monitoring-rules/prometheusrule-velero.yaml b/kubernetes/apps/monitoring-rules/prometheusrule-velero.yaml index 863250a5..6d4383cd 100644 --- a/kubernetes/apps/monitoring-rules/prometheusrule-velero.yaml +++ b/kubernetes/apps/monitoring-rules/prometheusrule-velero.yaml @@ -40,17 +40,31 @@ spec: description: "A Velero backup run for {{ $labels.schedule }} completed with Failed or PartiallyFailed status in the last hour. Inspect with `kubectl -n velero get backup.velero.io -l velero.io/schedule-name={{ $labels.schedule }}` and the velero pod logs." - alert: VeleroBackupStale - # Daily schedule with no successful backup in over 30h: covers - # the case where multiple consecutive runs failed (Garage flapping - # over the backup window) without the per-event alerts being - # acted on. + # Daily schedule with no successful backup in over 60h: covers the + # case where consecutive runs failed without the per-event alerts + # being acted on. + # + # 60h, not 30h. At 30h a single missed cycle pages, and a single + # missed cycle is the normal cost of touching the nodes: Velero moves + # data through pods, so draining a node kills whatever upload is in + # flight and the schedule is simply lost for that day — it is not + # retried. Rolling node work between 03:15 and 08:45 UTC therefore + # produced a guaranteed critical alert for every schedule it + # overlapped, which is noise about planned work rather than a signal + # about backups. + # + # 60h still catches what this is for. Two consecutive failures is no + # longer explicable by one maintenance window, and the per-event + # VeleroBackupFailed alert fires on the first failure regardless — so + # nothing is hidden, only the escalation to critical is deferred + # until a second cycle confirms it. expr: | - time() - velero_backup_last_successful_timestamp{schedule!=""} > 30 * 3600 + time() - velero_backup_last_successful_timestamp{schedule!=""} > 60 * 3600 and velero_backup_last_successful_timestamp{schedule!=""} > 0 for: 10m labels: severity: critical annotations: - summary: "Velero {{ $labels.schedule }} has no successful backup in 30h+" - description: "Last successful backup for {{ $labels.schedule }} was over 30 hours ago. With a daily schedule that means at least one full cycle missed. RPO is degrading." + summary: "Velero {{ $labels.schedule }} has no successful backup in 60h+" + description: "Last successful backup for {{ $labels.schedule }} was over 60 hours ago. With a daily schedule that is at least two consecutive cycles missed, which one maintenance window does not explain. RPO is degrading."