diff --git a/kubernetes/apps/monitoring-rules/kustomization.yaml b/kubernetes/apps/monitoring-rules/kustomization.yaml index d0f39b1e..17bcf6d4 100644 --- a/kubernetes/apps/monitoring-rules/kustomization.yaml +++ b/kubernetes/apps/monitoring-rules/kustomization.yaml @@ -13,6 +13,10 @@ resources: - prometheusrule-netbird-ingress.yaml - prometheusrule-velero.yaml - prometheusrule-woodpecker-rotation.yaml + # CNPG backup health. Seven Postgres clusters ship to Hetzner and nothing + # alerted if that stopped — HetznerBackupEndpointDown only probes the S3 + # endpoint, which proves reachability, not that anything was written. + - prometheusrule-cnpg-backup.yaml - alloy-servicemonitor.yaml - cnpg-podmonitors.yaml - home-assistant-servicemonitor.yaml diff --git a/kubernetes/apps/monitoring-rules/prometheusrule-cnpg-backup.yaml b/kubernetes/apps/monitoring-rules/prometheusrule-cnpg-backup.yaml new file mode 100644 index 00000000..b3c64c72 --- /dev/null +++ b/kubernetes/apps/monitoring-rules/prometheusrule-cnpg-backup.yaml @@ -0,0 +1,78 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: cnpg-backup + namespace: monitoring + labels: + # kube-prometheus-stack's Prometheus resource selects on this label; + # without it the rule is created but never loaded into Prometheus. + release: kube-prometheus-stack +spec: + groups: + - name: cnpg-backup.rules + rules: + # WHY THIS FILE EXISTS + # Seven Postgres clusters back up to Hetzner and NOTHING alerted if that + # stopped. The only related rule was HetznerBackupEndpointDown, which + # probes the S3 endpoint — it proves the bucket is reachable, not that + # anything was written to it. Same failure shape as the WordPress backup + # that failed nightly for three days against a bucket that was declared + # in tofu but never applied: the status surface looked fine because + # nothing checked the thing that mattered. + + - alert: CNPGWalArchiveBacklog + # `ready` counts WAL segments sitting in pg_wal/archive_status waiting + # to be archived. Steady state is 0 across every cluster (verified + # 2026-08-19). A sustained backlog means the archiver is failing: + # point-in-time recovery is silently broken from that moment, and + # pg_wal keeps growing until it fills the PVC and takes Postgres down. + # + # This is the load-bearing alert of the two. It depends only on the + # instance-level collector, so unlike the backup-age metrics below it + # is unaffected by the missing Backup CRs (#522). + # + # Threshold: >5 segments sustained for 30m. Brief non-zero values are + # normal under write bursts — a segment is 16 MiB, so 5 is roughly + # 80 MiB of un-archived WAL, well below anything that threatens the + # volume but far above idle noise. + expr: cnpg_collector_pg_wal_archive_status{value="ready"} > 5 + for: 30m + labels: + severity: warning + annotations: + summary: "CNPG WAL archiving is falling behind on {{ $labels.namespace }}/{{ $labels.pod }}" + description: "{{ $value }} WAL segments have been waiting to archive for over 30 minutes. Point-in-time recovery is broken for anything after the last archived segment, and pg_wal will grow until the PVC fills. Check the barman-cloud sidecar logs and the ObjectStore credentials: `kubectl -n {{ $labels.namespace }} logs {{ $labels.pod }} -c plugin-barman-cloud`." + + - alert: CNPGBackupFailed + # cnpg_collector_last_failed_backup_timestamp is a unix timestamp, 0 + # when no failure is recorded. Guard on > 0 so the epoch-zero default + # is not read as "failed in 1970". + expr: | + cnpg_collector_last_failed_backup_timestamp > 0 + and + (time() - cnpg_collector_last_failed_backup_timestamp) < 86400 + for: 5m + labels: + severity: warning + annotations: + summary: "CNPG backup failed in the last 24h on {{ $labels.namespace }}/{{ $labels.pod }}" + description: "A base backup for {{ $labels.namespace }}/{{ $labels.pod }} failed within the last 24 hours. Verify what is actually in the bucket rather than trusting CR status — see the barman-cloud-backup-list recipe in kubernetes/apps/home-assistant/MIGRATION-recorder-postgres.md." + + # DELIBERATELY ABSENT: a backup-age alert, which is the one you would + # most want here. + # + # The natural expression is + # (time() - cnpg_collector_last_available_backup_timestamp) > 48*3600 + # and it CANNOT be used today: that metric reads exactly 0 on every + # instance in the cluster (verified 2026-08-19 across gf-pg, ak-pg-1/2, + # pprl-pg-2/3, im-pg, wp-pg, ha-pg). CNPG has no record of any + # successful backup anywhere, because the Backup CRs it derives that + # from are being deleted within a minute of completing — see #522. + # + # Backups themselves DO work; that was verified independently by + # listing the bucket with barman-cloud-backup-list and restoring a WAL + # segment back out of S3. The gap is in the record, not the data. + # + # Adding the rule now would fire permanently for all seven clusters and + # be silenced within a day, which is worse than not having it. Add it as + # part of resolving #522, once the metric reflects reality.