kubernetes/apps/monitoring-rules/kustomization.yaml

@@ -13,6 +13,10 @@ resources:
   - prometheusrule-netbird-ingress.yaml
   - prometheusrule-velero.yaml
   - prometheusrule-woodpecker-rotation.yaml
+  # CNPG backup health. Seven Postgres clusters ship to Hetzner and nothing
+  # alerted if that stopped — HetznerBackupEndpointDown only probes the S3
+  # endpoint, which proves reachability, not that anything was written.
+  - prometheusrule-cnpg-backup.yaml
   - alloy-servicemonitor.yaml
   - cnpg-podmonitors.yaml
   - home-assistant-servicemonitor.yaml

kubernetes/apps/monitoring-rules/prometheusrule-cnpg-backup.yaml

@@ -0,0 +1,78 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: cnpg-backup
+  namespace: monitoring
+  labels:
+    # kube-prometheus-stack's Prometheus resource selects on this label;
+    # without it the rule is created but never loaded into Prometheus.
+    release: kube-prometheus-stack
+spec:
+  groups:
+    - name: cnpg-backup.rules
+      rules:
+        # WHY THIS FILE EXISTS
+        # Seven Postgres clusters back up to Hetzner and NOTHING alerted if that
+        # stopped. The only related rule was HetznerBackupEndpointDown, which
+        # probes the S3 endpoint — it proves the bucket is reachable, not that
+        # anything was written to it. Same failure shape as the WordPress backup
+        # that failed nightly for three days against a bucket that was declared
+        # in tofu but never applied: the status surface looked fine because
+        # nothing checked the thing that mattered.
+
+        - alert: CNPGWalArchiveBacklog
+          # `ready` counts WAL segments sitting in pg_wal/archive_status waiting
+          # to be archived. Steady state is 0 across every cluster (verified
+          # 2026-08-19). A sustained backlog means the archiver is failing:
+          # point-in-time recovery is silently broken from that moment, and
+          # pg_wal keeps growing until it fills the PVC and takes Postgres down.
+          #
+          # This is the load-bearing alert of the two. It depends only on the
+          # instance-level collector, so unlike the backup-age metrics below it
+          # is unaffected by the missing Backup CRs (#522).
+          #
+          # Threshold: >5 segments sustained for 30m. Brief non-zero values are
+          # normal under write bursts — a segment is 16 MiB, so 5 is roughly
+          # 80 MiB of un-archived WAL, well below anything that threatens the
+          # volume but far above idle noise.
+          expr: cnpg_collector_pg_wal_archive_status{value="ready"} > 5
+          for: 30m
+          labels:
+            severity: warning
+          annotations:
+            summary: "CNPG WAL archiving is falling behind on {{ $labels.namespace }}/{{ $labels.pod }}"
+            description: "{{ $value }} WAL segments have been waiting to archive for over 30 minutes. Point-in-time recovery is broken for anything after the last archived segment, and pg_wal will grow until the PVC fills. Check the barman-cloud sidecar logs and the ObjectStore credentials: `kubectl -n {{ $labels.namespace }} logs {{ $labels.pod }} -c plugin-barman-cloud`."
+
+        - alert: CNPGBackupFailed
+          # cnpg_collector_last_failed_backup_timestamp is a unix timestamp, 0
+          # when no failure is recorded. Guard on > 0 so the epoch-zero default
+          # is not read as "failed in 1970".
+          expr: |
+            cnpg_collector_last_failed_backup_timestamp > 0
+            and
+            (time() - cnpg_collector_last_failed_backup_timestamp) < 86400
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: "CNPG backup failed in the last 24h on {{ $labels.namespace }}/{{ $labels.pod }}"
+            description: "A base backup for {{ $labels.namespace }}/{{ $labels.pod }} failed within the last 24 hours. Verify what is actually in the bucket rather than trusting CR status — see the barman-cloud-backup-list recipe in kubernetes/apps/home-assistant/MIGRATION-recorder-postgres.md."
+
+        # DELIBERATELY ABSENT: a backup-age alert, which is the one you would
+        # most want here.
+        #
+        # The natural expression is
+        #   (time() - cnpg_collector_last_available_backup_timestamp) > 48*3600
+        # and it CANNOT be used today: that metric reads exactly 0 on every
+        # instance in the cluster (verified 2026-08-19 across gf-pg, ak-pg-1/2,
+        # pprl-pg-2/3, im-pg, wp-pg, ha-pg). CNPG has no record of any
+        # successful backup anywhere, because the Backup CRs it derives that
+        # from are being deleted within a minute of completing — see #522.
+        #
+        # Backups themselves DO work; that was verified independently by
+        # listing the bucket with barman-cloud-backup-list and restoring a WAL
+        # segment back out of S3. The gap is in the record, not the data.
+        #
+        # Adding the rule now would fire permanently for all seven clusters and
+        # be silenced within a day, which is worse than not having it. Add it as
+        # part of resolving #522, once the metric reflects reality.