2026-06-25 14:10:39 +00:00
7 changed files with 36 additions and 76 deletions
@@ -4,7 +4,7 @@ kind: Kustomization
 # come from the kube-prometheus-stack chart. Flux applies this Kustomization
 # only after `monitoring` is Ready (chart installed → CRDs present).
 resources:
-  - prometheusrule-garage.yaml
+  - prometheusrule-hetzner-backup.yaml
  - prometheusrule-loki.yaml
  - prometheusrule-velero.yaml
  - alloy-servicemonitor.yaml
@@ -24,6 +24,10 @@ spec:
        - https://prometheus.renner.uno
        - https://alerts.renner.uno
        - https://dash.renner.uno
+        # Offsite backup target (Hetzner Object Storage). Bare endpoint
+        # returns 200. CNPG WAL archiving + Velero both depend on it;
+        # this probe replaces the old GarageDown signal on nas-1.
+        - https://fsn1.your-objectstorage.com
 # blackbox-tcp probe deleted 2026-05-20 — only target was the legacy
 # postgres-ha keepalived VIP at 10.2.40.135:5432, decommissioned after
 # the authentik DB moved to in-cluster CNPG (ak-pg). Re-introduce a
@@ -1,35 +0,0 @@
-apiVersion: monitoring.coreos.com/v1
-kind: PrometheusRule
-metadata:
-  name: garage
-  namespace: monitoring
-  labels:
-    # kube-prometheus-stack's Prometheus resource selects on this label;
-    # without it the rule is created but never loaded into Prometheus.
-    release: kube-prometheus-stack
-spec:
-  groups:
-    - name: garage.rules
-      rules:
-        - alert: GarageDown
-          expr: up{job="garage"} == 0
-          for: 5m
-          labels:
-            severity: critical
-          annotations:
-            summary: "Garage S3 on nas-1 is unreachable"
-            description: "Prometheus has been unable to scrape job=garage (s3.nas-1.renner.uno:39033) for 5 minutes. K8s backups (Velero, CNPG Barman) and seed jobs depend on this endpoint."
-
-        - alert: GarageScrapeFailing
-          # scrape_samples_scraped is a gauge (sample count from the
-          # most-recent scrape, not a counter), so rate() of a stable
-          # value is 0 and the alert would fire whenever Garage is
-          # healthy. Express the actual intent: zero samples on the
-          # last scrape with the endpoint still up.
-          expr: scrape_samples_scraped{job="garage"} == 0 and up{job="garage"} == 1
-          for: 15m
-          labels:
-            severity: warning
-          annotations:
-            summary: "Garage metrics endpoint reachable but returning no samples"
-            description: "Endpoint is up but /metrics is returning zero samples for 15 minutes — likely an auth/token mismatch or Garage admin API misconfiguration."
@@ -0,0 +1,31 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: hetzner-backup
+  namespace: monitoring
+  labels:
+    # kube-prometheus-stack's Prometheus selects on this label.
+    release: kube-prometheus-stack
+spec:
+  groups:
+    - name: hetzner-backup.rules
+      rules:
+        - alert: HetznerBackupEndpointDown
+          # Reachability of the offsite S3 backup target, via the
+          # blackbox-http probe (probes-blackbox.yaml). When this is down,
+          # CNPG WAL archiving stalls (WAL queues on the DB PVCs and
+          # drains on recovery) and Velero backups fail. Replaces the old
+          # GarageDown alert now that backups live on Hetzner, not nas-1.
+          #
+          # CNPG has no metric-based archiving alert yet — its pod metrics
+          # are not scraped (enablePodMonitor=false). Enabling CNPG
+          # PodMonitors for a granular "archiving falling behind" alert is
+          # a follow-up. Velero's own VeleroBackupValidationFailure /
+          # VeleroBackupStale rules already cover backup-level failures.
+          expr: probe_success{instance="https://fsn1.your-objectstorage.com"} == 0
+          for: 10m
+          labels:
+            severity: critical
+          annotations:
+            summary: "Hetzner Object Storage (offsite backups) is unreachable"
+            description: "The blackbox probe of https://fsn1.your-objectstorage.com has been failing for 10 minutes. CNPG WAL archiving (ak-pg/pprl-pg/im-pg) and Velero backups go here — WAL is queuing on the DB PVCs and will drain when connectivity returns, but a prolonged outage risks filling pg_wal. Check the WAN/internet path and Hetzner status."
@@ -1,23 +0,0 @@
-apiVersion: ENC[AES256_GCM,data:Jdc=,iv:668OOOPScHDq9dKiLelY0GMP50aJXVZ/iSU2GwKdX94=,tag:Nxl18sffT7seWBgy78x6EA==,type:str]
-kind: ENC[AES256_GCM,data:iwYAV5hL,iv:tqVwG3dy0KgsD8N+8DVCDk8Z9F0Jup7x1kuKdmjlydc=,tag:hDR3GVdSOCczaTGCErWwFw==,type:str]
-metadata:
-    name: ENC[AES256_GCM,data:gPHggby5aymlr53sr/5Pi3K6uh8=,iv:Hd0AIB4rWp92H2R5BbOojCNy6VyDlaRg/Wl16uytmQQ=,tag:vV18QVASmsLby/g2ViSrYg==,type:str]
-    namespace: ENC[AES256_GCM,data:GyAesZuMR05seg==,iv:4PxHLOwwwwUOqMv4Rl0AZr3Iyq3MsjbIqYboUvQinAs=,tag:LZb+UjkqQOq97BEm0u3SLw==,type:str]
-type: ENC[AES256_GCM,data:z+4nmGzZ,iv:f7+jpue24up8HJ4A26RpN/stxLO3kPNW/k9G/sRqC/0=,tag:hqSuoJcNfYwRGnee/Yv7tA==,type:str]
-stringData:
-    token: ENC[AES256_GCM,data:agLDseUMv9EdPvKC8IxwG5aRJ32JUwuGRT2yVWMuww+aGhyhwkn148ECzWc=,iv:CcHCpV29Gae0T8VIroCmrLlNV/nvDi5V7MJ+7pNEOcY=,tag:bgx7izloa+5Xa4u6S11o5w==,type:str]
-sops:
-    age:
-        - recipient: age1yqjvqkwwwxdj7mr546zfs5ckzuwndp69ya2n2drd4vypwns8w4ys04qfy3
-          enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBIQjdUbmliWU5vVDE3U3d1
-            Z2Zjb1NKUGhDaVUxRE1nZzN6UUZRMkRqTVRzCk5YZWo5MU1JL3ZaM2hLbGRNdTli
-            S3NoVFd4S2NaYTJqZVhCZ0duNGNWc0kKLS0tIE9XU3F5eHVaanlHOFJBbDVsNmFS
-            V0NKelRLa1B3c0ErQzVxSHk1WlFOWlUKr+yV7yABbuq4Sk8Maoq4Ad3aga6Qryar
-            gwL1lcqLCUGNKn3+VIXjmp5UTE2yeNKsOrNA1nvUuflPfHA7lI4Z+A==
-            -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2026-04-26T14:30:58Z"
-    mac: ENC[AES256_GCM,data:NP0ueNFVAsKEQLLjQqArxq16i5D87EgiZYJC66Nw6S72FKRqHjxJU3GlRVZXiNKYkFKL2ko5+fLo4htviRoCt0XlmnHevJMnyeY0/YF16WqeRxIW36P1tkFJJA+VmDIuxPLfSqn5QEc0+wfWXNtGNCl5yUkDUDf2+1LYtse+bYw=,iv:giYYkUs465qZBl/gh3ksanY8aEA8otDgkwx0h2IRSr4=,tag:h/bddtGkHxWyebP40DTCQg==,type:str]
-    unencrypted_suffix: _unencrypted
-    version: 3.10.2
@@ -64,10 +64,6 @@ spec:
              resources:
                requests:
                  storage: 50Gi
-        # Mount additional secrets at /etc/prometheus/secrets/<name>/<key>
-        # so scrape configs can reference credential files.
-        secrets:
-          - garage-metrics-token
        # Scrape existing Proxmox infrastructure targets
        additionalScrapeConfigs:
          # pve-exporter is a blackbox-style exporter: Prometheus passes
@@ -107,18 +103,6 @@ spec:
              - targets: ["gitea.srv.renner.uno:3000"]
            metrics_path: /metrics

-          # Garage S3 admin/metrics on nas-1. DSM reverse proxy republishes
-          # 127.0.0.1:3903 (Garage admin port, plain HTTP, no native TLS) as
-          # http://s3.nas-1.renner.uno:39033. metrics_token enforces auth.
-          - job_name: garage
-            scheme: http
-            metrics_path: /metrics
-            static_configs:
-              - targets: ["s3.nas-1.renner.uno:39033"]
-            authorization:
-              type: Bearer
-              credentials_file: /etc/prometheus/secrets/garage-metrics-token/token
-
    # Grafana
    grafana:
      defaultDashboardsEnabled: true
@@ -2,7 +2,6 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
  - grafana-oauth.sops.yaml
-  - garage-metrics-token.sops.yaml
  - alertmanager-smtp.sops.yaml
  - pve-exporter-config.sops.yaml
  - pve-exporter-deployment.yaml