From fe23f18c865b6aee253cd996bc027d22c059ee91 Mon Sep 17 00:00:00 2001 From: Konstantin Renner Date: Thu, 25 Jun 2026 16:09:06 +0200 Subject: [PATCH] monitoring: retire Garage scrape, add Hetzner backup-endpoint alert MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Finish line of the backup migration — nas-1's Garage is no longer a cluster-backup dependency, so: - drop the prometheus `garage` scrape job (s3.nas-1.renner.uno:39033) + the garage-metrics-token secret (now orphaned) - delete prometheusrule-garage.yaml (GarageDown / GarageScrapeFailing) — it would fire every night once nas-1 powers off Replace it with outage visibility for the new offsite target: - add https://fsn1.your-objectstorage.com to the blackbox-http probe (bare endpoint returns 200) - new HetznerBackupEndpointDown alert on probe_success == 0 for 10m Velero outages are already covered by VeleroBackupValidationFailure / VeleroBackupStale. A granular CNPG "WAL archiving behind" alert needs CNPG PodMonitors (enablePodMonitor=false today, metrics not scraped) — left as a follow-up rather than shipping an alert on an absent metric. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../apps/monitoring-rules/kustomization.yaml | 2 +- .../monitoring-rules/probes-blackbox.yaml | 4 +++ .../prometheusrule-garage.yaml | 35 ------------------- .../prometheusrule-hetzner-backup.yaml | 31 ++++++++++++++++ .../monitoring/garage-metrics-token.sops.yaml | 23 ------------ .../monitoring/helmrelease-prometheus.yaml | 16 --------- kubernetes/apps/monitoring/kustomization.yaml | 1 - 7 files changed, 36 insertions(+), 76 deletions(-) delete mode 100644 kubernetes/apps/monitoring-rules/prometheusrule-garage.yaml create mode 100644 kubernetes/apps/monitoring-rules/prometheusrule-hetzner-backup.yaml delete mode 100644 kubernetes/apps/monitoring/garage-metrics-token.sops.yaml diff --git a/kubernetes/apps/monitoring-rules/kustomization.yaml b/kubernetes/apps/monitoring-rules/kustomization.yaml index d6bbdf3d..cb5f8cfb 100644 --- a/kubernetes/apps/monitoring-rules/kustomization.yaml +++ b/kubernetes/apps/monitoring-rules/kustomization.yaml @@ -4,7 +4,7 @@ kind: Kustomization # come from the kube-prometheus-stack chart. Flux applies this Kustomization # only after `monitoring` is Ready (chart installed → CRDs present). resources: - - prometheusrule-garage.yaml + - prometheusrule-hetzner-backup.yaml - prometheusrule-loki.yaml - prometheusrule-velero.yaml - alloy-servicemonitor.yaml diff --git a/kubernetes/apps/monitoring-rules/probes-blackbox.yaml b/kubernetes/apps/monitoring-rules/probes-blackbox.yaml index ca7c3c9c..26a93c5d 100644 --- a/kubernetes/apps/monitoring-rules/probes-blackbox.yaml +++ b/kubernetes/apps/monitoring-rules/probes-blackbox.yaml @@ -24,6 +24,10 @@ spec: - https://prometheus.renner.uno - https://alerts.renner.uno - https://dash.renner.uno + # Offsite backup target (Hetzner Object Storage). Bare endpoint + # returns 200. CNPG WAL archiving + Velero both depend on it; + # this probe replaces the old GarageDown signal on nas-1. + - https://fsn1.your-objectstorage.com # blackbox-tcp probe deleted 2026-05-20 — only target was the legacy # postgres-ha keepalived VIP at 10.2.40.135:5432, decommissioned after # the authentik DB moved to in-cluster CNPG (ak-pg). Re-introduce a diff --git a/kubernetes/apps/monitoring-rules/prometheusrule-garage.yaml b/kubernetes/apps/monitoring-rules/prometheusrule-garage.yaml deleted file mode 100644 index 920682b2..00000000 --- a/kubernetes/apps/monitoring-rules/prometheusrule-garage.yaml +++ /dev/null @@ -1,35 +0,0 @@ -apiVersion: monitoring.coreos.com/v1 -kind: PrometheusRule -metadata: - name: garage - namespace: monitoring - labels: - # kube-prometheus-stack's Prometheus resource selects on this label; - # without it the rule is created but never loaded into Prometheus. - release: kube-prometheus-stack -spec: - groups: - - name: garage.rules - rules: - - alert: GarageDown - expr: up{job="garage"} == 0 - for: 5m - labels: - severity: critical - annotations: - summary: "Garage S3 on nas-1 is unreachable" - description: "Prometheus has been unable to scrape job=garage (s3.nas-1.renner.uno:39033) for 5 minutes. K8s backups (Velero, CNPG Barman) and seed jobs depend on this endpoint." - - - alert: GarageScrapeFailing - # scrape_samples_scraped is a gauge (sample count from the - # most-recent scrape, not a counter), so rate() of a stable - # value is 0 and the alert would fire whenever Garage is - # healthy. Express the actual intent: zero samples on the - # last scrape with the endpoint still up. - expr: scrape_samples_scraped{job="garage"} == 0 and up{job="garage"} == 1 - for: 15m - labels: - severity: warning - annotations: - summary: "Garage metrics endpoint reachable but returning no samples" - description: "Endpoint is up but /metrics is returning zero samples for 15 minutes — likely an auth/token mismatch or Garage admin API misconfiguration." diff --git a/kubernetes/apps/monitoring-rules/prometheusrule-hetzner-backup.yaml b/kubernetes/apps/monitoring-rules/prometheusrule-hetzner-backup.yaml new file mode 100644 index 00000000..51be46f0 --- /dev/null +++ b/kubernetes/apps/monitoring-rules/prometheusrule-hetzner-backup.yaml @@ -0,0 +1,31 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: hetzner-backup + namespace: monitoring + labels: + # kube-prometheus-stack's Prometheus selects on this label. + release: kube-prometheus-stack +spec: + groups: + - name: hetzner-backup.rules + rules: + - alert: HetznerBackupEndpointDown + # Reachability of the offsite S3 backup target, via the + # blackbox-http probe (probes-blackbox.yaml). When this is down, + # CNPG WAL archiving stalls (WAL queues on the DB PVCs and + # drains on recovery) and Velero backups fail. Replaces the old + # GarageDown alert now that backups live on Hetzner, not nas-1. + # + # CNPG has no metric-based archiving alert yet — its pod metrics + # are not scraped (enablePodMonitor=false). Enabling CNPG + # PodMonitors for a granular "archiving falling behind" alert is + # a follow-up. Velero's own VeleroBackupValidationFailure / + # VeleroBackupStale rules already cover backup-level failures. + expr: probe_success{instance="https://fsn1.your-objectstorage.com"} == 0 + for: 10m + labels: + severity: critical + annotations: + summary: "Hetzner Object Storage (offsite backups) is unreachable" + description: "The blackbox probe of https://fsn1.your-objectstorage.com has been failing for 10 minutes. CNPG WAL archiving (ak-pg/pprl-pg/im-pg) and Velero backups go here — WAL is queuing on the DB PVCs and will drain when connectivity returns, but a prolonged outage risks filling pg_wal. Check the WAN/internet path and Hetzner status." diff --git a/kubernetes/apps/monitoring/garage-metrics-token.sops.yaml b/kubernetes/apps/monitoring/garage-metrics-token.sops.yaml deleted file mode 100644 index e08ac527..00000000 --- a/kubernetes/apps/monitoring/garage-metrics-token.sops.yaml +++ /dev/null @@ -1,23 +0,0 @@ -apiVersion: ENC[AES256_GCM,data:Jdc=,iv:668OOOPScHDq9dKiLelY0GMP50aJXVZ/iSU2GwKdX94=,tag:Nxl18sffT7seWBgy78x6EA==,type:str] -kind: ENC[AES256_GCM,data:iwYAV5hL,iv:tqVwG3dy0KgsD8N+8DVCDk8Z9F0Jup7x1kuKdmjlydc=,tag:hDR3GVdSOCczaTGCErWwFw==,type:str] -metadata: - name: ENC[AES256_GCM,data:gPHggby5aymlr53sr/5Pi3K6uh8=,iv:Hd0AIB4rWp92H2R5BbOojCNy6VyDlaRg/Wl16uytmQQ=,tag:vV18QVASmsLby/g2ViSrYg==,type:str] - namespace: ENC[AES256_GCM,data:GyAesZuMR05seg==,iv:4PxHLOwwwwUOqMv4Rl0AZr3Iyq3MsjbIqYboUvQinAs=,tag:LZb+UjkqQOq97BEm0u3SLw==,type:str] -type: ENC[AES256_GCM,data:z+4nmGzZ,iv:f7+jpue24up8HJ4A26RpN/stxLO3kPNW/k9G/sRqC/0=,tag:hqSuoJcNfYwRGnee/Yv7tA==,type:str] -stringData: - token: ENC[AES256_GCM,data:agLDseUMv9EdPvKC8IxwG5aRJ32JUwuGRT2yVWMuww+aGhyhwkn148ECzWc=,iv:CcHCpV29Gae0T8VIroCmrLlNV/nvDi5V7MJ+7pNEOcY=,tag:bgx7izloa+5Xa4u6S11o5w==,type:str] -sops: - age: - - recipient: age1yqjvqkwwwxdj7mr546zfs5ckzuwndp69ya2n2drd4vypwns8w4ys04qfy3 - enc: | - -----BEGIN AGE ENCRYPTED FILE----- - YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBIQjdUbmliWU5vVDE3U3d1 - Z2Zjb1NKUGhDaVUxRE1nZzN6UUZRMkRqTVRzCk5YZWo5MU1JL3ZaM2hLbGRNdTli - S3NoVFd4S2NaYTJqZVhCZ0duNGNWc0kKLS0tIE9XU3F5eHVaanlHOFJBbDVsNmFS - V0NKelRLa1B3c0ErQzVxSHk1WlFOWlUKr+yV7yABbuq4Sk8Maoq4Ad3aga6Qryar - gwL1lcqLCUGNKn3+VIXjmp5UTE2yeNKsOrNA1nvUuflPfHA7lI4Z+A== - -----END AGE ENCRYPTED FILE----- - lastmodified: "2026-04-26T14:30:58Z" - mac: ENC[AES256_GCM,data:NP0ueNFVAsKEQLLjQqArxq16i5D87EgiZYJC66Nw6S72FKRqHjxJU3GlRVZXiNKYkFKL2ko5+fLo4htviRoCt0XlmnHevJMnyeY0/YF16WqeRxIW36P1tkFJJA+VmDIuxPLfSqn5QEc0+wfWXNtGNCl5yUkDUDf2+1LYtse+bYw=,iv:giYYkUs465qZBl/gh3ksanY8aEA8otDgkwx0h2IRSr4=,tag:h/bddtGkHxWyebP40DTCQg==,type:str] - unencrypted_suffix: _unencrypted - version: 3.10.2 diff --git a/kubernetes/apps/monitoring/helmrelease-prometheus.yaml b/kubernetes/apps/monitoring/helmrelease-prometheus.yaml index e0ba8891..2236b115 100644 --- a/kubernetes/apps/monitoring/helmrelease-prometheus.yaml +++ b/kubernetes/apps/monitoring/helmrelease-prometheus.yaml @@ -64,10 +64,6 @@ spec: resources: requests: storage: 50Gi - # Mount additional secrets at /etc/prometheus/secrets// - # so scrape configs can reference credential files. - secrets: - - garage-metrics-token # Scrape existing Proxmox infrastructure targets additionalScrapeConfigs: # pve-exporter is a blackbox-style exporter: Prometheus passes @@ -107,18 +103,6 @@ spec: - targets: ["gitea.srv.renner.uno:3000"] metrics_path: /metrics - # Garage S3 admin/metrics on nas-1. DSM reverse proxy republishes - # 127.0.0.1:3903 (Garage admin port, plain HTTP, no native TLS) as - # http://s3.nas-1.renner.uno:39033. metrics_token enforces auth. - - job_name: garage - scheme: http - metrics_path: /metrics - static_configs: - - targets: ["s3.nas-1.renner.uno:39033"] - authorization: - type: Bearer - credentials_file: /etc/prometheus/secrets/garage-metrics-token/token - # Grafana grafana: defaultDashboardsEnabled: true diff --git a/kubernetes/apps/monitoring/kustomization.yaml b/kubernetes/apps/monitoring/kustomization.yaml index dfa3d5bd..cc241916 100644 --- a/kubernetes/apps/monitoring/kustomization.yaml +++ b/kubernetes/apps/monitoring/kustomization.yaml @@ -2,7 +2,6 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - grafana-oauth.sops.yaml - - garage-metrics-token.sops.yaml - alertmanager-smtp.sops.yaml - pve-exporter-config.sops.yaml - pve-exporter-deployment.yaml -- 2.54.0