kubernetes/apps/monitoring-rules/kustomization.yaml

@@ -5,6 +5,7 @@ kind: Kustomization
 # only after `monitoring` is Ready (chart installed → CRDs present).
 resources:
   - prometheusrule-ceph.yaml
+  - prometheusrule-ci-openbao.yaml
   - prometheusrule-cilium.yaml
   - prometheusrule-disk-wear.yaml
   - prometheusrule-hetzner-backup.yaml

kubernetes/apps/monitoring-rules/probes-blackbox.yaml

@@ -49,6 +49,23 @@ spec:
         # ports outside the hairpin's 80/443 — real WAN path.
         - http://proxy-1.renner.uno:9000/health
         - http://proxy-2.renner.uno:9000/health
+        # Woodpecker. Ungated, root returns 200 with no redirect. Every
+        # scheduled job in this repo runs here — the alert-monitor cron, the
+        # nightly scans, the deploy targets — so an outage here is silent by
+        # construction: the thing that would have told you is the thing that
+        # is down.
+        - https://ci.renner.uno
+        # OpenBao. Probes the health API rather than the UI on purpose: a
+        # SEALED instance still serves the UI (307 -> /ui/ -> 200) while every
+        # ExternalSecret in the cluster fails, which is exactly the
+        # green-while-broken shape this file warns about elsewhere.
+        #
+        # standbyok=true because the endpoint answers 200 only on the ACTIVE
+        # node and 429 on a standby — the external route currently lands on a
+        # standby, so the bare path would be permanently red. With the
+        # parameter, 200 means initialized, unsealed and serving, on whichever
+        # node answers; 503 sealed and 501 uninitialized both stay red.
+        - https://openbao.renner.uno/v1/sys/health?standbyok=true
         # APP-HEALTH probes for the gated monitoring UIs, from the correct
         # in-cluster vantage (monitoring-vantage-point-rule): the gate is
         # only on the ingress, so the ClusterIP /-/healthy endpoints are

kubernetes/apps/monitoring-rules/prometheusrule-ci-openbao.yaml

@@ -0,0 +1,47 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: ci-openbao
+  namespace: monitoring
+  labels:
+    # kube-prometheus-stack's Prometheus resource selects on this label;
+    # without it the rule is created but never loaded into Prometheus.
+    release: kube-prometheus-stack
+spec:
+  groups:
+    - name: ci-openbao.rules
+      rules:
+        - alert: WoodpeckerDown
+          # Probe target defined in probes-blackbox.yaml. Root, ungated, 200.
+          expr: probe_success{job="blackbox-http", instance="https://ci.renner.uno"} == 0
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Woodpecker is not answering on ci.renner.uno"
+            description: >-
+              Every scheduled job in this repo runs here: the alert-monitor
+              cron that files these issues, the nightly scans, and every deploy
+              target. Nothing else reports this outage, because the component
+              that would have reported it is the one that is down. Alertmanager
+              itself is unaffected, which is why this alert exists.
+        - alert: OpenBaoUnavailable
+          # Probes /v1/sys/health?standbyok=true — 200 on any unsealed node,
+          # 503 sealed, 501 uninitialized. The UI stays up when sealed, so a
+          # root probe would not catch this.
+          #
+          # Matched by prefix rather than the full string: the target carries a
+          # query parameter, and an exact match would break silently if the
+          # instance label ever encodes it differently.
+          expr: probe_success{job="blackbox-http", instance=~"https://openbao\\.renner\\.uno/v1/sys/health.*"} == 0
+          for: 5m
+          labels:
+            severity: critical
+          annotations:
+            summary: "OpenBao is sealed, uninitialized or unreachable"
+            description: >-
+              Every ExternalSecret in the cluster resolves through OpenBao.
+              Existing Secrets keep working, so nothing breaks at once — the
+              damage shows up at the next refresh interval or the next pod that
+              needs a fresh credential. Check whether it is sealed before
+              assuming a network fault.