diff --git a/kubernetes/apps/monitoring-rules/kustomization.yaml b/kubernetes/apps/monitoring-rules/kustomization.yaml index 748cc9d2..e226cfad 100644 --- a/kubernetes/apps/monitoring-rules/kustomization.yaml +++ b/kubernetes/apps/monitoring-rules/kustomization.yaml @@ -5,6 +5,7 @@ kind: Kustomization # only after `monitoring` is Ready (chart installed → CRDs present). resources: - prometheusrule-ceph.yaml + - prometheusrule-ci-openbao.yaml - prometheusrule-cilium.yaml - prometheusrule-disk-wear.yaml - prometheusrule-hetzner-backup.yaml diff --git a/kubernetes/apps/monitoring-rules/probes-blackbox.yaml b/kubernetes/apps/monitoring-rules/probes-blackbox.yaml index 5daa164f..950fabb2 100644 --- a/kubernetes/apps/monitoring-rules/probes-blackbox.yaml +++ b/kubernetes/apps/monitoring-rules/probes-blackbox.yaml @@ -49,6 +49,23 @@ spec: # ports outside the hairpin's 80/443 — real WAN path. - http://proxy-1.renner.uno:9000/health - http://proxy-2.renner.uno:9000/health + # Woodpecker. Ungated, root returns 200 with no redirect. Every + # scheduled job in this repo runs here — the alert-monitor cron, the + # nightly scans, the deploy targets — so an outage here is silent by + # construction: the thing that would have told you is the thing that + # is down. + - https://ci.renner.uno + # OpenBao. Probes the health API rather than the UI on purpose: a + # SEALED instance still serves the UI (307 -> /ui/ -> 200) while every + # ExternalSecret in the cluster fails, which is exactly the + # green-while-broken shape this file warns about elsewhere. + # + # standbyok=true because the endpoint answers 200 only on the ACTIVE + # node and 429 on a standby — the external route currently lands on a + # standby, so the bare path would be permanently red. With the + # parameter, 200 means initialized, unsealed and serving, on whichever + # node answers; 503 sealed and 501 uninitialized both stay red. + - https://openbao.renner.uno/v1/sys/health?standbyok=true # APP-HEALTH probes for the gated monitoring UIs, from the correct # in-cluster vantage (monitoring-vantage-point-rule): the gate is # only on the ingress, so the ClusterIP /-/healthy endpoints are diff --git a/kubernetes/apps/monitoring-rules/prometheusrule-ci-openbao.yaml b/kubernetes/apps/monitoring-rules/prometheusrule-ci-openbao.yaml new file mode 100644 index 00000000..6b74e33d --- /dev/null +++ b/kubernetes/apps/monitoring-rules/prometheusrule-ci-openbao.yaml @@ -0,0 +1,47 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: ci-openbao + namespace: monitoring + labels: + # kube-prometheus-stack's Prometheus resource selects on this label; + # without it the rule is created but never loaded into Prometheus. + release: kube-prometheus-stack +spec: + groups: + - name: ci-openbao.rules + rules: + - alert: WoodpeckerDown + # Probe target defined in probes-blackbox.yaml. Root, ungated, 200. + expr: probe_success{job="blackbox-http", instance="https://ci.renner.uno"} == 0 + for: 5m + labels: + severity: warning + annotations: + summary: "Woodpecker is not answering on ci.renner.uno" + description: >- + Every scheduled job in this repo runs here: the alert-monitor + cron that files these issues, the nightly scans, and every deploy + target. Nothing else reports this outage, because the component + that would have reported it is the one that is down. Alertmanager + itself is unaffected, which is why this alert exists. + - alert: OpenBaoUnavailable + # Probes /v1/sys/health?standbyok=true — 200 on any unsealed node, + # 503 sealed, 501 uninitialized. The UI stays up when sealed, so a + # root probe would not catch this. + # + # Matched by prefix rather than the full string: the target carries a + # query parameter, and an exact match would break silently if the + # instance label ever encodes it differently. + expr: probe_success{job="blackbox-http", instance=~"https://openbao\\.renner\\.uno/v1/sys/health.*"} == 0 + for: 5m + labels: + severity: critical + annotations: + summary: "OpenBao is sealed, uninitialized or unreachable" + description: >- + Every ExternalSecret in the cluster resolves through OpenBao. + Existing Secrets keep working, so nothing breaks at once — the + damage shows up at the next refresh interval or the next pod that + needs a fresh credential. Check whether it is sealed before + assuming a network fault.