fix: [alert] OpenBaoReplicaNotReady (openbao) (#808)

Reviewed-on: k.renner/homelab#809

kubernetes/apps/monitoring-rules/prometheusrule-openbao.yaml

@@ -27,11 +27,25 @@ spec:
         # not-Ready. The converse does not hold — a crash or a stuck
         # scheduling would also fire this — but both want a human too, so the
         # imprecision costs nothing.
+        #
+        # WHY THE TWO THRESHOLD ALERTS AGGREGATE AWAY THE SCRAPE TARGET
+        #
+        # kube-state-metrics stamps instance/pod/endpoint onto every series it
+        # exports, so a bare selector yields a *different* series each time the
+        # kube-state-metrics pod is replaced — and Prometheus tracks the `for:`
+        # timer per output series, so a replacement resets a pending alert to
+        # inactive. That is not hypothetical: during the 2026-09-07 quorum
+        # outage kube-state-metrics itself moved twice (the openbao replicas
+        # were on the same three nodes that were being rebooted), and the two
+        # alerts below fired ~14 min and ~6.5 min later than their `for:`
+        # windows say they should have. Aggregating to the identity that
+        # actually matters — the StatefulSet — makes the series survive a
+        # kube-state-metrics restart and the timer run continuously.
         - alert: OpenBaoReplicaNotReady
           # A replica that is sealed never recovers on its own, so the
           # duration is only there to ignore ordinary rescheduling: a healthy
           # pod moving between nodes is Ready again in well under a minute.
-          expr: kube_statefulset_status_replicas_ready{namespace="openbao", statefulset="openbao"} < 3
+          expr: max by (namespace, statefulset) (kube_statefulset_status_replicas_ready{namespace="openbao", statefulset="openbao"}) < 3
           for: 10m
           labels:
             severity: warning
@@ -56,7 +70,7 @@ spec:
           # Shorter window than the warning above because this is an outage,
           # not a degradation — but not zero, so a rolling restart that
           # briefly dips does not page.
-          expr: kube_statefulset_status_replicas_ready{namespace="openbao", statefulset="openbao"} < 2
+          expr: max by (namespace, statefulset) (kube_statefulset_status_replicas_ready{namespace="openbao", statefulset="openbao"}) < 2
           for: 5m
           labels:
             severity: critical

kubernetes/infrastructure/crowdsec/lapi-ingress.yaml

@@ -31,22 +31,12 @@
 # serves read-only decision streams to a valid bouncer key and nothing else.
 # The admin surface is cscli, in-cluster. If the source address ever becomes
 # deterministic, an ipAllowList can be added on top.
----
+#
 # The LAPI serves HTTPS only once tls.enabled is set — there is no plaintext
 # fallback and no second port; 8080 is the TLS port. Its certificate comes
 # from the chart's own self-signed CA, so Traefik needs that CA to verify the
-# backend. SANs include crowdsec-service.crowdsec.svc.cluster.local, which is
-# what the Service resolves to.
-apiVersion: traefik.io/v1alpha1
-kind: ServersTransport
-metadata:
-  name: crowdsec-lapi
-  namespace: crowdsec
-spec:
-  serverName: crowdsec-service.crowdsec.svc.cluster.local
-  rootCAs:
-    - secret: crowdsec-root-secret
----
+# backend; the ServersTransport carrying it is a traefik.io CRD consumer and
+# lives in post-crds/crowdsec-lapi-serverstransport.yaml.
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:

kubernetes/infrastructure/post-crds/kustomization.yaml

@@ -31,7 +31,5 @@ resources:
   - traefik-crowdsec-middleware.yaml
   - traefik-rate-limit-middleware.yaml
   - traefik-security-headers-middleware.yaml
-  # Phase 1 of a two-Kustomization move: also still in crowdsec/lapi-ingress.yaml
-  # until this layer has applied it once, so prune never runs ahead of apply.
   - crowdsec-lapi-serverstransport.yaml
   - ../network-policies/