fix: [alert] OpenBaoReplicaNotReady (openbao) (#808 )

2026-09-07 07:48:24 +00:00
1 changed files with 16 additions and 2 deletions
@@ -27,11 +27,25 @@ spec:
        # not-Ready. The converse does not hold — a crash or a stuck
        # scheduling would also fire this — but both want a human too, so the
        # imprecision costs nothing.
+        #
+        # WHY THE TWO THRESHOLD ALERTS AGGREGATE AWAY THE SCRAPE TARGET
+        #
+        # kube-state-metrics stamps instance/pod/endpoint onto every series it
+        # exports, so a bare selector yields a *different* series each time the
+        # kube-state-metrics pod is replaced — and Prometheus tracks the `for:`
+        # timer per output series, so a replacement resets a pending alert to
+        # inactive. That is not hypothetical: during the 2026-09-07 quorum
+        # outage kube-state-metrics itself moved twice (the openbao replicas
+        # were on the same three nodes that were being rebooted), and the two
+        # alerts below fired ~14 min and ~6.5 min later than their `for:`
+        # windows say they should have. Aggregating to the identity that
+        # actually matters — the StatefulSet — makes the series survive a
+        # kube-state-metrics restart and the timer run continuously.
        - alert: OpenBaoReplicaNotReady
          # A replica that is sealed never recovers on its own, so the
          # duration is only there to ignore ordinary rescheduling: a healthy
          # pod moving between nodes is Ready again in well under a minute.
-          expr: kube_statefulset_status_replicas_ready{namespace="openbao", statefulset="openbao"} < 3
+          expr: max by (namespace, statefulset) (kube_statefulset_status_replicas_ready{namespace="openbao", statefulset="openbao"}) < 3
          for: 10m
          labels:
            severity: warning
@@ -56,7 +70,7 @@ spec:
          # Shorter window than the warning above because this is an outage,
          # not a degradation — but not zero, so a rolling restart that
          # briefly dips does not page.
-          expr: kube_statefulset_status_replicas_ready{namespace="openbao", statefulset="openbao"} < 2
+          expr: max by (namespace, statefulset) (kube_statefulset_status_replicas_ready{namespace="openbao", statefulset="openbao"}) < 2
          for: 5m
          labels:
            severity: critical