From 87083c46e716052e91bb080e2d820d59faea23c6 Mon Sep 17 00:00:00 2001 From: Alert Fix Agent Date: Wed, 23 Sep 2026 12:42:58 +0000 Subject: [PATCH] fix: [alert] OpenBaoQuorumLost (openbao) (#920) --- .../apps/monitoring-rules/prometheusrule-openbao.yaml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/kubernetes/apps/monitoring-rules/prometheusrule-openbao.yaml b/kubernetes/apps/monitoring-rules/prometheusrule-openbao.yaml index e61bbb58..6804a3d2 100644 --- a/kubernetes/apps/monitoring-rules/prometheusrule-openbao.yaml +++ b/kubernetes/apps/monitoring-rules/prometheusrule-openbao.yaml @@ -57,10 +57,14 @@ spec: # the ClusterSecretStore goes InvalidProviderConfig and no # ExternalSecret can refresh. # - # No window: below quorum the secret store is down and every - # ExternalSecret stalls with it, so waiting only delays the page. + # Short window: readiness is only a proxy for quorum. A one-second + # node NotReady flap makes the node lifecycle controller mark that + # node's pods not-ready for ~30s while raft keeps its leader and + # serves (issue #920, during a rolling reboot with one replica + # already evicted). A really sealed or crashed majority persists, + # and existing Secrets keep their last values, so 2m costs little. expr: max by (namespace, statefulset) (kube_statefulset_status_replicas_ready{namespace="openbao", statefulset="openbao"}) < 2 - for: 0m + for: 2m labels: severity: critical annotations: -- 2.54.0