fix(secret-rotator): keep the replaced HA token valid until the next rotation

ha-token mode revoked the token it had just replaced right after storing the
new one in OpenBao. Consumers never read OpenBao: ESO copies it into the k8s
Secret on a 1h refreshInterval, and env-var consumers only see it after a
Reloader restart. On 2026-10-01 that left every consumer on a revoked token
for ~35 min: up{job="home-assistant"} was 0 from 02:01 to 02:35 UTC (401 on
the scrape -> HomeAssistantDown, TargetDown), and zone-configurator
crashlooped on "Auth invalid".

The rotator now reaps instead of revoking: after the store it lists HA's
long-lived tokens and revokes only those named "<HA_CLIENT_NAME> [" (the
rotator's own unique-name form) that are neither the stored-before token nor
the new one. The replaced token stays valid for one quarter and the next run
removes it. Same keep-the-previous pattern as gitea-token mode (59e804d9).
reap_targets is pure and unit-tested.

The HA rotation CronJobs were pinned to secret-rotator:2026-07-30 with no
automation, so a code fix would never deploy. Enrol the image in Flux image
automation (ImageRepository + ImagePolicy, newest YYYY-MM-DD tag) and mark the
three HA CronJobs only: staged, the other rotators (one runs daily) follow
once an off-cycle HA run has validated the new binary.

zone-configurator and speech-to-phrase read home-assistant-prometheus-token
as an env var with no Reloader annotation, so they would still hold the
replaced token when the next run reaps it. Add the annotation.

Closes #1084
Closes #1081

kubernetes/apps/speech-to-phrase/deployment.yaml

@@ -4,6 +4,8 @@ metadata:
   name: speech-to-phrase
   labels:
     app.kubernetes.io/name: speech-to-phrase
+  annotations:
+    secret.reloader.stakater.com/reload: home-assistant-prometheus-token
 spec:
   replicas: 1
   strategy:

kubernetes/apps/zone-configurator/deployment.yaml

@@ -4,6 +4,8 @@ metadata:
   name: zone-configurator
   labels:
     app.kubernetes.io/name: zone-configurator
+  annotations:
+    secret.reloader.stakater.com/reload: home-assistant-prometheus-token
 spec:
   replicas: 1
   strategy:

kubernetes/infrastructure/image-automation/README.md

@@ -45,6 +45,7 @@ existing `flux-system` GitRepository whose Secret was created by
 | wyoming-{whisper,openwakeword,speech-to-phrase} | `3.x` / `2.x` / `1.x` | |
 | authentik-gate-sync | `YYYY-MM-DD-HHMM` | In-house (build-ci-images.yml); newest tag wins |
 | kuma-sync | `YYYY-MM-DD-HHMM` | In-house (build-ci-images.yml); newest tag wins |
+| secret-rotator | `YYYY-MM-DD` | In-house (build-ci-images.yml); newest tag wins; only the Home Assistant rotation CronJobs carry the marker so far |
 | gatus | `5.x` | v-prefix filter |
 | velero-plugin-for-aws | `1.13.x` | Tight: must match velero server major.minor |
 | gotenberg | `8.x` | Plain-variant filter (rejects `-libreoffice`/`-chromium`/`-cloudrun`/`-aws-lambda`) |

kubernetes/infrastructure/image-automation/kustomization.yaml

@@ -25,6 +25,7 @@ resources:
   - mosquitto.yaml
   - redis.yaml
   - review-bot.yaml
+  - secret-rotator.yaml
   - shelly-manager.yaml
   - terminus.yaml
   - tika.yaml

kubernetes/infrastructure/image-automation/secret-rotator.yaml

@@ -0,0 +1,23 @@
+apiVersion: image.toolkit.fluxcd.io/v1
+kind: ImageRepository
+metadata:
+  name: secret-rotator
+  namespace: flux-system
+spec:
+  image: gitea.renner.uno/gitea_admin/secret-rotator
+  interval: 30m
+---
+# build-ci-images.yml tags each build YYYY-MM-DD; newest wins.
+apiVersion: image.toolkit.fluxcd.io/v1
+kind: ImagePolicy
+metadata:
+  name: secret-rotator
+  namespace: flux-system
+spec:
+  imageRepositoryRef:
+    name: secret-rotator
+  filterTags:
+    pattern: '^\d{4}-\d{2}-\d{2}(-\d{4})?$'
+  policy:
+    alphabetical:
+      order: asc

kubernetes/infrastructure/openbao/rotation/home-assistant.yaml

@@ -10,9 +10,13 @@
 # these run the secret-rotator binary (ha-token mode). It mints under a UNIQUE
 # name (base + random tag): HA refuses a second token with an existing
 # client_name, and make-before-break runs while the outgoing token is still
-# present. The outgoing token is identified by its JWT `iss`, never by name, so
-# the tag is free. Make-before-break: read -> mint -> store -> revoke; a failed
-# store leaves the old token valid and stored (a no-op, not an outage).
+# present. The token being replaced is identified by its JWT `iss`; older
+# generations by the rotator-only "<name> [" prefix, which no foreign token
+# ("Prometheus", "Config Reconciler", ...) carries. Make-before-break:
+# read -> mint -> store -> reap; a failed
+# store leaves the old token valid and stored (a no-op, not an outage). The
+# reap keeps the token just replaced (consumers still hold it until ESO and
+# Reloader propagate the new one) and revokes only older "<name> [" tokens.
 #
 # Reachability: HA :8123 (HA itself, not the :8124 XFF-anchor proxy) via the
 # home-assistant-bgp Service ClusterIP. Needs BOTH netpol halves —
@@ -68,7 +72,7 @@ spec:
               type: RuntimeDefault
           containers:
             - name: rotate
-              image: gitea.renner.uno/gitea_admin/secret-rotator:2026-07-30
+              image: gitea.renner.uno/gitea_admin/secret-rotator:2026-07-30 # {"$imagepolicy": "flux-system:secret-rotator"}
               args: ["ha-token"]
               securityContext:
                 allowPrivilegeEscalation: false
@@ -130,7 +134,7 @@ spec:
               type: RuntimeDefault
           containers:
             - name: rotate
-              image: gitea.renner.uno/gitea_admin/secret-rotator:2026-07-30
+              image: gitea.renner.uno/gitea_admin/secret-rotator:2026-07-30 # {"$imagepolicy": "flux-system:secret-rotator"}
               args: ["ha-token"]
               securityContext:
                 allowPrivilegeEscalation: false
@@ -192,7 +196,7 @@ spec:
               type: RuntimeDefault
           containers:
             - name: rotate
-              image: gitea.renner.uno/gitea_admin/secret-rotator:2026-07-30
+              image: gitea.renner.uno/gitea_admin/secret-rotator:2026-07-30 # {"$imagepolicy": "flux-system:secret-rotator"}
               args: ["ha-token"]
               securityContext:
                 allowPrivilegeEscalation: false

kubernetes/infrastructure/openbao/rotation/kustomization.yaml

@@ -14,6 +14,7 @@ kind: Kustomization
 #
 # Keep every rotator on the same tag: it is one binary containing all modes, so
 # splitting only makes the next bump partial.
+# Exception while staged: home-assistant.yaml follows ImagePolicy flux-system:secret-rotator.
 #
 # ADDING A ROTATOR: copy an existing file, and check you did not inherit a
 # digest pin from an older example — nothing enforces this automatically, so it

secret-rotator/src/ha.rs

@@ -133,16 +133,28 @@ impl Ha {
         }
     }
 
-    /// Confirm a refresh-token id exists and is a long-lived access token,
-    /// returning its client_name for the log line.
-    pub fn describe(&mut self, id: &str) -> Result<Option<String>, Box<dyn Error>> {
+    /// (id, client_name) of every long-lived access token HA lists.
+    pub fn long_lived_tokens(&mut self) -> Result<Vec<(String, String)>, Box<dyn Error>> {
         let result = self.call(serde_json::json!({ "type": "auth/refresh_tokens" }))?;
         let list = result.as_array().ok_or_else(|| HaError("refresh_tokens: not a list".into()))?;
         Ok(list
             .iter()
-            .find(|t| t["id"].as_str() == Some(id))
             .filter(|t| t["type"].as_str() == Some("long_lived_access_token"))
-            .map(|t| t["client_name"].as_str().unwrap_or("?").to_string()))
+            .filter_map(|t| {
+                let id = t["id"].as_str()?.to_string();
+                Some((id, t["client_name"].as_str().unwrap_or("?").to_string()))
+            })
+            .collect())
+    }
+
+    /// Confirm a refresh-token id exists and is a long-lived access token,
+    /// returning its client_name for the log line.
+    pub fn describe(&mut self, id: &str) -> Result<Option<String>, Box<dyn Error>> {
+        Ok(self
+            .long_lived_tokens()?
+            .into_iter()
+            .find(|(tid, _)| tid == id)
+            .map(|(_, name)| name))
     }
 
     /// Mint a new long-lived access token. Returns the token value.
@@ -167,10 +179,64 @@ impl Ha {
     }
 }
 
+/// The rotator's own tokens that are safe to revoke: long-lived tokens whose
+/// client_name starts with `prefix` (the rotator's `"<HA_CLIENT_NAME> ["`
+/// unique-name form), minus every id in `keep`. Tokens without the prefix were
+/// not minted by this rotator and are never touched — names like "Prometheus"
+/// or "Config Reconciler" belong to something else.
+pub fn reap_targets(tokens: &[(String, String)], prefix: &str, keep: &[&str]) -> Vec<(String, String)> {
+    tokens
+        .iter()
+        .filter(|(id, name)| name.starts_with(prefix) && !keep.contains(&id.as_str()))
+        .cloned()
+        .collect()
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
 
+    fn toks(v: &[(&str, &str)]) -> Vec<(String, String)> {
+        v.iter().map(|(i, n)| (i.to_string(), n.to_string())).collect()
+    }
+
+    #[test]
+    fn reap_keeps_current_and_new() {
+        let t = toks(&[
+            ("old", "ServiceMonitor [77adb8]"),
+            ("cur", "ServiceMonitor [fd7b91]"),
+            ("new", "ServiceMonitor [a1b2c3]"),
+        ]);
+        let r = reap_targets(&t, "ServiceMonitor [", &["cur", "new"]);
+        assert_eq!(r, toks(&[("old", "ServiceMonitor [77adb8]")]));
+    }
+
+    #[test]
+    fn reap_ignores_foreign_and_untagged_names() {
+        let t = toks(&[
+            ("p", "Prometheus"),
+            ("u", "ServiceMonitor"),
+            ("x", "ServiceMonitorX [123456]"),
+            ("o", "Config Sync [3fa243]"),
+        ]);
+        assert!(reap_targets(&t, "ServiceMonitor [", &[]).is_empty());
+    }
+
+    #[test]
+    fn reap_takes_every_stale_generation() {
+        let t = toks(&[
+            ("a", "Shelly Manager [000001]"),
+            ("b", "Shelly Manager [000002]"),
+            ("cur", "Shelly Manager [000003]"),
+            ("new", "Shelly Manager [000004]"),
+        ]);
+        let ids: Vec<String> = reap_targets(&t, "Shelly Manager [", &["cur", "new"])
+            .into_iter()
+            .map(|(i, _)| i)
+            .collect();
+        assert_eq!(ids, vec!["a", "b"]);
+    }
+
     /// jwt with payload {"iss":"abc123","iat":1}
     fn jwt(payload_b64: &str) -> String {
         format!("header.{payload_b64}.signature")

secret-rotator/src/main.rs

@@ -409,15 +409,25 @@ fn sync_woodpecker_secret() -> Result<String, Box<dyn Error>> {
 /// Rotate a Home Assistant long-lived access token, make-before-break.
 ///
 /// Ordering is deliberate and must not be reshuffled:
-///   1. list existing token ids for this client_name  (before minting, so the
-///      new token can never end up in the revocation set)
+///   1. read the stored token and identify it by its JWT `iss` (the token
+///      this run replaces; it is excluded from the reap in step 4)
 ///   2. mint the replacement
 ///   3. write it to OpenBao
-///   4. only then revoke the old ones
+///   4. only then reap the generation BEFORE the one just replaced
 ///
 /// If step 3 fails the old token is still valid and still in OpenBao, so the
 /// consumer keeps working — the run is a no-op rather than an outage. Revoking
 /// before writing would strand the consumer with a dead credential.
+///
+/// Step 4 deliberately does NOT revoke the token this run replaced. Consumers
+/// never read OpenBao: they read a k8s Secret that ESO refreshes on its
+/// refreshInterval, and env-var consumers only see it after a Reloader
+/// restart. Revoking the replaced token straight after the store left every
+/// consumer on a dead credential until that propagation finished (2026-10-01:
+/// 35 min of 401s on the HA scrape, HomeAssistantDown, zone-configurator
+/// crashloop; homelab #1081/#1084). So the replaced token stays valid until the
+/// NEXT run — a quarter later, long after propagation — which then reaps it.
+/// Same keep-the-previous pattern as gitea-token mode.
 fn rotate_ha_token() -> Result<String, Box<dyn Error>> {
     let bao_addr = env_or("BAO_ADDR", "http://openbao.openbao.svc.cluster.local:8200");
     let bao_role = env("BAO_ROLE")?;
@@ -463,11 +473,10 @@ fn rotate_ha_token() -> Result<String, Box<dyn Error>> {
     let unique_name = format!("{client_name} [{}]", secretgen::random_hex(3)?);
     let fresh = ha.mint(&unique_name, lifespan)?;
 
-    // If the store fails now, the freshly minted token is valid for its full
-    // lifespan, referenced nowhere, and will never be revoked automatically
-    // (revocation only ever targets the OLD stored token's id). Log the
-    // orphan's refresh id — recoverable from its own JWT — so it can be
-    // revoked by hand. Do NOT log the token value.
+    // If the store fails now, the freshly minted token is valid, referenced
+    // nowhere, and only reaped by the next successful run (it carries the
+    // prefix). Log the orphan's refresh id — recoverable from its own JWT — so
+    // it can be revoked by hand sooner. Do NOT log the token value.
     if let Err(e) = bao.patch_field(&bao_path, &field, &fresh) {
         let orphan = ha::token_refresh_id(&fresh)
             .unwrap_or_else(|_| "<unknown: could not decode new token>".to_string());
@@ -480,29 +489,43 @@ fn rotate_ha_token() -> Result<String, Box<dyn Error>> {
         return Err(e);
     }
 
-    // Revoke over a FRESH connection authenticated with the NEW token, not the
-    // one we've been using. HA binds the WebSocket session to the refresh token
-    // behind its access token, so deleting the OLD token over the OLD token's
-    // own session tears that session down before the delete-ack can be read —
-    // the delete succeeds server-side but surfaces as "connection closed by
-    // HA", a false failure. Reconnecting with the new token keeps the session
-    // alive through the revoke. (Verified live: the old flow reported failure
-    // while the old token was in fact already gone.)
+    // Reap over a FRESH connection authenticated with the NEW token. HA binds
+    // the WebSocket session to the refresh token behind its access token, so
+    // deleting a token over its own session tears the session down before the
+    // delete-ack can be read (verified live). The new token is never a reap
+    // target, so its session survives every revoke below.
     drop(ha);
+    let fresh_id = ha::token_refresh_id(&fresh)?;
+    let prefix = format!("{client_name} [");
+    if !old_name.starts_with(&prefix) {
+        // A pre-rotator token (e.g. the original "Config Sync") never matches
+        // the prefix, so no later run will reap it.
+        eprintln!(
+            "warning: replaced token '{old_name}' ({old_id}) does not carry the '{prefix}' prefix; \
+             no later run will revoke it — revoke it by hand once consumers use '{unique_name}'"
+        );
+    }
     let mut ha2 = ha::Ha::connect(&ha_url, &fresh)?;
-    match ha2.revoke(&old_id) {
-        Ok(()) => Ok(format!(
-            "rotated {bao_path}#{field}: minted '{unique_name}', revoked old '{old_name}' ({old_id}) — value not logged"
-        )),
-        Err(e) => {
-            // Reaching here means the new token is live and stored but the old
-            // one may genuinely still be valid. Now that the session is NOT the
-            // one being deleted, this is a real finding rather than the
-            // self-inflicted false negative above — surface it loudly.
-            eprintln!("warning: new token is live and stored, but revoking the OLD token '{old_name}' ({old_id}) failed and it MAY REMAIN VALID: {e}");
-            Ok(format!(
-                "rotated {bao_path}#{field}: minted '{unique_name}'; old token '{old_name}' ({old_id}) NOT confirmed revoked (see warning)"
-            ))
+    let targets = ha::reap_targets(&ha2.long_lived_tokens()?, &prefix, &[&old_id, &fresh_id]);
+    let mut reaped = Vec::new();
+    let mut failed = Vec::new();
+    for (id, name) in &targets {
+        match ha2.revoke(id) {
+            Ok(()) => reaped.push(format!("'{name}' ({id})")),
+            Err(e) => {
+                eprintln!("warning: revoking stale token '{name}' ({id}) failed and it MAY REMAIN VALID: {e}");
+                failed.push(format!("'{name}' ({id})"));
+            }
         }
     }
+    let reaped = if reaped.is_empty() { "none".to_string() } else { reaped.join(", ") };
+    let failed = if failed.is_empty() {
+        String::new()
+    } else {
+        format!("; NOT confirmed revoked: {} (see warnings)", failed.join(", "))
+    };
+    Ok(format!(
+        "rotated {bao_path}#{field}: minted '{unique_name}', kept replaced '{old_name}' ({old_id}) valid until the next run, \
+         reaped older: {reaped}{failed} — value not logged"
+    ))
 }