From 91133c94b044cc2d2827b9a3db901da15f2927e5 Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Thu, 30 Jul 2026 16:13:09 +0200 Subject: [PATCH 1/5] feat(mosquitto): enrol the health broker credential in Class-A rotation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 1 of PROPOSAL-mosquitto-credential-rotation.md. One of eight broker users; the other seven are untouched and keep their bootstrap Jobs. `health` goes first deliberately: it is the broker's OWN liveness/readiness probe credential, consumed by exec probes inside the mosquitto container via the same Secret that seeds the passwd file. Nothing outside the pod holds it, so rotating it needs no coordination with any client — the smallest blast radius for proving rotator -> OpenBao -> ESO -> Reloader end to end before touching credentials held by Shelly firmware, the edge host or the HA config entry. Adds, following rotation/paperless.yaml exactly: - rotator-mosquitto-health.hcl — per-secret policy, read+patch on that ONE path. Deliberately not a wildcard over mosquitto-*: the users are being enrolled one at a time, and a wildcard would silently grant this rotator write access to the other seven before they are ready. - rotation/mosquitto-health.yaml — SA + quarterly CronJob running secret-rotator in random mode. Scheduled one month off paperless so unrelated rotations do not share a window. - mosquitto-health-credentials-externalsecret.yaml — projects the value back to the SAME Secret name and key, so mosquitto-deployment.yaml's secretKeyRef and the probes reading $HEALTH_PASSWORD are unchanged. - A Reloader annotation on the mosquitto Deployment. Without it rotation is INERT: the startup shim rebuilds /mosquitto/auth/passwd from the Secrets on pod start, so a rotated Secret would sit unused until some unrelated restart while the old password stayed valid. Listed per-Secret rather than `auto: "true"` so enrolling each further user stays a visible edit. Removes only the health bootstrap Job (508 -> 440 lines); the remaining six Jobs are untouched. Deleting rather than editing also sidesteps Job spec.template immutability, which is what made the earlier attempt (#449) unmergeable. Verified before committing: - random mode fits unchanged: env contract is BAO_ROLE/BAO_PATH/BAO_FIELDS/ RANDOM_BYTES, and it calls assert_fields_exist FIRST — so it rotates an existing field and never bootstraps one. Seeding stays a documented one-time out-of-band step, exactly as every other rotator does it. - Charset is safe HERE but not universally: random mode emits standard base64 (+, /, =) whereas the old Jobs emitted alphanumeric via `tr -dc 'A-Za-z0-9'`. The probes pass it as -P "$HEALTH_PASSWORD" inside the container, so it is never parsed by a URL or config file. That must be re-checked per user before enrolling sml2mqtt/open3e (edge-host templates), shellies (device firmware) or homeassistant. - OpenBao path already holds a `password` field, and its value still matches the live Secret (both 23c10b3b), so ESO adoption on merge is a no-op and no client sees an auth change. - eso-read.hcl already grants homelab/data/*, so no ESO policy change. - Policy and kubernetes auth role applied out-of-band (they are not Kubernetes resources): bound to SA mosquitto-health-secret-rotator in ns openbao, ttl 15m. Not done here: the first actual rotation. It is safe (in-cluster only) but restarts the broker, so it is left as a deliberate post-merge acceptance test rather than bundled into a merge. Co-Authored-By: Claude Opus 5 (1M context) --- .../apps/home-assistant/kustomization.yaml | 3 + .../home-assistant/mosquitto-bootstrap.yaml | 68 ---------- .../home-assistant/mosquitto-deployment.yaml | 12 ++ ...tto-health-credentials-externalsecret.yaml | 35 ++++++ .../policies/rotator-mosquitto-health.hcl | 39 ++++++ .../openbao/rotation/kustomization.yaml | 1 + .../openbao/rotation/mosquitto-health.yaml | 118 ++++++++++++++++++ 7 files changed, 208 insertions(+), 68 deletions(-) create mode 100644 kubernetes/apps/home-assistant/mosquitto-health-credentials-externalsecret.yaml create mode 100644 kubernetes/infrastructure/openbao/policies/rotator-mosquitto-health.hcl create mode 100644 kubernetes/infrastructure/openbao/rotation/mosquitto-health.yaml diff --git a/kubernetes/apps/home-assistant/kustomization.yaml b/kubernetes/apps/home-assistant/kustomization.yaml index e774a2dc..bad7ab66 100644 --- a/kubernetes/apps/home-assistant/kustomization.yaml +++ b/kubernetes/apps/home-assistant/kustomization.yaml @@ -13,6 +13,9 @@ resources: - prometheus-token-externalsecret.yaml - config-sync-token-externalsecret.yaml - mosquitto-bootstrap.yaml + # health broker credential: OpenBao-backed + Class-A rotated (first of the + # eight users to migrate off mosquitto-bootstrap.yaml). + - mosquitto-health-credentials-externalsecret.yaml - mosquitto-config.yaml - mosquitto-certificate.yaml - mosquitto-service.yaml diff --git a/kubernetes/apps/home-assistant/mosquitto-bootstrap.yaml b/kubernetes/apps/home-assistant/mosquitto-bootstrap.yaml index e5f60cd6..44f34cf0 100644 --- a/kubernetes/apps/home-assistant/mosquitto-bootstrap.yaml +++ b/kubernetes/apps/home-assistant/mosquitto-bootstrap.yaml @@ -370,74 +370,6 @@ spec: limits: memory: 64Mi --- -# Broker user `health`: a dedicated, least-privilege credential used ONLY by -# mosquitto's own liveness/readiness probes (mosquitto-deployment.yaml). Its -# ACL is `write health/#` and nothing else — it cannot read any topic or -# write outside the throwaway probe subtree, so a leak is worthless (nothing -# subscribes to health/#). Kept on its own Secret, like every other user, so -# the probe never has to borrow the admin credential. -apiVersion: batch/v1 -kind: Job -metadata: - name: mosquitto-health-credentials-bootstrap - namespace: home-assistant -spec: - backoffLimit: 4 - template: - metadata: - labels: - app.kubernetes.io/name: mosquitto-bootstrap - spec: - serviceAccountName: mosquitto-bootstrap - restartPolicy: OnFailure - containers: - - name: bootstrap - image: alpine:3.21 - command: - - sh - - -c - - | - set -eu - apk add --no-cache curl jq >/dev/null - - NS=home-assistant - NAME=mosquitto-health-credentials - TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token) - CA=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt - API=https://kubernetes.default.svc - - if curl -fsSL --cacert "$CA" -H "Authorization: Bearer $TOKEN" \ - "$API/api/v1/namespaces/$NS/secrets/$NAME" >/dev/null 2>&1; then - echo "$NAME already exists; nothing to do." - exit 0 - fi - - PW=$(tr -dc 'A-Za-z0-9' /dev/null - - echo "generated $NAME" - resources: - requests: - cpu: 10m - memory: 32Mi - limits: - memory: 64Mi ---- # Broker user `zigbee2mqtt` — the Zigbee2MQTT bridge (ZHA replacement). Scoped # to its own `zigbee2mqtt/#` subtree plus `write homeassistant/#` (so it can # publish HA MQTT-discovery configs, exactly like the `shellies` user) and diff --git a/kubernetes/apps/home-assistant/mosquitto-deployment.yaml b/kubernetes/apps/home-assistant/mosquitto-deployment.yaml index c52e8119..0c28523a 100644 --- a/kubernetes/apps/home-assistant/mosquitto-deployment.yaml +++ b/kubernetes/apps/home-assistant/mosquitto-deployment.yaml @@ -5,6 +5,18 @@ metadata: namespace: home-assistant labels: app.kubernetes.io/name: mosquitto + annotations: + # REQUIRED for credential rotation to take effect at all. The startup shim + # rebuilds /mosquitto/auth/passwd from these Secrets on every pod start, so + # a rotated Secret is inert until the pod restarts — without this the new + # password would sit unused until some unrelated restart, while the old one + # stayed valid indefinitely. + # + # Listed per-Secret rather than using the blanket `reloader.stakater.com/ + # auto: "true"`, so that enrolling each remaining broker user in rotation is + # a visible, reviewable edit to this line rather than something that starts + # happening implicitly. + secret.reloader.stakater.com/reload: mosquitto-health-credentials spec: replicas: 1 strategy: diff --git a/kubernetes/apps/home-assistant/mosquitto-health-credentials-externalsecret.yaml b/kubernetes/apps/home-assistant/mosquitto-health-credentials-externalsecret.yaml new file mode 100644 index 00000000..df511112 --- /dev/null +++ b/kubernetes/apps/home-assistant/mosquitto-health-credentials-externalsecret.yaml @@ -0,0 +1,35 @@ +# Projects the mosquitto `health` broker credential from OpenBao into the Secret +# the broker already consumes. FIRST of the eight broker users to move off the +# bootstrap Jobs and onto Class-A rotation (see +# PROPOSAL-mosquitto-credential-rotation.md); the rotator lives at +# kubernetes/infrastructure/openbao/rotation/mosquitto-health.yaml. +# +# Secret name and key are UNCHANGED (`password`), so mosquitto-deployment.yaml's +# secretKeyRef and the exec probes that read $HEALTH_PASSWORD need no edit. +# +# refreshInterval is 10m rather than the 1h used by mosquitto-evcc-credentials: +# a rotation only lands once ESO re-reads, so 1h would mean up to an hour between +# the rotator running and the broker actually using the new password, with the +# old one still valid in the meantime. 10m keeps that window short without +# polling OpenBao pointlessly hard. +# +# ⚠️ OWNERSHIP: creationPolicy is implicitly Owner, so ESO sets an ownerReference +# on the Secret. Deleting this ExternalSecret therefore DELETES the live Secret +# and with it the broker's probe credential — the bootstrap Job it replaces had +# no such coupling. To detach without destroying, set creationPolicy: Orphan +# before removing it. +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: mosquitto-health-credentials + namespace: home-assistant +spec: + refreshInterval: 10m + secretStoreRef: + kind: ClusterSecretStore + name: openbao + target: + name: mosquitto-health-credentials + dataFrom: + - extract: + key: apps/home-assistant/mosquitto-health-credentials diff --git a/kubernetes/infrastructure/openbao/policies/rotator-mosquitto-health.hcl b/kubernetes/infrastructure/openbao/policies/rotator-mosquitto-health.hcl new file mode 100644 index 00000000..e918de4e --- /dev/null +++ b/kubernetes/infrastructure/openbao/policies/rotator-mosquitto-health.hcl @@ -0,0 +1,39 @@ +# OpenBao policy: rotator-mosquitto-health +# +# Least-privilege policy for the mosquitto `health` broker-credential rotation +# CronJob (kubernetes auth role `rotator-mosquitto-health`). PER-SECRET +# isolation, same template as rotator-paperless: one policy + one role per +# rotated secret, so a compromised rotation Job cannot touch any other secret. +# +# Scoped to the `health` credential ALONE, deliberately not to a wildcard over +# apps/home-assistant/mosquitto-*. The broker has eight users and they are being +# enrolled one at a time (see kubernetes/apps/home-assistant/ +# PROPOSAL-mosquitto-credential-rotation.md); a wildcard would silently grant +# the first rotator write access to the other seven before they are ready, which +# is exactly the blast radius this per-secret pattern exists to avoid. +# +# NOT applied by Flux — OpenBao policies/roles are not Kubernetes resources +# (same convention as eso-read.hcl and every other rotator policy). Apply ONCE +# with an admin token: +# +# bao policy write rotator-mosquitto-health \ +# kubernetes/infrastructure/openbao/policies/rotator-mosquitto-health.hcl +# +# bao write auth/kubernetes/role/rotator-mosquitto-health \ +# bound_service_account_names=mosquitto-health-secret-rotator \ +# bound_service_account_namespaces=openbao \ +# policies=rotator-mosquitto-health ttl=15m +# +# BOOTSTRAP: `random` mode calls assert_fields_exist BEFORE patching, so it +# ROTATES an existing field and never creates one. The path must already hold a +# `password` field or the Job fails safe without writing. Seed it once: +# +# bao kv put homelab/apps/home-assistant/mosquitto-health-credentials \ +# password="$(kubectl -n home-assistant get secret mosquitto-health-credentials \ +# -o jsonpath='{.data.password}' | base64 -d)" +# +# (Reads are already covered by eso-read.hcl's `homelab/data/*`, so the +# ExternalSecret needs no policy change.) +path "homelab/data/apps/home-assistant/mosquitto-health-credentials" { + capabilities = ["read", "patch"] +} diff --git a/kubernetes/infrastructure/openbao/rotation/kustomization.yaml b/kubernetes/infrastructure/openbao/rotation/kustomization.yaml index f2e61a94..a81b73b7 100644 --- a/kubernetes/infrastructure/openbao/rotation/kustomization.yaml +++ b/kubernetes/infrastructure/openbao/rotation/kustomization.yaml @@ -18,3 +18,4 @@ resources: - home-assistant.yaml - review-bot.yaml - claude-oauth.yaml + - mosquitto-health.yaml diff --git a/kubernetes/infrastructure/openbao/rotation/mosquitto-health.yaml b/kubernetes/infrastructure/openbao/rotation/mosquitto-health.yaml new file mode 100644 index 00000000..aa467596 --- /dev/null +++ b/kubernetes/infrastructure/openbao/rotation/mosquitto-health.yaml @@ -0,0 +1,118 @@ +--- +# Class-A automated rotation of the mosquitto `health` broker credential — a +# self-generated random secret. +# +# FIRST of the eight mosquitto broker users to be enrolled. The credentials were +# previously minted once by bootstrap Jobs and then never changed at all: no +# schedule, no compromise response, and they existed only in the Secret object, +# outside the OpenBao snapshot. See kubernetes/apps/home-assistant/ +# PROPOSAL-mosquitto-credential-rotation.md for the full plan and phasing. +# +# `health` deliberately goes first: it is the broker's OWN liveness/readiness +# probe credential, consumed by exec probes inside the mosquitto container via +# the same Secret that seeds the passwd file. Nothing outside the pod holds it, +# so a rotation needs no coordination with any client — the smallest possible +# blast radius for proving rotator -> OpenBao -> ESO -> Reloader end to end. +# +# Runs the secret-rotator binary (random mode): generate 32 random bytes, +# standard-base64 encode, and merge-patch into OpenBao. assert_fields_exist runs +# FIRST, so a misconfigured path fails safe (no rotation) rather than adding a +# stray field — which also means this ROTATES an existing value and never +# bootstraps one. Seeding is a one-time out-of-band step, documented in +# policies/rotator-mosquitto-health.hcl. +# +# Base64 output is safe for this credential specifically: the probes pass it as +# `-P "$HEALTH_PASSWORD"` to mosquitto_pub/sub inside the container, so the `+`, +# `/` and `=` characters are never parsed by a URL or a config file. That is NOT +# automatically true of the other seven users (sml2mqtt and open3e templates on +# the edge host, Shelly device firmware, the HA config entry), so the charset +# must be re-checked per user before enrolling each one — the old bootstrap Jobs +# generated alphanumeric-only passwords with `tr -dc 'A-Za-z0-9'`. +# +# openbao-ns pattern: the Job runs in the openbao namespace, reaching the +# OpenBao API intra-namespace — no NetworkPolicy needed. It writes OpenBao only; +# ESO syncs mosquitto-health-credentials (ns home-assistant) and Reloader +# restarts the broker so the passwd file is rebuilt from the new value. +# +# Rotation is not instant: it completes only after the ExternalSecret's +# refreshInterval elapses and the broker rolls. During that roll two broker pods +# with different passwd files sit behind the LB (maxUnavailable: 0/maxSurge: 1), +# so a probe against the outgoing pod can fail auth briefly. Self-healing. +# +# ON-DEMAND: kubectl -n openbao create job --from=cronjob/mosquitto-health-secret-rotate mosquitto-health-secret-rotate-now +apiVersion: v1 +kind: ServiceAccount +metadata: + name: mosquitto-health-secret-rotator + namespace: openbao + labels: + app.kubernetes.io/name: mosquitto + app.kubernetes.io/component: secret-rotator +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: mosquitto-health-secret-rotate + namespace: openbao + labels: + app.kubernetes.io/name: mosquitto + app.kubernetes.io/component: secret-rotator +spec: + # Quarterly, offset one month from paperless (which runs Jan/Apr/Jul/Oct) so + # unrelated rotations do not land in the same window and complicate diagnosis. + schedule: "0 4 1 2,5,8,11 *" + timeZone: Europe/Berlin + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 2 + failedJobsHistoryLimit: 5 + startingDeadlineSeconds: 3600 + jobTemplate: + spec: + backoffLimit: 2 + activeDeadlineSeconds: 300 + ttlSecondsAfterFinished: 259200 # 3 d + template: + metadata: + labels: + app.kubernetes.io/name: mosquitto + app.kubernetes.io/component: secret-rotator + spec: + serviceAccountName: mosquitto-health-secret-rotator + restartPolicy: Never + securityContext: + runAsNonRoot: true + runAsUser: 65534 + runAsGroup: 65534 + seccompProfile: + type: RuntimeDefault + containers: + - name: rotate + image: gitea.renner.uno/gitea_admin/secret-rotator@sha256:159e5888ff4e23f5fe7fb1284711d81591635f840888e61402af311b0a76ba92 + args: ["random"] + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + env: + - name: BAO_ADDR + value: http://openbao.openbao.svc.cluster.local:8200 + - name: BAO_ROLE + value: rotator-mosquitto-health + - name: BAO_PATH + value: homelab/apps/home-assistant/mosquitto-health-credentials + # Matches the key the ExternalSecret projects and the key the + # Secret already carries, so mosquitto-deployment.yaml's + # secretKeyRef is unchanged. + - name: BAO_FIELDS + value: "password" + # 32 bytes -> 44 base64 chars. The binary refuses anything below + # 16 bytes outright. + - name: RANDOM_BYTES + value: "32" + resources: + requests: + cpu: 25m + memory: 16Mi + limits: + memory: 48Mi -- 2.54.0 From 2d24d0b6d1d033b9887d37f1765cb9d7f082f49d Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Thu, 30 Jul 2026 16:19:02 +0200 Subject: [PATCH 2/5] docs(mosquitto): fix dangling proposal reference and stale Job comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review points on #455. The three new files referenced PROPOSAL-mosquitto-credential-rotation.md as though it were in the tree. It is not — it lives in PR #453, which is still open, so anyone following the reference (including whoever enrols broker user #2) would hit a 404. That is a sequencing mistake on my part: I wrote the implementation against a doc that had not landed. The references now say so explicitly rather than silently pointing at nothing. Also fixed two comments in mosquitto-deployment.yaml that described HEALTH_PASSWORD as coming from the mosquitto-health-credentials-bootstrap Job. This PR deletes that Job, so those comments described infrastructure that no longer exists. They now point at the ExternalSecret. Fixing here rather than deferring, since this PR is what made them stale. Deliberately did NOT touch the neighbouring zigbee2mqtt comment the review also flagged: that Job still exists (only health is enrolled in this phase), so the reference is still accurate. It becomes stale when zigbee2mqtt is enrolled, and should be fixed by that change. Co-Authored-By: Claude Opus 5 (1M context) --- .../apps/home-assistant/mosquitto-deployment.yaml | 6 ++++-- .../mosquitto-health-credentials-externalsecret.yaml | 3 ++- .../openbao/policies/rotator-mosquitto-health.hcl | 11 +++++++---- .../openbao/rotation/mosquitto-health.yaml | 3 ++- 4 files changed, 15 insertions(+), 8 deletions(-) diff --git a/kubernetes/apps/home-assistant/mosquitto-deployment.yaml b/kubernetes/apps/home-assistant/mosquitto-deployment.yaml index 0c28523a..6e10a88c 100644 --- a/kubernetes/apps/home-assistant/mosquitto-deployment.yaml +++ b/kubernetes/apps/home-assistant/mosquitto-deployment.yaml @@ -189,7 +189,9 @@ spec: secretKeyRef: name: mosquitto-evcc-credentials key: password - # Probe-only credential (mosquitto-health-credentials-bootstrap). + # Probe-only credential. Now OpenBao-backed and Class-A rotated (see + # mosquitto-health-credentials-externalsecret.yaml); its bootstrap Job + # is gone. # Present here for the same reason as the others — the startup # shim hashes it into the passwd file — and reused by the exec # probes below, which run inside this container. @@ -242,7 +244,7 @@ spec: # # Uses the dedicated least-privilege `health` user (ACL: readwrite # health/# only — see the passwd/acl build above and the - # mosquitto-health-credentials-bootstrap Job). Its password is in + # mosquitto-health-credentials ExternalSecret). Its password is in # this container's env (HEALTH_PASSWORD) because the startup shim # hashes every user into the passwd file; the exec probes run # inside this container and read it directly. A leak of this diff --git a/kubernetes/apps/home-assistant/mosquitto-health-credentials-externalsecret.yaml b/kubernetes/apps/home-assistant/mosquitto-health-credentials-externalsecret.yaml index df511112..4422d819 100644 --- a/kubernetes/apps/home-assistant/mosquitto-health-credentials-externalsecret.yaml +++ b/kubernetes/apps/home-assistant/mosquitto-health-credentials-externalsecret.yaml @@ -1,7 +1,8 @@ # Projects the mosquitto `health` broker credential from OpenBao into the Secret # the broker already consumes. FIRST of the eight broker users to move off the # bootstrap Jobs and onto Class-A rotation (see -# PROPOSAL-mosquitto-credential-rotation.md); the rotator lives at +# PROPOSAL-mosquitto-credential-rotation.md (PR #453 — not yet merged; this +# file lands with it)); the rotator lives at # kubernetes/infrastructure/openbao/rotation/mosquitto-health.yaml. # # Secret name and key are UNCHANGED (`password`), so mosquitto-deployment.yaml's diff --git a/kubernetes/infrastructure/openbao/policies/rotator-mosquitto-health.hcl b/kubernetes/infrastructure/openbao/policies/rotator-mosquitto-health.hcl index e918de4e..56d464a9 100644 --- a/kubernetes/infrastructure/openbao/policies/rotator-mosquitto-health.hcl +++ b/kubernetes/infrastructure/openbao/policies/rotator-mosquitto-health.hcl @@ -7,10 +7,13 @@ # # Scoped to the `health` credential ALONE, deliberately not to a wildcard over # apps/home-assistant/mosquitto-*. The broker has eight users and they are being -# enrolled one at a time (see kubernetes/apps/home-assistant/ -# PROPOSAL-mosquitto-credential-rotation.md); a wildcard would silently grant -# the first rotator write access to the other seven before they are ready, which -# is exactly the blast radius this per-secret pattern exists to avoid. +# enrolled one at a time; a wildcard would silently grant the first rotator write +# access to the other seven before they are ready, which is exactly the blast +# radius this per-secret pattern exists to avoid. +# +# Plan and phasing: kubernetes/apps/home-assistant/ +# PROPOSAL-mosquitto-credential-rotation.md — added by PR #453, which is not yet +# merged, so that path does not resolve until it lands. # # NOT applied by Flux — OpenBao policies/roles are not Kubernetes resources # (same convention as eso-read.hcl and every other rotator policy). Apply ONCE diff --git a/kubernetes/infrastructure/openbao/rotation/mosquitto-health.yaml b/kubernetes/infrastructure/openbao/rotation/mosquitto-health.yaml index aa467596..1d883099 100644 --- a/kubernetes/infrastructure/openbao/rotation/mosquitto-health.yaml +++ b/kubernetes/infrastructure/openbao/rotation/mosquitto-health.yaml @@ -6,7 +6,8 @@ # previously minted once by bootstrap Jobs and then never changed at all: no # schedule, no compromise response, and they existed only in the Secret object, # outside the OpenBao snapshot. See kubernetes/apps/home-assistant/ -# PROPOSAL-mosquitto-credential-rotation.md for the full plan and phasing. +# PROPOSAL-mosquitto-credential-rotation.md (PR #453 — not yet merged; this +# file lands with it) for the full plan and phasing. # # `health` deliberately goes first: it is the broker's OWN liveness/readiness # probe credential, consumed by exec probes inside the mosquitto container via -- 2.54.0 From 0817aaa24b72749c1aebacc85362c9e18a35dd46 Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Thu, 30 Jul 2026 16:23:42 +0200 Subject: [PATCH 3/5] docs(mosquitto): add the token_no_default_policy warning to the rotator policy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review point on #455. Five sibling rotator policies carry this line and mine omitted it: Do NOT set token_no_default_policy=true — the `bao kv patch` kv-v2 preflight needs the default policy (see rotator.hcl). It matters precisely because these policies are applied BY HAND, out of band: the warning exists to stop someone tightening the role in a later hardening pass and silently breaking the patch the rotator depends on. Omitting it made this file the one that looks safe to harden. Verified the live role is already correct — token_no_default_policy is false — so this documents an existing good state rather than fixing a broken one. Co-Authored-By: Claude Opus 5 (1M context) --- .../openbao/policies/rotator-mosquitto-health.hcl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/kubernetes/infrastructure/openbao/policies/rotator-mosquitto-health.hcl b/kubernetes/infrastructure/openbao/policies/rotator-mosquitto-health.hcl index 56d464a9..0fcde710 100644 --- a/kubernetes/infrastructure/openbao/policies/rotator-mosquitto-health.hcl +++ b/kubernetes/infrastructure/openbao/policies/rotator-mosquitto-health.hcl @@ -27,6 +27,10 @@ # bound_service_account_namespaces=openbao \ # policies=rotator-mosquitto-health ttl=15m # +# Do NOT set token_no_default_policy=true — the `bao kv patch` kv-v2 preflight +# needs the default policy (see rotator.hcl). The rotator writes via patch, so +# hardening the role that way would break rotation rather than tighten it. +# # BOOTSTRAP: `random` mode calls assert_fields_exist BEFORE patching, so it # ROTATES an existing field and never creates one. The path must already hold a # `password` field or the Job fails safe without writing. Seed it once: -- 2.54.0 From 4df89506fba15ddff24653b41b576676984bf7fc Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Thu, 30 Jul 2026 16:34:59 +0200 Subject: [PATCH 4/5] fix(mosquitto): pin a rotator image that actually exists MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The review asked whether copying paperless's rotator digest was safe. Checking it turned up something worse than a version-skew question: that digest does not exist in the registry at all. sha256:159e5888… ImagePullBackOff / NotFound Verified by pulling it, not by inspecting metadata. So this PR as written would have shipped a CronJob that fails on its first run in three months' time — silently, since a quarterly rotation failing looks like nothing happening. Now pinned to the digest :latest currently resolves to (sha256:dcf4267a…), verified to pull, to list `random` among its modes, and to carry the RANDOM_BYTES>=16 floor this file documents (RANDOM_BYTES=8 exits with "refusing to generate a weak secret" rather than generating one). BROADER PROBLEM, not introduced here and not fixed here: 11 of the 14 existing rotators are pinned to digests that are equally gone. Tested each: gone 159e5888 authentik, crowdsec-bouncer-key, paperless, shelly-manager, woodpecker, home-assistant gone 978fe53c cloudflare, external-dns gone 49bfddd8 netbird-ingress gone 97c22bcf pve-exporter present b41cc364 flux-git present 828bf486 claude-oauth, review-bot The two surviving digests belong to the rotators that run often; the gone ones are all quarterly or infrequent. Untagged manifests appear to be garbage-collected once :latest and the dated tag move past them, so a digest pinned today stops resolving after a couple of rebuilds. Every affected rotator will ImagePullBackOff on its next scheduled run unless the image happens to still be cached on whichever node it lands on. That means secret rotation across most of this cluster is probably already broken and would fail quietly. Raised separately rather than fixed here — it needs a decision (repin all, stop GC'ing, or tag-pin rather than digest-pin), not a drive-by edit inside a mosquitto PR. Co-Authored-By: Claude Opus 5 (1M context) --- .../openbao/rotation/mosquitto-health.yaml | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/kubernetes/infrastructure/openbao/rotation/mosquitto-health.yaml b/kubernetes/infrastructure/openbao/rotation/mosquitto-health.yaml index 1d883099..5002523b 100644 --- a/kubernetes/infrastructure/openbao/rotation/mosquitto-health.yaml +++ b/kubernetes/infrastructure/openbao/rotation/mosquitto-health.yaml @@ -88,7 +88,20 @@ spec: type: RuntimeDefault containers: - name: rotate - image: gitea.renner.uno/gitea_admin/secret-rotator@sha256:159e5888ff4e23f5fe7fb1284711d81591635f840888e61402af311b0a76ba92 + # Deliberately NOT the digest the sibling random-mode rotators use + # (sha256:159e5888…). That digest NO LONGER EXISTS in the registry — + # pulling it returns NotFound, verified by running it. Untagged + # manifests appear to get garbage-collected once :latest and the + # dated tag move on, and 11 of the 14 existing rotators are pinned + # to digests that have gone the same way. Copying the sibling pin + # would have shipped a CronJob that ImagePullBackOffs on its first + # run in three months' time. + # + # This is the digest :latest currently resolves to. Verified it + # pulls, that `random` is among its modes, and that it carries the + # RANDOM_BYTES>=16 floor documented below (RANDOM_BYTES=8 exits with + # "refusing to generate a weak secret"). + image: gitea.renner.uno/gitea_admin/secret-rotator@sha256:dcf4267a7b7c9db7614cfc10fe4a4ab1ba26ea29230552a24071ec810615fcfa args: ["random"] securityContext: allowPrivilegeEscalation: false -- 2.54.0 From 9e75ab6b4d4fef96aece7f714618199262f810ac Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Thu, 30 Jul 2026 18:47:58 +0200 Subject: [PATCH 5/5] fix(mosquitto): pin the rotator by dated tag, not digest MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Owner directive: our own images do not need digest pinning, and :latest is never to be used. Dated tag it is — matching ci-ansible:2026-04-29 and the alpine-jq / cni-plugins / alpine-openssl pins added earlier today. Now secret-rotator:2026-07-30. Verified before pinning that the tag exists, pulls, lists `random` among its modes, and carries the RANDOM_BYTES>=16 floor this file documents (RANDOM_BYTES=8 exits "refusing to generate a weak secret" rather than generating one). Worth recording why the tag is today's rather than 2026-07-23, the date of the last commit touching secret-rotator/: the build step also triggers on .woodpecker/build-ci-images.yml, which was edited several times today while adding the alpine-jq, cni-plugins and alpine-openssl images. So secret-rotator was rebuilt and republished today from unchanged source. 2026-07-23 resolves to a different, older image (sha256:828bf486…) that is still present; 2026-07-30 matches what :latest currently points at. That incidental rebuild is also part of why the digest pins decayed: every unrelated pipeline edit moves :latest and orphans the previous manifest, which then gets garbage-collected. Tracked in #458 for the other 11 rotators; this change keeps the new one out of that trap rather than joining it. Co-Authored-By: Claude Opus 5 (1M context) --- .../openbao/rotation/mosquitto-health.yaml | 29 ++++++++++--------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/kubernetes/infrastructure/openbao/rotation/mosquitto-health.yaml b/kubernetes/infrastructure/openbao/rotation/mosquitto-health.yaml index 5002523b..d61f1c10 100644 --- a/kubernetes/infrastructure/openbao/rotation/mosquitto-health.yaml +++ b/kubernetes/infrastructure/openbao/rotation/mosquitto-health.yaml @@ -88,20 +88,23 @@ spec: type: RuntimeDefault containers: - name: rotate - # Deliberately NOT the digest the sibling random-mode rotators use - # (sha256:159e5888…). That digest NO LONGER EXISTS in the registry — - # pulling it returns NotFound, verified by running it. Untagged - # manifests appear to get garbage-collected once :latest and the - # dated tag move on, and 11 of the 14 existing rotators are pinned - # to digests that have gone the same way. Copying the sibling pin - # would have shipped a CronJob that ImagePullBackOffs on its first - # run in three months' time. + # DATED TAG, not a digest and never :latest — this is our own + # image, so it does not need digest pinning, and a floating tag + # would make what runs here unreviewable. # - # This is the digest :latest currently resolves to. Verified it - # pulls, that `random` is among its modes, and that it carries the - # RANDOM_BYTES>=16 floor documented below (RANDOM_BYTES=8 exits with - # "refusing to generate a weak secret"). - image: gitea.renner.uno/gitea_admin/secret-rotator@sha256:dcf4267a7b7c9db7614cfc10fe4a4ab1ba26ea29230552a24071ec810615fcfa + # Deliberately NOT copying the sibling rotators' digest pins: those + # digests NO LONGER EXIST in the registry (verified by pulling them + # — NotFound), because untagged manifests are garbage-collected + # once :latest and the dated tags move past them. 11 of the 14 + # existing rotators are pinned that way and will ImagePullBackOff + # on their first run — see issue #458. Copying the convention would + # have shipped a CronJob broken from the day it was written. + # + # Verified for THIS tag: it pulls, `random` is among its modes, and + # it carries the RANDOM_BYTES>=16 floor documented below + # (RANDOM_BYTES=8 exits "refusing to generate a weak secret"). + # Bumping the date is a deliberate, reviewable edit. + image: gitea.renner.uno/gitea_admin/secret-rotator:2026-07-30 args: ["random"] securityContext: allowPrivilegeEscalation: false -- 2.54.0