diff --git a/kubernetes/apps/home-assistant/kustomization.yaml b/kubernetes/apps/home-assistant/kustomization.yaml index e774a2dc..bad7ab66 100644 --- a/kubernetes/apps/home-assistant/kustomization.yaml +++ b/kubernetes/apps/home-assistant/kustomization.yaml @@ -13,6 +13,9 @@ resources: - prometheus-token-externalsecret.yaml - config-sync-token-externalsecret.yaml - mosquitto-bootstrap.yaml + # health broker credential: OpenBao-backed + Class-A rotated (first of the + # eight users to migrate off mosquitto-bootstrap.yaml). + - mosquitto-health-credentials-externalsecret.yaml - mosquitto-config.yaml - mosquitto-certificate.yaml - mosquitto-service.yaml diff --git a/kubernetes/apps/home-assistant/mosquitto-bootstrap.yaml b/kubernetes/apps/home-assistant/mosquitto-bootstrap.yaml index e5f60cd6..44f34cf0 100644 --- a/kubernetes/apps/home-assistant/mosquitto-bootstrap.yaml +++ b/kubernetes/apps/home-assistant/mosquitto-bootstrap.yaml @@ -370,74 +370,6 @@ spec: limits: memory: 64Mi --- -# Broker user `health`: a dedicated, least-privilege credential used ONLY by -# mosquitto's own liveness/readiness probes (mosquitto-deployment.yaml). Its -# ACL is `write health/#` and nothing else — it cannot read any topic or -# write outside the throwaway probe subtree, so a leak is worthless (nothing -# subscribes to health/#). Kept on its own Secret, like every other user, so -# the probe never has to borrow the admin credential. -apiVersion: batch/v1 -kind: Job -metadata: - name: mosquitto-health-credentials-bootstrap - namespace: home-assistant -spec: - backoffLimit: 4 - template: - metadata: - labels: - app.kubernetes.io/name: mosquitto-bootstrap - spec: - serviceAccountName: mosquitto-bootstrap - restartPolicy: OnFailure - containers: - - name: bootstrap - image: alpine:3.21 - command: - - sh - - -c - - | - set -eu - apk add --no-cache curl jq >/dev/null - - NS=home-assistant - NAME=mosquitto-health-credentials - TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token) - CA=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt - API=https://kubernetes.default.svc - - if curl -fsSL --cacert "$CA" -H "Authorization: Bearer $TOKEN" \ - "$API/api/v1/namespaces/$NS/secrets/$NAME" >/dev/null 2>&1; then - echo "$NAME already exists; nothing to do." - exit 0 - fi - - PW=$(tr -dc 'A-Za-z0-9' /dev/null - - echo "generated $NAME" - resources: - requests: - cpu: 10m - memory: 32Mi - limits: - memory: 64Mi ---- # Broker user `zigbee2mqtt` — the Zigbee2MQTT bridge (ZHA replacement). Scoped # to its own `zigbee2mqtt/#` subtree plus `write homeassistant/#` (so it can # publish HA MQTT-discovery configs, exactly like the `shellies` user) and diff --git a/kubernetes/apps/home-assistant/mosquitto-deployment.yaml b/kubernetes/apps/home-assistant/mosquitto-deployment.yaml index c52e8119..6e10a88c 100644 --- a/kubernetes/apps/home-assistant/mosquitto-deployment.yaml +++ b/kubernetes/apps/home-assistant/mosquitto-deployment.yaml @@ -5,6 +5,18 @@ metadata: namespace: home-assistant labels: app.kubernetes.io/name: mosquitto + annotations: + # REQUIRED for credential rotation to take effect at all. The startup shim + # rebuilds /mosquitto/auth/passwd from these Secrets on every pod start, so + # a rotated Secret is inert until the pod restarts — without this the new + # password would sit unused until some unrelated restart, while the old one + # stayed valid indefinitely. + # + # Listed per-Secret rather than using the blanket `reloader.stakater.com/ + # auto: "true"`, so that enrolling each remaining broker user in rotation is + # a visible, reviewable edit to this line rather than something that starts + # happening implicitly. + secret.reloader.stakater.com/reload: mosquitto-health-credentials spec: replicas: 1 strategy: @@ -177,7 +189,9 @@ spec: secretKeyRef: name: mosquitto-evcc-credentials key: password - # Probe-only credential (mosquitto-health-credentials-bootstrap). + # Probe-only credential. Now OpenBao-backed and Class-A rotated (see + # mosquitto-health-credentials-externalsecret.yaml); its bootstrap Job + # is gone. # Present here for the same reason as the others — the startup # shim hashes it into the passwd file — and reused by the exec # probes below, which run inside this container. @@ -230,7 +244,7 @@ spec: # # Uses the dedicated least-privilege `health` user (ACL: readwrite # health/# only — see the passwd/acl build above and the - # mosquitto-health-credentials-bootstrap Job). Its password is in + # mosquitto-health-credentials ExternalSecret). Its password is in # this container's env (HEALTH_PASSWORD) because the startup shim # hashes every user into the passwd file; the exec probes run # inside this container and read it directly. A leak of this diff --git a/kubernetes/apps/home-assistant/mosquitto-health-credentials-externalsecret.yaml b/kubernetes/apps/home-assistant/mosquitto-health-credentials-externalsecret.yaml new file mode 100644 index 00000000..4422d819 --- /dev/null +++ b/kubernetes/apps/home-assistant/mosquitto-health-credentials-externalsecret.yaml @@ -0,0 +1,36 @@ +# Projects the mosquitto `health` broker credential from OpenBao into the Secret +# the broker already consumes. FIRST of the eight broker users to move off the +# bootstrap Jobs and onto Class-A rotation (see +# PROPOSAL-mosquitto-credential-rotation.md (PR #453 — not yet merged; this +# file lands with it)); the rotator lives at +# kubernetes/infrastructure/openbao/rotation/mosquitto-health.yaml. +# +# Secret name and key are UNCHANGED (`password`), so mosquitto-deployment.yaml's +# secretKeyRef and the exec probes that read $HEALTH_PASSWORD need no edit. +# +# refreshInterval is 10m rather than the 1h used by mosquitto-evcc-credentials: +# a rotation only lands once ESO re-reads, so 1h would mean up to an hour between +# the rotator running and the broker actually using the new password, with the +# old one still valid in the meantime. 10m keeps that window short without +# polling OpenBao pointlessly hard. +# +# ⚠️ OWNERSHIP: creationPolicy is implicitly Owner, so ESO sets an ownerReference +# on the Secret. Deleting this ExternalSecret therefore DELETES the live Secret +# and with it the broker's probe credential — the bootstrap Job it replaces had +# no such coupling. To detach without destroying, set creationPolicy: Orphan +# before removing it. +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: mosquitto-health-credentials + namespace: home-assistant +spec: + refreshInterval: 10m + secretStoreRef: + kind: ClusterSecretStore + name: openbao + target: + name: mosquitto-health-credentials + dataFrom: + - extract: + key: apps/home-assistant/mosquitto-health-credentials diff --git a/kubernetes/infrastructure/openbao/policies/rotator-mosquitto-health.hcl b/kubernetes/infrastructure/openbao/policies/rotator-mosquitto-health.hcl new file mode 100644 index 00000000..0fcde710 --- /dev/null +++ b/kubernetes/infrastructure/openbao/policies/rotator-mosquitto-health.hcl @@ -0,0 +1,46 @@ +# OpenBao policy: rotator-mosquitto-health +# +# Least-privilege policy for the mosquitto `health` broker-credential rotation +# CronJob (kubernetes auth role `rotator-mosquitto-health`). PER-SECRET +# isolation, same template as rotator-paperless: one policy + one role per +# rotated secret, so a compromised rotation Job cannot touch any other secret. +# +# Scoped to the `health` credential ALONE, deliberately not to a wildcard over +# apps/home-assistant/mosquitto-*. The broker has eight users and they are being +# enrolled one at a time; a wildcard would silently grant the first rotator write +# access to the other seven before they are ready, which is exactly the blast +# radius this per-secret pattern exists to avoid. +# +# Plan and phasing: kubernetes/apps/home-assistant/ +# PROPOSAL-mosquitto-credential-rotation.md — added by PR #453, which is not yet +# merged, so that path does not resolve until it lands. +# +# NOT applied by Flux — OpenBao policies/roles are not Kubernetes resources +# (same convention as eso-read.hcl and every other rotator policy). Apply ONCE +# with an admin token: +# +# bao policy write rotator-mosquitto-health \ +# kubernetes/infrastructure/openbao/policies/rotator-mosquitto-health.hcl +# +# bao write auth/kubernetes/role/rotator-mosquitto-health \ +# bound_service_account_names=mosquitto-health-secret-rotator \ +# bound_service_account_namespaces=openbao \ +# policies=rotator-mosquitto-health ttl=15m +# +# Do NOT set token_no_default_policy=true — the `bao kv patch` kv-v2 preflight +# needs the default policy (see rotator.hcl). The rotator writes via patch, so +# hardening the role that way would break rotation rather than tighten it. +# +# BOOTSTRAP: `random` mode calls assert_fields_exist BEFORE patching, so it +# ROTATES an existing field and never creates one. The path must already hold a +# `password` field or the Job fails safe without writing. Seed it once: +# +# bao kv put homelab/apps/home-assistant/mosquitto-health-credentials \ +# password="$(kubectl -n home-assistant get secret mosquitto-health-credentials \ +# -o jsonpath='{.data.password}' | base64 -d)" +# +# (Reads are already covered by eso-read.hcl's `homelab/data/*`, so the +# ExternalSecret needs no policy change.) +path "homelab/data/apps/home-assistant/mosquitto-health-credentials" { + capabilities = ["read", "patch"] +} diff --git a/kubernetes/infrastructure/openbao/rotation/kustomization.yaml b/kubernetes/infrastructure/openbao/rotation/kustomization.yaml index f2e61a94..a81b73b7 100644 --- a/kubernetes/infrastructure/openbao/rotation/kustomization.yaml +++ b/kubernetes/infrastructure/openbao/rotation/kustomization.yaml @@ -18,3 +18,4 @@ resources: - home-assistant.yaml - review-bot.yaml - claude-oauth.yaml + - mosquitto-health.yaml diff --git a/kubernetes/infrastructure/openbao/rotation/mosquitto-health.yaml b/kubernetes/infrastructure/openbao/rotation/mosquitto-health.yaml new file mode 100644 index 00000000..d61f1c10 --- /dev/null +++ b/kubernetes/infrastructure/openbao/rotation/mosquitto-health.yaml @@ -0,0 +1,135 @@ +--- +# Class-A automated rotation of the mosquitto `health` broker credential — a +# self-generated random secret. +# +# FIRST of the eight mosquitto broker users to be enrolled. The credentials were +# previously minted once by bootstrap Jobs and then never changed at all: no +# schedule, no compromise response, and they existed only in the Secret object, +# outside the OpenBao snapshot. See kubernetes/apps/home-assistant/ +# PROPOSAL-mosquitto-credential-rotation.md (PR #453 — not yet merged; this +# file lands with it) for the full plan and phasing. +# +# `health` deliberately goes first: it is the broker's OWN liveness/readiness +# probe credential, consumed by exec probes inside the mosquitto container via +# the same Secret that seeds the passwd file. Nothing outside the pod holds it, +# so a rotation needs no coordination with any client — the smallest possible +# blast radius for proving rotator -> OpenBao -> ESO -> Reloader end to end. +# +# Runs the secret-rotator binary (random mode): generate 32 random bytes, +# standard-base64 encode, and merge-patch into OpenBao. assert_fields_exist runs +# FIRST, so a misconfigured path fails safe (no rotation) rather than adding a +# stray field — which also means this ROTATES an existing value and never +# bootstraps one. Seeding is a one-time out-of-band step, documented in +# policies/rotator-mosquitto-health.hcl. +# +# Base64 output is safe for this credential specifically: the probes pass it as +# `-P "$HEALTH_PASSWORD"` to mosquitto_pub/sub inside the container, so the `+`, +# `/` and `=` characters are never parsed by a URL or a config file. That is NOT +# automatically true of the other seven users (sml2mqtt and open3e templates on +# the edge host, Shelly device firmware, the HA config entry), so the charset +# must be re-checked per user before enrolling each one — the old bootstrap Jobs +# generated alphanumeric-only passwords with `tr -dc 'A-Za-z0-9'`. +# +# openbao-ns pattern: the Job runs in the openbao namespace, reaching the +# OpenBao API intra-namespace — no NetworkPolicy needed. It writes OpenBao only; +# ESO syncs mosquitto-health-credentials (ns home-assistant) and Reloader +# restarts the broker so the passwd file is rebuilt from the new value. +# +# Rotation is not instant: it completes only after the ExternalSecret's +# refreshInterval elapses and the broker rolls. During that roll two broker pods +# with different passwd files sit behind the LB (maxUnavailable: 0/maxSurge: 1), +# so a probe against the outgoing pod can fail auth briefly. Self-healing. +# +# ON-DEMAND: kubectl -n openbao create job --from=cronjob/mosquitto-health-secret-rotate mosquitto-health-secret-rotate-now +apiVersion: v1 +kind: ServiceAccount +metadata: + name: mosquitto-health-secret-rotator + namespace: openbao + labels: + app.kubernetes.io/name: mosquitto + app.kubernetes.io/component: secret-rotator +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: mosquitto-health-secret-rotate + namespace: openbao + labels: + app.kubernetes.io/name: mosquitto + app.kubernetes.io/component: secret-rotator +spec: + # Quarterly, offset one month from paperless (which runs Jan/Apr/Jul/Oct) so + # unrelated rotations do not land in the same window and complicate diagnosis. + schedule: "0 4 1 2,5,8,11 *" + timeZone: Europe/Berlin + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 2 + failedJobsHistoryLimit: 5 + startingDeadlineSeconds: 3600 + jobTemplate: + spec: + backoffLimit: 2 + activeDeadlineSeconds: 300 + ttlSecondsAfterFinished: 259200 # 3 d + template: + metadata: + labels: + app.kubernetes.io/name: mosquitto + app.kubernetes.io/component: secret-rotator + spec: + serviceAccountName: mosquitto-health-secret-rotator + restartPolicy: Never + securityContext: + runAsNonRoot: true + runAsUser: 65534 + runAsGroup: 65534 + seccompProfile: + type: RuntimeDefault + containers: + - name: rotate + # DATED TAG, not a digest and never :latest — this is our own + # image, so it does not need digest pinning, and a floating tag + # would make what runs here unreviewable. + # + # Deliberately NOT copying the sibling rotators' digest pins: those + # digests NO LONGER EXIST in the registry (verified by pulling them + # — NotFound), because untagged manifests are garbage-collected + # once :latest and the dated tags move past them. 11 of the 14 + # existing rotators are pinned that way and will ImagePullBackOff + # on their first run — see issue #458. Copying the convention would + # have shipped a CronJob broken from the day it was written. + # + # Verified for THIS tag: it pulls, `random` is among its modes, and + # it carries the RANDOM_BYTES>=16 floor documented below + # (RANDOM_BYTES=8 exits "refusing to generate a weak secret"). + # Bumping the date is a deliberate, reviewable edit. + image: gitea.renner.uno/gitea_admin/secret-rotator:2026-07-30 + args: ["random"] + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + env: + - name: BAO_ADDR + value: http://openbao.openbao.svc.cluster.local:8200 + - name: BAO_ROLE + value: rotator-mosquitto-health + - name: BAO_PATH + value: homelab/apps/home-assistant/mosquitto-health-credentials + # Matches the key the ExternalSecret projects and the key the + # Secret already carries, so mosquitto-deployment.yaml's + # secretKeyRef is unchanged. + - name: BAO_FIELDS + value: "password" + # 32 bytes -> 44 base64 chars. The binary refuses anything below + # 16 bytes outright. + - name: RANDOM_BYTES + value: "32" + resources: + requests: + cpu: 25m + memory: 16Mi + limits: + memory: 48Mi