mosquitto: real MQTT round-trip probes instead of tcpSocket #150
@@ -369,3 +369,71 @@ spec:
|
||||
memory: 32Mi
|
||||
limits:
|
||||
memory: 64Mi
|
||||
---
|
||||
# Broker user `health`: a dedicated, least-privilege credential used ONLY by
|
||||
# mosquitto's own liveness/readiness probes (mosquitto-deployment.yaml). Its
|
||||
# ACL is `write health/#` and nothing else — it cannot read any topic or
|
||||
# write outside the throwaway probe subtree, so a leak is worthless (nothing
|
||||
# subscribes to health/#). Kept on its own Secret, like every other user, so
|
||||
# the probe never has to borrow the admin credential.
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: mosquitto-health-credentials-bootstrap
|
||||
namespace: home-assistant
|
||||
spec:
|
||||
backoffLimit: 4
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: mosquitto-bootstrap
|
||||
spec:
|
||||
serviceAccountName: mosquitto-bootstrap
|
||||
restartPolicy: OnFailure
|
||||
containers:
|
||||
- name: bootstrap
|
||||
image: alpine:3.21
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
set -eu
|
||||
apk add --no-cache curl jq >/dev/null
|
||||
|
||||
NS=home-assistant
|
||||
NAME=mosquitto-health-credentials
|
||||
TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)
|
||||
CA=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt
|
||||
API=https://kubernetes.default.svc
|
||||
|
||||
if curl -fsSL --cacert "$CA" -H "Authorization: Bearer $TOKEN" \
|
||||
"$API/api/v1/namespaces/$NS/secrets/$NAME" >/dev/null 2>&1; then
|
||||
echo "$NAME already exists; nothing to do."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
PW=$(tr -dc 'A-Za-z0-9' </dev/urandom | head -c 32)
|
||||
PW_B64=$(printf '%s' "$PW" | base64 | tr -d '\n')
|
||||
|
||||
body=$(jq -nc \
|
||||
--arg name "$NAME" \
|
||||
--arg ns "$NS" \
|
||||
--arg pw "$PW_B64" \
|
||||
'{apiVersion: "v1",
|
||||
kind: "Secret",
|
||||
metadata: {name: $name, namespace: $ns},
|
||||
type: "Opaque",
|
||||
data: {password: $pw}}')
|
||||
|
||||
curl -fsSL --cacert "$CA" -H "Authorization: Bearer $TOKEN" \
|
||||
-H "Content-Type: application/json" -X POST \
|
||||
--data "$body" \
|
||||
"$API/api/v1/namespaces/$NS/secrets" >/dev/null
|
||||
|
||||
echo "generated $NAME"
|
||||
resources:
|
||||
requests:
|
||||
cpu: 10m
|
||||
memory: 32Mi
|
||||
limits:
|
||||
memory: 64Mi
|
||||
|
||||
@@ -72,6 +72,7 @@ spec:
|
||||
mosquitto_passwd -b /mosquitto/auth/passwd sml2mqtt "$SML2MQTT_PASSWORD"
|
||||
mosquitto_passwd -b /mosquitto/auth/passwd open3e "$OPEN3E_PASSWORD"
|
||||
mosquitto_passwd -b /mosquitto/auth/passwd evcc "$EVCC_PASSWORD"
|
||||
mosquitto_passwd -b /mosquitto/auth/passwd health "$HEALTH_PASSWORD"
|
||||
# ACL: HA keeps full access (subscribes to everything); the
|
||||
# shellies user is `readwrite shellies/#` (devices publish state
|
||||
# and subscribe to their command topics) + `write homeassistant/#`
|
||||
@@ -119,6 +120,16 @@ spec:
|
||||
topic read open3e/#
|
||||
topic readwrite evcc/#
|
||||
topic read energy/#
|
||||
|
||||
# `health` is the broker's own probe credential (see the
|
||||
# liveness/readiness probes below). Scoped to health/# and
|
||||
# nothing else: the probes publish a random value retained and
|
||||
# read it back to prove the broker actually stores+delivers
|
||||
# (not just PUBACKs) — so the user needs read AND write, but
|
||||
# only under health/#. A leak can only touch the throwaway
|
||||
# probe subtree that nothing else subscribes to.
|
||||
user health
|
||||
topic readwrite health/#
|
||||
EOF
|
||||
chown 1883:1883 /mosquitto/auth/passwd /mosquitto/auth/acl
|
||||
chmod 0600 /mosquitto/auth/passwd /mosquitto/auth/acl
|
||||
@@ -154,6 +165,15 @@ spec:
|
||||
secretKeyRef:
|
||||
name: mosquitto-evcc-credentials
|
||||
key: password
|
||||
# Probe-only credential (mosquitto-health-credentials-bootstrap).
|
||||
# Present here for the same reason as the others — the startup
|
||||
# shim hashes it into the passwd file — and reused by the exec
|
||||
# probes below, which run inside this container.
|
||||
- name: HEALTH_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: mosquitto-health-credentials
|
||||
key: password
|
||||
ports:
|
||||
- name: mqtt
|
||||
containerPort: 1883
|
||||
@@ -170,15 +190,68 @@ spec:
|
||||
- name: tls
|
||||
mountPath: /mosquitto/tls
|
||||
readOnly: true
|
||||
livenessProbe:
|
||||
# Real MQTT round-trip probes, replacing the old tcpSocket checks.
|
||||
# A bare TCP accept proved only that a listener existed. Even a
|
||||
# QoS-1 PUBACK proves only that the broker acknowledged the publish
|
||||
# at the protocol level — a broker wedged so it ACKs writes but
|
||||
# drops them (a write blackhole) would still pass. So each probe
|
||||
# publishes a fresh RANDOM value retained and reads it back, and
|
||||
# succeeds only if the exact bytes round-trip. That proves the
|
||||
# broker authenticated, stored the message, and delivered it to a
|
||||
# subscriber — auth + write + storage + read + delivery. Verified
|
||||
# live before merge (retained round-trip returns the value;
|
||||
# subscribing to an empty topic times out and fails).
|
||||
#
|
||||
# Retained (not a live subscribe-then-publish) on purpose: the
|
||||
# subscriber always gets the stored value regardless of connect
|
||||
# timing, so there is no race that could false-fail the probe.
|
||||
# Liveness and readiness use SEPARATE topics (probe/live vs
|
||||
# probe/ready) so their rotating retained values never collide.
|
||||
#
|
||||
# Uses the dedicated least-privilege `health` user (ACL: readwrite
|
||||
# health/# only — see the passwd/acl build above and the
|
||||
# mosquitto-health-credentials-bootstrap Job). Its password is in
|
||||
# this container's env (HEALTH_PASSWORD) because the startup shim
|
||||
# hashes every user into the passwd file; the exec probes run
|
||||
# inside this container and read it directly. A leak of this
|
||||
# credential can only touch the health/# probe subtree, which
|
||||
# nothing else subscribes to.
|
||||
#
|
||||
# mosquitto is a single-replica SPOF (HA, Shelly, sml2mqtt, open3e,
|
||||
# evcc). Liveness is deliberately slow to fire — failureThreshold 6
|
||||
# x 30s = 3 min of sustained failure before a restart — so a
|
||||
# transient hiccup never bounces the broker every client depends
|
||||
# on. The startupProbe (tcpSocket) suspends both while the
|
||||
# container builds the passwd/acl files and mosquitto binds :1883.
|
||||
startupProbe:
|
||||
tcpSocket:
|
||||
port: mqtt
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 30
|
||||
periodSeconds: 5
|
||||
failureThreshold: 12
|
||||
readinessProbe:
|
||||
tcpSocket:
|
||||
port: mqtt
|
||||
periodSeconds: 10
|
||||
exec:
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
V="probe-$(head -c8 /dev/urandom | od -An -tx1 | tr -d ' ')"
|
||||
mosquitto_pub -h 127.0.0.1 -p 1883 -u health -P "$HEALTH_PASSWORD" -q 1 -r -t health/probe/ready -m "$V" &&
|
||||
[ "$(mosquitto_sub -h 127.0.0.1 -p 1883 -u health -P "$HEALTH_PASSWORD" -t health/probe/ready -C 1 -W 5)" = "$V" ]
|
||||
periodSeconds: 15
|
||||
timeoutSeconds: 10
|
||||
failureThreshold: 3
|
||||
livenessProbe:
|
||||
exec:
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
V="probe-$(head -c8 /dev/urandom | od -An -tx1 | tr -d ' ')"
|
||||
mosquitto_pub -h 127.0.0.1 -p 1883 -u health -P "$HEALTH_PASSWORD" -q 1 -r -t health/probe/live -m "$V" &&
|
||||
[ "$(mosquitto_sub -h 127.0.0.1 -p 1883 -u health -P "$HEALTH_PASSWORD" -t health/probe/live -C 1 -W 5)" = "$V" ]
|
||||
periodSeconds: 30
|
||||
timeoutSeconds: 10
|
||||
failureThreshold: 6
|
||||
resources:
|
||||
requests:
|
||||
cpu: 20m
|
||||
|
||||
Reference in New Issue
Block a user