diff --git a/kubernetes/apps/home-assistant/kustomization.yaml b/kubernetes/apps/home-assistant/kustomization.yaml index add9c4eb..e774a2dc 100644 --- a/kubernetes/apps/home-assistant/kustomization.yaml +++ b/kubernetes/apps/home-assistant/kustomization.yaml @@ -68,5 +68,13 @@ configMapGenerator: namespace: home-assistant files: - sync.sh=config-sync.sh + # Z2M re-interview watchdog sidecar script. Hash suffix disabled (name stays + # stable), so editing zigbee2mqtt-interview-watch.sh does not auto-roll the + # pod — restart manually with + # `kubectl rollout restart deploy/zigbee2mqtt -n home-assistant`. + - name: zigbee2mqtt-interview-watch-script + namespace: home-assistant + files: + - watch.sh=zigbee2mqtt-interview-watch.sh generatorOptions: disableNameSuffixHash: true diff --git a/kubernetes/apps/home-assistant/zigbee2mqtt-config.yaml b/kubernetes/apps/home-assistant/zigbee2mqtt-config.yaml index 2d5f3275..aebb342f 100644 --- a/kubernetes/apps/home-assistant/zigbee2mqtt-config.yaml +++ b/kubernetes/apps/home-assistant/zigbee2mqtt-config.yaml @@ -52,7 +52,34 @@ data: baudrate: 115200 advanced: + # SEED-ONLY, like the rest of this file (see the header): the two log_* + # keys below are NOT picked up by a running Z2M whose PVC is already + # seeded. Both were therefore also applied to the live + # /app/data/configuration.yaml via `zigbee2mqtt/bridge/request/options`, + # which Z2M persists. They are duplicated here so a fresh PVC comes up + # with the sidecar working; if you delete these, delete the sidecar too. + # Note log_symlink_current only takes effect on a Z2M restart, since the + # symlink is created when the log directory is opened. log_level: info + # zh:controller at debug emits, for every received frame, + # Received payload: clusterID=6, address=12468, ... + # which the zigbee2mqtt-interview-watch.sh sidecar uses as its "device is + # awake right now" signal to fire a forced interview (see that script's + # header for why nothing on MQTT works for this). Namespaced rather than a + # global `log_level: debug` on purpose: global debug adds the zh:zstack: + # unpi:parser byte-level firehose (~90 lines/min measured), the namespaced + # form costs ~6 lines/min. Drop this once every device is interviewed — + # the sidecar is then a no-op and can go too. + log_namespaced_levels: + zh:controller: debug + # Maintains log/current -> the active log// directory. + # REQUIRED by the interview-watch sidecar: without it there is no stable + # path to follow, and `tail -F` on the newest timestamped directory silently + # goes deaf the moment Z2M restarts into a new one (the old file just stops + # receiving writes, so nothing signals the change). Verified that BusyBox + # `tail -F` does follow this symlink when Z2M repoints it. Remove together + # with the sidecar. + log_symlink_current: true # EXPLICIT network params matching the live coordinator, so zigbee-herdsman # ADOPTS (resumes) the existing network instead of forming a new one — # every device stays joined, no re-pairing. herdsman compares these against diff --git a/kubernetes/apps/home-assistant/zigbee2mqtt-deployment.yaml b/kubernetes/apps/home-assistant/zigbee2mqtt-deployment.yaml index 0c7c5a4c..1d76cdd2 100644 --- a/kubernetes/apps/home-assistant/zigbee2mqtt-deployment.yaml +++ b/kubernetes/apps/home-assistant/zigbee2mqtt-deployment.yaml @@ -136,6 +136,78 @@ spec: memory: 128Mi limits: memory: 512Mi + # Fires a forced interview whenever a still-un-interviewed device + # transmits — i.e. at the only moment a sleepy battery device can + # actually complete one. Z2M itself only retries interviews on a ZDO + # announce, which these devices essentially never emit, so without this + # they stay "Not supported" indefinitely (16 of them did, for six days + # across 38 restarts). Full rationale in the script header. + # + # A SIDECAR rather than its own Deployment: it reads both Z2M's log file + # and database.db, and the PVC is RWO/single-writer, so it has to share + # this pod. It only ever reads the PVC. + # + # Depends on advanced.log_namespaced_levels {zh:controller: debug} in + # zigbee2mqtt-config.yaml. Remove both once every device is interviewed. + - name: interview-watch + # Reuses the broker image purely for its bundled mosquitto_pub, so + # nothing is installed at runtime — no `apk add`, hence no network + # dependency at start and no package-index drift. Everything the script + # needs was verified present in this exact image: sh, tail (incl. -F), + # awk (BusyBox match()/RSTART/RLENGTH), cat, date, mkdir, + # mosquitto_pub. Pinned deliberately WITHOUT a Flux $imagepolicy + # marker: this sidecar is temporary, and it must not drag the broker's + # image-automation policy along with it. + image: eclipse-mosquitto:2.0.22 + imagePullPolicy: IfNotPresent + command: [sh, /script/watch.sh] + env: + - name: MQTT_PASSWORD + valueFrom: + secretKeyRef: + name: mosquitto-zigbee2mqtt-credentials + key: password + # These three must track zigbee2mqtt-config.yaml: MQTT_HOST the + # mosquitto Service name, MQTT_USER and BASE_TOPIC the mqtt.user and + # mqtt.base_topic it connects with. Set explicitly rather than left + # to the script's defaults so the coupling is visible here. + - name: MQTT_HOST + value: mosquitto + - name: MQTT_USER + value: zigbee2mqtt + - name: BASE_TOPIC + value: zigbee2mqtt + - name: STATEDIR + value: /state + # Runs as the image's own mosquitto user (1883). Viable because + # nothing is installed at runtime and everything it reads under + # /app/data is world-readable (dirs 0755, files 0644 — verified), so + # root is not needed to tail the log or read database.db. + securityContext: + runAsNonRoot: true + runAsUser: 1883 + runAsGroup: 1883 + allowPrivilegeEscalation: false + capabilities: + drop: [ALL] + volumeMounts: + - name: data + mountPath: /app/data + readOnly: true + - name: interview-watch-script + mountPath: /script + readOnly: true + # Cooldown stamps. An explicit emptyDir rather than the image's + # /tmp, so it stays writable as non-root without depending on the + # container filesystem. + - name: interview-watch-state + mountPath: /state + resources: + requests: + cpu: 10m + memory: 16Mi + limits: + memory: 64Mi volumes: - name: data persistentVolumeClaim: @@ -146,3 +218,8 @@ spec: - name: coordinator-backup secret: secretName: zigbee2mqtt-coordinator-backup + - name: interview-watch-script + configMap: + name: zigbee2mqtt-interview-watch-script + - name: interview-watch-state + emptyDir: {} diff --git a/kubernetes/apps/home-assistant/zigbee2mqtt-interview-watch.sh b/kubernetes/apps/home-assistant/zigbee2mqtt-interview-watch.sh new file mode 100644 index 00000000..d7f0580e --- /dev/null +++ b/kubernetes/apps/home-assistant/zigbee2mqtt-interview-watch.sh @@ -0,0 +1,250 @@ +#!/bin/sh +# Re-interview watchdog for Zigbee2MQTT. +# +# PROBLEM +# The ZHA->Z2M cutover (2026-07-23) adopted the existing Zigbee network, so +# every device stayed joined but Z2M's database.db started empty. Each device +# must therefore be *interviewed* before Z2M knows its modelId and can match a +# converter; until then it logs ": Not supported (EndDevice)" and exposes +# only a generic "Automatically generated definition" (battery/linkquality), +# not the real entities (action / contact / moisture). +# +# Mains-powered routers interviewed immediately (always awake). Battery end +# devices did not, and they do NOT recover on their own, because: +# +# - zigbee-herdsman only retries an interview on a ZDO device *announce* +# (controller.js: interviewState Pending|Failed -> device.interview()), and +# sleepy battery devices almost never emit one spontaneously. +# - Z2M's Configure extension retries on any traffic (onLastSeenChanged) but +# is capped at 3 attempts per process (in-memory Map, reset on restart). +# +# So 16 devices sat PENDING for six days across 38 process restarts. +# +# APPROACH +# A device that transmits *is awake right now*, which is exactly when an +# interview can succeed. This watches for that moment and fires a forced +# interview request. Confirmed in practice: interviews completed in 11-34s when +# triggered this way, including Xiaomi/Aqara sensors whose wake window is short. +# +# WHY THE LOG FILE AND NOT MQTT +# An un-interviewed Xiaomi device publishes NOTHING to MQTT (its +# manufacturer-specific attributes cannot be mapped by the auto-generated +# definition), so MQTT state topics are not a usable signal for exactly the +# devices that need help. `zigbee2mqtt/bridge/logging` carries only `info`. +# The one reliable signal is the zh:controller debug line +# +# Received payload: clusterID=6, address=12468, ... +# +# which requires advanced.log_namespaced_levels {zh:controller: debug} — set in +# zigbee2mqtt-config.yaml. Root log_level stays `info`; enabling debug globally +# adds ~90 lines/min of UNPI frame noise, the namespaced form ~6 lines/min. +# +# This runs as a sidecar (not a separate Deployment) because it needs the same +# RWO PVC as Z2M for both the log file and database.db. +# +# NOTE: interviewing is only half the job. Philips devices additionally need +# `configure` to bind manuSpecificPhilips (0xFC00), and that cluster is only +# registered on Z2M's `start` event (deviceAddCustomCluster onEvent), so a +# device interviewed mid-run cannot be configured until the next restart. One +# Z2M restart after a batch of interviews completes the binds. + +set -u + +DATA=/app/data +DB="$DATA/database.db" +MQTT_HOST=${MQTT_HOST:-mosquitto} +MQTT_USER=${MQTT_USER:-zigbee2mqtt} +# Must track mqtt.base_topic in zigbee2mqtt-config.yaml. Overridable via env so +# the two can be kept in step from the Deployment rather than by editing this +# script; the MQTT ACL for this user is scoped to `zigbee2mqtt/#`, so a change +# here needs a matching ACL change too. +BASE_TOPIC=${BASE_TOPIC:-zigbee2mqtt} +COOLDOWN=${COOLDOWN:-300} # min seconds between interview attempts per device +PUB_TIMEOUT=${PUB_TIMEOUT:-10} # seconds; bounds a single mosquitto_pub +HEARTBEAT=${HEARTBEAT:-600} # seconds between "still N to go" log lines +STATEDIR=${STATEDIR:-/state} # emptyDir; writable as non-root + +mkdir -p "$STATEDIR" + +log() { printf '%s interview-watch: %s\n' "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" "$*"; } + +# Resolve a network address to an IEEE, but ONLY for devices that still need an +# interview. database.db is newline-delimited JSON (one device per line) despite +# the .db extension, so a per-line scan is enough and needs no jq. Read fresh on +# every frame so devices drop out the moment they succeed, and so changed +# network addresses are picked up automatically. +# +# Each field is matched INDEPENDENTLY rather than with one combined pattern: +# JSON guarantees no key order, so a single regex expecting "ieeeAddr" before +# "nwkAddr" would silently capture nothing (no error, no log — a device just +# never gets picked up) if herdsman's serialization order ever changed. +ieee_needing_interview() { + awk -v want="$1" ' + /"interviewState":"(PENDING|FAILED)"/ { + ieee = ""; nwk = "" + if (match($0, /"ieeeAddr":"[^"]*"/)) { + s = substr($0, RSTART, RLENGTH); sub(/^"ieeeAddr":"/, "", s); sub(/"$/, "", s); ieee = s + } + if (match($0, /"nwkAddr":[0-9]+/)) { + s = substr($0, RSTART, RLENGTH); sub(/^"nwkAddr":/, "", s); nwk = s + } + if (ieee != "" && nwk == want) { print ieee; exit } + }' "$DB" 2>/dev/null +} + +# NOTE the `|| true` and the sanity case: `grep -c` prints "0" AND exits 1 when +# nothing matches, so a naive `grep -c ... || echo 0` emits TWO lines ("0\n0"), +# which then blows up every `-eq` test with "bad number". That only shows up once +# no devices need interviewing — i.e. the steady end state this is meant to reach. +count_needing_interview() { + c=$(grep -cE '"interviewState":"(PENDING|FAILED)"' "$DB" 2>/dev/null || true) + case "$c" in + ''|*[!0-9]*) c=0 ;; # missing/unreadable DB, or unexpected output + esac + printf '%s' "$c" +} + +log "starting" +log "$(count_needing_interview) device(s) currently need an interview" + +# Follow the STABLE path, not the newest timestamped directory. +# +# Z2M writes logs to log//log.log — a fresh directory on every +# start. `tail -F` follows a specific pathname: it re-opens that name if the +# inode behind it changes, but it can never notice that Z2M has begun writing +# somewhere else entirely. Tailing the newest directory therefore breaks the +# moment Z2M restarts: the old file simply goes quiet, so the pipe never ends, +# the outer loop never re-resolves, "following " still shows the old path, +# and the sidecar looks healthy while seeing no frames again ever. That is not a +# rare edge case here — the Z2M container has restarted dozens of times on its +# own (DNS flaps), each time into a new directory, and the planned "interview a +# batch, then restart Z2M once" step hits it deliberately. +# +# Fixed at the source instead of by polling: advanced.log_symlink_current: true +# (set in zigbee2mqtt-config.yaml) makes Z2M maintain log/current -> the active +# directory, giving one stable path. Verified that BusyBox `tail -F` in this +# image does follow a symlink that gets repointed to a new target mid-follow, so +# a Z2M restart is picked up automatically with no watchdog. +# +# Deliberately NO fallback to the newest directory: falling back would risk +# silently tailing a frozen file, which is the exact failure being fixed. If the +# symlink is missing the sidecar says so, loudly and repeatedly, and waits. +LOGFILE="$DATA/log/current/log.log" +warned=0 +while [ ! -f "$LOGFILE" ]; do + if [ "$warned" -eq 0 ]; then + log "WARNING: $LOGFILE absent — Z2M needs advanced.log_symlink_current: true (and a restart to create it). Waiting." + warned=1 + fi + sleep 30 + log "still waiting for $LOGFILE" +done + +# Heartbeat, independent of frame arrival. The main loop is purely event-driven, +# so if the "Received payload: clusterID=" format ever changes (herdsman upgrade, +# log format change) it would match nothing forever while still looking healthy — +# and the per-frame recount cannot detect that either, because it only runs when +# a frame is matched. This gives an unconditional signal to notice regression +# against: if the count never falls while devices are demonstrably being used, +# the matcher is broken. +( + while :; do + sleep "$HEARTBEAT" + log "heartbeat: $(count_needing_interview) device(s) still need an interview" + done +) & + +# Kept as a loop purely so an unexpected end of stream re-establishes the tail +# rather than exiting. Exiting would incur kubelet crash-loop backoff (climbing +# to 5 min unless the container stays up >=10 min), which would bite precisely +# when a device's short wake window needs a prompt reaction. +while :; do + log "following $LOGFILE ($(count_needing_interview) device(s) still need an interview)" + + # + # Matching happens in the read loop rather than via a `grep` stage on purpose. + # This runs on an Alpine-based image whose `grep` is the BusyBox applet, and + # BusyBox grep does NOT support --line-buffered: + # grep: unrecognized option: line-buffered (exit 2) + # so a `tail -F | grep --line-buffered | while read` pipe yields ZERO lines + # forever while the sidecar still logs "following " and looks healthy — + # a silent no-op of the whole feature. Dropping the stage avoids both that and + # any block-buffering question, since `tail` writes per line and `read` + # consumes per line, with no extra package to install. + pending=$(count_needing_interview) + next_recount=0 + + tail -F "$LOGFILE" 2>/dev/null | + while IFS= read -r line; do + case "$line" in + *"Received payload: clusterID="*) ;; + *) continue ;; + esac + + # ... clusterID=6, address=12468, groupID=0, endpoint=1, ... + rest=${line#*"Received payload: clusterID="} + cluster=${rest%%,*} + rest=${rest#*address=} + nwk=${rest%%,*} + + # Guard against a format change silently yielding garbage. + case "$nwk" in + ''|*[!0-9]*) continue ;; + esac + + now=$(date +%s) + + # Short-circuit while nothing needs interviewing. The debug log covers ALL + # of zh:controller, so every device on the network produces frames here — + # roughly 30 devices' worth once the backlog is cleared. Without this the + # script would fork awk over the whole database.db for each of those frames + # forever, for no possible benefit. Recounted periodically (not per frame) + # so a device that later regresses to PENDING is still picked up. + if [ "$now" -ge "$next_recount" ]; then + pending=$(count_needing_interview) + next_recount=$((now + 60)) + [ "$pending" -eq 0 ] && log "no devices need an interview; idling (recheck every 60s)" + fi + [ "$pending" -eq 0 ] && continue + + ieee=$(ieee_needing_interview "$nwk") + [ -z "${ieee:-}" ] && continue # already interviewed, or not a device we track + + stamp="$STATEDIR/$ieee" + if [ -f "$stamp" ]; then + last=$(cat "$stamp" 2>/dev/null || echo 0) + # Suppress the interview's OWN traffic, which would otherwise re-trigger + # this on every attribute read it performs. + [ $((now - last)) -lt "$COOLDOWN" ] && continue + fi + + log "frame from $ieee (nwk=$nwk cluster=$cluster) while awake -> requesting interview" + # NOTE: mosquitto_pub can only take a password via argv (-P, or an -L URL); + # it has no --pw-file or env option, so this cannot be avoided with this + # client. Not a cross-container exposure: the pod does not set + # shareProcessNamespace, so /proc is not shared with the Z2M container. + # Bounded with `timeout`, which also covers a TCP connect that hangs rather + # than being refused. The read loop is single-threaded, so an unbounded + # publish would stall it and miss other devices' wake windows. + # + # NOT `mosquitto_pub -W`: that flag does not exist for mosquitto_pub (it is + # a mosquitto_sub option), and passing it makes every publish fail on a + # usage error — verified against the client in this image. + if timeout "$PUB_TIMEOUT" mosquitto_pub -h "$MQTT_HOST" -u "$MQTT_USER" -P "$MQTT_PASSWORD" \ + -t "$BASE_TOPIC/bridge/request/device/interview" \ + -m "{\"id\":\"$ieee\"}" 2>/dev/null; then + # Stamp only on a SUCCESSFUL publish. Stamping unconditionally would burn + # the device's wake window on a failed publish (broker hiccup, DNS blip) + # and then refuse to retry for the full cooldown, despite no interview + # ever having been requested. + printf '%s' "$now" > "$stamp" + log " interview requested for $ieee" + else + log " WARNING: mosquitto_pub failed for $ieee; not starting cooldown so the next frame retries" + fi + done + + # Stream ended: Z2M restarted into a new log directory, or the file vanished. + log "log stream ended; re-resolving current log file" + sleep 5 +done diff --git a/kubernetes/apps/home-assistant/zigbee2mqtt-migration-runbook.md b/kubernetes/apps/home-assistant/zigbee2mqtt-migration-runbook.md index a4b0fd3e..4d28a8da 100644 --- a/kubernetes/apps/home-assistant/zigbee2mqtt-migration-runbook.md +++ b/kubernetes/apps/home-assistant/zigbee2mqtt-migration-runbook.md @@ -144,6 +144,25 @@ empty, so each device must be re-interviewed before it reports fully. Order: Keep `permit_join` **off** (it is, by config) — this is re-interviewing, not re-pairing; nothing leaves and rejoins the network. +> **Item 2 above (battery/sleepy devices) is now automated — do not chase +> those by hand.** The +> `interview-watch` sidecar (`zigbee2mqtt-interview-watch.sh`) fires a forced +> interview whenever a still-`PENDING`/`FAILED` device transmits, which is the +> only moment a sleepy device can complete one. Devices heal as they are used or +> report naturally; several did so with no human interaction at all. Hand-timing +> a wake against a re-interview click is no longer necessary. +> +> Two caveats remain: +> - **Interviewing is only half of it.** Philips devices also need `configure` +> to bind `manuSpecificPhilips` (0xFC00), and that cluster is registered only +> on Z2M's `start` event, so a device interviewed mid-run stays unbound (no +> `action` entities) until the next restart. Restart Z2M once after a batch of +> interviews lands. +> - Devices that never transmit at all are **offline, not un-interviewed** — +> check `lastSeen` in `database.db` (absent + empty `endpoints[].clusters` +> means Z2M has never heard from it) before hunting for a Z2M-side fault. +> Flat batteries look exactly like a failed interview otherwise. + ## Step 6 — Home Assistant side Z2M publishes MQTT discovery under `homeassistant/#`; HA's MQTT integration