From b39a186b7edaf534672e4f5f713bfd0a7b8bdefc Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Thu, 30 Jul 2026 08:00:50 +0200 Subject: [PATCH 1/6] feat(zigbee2mqtt): add re-interview watchdog sidecar MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The ZHA->Z2M cutover adopted the existing Zigbee network, so devices stayed joined but database.db started empty — every device needs an interview before Z2M knows its modelId and can match a converter. Mains routers interviewed immediately. 16 battery end devices did not, and they do not self-heal: zigbee-herdsman only retries an interview on a ZDO device announce, which sleepy devices essentially never emit. They sat PENDING for six days across 38 process restarts, exposing only a generic "Automatically generated definition" (battery/linkquality) instead of action/contact/moisture entities. A device that transmits is awake, which is exactly when an interview can succeed. The sidecar watches Z2M's log for the zh:controller frame-receipt line and fires a forced interview for any device still PENDING/FAILED. Verified in practice: interviews completed in 11-34s, including Xiaomi/Aqara sensors, and two fired with no human interaction at all (a periodic OTA check and an illuminance report). Implementation notes: - Sidecar, not a separate Deployment: it reads Z2M's log file and database.db, and the PVC is RWO/single-writer. Mounted read-only. - Signal comes from the log file, not MQTT: an un-interviewed Xiaomi device publishes nothing at all (its manufacturer-specific attributes cannot be mapped by the auto-generated definition), and bridge/logging carries only info-level messages. - Needs advanced.log_namespaced_levels {zh:controller: debug}. Namespaced rather than global debug on purpose: global adds the unpi:parser byte-level firehose (~90 lines/min measured) vs ~6 lines/min here. - database.db is newline-delimited JSON despite the extension, so the nwk->IEEE lookup needs no jq and is re-read per frame, which keeps it correct across network-address changes and drops devices as they pass. Interviewing is only half the job: Philips devices also need configure to bind manuSpecificPhilips (0xFC00), and that cluster is registered only on Z2M's start event, so a device interviewed mid-run stays unbound until the next restart. One restart after a batch of interviews completes the binds. Both the sidecar and the namespaced log level are temporary — remove them once every device is interviewed. Co-Authored-By: Claude Opus 5 (1M context) --- .../apps/home-assistant/kustomization.yaml | 8 ++ .../home-assistant/zigbee2mqtt-config.yaml | 11 ++ .../zigbee2mqtt-deployment.yaml | 44 ++++++ .../zigbee2mqtt-interview-watch.sh | 126 ++++++++++++++++++ 4 files changed, 189 insertions(+) create mode 100644 kubernetes/apps/home-assistant/zigbee2mqtt-interview-watch.sh diff --git a/kubernetes/apps/home-assistant/kustomization.yaml b/kubernetes/apps/home-assistant/kustomization.yaml index add9c4eb..e774a2dc 100644 --- a/kubernetes/apps/home-assistant/kustomization.yaml +++ b/kubernetes/apps/home-assistant/kustomization.yaml @@ -68,5 +68,13 @@ configMapGenerator: namespace: home-assistant files: - sync.sh=config-sync.sh + # Z2M re-interview watchdog sidecar script. Hash suffix disabled (name stays + # stable), so editing zigbee2mqtt-interview-watch.sh does not auto-roll the + # pod — restart manually with + # `kubectl rollout restart deploy/zigbee2mqtt -n home-assistant`. + - name: zigbee2mqtt-interview-watch-script + namespace: home-assistant + files: + - watch.sh=zigbee2mqtt-interview-watch.sh generatorOptions: disableNameSuffixHash: true diff --git a/kubernetes/apps/home-assistant/zigbee2mqtt-config.yaml b/kubernetes/apps/home-assistant/zigbee2mqtt-config.yaml index 2d5f3275..d06c5fee 100644 --- a/kubernetes/apps/home-assistant/zigbee2mqtt-config.yaml +++ b/kubernetes/apps/home-assistant/zigbee2mqtt-config.yaml @@ -53,6 +53,17 @@ data: advanced: log_level: info + # zh:controller at debug emits, for every received frame, + # Received payload: clusterID=6, address=12468, ... + # which the zigbee2mqtt-interview-watch.sh sidecar uses as its "device is + # awake right now" signal to fire a forced interview (see that script's + # header for why nothing on MQTT works for this). Namespaced rather than a + # global `log_level: debug` on purpose: global debug adds the zh:zstack: + # unpi:parser byte-level firehose (~90 lines/min measured), the namespaced + # form costs ~6 lines/min. Drop this once every device is interviewed — + # the sidecar is then a no-op and can go too. + log_namespaced_levels: + zh:controller: debug # EXPLICIT network params matching the live coordinator, so zigbee-herdsman # ADOPTS (resumes) the existing network instead of forming a new one — # every device stays joined, no re-pairing. herdsman compares these against diff --git a/kubernetes/apps/home-assistant/zigbee2mqtt-deployment.yaml b/kubernetes/apps/home-assistant/zigbee2mqtt-deployment.yaml index 0c7c5a4c..f04e341b 100644 --- a/kubernetes/apps/home-assistant/zigbee2mqtt-deployment.yaml +++ b/kubernetes/apps/home-assistant/zigbee2mqtt-deployment.yaml @@ -136,6 +136,47 @@ spec: memory: 128Mi limits: memory: 512Mi + # Fires a forced interview whenever a still-un-interviewed device + # transmits — i.e. at the only moment a sleepy battery device can + # actually complete one. Z2M itself only retries interviews on a ZDO + # announce, which these devices essentially never emit, so without this + # they stay "Not supported" indefinitely (16 of them did, for six days + # across 38 restarts). Full rationale in the script header. + # + # A SIDECAR rather than its own Deployment: it reads both Z2M's log file + # and database.db, and the PVC is RWO/single-writer, so it has to share + # this pod. It only ever reads the PVC. + # + # Depends on advanced.log_namespaced_levels {zh:controller: debug} in + # zigbee2mqtt-config.yaml. Remove both once every device is interviewed. + - name: interview-watch + image: alpine:3.21 + command: + - sh + - -c + - | + set -eu + apk add --no-cache mosquitto-clients >/dev/null + exec sh /script/watch.sh + env: + - name: MQTT_PASSWORD + valueFrom: + secretKeyRef: + name: mosquitto-zigbee2mqtt-credentials + key: password + volumeMounts: + - name: data + mountPath: /app/data + readOnly: true + - name: interview-watch-script + mountPath: /script + readOnly: true + resources: + requests: + cpu: 10m + memory: 16Mi + limits: + memory: 64Mi volumes: - name: data persistentVolumeClaim: @@ -146,3 +187,6 @@ spec: - name: coordinator-backup secret: secretName: zigbee2mqtt-coordinator-backup + - name: interview-watch-script + configMap: + name: zigbee2mqtt-interview-watch-script diff --git a/kubernetes/apps/home-assistant/zigbee2mqtt-interview-watch.sh b/kubernetes/apps/home-assistant/zigbee2mqtt-interview-watch.sh new file mode 100644 index 00000000..44a8a140 --- /dev/null +++ b/kubernetes/apps/home-assistant/zigbee2mqtt-interview-watch.sh @@ -0,0 +1,126 @@ +#!/bin/sh +# Re-interview watchdog for Zigbee2MQTT. +# +# PROBLEM +# The ZHA->Z2M cutover (2026-07-23) adopted the existing Zigbee network, so +# every device stayed joined but Z2M's database.db started empty. Each device +# must therefore be *interviewed* before Z2M knows its modelId and can match a +# converter; until then it logs ": Not supported (EndDevice)" and exposes +# only a generic "Automatically generated definition" (battery/linkquality), +# not the real entities (action / contact / moisture). +# +# Mains-powered routers interviewed immediately (always awake). Battery end +# devices did not, and they do NOT recover on their own, because: +# +# - zigbee-herdsman only retries an interview on a ZDO device *announce* +# (controller.js: interviewState Pending|Failed -> device.interview()), and +# sleepy battery devices almost never emit one spontaneously. +# - Z2M's Configure extension retries on any traffic (onLastSeenChanged) but +# is capped at 3 attempts per process (in-memory Map, reset on restart). +# +# So 16 devices sat PENDING for six days across 38 process restarts. +# +# APPROACH +# A device that transmits *is awake right now*, which is exactly when an +# interview can succeed. This watches for that moment and fires a forced +# interview request. Confirmed in practice: interviews completed in 11-34s when +# triggered this way, including Xiaomi/Aqara sensors whose wake window is short. +# +# WHY THE LOG FILE AND NOT MQTT +# An un-interviewed Xiaomi device publishes NOTHING to MQTT (its +# manufacturer-specific attributes cannot be mapped by the auto-generated +# definition), so MQTT state topics are not a usable signal for exactly the +# devices that need help. `zigbee2mqtt/bridge/logging` carries only `info`. +# The one reliable signal is the zh:controller debug line +# +# Received payload: clusterID=6, address=12468, ... +# +# which requires advanced.log_namespaced_levels {zh:controller: debug} — set in +# zigbee2mqtt-config.yaml. Root log_level stays `info`; enabling debug globally +# adds ~90 lines/min of UNPI frame noise, the namespaced form ~6 lines/min. +# +# This runs as a sidecar (not a separate Deployment) because it needs the same +# RWO PVC as Z2M for both the log file and database.db. +# +# NOTE: interviewing is only half the job. Philips devices additionally need +# `configure` to bind manuSpecificPhilips (0xFC00), and that cluster is only +# registered on Z2M's `start` event (deviceAddCustomCluster onEvent), so a +# device interviewed mid-run cannot be configured until the next restart. One +# Z2M restart after a batch of interviews completes the binds. + +set -u + +DATA=/app/data +DB="$DATA/database.db" +MQTT_HOST=${MQTT_HOST:-mosquitto} +MQTT_USER=${MQTT_USER:-zigbee2mqtt} +COOLDOWN=${COOLDOWN:-300} # min seconds between interview attempts per device +STATEDIR=/tmp/interview-cooldown + +mkdir -p "$STATEDIR" + +log() { printf '%s interview-watch: %s\n' "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" "$*"; } + +# Resolve a network address to an IEEE, but ONLY for devices that still need an +# interview. database.db is newline-delimited JSON (one device per line) despite +# the .db extension, so a per-line grep+sed is enough and needs no jq. Read +# fresh on every frame so devices drop out the moment they succeed, and so +# changed network addresses are picked up automatically. +ieee_needing_interview() { + grep -E '"interviewState":"(PENDING|FAILED)"' "$DB" 2>/dev/null | + sed -n 's/.*"ieeeAddr":"\([^"]*\)".*"nwkAddr":\([0-9]*\).*/\2 \1/p' | + awk -v n="$1" '$1 == n { print $2; exit }' +} + +# Z2M creates a fresh log directory per start, so resolve the newest one. On a +# cold pod start Z2M may not have created it yet. +newest_log() { + d=$(ls -t "$DATA/log" 2>/dev/null | head -1) + [ -n "${d:-}" ] && printf '%s/log/%s/log.log' "$DATA" "$d" +} + +log "starting; waiting for Z2M log file" +while :; do + LOGFILE=$(newest_log) + [ -n "${LOGFILE:-}" ] && [ -f "$LOGFILE" ] && break + sleep 5 +done +log "following $LOGFILE" +log "$(grep -cE '"interviewState":"(PENDING|FAILED)"' "$DB" 2>/dev/null || echo 0) device(s) currently need an interview" + +# tail -F (--follow=name --retry) so a log rotation does not silently end the +# stream. +tail -F "$LOGFILE" 2>/dev/null | +grep --line-buffered -oE 'Received payload: clusterID=[0-9]+, address=[0-9]+' | +while IFS= read -r line; do + nwk=${line##*address=} + cluster=${line#*clusterID=} + cluster=${cluster%%,*} + + ieee=$(ieee_needing_interview "$nwk") + [ -z "${ieee:-}" ] && continue # already interviewed, or not a device we track + + stamp="$STATEDIR/$ieee" + now=$(date +%s) + if [ -f "$stamp" ]; then + last=$(cat "$stamp" 2>/dev/null || echo 0) + # Suppress the interview's OWN traffic, which would otherwise re-trigger + # this on every attribute read it performs. + [ $((now - last)) -lt "$COOLDOWN" ] && continue + fi + printf '%s' "$now" > "$stamp" + + log "frame from $ieee (nwk=$nwk cluster=$cluster) while awake -> requesting interview" + if mosquitto_pub -h "$MQTT_HOST" -u "$MQTT_USER" -P "$MQTT_PASSWORD" \ + -t 'zigbee2mqtt/bridge/request/device/interview' \ + -m "{\"id\":\"$ieee\"}" 2>/dev/null; then + log " interview requested for $ieee" + else + log " WARNING: mosquitto_pub failed for $ieee" + fi +done + +# tail/grep ended (log rotated away or Z2M restarted); exit non-zero so the +# kubelet restarts the sidecar and it re-resolves the current log file. +log "log stream ended; exiting for restart" +exit 1 -- 2.54.0 From ead4aa3f0174a44738a470135471c44d1e599576 Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Thu, 30 Jul 2026 08:10:50 +0200 Subject: [PATCH 2/6] fix(zigbee2mqtt): address review on interview watchdog MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review points from review-bot on #441. Stay resident instead of exiting on log-stream end. Previously the sidecar exit 1'd so the kubelet would restart it and re-resolve the log file, but that re-ran `apk add mosquitto-clients` over the network every time and, worse, incurred kubelet crash-loop backoff — which climbs to 5 minutes unless the container stays up >=10 min. Z2M restarts and log rotations land closer together than that (notably during the "interview a batch, then restart Z2M once" workflow), so backoff would have kicked in exactly when a device's short wake window needs a prompt reaction. The log file is now re-resolved in an outer loop, which also drops the post-startup network dependency. Chosen over baking a custom image because it removes the restart entirely rather than making restarts cheaper. Match ieeeAddr/nwkAddr independently rather than with one combined sed pattern. JSON does not guarantee key order, so the old pattern's assumption that "ieeeAddr" precedes "nwkAddr" held only by virtue of herdsman's current serialization. Had that changed, the regex would capture nothing and a device would silently never be picked up — no error, no log. Now extracted per-field with awk match(), with the rationale recorded in a comment. Verified against the live database.db under busybox awk (the sidecar runs alpine): resolves both still-pending devices, and correctly returns nothing for one that has since been interviewed. Flag Step 5 of the migration runbook as automated, so operators do not hand-chase devices the sidecar already handles. Includes the two caveats that cost real debugging time: interviewing does not bind manuSpecificPhilips (needs one Z2M restart), and a device that never transmits is offline rather than un-interviewed — check lastSeen before hunting a Z2M-side fault, because a flat battery presents identically. Not changed: mosquitto_pub takes a password only via argv (-P, or an -L URL) — it has no --pw-file or env option, verified against the client in the cluster — so the 🟢 nit cannot be resolved with this client. Not a cross-container exposure, as the pod does not set shareProcessNamespace; noted in a comment at the call site. Co-Authored-By: Claude Opus 5 (1M context) --- .../zigbee2mqtt-interview-watch.sh | 121 +++++++++++------- .../zigbee2mqtt-migration-runbook.md | 18 +++ 2 files changed, 93 insertions(+), 46 deletions(-) diff --git a/kubernetes/apps/home-assistant/zigbee2mqtt-interview-watch.sh b/kubernetes/apps/home-assistant/zigbee2mqtt-interview-watch.sh index 44a8a140..efcac6d9 100644 --- a/kubernetes/apps/home-assistant/zigbee2mqtt-interview-watch.sh +++ b/kubernetes/apps/home-assistant/zigbee2mqtt-interview-watch.sh @@ -63,13 +63,26 @@ log() { printf '%s interview-watch: %s\n' "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" "$* # Resolve a network address to an IEEE, but ONLY for devices that still need an # interview. database.db is newline-delimited JSON (one device per line) despite -# the .db extension, so a per-line grep+sed is enough and needs no jq. Read -# fresh on every frame so devices drop out the moment they succeed, and so -# changed network addresses are picked up automatically. +# the .db extension, so a per-line scan is enough and needs no jq. Read fresh on +# every frame so devices drop out the moment they succeed, and so changed +# network addresses are picked up automatically. +# +# Each field is matched INDEPENDENTLY rather than with one combined pattern: +# JSON guarantees no key order, so a single regex expecting "ieeeAddr" before +# "nwkAddr" would silently capture nothing (no error, no log — a device just +# never gets picked up) if herdsman's serialization order ever changed. ieee_needing_interview() { - grep -E '"interviewState":"(PENDING|FAILED)"' "$DB" 2>/dev/null | - sed -n 's/.*"ieeeAddr":"\([^"]*\)".*"nwkAddr":\([0-9]*\).*/\2 \1/p' | - awk -v n="$1" '$1 == n { print $2; exit }' + awk -v want="$1" ' + /"interviewState":"(PENDING|FAILED)"/ { + ieee = ""; nwk = "" + if (match($0, /"ieeeAddr":"[^"]*"/)) { + s = substr($0, RSTART, RLENGTH); sub(/^"ieeeAddr":"/, "", s); sub(/"$/, "", s); ieee = s + } + if (match($0, /"nwkAddr":[0-9]+/)) { + s = substr($0, RSTART, RLENGTH); sub(/^"nwkAddr":/, "", s); nwk = s + } + if (ieee != "" && nwk == want) { print ieee; exit } + }' "$DB" 2>/dev/null } # Z2M creates a fresh log directory per start, so resolve the newest one. On a @@ -79,48 +92,64 @@ newest_log() { [ -n "${d:-}" ] && printf '%s/log/%s/log.log' "$DATA" "$d" } -log "starting; waiting for Z2M log file" -while :; do - LOGFILE=$(newest_log) - [ -n "${LOGFILE:-}" ] && [ -f "$LOGFILE" ] && break - sleep 5 -done -log "following $LOGFILE" +log "starting" log "$(grep -cE '"interviewState":"(PENDING|FAILED)"' "$DB" 2>/dev/null || echo 0) device(s) currently need an interview" -# tail -F (--follow=name --retry) so a log rotation does not silently end the -# stream. -tail -F "$LOGFILE" 2>/dev/null | -grep --line-buffered -oE 'Received payload: clusterID=[0-9]+, address=[0-9]+' | -while IFS= read -r line; do - nwk=${line##*address=} - cluster=${line#*clusterID=} - cluster=${cluster%%,*} - - ieee=$(ieee_needing_interview "$nwk") - [ -z "${ieee:-}" ] && continue # already interviewed, or not a device we track - - stamp="$STATEDIR/$ieee" - now=$(date +%s) - if [ -f "$stamp" ]; then - last=$(cat "$stamp" 2>/dev/null || echo 0) - # Suppress the interview's OWN traffic, which would otherwise re-trigger - # this on every attribute read it performs. - [ $((now - last)) -lt "$COOLDOWN" ] && continue +# Re-resolve the log file IN-PROCESS rather than exiting and letting the kubelet +# restart us. Exiting would mean (a) re-running `apk add mosquitto-clients` over +# the network on every restart, and (b) kubelet crash-loop backoff, which climbs +# to 5 minutes unless the container has stayed up >=10 min. Z2M restarts and log +# rotations can easily land closer together than that (notably during the +# "interview a batch, then restart Z2M once" workflow), so backoff would kick in +# exactly when a device's short wake window needs a prompt reaction. Staying +# resident keeps the reaction time flat and the container network-independent +# after startup. +while :; do + LOGFILE=$(newest_log) + if [ -z "${LOGFILE:-}" ] || [ ! -f "$LOGFILE" ]; then + sleep 5 + continue fi - printf '%s' "$now" > "$stamp" - log "frame from $ieee (nwk=$nwk cluster=$cluster) while awake -> requesting interview" - if mosquitto_pub -h "$MQTT_HOST" -u "$MQTT_USER" -P "$MQTT_PASSWORD" \ - -t 'zigbee2mqtt/bridge/request/device/interview' \ - -m "{\"id\":\"$ieee\"}" 2>/dev/null; then - log " interview requested for $ieee" - else - log " WARNING: mosquitto_pub failed for $ieee" - fi + log "following $LOGFILE" + + # tail -F (--follow=name --retry) rides out an in-place rotation; the outer + # loop handles Z2M starting a whole new log directory. + tail -F "$LOGFILE" 2>/dev/null | + grep --line-buffered -oE 'Received payload: clusterID=[0-9]+, address=[0-9]+' | + while IFS= read -r line; do + nwk=${line##*address=} + cluster=${line#*clusterID=} + cluster=${cluster%%,*} + + ieee=$(ieee_needing_interview "$nwk") + [ -z "${ieee:-}" ] && continue # already interviewed, or not a device we track + + stamp="$STATEDIR/$ieee" + now=$(date +%s) + if [ -f "$stamp" ]; then + last=$(cat "$stamp" 2>/dev/null || echo 0) + # Suppress the interview's OWN traffic, which would otherwise re-trigger + # this on every attribute read it performs. + [ $((now - last)) -lt "$COOLDOWN" ] && continue + fi + printf '%s' "$now" > "$stamp" + + log "frame from $ieee (nwk=$nwk cluster=$cluster) while awake -> requesting interview" + # NOTE: mosquitto_pub can only take a password via argv (-P, or an -L URL); + # it has no --pw-file or env option, so this cannot be avoided with this + # client. Not a cross-container exposure: the pod does not set + # shareProcessNamespace, so /proc is not shared with the Z2M container. + if mosquitto_pub -h "$MQTT_HOST" -u "$MQTT_USER" -P "$MQTT_PASSWORD" \ + -t 'zigbee2mqtt/bridge/request/device/interview' \ + -m "{\"id\":\"$ieee\"}" 2>/dev/null; then + log " interview requested for $ieee" + else + log " WARNING: mosquitto_pub failed for $ieee" + fi + done + + # Stream ended: Z2M restarted into a new log directory, or the file vanished. + log "log stream ended; re-resolving current log file" + sleep 5 done - -# tail/grep ended (log rotated away or Z2M restarted); exit non-zero so the -# kubelet restarts the sidecar and it re-resolves the current log file. -log "log stream ended; exiting for restart" -exit 1 diff --git a/kubernetes/apps/home-assistant/zigbee2mqtt-migration-runbook.md b/kubernetes/apps/home-assistant/zigbee2mqtt-migration-runbook.md index a4b0fd3e..c987fc32 100644 --- a/kubernetes/apps/home-assistant/zigbee2mqtt-migration-runbook.md +++ b/kubernetes/apps/home-assistant/zigbee2mqtt-migration-runbook.md @@ -144,6 +144,24 @@ empty, so each device must be re-interviewed before it reports fully. Order: Keep `permit_join` **off** (it is, by config) — this is re-interviewing, not re-pairing; nothing leaves and rejoins the network. +> **Step 2 is now automated — do not chase these by hand.** The +> `interview-watch` sidecar (`zigbee2mqtt-interview-watch.sh`) fires a forced +> interview whenever a still-`PENDING`/`FAILED` device transmits, which is the +> only moment a sleepy device can complete one. Devices heal as they are used or +> report naturally; several did so with no human interaction at all. Hand-timing +> a wake against a re-interview click is no longer necessary. +> +> Two caveats remain: +> - **Interviewing is only half of it.** Philips devices also need `configure` +> to bind `manuSpecificPhilips` (0xFC00), and that cluster is registered only +> on Z2M's `start` event, so a device interviewed mid-run stays unbound (no +> `action` entities) until the next restart. Restart Z2M once after a batch of +> interviews lands. +> - Devices that never transmit at all are **offline, not un-interviewed** — +> check `lastSeen` in `database.db` (absent + empty `endpoints[].clusters` +> means Z2M has never heard from it) before hunting for a Z2M-side fault. +> Flat batteries look exactly like a failed interview otherwise. + ## Step 6 — Home Assistant side Z2M publishes MQTT discovery under `homeassistant/#`; HA's MQTT integration -- 2.54.0 From 3961640130b1372828c44059ef865e8fd84d0d61 Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Thu, 30 Jul 2026 08:22:40 +0200 Subject: [PATCH 3/6] fix(zigbee2mqtt): drop BusyBox-incompatible grep from watchdog MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Second review round on #441. The first point was a real blocking defect that would have shipped silently. BusyBox grep does not support --line-buffered. Confirmed by running it: $ grep --line-buffered -oE '...' <<< 'Received payload: ...' grep: unrecognized option: line-buffered GREP_EXIT=2 The sidecar runs alpine:3.21 and `apk add mosquitto-clients` does not pull GNU grep, so the `tail -F | grep --line-buffered | while read` pipe would have produced zero lines forever while the container still logged "following " and appeared healthy — a silent no-op of the entire feature, undetectable from the diff. Fixed by removing the grep stage rather than installing GNU grep: match in the read loop with a `case` glob and extract the fields with parameter expansion. `tail` writes per line and `read` consumes per line, so there is no buffering question and no extra package. Added a numeric guard on the extracted address so a future log-format change cannot feed garbage downstream. Also from this round: - Write the cooldown stamp only on a SUCCESSFUL publish. Stamping before the result was known burned a device's wake window whenever mosquitto_pub failed, then refused to retry for the full 300s despite no interview having been requested. - Derive the request topic from BASE_TOPIC (env, defaulting to zigbee2mqtt) instead of hardcoding it, so it cannot silently drift from mqtt.base_topic. Noted that the MQTT ACL is scoped to zigbee2mqtt/# and would need changing too. - Harden the sidecar with allowPrivilegeEscalation: false and capabilities drop ALL. Deliberately no runAsNonRoot: apk add needs root, and Z2M writes database.db and the log dir as root, so a non-root uid could not read either input. - Runbook callout said "Step 2" where it meant item 2 within Step 5. Verified end-to-end in an alpine:3.21 pod against this script (the earlier "verified live" evidence came from the session prototype, which resolved friendly names and is not this code — see PR discussion): counts only PENDING devices, fires on a frame from a PENDING device with mosquitto_pub returning success, ignores a frame from an already-SUCCESSFUL device, and survives a malformed address without firing or crashing. Co-Authored-By: Claude Opus 5 (1M context) --- .../zigbee2mqtt-deployment.yaml | 11 +++++ .../zigbee2mqtt-interview-watch.sh | 44 ++++++++++++++++--- .../zigbee2mqtt-migration-runbook.md | 3 +- 3 files changed, 50 insertions(+), 8 deletions(-) diff --git a/kubernetes/apps/home-assistant/zigbee2mqtt-deployment.yaml b/kubernetes/apps/home-assistant/zigbee2mqtt-deployment.yaml index f04e341b..a17e011d 100644 --- a/kubernetes/apps/home-assistant/zigbee2mqtt-deployment.yaml +++ b/kubernetes/apps/home-assistant/zigbee2mqtt-deployment.yaml @@ -164,6 +164,17 @@ spec: secretKeyRef: name: mosquitto-zigbee2mqtt-credentials key: password + # Must track mqtt.base_topic in zigbee2mqtt-config.yaml. + - name: BASE_TOPIC + value: zigbee2mqtt + # No runAsNonRoot: `apk add` needs root, and Z2M writes /app/data as + # root (database.db and the log dir are root-owned 0644/0755), so a + # non-root uid could not read either input. Dropping privilege + # escalation and all capabilities is still worth having. + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: [ALL] volumeMounts: - name: data mountPath: /app/data diff --git a/kubernetes/apps/home-assistant/zigbee2mqtt-interview-watch.sh b/kubernetes/apps/home-assistant/zigbee2mqtt-interview-watch.sh index efcac6d9..28b572dc 100644 --- a/kubernetes/apps/home-assistant/zigbee2mqtt-interview-watch.sh +++ b/kubernetes/apps/home-assistant/zigbee2mqtt-interview-watch.sh @@ -54,6 +54,11 @@ DATA=/app/data DB="$DATA/database.db" MQTT_HOST=${MQTT_HOST:-mosquitto} MQTT_USER=${MQTT_USER:-zigbee2mqtt} +# Must track mqtt.base_topic in zigbee2mqtt-config.yaml. Overridable via env so +# the two can be kept in step from the Deployment rather than by editing this +# script; the MQTT ACL for this user is scoped to `zigbee2mqtt/#`, so a change +# here needs a matching ACL change too. +BASE_TOPIC=${BASE_TOPIC:-zigbee2mqtt} COOLDOWN=${COOLDOWN:-300} # min seconds between interview attempts per device STATEDIR=/tmp/interview-cooldown @@ -115,12 +120,33 @@ while :; do # tail -F (--follow=name --retry) rides out an in-place rotation; the outer # loop handles Z2M starting a whole new log directory. + # + # Matching happens in the read loop rather than via a `grep` stage on purpose. + # This runs on alpine:3.21, whose `grep` is the BusyBox applet, and BusyBox + # grep does NOT support --line-buffered: + # grep: unrecognized option: line-buffered (exit 2) + # so a `tail -F | grep --line-buffered | while read` pipe yields ZERO lines + # forever while the sidecar still logs "following " and looks healthy — + # a silent no-op of the whole feature. Dropping the stage avoids both that and + # any block-buffering question, since `tail` writes per line and `read` + # consumes per line, with no extra package to install. tail -F "$LOGFILE" 2>/dev/null | - grep --line-buffered -oE 'Received payload: clusterID=[0-9]+, address=[0-9]+' | while IFS= read -r line; do - nwk=${line##*address=} - cluster=${line#*clusterID=} - cluster=${cluster%%,*} + case "$line" in + *"Received payload: clusterID="*) ;; + *) continue ;; + esac + + # ... clusterID=6, address=12468, groupID=0, endpoint=1, ... + rest=${line#*"Received payload: clusterID="} + cluster=${rest%%,*} + rest=${rest#*address=} + nwk=${rest%%,*} + + # Guard against a format change silently yielding garbage. + case "$nwk" in + ''|*[!0-9]*) continue ;; + esac ieee=$(ieee_needing_interview "$nwk") [ -z "${ieee:-}" ] && continue # already interviewed, or not a device we track @@ -133,7 +159,6 @@ while :; do # this on every attribute read it performs. [ $((now - last)) -lt "$COOLDOWN" ] && continue fi - printf '%s' "$now" > "$stamp" log "frame from $ieee (nwk=$nwk cluster=$cluster) while awake -> requesting interview" # NOTE: mosquitto_pub can only take a password via argv (-P, or an -L URL); @@ -141,11 +166,16 @@ while :; do # client. Not a cross-container exposure: the pod does not set # shareProcessNamespace, so /proc is not shared with the Z2M container. if mosquitto_pub -h "$MQTT_HOST" -u "$MQTT_USER" -P "$MQTT_PASSWORD" \ - -t 'zigbee2mqtt/bridge/request/device/interview' \ + -t "$BASE_TOPIC/bridge/request/device/interview" \ -m "{\"id\":\"$ieee\"}" 2>/dev/null; then + # Stamp only on a SUCCESSFUL publish. Stamping unconditionally would burn + # the device's wake window on a failed publish (broker hiccup, DNS blip) + # and then refuse to retry for the full cooldown, despite no interview + # ever having been requested. + printf '%s' "$now" > "$stamp" log " interview requested for $ieee" else - log " WARNING: mosquitto_pub failed for $ieee" + log " WARNING: mosquitto_pub failed for $ieee; not starting cooldown so the next frame retries" fi done diff --git a/kubernetes/apps/home-assistant/zigbee2mqtt-migration-runbook.md b/kubernetes/apps/home-assistant/zigbee2mqtt-migration-runbook.md index c987fc32..4d28a8da 100644 --- a/kubernetes/apps/home-assistant/zigbee2mqtt-migration-runbook.md +++ b/kubernetes/apps/home-assistant/zigbee2mqtt-migration-runbook.md @@ -144,7 +144,8 @@ empty, so each device must be re-interviewed before it reports fully. Order: Keep `permit_join` **off** (it is, by config) — this is re-interviewing, not re-pairing; nothing leaves and rejoins the network. -> **Step 2 is now automated — do not chase these by hand.** The +> **Item 2 above (battery/sleepy devices) is now automated — do not chase +> those by hand.** The > `interview-watch` sidecar (`zigbee2mqtt-interview-watch.sh`) fires a forced > interview whenever a still-`PENDING`/`FAILED` device transmits, which is the > only moment a sleepy device can complete one. Devices heal as they are used or -- 2.54.0 From 834ded5e9de496f23d923811019292503cc5c57a Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Thu, 30 Jul 2026 08:37:35 +0200 Subject: [PATCH 4/6] refactor(zigbee2mqtt): drop apk add from watchdog sidecar MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Owner directive: `apk add` at container start is not acceptable. Replace alpine:3.21 + `apk add mosquitto-clients` with the broker image eclipse-mosquitto:2.0.22, used purely for its bundled mosquitto_pub. No packages are installed at runtime, so the sidecar no longer depends on network or package-index availability at start. Every tool the script needs was verified present in that exact image rather than assumed: sh, tail (including -F, which BusyBox does support), awk with match()/RSTART/RLENGTH, cat, date, mkdir, mosquitto_pub. The awk extraction was re-run there and resolves a PENDING device, skips a SUCCESSFUL one, and skips an unknown address. Pinned WITHOUT a Flux $imagepolicy marker on purpose: this sidecar is temporary and must not drag the broker's image-automation policy with it. Now genuinely non-root (1883/1883, the image's own mosquitto user). The previous commit justified running as root because `apk add` needed it; with that gone the only remaining requirement is reading /app/data, and those paths are world-readable (dirs 0755, files 0644 — verified against the live PVC), so runAsNonRoot is viable. Cooldown stamps moved to an explicit emptyDir at /state rather than the image's /tmp, so they stay writable as non-root without relying on the container filesystem. Re-verified end-to-end in this image as uid 1883 with no package install: running as: uid=1883(mosquitto) gid=1883(mosquitto) interview-watch: 1 device(s) currently need an interview interview-watch: following /app/data/log/.../log.log interview-watch: frame from 0xdeadbeefdeadbeef (nwk=65001 cluster=6) ... interview-watch: interview requested for 0xdeadbeefdeadbeef with the negative paths also holding: a frame from an already-SUCCESSFUL device is ignored, and a malformed address neither fires nor crashes. NOTE: the pre-existing seed-data init container still runs `apk add --no-cache jq` (line 49). Untouched here as it is out of scope for this PR, but it violates the same directive and is raised separately. Co-Authored-By: Claude Opus 5 (1M context) --- .../zigbee2mqtt-deployment.yaml | 38 +++++++++++++------ .../zigbee2mqtt-interview-watch.sh | 14 +++---- 2 files changed, 32 insertions(+), 20 deletions(-) diff --git a/kubernetes/apps/home-assistant/zigbee2mqtt-deployment.yaml b/kubernetes/apps/home-assistant/zigbee2mqtt-deployment.yaml index a17e011d..7cc4561b 100644 --- a/kubernetes/apps/home-assistant/zigbee2mqtt-deployment.yaml +++ b/kubernetes/apps/home-assistant/zigbee2mqtt-deployment.yaml @@ -150,14 +150,16 @@ spec: # Depends on advanced.log_namespaced_levels {zh:controller: debug} in # zigbee2mqtt-config.yaml. Remove both once every device is interviewed. - name: interview-watch - image: alpine:3.21 - command: - - sh - - -c - - | - set -eu - apk add --no-cache mosquitto-clients >/dev/null - exec sh /script/watch.sh + # Reuses the broker image purely for its bundled mosquitto_pub, so + # nothing is installed at runtime — no `apk add`, hence no network + # dependency at start and no package-index drift. Everything the script + # needs was verified present in this exact image: sh, tail (incl. -F), + # awk (BusyBox match()/RSTART/RLENGTH), cat, date, mkdir, + # mosquitto_pub. Pinned deliberately WITHOUT a Flux $imagepolicy + # marker: this sidecar is temporary, and it must not drag the broker's + # image-automation policy along with it. + image: eclipse-mosquitto:2.0.22 + command: [sh, /script/watch.sh] env: - name: MQTT_PASSWORD valueFrom: @@ -167,11 +169,16 @@ spec: # Must track mqtt.base_topic in zigbee2mqtt-config.yaml. - name: BASE_TOPIC value: zigbee2mqtt - # No runAsNonRoot: `apk add` needs root, and Z2M writes /app/data as - # root (database.db and the log dir are root-owned 0644/0755), so a - # non-root uid could not read either input. Dropping privilege - # escalation and all capabilities is still worth having. + - name: STATEDIR + value: /state + # Runs as the image's own mosquitto user (1883). Viable because + # nothing is installed at runtime and everything it reads under + # /app/data is world-readable (dirs 0755, files 0644 — verified), so + # root is not needed to tail the log or read database.db. securityContext: + runAsNonRoot: true + runAsUser: 1883 + runAsGroup: 1883 allowPrivilegeEscalation: false capabilities: drop: [ALL] @@ -182,6 +189,11 @@ spec: - name: interview-watch-script mountPath: /script readOnly: true + # Cooldown stamps. An explicit emptyDir rather than the image's + # /tmp, so it stays writable as non-root without depending on the + # container filesystem. + - name: interview-watch-state + mountPath: /state resources: requests: cpu: 10m @@ -201,3 +213,5 @@ spec: - name: interview-watch-script configMap: name: zigbee2mqtt-interview-watch-script + - name: interview-watch-state + emptyDir: {} diff --git a/kubernetes/apps/home-assistant/zigbee2mqtt-interview-watch.sh b/kubernetes/apps/home-assistant/zigbee2mqtt-interview-watch.sh index 28b572dc..198f6af4 100644 --- a/kubernetes/apps/home-assistant/zigbee2mqtt-interview-watch.sh +++ b/kubernetes/apps/home-assistant/zigbee2mqtt-interview-watch.sh @@ -60,7 +60,7 @@ MQTT_USER=${MQTT_USER:-zigbee2mqtt} # here needs a matching ACL change too. BASE_TOPIC=${BASE_TOPIC:-zigbee2mqtt} COOLDOWN=${COOLDOWN:-300} # min seconds between interview attempts per device -STATEDIR=/tmp/interview-cooldown +STATEDIR=${STATEDIR:-/state} # emptyDir; writable as non-root mkdir -p "$STATEDIR" @@ -101,14 +101,12 @@ log "starting" log "$(grep -cE '"interviewState":"(PENDING|FAILED)"' "$DB" 2>/dev/null || echo 0) device(s) currently need an interview" # Re-resolve the log file IN-PROCESS rather than exiting and letting the kubelet -# restart us. Exiting would mean (a) re-running `apk add mosquitto-clients` over -# the network on every restart, and (b) kubelet crash-loop backoff, which climbs -# to 5 minutes unless the container has stayed up >=10 min. Z2M restarts and log +# restart us. Exiting would incur kubelet crash-loop backoff, which climbs to +# 5 minutes unless the container has stayed up >=10 min. Z2M restarts and log # rotations can easily land closer together than that (notably during the # "interview a batch, then restart Z2M once" workflow), so backoff would kick in # exactly when a device's short wake window needs a prompt reaction. Staying -# resident keeps the reaction time flat and the container network-independent -# after startup. +# resident keeps the reaction time flat. while :; do LOGFILE=$(newest_log) if [ -z "${LOGFILE:-}" ] || [ ! -f "$LOGFILE" ]; then @@ -122,8 +120,8 @@ while :; do # loop handles Z2M starting a whole new log directory. # # Matching happens in the read loop rather than via a `grep` stage on purpose. - # This runs on alpine:3.21, whose `grep` is the BusyBox applet, and BusyBox - # grep does NOT support --line-buffered: + # This runs on an Alpine-based image whose `grep` is the BusyBox applet, and + # BusyBox grep does NOT support --line-buffered: # grep: unrecognized option: line-buffered (exit 2) # so a `tail -F | grep --line-buffered | while read` pipe yields ZERO lines # forever while the sidecar still logs "following " and looks healthy — -- 2.54.0 From ee15cdeb13905b9432bc3d7e0292a0cfa28fcece Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Thu, 30 Jul 2026 09:03:10 +0200 Subject: [PATCH 5/6] fix(zigbee2mqtt): follow a stable log path across Z2M restarts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses the blocking review point on #441, which was correct. `tail -F` follows a PATHNAME: it re-opens that name when the inode behind it changes, but it can never notice that Z2M has started writing to a different directory. Z2M creates log//log.log fresh on every start, so tailing the newest directory breaks as soon as Z2M restarts — the old file merely goes quiet, so the pipe never ends, the outer loop never re-resolves, "following " still shows the stale path, and the sidecar looks healthy while never seeing another frame. Worse than the review assumed: this is not limited to the planned "interview a batch, then restart Z2M once" step. The Z2M *container* has restarted 38 times on its own from DNS flaps, each time into a new log directory, while the pod (and so the sidecar) kept running. Fixed at the source rather than with a polling watchdog: set advanced.log_symlink_current: true so Z2M maintains log/current -> the active directory, and follow that stable path. Verified that BusyBox `tail -F` in eclipse-mosquitto:2.0.22 does follow a symlink repointed to a new target mid-follow. No fallback to the newest directory, deliberately: falling back could silently tail a frozen file, which is the failure being fixed. A missing symlink is instead reported loudly and repeatedly while waiting. Also from this round: - Short-circuit while nothing needs interviewing. The namespaced debug log covers all of zh:controller, so every device on the network produces frames; without this the script forked awk over database.db for each of ~30 devices' traffic forever. Recounted at most once a minute, so a device regressing to PENDING is still picked up. - Bound mosquitto_pub with `timeout`, so a hanging TCP connect cannot stall the single-threaded read loop. - Wire MQTT_HOST and MQTT_USER explicitly from the Deployment, like BASE_TOPIC, instead of relying on the script's defaults. - imagePullPolicy: IfNotPresent, matching the sibling container. Two bugs found by testing this, not by inspection: - `mosquitto_pub -W` does not exist (it is a mosquitto_sub flag). Adding it made every publish fail on a usage error. Removed; `timeout` alone bounds the call. Same class as the earlier `grep --line-buffered` mistake, and again only visible by running it. - `grep -c` prints "0" AND exits 1 when nothing matches, so `grep -c ... || echo 0` emitted two lines ("0\n0") and every `-eq` test died with "bad number". This surfaced only once no devices needed an interview — the steady end state this feature exists to reach. Verified in eclipse-mosquitto:2.0.22 as uid 1883, simulating Z2M's layout and a restart (new directory + repointed symlink): frame before restart -> interview requested symlink repointed to a new directory frame after restart -> interview requested <- the reviewed failure case zero-pending fixture -> "no devices need an interview; idling", no errors Co-Authored-By: Claude Opus 5 (1M context) --- .../home-assistant/zigbee2mqtt-config.yaml | 8 ++ .../zigbee2mqtt-deployment.yaml | 10 +- .../zigbee2mqtt-interview-watch.sh | 98 ++++++++++++++----- 3 files changed, 92 insertions(+), 24 deletions(-) diff --git a/kubernetes/apps/home-assistant/zigbee2mqtt-config.yaml b/kubernetes/apps/home-assistant/zigbee2mqtt-config.yaml index d06c5fee..41460382 100644 --- a/kubernetes/apps/home-assistant/zigbee2mqtt-config.yaml +++ b/kubernetes/apps/home-assistant/zigbee2mqtt-config.yaml @@ -64,6 +64,14 @@ data: # the sidecar is then a no-op and can go too. log_namespaced_levels: zh:controller: debug + # Maintains log/current -> the active log// directory. + # REQUIRED by the interview-watch sidecar: without it there is no stable + # path to follow, and `tail -F` on the newest timestamped directory silently + # goes deaf the moment Z2M restarts into a new one (the old file just stops + # receiving writes, so nothing signals the change). Verified that BusyBox + # `tail -F` does follow this symlink when Z2M repoints it. Remove together + # with the sidecar. + log_symlink_current: true # EXPLICIT network params matching the live coordinator, so zigbee-herdsman # ADOPTS (resumes) the existing network instead of forming a new one — # every device stays joined, no re-pairing. herdsman compares these against diff --git a/kubernetes/apps/home-assistant/zigbee2mqtt-deployment.yaml b/kubernetes/apps/home-assistant/zigbee2mqtt-deployment.yaml index 7cc4561b..1d76cdd2 100644 --- a/kubernetes/apps/home-assistant/zigbee2mqtt-deployment.yaml +++ b/kubernetes/apps/home-assistant/zigbee2mqtt-deployment.yaml @@ -159,6 +159,7 @@ spec: # marker: this sidecar is temporary, and it must not drag the broker's # image-automation policy along with it. image: eclipse-mosquitto:2.0.22 + imagePullPolicy: IfNotPresent command: [sh, /script/watch.sh] env: - name: MQTT_PASSWORD @@ -166,7 +167,14 @@ spec: secretKeyRef: name: mosquitto-zigbee2mqtt-credentials key: password - # Must track mqtt.base_topic in zigbee2mqtt-config.yaml. + # These three must track zigbee2mqtt-config.yaml: MQTT_HOST the + # mosquitto Service name, MQTT_USER and BASE_TOPIC the mqtt.user and + # mqtt.base_topic it connects with. Set explicitly rather than left + # to the script's defaults so the coupling is visible here. + - name: MQTT_HOST + value: mosquitto + - name: MQTT_USER + value: zigbee2mqtt - name: BASE_TOPIC value: zigbee2mqtt - name: STATEDIR diff --git a/kubernetes/apps/home-assistant/zigbee2mqtt-interview-watch.sh b/kubernetes/apps/home-assistant/zigbee2mqtt-interview-watch.sh index 198f6af4..3c030297 100644 --- a/kubernetes/apps/home-assistant/zigbee2mqtt-interview-watch.sh +++ b/kubernetes/apps/home-assistant/zigbee2mqtt-interview-watch.sh @@ -60,6 +60,7 @@ MQTT_USER=${MQTT_USER:-zigbee2mqtt} # here needs a matching ACL change too. BASE_TOPIC=${BASE_TOPIC:-zigbee2mqtt} COOLDOWN=${COOLDOWN:-300} # min seconds between interview attempts per device +PUB_TIMEOUT=${PUB_TIMEOUT:-10} # seconds; bounds a single mosquitto_pub STATEDIR=${STATEDIR:-/state} # emptyDir; writable as non-root mkdir -p "$STATEDIR" @@ -90,34 +91,61 @@ ieee_needing_interview() { }' "$DB" 2>/dev/null } -# Z2M creates a fresh log directory per start, so resolve the newest one. On a -# cold pod start Z2M may not have created it yet. -newest_log() { - d=$(ls -t "$DATA/log" 2>/dev/null | head -1) - [ -n "${d:-}" ] && printf '%s/log/%s/log.log' "$DATA" "$d" +# NOTE the `|| true` and the sanity case: `grep -c` prints "0" AND exits 1 when +# nothing matches, so a naive `grep -c ... || echo 0` emits TWO lines ("0\n0"), +# which then blows up every `-eq` test with "bad number". That only shows up once +# no devices need interviewing — i.e. the steady end state this is meant to reach. +count_needing_interview() { + c=$(grep -cE '"interviewState":"(PENDING|FAILED)"' "$DB" 2>/dev/null || true) + case "$c" in + ''|*[!0-9]*) c=0 ;; # missing/unreadable DB, or unexpected output + esac + printf '%s' "$c" } log "starting" -log "$(grep -cE '"interviewState":"(PENDING|FAILED)"' "$DB" 2>/dev/null || echo 0) device(s) currently need an interview" +log "$(count_needing_interview) device(s) currently need an interview" -# Re-resolve the log file IN-PROCESS rather than exiting and letting the kubelet -# restart us. Exiting would incur kubelet crash-loop backoff, which climbs to -# 5 minutes unless the container has stayed up >=10 min. Z2M restarts and log -# rotations can easily land closer together than that (notably during the -# "interview a batch, then restart Z2M once" workflow), so backoff would kick in -# exactly when a device's short wake window needs a prompt reaction. Staying -# resident keeps the reaction time flat. -while :; do - LOGFILE=$(newest_log) - if [ -z "${LOGFILE:-}" ] || [ ! -f "$LOGFILE" ]; then - sleep 5 - continue +# Follow the STABLE path, not the newest timestamped directory. +# +# Z2M writes logs to log//log.log — a fresh directory on every +# start. `tail -F` follows a specific pathname: it re-opens that name if the +# inode behind it changes, but it can never notice that Z2M has begun writing +# somewhere else entirely. Tailing the newest directory therefore breaks the +# moment Z2M restarts: the old file simply goes quiet, so the pipe never ends, +# the outer loop never re-resolves, "following " still shows the old path, +# and the sidecar looks healthy while seeing no frames again ever. That is not a +# rare edge case here — the Z2M container has restarted dozens of times on its +# own (DNS flaps), each time into a new directory, and the planned "interview a +# batch, then restart Z2M once" step hits it deliberately. +# +# Fixed at the source instead of by polling: advanced.log_symlink_current: true +# (set in zigbee2mqtt-config.yaml) makes Z2M maintain log/current -> the active +# directory, giving one stable path. Verified that BusyBox `tail -F` in this +# image does follow a symlink that gets repointed to a new target mid-follow, so +# a Z2M restart is picked up automatically with no watchdog. +# +# Deliberately NO fallback to the newest directory: falling back would risk +# silently tailing a frozen file, which is the exact failure being fixed. If the +# symlink is missing the sidecar says so, loudly and repeatedly, and waits. +LOGFILE="$DATA/log/current/log.log" +warned=0 +while [ ! -f "$LOGFILE" ]; do + if [ "$warned" -eq 0 ]; then + log "WARNING: $LOGFILE absent — Z2M needs advanced.log_symlink_current: true (and a restart to create it). Waiting." + warned=1 fi + sleep 30 + log "still waiting for $LOGFILE" +done - log "following $LOGFILE" +# Kept as a loop purely so an unexpected end of stream re-establishes the tail +# rather than exiting. Exiting would incur kubelet crash-loop backoff (climbing +# to 5 min unless the container stays up >=10 min), which would bite precisely +# when a device's short wake window needs a prompt reaction. +while :; do + log "following $LOGFILE ($(count_needing_interview) device(s) still need an interview)" - # tail -F (--follow=name --retry) rides out an in-place rotation; the outer - # loop handles Z2M starting a whole new log directory. # # Matching happens in the read loop rather than via a `grep` stage on purpose. # This runs on an Alpine-based image whose `grep` is the BusyBox applet, and @@ -128,6 +156,9 @@ while :; do # a silent no-op of the whole feature. Dropping the stage avoids both that and # any block-buffering question, since `tail` writes per line and `read` # consumes per line, with no extra package to install. + pending=$(count_needing_interview) + next_recount=0 + tail -F "$LOGFILE" 2>/dev/null | while IFS= read -r line; do case "$line" in @@ -146,11 +177,25 @@ while :; do ''|*[!0-9]*) continue ;; esac + now=$(date +%s) + + # Short-circuit while nothing needs interviewing. The debug log covers ALL + # of zh:controller, so every device on the network produces frames here — + # roughly 30 devices' worth once the backlog is cleared. Without this the + # script would fork awk over the whole database.db for each of those frames + # forever, for no possible benefit. Recounted periodically (not per frame) + # so a device that later regresses to PENDING is still picked up. + if [ "$now" -ge "$next_recount" ]; then + pending=$(count_needing_interview) + next_recount=$((now + 60)) + [ "$pending" -eq 0 ] && log "no devices need an interview; idling (recheck every 60s)" + fi + [ "$pending" -eq 0 ] && continue + ieee=$(ieee_needing_interview "$nwk") [ -z "${ieee:-}" ] && continue # already interviewed, or not a device we track stamp="$STATEDIR/$ieee" - now=$(date +%s) if [ -f "$stamp" ]; then last=$(cat "$stamp" 2>/dev/null || echo 0) # Suppress the interview's OWN traffic, which would otherwise re-trigger @@ -163,7 +208,14 @@ while :; do # it has no --pw-file or env option, so this cannot be avoided with this # client. Not a cross-container exposure: the pod does not set # shareProcessNamespace, so /proc is not shared with the Z2M container. - if mosquitto_pub -h "$MQTT_HOST" -u "$MQTT_USER" -P "$MQTT_PASSWORD" \ + # Bounded with `timeout`, which also covers a TCP connect that hangs rather + # than being refused. The read loop is single-threaded, so an unbounded + # publish would stall it and miss other devices' wake windows. + # + # NOT `mosquitto_pub -W`: that flag does not exist for mosquitto_pub (it is + # a mosquitto_sub option), and passing it makes every publish fail on a + # usage error — verified against the client in this image. + if timeout "$PUB_TIMEOUT" mosquitto_pub -h "$MQTT_HOST" -u "$MQTT_USER" -P "$MQTT_PASSWORD" \ -t "$BASE_TOPIC/bridge/request/device/interview" \ -m "{\"id\":\"$ieee\"}" 2>/dev/null; then # Stamp only on a SUCCESSFUL publish. Stamping unconditionally would burn -- 2.54.0 From a75c482f7dc1e42aa4ca1a261e3d0b37eeca753c Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Thu, 30 Jul 2026 09:07:32 +0200 Subject: [PATCH 6/6] feat(zigbee2mqtt): add watchdog heartbeat and document seed-only log keys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses the round-3 review (commit 39616401), which I failed to reply to before pushing the next change. Add an unconditional heartbeat logging the outstanding PENDING/FAILED count. The main loop is purely event-driven, so if the "Received payload: clusterID=" format ever changes (herdsman upgrade, log format change) it would match nothing forever while still logging "following " and looking healthy. The per-frame recount cannot detect that either, because it only runs when a frame is matched. The heartbeat runs off a timer instead, so a count that never falls while devices are demonstrably in use is a visible signal that the matcher broke. Document, in the ConfigMap itself rather than only in the PR body, that the two log_* keys are seed-only and do not reach a running Z2M whose PVC is already seeded. Both were therefore also applied to the live configuration.yaml via bridge/request/options; verified they persist there (lines 29-31 of the on-PVC file) so they survive a restart. Also noted that log_symlink_current only takes effect on a Z2M restart, since the symlink is created when the log directory is opened — until then the sidecar waits and says so. Not changed: resources.limits.cpu is still unset on the sidecar. The review flagged it as consistent with the sibling containers in this file and asked only whether it was intentional — it is. A CPU limit on a mostly-idle process whose value is reacting inside a device's short wake window buys nothing and risks throttling at exactly the wrong moment; the memory limit already bounds the real risk. Verified in eclipse-mosquitto:2.0.22 as uid 1883 with HEARTBEAT=5: the heartbeat interleaves with frame handling and the simulated Z2M restart without disturbing either. Co-Authored-By: Claude Opus 5 (1M context) --- .../apps/home-assistant/zigbee2mqtt-config.yaml | 8 ++++++++ .../home-assistant/zigbee2mqtt-interview-watch.sh | 15 +++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/kubernetes/apps/home-assistant/zigbee2mqtt-config.yaml b/kubernetes/apps/home-assistant/zigbee2mqtt-config.yaml index 41460382..aebb342f 100644 --- a/kubernetes/apps/home-assistant/zigbee2mqtt-config.yaml +++ b/kubernetes/apps/home-assistant/zigbee2mqtt-config.yaml @@ -52,6 +52,14 @@ data: baudrate: 115200 advanced: + # SEED-ONLY, like the rest of this file (see the header): the two log_* + # keys below are NOT picked up by a running Z2M whose PVC is already + # seeded. Both were therefore also applied to the live + # /app/data/configuration.yaml via `zigbee2mqtt/bridge/request/options`, + # which Z2M persists. They are duplicated here so a fresh PVC comes up + # with the sidecar working; if you delete these, delete the sidecar too. + # Note log_symlink_current only takes effect on a Z2M restart, since the + # symlink is created when the log directory is opened. log_level: info # zh:controller at debug emits, for every received frame, # Received payload: clusterID=6, address=12468, ... diff --git a/kubernetes/apps/home-assistant/zigbee2mqtt-interview-watch.sh b/kubernetes/apps/home-assistant/zigbee2mqtt-interview-watch.sh index 3c030297..d7f0580e 100644 --- a/kubernetes/apps/home-assistant/zigbee2mqtt-interview-watch.sh +++ b/kubernetes/apps/home-assistant/zigbee2mqtt-interview-watch.sh @@ -61,6 +61,7 @@ MQTT_USER=${MQTT_USER:-zigbee2mqtt} BASE_TOPIC=${BASE_TOPIC:-zigbee2mqtt} COOLDOWN=${COOLDOWN:-300} # min seconds between interview attempts per device PUB_TIMEOUT=${PUB_TIMEOUT:-10} # seconds; bounds a single mosquitto_pub +HEARTBEAT=${HEARTBEAT:-600} # seconds between "still N to go" log lines STATEDIR=${STATEDIR:-/state} # emptyDir; writable as non-root mkdir -p "$STATEDIR" @@ -139,6 +140,20 @@ while [ ! -f "$LOGFILE" ]; do log "still waiting for $LOGFILE" done +# Heartbeat, independent of frame arrival. The main loop is purely event-driven, +# so if the "Received payload: clusterID=" format ever changes (herdsman upgrade, +# log format change) it would match nothing forever while still looking healthy — +# and the per-frame recount cannot detect that either, because it only runs when +# a frame is matched. This gives an unconditional signal to notice regression +# against: if the count never falls while devices are demonstrably being used, +# the matcher is broken. +( + while :; do + sleep "$HEARTBEAT" + log "heartbeat: $(count_needing_interview) device(s) still need an interview" + done +) & + # Kept as a loop purely so an unexpected end of stream re-establishes the tail # rather than exiting. Exiting would incur kubelet crash-loop backoff (climbing # to 5 min unless the container stays up >=10 min), which would bite precisely -- 2.54.0