2026-07-30 07:11:25 +00:00
5 changed files with 381 additions and 0 deletions
@@ -68,5 +68,13 @@ configMapGenerator:
    namespace: home-assistant
    files:
      - sync.sh=config-sync.sh
+  # Z2M re-interview watchdog sidecar script. Hash suffix disabled (name stays
+  # stable), so editing zigbee2mqtt-interview-watch.sh does not auto-roll the
+  # pod — restart manually with
+  # `kubectl rollout restart deploy/zigbee2mqtt -n home-assistant`.
+  - name: zigbee2mqtt-interview-watch-script
+    namespace: home-assistant
+    files:
+      - watch.sh=zigbee2mqtt-interview-watch.sh
 generatorOptions:
  disableNameSuffixHash: true
@@ -52,7 +52,34 @@ data:
      baudrate: 115200

    advanced:
+      # SEED-ONLY, like the rest of this file (see the header): the two log_*
+      # keys below are NOT picked up by a running Z2M whose PVC is already
+      # seeded. Both were therefore also applied to the live
+      # /app/data/configuration.yaml via `zigbee2mqtt/bridge/request/options`,
+      # which Z2M persists. They are duplicated here so a fresh PVC comes up
+      # with the sidecar working; if you delete these, delete the sidecar too.
+      # Note log_symlink_current only takes effect on a Z2M restart, since the
+      # symlink is created when the log directory is opened.
      log_level: info
+      # zh:controller at debug emits, for every received frame,
+      #   Received payload: clusterID=6, address=12468, ...
+      # which the zigbee2mqtt-interview-watch.sh sidecar uses as its "device is
+      # awake right now" signal to fire a forced interview (see that script's
+      # header for why nothing on MQTT works for this). Namespaced rather than a
+      # global `log_level: debug` on purpose: global debug adds the zh:zstack:
+      # unpi:parser byte-level firehose (~90 lines/min measured), the namespaced
+      # form costs ~6 lines/min. Drop this once every device is interviewed —
+      # the sidecar is then a no-op and can go too.
+      log_namespaced_levels:
+        zh:controller: debug
+      # Maintains log/current -> the active log/<start-timestamp>/ directory.
+      # REQUIRED by the interview-watch sidecar: without it there is no stable
+      # path to follow, and `tail -F` on the newest timestamped directory silently
+      # goes deaf the moment Z2M restarts into a new one (the old file just stops
+      # receiving writes, so nothing signals the change). Verified that BusyBox
+      # `tail -F` does follow this symlink when Z2M repoints it. Remove together
+      # with the sidecar.
+      log_symlink_current: true
      # EXPLICIT network params matching the live coordinator, so zigbee-herdsman
      # ADOPTS (resumes) the existing network instead of forming a new one —
      # every device stays joined, no re-pairing. herdsman compares these against
@@ -136,6 +136,78 @@ spec:
              memory: 128Mi
            limits:
              memory: 512Mi
+        # Fires a forced interview whenever a still-un-interviewed device
+        # transmits — i.e. at the only moment a sleepy battery device can
+        # actually complete one. Z2M itself only retries interviews on a ZDO
+        # announce, which these devices essentially never emit, so without this
+        # they stay "Not supported" indefinitely (16 of them did, for six days
+        # across 38 restarts). Full rationale in the script header.
+        #
+        # A SIDECAR rather than its own Deployment: it reads both Z2M's log file
+        # and database.db, and the PVC is RWO/single-writer, so it has to share
+        # this pod. It only ever reads the PVC.
+        #
+        # Depends on advanced.log_namespaced_levels {zh:controller: debug} in
+        # zigbee2mqtt-config.yaml. Remove both once every device is interviewed.
+        - name: interview-watch
+          # Reuses the broker image purely for its bundled mosquitto_pub, so
+          # nothing is installed at runtime — no `apk add`, hence no network
+          # dependency at start and no package-index drift. Everything the script
+          # needs was verified present in this exact image: sh, tail (incl. -F),
+          # awk (BusyBox match()/RSTART/RLENGTH), cat, date, mkdir,
+          # mosquitto_pub. Pinned deliberately WITHOUT a Flux $imagepolicy
+          # marker: this sidecar is temporary, and it must not drag the broker's
+          # image-automation policy along with it.
+          image: eclipse-mosquitto:2.0.22
+          imagePullPolicy: IfNotPresent
+          command: [sh, /script/watch.sh]
+          env:
+            - name: MQTT_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: mosquitto-zigbee2mqtt-credentials
+                  key: password
+            # These three must track zigbee2mqtt-config.yaml: MQTT_HOST the
+            # mosquitto Service name, MQTT_USER and BASE_TOPIC the mqtt.user and
+            # mqtt.base_topic it connects with. Set explicitly rather than left
+            # to the script's defaults so the coupling is visible here.
+            - name: MQTT_HOST
+              value: mosquitto
+            - name: MQTT_USER
+              value: zigbee2mqtt
+            - name: BASE_TOPIC
+              value: zigbee2mqtt
+            - name: STATEDIR
+              value: /state
+          # Runs as the image's own mosquitto user (1883). Viable because
+          # nothing is installed at runtime and everything it reads under
+          # /app/data is world-readable (dirs 0755, files 0644 — verified), so
+          # root is not needed to tail the log or read database.db.
+          securityContext:
+            runAsNonRoot: true
+            runAsUser: 1883
+            runAsGroup: 1883
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: [ALL]
+          volumeMounts:
+            - name: data
+              mountPath: /app/data
+              readOnly: true
+            - name: interview-watch-script
+              mountPath: /script
+              readOnly: true
+            # Cooldown stamps. An explicit emptyDir rather than the image's
+            # /tmp, so it stays writable as non-root without depending on the
+            # container filesystem.
+            - name: interview-watch-state
+              mountPath: /state
+          resources:
+            requests:
+              cpu: 10m
+              memory: 16Mi
+            limits:
+              memory: 64Mi
      volumes:
        - name: data
          persistentVolumeClaim:
@@ -146,3 +218,8 @@ spec:
        - name: coordinator-backup
          secret:
            secretName: zigbee2mqtt-coordinator-backup
+        - name: interview-watch-script
+          configMap:
+            name: zigbee2mqtt-interview-watch-script
+        - name: interview-watch-state
+          emptyDir: {}
@@ -0,0 +1,250 @@
+#!/bin/sh
+# Re-interview watchdog for Zigbee2MQTT.
+#
+# PROBLEM
+# The ZHA->Z2M cutover (2026-07-23) adopted the existing Zigbee network, so
+# every device stayed joined but Z2M's database.db started empty. Each device
+# must therefore be *interviewed* before Z2M knows its modelId and can match a
+# converter; until then it logs "<name>: Not supported (EndDevice)" and exposes
+# only a generic "Automatically generated definition" (battery/linkquality),
+# not the real entities (action / contact / moisture).
+#
+# Mains-powered routers interviewed immediately (always awake). Battery end
+# devices did not, and they do NOT recover on their own, because:
+#
+#   - zigbee-herdsman only retries an interview on a ZDO device *announce*
+#     (controller.js: interviewState Pending|Failed -> device.interview()), and
+#     sleepy battery devices almost never emit one spontaneously.
+#   - Z2M's Configure extension retries on any traffic (onLastSeenChanged) but
+#     is capped at 3 attempts per process (in-memory Map, reset on restart).
+#
+# So 16 devices sat PENDING for six days across 38 process restarts.
+#
+# APPROACH
+# A device that transmits *is awake right now*, which is exactly when an
+# interview can succeed. This watches for that moment and fires a forced
+# interview request. Confirmed in practice: interviews completed in 11-34s when
+# triggered this way, including Xiaomi/Aqara sensors whose wake window is short.
+#
+# WHY THE LOG FILE AND NOT MQTT
+# An un-interviewed Xiaomi device publishes NOTHING to MQTT (its
+# manufacturer-specific attributes cannot be mapped by the auto-generated
+# definition), so MQTT state topics are not a usable signal for exactly the
+# devices that need help. `zigbee2mqtt/bridge/logging` carries only `info`.
+# The one reliable signal is the zh:controller debug line
+#
+#   Received payload: clusterID=6, address=12468, ...
+#
+# which requires advanced.log_namespaced_levels {zh:controller: debug} — set in
+# zigbee2mqtt-config.yaml. Root log_level stays `info`; enabling debug globally
+# adds ~90 lines/min of UNPI frame noise, the namespaced form ~6 lines/min.
+#
+# This runs as a sidecar (not a separate Deployment) because it needs the same
+# RWO PVC as Z2M for both the log file and database.db.
+#
+# NOTE: interviewing is only half the job. Philips devices additionally need
+# `configure` to bind manuSpecificPhilips (0xFC00), and that cluster is only
+# registered on Z2M's `start` event (deviceAddCustomCluster onEvent), so a
+# device interviewed mid-run cannot be configured until the next restart. One
+# Z2M restart after a batch of interviews completes the binds.
+
+set -u
+
+DATA=/app/data
+DB="$DATA/database.db"
+MQTT_HOST=${MQTT_HOST:-mosquitto}
+MQTT_USER=${MQTT_USER:-zigbee2mqtt}
+# Must track mqtt.base_topic in zigbee2mqtt-config.yaml. Overridable via env so
+# the two can be kept in step from the Deployment rather than by editing this
+# script; the MQTT ACL for this user is scoped to `zigbee2mqtt/#`, so a change
+# here needs a matching ACL change too.
+BASE_TOPIC=${BASE_TOPIC:-zigbee2mqtt}
+COOLDOWN=${COOLDOWN:-300}     # min seconds between interview attempts per device
+PUB_TIMEOUT=${PUB_TIMEOUT:-10} # seconds; bounds a single mosquitto_pub
+HEARTBEAT=${HEARTBEAT:-600}   # seconds between "still N to go" log lines
+STATEDIR=${STATEDIR:-/state}  # emptyDir; writable as non-root
+
+mkdir -p "$STATEDIR"
+
+log() { printf '%s interview-watch: %s\n' "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" "$*"; }
+
+# Resolve a network address to an IEEE, but ONLY for devices that still need an
+# interview. database.db is newline-delimited JSON (one device per line) despite
+# the .db extension, so a per-line scan is enough and needs no jq. Read fresh on
+# every frame so devices drop out the moment they succeed, and so changed
+# network addresses are picked up automatically.
+#
+# Each field is matched INDEPENDENTLY rather than with one combined pattern:
+# JSON guarantees no key order, so a single regex expecting "ieeeAddr" before
+# "nwkAddr" would silently capture nothing (no error, no log — a device just
+# never gets picked up) if herdsman's serialization order ever changed.
+ieee_needing_interview() {
+  awk -v want="$1" '
+    /"interviewState":"(PENDING|FAILED)"/ {
+      ieee = ""; nwk = ""
+      if (match($0, /"ieeeAddr":"[^"]*"/)) {
+        s = substr($0, RSTART, RLENGTH); sub(/^"ieeeAddr":"/, "", s); sub(/"$/, "", s); ieee = s
+      }
+      if (match($0, /"nwkAddr":[0-9]+/)) {
+        s = substr($0, RSTART, RLENGTH); sub(/^"nwkAddr":/, "", s); nwk = s
+      }
+      if (ieee != "" && nwk == want) { print ieee; exit }
+    }' "$DB" 2>/dev/null
+}
+
+# NOTE the `|| true` and the sanity case: `grep -c` prints "0" AND exits 1 when
+# nothing matches, so a naive `grep -c ... || echo 0` emits TWO lines ("0\n0"),
+# which then blows up every `-eq` test with "bad number". That only shows up once
+# no devices need interviewing — i.e. the steady end state this is meant to reach.
+count_needing_interview() {
+  c=$(grep -cE '"interviewState":"(PENDING|FAILED)"' "$DB" 2>/dev/null || true)
+  case "$c" in
+    ''|*[!0-9]*) c=0 ;;   # missing/unreadable DB, or unexpected output
+  esac
+  printf '%s' "$c"
+}
+
+log "starting"
+log "$(count_needing_interview) device(s) currently need an interview"
+
+# Follow the STABLE path, not the newest timestamped directory.
+#
+# Z2M writes logs to log/<start-timestamp>/log.log — a fresh directory on every
+# start. `tail -F` follows a specific pathname: it re-opens that name if the
+# inode behind it changes, but it can never notice that Z2M has begun writing
+# somewhere else entirely. Tailing the newest directory therefore breaks the
+# moment Z2M restarts: the old file simply goes quiet, so the pipe never ends,
+# the outer loop never re-resolves, "following <file>" still shows the old path,
+# and the sidecar looks healthy while seeing no frames again ever. That is not a
+# rare edge case here — the Z2M container has restarted dozens of times on its
+# own (DNS flaps), each time into a new directory, and the planned "interview a
+# batch, then restart Z2M once" step hits it deliberately.
+#
+# Fixed at the source instead of by polling: advanced.log_symlink_current: true
+# (set in zigbee2mqtt-config.yaml) makes Z2M maintain log/current -> the active
+# directory, giving one stable path. Verified that BusyBox `tail -F` in this
+# image does follow a symlink that gets repointed to a new target mid-follow, so
+# a Z2M restart is picked up automatically with no watchdog.
+#
+# Deliberately NO fallback to the newest directory: falling back would risk
+# silently tailing a frozen file, which is the exact failure being fixed. If the
+# symlink is missing the sidecar says so, loudly and repeatedly, and waits.
+LOGFILE="$DATA/log/current/log.log"
+warned=0
+while [ ! -f "$LOGFILE" ]; do
+  if [ "$warned" -eq 0 ]; then
+    log "WARNING: $LOGFILE absent — Z2M needs advanced.log_symlink_current: true (and a restart to create it). Waiting."
+    warned=1
+  fi
+  sleep 30
+  log "still waiting for $LOGFILE"
+done
+
+# Heartbeat, independent of frame arrival. The main loop is purely event-driven,
+# so if the "Received payload: clusterID=" format ever changes (herdsman upgrade,
+# log format change) it would match nothing forever while still looking healthy —
+# and the per-frame recount cannot detect that either, because it only runs when
+# a frame is matched. This gives an unconditional signal to notice regression
+# against: if the count never falls while devices are demonstrably being used,
+# the matcher is broken.
+(
+  while :; do
+    sleep "$HEARTBEAT"
+    log "heartbeat: $(count_needing_interview) device(s) still need an interview"
+  done
+) &
+
+# Kept as a loop purely so an unexpected end of stream re-establishes the tail
+# rather than exiting. Exiting would incur kubelet crash-loop backoff (climbing
+# to 5 min unless the container stays up >=10 min), which would bite precisely
+# when a device's short wake window needs a prompt reaction.
+while :; do
+  log "following $LOGFILE ($(count_needing_interview) device(s) still need an interview)"
+
+  #
+  # Matching happens in the read loop rather than via a `grep` stage on purpose.
+  # This runs on an Alpine-based image whose `grep` is the BusyBox applet, and
+  # BusyBox grep does NOT support --line-buffered:
+  #     grep: unrecognized option: line-buffered   (exit 2)
+  # so a `tail -F | grep --line-buffered | while read` pipe yields ZERO lines
+  # forever while the sidecar still logs "following <file>" and looks healthy —
+  # a silent no-op of the whole feature. Dropping the stage avoids both that and
+  # any block-buffering question, since `tail` writes per line and `read`
+  # consumes per line, with no extra package to install.
+  pending=$(count_needing_interview)
+  next_recount=0
+
+  tail -F "$LOGFILE" 2>/dev/null |
+  while IFS= read -r line; do
+    case "$line" in
+      *"Received payload: clusterID="*) ;;
+      *) continue ;;
+    esac
+
+    # ... clusterID=6, address=12468, groupID=0, endpoint=1, ...
+    rest=${line#*"Received payload: clusterID="}
+    cluster=${rest%%,*}
+    rest=${rest#*address=}
+    nwk=${rest%%,*}
+
+    # Guard against a format change silently yielding garbage.
+    case "$nwk" in
+      ''|*[!0-9]*) continue ;;
+    esac
+
+    now=$(date +%s)
+
+    # Short-circuit while nothing needs interviewing. The debug log covers ALL
+    # of zh:controller, so every device on the network produces frames here —
+    # roughly 30 devices' worth once the backlog is cleared. Without this the
+    # script would fork awk over the whole database.db for each of those frames
+    # forever, for no possible benefit. Recounted periodically (not per frame)
+    # so a device that later regresses to PENDING is still picked up.
+    if [ "$now" -ge "$next_recount" ]; then
+      pending=$(count_needing_interview)
+      next_recount=$((now + 60))
+      [ "$pending" -eq 0 ] && log "no devices need an interview; idling (recheck every 60s)"
+    fi
+    [ "$pending" -eq 0 ] && continue
+
+    ieee=$(ieee_needing_interview "$nwk")
+    [ -z "${ieee:-}" ] && continue      # already interviewed, or not a device we track
+
+    stamp="$STATEDIR/$ieee"
+    if [ -f "$stamp" ]; then
+      last=$(cat "$stamp" 2>/dev/null || echo 0)
+      # Suppress the interview's OWN traffic, which would otherwise re-trigger
+      # this on every attribute read it performs.
+      [ $((now - last)) -lt "$COOLDOWN" ] && continue
+    fi
+
+    log "frame from $ieee (nwk=$nwk cluster=$cluster) while awake -> requesting interview"
+    # NOTE: mosquitto_pub can only take a password via argv (-P, or an -L URL);
+    # it has no --pw-file or env option, so this cannot be avoided with this
+    # client. Not a cross-container exposure: the pod does not set
+    # shareProcessNamespace, so /proc is not shared with the Z2M container.
+    # Bounded with `timeout`, which also covers a TCP connect that hangs rather
+    # than being refused. The read loop is single-threaded, so an unbounded
+    # publish would stall it and miss other devices' wake windows.
+    #
+    # NOT `mosquitto_pub -W`: that flag does not exist for mosquitto_pub (it is
+    # a mosquitto_sub option), and passing it makes every publish fail on a
+    # usage error — verified against the client in this image.
+    if timeout "$PUB_TIMEOUT" mosquitto_pub -h "$MQTT_HOST" -u "$MQTT_USER" -P "$MQTT_PASSWORD" \
+         -t "$BASE_TOPIC/bridge/request/device/interview" \
+         -m "{\"id\":\"$ieee\"}" 2>/dev/null; then
+      # Stamp only on a SUCCESSFUL publish. Stamping unconditionally would burn
+      # the device's wake window on a failed publish (broker hiccup, DNS blip)
+      # and then refuse to retry for the full cooldown, despite no interview
+      # ever having been requested.
+      printf '%s' "$now" > "$stamp"
+      log "  interview requested for $ieee"
+    else
+      log "  WARNING: mosquitto_pub failed for $ieee; not starting cooldown so the next frame retries"
+    fi
+  done
+
+  # Stream ended: Z2M restarted into a new log directory, or the file vanished.
+  log "log stream ended; re-resolving current log file"
+  sleep 5
+done
@@ -144,6 +144,25 @@ empty, so each device must be re-interviewed before it reports fully. Order:
 Keep `permit_join` **off** (it is, by config) — this is re-interviewing, not
 re-pairing; nothing leaves and rejoins the network.

+> **Item 2 above (battery/sleepy devices) is now automated — do not chase
+> those by hand.** The
+> `interview-watch` sidecar (`zigbee2mqtt-interview-watch.sh`) fires a forced
+> interview whenever a still-`PENDING`/`FAILED` device transmits, which is the
+> only moment a sleepy device can complete one. Devices heal as they are used or
+> report naturally; several did so with no human interaction at all. Hand-timing
+> a wake against a re-interview click is no longer necessary.
+>
+> Two caveats remain:
+> - **Interviewing is only half of it.** Philips devices also need `configure`
+>   to bind `manuSpecificPhilips` (0xFC00), and that cluster is registered only
+>   on Z2M's `start` event, so a device interviewed mid-run stays unbound (no
+>   `action` entities) until the next restart. Restart Z2M once after a batch of
+>   interviews lands.
+> - Devices that never transmit at all are **offline, not un-interviewed** —
+>   check `lastSeen` in `database.db` (absent + empty `endpoints[].clusters`
+>   means Z2M has never heard from it) before hunting for a Z2M-side fault.
+>   Flat batteries look exactly like a failed interview otherwise.
+
 ## Step 6 — Home Assistant side

 Z2M publishes MQTT discovery under `homeassistant/#`; HA's MQTT integration