diff --git a/kubernetes/apps/otbr/deployment.yaml b/kubernetes/apps/otbr/deployment.yaml index f8cd9dd1..6792573d 100644 --- a/kubernetes/apps/otbr/deployment.yaml +++ b/kubernetes/apps/otbr/deployment.yaml @@ -236,20 +236,92 @@ spec: # and re-election on every restart). - name: data mountPath: /var/lib/thread + # Probes must watch the *protocol*, not just the process. + # # The upstream entrypoint backgrounds otbr-agent and ends in # `tail -f /var/log/syslog`, so PID 1 stays alive even when - # otbr-agent has died — the pod stays Ready while the radio - # is broken. A liveness probe restores honest signaling: if - # otbr-agent isn't running, the container is restarted and - # CrashLoopBackOff surfaces the failure. + # otbr-agent has died. `pgrep -x otbr-agent` fixed that half. + # It does not fix the other half: on 2026-06-27 otbr-agent ran + # happily for twelve days with `ot-ctl state` = `disabled` and + # the border router `stopped`, and the pod reported Running 1/1 + # the entire time. A live process is not a live Thread mesh. + # + # All three probes must strip CR: ot-ctl speaks a serial-console + # protocol and terminates lines with CRLF, so `ot-ctl state` + # yields `leader\r`, which matches no shell glob (see the + # entrypoint's attach check and PR #140). curl/wget are absent + # from this image, so probes go through ot-ctl, not REST. + # + # startupProbe: the agent's control socket answers. Covers the + # apt install, the socat TCP↔PTY bridge to the SLZB radio, and + # the spinel handshake. Generous: 10s × 60 = 10 minutes. While + # it runs, the other two probes are suspended, so the entrypoint + # gets to finish activating Thread before liveness may act. + startupProbe: + exec: + command: + - sh + - -c + - ot-ctl state >/dev/null 2>&1 + periodSeconds: 10 + timeoutSeconds: 10 + failureThreshold: 60 + # readinessProbe: Thread is up. `detached` counts as ready — + # it means the stack is running and trying to attach, REST + # still serves, and treating normal mesh churn as NotReady + # would flap HA's OTBR integration for nothing. `disabled` does + # NOT count: that is the twelve-day failure, and it is the one + # state that never resolves on its own. + # + # Removing endpoints only affects HA's OTBR REST integration on + # :8081. matter-server does not consume this Service — it learns + # the OMR prefix by RA on its own macvlan interface. + readinessProbe: + exec: + command: + - sh + - -c + - | + S=$(ot-ctl state 2>/dev/null | head -1 | tr -d '\r') + case "$S" in + leader|router|child|detached) exit 0 ;; + esac + exit 1 + periodSeconds: 30 + timeoutSeconds: 10 + failureThreshold: 2 + # livenessProbe: the agent process, plus the one protocol state + # a restart actually repairs. `disabled` WITH a persisted dataset + # means the entrypoint's activation never ran (or was undone) — + # restarting re-runs it, and since #139 that path polls for the + # dataset and starts Thread. Self-healing. + # + # The dataset guard is load-bearing. On a clean install there is + # no dataset and `disabled` is the correct state until the + # otbr-dataset-bootstrap Job seeds one over `kubectl exec`. + # Restarting there would fight the Job — which has already been + # seen to die with BackoffLimitExceeded against a CrashLoopBackOff. + # + # If ot-ctl does not answer, S is empty, the state check falls + # through, and liveness rests on pgrep alone. A busy or wedged + # control socket must not trigger a restart storm. + # + # Caveat: an operator running `ot-ctl thread stop` by hand on a + # provisioned node gets the container restarted ~3 minutes later. + # That is deliberate — `disabled` with a dataset is a fault here. livenessProbe: exec: command: - sh - -c - - pgrep -x otbr-agent >/dev/null - initialDelaySeconds: 60 - periodSeconds: 15 + - | + pgrep -x otbr-agent >/dev/null || exit 1 + S=$(ot-ctl state 2>/dev/null | head -1 | tr -d '\r') + [ "$S" = "disabled" ] || exit 0 + ot-ctl dataset active 2>&1 | grep -qE "Active Timestamp|Network Name" || exit 0 + exit 1 + periodSeconds: 60 + timeoutSeconds: 15 failureThreshold: 3 volumes: - name: data