Merge pull request 'otbr: make the probes watch Thread, not just the process' (#141) from fix/otbr-protocol-aware-probes into master

Reviewed-on: gitea_admin/homelab#141

kubernetes/apps/otbr/deployment.yaml

@@ -236,20 +236,92 @@ spec:
             # and re-election on every restart).
             - name: data
               mountPath: /var/lib/thread
+          # Probes must watch the *protocol*, not just the process.
+          #
           # The upstream entrypoint backgrounds otbr-agent and ends in
           # `tail -f /var/log/syslog`, so PID 1 stays alive even when
-          # otbr-agent has died — the pod stays Ready while the radio
-          # is broken. A liveness probe restores honest signaling: if
-          # otbr-agent isn't running, the container is restarted and
-          # CrashLoopBackOff surfaces the failure.
+          # otbr-agent has died. `pgrep -x otbr-agent` fixed that half.
+          # It does not fix the other half: on 2026-06-27 otbr-agent ran
+          # happily for twelve days with `ot-ctl state` = `disabled` and
+          # the border router `stopped`, and the pod reported Running 1/1
+          # the entire time. A live process is not a live Thread mesh.
+          #
+          # All three probes must strip CR: ot-ctl speaks a serial-console
+          # protocol and terminates lines with CRLF, so `ot-ctl state`
+          # yields `leader\r`, which matches no shell glob (see the
+          # entrypoint's attach check and PR #140). curl/wget are absent
+          # from this image, so probes go through ot-ctl, not REST.
+          #
+          # startupProbe: the agent's control socket answers. Covers the
+          # apt install, the socat TCP↔PTY bridge to the SLZB radio, and
+          # the spinel handshake. Generous: 10s × 60 = 10 minutes. While
+          # it runs, the other two probes are suspended, so the entrypoint
+          # gets to finish activating Thread before liveness may act.
+          startupProbe:
+            exec:
+              command:
+                - sh
+                - -c
+                - ot-ctl state >/dev/null 2>&1
+            periodSeconds: 10
+            timeoutSeconds: 10
+            failureThreshold: 60
+          # readinessProbe: Thread is up. `detached` counts as ready —
+          # it means the stack is running and trying to attach, REST
+          # still serves, and treating normal mesh churn as NotReady
+          # would flap HA's OTBR integration for nothing. `disabled` does
+          # NOT count: that is the twelve-day failure, and it is the one
+          # state that never resolves on its own.
+          #
+          # Removing endpoints only affects HA's OTBR REST integration on
+          # :8081. matter-server does not consume this Service — it learns
+          # the OMR prefix by RA on its own macvlan interface.
+          readinessProbe:
+            exec:
+              command:
+                - sh
+                - -c
+                - |
+                  S=$(ot-ctl state 2>/dev/null | head -1 | tr -d '\r')
+                  case "$S" in
+                    leader|router|child|detached) exit 0 ;;
+                  esac
+                  exit 1
+            periodSeconds: 30
+            timeoutSeconds: 10
+            failureThreshold: 2
+          # livenessProbe: the agent process, plus the one protocol state
+          # a restart actually repairs. `disabled` WITH a persisted dataset
+          # means the entrypoint's activation never ran (or was undone) —
+          # restarting re-runs it, and since #139 that path polls for the
+          # dataset and starts Thread. Self-healing.
+          #
+          # The dataset guard is load-bearing. On a clean install there is
+          # no dataset and `disabled` is the correct state until the
+          # otbr-dataset-bootstrap Job seeds one over `kubectl exec`.
+          # Restarting there would fight the Job — which has already been
+          # seen to die with BackoffLimitExceeded against a CrashLoopBackOff.
+          #
+          # If ot-ctl does not answer, S is empty, the state check falls
+          # through, and liveness rests on pgrep alone. A busy or wedged
+          # control socket must not trigger a restart storm.
+          #
+          # Caveat: an operator running `ot-ctl thread stop` by hand on a
+          # provisioned node gets the container restarted ~3 minutes later.
+          # That is deliberate — `disabled` with a dataset is a fault here.
           livenessProbe:
             exec:
               command:
                 - sh
                 - -c
-                - pgrep -x otbr-agent >/dev/null
-            initialDelaySeconds: 60
-            periodSeconds: 15
+                - |
+                  pgrep -x otbr-agent >/dev/null || exit 1
+                  S=$(ot-ctl state 2>/dev/null | head -1 | tr -d '\r')
+                  [ "$S" = "disabled" ] || exit 0
+                  ot-ctl dataset active 2>&1 | grep -qE "Active Timestamp|Network Name" || exit 0
+                  exit 1
+            periodSeconds: 60
+            timeoutSeconds: 15
             failureThreshold: 3
       volumes:
         - name: data