diff --git a/kubernetes/apps/home-assistant/deployment.yaml b/kubernetes/apps/home-assistant/deployment.yaml index cf9b4342..4b693f9e 100644 --- a/kubernetes/apps/home-assistant/deployment.yaml +++ b/kubernetes/apps/home-assistant/deployment.yaml @@ -210,6 +210,28 @@ spec: - name: vscode containerPort: 8080 protocol: TCP + # /healthz answers 200 whenever the HTTP server is up; its body + # reports session heartbeat state, which is not health — an idle + # editor reports "expired" and is perfectly fine. So this asserts + # that code-server is serving, not that anyone is using it. + # + # Liveness matters here more than elsewhere in this pod: code-server + # shares the Home Assistant config volume, and a wedged editor is + # otherwise invisible until someone tries to open it. + livenessProbe: + httpGet: + path: /healthz + port: vscode + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 5 + readinessProbe: + httpGet: + path: /healthz + port: vscode + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 5 volumeMounts: - name: config mountPath: /config diff --git a/kubernetes/apps/openwakeword/deployment.yaml b/kubernetes/apps/openwakeword/deployment.yaml index f9a9e548..a6a14cef 100644 --- a/kubernetes/apps/openwakeword/deployment.yaml +++ b/kubernetes/apps/openwakeword/deployment.yaml @@ -24,6 +24,22 @@ spec: - name: wyoming containerPort: 10400 protocol: TCP + # Wyoming is raw TCP with no HTTP surface and no health verb, so + # accepting a connection is all that can be asserted from outside the + # process. Weaker than an HTTP probe, but the previous state was none + # at all: a dead server stayed Ready and kept being routed to. + livenessProbe: + tcpSocket: + port: wyoming + initialDelaySeconds: 60 + periodSeconds: 30 + timeoutSeconds: 5 + readinessProbe: + tcpSocket: + port: wyoming + initialDelaySeconds: 15 + periodSeconds: 10 + timeoutSeconds: 5 resources: requests: cpu: 100m diff --git a/kubernetes/apps/paperless/ftp-scanner.yaml b/kubernetes/apps/paperless/ftp-scanner.yaml index c6d005de..34e9dee0 100644 --- a/kubernetes/apps/paperless/ftp-scanner.yaml +++ b/kubernetes/apps/paperless/ftp-scanner.yaml @@ -70,6 +70,27 @@ spec: - name: consume mountPath: /home/scanner subPath: scanner + # FTP has no health verb, so the control port accepting a connection + # is what can be asserted. It is enough for the failure that matters: + # if vsftpd dies the scanner cannot deposit anything and nothing else + # in the chain notices, because paperless only ever sees an empty + # consume directory — which is indistinguishable from nobody having + # scanned. + # + # Only the control port is probed. The passive range is negotiated + # per transfer and is not listening at rest. + livenessProbe: + tcpSocket: + port: ftp-ctrl + initialDelaySeconds: 15 + periodSeconds: 30 + timeoutSeconds: 5 + readinessProbe: + tcpSocket: + port: ftp-ctrl + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 5 resources: requests: cpu: 20m diff --git a/kubernetes/apps/speech-to-phrase/deployment.yaml b/kubernetes/apps/speech-to-phrase/deployment.yaml index 7c236b99..10f1f26b 100644 --- a/kubernetes/apps/speech-to-phrase/deployment.yaml +++ b/kubernetes/apps/speech-to-phrase/deployment.yaml @@ -34,6 +34,32 @@ spec: - name: wyoming containerPort: 10300 protocol: TCP + # Wyoming is raw TCP with no HTTP surface and no health verb, so + # accepting a connection is all that can be asserted from outside the + # process — weaker than an HTTP probe, but the previous state was + # none at all. + # + # A startupProbe rather than a long liveness delay: this container + # runs with --retrain-on-start, and retraining against the Home + # Assistant entity list takes as long as it takes. The startup gate + # allows up to 10 minutes (60 x 10s) and only then hands over to + # liveness, so a slow retrain cannot be mistaken for a hang, and a + # genuine hang after startup is still caught within a minute. + startupProbe: + tcpSocket: + port: wyoming + periodSeconds: 10 + failureThreshold: 60 + livenessProbe: + tcpSocket: + port: wyoming + periodSeconds: 30 + timeoutSeconds: 5 + readinessProbe: + tcpSocket: + port: wyoming + periodSeconds: 10 + timeoutSeconds: 5 volumeMounts: - name: models mountPath: /models diff --git a/kubernetes/apps/whisper/deployment.yaml b/kubernetes/apps/whisper/deployment.yaml index 80f9e3ae..5c1c1bea 100644 --- a/kubernetes/apps/whisper/deployment.yaml +++ b/kubernetes/apps/whisper/deployment.yaml @@ -29,6 +29,27 @@ spec: - name: wyoming containerPort: 10300 protocol: TCP + # Wyoming is a raw TCP protocol with no HTTP surface and no health + # verb, so accepting a connection is the only thing that can be + # asserted from outside the process. That is weaker than an HTTP + # probe but strictly better than the previous state of none at all, + # where a dead server stayed Ready and Home Assistant kept routing + # speech to it. + # + # The liveness delay is generous because the model is loaded at + # startup from the data volume. + livenessProbe: + tcpSocket: + port: wyoming + initialDelaySeconds: 90 + periodSeconds: 30 + timeoutSeconds: 5 + readinessProbe: + tcpSocket: + port: wyoming + initialDelaySeconds: 20 + periodSeconds: 10 + timeoutSeconds: 5 volumeMounts: - name: data mountPath: /data diff --git a/kubernetes/infrastructure/external-secrets/helmrelease.yaml b/kubernetes/infrastructure/external-secrets/helmrelease.yaml index 6c001b61..6f2028d3 100644 --- a/kubernetes/infrastructure/external-secrets/helmrelease.yaml +++ b/kubernetes/infrastructure/external-secrets/helmrelease.yaml @@ -33,3 +33,16 @@ spec: crds: CreateReplace remediation: retries: 3 + values: + # The chart ships these disabled, so the controller ran with no probe of + # any kind: a wedged reconciler stayed Ready and every ExternalSecret would + # simply stop refreshing, which surfaces only when a consumer needs a + # credential that has since rotated. + # + # Enabling them also starts the health server — the chart derives + # --health-probe-bind-address from these values, and without them the + # controller exposes nothing but its metrics port. + livenessProbe: + enabled: true + readinessProbe: + enabled: true