From 9738f6fd6ee057114c6ebe4d2c38760a1ab7df7a Mon Sep 17 00:00:00 2001 From: Konstantin Renner Date: Wed, 8 Jul 2026 10:29:42 +0200 Subject: [PATCH] =?UTF-8?q?mosquitto:=20availability=20hardening=20?= =?UTF-8?q?=E2=80=94=20stateless=20broker=20+=20priority=20+=20fast=20fail?= =?UTF-8?q?over?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The broker is now load-bearing (heat pump, energy meters, EV charging, HA all depend on it) but was a slow-to-recover SPOF: single replica on a Ceph RWO PVC with no priority and default 5-min fault tolerations. mosquitto can't cluster, so instead of faking HA this minimizes recovery time and removes the storage coupling. - persistence false + drop the Ceph PVC. The db was only ~178 KB of retained messages (evcc state, HA discovery, Gen1 Shelly announce/state) — all of which clients re-publish on reconnect. Removing it means the broker no longer depends on Ceph to start, and reschedules with no RWO volume to fence/detach/reattach. Trade-off: a restart briefly empties retained state until clients reconnect (self-healing). - PriorityClass `home-critical` (1e6, below system-cluster-critical) so the broker is evicted last and reschedules ahead of ordinary workloads. - not-ready/unreachable tolerations at 30s (was default 300s) → node-failure reschedule in ~30s instead of ~5min. - terminationGracePeriodSeconds 10 for faster Recreate rollovers. Single replica + Recreate retained (two brokers would split clients/state). Apply notes: - Recreates the pod (brief MQTT reconnect; retained state re-populates). - The old PVC has prune:disabled, so Flux won't delete it — reclaim manually with `kubectl -n home-assistant delete pvc mosquitto-data` once happy. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../apps/home-assistant/kustomization.yaml | 1 - .../apps/home-assistant/mosquitto-config.yaml | 10 +++++-- .../home-assistant/mosquitto-deployment.yaml | 27 +++++++++++++------ .../apps/home-assistant/mosquitto-pvc.yaml | 15 ----------- .../cluster-settings/kustomization.yaml | 1 + .../cluster-settings/priority-classes.yaml | 11 ++++++++ 6 files changed, 39 insertions(+), 26 deletions(-) delete mode 100644 kubernetes/apps/home-assistant/mosquitto-pvc.yaml create mode 100644 kubernetes/infrastructure/cluster-settings/priority-classes.yaml diff --git a/kubernetes/apps/home-assistant/kustomization.yaml b/kubernetes/apps/home-assistant/kustomization.yaml index 46c0343f..da35f876 100644 --- a/kubernetes/apps/home-assistant/kustomization.yaml +++ b/kubernetes/apps/home-assistant/kustomization.yaml @@ -13,7 +13,6 @@ resources: - config-sync-token.sops.yaml - mosquitto-bootstrap.yaml - mosquitto-config.yaml - - mosquitto-pvc.yaml - mosquitto-certificate.yaml - mosquitto-service.yaml - mosquitto-iot-service.yaml diff --git a/kubernetes/apps/home-assistant/mosquitto-config.yaml b/kubernetes/apps/home-assistant/mosquitto-config.yaml index a7b3d7e2..c3895a6a 100644 --- a/kubernetes/apps/home-assistant/mosquitto-config.yaml +++ b/kubernetes/apps/home-assistant/mosquitto-config.yaml @@ -5,8 +5,14 @@ metadata: namespace: home-assistant data: mosquitto.conf: | - persistence true - persistence_location /mosquitto/data/ + # No on-disk persistence. Retained messages + sessions are re-populated by + # clients on reconnect (evcc republishes state, Shellies re-send their + # announce/online, HA re-publishes discovery), so the broker keeps no + # durable state. This drops the Ceph PVC dependency (the broker starts even + # if Ceph is degraded) and lets it reschedule anywhere in seconds — no RWO + # volume to fence/detach/reattach. Trade-off: a broker restart briefly + # empties retained state until clients reconnect (self-healing). + persistence false log_dest stdout log_type error log_type warning diff --git a/kubernetes/apps/home-assistant/mosquitto-deployment.yaml b/kubernetes/apps/home-assistant/mosquitto-deployment.yaml index 88c4a0b4..e907c4fa 100644 --- a/kubernetes/apps/home-assistant/mosquitto-deployment.yaml +++ b/kubernetes/apps/home-assistant/mosquitto-deployment.yaml @@ -17,9 +17,25 @@ spec: labels: app.kubernetes.io/name: mosquitto spec: - # No nodeSelector / tolerations — mosquitto-iot uses ETP=Cluster, so - # IoT-VLAN traffic landing on k8s-edge can be forwarded to a broker - # pod on any node. + # Runs on any node (no nodeSelector); the mosquitto-iot LB forwards + # IoT-VLAN traffic to the broker wherever it lands. + # Availability: home-critical priority (never preempted, reschedules + # first) + short fault tolerations so a node failure reschedules the + # broker in ~30s instead of the default ~5min. Single replica + Recreate + # stays (mosquitto can't run two brokers at once), but with persistence + # off there is no PVC to fence/detach/reattach, so it can restart + # anywhere instantly and no longer depends on Ceph being healthy. + priorityClassName: home-critical + terminationGracePeriodSeconds: 10 + tolerations: + - key: node.kubernetes.io/not-ready + operator: Exists + effect: NoExecute + tolerationSeconds: 30 + - key: node.kubernetes.io/unreachable + operator: Exists + effect: NoExecute + tolerationSeconds: 30 securityContext: fsGroup: 1883 containers: @@ -136,8 +152,6 @@ spec: - name: config mountPath: /mosquitto/config readOnly: true - - name: data - mountPath: /mosquitto/data - name: auth mountPath: /mosquitto/auth - name: tls @@ -162,9 +176,6 @@ spec: - name: config configMap: name: mosquitto-config - - name: data - persistentVolumeClaim: - claimName: mosquitto-data - name: auth emptyDir: {} - name: tls diff --git a/kubernetes/apps/home-assistant/mosquitto-pvc.yaml b/kubernetes/apps/home-assistant/mosquitto-pvc.yaml deleted file mode 100644 index cc796f87..00000000 --- a/kubernetes/apps/home-assistant/mosquitto-pvc.yaml +++ /dev/null @@ -1,15 +0,0 @@ ---- -apiVersion: v1 -kind: PersistentVolumeClaim -metadata: - name: mosquitto-data - namespace: home-assistant - annotations: - kustomize.toolkit.fluxcd.io/prune: disabled -spec: - accessModes: - - ReadWriteOnce - storageClassName: ceph-block - resources: - requests: - storage: 1Gi diff --git a/kubernetes/infrastructure/cluster-settings/kustomization.yaml b/kubernetes/infrastructure/cluster-settings/kustomization.yaml index 4e94b2d2..68557c12 100644 --- a/kubernetes/infrastructure/cluster-settings/kustomization.yaml +++ b/kubernetes/infrastructure/cluster-settings/kustomization.yaml @@ -2,3 +2,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - cluster-settings.yaml + - priority-classes.yaml diff --git a/kubernetes/infrastructure/cluster-settings/priority-classes.yaml b/kubernetes/infrastructure/cluster-settings/priority-classes.yaml new file mode 100644 index 00000000..ba037848 --- /dev/null +++ b/kubernetes/infrastructure/cluster-settings/priority-classes.yaml @@ -0,0 +1,11 @@ +apiVersion: scheduling.k8s.io/v1 +kind: PriorityClass +metadata: + name: home-critical +value: 1000000 +globalDefault: false +description: >- + Load-bearing home services (e.g. the MQTT broker the heat pump, energy + meters, EV charging and Home Assistant all depend on) that must reschedule + ahead of ordinary workloads and be evicted last under node pressure. Sits + well below system-cluster-critical (2e9) and above the default (0). -- 2.54.0