From 2155edd3b41386831e619c082b68ea8061e6592b Mon Sep 17 00:00:00 2001 From: Konstantin Renner Date: Wed, 8 Jul 2026 11:10:41 +0200 Subject: [PATCH 1/2] mosquitto: put the broker image under Flux image-automation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The broker ran on a floating `eclipse-mosquitto:2` tag — no reproducibility, and a reschedule could pull a changed/broken image. Rather than hand-pin a digest (which would then never get patched), add it to the repo's Flux image-automation like redis et al.: - ImageRepository + ImagePolicy tracking eclipse-mosquitto, semver range 2.x (auto-bump minor+patch; a 3.0 major stays a deliberate migration), matching plain-semver tags only (excludes -openssl/latest/bare 2). - The deployment image carries the pinned tag + the $imagepolicy marker, so each bump lands as a git commit and rolls the broker (brief restart; discovery self-heals via HA). Net: the broker is both pinned (reproducible) and auto-patched. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../home-assistant/mosquitto-deployment.yaml | 6 ++++- .../image-automation/kustomization.yaml | 1 + .../image-automation/mosquitto.yaml | 26 +++++++++++++++++++ 3 files changed, 32 insertions(+), 1 deletion(-) create mode 100644 kubernetes/infrastructure/image-automation/mosquitto.yaml diff --git a/kubernetes/apps/home-assistant/mosquitto-deployment.yaml b/kubernetes/apps/home-assistant/mosquitto-deployment.yaml index e907c4fa..b23e18ab 100644 --- a/kubernetes/apps/home-assistant/mosquitto-deployment.yaml +++ b/kubernetes/apps/home-assistant/mosquitto-deployment.yaml @@ -40,7 +40,11 @@ spec: fsGroup: 1883 containers: - name: mosquitto - image: eclipse-mosquitto:2 + # Pinned tag, auto-bumped within the 2.x line by Flux image-automation + # (infrastructure/image-automation/mosquitto.yaml). The marker tells + # Flux where to write; patch/minor updates land as git commits and + # roll the broker (brief restart; discovery self-heals via HA). + image: eclipse-mosquitto:2.1.2 # {"$imagepolicy": "flux-system:mosquitto"} # The shim runs as root (image sets no USER) and prepares the # auth files for the broker, which drops to user mosquitto # (UID 1883) before reading them — so files must be owned by diff --git a/kubernetes/infrastructure/image-automation/kustomization.yaml b/kubernetes/infrastructure/image-automation/kustomization.yaml index 8e26aae6..d581d942 100644 --- a/kubernetes/infrastructure/image-automation/kustomization.yaml +++ b/kubernetes/infrastructure/image-automation/kustomization.yaml @@ -17,6 +17,7 @@ resources: - paperless-ngx.yaml - protonmail-bridge.yaml - pve-exporter.yaml + - mosquitto.yaml - redis.yaml - shelly-manager.yaml - tika.yaml diff --git a/kubernetes/infrastructure/image-automation/mosquitto.yaml b/kubernetes/infrastructure/image-automation/mosquitto.yaml new file mode 100644 index 00000000..7001893f --- /dev/null +++ b/kubernetes/infrastructure/image-automation/mosquitto.yaml @@ -0,0 +1,26 @@ +apiVersion: image.toolkit.fluxcd.io/v1 +kind: ImageRepository +metadata: + name: mosquitto + namespace: flux-system +spec: + image: eclipse-mosquitto + interval: 1h +--- +# Stay on the mosquitto 2.x line — auto-bump minor+patch releases; a 3.0 major +# would be a deliberate migration. Plain-semver tags only, which excludes the +# `-openssl`, `latest`, and bare `2` tags. +apiVersion: image.toolkit.fluxcd.io/v1 +kind: ImagePolicy +metadata: + name: mosquitto + namespace: flux-system +spec: + imageRepositoryRef: + name: mosquitto + filterTags: + pattern: '^(?P\d+\.\d+\.\d+)$' + extract: '$ver' + policy: + semver: + range: "2.x" -- 2.54.0 From d0c8044a674e24c188cd3bd584668eca0d04cb21 Mon Sep 17 00:00:00 2001 From: Konstantin Renner Date: Wed, 8 Jul 2026 11:44:47 +0200 Subject: [PATCH 2/2] mosquitto: alert on a bad image bump + roll without a downtime gap MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit image-automation commits tag bumps straight to master with no CI gate, and the broker is a single pod — so an image that fails to come up would be silent. Two changes make that safe: - gatus TCP-checks the broker's ClusterIP on :1883, so a pod that never comes up alerts within one interval. Reachability needed a scoped egress (apps ns) + ingress (home-assistant ns, gatus pod on :1883 only), mirroring the existing evcc -> mosquitto:1883 allow. - strategy Recreate -> RollingUpdate (maxUnavailable:0, maxSurge:1). With the PVC gone there is no RWO volume to fence, so the new broker comes up and passes its readiness probe before the old one is torn down — no full-down window on a bump/reschedule; clients on the old pod just reconnect. The brief two-pod overlap is two independent stateless brokers behind the LB, fine with no shared state + self-healing discovery. Co-Authored-By: Claude Opus 4.8 (1M context) --- kubernetes/apps/gatus/configmap.yaml | 12 ++++++++++++ .../home-assistant/mosquitto-deployment.yaml | 19 ++++++++++++++----- .../network-policies/default-deny.yaml | 11 +++++++++++ .../network-policies/home-assistant-ns.yaml | 12 ++++++++++++ 4 files changed, 49 insertions(+), 5 deletions(-) diff --git a/kubernetes/apps/gatus/configmap.yaml b/kubernetes/apps/gatus/configmap.yaml index a944942e..ca5a3672 100644 --- a/kubernetes/apps/gatus/configmap.yaml +++ b/kubernetes/apps/gatus/configmap.yaml @@ -124,5 +124,17 @@ data: - "[DNS_RCODE] == NOERROR" - "[RESPONSE_TIME] < 500" + # Mosquitto MQTT broker (ClusterIP, home-assistant ns). A TCP-connect + # check on :1883: the broker's image tag is auto-bumped by Flux + # image-automation straight to master (no CI gate), and the deployment + # is a single Recreate/rolling pod, so a bad bump that fails to come up + # would otherwise be silent. This turns that into an alert. + - name: Mosquitto (MQTT broker) + group: infrastructure + url: tcp://mosquitto.home-assistant.svc.cluster.local:1883 + interval: 60s + conditions: + - "[CONNECTED] == true" + # Removed 2026-05-21: PostgreSQL HA (VIP) tcp://10.2.40.135:5432 — # VIP no longer responds (no TCP, no ICMP). Re-add if it returns. diff --git a/kubernetes/apps/home-assistant/mosquitto-deployment.yaml b/kubernetes/apps/home-assistant/mosquitto-deployment.yaml index b23e18ab..4d331cc4 100644 --- a/kubernetes/apps/home-assistant/mosquitto-deployment.yaml +++ b/kubernetes/apps/home-assistant/mosquitto-deployment.yaml @@ -8,7 +8,16 @@ metadata: spec: replicas: 1 strategy: - type: Recreate + # No PVC anymore, so nothing to fence: the new broker can come up and pass + # its readiness probe before the old one is torn down. maxUnavailable:0 keeps + # a broker serving throughout an image bump / reschedule (clients on the old + # pod just reconnect to the new one); maxSurge:1 permits the brief two-pod + # overlap — two independent stateless brokers behind the LB, fine since there + # is no shared state and discovery self-heals. + type: RollingUpdate + rollingUpdate: + maxUnavailable: 0 + maxSurge: 1 selector: matchLabels: app.kubernetes.io/name: mosquitto @@ -21,10 +30,10 @@ spec: # IoT-VLAN traffic to the broker wherever it lands. # Availability: home-critical priority (never preempted, reschedules # first) + short fault tolerations so a node failure reschedules the - # broker in ~30s instead of the default ~5min. Single replica + Recreate - # stays (mosquitto can't run two brokers at once), but with persistence - # off there is no PVC to fence/detach/reattach, so it can restart - # anywhere instantly and no longer depends on Ceph being healthy. + # broker in ~30s instead of the default ~5min. With persistence off there + # is no PVC to fence/detach/reattach, so the broker can (re)start anywhere + # instantly, no longer depends on Ceph being healthy, and rolls without a + # tear-down-first gap (see the RollingUpdate strategy above). priorityClassName: home-critical terminationGracePeriodSeconds: 10 tolerations: diff --git a/kubernetes/infrastructure/network-policies/default-deny.yaml b/kubernetes/infrastructure/network-policies/default-deny.yaml index 50ee7ff5..b47ff68c 100644 --- a/kubernetes/infrastructure/network-policies/default-deny.yaml +++ b/kubernetes/infrastructure/network-policies/default-deny.yaml @@ -34,6 +34,17 @@ spec: - ports: - port: "53" protocol: UDP + # gatus health-checks the mosquitto broker (home-assistant ns) on :1883. + # The broker's own ingress policy scopes this to the gatus pod, so this + # namespace-wide egress allow is gatus-only in effect. + - toEndpoints: + - matchLabels: + "k8s:io.kubernetes.pod.namespace": home-assistant + app.kubernetes.io/name: mosquitto + toPorts: + - ports: + - port: "1883" + protocol: TCP - toEntities: - host - remote-node diff --git a/kubernetes/infrastructure/network-policies/home-assistant-ns.yaml b/kubernetes/infrastructure/network-policies/home-assistant-ns.yaml index 7e84d73d..0c7fdfb4 100644 --- a/kubernetes/infrastructure/network-policies/home-assistant-ns.yaml +++ b/kubernetes/infrastructure/network-policies/home-assistant-ns.yaml @@ -41,6 +41,18 @@ spec: - ports: - port: "1883" protocol: TCP + # gatus (apps ns) TCP-checks that the broker is listening on :1883 — an + # availability probe that fires after image-automation bumps the + # eclipse-mosquitto tag, so an image that fails to come up alerts instead + # of failing silently. Scoped to the gatus pod on :1883. + - fromEndpoints: + - matchLabels: + "k8s:io.kubernetes.pod.namespace": apps + app: gatus + toPorts: + - ports: + - port: "1883" + protocol: TCP # Kubelet probes (host) and hostNetwork siblings (remote-node). - fromEntities: - host -- 2.54.0