diff --git a/kubernetes/clusters/homelab/uptime-monitor.yaml b/kubernetes/clusters/homelab/uptime-monitor.yaml index ea351bcc..395c29cd 100644 --- a/kubernetes/clusters/homelab/uptime-monitor.yaml +++ b/kubernetes/clusters/homelab/uptime-monitor.yaml @@ -14,9 +14,28 @@ # in the critical path of most of the cluster. Here, the only thing a missing or # misnamed key can stall is this Kustomization, which nothing depends on. # -# Same reason `wait: true` is kept below rather than dropped: the wait is how a -# key that failed to sync becomes visible, and it is safe precisely because the -# blast radius is this Kustomization alone. +# `wait: true` is kept below rather than dropped, but ON ITS OWN IT DOES NOT +# MAKE A FAILED SYNC VISIBLE — an earlier revision of this comment claimed it +# did, and that was wrong (REN-100). Flux ships no status reader for +# `ExternalSecret`, so kstatus falls back to its generic rule, which looks only +# at `status.observedGeneration` and at conditions of type `Reconciling` / +# `Stalled`. ESO sets none of those three: a failed sync is reported as +# `Ready=False, reason=SecretSyncedError`. kstatus therefore returns Current and +# the Kustomization goes Ready with no Secret behind it. +# +# In-repo proof rather than a doc citation: `llama-server` is also `wait: true` +# and carries the `ai/wiki-embed-basic-auth` ExternalSecret +# (apps/llama-server/wiki-embed-ingress.yaml). That key sat in ESO retry backoff +# through ~10 failures on 2026-09-20 and needed a manual `force-sync`. Its +# Kustomization stayed Ready throughout, which is why a human had to notice. +# +# So `wait: true` is paired with the `healthCheckExprs` below, which is the +# mechanism this repo already uses for CRs kstatus cannot read — CNPG `Cluster` +# (paperless, immich, paperclip, authentik, woodpecker, monitoring), MariaDB +# (wordpress) and the Ceph CRs (infrastructure). Now a key that fails to sync +# holds this Kustomization NotReady, and it is safe to fail loudly here +# precisely because the blast radius is this Kustomization alone: nothing +# `dependsOn` it. apiVersion: kustomize.toolkit.fluxcd.io/v1 kind: Kustomization metadata: @@ -35,6 +54,30 @@ spec: path: ./kubernetes/apps/uptime-monitor prune: true wait: true + # No name/namespace qualifier, so this is evaluated against every + # ExternalSecret in this Kustomization's inventory. Today that is exactly one + # (apps/uptime-monitor/externalsecret.yaml); when the IMC Deployment joins in + # REN-6 it is still one, and any ExternalSecret added here later should be held + # to the same bar anyway. + # + # `external-secrets.io/v1` is the version every one of the 54 ExternalSecret + # manifests in this repo is written against, so it is the GVK Flux records in + # the inventory and the one that has to be matched here. + # + # Both expressions are guarded with `has(status.conditions)`, copying + # wordpress.yaml. Without the guard, CEL errors instead of evaluating while + # `status` is still empty — which is the state a freshly applied ExternalSecret + # is in, i.e. exactly the case this check exists for. Unguarded, the + # Kustomization would report a CEL evaluation error rather than a clean + # in-progress. + # + # No `inProgress` expression: Flux treats "neither failed nor current" as + # in-progress, which is the behaviour wanted here. + healthCheckExprs: + - apiVersion: external-secrets.io/v1 + kind: ExternalSecret + failed: "has(status.conditions) && status.conditions.exists(c, c.type == 'Ready' && c.status == 'False')" + current: "has(status.conditions) && status.conditions.exists(c, c.type == 'Ready' && c.status == 'True')" dependsOn: # ESO itself, and the `uptime-monitor` namespace (declared centrally in # kubernetes/infrastructure/namespaces/namespaces.yaml).