kubernetes/infrastructure/crowdsec/helmrelease.yaml

@@ -17,7 +17,12 @@ spec:
   chart:
     spec:
       chart: crowdsec
-      version: ">=0.12.0"
+      # Pinned to the 0.24.x line: patch updates allowed, minor/major bumps
+      # not. The old floating ">=0.12.0" silently rolled the release forward to
+      # 0.24.0 (whose chart defaults differ, e.g. lapi.persistentVolume now
+      # defaults on) with no review — a surprise upgrade that fed into the #340
+      # outage. Bump this deliberately after checking the chart changelog.
+      version: "0.24.x"
       sourceRef:
         kind: HelmRepository
         name: crowdsec
@@ -31,6 +36,46 @@ spec:
   values:
     container_runtime: containerd
 
+    # --- mTLS between agents and the LAPI: the durable fix for the recurring
+    # KubeDaemonSetRolloutStuck auth loop (#340) ---
+    #
+    # The LAPI runs WITHOUT persistence (see persistentVolume note below), so
+    # its sqlite machine-registration DB is ephemeral and every LAPI pod restart
+    # wiped it. Under the old password flow the agents registered exactly ONCE
+    # (init container running `cscli lapi register --token`) and cached a
+    # password the wiped LAPI no longer recognised -> 401 "machine not found"
+    # -> CrashLoopBackOff on all three agents -> DaemonSet never Ready -> the
+    # crowdsec HelmRelease upgrade `--wait` times out forever -> the whole
+    # `infrastructure` Kustomization (and every app depending on it) stays
+    # unhealthy. That is the #340 outage.
+    #
+    # With client-cert auth the agent init container no longer registers at all
+    # (verified via `helm template`: the `cscli lapi register` step is gone).
+    # The agent presents a cert whose OU (agent-ou) the LAPI trusts, and a
+    # freshly restarted LAPI with an EMPTY DB auto-recreates each machine record
+    # from the presented cert on connect. So the system self-heals across LAPI
+    # restarts with no PVC, no Ceph, and no manual re-registration.
+    #
+    # cert-manager (already reconciled earlier in this same `infrastructure`
+    # stage) provisions a fully self-contained self-signed CA (selfSigned
+    # ClusterIssuer -> CA cert -> CA Issuer -> lapi/agent/bouncer leaf certs);
+    # empty issuerRef => the chart mints that CA itself, so nothing external is
+    # required.
+    #
+    # NOTE: enabling this makes the LAPI serve HTTPS-only. The Traefik ingress
+    # bouncer is currently INERT (detached, see traefik/crowdsec-middleware.yaml)
+    # so there is no live impact, but re-enabling it will require pointing the
+    # plugin at https + trusting this CA (or giving it a bouncer-ou client cert).
+    tls:
+      enabled: true
+      caBundle: true
+      certManager:
+        enabled: true
+        # Empty => chart provisions its own self-signed CA (self-contained).
+        issuerRef: {}
+      agent:
+        tlsClientAuth: true
+
     lapi:
       replicas: 1
       podAnnotations:
@@ -40,12 +85,22 @@ spec:
         secret.reloader.stakater.com/reload: crowdsec-bouncer-key
       persistentVolume:
         data:
-          # Must survive LAPI pod replacement: the sqlite/ent DB here holds
-          # machine (agent) and bouncer registrations. Without this, any LAPI
-          # restart wipes the DB while agents keep retrying login with their
-          # now-invalid cached credentials, exiting fatally instead of
-          # re-registering (see issue #337).
-          enabled: true
+          # Kept OFF deliberately. The #337 attempt to set this `true` (persist
+          # the sqlite/ent machine+bouncer DB across LAPI restarts) is the
+          # change that thrashed in the #340 outage: it reintroduces the exact
+          # Ceph bootstrap circular-dependency 629a303d removed (crowdsec is in
+          # the `infrastructure` stage that must go Ready before the CephCluster
+          # — infrastructure-ceph-cluster dependsOn: infrastructure — so a
+          # ceph-block PVC here deadlocks a cold boot), and it doesn't even heal
+          # the agents (a fresh persistent volume starts with an EMPTY DB, so
+          # already-running agents still can't authenticate).
+          #
+          # The durable fix is mTLS auth (tls: block above): agent credentials
+          # are cert-based, not DB-stored, so the ephemeral DB is now harmless —
+          # a wiped/restarted LAPI re-derives each machine from the client cert
+          # on connect, and the bouncer key is re-registered from
+          # BOUNCER_KEY_ingress on every LAPI start. No PVC, no Ceph dependency.
+          enabled: false
         config:
           enabled: false
       env:

kubernetes/infrastructure/traefik/crowdsec-middleware.yaml

@@ -42,6 +42,13 @@ spec:
       forwardedHeadersTrustedIPs:
         - "10.244.0.0/16"
         - "fd10:244::/56"
+      # ⚠️ LANDMINE #2: the LAPI now serves HTTPS-only (mTLS was enabled on the
+      # crowdsec HelmRelease to fix the agent auth loop, #340). This `http`
+      # scheme is therefore stale and will fail once this middleware is
+      # re-attached. BEFORE re-enabling: switch to `https` and give the plugin
+      # the CA trust (crowdsecLapiTLSCertificateAuthority from the crowdsec CA,
+      # secret crowdsec-root-secret / key ca.crt) — or issue it a bouncer-ou
+      # client cert. Harmless today only because this middleware is INERT.
       crowdsecLapiScheme: http
       crowdsecLapiHost: crowdsec-service.crowdsec.svc.cluster.local:8080
       # Static pre-shared bouncer key, pre-registered with the LAPI via the