diff --git a/kubernetes/infrastructure/crowdsec/helmrelease.yaml b/kubernetes/infrastructure/crowdsec/helmrelease.yaml index d3b4ec00..df3d5c6f 100644 --- a/kubernetes/infrastructure/crowdsec/helmrelease.yaml +++ b/kubernetes/infrastructure/crowdsec/helmrelease.yaml @@ -17,7 +17,12 @@ spec: chart: spec: chart: crowdsec - version: ">=0.12.0" + # Pinned to the 0.24.x line: patch updates allowed, minor/major bumps + # not. The old floating ">=0.12.0" silently rolled the release forward to + # 0.24.0 (whose chart defaults differ, e.g. lapi.persistentVolume now + # defaults on) with no review — a surprise upgrade that fed into the #340 + # outage. Bump this deliberately after checking the chart changelog. + version: "0.24.x" sourceRef: kind: HelmRepository name: crowdsec @@ -31,6 +36,46 @@ spec: values: container_runtime: containerd + # --- mTLS between agents and the LAPI: the durable fix for the recurring + # KubeDaemonSetRolloutStuck auth loop (#340) --- + # + # The LAPI runs WITHOUT persistence (see persistentVolume note below), so + # its sqlite machine-registration DB is ephemeral and every LAPI pod restart + # wiped it. Under the old password flow the agents registered exactly ONCE + # (init container running `cscli lapi register --token`) and cached a + # password the wiped LAPI no longer recognised -> 401 "machine not found" + # -> CrashLoopBackOff on all three agents -> DaemonSet never Ready -> the + # crowdsec HelmRelease upgrade `--wait` times out forever -> the whole + # `infrastructure` Kustomization (and every app depending on it) stays + # unhealthy. That is the #340 outage. + # + # With client-cert auth the agent init container no longer registers at all + # (verified via `helm template`: the `cscli lapi register` step is gone). + # The agent presents a cert whose OU (agent-ou) the LAPI trusts, and a + # freshly restarted LAPI with an EMPTY DB auto-recreates each machine record + # from the presented cert on connect. So the system self-heals across LAPI + # restarts with no PVC, no Ceph, and no manual re-registration. + # + # cert-manager (already reconciled earlier in this same `infrastructure` + # stage) provisions a fully self-contained self-signed CA (selfSigned + # ClusterIssuer -> CA cert -> CA Issuer -> lapi/agent/bouncer leaf certs); + # empty issuerRef => the chart mints that CA itself, so nothing external is + # required. + # + # NOTE: enabling this makes the LAPI serve HTTPS-only. The Traefik ingress + # bouncer is currently INERT (detached, see traefik/crowdsec-middleware.yaml) + # so there is no live impact, but re-enabling it will require pointing the + # plugin at https + trusting this CA (or giving it a bouncer-ou client cert). + tls: + enabled: true + caBundle: true + certManager: + enabled: true + # Empty => chart provisions its own self-signed CA (self-contained). + issuerRef: {} + agent: + tlsClientAuth: true + lapi: replicas: 1 podAnnotations: @@ -40,12 +85,22 @@ spec: secret.reloader.stakater.com/reload: crowdsec-bouncer-key persistentVolume: data: - # Must survive LAPI pod replacement: the sqlite/ent DB here holds - # machine (agent) and bouncer registrations. Without this, any LAPI - # restart wipes the DB while agents keep retrying login with their - # now-invalid cached credentials, exiting fatally instead of - # re-registering (see issue #337). - enabled: true + # Kept OFF deliberately. The #337 attempt to set this `true` (persist + # the sqlite/ent machine+bouncer DB across LAPI restarts) is the + # change that thrashed in the #340 outage: it reintroduces the exact + # Ceph bootstrap circular-dependency 629a303d removed (crowdsec is in + # the `infrastructure` stage that must go Ready before the CephCluster + # — infrastructure-ceph-cluster dependsOn: infrastructure — so a + # ceph-block PVC here deadlocks a cold boot), and it doesn't even heal + # the agents (a fresh persistent volume starts with an EMPTY DB, so + # already-running agents still can't authenticate). + # + # The durable fix is mTLS auth (tls: block above): agent credentials + # are cert-based, not DB-stored, so the ephemeral DB is now harmless — + # a wiped/restarted LAPI re-derives each machine from the client cert + # on connect, and the bouncer key is re-registered from + # BOUNCER_KEY_ingress on every LAPI start. No PVC, no Ceph dependency. + enabled: false config: enabled: false env: diff --git a/kubernetes/infrastructure/traefik/crowdsec-middleware.yaml b/kubernetes/infrastructure/traefik/crowdsec-middleware.yaml index dd0ed25f..843d360a 100644 --- a/kubernetes/infrastructure/traefik/crowdsec-middleware.yaml +++ b/kubernetes/infrastructure/traefik/crowdsec-middleware.yaml @@ -42,6 +42,13 @@ spec: forwardedHeadersTrustedIPs: - "10.244.0.0/16" - "fd10:244::/56" + # ⚠️ LANDMINE #2: the LAPI now serves HTTPS-only (mTLS was enabled on the + # crowdsec HelmRelease to fix the agent auth loop, #340). This `http` + # scheme is therefore stale and will fail once this middleware is + # re-attached. BEFORE re-enabling: switch to `https` and give the plugin + # the CA trust (crowdsecLapiTLSCertificateAuthority from the crowdsec CA, + # secret crowdsec-root-secret / key ca.crt) — or issue it a bouncer-ou + # client cert. Harmless today only because this middleware is INERT. crowdsecLapiScheme: http crowdsecLapiHost: crowdsec-service.crowdsec.svc.cluster.local:8080 # Static pre-shared bouncer key, pre-registered with the LAPI via the