apiVersion: source.toolkit.fluxcd.io/v1 kind: HelmRepository metadata: name: crowdsec namespace: crowdsec spec: interval: 24h url: https://crowdsecurity.github.io/helm-charts --- apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: name: crowdsec namespace: crowdsec spec: interval: 30m chart: spec: chart: crowdsec # Pinned to the 0.24.x line: patch updates allowed, minor/major bumps # not. The old floating ">=0.12.0" silently rolled the release forward to # 0.24.0 (whose chart defaults differ, e.g. lapi.persistentVolume now # defaults on) with no review — a surprise upgrade that fed into the #340 # outage. Bump this deliberately after checking the chart changelog. version: "0.24.x" sourceRef: kind: HelmRepository name: crowdsec namespace: crowdsec install: remediation: retries: 3 upgrade: remediation: retries: 3 values: container_runtime: containerd # Addresses a community blocklist must never be allowed to ban. # # This is the ONLY declarative allowlist the chart offers. `cscli # allowlists` objects live in the LAPI database, and that database runs # without persistence here on purpose (see lapi.persistentVolume below), # so anything created through cscli is gone at the next restart. This file # is a ConfigMap mount, so it survives by construction. Upstream marks it # deprecated in favour of centralized allowlists; it is still the only # thing that works against a LAPI with no PVC. # # Scope: CAPI decisions only. Locally generated decisions are not filtered # here — they currently carry real client IPs (verified), because Traefik # resolves X-Forwarded-For past the netbird-ingress pod IP. # # The home WAN address is deliberately absent and is not needed: LAN # clients never present it. opentofu/unifi_nat_hairpin.tf DNATs the public # proxy addresses to the Traefik VIP on the UDM, so on-LAN traffic reaches # the cluster directly with a LAN source and never traverses the proxies. # Private ranges cannot appear in a community blocklist either way. config: capi_whitelists.yaml: | cidrs: # The proxy hosts themselves. Same three addresses as the # netbird-ingress-egress toCIDRSet and the hairpin endpoints — # keep all three lists in sync. proxy-1 is the v6-only member, # proxy-2 the dual-stack one. - 45.157.177.26/32 - 2a03:4000:4b:c1e:4474:43ff:fedf:8f85/128 - 2a03:4000:28:f6e:a4c7:59ff:fe1e:9a29/128 # netbird.renner.uno — management, signal, relay and TURN. # # Not optional once a bouncer is armed. Its chain sits on the INPUT # hook, the proxy hosts run no other firewall, and there is therefore # no conntrack rule accepting established flows ahead of it. A ban on # this address would drop the RETURN traffic of the proxies' own # outbound control connection and take them off the mesh — with no # inbound attack involved at all. # # Checked when this was added: none of these addresses appear in the # current decision set. That is the state today, not a guarantee — # these are rented addresses on a hoster whose ranges do end up on # community blocklists. - 162.55.50.97/32 - 2a01:4f8:c2c:4579::1/128 # mTLS between agents and the LAPI. Do not switch back to password auth: # the LAPI runs without persistence, so its machine-registration DB is # wiped on every restart, and password-registered agents then get 401 # "machine not found" and crashloop until re-registered by hand — which # wedges the `crowdsec` Kustomization behind a HelmRelease # that never goes Ready (#340). # # With client certs the agent never registers: a restarted LAPI with an # empty DB recreates each machine record from the presented cert, so this # self-heals with no PVC and no intervention. # # cert-manager (in `infrastructure`, which the `crowdsec` Kustomization # depends on) provisions a fully self-contained self-signed CA (selfSigned # ClusterIssuer -> CA cert -> CA Issuer -> lapi/agent/bouncer leaf certs); # empty issuerRef => the chart mints that CA itself, so nothing external is # required. # # NOTE: enabling this makes the LAPI serve HTTPS-only. The Traefik ingress # bouncer is currently INERT (detached, see post-crds/traefik-crowdsec-middleware.yaml) # so there is no live impact, but re-enabling it will require pointing the # plugin at https + trusting this CA (or giving it a bouncer-ou client cert). tls: enabled: true caBundle: true certManager: enabled: true # Empty => chart provisions its own self-signed CA (self-contained). issuerRef: {} # Chart 0.24.0 bug: the agent/bouncer/appsec Certificate templates emit # `secretTemplate:` unconditionally while its child blocks are guarded, # so with nothing set it renders as null and cert-manager's webhook # rejects it ("spec.secretTemplate ... must be of type object"). (The # lapi cert template guards the whole key, which is why only the others # failed.) Setting a label makes secretTemplate a valid non-null object. secretTemplate: labels: app.kubernetes.io/part-of: crowdsec agent: tlsClientAuth: true lapi: replicas: 1 service: # Backend wiring for the public LAPI Ingress (lapi-ingress.yaml). # # These belong on the SERVICE, not on the Ingress: under the Kubernetes # Ingress provider `traefik.ingress.kubernetes.io/service.*` are service # annotations, and on an Ingress they are silently ignored. The symptom # is Traefik speaking plaintext to a TLS-only backend, which the LAPI # answers with "Client sent an HTTP request to an HTTPS server." and a # 400 — an error that looks like a malformed request rather than a # scheme mismatch. # # With tls.enabled the LAPI serves HTTPS on 8080 and offers no # plaintext port, so https here is mandatory rather than a hardening # choice. The ServersTransport carries the chart's own CA, since that # certificate is self-signed. annotations: traefik.ingress.kubernetes.io/service.serversscheme: https traefik.ingress.kubernetes.io/service.serverstransport: crowdsec-crowdsec-lapi@kubernetescrd # Far below the chart default, which reserves half a core per LAPI for a # workload that draws single-digit millicores — enough to leave the # cluster unschedulable with one node out. Only the reservation shrinks; # the limit stays at the chart default so a log flood can still burst. resources: requests: cpu: 50m podAnnotations: # Reloader restarts the LAPI when either bouncer key changes. Both are # consumed as env vars from a secretKeyRef, and env is fixed at pod # start — without a restart the LAPI keeps trusting the old value while # the bouncer already presents the new one, which is a 401 at exactly # the moment a rotation is supposed to be seamless. # # crowdsec-bouncer-key rotates on a schedule, see # kubernetes/infrastructure/openbao/rotation/crowdsec-bouncer-key.yaml. # crowdsec-edge-bouncer-key rotates by hand together with # ansible/plays/install-netbird-proxy.yml; it still needs to be listed # here, because the LAPI side of that rotation is this env var. secret.reloader.stakater.com/reload: crowdsec-bouncer-key,crowdsec-edge-bouncer-key persistentVolume: data: # Kept OFF deliberately. The #337 attempt to set this `true` (persist # the sqlite/ent machine+bouncer DB across LAPI restarts) is the # change that thrashed in the #340 outage: it reintroduces the exact # Ceph bootstrap circular-dependency 629a303d removed (crowdsec was # then in the `infrastructure` stage that must go Ready before the # CephCluster, so a ceph-block PVC deadlocked a cold boot; its own # Kustomization does not depend on infrastructure-ceph-cluster # either), and it doesn't even heal # the agents (a fresh persistent volume starts with an EMPTY DB, so # already-running agents still can't authenticate). # # The durable fix is mTLS auth (tls: block above): agent credentials # are cert-based, not DB-stored, so the ephemeral DB is now harmless — # a wiped/restarted LAPI re-derives each machine from the client cert # on connect, and the bouncer key is re-registered from # BOUNCER_KEY_ingress on every LAPI start. No PVC, no Ceph dependency. enabled: false config: enabled: false env: - name: ENROLL_INSTANCE_NAME value: "k8s-homelab" - name: BOUNCER_KEY_ingress valueFrom: secretKeyRef: name: crowdsec-bouncer-key key: key # The firewall bouncers on proxy-1/proxy-2, over the public LAPI # endpoint (lapi-ingress.yaml). The chart's entrypoint re-runs # `register_bouncer` for every BOUNCER_KEY_* on each start and skips # names that already exist, so this survives the LAPI's wiped # registration DB the same way BOUNCER_KEY_ingress does — the env var # is the source of truth, not the database. - name: BOUNCER_KEY_edge valueFrom: secretKeyRef: name: crowdsec-edge-bouncer-key key: key agent: # Same reasoning as lapi.resources above, multiplied by three: this is a # DaemonSet, so the chart's 500m default reserves half a core on every # node to run a process that measures 1-2m. resources: requests: cpu: 50m # The agent parses the ingress controller's access logs (read from # the pod's stdout) to detect attacks. Repointed from the retired # ingress-nginx (decommissioned 2026-05-17) to Traefik. The # crowdsecurity/traefik collection bundles the matching log parser, # so no separate PARSERS entry is needed. acquisition: - namespace: ingress podName: traefik-* program: traefik # Read by polling, not inotify. Every path under # /var/log/containers is a symlink into /var/log/pods, and kubelet # rotates the target once it reaches containerLogMaxSize (10Mi, # roughly every eight hours for Traefik here). An inotify tailer # does not survive that rotation: it reopens the path, waits for a # file that is in fact already present, and never reads another # line. Nothing about that state looks broken from outside — the # pod stays Running and Ready, the LAPI keeps answering, and the # cumulative counters keep displaying whatever they reached, so the # only symptom is that no scenario ever fires again. # # CrowdSec detects the hazard itself and logs it at startup, with # this exact remedy in the message: "is a symlink, but inotify # polling is enabled. Crowdsec will not be able to detect rotation. # Consider setting poll_without_inotify to true". Upstream carries # the same case as crowdsec#2449 (k3s + Traefik), and the chart # gained this field in helm-charts#109 specifically to fix it. # # Orthogonal to the force_inotify: true that the chart hardcodes # into acquis.yaml and offers no way to unset. That one governs the # fsnotify watch on the directory, i.e. discovering the log of a # replacement Traefik pod; this one governs how an already # discovered file is read. Both true is the supported combination. # # Costs CPU per open file per upstream docs — two files per node. poll_without_inotify: true env: - name: COLLECTIONS value: "crowdsecurity/traefik crowdsecurity/http-cve crowdsecurity/appsec-virtual-patching crowdsecurity/appsec-generic-rules"