Merge pull request 'feat(crowdsec): automated bouncer-key rotation (openbao-ns)' (#303) from feat/crowdsec-bouncer-key-rotation into master

Reviewed-on: k.renner/homelab#303

kubernetes/infrastructure/crowdsec/bouncer-key-externalsecret.yaml

@@ -9,7 +9,7 @@ metadata:
   name: crowdsec-bouncer-key
   namespace: crowdsec
 spec:
-  refreshInterval: 1h
+  refreshInterval: 10m # 10m so an automated rotation propagates within ~10m
   secretStoreRef:
     kind: ClusterSecretStore
     name: openbao
@@ -25,7 +25,7 @@ metadata:
   name: crowdsec-bouncer-key
   namespace: ingress
 spec:
-  refreshInterval: 1h
+  refreshInterval: 10m # 10m so an automated rotation propagates within ~10m
   secretStoreRef:
     kind: ClusterSecretStore
     name: openbao

kubernetes/infrastructure/crowdsec/helmrelease.yaml

@@ -33,6 +33,11 @@ spec:
 
     lapi:
       replicas: 1
+      podAnnotations:
+        # Reloader restarts the LAPI when crowdsec-bouncer-key changes (it's an
+        # env-var secret, so a restart is required to pick up a rotated key). See
+        # kubernetes/infrastructure/openbao/rotation/crowdsec-bouncer-key.yaml.
+        secret.reloader.stakater.com/reload: crowdsec-bouncer-key
       persistentVolume:
         data:
           enabled: false

kubernetes/infrastructure/openbao/policies/rotator-crowdsec-bouncer.hcl

@@ -0,0 +1,22 @@
+# OpenBao policy: rotator-crowdsec-bouncer
+#
+# Least-privilege policy for the crowdsec-bouncer-key rotation CronJob (kubernetes
+# auth role `rotator-crowdsec-bouncer`). Per-secret isolation — scoped to only the
+# crowdsec-bouncer-key KV path (issue #213 blast-radius, #202 per-app segregation).
+# openbao-ns rotation pattern; see rotator-paperless.hcl for the template.
+#
+# NOT applied by Flux. Apply ONCE with an admin token:
+#
+#   bao policy write rotator-crowdsec-bouncer kubernetes/infrastructure/openbao/policies/rotator-crowdsec-bouncer.hcl
+#   bao write auth/kubernetes/role/rotator-crowdsec-bouncer \
+#     bound_service_account_names=crowdsec-secret-rotator \
+#     bound_service_account_namespaces=openbao \
+#     policies=rotator-crowdsec-bouncer ttl=15m
+#
+# Do NOT set token_no_default_policy=true (kv-v2 preflight needs the default policy).
+#
+# NOTE: this KV entry lives under infrastructure/, not apps/. Its `key` field
+# feeds both the crowdsec and ingress namespace Secrets (same KV path).
+path "homelab/data/infrastructure/crowdsec-bouncer-key" {
+  capabilities = ["read", "patch"]
+}

kubernetes/infrastructure/openbao/rotation/crowdsec-bouncer-key.yaml

@@ -0,0 +1,95 @@
+---
+# Automated rotation of crowdsec-bouncer-key (the pre-shared key the CrowdSec
+# LAPI trusts for the ingress bouncer). openbao-ns pattern (intra-namespace
+# OpenBao access, no NetworkPolicy). Writes OpenBao; the single KV path feeds
+# BOTH the crowdsec- and ingress-namespace Secrets via ESO (10m refresh), and
+# Reloader restarts the crowdsec-lapi Deployment (env-var secret → restart needed;
+# the ingress-ns copy has no live consumer today).
+#
+# ⚠️ There is a THIRD, hardcoded copy of this key in
+# kubernetes/infrastructure/traefik/crowdsec-middleware.yaml (crowdsecLapiKey),
+# NOT sourced from the Secret. That middleware is currently INERT; a guard comment
+# there warns it must be refreshed from OpenBao before ever being re-enabled.
+#
+# ONE-TIME BOOTSTRAP (per rotator-crowdsec-bouncer.hcl): apply the policy + role.
+# ON-DEMAND: kubectl -n openbao create job --from=cronjob/crowdsec-bouncer-key-rotate crowdsec-bouncer-key-rotate-now
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: crowdsec-secret-rotator
+  namespace: openbao
+  labels:
+    app.kubernetes.io/name: crowdsec
+    app.kubernetes.io/component: secret-rotator
+  # OpenBao: auth/kubernetes/role/rotator-crowdsec-bouncer binds this SA to the
+  # rotator-crowdsec-bouncer policy. Talks only to OpenBao → no k8s RBAC needed.
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: crowdsec-bouncer-key-rotate
+  namespace: openbao
+  labels:
+    app.kubernetes.io/name: crowdsec
+    app.kubernetes.io/component: secret-rotator
+spec:
+  schedule: "0 4 1 1,4,7,10 *" # quarterly, Europe/Berlin
+  timeZone: Europe/Berlin
+  concurrencyPolicy: Forbid
+  successfulJobsHistoryLimit: 2
+  failedJobsHistoryLimit: 5
+  startingDeadlineSeconds: 3600
+  jobTemplate:
+    spec:
+      backoffLimit: 2
+      activeDeadlineSeconds: 300
+      ttlSecondsAfterFinished: 259200 # 3 d
+      template:
+        metadata:
+          labels:
+            app.kubernetes.io/name: crowdsec
+            app.kubernetes.io/component: secret-rotator
+        spec:
+          serviceAccountName: crowdsec-secret-rotator
+          restartPolicy: Never
+          securityContext:
+            runAsNonRoot: true
+            runAsUser: 100
+            runAsGroup: 1000
+            fsGroup: 1000
+            seccompProfile:
+              type: RuntimeDefault
+          containers:
+            - name: rotate
+              image: quay.io/openbao/openbao:2.5.5
+              securityContext:
+                allowPrivilegeEscalation: false
+                readOnlyRootFilesystem: true
+                capabilities:
+                  drop: ["ALL"]
+              env:
+                - name: HOME
+                  value: /tmp
+              command: ["/bin/sh", "-ec"]
+              args:
+                - |
+                  export BAO_ADDR=http://openbao.openbao.svc.cluster.local:8200
+                  BAO_TOKEN=$(bao write -field=token auth/kubernetes/login \
+                    role=rotator-crowdsec-bouncer \
+                    jwt="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)")
+                  export BAO_TOKEN
+                  NEW=$(head -c 32 /dev/urandom | base64 | tr -d '\n')
+                  bao kv patch homelab/infrastructure/crowdsec-bouncer-key "key=${NEW}"
+                  echo "rotated key at homelab/infrastructure/crowdsec-bouncer-key (value not logged)"
+              volumeMounts:
+                - name: tmp
+                  mountPath: /tmp
+              resources:
+                requests:
+                  cpu: 25m
+                  memory: 32Mi
+                limits:
+                  memory: 64Mi
+          volumes:
+            - name: tmp
+              emptyDir: {}

kubernetes/infrastructure/openbao/rotation/kustomization.yaml

@@ -9,3 +9,4 @@ resources:
   - woodpecker.yaml
   - shelly-manager.yaml
   - authentik.yaml
+  - crowdsec-bouncer-key.yaml

kubernetes/infrastructure/traefik/crowdsec-middleware.yaml

@@ -46,6 +46,13 @@ spec:
       crowdsecLapiHost: crowdsec-service.crowdsec.svc.cluster.local:8080
       # Static pre-shared bouncer key, pre-registered with the LAPI via the
       # crowdsec HelmRelease env BOUNCER_KEY_ingress. Same value as the
-      # crowdsec-bouncer-key Secret (already committed in plaintext as a
-      # low-sensitivity internal token).
+      # crowdsec-bouncer-key Secret.
+      #
+      # ⚠️ LANDMINE: crowdsec-bouncer-key now AUTO-ROTATES (see
+      # kubernetes/infrastructure/openbao/rotation/crowdsec-bouncer-key.yaml).
+      # This literal is NOT sourced from that Secret, so it goes stale on every
+      # rotation. This middleware is currently INERT (not attached to any router),
+      # so nothing breaks today — but BEFORE re-enabling it, refresh this value
+      # from the current OpenBao key (`bao kv get homelab/infrastructure/crowdsec-bouncer-key`)
+      # or the bouncer will 401/403 against the LAPI with a stale key.
       crowdsecLapiKey: "k8s-homelab-ingress-bouncer-key-v1"