fix: [alert] KubeDeploymentRolloutStuck (ai) (#750)

.woodpecker/deploy-netbird-proxy.yml

@@ -7,8 +7,6 @@ when:
       # install-netbird-proxy.yml also applies the netbird-proxy-watchdog role,
       # so changes to it must redeploy this host too.
       - "ansible/roles/netbird-proxy-watchdog/**"
-      # install-netbird-proxy.yml also applies the edge bouncer.
-      - "ansible/roles/crowdsec-edge-bouncer/**"
       - "ansible/plays/install-netbird-proxy.yml"
       - ".woodpecker/deploy-netbird-proxy.yml"
   - event: deployment

ansible/inventory/group_vars/netbird_proxy/vault.yml

@@ -1,23 +0,0 @@
----
-# Bouncer key for crowdsec-edge-bouncer, scoped to the proxy hosts.
-#
-# A single encrypted VALUE rather than a whole-file vault like its siblings in
-# group_vars/all and group_vars/edge_host. Deliberate: adding this variable to
-# an existing whole-file vault means decrypting that file first, which exposes
-# every other secret in it for the sake of appending one line. encrypt_string
-# keeps the blast radius at this one value.
-#
-# Same value as OpenBao homelab/infrastructure/crowdsec-edge-bouncer-key, which
-# the crowdsec-edge-bouncer-key ExternalSecret feeds to BOUNCER_KEY_edge on the
-# LAPI. Both sides must move together — and unlike crowdsec-bouncer-key, this
-# one is NOT in the quarterly rotator, because nothing on these hosts re-reads
-# OpenBao on a schedule. Rotating it means updating OpenBao and re-running
-# plays/install-netbird-proxy.yml.
-vault_crowdsec_edge_bouncer_key: !vault |
-          $ANSIBLE_VAULT;1.1;AES256
-          32653764333536316163303838333839616562663862666637626165303837633866643033393239
-          3363376538656236643161386432326437633232643564630a633635306365323537346139626364
-          36613033343938363366306239363638383466396337623333623136313234656366333366623433
-          6437656632373364340a346566343866366433306533353431383064356362346536383136323663
-          39333864643534326433653233303137373162663066633132623931316135323733333863306330
-          6366623463666435636362376238313639666263323163303733

ansible/plays/install-netbird-proxy.yml

@@ -26,7 +26,3 @@
     - docker-install
     - netbird-proxy
     - netbird-proxy-watchdog
-    # Enforces the cluster's CrowdSec decisions here, where public traffic
-    # enters. Deployed in dry-run: it logs what it would block and creates no
-    # nftables rules until crowdsec_bouncer_mode is flipped per host.
-    - crowdsec-edge-bouncer

ansible/roles/crowdsec-edge-bouncer/defaults/main.yml

@@ -1,47 +0,0 @@
----
-# CrowdSec firewall bouncer on the NetBird proxy hosts.
-#
-# Enforces the decisions the in-cluster CrowdSec already detects, at the point
-# where public traffic actually enters: these hosts terminate TLS for every
-# *.renner.uno name. A drop here never reaches the mesh, the relay or the
-# cluster.
-#
-# Detection stays in the cluster. The reverse-proxy produces no parseable
-# access log and there is no CrowdSec collection for it, while Traefik has a
-# working one — and every scenario in use needs HTTP detail (paths, status
-# codes, user agents) that does not exist at this layer.
-
-# Pinned. A release tarball URL is immutable, and this checksum is of the
-# amd64 asset for exactly this tag.
-crowdsec_bouncer_version: v0.0.36
-crowdsec_bouncer_sha256: f86e4b72693549d99f40a9402abefb894108f047a3fbe6e72fada25ee17ce88b
-
-# The LAPI over the public path these hosts already serve, see
-# kubernetes/infrastructure/crowdsec/lapi-ingress.yaml. The proxies cannot
-# reach it over the mesh: the reverse-proxy container runs non-root, so its
-# embedded NetBird client terminates the tunnel in userspace and the tunnel
-# belongs to that process rather than to the host netns.
-#
-# No ca_cert_path: Traefik serves a Let's Encrypt certificate for this name,
-# and the bouncer adds ca_cert_path to the system pool rather than replacing
-# it — the system pool already trusts LE.
-crowdsec_bouncer_api_url: https://crowdsec.renner.uno/
-crowdsec_bouncer_api_key: "{{ vault_crowdsec_edge_bouncer_key }}"
-
-# STARTS INERT, and that is the point.
-#
-# dry-run pulls decisions and logs what it would block without creating a
-# single nftables rule. The first bouncer to arm anywhere enforces ~24k CAPI
-# decisions at once on the only external entry path, so the false-positive
-# window has to come before the enforcement, not after it. Flip to `nftables`
-# one host at a time once the log looks right.
-crowdsec_bouncer_mode: dry-run
-
-crowdsec_bouncer_update_frequency: 10s
-crowdsec_bouncer_log_level: info
-
-# Bound to loopback deliberately. These hosts are public and have no metrics
-# collection path at all today — opening :60601 to the internet to gain a
-# scrape target is the wrong trade. Revisit when host metrics exist.
-crowdsec_bouncer_metrics_listen: 127.0.0.1
-crowdsec_bouncer_metrics_port: 60601

ansible/roles/crowdsec-edge-bouncer/handlers/main.yml

@@ -1,10 +0,0 @@
----
-# A restart drops both nftables tables and rebuilds them from the next
-# successful LAPI pull, so there is a short window with no blocklist. That is
-# the documented behaviour of a clean shutdown, not a defect — and it is why
-# retry_initial_connect matters.
-- name: Restart crowdsec firewall bouncer
-  ansible.builtin.systemd:
-    name: crowdsec-firewall-bouncer.service
-    state: restarted
-    daemon_reload: true

ansible/roles/crowdsec-edge-bouncer/tasks/main.yml

@@ -1,111 +0,0 @@
----
-# See defaults/main.yml for why this exists and why it starts inert.
-#
-# The static release tarball, not a distribution package: the AUR package
-# `crowdsec-firewall-bouncer-nftables` depends on `crowdsec` and would pull the
-# whole Security Engine onto hosts that only need the bouncer, the group is
-# mixed (proxy-1 Arch, VPS-1 Debian), and the AUR would be a new pattern this
-# repo does not otherwise have. The tarball is the same artefact the AUR
-# package downloads.
-
-- name: Ensure nftables is present (the enforcing backend)
-  ansible.builtin.package:
-    name: nftables
-    state: present
-
-- name: Ensure the bouncer config directory exists
-  ansible.builtin.file:
-    path: /etc/crowdsec/bouncers
-    state: directory
-    mode: "0755"
-
-# Fetched on the CONTROLLER and pushed, not pulled by the host.
-#
-# proxy-1 is IPv6-only, and neither github.com nor objects.githubusercontent.com
-# publishes an AAAA record — a get_url on the host fails with "Network is
-# unreachable" and no amount of retrying fixes it. Mirroring into
-# gitea.renner.uno is no help either: its public records are the internal LB
-# VIP (10.2.99.86 / fd99::86), so it is unreachable from outside the LAN. And
-# upstream ships no container image for this bouncer, which is expected —
-# it manipulates host netfilter.
-#
-# The CI runner has v4, so it does the fetching. The checksum is verified
-# there, before anything is copied to a host.
-- name: Fetch the pinned bouncer release on the controller
-  ansible.builtin.get_url:
-    url: >-
-      https://github.com/crowdsecurity/cs-firewall-bouncer/releases/download/{{
-      crowdsec_bouncer_version }}/crowdsec-firewall-bouncer-linux-amd64.tgz
-    dest: /tmp/crowdsec-firewall-bouncer-{{ crowdsec_bouncer_version }}.tgz
-    checksum: sha256:{{ crowdsec_bouncer_sha256 }}
-    mode: "0644"
-  delegate_to: localhost
-  become: false
-  run_once: true
-
-# Only the download had to move to the controller — the extraction stays on
-# the host. The ci-ansible image ships a tar that ansible rejects ("detected as
-# tar type None. GNU tar required"), while both host distributions have GNU
-# tar. Shipping the archive rather than the unpacked binary also keeps the
-# artefact that the checksum was verified against intact all the way to the
-# host.
-- name: Copy the bouncer archive to the host
-  ansible.builtin.copy:
-    src: /tmp/crowdsec-firewall-bouncer-{{ crowdsec_bouncer_version }}.tgz
-    dest: /opt/crowdsec-firewall-bouncer-{{ crowdsec_bouncer_version }}.tgz
-    mode: "0644"
-
-# Only the binary. The tarball also carries install.sh / upgrade.sh /
-# uninstall.sh, which manage systemd and config themselves — that is this
-# role's job, and letting them run would leave two owners for the same files.
-- name: Unpack the bouncer binary
-  ansible.builtin.unarchive:
-    src: /opt/crowdsec-firewall-bouncer-{{ crowdsec_bouncer_version }}.tgz
-    dest: /opt
-    remote_src: true
-    include:
-      - crowdsec-firewall-bouncer-{{ crowdsec_bouncer_version }}/crowdsec-firewall-bouncer
-    creates: /opt/crowdsec-firewall-bouncer-{{ crowdsec_bouncer_version }}/crowdsec-firewall-bouncer
-
-- name: Install the bouncer binary
-  ansible.builtin.copy:
-    src: /opt/crowdsec-firewall-bouncer-{{ crowdsec_bouncer_version }}/crowdsec-firewall-bouncer
-    dest: /usr/local/bin/crowdsec-firewall-bouncer
-    remote_src: true
-    mode: "0755"
-  notify: Restart crowdsec firewall bouncer
-
-# 0600: this file carries the LAPI key.
-- name: Deploy the bouncer configuration
-  ansible.builtin.template:
-    src: crowdsec-firewall-bouncer.yaml.j2
-    dest: /etc/crowdsec/bouncers/crowdsec-firewall-bouncer.yaml
-    mode: "0600"
-  notify: Restart crowdsec firewall bouncer
-
-- name: Deploy the bouncer systemd unit
-  ansible.builtin.template:
-    src: crowdsec-firewall-bouncer.service.j2
-    dest: /etc/systemd/system/crowdsec-firewall-bouncer.service
-    mode: "0644"
-  notify: Restart crowdsec firewall bouncer
-
-- name: Enable and start the bouncer
-  ansible.builtin.systemd:
-    name: crowdsec-firewall-bouncer.service
-    enabled: true
-    state: started
-    daemon_reload: true
-
-- name: Report how to read it
-  ansible.builtin.debug:
-    msg: |
-      crowdsec-firewall-bouncer deployed in mode "{{ crowdsec_bouncer_mode }}".
-
-      What it would block:  journalctl -u crowdsec-firewall-bouncer -f
-      What it does block:   nft list table ip crowdsec ; nft list table ip6 crowdsec6
-      Metrics:              curl -s {{ crowdsec_bouncer_metrics_listen }}:{{ crowdsec_bouncer_metrics_port }}/metrics
-
-      Rollback: systemctl stop crowdsec-firewall-bouncer. A clean stop deletes
-      both nftables tables by itself. Only after a kill -9 do they need
-      removing by hand: nft delete table ip crowdsec ; nft delete table ip6 crowdsec6

ansible/roles/crowdsec-edge-bouncer/templates/crowdsec-firewall-bouncer.service.j2

@@ -1,24 +0,0 @@
-# Managed by Ansible (crowdsec-edge-bouncer role) — do not edit.
-#
-# Upstream's shipped unit orders itself After=crowdsec.service. There is no
-# local Security Engine on these hosts — the LAPI is in the cluster — so that
-# dependency is dropped; nss-lookup instead, because api_url is a public name
-# that has to resolve before the first connect.
-[Unit]
-Description=CrowdSec firewall bouncer (edge enforcement for the NetBird proxy)
-After=network-online.target nss-lookup.target
-Wants=network-online.target
-
-[Service]
-Type=notify
-# -t validates the config and exits non-zero on a bad one, so a broken config
-# fails the start instead of taking the running rules down with it.
-ExecStartPre=/usr/local/bin/crowdsec-firewall-bouncer -c /etc/crowdsec/bouncers/crowdsec-firewall-bouncer.yaml -t
-ExecStart=/usr/local/bin/crowdsec-firewall-bouncer -c /etc/crowdsec/bouncers/crowdsec-firewall-bouncer.yaml
-Restart=always
-RestartSec=10
-LimitNOFILE=65536
-KillMode=mixed
-
-[Install]
-WantedBy=multi-user.target

ansible/roles/crowdsec-edge-bouncer/templates/crowdsec-firewall-bouncer.yaml.j2

@@ -1,55 +0,0 @@
-# Managed by Ansible (crowdsec-edge-bouncer role) — do not edit.
-mode: {{ crowdsec_bouncer_mode }}
-update_frequency: {{ crowdsec_bouncer_update_frequency }}
-
-# stdout, so journald owns the log and there is no second rotation policy to
-# keep in sync. The shipped example writes to /var/log; that only matters for
-# hosts without journald.
-log_mode: stdout
-log_level: {{ crowdsec_bouncer_log_level }}
-
-api_url: {{ crowdsec_bouncer_api_url }}
-api_key: {{ crowdsec_bouncer_api_key }}
-
-# Without this the process EXITS when the LAPI is unreachable at startup —
-# there is no retry on the first connect. And because a clean shutdown deletes
-# the nftables tables, the systemd restart loop would leave the host with no
-# blocklist at all until the first successful connect. Present in the code,
-# absent from the shipped example config.
-retry_initial_connect: true
-
-disable_ipv4: false
-disable_ipv6: false
-deny_action: DROP
-
-supported_decisions_types:
-  - ban
-
-# Separate tables per family; the bouncer never touches an existing table or
-# chain, and never sets a chain policy. Priority -10 puts it ahead of anything
-# at the default filter priority.
-nftables:
-  ipv4:
-    enabled: true
-    set-only: false
-    table: crowdsec
-    chain: crowdsec-chain
-    priority: -10
-  ipv6:
-    enabled: true
-    set-only: false
-    table: crowdsec6
-    chain: crowdsec6-chain
-    priority: -10
-
-# input is the traffic this exists for: the reverse-proxy is host-networked and
-# owns :443, so its packets traverse INPUT. forward covers the bridge-networked
-# http-redirect sidecar, whose packets never do.
-nftables_hooks:
-  - input
-  - forward
-
-prometheus:
-  enabled: true
-  listen_addr: {{ crowdsec_bouncer_metrics_listen }}
-  listen_port: {{ crowdsec_bouncer_metrics_port }}

kubernetes/apps/llama-server/deployment.yaml

@@ -25,6 +25,13 @@ metadata:
     app.kubernetes.io/name: llama-server
 spec:
   replicas: 1
+  # The startupProbe below sanctions a 40 min cold download, but the default
+  # progressDeadlineSeconds is 600. Left implicit the Deployment declares
+  # ProgressDeadlineExceeded at 10 minutes while the probe is still legitimately
+  # waiting, and KubeDeploymentRolloutStuck fires 15 min after that on a rollout
+  # that is fine. Kept above the startup budget, with room for the image pull
+  # and scheduling that happen before the first probe.
+  progressDeadlineSeconds: 3000
   # A rolling update would load a second copy of the weights onto the same
   # iGPU and into the same node's RAM before releasing the first.
   strategy:

kubernetes/apps/llama-server/embed.yaml

@@ -19,6 +19,11 @@ metadata:
     app.kubernetes.io/name: llama-embed
 spec:
   replicas: 1
+  # Same reasoning as llama-server: the startupProbe budget (600s) must stay
+  # below progressDeadlineSeconds, whose default is also 600 — exactly equal, so
+  # any cold download that uses most of its budget trips
+  # KubeDeploymentRolloutStuck on a rollout that is still progressing normally.
+  progressDeadlineSeconds: 900
   strategy:
     type: Recreate
   selector:
@@ -41,7 +46,7 @@ spec:
           image: ghcr.io/ggml-org/llama.cpp:server-vulkan-b10731
           args:
             - --hf-repo
-            - gpustack/bge-m3-GGUF:Q8_0
+            - Qwen/Qwen3-Embedding-0.6B-GGUF:Q8_0
             # Serves POST /v1/embeddings.
             - --embeddings
             # A short, stable id for consumers. Without it the model is
@@ -50,41 +55,28 @@ spec:
             # One name only: llama-server keeps aliases in a sorted set and
             # reports the first as the id, so a second name can steal it.
             - --alias
-            - bge-m3
-            # bge-m3 is trained for CLS pooling — a different type from the
-            # Qwen family this replaces, which uses last-token. Left unspecified
-            # llama-server takes the GGUF default, which is not visible from
-            # outside and would silently produce vectors the model was not
-            # trained to emit. Read from the model's own 1_Pooling/config.json
-            # rather than assumed; of the candidates measured, three different
-            # pooling types were needed.
+            - qwen3-embedding-0.6b
+            # Qwen3-Embedding is trained for last-token pooling. Left
+            # unspecified llama-server takes the GGUF's default, which is not
+            # visible from outside and would silently produce vectors the model
+            # was not trained to emit. Set before anything is indexed: changing
+            # pooling later invalidates every stored embedding.
             - --pooling
-            - cls
+            - last
             - --n-gpu-layers
             - "99"
             - --threads
             - "2"
-            # Paperless chunks at 1024 tokens, but counts them with its own
-            # tokenizer while this server measures the batch in bge-m3's.
-            # Those disagree: XLM-RoBERTa splits German into more pieces than
-            # llama_index's counter does, so a chunk Paperless considers full
-            # at 1024 arrived here as 1069 and the request failed outright —
-            # "input (1069 tokens) is too large to process".
-            #
-            # The headroom therefore has to absorb a tokenizer ratio, not just
-            # a rounding error, and 1.5x covers far more than the 1.044 that
-            # broke it. The defaults are batch 2048 / ubatch 512, and with
-            # --embeddings llama-server clamps both to the smaller rather than
-            # run them mismatched, so both are set.
-            #
-            # 2048 stays avoided: that is where the chat model met
-            # ErrorDeviceLost on this Vulkan stack. That failure came under a
-            # 20 GB model's memory pressure and may not bind a 600 MB one, but
-            # this does not need to find out.
+            # Paperless chunks documents at 1024 tokens. The defaults are
+            # batch 2048 / ubatch 512, and with --embeddings llama-server
+            # clamps both down to 512 rather than run them mismatched — below
+            # the chunk size. Matching both to 1024 fits a whole chunk in one
+            # pass; 2048 is deliberately avoided, since that is where this
+            # Vulkan stack fails with ErrorDeviceLost.
             - --batch-size
-            - "1536"
+            - "1024"
             - --ubatch-size
-            - "1536"
+            - "1024"
             # Room for a full chunk plus overhead rather than exactly one.
             - --ctx-size
             - "4096"
@@ -104,14 +96,6 @@ spec:
             capabilities:
               drop:
                 - ALL
-          # Measured, not estimated: embedding 400 real documents peaked at
-          # 1.6 GiB (Prometheus max_over_time on the working set). The limit is
-          # well above that because the footprint is dominated by the four slots
-          # at 4096 context and the Vulkan working set rather than by the
-          # weights — the Qwen model this replaces peaked at 10.6 GiB on the
-          # same settings for the same 600 MiB of weights, and several
-          # candidates were OOMKilled at limits that looked generous against
-          # their model size.
           resources:
             requests:
               cpu: 250m
@@ -119,7 +103,7 @@ spec:
               gpu.intel.com/i915: "1"
             limits:
               cpu: "2"
-              memory: 8Gi
+              memory: 4Gi
               gpu.intel.com/i915: "1"
           # ~600 MiB rather than ~5 GiB, so a much shorter budget than the chat
           # model needs, but still sized for a cold download.

kubernetes/apps/paperless/paperless.yaml

@@ -82,10 +82,7 @@ data:
   # primarily English archives — the better fit for a German corpus.
   PAPERLESS_AI_LLM_EMBEDDING_BACKEND: "openai-like"
   PAPERLESS_AI_LLM_EMBEDDING_ENDPOINT: "http://llama-embed.ai.svc.cluster.local:8080/v1"
-  # Must match --alias in kubernetes/apps/llama-server/embed.yaml. Changing the
-  # embedding model invalidates every stored vector, so a swap means the LLM
-  # index has to be rebuilt.
-  PAPERLESS_AI_LLM_EMBEDDING_MODEL: "bge-m3"
+  PAPERLESS_AI_LLM_EMBEDDING_MODEL: "qwen3-embedding-0.6b"
   PAPERLESS_AI_LLM_EMBEDDING_CHUNK_SIZE: "1024"
 ---
 apiVersion: apps/v1

kubernetes/infrastructure/crowdsec/edge-bouncer-key-externalsecret.yaml

@@ -1,27 +0,0 @@
-# Bouncer key for the firewall bouncers on proxy-1/proxy-2.
-#
-# Separate from crowdsec-bouncer-key ON PURPOSE. That key is rotated
-# quarterly by kubernetes/infrastructure/openbao/rotation/crowdsec-bouncer-key.yaml,
-# and both of its consumers live in the cluster, so ESO propagates a rotation
-# within 10m. The edge bouncers are Ansible-managed hosts: nothing there
-# re-reads OpenBao on a schedule, so a rotated key would leave them 401ing
-# against the LAPI until someone runs the playbook. This key is therefore
-# deliberately NOT enrolled in the rotator — it rotates when the playbook runs,
-# and the playbook is what carries the new value to the hosts.
-#
-# The value must match ansible vault_crowdsec_edge_bouncer_key.
-apiVersion: external-secrets.io/v1
-kind: ExternalSecret
-metadata:
-  name: crowdsec-edge-bouncer-key
-  namespace: crowdsec
-spec:
-  refreshInterval: 10m
-  secretStoreRef:
-    kind: ClusterSecretStore
-    name: openbao
-  target:
-    name: crowdsec-edge-bouncer-key
-  dataFrom:
-    - extract:
-        key: infrastructure/crowdsec-edge-bouncer-key

kubernetes/infrastructure/crowdsec/helmrelease.yaml

@@ -36,36 +36,6 @@ spec:
   values:
     container_runtime: containerd
 
-    # Addresses a community blocklist must never be allowed to ban.
-    #
-    # This is the ONLY declarative allowlist the chart offers. `cscli
-    # allowlists` objects live in the LAPI database, and that database runs
-    # without persistence here on purpose (see lapi.persistentVolume below),
-    # so anything created through cscli is gone at the next restart. This file
-    # is a ConfigMap mount, so it survives by construction. Upstream marks it
-    # deprecated in favour of centralized allowlists; it is still the only
-    # thing that works against a LAPI with no PVC.
-    #
-    # Scope: CAPI decisions only. Locally generated decisions are not filtered
-    # here — they currently carry real client IPs (verified), because Traefik
-    # resolves X-Forwarded-For past the netbird-ingress pod IP.
-    #
-    # The home WAN address is deliberately absent and is not needed: LAN
-    # clients never present it. opentofu/unifi_nat_hairpin.tf DNATs the public
-    # proxy addresses to the Traefik VIP on the UDM, so on-LAN traffic reaches
-    # the cluster directly with a LAN source and never traverses the proxies.
-    # Private ranges cannot appear in a community blocklist either way.
-    config:
-      capi_whitelists.yaml: |
-        cidrs:
-          # The proxy hosts themselves. Same three addresses as the
-          # netbird-ingress-egress toCIDRSet and the hairpin endpoints —
-          # keep all three lists in sync. proxy-1 is the v6-only member,
-          # proxy-2 the dual-stack one.
-          - 45.157.177.26/32
-          - 2a03:4000:4b:c1e:4474:43ff:fedf:8f85/128
-          - 2a03:4000:28:f6e:a4c7:59ff:fe1e:9a29/128
-
     # mTLS between agents and the LAPI. Do not switch back to password auth:
     # the LAPI runs without persistence, so its machine-registration DB is
     # wiped on every restart, and password-registered agents then get 401
@@ -116,18 +86,10 @@ spec:
         requests:
           cpu: 50m
       podAnnotations:
-        # Reloader restarts the LAPI when either bouncer key changes. Both are
-        # consumed as env vars from a secretKeyRef, and env is fixed at pod
-        # start — without a restart the LAPI keeps trusting the old value while
-        # the bouncer already presents the new one, which is a 401 at exactly
-        # the moment a rotation is supposed to be seamless.
-        #
-        # crowdsec-bouncer-key rotates on a schedule, see
+        # Reloader restarts the LAPI when crowdsec-bouncer-key changes (it's an
+        # env-var secret, so a restart is required to pick up a rotated key). See
         # kubernetes/infrastructure/openbao/rotation/crowdsec-bouncer-key.yaml.
-        # crowdsec-edge-bouncer-key rotates by hand together with
-        # ansible/plays/install-netbird-proxy.yml; it still needs to be listed
-        # here, because the LAPI side of that rotation is this env var.
-        secret.reloader.stakater.com/reload: crowdsec-bouncer-key,crowdsec-edge-bouncer-key
+        secret.reloader.stakater.com/reload: crowdsec-bouncer-key
       persistentVolume:
         data:
           # Kept OFF deliberately. The #337 attempt to set this `true` (persist
@@ -156,17 +118,6 @@ spec:
             secretKeyRef:
               name: crowdsec-bouncer-key
               key: key
-        # The firewall bouncers on proxy-1/proxy-2, over the public LAPI
-        # endpoint (lapi-ingress.yaml). The chart's entrypoint re-runs
-        # `register_bouncer` for every BOUNCER_KEY_* on each start and skips
-        # names that already exist, so this survives the LAPI's wiped
-        # registration DB the same way BOUNCER_KEY_ingress does — the env var
-        # is the source of truth, not the database.
-        - name: BOUNCER_KEY_edge
-          valueFrom:
-            secretKeyRef:
-              name: crowdsec-edge-bouncer-key
-              key: key
 
     agent:
       # Same reasoning as lapi.resources above, multiplied by three: this is a

kubernetes/infrastructure/crowdsec/kustomization.yaml

@@ -2,6 +2,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
   - bouncer-key-externalsecret.yaml
-  - edge-bouncer-key-externalsecret.yaml
-  - lapi-ingress.yaml
   - helmrelease.yaml

kubernetes/infrastructure/crowdsec/lapi-ingress.yaml

@@ -1,81 +0,0 @@
-# Public LAPI endpoint for the firewall bouncers on proxy-1/proxy-2.
-#
-# WHY PUBLIC AT ALL. The bouncers run as host processes on the proxies, and
-# the proxies have no mesh access: the netbird-proxy container runs non-root
-# (see the ip_unprivileged_port_start task in the netbird-proxy role), so it
-# has no NET_ADMIN and no /dev/net/tun, and its embedded NetBird client
-# therefore terminates the tunnel in userspace. The tunnel belongs to that
-# process, not to the host netns — a second process on the same host cannot
-# route through it. The alternative was a host-level NetBird client plus the
-# LAPI on a LoadBalancer VIP; this reuses the path every other service already
-# takes instead of adding two peers and an ACL policy.
-#
-# NO IP ALLOWLIST, deliberately. The bouncer key is the access control.
-#
-# An allowlist was the obvious second factor and it does not work here. A
-# request originating on a proxy host does not arrive carrying that host's
-# public address: the client connects to the local proxy on 127.0.0.1, the
-# reverse-proxy puts what it saw into X-Forwarded-For, and Traefik resolves
-# that. Measured against the netbird-proxy-watchdog, which makes exactly this
-# kind of request every round:
-#
-#   172.18.0.3 - - [02/Sep/2026:10:12:01] "GET /" 200 websecure-immich-...
-#   172.18.0.3 - - [02/Sep/2026:10:12:01] "GET /" 200 websecure-home-...
-#
-# 172.18.0.3 is a Docker bridge address on the proxy side. Allowlisting the
-# published proxy addresses would have rejected the bouncer with 403;
-# allowlisting 172.18.0.0/16 would be a private range on a host we do not
-# control the addressing of, which is not a security control.
-#
-# What remains is the API key, which is the real control anyway: this endpoint
-# serves read-only decision streams to a valid bouncer key and nothing else.
-# The admin surface is cscli, in-cluster. If the source address ever becomes
-# deterministic, an ipAllowList can be added on top.
----
-# The LAPI serves HTTPS only once tls.enabled is set — there is no plaintext
-# fallback and no second port; 8080 is the TLS port. Its certificate comes
-# from the chart's own self-signed CA, so Traefik needs that CA to verify the
-# backend. SANs include crowdsec-service.crowdsec.svc.cluster.local, which is
-# what the Service resolves to.
-apiVersion: traefik.io/v1alpha1
-kind: ServersTransport
-metadata:
-  name: crowdsec-lapi
-  namespace: crowdsec
-spec:
-  serverName: crowdsec-service.crowdsec.svc.cluster.local
-  rootCAsSecrets:
-    - crowdsec-root-secret
----
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
-  name: crowdsec-lapi
-  namespace: crowdsec
-  labels:
-    # Public DNS via Cloudflare; the proxies resolve it from outside. The UDM
-    # external-dns instance skips it, LAN reaches it via the hairpin NAT.
-    dns.renner.uno/public: "true"
-  annotations:
-    cert-manager.io/cluster-issuer: letsencrypt-cluster
-    # NOT behind the Authentik gate: the client is a daemon with an API key,
-    # not a browser that can complete an OIDC flow.
-    traefik.ingress.kubernetes.io/service.serversscheme: https
-    traefik.ingress.kubernetes.io/service.serverstransport: crowdsec-crowdsec-lapi@kubernetescrd
-spec:
-  ingressClassName: traefik
-  rules:
-    - host: crowdsec.renner.uno
-      http:
-        paths:
-          - path: /
-            pathType: Prefix
-            backend:
-              service:
-                name: crowdsec-service
-                port:
-                  number: 8080
-  tls:
-    - secretName: crowdsec-lapi-public-tls
-      hosts:
-        - crowdsec.renner.uno