fix(llama-embed): raise the memory limit to match its own context and batch

#739 raised the context from 2048 to 4096 and the micro-batch from an effective
512 to 1024, both to match how Paperless chunks documents. It left memory at
2Gi/4Gi, which was sized for the smaller values.

The pod ran fine interactively and was OOMKilled (exit 137) partway through the
first sustained run, around 300 documents into an embedding pass. Nothing in the
server log marked it — the process was killed outright, so the last line is a
half-finished task and the reason is only visible in the container's
lastState.terminated.

This would have surfaced when Paperless first indexed the archive, i.e. on the
first real use rather than under test.

The weights are only ~600 MiB. What sets the footprint is four slots at 4096
context with a 1024-token micro-batch, plus Vulkan allocating the iGPU working
set from system RAM where it counts against this cgroup — the same effect that
put llama-server at 32Gi for 19 GiB of weights.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

.woodpecker/deploy-netbird-proxy.yml

@@ -7,8 +7,6 @@ when:
       # install-netbird-proxy.yml also applies the netbird-proxy-watchdog role,
       # so changes to it must redeploy this host too.
       - "ansible/roles/netbird-proxy-watchdog/**"
-      # install-netbird-proxy.yml also applies the edge bouncer.
-      - "ansible/roles/crowdsec-edge-bouncer/**"
       - "ansible/plays/install-netbird-proxy.yml"
       - ".woodpecker/deploy-netbird-proxy.yml"
   - event: deployment

ansible/inventory/group_vars/netbird_proxy/vault.yml

@@ -1,23 +0,0 @@
----
-# Bouncer key for crowdsec-edge-bouncer, scoped to the proxy hosts.
-#
-# A single encrypted VALUE rather than a whole-file vault like its siblings in
-# group_vars/all and group_vars/edge_host. Deliberate: adding this variable to
-# an existing whole-file vault means decrypting that file first, which exposes
-# every other secret in it for the sake of appending one line. encrypt_string
-# keeps the blast radius at this one value.
-#
-# Same value as OpenBao homelab/infrastructure/crowdsec-edge-bouncer-key, which
-# the crowdsec-edge-bouncer-key ExternalSecret feeds to BOUNCER_KEY_edge on the
-# LAPI. Both sides must move together — and unlike crowdsec-bouncer-key, this
-# one is NOT in the quarterly rotator, because nothing on these hosts re-reads
-# OpenBao on a schedule. Rotating it means updating OpenBao and re-running
-# plays/install-netbird-proxy.yml.
-vault_crowdsec_edge_bouncer_key: !vault |
-          $ANSIBLE_VAULT;1.1;AES256
-          32653764333536316163303838333839616562663862666637626165303837633866643033393239
-          3363376538656236643161386432326437633232643564630a633635306365323537346139626364
-          36613033343938363366306239363638383466396337623333623136313234656366333366623433
-          6437656632373364340a346566343866366433306533353431383064356362346536383136323663
-          39333864643534326433653233303137373162663066633132623931316135323733333863306330
-          6366623463666435636362376238313639666263323163303733

ansible/plays/install-netbird-proxy.yml

@@ -26,7 +26,3 @@
     - docker-install
     - netbird-proxy
     - netbird-proxy-watchdog
-    # Enforces the cluster's CrowdSec decisions here, where public traffic
-    # enters. Deployed in dry-run: it logs what it would block and creates no
-    # nftables rules until crowdsec_bouncer_mode is flipped per host.
-    - crowdsec-edge-bouncer

ansible/roles/crowdsec-edge-bouncer/defaults/main.yml

@@ -1,47 +0,0 @@
----
-# CrowdSec firewall bouncer on the NetBird proxy hosts.
-#
-# Enforces the decisions the in-cluster CrowdSec already detects, at the point
-# where public traffic actually enters: these hosts terminate TLS for every
-# *.renner.uno name. A drop here never reaches the mesh, the relay or the
-# cluster.
-#
-# Detection stays in the cluster. The reverse-proxy produces no parseable
-# access log and there is no CrowdSec collection for it, while Traefik has a
-# working one — and every scenario in use needs HTTP detail (paths, status
-# codes, user agents) that does not exist at this layer.
-
-# Pinned. A release tarball URL is immutable, and this checksum is of the
-# amd64 asset for exactly this tag.
-crowdsec_bouncer_version: v0.0.36
-crowdsec_bouncer_sha256: f86e4b72693549d99f40a9402abefb894108f047a3fbe6e72fada25ee17ce88b
-
-# The LAPI over the public path these hosts already serve, see
-# kubernetes/infrastructure/crowdsec/lapi-ingress.yaml. The proxies cannot
-# reach it over the mesh: the reverse-proxy container runs non-root, so its
-# embedded NetBird client terminates the tunnel in userspace and the tunnel
-# belongs to that process rather than to the host netns.
-#
-# No ca_cert_path: Traefik serves a Let's Encrypt certificate for this name,
-# and the bouncer adds ca_cert_path to the system pool rather than replacing
-# it — the system pool already trusts LE.
-crowdsec_bouncer_api_url: https://crowdsec.renner.uno/
-crowdsec_bouncer_api_key: "{{ vault_crowdsec_edge_bouncer_key }}"
-
-# STARTS INERT, and that is the point.
-#
-# dry-run pulls decisions and logs what it would block without creating a
-# single nftables rule. The first bouncer to arm anywhere enforces ~24k CAPI
-# decisions at once on the only external entry path, so the false-positive
-# window has to come before the enforcement, not after it. Flip to `nftables`
-# one host at a time once the log looks right.
-crowdsec_bouncer_mode: dry-run
-
-crowdsec_bouncer_update_frequency: 10s
-crowdsec_bouncer_log_level: info
-
-# Bound to loopback deliberately. These hosts are public and have no metrics
-# collection path at all today — opening :60601 to the internet to gain a
-# scrape target is the wrong trade. Revisit when host metrics exist.
-crowdsec_bouncer_metrics_listen: 127.0.0.1
-crowdsec_bouncer_metrics_port: 60601

ansible/roles/crowdsec-edge-bouncer/handlers/main.yml

@@ -1,10 +0,0 @@
----
-# A restart drops both nftables tables and rebuilds them from the next
-# successful LAPI pull, so there is a short window with no blocklist. That is
-# the documented behaviour of a clean shutdown, not a defect — and it is why
-# retry_initial_connect matters.
-- name: Restart crowdsec firewall bouncer
-  ansible.builtin.systemd:
-    name: crowdsec-firewall-bouncer.service
-    state: restarted
-    daemon_reload: true

ansible/roles/crowdsec-edge-bouncer/tasks/main.yml

@@ -1,84 +0,0 @@
----
-# See defaults/main.yml for why this exists and why it starts inert.
-#
-# The static release tarball, not a distribution package: the AUR package
-# `crowdsec-firewall-bouncer-nftables` depends on `crowdsec` and would pull the
-# whole Security Engine onto hosts that only need the bouncer, the group is
-# mixed (proxy-1 Arch, VPS-1 Debian), and the AUR would be a new pattern this
-# repo does not otherwise have. The tarball is the same artefact the AUR
-# package downloads.
-
-- name: Ensure nftables is present (the enforcing backend)
-  ansible.builtin.package:
-    name: nftables
-    state: present
-
-- name: Ensure the bouncer config directory exists
-  ansible.builtin.file:
-    path: /etc/crowdsec/bouncers
-    state: directory
-    mode: "0755"
-
-- name: Fetch the pinned bouncer release
-  ansible.builtin.get_url:
-    url: >-
-      https://github.com/crowdsecurity/cs-firewall-bouncer/releases/download/{{
-      crowdsec_bouncer_version }}/crowdsec-firewall-bouncer-linux-amd64.tgz
-    dest: /opt/crowdsec-firewall-bouncer-{{ crowdsec_bouncer_version }}.tgz
-    checksum: sha256:{{ crowdsec_bouncer_sha256 }}
-    mode: "0644"
-
-# Only the binary. The tarball also carries install.sh / upgrade.sh /
-# uninstall.sh, which manage systemd and config themselves — that is this
-# role's job, and letting them run would leave two owners for the same files.
-- name: Unpack the bouncer binary
-  ansible.builtin.unarchive:
-    src: /opt/crowdsec-firewall-bouncer-{{ crowdsec_bouncer_version }}.tgz
-    dest: /opt
-    remote_src: true
-    include:
-      - crowdsec-firewall-bouncer-{{ crowdsec_bouncer_version }}/crowdsec-firewall-bouncer
-    creates: /opt/crowdsec-firewall-bouncer-{{ crowdsec_bouncer_version }}/crowdsec-firewall-bouncer
-
-- name: Install the bouncer binary
-  ansible.builtin.copy:
-    src: /opt/crowdsec-firewall-bouncer-{{ crowdsec_bouncer_version }}/crowdsec-firewall-bouncer
-    dest: /usr/local/bin/crowdsec-firewall-bouncer
-    remote_src: true
-    mode: "0755"
-  notify: Restart crowdsec firewall bouncer
-
-# 0600: this file carries the LAPI key.
-- name: Deploy the bouncer configuration
-  ansible.builtin.template:
-    src: crowdsec-firewall-bouncer.yaml.j2
-    dest: /etc/crowdsec/bouncers/crowdsec-firewall-bouncer.yaml
-    mode: "0600"
-  notify: Restart crowdsec firewall bouncer
-
-- name: Deploy the bouncer systemd unit
-  ansible.builtin.template:
-    src: crowdsec-firewall-bouncer.service.j2
-    dest: /etc/systemd/system/crowdsec-firewall-bouncer.service
-    mode: "0644"
-  notify: Restart crowdsec firewall bouncer
-
-- name: Enable and start the bouncer
-  ansible.builtin.systemd:
-    name: crowdsec-firewall-bouncer.service
-    enabled: true
-    state: started
-    daemon_reload: true
-
-- name: Report how to read it
-  ansible.builtin.debug:
-    msg: |
-      crowdsec-firewall-bouncer deployed in mode "{{ crowdsec_bouncer_mode }}".
-
-      What it would block:  journalctl -u crowdsec-firewall-bouncer -f
-      What it does block:   nft list table ip crowdsec ; nft list table ip6 crowdsec6
-      Metrics:              curl -s {{ crowdsec_bouncer_metrics_listen }}:{{ crowdsec_bouncer_metrics_port }}/metrics
-
-      Rollback: systemctl stop crowdsec-firewall-bouncer. A clean stop deletes
-      both nftables tables by itself. Only after a kill -9 do they need
-      removing by hand: nft delete table ip crowdsec ; nft delete table ip6 crowdsec6

ansible/roles/crowdsec-edge-bouncer/templates/crowdsec-firewall-bouncer.service.j2

@@ -1,24 +0,0 @@
-# Managed by Ansible (crowdsec-edge-bouncer role) — do not edit.
-#
-# Upstream's shipped unit orders itself After=crowdsec.service. There is no
-# local Security Engine on these hosts — the LAPI is in the cluster — so that
-# dependency is dropped; nss-lookup instead, because api_url is a public name
-# that has to resolve before the first connect.
-[Unit]
-Description=CrowdSec firewall bouncer (edge enforcement for the NetBird proxy)
-After=network-online.target nss-lookup.target
-Wants=network-online.target
-
-[Service]
-Type=notify
-# -t validates the config and exits non-zero on a bad one, so a broken config
-# fails the start instead of taking the running rules down with it.
-ExecStartPre=/usr/local/bin/crowdsec-firewall-bouncer -c /etc/crowdsec/bouncers/crowdsec-firewall-bouncer.yaml -t
-ExecStart=/usr/local/bin/crowdsec-firewall-bouncer -c /etc/crowdsec/bouncers/crowdsec-firewall-bouncer.yaml
-Restart=always
-RestartSec=10
-LimitNOFILE=65536
-KillMode=mixed
-
-[Install]
-WantedBy=multi-user.target

ansible/roles/crowdsec-edge-bouncer/templates/crowdsec-firewall-bouncer.yaml.j2

@@ -1,55 +0,0 @@
-# Managed by Ansible (crowdsec-edge-bouncer role) — do not edit.
-mode: {{ crowdsec_bouncer_mode }}
-update_frequency: {{ crowdsec_bouncer_update_frequency }}
-
-# stdout, so journald owns the log and there is no second rotation policy to
-# keep in sync. The shipped example writes to /var/log; that only matters for
-# hosts without journald.
-log_mode: stdout
-log_level: {{ crowdsec_bouncer_log_level }}
-
-api_url: {{ crowdsec_bouncer_api_url }}
-api_key: {{ crowdsec_bouncer_api_key }}
-
-# Without this the process EXITS when the LAPI is unreachable at startup —
-# there is no retry on the first connect. And because a clean shutdown deletes
-# the nftables tables, the systemd restart loop would leave the host with no
-# blocklist at all until the first successful connect. Present in the code,
-# absent from the shipped example config.
-retry_initial_connect: true
-
-disable_ipv4: false
-disable_ipv6: false
-deny_action: DROP
-
-supported_decisions_types:
-  - ban
-
-# Separate tables per family; the bouncer never touches an existing table or
-# chain, and never sets a chain policy. Priority -10 puts it ahead of anything
-# at the default filter priority.
-nftables:
-  ipv4:
-    enabled: true
-    set-only: false
-    table: crowdsec
-    chain: crowdsec-chain
-    priority: -10
-  ipv6:
-    enabled: true
-    set-only: false
-    table: crowdsec6
-    chain: crowdsec6-chain
-    priority: -10
-
-# input is the traffic this exists for: the reverse-proxy is host-networked and
-# owns :443, so its packets traverse INPUT. forward covers the bridge-networked
-# http-redirect sidecar, whose packets never do.
-nftables_hooks:
-  - input
-  - forward
-
-prometheus:
-  enabled: true
-  listen_addr: {{ crowdsec_bouncer_metrics_listen }}
-  listen_port: {{ crowdsec_bouncer_metrics_port }}

kubernetes/RUNBOOK-baremetal-conversion.md

@@ -2,12 +2,7 @@
 
 Converts one MS-01 from "Proxmox host running a Talos VM" to "Talos on bare
 metal", and swaps its Ceph OSD onto a Samsung PM983 at the same time. Written
-after doing jupiter on 2026-08-25.
-
-All three nodes are now on bare metal — `talosctl get systeminformation`
-reports Minisforum MS-01 hardware with distinct chassis serials for k8s-1,
-k8s-2 and k8s-3, with no hypervisor in between. Kept as the procedure for a
-rebuild, not as outstanding work.
+after doing jupiter on 2026-08-25; saturn and neptune remain.
 
 Everything here that reads like an over-specific warning is something that
 actually happened. The conversion itself is straightforward — the failures were

kubernetes/apps/home-assistant/MIGRATION-recorder-postgres.md

@@ -111,23 +111,9 @@ statistics             max_id=13323904    max_start_ts=1787162400.0
 statistics_short_term  max_id=159205762   max_start_ts=1787166000.0
 ```
 
-## Phase 2c — cutover: DONE
+## Phase 2c — cutover (REMAINING)
 
-Verified 2026-09-02 against the live database:
-
-| table | rows | span |
-|---|---|---|
-| `statistics` | 11,092,586 | 2024-03-13 20:00 → current |
-| `statistics_short_term` | 1,883,709 | rolling ~7 days |
-| `statistics_meta` | 1,480 | |
-
-The 2024-03-13 origin is the proof that matters. It is unchanged from the
-pre-migration count in "What is at risk" above, so the history was copied
-across engines rather than started from zero — which is the failure this
-document exists to prevent. `states` carries current rows, so `db_url` points
-at Postgres.
-
-The procedure below is kept as the record of what was done.
+Do not start until phase 1 is verified.
 
 **Migrate statistics only.** Copying all 18M `states` rows is slow, needs a long
 HA outage, and buys a table that is discarded within 7 days anyway.

kubernetes/apps/home-assistant/deployment.yaml

@@ -56,9 +56,10 @@ spec:
         # Re-rendered on every pod start, so a CNPG password rotation takes
         # effect rather than silently leaving a stale URI behind.
         #
-        # Load-bearing: `db_url: !secret recorder_db_url` in the homeassistant
-        # repo's configuration.yaml reads this key, so HA does not start
-        # without it. Removing this init container takes the recorder down.
+        # INERT until the recorder is repointed: nothing reads this key until
+        # `db_url: !secret recorder_db_url` lands in configuration.yaml in the
+        # homeassistant repo. That ordering is required — HA refuses to start
+        # when a referenced secret is missing.
         - name: render-db-url
           image: busybox:1.36
           command:

kubernetes/apps/home-assistant/ha-pg-cluster.yaml

@@ -18,12 +18,12 @@
 # That is the point of the move — it attacks the multiplier rather than trimming
 # what gets recorded, which would only hide the problem.
 #
-# This is the live recorder database. HA writes here; the SQLite file is no
-# longer the system of record. The 2.4 years of long-term statistics were
-# copied across engines rather than regenerated — HA does not migrate data
-# between database engines, so an empty Postgres would have silently started
-# from zero and taken the energy dashboard's history with it.
-# MIGRATION-recorder-postgres.md holds the sequence that was followed.
+# STAGING
+# Phase 1 (this manifest): the cluster comes up EMPTY and HA keeps using SQLite.
+# Nothing depends on it, so a failure here is inert. Verify Ready, WAL archiving
+# to Hetzner, and a successful base backup before going further.
+# Phase 2: migrate the statistics tables and repoint recorder.db_url.
+# See MIGRATION-recorder-postgres.md for the sequence and the rollback.
 apiVersion: postgresql.cnpg.io/v1
 kind: Cluster
 metadata:

kubernetes/apps/home-assistant/kustomization.yaml

@@ -60,8 +60,9 @@ resources:
   - zigbee2mqtt-gate-service.yaml
   - zigbee2mqtt-ingress.yaml
   # Postgres backend for the recorder (MIGRATION-recorder-postgres.md).
-  # Live: recorder.db_url in the homeassistant repo points here, and the
-  # long-term statistics carry their full history from 2024.
+  # PHASE 1: this cluster comes up EMPTY and nothing consumes it — HA is still
+  # on SQLite. Deploying it is therefore inert; phase 2 migrates the statistics
+  # tables and repoints recorder.db_url in the homeassistant repo.
   - ha-pg-hetzner-creds-externalsecret.yaml
   - ha-pg-objectstore.yaml
   - ha-pg-cluster.yaml

kubernetes/apps/immich/cluster.yaml

@@ -10,15 +10,15 @@ metadata:
     # Kept permanently as a standing guard.
     kustomize.toolkit.fluxcd.io/prune: disabled
 spec:
-  # A replica is viable because nothing here writes pg_vectors any more. Both
-  # vector indexes run on vchordrq and the vectors extension is gone, so the
-  # long pg_vectors/indexes/<uuid> paths are absent — pg_basebackup's tar
-  # format cannot represent them, and that is what blocks a replica join.
-  instances: 2
-  # See ak-pg cluster.yaml for rationale. The barman-plugin shutdown wedge
-  # (cloudnative-pg#9770) holds an instance up until SIGKILL, so bounding
-  # terminationGracePeriod (stopDelay) is what keeps a failover from waiting
-  # on it. smartShutdownTimeout keeps pod restarts quick.
+  # Single instance during pgvecto.rs → VectorChord migration window;
+  # pg_basebackup tar format chokes on the long pg_vectors/indexes/<uuid>
+  # paths, blocking replica join. Bump back to 2 once Immich's in-app
+  # vchord migration drops the vectors extension.
+  instances: 1
+  # See ak-pg cluster.yaml for rationale. Single-instance, so no failover, but
+  # the barman-plugin shutdown wedge (cloudnative-pg#9770) takes the whole DB
+  # down until SIGKILL — bounding terminationGracePeriod (stopDelay) matters
+  # even without a standby. smartShutdownTimeout keeps pod restarts quick.
   smartShutdownTimeout: 10
   stopDelay: 60
   imageName: ghcr.io/corentingiraud/cnpg-pgvector-vectorchord:16-migration
@@ -36,10 +36,8 @@ spec:
       database: immich
       owner: immich
   postgresql:
-    # vchord only. The image also ships pgvecto.rs (vectors.so), but its
-    # extension is not installed and preloading it would recreate pg_vectors
-    # in PGDATA for nothing. pgvector needs no preloading.
     shared_preload_libraries:
+      - vectors.so
       - vchord.so
     parameters:
       shared_buffers: "512MB"

kubernetes/apps/llama-server/embed.yaml

@@ -91,14 +91,24 @@ spec:
             capabilities:
               drop:
                 - ALL
+          # The weights are only ~600 MiB, but that is not what sets the
+          # footprint: four slots at 4096 context with a 1024-token micro-batch,
+          # and Vulkan allocating the iGPU's working set from system RAM where
+          # it counts against this cgroup.
+          #
+          # 4Gi was sized for the earlier 2048 context and 512 micro-batch. Both
+          # were raised to match Paperless's chunking without raising this, and
+          # the pod then survived interactive use and was OOMKilled (137) partway
+          # through the first sustained run — roughly 300 documents in. Indexing
+          # the archive would have hit it on day one.
           resources:
             requests:
               cpu: 250m
-              memory: 2Gi
+              memory: 4Gi
               gpu.intel.com/i915: "1"
             limits:
               cpu: "2"
-              memory: 4Gi
+              memory: 10Gi
               gpu.intel.com/i915: "1"
           # ~600 MiB rather than ~5 GiB, so a much shorter budget than the chat
           # model needs, but still sized for a cold download.

kubernetes/apps/llama-server/pvc.yaml

@@ -1,20 +1,10 @@
 ---
-# Holds the GGUF weights, downloaded once by llama-server itself via --hf-repo.
-# Prune-disabled so retiring this Kustomization does not throw away tens of GiB
-# of model files that would have to be pulled from Hugging Face again.
+# Holds the GGUF weights, downloaded once by the Deployment's initContainer.
+# Prune-disabled so retiring this Kustomization does not throw away several
+# GiB of model files that would have to be pulled from Hugging Face again.
 #
-# ReadWriteMany so a second replica, a one-off benchmark Job, or a candidate
-# model being evaluated can mount the same weights; the models are read-only at
-# inference time.
-#
-# Sized well beyond the two deployed models. The cache is shared by every
-# candidate under evaluation (scripts/embed-eval-real.sh), and on a slow
-# connection the download is the long pole — a shared cache lets the large
-# candidates pull in the background while the small ones are already being
-# measured, and means a model is never fetched twice across runs.
-#
-# CephFS supports online expansion (allowVolumeExpansion on the storage class),
-# so growing this does not disturb the running pods.
+# ReadWriteMany so a second replica or a one-off benchmark Job can mount the
+# same weights; the models are read-only at inference time.
 apiVersion: v1
 kind: PersistentVolumeClaim
 metadata:
@@ -28,4 +18,4 @@ spec:
   storageClassName: ceph-filesystem
   resources:
     requests:
-      storage: 120Gi
+      storage: 40Gi

kubernetes/infrastructure/crowdsec/edge-bouncer-key-externalsecret.yaml

@@ -1,27 +0,0 @@
-# Bouncer key for the firewall bouncers on proxy-1/proxy-2.
-#
-# Separate from crowdsec-bouncer-key ON PURPOSE. That key is rotated
-# quarterly by kubernetes/infrastructure/openbao/rotation/crowdsec-bouncer-key.yaml,
-# and both of its consumers live in the cluster, so ESO propagates a rotation
-# within 10m. The edge bouncers are Ansible-managed hosts: nothing there
-# re-reads OpenBao on a schedule, so a rotated key would leave them 401ing
-# against the LAPI until someone runs the playbook. This key is therefore
-# deliberately NOT enrolled in the rotator — it rotates when the playbook runs,
-# and the playbook is what carries the new value to the hosts.
-#
-# The value must match ansible vault_crowdsec_edge_bouncer_key.
-apiVersion: external-secrets.io/v1
-kind: ExternalSecret
-metadata:
-  name: crowdsec-edge-bouncer-key
-  namespace: crowdsec
-spec:
-  refreshInterval: 10m
-  secretStoreRef:
-    kind: ClusterSecretStore
-    name: openbao
-  target:
-    name: crowdsec-edge-bouncer-key
-  dataFrom:
-    - extract:
-        key: infrastructure/crowdsec-edge-bouncer-key

kubernetes/infrastructure/crowdsec/helmrelease.yaml

@@ -36,36 +36,6 @@ spec:
   values:
     container_runtime: containerd
 
-    # Addresses a community blocklist must never be allowed to ban.
-    #
-    # This is the ONLY declarative allowlist the chart offers. `cscli
-    # allowlists` objects live in the LAPI database, and that database runs
-    # without persistence here on purpose (see lapi.persistentVolume below),
-    # so anything created through cscli is gone at the next restart. This file
-    # is a ConfigMap mount, so it survives by construction. Upstream marks it
-    # deprecated in favour of centralized allowlists; it is still the only
-    # thing that works against a LAPI with no PVC.
-    #
-    # Scope: CAPI decisions only. Locally generated decisions are not filtered
-    # here — they currently carry real client IPs (verified), because Traefik
-    # resolves X-Forwarded-For past the netbird-ingress pod IP.
-    #
-    # The home WAN address is deliberately absent and is not needed: LAN
-    # clients never present it. opentofu/unifi_nat_hairpin.tf DNATs the public
-    # proxy addresses to the Traefik VIP on the UDM, so on-LAN traffic reaches
-    # the cluster directly with a LAN source and never traverses the proxies.
-    # Private ranges cannot appear in a community blocklist either way.
-    config:
-      capi_whitelists.yaml: |
-        cidrs:
-          # The proxy hosts themselves. Same three addresses as the
-          # netbird-ingress-egress toCIDRSet and the hairpin endpoints —
-          # keep all three lists in sync. proxy-1 is the v6-only member,
-          # proxy-2 the dual-stack one.
-          - 45.157.177.26/32
-          - 2a03:4000:4b:c1e:4474:43ff:fedf:8f85/128
-          - 2a03:4000:28:f6e:a4c7:59ff:fe1e:9a29/128
-
     # mTLS between agents and the LAPI. Do not switch back to password auth:
     # the LAPI runs without persistence, so its machine-registration DB is
     # wiped on every restart, and password-registered agents then get 401
@@ -116,18 +86,10 @@ spec:
         requests:
           cpu: 50m
       podAnnotations:
-        # Reloader restarts the LAPI when either bouncer key changes. Both are
-        # consumed as env vars from a secretKeyRef, and env is fixed at pod
-        # start — without a restart the LAPI keeps trusting the old value while
-        # the bouncer already presents the new one, which is a 401 at exactly
-        # the moment a rotation is supposed to be seamless.
-        #
-        # crowdsec-bouncer-key rotates on a schedule, see
+        # Reloader restarts the LAPI when crowdsec-bouncer-key changes (it's an
+        # env-var secret, so a restart is required to pick up a rotated key). See
         # kubernetes/infrastructure/openbao/rotation/crowdsec-bouncer-key.yaml.
-        # crowdsec-edge-bouncer-key rotates by hand together with
-        # ansible/plays/install-netbird-proxy.yml; it still needs to be listed
-        # here, because the LAPI side of that rotation is this env var.
-        secret.reloader.stakater.com/reload: crowdsec-bouncer-key,crowdsec-edge-bouncer-key
+        secret.reloader.stakater.com/reload: crowdsec-bouncer-key
       persistentVolume:
         data:
           # Kept OFF deliberately. The #337 attempt to set this `true` (persist
@@ -156,17 +118,6 @@ spec:
             secretKeyRef:
               name: crowdsec-bouncer-key
               key: key
-        # The firewall bouncers on proxy-1/proxy-2, over the public LAPI
-        # endpoint (lapi-ingress.yaml). The chart's entrypoint re-runs
-        # `register_bouncer` for every BOUNCER_KEY_* on each start and skips
-        # names that already exist, so this survives the LAPI's wiped
-        # registration DB the same way BOUNCER_KEY_ingress does — the env var
-        # is the source of truth, not the database.
-        - name: BOUNCER_KEY_edge
-          valueFrom:
-            secretKeyRef:
-              name: crowdsec-edge-bouncer-key
-              key: key
 
     agent:
       # Same reasoning as lapi.resources above, multiplied by three: this is a

kubernetes/infrastructure/crowdsec/kustomization.yaml

@@ -2,6 +2,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
   - bouncer-key-externalsecret.yaml
-  - edge-bouncer-key-externalsecret.yaml
-  - lapi-ingress.yaml
   - helmrelease.yaml

kubernetes/infrastructure/crowdsec/lapi-ingress.yaml

@@ -1,81 +0,0 @@
-# Public LAPI endpoint for the firewall bouncers on proxy-1/proxy-2.
-#
-# WHY PUBLIC AT ALL. The bouncers run as host processes on the proxies, and
-# the proxies have no mesh access: the netbird-proxy container runs non-root
-# (see the ip_unprivileged_port_start task in the netbird-proxy role), so it
-# has no NET_ADMIN and no /dev/net/tun, and its embedded NetBird client
-# therefore terminates the tunnel in userspace. The tunnel belongs to that
-# process, not to the host netns — a second process on the same host cannot
-# route through it. The alternative was a host-level NetBird client plus the
-# LAPI on a LoadBalancer VIP; this reuses the path every other service already
-# takes instead of adding two peers and an ACL policy.
-#
-# NO IP ALLOWLIST, deliberately. The bouncer key is the access control.
-#
-# An allowlist was the obvious second factor and it does not work here. A
-# request originating on a proxy host does not arrive carrying that host's
-# public address: the client connects to the local proxy on 127.0.0.1, the
-# reverse-proxy puts what it saw into X-Forwarded-For, and Traefik resolves
-# that. Measured against the netbird-proxy-watchdog, which makes exactly this
-# kind of request every round:
-#
-#   172.18.0.3 - - [02/Sep/2026:10:12:01] "GET /" 200 websecure-immich-...
-#   172.18.0.3 - - [02/Sep/2026:10:12:01] "GET /" 200 websecure-home-...
-#
-# 172.18.0.3 is a Docker bridge address on the proxy side. Allowlisting the
-# published proxy addresses would have rejected the bouncer with 403;
-# allowlisting 172.18.0.0/16 would be a private range on a host we do not
-# control the addressing of, which is not a security control.
-#
-# What remains is the API key, which is the real control anyway: this endpoint
-# serves read-only decision streams to a valid bouncer key and nothing else.
-# The admin surface is cscli, in-cluster. If the source address ever becomes
-# deterministic, an ipAllowList can be added on top.
----
-# The LAPI serves HTTPS only once tls.enabled is set — there is no plaintext
-# fallback and no second port; 8080 is the TLS port. Its certificate comes
-# from the chart's own self-signed CA, so Traefik needs that CA to verify the
-# backend. SANs include crowdsec-service.crowdsec.svc.cluster.local, which is
-# what the Service resolves to.
-apiVersion: traefik.io/v1alpha1
-kind: ServersTransport
-metadata:
-  name: crowdsec-lapi
-  namespace: crowdsec
-spec:
-  serverName: crowdsec-service.crowdsec.svc.cluster.local
-  rootCAsSecrets:
-    - crowdsec-root-secret
----
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
-  name: crowdsec-lapi
-  namespace: crowdsec
-  labels:
-    # Public DNS via Cloudflare; the proxies resolve it from outside. The UDM
-    # external-dns instance skips it, LAN reaches it via the hairpin NAT.
-    dns.renner.uno/public: "true"
-  annotations:
-    cert-manager.io/cluster-issuer: letsencrypt-cluster
-    # NOT behind the Authentik gate: the client is a daemon with an API key,
-    # not a browser that can complete an OIDC flow.
-    traefik.ingress.kubernetes.io/service.serversscheme: https
-    traefik.ingress.kubernetes.io/service.serverstransport: crowdsec-crowdsec-lapi@kubernetescrd
-spec:
-  ingressClassName: traefik
-  rules:
-    - host: crowdsec.renner.uno
-      http:
-        paths:
-          - path: /
-            pathType: Prefix
-            backend:
-              service:
-                name: crowdsec-service
-                port:
-                  number: 8080
-  tls:
-    - secretName: crowdsec-lapi-public-tls
-      hosts:
-        - crowdsec.renner.uno

scripts/paperless-eval.sh

@@ -1,207 +0,0 @@
-#!/usr/bin/env bash
-# End-to-end evaluation of Paperless document suggestions against the real
-# archive, comparing the two engines Paperless ships:
-#
-#   /api/documents/<id>/suggestions/      the built-in scikit-learn classifier,
-#                                         trained on the existing corpus
-#   /api/documents/<id>/ai_suggestions/   the LLM path, optionally with RAG
-#
-# WHY THIS EXISTS
-#
-# scripts/embed-eval-real.sh measures whether a document's nearest neighbours
-# carry its labels. That is a proxy: Paperless does not act on the neighbours
-# directly, it hands them to an LLM as context, alongside the document text and
-# a candidate list of existing tags and correspondents. The LLM can override
-# poor context, or fail with good context. So a better embedding score does not
-# automatically mean better suggestions, and nothing so far has verified that it
-# does.
-#
-# This measures the thing that is actually wanted: does Paperless suggest the
-# labels a human already chose? Everything else — MTEB scores, retrieval
-# accuracy — is only interesting insofar as it predicts this number.
-#
-# It also answers a question that predates the LLM work: with ten years of
-# hand-curated labels, the built-in classifier has excellent training data. It
-# cannot invent a new tag or a title, which is the LLM's advantage — but for
-# assigning labels that already exist it may simply be better. Running both on
-# the same documents settles that rather than assuming it.
-#
-# Ground truth is the current filing. Where the two disagree with it, that is
-# not automatically an error: over ten years filing conventions change, and an
-# old document may carry an older convention. Treat the absolute numbers as
-# indicative and the comparison between engines as the result.
-#
-# Requires a Paperless API token in PAPERLESS_TOKEN. The API is only reachable
-# from inside the cluster (the namespace policy admits the Authentik gate
-# outpost alone), so requests are issued from within the paperless pod.
-#
-# Usage:
-#   PAPERLESS_TOKEN=... scripts/paperless-eval.sh [-n 100] [--engine both|classic|ai]
-set -uo pipefail
-
-SAMPLE="${PAPERLESS_EVAL_SAMPLE:-100}"
-ENGINE=both
-PG_POD="${PAPERLESS_EVAL_PG_POD:-pprl-pg-2}"
-APP_POD="${PAPERLESS_EVAL_APP_POD:-deploy/paperless}"
-
-while [ $# -gt 0 ]; do
-    case "$1" in
-        -n|--sample) SAMPLE="$2"; shift 2 ;;
-        --engine)    ENGINE="$2"; shift 2 ;;
-        -h|--help)   sed -n '2,40p' "$0"; exit 0 ;;
-        *) echo "unknown argument: $1 (try --help)" >&2; exit 2 ;;
-    esac
-done
-
-if [ -z "${PAPERLESS_TOKEN:-}" ]; then
-    echo "PAPERLESS_TOKEN is not set." >&2
-    echo "Create one for an existing user, then remove it when finished." >&2
-    exit 2
-fi
-
-WORK="$(mktemp -d)"
-trap 'rm -rf "$WORK"' EXIT
-
-# Deterministic sample, so repeated runs and different engines see the same
-# documents. Only documents that carry both labels are useful as ground truth.
-echo "reading $SAMPLE labelled documents from postgres ($PG_POD)..."
-# Names come along with the ids because the AI engine reports an unresolved
-# guess as a name string rather than an id; see the lenient scoring below.
-# Tab-separated rather than '|', which occurs in correspondent names.
-kubectl -n paperless exec "$PG_POD" -- psql -U postgres -d paperless -tA -F$'\t' -c \
-  "select d.id, d.correspondent_id, d.document_type_id, c.name, t.name
-   from documents_document d
-   join documents_correspondent c on c.id = d.correspondent_id
-   join documents_documenttype  t on t.id = d.document_type_id
-   where d.correspondent_id is not null
-     and d.document_type_id is not null
-     and length(d.content) > 400
-   order by md5(d.id::text)
-   limit $SAMPLE;" > "$WORK/docs.txt" 2>"$WORK/pg-err.txt"
-if ! [ -s "$WORK/docs.txt" ]; then
-    echo "no documents returned from postgres." >&2
-    [ -s "$WORK/pg-err.txt" ] && cat "$WORK/pg-err.txt" >&2
-    exit 2
-fi
-N=$(grep -c . "$WORK/docs.txt")
-
-# The floor: always guessing the most frequent label. Without it a number like
-# "70% correct" cannot be read, because a skewed archive gives that away for
-# free — one correspondent holds several hundred of ~3900 documents.
-kubectl -n paperless exec "$PG_POD" -- psql -U postgres -d paperless -tAc \
-  "select max(c) from (select count(*) c from documents_document
-   where correspondent_id is not null group by correspondent_id) x;" \
-  > "$WORK/corr_max.txt" 2>"$WORK/pg-err.txt"
-kubectl -n paperless exec "$PG_POD" -- psql -U postgres -d paperless -tAc \
-  "select count(*) from documents_document where correspondent_id is not null;" \
-  > "$WORK/corr_total.txt" 2>"$WORK/pg-err.txt"
-
-api() { # path -> body on stdout
-    kubectl -n paperless exec "$APP_POD" -c paperless -- \
-        curl -sS -m 300 -H "Authorization: Token $PAPERLESS_TOKEN" \
-        "http://127.0.0.1:8000$1" 2>"$WORK/err.txt"
-}
-
-# A hit under two definitions, because the AI engine can be right in a way the
-# API response does not record as a suggestion:
-#
-#   strict   the true label's id appears in `.correspondents` /
-#            `.document_types`. This is what the UI actually offers, so it is
-#            the number that describes the feature as experienced.
-#
-#   lenient  strict, OR the true label's name is contained wordwise in one of
-#            `.suggested_correspondents` / `.suggested_document_types`.
-#
-# The gap between them is one specific failure: the LLM names the right entity
-# but paperless_ai.matching cannot resolve the name to the existing object,
-# because difflib.get_close_matches runs at MATCH_THRESHOLD = 0.8 and a correct
-# but more verbose answer falls below it — observed with "Generali Deutschland
-# Krankenversicherung AG" against the existing "Generali AG", which yielded an
-# empty `correspondents` list.
-#
-# Reporting only the strict number would blame the model for a matcher problem;
-# reporting only the lenient one would claim a usefulness the UI does not
-# deliver. The classic engine emits no `suggested_*` fields at all, so both
-# numbers coincide for it and the comparison stays fair in either column.
-LENIENT_JQ='
-def norm: ascii_downcase | gsub("[^\\p{L}\\p{N}]+"; " ") | split(" ") | map(select(length > 0));
-def contains_all($want): . as $have | $want | all(. as $t | $have | index([$t]) != null);
-($want_name | norm) as $w
-| ((.[$id_field] // []) | index($want_id) != null)
-  or ([(.[$name_field] // [])[] | norm] | any(contains_all($w)))
-'
-
-# score <engine> <url-suffix>
-score() {
-    local engine="$1" suffix="$2"
-    local n=0 corr_hit=0 type_hit=0 corr_len=0 type_len=0 fails=0 start elapsed
-    echo ""
-    echo "=== $engine"
-    start=$(date +%s)
-    while IFS=$'\t' read -r id want_corr want_type want_corr_name want_type_name; do
-        [ -n "$id" ] || continue
-        n=$((n + 1))
-        body=$(api "/api/documents/$id/$suffix")
-        if ! printf '%s' "$body" | jq -e . >/dev/null 2>&1; then
-            fails=$((fails + 1))
-            [ "$fails" -le 3 ] && printf '    %s: keine gueltige Antwort: %s\n' \
-                "$id" "$(printf '%s' "$body" | tr '\n' ' ' | head -c 120)" >&2
-            continue
-        fi
-        if printf '%s' "$body" | jq -e --argjson w "$want_corr" \
-               '.correspondents // [] | index($w) != null' >/dev/null 2>&1; then
-            corr_hit=$((corr_hit + 1))
-        fi
-        if printf '%s' "$body" | jq -e --argjson w "$want_type" \
-               '.document_types // [] | index($w) != null' >/dev/null 2>&1; then
-            type_hit=$((type_hit + 1))
-        fi
-        if printf '%s' "$body" | jq -e \
-               --argjson want_id "$want_corr" --arg want_name "$want_corr_name" \
-               --arg id_field correspondents --arg name_field suggested_correspondents \
-               "$LENIENT_JQ" >/dev/null 2>&1; then
-            corr_len=$((corr_len + 1))
-        fi
-        if printf '%s' "$body" | jq -e \
-               --argjson want_id "$want_type" --arg want_name "$want_type_name" \
-               --arg id_field document_types --arg name_field suggested_document_types \
-               "$LENIENT_JQ" >/dev/null 2>&1; then
-            type_len=$((type_len + 1))
-        fi
-        [ $((n % 25)) -eq 0 ] && echo "    $n/$N"
-    done < "$WORK/docs.txt"
-    elapsed=$(( $(date +%s) - start ))
-    pct() { awk -v a="$1" -v b="$2" 'BEGIN{if (b>0) printf "%.1f", 100*a/b; else print "0.0"}'; }
-    printf '    correspondent, offered  : %5s%%   recognised: %5s%%\n' \
-        "$(pct "$corr_hit" "$n")" "$(pct "$corr_len" "$n")"
-    printf '    document type, offered  : %5s%%   recognised: %5s%%\n' \
-        "$(pct "$type_hit" "$n")" "$(pct "$type_len" "$n")"
-    printf '    no usable answer        : %s of %s\n' "$fails" "$n"
-    printf '    %ss total, %.1fs per document\n' "$elapsed" "$(awk -v e="$elapsed" -v n="$n" 'BEGIN{print e/n}')"
-    printf '%-9s %7s %7s %7s %7s %6s %8ss\n' "$engine" \
-        "$(pct "$corr_hit" "$n")" "$(pct "$corr_len" "$n")" \
-        "$(pct "$type_hit" "$n")" "$(pct "$type_len" "$n")" \
-        "$fails" "$elapsed" >> "$WORK/summary.txt"
-}
-
-: > "$WORK/summary.txt"
-FLOOR=$(awk -v m="$(cat "$WORK/corr_max.txt")" -v t="$(cat "$WORK/corr_total.txt")" \
-    'BEGIN{if (t>0) printf "%.1f", 100*m/t; else print "?"}')
-echo "documents: $N   most-frequent-correspondent floor: ${FLOOR}%"
-
-case "$ENGINE" in
-    both)    score classic suggestions/; score ai ai_suggestions/ ;;
-    classic) score classic suggestions/ ;;
-    ai)      score ai ai_suggestions/ ;;
-    *) echo "unknown engine: $ENGINE" >&2; exit 2 ;;
-esac
-
-echo ""
-echo "================================================================"
-printf '%-9s %7s %7s %7s %7s %6s %9s\n' \
-    ENGINE CORR CORR+ TYPE TYPE+ FAILS TIME
-cat "$WORK/summary.txt"
-printf '%-9s %7s\n' "floor" "$FLOOR"
-echo "================================================================"
-echo "CORR/TYPE  = true label offered as a resolved id (what the UI shows)"
-echo "CORR+/TYPE+ = also counting names the LLM got right but matching dropped"