fix(llama-embed): raise the memory limit to match its own context and batch

#739 raised the context from 2048 to 4096 and the micro-batch from an effective 512 to 1024, both to match how Paperless chunks documents. It left memory at 2Gi/4Gi, which was sized for the smaller values. The pod ran fine interactively and was OOMKilled (exit 137) partway through the first sustained run, around 300 documents into an embedding pass. Nothing in the server log marked it — the process was killed outright, so the last line is a half-finished task and the reason is only visible in the container's lastState.terminated. This would have surfaced when Paperless first indexed the archive, i.e. on the first real use rather than under test. The weights are only ~600 MiB. What sets the footprint is four slots at 4096 context with a 1024-token micro-batch, plus Vulkan allocating the iGPU working set from system RAM where it counts against this cgroup — the same effect that put llama-server at 32Gi for 19 GiB of weights. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-02 08:14:19 +02:00
38 changed files with 68 additions and 1612 deletions
@@ -7,21 +7,7 @@ when:
      # install-netbird-proxy.yml also applies the netbird-proxy-watchdog role,
      # so changes to it must redeploy this host too.
      - "ansible/roles/netbird-proxy-watchdog/**"
      # install-netbird-proxy.yml also applies the edge bouncer.
      - "ansible/roles/crowdsec-edge-bouncer/**"
      - "ansible/plays/install-netbird-proxy.yml"
      # The variables that steer those roles, not just the roles themselves.
      # Without these two, a change that only touches inventory never deploys
      # and nothing says so: arming the bouncer lives in host_vars, and its
      # LAPI key lives in group_vars, so both would sit in git looking applied
      # while the hosts kept running the previous configuration.
      #
      # pve_hosts.yml is deliberately NOT listed. It is the single inventory
      # file for the whole fleet, so listing it would redeploy the proxies for
      # any unrelated host edit. The cost is that a change to [netbird_proxy]
      # group membership needs a manual deployment run.
      - "ansible/inventory/host_vars/proxy-*.yml"
      - "ansible/inventory/group_vars/netbird_proxy/**"
      - ".woodpecker/deploy-netbird-proxy.yml"
  - event: deployment
    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "deploy-netbird-proxy"'
@@ -1,23 +0,0 @@
 ---
 # Bouncer key for crowdsec-edge-bouncer, scoped to the proxy hosts.
 #
 # A single encrypted VALUE rather than a whole-file vault like its siblings in
 # group_vars/all and group_vars/edge_host. Deliberate: adding this variable to
 # an existing whole-file vault means decrypting that file first, which exposes
 # every other secret in it for the sake of appending one line. encrypt_string
 # keeps the blast radius at this one value.
 #
 # Same value as OpenBao homelab/infrastructure/crowdsec-edge-bouncer-key, which
 # the crowdsec-edge-bouncer-key ExternalSecret feeds to BOUNCER_KEY_edge on the
 # LAPI. Both sides must move together — and unlike crowdsec-bouncer-key, this
 # one is NOT in the quarterly rotator, because nothing on these hosts re-reads
 # OpenBao on a schedule. Rotating it means updating OpenBao and re-running
 # plays/install-netbird-proxy.yml.
 vault_crowdsec_edge_bouncer_key: !vault |
          $ANSIBLE_VAULT;1.1;AES256
          32653764333536316163303838333839616562663862666637626165303837633866643033393239
          3363376538656236643161386432326437633232643564630a633635306365323537346139626364
          36613033343938363366306239363638383466396337623333623136313234656366333366623433
          6437656632373364340a346566343866366433306533353431383064356362346536383136323663
          39333864643534326433653233303137373162663066633132623931316135323733333863306330
          6366623463666435636362376238313639666263323163303733
@@ -26,7 +26,3 @@
    - docker-install
    - netbird-proxy
    - netbird-proxy-watchdog
    # Enforces the cluster's CrowdSec decisions here, where public traffic
    # enters. Deployed in dry-run: it logs what it would block and creates no
    # nftables rules until crowdsec_bouncer_mode is flipped per host.
    - crowdsec-edge-bouncer
@@ -1,56 +0,0 @@
 ---
 # CrowdSec firewall bouncer on the NetBird proxy hosts.
 #
 # Enforces the decisions the in-cluster CrowdSec already detects, at the point
 # where public traffic actually enters: these hosts terminate TLS for every
 # *.renner.uno name. A drop here never reaches the mesh, the relay or the
 # cluster.
 #
 # Detection stays in the cluster. The reverse-proxy produces no parseable
 # access log and there is no CrowdSec collection for it, while Traefik has a
 # working one — and every scenario in use needs HTTP detail (paths, status
 # codes, user agents) that does not exist at this layer.
 # Pinned. A release tarball URL is immutable, and this checksum is of the
 # amd64 asset for exactly this tag.
 crowdsec_bouncer_version: v0.0.36
 crowdsec_bouncer_sha256: f86e4b72693549d99f40a9402abefb894108f047a3fbe6e72fada25ee17ce88b
 # The LAPI over the public path these hosts already serve, see
 # kubernetes/infrastructure/crowdsec/lapi-ingress.yaml. The proxies cannot
 # reach it over the mesh: the reverse-proxy container runs non-root, so its
 # embedded NetBird client terminates the tunnel in userspace and the tunnel
 # belongs to that process rather than to the host netns.
 #
 # No ca_cert_path: Traefik serves a Let's Encrypt certificate for this name,
 # and the bouncer adds ca_cert_path to the system pool rather than replacing
 # it — the system pool already trusts LE.
 crowdsec_bouncer_api_url: https://crowdsec.renner.uno/
 crowdsec_bouncer_api_key: "{{ vault_crowdsec_edge_bouncer_key }}"
 # Enforcing. The bouncer creates two nftables tables (ip crowdsec, ip6
 # crowdsec6) holding the decisions it pulled, touches no existing chain and
 # sets no chain policy — SSH and :443 stay open unless the source is banned.
 #
 # `dry-run` is the other value: it pulls and logs what it would block without
 # creating a single rule. That is what a new host should be given before it
 # enforces, and what an existing one can be dropped back to, per host through
 # inventory/host_vars, without disarming the rest.
 #
 # Worth knowing before arming anything new: the first bouncer on a path
 # enforces the whole CAPI set at once — around 15k addresses today — so the
 # false-positive window belongs before the switch, not after it. And because
 # this chain sits on the INPUT hook while these hosts run no other firewall,
 # there is no conntrack rule accepting established flows ahead of it: a banned
 # address is dropped even when it is answering our own outbound connection.
 # That is why capi_whitelists.yaml carries the proxies and netbird.renner.uno.
 crowdsec_bouncer_mode: nftables
 crowdsec_bouncer_update_frequency: 10s
 crowdsec_bouncer_log_level: info
 # Bound to loopback deliberately. These hosts are public and have no metrics
 # collection path at all today — opening :60601 to the internet to gain a
 # scrape target is the wrong trade. Revisit when host metrics exist.
 crowdsec_bouncer_metrics_listen: 127.0.0.1
 crowdsec_bouncer_metrics_port: 60601
@@ -1,10 +0,0 @@
 ---
 # A restart drops both nftables tables and rebuilds them from the next
 # successful LAPI pull, so there is a short window with no blocklist. That is
 # the documented behaviour of a clean shutdown, not a defect — and it is why
 # retry_initial_connect matters.
 - name: Restart crowdsec firewall bouncer
  ansible.builtin.systemd:
    name: crowdsec-firewall-bouncer.service
    state: restarted
    daemon_reload: true
@@ -1,111 +0,0 @@
 ---
 # See defaults/main.yml for why this exists and why it starts inert.
 #
 # The static release tarball, not a distribution package: the AUR package
 # `crowdsec-firewall-bouncer-nftables` depends on `crowdsec` and would pull the
 # whole Security Engine onto hosts that only need the bouncer, the group is
 # mixed (proxy-1 Arch, VPS-1 Debian), and the AUR would be a new pattern this
 # repo does not otherwise have. The tarball is the same artefact the AUR
 # package downloads.
 - name: Ensure nftables is present (the enforcing backend)
  ansible.builtin.package:
    name: nftables
    state: present
 - name: Ensure the bouncer config directory exists
  ansible.builtin.file:
    path: /etc/crowdsec/bouncers
    state: directory
    mode: "0755"
 # Fetched on the CONTROLLER and pushed, not pulled by the host.
 #
 # proxy-1 is IPv6-only, and neither github.com nor objects.githubusercontent.com
 # publishes an AAAA record — a get_url on the host fails with "Network is
 # unreachable" and no amount of retrying fixes it. Mirroring into
 # gitea.renner.uno is no help either: its public records are the internal LB
 # VIP (10.2.99.86 / fd99::86), so it is unreachable from outside the LAN. And
 # upstream ships no container image for this bouncer, which is expected —
 # it manipulates host netfilter.
 #
 # The CI runner has v4, so it does the fetching. The checksum is verified
 # there, before anything is copied to a host.
 - name: Fetch the pinned bouncer release on the controller
  ansible.builtin.get_url:
    url: >-
      https://github.com/crowdsecurity/cs-firewall-bouncer/releases/download/{{
      crowdsec_bouncer_version }}/crowdsec-firewall-bouncer-linux-amd64.tgz
    dest: /tmp/crowdsec-firewall-bouncer-{{ crowdsec_bouncer_version }}.tgz
    checksum: sha256:{{ crowdsec_bouncer_sha256 }}
    mode: "0644"
  delegate_to: localhost
  become: false
  run_once: true
 # Only the download had to move to the controller — the extraction stays on
 # the host. The ci-ansible image ships a tar that ansible rejects ("detected as
 # tar type None. GNU tar required"), while both host distributions have GNU
 # tar. Shipping the archive rather than the unpacked binary also keeps the
 # artefact that the checksum was verified against intact all the way to the
 # host.
 - name: Copy the bouncer archive to the host
  ansible.builtin.copy:
    src: /tmp/crowdsec-firewall-bouncer-{{ crowdsec_bouncer_version }}.tgz
    dest: /opt/crowdsec-firewall-bouncer-{{ crowdsec_bouncer_version }}.tgz
    mode: "0644"
 # Only the binary. The tarball also carries install.sh / upgrade.sh /
 # uninstall.sh, which manage systemd and config themselves — that is this
 # role's job, and letting them run would leave two owners for the same files.
 - name: Unpack the bouncer binary
  ansible.builtin.unarchive:
    src: /opt/crowdsec-firewall-bouncer-{{ crowdsec_bouncer_version }}.tgz
    dest: /opt
    remote_src: true
    include:
      - crowdsec-firewall-bouncer-{{ crowdsec_bouncer_version }}/crowdsec-firewall-bouncer
    creates: /opt/crowdsec-firewall-bouncer-{{ crowdsec_bouncer_version }}/crowdsec-firewall-bouncer
 - name: Install the bouncer binary
  ansible.builtin.copy:
    src: /opt/crowdsec-firewall-bouncer-{{ crowdsec_bouncer_version }}/crowdsec-firewall-bouncer
    dest: /usr/local/bin/crowdsec-firewall-bouncer
    remote_src: true
    mode: "0755"
  notify: Restart crowdsec firewall bouncer
 # 0600: this file carries the LAPI key.
 - name: Deploy the bouncer configuration
  ansible.builtin.template:
    src: crowdsec-firewall-bouncer.yaml.j2
    dest: /etc/crowdsec/bouncers/crowdsec-firewall-bouncer.yaml
    mode: "0600"
  notify: Restart crowdsec firewall bouncer
 - name: Deploy the bouncer systemd unit
  ansible.builtin.template:
    src: crowdsec-firewall-bouncer.service.j2
    dest: /etc/systemd/system/crowdsec-firewall-bouncer.service
    mode: "0644"
  notify: Restart crowdsec firewall bouncer
 - name: Enable and start the bouncer
  ansible.builtin.systemd:
    name: crowdsec-firewall-bouncer.service
    enabled: true
    state: started
    daemon_reload: true
 - name: Report how to read it
  ansible.builtin.debug:
    msg: |
      crowdsec-firewall-bouncer deployed in mode "{{ crowdsec_bouncer_mode }}".
      What it would block:  journalctl -u crowdsec-firewall-bouncer -f
      What it does block:   nft list table ip crowdsec ; nft list table ip6 crowdsec6
      Metrics:              curl -s {{ crowdsec_bouncer_metrics_listen }}:{{ crowdsec_bouncer_metrics_port }}/metrics
      Rollback: systemctl stop crowdsec-firewall-bouncer. A clean stop deletes
      both nftables tables by itself. Only after a kill -9 do they need
      removing by hand: nft delete table ip crowdsec ; nft delete table ip6 crowdsec6
@@ -1,24 +0,0 @@
 # Managed by Ansible (crowdsec-edge-bouncer role) — do not edit.
 #
 # Upstream's shipped unit orders itself After=crowdsec.service. There is no
 # local Security Engine on these hosts — the LAPI is in the cluster — so that
 # dependency is dropped; nss-lookup instead, because api_url is a public name
 # that has to resolve before the first connect.
 [Unit]
 Description=CrowdSec firewall bouncer (edge enforcement for the NetBird proxy)
 After=network-online.target nss-lookup.target
 Wants=network-online.target
 [Service]
 Type=notify
 # -t validates the config and exits non-zero on a bad one, so a broken config
 # fails the start instead of taking the running rules down with it.
 ExecStartPre=/usr/local/bin/crowdsec-firewall-bouncer -c /etc/crowdsec/bouncers/crowdsec-firewall-bouncer.yaml -t
 ExecStart=/usr/local/bin/crowdsec-firewall-bouncer -c /etc/crowdsec/bouncers/crowdsec-firewall-bouncer.yaml
 Restart=always
 RestartSec=10
 LimitNOFILE=65536
 KillMode=mixed
 [Install]
 WantedBy=multi-user.target
@@ -1,55 +0,0 @@
 # Managed by Ansible (crowdsec-edge-bouncer role) — do not edit.
 mode: {{ crowdsec_bouncer_mode }}
 update_frequency: {{ crowdsec_bouncer_update_frequency }}
 # stdout, so journald owns the log and there is no second rotation policy to
 # keep in sync. The shipped example writes to /var/log; that only matters for
 # hosts without journald.
 log_mode: stdout
 log_level: {{ crowdsec_bouncer_log_level }}
 api_url: {{ crowdsec_bouncer_api_url }}
 api_key: {{ crowdsec_bouncer_api_key }}
 # Without this the process EXITS when the LAPI is unreachable at startup —
 # there is no retry on the first connect. And because a clean shutdown deletes
 # the nftables tables, the systemd restart loop would leave the host with no
 # blocklist at all until the first successful connect. Present in the code,
 # absent from the shipped example config.
 retry_initial_connect: true
 disable_ipv4: false
 disable_ipv6: false
 deny_action: DROP
 supported_decisions_types:
  - ban
 # Separate tables per family; the bouncer never touches an existing table or
 # chain, and never sets a chain policy. Priority -10 puts it ahead of anything
 # at the default filter priority.
 nftables:
  ipv4:
    enabled: true
    set-only: false
    table: crowdsec
    chain: crowdsec-chain
    priority: -10
  ipv6:
    enabled: true
    set-only: false
    table: crowdsec6
    chain: crowdsec6-chain
    priority: -10
 # input is the traffic this exists for: the reverse-proxy is host-networked and
 # owns :443, so its packets traverse INPUT. forward covers the bridge-networked
 # http-redirect sidecar, whose packets never do.
 nftables_hooks:
  - input
  - forward
 prometheus:
  enabled: true
  listen_addr: {{ crowdsec_bouncer_metrics_listen }}
  listen_port: {{ crowdsec_bouncer_metrics_port }}
@@ -2,12 +2,7 @@
 Converts one MS-01 from "Proxmox host running a Talos VM" to "Talos on bare
 metal", and swaps its Ceph OSD onto a Samsung PM983 at the same time. Written
-after doing jupiter on 2026-08-25.
+after doing jupiter on 2026-08-25; saturn and neptune remain.
 All three nodes are now on bare metal — `talosctl get systeminformation`
 reports Minisforum MS-01 hardware with distinct chassis serials for k8s-1,
 k8s-2 and k8s-3, with no hypervisor in between. Kept as the procedure for a
 rebuild, not as outstanding work.
 Everything here that reads like an over-specific warning is something that
 actually happened. The conversion itself is straightforward — the failures were
@@ -111,23 +111,9 @@ statistics             max_id=13323904    max_start_ts=1787162400.0
 statistics_short_term  max_id=159205762   max_start_ts=1787166000.0
 ```
-## Phase 2c — cutover: DONE
+## Phase 2c — cutover (REMAINING)
-Verified 2026-09-02 against the live database:
+Do not start until phase 1 is verified.
 | table | rows | span |
 |---|---|---|
 | `statistics` | 11,092,586 | 2024-03-13 20:00 → current |
 | `statistics_short_term` | 1,883,709 | rolling ~7 days |
 | `statistics_meta` | 1,480 | |
 The 2024-03-13 origin is the proof that matters. It is unchanged from the
 pre-migration count in "What is at risk" above, so the history was copied
 across engines rather than started from zero — which is the failure this
 document exists to prevent. `states` carries current rows, so `db_url` points
 at Postgres.
 The procedure below is kept as the record of what was done.
 **Migrate statistics only.** Copying all 18M `states` rows is slow, needs a long
 HA outage, and buys a table that is discarded within 7 days anyway.
@@ -56,9 +56,10 @@ spec:
        # Re-rendered on every pod start, so a CNPG password rotation takes
        # effect rather than silently leaving a stale URI behind.
        #
-        # Load-bearing: `db_url: !secret recorder_db_url` in the homeassistant
+        # INERT until the recorder is repointed: nothing reads this key until
-        # repo's configuration.yaml reads this key, so HA does not start
+        # `db_url: !secret recorder_db_url` lands in configuration.yaml in the
-        # without it. Removing this init container takes the recorder down.
+        # homeassistant repo. That ordering is required — HA refuses to start
        # when a referenced secret is missing.
        - name: render-db-url
          image: busybox:1.36
          command:
@@ -95,7 +96,7 @@ spec:
              memory: 32Mi
      containers:
        - name: home-assistant
-          image: ghcr.io/home-assistant/home-assistant:2026.9.0 # {"$imagepolicy": "flux-system:home-assistant"}
+          image: ghcr.io/home-assistant/home-assistant:2026.8.3 # {"$imagepolicy": "flux-system:home-assistant"}
          env:
            - name: TZ
              value: Europe/Berlin
@@ -18,12 +18,12 @@
 # That is the point of the move — it attacks the multiplier rather than trimming
 # what gets recorded, which would only hide the problem.
 #
-# This is the live recorder database. HA writes here; the SQLite file is no
+# STAGING
-# longer the system of record. The 2.4 years of long-term statistics were
+# Phase 1 (this manifest): the cluster comes up EMPTY and HA keeps using SQLite.
-# copied across engines rather than regenerated — HA does not migrate data
+# Nothing depends on it, so a failure here is inert. Verify Ready, WAL archiving
-# between database engines, so an empty Postgres would have silently started
+# to Hetzner, and a successful base backup before going further.
-# from zero and taken the energy dashboard's history with it.
+# Phase 2: migrate the statistics tables and repoint recorder.db_url.
-# MIGRATION-recorder-postgres.md holds the sequence that was followed.
+# See MIGRATION-recorder-postgres.md for the sequence and the rollback.
 apiVersion: postgresql.cnpg.io/v1
 kind: Cluster
 metadata:
@@ -60,8 +60,9 @@ resources:
  - zigbee2mqtt-gate-service.yaml
  - zigbee2mqtt-ingress.yaml
  # Postgres backend for the recorder (MIGRATION-recorder-postgres.md).
-  # Live: recorder.db_url in the homeassistant repo points here, and the
+  # PHASE 1: this cluster comes up EMPTY and nothing consumes it — HA is still
-  # long-term statistics carry their full history from 2024.
+  # on SQLite. Deploying it is therefore inert; phase 2 migrates the statistics
  # tables and repoints recorder.db_url in the homeassistant repo.
  - ha-pg-hetzner-creds-externalsecret.yaml
  - ha-pg-objectstore.yaml
  - ha-pg-cluster.yaml
@@ -10,15 +10,15 @@ metadata:
    # Kept permanently as a standing guard.
    kustomize.toolkit.fluxcd.io/prune: disabled
 spec:
-  # A replica is viable because nothing here writes pg_vectors any more. Both
+  # Single instance during pgvecto.rs → VectorChord migration window;
-  # vector indexes run on vchordrq and the vectors extension is gone, so the
+  # pg_basebackup tar format chokes on the long pg_vectors/indexes/<uuid>
-  # long pg_vectors/indexes/<uuid> paths are absent — pg_basebackup's tar
+  # paths, blocking replica join. Bump back to 2 once Immich's in-app
-  # format cannot represent them, and that is what blocks a replica join.
+  # vchord migration drops the vectors extension.
-  instances: 2
+  instances: 1
-  # See ak-pg cluster.yaml for rationale. The barman-plugin shutdown wedge
+  # See ak-pg cluster.yaml for rationale. Single-instance, so no failover, but
-  # (cloudnative-pg#9770) holds an instance up until SIGKILL, so bounding
+  # the barman-plugin shutdown wedge (cloudnative-pg#9770) takes the whole DB
-  # terminationGracePeriod (stopDelay) is what keeps a failover from waiting
+  # down until SIGKILL — bounding terminationGracePeriod (stopDelay) matters
-  # on it. smartShutdownTimeout keeps pod restarts quick.
+  # even without a standby. smartShutdownTimeout keeps pod restarts quick.
  smartShutdownTimeout: 10
  stopDelay: 60
  imageName: ghcr.io/corentingiraud/cnpg-pgvector-vectorchord:16-migration
@@ -36,10 +36,8 @@ spec:
      database: immich
      owner: immich
  postgresql:
    # vchord only. The image also ships pgvecto.rs (vectors.so), but its
    # extension is not installed and preloading it would recreate pg_vectors
    # in PGDATA for nothing. pgvector needs no preloading.
    shared_preload_libraries:
      - vectors.so
      - vchord.so
    parameters:
      shared_buffers: "512MB"
@@ -75,44 +75,11 @@ spec:
            # many small per-expert matmuls that sets the pace, not bandwidth.
            - --threads
            - "4"
-            # 16k, and going above it corrupts inference rather than failing.
+            # 16k rather than 8k because the measured prefill falloff is mild
-            # At 32768 per slot this server answered a 650-token classification
+            # and this model's KV cache is unusually cheap — only 10 of its 40
-            # prompt with 1025 tokens of "!" — the KV cache producing numbers
+            # layers use full attention.
            # rather than language. It does not crash, it does not warn, and
            # the OpenAI response looks well-formed, so the damage only shows
            # in what callers get back: Paperless ran past its 300 s timeout,
            # and Home Assistant got an empty completion and a hallucinated
            # device list.
            #
            # The same Vulkan/Mesa stack already fails hard at --ubatch-size
            # 2048; this is the quiet version of that limit. Raise this only
            # against a generation that is read, not against a server that
            # merely starts.
            #
            - --ctx-size
            - "16384"
            # One slot, so the 16384 belongs to one request at a time.
            #
            # Left on auto, llama.cpp opens four slots and — because auto is
            # also what enables --kv-unified — has them share a single buffer
            # of this size. Four Paperless requests at once wanted 6209, 6472,
            # 4303 and 1271 tokens against those 16384, and all four died with
            # "Context size has been exceeded".
            #
            # They were concurrent because a client timeout does not stop the
            # generation behind it: each abandoned request keeps consuming the
            # shared budget the next one needs, so one slow document is enough
            # to start the pile-up. Queueing is the behaviour worth having
            # here — nothing calls this server in parallel on purpose, and
            # Paperless's celery worker runs at concurrency 1.
            #
            # No liveness probe change goes with this. The server recovered on
            # its own once the batch drained, so a probe that restarted the pod
            # would have made the incident worse; with a single slot, one that
            # asked for a real completion would also queue behind live work and
            # kill healthy pods during a long document.
            - --parallel
            - "1"
            # A micro-batch of 2048 or more makes this Vulkan/Mesa stack fail
            # with ErrorDeviceLost.
            - --ubatch-size
@@ -41,7 +41,7 @@ spec:
          image: ghcr.io/ggml-org/llama.cpp:server-vulkan-b10731
          args:
            - --hf-repo
-            - gpustack/bge-m3-GGUF:Q8_0
+            - Qwen/Qwen3-Embedding-0.6B-GGUF:Q8_0
            # Serves POST /v1/embeddings.
            - --embeddings
            # A short, stable id for consumers. Without it the model is
@@ -50,41 +50,28 @@ spec:
            # One name only: llama-server keeps aliases in a sorted set and
            # reports the first as the id, so a second name can steal it.
            - --alias
-            - bge-m3
+            - qwen3-embedding-0.6b
-            # bge-m3 is trained for CLS pooling — a different type from the
+            # Qwen3-Embedding is trained for last-token pooling. Left
-            # Qwen family this replaces, which uses last-token. Left unspecified
+            # unspecified llama-server takes the GGUF's default, which is not
-            # llama-server takes the GGUF default, which is not visible from
+            # visible from outside and would silently produce vectors the model
-            # outside and would silently produce vectors the model was not
+            # was not trained to emit. Set before anything is indexed: changing
-            # trained to emit. Read from the model's own 1_Pooling/config.json
+            # pooling later invalidates every stored embedding.
            # rather than assumed; of the candidates measured, three different
            # pooling types were needed.
            - --pooling
-            - cls
+            - last
            - --n-gpu-layers
            - "99"
            - --threads
            - "2"
-            # Paperless chunks at 1024 tokens, but counts them with its own
+            # Paperless chunks documents at 1024 tokens. The defaults are
-            # tokenizer while this server measures the batch in bge-m3's.
+            # batch 2048 / ubatch 512, and with --embeddings llama-server
-            # Those disagree: XLM-RoBERTa splits German into more pieces than
+            # clamps both down to 512 rather than run them mismatched — below
-            # llama_index's counter does, so a chunk Paperless considers full
+            # the chunk size. Matching both to 1024 fits a whole chunk in one
-            # at 1024 arrived here as 1069 and the request failed outright —
+            # pass; 2048 is deliberately avoided, since that is where this
-            # "input (1069 tokens) is too large to process".
+            # Vulkan stack fails with ErrorDeviceLost.
            #
            # The headroom therefore has to absorb a tokenizer ratio, not just
            # a rounding error, and 1.5x covers far more than the 1.044 that
            # broke it. The defaults are batch 2048 / ubatch 512, and with
            # --embeddings llama-server clamps both to the smaller rather than
            # run them mismatched, so both are set.
            #
            # 2048 stays avoided: that is where the chat model met
            # ErrorDeviceLost on this Vulkan stack. That failure came under a
            # 20 GB model's memory pressure and may not bind a 600 MB one, but
            # this does not need to find out.
            - --batch-size
-            - "1536"
+            - "1024"
            - --ubatch-size
-            - "1536"
+            - "1024"
            # Room for a full chunk plus overhead rather than exactly one.
            - --ctx-size
            - "4096"
@@ -104,22 +91,24 @@ spec:
            capabilities:
              drop:
                - ALL
-          # Measured, not estimated: embedding 400 real documents peaked at
+          # The weights are only ~600 MiB, but that is not what sets the
-          # 1.6 GiB (Prometheus max_over_time on the working set). The limit is
+          # footprint: four slots at 4096 context with a 1024-token micro-batch,
-          # well above that because the footprint is dominated by the four slots
+          # and Vulkan allocating the iGPU's working set from system RAM where
-          # at 4096 context and the Vulkan working set rather than by the
+          # it counts against this cgroup.
-          # weights — the Qwen model this replaces peaked at 10.6 GiB on the
+          #
-          # same settings for the same 600 MiB of weights, and several
+          # 4Gi was sized for the earlier 2048 context and 512 micro-batch. Both
-          # candidates were OOMKilled at limits that looked generous against
+          # were raised to match Paperless's chunking without raising this, and
-          # their model size.
+          # the pod then survived interactive use and was OOMKilled (137) partway
          # through the first sustained run — roughly 300 documents in. Indexing
          # the archive would have hit it on day one.
          resources:
            requests:
              cpu: 250m
-              memory: 2Gi
+              memory: 4Gi
              gpu.intel.com/i915: "1"
            limits:
              cpu: "2"
-              memory: 8Gi
+              memory: 10Gi
              gpu.intel.com/i915: "1"
          # ~600 MiB rather than ~5 GiB, so a much shorter budget than the chat
          # model needs, but still sized for a cold download.
@@ -1,20 +1,10 @@
 ---
-# Holds the GGUF weights, downloaded once by llama-server itself via --hf-repo.
+# Holds the GGUF weights, downloaded once by the Deployment's initContainer.
-# Prune-disabled so retiring this Kustomization does not throw away tens of GiB
+# Prune-disabled so retiring this Kustomization does not throw away several
-# of model files that would have to be pulled from Hugging Face again.
+# GiB of model files that would have to be pulled from Hugging Face again.
 #
-# ReadWriteMany so a second replica, a one-off benchmark Job, or a candidate
+# ReadWriteMany so a second replica or a one-off benchmark Job can mount the
-# model being evaluated can mount the same weights; the models are read-only at
+# same weights; the models are read-only at inference time.
 # inference time.
 #
 # Sized well beyond the two deployed models. The cache is shared by every
 # candidate under evaluation (scripts/embed-eval-real.sh), and on a slow
 # connection the download is the long pole — a shared cache lets the large
 # candidates pull in the background while the small ones are already being
 # measured, and means a model is never fetched twice across runs.
 #
 # CephFS supports online expansion (allowVolumeExpansion on the storage class),
 # so growing this does not disturb the running pods.
 apiVersion: v1
 kind: PersistentVolumeClaim
 metadata:
@@ -28,4 +18,4 @@ spec:
  storageClassName: ceph-filesystem
  resources:
    requests:
-      storage: 120Gi
+      storage: 40Gi
@@ -1,36 +0,0 @@
 # Scrapes CrowdSec. Until this landed, nothing did: `up{namespace="crowdsec"}`
 # returned zero series and Prometheus held no cs_* metric at all, so the
 # detection side of the ingress path was invisible — including whether the
 # agents were reading logs and whether any bouncer was pulling decisions.
 #
 # One monitor for both services. They carry the same shape (a `metrics` port on
 # 6060) and differ only in `app`, so a matchExpressions selector keeps this to
 # a single object rather than two near-identical ones:
 #
 #   crowdsec-service        the LAPI      — decisions, bouncer + machine requests
 #   crowdsec-agent-service  the agents    — acquisition, parsers, scenarios
 #
 # The crowdsec-ns CiliumNetworkPolicy already admits the monitoring namespace
 # without a port restriction, so no policy change is needed.
 #
 # `release: kube-prometheus-stack` is what the chart's Prometheus selects on.
 apiVersion: monitoring.coreos.com/v1
 kind: ServiceMonitor
 metadata:
  name: crowdsec
  namespace: crowdsec
  labels:
    app.kubernetes.io/name: crowdsec
    release: kube-prometheus-stack
 spec:
  selector:
    matchExpressions:
      - key: app
        operator: In
        values:
          - crowdsec-service
          - crowdsec-agent-service
  endpoints:
    - port: metrics
      path: /metrics
      interval: 30s
@@ -23,7 +23,6 @@ resources:
  - prometheusrule-cnpg-backup.yaml
  - alloy-servicemonitor.yaml
  - cnpg-podmonitors.yaml
  - crowdsec-servicemonitor.yaml
  - home-assistant-servicemonitor.yaml
  - immich-servicemonitor.yaml
  - loki-servicemonitor.yaml
@@ -1,163 +0,0 @@
 {
  "annotations": {"list": []},
  "description": "CrowdSec: Erkennung im Cluster, Durchsetzung am NetBird-Edge",
  "editable": true,
  "fiscalYearStartMonth": 0,
  "graphTooltip": 1,
  "id": null,
  "links": [],
  "panels": [
    {
      "datasource": {"type": "prometheus", "uid": "prometheus"},
      "description": "Aktive Ban-Entscheidungen in der LAPI, ganz überwiegend aus den CAPI-Community-Blocklisten. Das ist die Menge, die ein scharfgeschalteter Bouncer durchsetzt.",
      "fieldConfig": {
        "defaults": {"color": {"mode": "thresholds"}, "mappings": [], "thresholds": {"mode": "absolute", "steps": [{"color": "text", "value": null}]}, "unit": "short"},
        "overrides": []
      },
      "gridPos": {"h": 4, "w": 6, "x": 0, "y": 0},
      "id": 1,
      "options": {"colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto"},
      "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "sum(cs_active_decisions)", "refId": "A"}],
      "title": "Aktive Decisions",
      "type": "stat"
    },
    {
      "datasource": {"type": "prometheus", "uid": "prometheus"},
      "description": "Bouncer, die in den letzten 10 Minuten Decisions abgeholt haben. Fällt das unter die erwartete Zahl, holt ein Bouncer nichts mehr — und setzt damit einen veralteten Stand durch, ohne dass irgendetwas ausfällt.",
      "fieldConfig": {
        "defaults": {"color": {"mode": "thresholds"}, "mappings": [], "thresholds": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "green", "value": 1}]}, "unit": "short"},
        "overrides": []
      },
      "gridPos": {"h": 4, "w": 6, "x": 6, "y": 0},
      "id": 2,
      "options": {"colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto"},
      "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "count(sum by (bouncer) (rate(cs_lapi_bouncer_requests_total[10m])) > 0)", "refId": "A"}],
      "title": "Bouncer, die abholen",
      "type": "stat"
    },
    {
      "datasource": {"type": "prometheus", "uid": "prometheus"},
      "description": "Vom Agent gelesene Logzeilen pro Sekunde. Steht das auf 0, detektiert CrowdSec nichts mehr — der häufigste stille Ausfall, weil alles andere weiterläuft.",
      "fieldConfig": {
        "defaults": {"color": {"mode": "thresholds"}, "mappings": [], "thresholds": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "green", "value": 0.01}]}, "unit": "reqps"},
        "overrides": []
      },
      "gridPos": {"h": 4, "w": 6, "x": 12, "y": 0},
      "id": 3,
      "options": {"colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto"},
      "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "sum(rate(cs_filesource_hits_total[5m]))", "refId": "A"}],
      "title": "Gelesene Logzeilen/s",
      "type": "stat"
    },
    {
      "datasource": {"type": "prometheus", "uid": "prometheus"},
      "description": "Szenario-Overflows der letzten 24 h — also lokal erkannte Angriffe, im Gegensatz zu den von CAPI übernommenen.",
      "fieldConfig": {
        "defaults": {"color": {"mode": "thresholds"}, "mappings": [], "thresholds": {"mode": "absolute", "steps": [{"color": "text", "value": null}]}, "unit": "short"},
        "overrides": []
      },
      "gridPos": {"h": 4, "w": 6, "x": 18, "y": 0},
      "id": 4,
      "options": {"colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto"},
      "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "sum(increase(cs_bucket_overflowed_total[24h]))", "refId": "A"}],
      "title": "Lokale Overflows (24 h)",
      "type": "stat"
    },
    {
      "datasource": {"type": "prometheus", "uid": "prometheus"},
      "description": "Abholrate je Bouncer. Der Firewall-Bouncer auf den Proxies fragt alle 10 s. Die LAPI legt bei jeder neuen beobachteten Client-Adresse einen Eintrag `<name>@<ip>` an, und beobachtet wird die Pod-IP des netbird-ingress-Peers — deshalb erscheint ein logischer Bouncer hier mehrfach und nach Pod-Rollen unter neuen Namen.",
      "fieldConfig": {
        "defaults": {"color": {"mode": "palette-classic"}, "custom": {"axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": {"legend": false, "tooltip": false, "viz": false}, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": {"type": "linear"}, "showPoints": "never", "spanNulls": false, "stacking": {"group": "A", "mode": "none"}, "thresholdsStyle": {"mode": "off"}}, "mappings": [], "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}]}, "unit": "reqps"},
        "overrides": []
      },
      "gridPos": {"h": 8, "w": 12, "x": 0, "y": 4},
      "id": 5,
      "options": {"legend": {"calcs": ["lastNotNull"], "displayMode": "table", "placement": "right", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
      "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "sum by (bouncer) (rate(cs_lapi_bouncer_requests_total[5m]))", "legendFormat": "{{bouncer}}", "refId": "A"}],
      "title": "Bouncer-Abholrate",
      "type": "timeseries"
    },
    {
      "datasource": {"type": "prometheus", "uid": "prometheus"},
      "description": "Aktive Decisions nach Grund. `http:scan` dominiert und stammt aus den CAPI-Blocklisten.",
      "fieldConfig": {
        "defaults": {"color": {"mode": "palette-classic"}, "custom": {"axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", "hideFrom": {"legend": false, "tooltip": false, "viz": false}, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": {"type": "linear"}, "showPoints": "never", "spanNulls": false, "stacking": {"group": "A", "mode": "normal"}, "thresholdsStyle": {"mode": "off"}}, "mappings": [], "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}]}, "unit": "short"},
        "overrides": []
      },
      "gridPos": {"h": 8, "w": 12, "x": 12, "y": 4},
      "id": 6,
      "options": {"legend": {"calcs": ["lastNotNull"], "displayMode": "table", "placement": "right", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
      "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "sum by (reason) (cs_active_decisions)", "legendFormat": "{{reason}}", "refId": "A"}],
      "title": "Aktive Decisions nach Grund",
      "type": "timeseries"
    },
    {
      "datasource": {"type": "prometheus", "uid": "prometheus"},
      "description": "Welche Szenarien lokal auslösen. Das ist das, was CrowdSec selbst aus den Traefik-Logs erkennt.",
      "fieldConfig": {
        "defaults": {"color": {"mode": "palette-classic"}, "custom": {"axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "bars", "fillOpacity": 70, "gradientMode": "none", "hideFrom": {"legend": false, "tooltip": false, "viz": false}, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": {"type": "linear"}, "showPoints": "never", "spanNulls": false, "stacking": {"group": "A", "mode": "normal"}, "thresholdsStyle": {"mode": "off"}}, "mappings": [], "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}]}, "unit": "short"},
        "overrides": []
      },
      "gridPos": {"h": 8, "w": 12, "x": 0, "y": 12},
      "id": 7,
      "options": {"legend": {"calcs": ["sum"], "displayMode": "table", "placement": "right", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
      "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "sum by (name) (increase(cs_bucket_overflowed_total[1h]))", "legendFormat": "{{name}}", "refId": "A"}],
      "title": "Szenario-Overflows (je Stunde)",
      "type": "timeseries"
    },
    {
      "datasource": {"type": "prometheus", "uid": "prometheus"},
      "description": "Geparste gegen nicht geparste Zeilen. Ein Anstieg bei `ko` heißt meist, dass sich ein Logformat geändert hat und der Parser nicht mehr greift — CrowdSec liest dann weiter, erkennt aber nichts mehr.",
      "fieldConfig": {
        "defaults": {"color": {"mode": "palette-classic"}, "custom": {"axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": {"legend": false, "tooltip": false, "viz": false}, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": {"type": "linear"}, "showPoints": "never", "spanNulls": false, "stacking": {"group": "A", "mode": "none"}, "thresholdsStyle": {"mode": "off"}}, "mappings": [], "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}]}, "unit": "reqps"},
        "overrides": []
      },
      "gridPos": {"h": 8, "w": 12, "x": 12, "y": 12},
      "id": 8,
      "options": {"legend": {"calcs": ["lastNotNull"], "displayMode": "list", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
      "targets": [
        {"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "sum(rate(cs_parser_hits_ok_total[5m]))", "legendFormat": "geparst", "refId": "A"},
        {"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "sum(rate(cs_parser_hits_ko_total[5m]))", "legendFormat": "nicht geparst", "refId": "B"}
      ],
      "title": "Parser",
      "type": "timeseries"
    },
    {
      "datasource": {"type": "prometheus", "uid": "prometheus"},
      "description": "Whitelist-Treffer nach Grund. Nützlich beim Scharfschalten: was hier zählt, wäre sonst ein Kandidat für einen Ban gewesen.",
      "fieldConfig": {
        "defaults": {"color": {"mode": "palette-classic"}, "custom": {"axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": {"legend": false, "tooltip": false, "viz": false}, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": {"type": "linear"}, "showPoints": "never", "spanNulls": false, "stacking": {"group": "A", "mode": "none"}, "thresholdsStyle": {"mode": "off"}}, "mappings": [], "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}]}, "unit": "reqps"},
        "overrides": []
      },
      "gridPos": {"h": 8, "w": 12, "x": 0, "y": 20},
      "id": 9,
      "options": {"legend": {"calcs": ["lastNotNull"], "displayMode": "table", "placement": "right", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
      "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "sum by (reason) (rate(cs_node_wl_hits_ok_total[5m]))", "legendFormat": "{{reason}}", "refId": "A"}],
      "title": "Whitelist-Treffer",
      "type": "timeseries"
    },
    {
      "datasource": {"type": "prometheus", "uid": "prometheus"},
      "gridPos": {"h": 8, "w": 12, "x": 12, "y": 20},
      "id": 10,
      "options": {
        "code": {"language": "plaintext", "showLineNumbers": false, "showMiniMap": false},
        "content": "## Was dieses Dashboard NICHT zeigt\n\nAlle Panels stammen aus der LAPI und den Agents im Cluster. Die **Bouncer auf proxy-1 und proxy-2 exportieren eigene Metriken**, die hier fehlen:\n\n- `fw_bouncer_active_decisions` — was tatsächlich in den nftables-Sets steht\n- `fw_bouncer_dropped_packets` / `_bytes` — was wirklich verworfen wurde\n- `lapi_requests_total` / `_failures_total` — aus Sicht des Bouncers\n\nSie liegen auf `127.0.0.1:60601` der Proxy-Hosts. Die Hosts sind öffentlich und haben bisher keinerlei Metrik-Pfad; den Port ins Internet zu öffnen, nur um ein Scrape-Ziel zu bekommen, wäre der falsche Tausch.\n\n**Konsequenz:** „Bouncer-Abholrate\" oben zeigt, dass ein Bouncer *fragt* — nicht, dass er die Antwort auch angewandt hat. Solange der Modus `dry-run` ist, wendet er ohnehin nichts an.\n\nEin Scrape-Pfad für die Hosts ist eigene Arbeit und Voraussetzung dafür, dass „was wird wirklich geblockt\" beantwortbar wird."
      },
      "title": "Blinder Fleck",
      "type": "text"
    }
  ],
  "refresh": "30s",
  "schemaVersion": 38,
  "style": "dark",
  "tags": ["homelab", "crowdsec", "security"],
  "templating": {"list": []},
  "time": {"from": "now-6h", "to": "now"},
  "timepicker": {},
  "timezone": "",
  "title": "CrowdSec Overview",
  "uid": "crowdsec-overview",
  "version": 1,
  "weekStart": ""
 }
@@ -38,9 +38,6 @@ configMapGenerator:
  - name: dashboard-blackbox-overview
    files:
      - blackbox-overview.json
  - name: dashboard-crowdsec-overview
    files:
      - crowdsec-overview.json
  - name: dashboard-disk-health
    files:
      - disk-health.json
@@ -1,116 +0,0 @@
 ---
 # A read-only view of paperless's backup store, used as a recovery source and
 # nothing else.
 #
 # It points at the production prefix, which is the whole point — the eval
 # instance is a copy of the real archive's metadata — and that makes what is
 # ABSENT here the safety-critical part:
 #
 #   no retentionPolicy   The production store has "30d". Without one here
 #                        there is no policy to enforce, and enforcement is not
 #                        tied to taking a backup: the plugin runs it from the
 #                        instance sidecar on a timer
 #                        (instanceSidecarConfiguration.retentionPolicyIntervalSeconds,
 #                        default 1800). So "this cluster never backs up" would
 #                        not have been enough on its own.
 #
 # A second property makes it belt-and-braces. Retention runs
 # barman-cloud-backup-delete, which takes a server name and works inside
 # <destinationPath>/<serverName> — the store's own status keys its recovery
 # window by "pprl-pg", confirming that layout. This cluster is pprleval-pg, so
 # even an accidental retention policy would address a prefix that holds nothing
 # rather than production's backups.
 #
 # The cluster below completes the guarantee with isWALArchiver: false, so
 # nothing is written back either. It does load the plugin — that is required
 # for the restore to work at all — and read-only comes from the flag rather
 # than from leaving the plugin out.
 apiVersion: barmancloud.cnpg.io/v1
 kind: ObjectStore
 metadata:
  name: pprleval-pg-source
  namespace: paperless-eval
 spec:
  configuration:
    destinationPath: s3://renner-uno-k8s-cnpg/pprl-pg
    endpointURL: https://fsn1.your-objectstorage.com
    s3Credentials:
      accessKeyId:
        name: hetzner-s3-creds
        key: ACCESS_KEY_ID
      secretAccessKey:
        name: hetzner-s3-creds
        key: ACCESS_SECRET_KEY
      region:
        name: hetzner-s3-creds
        key: REGION
    wal:
      compression: gzip
    data:
      compression: gzip
 ---
 # A disposable clone of the paperless database, for measuring how well the
 # local LLM classifies documents without running the experiment against the
 # live archive.
 #
 # Losing this cluster costs a re-restore and nothing else, which is the intent
 # — it is rebuilt whenever a test needs a clean starting point.
 #
 # One instance, not two. Replication buys availability, and an eval instance
 # that is down is simply an eval that waits.
 apiVersion: postgresql.cnpg.io/v1
 kind: Cluster
 metadata:
  name: pprleval-pg
  namespace: paperless-eval
 spec:
  instances: 1
  imageName: ghcr.io/cloudnative-pg/postgresql:16
  enableSuperuserAccess: true
  storage:
    # Delete-reclaim, unlike the Retain default: this volume is a copy that
    # gets rebuilt, so it should vanish with the cluster rather than leave a
    # Released PV and an orphaned RBD image behind (issue #262).
    storageClass: ceph-block-ephemeral
    size: 10Gi
  # Declared with isWALArchiver: false, which is what kubernetes/RECOVERY.md
  # arrived at after an earlier version of that runbook said to omit it: the
  # instance manager loads this plugin to replay WAL during the restore, and
  # without it the instance never starts ("Unknown plugin:
  # barman-cloud.cloudnative-pg.io").
  #
  # externalClusters below only says WHERE the backup is; it does not get the
  # plugin injected into the pod. So "no plugin" is not the way to keep this
  # read-only — "plugin, not archiving" is, and it is the combination verified
  # on im-pg on 2026-07-25.
  plugins:
    - name: barman-cloud.cloudnative-pg.io
      enabled: true
      isWALArchiver: false
      parameters:
        barmanObjectName: pprleval-pg-source
  bootstrap:
    recovery:
      source: pprl-pg
      database: paperless
      owner: paperless
  externalClusters:
    - name: pprl-pg
      plugin:
        name: barman-cloud.cloudnative-pg.io
        parameters:
          barmanObjectName: pprleval-pg-source
          serverName: pprl-pg
  postgresql:
    parameters:
      shared_buffers: "256MB"
      max_connections: "100"
      # The production tuning for write reduction is deliberately not copied.
      # It exists to spare the SSDs under a database that runs for years; this
      # one is read-only for hours at a time and then deleted.
  resources:
    requests:
      cpu: 200m
      memory: 512Mi
    limits:
      memory: 1Gi
@@ -1,9 +0,0 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 namespace: paperless-eval
 resources:
  - secrets.yaml
  - cluster.yaml
  - pvcs.yaml
  - redis.yaml
  - paperless.yaml
@@ -1,193 +0,0 @@
 ---
 # Deliberately much smaller than the production ConfigMap. Everything that
 # exists to serve people is gone: no remote-user auth, no CSRF origins, no
 # barcode or mail consumption, no Tika/Gotenberg. Nothing consumes documents
 # here, and nothing signs in.
 #
 # What remains is the database connection and the AI settings, because those
 # are what the evaluation measures.
 apiVersion: v1
 kind: ConfigMap
 metadata:
  name: paperless-eval-config
  namespace: paperless-eval
 data:
  PAPERLESS_URL: "http://paperless.paperless-eval.svc.cluster.local"
  PAPERLESS_ALLOWED_HOSTS: "*"
  PAPERLESS_TIME_ZONE: "Europe/Berlin"
  PAPERLESS_REDIS: "redis://redis:6379"
  PAPERLESS_DBENGINE: "postgresql"
  PAPERLESS_DBHOST: "pprleval-pg-rw"
  PAPERLESS_DBNAME: "paperless"
  # Polling off, so a pod that finds an empty consume volume does not spend a
  # thread looking at it every ten seconds for the length of a run.
  PAPERLESS_CONSUMER_POLLING_INTERVAL: "0"
  # The AI settings under test. The endpoints are placeholders that each eval
  # job overrides for the model it is measuring — a job runs its own
  # llama-server beside paperless in the same pod, so the value that matters at
  # run time is the one in the job, not the one here.
  #
  # Same caveat as production, and sharper here: paperless reads these from an
  # ApplicationConfiguration row and falls back to the environment only when
  # that row is empty. The row comes across in the restored database, so
  # anything configured in the production admin UI arrives with it and silently
  # wins over this ConfigMap — an eval would then measure production's settings
  # while appearing to measure the job's.
  #
  # Check it before trusting any result, and after every fresh restore:
  #
  #   select llm_backend, llm_model, llm_embedding_backend, llm_embedding_model
  #     from paperless_applicationconfiguration;
  #
  # Every column must be null for the environment to apply.
  PAPERLESS_AI_ENABLED: "true"
  PAPERLESS_AI_LLM_BACKEND: "openai-like"
  PAPERLESS_AI_LLM_ENDPOINT: "http://127.0.0.1:8080/v1"
  PAPERLESS_AI_LLM_MODEL: "qwen3.6-35b-a3b"
  PAPERLESS_AI_LLM_ALLOW_INTERNAL_ENDPOINTS: "true"
  PAPERLESS_AI_LLM_CONTEXT_SIZE: "12288"
  PAPERLESS_AI_LLM_REQUEST_TIMEOUT: "900"
  PAPERLESS_AI_LLM_OUTPUT_LANGUAGE: "de-de"
  PAPERLESS_AI_LLM_EMBEDDING_BACKEND: "openai-like"
  PAPERLESS_AI_LLM_EMBEDDING_ENDPOINT: "http://llama-embed.ai.svc.cluster.local:8080/v1"
  PAPERLESS_AI_LLM_EMBEDDING_MODEL: "bge-m3"
  PAPERLESS_AI_LLM_EMBEDDING_CHUNK_SIZE: "1024"
 ---
 # A base instance that proves the restored database is usable and that the
 # search index and classifier build against it. The evaluation itself does not
 # run here — each job brings its own paperless container so it can point at its
 # own model — but this is what makes a broken clone visible before a job spends
 # hours on it.
 apiVersion: apps/v1
 kind: Deployment
 metadata:
  name: paperless
  namespace: paperless-eval
  labels:
    app.kubernetes.io/name: paperless
  annotations:
    configmap.reloader.stakater.com/reload: paperless-eval-config
    # Both, like production: the secret refreshes on the same 10m cycle, and
    # without this the pod would keep a rotated key until something else
    # restarted it — which defeats the point of matching the interval.
    secret.reloader.stakater.com/reload: paperless-eval-secrets
 spec:
  replicas: 1
  strategy:
    type: Recreate
  selector:
    matchLabels:
      app.kubernetes.io/name: paperless
  template:
    metadata:
      labels:
        app.kubernetes.io/name: paperless
    spec:
      enableServiceLinks: false
      securityContext:
        fsGroup: 1000
        fsGroupChangePolicy: OnRootMismatch
        seccompProfile:
          type: RuntimeDefault
      containers:
        - name: paperless
          # No $imagepolicy marker, unlike apps/paperless: a measurement is
          # only comparable against a fixed version, so this tag moves when
          # somebody decides to re-measure, not when a release lands.
          image: ghcr.io/paperless-ngx/paperless-ngx:3.1.2
          ports:
            - name: http
              containerPort: 8000
          envFrom:
            - configMapRef:
                name: paperless-eval-config
          env:
            - name: PAPERLESS_SECRET_KEY
              valueFrom:
                secretKeyRef:
                  name: paperless-eval-secrets
                  key: secret-key
            - name: PAPERLESS_DBUSER
              valueFrom:
                secretKeyRef:
                  name: pprleval-pg-app
                  key: username
            - name: PAPERLESS_DBPASS
              valueFrom:
                secretKeyRef:
                  name: pprleval-pg-app
                  key: password
          volumeMounts:
            - name: data
              mountPath: /usr/src/paperless/data
            - name: media
              mountPath: /usr/src/paperless/media
            - name: consume
              mountPath: /usr/src/paperless/consume
          # 60 minutes, against production's 10, and matching this app's Flux
          # Kustomization timeout.
          #
          # Production's budget was sized for a from-scratch boot on this same
          # corpus and held, so 10 would probably do — but there the volume is
          # Retain and the index survives, making that a one-off. Here the
          # volume is ephemeral and the index is rebuilt on every re-test, so
          # the expensive path is the normal one. A killed pod starts the work
          # over rather than resuming, which turns a slow boot into a
          # crash-loop that stalls Flux behind wait: true.
          #
          # The LLM index is not in this budget: it is a celery task, and boot
          # only runs init-llmindex-migrate, a migration check.
          startupProbe:
            httpGet:
              path: /accounts/login/
              port: http
            periodSeconds: 20
            timeoutSeconds: 5
            failureThreshold: 180
          livenessProbe:
            httpGet:
              path: /accounts/login/
              port: http
            initialDelaySeconds: 120
            periodSeconds: 30
            timeoutSeconds: 10
          readinessProbe:
            httpGet:
              path: /accounts/login/
              port: http
            initialDelaySeconds: 30
            periodSeconds: 10
            timeoutSeconds: 5
          resources:
            requests:
              cpu: 200m
              memory: 512Mi
            limits:
              memory: 2Gi
          securityContext:
            allowPrivilegeEscalation: false
            capabilities:
              drop: ["ALL"]
      volumes:
        - name: data
          persistentVolumeClaim:
            claimName: paperless-eval-data
        - name: media
          persistentVolumeClaim:
            claimName: paperless-eval-media
        - name: consume
          persistentVolumeClaim:
            claimName: paperless-eval-consume
 ---
 apiVersion: v1
 kind: Service
 metadata:
  name: paperless
  namespace: paperless-eval
 spec:
  selector:
    app.kubernetes.io/name: paperless
  ports:
    - port: 80
      targetPort: http
@@ -1,80 +0,0 @@
 ---
 # All four use ceph-block-ephemeral rather than the Retain default. Everything
 # here is derived — a restored database, an index built from it, a cache —  so
 # the volumes should go when the namespace does instead of leaving Released PVs
 # and orphaned RBD images to reclaim by hand (issue #262).
 #
 # /data holds the Tantivy search index and the classifier pickle, both rebuilt
 # from the database, and the LLM index — which is the expensive one at roughly
 # three hours for the full corpus.
 #
 # Measured on production for the same ~3900 documents: 36 MiB search index,
 # 125 MiB LLM index. 10Gi is far more than that needs, and deliberately so —
 # an embedding model with wider vectors than bge-m3's 1024 dimensions is
 # exactly what phase two varies, and running out of space mid-run would cost
 # the whole rebuild.
 apiVersion: v1
 kind: PersistentVolumeClaim
 metadata:
  name: paperless-eval-data
  namespace: paperless-eval
  labels:
    velero.io/exclude-from-backup: "true"
 spec:
  accessModes:
    - ReadWriteOnce
  storageClassName: ceph-block-ephemeral
  resources:
    requests:
      storage: 10Gi
 ---
 # Empty, and stays that way. Paperless refuses to start without the path, but
 # the AI path reads title, content and filename out of the database and never
 # opens a file — so the documents themselves are not copied here. That is what
 # keeps this instance cheap and keeps a second copy of the archive off the
 # cluster.
 apiVersion: v1
 kind: PersistentVolumeClaim
 metadata:
  name: paperless-eval-media
  namespace: paperless-eval
  labels:
    velero.io/exclude-from-backup: "true"
 spec:
  accessModes:
    - ReadWriteOnce
  storageClassName: ceph-block-ephemeral
  resources:
    requests:
      storage: 1Gi
 ---
 # Also empty: nothing is consumed here, but the path has to exist.
 apiVersion: v1
 kind: PersistentVolumeClaim
 metadata:
  name: paperless-eval-consume
  namespace: paperless-eval
  labels:
    velero.io/exclude-from-backup: "true"
 spec:
  accessModes:
    - ReadWriteOnce
  storageClassName: ceph-block-ephemeral
  resources:
    requests:
      storage: 1Gi
 ---
 apiVersion: v1
 kind: PersistentVolumeClaim
 metadata:
  name: paperless-eval-redis
  namespace: paperless-eval
  labels:
    velero.io/exclude-from-backup: "true"
 spec:
  accessModes:
    - ReadWriteOnce
  storageClassName: ceph-block-ephemeral
  resources:
    requests:
      storage: 1Gi
@@ -1,81 +0,0 @@
 ---
 apiVersion: apps/v1
 kind: Deployment
 metadata:
  name: redis
  namespace: paperless-eval
  labels:
    app.kubernetes.io/name: redis
 spec:
  replicas: 1
  strategy:
    type: Recreate
  selector:
    matchLabels:
      app.kubernetes.io/name: redis
  template:
    metadata:
      labels:
        app.kubernetes.io/name: redis
    spec:
      securityContext:
        runAsNonRoot: true
        runAsUser: 999
        fsGroup: 999
        seccompProfile:
          type: RuntimeDefault
      containers:
        - name: redis
          # No $imagepolicy marker; see the note on the paperless image.
          image: redis:8.10.1-alpine
          # No --save: this holds the celery queue and the suggestion cache,
          # both of which are regenerated. Persisting them across a restart of
          # a throwaway instance would only preserve stale suggestions.
          args:
            - redis-server
            - --save
            - ""
            - --loglevel
            - warning
          ports:
            - name: redis
              containerPort: 6379
          volumeMounts:
            - name: data
              mountPath: /data
          livenessProbe:
            tcpSocket:
              port: redis
            initialDelaySeconds: 10
          readinessProbe:
            exec:
              command: ["redis-cli", "ping"]
            initialDelaySeconds: 5
            periodSeconds: 10
          resources:
            requests:
              cpu: 20m
              memory: 64Mi
            limits:
              memory: 256Mi
          securityContext:
            allowPrivilegeEscalation: false
            readOnlyRootFilesystem: true
            capabilities:
              drop: ["ALL"]
      volumes:
        - name: data
          persistentVolumeClaim:
            claimName: paperless-eval-redis
 ---
 apiVersion: v1
 kind: Service
 metadata:
  name: redis
  namespace: paperless-eval
 spec:
  selector:
    app.kubernetes.io/name: redis
  ports:
    - port: 6379
      targetPort: redis
@@ -1,45 +0,0 @@
 ---
 # The same Hetzner S3 key every CNPG-using namespace pulls, needed here only so
 # the recovery source can be read. See apps/paperless for why one shared key
 # rather than per-app paths.
 apiVersion: external-secrets.io/v1
 kind: ExternalSecret
 metadata:
  name: hetzner-s3-creds
  namespace: paperless-eval
 spec:
  refreshInterval: 1h
  secretStoreRef:
    kind: ClusterSecretStore
    name: openbao
  target:
    name: hetzner-s3-creds
  dataFrom:
    - extract:
        key: infrastructure/hetzner-s3
 ---
 # Django's secret key, shared with the production instance rather than given
 # its own vault entry.
 #
 # It signs sessions and CSRF tokens, and this instance has no ingress and no
 # users — nothing signs in, so nothing is at stake in sharing it. A separate
 # entry would mean a vault write before the namespace can start, for a
 # throwaway instance that gets rebuilt.
 apiVersion: external-secrets.io/v1
 kind: ExternalSecret
 metadata:
  name: paperless-eval-secrets
  namespace: paperless-eval
 spec:
  # 10m, matching apps/paperless rather than the 1h default: the point of
  # sharing the key is that the two hold the same value, and a slower refresh
  # here would let them diverge for up to an hour after a rotation.
  refreshInterval: 10m
  secretStoreRef:
    kind: ClusterSecretStore
    name: openbao
  target:
    name: paperless-eval-secrets
  dataFrom:
    - extract:
        key: apps/paperless
@@ -82,10 +82,7 @@ data:
  # primarily English archives — the better fit for a German corpus.
  PAPERLESS_AI_LLM_EMBEDDING_BACKEND: "openai-like"
  PAPERLESS_AI_LLM_EMBEDDING_ENDPOINT: "http://llama-embed.ai.svc.cluster.local:8080/v1"
-  # Must match --alias in kubernetes/apps/llama-server/embed.yaml. Changing the
+  PAPERLESS_AI_LLM_EMBEDDING_MODEL: "qwen3-embedding-0.6b"
  # embedding model invalidates every stored vector, so a swap means the LLM
  # index has to be rebuilt.
  PAPERLESS_AI_LLM_EMBEDDING_MODEL: "bge-m3"
  PAPERLESS_AI_LLM_EMBEDDING_CHUNK_SIZE: "1024"
 ---
 apiVersion: apps/v1
@@ -1,34 +0,0 @@
 apiVersion: kustomize.toolkit.fluxcd.io/v1
 kind: Kustomization
 metadata:
  name: paperless-eval
  namespace: flux-system
 spec:
  # A throwaway copy of paperless for measuring the local LLM against the real
  # corpus. Its Cluster restores from paperless's backup store and archives
  # nothing back; see kubernetes/apps/paperless-eval/cluster.yaml.
  #
  # prune: true and no prune-disabled annotations anywhere underneath — the
  # data here is a copy, and being able to delete the whole namespace by
  # removing this file is the point.
  interval: 10m
  retryInterval: 1m
  # 30m is not enough: the first start restores the database and then builds
  # the search index over ~3900 documents before the webserver answers.
  timeout: 60m
  sourceRef:
    kind: GitRepository
    name: flux-system
  path: ./kubernetes/apps/paperless-eval
  prune: true
  wait: true
  dependsOn:
    - name: infrastructure
    - name: infrastructure-crds
    - name: infrastructure-ceph-cluster
    - name: infrastructure-cnpg-barman
  healthCheckExprs:
    - apiVersion: postgresql.cnpg.io/v1
      kind: Cluster
      failed: "has(status.phase) && status.phase.startsWith('Failed')"
      current: "has(status.phase) && (status.phase == 'Cluster in healthy state' || status.readyInstances > 0)"
@@ -1,27 +0,0 @@
 # Bouncer key for the firewall bouncers on proxy-1/proxy-2.
 #
 # Separate from crowdsec-bouncer-key ON PURPOSE. That key is rotated
 # quarterly by kubernetes/infrastructure/openbao/rotation/crowdsec-bouncer-key.yaml,
 # and both of its consumers live in the cluster, so ESO propagates a rotation
 # within 10m. The edge bouncers are Ansible-managed hosts: nothing there
 # re-reads OpenBao on a schedule, so a rotated key would leave them 401ing
 # against the LAPI until someone runs the playbook. This key is therefore
 # deliberately NOT enrolled in the rotator — it rotates when the playbook runs,
 # and the playbook is what carries the new value to the hosts.
 #
 # The value must match ansible vault_crowdsec_edge_bouncer_key.
 apiVersion: external-secrets.io/v1
 kind: ExternalSecret
 metadata:
  name: crowdsec-edge-bouncer-key
  namespace: crowdsec
 spec:
  refreshInterval: 10m
  secretStoreRef:
    kind: ClusterSecretStore
    name: openbao
  target:
    name: crowdsec-edge-bouncer-key
  dataFrom:
    - extract:
        key: infrastructure/crowdsec-edge-bouncer-key
@@ -36,51 +36,6 @@ spec:
  values:
    container_runtime: containerd
    # Addresses a community blocklist must never be allowed to ban.
    #
    # This is the ONLY declarative allowlist the chart offers. `cscli
    # allowlists` objects live in the LAPI database, and that database runs
    # without persistence here on purpose (see lapi.persistentVolume below),
    # so anything created through cscli is gone at the next restart. This file
    # is a ConfigMap mount, so it survives by construction. Upstream marks it
    # deprecated in favour of centralized allowlists; it is still the only
    # thing that works against a LAPI with no PVC.
    #
    # Scope: CAPI decisions only. Locally generated decisions are not filtered
    # here — they currently carry real client IPs (verified), because Traefik
    # resolves X-Forwarded-For past the netbird-ingress pod IP.
    #
    # The home WAN address is deliberately absent and is not needed: LAN
    # clients never present it. opentofu/unifi_nat_hairpin.tf DNATs the public
    # proxy addresses to the Traefik VIP on the UDM, so on-LAN traffic reaches
    # the cluster directly with a LAN source and never traverses the proxies.
    # Private ranges cannot appear in a community blocklist either way.
    config:
      capi_whitelists.yaml: |
        cidrs:
          # The proxy hosts themselves. Same three addresses as the
          # netbird-ingress-egress toCIDRSet and the hairpin endpoints —
          # keep all three lists in sync. proxy-1 is the v6-only member,
          # proxy-2 the dual-stack one.
          - 45.157.177.26/32
          - 2a03:4000:4b:c1e:4474:43ff:fedf:8f85/128
          - 2a03:4000:28:f6e:a4c7:59ff:fe1e:9a29/128
          # netbird.renner.uno — management, signal, relay and TURN.
          #
          # Not optional once a bouncer is armed. Its chain sits on the INPUT
          # hook, the proxy hosts run no other firewall, and there is therefore
          # no conntrack rule accepting established flows ahead of it. A ban on
          # this address would drop the RETURN traffic of the proxies' own
          # outbound control connection and take them off the mesh — with no
          # inbound attack involved at all.
          #
          # Checked when this was added: none of these addresses appear in the
          # current decision set. That is the state today, not a guarantee —
          # these are rented addresses on a hoster whose ranges do end up on
          # community blocklists.
          - 162.55.50.97/32
          - 2a01:4f8:c2c:4579::1/128
    # mTLS between agents and the LAPI. Do not switch back to password auth:
    # the LAPI runs without persistence, so its machine-registration DB is
    # wiped on every restart, and password-registered agents then get 401
@@ -123,24 +78,6 @@ spec:
    lapi:
      replicas: 1
      service:
        # Backend wiring for the public LAPI Ingress (lapi-ingress.yaml).
        #
        # These belong on the SERVICE, not on the Ingress: under the Kubernetes
        # Ingress provider `traefik.ingress.kubernetes.io/service.*` are service
        # annotations, and on an Ingress they are silently ignored. The symptom
        # is Traefik speaking plaintext to a TLS-only backend, which the LAPI
        # answers with "Client sent an HTTP request to an HTTPS server." and a
        # 400 — an error that looks like a malformed request rather than a
        # scheme mismatch.
        #
        # With tls.enabled the LAPI serves HTTPS on 8080 and offers no
        # plaintext port, so https here is mandatory rather than a hardening
        # choice. The ServersTransport carries the chart's own CA, since that
        # certificate is self-signed.
        annotations:
          traefik.ingress.kubernetes.io/service.serversscheme: https
          traefik.ingress.kubernetes.io/service.serverstransport: crowdsec-crowdsec-lapi@kubernetescrd
      # Far below the chart default, which reserves half a core per LAPI for a
      # workload that draws single-digit millicores — enough to leave the
      # cluster unschedulable with one node out. Only the reservation shrinks;
@@ -149,18 +86,10 @@ spec:
        requests:
          cpu: 50m
      podAnnotations:
-        # Reloader restarts the LAPI when either bouncer key changes. Both are
+        # Reloader restarts the LAPI when crowdsec-bouncer-key changes (it's an
-        # consumed as env vars from a secretKeyRef, and env is fixed at pod
+        # env-var secret, so a restart is required to pick up a rotated key). See
        # start — without a restart the LAPI keeps trusting the old value while
        # the bouncer already presents the new one, which is a 401 at exactly
        # the moment a rotation is supposed to be seamless.
        #
        # crowdsec-bouncer-key rotates on a schedule, see
        # kubernetes/infrastructure/openbao/rotation/crowdsec-bouncer-key.yaml.
-        # crowdsec-edge-bouncer-key rotates by hand together with
+        secret.reloader.stakater.com/reload: crowdsec-bouncer-key
        # ansible/plays/install-netbird-proxy.yml; it still needs to be listed
        # here, because the LAPI side of that rotation is this env var.
        secret.reloader.stakater.com/reload: crowdsec-bouncer-key,crowdsec-edge-bouncer-key
      persistentVolume:
        data:
          # Kept OFF deliberately. The #337 attempt to set this `true` (persist
@@ -189,17 +118,6 @@ spec:
            secretKeyRef:
              name: crowdsec-bouncer-key
              key: key
        # The firewall bouncers on proxy-1/proxy-2, over the public LAPI
        # endpoint (lapi-ingress.yaml). The chart's entrypoint re-runs
        # `register_bouncer` for every BOUNCER_KEY_* on each start and skips
        # names that already exist, so this survives the LAPI's wiped
        # registration DB the same way BOUNCER_KEY_ingress does — the env var
        # is the source of truth, not the database.
        - name: BOUNCER_KEY_edge
          valueFrom:
            secretKeyRef:
              name: crowdsec-edge-bouncer-key
              key: key
    agent:
      # Same reasoning as lapi.resources above, multiplied by three: this is a
@@ -217,32 +135,6 @@ spec:
        - namespace: ingress
          podName: traefik-*
          program: traefik
          # Read by polling, not inotify. Every path under
          # /var/log/containers is a symlink into /var/log/pods, and kubelet
          # rotates the target once it reaches containerLogMaxSize (10Mi,
          # roughly every eight hours for Traefik here). An inotify tailer
          # does not survive that rotation: it reopens the path, waits for a
          # file that is in fact already present, and never reads another
          # line. Nothing about that state looks broken from outside — the
          # pod stays Running and Ready, the LAPI keeps answering, and the
          # cumulative counters keep displaying whatever they reached, so the
          # only symptom is that no scenario ever fires again.
          #
          # CrowdSec detects the hazard itself and logs it at startup, with
          # this exact remedy in the message: "is a symlink, but inotify
          # polling is enabled. Crowdsec will not be able to detect rotation.
          # Consider setting poll_without_inotify to true". Upstream carries
          # the same case as crowdsec#2449 (k3s + Traefik), and the chart
          # gained this field in helm-charts#109 specifically to fix it.
          #
          # Orthogonal to the force_inotify: true that the chart hardcodes
          # into acquis.yaml and offers no way to unset. That one governs the
          # fsnotify watch on the directory, i.e. discovering the log of a
          # replacement Traefik pod; this one governs how an already
          # discovered file is read. Both true is the supported combination.
          #
          # Costs CPU per open file per upstream docs — two files per node.
          poll_without_inotify: true
      env:
        - name: COLLECTIONS
          value: "crowdsecurity/traefik crowdsecurity/http-cve crowdsecurity/appsec-virtual-patching crowdsecurity/appsec-generic-rules"
@@ -2,6 +2,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
  - bouncer-key-externalsecret.yaml
  - edge-bouncer-key-externalsecret.yaml
  - lapi-ingress.yaml
  - helmrelease.yaml
@@ -1,85 +0,0 @@
 # Public LAPI endpoint for the firewall bouncers on proxy-1/proxy-2.
 #
 # WHY PUBLIC AT ALL. The bouncers run as host processes on the proxies, and
 # the proxies have no mesh access: the netbird-proxy container runs non-root
 # (see the ip_unprivileged_port_start task in the netbird-proxy role), so it
 # has no NET_ADMIN and no /dev/net/tun, and its embedded NetBird client
 # therefore terminates the tunnel in userspace. The tunnel belongs to that
 # process, not to the host netns — a second process on the same host cannot
 # route through it. The alternative was a host-level NetBird client plus the
 # LAPI on a LoadBalancer VIP; this reuses the path every other service already
 # takes instead of adding two peers and an ACL policy.
 #
 # NO IP ALLOWLIST, deliberately. The bouncer key is the access control.
 #
 # An allowlist was the obvious second factor and it does not work here. A
 # request originating on a proxy host does not arrive carrying that host's
 # public address: the client connects to the local proxy on 127.0.0.1, the
 # reverse-proxy puts what it saw into X-Forwarded-For, and Traefik resolves
 # that. Measured against the netbird-proxy-watchdog, which makes exactly this
 # kind of request every round:
 #
 #   172.18.0.3 - - [02/Sep/2026:10:12:01] "GET /" 200 websecure-immich-...
 #   172.18.0.3 - - [02/Sep/2026:10:12:01] "GET /" 200 websecure-home-...
 #
 # 172.18.0.3 is a Docker bridge address on the proxy side. Allowlisting the
 # published proxy addresses would have rejected the bouncer with 403;
 # allowlisting 172.18.0.0/16 would be a private range on a host we do not
 # control the addressing of, which is not a security control.
 #
 # What remains is the API key, which is the real control anyway: this endpoint
 # serves read-only decision streams to a valid bouncer key and nothing else.
 # The admin surface is cscli, in-cluster. If the source address ever becomes
 # deterministic, an ipAllowList can be added on top.
 ---
 # The LAPI serves HTTPS only once tls.enabled is set — there is no plaintext
 # fallback and no second port; 8080 is the TLS port. Its certificate comes
 # from the chart's own self-signed CA, so Traefik needs that CA to verify the
 # backend. SANs include crowdsec-service.crowdsec.svc.cluster.local, which is
 # what the Service resolves to.
 apiVersion: traefik.io/v1alpha1
 kind: ServersTransport
 metadata:
  name: crowdsec-lapi
  namespace: crowdsec
 spec:
  serverName: crowdsec-service.crowdsec.svc.cluster.local
  rootCAs:
    - secret: crowdsec-root-secret
 ---
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
  name: crowdsec-lapi
  namespace: crowdsec
  labels:
    # Public DNS via Cloudflare; the proxies resolve it from outside. The UDM
    # external-dns instance skips it, LAN reaches it via the hairpin NAT.
    dns.renner.uno/public: "true"
  annotations:
    cert-manager.io/cluster-issuer: letsencrypt-cluster
    # NOT behind the Authentik gate: the client is a daemon with an API key,
    # not a browser that can complete an OIDC flow.
    #
    # The backend scheme and ServersTransport are NOT set here. Under the
    # Kubernetes Ingress provider, `traefik.ingress.kubernetes.io/service.*`
    # are SERVICE annotations — on an Ingress they are silently ignored, and
    # Traefik then talks plaintext to a TLS-only backend. They live on the
    # chart's Service instead, via lapi.service.annotations in helmrelease.yaml.
 spec:
  ingressClassName: traefik
  rules:
    - host: crowdsec.renner.uno
      http:
        paths:
          - path: /
            pathType: Prefix
            backend:
              service:
                name: crowdsec-service
                port:
                  number: 8080
  tls:
    - secretName: crowdsec-lapi-public-tls
      hosts:
        - crowdsec.renner.uno
@@ -69,13 +69,6 @@ kind: Namespace
 metadata:
  name: paperless
 ---
 # A disposable copy of paperless's database, for measuring how the local LLM
 # classifies documents without experimenting on the live archive.
 apiVersion: v1
 kind: Namespace
 metadata:
  name: paperless-eval
 ---
 apiVersion: v1
 kind: Namespace
 metadata:
@@ -34,33 +34,6 @@ spec:
        - ports:
            - port: "8080"
              protocol: TCP
    # The paperless-eval namespace reaches the embedding endpoint for the same
    # reason paperless does; its chat model runs inside each eval job's own pod
    # and needs no grant here.
    #
    # Scoped to the namespace rather than to an app label, unlike the grants
    # around it: that namespace exists only to run evaluations, and its
    # workload is jobs created per measurement whose labels vary with what is
    # being measured. A label match would be a trap that presents as an
    # embedding call silently dropped.
    - fromEndpoints:
        - matchLabels:
            "k8s:io.kubernetes.pod.namespace": paperless-eval
      toPorts:
        - ports:
            - port: "8080"
              protocol: TCP
    # Home Assistant's llama.cpp conversation agent calls the chat model only —
    # it has no use for the embedding endpoint, so unlike paperless above this
    # grant does not extend to llama-embed.
    - fromEndpoints:
        - matchLabels:
            "k8s:io.kubernetes.pod.namespace": home-assistant
            app.kubernetes.io/name: home-assistant
      toPorts:
        - ports:
            - port: "8080"
              protocol: TCP
    - fromEndpoints:
        - matchLabels:
            "k8s:io.kubernetes.pod.namespace": monitoring
@@ -167,18 +167,6 @@ spec:
    - toEndpoints:
        - matchLabels:
            "k8s:io.kubernetes.pod.namespace": influxdb
    # The cluster's own chat model, for the llama.cpp conversation agent that
    # backs Assist. Scoped to the one pod and port rather than the ai namespace
    # as a whole, so this grant cannot widen if something else is placed there —
    # the embedding endpoint next to it is deliberately not included.
    - toEndpoints:
        - matchLabels:
            "k8s:io.kubernetes.pod.namespace": ai
            app.kubernetes.io/name: llama-server
      toPorts:
        - ports:
            - port: "8080"
              protocol: TCP
    # traefik for east-west OIDC roundtrips that go through the
    # public hostname via k8s_gateway.
    - toEndpoints:
@@ -6,7 +6,6 @@ resources:
  - ingress-isolation.yaml
  - authentik-ns.yaml
  - paperless-ns.yaml
  - paperless-eval-ns.yaml
  - audiobookshelf-ns.yaml
  - home-assistant-ns.yaml
  - spegel-ns.yaml
@@ -1,141 +0,0 @@
 # Narrower than paperless-ns.yaml, because this namespace serves nobody.
 #
 # There is no ingress, no gate outpost and no hostname: the only things that
 # talk to it are its own pods. Monitoring is admitted so a run can be watched,
 # and host/remote-node for the kubelet probes.
 #
 # Egress reaches the ai namespace for the embedding endpoint, which is the one
 # dependency an eval job has outside its own pod, and world:443 so the CNPG
 # recovery can read the backup store on Hetzner.
 apiVersion: cilium.io/v2
 kind: CiliumNetworkPolicy
 metadata:
  name: default-deny-ingress
  namespace: paperless-eval
 spec:
  endpointSelector: {}
  ingress:
    - fromEndpoints:
        - matchLabels:
            "k8s:io.kubernetes.pod.namespace": paperless-eval
    - fromEndpoints:
        - matchLabels:
            "k8s:io.kubernetes.pod.namespace": monitoring
    - fromEntities:
        - host
        - remote-node
    # ICMP — all standard ICMPv4 + ICMPv6 from any cluster pod. See
    # home-assistant-ns.yaml for the rationale.
    - fromEndpoints:
        - matchExpressions:
            - key: io.kubernetes.pod.namespace
              operator: Exists
      icmps:
        - fields:
            - { type: 0,   family: IPv4 }
            - { type: 3,   family: IPv4 }
            - { type: 8,   family: IPv4 }
            - { type: 11,  family: IPv4 }
            - { type: 12,  family: IPv4 }
            - { type: 1,   family: IPv6 }
            - { type: 2,   family: IPv6 }
            - { type: 3,   family: IPv6 }
            - { type: 4,   family: IPv6 }
            - { type: 128, family: IPv6 }
            - { type: 129, family: IPv6 }
            - { type: 133, family: IPv6 }
            - { type: 134, family: IPv6 }
            - { type: 135, family: IPv6 }
            - { type: 136, family: IPv6 }
            - { type: 143, family: IPv6 }
  egress:
    - toEndpoints:
        - matchLabels:
            "k8s:io.kubernetes.pod.namespace": paperless-eval
    # The embedding endpoint. The chat model runs inside each job's own pod, so
    # it needs no grant; the embedder is shared with production because it is
    # read-only and cheap, and swapping it is a phase-two concern.
    - toEndpoints:
        - matchLabels:
            "k8s:io.kubernetes.pod.namespace": ai
            app.kubernetes.io/name: llama-embed
      toPorts:
        - ports:
            - port: "8080"
              protocol: TCP
    - toEndpoints:
        - matchLabels:
            "k8s:io.kubernetes.pod.namespace": rook-ceph
    - toEndpoints:
        - matchLabels:
            "k8s:io.kubernetes.pod.namespace": kube-system
            k8s-app: kube-dns
      toPorts:
        - ports:
            - port: "53"
              protocol: UDP
    - toEntities:
        - host
        - remote-node
    # world:443 — the CNPG recovery reads the backup store on Hetzner Object
    # Storage, and image pulls.
    - toEntities:
        - world
      toPorts:
        - ports:
            - port: "443"
              protocol: TCP
    # ICMP — egress mirror; see ingress block above.
    - toEndpoints:
        - matchExpressions:
            - key: io.kubernetes.pod.namespace
              operator: Exists
      icmps:
        - fields:
            - { type: 0,   family: IPv4 }
            - { type: 3,   family: IPv4 }
            - { type: 8,   family: IPv4 }
            - { type: 11,  family: IPv4 }
            - { type: 12,  family: IPv4 }
            - { type: 1,   family: IPv6 }
            - { type: 2,   family: IPv6 }
            - { type: 3,   family: IPv6 }
            - { type: 4,   family: IPv6 }
            - { type: 128, family: IPv6 }
            - { type: 129, family: IPv6 }
            - { type: 133, family: IPv6 }
            - { type: 134, family: IPv6 }
            - { type: 135, family: IPv6 }
            - { type: 136, family: IPv6 }
            - { type: 143, family: IPv6 }
 ---
 # The CNPG operator polls each instance's status endpoint over HTTPS
 # (https://<pod-ip>:8000/pg/status) to populate Cluster status. Without it the
 # Cluster reports "Instance Status Extraction Error: HTTP communication issue"
 # while Postgres serves normally — issue #433, where the operator's dial timed
 # out because the apps baseline never allowed the database namespace.
 #
 # It matters here even for a throwaway: the Flux Kustomization waits on
 # Cluster status, and a smoke test that cannot report its own health is not one.
 #
 # Scoped to the pprleval-pg endpoints and that one port, like the equivalent
 # rules for wp-pg and ha-pg: the barman and mariadb operators sharing the
 # database namespace have no reason to reach anything here.
 apiVersion: cilium.io/v2
 kind: CiliumNetworkPolicy
 metadata:
  name: pprleval-pg-allow-cnpg-operator
  namespace: paperless-eval
 spec:
  endpointSelector:
    matchLabels:
      cnpg.io/cluster: pprleval-pg
  ingress:
    - fromEndpoints:
        - matchLabels:
            "k8s:io.kubernetes.pod.namespace": database
            app.kubernetes.io/name: cloudnative-pg
      toPorts:
        - ports:
            - port: "8000"
              protocol: TCP