fix(paperless-eval): score a right answer the matcher threw away

The AI engine reports an unresolved guess as a name in
`suggested_correspondents` rather than an id in `correspondents`, and
paperless_ai.matching only resolves it when difflib.get_close_matches
clears MATCH_THRESHOLD = 0.8. A correct but more verbose answer does not:
"Generali Deutschland Krankenversicherung AG" fails against the existing
"Generali AG", leaving `correspondents` empty.

Scored as it was, that counts as a miss, which would have blamed the
model for a matcher limit. Score both instead — the id as offered, and
the name matched wordwise against the true label — so the two causes stay
separable. The classic engine emits no `suggested_*` fields, so its two
columns coincide and the comparison stays fair either way.

Names now come out of postgres alongside the ids, tab-separated because
'|' occurs in correspondent names.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

.woodpecker/deploy-netbird-proxy.yml

@@ -10,18 +10,6 @@ when:
       # install-netbird-proxy.yml also applies the edge bouncer.
       - "ansible/roles/crowdsec-edge-bouncer/**"
       - "ansible/plays/install-netbird-proxy.yml"
-      # The variables that steer those roles, not just the roles themselves.
-      # Without these two, a change that only touches inventory never deploys
-      # and nothing says so: arming the bouncer lives in host_vars, and its
-      # LAPI key lives in group_vars, so both would sit in git looking applied
-      # while the hosts kept running the previous configuration.
-      #
-      # pve_hosts.yml is deliberately NOT listed. It is the single inventory
-      # file for the whole fleet, so listing it would redeploy the proxies for
-      # any unrelated host edit. The cost is that a change to [netbird_proxy]
-      # group membership needs a manual deployment run.
-      - "ansible/inventory/host_vars/proxy-*.yml"
-      - "ansible/inventory/group_vars/netbird_proxy/**"
       - ".woodpecker/deploy-netbird-proxy.yml"
   - event: deployment
     evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "deploy-netbird-proxy"'

ansible/roles/crowdsec-edge-bouncer/defaults/main.yml

@@ -28,23 +28,14 @@ crowdsec_bouncer_sha256: f86e4b72693549d99f40a9402abefb894108f047a3fbe6e72fada25
 crowdsec_bouncer_api_url: https://crowdsec.renner.uno/
 crowdsec_bouncer_api_key: "{{ vault_crowdsec_edge_bouncer_key }}"
 
-# Enforcing. The bouncer creates two nftables tables (ip crowdsec, ip6
-# crowdsec6) holding the decisions it pulled, touches no existing chain and
-# sets no chain policy — SSH and :443 stay open unless the source is banned.
+# STARTS INERT, and that is the point.
 #
-# `dry-run` is the other value: it pulls and logs what it would block without
-# creating a single rule. That is what a new host should be given before it
-# enforces, and what an existing one can be dropped back to, per host through
-# inventory/host_vars, without disarming the rest.
-#
-# Worth knowing before arming anything new: the first bouncer on a path
-# enforces the whole CAPI set at once — around 15k addresses today — so the
-# false-positive window belongs before the switch, not after it. And because
-# this chain sits on the INPUT hook while these hosts run no other firewall,
-# there is no conntrack rule accepting established flows ahead of it: a banned
-# address is dropped even when it is answering our own outbound connection.
-# That is why capi_whitelists.yaml carries the proxies and netbird.renner.uno.
-crowdsec_bouncer_mode: nftables
+# dry-run pulls decisions and logs what it would block without creating a
+# single nftables rule. The first bouncer to arm anywhere enforces ~24k CAPI
+# decisions at once on the only external entry path, so the false-positive
+# window has to come before the enforcement, not after it. Flip to `nftables`
+# one host at a time once the log looks right.
+crowdsec_bouncer_mode: dry-run
 
 crowdsec_bouncer_update_frequency: 10s
 crowdsec_bouncer_log_level: info

ansible/roles/crowdsec-edge-bouncer/tasks/main.yml

@@ -19,40 +19,13 @@
     state: directory
     mode: "0755"
 
-# Fetched on the CONTROLLER and pushed, not pulled by the host.
-#
-# proxy-1 is IPv6-only, and neither github.com nor objects.githubusercontent.com
-# publishes an AAAA record — a get_url on the host fails with "Network is
-# unreachable" and no amount of retrying fixes it. Mirroring into
-# gitea.renner.uno is no help either: its public records are the internal LB
-# VIP (10.2.99.86 / fd99::86), so it is unreachable from outside the LAN. And
-# upstream ships no container image for this bouncer, which is expected —
-# it manipulates host netfilter.
-#
-# The CI runner has v4, so it does the fetching. The checksum is verified
-# there, before anything is copied to a host.
-- name: Fetch the pinned bouncer release on the controller
+- name: Fetch the pinned bouncer release
   ansible.builtin.get_url:
     url: >-
       https://github.com/crowdsecurity/cs-firewall-bouncer/releases/download/{{
       crowdsec_bouncer_version }}/crowdsec-firewall-bouncer-linux-amd64.tgz
-    dest: /tmp/crowdsec-firewall-bouncer-{{ crowdsec_bouncer_version }}.tgz
-    checksum: sha256:{{ crowdsec_bouncer_sha256 }}
-    mode: "0644"
-  delegate_to: localhost
-  become: false
-  run_once: true
-
-# Only the download had to move to the controller — the extraction stays on
-# the host. The ci-ansible image ships a tar that ansible rejects ("detected as
-# tar type None. GNU tar required"), while both host distributions have GNU
-# tar. Shipping the archive rather than the unpacked binary also keeps the
-# artefact that the checksum was verified against intact all the way to the
-# host.
-- name: Copy the bouncer archive to the host
-  ansible.builtin.copy:
-    src: /tmp/crowdsec-firewall-bouncer-{{ crowdsec_bouncer_version }}.tgz
     dest: /opt/crowdsec-firewall-bouncer-{{ crowdsec_bouncer_version }}.tgz
+    checksum: sha256:{{ crowdsec_bouncer_sha256 }}
     mode: "0644"
 
 # Only the binary. The tarball also carries install.sh / upgrade.sh /

kubernetes/apps/home-assistant/deployment.yaml

@@ -95,7 +95,7 @@ spec:
               memory: 32Mi
       containers:
         - name: home-assistant
-          image: ghcr.io/home-assistant/home-assistant:2026.9.0 # {"$imagepolicy": "flux-system:home-assistant"}
+          image: ghcr.io/home-assistant/home-assistant:2026.8.3 # {"$imagepolicy": "flux-system:home-assistant"}
           env:
             - name: TZ
               value: Europe/Berlin

kubernetes/apps/llama-server/deployment.yaml

@@ -75,44 +75,11 @@ spec:
             # many small per-expert matmuls that sets the pace, not bandwidth.
             - --threads
             - "4"
-            # 16k, and going above it corrupts inference rather than failing.
-            # At 32768 per slot this server answered a 650-token classification
-            # prompt with 1025 tokens of "!" — the KV cache producing numbers
-            # rather than language. It does not crash, it does not warn, and
-            # the OpenAI response looks well-formed, so the damage only shows
-            # in what callers get back: Paperless ran past its 300 s timeout,
-            # and Home Assistant got an empty completion and a hallucinated
-            # device list.
-            #
-            # The same Vulkan/Mesa stack already fails hard at --ubatch-size
-            # 2048; this is the quiet version of that limit. Raise this only
-            # against a generation that is read, not against a server that
-            # merely starts.
-            #
+            # 16k rather than 8k because the measured prefill falloff is mild
+            # and this model's KV cache is unusually cheap — only 10 of its 40
+            # layers use full attention.
             - --ctx-size
             - "16384"
-            # One slot, so the 16384 belongs to one request at a time.
-            #
-            # Left on auto, llama.cpp opens four slots and — because auto is
-            # also what enables --kv-unified — has them share a single buffer
-            # of this size. Four Paperless requests at once wanted 6209, 6472,
-            # 4303 and 1271 tokens against those 16384, and all four died with
-            # "Context size has been exceeded".
-            #
-            # They were concurrent because a client timeout does not stop the
-            # generation behind it: each abandoned request keeps consuming the
-            # shared budget the next one needs, so one slow document is enough
-            # to start the pile-up. Queueing is the behaviour worth having
-            # here — nothing calls this server in parallel on purpose, and
-            # Paperless's celery worker runs at concurrency 1.
-            #
-            # No liveness probe change goes with this. The server recovered on
-            # its own once the batch drained, so a probe that restarted the pod
-            # would have made the incident worse; with a single slot, one that
-            # asked for a real completion would also queue behind live work and
-            # kill healthy pods during a long document.
-            - --parallel
-            - "1"
             # A micro-batch of 2048 or more makes this Vulkan/Mesa stack fail
             # with ErrorDeviceLost.
             - --ubatch-size

kubernetes/apps/llama-server/embed.yaml

@@ -41,7 +41,7 @@ spec:
           image: ghcr.io/ggml-org/llama.cpp:server-vulkan-b10731
           args:
             - --hf-repo
-            - gpustack/bge-m3-GGUF:Q8_0
+            - Qwen/Qwen3-Embedding-0.6B-GGUF:Q8_0
             # Serves POST /v1/embeddings.
             - --embeddings
             # A short, stable id for consumers. Without it the model is
@@ -50,41 +50,28 @@ spec:
             # One name only: llama-server keeps aliases in a sorted set and
             # reports the first as the id, so a second name can steal it.
             - --alias
-            - bge-m3
-            # bge-m3 is trained for CLS pooling — a different type from the
-            # Qwen family this replaces, which uses last-token. Left unspecified
-            # llama-server takes the GGUF default, which is not visible from
-            # outside and would silently produce vectors the model was not
-            # trained to emit. Read from the model's own 1_Pooling/config.json
-            # rather than assumed; of the candidates measured, three different
-            # pooling types were needed.
+            - qwen3-embedding-0.6b
+            # Qwen3-Embedding is trained for last-token pooling. Left
+            # unspecified llama-server takes the GGUF's default, which is not
+            # visible from outside and would silently produce vectors the model
+            # was not trained to emit. Set before anything is indexed: changing
+            # pooling later invalidates every stored embedding.
             - --pooling
-            - cls
+            - last
             - --n-gpu-layers
             - "99"
             - --threads
             - "2"
-            # Paperless chunks at 1024 tokens, but counts them with its own
-            # tokenizer while this server measures the batch in bge-m3's.
-            # Those disagree: XLM-RoBERTa splits German into more pieces than
-            # llama_index's counter does, so a chunk Paperless considers full
-            # at 1024 arrived here as 1069 and the request failed outright —
-            # "input (1069 tokens) is too large to process".
-            #
-            # The headroom therefore has to absorb a tokenizer ratio, not just
-            # a rounding error, and 1.5x covers far more than the 1.044 that
-            # broke it. The defaults are batch 2048 / ubatch 512, and with
-            # --embeddings llama-server clamps both to the smaller rather than
-            # run them mismatched, so both are set.
-            #
-            # 2048 stays avoided: that is where the chat model met
-            # ErrorDeviceLost on this Vulkan stack. That failure came under a
-            # 20 GB model's memory pressure and may not bind a 600 MB one, but
-            # this does not need to find out.
+            # Paperless chunks documents at 1024 tokens. The defaults are
+            # batch 2048 / ubatch 512, and with --embeddings llama-server
+            # clamps both down to 512 rather than run them mismatched — below
+            # the chunk size. Matching both to 1024 fits a whole chunk in one
+            # pass; 2048 is deliberately avoided, since that is where this
+            # Vulkan stack fails with ErrorDeviceLost.
             - --batch-size
-            - "1536"
+            - "1024"
             - --ubatch-size
-            - "1536"
+            - "1024"
             # Room for a full chunk plus overhead rather than exactly one.
             - --ctx-size
             - "4096"
@@ -104,14 +91,6 @@ spec:
             capabilities:
               drop:
                 - ALL
-          # Measured, not estimated: embedding 400 real documents peaked at
-          # 1.6 GiB (Prometheus max_over_time on the working set). The limit is
-          # well above that because the footprint is dominated by the four slots
-          # at 4096 context and the Vulkan working set rather than by the
-          # weights — the Qwen model this replaces peaked at 10.6 GiB on the
-          # same settings for the same 600 MiB of weights, and several
-          # candidates were OOMKilled at limits that looked generous against
-          # their model size.
           resources:
             requests:
               cpu: 250m
@@ -119,7 +98,7 @@ spec:
               gpu.intel.com/i915: "1"
             limits:
               cpu: "2"
-              memory: 8Gi
+              memory: 4Gi
               gpu.intel.com/i915: "1"
           # ~600 MiB rather than ~5 GiB, so a much shorter budget than the chat
           # model needs, but still sized for a cold download.

kubernetes/apps/monitoring-rules/crowdsec-servicemonitor.yaml

@@ -1,36 +0,0 @@
-# Scrapes CrowdSec. Until this landed, nothing did: `up{namespace="crowdsec"}`
-# returned zero series and Prometheus held no cs_* metric at all, so the
-# detection side of the ingress path was invisible — including whether the
-# agents were reading logs and whether any bouncer was pulling decisions.
-#
-# One monitor for both services. They carry the same shape (a `metrics` port on
-# 6060) and differ only in `app`, so a matchExpressions selector keeps this to
-# a single object rather than two near-identical ones:
-#
-#   crowdsec-service        the LAPI      — decisions, bouncer + machine requests
-#   crowdsec-agent-service  the agents    — acquisition, parsers, scenarios
-#
-# The crowdsec-ns CiliumNetworkPolicy already admits the monitoring namespace
-# without a port restriction, so no policy change is needed.
-#
-# `release: kube-prometheus-stack` is what the chart's Prometheus selects on.
-apiVersion: monitoring.coreos.com/v1
-kind: ServiceMonitor
-metadata:
-  name: crowdsec
-  namespace: crowdsec
-  labels:
-    app.kubernetes.io/name: crowdsec
-    release: kube-prometheus-stack
-spec:
-  selector:
-    matchExpressions:
-      - key: app
-        operator: In
-        values:
-          - crowdsec-service
-          - crowdsec-agent-service
-  endpoints:
-    - port: metrics
-      path: /metrics
-      interval: 30s

kubernetes/apps/monitoring-rules/kustomization.yaml

@@ -23,7 +23,6 @@ resources:
   - prometheusrule-cnpg-backup.yaml
   - alloy-servicemonitor.yaml
   - cnpg-podmonitors.yaml
-  - crowdsec-servicemonitor.yaml
   - home-assistant-servicemonitor.yaml
   - immich-servicemonitor.yaml
   - loki-servicemonitor.yaml

kubernetes/apps/monitoring/dashboards/crowdsec-overview.json

@@ -1,163 +0,0 @@
-{
-  "annotations": {"list": []},
-  "description": "CrowdSec: Erkennung im Cluster, Durchsetzung am NetBird-Edge",
-  "editable": true,
-  "fiscalYearStartMonth": 0,
-  "graphTooltip": 1,
-  "id": null,
-  "links": [],
-  "panels": [
-    {
-      "datasource": {"type": "prometheus", "uid": "prometheus"},
-      "description": "Aktive Ban-Entscheidungen in der LAPI, ganz überwiegend aus den CAPI-Community-Blocklisten. Das ist die Menge, die ein scharfgeschalteter Bouncer durchsetzt.",
-      "fieldConfig": {
-        "defaults": {"color": {"mode": "thresholds"}, "mappings": [], "thresholds": {"mode": "absolute", "steps": [{"color": "text", "value": null}]}, "unit": "short"},
-        "overrides": []
-      },
-      "gridPos": {"h": 4, "w": 6, "x": 0, "y": 0},
-      "id": 1,
-      "options": {"colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto"},
-      "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "sum(cs_active_decisions)", "refId": "A"}],
-      "title": "Aktive Decisions",
-      "type": "stat"
-    },
-    {
-      "datasource": {"type": "prometheus", "uid": "prometheus"},
-      "description": "Bouncer, die in den letzten 10 Minuten Decisions abgeholt haben. Fällt das unter die erwartete Zahl, holt ein Bouncer nichts mehr — und setzt damit einen veralteten Stand durch, ohne dass irgendetwas ausfällt.",
-      "fieldConfig": {
-        "defaults": {"color": {"mode": "thresholds"}, "mappings": [], "thresholds": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "green", "value": 1}]}, "unit": "short"},
-        "overrides": []
-      },
-      "gridPos": {"h": 4, "w": 6, "x": 6, "y": 0},
-      "id": 2,
-      "options": {"colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto"},
-      "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "count(sum by (bouncer) (rate(cs_lapi_bouncer_requests_total[10m])) > 0)", "refId": "A"}],
-      "title": "Bouncer, die abholen",
-      "type": "stat"
-    },
-    {
-      "datasource": {"type": "prometheus", "uid": "prometheus"},
-      "description": "Vom Agent gelesene Logzeilen pro Sekunde. Steht das auf 0, detektiert CrowdSec nichts mehr — der häufigste stille Ausfall, weil alles andere weiterläuft.",
-      "fieldConfig": {
-        "defaults": {"color": {"mode": "thresholds"}, "mappings": [], "thresholds": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "green", "value": 0.01}]}, "unit": "reqps"},
-        "overrides": []
-      },
-      "gridPos": {"h": 4, "w": 6, "x": 12, "y": 0},
-      "id": 3,
-      "options": {"colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto"},
-      "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "sum(rate(cs_filesource_hits_total[5m]))", "refId": "A"}],
-      "title": "Gelesene Logzeilen/s",
-      "type": "stat"
-    },
-    {
-      "datasource": {"type": "prometheus", "uid": "prometheus"},
-      "description": "Szenario-Overflows der letzten 24 h — also lokal erkannte Angriffe, im Gegensatz zu den von CAPI übernommenen.",
-      "fieldConfig": {
-        "defaults": {"color": {"mode": "thresholds"}, "mappings": [], "thresholds": {"mode": "absolute", "steps": [{"color": "text", "value": null}]}, "unit": "short"},
-        "overrides": []
-      },
-      "gridPos": {"h": 4, "w": 6, "x": 18, "y": 0},
-      "id": 4,
-      "options": {"colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto"},
-      "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "sum(increase(cs_bucket_overflowed_total[24h]))", "refId": "A"}],
-      "title": "Lokale Overflows (24 h)",
-      "type": "stat"
-    },
-    {
-      "datasource": {"type": "prometheus", "uid": "prometheus"},
-      "description": "Abholrate je Bouncer. Der Firewall-Bouncer auf den Proxies fragt alle 10 s. Die LAPI legt bei jeder neuen beobachteten Client-Adresse einen Eintrag `<name>@<ip>` an, und beobachtet wird die Pod-IP des netbird-ingress-Peers — deshalb erscheint ein logischer Bouncer hier mehrfach und nach Pod-Rollen unter neuen Namen.",
-      "fieldConfig": {
-        "defaults": {"color": {"mode": "palette-classic"}, "custom": {"axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": {"legend": false, "tooltip": false, "viz": false}, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": {"type": "linear"}, "showPoints": "never", "spanNulls": false, "stacking": {"group": "A", "mode": "none"}, "thresholdsStyle": {"mode": "off"}}, "mappings": [], "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}]}, "unit": "reqps"},
-        "overrides": []
-      },
-      "gridPos": {"h": 8, "w": 12, "x": 0, "y": 4},
-      "id": 5,
-      "options": {"legend": {"calcs": ["lastNotNull"], "displayMode": "table", "placement": "right", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
-      "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "sum by (bouncer) (rate(cs_lapi_bouncer_requests_total[5m]))", "legendFormat": "{{bouncer}}", "refId": "A"}],
-      "title": "Bouncer-Abholrate",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {"type": "prometheus", "uid": "prometheus"},
-      "description": "Aktive Decisions nach Grund. `http:scan` dominiert und stammt aus den CAPI-Blocklisten.",
-      "fieldConfig": {
-        "defaults": {"color": {"mode": "palette-classic"}, "custom": {"axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", "hideFrom": {"legend": false, "tooltip": false, "viz": false}, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": {"type": "linear"}, "showPoints": "never", "spanNulls": false, "stacking": {"group": "A", "mode": "normal"}, "thresholdsStyle": {"mode": "off"}}, "mappings": [], "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}]}, "unit": "short"},
-        "overrides": []
-      },
-      "gridPos": {"h": 8, "w": 12, "x": 12, "y": 4},
-      "id": 6,
-      "options": {"legend": {"calcs": ["lastNotNull"], "displayMode": "table", "placement": "right", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
-      "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "sum by (reason) (cs_active_decisions)", "legendFormat": "{{reason}}", "refId": "A"}],
-      "title": "Aktive Decisions nach Grund",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {"type": "prometheus", "uid": "prometheus"},
-      "description": "Welche Szenarien lokal auslösen. Das ist das, was CrowdSec selbst aus den Traefik-Logs erkennt.",
-      "fieldConfig": {
-        "defaults": {"color": {"mode": "palette-classic"}, "custom": {"axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "bars", "fillOpacity": 70, "gradientMode": "none", "hideFrom": {"legend": false, "tooltip": false, "viz": false}, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": {"type": "linear"}, "showPoints": "never", "spanNulls": false, "stacking": {"group": "A", "mode": "normal"}, "thresholdsStyle": {"mode": "off"}}, "mappings": [], "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}]}, "unit": "short"},
-        "overrides": []
-      },
-      "gridPos": {"h": 8, "w": 12, "x": 0, "y": 12},
-      "id": 7,
-      "options": {"legend": {"calcs": ["sum"], "displayMode": "table", "placement": "right", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
-      "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "sum by (name) (increase(cs_bucket_overflowed_total[1h]))", "legendFormat": "{{name}}", "refId": "A"}],
-      "title": "Szenario-Overflows (je Stunde)",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {"type": "prometheus", "uid": "prometheus"},
-      "description": "Geparste gegen nicht geparste Zeilen. Ein Anstieg bei `ko` heißt meist, dass sich ein Logformat geändert hat und der Parser nicht mehr greift — CrowdSec liest dann weiter, erkennt aber nichts mehr.",
-      "fieldConfig": {
-        "defaults": {"color": {"mode": "palette-classic"}, "custom": {"axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": {"legend": false, "tooltip": false, "viz": false}, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": {"type": "linear"}, "showPoints": "never", "spanNulls": false, "stacking": {"group": "A", "mode": "none"}, "thresholdsStyle": {"mode": "off"}}, "mappings": [], "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}]}, "unit": "reqps"},
-        "overrides": []
-      },
-      "gridPos": {"h": 8, "w": 12, "x": 12, "y": 12},
-      "id": 8,
-      "options": {"legend": {"calcs": ["lastNotNull"], "displayMode": "list", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
-      "targets": [
-        {"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "sum(rate(cs_parser_hits_ok_total[5m]))", "legendFormat": "geparst", "refId": "A"},
-        {"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "sum(rate(cs_parser_hits_ko_total[5m]))", "legendFormat": "nicht geparst", "refId": "B"}
-      ],
-      "title": "Parser",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {"type": "prometheus", "uid": "prometheus"},
-      "description": "Whitelist-Treffer nach Grund. Nützlich beim Scharfschalten: was hier zählt, wäre sonst ein Kandidat für einen Ban gewesen.",
-      "fieldConfig": {
-        "defaults": {"color": {"mode": "palette-classic"}, "custom": {"axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": {"legend": false, "tooltip": false, "viz": false}, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": {"type": "linear"}, "showPoints": "never", "spanNulls": false, "stacking": {"group": "A", "mode": "none"}, "thresholdsStyle": {"mode": "off"}}, "mappings": [], "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}]}, "unit": "reqps"},
-        "overrides": []
-      },
-      "gridPos": {"h": 8, "w": 12, "x": 0, "y": 20},
-      "id": 9,
-      "options": {"legend": {"calcs": ["lastNotNull"], "displayMode": "table", "placement": "right", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
-      "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "sum by (reason) (rate(cs_node_wl_hits_ok_total[5m]))", "legendFormat": "{{reason}}", "refId": "A"}],
-      "title": "Whitelist-Treffer",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {"type": "prometheus", "uid": "prometheus"},
-      "gridPos": {"h": 8, "w": 12, "x": 12, "y": 20},
-      "id": 10,
-      "options": {
-        "code": {"language": "plaintext", "showLineNumbers": false, "showMiniMap": false},
-        "content": "## Was dieses Dashboard NICHT zeigt\n\nAlle Panels stammen aus der LAPI und den Agents im Cluster. Die **Bouncer auf proxy-1 und proxy-2 exportieren eigene Metriken**, die hier fehlen:\n\n- `fw_bouncer_active_decisions` — was tatsächlich in den nftables-Sets steht\n- `fw_bouncer_dropped_packets` / `_bytes` — was wirklich verworfen wurde\n- `lapi_requests_total` / `_failures_total` — aus Sicht des Bouncers\n\nSie liegen auf `127.0.0.1:60601` der Proxy-Hosts. Die Hosts sind öffentlich und haben bisher keinerlei Metrik-Pfad; den Port ins Internet zu öffnen, nur um ein Scrape-Ziel zu bekommen, wäre der falsche Tausch.\n\n**Konsequenz:** „Bouncer-Abholrate\" oben zeigt, dass ein Bouncer *fragt* — nicht, dass er die Antwort auch angewandt hat. Solange der Modus `dry-run` ist, wendet er ohnehin nichts an.\n\nEin Scrape-Pfad für die Hosts ist eigene Arbeit und Voraussetzung dafür, dass „was wird wirklich geblockt\" beantwortbar wird."
-      },
-      "title": "Blinder Fleck",
-      "type": "text"
-    }
-  ],
-  "refresh": "30s",
-  "schemaVersion": 38,
-  "style": "dark",
-  "tags": ["homelab", "crowdsec", "security"],
-  "templating": {"list": []},
-  "time": {"from": "now-6h", "to": "now"},
-  "timepicker": {},
-  "timezone": "",
-  "title": "CrowdSec Overview",
-  "uid": "crowdsec-overview",
-  "version": 1,
-  "weekStart": ""
-}

kubernetes/apps/monitoring/dashboards/kustomization.yaml

@@ -38,9 +38,6 @@ configMapGenerator:
   - name: dashboard-blackbox-overview
     files:
       - blackbox-overview.json
-  - name: dashboard-crowdsec-overview
-    files:
-      - crowdsec-overview.json
   - name: dashboard-disk-health
     files:
       - disk-health.json

kubernetes/apps/paperless-eval/cluster.yaml

@@ -1,116 +0,0 @@
----
-# A read-only view of paperless's backup store, used as a recovery source and
-# nothing else.
-#
-# It points at the production prefix, which is the whole point — the eval
-# instance is a copy of the real archive's metadata — and that makes what is
-# ABSENT here the safety-critical part:
-#
-#   no retentionPolicy   The production store has "30d". Without one here
-#                        there is no policy to enforce, and enforcement is not
-#                        tied to taking a backup: the plugin runs it from the
-#                        instance sidecar on a timer
-#                        (instanceSidecarConfiguration.retentionPolicyIntervalSeconds,
-#                        default 1800). So "this cluster never backs up" would
-#                        not have been enough on its own.
-#
-# A second property makes it belt-and-braces. Retention runs
-# barman-cloud-backup-delete, which takes a server name and works inside
-# <destinationPath>/<serverName> — the store's own status keys its recovery
-# window by "pprl-pg", confirming that layout. This cluster is pprleval-pg, so
-# even an accidental retention policy would address a prefix that holds nothing
-# rather than production's backups.
-#
-# The cluster below completes the guarantee with isWALArchiver: false, so
-# nothing is written back either. It does load the plugin — that is required
-# for the restore to work at all — and read-only comes from the flag rather
-# than from leaving the plugin out.
-apiVersion: barmancloud.cnpg.io/v1
-kind: ObjectStore
-metadata:
-  name: pprleval-pg-source
-  namespace: paperless-eval
-spec:
-  configuration:
-    destinationPath: s3://renner-uno-k8s-cnpg/pprl-pg
-    endpointURL: https://fsn1.your-objectstorage.com
-    s3Credentials:
-      accessKeyId:
-        name: hetzner-s3-creds
-        key: ACCESS_KEY_ID
-      secretAccessKey:
-        name: hetzner-s3-creds
-        key: ACCESS_SECRET_KEY
-      region:
-        name: hetzner-s3-creds
-        key: REGION
-    wal:
-      compression: gzip
-    data:
-      compression: gzip
----
-# A disposable clone of the paperless database, for measuring how well the
-# local LLM classifies documents without running the experiment against the
-# live archive.
-#
-# Losing this cluster costs a re-restore and nothing else, which is the intent
-# — it is rebuilt whenever a test needs a clean starting point.
-#
-# One instance, not two. Replication buys availability, and an eval instance
-# that is down is simply an eval that waits.
-apiVersion: postgresql.cnpg.io/v1
-kind: Cluster
-metadata:
-  name: pprleval-pg
-  namespace: paperless-eval
-spec:
-  instances: 1
-  imageName: ghcr.io/cloudnative-pg/postgresql:16
-  enableSuperuserAccess: true
-  storage:
-    # Delete-reclaim, unlike the Retain default: this volume is a copy that
-    # gets rebuilt, so it should vanish with the cluster rather than leave a
-    # Released PV and an orphaned RBD image behind (issue #262).
-    storageClass: ceph-block-ephemeral
-    size: 10Gi
-  # Declared with isWALArchiver: false, which is what kubernetes/RECOVERY.md
-  # arrived at after an earlier version of that runbook said to omit it: the
-  # instance manager loads this plugin to replay WAL during the restore, and
-  # without it the instance never starts ("Unknown plugin:
-  # barman-cloud.cloudnative-pg.io").
-  #
-  # externalClusters below only says WHERE the backup is; it does not get the
-  # plugin injected into the pod. So "no plugin" is not the way to keep this
-  # read-only — "plugin, not archiving" is, and it is the combination verified
-  # on im-pg on 2026-07-25.
-  plugins:
-    - name: barman-cloud.cloudnative-pg.io
-      enabled: true
-      isWALArchiver: false
-      parameters:
-        barmanObjectName: pprleval-pg-source
-  bootstrap:
-    recovery:
-      source: pprl-pg
-      database: paperless
-      owner: paperless
-  externalClusters:
-    - name: pprl-pg
-      plugin:
-        name: barman-cloud.cloudnative-pg.io
-        parameters:
-          barmanObjectName: pprleval-pg-source
-          serverName: pprl-pg
-  postgresql:
-    parameters:
-      shared_buffers: "256MB"
-      max_connections: "100"
-      # The production tuning for write reduction is deliberately not copied.
-      # It exists to spare the SSDs under a database that runs for years; this
-      # one is read-only for hours at a time and then deleted.
-  resources:
-    requests:
-      cpu: 200m
-      memory: 512Mi
-    limits:
-      memory: 1Gi

kubernetes/apps/paperless-eval/kustomization.yaml

@@ -1,9 +0,0 @@
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-namespace: paperless-eval
-resources:
-  - secrets.yaml
-  - cluster.yaml
-  - pvcs.yaml
-  - redis.yaml
-  - paperless.yaml

kubernetes/apps/paperless-eval/paperless.yaml

@@ -1,203 +0,0 @@
----
-# Deliberately much smaller than the production ConfigMap. Everything that
-# exists to serve people is gone: no remote-user auth, no CSRF origins, no
-# barcode or mail consumption, no Tika/Gotenberg. Nothing consumes documents
-# here, and nothing signs in.
-#
-# What remains is the database connection and the AI settings, because those
-# are what the evaluation measures.
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: paperless-eval-config
-  namespace: paperless-eval
-data:
-  PAPERLESS_URL: "http://paperless.paperless-eval.svc.cluster.local"
-  PAPERLESS_ALLOWED_HOSTS: "*"
-  PAPERLESS_TIME_ZONE: "Europe/Berlin"
-  PAPERLESS_REDIS: "redis://redis:6379"
-  PAPERLESS_DBENGINE: "postgresql"
-  PAPERLESS_DBHOST: "pprleval-pg-rw"
-  PAPERLESS_DBNAME: "paperless"
-  # Polling off, so a pod that finds an empty consume volume does not spend a
-  # thread looking at it every ten seconds for the length of a run.
-  PAPERLESS_CONSUMER_POLLING_INTERVAL: "0"
-
-  # The AI settings under test. The endpoints are placeholders that each eval
-  # job overrides for the model it is measuring — a job runs its own
-  # llama-server beside paperless in the same pod, so the value that matters at
-  # run time is the one in the job, not the one here.
-  #
-  # Same caveat as production, and sharper here: paperless reads these from an
-  # ApplicationConfiguration row and falls back to the environment only when
-  # that row is empty. The row comes across in the restored database, so
-  # anything configured in the production admin UI arrives with it and silently
-  # wins over this ConfigMap — an eval would then measure production's settings
-  # while appearing to measure the job's.
-  #
-  # Check it before trusting any result, and after every fresh restore:
-  #
-  #   select llm_backend, llm_model, llm_embedding_backend, llm_embedding_model
-  #     from paperless_applicationconfiguration;
-  #
-  # Every column must be null for the environment to apply.
-  PAPERLESS_AI_ENABLED: "true"
-  PAPERLESS_AI_LLM_BACKEND: "openai-like"
-  PAPERLESS_AI_LLM_ENDPOINT: "http://127.0.0.1:8080/v1"
-  PAPERLESS_AI_LLM_MODEL: "qwen3.6-35b-a3b"
-  PAPERLESS_AI_LLM_ALLOW_INTERNAL_ENDPOINTS: "true"
-  PAPERLESS_AI_LLM_CONTEXT_SIZE: "12288"
-  PAPERLESS_AI_LLM_REQUEST_TIMEOUT: "900"
-  PAPERLESS_AI_LLM_OUTPUT_LANGUAGE: "de-de"
-  PAPERLESS_AI_LLM_EMBEDDING_BACKEND: "openai-like"
-  PAPERLESS_AI_LLM_EMBEDDING_ENDPOINT: "http://llama-embed.ai.svc.cluster.local:8080/v1"
-  PAPERLESS_AI_LLM_EMBEDDING_MODEL: "bge-m3"
-  PAPERLESS_AI_LLM_EMBEDDING_CHUNK_SIZE: "1024"
----
-# A base instance that proves the restored database is usable and that the
-# search index and classifier build against it. The evaluation itself does not
-# run here — each job brings its own paperless container so it can point at its
-# own model — but this is what makes a broken clone visible before a job spends
-# hours on it.
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: paperless
-  namespace: paperless-eval
-  labels:
-    app.kubernetes.io/name: paperless
-  annotations:
-    configmap.reloader.stakater.com/reload: paperless-eval-config
-    # Both, like production: the secret refreshes on the same 10m cycle, and
-    # without this the pod would keep a rotated key until something else
-    # restarted it — which defeats the point of matching the interval.
-    secret.reloader.stakater.com/reload: paperless-eval-secrets
-spec:
-  replicas: 1
-  strategy:
-    type: Recreate
-  selector:
-    matchLabels:
-      app.kubernetes.io/name: paperless
-  template:
-    metadata:
-      labels:
-        app.kubernetes.io/name: paperless
-    spec:
-      enableServiceLinks: false
-      securityContext:
-        fsGroup: 1000
-        fsGroupChangePolicy: OnRootMismatch
-        seccompProfile:
-          type: RuntimeDefault
-      containers:
-        - name: paperless
-          # No $imagepolicy marker, unlike apps/paperless: a measurement is
-          # only comparable against a fixed version, so this tag moves when
-          # somebody decides to re-measure, not when a release lands.
-          image: ghcr.io/paperless-ngx/paperless-ngx:3.1.2
-          ports:
-            - name: http
-              containerPort: 8000
-          envFrom:
-            - configMapRef:
-                name: paperless-eval-config
-          env:
-            - name: PAPERLESS_SECRET_KEY
-              valueFrom:
-                secretKeyRef:
-                  name: paperless-eval-secrets
-                  key: secret-key
-            - name: PAPERLESS_DBUSER
-              valueFrom:
-                secretKeyRef:
-                  name: pprleval-pg-app
-                  key: username
-            - name: PAPERLESS_DBPASS
-              valueFrom:
-                secretKeyRef:
-                  name: pprleval-pg-app
-                  key: password
-          volumeMounts:
-            - name: data
-              mountPath: /usr/src/paperless/data
-            - name: media
-              mountPath: /usr/src/paperless/media
-            - name: consume
-              mountPath: /usr/src/paperless/consume
-          # 60 minutes, against production's 10, and matching this app's Flux
-          # Kustomization timeout.
-          #
-          # Production's budget was sized for a from-scratch boot on this same
-          # corpus and held, so 10 would probably do — but there the volume is
-          # Retain and the index survives, making that a one-off. Here the
-          # volume is ephemeral and the index is rebuilt on every re-test, so
-          # the expensive path is the normal one. A killed pod starts the work
-          # over rather than resuming, which turns a slow boot into a
-          # crash-loop that stalls Flux behind wait: true.
-          #
-          # The LLM index is not in this budget: it is a celery task, and boot
-          # only runs init-llmindex-migrate, a migration check.
-          startupProbe:
-            httpGet:
-              path: /accounts/login/
-              port: http
-            periodSeconds: 20
-            timeoutSeconds: 5
-            failureThreshold: 180
-          livenessProbe:
-            httpGet:
-              path: /accounts/login/
-              port: http
-            initialDelaySeconds: 120
-            periodSeconds: 30
-            timeoutSeconds: 10
-          readinessProbe:
-            httpGet:
-              path: /accounts/login/
-              port: http
-            initialDelaySeconds: 30
-            periodSeconds: 10
-            timeoutSeconds: 5
-          resources:
-            requests:
-              cpu: 200m
-              memory: 512Mi
-            limits:
-              memory: 2Gi
-          # Production's context verbatim, and it has to be: the paperless-ngx
-          # entrypoint starts as root, chowns the mounted data/media/consume
-          # volumes and then drops to uid 1000 via setgid/setuid. Without these
-          # the same image fails that sequence with EPERM.
-          #
-          # It bites harder here than in production, where those volumes were
-          # chowned once years ago and keep their ownership. These are fresh on
-          # every rebuild, so the chown is not a no-op.
-          securityContext:
-            runAsNonRoot: false
-            allowPrivilegeEscalation: true
-            capabilities:
-              drop: ["ALL"]
-              add: ["CHOWN", "DAC_OVERRIDE", "FOWNER", "SETGID", "SETUID"]
-      volumes:
-        - name: data
-          persistentVolumeClaim:
-            claimName: paperless-eval-data
-        - name: media
-          persistentVolumeClaim:
-            claimName: paperless-eval-media
-        - name: consume
-          persistentVolumeClaim:
-            claimName: paperless-eval-consume
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: paperless
-  namespace: paperless-eval
-spec:
-  selector:
-    app.kubernetes.io/name: paperless
-  ports:
-    - port: 80
-      targetPort: http

kubernetes/apps/paperless-eval/pvcs.yaml

@@ -1,80 +0,0 @@
----
-# All four use ceph-block-ephemeral rather than the Retain default. Everything
-# here is derived — a restored database, an index built from it, a cache —  so
-# the volumes should go when the namespace does instead of leaving Released PVs
-# and orphaned RBD images to reclaim by hand (issue #262).
-#
-# /data holds the Tantivy search index and the classifier pickle, both rebuilt
-# from the database, and the LLM index — which is the expensive one at roughly
-# three hours for the full corpus.
-#
-# Measured on production for the same ~3900 documents: 36 MiB search index,
-# 125 MiB LLM index. 10Gi is far more than that needs, and deliberately so —
-# an embedding model with wider vectors than bge-m3's 1024 dimensions is
-# exactly what phase two varies, and running out of space mid-run would cost
-# the whole rebuild.
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: paperless-eval-data
-  namespace: paperless-eval
-  labels:
-    velero.io/exclude-from-backup: "true"
-spec:
-  accessModes:
-    - ReadWriteOnce
-  storageClassName: ceph-block-ephemeral
-  resources:
-    requests:
-      storage: 10Gi
----
-# Empty, and stays that way. Paperless refuses to start without the path, but
-# the AI path reads title, content and filename out of the database and never
-# opens a file — so the documents themselves are not copied here. That is what
-# keeps this instance cheap and keeps a second copy of the archive off the
-# cluster.
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: paperless-eval-media
-  namespace: paperless-eval
-  labels:
-    velero.io/exclude-from-backup: "true"
-spec:
-  accessModes:
-    - ReadWriteOnce
-  storageClassName: ceph-block-ephemeral
-  resources:
-    requests:
-      storage: 1Gi
----
-# Also empty: nothing is consumed here, but the path has to exist.
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: paperless-eval-consume
-  namespace: paperless-eval
-  labels:
-    velero.io/exclude-from-backup: "true"
-spec:
-  accessModes:
-    - ReadWriteOnce
-  storageClassName: ceph-block-ephemeral
-  resources:
-    requests:
-      storage: 1Gi
----
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: paperless-eval-redis
-  namespace: paperless-eval
-  labels:
-    velero.io/exclude-from-backup: "true"
-spec:
-  accessModes:
-    - ReadWriteOnce
-  storageClassName: ceph-block-ephemeral
-  resources:
-    requests:
-      storage: 1Gi

kubernetes/apps/paperless-eval/redis.yaml

@@ -1,81 +0,0 @@
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: redis
-  namespace: paperless-eval
-  labels:
-    app.kubernetes.io/name: redis
-spec:
-  replicas: 1
-  strategy:
-    type: Recreate
-  selector:
-    matchLabels:
-      app.kubernetes.io/name: redis
-  template:
-    metadata:
-      labels:
-        app.kubernetes.io/name: redis
-    spec:
-      securityContext:
-        runAsNonRoot: true
-        runAsUser: 999
-        fsGroup: 999
-        seccompProfile:
-          type: RuntimeDefault
-      containers:
-        - name: redis
-          # No $imagepolicy marker; see the note on the paperless image.
-          image: redis:8.10.1-alpine
-          # No --save: this holds the celery queue and the suggestion cache,
-          # both of which are regenerated. Persisting them across a restart of
-          # a throwaway instance would only preserve stale suggestions.
-          args:
-            - redis-server
-            - --save
-            - ""
-            - --loglevel
-            - warning
-          ports:
-            - name: redis
-              containerPort: 6379
-          volumeMounts:
-            - name: data
-              mountPath: /data
-          livenessProbe:
-            tcpSocket:
-              port: redis
-            initialDelaySeconds: 10
-          readinessProbe:
-            exec:
-              command: ["redis-cli", "ping"]
-            initialDelaySeconds: 5
-            periodSeconds: 10
-          resources:
-            requests:
-              cpu: 20m
-              memory: 64Mi
-            limits:
-              memory: 256Mi
-          securityContext:
-            allowPrivilegeEscalation: false
-            readOnlyRootFilesystem: true
-            capabilities:
-              drop: ["ALL"]
-      volumes:
-        - name: data
-          persistentVolumeClaim:
-            claimName: paperless-eval-redis
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: redis
-  namespace: paperless-eval
-spec:
-  selector:
-    app.kubernetes.io/name: redis
-  ports:
-    - port: 6379
-      targetPort: redis

kubernetes/apps/paperless-eval/secrets.yaml

@@ -1,45 +0,0 @@
----
-# The same Hetzner S3 key every CNPG-using namespace pulls, needed here only so
-# the recovery source can be read. See apps/paperless for why one shared key
-# rather than per-app paths.
-apiVersion: external-secrets.io/v1
-kind: ExternalSecret
-metadata:
-  name: hetzner-s3-creds
-  namespace: paperless-eval
-spec:
-  refreshInterval: 1h
-  secretStoreRef:
-    kind: ClusterSecretStore
-    name: openbao
-  target:
-    name: hetzner-s3-creds
-  dataFrom:
-    - extract:
-        key: infrastructure/hetzner-s3
----
-# Django's secret key, shared with the production instance rather than given
-# its own vault entry.
-#
-# It signs sessions and CSRF tokens, and this instance has no ingress and no
-# users — nothing signs in, so nothing is at stake in sharing it. A separate
-# entry would mean a vault write before the namespace can start, for a
-# throwaway instance that gets rebuilt.
-apiVersion: external-secrets.io/v1
-kind: ExternalSecret
-metadata:
-  name: paperless-eval-secrets
-  namespace: paperless-eval
-spec:
-  # 10m, matching apps/paperless rather than the 1h default: the point of
-  # sharing the key is that the two hold the same value, and a slower refresh
-  # here would let them diverge for up to an hour after a rotation.
-  refreshInterval: 10m
-  secretStoreRef:
-    kind: ClusterSecretStore
-    name: openbao
-  target:
-    name: paperless-eval-secrets
-  dataFrom:
-    - extract:
-        key: apps/paperless

kubernetes/apps/paperless/paperless.yaml

@@ -82,10 +82,7 @@ data:
   # primarily English archives — the better fit for a German corpus.
   PAPERLESS_AI_LLM_EMBEDDING_BACKEND: "openai-like"
   PAPERLESS_AI_LLM_EMBEDDING_ENDPOINT: "http://llama-embed.ai.svc.cluster.local:8080/v1"
-  # Must match --alias in kubernetes/apps/llama-server/embed.yaml. Changing the
-  # embedding model invalidates every stored vector, so a swap means the LLM
-  # index has to be rebuilt.
-  PAPERLESS_AI_LLM_EMBEDDING_MODEL: "bge-m3"
+  PAPERLESS_AI_LLM_EMBEDDING_MODEL: "qwen3-embedding-0.6b"
   PAPERLESS_AI_LLM_EMBEDDING_CHUNK_SIZE: "1024"
 ---
 apiVersion: apps/v1

kubernetes/clusters/homelab/paperless-eval.yaml

@@ -1,34 +0,0 @@
-apiVersion: kustomize.toolkit.fluxcd.io/v1
-kind: Kustomization
-metadata:
-  name: paperless-eval
-  namespace: flux-system
-spec:
-  # A throwaway copy of paperless for measuring the local LLM against the real
-  # corpus. Its Cluster restores from paperless's backup store and archives
-  # nothing back; see kubernetes/apps/paperless-eval/cluster.yaml.
-  #
-  # prune: true and no prune-disabled annotations anywhere underneath — the
-  # data here is a copy, and being able to delete the whole namespace by
-  # removing this file is the point.
-  interval: 10m
-  retryInterval: 1m
-  # 30m is not enough: the first start restores the database and then builds
-  # the search index over ~3900 documents before the webserver answers.
-  timeout: 60m
-  sourceRef:
-    kind: GitRepository
-    name: flux-system
-  path: ./kubernetes/apps/paperless-eval
-  prune: true
-  wait: true
-  dependsOn:
-    - name: infrastructure
-    - name: infrastructure-crds
-    - name: infrastructure-ceph-cluster
-    - name: infrastructure-cnpg-barman
-  healthCheckExprs:
-    - apiVersion: postgresql.cnpg.io/v1
-      kind: Cluster
-      failed: "has(status.phase) && status.phase.startsWith('Failed')"
-      current: "has(status.phase) && (status.phase == 'Cluster in healthy state' || status.readyInstances > 0)"

kubernetes/infrastructure/crowdsec/helmrelease.yaml

@@ -65,21 +65,6 @@ spec:
           - 45.157.177.26/32
           - 2a03:4000:4b:c1e:4474:43ff:fedf:8f85/128
           - 2a03:4000:28:f6e:a4c7:59ff:fe1e:9a29/128
-          # netbird.renner.uno — management, signal, relay and TURN.
-          #
-          # Not optional once a bouncer is armed. Its chain sits on the INPUT
-          # hook, the proxy hosts run no other firewall, and there is therefore
-          # no conntrack rule accepting established flows ahead of it. A ban on
-          # this address would drop the RETURN traffic of the proxies' own
-          # outbound control connection and take them off the mesh — with no
-          # inbound attack involved at all.
-          #
-          # Checked when this was added: none of these addresses appear in the
-          # current decision set. That is the state today, not a guarantee —
-          # these are rented addresses on a hoster whose ranges do end up on
-          # community blocklists.
-          - 162.55.50.97/32
-          - 2a01:4f8:c2c:4579::1/128
 
     # mTLS between agents and the LAPI. Do not switch back to password auth:
     # the LAPI runs without persistence, so its machine-registration DB is
@@ -123,24 +108,6 @@ spec:
 
     lapi:
       replicas: 1
-      service:
-        # Backend wiring for the public LAPI Ingress (lapi-ingress.yaml).
-        #
-        # These belong on the SERVICE, not on the Ingress: under the Kubernetes
-        # Ingress provider `traefik.ingress.kubernetes.io/service.*` are service
-        # annotations, and on an Ingress they are silently ignored. The symptom
-        # is Traefik speaking plaintext to a TLS-only backend, which the LAPI
-        # answers with "Client sent an HTTP request to an HTTPS server." and a
-        # 400 — an error that looks like a malformed request rather than a
-        # scheme mismatch.
-        #
-        # With tls.enabled the LAPI serves HTTPS on 8080 and offers no
-        # plaintext port, so https here is mandatory rather than a hardening
-        # choice. The ServersTransport carries the chart's own CA, since that
-        # certificate is self-signed.
-        annotations:
-          traefik.ingress.kubernetes.io/service.serversscheme: https
-          traefik.ingress.kubernetes.io/service.serverstransport: crowdsec-crowdsec-lapi@kubernetescrd
       # Far below the chart default, which reserves half a core per LAPI for a
       # workload that draws single-digit millicores — enough to leave the
       # cluster unschedulable with one node out. Only the reservation shrinks;
@@ -217,32 +184,6 @@ spec:
         - namespace: ingress
           podName: traefik-*
           program: traefik
-          # Read by polling, not inotify. Every path under
-          # /var/log/containers is a symlink into /var/log/pods, and kubelet
-          # rotates the target once it reaches containerLogMaxSize (10Mi,
-          # roughly every eight hours for Traefik here). An inotify tailer
-          # does not survive that rotation: it reopens the path, waits for a
-          # file that is in fact already present, and never reads another
-          # line. Nothing about that state looks broken from outside — the
-          # pod stays Running and Ready, the LAPI keeps answering, and the
-          # cumulative counters keep displaying whatever they reached, so the
-          # only symptom is that no scenario ever fires again.
-          #
-          # CrowdSec detects the hazard itself and logs it at startup, with
-          # this exact remedy in the message: "is a symlink, but inotify
-          # polling is enabled. Crowdsec will not be able to detect rotation.
-          # Consider setting poll_without_inotify to true". Upstream carries
-          # the same case as crowdsec#2449 (k3s + Traefik), and the chart
-          # gained this field in helm-charts#109 specifically to fix it.
-          #
-          # Orthogonal to the force_inotify: true that the chart hardcodes
-          # into acquis.yaml and offers no way to unset. That one governs the
-          # fsnotify watch on the directory, i.e. discovering the log of a
-          # replacement Traefik pod; this one governs how an already
-          # discovered file is read. Both true is the supported combination.
-          #
-          # Costs CPU per open file per upstream docs — two files per node.
-          poll_without_inotify: true
       env:
         - name: COLLECTIONS
           value: "crowdsecurity/traefik crowdsecurity/http-cve crowdsecurity/appsec-virtual-patching crowdsecurity/appsec-generic-rules"

kubernetes/infrastructure/crowdsec/lapi-ingress.yaml

@@ -44,8 +44,8 @@ metadata:
   namespace: crowdsec
 spec:
   serverName: crowdsec-service.crowdsec.svc.cluster.local
-  rootCAs:
-    - secret: crowdsec-root-secret
+  rootCAsSecrets:
+    - crowdsec-root-secret
 ---
 apiVersion: networking.k8s.io/v1
 kind: Ingress
@@ -60,12 +60,8 @@ metadata:
     cert-manager.io/cluster-issuer: letsencrypt-cluster
     # NOT behind the Authentik gate: the client is a daemon with an API key,
     # not a browser that can complete an OIDC flow.
-    #
-    # The backend scheme and ServersTransport are NOT set here. Under the
-    # Kubernetes Ingress provider, `traefik.ingress.kubernetes.io/service.*`
-    # are SERVICE annotations — on an Ingress they are silently ignored, and
-    # Traefik then talks plaintext to a TLS-only backend. They live on the
-    # chart's Service instead, via lapi.service.annotations in helmrelease.yaml.
+    traefik.ingress.kubernetes.io/service.serversscheme: https
+    traefik.ingress.kubernetes.io/service.serverstransport: crowdsec-crowdsec-lapi@kubernetescrd
 spec:
   ingressClassName: traefik
   rules:

kubernetes/infrastructure/namespaces/namespaces.yaml

@@ -69,13 +69,6 @@ kind: Namespace
 metadata:
   name: paperless
 ---
-# A disposable copy of paperless's database, for measuring how the local LLM
-# classifies documents without experimenting on the live archive.
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: paperless-eval
----
 apiVersion: v1
 kind: Namespace
 metadata:

kubernetes/infrastructure/network-policies/ai-ns.yaml

@@ -34,33 +34,6 @@ spec:
         - ports:
             - port: "8080"
               protocol: TCP
-    # The paperless-eval namespace reaches the embedding endpoint for the same
-    # reason paperless does; its chat model runs inside each eval job's own pod
-    # and needs no grant here.
-    #
-    # Scoped to the namespace rather than to an app label, unlike the grants
-    # around it: that namespace exists only to run evaluations, and its
-    # workload is jobs created per measurement whose labels vary with what is
-    # being measured. A label match would be a trap that presents as an
-    # embedding call silently dropped.
-    - fromEndpoints:
-        - matchLabels:
-            "k8s:io.kubernetes.pod.namespace": paperless-eval
-      toPorts:
-        - ports:
-            - port: "8080"
-              protocol: TCP
-    # Home Assistant's llama.cpp conversation agent calls the chat model only —
-    # it has no use for the embedding endpoint, so unlike paperless above this
-    # grant does not extend to llama-embed.
-    - fromEndpoints:
-        - matchLabels:
-            "k8s:io.kubernetes.pod.namespace": home-assistant
-            app.kubernetes.io/name: home-assistant
-      toPorts:
-        - ports:
-            - port: "8080"
-              protocol: TCP
     - fromEndpoints:
         - matchLabels:
             "k8s:io.kubernetes.pod.namespace": monitoring

kubernetes/infrastructure/network-policies/home-assistant-ns.yaml

@@ -167,18 +167,6 @@ spec:
     - toEndpoints:
         - matchLabels:
             "k8s:io.kubernetes.pod.namespace": influxdb
-    # The cluster's own chat model, for the llama.cpp conversation agent that
-    # backs Assist. Scoped to the one pod and port rather than the ai namespace
-    # as a whole, so this grant cannot widen if something else is placed there —
-    # the embedding endpoint next to it is deliberately not included.
-    - toEndpoints:
-        - matchLabels:
-            "k8s:io.kubernetes.pod.namespace": ai
-            app.kubernetes.io/name: llama-server
-      toPorts:
-        - ports:
-            - port: "8080"
-              protocol: TCP
     # traefik for east-west OIDC roundtrips that go through the
     # public hostname via k8s_gateway.
     - toEndpoints:

kubernetes/infrastructure/network-policies/kustomization.yaml

@@ -6,7 +6,6 @@ resources:
   - ingress-isolation.yaml
   - authentik-ns.yaml
   - paperless-ns.yaml
-  - paperless-eval-ns.yaml
   - audiobookshelf-ns.yaml
   - home-assistant-ns.yaml
   - spegel-ns.yaml

kubernetes/infrastructure/network-policies/paperless-eval-ns.yaml

@@ -1,141 +0,0 @@
-# Narrower than paperless-ns.yaml, because this namespace serves nobody.
-#
-# There is no ingress, no gate outpost and no hostname: the only things that
-# talk to it are its own pods. Monitoring is admitted so a run can be watched,
-# and host/remote-node for the kubelet probes.
-#
-# Egress reaches the ai namespace for the embedding endpoint, which is the one
-# dependency an eval job has outside its own pod, and world:443 so the CNPG
-# recovery can read the backup store on Hetzner.
-apiVersion: cilium.io/v2
-kind: CiliumNetworkPolicy
-metadata:
-  name: default-deny-ingress
-  namespace: paperless-eval
-spec:
-  endpointSelector: {}
-  ingress:
-    - fromEndpoints:
-        - matchLabels:
-            "k8s:io.kubernetes.pod.namespace": paperless-eval
-    - fromEndpoints:
-        - matchLabels:
-            "k8s:io.kubernetes.pod.namespace": monitoring
-    - fromEntities:
-        - host
-        - remote-node
-    # ICMP — all standard ICMPv4 + ICMPv6 from any cluster pod. See
-    # home-assistant-ns.yaml for the rationale.
-    - fromEndpoints:
-        - matchExpressions:
-            - key: io.kubernetes.pod.namespace
-              operator: Exists
-      icmps:
-        - fields:
-            - { type: 0,   family: IPv4 }
-            - { type: 3,   family: IPv4 }
-            - { type: 8,   family: IPv4 }
-            - { type: 11,  family: IPv4 }
-            - { type: 12,  family: IPv4 }
-            - { type: 1,   family: IPv6 }
-            - { type: 2,   family: IPv6 }
-            - { type: 3,   family: IPv6 }
-            - { type: 4,   family: IPv6 }
-            - { type: 128, family: IPv6 }
-            - { type: 129, family: IPv6 }
-            - { type: 133, family: IPv6 }
-            - { type: 134, family: IPv6 }
-            - { type: 135, family: IPv6 }
-            - { type: 136, family: IPv6 }
-            - { type: 143, family: IPv6 }
-  egress:
-    - toEndpoints:
-        - matchLabels:
-            "k8s:io.kubernetes.pod.namespace": paperless-eval
-    # The embedding endpoint. The chat model runs inside each job's own pod, so
-    # it needs no grant; the embedder is shared with production because it is
-    # read-only and cheap, and swapping it is a phase-two concern.
-    - toEndpoints:
-        - matchLabels:
-            "k8s:io.kubernetes.pod.namespace": ai
-            app.kubernetes.io/name: llama-embed
-      toPorts:
-        - ports:
-            - port: "8080"
-              protocol: TCP
-    - toEndpoints:
-        - matchLabels:
-            "k8s:io.kubernetes.pod.namespace": rook-ceph
-    - toEndpoints:
-        - matchLabels:
-            "k8s:io.kubernetes.pod.namespace": kube-system
-            k8s-app: kube-dns
-      toPorts:
-        - ports:
-            - port: "53"
-              protocol: UDP
-    - toEntities:
-        - host
-        - remote-node
-    # world:443 — the CNPG recovery reads the backup store on Hetzner Object
-    # Storage, and image pulls.
-    - toEntities:
-        - world
-      toPorts:
-        - ports:
-            - port: "443"
-              protocol: TCP
-    # ICMP — egress mirror; see ingress block above.
-    - toEndpoints:
-        - matchExpressions:
-            - key: io.kubernetes.pod.namespace
-              operator: Exists
-      icmps:
-        - fields:
-            - { type: 0,   family: IPv4 }
-            - { type: 3,   family: IPv4 }
-            - { type: 8,   family: IPv4 }
-            - { type: 11,  family: IPv4 }
-            - { type: 12,  family: IPv4 }
-            - { type: 1,   family: IPv6 }
-            - { type: 2,   family: IPv6 }
-            - { type: 3,   family: IPv6 }
-            - { type: 4,   family: IPv6 }
-            - { type: 128, family: IPv6 }
-            - { type: 129, family: IPv6 }
-            - { type: 133, family: IPv6 }
-            - { type: 134, family: IPv6 }
-            - { type: 135, family: IPv6 }
-            - { type: 136, family: IPv6 }
-            - { type: 143, family: IPv6 }
----
-# The CNPG operator polls each instance's status endpoint over HTTPS
-# (https://<pod-ip>:8000/pg/status) to populate Cluster status. Without it the
-# Cluster reports "Instance Status Extraction Error: HTTP communication issue"
-# while Postgres serves normally — issue #433, where the operator's dial timed
-# out because the apps baseline never allowed the database namespace.
-#
-# It matters here even for a throwaway: the Flux Kustomization waits on
-# Cluster status, and a smoke test that cannot report its own health is not one.
-#
-# Scoped to the pprleval-pg endpoints and that one port, like the equivalent
-# rules for wp-pg and ha-pg: the barman and mariadb operators sharing the
-# database namespace have no reason to reach anything here.
-apiVersion: cilium.io/v2
-kind: CiliumNetworkPolicy
-metadata:
-  name: pprleval-pg-allow-cnpg-operator
-  namespace: paperless-eval
-spec:
-  endpointSelector:
-    matchLabels:
-      cnpg.io/cluster: pprleval-pg
-  ingress:
-    - fromEndpoints:
-        - matchLabels:
-            "k8s:io.kubernetes.pod.namespace": database
-            app.kubernetes.io/name: cloudnative-pg
-      toPorts:
-        - ports:
-            - port: "8000"
-              protocol: TCP

kubernetes/infrastructure/post-crds/openbao-clustersecretstore.yaml

@@ -27,7 +27,6 @@ spec:
         - ingress
         - monitoring
         - paperless
-        - paperless-eval # same apps/paperless key plus the shared barman S3 creds
         - trmnl
         - velero
         - woodpecker # wp-pg barman creds; woodpecker moving out of `apps` (#433)

scripts/paperless-eval.sh

@@ -0,0 +1,207 @@
+#!/usr/bin/env bash
+# End-to-end evaluation of Paperless document suggestions against the real
+# archive, comparing the two engines Paperless ships:
+#
+#   /api/documents/<id>/suggestions/      the built-in scikit-learn classifier,
+#                                         trained on the existing corpus
+#   /api/documents/<id>/ai_suggestions/   the LLM path, optionally with RAG
+#
+# WHY THIS EXISTS
+#
+# scripts/embed-eval-real.sh measures whether a document's nearest neighbours
+# carry its labels. That is a proxy: Paperless does not act on the neighbours
+# directly, it hands them to an LLM as context, alongside the document text and
+# a candidate list of existing tags and correspondents. The LLM can override
+# poor context, or fail with good context. So a better embedding score does not
+# automatically mean better suggestions, and nothing so far has verified that it
+# does.
+#
+# This measures the thing that is actually wanted: does Paperless suggest the
+# labels a human already chose? Everything else — MTEB scores, retrieval
+# accuracy — is only interesting insofar as it predicts this number.
+#
+# It also answers a question that predates the LLM work: with ten years of
+# hand-curated labels, the built-in classifier has excellent training data. It
+# cannot invent a new tag or a title, which is the LLM's advantage — but for
+# assigning labels that already exist it may simply be better. Running both on
+# the same documents settles that rather than assuming it.
+#
+# Ground truth is the current filing. Where the two disagree with it, that is
+# not automatically an error: over ten years filing conventions change, and an
+# old document may carry an older convention. Treat the absolute numbers as
+# indicative and the comparison between engines as the result.
+#
+# Requires a Paperless API token in PAPERLESS_TOKEN. The API is only reachable
+# from inside the cluster (the namespace policy admits the Authentik gate
+# outpost alone), so requests are issued from within the paperless pod.
+#
+# Usage:
+#   PAPERLESS_TOKEN=... scripts/paperless-eval.sh [-n 100] [--engine both|classic|ai]
+set -uo pipefail
+
+SAMPLE="${PAPERLESS_EVAL_SAMPLE:-100}"
+ENGINE=both
+PG_POD="${PAPERLESS_EVAL_PG_POD:-pprl-pg-2}"
+APP_POD="${PAPERLESS_EVAL_APP_POD:-deploy/paperless}"
+
+while [ $# -gt 0 ]; do
+    case "$1" in
+        -n|--sample) SAMPLE="$2"; shift 2 ;;
+        --engine)    ENGINE="$2"; shift 2 ;;
+        -h|--help)   sed -n '2,40p' "$0"; exit 0 ;;
+        *) echo "unknown argument: $1 (try --help)" >&2; exit 2 ;;
+    esac
+done
+
+if [ -z "${PAPERLESS_TOKEN:-}" ]; then
+    echo "PAPERLESS_TOKEN is not set." >&2
+    echo "Create one for an existing user, then remove it when finished." >&2
+    exit 2
+fi
+
+WORK="$(mktemp -d)"
+trap 'rm -rf "$WORK"' EXIT
+
+# Deterministic sample, so repeated runs and different engines see the same
+# documents. Only documents that carry both labels are useful as ground truth.
+echo "reading $SAMPLE labelled documents from postgres ($PG_POD)..."
+# Names come along with the ids because the AI engine reports an unresolved
+# guess as a name string rather than an id; see the lenient scoring below.
+# Tab-separated rather than '|', which occurs in correspondent names.
+kubectl -n paperless exec "$PG_POD" -- psql -U postgres -d paperless -tA -F$'\t' -c \
+  "select d.id, d.correspondent_id, d.document_type_id, c.name, t.name
+   from documents_document d
+   join documents_correspondent c on c.id = d.correspondent_id
+   join documents_documenttype  t on t.id = d.document_type_id
+   where d.correspondent_id is not null
+     and d.document_type_id is not null
+     and length(d.content) > 400
+   order by md5(d.id::text)
+   limit $SAMPLE;" > "$WORK/docs.txt" 2>"$WORK/pg-err.txt"
+if ! [ -s "$WORK/docs.txt" ]; then
+    echo "no documents returned from postgres." >&2
+    [ -s "$WORK/pg-err.txt" ] && cat "$WORK/pg-err.txt" >&2
+    exit 2
+fi
+N=$(grep -c . "$WORK/docs.txt")
+
+# The floor: always guessing the most frequent label. Without it a number like
+# "70% correct" cannot be read, because a skewed archive gives that away for
+# free — one correspondent holds several hundred of ~3900 documents.
+kubectl -n paperless exec "$PG_POD" -- psql -U postgres -d paperless -tAc \
+  "select max(c) from (select count(*) c from documents_document
+   where correspondent_id is not null group by correspondent_id) x;" \
+  > "$WORK/corr_max.txt" 2>"$WORK/pg-err.txt"
+kubectl -n paperless exec "$PG_POD" -- psql -U postgres -d paperless -tAc \
+  "select count(*) from documents_document where correspondent_id is not null;" \
+  > "$WORK/corr_total.txt" 2>"$WORK/pg-err.txt"
+
+api() { # path -> body on stdout
+    kubectl -n paperless exec "$APP_POD" -c paperless -- \
+        curl -sS -m 300 -H "Authorization: Token $PAPERLESS_TOKEN" \
+        "http://127.0.0.1:8000$1" 2>"$WORK/err.txt"
+}
+
+# A hit under two definitions, because the AI engine can be right in a way the
+# API response does not record as a suggestion:
+#
+#   strict   the true label's id appears in `.correspondents` /
+#            `.document_types`. This is what the UI actually offers, so it is
+#            the number that describes the feature as experienced.
+#
+#   lenient  strict, OR the true label's name is contained wordwise in one of
+#            `.suggested_correspondents` / `.suggested_document_types`.
+#
+# The gap between them is one specific failure: the LLM names the right entity
+# but paperless_ai.matching cannot resolve the name to the existing object,
+# because difflib.get_close_matches runs at MATCH_THRESHOLD = 0.8 and a correct
+# but more verbose answer falls below it — observed with "Generali Deutschland
+# Krankenversicherung AG" against the existing "Generali AG", which yielded an
+# empty `correspondents` list.
+#
+# Reporting only the strict number would blame the model for a matcher problem;
+# reporting only the lenient one would claim a usefulness the UI does not
+# deliver. The classic engine emits no `suggested_*` fields at all, so both
+# numbers coincide for it and the comparison stays fair in either column.
+LENIENT_JQ='
+def norm: ascii_downcase | gsub("[^\\p{L}\\p{N}]+"; " ") | split(" ") | map(select(length > 0));
+def contains_all($want): . as $have | $want | all(. as $t | $have | index([$t]) != null);
+($want_name | norm) as $w
+| ((.[$id_field] // []) | index($want_id) != null)
+  or ([(.[$name_field] // [])[] | norm] | any(contains_all($w)))
+'
+
+# score <engine> <url-suffix>
+score() {
+    local engine="$1" suffix="$2"
+    local n=0 corr_hit=0 type_hit=0 corr_len=0 type_len=0 fails=0 start elapsed
+    echo ""
+    echo "=== $engine"
+    start=$(date +%s)
+    while IFS=$'\t' read -r id want_corr want_type want_corr_name want_type_name; do
+        [ -n "$id" ] || continue
+        n=$((n + 1))
+        body=$(api "/api/documents/$id/$suffix")
+        if ! printf '%s' "$body" | jq -e . >/dev/null 2>&1; then
+            fails=$((fails + 1))
+            [ "$fails" -le 3 ] && printf '    %s: keine gueltige Antwort: %s\n' \
+                "$id" "$(printf '%s' "$body" | tr '\n' ' ' | head -c 120)" >&2
+            continue
+        fi
+        if printf '%s' "$body" | jq -e --argjson w "$want_corr" \
+               '.correspondents // [] | index($w) != null' >/dev/null 2>&1; then
+            corr_hit=$((corr_hit + 1))
+        fi
+        if printf '%s' "$body" | jq -e --argjson w "$want_type" \
+               '.document_types // [] | index($w) != null' >/dev/null 2>&1; then
+            type_hit=$((type_hit + 1))
+        fi
+        if printf '%s' "$body" | jq -e \
+               --argjson want_id "$want_corr" --arg want_name "$want_corr_name" \
+               --arg id_field correspondents --arg name_field suggested_correspondents \
+               "$LENIENT_JQ" >/dev/null 2>&1; then
+            corr_len=$((corr_len + 1))
+        fi
+        if printf '%s' "$body" | jq -e \
+               --argjson want_id "$want_type" --arg want_name "$want_type_name" \
+               --arg id_field document_types --arg name_field suggested_document_types \
+               "$LENIENT_JQ" >/dev/null 2>&1; then
+            type_len=$((type_len + 1))
+        fi
+        [ $((n % 25)) -eq 0 ] && echo "    $n/$N"
+    done < "$WORK/docs.txt"
+    elapsed=$(( $(date +%s) - start ))
+    pct() { awk -v a="$1" -v b="$2" 'BEGIN{if (b>0) printf "%.1f", 100*a/b; else print "0.0"}'; }
+    printf '    correspondent, offered  : %5s%%   recognised: %5s%%\n' \
+        "$(pct "$corr_hit" "$n")" "$(pct "$corr_len" "$n")"
+    printf '    document type, offered  : %5s%%   recognised: %5s%%\n' \
+        "$(pct "$type_hit" "$n")" "$(pct "$type_len" "$n")"
+    printf '    no usable answer        : %s of %s\n' "$fails" "$n"
+    printf '    %ss total, %.1fs per document\n' "$elapsed" "$(awk -v e="$elapsed" -v n="$n" 'BEGIN{print e/n}')"
+    printf '%-9s %7s %7s %7s %7s %6s %8ss\n' "$engine" \
+        "$(pct "$corr_hit" "$n")" "$(pct "$corr_len" "$n")" \
+        "$(pct "$type_hit" "$n")" "$(pct "$type_len" "$n")" \
+        "$fails" "$elapsed" >> "$WORK/summary.txt"
+}
+
+: > "$WORK/summary.txt"
+FLOOR=$(awk -v m="$(cat "$WORK/corr_max.txt")" -v t="$(cat "$WORK/corr_total.txt")" \
+    'BEGIN{if (t>0) printf "%.1f", 100*m/t; else print "?"}')
+echo "documents: $N   most-frequent-correspondent floor: ${FLOOR}%"
+
+case "$ENGINE" in
+    both)    score classic suggestions/; score ai ai_suggestions/ ;;
+    classic) score classic suggestions/ ;;
+    ai)      score ai ai_suggestions/ ;;
+    *) echo "unknown engine: $ENGINE" >&2; exit 2 ;;
+esac
+
+echo ""
+echo "================================================================"
+printf '%-9s %7s %7s %7s %7s %6s %9s\n' \
+    ENGINE CORR CORR+ TYPE TYPE+ FAILS TIME
+cat "$WORK/summary.txt"
+printf '%-9s %7s\n' "floor" "$FLOOR"
+echo "================================================================"
+echo "CORR/TYPE  = true label offered as a resolved id (what the UI shows)"
+echo "CORR+/TYPE+ = also counting names the LLM got right but matching dropped"