fix: [alert] KubeHpaMaxedOut (authentik) (#761 )

2026-09-02 13:45:55 +00:00
33 changed files with 36 additions and 2889 deletions
@@ -10,18 +10,6 @@ when:
      # install-netbird-proxy.yml also applies the edge bouncer.
      - "ansible/roles/crowdsec-edge-bouncer/**"
      - "ansible/plays/install-netbird-proxy.yml"
-      # The variables that steer those roles, not just the roles themselves.
-      # Without these two, a change that only touches inventory never deploys
-      # and nothing says so: arming the bouncer lives in host_vars, and its
-      # LAPI key lives in group_vars, so both would sit in git looking applied
-      # while the hosts kept running the previous configuration.
-      #
-      # pve_hosts.yml is deliberately NOT listed. It is the single inventory
-      # file for the whole fleet, so listing it would redeploy the proxies for
-      # any unrelated host edit. The cost is that a change to [netbird_proxy]
-      # group membership needs a manual deployment run.
-      - "ansible/inventory/host_vars/proxy-*.yml"
-      - "ansible/inventory/group_vars/netbird_proxy/**"
      - ".woodpecker/deploy-netbird-proxy.yml"
  - event: deployment
    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "deploy-netbird-proxy"'
@@ -28,23 +28,14 @@ crowdsec_bouncer_sha256: f86e4b72693549d99f40a9402abefb894108f047a3fbe6e72fada25
 crowdsec_bouncer_api_url: https://crowdsec.renner.uno/
 crowdsec_bouncer_api_key: "{{ vault_crowdsec_edge_bouncer_key }}"

-# Enforcing. The bouncer creates two nftables tables (ip crowdsec, ip6
-# crowdsec6) holding the decisions it pulled, touches no existing chain and
-# sets no chain policy — SSH and :443 stay open unless the source is banned.
+# STARTS INERT, and that is the point.
 #
-# `dry-run` is the other value: it pulls and logs what it would block without
-# creating a single rule. That is what a new host should be given before it
-# enforces, and what an existing one can be dropped back to, per host through
-# inventory/host_vars, without disarming the rest.
-#
-# Worth knowing before arming anything new: the first bouncer on a path
-# enforces the whole CAPI set at once — around 15k addresses today — so the
-# false-positive window belongs before the switch, not after it. And because
-# this chain sits on the INPUT hook while these hosts run no other firewall,
-# there is no conntrack rule accepting established flows ahead of it: a banned
-# address is dropped even when it is answering our own outbound connection.
-# That is why capi_whitelists.yaml carries the proxies and netbird.renner.uno.
-crowdsec_bouncer_mode: nftables
+# dry-run pulls decisions and logs what it would block without creating a
+# single nftables rule. The first bouncer to arm anywhere enforces ~24k CAPI
+# decisions at once on the only external entry path, so the false-positive
+# window has to come before the enforcement, not after it. Flip to `nftables`
+# one host at a time once the log looks right.
+crowdsec_bouncer_mode: dry-run

 crowdsec_bouncer_update_frequency: 10s
 crowdsec_bouncer_log_level: info
@@ -120,7 +120,14 @@ spec:
    server:
      resources:
        requests:
-          cpu: 200m
+          # 500m, not 200m: the HPA below targets 70% of *this* request, so at
+          # 200m the scale-up threshold sat at 140m — below the pod's own
+          # ordinary peak. Measured over 7d, a warm server pod p99s at ~108m
+          # and peaks at ~190m, and a freshly-started one burns ~300m booting
+          # Django. That made the HPA oscillate 2<->3<->4 all day and pin at
+          # maxReplicas (KubeHpaMaxedOut, issue #761). At 500m the threshold
+          # is 350m/pod, above the boot burst, so only real load scales out.
+          cpu: 500m
          memory: 1Gi
        limits:
          memory: 2Gi
@@ -154,7 +161,11 @@ spec:
    worker:
      resources:
        requests:
-          cpu: 200m
+          # Same reasoning as server above, and the worker is burstier still:
+          # over 7d its per-pod p99 reaches ~336m and peaks hit ~630m while
+          # running blueprint applies and scheduled tasks. 200m understated
+          # that badly enough that routine task batches read as saturation.
+          cpu: 500m
          memory: 1Gi
        limits:
          memory: 2Gi
@@ -95,7 +95,7 @@ spec:
              memory: 32Mi
      containers:
        - name: home-assistant
-          image: ghcr.io/home-assistant/home-assistant:2026.9.0 # {"$imagepolicy": "flux-system:home-assistant"}
+          image: ghcr.io/home-assistant/home-assistant:2026.8.3 # {"$imagepolicy": "flux-system:home-assistant"}
          env:
            - name: TZ
              value: Europe/Berlin
@@ -75,62 +75,11 @@ spec:
            # many small per-expert matmuls that sets the pace, not bandwidth.
            - --threads
            - "4"
-            # 16k is room to spare, not a safety limit. The limit that matters
-            # is not set here at all.
-            #
-            # Isolated by bisection on a scratch pod: this stack breaks on
-            # PROMPT length, and the value of --ctx-size makes no difference to
-            # it. Measured at --ctx-size 65536 with a single slot, so with four
-            # times the headroom:
-            #
-            #     13588 prompt tokens   coherent answer
-            #     15086                 one token, immediate EOS, empty content
-            #     16088                 CJK characters and digit runs
-            #
-            # A single request past that boundary then poisons the slot: the
-            # same 13588-token prompt that answered cleanly a minute earlier
-            # returns one empty token afterwards, and stays broken until the
-            # pod restarts. That is why it looked intermittent, and why a
-            # 650-token classification once came back as 1025 tokens of "!" —
-            # it followed Home Assistant's ~19k prompts through the same slot.
-            #
-            # Nothing reports any of it: finish_reason "stop", a well-formed
-            # OpenAI response, HTTP 200, /health green.
-            #
-            # So the operating rule is about callers, not this flag. Paperless
-            # sits near 5k and is safe; Home Assistant's Assist prompt measured
-            # 18910 and never had a chance. An earlier version of this comment
-            # blamed --ctx-size 32768, which the bisection disproves — 32768
-            # and 65536 both answer correctly as long as the prompt stays under
-            # the boundary.
-            #
-            # The same stack also fails hard at --ubatch-size 2048, which is
-            # the loud version of a limit this one keeps quiet about.
-            #
+            # 16k rather than 8k because the measured prefill falloff is mild
+            # and this model's KV cache is unusually cheap — only 10 of its 40
+            # layers use full attention.
            - --ctx-size
            - "16384"
-            # One slot, so the 16384 belongs to one request at a time.
-            #
-            # Left on auto, llama.cpp opens four slots and — because auto is
-            # also what enables --kv-unified — has them share a single buffer
-            # of this size. Four Paperless requests at once wanted 6209, 6472,
-            # 4303 and 1271 tokens against those 16384, and all four died with
-            # "Context size has been exceeded".
-            #
-            # They were concurrent because a client timeout does not stop the
-            # generation behind it: each abandoned request keeps consuming the
-            # shared budget the next one needs, so one slow document is enough
-            # to start the pile-up. Queueing is the behaviour worth having
-            # here — nothing calls this server in parallel on purpose, and
-            # Paperless's celery worker runs at concurrency 1.
-            #
-            # No liveness probe change goes with this. The server recovered on
-            # its own once the batch drained, so a probe that restarted the pod
-            # would have made the incident worse; with a single slot, one that
-            # asked for a real completion would also queue behind live work and
-            # kill healthy pods during a long document.
-            - --parallel
-            - "1"
            # A micro-batch of 2048 or more makes this Vulkan/Mesa stack fail
            # with ErrorDeviceLost.
            - --ubatch-size
@@ -64,27 +64,16 @@ spec:
            - "99"
            - --threads
            - "2"
-            # Paperless chunks at 1024 tokens, but counts them with its own
-            # tokenizer while this server measures the batch in bge-m3's.
-            # Those disagree: XLM-RoBERTa splits German into more pieces than
-            # llama_index's counter does, so a chunk Paperless considers full
-            # at 1024 arrived here as 1069 and the request failed outright —
-            # "input (1069 tokens) is too large to process".
-            #
-            # The headroom therefore has to absorb a tokenizer ratio, not just
-            # a rounding error, and 1.5x covers far more than the 1.044 that
-            # broke it. The defaults are batch 2048 / ubatch 512, and with
-            # --embeddings llama-server clamps both to the smaller rather than
-            # run them mismatched, so both are set.
-            #
-            # 2048 stays avoided: that is where the chat model met
-            # ErrorDeviceLost on this Vulkan stack. That failure came under a
-            # 20 GB model's memory pressure and may not bind a 600 MB one, but
-            # this does not need to find out.
+            # Paperless chunks documents at 1024 tokens. The defaults are
+            # batch 2048 / ubatch 512, and with --embeddings llama-server
+            # clamps both down to 512 rather than run them mismatched — below
+            # the chunk size. Matching both to 1024 fits a whole chunk in one
+            # pass; 2048 is deliberately avoided, since that is where this
+            # Vulkan stack fails with ErrorDeviceLost.
            - --batch-size
-            - "1536"
+            - "1024"
            - --ubatch-size
-            - "1536"
+            - "1024"
            # Room for a full chunk plus overhead rather than exactly one.
            - --ctx-size
            - "4096"
@@ -1,36 +0,0 @@
-# Scrapes CrowdSec. Until this landed, nothing did: `up{namespace="crowdsec"}`
-# returned zero series and Prometheus held no cs_* metric at all, so the
-# detection side of the ingress path was invisible — including whether the
-# agents were reading logs and whether any bouncer was pulling decisions.
-#
-# One monitor for both services. They carry the same shape (a `metrics` port on
-# 6060) and differ only in `app`, so a matchExpressions selector keeps this to
-# a single object rather than two near-identical ones:
-#
-#   crowdsec-service        the LAPI      — decisions, bouncer + machine requests
-#   crowdsec-agent-service  the agents    — acquisition, parsers, scenarios
-#
-# The crowdsec-ns CiliumNetworkPolicy already admits the monitoring namespace
-# without a port restriction, so no policy change is needed.
-#
-# `release: kube-prometheus-stack` is what the chart's Prometheus selects on.
-apiVersion: monitoring.coreos.com/v1
-kind: ServiceMonitor
-metadata:
-  name: crowdsec
-  namespace: crowdsec
-  labels:
-    app.kubernetes.io/name: crowdsec
-    release: kube-prometheus-stack
-spec:
-  selector:
-    matchExpressions:
-      - key: app
-        operator: In
-        values:
-          - crowdsec-service
-          - crowdsec-agent-service
-  endpoints:
-    - port: metrics
-      path: /metrics
-      interval: 30s
@@ -23,7 +23,6 @@ resources:
  - prometheusrule-cnpg-backup.yaml
  - alloy-servicemonitor.yaml
  - cnpg-podmonitors.yaml
-  - crowdsec-servicemonitor.yaml
  - home-assistant-servicemonitor.yaml
  - immich-servicemonitor.yaml
  - loki-servicemonitor.yaml
@@ -1,163 +0,0 @@
-{
-  "annotations": {"list": []},
-  "description": "CrowdSec: Erkennung im Cluster, Durchsetzung am NetBird-Edge",
-  "editable": true,
-  "fiscalYearStartMonth": 0,
-  "graphTooltip": 1,
-  "id": null,
-  "links": [],
-  "panels": [
-    {
-      "datasource": {"type": "prometheus", "uid": "prometheus"},
-      "description": "Aktive Ban-Entscheidungen in der LAPI, ganz überwiegend aus den CAPI-Community-Blocklisten. Das ist die Menge, die ein scharfgeschalteter Bouncer durchsetzt.",
-      "fieldConfig": {
-        "defaults": {"color": {"mode": "thresholds"}, "mappings": [], "thresholds": {"mode": "absolute", "steps": [{"color": "text", "value": null}]}, "unit": "short"},
-        "overrides": []
-      },
-      "gridPos": {"h": 4, "w": 6, "x": 0, "y": 0},
-      "id": 1,
-      "options": {"colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto"},
-      "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "sum(cs_active_decisions)", "refId": "A"}],
-      "title": "Aktive Decisions",
-      "type": "stat"
-    },
-    {
-      "datasource": {"type": "prometheus", "uid": "prometheus"},
-      "description": "Bouncer, die in den letzten 10 Minuten Decisions abgeholt haben. Fällt das unter die erwartete Zahl, holt ein Bouncer nichts mehr — und setzt damit einen veralteten Stand durch, ohne dass irgendetwas ausfällt.",
-      "fieldConfig": {
-        "defaults": {"color": {"mode": "thresholds"}, "mappings": [], "thresholds": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "green", "value": 1}]}, "unit": "short"},
-        "overrides": []
-      },
-      "gridPos": {"h": 4, "w": 6, "x": 6, "y": 0},
-      "id": 2,
-      "options": {"colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto"},
-      "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "count(sum by (bouncer) (rate(cs_lapi_bouncer_requests_total[10m])) > 0)", "refId": "A"}],
-      "title": "Bouncer, die abholen",
-      "type": "stat"
-    },
-    {
-      "datasource": {"type": "prometheus", "uid": "prometheus"},
-      "description": "Vom Agent gelesene Logzeilen pro Sekunde. Steht das auf 0, detektiert CrowdSec nichts mehr — der häufigste stille Ausfall, weil alles andere weiterläuft.",
-      "fieldConfig": {
-        "defaults": {"color": {"mode": "thresholds"}, "mappings": [], "thresholds": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "green", "value": 0.01}]}, "unit": "reqps"},
-        "overrides": []
-      },
-      "gridPos": {"h": 4, "w": 6, "x": 12, "y": 0},
-      "id": 3,
-      "options": {"colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto"},
-      "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "sum(rate(cs_filesource_hits_total[5m]))", "refId": "A"}],
-      "title": "Gelesene Logzeilen/s",
-      "type": "stat"
-    },
-    {
-      "datasource": {"type": "prometheus", "uid": "prometheus"},
-      "description": "Szenario-Overflows der letzten 24 h — also lokal erkannte Angriffe, im Gegensatz zu den von CAPI übernommenen.",
-      "fieldConfig": {
-        "defaults": {"color": {"mode": "thresholds"}, "mappings": [], "thresholds": {"mode": "absolute", "steps": [{"color": "text", "value": null}]}, "unit": "short"},
-        "overrides": []
-      },
-      "gridPos": {"h": 4, "w": 6, "x": 18, "y": 0},
-      "id": 4,
-      "options": {"colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto"},
-      "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "sum(increase(cs_bucket_overflowed_total[24h]))", "refId": "A"}],
-      "title": "Lokale Overflows (24 h)",
-      "type": "stat"
-    },
-    {
-      "datasource": {"type": "prometheus", "uid": "prometheus"},
-      "description": "Abholrate je Bouncer. Der Firewall-Bouncer auf den Proxies fragt alle 10 s. Die LAPI legt bei jeder neuen beobachteten Client-Adresse einen Eintrag `<name>@<ip>` an, und beobachtet wird die Pod-IP des netbird-ingress-Peers — deshalb erscheint ein logischer Bouncer hier mehrfach und nach Pod-Rollen unter neuen Namen.",
-      "fieldConfig": {
-        "defaults": {"color": {"mode": "palette-classic"}, "custom": {"axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": {"legend": false, "tooltip": false, "viz": false}, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": {"type": "linear"}, "showPoints": "never", "spanNulls": false, "stacking": {"group": "A", "mode": "none"}, "thresholdsStyle": {"mode": "off"}}, "mappings": [], "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}]}, "unit": "reqps"},
-        "overrides": []
-      },
-      "gridPos": {"h": 8, "w": 12, "x": 0, "y": 4},
-      "id": 5,
-      "options": {"legend": {"calcs": ["lastNotNull"], "displayMode": "table", "placement": "right", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
-      "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "sum by (bouncer) (rate(cs_lapi_bouncer_requests_total[5m]))", "legendFormat": "{{bouncer}}", "refId": "A"}],
-      "title": "Bouncer-Abholrate",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {"type": "prometheus", "uid": "prometheus"},
-      "description": "Aktive Decisions nach Grund. `http:scan` dominiert und stammt aus den CAPI-Blocklisten.",
-      "fieldConfig": {
-        "defaults": {"color": {"mode": "palette-classic"}, "custom": {"axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", "hideFrom": {"legend": false, "tooltip": false, "viz": false}, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": {"type": "linear"}, "showPoints": "never", "spanNulls": false, "stacking": {"group": "A", "mode": "normal"}, "thresholdsStyle": {"mode": "off"}}, "mappings": [], "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}]}, "unit": "short"},
-        "overrides": []
-      },
-      "gridPos": {"h": 8, "w": 12, "x": 12, "y": 4},
-      "id": 6,
-      "options": {"legend": {"calcs": ["lastNotNull"], "displayMode": "table", "placement": "right", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
-      "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "sum by (reason) (cs_active_decisions)", "legendFormat": "{{reason}}", "refId": "A"}],
-      "title": "Aktive Decisions nach Grund",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {"type": "prometheus", "uid": "prometheus"},
-      "description": "Welche Szenarien lokal auslösen. Das ist das, was CrowdSec selbst aus den Traefik-Logs erkennt.",
-      "fieldConfig": {
-        "defaults": {"color": {"mode": "palette-classic"}, "custom": {"axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "bars", "fillOpacity": 70, "gradientMode": "none", "hideFrom": {"legend": false, "tooltip": false, "viz": false}, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": {"type": "linear"}, "showPoints": "never", "spanNulls": false, "stacking": {"group": "A", "mode": "normal"}, "thresholdsStyle": {"mode": "off"}}, "mappings": [], "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}]}, "unit": "short"},
-        "overrides": []
-      },
-      "gridPos": {"h": 8, "w": 12, "x": 0, "y": 12},
-      "id": 7,
-      "options": {"legend": {"calcs": ["sum"], "displayMode": "table", "placement": "right", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
-      "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "sum by (name) (increase(cs_bucket_overflowed_total[1h]))", "legendFormat": "{{name}}", "refId": "A"}],
-      "title": "Szenario-Overflows (je Stunde)",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {"type": "prometheus", "uid": "prometheus"},
-      "description": "Geparste gegen nicht geparste Zeilen. Ein Anstieg bei `ko` heißt meist, dass sich ein Logformat geändert hat und der Parser nicht mehr greift — CrowdSec liest dann weiter, erkennt aber nichts mehr.",
-      "fieldConfig": {
-        "defaults": {"color": {"mode": "palette-classic"}, "custom": {"axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": {"legend": false, "tooltip": false, "viz": false}, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": {"type": "linear"}, "showPoints": "never", "spanNulls": false, "stacking": {"group": "A", "mode": "none"}, "thresholdsStyle": {"mode": "off"}}, "mappings": [], "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}]}, "unit": "reqps"},
-        "overrides": []
-      },
-      "gridPos": {"h": 8, "w": 12, "x": 12, "y": 12},
-      "id": 8,
-      "options": {"legend": {"calcs": ["lastNotNull"], "displayMode": "list", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
-      "targets": [
-        {"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "sum(rate(cs_parser_hits_ok_total[5m]))", "legendFormat": "geparst", "refId": "A"},
-        {"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "sum(rate(cs_parser_hits_ko_total[5m]))", "legendFormat": "nicht geparst", "refId": "B"}
-      ],
-      "title": "Parser",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {"type": "prometheus", "uid": "prometheus"},
-      "description": "Whitelist-Treffer nach Grund. Nützlich beim Scharfschalten: was hier zählt, wäre sonst ein Kandidat für einen Ban gewesen.",
-      "fieldConfig": {
-        "defaults": {"color": {"mode": "palette-classic"}, "custom": {"axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": {"legend": false, "tooltip": false, "viz": false}, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": {"type": "linear"}, "showPoints": "never", "spanNulls": false, "stacking": {"group": "A", "mode": "none"}, "thresholdsStyle": {"mode": "off"}}, "mappings": [], "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}]}, "unit": "reqps"},
-        "overrides": []
-      },
-      "gridPos": {"h": 8, "w": 12, "x": 0, "y": 20},
-      "id": 9,
-      "options": {"legend": {"calcs": ["lastNotNull"], "displayMode": "table", "placement": "right", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
-      "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "sum by (reason) (rate(cs_node_wl_hits_ok_total[5m]))", "legendFormat": "{{reason}}", "refId": "A"}],
-      "title": "Whitelist-Treffer",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {"type": "prometheus", "uid": "prometheus"},
-      "gridPos": {"h": 8, "w": 12, "x": 12, "y": 20},
-      "id": 10,
-      "options": {
-        "code": {"language": "plaintext", "showLineNumbers": false, "showMiniMap": false},
-        "content": "## Was dieses Dashboard NICHT zeigt\n\nAlle Panels stammen aus der LAPI und den Agents im Cluster. Die **Bouncer auf proxy-1 und proxy-2 exportieren eigene Metriken**, die hier fehlen:\n\n- `fw_bouncer_active_decisions` — was tatsächlich in den nftables-Sets steht\n- `fw_bouncer_dropped_packets` / `_bytes` — was wirklich verworfen wurde\n- `lapi_requests_total` / `_failures_total` — aus Sicht des Bouncers\n\nSie liegen auf `127.0.0.1:60601` der Proxy-Hosts. Die Hosts sind öffentlich und haben bisher keinerlei Metrik-Pfad; den Port ins Internet zu öffnen, nur um ein Scrape-Ziel zu bekommen, wäre der falsche Tausch.\n\n**Konsequenz:** „Bouncer-Abholrate\" oben zeigt, dass ein Bouncer *fragt* — nicht, dass er die Antwort auch angewandt hat. Solange der Modus `dry-run` ist, wendet er ohnehin nichts an.\n\nEin Scrape-Pfad für die Hosts ist eigene Arbeit und Voraussetzung dafür, dass „was wird wirklich geblockt\" beantwortbar wird."
-      },
-      "title": "Blinder Fleck",
-      "type": "text"
-    }
-  ],
-  "refresh": "30s",
-  "schemaVersion": 38,
-  "style": "dark",
-  "tags": ["homelab", "crowdsec", "security"],
-  "templating": {"list": []},
-  "time": {"from": "now-6h", "to": "now"},
-  "timepicker": {},
-  "timezone": "",
-  "title": "CrowdSec Overview",
-  "uid": "crowdsec-overview",
-  "version": 1,
-  "weekStart": ""
-}
@@ -38,9 +38,6 @@ configMapGenerator:
  - name: dashboard-blackbox-overview
    files:
      - blackbox-overview.json
-  - name: dashboard-crowdsec-overview
-    files:
-      - crowdsec-overview.json
  - name: dashboard-disk-health
    files:
      - disk-health.json
@@ -1,116 +0,0 @@
---
-# A read-only view of paperless's backup store, used as a recovery source and
-# nothing else.
-#
-# It points at the production prefix, which is the whole point — the eval
-# instance is a copy of the real archive's metadata — and that makes what is
-# ABSENT here the safety-critical part:
-#
-#   no retentionPolicy   The production store has "30d". Without one here
-#                        there is no policy to enforce, and enforcement is not
-#                        tied to taking a backup: the plugin runs it from the
-#                        instance sidecar on a timer
-#                        (instanceSidecarConfiguration.retentionPolicyIntervalSeconds,
-#                        default 1800). So "this cluster never backs up" would
-#                        not have been enough on its own.
-#
-# A second property makes it belt-and-braces. Retention runs
-# barman-cloud-backup-delete, which takes a server name and works inside
-# <destinationPath>/<serverName> — the store's own status keys its recovery
-# window by "pprl-pg", confirming that layout. This cluster is pprleval-pg, so
-# even an accidental retention policy would address a prefix that holds nothing
-# rather than production's backups.
-#
-# The cluster below completes the guarantee with isWALArchiver: false, so
-# nothing is written back either. It does load the plugin — that is required
-# for the restore to work at all — and read-only comes from the flag rather
-# than from leaving the plugin out.
-apiVersion: barmancloud.cnpg.io/v1
-kind: ObjectStore
-metadata:
-  name: pprleval-pg-source
-  namespace: paperless-eval
-spec:
-  configuration:
-    destinationPath: s3://renner-uno-k8s-cnpg/pprl-pg
-    endpointURL: https://fsn1.your-objectstorage.com
-    s3Credentials:
-      accessKeyId:
-        name: hetzner-s3-creds
-        key: ACCESS_KEY_ID
-      secretAccessKey:
-        name: hetzner-s3-creds
-        key: ACCESS_SECRET_KEY
-      region:
-        name: hetzner-s3-creds
-        key: REGION
-    wal:
-      compression: gzip
-    data:
-      compression: gzip
---
-# A disposable clone of the paperless database, for measuring how well the
-# local LLM classifies documents without running the experiment against the
-# live archive.
-#
-# Losing this cluster costs a re-restore and nothing else, which is the intent
-# — it is rebuilt whenever a test needs a clean starting point.
-#
-# One instance, not two. Replication buys availability, and an eval instance
-# that is down is simply an eval that waits.
-apiVersion: postgresql.cnpg.io/v1
-kind: Cluster
-metadata:
-  name: pprleval-pg
-  namespace: paperless-eval
-spec:
-  instances: 1
-  imageName: ghcr.io/cloudnative-pg/postgresql:16
-  enableSuperuserAccess: true
-  storage:
-    # Delete-reclaim, unlike the Retain default: this volume is a copy that
-    # gets rebuilt, so it should vanish with the cluster rather than leave a
-    # Released PV and an orphaned RBD image behind (issue #262).
-    storageClass: ceph-block-ephemeral
-    size: 10Gi
-  # Declared with isWALArchiver: false, which is what kubernetes/RECOVERY.md
-  # arrived at after an earlier version of that runbook said to omit it: the
-  # instance manager loads this plugin to replay WAL during the restore, and
-  # without it the instance never starts ("Unknown plugin:
-  # barman-cloud.cloudnative-pg.io").
-  #
-  # externalClusters below only says WHERE the backup is; it does not get the
-  # plugin injected into the pod. So "no plugin" is not the way to keep this
-  # read-only — "plugin, not archiving" is, and it is the combination verified
-  # on im-pg on 2026-07-25.
-  plugins:
-    - name: barman-cloud.cloudnative-pg.io
-      enabled: true
-      isWALArchiver: false
-      parameters:
-        barmanObjectName: pprleval-pg-source
-  bootstrap:
-    recovery:
-      source: pprl-pg
-      database: paperless
-      owner: paperless
-  externalClusters:
-    - name: pprl-pg
-      plugin:
-        name: barman-cloud.cloudnative-pg.io
-        parameters:
-          barmanObjectName: pprleval-pg-source
-          serverName: pprl-pg
-  postgresql:
-    parameters:
-      shared_buffers: "256MB"
-      max_connections: "100"
-      # The production tuning for write reduction is deliberately not copied.
-      # It exists to spare the SSDs under a database that runs for years; this
-      # one is read-only for hours at a time and then deleted.
-  resources:
-    requests:
-      cpu: 200m
-      memory: 512Mi
-    limits:
-      memory: 1Gi
@@ -1,13 +0,0 @@
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-namespace: paperless-eval
-# The instance only. The measurement harness lives in scripts/paperless-eval/
-# and is applied by hand — a Job is immutable, re-running one means deleting and
-# recreating it, and Flux would just put the old one back. Nothing here should
-# converge on a finished measurement.
-resources:
-  - secrets.yaml
-  - cluster.yaml
-  - pvcs.yaml
-  - redis.yaml
-  - paperless.yaml
@@ -1,203 +0,0 @@
---
-# Deliberately much smaller than the production ConfigMap. Everything that
-# exists to serve people is gone: no remote-user auth, no CSRF origins, no
-# barcode or mail consumption, no Tika/Gotenberg. Nothing consumes documents
-# here, and nothing signs in.
-#
-# What remains is the database connection and the AI settings, because those
-# are what the evaluation measures.
-apiVersion: v1
-kind: ConfigMap
-metadata:
-  name: paperless-eval-config
-  namespace: paperless-eval
-data:
-  PAPERLESS_URL: "http://paperless.paperless-eval.svc.cluster.local"
-  PAPERLESS_ALLOWED_HOSTS: "*"
-  PAPERLESS_TIME_ZONE: "Europe/Berlin"
-  PAPERLESS_REDIS: "redis://redis:6379"
-  PAPERLESS_DBENGINE: "postgresql"
-  PAPERLESS_DBHOST: "pprleval-pg-rw"
-  PAPERLESS_DBNAME: "paperless"
-  # Polling off, so a pod that finds an empty consume volume does not spend a
-  # thread looking at it every ten seconds for the length of a run.
-  PAPERLESS_CONSUMER_POLLING_INTERVAL: "0"
-
-  # The AI settings under test. The endpoints are placeholders that each eval
-  # job overrides for the model it is measuring — a job runs its own
-  # llama-server beside paperless in the same pod, so the value that matters at
-  # run time is the one in the job, not the one here.
-  #
-  # Same caveat as production, and sharper here: paperless reads these from an
-  # ApplicationConfiguration row and falls back to the environment only when
-  # that row is empty. The row comes across in the restored database, so
-  # anything configured in the production admin UI arrives with it and silently
-  # wins over this ConfigMap — an eval would then measure production's settings
-  # while appearing to measure the job's.
-  #
-  # Check it before trusting any result, and after every fresh restore:
-  #
-  #   select llm_backend, llm_model, llm_embedding_backend, llm_embedding_model
-  #     from paperless_applicationconfiguration;
-  #
-  # Every column must be null for the environment to apply.
-  PAPERLESS_AI_ENABLED: "true"
-  PAPERLESS_AI_LLM_BACKEND: "openai-like"
-  PAPERLESS_AI_LLM_ENDPOINT: "http://127.0.0.1:8080/v1"
-  PAPERLESS_AI_LLM_MODEL: "qwen3.6-35b-a3b"
-  PAPERLESS_AI_LLM_ALLOW_INTERNAL_ENDPOINTS: "true"
-  PAPERLESS_AI_LLM_CONTEXT_SIZE: "12288"
-  PAPERLESS_AI_LLM_REQUEST_TIMEOUT: "900"
-  PAPERLESS_AI_LLM_OUTPUT_LANGUAGE: "de-de"
-  PAPERLESS_AI_LLM_EMBEDDING_BACKEND: "openai-like"
-  PAPERLESS_AI_LLM_EMBEDDING_ENDPOINT: "http://llama-embed.ai.svc.cluster.local:8080/v1"
-  PAPERLESS_AI_LLM_EMBEDDING_MODEL: "bge-m3"
-  PAPERLESS_AI_LLM_EMBEDDING_CHUNK_SIZE: "1024"
---
-# A base instance that proves the restored database is usable and that the
-# search index and classifier build against it. The evaluation itself does not
-# run here — each job brings its own paperless container so it can point at its
-# own model — but this is what makes a broken clone visible before a job spends
-# hours on it.
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: paperless
-  namespace: paperless-eval
-  labels:
-    app.kubernetes.io/name: paperless
-  annotations:
-    configmap.reloader.stakater.com/reload: paperless-eval-config
-    # Both, like production: the secret refreshes on the same 10m cycle, and
-    # without this the pod would keep a rotated key until something else
-    # restarted it — which defeats the point of matching the interval.
-    secret.reloader.stakater.com/reload: paperless-eval-secrets
-spec:
-  replicas: 1
-  strategy:
-    type: Recreate
-  selector:
-    matchLabels:
-      app.kubernetes.io/name: paperless
-  template:
-    metadata:
-      labels:
-        app.kubernetes.io/name: paperless
-    spec:
-      enableServiceLinks: false
-      securityContext:
-        fsGroup: 1000
-        fsGroupChangePolicy: OnRootMismatch
-        seccompProfile:
-          type: RuntimeDefault
-      containers:
-        - name: paperless
-          # No $imagepolicy marker, unlike apps/paperless: a measurement is
-          # only comparable against a fixed version, so this tag moves when
-          # somebody decides to re-measure, not when a release lands.
-          image: ghcr.io/paperless-ngx/paperless-ngx:3.1.2
-          ports:
-            - name: http
-              containerPort: 8000
-          envFrom:
-            - configMapRef:
-                name: paperless-eval-config
-          env:
-            - name: PAPERLESS_SECRET_KEY
-              valueFrom:
-                secretKeyRef:
-                  name: paperless-eval-secrets
-                  key: secret-key
-            - name: PAPERLESS_DBUSER
-              valueFrom:
-                secretKeyRef:
-                  name: pprleval-pg-app
-                  key: username
-            - name: PAPERLESS_DBPASS
-              valueFrom:
-                secretKeyRef:
-                  name: pprleval-pg-app
-                  key: password
-          volumeMounts:
-            - name: data
-              mountPath: /usr/src/paperless/data
-            - name: media
-              mountPath: /usr/src/paperless/media
-            - name: consume
-              mountPath: /usr/src/paperless/consume
-          # 60 minutes, against production's 10, and matching this app's Flux
-          # Kustomization timeout.
-          #
-          # Production's budget was sized for a from-scratch boot on this same
-          # corpus and held, so 10 would probably do — but there the volume is
-          # Retain and the index survives, making that a one-off. Here the
-          # volume is ephemeral and the index is rebuilt on every re-test, so
-          # the expensive path is the normal one. A killed pod starts the work
-          # over rather than resuming, which turns a slow boot into a
-          # crash-loop that stalls Flux behind wait: true.
-          #
-          # The LLM index is not in this budget: it is a celery task, and boot
-          # only runs init-llmindex-migrate, a migration check.
-          startupProbe:
-            httpGet:
-              path: /accounts/login/
-              port: http
-            periodSeconds: 20
-            timeoutSeconds: 5
-            failureThreshold: 180
-          livenessProbe:
-            httpGet:
-              path: /accounts/login/
-              port: http
-            initialDelaySeconds: 120
-            periodSeconds: 30
-            timeoutSeconds: 10
-          readinessProbe:
-            httpGet:
-              path: /accounts/login/
-              port: http
-            initialDelaySeconds: 30
-            periodSeconds: 10
-            timeoutSeconds: 5
-          resources:
-            requests:
-              cpu: 200m
-              memory: 512Mi
-            limits:
-              memory: 2Gi
-          # Production's context verbatim, and it has to be: the paperless-ngx
-          # entrypoint starts as root, chowns the mounted data/media/consume
-          # volumes and then drops to uid 1000 via setgid/setuid. Without these
-          # the same image fails that sequence with EPERM.
-          #
-          # It bites harder here than in production, where those volumes were
-          # chowned once years ago and keep their ownership. These are fresh on
-          # every rebuild, so the chown is not a no-op.
-          securityContext:
-            runAsNonRoot: false
-            allowPrivilegeEscalation: true
-            capabilities:
-              drop: ["ALL"]
-              add: ["CHOWN", "DAC_OVERRIDE", "FOWNER", "SETGID", "SETUID"]
-      volumes:
-        - name: data
-          persistentVolumeClaim:
-            claimName: paperless-eval-golden
-        - name: media
-          persistentVolumeClaim:
-            claimName: paperless-eval-media
-        - name: consume
-          persistentVolumeClaim:
-            claimName: paperless-eval-consume
---
-apiVersion: v1
-kind: Service
-metadata:
-  name: paperless
-  namespace: paperless-eval
-spec:
-  selector:
-    app.kubernetes.io/name: paperless
-  ports:
-    - port: 80
-      targetPort: http
@@ -1,129 +0,0 @@
---
-# All four use ceph-block-ephemeral rather than the Retain default. Everything
-# here is derived — a restored database, an index built from it, a cache —  so
-# the volumes should go when the namespace does instead of leaving Released PVs
-# and orphaned RBD images to reclaim by hand (issue #262).
-#
-# /data holds the Tantivy search index and the classifier pickle, both rebuilt
-# from the database, and the LLM index — which is the expensive one at roughly
-# three hours for the full corpus.
-#
-# Measured on production for the same ~3900 documents: 36 MiB search index,
-# 125 MiB LLM index. 10Gi is far more than that needs, and deliberately so —
-# an embedding model with wider vectors than bge-m3's 1024 dimensions is
-# exactly what phase two varies, and running out of space mid-run would cost
-# the whole rebuild.
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  # Named for its role rather than its mount point: the smoke-test deployment
-  # builds /data here, and every eval job copies it out. Replaces the earlier
-  # RWO paperless-eval-data, whose access mode could not be widened in place —
-  # a bound claim's spec is immutable.
-  name: paperless-eval-golden
-  namespace: paperless-eval
-  labels:
-    velero.io/exclude-from-backup: "true"
-spec:
-  # RWX, because the eval jobs read it too — each copies it into its own
-  # emptyDir rather than mounting it live. The vector store is SQLite, and
-  # SQLite locking across pods over CephFS is not something to rely on.
-  accessModes:
-    - ReadWriteMany
-  storageClassName: ceph-filesystem-ephemeral
-  resources:
-    requests:
-      storage: 10Gi
---
-# Empty, and stays that way. Paperless refuses to start without the path, but
-# the AI path reads title, content and filename out of the database and never
-# opens a file — so the documents themselves are not copied here. That is what
-# keeps this instance cheap and keeps a second copy of the archive off the
-# cluster.
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: paperless-eval-media
-  namespace: paperless-eval
-  labels:
-    velero.io/exclude-from-backup: "true"
-spec:
-  accessModes:
-    - ReadWriteOnce
-  storageClassName: ceph-block-ephemeral
-  resources:
-    requests:
-      storage: 1Gi
---
-# Also empty: nothing is consumed here, but the path has to exist.
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: paperless-eval-consume
-  namespace: paperless-eval
-  labels:
-    velero.io/exclude-from-backup: "true"
-spec:
-  accessModes:
-    - ReadWriteOnce
-  storageClassName: ceph-block-ephemeral
-  resources:
-    requests:
-      storage: 1Gi
---
-# The eval jobs' model cache, RWX so several jobs can read the same weights at
-# once — that is the whole point of measuring two candidates in parallel.
-#
-# Separate from the ai namespace's llama-server-models despite holding some of
-# the same files: PVCs do not cross namespaces. A model therefore downloads
-# once here and is then shared by every job, which is the cost a new candidate
-# carries anyway.
-#
-# NOT ephemeral, unlike everything else in this namespace: re-downloading tens
-# of gigabytes on a slow uplink is a worse outcome than an orphaned volume.
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: llama-models
-  namespace: paperless-eval
-  labels:
-    velero.io/exclude-from-backup: "true"
-spec:
-  accessModes:
-    - ReadWriteMany
-  storageClassName: ceph-filesystem
-  resources:
-    requests:
-      storage: 120Gi
---
-# Results outlive the jobs that produce them: a Job is deleted and recreated to
-# re-run a configuration, and comparing configurations is the entire exercise.
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: paperless-eval-results
-  namespace: paperless-eval
-  labels:
-    velero.io/exclude-from-backup: "true"
-spec:
-  accessModes:
-    - ReadWriteMany
-  storageClassName: ceph-filesystem
-  resources:
-    requests:
-      storage: 1Gi
---
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: paperless-eval-redis
-  namespace: paperless-eval
-  labels:
-    velero.io/exclude-from-backup: "true"
-spec:
-  accessModes:
-    - ReadWriteOnce
-  storageClassName: ceph-block-ephemeral
-  resources:
-    requests:
-      storage: 1Gi
@@ -1,81 +0,0 @@
---
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: redis
-  namespace: paperless-eval
-  labels:
-    app.kubernetes.io/name: redis
-spec:
-  replicas: 1
-  strategy:
-    type: Recreate
-  selector:
-    matchLabels:
-      app.kubernetes.io/name: redis
-  template:
-    metadata:
-      labels:
-        app.kubernetes.io/name: redis
-    spec:
-      securityContext:
-        runAsNonRoot: true
-        runAsUser: 999
-        fsGroup: 999
-        seccompProfile:
-          type: RuntimeDefault
-      containers:
-        - name: redis
-          # No $imagepolicy marker; see the note on the paperless image.
-          image: redis:8.10.1-alpine
-          # No --save: this holds the celery queue and the suggestion cache,
-          # both of which are regenerated. Persisting them across a restart of
-          # a throwaway instance would only preserve stale suggestions.
-          args:
-            - redis-server
-            - --save
-            - ""
-            - --loglevel
-            - warning
-          ports:
-            - name: redis
-              containerPort: 6379
-          volumeMounts:
-            - name: data
-              mountPath: /data
-          livenessProbe:
-            tcpSocket:
-              port: redis
-            initialDelaySeconds: 10
-          readinessProbe:
-            exec:
-              command: ["redis-cli", "ping"]
-            initialDelaySeconds: 5
-            periodSeconds: 10
-          resources:
-            requests:
-              cpu: 20m
-              memory: 64Mi
-            limits:
-              memory: 256Mi
-          securityContext:
-            allowPrivilegeEscalation: false
-            readOnlyRootFilesystem: true
-            capabilities:
-              drop: ["ALL"]
-      volumes:
-        - name: data
-          persistentVolumeClaim:
-            claimName: paperless-eval-redis
---
-apiVersion: v1
-kind: Service
-metadata:
-  name: redis
-  namespace: paperless-eval
-spec:
-  selector:
-    app.kubernetes.io/name: redis
-  ports:
-    - port: 6379
-      targetPort: redis
@@ -1,45 +0,0 @@
---
-# The same Hetzner S3 key every CNPG-using namespace pulls, needed here only so
-# the recovery source can be read. See apps/paperless for why one shared key
-# rather than per-app paths.
-apiVersion: external-secrets.io/v1
-kind: ExternalSecret
-metadata:
-  name: hetzner-s3-creds
-  namespace: paperless-eval
-spec:
-  refreshInterval: 1h
-  secretStoreRef:
-    kind: ClusterSecretStore
-    name: openbao
-  target:
-    name: hetzner-s3-creds
-  dataFrom:
-    - extract:
-        key: infrastructure/hetzner-s3
---
-# Django's secret key, shared with the production instance rather than given
-# its own vault entry.
-#
-# It signs sessions and CSRF tokens, and this instance has no ingress and no
-# users — nothing signs in, so nothing is at stake in sharing it. A separate
-# entry would mean a vault write before the namespace can start, for a
-# throwaway instance that gets rebuilt.
-apiVersion: external-secrets.io/v1
-kind: ExternalSecret
-metadata:
-  name: paperless-eval-secrets
-  namespace: paperless-eval
-spec:
-  # 10m, matching apps/paperless rather than the 1h default: the point of
-  # sharing the key is that the two hold the same value, and a slower refresh
-  # here would let them diverge for up to an hour after a rotation.
-  refreshInterval: 10m
-  secretStoreRef:
-    kind: ClusterSecretStore
-    name: openbao
-  target:
-    name: paperless-eval-secrets
-  dataFrom:
-    - extract:
-        key: apps/paperless
@@ -1,34 +0,0 @@
-apiVersion: kustomize.toolkit.fluxcd.io/v1
-kind: Kustomization
-metadata:
-  name: paperless-eval
-  namespace: flux-system
-spec:
-  # A throwaway copy of paperless for measuring the local LLM against the real
-  # corpus. Its Cluster restores from paperless's backup store and archives
-  # nothing back; see kubernetes/apps/paperless-eval/cluster.yaml.
-  #
-  # prune: true and no prune-disabled annotations anywhere underneath — the
-  # data here is a copy, and being able to delete the whole namespace by
-  # removing this file is the point.
-  interval: 10m
-  retryInterval: 1m
-  # 30m is not enough: the first start restores the database and then builds
-  # the search index over ~3900 documents before the webserver answers.
-  timeout: 60m
-  sourceRef:
-    kind: GitRepository
-    name: flux-system
-  path: ./kubernetes/apps/paperless-eval
-  prune: true
-  wait: true
-  dependsOn:
-    - name: infrastructure
-    - name: infrastructure-crds
-    - name: infrastructure-ceph-cluster
-    - name: infrastructure-cnpg-barman
-  healthCheckExprs:
-    - apiVersion: postgresql.cnpg.io/v1
-      kind: Cluster
-      failed: "has(status.phase) && status.phase.startsWith('Failed')"
-      current: "has(status.phase) && (status.phase == 'Cluster in healthy state' || status.readyInstances > 0)"
@@ -65,21 +65,6 @@ spec:
          - 45.157.177.26/32
          - 2a03:4000:4b:c1e:4474:43ff:fedf:8f85/128
          - 2a03:4000:28:f6e:a4c7:59ff:fe1e:9a29/128
-          # netbird.renner.uno — management, signal, relay and TURN.
-          #
-          # Not optional once a bouncer is armed. Its chain sits on the INPUT
-          # hook, the proxy hosts run no other firewall, and there is therefore
-          # no conntrack rule accepting established flows ahead of it. A ban on
-          # this address would drop the RETURN traffic of the proxies' own
-          # outbound control connection and take them off the mesh — with no
-          # inbound attack involved at all.
-          #
-          # Checked when this was added: none of these addresses appear in the
-          # current decision set. That is the state today, not a guarantee —
-          # these are rented addresses on a hoster whose ranges do end up on
-          # community blocklists.
-          - 162.55.50.97/32
-          - 2a01:4f8:c2c:4579::1/128

    # mTLS between agents and the LAPI. Do not switch back to password auth:
    # the LAPI runs without persistence, so its machine-registration DB is
@@ -123,24 +108,6 @@ spec:

    lapi:
      replicas: 1
-      service:
-        # Backend wiring for the public LAPI Ingress (lapi-ingress.yaml).
-        #
-        # These belong on the SERVICE, not on the Ingress: under the Kubernetes
-        # Ingress provider `traefik.ingress.kubernetes.io/service.*` are service
-        # annotations, and on an Ingress they are silently ignored. The symptom
-        # is Traefik speaking plaintext to a TLS-only backend, which the LAPI
-        # answers with "Client sent an HTTP request to an HTTPS server." and a
-        # 400 — an error that looks like a malformed request rather than a
-        # scheme mismatch.
-        #
-        # With tls.enabled the LAPI serves HTTPS on 8080 and offers no
-        # plaintext port, so https here is mandatory rather than a hardening
-        # choice. The ServersTransport carries the chart's own CA, since that
-        # certificate is self-signed.
-        annotations:
-          traefik.ingress.kubernetes.io/service.serversscheme: https
-          traefik.ingress.kubernetes.io/service.serverstransport: crowdsec-crowdsec-lapi@kubernetescrd
      # Far below the chart default, which reserves half a core per LAPI for a
      # workload that draws single-digit millicores — enough to leave the
      # cluster unschedulable with one node out. Only the reservation shrinks;
@@ -217,32 +184,6 @@ spec:
        - namespace: ingress
          podName: traefik-*
          program: traefik
-          # Read by polling, not inotify. Every path under
-          # /var/log/containers is a symlink into /var/log/pods, and kubelet
-          # rotates the target once it reaches containerLogMaxSize (10Mi,
-          # roughly every eight hours for Traefik here). An inotify tailer
-          # does not survive that rotation: it reopens the path, waits for a
-          # file that is in fact already present, and never reads another
-          # line. Nothing about that state looks broken from outside — the
-          # pod stays Running and Ready, the LAPI keeps answering, and the
-          # cumulative counters keep displaying whatever they reached, so the
-          # only symptom is that no scenario ever fires again.
-          #
-          # CrowdSec detects the hazard itself and logs it at startup, with
-          # this exact remedy in the message: "is a symlink, but inotify
-          # polling is enabled. Crowdsec will not be able to detect rotation.
-          # Consider setting poll_without_inotify to true". Upstream carries
-          # the same case as crowdsec#2449 (k3s + Traefik), and the chart
-          # gained this field in helm-charts#109 specifically to fix it.
-          #
-          # Orthogonal to the force_inotify: true that the chart hardcodes
-          # into acquis.yaml and offers no way to unset. That one governs the
-          # fsnotify watch on the directory, i.e. discovering the log of a
-          # replacement Traefik pod; this one governs how an already
-          # discovered file is read. Both true is the supported combination.
-          #
-          # Costs CPU per open file per upstream docs — two files per node.
-          poll_without_inotify: true
      env:
        - name: COLLECTIONS
          value: "crowdsecurity/traefik crowdsecurity/http-cve crowdsecurity/appsec-virtual-patching crowdsecurity/appsec-generic-rules"
@@ -44,8 +44,8 @@ metadata:
  namespace: crowdsec
 spec:
  serverName: crowdsec-service.crowdsec.svc.cluster.local
-  rootCAs:
-    - secret: crowdsec-root-secret
+  rootCAsSecrets:
+    - crowdsec-root-secret
 ---
 apiVersion: networking.k8s.io/v1
 kind: Ingress
@@ -60,12 +60,8 @@ metadata:
    cert-manager.io/cluster-issuer: letsencrypt-cluster
    # NOT behind the Authentik gate: the client is a daemon with an API key,
    # not a browser that can complete an OIDC flow.
-    #
-    # The backend scheme and ServersTransport are NOT set here. Under the
-    # Kubernetes Ingress provider, `traefik.ingress.kubernetes.io/service.*`
-    # are SERVICE annotations — on an Ingress they are silently ignored, and
-    # Traefik then talks plaintext to a TLS-only backend. They live on the
-    # chart's Service instead, via lapi.service.annotations in helmrelease.yaml.
+    traefik.ingress.kubernetes.io/service.serversscheme: https
+    traefik.ingress.kubernetes.io/service.serverstransport: crowdsec-crowdsec-lapi@kubernetescrd
 spec:
  ingressClassName: traefik
  rules:
@@ -69,13 +69,6 @@ kind: Namespace
 metadata:
  name: paperless
 ---
-# A disposable copy of paperless's database, for measuring how the local LLM
-# classifies documents without experimenting on the live archive.
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: paperless-eval
---
 apiVersion: v1
 kind: Namespace
 metadata:
@@ -34,33 +34,6 @@ spec:
        - ports:
            - port: "8080"
              protocol: TCP
-    # The paperless-eval namespace reaches the embedding endpoint for the same
-    # reason paperless does; its chat model runs inside each eval job's own pod
-    # and needs no grant here.
-    #
-    # Scoped to the namespace rather than to an app label, unlike the grants
-    # around it: that namespace exists only to run evaluations, and its
-    # workload is jobs created per measurement whose labels vary with what is
-    # being measured. A label match would be a trap that presents as an
-    # embedding call silently dropped.
-    - fromEndpoints:
-        - matchLabels:
-            "k8s:io.kubernetes.pod.namespace": paperless-eval
-      toPorts:
-        - ports:
-            - port: "8080"
-              protocol: TCP
-    # Home Assistant's llama.cpp conversation agent calls the chat model only —
-    # it has no use for the embedding endpoint, so unlike paperless above this
-    # grant does not extend to llama-embed.
-    - fromEndpoints:
-        - matchLabels:
-            "k8s:io.kubernetes.pod.namespace": home-assistant
-            app.kubernetes.io/name: home-assistant
-      toPorts:
-        - ports:
-            - port: "8080"
-              protocol: TCP
    - fromEndpoints:
        - matchLabels:
            "k8s:io.kubernetes.pod.namespace": monitoring
@@ -167,18 +167,6 @@ spec:
    - toEndpoints:
        - matchLabels:
            "k8s:io.kubernetes.pod.namespace": influxdb
-    # The cluster's own chat model, for the llama.cpp conversation agent that
-    # backs Assist. Scoped to the one pod and port rather than the ai namespace
-    # as a whole, so this grant cannot widen if something else is placed there —
-    # the embedding endpoint next to it is deliberately not included.
-    - toEndpoints:
-        - matchLabels:
-            "k8s:io.kubernetes.pod.namespace": ai
-            app.kubernetes.io/name: llama-server
-      toPorts:
-        - ports:
-            - port: "8080"
-              protocol: TCP
    # traefik for east-west OIDC roundtrips that go through the
    # public hostname via k8s_gateway.
    - toEndpoints:
@@ -6,7 +6,6 @@ resources:
  - ingress-isolation.yaml
  - authentik-ns.yaml
  - paperless-ns.yaml
-  - paperless-eval-ns.yaml
  - audiobookshelf-ns.yaml
  - home-assistant-ns.yaml
  - spegel-ns.yaml
@@ -1,141 +0,0 @@
-# Narrower than paperless-ns.yaml, because this namespace serves nobody.
-#
-# There is no ingress, no gate outpost and no hostname: the only things that
-# talk to it are its own pods. Monitoring is admitted so a run can be watched,
-# and host/remote-node for the kubelet probes.
-#
-# Egress reaches the ai namespace for the embedding endpoint, which is the one
-# dependency an eval job has outside its own pod, and world:443 so the CNPG
-# recovery can read the backup store on Hetzner.
-apiVersion: cilium.io/v2
-kind: CiliumNetworkPolicy
-metadata:
-  name: default-deny-ingress
-  namespace: paperless-eval
-spec:
-  endpointSelector: {}
-  ingress:
-    - fromEndpoints:
-        - matchLabels:
-            "k8s:io.kubernetes.pod.namespace": paperless-eval
-    - fromEndpoints:
-        - matchLabels:
-            "k8s:io.kubernetes.pod.namespace": monitoring
-    - fromEntities:
-        - host
-        - remote-node
-    # ICMP — all standard ICMPv4 + ICMPv6 from any cluster pod. See
-    # home-assistant-ns.yaml for the rationale.
-    - fromEndpoints:
-        - matchExpressions:
-            - key: io.kubernetes.pod.namespace
-              operator: Exists
-      icmps:
-        - fields:
-            - { type: 0,   family: IPv4 }
-            - { type: 3,   family: IPv4 }
-            - { type: 8,   family: IPv4 }
-            - { type: 11,  family: IPv4 }
-            - { type: 12,  family: IPv4 }
-            - { type: 1,   family: IPv6 }
-            - { type: 2,   family: IPv6 }
-            - { type: 3,   family: IPv6 }
-            - { type: 4,   family: IPv6 }
-            - { type: 128, family: IPv6 }
-            - { type: 129, family: IPv6 }
-            - { type: 133, family: IPv6 }
-            - { type: 134, family: IPv6 }
-            - { type: 135, family: IPv6 }
-            - { type: 136, family: IPv6 }
-            - { type: 143, family: IPv6 }
-  egress:
-    - toEndpoints:
-        - matchLabels:
-            "k8s:io.kubernetes.pod.namespace": paperless-eval
-    # The embedding endpoint. The chat model runs inside each job's own pod, so
-    # it needs no grant; the embedder is shared with production because it is
-    # read-only and cheap, and swapping it is a phase-two concern.
-    - toEndpoints:
-        - matchLabels:
-            "k8s:io.kubernetes.pod.namespace": ai
-            app.kubernetes.io/name: llama-embed
-      toPorts:
-        - ports:
-            - port: "8080"
-              protocol: TCP
-    - toEndpoints:
-        - matchLabels:
-            "k8s:io.kubernetes.pod.namespace": rook-ceph
-    - toEndpoints:
-        - matchLabels:
-            "k8s:io.kubernetes.pod.namespace": kube-system
-            k8s-app: kube-dns
-      toPorts:
-        - ports:
-            - port: "53"
-              protocol: UDP
-    - toEntities:
-        - host
-        - remote-node
-    # world:443 — the CNPG recovery reads the backup store on Hetzner Object
-    # Storage, and image pulls.
-    - toEntities:
-        - world
-      toPorts:
-        - ports:
-            - port: "443"
-              protocol: TCP
-    # ICMP — egress mirror; see ingress block above.
-    - toEndpoints:
-        - matchExpressions:
-            - key: io.kubernetes.pod.namespace
-              operator: Exists
-      icmps:
-        - fields:
-            - { type: 0,   family: IPv4 }
-            - { type: 3,   family: IPv4 }
-            - { type: 8,   family: IPv4 }
-            - { type: 11,  family: IPv4 }
-            - { type: 12,  family: IPv4 }
-            - { type: 1,   family: IPv6 }
-            - { type: 2,   family: IPv6 }
-            - { type: 3,   family: IPv6 }
-            - { type: 4,   family: IPv6 }
-            - { type: 128, family: IPv6 }
-            - { type: 129, family: IPv6 }
-            - { type: 133, family: IPv6 }
-            - { type: 134, family: IPv6 }
-            - { type: 135, family: IPv6 }
-            - { type: 136, family: IPv6 }
-            - { type: 143, family: IPv6 }
---
-# The CNPG operator polls each instance's status endpoint over HTTPS
-# (https://<pod-ip>:8000/pg/status) to populate Cluster status. Without it the
-# Cluster reports "Instance Status Extraction Error: HTTP communication issue"
-# while Postgres serves normally — issue #433, where the operator's dial timed
-# out because the apps baseline never allowed the database namespace.
-#
-# It matters here even for a throwaway: the Flux Kustomization waits on
-# Cluster status, and a smoke test that cannot report its own health is not one.
-#
-# Scoped to the pprleval-pg endpoints and that one port, like the equivalent
-# rules for wp-pg and ha-pg: the barman and mariadb operators sharing the
-# database namespace have no reason to reach anything here.
-apiVersion: cilium.io/v2
-kind: CiliumNetworkPolicy
-metadata:
-  name: pprleval-pg-allow-cnpg-operator
-  namespace: paperless-eval
-spec:
-  endpointSelector:
-    matchLabels:
-      cnpg.io/cluster: pprleval-pg
-  ingress:
-    - fromEndpoints:
-        - matchLabels:
-            "k8s:io.kubernetes.pod.namespace": database
-            app.kubernetes.io/name: cloudnative-pg
-      toPorts:
-        - ports:
-            - port: "8000"
-              protocol: TCP
@@ -27,7 +27,6 @@ spec:
        - ingress
        - monitoring
        - paperless
-        - paperless-eval # same apps/paperless key plus the shared barman S3 creds
        - trmnl
        - velero
        - woodpecker # wp-pg barman creds; woodpecker moving out of `apps` (#433)
@@ -1,198 +0,0 @@
-# Comparing chat models on Paperless suggestions
-
-Which local LLM is worth running behind paperless-ngx's AI suggestions, measured
-against documents a human already filed, in the `paperless-eval` namespace
-rather than against the live archive.
-
-**Applied by hand, not by Flux.** A Job is immutable, so re-running a
-configuration means deleting and recreating it, and a Flux-managed Job would
-simply come back. The namespace itself *is* managed
-(`kubernetes/apps/paperless-eval/`); only the measurement is not.
-
-## Running one
-
-Screen first, always:
-
-```sh
-./run-candidate.sh qwen3-8b 5          # ~5 min to a rejection, ~10 to a pass
-kubectl -n paperless-eval logs -f job/eval-qwen3-8b -c eval
-
-kubectl -n paperless-eval delete job eval-qwen3-8b
-./run-candidate.sh qwen3-8b            # the real measurement, 50 documents
-```
-
-Three of the first four candidates could not do the task at all, and each was
-left running for an hour before anyone looked. eval.py now stops after three
-consecutive failures from the first document, so a screening run rejects a
-hopeless candidate in about five minutes.
-
-Screening uses genuine paperless requests. That matters — `probe-toolcall.sh`
-is cheaper still and worth a first pass, but it passed two candidates that then
-failed on document one, because a synthesised request cannot reproduce the RAG
-neighbours and taxonomy candidate lists that make a real prompt what it is.
-
-Never compare numbers across sample sizes. A screen is a gate; the comparison
-happens at 50.
-
-Candidates live in `candidates.tsv`; the runner substitutes the model-specific
-values into `job-template.yaml` and refreshes the `eval.py` ConfigMap. Adding a
-candidate is a line in that file, not a copy of the manifest — the sidecar
-configuration deliberately exists once.
-
-Two run at a time: the pod anti-affinity puts one per node, and only two nodes
-have a free iGPU. A third waits.
-
-Re-running after an edit needs the old Job gone first:
-`kubectl -n paperless-eval delete job eval-<label>`.
-
-Results land on the `paperless-eval-results` PVC as `<label>.jsonl` (one row per
-document) and `<label>.summary.json`, and in the pod log.
-
-## What is being compared
-
-The candidates against each other. The scikit-learn classifier runs in every
-job as a fixed reference line — it costs 15 seconds and keeps running in
-production whatever the LLM does, so `llm_rescued` against `llm_only_wrong`
-says what *this* model adds on top of it.
-
-Do not read the classifier's own score as a quality bar. `train()` fits on every
-non-inbox document with no holdout, so the evaluation documents are in its
-training set and its numbers are memorisation. That does not matter for ranking
-models — it is the same constant under all of them.
-
-## Reading the output
-
-`correspondent_offered` against `correspondent_recognised` is the
-`MATCH_THRESHOLD = 0.8` gap: the model names the right entity and difflib
-rejects it. Measured, not hypothetical — "GOTOOLS GmbH" against the stored
-"GOTOOLS" scores ~0.74 and is thrown away despite being an exact prefix.
-
-`type_specific` against `type_catchall` splits on whether the true type is
-"Sonstige", which holds 1133 of ~3900 documents and is the one label with no
-semantic content. A model reading the document has to infer "none of the above".
-
-`failed` counts documents where the model produced nothing usable. With
-`--n-predict 768` in place, that is mostly non-termination: paperless reports
-the truncated response as a flat `Invalid AI configuration` and discards the
-content, so what the model was doing is visible only in the llama-server log.
-
-## Results so far
-
-| model | corr. offered | corr. recognised | type | specific | catchall | failed | seconds |
-|---|---|---|---|---|---|---|---|
-| *classifier (reference)* | 72.0 | 72.0 | 100.0 | 100.0 | 100.0 | 0 | 18 |
-| Qwen3.6-35B-A3B Q4_K_M | 72.9 | 77.1 | 85.4 | 94.9 | 44.4 | 2 | 5494 |
-| Ministral-3 3B Q4_K_M | 71.4 | 76.2 | 76.2 | 83.3 | 33.3 | **29** | 5511 |
-| Ministral-3 8B Q4_K_M | — | — | — | — | — | 3/3 | aborted |
-| Granite 3.3 8B Q4_K_M | — | — | — | — | — | all | not measurable |
-
-Read the Ministral-3 3B row with its failure count, not without it. It scored 21
-documents and failed 29 — and on those 21 its correspondent numbers sit within
-the noise floor of Qwen3.6's. That is not a near-tie: the 21 are the documents
-it could answer at all, which is a set selected for being easy, and the
-classifier does better on them too (11 of 21 both right against 30 of 48).
-A model that declines three documents in five has no usable accuracy number.
-
-Qwen3.6 complement: 30 both right, 5 rescued, 4 broken, 9 both wrong — and two
-of those four "broken" are the difflib gap above, not model errors.
-
-### How big a difference means anything
-
-Qwen3.6 was measured twice over the same 50 documents, once with the deprecated
--chat-template-kwargs and once with --reasoning off:
-
-    70.8 / 75.0 / 83.3 / 94.9 / 33.3   two failures
-    72.9 / 77.1 / 85.4 / 94.9 / 44.4   two failures
-
-Every one of those gaps is a single document out of 48 — the catchall one is
-one document out of nine. So that is the noise floor: **a difference between
-two candidates smaller than about two documents says nothing.** Decoding is
-sampled, and the same model on the same input does not give the same answer
-twice.
-
-Worth knowing before reading a 2-point lead as a result.
-
-## The instrument is checked, not assumed
-
-Every AI run asks the model "Nenne die Hauptstadt von Frankreich" before the
-documents, every ten documents, and again if it aborts. If Paris does not come
-back, the run stops and says so.
-
-This exists because a whole day of results turned out to be measuring a broken
-instrument. On this Vulkan stack a prompt past some model-specific length stops
-llama.cpp producing language — it repeats one token, " de de de" or
-" phosphorus phosphorus" or whitespace — at HTTP 200 with finish_reason "stop"
-and nothing in the log. The slot then stays broken for every request after it
-until the pod restarts.
-
-So a run against a poisoned slot reports fifty failures that are one event, and
-reads exactly like a model that cannot classify. Six candidates were written up
-that way. gemma-4-26B-A4B answers "Die Hauptstadt von Frankreich ist Paris."
-perfectly on a fresh pod and produces whitespace at a 2334-token prompt; its
-recorded classification scores described a corrupted slot and nothing else.
-
-**A run whose canary breaks is void, not a result.** Failures adjacent to the
-break are one fault. Use `probe-length.sh` to find where the model's boundary
-actually sits before judging it — paperless needs ~5300 tokens.
-
-## Bounds of the harness, not of the model
-
-Two numbers in here are mine and both have produced misleading results:
-
-`--n-predict` caps generation server-side, default 768. A candidate that stops
-exactly there has hit MY bound. Six candidates were written up as incapable on
-that signal before anyone checked whether they would have finished at 900
-tokens. A working answer takes 241-291 tokens, so 768 looked generous — and
-"looked generous" is not evidence.
-
-`EVAL_TIMEOUT` bounds the client, default 300 s. Raising the token cap alone
-just moves the failure here: at 6 tok/s a 4096-token budget needs 620 s of
-generation, so the run fails with TimeoutError instead. Raise both together.
-
-    ./run-candidate.sh <label> <sample> <n_predict> <timeout>
-    ./run-candidate.sh gemma4-26b-a4b 3 4096 900
-
-The Q8 precision comparison had to be thrown away over this. Q8's uncapped
-answers run 460-520 tokens against Q4's 280-443 — about 60% longer for the same
-task — so 20 of 50 documents hit the 768 cap against Q4's one, and the run
-looked like "Q8 classifies worse" when it measured "Q8 is wordier than my
-bound". Before comparing two configurations, check that neither is sitting on
-the cap.
-
-## What to watch for
-
-**Prompt length.** Past roughly 14k tokens this stack stops producing language
-and starts producing garbage or nothing, without any error — and a single such
-request poisons the llama.cpp slot until the pod restarts. Paperless prompts sit
-near 5k, but the 252 documents over 14000 characters push toward the boundary
-with their RAG neighbours. A run that starts scoring zeros mid-way is more
-likely this than the model.
-
-**Ground truth.** It is the current filing, and over ten years conventions
-change. A disagreement with an old document is not automatically an error.
-
-## The models volume fills up
-
-Each candidate leaves its weights on the shared `llama-models` PVC — 2.5 GB for
-a small dense model, 17-25 GB for an MoE, 34 GB for a Q8. The volume is 120 GiB
-and nine screened candidates filled it completely.
-
-A full volume does not announce itself. llama-server cannot write its metadata
-cache, so it cannot resolve the repository listing, and says:
-
-    no GGUF files found in repository <repo>
-    Available GGUF files:
-      - <whatever it had cached before the disk filled>
-
-which reads as a wrong model reference. The pod then sits in
-`Init:CrashLoopBackOff`, which from outside is indistinguishable from a slow
-34 GB download. Two jobs spent three hours that way.
-
-**Check `df -h /models` before believing anything llama-server says about a
-repository.** Clear out rejected candidates when the volume gets tight — the
-weights are re-downloadable, so deleting them costs only bandwidth:
-
-```sh
-kubectl -n paperless-eval exec <a pod mounting llama-models> -- \
-  sh -c 'cd /models && du -sh * | sort -h'
-```
@@ -1,238 +0,0 @@
-# label	hf-repo	hf-file	alias	mem-request	mem-limit
-#
-# Every candidate must do function calling: paperless calls the model through
-# chat_with_tools(..., tool_required=True) with DocumentClassifierSchema as a
-# tool, not via JSON mode. A model whose chat template has no tool support does
-# not score badly here, it fails outright.
-#
-# Instruct variants, never Reasoning ones. Qwen3.6 spent two of fifty documents
-# generating until something cut it off, and a model tuned to think first does
-# that by design. The one exception is Qwen3.6 itself, which is measured with
-# thinking switched off.
-#
-# Sizes. An earlier version of this note said "a smaller dense model gains at
-# both ends" and chose the Ministral pair on that basis. Measured, it is wrong:
-#
-#     Ministral-3 8B dense, ~4.7 GB weights   7.1-8.2 tok/s
-#     Qwen3.6 35B-A3B MoE,  ~20 GB weights    11 tok/s
-#
-# The largest model is the fastest one here. But the bandwidth explanation I
-# gave for it is wrong, and measuring Q8 disproved it:
-#
-#     Qwen3.6-35B-A3B Q4_K_M   ~21 GB   11.0 tok/s
-#     Qwen3.6-35B-A3B Q8_0     ~34 GB   10.3 tok/s
-#
-# Doubling the bytes per active parameter cost ten percent, not half. So decode
-# is NOT bandwidth-bound for this MoE — it is compute-bound on many small
-# per-expert matmuls, which also explains the earlier measurement of 21.4 GB/s
-# effective against a 53.9 GB/s ceiling: it was already leaving 60% of the bus
-# unused.
-#
-# Consequences: the bytes-per-token formula predicts dense models and not this
-# one, and higher precision is nearly free on an MoE here — about 10% slower
-# for double the weight precision.
-#
-# So the order to prefer is: MoE with few active parameters, then dense 3-9B,
-# then dense 10-14B only for an exceptional model (~4 tok/s, roughly three
-# hours for 50 documents), and nothing dense above 14B.
-qwen35	ggml-org/Qwen3.6-35B-A3B-GGUF	Qwen3.6-35B-A3B-Q4_K_M.gguf	qwen3.6-35b-a3b	24Gi	32Gi
-#
-# The same model at higher precision, to find out what Q4 costs. Everything in
-# this file is Q4_K_M, so the whole comparison could be sitting on quantisation
-# damage nobody has measured — and there is a concrete suspicion, not just a
-# worry: Ministral-3's leak of [THINK] and [TOOL_CALLS] as literal text is the
-# kind of fault a 4-bit quantisation can produce, and Q8 would separate "the
-# model cannot do this" from "Q4 broke it".
-#
-# ~34 GB resident against Q4's ~21, and roughly 6 tok/s against 11, so about
-# double the wall clock. Read the result against the noise floor in README.md:
-# below about two documents of difference, this says nothing.
-qwen35-q8	ggml-org/Qwen3.6-35B-A3B-GGUF	Qwen3.6-35B-A3B-Q8_0.gguf	qwen3.6-35b-a3b-q8	40Gi	48Gi
-#
-# --- MoE, few active parameters: the shape that wins on this hardware ---
-qwen3-30b-a3b	bartowski/Qwen_Qwen3-30B-A3B-Instruct-2507-GGUF	Qwen_Qwen3-30B-A3B-Instruct-2507-Q4_K_M.gguf	qwen3-30b-a3b	22Gi	28Gi
-nemotron3-30b-a3b	bartowski/nvidia_Nemotron-3-Nano-30B-A3B-GGUF	nvidia_Nemotron-3-Nano-30B-A3B-Q4_K_M.gguf	nemotron3-30b-a3b	28Gi	34Gi
-glm47-flash	bartowski/zai-org_GLM-4.7-Flash-GGUF	zai-org_GLM-4.7-Flash-Q4_K_M.gguf	glm-4.7-flash	20Gi	26Gi
-gemma4-26b-a4b	unsloth/gemma-4-26B-A4B-it-GGUF	gemma-4-26B-A4B-it-UD-Q4_K_M.gguf	gemma-4-26b-a4b	20Gi	26Gi
-qwen35-35b-a3b	bartowski/Qwen_Qwen3.5-35B-A3B-GGUF	Qwen_Qwen3.5-35B-A3B-Q4_K_M.gguf	qwen3.5-35b-a3b	24Gi	30Gi
-nemotron35-30b-a3b	bartowski/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-GGUF	NVIDIA-Nemotron-3.5-Lightning-30B-A3B-Q4_K_M.gguf	nemotron3.5-30b-a3b	28Gi	34Gi
-#
-# --- dense, small, with German named in the model's own card ---
-glm4-9b	bartowski/glm-4-9b-chat-GGUF	glm-4-9b-chat-Q4_K_M.gguf	glm-4-9b	10Gi	14Gi
-commandr7b	bartowski/c4ai-command-r7b-12-2024-GGUF	c4ai-command-r7b-12-2024-Q4_K_M.gguf	command-r7b	10Gi	14Gi
-qwen25-7b	bartowski/Qwen2.5-7B-Instruct-GGUF	Qwen2.5-7B-Instruct-Q4_K_M.gguf	qwen2.5-7b	10Gi	14Gi
-llama31-8b	bartowski/Meta-Llama-3.1-8B-Instruct-GGUF	Meta-Llama-3.1-8B-Instruct-Q4_K_M.gguf	llama-3.1-8b	10Gi	14Gi
-nemotron-nano-12b	bartowski/nvidia_NVIDIA-Nemotron-Nano-12B-v2-GGUF	nvidia_NVIDIA-Nemotron-Nano-12B-v2-Q4_K_M.gguf	nemotron-nano-12b	12Gi	16Gi
-phi4-mini	bartowski/microsoft_Phi-4-mini-instruct-GGUF	microsoft_Phi-4-mini-instruct-Q4_K_M.gguf	phi-4-mini	8Gi	12Gi
-#
-# --- dense, small, no German claim but no thinking risk either ---
-qwen3-4b-2507	bartowski/Qwen_Qwen3-4B-Instruct-2507-GGUF	Qwen_Qwen3-4B-Instruct-2507-Q4_K_M.gguf	qwen3-4b	8Gi	12Gi
-qwen35-9b	bartowski/Qwen_Qwen3.5-9B-GGUF	Qwen_Qwen3.5-9B-Q4_K_M.gguf	qwen3.5-9b	10Gi	14Gi
-#
-# WHAT THE RESEARCH RULED OUT, so nobody re-derives it
-#
-# Hard-excluded on tool calling, chat template inspected directly rather than
-# assumed — the string "tool" appears zero times in any of them:
-#   Gemma 3, every size (4B/12B/27B)   phi-4 14B (the mini DOES have it)
-#   Cohere Aya, all variants           InternLM 2.5 / 3
-#
-# Hard-excluded on German, stated by the vendor rather than merely missing:
-#   Falcon 3 — "supports EN, FR, ES, PT", German is not on the list
-#   EXAONE — the small ones are English/Korean; the ones with German are 33B+
-#
-# German fine-tunes are a dead end, and for one consistent reason: SauerkrautLM,
-# DiscoLM, LeoLM, Occiglot and Teuken all overwrite their base model's chat
-# template with a minimal ChatML loop that drops tool calling. VAGOsolutions
-# does this even on bases that have it (Llama-3.1, Qwen2.5). Assume any German
-# tune is tool-incapable until its own tokenizer_config.json says otherwise.
-#
-# On llama.cpp's tool parsing, which explains the Granite failure: current
-# builds have no per-vendor parser table any more. common/chat.cpp exposes
-# PEG_SIMPLE / PEG_NATIVE / PEG_GEMMA4 / PEG_MINIMAX_M3 and derives the grammar
-# from the model's own template. Hermes-style <tool_call> JSON is a first-class
-# tested fixture in tests/test-chat.cpp; Granite's format is not. So this is not
-# "Granite is unsupported" so much as "Granite's template shape is outside what
-# the auto-parser was validated against".
-#
-# SCREENING RESULTS, and a pattern that reframes the whole search
-#
-#   qwen35        MoE 35B total / 3B active   works, 2 of 50 failed
-#   qwen3-8b      dense 8B                    4 of 5 failed
-#   hermes4-14b   dense 14B                   3 of 3 failed, aborted
-#   ministral8b   dense 8B                    3 of 3 failed, aborted
-#   ministral3b   dense 3B                    29 of 50 failed
-#   granite8b     dense 8B                    every document failed
-#
-# Five dense models from 3B to 14B cannot produce a valid call against
-# paperless's 14-field schema; the one mixture-of-experts model can. Hermes is
-# the cleanest of these failures: its <tool_call> JSON is an explicitly tested
-# fixture in llama.cpp's own test suite, so the parser demonstrably handles the
-# format and the failure has to be the model.
-#
-# Working hypothesis: holding a long grammar-constrained structure needs model
-# capacity, and what supplies it is TOTAL parameters, not active ones. If that
-# is right, "go smaller for speed" is a dead end here — and the consolation is
-# that the fastest shape on this hardware, MoE with few active parameters, is
-# also the only one that works.
-#
-# qwen3-30b-a3b tests it directly: 30B total, 3B active.
-#
-# CORRECTION: Gemma is not excluded after all. Gemma *3* has no tool support —
-# "tool" appears zero times in its chat template at every size, checked
-# directly. Gemma *4* 26B-A4B does have it, documented in the template, plus
-# "140+ languages pretrained, 35+ out of the box". Excluding a family by its
-# previous generation was sloppy.
-#
-# Two of these carry a Vulkan risk worth checking in the llama-server log
-# before trusting a run: Nemotron-3.5 interleaves real Mamba2 layers and
-# Qwen3.5 uses DeltaNet linear attention, and llama.cpp's Vulkan SSM shaders
-# are less exercised on Iris Xe than on Arc. Issue #19957 reported exactly this
-# for Qwen3.5-35B-A3B and it turned out to be a stale build, fixed months
-# before b10731 — but confirm layers are actually offloaded rather than
-# silently falling back to CPU, which would read as a slow model.
-#
-# Ruled out after checking: ERNIE-4.5-21B-A3B (only the Thinking variant has
-# tool calling; the instruct one has none), Ling-3.0-flash (124B total),
-# Granite-4.0-H-Small (9B active), DeepSeek-V2-Lite and OLMoE (under 20B
-# total), Cohere North Mini (code-only, no ggml-org/bartowski conversion,
-# cohere2_moe support in llama.cpp unverified), gpt-oss-20b (a reasoning-effort
-# model in Harmony format rather than plain instruct).
-#
-# THE MoE HYPOTHESIS IS DEAD. qwen3-30b-a3b — MoE, 30B total, 3B active, the
-# shape that was supposed to explain everything — scored 1 of 5 and failed 4,
-# indistinguishable from the dense 8B.
-#
-# So neither architecture nor size predicts success:
-#
-#   Qwen3.6-35B-A3B   MoE 35B/3B   Apr 2026   2 of 50 failed
-#   Qwen3-30B-A3B     MoE 30B/3B   older      4 of 5 failed
-#   Qwen3-8B          dense 8B     older      4 of 5 failed
-#   Hermes-4-14B      dense 14B    —          3 of 3 failed
-#
-# The only thing correlating with success so far is how new and how strong the
-# model is. That is an unsatisfying hypothesis because it predicts nothing
-# cheaply — but it is what the data says, and two architectural theories have
-# already died here.
-#
-# qwen35-35b-a3b isolates it: same architecture, same total and active
-# parameters as the working baseline, one generation older. If it fails, the
-# generation is the variable and shape is noise.
-#
-# qwen35-35b-a3b PASSES the screen: 5 of 5, no failures, 584 s. So the
-# generation hypothesis is dead too — Qwen3.5 is a generation OLDER than the
-# working Qwen3.6 and handles the schema cleanly.
-#
-# What the two working models share is 35B total parameters; the one failing
-# MoE has 30B. But that is confounded — Qwen3-30B-A3B is both smaller and
-# older, and three data points cannot separate the two.
-#
-#   Qwen3.6-35B-A3B   35B total   newest   works
-#   Qwen3.5-35B-A3B   35B total   older    works
-#   Qwen3-30B-A3B     30B total   oldest   4 of 5 failed
-#
-# glm47-flash separates them: 30B total, newest generation. If it works,
-# generation is what matters and 35B is not a threshold. If it fails, total
-# size is, and the search should stop looking below ~35B.
-#
-# PREFILL SPEED IS A SELECTION CRITERION AND I WAS NOT TRACKING IT.
-# Measured on this hardware, prompt processing varies more than sixfold across
-# architectures, and it is roughly half of a run's wall clock:
-#
-#   dense 8B (Ministral)        240 tok/s
-#   Qwen3.6-35B-A3B             100 tok/s
-#   GLM-4.7-Flash                31 tok/s   decode 4.9 tok/s
-#
-# GLM-4.7-Flash is disqualified on that alone. At 31 tok/s a 5300-token prompt
-# takes 168 s, times ~2.6 requests per document, is over seven minutes per
-# document and about six hours for a run — before considering whether its
-# answers are any good.
-#
-# It also shows why the 300 s REQUEST_TIMEOUT needs care: 4453 prompt tokens at
-# 31.5 tok/s plus 768 generated at 4.9 tok/s is 298 s, so a slow model turns a
-# content failure into a TimeoutError and muddies the diagnosis. Read the
-# llama-server timings, not just the eval error.
-#
-# glm47-flash: aborted, 3 of 3 — but every one a TimeoutError, not an invalid
-# answer, so this does NOT settle the size-versus-generation question it was
-# started to settle. It failed on speed. Document 1 did reach the --n-predict
-# cap without producing a call, but 2 and 3 were cut off by the 300 s client
-# bound before anything could be concluded about them.
-#
-# Disqualified regardless: 31 tok/s prefill and 4.9 tok/s decode is over seven
-# minutes per document and about six hours for a full run.
-#
-# gemma4-26b-a4b is the better discriminator — 26B total, well under 35, and
-# newest generation. Works means generation and quality matter rather than a
-# 35B threshold; fails is consistent with total size being the line.
-#
-# gemma4-26b-a4b: aborted, 3 of 3, all HTTP 400 rather than timeouts, and the
-# server log shows it ran to the --n-predict cap on every one. So it is the
-# same capability failure as the small dense models, not a template mismatch.
-# (Prefill 45-50 tok/s, decode 6 tok/s — also not fast.)
-#
-# THE SIZE THRESHOLD SURVIVED ITS FIRST REAL TEST. gemma-4 was chosen
-# specifically because it could refute it — newest generation, 26B total — and
-# it did not:
-#
-#   Qwen3.6-35B-A3B   35B   newest    works
-#   Qwen3.5-35B-A3B   35B   older     works
-#   Qwen3-30B-A3B     30B   oldest    fails
-#   gemma-4-26B-A4B   26B   NEWEST    fails
-#   Hermes-4-14B      14B   —         fails
-#   dense 3-8B        —     —         fail
-#
-# Two generations work at 35B total; everything at 30B and below fails,
-# including the newest model in the field. Operating rule until something
-# contradicts it: do not spend a screening slot on anything under ~35B total.
-#
-# Not pinned further on purpose. Whether the line sits at 31B or 34B is
-# interesting and not decision-relevant — there are two working candidates and
-# the useful next number is a full 50-document run of the second one.
-#
-# Re-enabled for length probing, not for scoring. Their classification numbers
-# were taken against a slot that may already have been corrupted, so the only
-# honest thing to do with them is find out where each one's boundary sits.
-hermes4-14b	bartowski/NousResearch_Hermes-4-14B-GGUF	NousResearch_Hermes-4-14B-Q4_K_M.gguf	hemes-4-14b	14Gi	20Gi
-qwen3-8b	bartowski/Qwen_Qwen3-8B-GGUF	Qwen_Qwen3-8B-Q4_K_M.gguf	qwen3-8b	12Gi	16Gi
-ministral8b	bartowski/mistralai_Ministral-3-8B-Instruct-2512-GGUF	mistralai_Ministral-3-8B-Instruct-2512-Q4_K_M.gguf	ministral-3-8b	12Gi	16Gi
-granite8b	bartowski/ibm-granite_granite-3.3-8b-instruct-GGUF	ibm-granite_granite-3.3-8b-instruct-Q4_K_M.gguf	granite-3.3-8b	12Gi	16Gi
@@ -1,544 +0,0 @@
-#!/usr/bin/env python3
-"""Measure what Paperless suggests for documents whose labels a human already chose.
-
-Runs both engines against the same documents:
-
-    /api/documents/<id>/suggestions/      the built-in scikit-learn classifier
-    /api/documents/<id>/ai_suggestions/   the LLM path, with RAG
-
-WHAT THIS ANSWERS, AND WHAT AN EARLIER VERSION DID NOT
-
-The two engines are not alternatives — the classifier costs 0.6 s per document
-and keeps running whatever the LLM does. So the useful question is not "which
-scores higher" but "what does the LLM add where the classifier is wrong", and
-that cannot be recovered from two aggregate percentages: 74% and 72% could
-overlap almost entirely or barely at all, and the difference between those two
-worlds is the entire value of running an LLM.
-
-So every document is written out individually, with both engines' verdicts, and
-the summary is derived from that rather than counted along the way.
-
-SCORING
-
-Correspondent and document type are each scored twice:
-
-    offered      the true label's id came back in `correspondents` /
-                 `document_types` — what the UI actually presents
-    recognised   also counting the true label's name appearing wordwise in
-                 `suggested_correspondents` / `suggested_document_types`
-
-The gap between them is one specific failure: the model names the right entity
-but paperless_ai.matching cannot resolve it, because difflib runs at
-MATCH_THRESHOLD = 0.8 and a correct but more verbose answer falls below it —
-"Generali Deutschland Krankenversicherung AG" against the existing "Generali
-AG". Reporting only the first blames the model for a matcher problem; only the
-second claims a usefulness the UI does not deliver.
-
-Document type is additionally split on whether the true type is "Sonstige",
-which holds 1133 of ~3900 documents and is the one label with no semantic
-content — a restaurant reservation lands there because nothing else fits. The
-classifier can memorise that; a model reading the document has to infer "none
-of the above". Scored separately rather than dropped, because the model does
-sometimes get it right.
-
-GROUND TRUTH
-
-The current filing, which deserves care: over ten years conventions change, and
-a disagreement with an old document is not automatically an error. Treat the
-comparison between engines as the result and the absolute numbers as
-indicative.
-
-Env:
-    PAPERLESS_URL     default http://127.0.0.1:8000
-    PGHOST/PGDATABASE standard libpq, for ground truth and the API token
-    EVAL_SAMPLE       documents to score, default 50
-    EVAL_ENGINES      comma-separated subset of classic,ai — default both
-    EVAL_LABEL        free-text name for this configuration, recorded in the
-                      output so results from different jobs stay distinguishable
-    EVAL_OUT          directory for results, default /results
-"""
-
-from __future__ import annotations
-
-import hashlib
-import json
-import os
-import re
-import secrets
-import sys
-import time
-import urllib.error
-import urllib.request
-
-import psycopg
-
-PAPERLESS_URL = os.environ.get("PAPERLESS_URL", "http://127.0.0.1:8000")
-SAMPLE = int(os.environ.get("EVAL_SAMPLE", "50"))
-ENGINES = os.environ.get("EVAL_ENGINES", "classic,ai").split(",")
-LABEL = os.environ.get("EVAL_LABEL", "unlabelled")
-OUT_DIR = os.environ.get("EVAL_OUT", "/results")
-
-# A backstop, not the real bound. Generation is capped server-side by
-# --n-predict on the llama-server sidecar, which is what actually stops a model
-# that will not terminate; this catches everything else — a wedged sidecar, a
-# request that never reaches it.
-#
-# 300 s against the ~73 s a healthy document takes end to end, prompt included.
-# It does work: llama-server logs `stop: cancel task` on the disconnect and
-# starts the next request 0.4 s later.
-REQUEST_TIMEOUT = int(os.environ.get("EVAL_TIMEOUT", "300"))
-
-# Consecutive failures from the very first document that mean the engine is
-# unusable rather than merely weak. Three, because one could be a single hard
-# document and two could be coincidence.
-ABORT_AFTER = 3
-
-CATCHALL_TYPE = "Sonstige"
-
-
-def log(msg: str) -> None:
-    print(f"[eval] {msg}", flush=True)
-
-
-def api_token(conn) -> str:
-    """A token for this run, against a user belonging to this run alone.
-
-    The API needs one and this database is a disposable clone, so the run
-    issues its own rather than depending on a token that happened to come
-    across in the restore.
-
-    The per-label user is not tidiness. Django's authtoken holds one token per
-    user, so an earlier version took the first superuser and deleted its
-    existing token first — which meant the second of two candidates running
-    concurrently silently revoked the first one's credentials. That surfaced as
-    a mid-run `HTTP 401: Invalid token` sitting among genuine model failures,
-    where it reads like one more broken document rather than the harness
-    sabotaging itself. Both jobs share this database; only the user can differ.
-    """
-    token = secrets.token_hex(20)
-    username = f"eval-{LABEL}"
-    with conn.cursor() as cur:
-        cur.execute("select id from auth_user where username = %s", (username,))
-        row = cur.fetchone()
-        if row is None:
-            cur.execute(
-                """
-                insert into auth_user (password, is_superuser, username,
-                                       first_name, last_name, email, is_staff,
-                                       is_active, date_joined)
-                values ('!unusable', true, %s, '', '', '', true, true, now())
-                returning id
-                """,
-                (username,),
-            )
-            row = cur.fetchone()
-        user_id = row[0]
-        cur.execute("delete from authtoken_token where user_id = %s", (user_id,))
-        cur.execute(
-            "insert into authtoken_token (key, created, user_id) values (%s, now(), %s)",
-            (token, user_id),
-        )
-    conn.commit()
-    return token
-
-
-def sample(conn) -> list[dict]:
-    """The documents to score, ordered by md5 of the id.
-
-    Deterministic, so every engine and every configuration sees the same
-    documents and results stay comparable across jobs. Only documents carrying
-    both labels are usable as ground truth.
-    """
-    with conn.cursor() as cur:
-        cur.execute(
-            """
-            select d.id, d.correspondent_id, d.document_type_id, c.name, t.name
-              from documents_document d
-              join documents_correspondent c on c.id = d.correspondent_id
-              join documents_documenttype  t on t.id = d.document_type_id
-             where d.correspondent_id is not null
-               and d.document_type_id is not null
-               and length(d.content) > 400
-             order by md5(d.id::text)
-             limit %s
-            """,
-            (SAMPLE,),
-        )
-        return [
-            {
-                "id": r[0],
-                "correspondent_id": r[1],
-                "document_type_id": r[2],
-                "correspondent_name": r[3],
-                "document_type_name": r[4],
-            }
-            for r in cur.fetchall()
-        ]
-
-
-def floor(conn) -> float:
-    """Always guessing the most frequent correspondent.
-
-    Without it a number like "70% correct" cannot be read: one correspondent
-    holds several hundred of ~3900 documents.
-    """
-    with conn.cursor() as cur:
-        cur.execute(
-            """
-            select max(c), sum(c) from (
-                select count(*) c from documents_document
-                 where correspondent_id is not null group by correspondent_id
-            ) x
-            """,
-        )
-        top, total = cur.fetchone()
-    # float() on both: psycopg returns Postgres numerics as Decimal, and
-    # mixing that with a float literal raises TypeError rather than coercing.
-    return 100.0 * float(top) / float(total) if total else 0.0
-
-
-def fetch(path: str, token: str) -> tuple[dict | None, str | None, float]:
-    req = urllib.request.Request(
-        f"{PAPERLESS_URL}{path}",
-        headers={"Authorization": f"Token {token}"},
-    )
-    start = time.monotonic()
-    try:
-        with urllib.request.urlopen(req, timeout=REQUEST_TIMEOUT) as resp:
-            body = resp.read()
-        return json.loads(body), None, time.monotonic() - start
-    except urllib.error.HTTPError as err:
-        return None, f"HTTP {err.code}: {err.read()[:200]!r}", time.monotonic() - start
-    except Exception as err:  # noqa: BLE001 — a failed document is data, not a crash
-        return None, f"{type(err).__name__}: {err}", time.monotonic() - start
-
-
-def normalise(text: str) -> set[str]:
-    return {w for w in re.split(r"[^\w]+", text.lower()) if w}
-
-
-def recognised(payload: dict, field: str, want_name: str) -> bool:
-    """True when a suggested name contains every word of the true label's name.
-
-    Handles the observed failure directly: the model answers with a superset of
-    the stored name, which difflib rejects at 0.8 but a reader would accept.
-    """
-    want = normalise(want_name)
-    return any(want <= normalise(name) for name in payload.get(field) or [])
-
-
-def degenerate(payload: dict) -> str | None:
-    """Whether this looks like broken inference rather than a wrong guess.
-
-    Past roughly 14k prompt tokens this stack stops producing language — first
-    empty completions, then runs of digits and CJK characters — and one such
-    request poisons the llama.cpp slot until the pod restarts. Nothing in the
-    HTTP response says so: status 200, finish_reason "stop", well-formed JSON.
-
-    Scored as a wrong answer it would silently depress a candidate's numbers,
-    and because the slot stays poisoned it would take the rest of the run with
-    it. Flagged instead, so the failure is visible as itself.
-    """
-    title = payload.get("title") or ""
-    if not title.strip():
-        return "empty title"
-    # Digits, punctuation and whitespace are normal in a document title —
-    # "Rechnung 2025-5425541" is 8 letters in 21 characters and tripped an
-    # earlier version of this check. What broken inference produces is
-    # characters from other scripts, so look for those directly rather than
-    # inferring them from a shortage of Latin ones.
-    foreign = sum(c.isalpha() and not (c.isascii() or c in "äöüÄÖÜßáéèêíóúàçñ") for c in title)
-    if foreign > 0.2 * len(title):
-        return f"non-latin title: {title[:40]!r}"
-    return None
-
-
-def score_document(payload: dict, doc: dict) -> dict:
-    offered_corr = doc["correspondent_id"] in (payload.get("correspondents") or [])
-    offered_type = doc["document_type_id"] in (payload.get("document_types") or [])
-    return {
-        "correspondent_offered": offered_corr,
-        "correspondent_recognised": offered_corr
-        or recognised(payload, "suggested_correspondents", doc["correspondent_name"]),
-        "type_offered": offered_type,
-        "type_recognised": offered_type
-        or recognised(payload, "suggested_document_types", doc["document_type_name"]),
-        "suggested_correspondents": payload.get("suggested_correspondents") or [],
-        "suggested_document_types": payload.get("suggested_document_types") or [],
-        "title": payload.get("title"),
-    }
-
-
-LLM_ENDPOINT = os.environ.get("PAPERLESS_AI_LLM_ENDPOINT", "http://127.0.0.1:8080/v1")
-
-
-def canary() -> tuple[bool, str]:
-    """Ask the model something it cannot get wrong, and check it did not.
-
-    The instrument has to be shown working before its readings mean anything,
-    and this one can fail silently in a way that looks like a bad model. On
-    this Vulkan stack a prompt past some model-specific length stops llama.cpp
-    producing language: it repeats one token — " de de de", " phosphorus
-    phosphorus", whitespace — at HTTP 200 with finish_reason "stop". The slot
-    stays broken for every request after it until the pod restarts.
-
-    So a run that measured 50 documents against a poisoned slot reports 50
-    failures that are really one event, and reads as a model that cannot
-    classify. That is exactly what happened before this existed.
-
-    Run before the documents and again after: healthy then broken means the
-    failures in between are one fault, not evidence about the model.
-    """
-    body = json.dumps({
-        "model": os.environ.get("PAPERLESS_AI_LLM_MODEL", "probe"),
-        "max_tokens": 40,
-        "messages": [{"role": "user", "content": "Nenne die Hauptstadt von Frankreich."}],
-    }).encode()
-    req = urllib.request.Request(
-        f"{LLM_ENDPOINT}/chat/completions",
-        data=body,
-        headers={"Content-Type": "application/json"},
-    )
-    try:
-        with urllib.request.urlopen(req, timeout=120) as resp:
-            payload = json.loads(resp.read())
-        text = payload["choices"][0]["message"].get("content") or ""
-        text += payload["choices"][0]["message"].get("reasoning_content") or ""
-        return ("Paris" in text), text[:80]
-    except Exception as err:  # noqa: BLE001
-        return False, f"{type(err).__name__}: {err}"
-
-
-def run(engine: str, suffix: str, docs: list[dict], token: str) -> list[dict]:
-    log(f"=== {engine}: {len(docs)} documents")
-    rows = []
-    if engine == "ai":
-        ok, text = canary()
-        log(f"  canary before: {'OK' if ok else 'BROKEN'} — {text!r}")
-        if not ok:
-            log("  The model is not answering a trivial question correctly, so")
-            log("  nothing measured after this describes its classification.")
-            return [{"engine": engine, "canary_before": False, "canary_text": text}]
-    for n, doc in enumerate(docs, 1):
-        payload, error, elapsed = fetch(
-            f"/api/documents/{doc['id']}/{suffix}",
-            token,
-        )
-        row = {
-            "engine": engine,
-            "document_id": doc["id"],
-            "seconds": round(elapsed, 1),
-            "truth_correspondent": doc["correspondent_name"],
-            "truth_type": doc["document_type_name"],
-            "catchall_type": doc["document_type_name"] == CATCHALL_TYPE,
-        }
-        if payload is None:
-            # An authentication failure is never a fact about the model, and
-            # counted alongside real ones it reads like one more broken
-            # document. Stop loudly instead: something revoked this run's
-            # credentials mid-flight, and every number after it is worthless.
-            if error and error.startswith("HTTP 401"):
-                raise SystemExit(
-                    f"[eval] FATAL: {error}\n"
-                    f"[eval] This run's token stopped working part-way through. "
-                    f"Django's authtoken holds one token per user, so another "
-                    f"job sharing this database and this user revoked it.",
-                )
-            row["error"] = error
-            log(f"  {doc['id']}: {error}")
-        else:
-            row.update(score_document(payload, doc))
-            # Only the AI path returns a title; the classifier answers with ids
-            # alone, so an "empty title" there is normal rather than broken.
-            if engine == "ai" and (broken := degenerate(payload)) is not None:
-                row["degenerate"] = broken
-                log(f"  {doc['id']}: DEGENERATE — {broken}")
-        rows.append(row)
-
-        # Clear the slot between documents so the next one is measured on a
-        # clean server. The corruption this guards against is stateful: one bad
-        # request leaves the slot returning a single repeated token for
-        # everything after it, and without this a run counts that once per
-        # remaining document.
-        if engine == "ai":
-            try:
-                urllib.request.urlopen(
-                    urllib.request.Request(
-                        f"{LLM_ENDPOINT.rsplit('/v1', 1)[0]}/slots/0?action=erase",
-                        data=b"",
-                        method="POST",
-                    ),
-                    timeout=30,
-                )
-            except Exception as err:  # noqa: BLE001 — best effort, never fatal
-                if n == 1:
-                    log(f"  slot erase unavailable ({type(err).__name__}) — "
-                        f"failures after a corrupt document will not be independent")
-
-        if n % 10 == 0:
-            log(f"  {n}/{len(docs)}")
-            # Re-check the instrument, not just the progress. A slot corrupted
-            # part-way through turns every remaining document into a failure
-            # that says nothing about the model, and scattered failures — 29 of
-            # 50 in one earlier run — are exactly the shape that a
-            # start-of-run check alone would miss.
-            if engine == "ai":
-                ok, text = canary()
-                if not ok:
-                    log(f"  CANARY BROKEN at {n}/{len(docs)} — {text!r}")
-                    log("  The model can no longer answer a trivial question, so")
-                    log("  everything from here would measure a dead slot. Stopping.")
-                    log("  Failures recorded before this point are still usable;")
-                    log("  the ones adjacent to it are one fault, not several.")
-                    rows.append({"engine": engine, "canary_failed_at": n, "canary_text": text})
-                    break
-
-        # Stop early when a model cannot do the task at all, rather than
-        # spending an hour writing the same error fifty times. Three candidates
-        # in a row failed this way — Granite because llama.cpp cannot read its
-        # tool calls back, both Ministral sizes because they fill the tool
-        # call's first string with their own [THINK]/[TOOL_CALLS] control
-        # tokens until the token cap. In each case document one already said
-        # so, and the remaining forty-nine said nothing new.
-        #
-        # Deliberately not a ratio: a model that fails a third of the time is a
-        # result worth measuring in full. This only catches the case where
-        # nothing has ever worked.
-        if len(rows) == ABORT_AFTER and all("error" in r for r in rows):
-            ok, text = canary()
-            log(f"  canary after {ABORT_AFTER} failures: {'OK' if ok else 'BROKEN'} — {text!r}")
-            if not ok:
-                log("  The slot is corrupted, so these are ONE fault and not")
-                log(f"  {ABORT_AFTER} independent results. Probe the model's prompt-length")
-                log("  boundary with probe-length.sh before judging it.")
-            log(f"  ABORT — first {ABORT_AFTER} documents all failed:")
-            log(f"    {rows[0]['error']}")
-            log("  Nothing has succeeded, so the rest would measure nothing.")
-            log("  Check the model's tool calling before re-running:")
-            log("    curl -s $ENDPOINT/v1/chat/completions -d @probe.json")
-            log("  A usable candidate answers with finish_reason 'tool_calls'.")
-            break
-
-    # And once at the end, whatever happened. A short run whose first
-    # document succeeds trips neither the abort check nor the every-tenth
-    # one, so without this a five-document screen can finish having never
-    # re-verified the instrument it just used.
-    if engine == "ai":
-        ok, text = canary()
-        log(f"  canary after: {'OK' if ok else 'BROKEN'} — {text!r}")
-        if not ok:
-            log("  Healthy before, broken after: the failures in this run are")
-            log("  one fault and not independent results.")
-            rows.append({"engine": engine, "canary_after": False, "canary_text": text})
-    return rows
-
-
-def pct(hits: int, total: int) -> float:
-    return round(100.0 * hits / total, 1) if total else 0.0
-
-
-def summarise(rows: list[dict], engine: str) -> dict:
-    mine = [r for r in rows if r["engine"] == engine]
-    # Degenerate rows are excluded from the accuracy figures: they measure the
-    # backend giving up, not the model choosing wrongly, and leaving them in
-    # would quietly punish whichever candidate happened to trip the slot.
-    scored = [r for r in mine if "error" not in r and "degenerate" not in r]
-    failed = [r for r in mine if "error" in r]
-    broken = [r for r in mine if "degenerate" in r]
-    specific = [r for r in scored if not r["catchall_type"]]
-    catchall = [r for r in scored if r["catchall_type"]]
-    return {
-        "engine": engine,
-        "scored": len(scored),
-        "failed": len(failed),
-        # Non-zero means the run is suspect from that point on, not that the
-        # model is bad — the slot stays poisoned until the pod restarts.
-        "degenerate": len(broken),
-        "seconds_total": round(sum(r["seconds"] for r in mine)),
-        "correspondent_offered": pct(
-            sum(r["correspondent_offered"] for r in scored), len(scored)
-        ),
-        "correspondent_recognised": pct(
-            sum(r["correspondent_recognised"] for r in scored), len(scored)
-        ),
-        "type_offered": pct(sum(r["type_offered"] for r in scored), len(scored)),
-        "type_specific": pct(
-            sum(r["type_offered"] for r in specific), len(specific)
-        ),
-        "type_catchall": pct(
-            sum(r["type_offered"] for r in catchall), len(catchall)
-        ),
-    }
-
-
-def complement(rows: list[dict]) -> dict | None:
-    """What the LLM adds where the classifier is wrong, and what it costs.
-
-    The reason this script writes per-document rows at all. Only computed when
-    both engines ran, and only over documents both actually scored.
-    """
-    by_engine = {"classic": {}, "ai": {}}
-    for row in rows:
-        usable = "error" not in row and "degenerate" not in row
-        if usable and row["engine"] in by_engine:
-            by_engine[row["engine"]][row["document_id"]] = row
-    both = set(by_engine["classic"]) & set(by_engine["ai"])
-    if not both or not by_engine["ai"]:
-        return None
-
-    rescued = missed = broken = agreed = 0
-    for doc_id in both:
-        classic_hit = by_engine["classic"][doc_id]["correspondent_offered"]
-        ai_hit = by_engine["ai"][doc_id]["correspondent_offered"]
-        if classic_hit and ai_hit:
-            agreed += 1
-        elif not classic_hit and ai_hit:
-            rescued += 1
-        elif classic_hit and not ai_hit:
-            broken += 1
-        else:
-            missed += 1
-    return {
-        "documents": len(both),
-        "both_right": agreed,
-        "llm_rescued": rescued,
-        "llm_only_wrong": broken,
-        "both_wrong": missed,
-    }
-
-
-def main() -> int:
-    os.makedirs(OUT_DIR, exist_ok=True)
-    with psycopg.connect() as conn:
-        token = api_token(conn)
-        docs = sample(conn)
-        guess_floor = floor(conn)
-    log(f"label={LABEL} documents={len(docs)} floor={guess_floor:.1f}%")
-
-    rows: list[dict] = []
-    for engine, suffix in (("classic", "suggestions/"), ("ai", "ai_suggestions/")):
-        if engine in ENGINES:
-            rows.extend(run(engine, suffix, docs, token))
-
-    result = {
-        "label": LABEL,
-        "sample": len(docs),
-        "floor": round(guess_floor, 1),
-        "summaries": [summarise(rows, e) for e in ENGINES if e in ("classic", "ai")],
-        "complement": complement(rows),
-    }
-
-    with open(f"{OUT_DIR}/{LABEL}.jsonl", "w") as handle:
-        for row in rows:
-            handle.write(json.dumps(row, ensure_ascii=False) + "\n")
-    with open(f"{OUT_DIR}/{LABEL}.summary.json", "w") as handle:
-        json.dump(result, handle, ensure_ascii=False, indent=2)
-
-    # Also to stdout, so the numbers are in the pod log and reachable via Loki
-    # without mounting the volume.
-    log(json.dumps(result, ensure_ascii=False, indent=2))
-    return 0
-
-
-if __name__ == "__main__":
-    sys.exit(main())
@@ -1,373 +0,0 @@
---
-# One configuration under test, complete in one pod.
-#
-# A paperless instance can only point at one chat endpoint, so measuring two
-# candidates at once means two paperless instances — which makes the pod, not
-# the namespace, the right unit. Everything a measurement needs is here except
-# the database and the embedding endpoint, both of which are read-only and
-# shared.
-#
-# The three services are native sidecars (initContainers with
-# restartPolicy: Always), not regular containers. That matters twice: they
-# start in order and each must pass its probes before the next begins, which
-# is what keeps paperless from racing the database; and when the eval
-# container exits they are torn down, so the Job actually completes instead of
-# running forever behind a llama-server that never returns.
-apiVersion: batch/v1
-kind: Job
-metadata:
-  name: eval-__LABEL__
-  namespace: paperless-eval
-  labels:
-    app.kubernetes.io/name: paperless-eval
-    app.kubernetes.io/component: eval
-spec:
-  # A failed run is re-read, not retried blind: a retry would spend another
-  # hour reproducing whatever went wrong.
-  backoffLimit: 0
-  template:
-    metadata:
-      labels:
-        app.kubernetes.io/name: paperless-eval
-        app.kubernetes.io/component: eval
-    spec:
-      restartPolicy: Never
-      enableServiceLinks: false
-      # One measurement per node. Two jobs sharing a node would share its iGPU,
-      # and every timing in the result would then describe contention rather
-      # than the model — which is exactly how a decode rate of 4.7 tokens/s got
-      # attributed to context length once already.
-      affinity:
-        podAntiAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-            - labelSelector:
-                matchLabels:
-                  app.kubernetes.io/component: eval
-              topologyKey: kubernetes.io/hostname
-      securityContext:
-        fsGroup: 1000
-        fsGroupChangePolicy: OnRootMismatch
-        seccompProfile:
-          type: RuntimeDefault
-      initContainers:
-        # A private copy of /data — the search index and, the expensive part,
-        # the LLM index the RAG path needs.
-        #
-        # It cannot be mounted live and shared: LLM_INDEX_DIR is hardcoded to
-        # DATA_DIR / "llm_index", the store behind it is SQLite, and SQLite
-        # locking across pods over CephFS is not dependable. Nor can it be left
-        # out — an empty llm_index makes retrieve_similar_nodes return nothing,
-        # and the job would quietly measure the no-RAG path while reporting a
-        # RAG result.
-        #
-        # ~160 MiB, so seconds. A regular initContainer: it must finish before
-        # the sidecars start.
-        - name: seed-data
-          image: ghcr.io/paperless-ngx/paperless-ngx:3.1.2
-          command:
-            - sh
-            - -c
-            - |
-              set -e
-              # -r, not -a: preserving ownership needs CHOWN, which this
-              # container does not have, and cp then exits non-zero after
-              # copying the files anyway. Running as 1000 makes the copy land
-              # with the ownership paperless needs — it writes to the LLM index
-              # while serving.
-              cp -r /golden/. /data/
-              du -sh /data/index /data/llm_index
-              # Refuse to start rather than measure the wrong thing. An empty
-              # llm_index makes retrieve_similar_nodes return nothing, and the
-              # run would report a RAG result having never retrieved anything —
-              # which is exactly what the first attempt at this job did, because
-              # the golden volume had not been populated yet. The index is
-              # ~125 MiB once built; anything under 1 MiB is not one.
-              bytes=$(du -sb /data/llm_index | cut -f1)
-              if [ "$bytes" -lt 1000000 ]; then
-                echo "FATAL: llm_index is ${bytes} bytes — build it on the" >&2
-                echo "smoke-test deployment first:" >&2
-                echo "  kubectl -n paperless-eval exec deploy/paperless -- \\" >&2
-                echo "    python manage.py document_llmindex rebuild" >&2
-                exit 1
-              fi
-          volumeMounts:
-            - name: golden
-              mountPath: /golden
-              readOnly: true
-            - name: data
-              mountPath: /data
-          securityContext:
-            runAsUser: 1000
-            runAsGroup: 1000
-            allowPrivilegeEscalation: false
-            capabilities:
-              drop: ["ALL"]
-        - name: llama-server
-          image: ghcr.io/ggml-org/llama.cpp:server-vulkan-b10731
-          restartPolicy: Always
-          args:
-            - --hf-repo
-            - __HF_REPO__
-            # The exact filename rather than the :QUANT shorthand. The
-            # shorthand matches by substring across the whole repo and repos do
-            # carry several files under one tag — ggml-org ships mmproj-, mtp-
-            # and dflash- variants at Q8_0 beside the model — so naming the file
-            # removes a real ambiguity and costs nothing.
-            #
-            # It is NOT what caused the three-hour crashloop that prompted this,
-            # and an earlier version of this comment claimed it was. The real
-            # cause was the llama-models volume being 100% full: llama-server
-            # could not write its metadata cache, so it could not resolve the
-            # repo listing, and reported
-            #     no GGUF files found in repository <repo>
-            # followed by a truncated list of what it had cached before the
-            # disk filled. That reads as a bad reference and is a full disk.
-            #
-            # If a model will not load, check `df -h /models` before believing
-            # anything llama-server says about the repository.
-            - --hf-file
-            - __HF_FILE__
-            - --alias
-            - __ALIAS__
-            - --n-gpu-layers
-            - "99"
-            - --threads
-            - "4"
-            # The production ceiling. Not because the flag value itself breaks
-            # anything — bisection showed the corruption boundary is prompt
-            # length, around 14000 tokens, whatever --ctx-size says — but
-            # because measuring against a different ceiling than production
-            # runs would not describe production.
-            - --ctx-size
-            - "16384"
-            - --parallel
-            - "1"
-            - --ubatch-size
-            - "1024"
-            # The default is -1, infinity: nothing on the server caps how long
-            # a request generates, and paperless sets no budget of its own and
-            # waits. A classification that needs ~300 tokens ran to 10269 at a
-            # healthy 11 tok/s, and three of the first eleven documents did it
-            # — 900 s each, most of an hour spent on four documents.
-            #
-            # 768 is generous against the ~300 a correct answer takes. Note
-            # that paperless reports a truncated response as a flat
-            # "Invalid AI configuration" and discards the content, so what the
-            # model was actually doing is visible only in this container's log.
-            - --n-predict
-            - "__N_PREDICT__"
-            # Without this, Qwen3.6 spends the response on reasoning_content
-            # and returns empty content — 983 tokens into a classification that
-            # needs a hundred, at a perfectly healthy 11 tok/s. It reads as a
-            # slow or broken model and is neither.
-            #
-            # NOT --chat-template-kwargs '{"enable_thinking": false}'. That is
-            # deprecated, llama-server says so at startup, and it is not the
-            # harmless no-op it looks like on a template that never reads the
-            # variable: injecting it into Mistral's render context produced
-            # token salad — "1UnComplete1Self1UnCompleteSDirect11S..." — on a
-            # 681-token prompt, while the identical request against a server
-            # without the flag returned a clean tool call. It reads exactly
-            # like a broken model and is a broken prompt.
-            - --reasoning
-            - "off"
-            # Enables POST /slots/0?action=erase, which the eval calls between
-            # documents. Without it the endpoint answers 501.
-            #
-            # This is what makes the measurements independent. A request that
-            # corrupts the slot otherwise takes every document after it, so a
-            # run reports N failures for one fault — "4 of 5 failed" was really
-            # "document 2 broke the slot". Erasing costs the prefix cache and
-            # some prefill speed, which is the right trade for numbers that
-            # mean what they say.
-            - --slot-save-path
-            - /tmp/slots
-            - --no-webui
-          env:
-            - name: LLAMA_CACHE
-              value: /models
-          ports:
-            - name: llm
-              containerPort: 8080
-          volumeMounts:
-            - name: models
-              mountPath: /models
-          # 60 minutes. Sized for a FIRST DOWNLOAD from HuggingFace, not for
-          # reading a warm PVC — an earlier version allowed 20 minutes on the
-          # latter reasoning and killed GLM-4.7-Flash mid-download at 17.8 GB.
-          # The container then restarts and begins again, so too small a budget
-          # is not "slow start", it is a loop that never finishes.
-          #
-          # The kill shows up as exitCode 137 with reason "Error" rather than
-          # OOMKilled, which is how to tell a probe kill from a memory limit.
-          startupProbe:
-            httpGet:
-              path: /health
-              port: llm
-            periodSeconds: 20
-            failureThreshold: 180
-          resources:
-            requests:
-              cpu: "1"
-              memory: __MEM_REQ__
-              gpu.intel.com/i915: "1"
-            limits:
-              memory: __MEM_LIM__
-              gpu.intel.com/i915: "1"
-          securityContext:
-            allowPrivilegeEscalation: false
-            capabilities:
-              drop: ["ALL"]
-        - name: redis
-          # Its own, not the namespace's: two paperless instances sharing a
-          # celery broker would take each other's tasks.
-          image: redis:8.10.1-alpine
-          restartPolicy: Always
-          args: ["redis-server", "--save", "", "--loglevel", "warning"]
-          startupProbe:
-            tcpSocket:
-              port: 6379
-            periodSeconds: 2
-            failureThreshold: 30
-          resources:
-            requests:
-              cpu: 20m
-              memory: 64Mi
-            limits:
-              memory: 256Mi
-          securityContext:
-            allowPrivilegeEscalation: false
-            readOnlyRootFilesystem: true
-            capabilities:
-              drop: ["ALL"]
-        - name: paperless
-          image: ghcr.io/paperless-ngx/paperless-ngx:3.1.2
-          restartPolicy: Always
-          envFrom:
-            - configMapRef:
-                name: paperless-eval-config
-          env:
-            # Overrides the namespace ConfigMap: this pod's own llama-server,
-            # and its own redis rather than the shared Service.
-            - name: PAPERLESS_AI_LLM_ENDPOINT
-              value: http://127.0.0.1:8080/v1
-            - name: PAPERLESS_AI_LLM_MODEL
-              value: __ALIAS__
-            - name: PAPERLESS_REDIS
-              value: redis://127.0.0.1:6379
-            - name: PAPERLESS_SECRET_KEY
-              valueFrom:
-                secretKeyRef:
-                  name: paperless-eval-secrets
-                  key: secret-key
-            - name: PAPERLESS_DBUSER
-              valueFrom:
-                secretKeyRef:
-                  name: pprleval-pg-app
-                  key: username
-            - name: PAPERLESS_DBPASS
-              valueFrom:
-                secretKeyRef:
-                  name: pprleval-pg-app
-                  key: password
-          ports:
-            - name: http
-              containerPort: 8000
-          volumeMounts:
-            - name: data
-              mountPath: /usr/src/paperless/data
-            - name: media
-              mountPath: /usr/src/paperless/media
-            - name: consume
-              mountPath: /usr/src/paperless/consume
-          startupProbe:
-            httpGet:
-              path: /accounts/login/
-              port: http
-            periodSeconds: 20
-            failureThreshold: 180
-          # Same as production, and required: the entrypoint starts as root,
-          # chowns the volumes and drops to uid 1000 via setgid/setuid.
-          securityContext:
-            runAsNonRoot: false
-            allowPrivilegeEscalation: true
-            capabilities:
-              drop: ["ALL"]
-              add: ["CHOWN", "DAC_OVERRIDE", "FOWNER", "SETGID", "SETUID"]
-          resources:
-            requests:
-              cpu: 200m
-              memory: 512Mi
-            limits:
-              memory: 2Gi
-      containers:
-        - name: eval
-          # The paperless image again, for psycopg and a python that matches
-          # the application — not because the script needs paperless itself.
-          image: ghcr.io/paperless-ngx/paperless-ngx:3.1.2
-          command: ["python3", "/eval/eval.py"]
-          env:
-            - name: EVAL_LABEL
-              value: __LABEL__
-            - name: EVAL_SAMPLE
-              value: "__SAMPLE__"
-            # Raised alongside --n-predict for a diagnostic run. The two bounds
-            # interact: lifting the token cap alone just moves the failure from
-            # "hit the cap" to "hit the clock", which is what happened the first
-            # time gemma-4 was retried at 4096.
-            - name: EVAL_TIMEOUT
-              value: "__TIMEOUT__"
-            - name: PGHOST
-              value: pprleval-pg-rw
-            - name: PGDATABASE
-              value: paperless
-            - name: PGUSER
-              valueFrom:
-                secretKeyRef:
-                  name: pprleval-pg-app
-                  key: username
-            - name: PGPASSWORD
-              valueFrom:
-                secretKeyRef:
-                  name: pprleval-pg-app
-                  key: password
-          volumeMounts:
-            - name: script
-              mountPath: /eval
-            - name: results
-              mountPath: /results
-          resources:
-            requests:
-              cpu: 100m
-              memory: 256Mi
-            limits:
-              memory: 512Mi
-          securityContext:
-            allowPrivilegeEscalation: false
-            capabilities:
-              drop: ["ALL"]
-      volumes:
-        - name: models
-          persistentVolumeClaim:
-            claimName: llama-models
-        - name: results
-          persistentVolumeClaim:
-            claimName: paperless-eval-results
-        - name: script
-          configMap:
-            name: paperless-eval-script
-        # The namespace's /data, mounted read-only and copied out by seed-data
-        # above. Never mounted as the live /data of a job.
-        - name: golden
-          persistentVolumeClaim:
-            claimName: paperless-eval-golden
-            readOnly: true
-        # paperless insists on all three paths existing. media and consume stay
-        # empty — nothing is consumed during a measurement — while data is
-        # seeded with the copy.
-        - name: data
-          emptyDir: {}
-        - name: media
-          emptyDir: {}
-        - name: consume
-          emptyDir: {}
@@ -1,149 +0,0 @@
-#!/bin/sh
-# Where does this model's output stop being language?
-#
-#   ./probe-length.sh <label>
-#
-# THE TEST THAT SHOULD HAVE COME FIRST. On this Vulkan/Iris Xe stack every
-# model has a prompt length past which llama.cpp stops producing language and
-# starts repeating a single token — " de de de", " the the the", " phosphorus
-# phosphorus", or plain whitespace. It is silent: HTTP 200, finish_reason
-# "stop" or "length", nothing in the log.
-#
-# It is also STATEFUL. One request past the boundary leaves the slot broken for
-# every request after it, until the pod restarts. So in a 50-document run the
-# first bad document takes the rest with it, and "failed 3 of 3" is one
-# measurement, not three.
-#
-# A real paperless request is ~5300 tokens once the document, its RAG
-# neighbours and the taxonomy lists are in. So the only question that matters
-# about a candidate is whether its boundary is above that — and answering it
-# takes minutes, against ninety for a classification run that would have
-# measured the same thing badly.
-#
-# Measured so far:
-#   Qwen3.6-35B-A3B   coherent at 13588, broken at 15086
-#   gemma-4-26B-A4B   coherent at ~350, whitespace at 2334
-set -eu
-
-label="${1:?usage: probe-length.sh <label>   (labels are in candidates.tsv)}"
-here="$(dirname "$0")"
-ns=paperless-eval
-pod="lenprobe-$(echo "$label" | tr 'A-Z._' 'a-z--' | cut -c1-40)"
-
-row="$(grep -v '^#' "$here/candidates.tsv" | awk -v l="$label" '$1==l')"
-[ -n "$row" ] || { echo "unknown candidate '$label'" >&2; exit 1; }
-repo="$(echo "$row"   | cut -f2)"
-hffile="$(echo "$row" | cut -f3)"
-memreq="$(echo "$row" | cut -f5)"
-memlim="$(echo "$row" | cut -f6)"
-
-cleanup() { kubectl -n "$ns" delete pod "$pod" --wait=false >/dev/null 2>&1 || true; }
-trap cleanup EXIT INT TERM
-cleanup; sleep 2
-
-kubectl -n "$ns" apply -f - >/dev/null <<EOF
-apiVersion: v1
-kind: Pod
-metadata: {name: $pod, namespace: $ns}
-spec:
-  restartPolicy: Never
-  containers:
-    - name: llama-server
-      image: ghcr.io/ggml-org/llama.cpp:server-vulkan-b10731
-      args: ["--hf-repo","$repo","--hf-file","$hffile","--alias","probe",
-             "--n-gpu-layers","99","--threads","4","--ctx-size","16384",
-             "--parallel","1","--ubatch-size","1024","--n-predict","200","--no-webui"]
-      env: [{name: LLAMA_CACHE, value: /models}]
-      volumeMounts: [{name: models, mountPath: /models}]
-      resources:
-        requests: {cpu: "1", memory: $memreq, gpu.intel.com/i915: "1"}
-        limits: {memory: $memlim, gpu.intel.com/i915: "1"}
-      securityContext: {allowPrivilegeEscalation: false, capabilities: {drop: ["ALL"]}}
-  volumes:
-    - name: models
-      persistentVolumeClaim: {claimName: llama-models}
-EOF
-
-# Say which of the two it is doing. There are only two iGPUs, so a probe
-# started while both are busy sits Pending indefinitely — and an earlier run
-# spent an hour reporting nothing before giving up with "never became healthy",
-# which reads as a broken model rather than a queue.
-echo "waiting for $label"
-i=0
-until kubectl -n "$ns" exec "$pod" -- curl -sf --max-time 10 http://127.0.0.1:8080/health >/dev/null 2>&1; do
-  i=$((i+1))
-  if [ $(( i % 8 )) -eq 0 ]; then
-    ph=$(kubectl -n "$ns" get pod "$pod" -o jsonpath='{.status.phase}' 2>&1)
-    why=$(kubectl -n "$ns" get pod "$pod" -o jsonpath='{.status.conditions[?(@.type=="PodScheduled")].message}' 2>&1)
-    echo "  ${ph}${why:+ — $why}"
-    [ "$ph" = "Pending" ] && [ -n "$why" ] && { echo "FAIL: cannot schedule — free a GPU first"; exit 1; }
-  fi
-  [ "$i" -gt 240 ] && { echo "FAIL: never became healthy after 60 min"; exit 1; }
-  sleep 15
-done
-
-# Fresh pod per size. The defect poisons the slot, so probing sizes in sequence
-# against one server would measure the first bad request and nothing after it.
-for reps in 4 20 40 70 110 160; do
-  kubectl -n "$ns" delete pod "$pod" >/dev/null 2>&1 || true
-  sleep 2
-  kubectl -n "$ns" apply -f - >/dev/null <<EOF2
-apiVersion: v1
-kind: Pod
-metadata: {name: $pod, namespace: $ns}
-spec:
-  restartPolicy: Never
-  containers:
-    - name: llama-server
-      image: ghcr.io/ggml-org/llama.cpp:server-vulkan-b10731
-      args: ["--hf-repo","$repo","--hf-file","$hffile","--alias","probe",
-             "--n-gpu-layers","99","--threads","4","--ctx-size","16384",
-             "--parallel","1","--ubatch-size","1024","--n-predict","200","--no-webui"]
-      env: [{name: LLAMA_CACHE, value: /models}]
-      volumeMounts: [{name: models, mountPath: /models}]
-      resources:
-        requests: {cpu: "1", memory: $memreq, gpu.intel.com/i915: "1"}
-        limits: {memory: $memlim, gpu.intel.com/i915: "1"}
-      securityContext: {allowPrivilegeEscalation: false, capabilities: {drop: ["ALL"]}}
-  volumes:
-    - name: models
-      persistentVolumeClaim: {claimName: llama-models}
-EOF2
-  until kubectl -n "$ns" exec "$pod" -- curl -sf --max-time 10 http://127.0.0.1:8080/health >/dev/null 2>&1; do sleep 10; done
-
-  filler=""; n=0
-  while [ $n -lt $reps ]; do
-    filler="${filler}Die Rechnung wurde am fuenften Maerz ausgestellt und umfasst mehrere Positionen aus dem laufenden Wartungsvertrag. "
-    n=$((n+1))
-  done
-  printf '{"model":"probe","max_tokens":200,"messages":[{"role":"user","content":"%s Nenne die Hauptstadt von Frankreich."}]}' "$filler" > /tmp/lp.json
-  kubectl -n "$ns" cp /tmp/lp.json "$pod":/tmp/lp.json >/dev/null 2>&1 || true
-  r=$(kubectl -n "$ns" exec "$pod" -- curl -s --max-time 400 -X POST http://127.0.0.1:8080/v1/chat/completions \
-        -H 'Content-Type: application/json' -d @/tmp/lp.json)
-  pt=$(printf '%s' "$r" | grep -oE '"prompt_tokens":[0-9]+' | head -1 | cut -d: -f2)
-  # Test for DEGENERATION, not for the right answer. Two earlier versions of
-  # this check asked whether "Paris" came back and failed a working model
-  # twice: once because a thinking model leaves `content` empty and puts its
-  # answer in `reasoning_content`, then because 64 tokens is not enough for one
-  # to think its way to the answer at all.
-  #
-  # What actually marks the defect is a single token repeated — " de de de",
-  # " phosphorus phosphorus", whitespace. That signature survives truncation
-  # and does not care which field the model writes to or whether it finished,
-  # so it tests the thing itself instead of a proxy for it.
-  txt=$(printf '%s' "$r" | grep -oE '"(reasoning_)?content":"[^"]*' | sed 's/"[a-z_]*content":"//' | tr '\n' ' ')
-  rep=$(printf '%s' "$txt" | tr -s ' ' '\n' | grep -c . || echo 0)
-  top=$(printf '%s' "$txt" | tr -s ' ' '\n' | sort | uniq -c | sort -rn | head -1 | awk '{print $1}')
-  : "${top:=0}"
-  if [ "$rep" -gt 8 ] && [ $(( top * 2 )) -gt "$rep" ]; then
-    verdict="BROKEN"      # one token is more than half the output
-  elif [ "$rep" -le 1 ]; then
-    verdict="EMPTY"       # nothing came back at all — also not healthy
-  else
-    verdict="OK"
-  fi
-  printf '%-6s prompt=%-6s %s  %s\n' "$verdict" "${pt:-?}" "$(printf '%.50s' "$txt")" ""
-  [ "$verdict" != "OK" ] && { echo; echo "boundary is below ${pt:-?} prompt tokens"; echo "paperless needs ~5300 — $([ "${pt:-0}" -lt 6000 ] && echo 'DISQUALIFIED' || echo 'still above the bar')"; exit 0; }
-done
-echo
-echo "coherent through the largest size probed — boundary is above that"
@@ -1,95 +0,0 @@
-#!/bin/sh
-# Can this model make one valid tool call against paperless's real schema?
-#
-#   ./probe-toolcall.sh ggml-org/Ministral-3-8B-Instruct-2512-GGUF:Q8_0
-#
-# Two minutes, against ninety for a full run. Every candidate must pass this
-# before it is worth measuring, because three in a row failed here and the
-# remaining forty-nine documents of each run said nothing the first one had not.
-#
-# WHAT THIS CAN AND CANNOT TELL YOU
-#
-# A FAIL is conclusive: the model cannot make the call, do not spend a run.
-# A PASS is not. This payload has been too easy twice — first with three fields
-# instead of fourteen, then with a 1191-token prompt instead of production's
-# ~5300 — and each time a pass sent a ninety-minute run after a model that
-# failed on document one.
-#
-# It carries the real DocumentClassifierSchema and ~5000 tokens now, but it
-# still cannot reproduce what actually inflates a paperless prompt: the RAG
-# neighbours and the full taxonomy candidate lists, hundreds of names with IDs
-# the model must copy exactly and align by position. Document 1970 is 1976
-# characters and still produced a 5333-token prompt.
-#
-# So treat this as a cheap smoke test, not a gate. The real gate is
-# ABORT_AFTER in eval.py, which stops a run after three failed documents —
-# five minutes lost instead of ninety, using the genuine request.
-set -eu
-
-repo="${1:?usage: probe-toolcall.sh <hf-repo:quant> [extra llama-server args...]}"
-shift || true
-here="$(dirname "$0")"
-ns=paperless-eval
-pod="toolprobe-$(echo "$repo" | tr 'A-Z/:._' 'a-z----' | cut -c1-40 | sed 's/-*$//')"
-
-cleanup() { kubectl -n "$ns" delete pod "$pod" --wait=false >/dev/null 2>&1 || true; }
-trap cleanup EXIT INT TERM
-cleanup
-sleep 2
-
-echo "probing $repo"
-kubectl -n "$ns" apply -f - >/dev/null <<EOF
-apiVersion: v1
-kind: Pod
-metadata:
-  name: $pod
-spec:
-  restartPolicy: Never
-  containers:
-    - name: llama-server
-      image: ghcr.io/ggml-org/llama.cpp:server-vulkan-b10731
-      args: ["--hf-repo","$repo","--alias","probe","--n-gpu-layers","99",
-             "--threads","4","--ctx-size","16384","--parallel","1",
-             "--ubatch-size","1024","--n-predict","768","--reasoning","off",
-             "--no-webui"$(for a in "$@"; do printf ',"%s"' "$a"; done)]
-      env: [{name: LLAMA_CACHE, value: /models}]
-      volumeMounts: [{name: models, mountPath: /models}]
-      resources:
-        requests: {cpu: "1", memory: 12Gi, gpu.intel.com/i915: "1"}
-        limits: {memory: 20Gi, gpu.intel.com/i915: "1"}
-      securityContext:
-        allowPrivilegeEscalation: false
-        capabilities: {drop: ["ALL"]}
-  volumes:
-    - name: models
-      persistentVolumeClaim: {claimName: llama-models}
-EOF
-
-echo "waiting for the model to load (a first download takes minutes)"
-i=0
-until kubectl -n "$ns" exec "$pod" -- curl -sf --max-time 10 http://127.0.0.1:8080/health >/dev/null 2>&1; do
-  i=$((i + 1))
-  [ "$i" -gt 120 ] && { echo "FAIL: server never became healthy"; kubectl -n "$ns" logs "$pod" --tail=20; exit 1; }
-  sleep 10
-done
-
-sed "s/\"MODEL\"/\"probe\"/" "$here/toolcall-probe.json" > /tmp/tcp.json
-kubectl -n "$ns" cp /tmp/tcp.json "$pod":/tmp/probe.json >/dev/null 2>&1 || true
-r=$(kubectl -n "$ns" exec "$pod" -- curl -s --max-time 300 \
-      -X POST http://127.0.0.1:8080/v1/chat/completions \
-      -H 'Content-Type: application/json' -d @/tmp/probe.json)
-
-finish=$(printf '%s' "$r" | grep -oE '"finish_reason":"[a-z_]+"' | head -1 | cut -d'"' -f4)
-echo
-echo "finish_reason: ${finish:-<none>}"
-printf '%s' "$r" | grep -oE '"arguments":"[^"]{0,300}' | head -1 || true
-echo
-
-# finish_reason "tool_calls" is the whole test. "length" means it never stopped,
-# "stop" with no tool_calls means the server could not read the call back.
-if [ "$finish" = "tool_calls" ]; then
-  echo "PASS — usable, worth a full run"
-else
-  echo "FAIL — do not spend a run on this until the cause is understood"
-  exit 1
-fi
@@ -1,90 +0,0 @@
-#!/bin/sh
-# Start one candidate's measurement.
-#
-#   ./run-candidate.sh ministral8b
-#
-# Exists so the sidecar configuration lives in one file. Four copies of a
-# 300-line Job manifest is how --chat-template-kwargs ended up on the isolation
-# pod but not on the first eval job, and the run that followed measured empty
-# completions for two hours before anyone noticed.
-set -eu
-
-label="${1:?usage: run-candidate.sh <label> [sample]   (labels are in candidates.tsv)}"
-
-# Screen before you measure. A candidate that cannot do the task at all is
-# rejected by eval.py's ABORT_AFTER within three documents, so a screening run
-# costs about five minutes; a full run costs ninety. Three of the first four
-# candidates were rejections, and each one was allowed to run for an hour or
-# more before anyone looked.
-#
-#     ./run-candidate.sh <label> 5     screen
-#     ./run-candidate.sh <label>       measure, 50 documents
-#
-# Screening uses genuine paperless requests, which matters: the standalone
-# probe-toolcall.sh passed two candidates that then failed on document one,
-# because a synthesised request cannot reproduce the RAG neighbours and the
-# taxonomy candidate lists that make a real prompt what it is.
-#
-# Never compare numbers across sample sizes. A screen is a gate, not a
-# measurement; the comparison happens at 50.
-sample="${2:-50}"
-
-# Third argument raises the server-side generation cap for a diagnostic run.
-#
-# The default 768 is MINE, not a property of any model, and every candidate
-# recorded as failing at exactly 768 tokens is failing against my bound rather
-# than demonstrably failing the task. A model that needs 900 tokens for a valid
-# call looks identical to one that never terminates. Raise it and find out
-# before calling a model incapable:
-#
-#     ./run-candidate.sh gemma4-26b-a4b 5 4096
-npredict="${3:-768}"
-
-# Client-side bound, raised with the cap. At 6 tok/s a 4096-token budget needs
-# 620 s of generation alone, so leaving this at 300 turns the diagnostic into a
-# different failure rather than an answer.
-timeout="${4:-300}"
-here="$(dirname "$0")"
-
-row="$(grep -v '^#' "$here/candidates.tsv" | awk -v l="$label" '$1==l')"
-if [ -z "$row" ]; then
-  echo "unknown candidate '$label'. Known:" >&2
-  grep -v '^#' "$here/candidates.tsv" | awk 'NF{print "  " $1}' >&2
-  exit 1
-fi
-
-repo="$(echo "$row"   | cut -f2)"
-hffile="$(echo "$row" | cut -f3)"
-alias="$(echo "$row"  | cut -f4)"
-memreq="$(echo "$row" | cut -f5)"
-memlim="$(echo "$row" | cut -f6)"
-
-if [ -z "$hffile" ]; then
-  echo "candidate '$label' has no hf-file column — refusing to guess." >&2
-  echo "Resolve the exact .gguf filename from the HF API and add it." >&2
-  exit 1
-fi
-
-# The script is mounted from a ConfigMap, so it has to be current before the
-# Job starts. Shared by every candidate — they run the same measurement.
-kubectl -n paperless-eval create configmap paperless-eval-script \
-  --from-file=eval.py="$here/eval.py" --dry-run=client -o yaml | kubectl apply -f -
-
-if [ "$sample" -lt 50 ]; then
-  echo "SCREENING $label at $sample documents — a gate, not a measurement"
-fi
-echo "candidate=$label repo=$repo alias=$alias mem=$memreq/$memlim sample=$sample n_predict=$npredict timeout=$timeout"
-sed -e "s#__LABEL__#$label#g" \
-    -e "s#__HF_REPO__#$repo#g" \
-    -e "s#__ALIAS__#$alias#g" \
-    -e "s#__MEM_REQ__#$memreq#g" \
-    -e "s#__MEM_LIM__#$memlim#g" \
-    -e "s#__SAMPLE__#$sample#g" \
-    -e "s#__HF_FILE__#$hffile#g" \
-    -e "s#__N_PREDICT__#$npredict#g" \
-    -e "s#__TIMEOUT__#$timeout#g" \
-    "$here/job-template.yaml" | kubectl apply -f -
-
-echo
-echo "follow:  kubectl -n paperless-eval logs -f job/eval-$label -c eval"
-echo "re-run:  kubectl -n paperless-eval delete job eval-$label   # a Job is immutable"