fix: [alert] KubeDeploymentRolloutStuck (ai) (#750)

.woodpecker/deploy-netbird-proxy.yml

@@ -7,8 +7,6 @@ when:
       # install-netbird-proxy.yml also applies the netbird-proxy-watchdog role,
       # so changes to it must redeploy this host too.
       - "ansible/roles/netbird-proxy-watchdog/**"
-      # install-netbird-proxy.yml also applies the edge bouncer.
-      - "ansible/roles/crowdsec-edge-bouncer/**"
       - "ansible/plays/install-netbird-proxy.yml"
       - ".woodpecker/deploy-netbird-proxy.yml"
   - event: deployment

ansible/inventory/group_vars/netbird_proxy/vault.yml

@@ -1,23 +0,0 @@
----
-# Bouncer key for crowdsec-edge-bouncer, scoped to the proxy hosts.
-#
-# A single encrypted VALUE rather than a whole-file vault like its siblings in
-# group_vars/all and group_vars/edge_host. Deliberate: adding this variable to
-# an existing whole-file vault means decrypting that file first, which exposes
-# every other secret in it for the sake of appending one line. encrypt_string
-# keeps the blast radius at this one value.
-#
-# Same value as OpenBao homelab/infrastructure/crowdsec-edge-bouncer-key, which
-# the crowdsec-edge-bouncer-key ExternalSecret feeds to BOUNCER_KEY_edge on the
-# LAPI. Both sides must move together — and unlike crowdsec-bouncer-key, this
-# one is NOT in the quarterly rotator, because nothing on these hosts re-reads
-# OpenBao on a schedule. Rotating it means updating OpenBao and re-running
-# plays/install-netbird-proxy.yml.
-vault_crowdsec_edge_bouncer_key: !vault |
-          $ANSIBLE_VAULT;1.1;AES256
-          32653764333536316163303838333839616562663862666637626165303837633866643033393239
-          3363376538656236643161386432326437633232643564630a633635306365323537346139626364
-          36613033343938363366306239363638383466396337623333623136313234656366333366623433
-          6437656632373364340a346566343866366433306533353431383064356362346536383136323663
-          39333864643534326433653233303137373162663066633132623931316135323733333863306330
-          6366623463666435636362376238313639666263323163303733

ansible/plays/install-netbird-proxy.yml

@@ -26,7 +26,3 @@
     - docker-install
     - netbird-proxy
     - netbird-proxy-watchdog
-    # Enforces the cluster's CrowdSec decisions here, where public traffic
-    # enters. Deployed in dry-run: it logs what it would block and creates no
-    # nftables rules until crowdsec_bouncer_mode is flipped per host.
-    - crowdsec-edge-bouncer

ansible/roles/crowdsec-edge-bouncer/defaults/main.yml

@@ -1,47 +0,0 @@
----
-# CrowdSec firewall bouncer on the NetBird proxy hosts.
-#
-# Enforces the decisions the in-cluster CrowdSec already detects, at the point
-# where public traffic actually enters: these hosts terminate TLS for every
-# *.renner.uno name. A drop here never reaches the mesh, the relay or the
-# cluster.
-#
-# Detection stays in the cluster. The reverse-proxy produces no parseable
-# access log and there is no CrowdSec collection for it, while Traefik has a
-# working one — and every scenario in use needs HTTP detail (paths, status
-# codes, user agents) that does not exist at this layer.
-
-# Pinned. A release tarball URL is immutable, and this checksum is of the
-# amd64 asset for exactly this tag.
-crowdsec_bouncer_version: v0.0.36
-crowdsec_bouncer_sha256: f86e4b72693549d99f40a9402abefb894108f047a3fbe6e72fada25ee17ce88b
-
-# The LAPI over the public path these hosts already serve, see
-# kubernetes/infrastructure/crowdsec/lapi-ingress.yaml. The proxies cannot
-# reach it over the mesh: the reverse-proxy container runs non-root, so its
-# embedded NetBird client terminates the tunnel in userspace and the tunnel
-# belongs to that process rather than to the host netns.
-#
-# No ca_cert_path: Traefik serves a Let's Encrypt certificate for this name,
-# and the bouncer adds ca_cert_path to the system pool rather than replacing
-# it — the system pool already trusts LE.
-crowdsec_bouncer_api_url: https://crowdsec.renner.uno/
-crowdsec_bouncer_api_key: "{{ vault_crowdsec_edge_bouncer_key }}"
-
-# STARTS INERT, and that is the point.
-#
-# dry-run pulls decisions and logs what it would block without creating a
-# single nftables rule. The first bouncer to arm anywhere enforces ~24k CAPI
-# decisions at once on the only external entry path, so the false-positive
-# window has to come before the enforcement, not after it. Flip to `nftables`
-# one host at a time once the log looks right.
-crowdsec_bouncer_mode: dry-run
-
-crowdsec_bouncer_update_frequency: 10s
-crowdsec_bouncer_log_level: info
-
-# Bound to loopback deliberately. These hosts are public and have no metrics
-# collection path at all today — opening :60601 to the internet to gain a
-# scrape target is the wrong trade. Revisit when host metrics exist.
-crowdsec_bouncer_metrics_listen: 127.0.0.1
-crowdsec_bouncer_metrics_port: 60601

ansible/roles/crowdsec-edge-bouncer/handlers/main.yml

@@ -1,10 +0,0 @@
----
-# A restart drops both nftables tables and rebuilds them from the next
-# successful LAPI pull, so there is a short window with no blocklist. That is
-# the documented behaviour of a clean shutdown, not a defect — and it is why
-# retry_initial_connect matters.
-- name: Restart crowdsec firewall bouncer
-  ansible.builtin.systemd:
-    name: crowdsec-firewall-bouncer.service
-    state: restarted
-    daemon_reload: true

ansible/roles/crowdsec-edge-bouncer/tasks/main.yml

@@ -1,84 +0,0 @@
----
-# See defaults/main.yml for why this exists and why it starts inert.
-#
-# The static release tarball, not a distribution package: the AUR package
-# `crowdsec-firewall-bouncer-nftables` depends on `crowdsec` and would pull the
-# whole Security Engine onto hosts that only need the bouncer, the group is
-# mixed (proxy-1 Arch, VPS-1 Debian), and the AUR would be a new pattern this
-# repo does not otherwise have. The tarball is the same artefact the AUR
-# package downloads.
-
-- name: Ensure nftables is present (the enforcing backend)
-  ansible.builtin.package:
-    name: nftables
-    state: present
-
-- name: Ensure the bouncer config directory exists
-  ansible.builtin.file:
-    path: /etc/crowdsec/bouncers
-    state: directory
-    mode: "0755"
-
-- name: Fetch the pinned bouncer release
-  ansible.builtin.get_url:
-    url: >-
-      https://github.com/crowdsecurity/cs-firewall-bouncer/releases/download/{{
-      crowdsec_bouncer_version }}/crowdsec-firewall-bouncer-linux-amd64.tgz
-    dest: /opt/crowdsec-firewall-bouncer-{{ crowdsec_bouncer_version }}.tgz
-    checksum: sha256:{{ crowdsec_bouncer_sha256 }}
-    mode: "0644"
-
-# Only the binary. The tarball also carries install.sh / upgrade.sh /
-# uninstall.sh, which manage systemd and config themselves — that is this
-# role's job, and letting them run would leave two owners for the same files.
-- name: Unpack the bouncer binary
-  ansible.builtin.unarchive:
-    src: /opt/crowdsec-firewall-bouncer-{{ crowdsec_bouncer_version }}.tgz
-    dest: /opt
-    remote_src: true
-    include:
-      - crowdsec-firewall-bouncer-{{ crowdsec_bouncer_version }}/crowdsec-firewall-bouncer
-    creates: /opt/crowdsec-firewall-bouncer-{{ crowdsec_bouncer_version }}/crowdsec-firewall-bouncer
-
-- name: Install the bouncer binary
-  ansible.builtin.copy:
-    src: /opt/crowdsec-firewall-bouncer-{{ crowdsec_bouncer_version }}/crowdsec-firewall-bouncer
-    dest: /usr/local/bin/crowdsec-firewall-bouncer
-    remote_src: true
-    mode: "0755"
-  notify: Restart crowdsec firewall bouncer
-
-# 0600: this file carries the LAPI key.
-- name: Deploy the bouncer configuration
-  ansible.builtin.template:
-    src: crowdsec-firewall-bouncer.yaml.j2
-    dest: /etc/crowdsec/bouncers/crowdsec-firewall-bouncer.yaml
-    mode: "0600"
-  notify: Restart crowdsec firewall bouncer
-
-- name: Deploy the bouncer systemd unit
-  ansible.builtin.template:
-    src: crowdsec-firewall-bouncer.service.j2
-    dest: /etc/systemd/system/crowdsec-firewall-bouncer.service
-    mode: "0644"
-  notify: Restart crowdsec firewall bouncer
-
-- name: Enable and start the bouncer
-  ansible.builtin.systemd:
-    name: crowdsec-firewall-bouncer.service
-    enabled: true
-    state: started
-    daemon_reload: true
-
-- name: Report how to read it
-  ansible.builtin.debug:
-    msg: |
-      crowdsec-firewall-bouncer deployed in mode "{{ crowdsec_bouncer_mode }}".
-
-      What it would block:  journalctl -u crowdsec-firewall-bouncer -f
-      What it does block:   nft list table ip crowdsec ; nft list table ip6 crowdsec6
-      Metrics:              curl -s {{ crowdsec_bouncer_metrics_listen }}:{{ crowdsec_bouncer_metrics_port }}/metrics
-
-      Rollback: systemctl stop crowdsec-firewall-bouncer. A clean stop deletes
-      both nftables tables by itself. Only after a kill -9 do they need
-      removing by hand: nft delete table ip crowdsec ; nft delete table ip6 crowdsec6

ansible/roles/crowdsec-edge-bouncer/templates/crowdsec-firewall-bouncer.service.j2

@@ -1,24 +0,0 @@
-# Managed by Ansible (crowdsec-edge-bouncer role) — do not edit.
-#
-# Upstream's shipped unit orders itself After=crowdsec.service. There is no
-# local Security Engine on these hosts — the LAPI is in the cluster — so that
-# dependency is dropped; nss-lookup instead, because api_url is a public name
-# that has to resolve before the first connect.
-[Unit]
-Description=CrowdSec firewall bouncer (edge enforcement for the NetBird proxy)
-After=network-online.target nss-lookup.target
-Wants=network-online.target
-
-[Service]
-Type=notify
-# -t validates the config and exits non-zero on a bad one, so a broken config
-# fails the start instead of taking the running rules down with it.
-ExecStartPre=/usr/local/bin/crowdsec-firewall-bouncer -c /etc/crowdsec/bouncers/crowdsec-firewall-bouncer.yaml -t
-ExecStart=/usr/local/bin/crowdsec-firewall-bouncer -c /etc/crowdsec/bouncers/crowdsec-firewall-bouncer.yaml
-Restart=always
-RestartSec=10
-LimitNOFILE=65536
-KillMode=mixed
-
-[Install]
-WantedBy=multi-user.target

ansible/roles/crowdsec-edge-bouncer/templates/crowdsec-firewall-bouncer.yaml.j2

@@ -1,55 +0,0 @@
-# Managed by Ansible (crowdsec-edge-bouncer role) — do not edit.
-mode: {{ crowdsec_bouncer_mode }}
-update_frequency: {{ crowdsec_bouncer_update_frequency }}
-
-# stdout, so journald owns the log and there is no second rotation policy to
-# keep in sync. The shipped example writes to /var/log; that only matters for
-# hosts without journald.
-log_mode: stdout
-log_level: {{ crowdsec_bouncer_log_level }}
-
-api_url: {{ crowdsec_bouncer_api_url }}
-api_key: {{ crowdsec_bouncer_api_key }}
-
-# Without this the process EXITS when the LAPI is unreachable at startup —
-# there is no retry on the first connect. And because a clean shutdown deletes
-# the nftables tables, the systemd restart loop would leave the host with no
-# blocklist at all until the first successful connect. Present in the code,
-# absent from the shipped example config.
-retry_initial_connect: true
-
-disable_ipv4: false
-disable_ipv6: false
-deny_action: DROP
-
-supported_decisions_types:
-  - ban
-
-# Separate tables per family; the bouncer never touches an existing table or
-# chain, and never sets a chain policy. Priority -10 puts it ahead of anything
-# at the default filter priority.
-nftables:
-  ipv4:
-    enabled: true
-    set-only: false
-    table: crowdsec
-    chain: crowdsec-chain
-    priority: -10
-  ipv6:
-    enabled: true
-    set-only: false
-    table: crowdsec6
-    chain: crowdsec6-chain
-    priority: -10
-
-# input is the traffic this exists for: the reverse-proxy is host-networked and
-# owns :443, so its packets traverse INPUT. forward covers the bridge-networked
-# http-redirect sidecar, whose packets never do.
-nftables_hooks:
-  - input
-  - forward
-
-prometheus:
-  enabled: true
-  listen_addr: {{ crowdsec_bouncer_metrics_listen }}
-  listen_port: {{ crowdsec_bouncer_metrics_port }}

kubernetes/apps/llama-server/deployment.yaml

@@ -25,6 +25,13 @@ metadata:
     app.kubernetes.io/name: llama-server
 spec:
   replicas: 1
+  # The startupProbe below sanctions a 40 min cold download, but the default
+  # progressDeadlineSeconds is 600. Left implicit the Deployment declares
+  # ProgressDeadlineExceeded at 10 minutes while the probe is still legitimately
+  # waiting, and KubeDeploymentRolloutStuck fires 15 min after that on a rollout
+  # that is fine. Kept above the startup budget, with room for the image pull
+  # and scheduling that happen before the first probe.
+  progressDeadlineSeconds: 3000
   # A rolling update would load a second copy of the weights onto the same
   # iGPU and into the same node's RAM before releasing the first.
   strategy:

kubernetes/apps/llama-server/embed.yaml

@@ -19,6 +19,11 @@ metadata:
     app.kubernetes.io/name: llama-embed
 spec:
   replicas: 1
+  # Same reasoning as llama-server: the startupProbe budget (600s) must stay
+  # below progressDeadlineSeconds, whose default is also 600 — exactly equal, so
+  # any cold download that uses most of its budget trips
+  # KubeDeploymentRolloutStuck on a rollout that is still progressing normally.
+  progressDeadlineSeconds: 900
   strategy:
     type: Recreate
   selector:

kubernetes/infrastructure/crowdsec/edge-bouncer-key-externalsecret.yaml

@@ -1,27 +0,0 @@
-# Bouncer key for the firewall bouncers on proxy-1/proxy-2.
-#
-# Separate from crowdsec-bouncer-key ON PURPOSE. That key is rotated
-# quarterly by kubernetes/infrastructure/openbao/rotation/crowdsec-bouncer-key.yaml,
-# and both of its consumers live in the cluster, so ESO propagates a rotation
-# within 10m. The edge bouncers are Ansible-managed hosts: nothing there
-# re-reads OpenBao on a schedule, so a rotated key would leave them 401ing
-# against the LAPI until someone runs the playbook. This key is therefore
-# deliberately NOT enrolled in the rotator — it rotates when the playbook runs,
-# and the playbook is what carries the new value to the hosts.
-#
-# The value must match ansible vault_crowdsec_edge_bouncer_key.
-apiVersion: external-secrets.io/v1
-kind: ExternalSecret
-metadata:
-  name: crowdsec-edge-bouncer-key
-  namespace: crowdsec
-spec:
-  refreshInterval: 10m
-  secretStoreRef:
-    kind: ClusterSecretStore
-    name: openbao
-  target:
-    name: crowdsec-edge-bouncer-key
-  dataFrom:
-    - extract:
-        key: infrastructure/crowdsec-edge-bouncer-key

kubernetes/infrastructure/crowdsec/helmrelease.yaml

@@ -36,36 +36,6 @@ spec:
   values:
     container_runtime: containerd
 
-    # Addresses a community blocklist must never be allowed to ban.
-    #
-    # This is the ONLY declarative allowlist the chart offers. `cscli
-    # allowlists` objects live in the LAPI database, and that database runs
-    # without persistence here on purpose (see lapi.persistentVolume below),
-    # so anything created through cscli is gone at the next restart. This file
-    # is a ConfigMap mount, so it survives by construction. Upstream marks it
-    # deprecated in favour of centralized allowlists; it is still the only
-    # thing that works against a LAPI with no PVC.
-    #
-    # Scope: CAPI decisions only. Locally generated decisions are not filtered
-    # here — they currently carry real client IPs (verified), because Traefik
-    # resolves X-Forwarded-For past the netbird-ingress pod IP.
-    #
-    # The home WAN address is deliberately absent and is not needed: LAN
-    # clients never present it. opentofu/unifi_nat_hairpin.tf DNATs the public
-    # proxy addresses to the Traefik VIP on the UDM, so on-LAN traffic reaches
-    # the cluster directly with a LAN source and never traverses the proxies.
-    # Private ranges cannot appear in a community blocklist either way.
-    config:
-      capi_whitelists.yaml: |
-        cidrs:
-          # The proxy hosts themselves. Same three addresses as the
-          # netbird-ingress-egress toCIDRSet and the hairpin endpoints —
-          # keep all three lists in sync. proxy-1 is the v6-only member,
-          # proxy-2 the dual-stack one.
-          - 45.157.177.26/32
-          - 2a03:4000:4b:c1e:4474:43ff:fedf:8f85/128
-          - 2a03:4000:28:f6e:a4c7:59ff:fe1e:9a29/128
-
     # mTLS between agents and the LAPI. Do not switch back to password auth:
     # the LAPI runs without persistence, so its machine-registration DB is
     # wiped on every restart, and password-registered agents then get 401
@@ -116,18 +86,10 @@ spec:
         requests:
           cpu: 50m
       podAnnotations:
-        # Reloader restarts the LAPI when either bouncer key changes. Both are
-        # consumed as env vars from a secretKeyRef, and env is fixed at pod
-        # start — without a restart the LAPI keeps trusting the old value while
-        # the bouncer already presents the new one, which is a 401 at exactly
-        # the moment a rotation is supposed to be seamless.
-        #
-        # crowdsec-bouncer-key rotates on a schedule, see
+        # Reloader restarts the LAPI when crowdsec-bouncer-key changes (it's an
+        # env-var secret, so a restart is required to pick up a rotated key). See
         # kubernetes/infrastructure/openbao/rotation/crowdsec-bouncer-key.yaml.
-        # crowdsec-edge-bouncer-key rotates by hand together with
-        # ansible/plays/install-netbird-proxy.yml; it still needs to be listed
-        # here, because the LAPI side of that rotation is this env var.
-        secret.reloader.stakater.com/reload: crowdsec-bouncer-key,crowdsec-edge-bouncer-key
+        secret.reloader.stakater.com/reload: crowdsec-bouncer-key
       persistentVolume:
         data:
           # Kept OFF deliberately. The #337 attempt to set this `true` (persist
@@ -156,17 +118,6 @@ spec:
             secretKeyRef:
               name: crowdsec-bouncer-key
               key: key
-        # The firewall bouncers on proxy-1/proxy-2, over the public LAPI
-        # endpoint (lapi-ingress.yaml). The chart's entrypoint re-runs
-        # `register_bouncer` for every BOUNCER_KEY_* on each start and skips
-        # names that already exist, so this survives the LAPI's wiped
-        # registration DB the same way BOUNCER_KEY_ingress does — the env var
-        # is the source of truth, not the database.
-        - name: BOUNCER_KEY_edge
-          valueFrom:
-            secretKeyRef:
-              name: crowdsec-edge-bouncer-key
-              key: key
 
     agent:
       # Same reasoning as lapi.resources above, multiplied by three: this is a

kubernetes/infrastructure/crowdsec/kustomization.yaml

@@ -2,6 +2,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
   - bouncer-key-externalsecret.yaml
-  - edge-bouncer-key-externalsecret.yaml
-  - lapi-ingress.yaml
   - helmrelease.yaml

kubernetes/infrastructure/crowdsec/lapi-ingress.yaml

@@ -1,81 +0,0 @@
-# Public LAPI endpoint for the firewall bouncers on proxy-1/proxy-2.
-#
-# WHY PUBLIC AT ALL. The bouncers run as host processes on the proxies, and
-# the proxies have no mesh access: the netbird-proxy container runs non-root
-# (see the ip_unprivileged_port_start task in the netbird-proxy role), so it
-# has no NET_ADMIN and no /dev/net/tun, and its embedded NetBird client
-# therefore terminates the tunnel in userspace. The tunnel belongs to that
-# process, not to the host netns — a second process on the same host cannot
-# route through it. The alternative was a host-level NetBird client plus the
-# LAPI on a LoadBalancer VIP; this reuses the path every other service already
-# takes instead of adding two peers and an ACL policy.
-#
-# NO IP ALLOWLIST, deliberately. The bouncer key is the access control.
-#
-# An allowlist was the obvious second factor and it does not work here. A
-# request originating on a proxy host does not arrive carrying that host's
-# public address: the client connects to the local proxy on 127.0.0.1, the
-# reverse-proxy puts what it saw into X-Forwarded-For, and Traefik resolves
-# that. Measured against the netbird-proxy-watchdog, which makes exactly this
-# kind of request every round:
-#
-#   172.18.0.3 - - [02/Sep/2026:10:12:01] "GET /" 200 websecure-immich-...
-#   172.18.0.3 - - [02/Sep/2026:10:12:01] "GET /" 200 websecure-home-...
-#
-# 172.18.0.3 is a Docker bridge address on the proxy side. Allowlisting the
-# published proxy addresses would have rejected the bouncer with 403;
-# allowlisting 172.18.0.0/16 would be a private range on a host we do not
-# control the addressing of, which is not a security control.
-#
-# What remains is the API key, which is the real control anyway: this endpoint
-# serves read-only decision streams to a valid bouncer key and nothing else.
-# The admin surface is cscli, in-cluster. If the source address ever becomes
-# deterministic, an ipAllowList can be added on top.
----
-# The LAPI serves HTTPS only once tls.enabled is set — there is no plaintext
-# fallback and no second port; 8080 is the TLS port. Its certificate comes
-# from the chart's own self-signed CA, so Traefik needs that CA to verify the
-# backend. SANs include crowdsec-service.crowdsec.svc.cluster.local, which is
-# what the Service resolves to.
-apiVersion: traefik.io/v1alpha1
-kind: ServersTransport
-metadata:
-  name: crowdsec-lapi
-  namespace: crowdsec
-spec:
-  serverName: crowdsec-service.crowdsec.svc.cluster.local
-  rootCAsSecrets:
-    - crowdsec-root-secret
----
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
-  name: crowdsec-lapi
-  namespace: crowdsec
-  labels:
-    # Public DNS via Cloudflare; the proxies resolve it from outside. The UDM
-    # external-dns instance skips it, LAN reaches it via the hairpin NAT.
-    dns.renner.uno/public: "true"
-  annotations:
-    cert-manager.io/cluster-issuer: letsencrypt-cluster
-    # NOT behind the Authentik gate: the client is a daemon with an API key,
-    # not a browser that can complete an OIDC flow.
-    traefik.ingress.kubernetes.io/service.serversscheme: https
-    traefik.ingress.kubernetes.io/service.serverstransport: crowdsec-crowdsec-lapi@kubernetescrd
-spec:
-  ingressClassName: traefik
-  rules:
-    - host: crowdsec.renner.uno
-      http:
-        paths:
-          - path: /
-            pathType: Prefix
-            backend:
-              service:
-                name: crowdsec-service
-                port:
-                  number: 8080
-  tls:
-    - secretName: crowdsec-lapi-public-tls
-      hosts:
-        - crowdsec.renner.uno

scripts/paperless-eval.sh

@@ -1,207 +0,0 @@
-#!/usr/bin/env bash
-# End-to-end evaluation of Paperless document suggestions against the real
-# archive, comparing the two engines Paperless ships:
-#
-#   /api/documents/<id>/suggestions/      the built-in scikit-learn classifier,
-#                                         trained on the existing corpus
-#   /api/documents/<id>/ai_suggestions/   the LLM path, optionally with RAG
-#
-# WHY THIS EXISTS
-#
-# scripts/embed-eval-real.sh measures whether a document's nearest neighbours
-# carry its labels. That is a proxy: Paperless does not act on the neighbours
-# directly, it hands them to an LLM as context, alongside the document text and
-# a candidate list of existing tags and correspondents. The LLM can override
-# poor context, or fail with good context. So a better embedding score does not
-# automatically mean better suggestions, and nothing so far has verified that it
-# does.
-#
-# This measures the thing that is actually wanted: does Paperless suggest the
-# labels a human already chose? Everything else — MTEB scores, retrieval
-# accuracy — is only interesting insofar as it predicts this number.
-#
-# It also answers a question that predates the LLM work: with ten years of
-# hand-curated labels, the built-in classifier has excellent training data. It
-# cannot invent a new tag or a title, which is the LLM's advantage — but for
-# assigning labels that already exist it may simply be better. Running both on
-# the same documents settles that rather than assuming it.
-#
-# Ground truth is the current filing. Where the two disagree with it, that is
-# not automatically an error: over ten years filing conventions change, and an
-# old document may carry an older convention. Treat the absolute numbers as
-# indicative and the comparison between engines as the result.
-#
-# Requires a Paperless API token in PAPERLESS_TOKEN. The API is only reachable
-# from inside the cluster (the namespace policy admits the Authentik gate
-# outpost alone), so requests are issued from within the paperless pod.
-#
-# Usage:
-#   PAPERLESS_TOKEN=... scripts/paperless-eval.sh [-n 100] [--engine both|classic|ai]
-set -uo pipefail
-
-SAMPLE="${PAPERLESS_EVAL_SAMPLE:-100}"
-ENGINE=both
-PG_POD="${PAPERLESS_EVAL_PG_POD:-pprl-pg-2}"
-APP_POD="${PAPERLESS_EVAL_APP_POD:-deploy/paperless}"
-
-while [ $# -gt 0 ]; do
-    case "$1" in
-        -n|--sample) SAMPLE="$2"; shift 2 ;;
-        --engine)    ENGINE="$2"; shift 2 ;;
-        -h|--help)   sed -n '2,40p' "$0"; exit 0 ;;
-        *) echo "unknown argument: $1 (try --help)" >&2; exit 2 ;;
-    esac
-done
-
-if [ -z "${PAPERLESS_TOKEN:-}" ]; then
-    echo "PAPERLESS_TOKEN is not set." >&2
-    echo "Create one for an existing user, then remove it when finished." >&2
-    exit 2
-fi
-
-WORK="$(mktemp -d)"
-trap 'rm -rf "$WORK"' EXIT
-
-# Deterministic sample, so repeated runs and different engines see the same
-# documents. Only documents that carry both labels are useful as ground truth.
-echo "reading $SAMPLE labelled documents from postgres ($PG_POD)..."
-# Names come along with the ids because the AI engine reports an unresolved
-# guess as a name string rather than an id; see the lenient scoring below.
-# Tab-separated rather than '|', which occurs in correspondent names.
-kubectl -n paperless exec "$PG_POD" -- psql -U postgres -d paperless -tA -F$'\t' -c \
-  "select d.id, d.correspondent_id, d.document_type_id, c.name, t.name
-   from documents_document d
-   join documents_correspondent c on c.id = d.correspondent_id
-   join documents_documenttype  t on t.id = d.document_type_id
-   where d.correspondent_id is not null
-     and d.document_type_id is not null
-     and length(d.content) > 400
-   order by md5(d.id::text)
-   limit $SAMPLE;" > "$WORK/docs.txt" 2>"$WORK/pg-err.txt"
-if ! [ -s "$WORK/docs.txt" ]; then
-    echo "no documents returned from postgres." >&2
-    [ -s "$WORK/pg-err.txt" ] && cat "$WORK/pg-err.txt" >&2
-    exit 2
-fi
-N=$(grep -c . "$WORK/docs.txt")
-
-# The floor: always guessing the most frequent label. Without it a number like
-# "70% correct" cannot be read, because a skewed archive gives that away for
-# free — one correspondent holds several hundred of ~3900 documents.
-kubectl -n paperless exec "$PG_POD" -- psql -U postgres -d paperless -tAc \
-  "select max(c) from (select count(*) c from documents_document
-   where correspondent_id is not null group by correspondent_id) x;" \
-  > "$WORK/corr_max.txt" 2>"$WORK/pg-err.txt"
-kubectl -n paperless exec "$PG_POD" -- psql -U postgres -d paperless -tAc \
-  "select count(*) from documents_document where correspondent_id is not null;" \
-  > "$WORK/corr_total.txt" 2>"$WORK/pg-err.txt"
-
-api() { # path -> body on stdout
-    kubectl -n paperless exec "$APP_POD" -c paperless -- \
-        curl -sS -m 300 -H "Authorization: Token $PAPERLESS_TOKEN" \
-        "http://127.0.0.1:8000$1" 2>"$WORK/err.txt"
-}
-
-# A hit under two definitions, because the AI engine can be right in a way the
-# API response does not record as a suggestion:
-#
-#   strict   the true label's id appears in `.correspondents` /
-#            `.document_types`. This is what the UI actually offers, so it is
-#            the number that describes the feature as experienced.
-#
-#   lenient  strict, OR the true label's name is contained wordwise in one of
-#            `.suggested_correspondents` / `.suggested_document_types`.
-#
-# The gap between them is one specific failure: the LLM names the right entity
-# but paperless_ai.matching cannot resolve the name to the existing object,
-# because difflib.get_close_matches runs at MATCH_THRESHOLD = 0.8 and a correct
-# but more verbose answer falls below it — observed with "Generali Deutschland
-# Krankenversicherung AG" against the existing "Generali AG", which yielded an
-# empty `correspondents` list.
-#
-# Reporting only the strict number would blame the model for a matcher problem;
-# reporting only the lenient one would claim a usefulness the UI does not
-# deliver. The classic engine emits no `suggested_*` fields at all, so both
-# numbers coincide for it and the comparison stays fair in either column.
-LENIENT_JQ='
-def norm: ascii_downcase | gsub("[^\\p{L}\\p{N}]+"; " ") | split(" ") | map(select(length > 0));
-def contains_all($want): . as $have | $want | all(. as $t | $have | index([$t]) != null);
-($want_name | norm) as $w
-| ((.[$id_field] // []) | index($want_id) != null)
-  or ([(.[$name_field] // [])[] | norm] | any(contains_all($w)))
-'
-
-# score <engine> <url-suffix>
-score() {
-    local engine="$1" suffix="$2"
-    local n=0 corr_hit=0 type_hit=0 corr_len=0 type_len=0 fails=0 start elapsed
-    echo ""
-    echo "=== $engine"
-    start=$(date +%s)
-    while IFS=$'\t' read -r id want_corr want_type want_corr_name want_type_name; do
-        [ -n "$id" ] || continue
-        n=$((n + 1))
-        body=$(api "/api/documents/$id/$suffix")
-        if ! printf '%s' "$body" | jq -e . >/dev/null 2>&1; then
-            fails=$((fails + 1))
-            [ "$fails" -le 3 ] && printf '    %s: keine gueltige Antwort: %s\n' \
-                "$id" "$(printf '%s' "$body" | tr '\n' ' ' | head -c 120)" >&2
-            continue
-        fi
-        if printf '%s' "$body" | jq -e --argjson w "$want_corr" \
-               '.correspondents // [] | index($w) != null' >/dev/null 2>&1; then
-            corr_hit=$((corr_hit + 1))
-        fi
-        if printf '%s' "$body" | jq -e --argjson w "$want_type" \
-               '.document_types // [] | index($w) != null' >/dev/null 2>&1; then
-            type_hit=$((type_hit + 1))
-        fi
-        if printf '%s' "$body" | jq -e \
-               --argjson want_id "$want_corr" --arg want_name "$want_corr_name" \
-               --arg id_field correspondents --arg name_field suggested_correspondents \
-               "$LENIENT_JQ" >/dev/null 2>&1; then
-            corr_len=$((corr_len + 1))
-        fi
-        if printf '%s' "$body" | jq -e \
-               --argjson want_id "$want_type" --arg want_name "$want_type_name" \
-               --arg id_field document_types --arg name_field suggested_document_types \
-               "$LENIENT_JQ" >/dev/null 2>&1; then
-            type_len=$((type_len + 1))
-        fi
-        [ $((n % 25)) -eq 0 ] && echo "    $n/$N"
-    done < "$WORK/docs.txt"
-    elapsed=$(( $(date +%s) - start ))
-    pct() { awk -v a="$1" -v b="$2" 'BEGIN{if (b>0) printf "%.1f", 100*a/b; else print "0.0"}'; }
-    printf '    correspondent, offered  : %5s%%   recognised: %5s%%\n' \
-        "$(pct "$corr_hit" "$n")" "$(pct "$corr_len" "$n")"
-    printf '    document type, offered  : %5s%%   recognised: %5s%%\n' \
-        "$(pct "$type_hit" "$n")" "$(pct "$type_len" "$n")"
-    printf '    no usable answer        : %s of %s\n' "$fails" "$n"
-    printf '    %ss total, %.1fs per document\n' "$elapsed" "$(awk -v e="$elapsed" -v n="$n" 'BEGIN{print e/n}')"
-    printf '%-9s %7s %7s %7s %7s %6s %8ss\n' "$engine" \
-        "$(pct "$corr_hit" "$n")" "$(pct "$corr_len" "$n")" \
-        "$(pct "$type_hit" "$n")" "$(pct "$type_len" "$n")" \
-        "$fails" "$elapsed" >> "$WORK/summary.txt"
-}
-
-: > "$WORK/summary.txt"
-FLOOR=$(awk -v m="$(cat "$WORK/corr_max.txt")" -v t="$(cat "$WORK/corr_total.txt")" \
-    'BEGIN{if (t>0) printf "%.1f", 100*m/t; else print "?"}')
-echo "documents: $N   most-frequent-correspondent floor: ${FLOOR}%"
-
-case "$ENGINE" in
-    both)    score classic suggestions/; score ai ai_suggestions/ ;;
-    classic) score classic suggestions/ ;;
-    ai)      score ai ai_suggestions/ ;;
-    *) echo "unknown engine: $ENGINE" >&2; exit 2 ;;
-esac
-
-echo ""
-echo "================================================================"
-printf '%-9s %7s %7s %7s %7s %6s %9s\n' \
-    ENGINE CORR CORR+ TYPE TYPE+ FAILS TIME
-cat "$WORK/summary.txt"
-printf '%-9s %7s\n' "floor" "$FLOOR"
-echo "================================================================"
-echo "CORR/TYPE  = true label offered as a resolved id (what the UI shows)"
-echo "CORR+/TYPE+ = also counting names the LLM got right but matching dropped"