chore(paperless-eval): drop a __pycache__ left by a syntax check

Committed by accident from running py_compile against eval.py.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

kubernetes/apps/paperless-eval/eval.py

@@ -0,0 +1,345 @@
+#!/usr/bin/env python3
+"""Measure what Paperless suggests for documents whose labels a human already chose.
+
+Runs both engines against the same documents:
+
+    /api/documents/<id>/suggestions/      the built-in scikit-learn classifier
+    /api/documents/<id>/ai_suggestions/   the LLM path, with RAG
+
+WHAT THIS ANSWERS, AND WHAT AN EARLIER VERSION DID NOT
+
+The two engines are not alternatives — the classifier costs 0.6 s per document
+and keeps running whatever the LLM does. So the useful question is not "which
+scores higher" but "what does the LLM add where the classifier is wrong", and
+that cannot be recovered from two aggregate percentages: 74% and 72% could
+overlap almost entirely or barely at all, and the difference between those two
+worlds is the entire value of running an LLM.
+
+So every document is written out individually, with both engines' verdicts, and
+the summary is derived from that rather than counted along the way.
+
+SCORING
+
+Correspondent and document type are each scored twice:
+
+    offered      the true label's id came back in `correspondents` /
+                 `document_types` — what the UI actually presents
+    recognised   also counting the true label's name appearing wordwise in
+                 `suggested_correspondents` / `suggested_document_types`
+
+The gap between them is one specific failure: the model names the right entity
+but paperless_ai.matching cannot resolve it, because difflib runs at
+MATCH_THRESHOLD = 0.8 and a correct but more verbose answer falls below it —
+"Generali Deutschland Krankenversicherung AG" against the existing "Generali
+AG". Reporting only the first blames the model for a matcher problem; only the
+second claims a usefulness the UI does not deliver.
+
+Document type is additionally split on whether the true type is "Sonstige",
+which holds 1133 of ~3900 documents and is the one label with no semantic
+content — a restaurant reservation lands there because nothing else fits. The
+classifier can memorise that; a model reading the document has to infer "none
+of the above". Scored separately rather than dropped, because the model does
+sometimes get it right.
+
+GROUND TRUTH
+
+The current filing, which deserves care: over ten years conventions change, and
+a disagreement with an old document is not automatically an error. Treat the
+comparison between engines as the result and the absolute numbers as
+indicative.
+
+Env:
+    PAPERLESS_URL     default http://127.0.0.1:8000
+    PGHOST/PGDATABASE standard libpq, for ground truth and the API token
+    EVAL_SAMPLE       documents to score, default 50
+    EVAL_ENGINES      comma-separated subset of classic,ai — default both
+    EVAL_LABEL        free-text name for this configuration, recorded in the
+                      output so results from different jobs stay distinguishable
+    EVAL_OUT          directory for results, default /results
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import re
+import secrets
+import sys
+import time
+import urllib.error
+import urllib.request
+
+import psycopg
+
+PAPERLESS_URL = os.environ.get("PAPERLESS_URL", "http://127.0.0.1:8000")
+SAMPLE = int(os.environ.get("EVAL_SAMPLE", "50"))
+ENGINES = os.environ.get("EVAL_ENGINES", "classic,ai").split(",")
+LABEL = os.environ.get("EVAL_LABEL", "unlabelled")
+OUT_DIR = os.environ.get("EVAL_OUT", "/results")
+
+# Long enough that a slow document is measured rather than abandoned. A client
+# timeout does not stop the generation behind it, and with a single llama.cpp
+# slot an abandoned request holds the queue for everything after it.
+REQUEST_TIMEOUT = 900
+
+CATCHALL_TYPE = "Sonstige"
+
+
+def log(msg: str) -> None:
+    print(f"[eval] {msg}", flush=True)
+
+
+def api_token(conn) -> str:
+    """A token for this run, created in the database.
+
+    The API needs one and this database is a disposable clone, so the run
+    issues its own against a superuser rather than depending on a token that
+    happened to come across in the restore.
+    """
+    token = secrets.token_hex(20)
+    with conn.cursor() as cur:
+        cur.execute(
+            "select id from auth_user where is_superuser and is_active order by id limit 1",
+        )
+        row = cur.fetchone()
+        if row is None:
+            raise SystemExit("no active superuser in the restored database")
+        user_id = row[0]
+        cur.execute("delete from authtoken_token where user_id = %s", (user_id,))
+        cur.execute(
+            "insert into authtoken_token (key, created, user_id) values (%s, now(), %s)",
+            (token, user_id),
+        )
+    conn.commit()
+    return token
+
+
+def sample(conn) -> list[dict]:
+    """The documents to score, ordered by md5 of the id.
+
+    Deterministic, so every engine and every configuration sees the same
+    documents and results stay comparable across jobs. Only documents carrying
+    both labels are usable as ground truth.
+    """
+    with conn.cursor() as cur:
+        cur.execute(
+            """
+            select d.id, d.correspondent_id, d.document_type_id, c.name, t.name
+              from documents_document d
+              join documents_correspondent c on c.id = d.correspondent_id
+              join documents_documenttype  t on t.id = d.document_type_id
+             where d.correspondent_id is not null
+               and d.document_type_id is not null
+               and length(d.content) > 400
+             order by md5(d.id::text)
+             limit %s
+            """,
+            (SAMPLE,),
+        )
+        return [
+            {
+                "id": r[0],
+                "correspondent_id": r[1],
+                "document_type_id": r[2],
+                "correspondent_name": r[3],
+                "document_type_name": r[4],
+            }
+            for r in cur.fetchall()
+        ]
+
+
+def floor(conn) -> float:
+    """Always guessing the most frequent correspondent.
+
+    Without it a number like "70% correct" cannot be read: one correspondent
+    holds several hundred of ~3900 documents.
+    """
+    with conn.cursor() as cur:
+        cur.execute(
+            """
+            select max(c), sum(c) from (
+                select count(*) c from documents_document
+                 where correspondent_id is not null group by correspondent_id
+            ) x
+            """,
+        )
+        top, total = cur.fetchone()
+    return 100.0 * top / total if total else 0.0
+
+
+def fetch(path: str, token: str) -> tuple[dict | None, str | None, float]:
+    req = urllib.request.Request(
+        f"{PAPERLESS_URL}{path}",
+        headers={"Authorization": f"Token {token}"},
+    )
+    start = time.monotonic()
+    try:
+        with urllib.request.urlopen(req, timeout=REQUEST_TIMEOUT) as resp:
+            body = resp.read()
+        return json.loads(body), None, time.monotonic() - start
+    except urllib.error.HTTPError as err:
+        return None, f"HTTP {err.code}: {err.read()[:200]!r}", time.monotonic() - start
+    except Exception as err:  # noqa: BLE001 — a failed document is data, not a crash
+        return None, f"{type(err).__name__}: {err}", time.monotonic() - start
+
+
+def normalise(text: str) -> set[str]:
+    return {w for w in re.split(r"[^\w]+", text.lower()) if w}
+
+
+def recognised(payload: dict, field: str, want_name: str) -> bool:
+    """True when a suggested name contains every word of the true label's name.
+
+    Handles the observed failure directly: the model answers with a superset of
+    the stored name, which difflib rejects at 0.8 but a reader would accept.
+    """
+    want = normalise(want_name)
+    return any(want <= normalise(name) for name in payload.get(field) or [])
+
+
+def score_document(payload: dict, doc: dict) -> dict:
+    offered_corr = doc["correspondent_id"] in (payload.get("correspondents") or [])
+    offered_type = doc["document_type_id"] in (payload.get("document_types") or [])
+    return {
+        "correspondent_offered": offered_corr,
+        "correspondent_recognised": offered_corr
+        or recognised(payload, "suggested_correspondents", doc["correspondent_name"]),
+        "type_offered": offered_type,
+        "type_recognised": offered_type
+        or recognised(payload, "suggested_document_types", doc["document_type_name"]),
+        "suggested_correspondents": payload.get("suggested_correspondents") or [],
+        "suggested_document_types": payload.get("suggested_document_types") or [],
+        "title": payload.get("title"),
+    }
+
+
+def run(engine: str, suffix: str, docs: list[dict], token: str) -> list[dict]:
+    log(f"=== {engine}: {len(docs)} documents")
+    rows = []
+    for n, doc in enumerate(docs, 1):
+        payload, error, elapsed = fetch(
+            f"/api/documents/{doc['id']}/{suffix}",
+            token,
+        )
+        row = {
+            "engine": engine,
+            "document_id": doc["id"],
+            "seconds": round(elapsed, 1),
+            "truth_correspondent": doc["correspondent_name"],
+            "truth_type": doc["document_type_name"],
+            "catchall_type": doc["document_type_name"] == CATCHALL_TYPE,
+        }
+        if payload is None:
+            row["error"] = error
+            log(f"  {doc['id']}: {error}")
+        else:
+            row.update(score_document(payload, doc))
+        rows.append(row)
+        if n % 10 == 0:
+            log(f"  {n}/{len(docs)}")
+    return rows
+
+
+def pct(hits: int, total: int) -> float:
+    return round(100.0 * hits / total, 1) if total else 0.0
+
+
+def summarise(rows: list[dict], engine: str) -> dict:
+    scored = [r for r in rows if r["engine"] == engine and "error" not in r]
+    failed = [r for r in rows if r["engine"] == engine and "error" in r]
+    specific = [r for r in scored if not r["catchall_type"]]
+    catchall = [r for r in scored if r["catchall_type"]]
+    return {
+        "engine": engine,
+        "scored": len(scored),
+        "failed": len(failed),
+        "seconds_total": round(
+            sum(r["seconds"] for r in rows if r["engine"] == engine),
+        ),
+        "correspondent_offered": pct(
+            sum(r["correspondent_offered"] for r in scored), len(scored)
+        ),
+        "correspondent_recognised": pct(
+            sum(r["correspondent_recognised"] for r in scored), len(scored)
+        ),
+        "type_offered": pct(sum(r["type_offered"] for r in scored), len(scored)),
+        "type_specific": pct(
+            sum(r["type_offered"] for r in specific), len(specific)
+        ),
+        "type_catchall": pct(
+            sum(r["type_offered"] for r in catchall), len(catchall)
+        ),
+    }
+
+
+def complement(rows: list[dict]) -> dict | None:
+    """What the LLM adds where the classifier is wrong, and what it costs.
+
+    The reason this script writes per-document rows at all. Only computed when
+    both engines ran, and only over documents both actually scored.
+    """
+    by_engine = {"classic": {}, "ai": {}}
+    for row in rows:
+        if "error" not in row and row["engine"] in by_engine:
+            by_engine[row["engine"]][row["document_id"]] = row
+    both = set(by_engine["classic"]) & set(by_engine["ai"])
+    if not both or not by_engine["ai"]:
+        return None
+
+    rescued = missed = broken = agreed = 0
+    for doc_id in both:
+        classic_hit = by_engine["classic"][doc_id]["correspondent_offered"]
+        ai_hit = by_engine["ai"][doc_id]["correspondent_offered"]
+        if classic_hit and ai_hit:
+            agreed += 1
+        elif not classic_hit and ai_hit:
+            rescued += 1
+        elif classic_hit and not ai_hit:
+            broken += 1
+        else:
+            missed += 1
+    return {
+        "documents": len(both),
+        "both_right": agreed,
+        "llm_rescued": rescued,
+        "llm_only_wrong": broken,
+        "both_wrong": missed,
+    }
+
+
+def main() -> int:
+    os.makedirs(OUT_DIR, exist_ok=True)
+    with psycopg.connect() as conn:
+        token = api_token(conn)
+        docs = sample(conn)
+        guess_floor = floor(conn)
+    log(f"label={LABEL} documents={len(docs)} floor={guess_floor:.1f}%")
+
+    rows: list[dict] = []
+    for engine, suffix in (("classic", "suggestions/"), ("ai", "ai_suggestions/")):
+        if engine in ENGINES:
+            rows.extend(run(engine, suffix, docs, token))
+
+    result = {
+        "label": LABEL,
+        "sample": len(docs),
+        "floor": round(guess_floor, 1),
+        "summaries": [summarise(rows, e) for e in ENGINES if e in ("classic", "ai")],
+        "complement": complement(rows),
+    }
+
+    with open(f"{OUT_DIR}/{LABEL}.jsonl", "w") as handle:
+        for row in rows:
+            handle.write(json.dumps(row, ensure_ascii=False) + "\n")
+    with open(f"{OUT_DIR}/{LABEL}.summary.json", "w") as handle:
+        json.dump(result, handle, ensure_ascii=False, indent=2)
+
+    # Also to stdout, so the numbers are in the pod log and reachable via Loki
+    # without mounting the volume.
+    log(json.dumps(result, ensure_ascii=False, indent=2))
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

kubernetes/apps/paperless-eval/job-qwen35.yaml

@@ -0,0 +1,283 @@
+---
+# One configuration under test, complete in one pod.
+#
+# A paperless instance can only point at one chat endpoint, so measuring two
+# candidates at once means two paperless instances — which makes the pod, not
+# the namespace, the right unit. Everything a measurement needs is here except
+# the database and the embedding endpoint, both of which are read-only and
+# shared.
+#
+# The three services are native sidecars (initContainers with
+# restartPolicy: Always), not regular containers. That matters twice: they
+# start in order and each must pass its probes before the next begins, which
+# is what keeps paperless from racing the database; and when the eval
+# container exits they are torn down, so the Job actually completes instead of
+# running forever behind a llama-server that never returns.
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: eval-qwen35
+  namespace: paperless-eval
+  labels:
+    app.kubernetes.io/name: paperless-eval
+    app.kubernetes.io/component: eval
+spec:
+  # A failed run is re-read, not retried blind: a retry would spend another
+  # hour reproducing whatever went wrong.
+  backoffLimit: 0
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: paperless-eval
+        app.kubernetes.io/component: eval
+    spec:
+      restartPolicy: Never
+      enableServiceLinks: false
+      # One measurement per node. Two jobs sharing a node would share its iGPU,
+      # and every timing in the result would then describe contention rather
+      # than the model — which is exactly how a decode rate of 4.7 tokens/s got
+      # attributed to context length once already.
+      affinity:
+        podAntiAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            - labelSelector:
+                matchLabels:
+                  app.kubernetes.io/component: eval
+              topologyKey: kubernetes.io/hostname
+      securityContext:
+        fsGroup: 1000
+        fsGroupChangePolicy: OnRootMismatch
+        seccompProfile:
+          type: RuntimeDefault
+      initContainers:
+        # A private copy of /data — the search index and, the expensive part,
+        # the LLM index the RAG path needs.
+        #
+        # It cannot be mounted live and shared: LLM_INDEX_DIR is hardcoded to
+        # DATA_DIR / "llm_index", the store behind it is SQLite, and SQLite
+        # locking across pods over CephFS is not dependable. Nor can it be left
+        # out — an empty llm_index makes retrieve_similar_nodes return nothing,
+        # and the job would quietly measure the no-RAG path while reporting a
+        # RAG result.
+        #
+        # ~160 MiB, so seconds. A regular initContainer: it must finish before
+        # the sidecars start.
+        - name: seed-data
+          image: ghcr.io/paperless-ngx/paperless-ngx:3.1.2
+          command:
+            - sh
+            - -c
+            - |
+              set -e
+              cp -a /golden/. /data/
+              echo "seeded /data:"
+              du -sh /data/index /data/llm_index 2>/dev/null || true
+          volumeMounts:
+            - name: golden
+              mountPath: /golden
+              readOnly: true
+            - name: data
+              mountPath: /data
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: ["ALL"]
+        - name: llama-server
+          image: ghcr.io/ggml-org/llama.cpp:server-vulkan-b10731
+          restartPolicy: Always
+          args:
+            - --hf-repo
+            - ggml-org/Qwen3.6-35B-A3B-GGUF:Q4_K_M
+            - --alias
+            - qwen3.6-35b-a3b
+            - --n-gpu-layers
+            - "99"
+            - --threads
+            - "4"
+            # The production ceiling, and for the same reason: 32768 per slot
+            # corrupts inference on this Vulkan stack — 1025 tokens of "!"
+            # against a 650-token prompt, with /props still reporting the
+            # requested size. A candidate needing more context than this cannot
+            # be measured here, and that is itself a result.
+            - --ctx-size
+            - "16384"
+            - --parallel
+            - "1"
+            - --ubatch-size
+            - "1024"
+            - --no-webui
+          env:
+            - name: LLAMA_CACHE
+              value: /models
+          ports:
+            - name: llm
+              containerPort: 8080
+          volumeMounts:
+            - name: models
+              mountPath: /models
+          # ~20 GB of weights off CephFS, so a generous budget: 60 * 20s.
+          startupProbe:
+            httpGet:
+              path: /health
+              port: llm
+            periodSeconds: 20
+            failureThreshold: 60
+          resources:
+            requests:
+              cpu: "1"
+              memory: 24Gi
+              gpu.intel.com/i915: "1"
+            limits:
+              memory: 32Gi
+              gpu.intel.com/i915: "1"
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: ["ALL"]
+        - name: redis
+          # Its own, not the namespace's: two paperless instances sharing a
+          # celery broker would take each other's tasks.
+          image: redis:8.10.1-alpine
+          restartPolicy: Always
+          args: ["redis-server", "--save", "", "--loglevel", "warning"]
+          startupProbe:
+            tcpSocket:
+              port: 6379
+            periodSeconds: 2
+            failureThreshold: 30
+          resources:
+            requests:
+              cpu: 20m
+              memory: 64Mi
+            limits:
+              memory: 256Mi
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop: ["ALL"]
+        - name: paperless
+          image: ghcr.io/paperless-ngx/paperless-ngx:3.1.2
+          restartPolicy: Always
+          envFrom:
+            - configMapRef:
+                name: paperless-eval-config
+          env:
+            # Overrides the namespace ConfigMap: this pod's own llama-server,
+            # and its own redis rather than the shared Service.
+            - name: PAPERLESS_AI_LLM_ENDPOINT
+              value: http://127.0.0.1:8080/v1
+            - name: PAPERLESS_AI_LLM_MODEL
+              value: qwen3.6-35b-a3b
+            - name: PAPERLESS_REDIS
+              value: redis://127.0.0.1:6379
+            - name: PAPERLESS_SECRET_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: paperless-eval-secrets
+                  key: secret-key
+            - name: PAPERLESS_DBUSER
+              valueFrom:
+                secretKeyRef:
+                  name: pprleval-pg-app
+                  key: username
+            - name: PAPERLESS_DBPASS
+              valueFrom:
+                secretKeyRef:
+                  name: pprleval-pg-app
+                  key: password
+          ports:
+            - name: http
+              containerPort: 8000
+          volumeMounts:
+            - name: data
+              mountPath: /usr/src/paperless/data
+            - name: media
+              mountPath: /usr/src/paperless/media
+            - name: consume
+              mountPath: /usr/src/paperless/consume
+          startupProbe:
+            httpGet:
+              path: /accounts/login/
+              port: http
+            periodSeconds: 20
+            failureThreshold: 180
+          # Same as production, and required: the entrypoint starts as root,
+          # chowns the volumes and drops to uid 1000 via setgid/setuid.
+          securityContext:
+            runAsNonRoot: false
+            allowPrivilegeEscalation: true
+            capabilities:
+              drop: ["ALL"]
+              add: ["CHOWN", "DAC_OVERRIDE", "FOWNER", "SETGID", "SETUID"]
+          resources:
+            requests:
+              cpu: 200m
+              memory: 512Mi
+            limits:
+              memory: 2Gi
+      containers:
+        - name: eval
+          # The paperless image again, for psycopg and a python that matches
+          # the application — not because the script needs paperless itself.
+          image: ghcr.io/paperless-ngx/paperless-ngx:3.1.2
+          command: ["python3", "/eval/eval.py"]
+          env:
+            - name: EVAL_LABEL
+              value: qwen35
+            - name: EVAL_SAMPLE
+              value: "50"
+            - name: PGHOST
+              value: pprleval-pg-rw
+            - name: PGDATABASE
+              value: paperless
+            - name: PGUSER
+              valueFrom:
+                secretKeyRef:
+                  name: pprleval-pg-app
+                  key: username
+            - name: PGPASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: pprleval-pg-app
+                  key: password
+          volumeMounts:
+            - name: script
+              mountPath: /eval
+            - name: results
+              mountPath: /results
+          resources:
+            requests:
+              cpu: 100m
+              memory: 256Mi
+            limits:
+              memory: 512Mi
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: ["ALL"]
+      volumes:
+        - name: models
+          persistentVolumeClaim:
+            claimName: llama-models
+        - name: results
+          persistentVolumeClaim:
+            claimName: paperless-eval-results
+        - name: script
+          configMap:
+            name: paperless-eval-script
+        # The namespace's /data, mounted read-only and copied out by seed-data
+        # above. Never mounted as the live /data of a job.
+        - name: golden
+          persistentVolumeClaim:
+            claimName: paperless-eval-golden
+            readOnly: true
+        # paperless insists on all three paths existing. media and consume stay
+        # empty — nothing is consumed during a measurement — while data is
+        # seeded with the copy.
+        - name: data
+          emptyDir: {}
+        - name: media
+          emptyDir: {}
+        - name: consume
+          emptyDir: {}

kubernetes/apps/paperless-eval/kustomization.yaml

@@ -7,3 +7,11 @@ resources:
   - pvcs.yaml
   - redis.yaml
   - paperless.yaml
+  - job-qwen35.yaml
+configMapGenerator:
+  # Hash-suffixed, so editing eval.py produces a new ConfigMap name and any Job
+  # referencing it has to be recreated rather than silently running the old
+  # script against a new name.
+  - name: paperless-eval-script
+    files:
+      - eval.py

kubernetes/apps/paperless-eval/paperless.yaml

@@ -165,14 +165,24 @@ spec:
               memory: 512Mi
             limits:
               memory: 2Gi
+          # Production's context verbatim, and it has to be: the paperless-ngx
+          # entrypoint starts as root, chowns the mounted data/media/consume
+          # volumes and then drops to uid 1000 via setgid/setuid. Without these
+          # the same image fails that sequence with EPERM.
+          #
+          # It bites harder here than in production, where those volumes were
+          # chowned once years ago and keep their ownership. These are fresh on
+          # every rebuild, so the chown is not a no-op.
           securityContext:
-            allowPrivilegeEscalation: false
+            runAsNonRoot: false
+            allowPrivilegeEscalation: true
             capabilities:
               drop: ["ALL"]
+              add: ["CHOWN", "DAC_OVERRIDE", "FOWNER", "SETGID", "SETUID"]
       volumes:
         - name: data
           persistentVolumeClaim:
-            claimName: paperless-eval-data
+            claimName: paperless-eval-golden
         - name: media
           persistentVolumeClaim:
             claimName: paperless-eval-media

kubernetes/apps/paperless-eval/pvcs.yaml

@@ -16,14 +16,21 @@
 apiVersion: v1
 kind: PersistentVolumeClaim
 metadata:
-  name: paperless-eval-data
+  # Named for its role rather than its mount point: the smoke-test deployment
+  # builds /data here, and every eval job copies it out. Replaces the earlier
+  # RWO paperless-eval-data, whose access mode could not be widened in place —
+  # a bound claim's spec is immutable.
+  name: paperless-eval-golden
   namespace: paperless-eval
   labels:
     velero.io/exclude-from-backup: "true"
 spec:
+  # RWX, because the eval jobs read it too — each copies it into its own
+  # emptyDir rather than mounting it live. The vector store is SQLite, and
+  # SQLite locking across pods over CephFS is not something to rely on.
   accessModes:
-    - ReadWriteOnce
-  storageClassName: ceph-block-ephemeral
+    - ReadWriteMany
+  storageClassName: ceph-filesystem-ephemeral
   resources:
     requests:
       storage: 10Gi
@@ -64,6 +71,48 @@ spec:
     requests:
       storage: 1Gi
 ---
+# The eval jobs' model cache, RWX so several jobs can read the same weights at
+# once — that is the whole point of measuring two candidates in parallel.
+#
+# Separate from the ai namespace's llama-server-models despite holding some of
+# the same files: PVCs do not cross namespaces. A model therefore downloads
+# once here and is then shared by every job, which is the cost a new candidate
+# carries anyway.
+#
+# NOT ephemeral, unlike everything else in this namespace: re-downloading tens
+# of gigabytes on a slow uplink is a worse outcome than an orphaned volume.
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: llama-models
+  namespace: paperless-eval
+  labels:
+    velero.io/exclude-from-backup: "true"
+spec:
+  accessModes:
+    - ReadWriteMany
+  storageClassName: ceph-filesystem
+  resources:
+    requests:
+      storage: 120Gi
+---
+# Results outlive the jobs that produce them: a Job is deleted and recreated to
+# re-run a configuration, and comparing configurations is the entire exercise.
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: paperless-eval-results
+  namespace: paperless-eval
+  labels:
+    velero.io/exclude-from-backup: "true"
+spec:
+  accessModes:
+    - ReadWriteMany
+  storageClassName: ceph-filesystem
+  resources:
+    requests:
+      storage: 1Gi
+---
 apiVersion: v1
 kind: PersistentVolumeClaim
 metadata:

kubernetes/infrastructure/post-crds/openbao-clustersecretstore.yaml

@@ -27,6 +27,7 @@ spec:
         - ingress
         - monitoring
         - paperless
+        - paperless-eval # same apps/paperless key plus the shared barman S3 creds
         - trmnl
         - velero
         - woodpecker # wp-pg barman creds; woodpecker moving out of `apps` (#433)