Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
  claude-botandClaude Opus 5
|
be27e33735 | ||
|
claude-botandClaude Opus 5
|
ba1125b1e3 |
Executable
+207
@@ -0,0 +1,207 @@
|
||||
#!/usr/bin/env bash
|
||||
# End-to-end evaluation of Paperless document suggestions against the real
|
||||
# archive, comparing the two engines Paperless ships:
|
||||
#
|
||||
# /api/documents/<id>/suggestions/ the built-in scikit-learn classifier,
|
||||
# trained on the existing corpus
|
||||
# /api/documents/<id>/ai_suggestions/ the LLM path, optionally with RAG
|
||||
#
|
||||
# WHY THIS EXISTS
|
||||
#
|
||||
# scripts/embed-eval-real.sh measures whether a document's nearest neighbours
|
||||
# carry its labels. That is a proxy: Paperless does not act on the neighbours
|
||||
# directly, it hands them to an LLM as context, alongside the document text and
|
||||
# a candidate list of existing tags and correspondents. The LLM can override
|
||||
# poor context, or fail with good context. So a better embedding score does not
|
||||
# automatically mean better suggestions, and nothing so far has verified that it
|
||||
# does.
|
||||
#
|
||||
# This measures the thing that is actually wanted: does Paperless suggest the
|
||||
# labels a human already chose? Everything else — MTEB scores, retrieval
|
||||
# accuracy — is only interesting insofar as it predicts this number.
|
||||
#
|
||||
# It also answers a question that predates the LLM work: with ten years of
|
||||
# hand-curated labels, the built-in classifier has excellent training data. It
|
||||
# cannot invent a new tag or a title, which is the LLM's advantage — but for
|
||||
# assigning labels that already exist it may simply be better. Running both on
|
||||
# the same documents settles that rather than assuming it.
|
||||
#
|
||||
# Ground truth is the current filing. Where the two disagree with it, that is
|
||||
# not automatically an error: over ten years filing conventions change, and an
|
||||
# old document may carry an older convention. Treat the absolute numbers as
|
||||
# indicative and the comparison between engines as the result.
|
||||
#
|
||||
# Requires a Paperless API token in PAPERLESS_TOKEN. The API is only reachable
|
||||
# from inside the cluster (the namespace policy admits the Authentik gate
|
||||
# outpost alone), so requests are issued from within the paperless pod.
|
||||
#
|
||||
# Usage:
|
||||
# PAPERLESS_TOKEN=... scripts/paperless-eval.sh [-n 100] [--engine both|classic|ai]
|
||||
set -uo pipefail
|
||||
|
||||
SAMPLE="${PAPERLESS_EVAL_SAMPLE:-100}"
|
||||
ENGINE=both
|
||||
PG_POD="${PAPERLESS_EVAL_PG_POD:-pprl-pg-2}"
|
||||
APP_POD="${PAPERLESS_EVAL_APP_POD:-deploy/paperless}"
|
||||
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
-n|--sample) SAMPLE="$2"; shift 2 ;;
|
||||
--engine) ENGINE="$2"; shift 2 ;;
|
||||
-h|--help) sed -n '2,40p' "$0"; exit 0 ;;
|
||||
*) echo "unknown argument: $1 (try --help)" >&2; exit 2 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [ -z "${PAPERLESS_TOKEN:-}" ]; then
|
||||
echo "PAPERLESS_TOKEN is not set." >&2
|
||||
echo "Create one for an existing user, then remove it when finished." >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
WORK="$(mktemp -d)"
|
||||
trap 'rm -rf "$WORK"' EXIT
|
||||
|
||||
# Deterministic sample, so repeated runs and different engines see the same
|
||||
# documents. Only documents that carry both labels are useful as ground truth.
|
||||
echo "reading $SAMPLE labelled documents from postgres ($PG_POD)..."
|
||||
# Names come along with the ids because the AI engine reports an unresolved
|
||||
# guess as a name string rather than an id; see the lenient scoring below.
|
||||
# Tab-separated rather than '|', which occurs in correspondent names.
|
||||
kubectl -n paperless exec "$PG_POD" -- psql -U postgres -d paperless -tA -F$'\t' -c \
|
||||
"select d.id, d.correspondent_id, d.document_type_id, c.name, t.name
|
||||
from documents_document d
|
||||
join documents_correspondent c on c.id = d.correspondent_id
|
||||
join documents_documenttype t on t.id = d.document_type_id
|
||||
where d.correspondent_id is not null
|
||||
and d.document_type_id is not null
|
||||
and length(d.content) > 400
|
||||
order by md5(d.id::text)
|
||||
limit $SAMPLE;" > "$WORK/docs.txt" 2>"$WORK/pg-err.txt"
|
||||
if ! [ -s "$WORK/docs.txt" ]; then
|
||||
echo "no documents returned from postgres." >&2
|
||||
[ -s "$WORK/pg-err.txt" ] && cat "$WORK/pg-err.txt" >&2
|
||||
exit 2
|
||||
fi
|
||||
N=$(grep -c . "$WORK/docs.txt")
|
||||
|
||||
# The floor: always guessing the most frequent label. Without it a number like
|
||||
# "70% correct" cannot be read, because a skewed archive gives that away for
|
||||
# free — one correspondent holds several hundred of ~3900 documents.
|
||||
kubectl -n paperless exec "$PG_POD" -- psql -U postgres -d paperless -tAc \
|
||||
"select max(c) from (select count(*) c from documents_document
|
||||
where correspondent_id is not null group by correspondent_id) x;" \
|
||||
> "$WORK/corr_max.txt" 2>"$WORK/pg-err.txt"
|
||||
kubectl -n paperless exec "$PG_POD" -- psql -U postgres -d paperless -tAc \
|
||||
"select count(*) from documents_document where correspondent_id is not null;" \
|
||||
> "$WORK/corr_total.txt" 2>"$WORK/pg-err.txt"
|
||||
|
||||
api() { # path -> body on stdout
|
||||
kubectl -n paperless exec "$APP_POD" -c paperless -- \
|
||||
curl -sS -m 300 -H "Authorization: Token $PAPERLESS_TOKEN" \
|
||||
"http://127.0.0.1:8000$1" 2>"$WORK/err.txt"
|
||||
}
|
||||
|
||||
# A hit under two definitions, because the AI engine can be right in a way the
|
||||
# API response does not record as a suggestion:
|
||||
#
|
||||
# strict the true label's id appears in `.correspondents` /
|
||||
# `.document_types`. This is what the UI actually offers, so it is
|
||||
# the number that describes the feature as experienced.
|
||||
#
|
||||
# lenient strict, OR the true label's name is contained wordwise in one of
|
||||
# `.suggested_correspondents` / `.suggested_document_types`.
|
||||
#
|
||||
# The gap between them is one specific failure: the LLM names the right entity
|
||||
# but paperless_ai.matching cannot resolve the name to the existing object,
|
||||
# because difflib.get_close_matches runs at MATCH_THRESHOLD = 0.8 and a correct
|
||||
# but more verbose answer falls below it — observed with "Generali Deutschland
|
||||
# Krankenversicherung AG" against the existing "Generali AG", which yielded an
|
||||
# empty `correspondents` list.
|
||||
#
|
||||
# Reporting only the strict number would blame the model for a matcher problem;
|
||||
# reporting only the lenient one would claim a usefulness the UI does not
|
||||
# deliver. The classic engine emits no `suggested_*` fields at all, so both
|
||||
# numbers coincide for it and the comparison stays fair in either column.
|
||||
LENIENT_JQ='
|
||||
def norm: ascii_downcase | gsub("[^\\p{L}\\p{N}]+"; " ") | split(" ") | map(select(length > 0));
|
||||
def contains_all($want): . as $have | $want | all(. as $t | $have | index([$t]) != null);
|
||||
($want_name | norm) as $w
|
||||
| ((.[$id_field] // []) | index($want_id) != null)
|
||||
or ([(.[$name_field] // [])[] | norm] | any(contains_all($w)))
|
||||
'
|
||||
|
||||
# score <engine> <url-suffix>
|
||||
score() {
|
||||
local engine="$1" suffix="$2"
|
||||
local n=0 corr_hit=0 type_hit=0 corr_len=0 type_len=0 fails=0 start elapsed
|
||||
echo ""
|
||||
echo "=== $engine"
|
||||
start=$(date +%s)
|
||||
while IFS=$'\t' read -r id want_corr want_type want_corr_name want_type_name; do
|
||||
[ -n "$id" ] || continue
|
||||
n=$((n + 1))
|
||||
body=$(api "/api/documents/$id/$suffix")
|
||||
if ! printf '%s' "$body" | jq -e . >/dev/null 2>&1; then
|
||||
fails=$((fails + 1))
|
||||
[ "$fails" -le 3 ] && printf ' %s: keine gueltige Antwort: %s\n' \
|
||||
"$id" "$(printf '%s' "$body" | tr '\n' ' ' | head -c 120)" >&2
|
||||
continue
|
||||
fi
|
||||
if printf '%s' "$body" | jq -e --argjson w "$want_corr" \
|
||||
'.correspondents // [] | index($w) != null' >/dev/null 2>&1; then
|
||||
corr_hit=$((corr_hit + 1))
|
||||
fi
|
||||
if printf '%s' "$body" | jq -e --argjson w "$want_type" \
|
||||
'.document_types // [] | index($w) != null' >/dev/null 2>&1; then
|
||||
type_hit=$((type_hit + 1))
|
||||
fi
|
||||
if printf '%s' "$body" | jq -e \
|
||||
--argjson want_id "$want_corr" --arg want_name "$want_corr_name" \
|
||||
--arg id_field correspondents --arg name_field suggested_correspondents \
|
||||
"$LENIENT_JQ" >/dev/null 2>&1; then
|
||||
corr_len=$((corr_len + 1))
|
||||
fi
|
||||
if printf '%s' "$body" | jq -e \
|
||||
--argjson want_id "$want_type" --arg want_name "$want_type_name" \
|
||||
--arg id_field document_types --arg name_field suggested_document_types \
|
||||
"$LENIENT_JQ" >/dev/null 2>&1; then
|
||||
type_len=$((type_len + 1))
|
||||
fi
|
||||
[ $((n % 25)) -eq 0 ] && echo " $n/$N"
|
||||
done < "$WORK/docs.txt"
|
||||
elapsed=$(( $(date +%s) - start ))
|
||||
pct() { awk -v a="$1" -v b="$2" 'BEGIN{if (b>0) printf "%.1f", 100*a/b; else print "0.0"}'; }
|
||||
printf ' correspondent, offered : %5s%% recognised: %5s%%\n' \
|
||||
"$(pct "$corr_hit" "$n")" "$(pct "$corr_len" "$n")"
|
||||
printf ' document type, offered : %5s%% recognised: %5s%%\n' \
|
||||
"$(pct "$type_hit" "$n")" "$(pct "$type_len" "$n")"
|
||||
printf ' no usable answer : %s of %s\n' "$fails" "$n"
|
||||
printf ' %ss total, %.1fs per document\n' "$elapsed" "$(awk -v e="$elapsed" -v n="$n" 'BEGIN{print e/n}')"
|
||||
printf '%-9s %7s %7s %7s %7s %6s %8ss\n' "$engine" \
|
||||
"$(pct "$corr_hit" "$n")" "$(pct "$corr_len" "$n")" \
|
||||
"$(pct "$type_hit" "$n")" "$(pct "$type_len" "$n")" \
|
||||
"$fails" "$elapsed" >> "$WORK/summary.txt"
|
||||
}
|
||||
|
||||
: > "$WORK/summary.txt"
|
||||
FLOOR=$(awk -v m="$(cat "$WORK/corr_max.txt")" -v t="$(cat "$WORK/corr_total.txt")" \
|
||||
'BEGIN{if (t>0) printf "%.1f", 100*m/t; else print "?"}')
|
||||
echo "documents: $N most-frequent-correspondent floor: ${FLOOR}%"
|
||||
|
||||
case "$ENGINE" in
|
||||
both) score classic suggestions/; score ai ai_suggestions/ ;;
|
||||
classic) score classic suggestions/ ;;
|
||||
ai) score ai ai_suggestions/ ;;
|
||||
*) echo "unknown engine: $ENGINE" >&2; exit 2 ;;
|
||||
esac
|
||||
|
||||
echo ""
|
||||
echo "================================================================"
|
||||
printf '%-9s %7s %7s %7s %7s %6s %9s\n' \
|
||||
ENGINE CORR CORR+ TYPE TYPE+ FAILS TIME
|
||||
cat "$WORK/summary.txt"
|
||||
printf '%-9s %7s\n' "floor" "$FLOOR"
|
||||
echo "================================================================"
|
||||
echo "CORR/TYPE = true label offered as a resolved id (what the UI shows)"
|
||||
echo "CORR+/TYPE+ = also counting names the LLM got right but matching dropped"
|
||||
Reference in New Issue
Block a user