homelab/scripts/paperless-eval/candidates.tsv

1	# label	hf-repo	hf-file	alias	mem-request	mem-limit
2	#
3	# Every candidate must do function calling: paperless calls the model through
4	# chat_with_tools(..., tool_required=True) with DocumentClassifierSchema as a
5	# tool, not via JSON mode. A model whose chat template has no tool support does
6	# not score badly here, it fails outright.
7	#
8	# Instruct variants, never Reasoning ones. Qwen3.6 spent two of fifty documents
9	# generating until something cut it off, and a model tuned to think first does
10	# that by design. The one exception is Qwen3.6 itself, which is measured with
11	# thinking switched off.
12	#
13	#
14	# Ministral-3 8B dense, ~4.7 GB weights 7.1-8.2 tok/s
15	# Qwen3.6 35B-A3B MoE, ~20 GB weights 11 tok/s
16	#
17	# The largest model is the fastest one here. But the bandwidth explanation I
18	# gave for it is wrong, and measuring Q8 disproved it:
19	#
20	# Qwen3.6-35B-A3B Q4_K_M ~21 GB 11.0 tok/s
21	# Qwen3.6-35B-A3B Q8_0 ~34 GB 10.3 tok/s
22	#
23	# Doubling the bytes per active parameter cost ten percent, not half. So decode
24	# is NOT bandwidth-bound for this MoE — it is compute-bound on many small
25	# per-expert matmuls, which also explains the earlier measurement of 21.4 GB/s
26	# effective against a 53.9 GB/s ceiling: it was already leaving 60% of the bus
27	# unused.
28	#
29	# Consequences: the bytes-per-token formula predicts dense models and not this
30	# one, and higher precision is nearly free on an MoE here — about 10% slower
31	# for double the weight precision.
32	#
33	# So the order to prefer is: MoE with few active parameters, then dense 3-9B,
34	# then dense 10-14B only for an exceptional model (~4 tok/s, roughly three
35	# hours for 50 documents), and nothing dense above 14B.
36	qwen35	ggml-org/Qwen3.6-35B-A3B-GGUF	Qwen3.6-35B-A3B-Q4_K_M.gguf	qwen3.6-35b-a3b	24Gi	32Gi
37	#
38	# The same model at higher precision, to find out what Q4 costs. Everything in
39	# this file is Q4_K_M, so the whole comparison could be sitting on quantisation
40	# damage nobody has measured — and there is a concrete suspicion, not just a
41	# worry: Ministral-3's leak of [THINK] and [TOOL_CALLS] as literal text is the
42	#
43	# ~34 GB resident against Q4's ~21, and roughly 6 tok/s against 11, so about
44	# double the wall clock. Read the result against the noise floor in README.md:
45	# below about two documents of difference, this says nothing.
46	qwen35-q8	ggml-org/Qwen3.6-35B-A3B-GGUF	Qwen3.6-35B-A3B-Q8_0.gguf	qwen3.6-35b-a3b-q8	40Gi	48Gi
47	#
48	# --- MoE, few active parameters: the shape that wins on this hardware ---
49	qwen3-30b-a3b	bartowski/Qwen_Qwen3-30B-A3B-Instruct-2507-GGUF	Qwen_Qwen3-30B-A3B-Instruct-2507-Q4_K_M.gguf	qwen3-30b-a3b	22Gi	28Gi
50	nemotron3-30b-a3b	bartowski/nvidia_Nemotron-3-Nano-30B-A3B-GGUF	nvidia_Nemotron-3-Nano-30B-A3B-Q4_K_M.gguf	nemotron3-30b-a3b	28Gi	34Gi
51	glm47-flash	bartowski/zai-org_GLM-4.7-Flash-GGUF	zai-org_GLM-4.7-Flash-Q4_K_M.gguf	glm-4.7-flash	20Gi	26Gi
52	gemma4-26b-a4b	unsloth/gemma-4-26B-A4B-it-GGUF	gemma-4-26B-A4B-it-UD-Q4_K_M.gguf	gemma-4-26b-a4b	20Gi	26Gi
53	qwen35-35b-a3b	bartowski/Qwen_Qwen3.5-35B-A3B-GGUF	Qwen_Qwen3.5-35B-A3B-Q4_K_M.gguf	qwen3.5-35b-a3b	24Gi	30Gi
54	nemotron35-30b-a3b	bartowski/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-GGUF	NVIDIA-Nemotron-3.5-Lightning-30B-A3B-Q4_K_M.gguf	nemotron3.5-30b-a3b	28Gi	34Gi
55	#
56	# --- dense, small, with German named in the model's own card ---
57	glm4-9b	bartowski/glm-4-9b-chat-GGUF	glm-4-9b-chat-Q4_K_M.gguf	glm-4-9b	10Gi	14Gi
58	commandr7b	bartowski/c4ai-command-r7b-12-2024-GGUF	c4ai-command-r7b-12-2024-Q4_K_M.gguf	command-r7b	10Gi	14Gi
59	qwen25-7b	bartowski/Qwen2.5-7B-Instruct-GGUF	Qwen2.5-7B-Instruct-Q4_K_M.gguf	qwen2.5-7b	10Gi	14Gi
60	llama31-8b	bartowski/Meta-Llama-3.1-8B-Instruct-GGUF	Meta-Llama-3.1-8B-Instruct-Q4_K_M.gguf	llama-3.1-8b	10Gi	14Gi
61	nemotron-nano-12b	bartowski/nvidia_NVIDIA-Nemotron-Nano-12B-v2-GGUF	nvidia_NVIDIA-Nemotron-Nano-12B-v2-Q4_K_M.gguf	nemotron-nano-12b	12Gi	16Gi
62	phi4-mini	bartowski/microsoft_Phi-4-mini-instruct-GGUF	microsoft_Phi-4-mini-instruct-Q4_K_M.gguf	phi-4-mini	8Gi	12Gi
63	#
64	# --- dense, small, no German claim but no thinking risk either ---
65	qwen3-4b-2507	bartowski/Qwen_Qwen3-4B-Instruct-2507-GGUF	Qwen_Qwen3-4B-Instruct-2507-Q4_K_M.gguf	qwen3-4b	8Gi	12Gi
66	qwen35-9b	bartowski/Qwen_Qwen3.5-9B-GGUF	Qwen_Qwen3.5-9B-Q4_K_M.gguf	qwen3.5-9b	10Gi	14Gi
67	#
68	# WHAT THE RESEARCH RULED OUT, so nobody re-derives it
69	#
70	# Hard-excluded on tool calling, chat template inspected directly rather than
71	# Gemma 3, every size (4B/12B/27B) phi-4 14B (the mini DOES have it)
72	# Cohere Aya, all variants InternLM 2.5 / 3
73	#
74	# Hard-excluded on German, stated by the vendor rather than merely missing:
75	# EXAONE — the small ones are English/Korean; the ones with German are 33B+
76	#
77	# German fine-tunes are a dead end, and for one consistent reason: SauerkrautLM,
78	# DiscoLM, LeoLM, Occiglot and Teuken all overwrite their base model's chat
79	# template with a minimal ChatML loop that drops tool calling. VAGOsolutions
80	# does this even on bases that have it (Llama-3.1, Qwen2.5). Assume any German
81	# tune is tool-incapable until its own tokenizer_config.json says otherwise.
82	#
83	# On llama.cpp's tool parsing, which explains the Granite failure: current
84	# builds have no per-vendor parser table any more. common/chat.cpp exposes
85	# PEG_SIMPLE / PEG_NATIVE / PEG_GEMMA4 / PEG_MINIMAX_M3 and derives the grammar
86	# from the model's own template. Hermes-style <tool_call> JSON is a first-class
87	# tested fixture in tests/test-chat.cpp; Granite's format is not. So this is not
88	#
89	# SCREENING RESULTS, and a pattern that reframes the whole search
90	#
91	# qwen35 MoE 35B total / 3B active works, 2 of 50 failed
92	# qwen3-8b dense 8B 4 of 5 failed
93	# hermes4-14b dense 14B 3 of 3 failed, aborted
94	# ministral8b dense 8B 3 of 3 failed, aborted
95	# ministral3b dense 3B 29 of 50 failed
96	# granite8b dense 8B every document failed
97	#
98	# Five dense models from 3B to 14B cannot produce a valid call against
99	# paperless's 14-field schema; the one mixture-of-experts model can. Hermes is
100	# the cleanest of these failures: its <tool_call> JSON is an explicitly tested
101	# fixture in llama.cpp's own test suite, so the parser demonstrably handles the
102	# format and the failure has to be the model.
103	#
104	# Working hypothesis: holding a long grammar-constrained structure needs model
105	# capacity, and what supplies it is TOTAL parameters, not active ones. If that
106	# that the fastest shape on this hardware, MoE with few active parameters, is
107	# also the only one that works.
108	#
109	# qwen3-30b-a3b tests it directly: 30B total, 3B active.
110	#
111	# CORRECTION: Gemma is not excluded after all. Gemma *3* has no tool support —
112	# directly. Gemma *4* 26B-A4B does have it, documented in the template, plus
113	# previous generation was sloppy.
114	#
115	# Two of these carry a Vulkan risk worth checking in the llama-server log
116	# before trusting a run: Nemotron-3.5 interleaves real Mamba2 layers and
117	# Qwen3.5 uses DeltaNet linear attention, and llama.cpp's Vulkan SSM shaders
118	# are less exercised on Iris Xe than on Arc. Issue #19957 reported exactly this
119	# for Qwen3.5-35B-A3B and it turned out to be a stale build, fixed months
120	# before b10731 — but confirm layers are actually offloaded rather than
121	# silently falling back to CPU, which would read as a slow model.
122	#
123	# Ruled out after checking: ERNIE-4.5-21B-A3B (only the Thinking variant has
124	# tool calling; the instruct one has none), Ling-3.0-flash (124B total),
125	# Granite-4.0-H-Small (9B active), DeepSeek-V2-Lite and OLMoE (under 20B
126	# total), Cohere North Mini (code-only, no ggml-org/bartowski conversion,
127	# cohere2_moe support in llama.cpp unverified), gpt-oss-20b (a reasoning-effort
128	# model in Harmony format rather than plain instruct).
129	#
130	# THE MoE HYPOTHESIS IS DEAD. qwen3-30b-a3b — MoE, 30B total, 3B active, the
131	# shape that was supposed to explain everything — scored 1 of 5 and failed 4,
132	# indistinguishable from the dense 8B.
133	#
134	# So neither architecture nor size predicts success:
135	#
136	# Qwen3.6-35B-A3B MoE 35B/3B Apr 2026 2 of 50 failed
137	# Qwen3-30B-A3B MoE 30B/3B older 4 of 5 failed
138	# Qwen3-8B dense 8B older 4 of 5 failed
139	# Hermes-4-14B dense 14B — 3 of 3 failed
140	#
141	# The only thing correlating with success so far is how new and how strong the
142	# model is. That is an unsatisfying hypothesis because it predicts nothing
143	# cheaply — but it is what the data says, and two architectural theories have
144	# already died here.
145	#
146	# qwen35-35b-a3b isolates it: same architecture, same total and active
147	# parameters as the working baseline, one generation older. If it fails, the
148	# generation is the variable and shape is noise.
149	#
150	# qwen35-35b-a3b PASSES the screen: 5 of 5, no failures, 584 s. So the
151	# generation hypothesis is dead too — Qwen3.5 is a generation OLDER than the
152	# working Qwen3.6 and handles the schema cleanly.
153	#
154	# What the two working models share is 35B total parameters; the one failing
155	# MoE has 30B. But that is confounded — Qwen3-30B-A3B is both smaller and
156	# older, and three data points cannot separate the two.
157	#
158	# Qwen3.6-35B-A3B 35B total newest works
159	# Qwen3.5-35B-A3B 35B total older works
160	# Qwen3-30B-A3B 30B total oldest 4 of 5 failed
161	#
162	# glm47-flash separates them: 30B total, newest generation. If it works,
163	# generation is what matters and 35B is not a threshold. If it fails, total
164	# size is, and the search should stop looking below ~35B.
165	#
166	# PREFILL SPEED IS A SELECTION CRITERION AND I WAS NOT TRACKING IT.
167	# Measured on this hardware, prompt processing varies more than sixfold across
168	# architectures, and it is roughly half of a run's wall clock:
169	#
170	# dense 8B (Ministral) 240 tok/s
171	# Qwen3.6-35B-A3B 100 tok/s
172	# GLM-4.7-Flash 31 tok/s decode 4.9 tok/s
173	#
174	# GLM-4.7-Flash is disqualified on that alone. At 31 tok/s a 5300-token prompt
175	# takes 168 s, times ~2.6 requests per document, is over seven minutes per
176	# document and about six hours for a run — before considering whether its
177	# answers are any good.
178	#
179	# It also shows why the 300 s REQUEST_TIMEOUT needs care: 4453 prompt tokens at
180	# 31.5 tok/s plus 768 generated at 4.9 tok/s is 298 s, so a slow model turns a
181	# content failure into a TimeoutError and muddies the diagnosis. Read the
182	# llama-server timings, not just the eval error.
183	#
184	# glm47-flash: aborted, 3 of 3 — but every one a TimeoutError, not an invalid
185	# answer, so this does NOT settle the size-versus-generation question it was
186	# started to settle. It failed on speed. Document 1 did reach the --n-predict
187	# cap without producing a call, but 2 and 3 were cut off by the 300 s client
188	# bound before anything could be concluded about them.
189	#
190	# Disqualified regardless: 31 tok/s prefill and 4.9 tok/s decode is over seven
191	# minutes per document and about six hours for a full run.
192	#
193	# gemma4-26b-a4b is the better discriminator — 26B total, well under 35, and
194	# newest generation. Works means generation and quality matter rather than a
195	# 35B threshold; fails is consistent with total size being the line.
196	#
197	# gemma4-26b-a4b: aborted, 3 of 3, all HTTP 400 rather than timeouts, and the
198	# server log shows it ran to the --n-predict cap on every one. So it is the
199	# same capability failure as the small dense models, not a template mismatch.
200	# (Prefill 45-50 tok/s, decode 6 tok/s — also not fast.)
201	#
202	# THE SIZE THRESHOLD SURVIVED ITS FIRST REAL TEST. gemma-4 was chosen
203	# specifically because it could refute it — newest generation, 26B total — and
204	# it did not:
205	#
206	# Qwen3.6-35B-A3B 35B newest works
207	# Qwen3.5-35B-A3B 35B older works
208	# Qwen3-30B-A3B 30B oldest fails
209	# gemma-4-26B-A4B 26B NEWEST fails
210	# Hermes-4-14B 14B — fails
211	# dense 3-8B — — fail
212	#
213	# Two generations work at 35B total; everything at 30B and below fails,
214	# including the newest model in the field. Operating rule until something
215	# contradicts it: do not spend a screening slot on anything under ~35B total.
216	#
217	# Not pinned further on purpose. Whether the line sits at 31B or 34B is
218	# interesting and not decision-relevant — there are two working candidates and
219	# the useful next number is a full 50-document run of the second one.
220	#
221	# Re-enabled for length probing, not for scoring. Their classification numbers
222	# were taken against a slot that may already have been corrupted, so the only
223	# honest thing to do with them is find out where each one's boundary sits.
224	hermes4-14b	bartowski/NousResearch_Hermes-4-14B-GGUF	NousResearch_Hermes-4-14B-Q4_K_M.gguf	hemes-4-14b	14Gi	20Gi
225	qwen3-8b	bartowski/Qwen_Qwen3-8B-GGUF	Qwen_Qwen3-8B-Q4_K_M.gguf	qwen3-8b	12Gi	16Gi
226	ministral8b	bartowski/mistralai_Ministral-3-8B-Instruct-2512-GGUF	mistralai_Ministral-3-8B-Instruct-2512-Q4_K_M.gguf	ministral-3-8b	12Gi	16Gi
227	granite8b	bartowski/ibm-granite_granite-3.3-8b-instruct-GGUF	ibm-granite_granite-3.3-8b-instruct-Q4_K_M.gguf	granite-3.3-8b	12Gi	16Gi