From 3bde2a4316b005b89e08a039adc317e833305c42 Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Wed, 2 Sep 2026 13:39:36 +0200 Subject: [PATCH] feat(llama-embed): switch to bge-m3, activating the Paperless AI config MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit First candidate to be carried through to an end-to-end measurement. The embedding comparison on 400 real archive documents puts bge-m3 level with the deployed Qwen3-Embedding-0.6B where it matters and ahead everywhere else: correspondent 74.0% vs 75.0% (ceiling 82.5%, so 90% vs 91% of achievable) document type 70.0% vs 68.2% time 147s vs 380s for 400 documents peak memory 1.6 GiB vs 10.6 GiB Same 1024 dimensions, so nothing downstream changes shape. Note the pooling type differs — bge-m3 wants CLS where the Qwen family wants last-token. Read from the model's own 1_Pooling/config.json rather than assumed; across the candidates measured, three different pooling types were required, and a wrong one produces plausible but untrained vectors with no error anywhere. The memory limit is now measured rather than estimated. It also supersedes #745, which raised the limit for the model being replaced: the footprint is dominated by four slots at 4096 context plus the Vulkan working set, not by the weights — which is why several candidates were OOMKilled at limits that looked generous against their model size. This is not a final answer. Every candidate has to go through the end-to-end test (scripts/paperless-eval.sh) before the embedding choice means anything, because nothing has yet verified that the retrieval metric predicts what Paperless actually suggests. bge-m3 is simply the one to start with. Changing paperless-config is also what lets that test run at all: the AI settings from #739 have never reached the running pod, which predates the Reloader annotation from #741. This edit gives Reloader something real to react to, rather than forcing a restart by hand or inventing a change to trigger one. Co-Authored-By: Claude Opus 5 (1M context) --- kubernetes/apps/llama-server/embed.yaml | 28 ++++++++++++++++-------- kubernetes/apps/paperless/paperless.yaml | 5 ++++- 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/kubernetes/apps/llama-server/embed.yaml b/kubernetes/apps/llama-server/embed.yaml index 9c6676a8..515fb988 100644 --- a/kubernetes/apps/llama-server/embed.yaml +++ b/kubernetes/apps/llama-server/embed.yaml @@ -41,7 +41,7 @@ spec: image: ghcr.io/ggml-org/llama.cpp:server-vulkan-b10731 args: - --hf-repo - - Qwen/Qwen3-Embedding-0.6B-GGUF:Q8_0 + - gpustack/bge-m3-GGUF:Q8_0 # Serves POST /v1/embeddings. - --embeddings # A short, stable id for consumers. Without it the model is @@ -50,14 +50,16 @@ spec: # One name only: llama-server keeps aliases in a sorted set and # reports the first as the id, so a second name can steal it. - --alias - - qwen3-embedding-0.6b - # Qwen3-Embedding is trained for last-token pooling. Left - # unspecified llama-server takes the GGUF's default, which is not - # visible from outside and would silently produce vectors the model - # was not trained to emit. Set before anything is indexed: changing - # pooling later invalidates every stored embedding. + - bge-m3 + # bge-m3 is trained for CLS pooling — a different type from the + # Qwen family this replaces, which uses last-token. Left unspecified + # llama-server takes the GGUF default, which is not visible from + # outside and would silently produce vectors the model was not + # trained to emit. Read from the model's own 1_Pooling/config.json + # rather than assumed; of the candidates measured, three different + # pooling types were needed. - --pooling - - last + - cls - --n-gpu-layers - "99" - --threads @@ -91,6 +93,14 @@ spec: capabilities: drop: - ALL + # Measured, not estimated: embedding 400 real documents peaked at + # 1.6 GiB (Prometheus max_over_time on the working set). The limit is + # well above that because the footprint is dominated by the four slots + # at 4096 context and the Vulkan working set rather than by the + # weights — the Qwen model this replaces peaked at 10.6 GiB on the + # same settings for the same 600 MiB of weights, and several + # candidates were OOMKilled at limits that looked generous against + # their model size. resources: requests: cpu: 250m @@ -98,7 +108,7 @@ spec: gpu.intel.com/i915: "1" limits: cpu: "2" - memory: 4Gi + memory: 8Gi gpu.intel.com/i915: "1" # ~600 MiB rather than ~5 GiB, so a much shorter budget than the chat # model needs, but still sized for a cold download. diff --git a/kubernetes/apps/paperless/paperless.yaml b/kubernetes/apps/paperless/paperless.yaml index e5eb765e..6a5354ee 100644 --- a/kubernetes/apps/paperless/paperless.yaml +++ b/kubernetes/apps/paperless/paperless.yaml @@ -82,7 +82,10 @@ data: # primarily English archives — the better fit for a German corpus. PAPERLESS_AI_LLM_EMBEDDING_BACKEND: "openai-like" PAPERLESS_AI_LLM_EMBEDDING_ENDPOINT: "http://llama-embed.ai.svc.cluster.local:8080/v1" - PAPERLESS_AI_LLM_EMBEDDING_MODEL: "qwen3-embedding-0.6b" + # Must match --alias in kubernetes/apps/llama-server/embed.yaml. Changing the + # embedding model invalidates every stored vector, so a swap means the LLM + # index has to be rebuilt. + PAPERLESS_AI_LLM_EMBEDDING_MODEL: "bge-m3" PAPERLESS_AI_LLM_EMBEDDING_CHUNK_SIZE: "1024" --- apiVersion: apps/v1 -- 2.54.0