diff --git a/kubernetes/apps/llama-server/embed.yaml b/kubernetes/apps/llama-server/embed.yaml index 9c6676a8..515fb988 100644 --- a/kubernetes/apps/llama-server/embed.yaml +++ b/kubernetes/apps/llama-server/embed.yaml @@ -41,7 +41,7 @@ spec: image: ghcr.io/ggml-org/llama.cpp:server-vulkan-b10731 args: - --hf-repo - - Qwen/Qwen3-Embedding-0.6B-GGUF:Q8_0 + - gpustack/bge-m3-GGUF:Q8_0 # Serves POST /v1/embeddings. - --embeddings # A short, stable id for consumers. Without it the model is @@ -50,14 +50,16 @@ spec: # One name only: llama-server keeps aliases in a sorted set and # reports the first as the id, so a second name can steal it. - --alias - - qwen3-embedding-0.6b - # Qwen3-Embedding is trained for last-token pooling. Left - # unspecified llama-server takes the GGUF's default, which is not - # visible from outside and would silently produce vectors the model - # was not trained to emit. Set before anything is indexed: changing - # pooling later invalidates every stored embedding. + - bge-m3 + # bge-m3 is trained for CLS pooling — a different type from the + # Qwen family this replaces, which uses last-token. Left unspecified + # llama-server takes the GGUF default, which is not visible from + # outside and would silently produce vectors the model was not + # trained to emit. Read from the model's own 1_Pooling/config.json + # rather than assumed; of the candidates measured, three different + # pooling types were needed. - --pooling - - last + - cls - --n-gpu-layers - "99" - --threads @@ -91,6 +93,14 @@ spec: capabilities: drop: - ALL + # Measured, not estimated: embedding 400 real documents peaked at + # 1.6 GiB (Prometheus max_over_time on the working set). The limit is + # well above that because the footprint is dominated by the four slots + # at 4096 context and the Vulkan working set rather than by the + # weights — the Qwen model this replaces peaked at 10.6 GiB on the + # same settings for the same 600 MiB of weights, and several + # candidates were OOMKilled at limits that looked generous against + # their model size. resources: requests: cpu: 250m @@ -98,7 +108,7 @@ spec: gpu.intel.com/i915: "1" limits: cpu: "2" - memory: 4Gi + memory: 8Gi gpu.intel.com/i915: "1" # ~600 MiB rather than ~5 GiB, so a much shorter budget than the chat # model needs, but still sized for a cold download. diff --git a/kubernetes/apps/paperless/paperless.yaml b/kubernetes/apps/paperless/paperless.yaml index e5eb765e..6a5354ee 100644 --- a/kubernetes/apps/paperless/paperless.yaml +++ b/kubernetes/apps/paperless/paperless.yaml @@ -82,7 +82,10 @@ data: # primarily English archives — the better fit for a German corpus. PAPERLESS_AI_LLM_EMBEDDING_BACKEND: "openai-like" PAPERLESS_AI_LLM_EMBEDDING_ENDPOINT: "http://llama-embed.ai.svc.cluster.local:8080/v1" - PAPERLESS_AI_LLM_EMBEDDING_MODEL: "qwen3-embedding-0.6b" + # Must match --alias in kubernetes/apps/llama-server/embed.yaml. Changing the + # embedding model invalidates every stored vector, so a swap means the LLM + # index has to be rebuilt. + PAPERLESS_AI_LLM_EMBEDDING_MODEL: "bge-m3" PAPERLESS_AI_LLM_EMBEDDING_CHUNK_SIZE: "1024" --- apiVersion: apps/v1