diff --git a/kubernetes/apps/llama-server/embed.yaml b/kubernetes/apps/llama-server/embed.yaml index 515fb988..a3060e5b 100644 --- a/kubernetes/apps/llama-server/embed.yaml +++ b/kubernetes/apps/llama-server/embed.yaml @@ -64,16 +64,27 @@ spec: - "99" - --threads - "2" - # Paperless chunks documents at 1024 tokens. The defaults are - # batch 2048 / ubatch 512, and with --embeddings llama-server - # clamps both down to 512 rather than run them mismatched — below - # the chunk size. Matching both to 1024 fits a whole chunk in one - # pass; 2048 is deliberately avoided, since that is where this - # Vulkan stack fails with ErrorDeviceLost. + # Paperless chunks at 1024 tokens, but counts them with its own + # tokenizer while this server measures the batch in bge-m3's. + # Those disagree: XLM-RoBERTa splits German into more pieces than + # llama_index's counter does, so a chunk Paperless considers full + # at 1024 arrived here as 1069 and the request failed outright — + # "input (1069 tokens) is too large to process". + # + # The headroom therefore has to absorb a tokenizer ratio, not just + # a rounding error, and 1.5x covers far more than the 1.044 that + # broke it. The defaults are batch 2048 / ubatch 512, and with + # --embeddings llama-server clamps both to the smaller rather than + # run them mismatched, so both are set. + # + # 2048 stays avoided: that is where the chat model met + # ErrorDeviceLost on this Vulkan stack. That failure came under a + # 20 GB model's memory pressure and may not bind a 600 MB one, but + # this does not need to find out. - --batch-size - - "1024" + - "1536" - --ubatch-size - - "1024" + - "1536" # Room for a full chunk plus overhead rather than exactly one. - --ctx-size - "4096"