diff --git a/kubernetes/apps/llama-server/deployment.yaml b/kubernetes/apps/llama-server/deployment.yaml index 1a947975..c15be8e5 100644 --- a/kubernetes/apps/llama-server/deployment.yaml +++ b/kubernetes/apps/llama-server/deployment.yaml @@ -75,31 +75,26 @@ spec: # many small per-expert matmuls that sets the pace, not bandwidth. - --threads - "4" - # 32k because Home Assistant's Assist agent measured 18910 tokens - # for a single request: ~12k of exposed-entity overview plus ~7k of - # base instructions and tool schemas, whose parameters enumerate - # every exposed name, area and floor. At 16384 the request was - # refused outright. + # 16k, and going above it corrupts inference rather than failing. + # At 32768 per slot this server answered a 650-token classification + # prompt with 1025 tokens of "!" — the KV cache producing numbers + # rather than language. It does not crash, it does not warn, and + # the OpenAI response looks well-formed, so the damage only shows + # in what callers get back: Paperless ran past its 300 s timeout, + # and Home Assistant got an empty completion and a hallucinated + # device list. # - # --ctx-size is the total and llama.cpp divides it across the - # slots, so 65536 over two slots is 32768 each. + # The same Vulkan/Mesa stack already fails hard at --ubatch-size + # 2048; this is the quiet version of that limit. Raise this only + # against a generation that is read, not against a server that + # merely starts. # - # It does not read that way without --parallel, and the reason is - # in `llama-server --help`: --kv-unified defaults to enabled only - # "if number of slots is auto". Leaving --parallel unset therefore - # bought a single KV buffer shared across sequences, where every - # slot could draw the full --ctx-size — which is why this server - # once reported n_ctx_slot 16384 across four slots. Naming a slot - # count turns that off and splits the context instead. - # - # Two slots is what is actually in use: one holds Home Assistant's - # prompt, one holds Paperless's. A third caller would queue rather - # than evict either, which is the better failure mode given a cold - # prefill of this prompt costs around two minutes. + # Slots stay unset, which is also what keeps --kv-unified on + # (`llama-server --help`: enabled "if number of slots is auto"), so + # the four auto slots share one buffer and each may draw the full + # 16384 rather than a quarter of it. - --ctx-size - - "65536" - - --parallel - - "2" + - "16384" # A micro-batch of 2048 or more makes this Vulkan/Mesa stack fail # with ErrorDeviceLost. - --ubatch-size