diff --git a/kubernetes/apps/llama-server/deployment.yaml b/kubernetes/apps/llama-server/deployment.yaml index 1b1a0be4..16e239ed 100644 --- a/kubernetes/apps/llama-server/deployment.yaml +++ b/kubernetes/apps/llama-server/deployment.yaml @@ -75,11 +75,27 @@ spec: # many small per-expert matmuls that sets the pace, not bandwidth. - --threads - "4" - # 16k rather than 8k because the measured prefill falloff is mild - # and this model's KV cache is unusually cheap — only 10 of its 40 - # layers use full attention. + # 32k because Home Assistant's Assist agent measured 18910 tokens + # for a single request: ~12k of exposed-entity overview plus ~7k of + # base instructions and tool schemas, whose parameters enumerate + # every exposed name, area and floor. At 16384 the request was + # refused outright. + # + # --ctx-size is per slot, not the total: /props reports + # n_ctx_slot equal to this value while running four slots, where a + # total would have divided down to 4096. Doubling it alone would + # therefore double the KV cache, so the slot count halves in step + # and the allocation stays where it is today — 24979 MiB of the + # 32Gi limit, most of it the 19.5 GiB of weights. + # + # Two slots is what is actually in use: one holds Home Assistant's + # prompt, one holds Paperless's. A third caller would queue rather + # than evict either, which is the better failure mode given a cold + # prefill of this prompt costs around two minutes. - --ctx-size - - "16384" + - "32768" + - --parallel + - "2" # A micro-batch of 2048 or more makes this Vulkan/Mesa stack fail # with ErrorDeviceLost. - --ubatch-size