test(e2e): assert the alias where it is observable, not by posting to it

The check posted a completion under the descriptive model name and asserted a
reply came back. That proves nothing: a single-model llama-server ignores the
request's `model` field entirely. Confirmed against the live endpoint — it
answers to "voellig-erfundenes-modell-xyz" and reports "model":"chat" — so the
check passed whether or not the alias existed.

/v1/models is the only place the alias is observable, and it is what a client
reads to populate a picker, so both names are asserted there instead.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

ansible/roles/netbird-server/files/netbird-backup-dump

@@ -1,10 +1,10 @@
 #!/bin/sh
 # Forced-command target for the netbird-backup SSH key (see the
 # "Authorize the cluster backup key" task): takes a CONSISTENT snapshot
-# of the management SQLite store via the online-backup API (safe against
-# a live writer, unlike copying store.db) and streams a gzipped tar to
-# stdout. The key's authorized_keys entry is restricted to exactly this
-# script — the in-cluster puller cannot run anything else.
+# of the management SQLite store (safe against a live writer, unlike
+# copying store.db) and streams a gzipped tar to stdout. The key's
+# authorized_keys entry is restricted to exactly this script — the
+# in-cluster puller cannot run anything else.
 #
 # Deliberately store.db ONLY: /opt/netbird/config is fully regenerable
 # from the netbird-server role's templates + vault, and including it
@@ -14,5 +14,23 @@
 set -eu
 tmp=$(mktemp -d)
 trap 'rm -rf "$tmp"' EXIT
-sqlite3 /opt/netbird/management/store.db ".backup '$tmp/store.db'"
+# NetBird sets no journal_mode on the store (its DSN carries only
+# _busy_timeout and cache=shared), so store.db uses the default rollback
+# journal, where the management container's write lock blocks a reader
+# outright rather than letting it through as WAL would. Two things follow.
+#
+# .timeout arms a busy handler on the source connection, which is the
+# contended side; the destination is a fresh file in $tmp that nothing else
+# touches. Without it sqlite3 uses its 0 ms default and gives up the moment
+# it cannot take the lock. 30 s sits well inside the puller's 600 s
+# activeDeadlineSeconds.
+#
+# VACUUM INTO rather than .backup: the shell's .backup copies 100 pages per
+# sqlite3_backup_step(), and SQLite restarts the copy whenever the source is
+# written between steps, so a steady writer drags it out for as long as the
+# writes keep coming. VACUUM INTO reads one transaction in a single pass and
+# is documented as an alternative to the backup API for exactly this. It
+# also compacts, so the dump is smaller than the live file.
+sqlite3 -cmd '.timeout 30000' /opt/netbird/management/store.db \
+    "VACUUM INTO '$tmp/store.db'"
 tar -C "$tmp" -czf - store.db

kubernetes/apps/llama-server/deployment.yaml

@@ -90,6 +90,25 @@ spec:
             # to it server-side; the WebUI is same-origin and unaffected.
             - --cors-origins
             - localhost
+            # Two names for one model. Without this the id is the whole
+            # --hf-repo string, which is unreadable in a model picker and in
+            # logs.
+            #
+            # The first is what callers see: it says which model is actually
+            # answering, which a bare "chat" hides and which matters as soon as
+            # a second chat model exists. The second is a stable handle for
+            # machine consumers — Home Assistant, Paperless, CI — so their
+            # config survives the next model swap without an edit.
+            - --alias
+            - qwen3.6-35b-a3b,chat
+            # Qwen3.6 enables thinking by default. At ~11.6 tok/s that spends
+            # minutes reasoning before answering, and a request with a modest
+            # max_tokens returns an empty `content` with finish_reason "length"
+            # because the whole budget went to reasoning_content. Off by default
+            # keeps answers usable; a client that wants it can send
+            # chat_template_kwargs per request.
+            - --chat-template-kwargs
+            - '{"enable_thinking": false}'
           env:
             # Where --hf-repo caches the weights. Pointing it at the PVC makes
             # the download a one-time cost, shared with llama-embed, and keeps

kubernetes/e2e-test.sh

@@ -524,6 +524,114 @@ check "alertmanager: no severity=critical alerts active" bash -c '
     test "$body" = "[]"
 '
 
+# ---------------------------------------------------------------------------
+# Local AI inference. Every check here corresponds to a defect that actually
+# shipped, because none of them are catchable by manifest validation — the
+# manifests were valid in each case and the endpoint still misbehaved.
+# ---------------------------------------------------------------------------
+
+# llm_chat: POST a chat completion through the ingress. Body lands in
+# /tmp/llm_body for the caller to assert against. Generous --max-time: the
+# model runs on an iGPU at ~12 tok/s, so a short answer still takes seconds.
+llm_chat() {
+    local model="$1" prompt="$2" max="${3:-64}" out body code payload
+    payload=$(jq -nc --arg m "$model" --arg p "$prompt" --argjson n "$max" \
+        '{model:$m,messages:[{role:"user",content:$p}],max_tokens:$n,temperature:0}')
+    out=$(curl -sk --resolve "llm.renner.uno:443:$LB_IP" --max-time 300 \
+          -H 'Content-Type: application/json' -d "$payload" \
+          -w $'\n__CODE__:%{http_code}' \
+          "https://llm.renner.uno/v1/chat/completions" 2>&1)
+    code=${out##*__CODE__:}
+    body=${out%$'\n'__CODE__:*}
+    printf 'code=%s body=%s\n' "$code" "${body:0:300}"
+    [[ "$code" == "200" ]] || return 1
+    printf '%s' "$body" > /tmp/llm_body
+}
+export -f llm_chat
+export LB_IP
+
+# helper_post: POST JSON from inside e2e-helper, for Services with no ingress.
+# kubectl exec passes argv directly, so the JSON needs no shell quoting.
+helper_post() {
+    local url="$1" data="$2" out body code attempt
+    for attempt in 1 2 3; do
+        out=$(kubectl -n monitoring exec e2e-helper -- \
+              curl -sk --max-time 60 -H 'Content-Type: application/json' \
+              -d "$data" -w $'\n__CODE__:%{http_code}' "$url" 2>&1)
+        body=${out%$'\n'__CODE__:*}
+        code=${out##*__CODE__:}
+        printf 'attempt=%s code=%s body=%s\n' "$attempt" "$code" "${body:0:200}"
+        if [[ "$code" =~ ^2[0-9][0-9]$ ]]; then
+            printf '%s' "$body" > /tmp/helper_body
+            return 0
+        fi
+        sleep 5
+    done
+    return 1
+}
+export -f helper_post
+
+echo "=== Local AI inference ==="
+
+check "llama-server /health returns 200" \
+    http_code "https://llm.renner.uno/health" 200
+
+# The chat model is named for what it is, so a picker and a log line say which
+# model answered. `chat` stays valid alongside it as a stable handle, so a
+# consumer's config survives the next model swap.
+check "model list advertises qwen3.6-35b-a3b" \
+    http_body "https://llm.renner.uno/v1/models" 'qwen3.6-35b-a3b'
+
+# Regression: the embedding model used to share this port. It appeared in the
+# chat picker, and selecting it returned fluent nonsense ("capital capital
+# capital ...") instead of an error, because --embeddings does not refuse chat
+# completions. It now lives on a ClusterIP-only Service; if it ever reappears
+# here, it is selectable again and this must fail.
+check "embedding model absent from chat endpoint" \
+    bash -c '! curl -sk --resolve "llm.renner.uno:443:$LB_IP" --max-time 15 \
+        "https://llm.renner.uno/v1/models" | grep -qi "embed"'
+
+check "embeddings not exposed through the ingress" \
+    http_code "https://llm.renner.uno/v1/embeddings" 501
+
+# Regression: with thinking enabled by default the model spent its whole token
+# budget on reasoning_content and returned an EMPTY content with
+# finish_reason "length" — a 200 response that answers nothing. Asserting on
+# the status code alone would have passed.
+check "chat completion returns non-empty content" \
+    bash -c 'llm_chat chat "Nenne die Hauptstadt von Frankreich." 64 &&
+             jq -e ".choices[0].message.content | length > 0" /tmp/llm_body >/dev/null'
+
+check "chat completion stops rather than truncating" \
+    bash -c 'jq -e ".choices[0].finish_reason == \"stop\"" /tmp/llm_body >/dev/null'
+
+# Coherence, not just liveness. This is what catches a model answering from the
+# wrong weights or a numerically broken backend — llama-bench measures
+# throughput and would report excellent numbers while emitting garbage.
+check "chat answer is coherent (knows Paris)" \
+    bash -c 'jq -re ".choices[0].message.content" /tmp/llm_body | grep -qi paris'
+
+# Both names must be advertised, because advertising is all that a single-model
+# server does with them: it ignores the request's `model` field entirely and
+# answers to any string, so posting a name and getting a reply proves nothing.
+# /v1/models is what a client reads to populate a picker, and it is the only
+# place the alias is observable.
+check "model list also offers the chat handle" \
+    http_body "https://llm.renner.uno/v1/models" '"chat"'
+
+# Reached from the monitoring namespace, which ai-ns.yaml admits; there is no
+# ingress for this Service by design.
+check "llama-embed returns an embedding vector" \
+    bash -c 'helper_post "http://llama-embed.ai.svc.cluster.local:8080/v1/embeddings" \
+                 "{\"model\":\"embed\",\"input\":\"Testsatz\"}" &&
+             jq -e ".data[0].embedding | length > 0" /tmp/helper_body >/dev/null'
+
+# Guards the resource request silently disappearing: without it the pod still
+# runs, just on CPU, and only a throughput measurement would notice.
+check "llama-server holds an iGPU allocation" \
+    bash -c 'kubectl -n ai get deploy llama-server -o json |
+             jq -e ".spec.template.spec.containers[0].resources.limits[\"gpu.intel.com/i915\"]" >/dev/null'
+
 echo "=== Authentik OIDC discovery (per-app) ==="
 # A reachable, parseable .well-known proves Authentik has an OIDC provider
 # bound to that application slug. If the app config is missing or Authentik