test(e2e): assert the alias where it is observable, not by posting to it

The check posted a completion under the descriptive model name and asserted a reply came back. That proves nothing: a single-model llama-server ignores the request's `model` field entirely. Confirmed against the live endpoint — it answers to "voellig-erfundenes-modell-xyz" and reports "model":"chat" — so the check passed whether or not the alias existed. /v1/models is the only place the alias is observable, and it is what a client reads to populate a picker, so both names are asserted there instead. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
test(e2e): regression suite for the local AI endpoint
2026-09-01 17:22:47 +02:00 · 2026-09-01 17:05:12 +02:00 · 2026-09-01 16:55:01 +02:00
2 changed files with 118 additions and 5 deletions
@@ -90,12 +90,17 @@ spec:
            # to it server-side; the WebUI is same-origin and unaffected.
            - --cors-origins
            - localhost
-            # Stable name for the API, so consumers ask for "chat" rather than
-            # the full repo string. Without it the model id is the value of
-            # --hf-repo, and every consumer's config would break the next time
-            # the model is swapped.
+            # Two names for one model. Without this the id is the whole
+            # --hf-repo string, which is unreadable in a model picker and in
+            # logs.
+            #
+            # The first is what callers see: it says which model is actually
+            # answering, which a bare "chat" hides and which matters as soon as
+            # a second chat model exists. The second is a stable handle for
+            # machine consumers — Home Assistant, Paperless, CI — so their
+            # config survives the next model swap without an edit.
            - --alias
-            - chat
+            - qwen3.6-35b-a3b,chat
            # Qwen3.6 enables thinking by default. At ~11.6 tok/s that spends
            # minutes reasoning before answering, and a request with a modest
            # max_tokens returns an empty `content` with finish_reason "length"
@@ -524,6 +524,114 @@ check "alertmanager: no severity=critical alerts active" bash -c '
    test "$body" = "[]"
 '

+# ---------------------------------------------------------------------------
+# Local AI inference. Every check here corresponds to a defect that actually
+# shipped, because none of them are catchable by manifest validation — the
+# manifests were valid in each case and the endpoint still misbehaved.
+# ---------------------------------------------------------------------------
+
+# llm_chat: POST a chat completion through the ingress. Body lands in
+# /tmp/llm_body for the caller to assert against. Generous --max-time: the
+# model runs on an iGPU at ~12 tok/s, so a short answer still takes seconds.
+llm_chat() {
+    local model="$1" prompt="$2" max="${3:-64}" out body code payload
+    payload=$(jq -nc --arg m "$model" --arg p "$prompt" --argjson n "$max" \
+        '{model:$m,messages:[{role:"user",content:$p}],max_tokens:$n,temperature:0}')
+    out=$(curl -sk --resolve "llm.renner.uno:443:$LB_IP" --max-time 300 \
+          -H 'Content-Type: application/json' -d "$payload" \
+          -w $'\n__CODE__:%{http_code}' \
+          "https://llm.renner.uno/v1/chat/completions" 2>&1)
+    code=${out##*__CODE__:}
+    body=${out%$'\n'__CODE__:*}
+    printf 'code=%s body=%s\n' "$code" "${body:0:300}"
+    [[ "$code" == "200" ]] || return 1
+    printf '%s' "$body" > /tmp/llm_body
+}
+export -f llm_chat
+export LB_IP
+
+# helper_post: POST JSON from inside e2e-helper, for Services with no ingress.
+# kubectl exec passes argv directly, so the JSON needs no shell quoting.
+helper_post() {
+    local url="$1" data="$2" out body code attempt
+    for attempt in 1 2 3; do
+        out=$(kubectl -n monitoring exec e2e-helper -- \
+              curl -sk --max-time 60 -H 'Content-Type: application/json' \
+              -d "$data" -w $'\n__CODE__:%{http_code}' "$url" 2>&1)
+        body=${out%$'\n'__CODE__:*}
+        code=${out##*__CODE__:}
+        printf 'attempt=%s code=%s body=%s\n' "$attempt" "$code" "${body:0:200}"
+        if [[ "$code" =~ ^2[0-9][0-9]$ ]]; then
+            printf '%s' "$body" > /tmp/helper_body
+            return 0
+        fi
+        sleep 5
+    done
+    return 1
+}
+export -f helper_post
+
+echo "=== Local AI inference ==="
+
+check "llama-server /health returns 200" \
+    http_code "https://llm.renner.uno/health" 200
+
+# The chat model is named for what it is, so a picker and a log line say which
+# model answered. `chat` stays valid alongside it as a stable handle, so a
+# consumer's config survives the next model swap.
+check "model list advertises qwen3.6-35b-a3b" \
+    http_body "https://llm.renner.uno/v1/models" 'qwen3.6-35b-a3b'
+
+# Regression: the embedding model used to share this port. It appeared in the
+# chat picker, and selecting it returned fluent nonsense ("capital capital
+# capital ...") instead of an error, because --embeddings does not refuse chat
+# completions. It now lives on a ClusterIP-only Service; if it ever reappears
+# here, it is selectable again and this must fail.
+check "embedding model absent from chat endpoint" \
+    bash -c '! curl -sk --resolve "llm.renner.uno:443:$LB_IP" --max-time 15 \
+        "https://llm.renner.uno/v1/models" | grep -qi "embed"'
+
+check "embeddings not exposed through the ingress" \
+    http_code "https://llm.renner.uno/v1/embeddings" 501
+
+# Regression: with thinking enabled by default the model spent its whole token
+# budget on reasoning_content and returned an EMPTY content with
+# finish_reason "length" — a 200 response that answers nothing. Asserting on
+# the status code alone would have passed.
+check "chat completion returns non-empty content" \
+    bash -c 'llm_chat chat "Nenne die Hauptstadt von Frankreich." 64 &&
+             jq -e ".choices[0].message.content | length > 0" /tmp/llm_body >/dev/null'
+
+check "chat completion stops rather than truncating" \
+    bash -c 'jq -e ".choices[0].finish_reason == \"stop\"" /tmp/llm_body >/dev/null'
+
+# Coherence, not just liveness. This is what catches a model answering from the
+# wrong weights or a numerically broken backend — llama-bench measures
+# throughput and would report excellent numbers while emitting garbage.
+check "chat answer is coherent (knows Paris)" \
+    bash -c 'jq -re ".choices[0].message.content" /tmp/llm_body | grep -qi paris'
+
+# Both names must be advertised, because advertising is all that a single-model
+# server does with them: it ignores the request's `model` field entirely and
+# answers to any string, so posting a name and getting a reply proves nothing.
+# /v1/models is what a client reads to populate a picker, and it is the only
+# place the alias is observable.
+check "model list also offers the chat handle" \
+    http_body "https://llm.renner.uno/v1/models" '"chat"'
+
+# Reached from the monitoring namespace, which ai-ns.yaml admits; there is no
+# ingress for this Service by design.
+check "llama-embed returns an embedding vector" \
+    bash -c 'helper_post "http://llama-embed.ai.svc.cluster.local:8080/v1/embeddings" \
+                 "{\"model\":\"embed\",\"input\":\"Testsatz\"}" &&
+             jq -e ".data[0].embedding | length > 0" /tmp/helper_body >/dev/null'
+
+# Guards the resource request silently disappearing: without it the pod still
+# runs, just on CPU, and only a throughput measurement would notice.
+check "llama-server holds an iGPU allocation" \
+    bash -c 'kubectl -n ai get deploy llama-server -o json |
+             jq -e ".spec.template.spec.containers[0].resources.limits[\"gpu.intel.com/i915\"]" >/dev/null'
+
 echo "=== Authentik OIDC discovery (per-app) ==="
 # A reachable, parseable .well-known proves Authentik has an OIDC provider
 # bound to that application slug. If the app config is missing or Authentik