claude-botandClaude Opus 5
8123021ea4
The Hermes probe reported "never became healthy" after an hour. It had not failed to load — it had never been scheduled, because both iGPUs were busy with other jobs and there are only two. Same blind spot as the eval pods sitting in Init: from outside, "waiting" and "will never start" look identical, and the message blamed the model. It now prints the pod phase every two minutes and exits immediately with the scheduler's own reason when it cannot place the pod. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
150 lines
7.0 KiB
Bash
Executable File
150 lines
7.0 KiB
Bash
Executable File
#!/bin/sh
|
|
# Where does this model's output stop being language?
|
|
#
|
|
# ./probe-length.sh <label>
|
|
#
|
|
# THE TEST THAT SHOULD HAVE COME FIRST. On this Vulkan/Iris Xe stack every
|
|
# model has a prompt length past which llama.cpp stops producing language and
|
|
# starts repeating a single token — " de de de", " the the the", " phosphorus
|
|
# phosphorus", or plain whitespace. It is silent: HTTP 200, finish_reason
|
|
# "stop" or "length", nothing in the log.
|
|
#
|
|
# It is also STATEFUL. One request past the boundary leaves the slot broken for
|
|
# every request after it, until the pod restarts. So in a 50-document run the
|
|
# first bad document takes the rest with it, and "failed 3 of 3" is one
|
|
# measurement, not three.
|
|
#
|
|
# A real paperless request is ~5300 tokens once the document, its RAG
|
|
# neighbours and the taxonomy lists are in. So the only question that matters
|
|
# about a candidate is whether its boundary is above that — and answering it
|
|
# takes minutes, against ninety for a classification run that would have
|
|
# measured the same thing badly.
|
|
#
|
|
# Measured so far:
|
|
# Qwen3.6-35B-A3B coherent at 13588, broken at 15086
|
|
# gemma-4-26B-A4B coherent at ~350, whitespace at 2334
|
|
set -eu
|
|
|
|
label="${1:?usage: probe-length.sh <label> (labels are in candidates.tsv)}"
|
|
here="$(dirname "$0")"
|
|
ns=paperless-eval
|
|
pod="lenprobe-$(echo "$label" | tr 'A-Z._' 'a-z--' | cut -c1-40)"
|
|
|
|
row="$(grep -v '^#' "$here/candidates.tsv" | awk -v l="$label" '$1==l')"
|
|
[ -n "$row" ] || { echo "unknown candidate '$label'" >&2; exit 1; }
|
|
repo="$(echo "$row" | cut -f2)"
|
|
hffile="$(echo "$row" | cut -f3)"
|
|
memreq="$(echo "$row" | cut -f5)"
|
|
memlim="$(echo "$row" | cut -f6)"
|
|
|
|
cleanup() { kubectl -n "$ns" delete pod "$pod" --wait=false >/dev/null 2>&1 || true; }
|
|
trap cleanup EXIT INT TERM
|
|
cleanup; sleep 2
|
|
|
|
kubectl -n "$ns" apply -f - >/dev/null <<EOF
|
|
apiVersion: v1
|
|
kind: Pod
|
|
metadata: {name: $pod, namespace: $ns}
|
|
spec:
|
|
restartPolicy: Never
|
|
containers:
|
|
- name: llama-server
|
|
image: ghcr.io/ggml-org/llama.cpp:server-vulkan-b10731
|
|
args: ["--hf-repo","$repo","--hf-file","$hffile","--alias","probe",
|
|
"--n-gpu-layers","99","--threads","4","--ctx-size","16384",
|
|
"--parallel","1","--ubatch-size","1024","--n-predict","200","--no-webui"]
|
|
env: [{name: LLAMA_CACHE, value: /models}]
|
|
volumeMounts: [{name: models, mountPath: /models}]
|
|
resources:
|
|
requests: {cpu: "1", memory: $memreq, gpu.intel.com/i915: "1"}
|
|
limits: {memory: $memlim, gpu.intel.com/i915: "1"}
|
|
securityContext: {allowPrivilegeEscalation: false, capabilities: {drop: ["ALL"]}}
|
|
volumes:
|
|
- name: models
|
|
persistentVolumeClaim: {claimName: llama-models}
|
|
EOF
|
|
|
|
# Say which of the two it is doing. There are only two iGPUs, so a probe
|
|
# started while both are busy sits Pending indefinitely — and an earlier run
|
|
# spent an hour reporting nothing before giving up with "never became healthy",
|
|
# which reads as a broken model rather than a queue.
|
|
echo "waiting for $label"
|
|
i=0
|
|
until kubectl -n "$ns" exec "$pod" -- curl -sf --max-time 10 http://127.0.0.1:8080/health >/dev/null 2>&1; do
|
|
i=$((i+1))
|
|
if [ $(( i % 8 )) -eq 0 ]; then
|
|
ph=$(kubectl -n "$ns" get pod "$pod" -o jsonpath='{.status.phase}' 2>&1)
|
|
why=$(kubectl -n "$ns" get pod "$pod" -o jsonpath='{.status.conditions[?(@.type=="PodScheduled")].message}' 2>&1)
|
|
echo " ${ph}${why:+ — $why}"
|
|
[ "$ph" = "Pending" ] && [ -n "$why" ] && { echo "FAIL: cannot schedule — free a GPU first"; exit 1; }
|
|
fi
|
|
[ "$i" -gt 240 ] && { echo "FAIL: never became healthy after 60 min"; exit 1; }
|
|
sleep 15
|
|
done
|
|
|
|
# Fresh pod per size. The defect poisons the slot, so probing sizes in sequence
|
|
# against one server would measure the first bad request and nothing after it.
|
|
for reps in 4 20 40 70 110 160; do
|
|
kubectl -n "$ns" delete pod "$pod" >/dev/null 2>&1 || true
|
|
sleep 2
|
|
kubectl -n "$ns" apply -f - >/dev/null <<EOF2
|
|
apiVersion: v1
|
|
kind: Pod
|
|
metadata: {name: $pod, namespace: $ns}
|
|
spec:
|
|
restartPolicy: Never
|
|
containers:
|
|
- name: llama-server
|
|
image: ghcr.io/ggml-org/llama.cpp:server-vulkan-b10731
|
|
args: ["--hf-repo","$repo","--hf-file","$hffile","--alias","probe",
|
|
"--n-gpu-layers","99","--threads","4","--ctx-size","16384",
|
|
"--parallel","1","--ubatch-size","1024","--n-predict","200","--no-webui"]
|
|
env: [{name: LLAMA_CACHE, value: /models}]
|
|
volumeMounts: [{name: models, mountPath: /models}]
|
|
resources:
|
|
requests: {cpu: "1", memory: $memreq, gpu.intel.com/i915: "1"}
|
|
limits: {memory: $memlim, gpu.intel.com/i915: "1"}
|
|
securityContext: {allowPrivilegeEscalation: false, capabilities: {drop: ["ALL"]}}
|
|
volumes:
|
|
- name: models
|
|
persistentVolumeClaim: {claimName: llama-models}
|
|
EOF2
|
|
until kubectl -n "$ns" exec "$pod" -- curl -sf --max-time 10 http://127.0.0.1:8080/health >/dev/null 2>&1; do sleep 10; done
|
|
|
|
filler=""; n=0
|
|
while [ $n -lt $reps ]; do
|
|
filler="${filler}Die Rechnung wurde am fuenften Maerz ausgestellt und umfasst mehrere Positionen aus dem laufenden Wartungsvertrag. "
|
|
n=$((n+1))
|
|
done
|
|
printf '{"model":"probe","max_tokens":200,"messages":[{"role":"user","content":"%s Nenne die Hauptstadt von Frankreich."}]}' "$filler" > /tmp/lp.json
|
|
kubectl -n "$ns" cp /tmp/lp.json "$pod":/tmp/lp.json >/dev/null 2>&1 || true
|
|
r=$(kubectl -n "$ns" exec "$pod" -- curl -s --max-time 400 -X POST http://127.0.0.1:8080/v1/chat/completions \
|
|
-H 'Content-Type: application/json' -d @/tmp/lp.json)
|
|
pt=$(printf '%s' "$r" | grep -oE '"prompt_tokens":[0-9]+' | head -1 | cut -d: -f2)
|
|
# Test for DEGENERATION, not for the right answer. Two earlier versions of
|
|
# this check asked whether "Paris" came back and failed a working model
|
|
# twice: once because a thinking model leaves `content` empty and puts its
|
|
# answer in `reasoning_content`, then because 64 tokens is not enough for one
|
|
# to think its way to the answer at all.
|
|
#
|
|
# What actually marks the defect is a single token repeated — " de de de",
|
|
# " phosphorus phosphorus", whitespace. That signature survives truncation
|
|
# and does not care which field the model writes to or whether it finished,
|
|
# so it tests the thing itself instead of a proxy for it.
|
|
txt=$(printf '%s' "$r" | grep -oE '"(reasoning_)?content":"[^"]*' | sed 's/"[a-z_]*content":"//' | tr '\n' ' ')
|
|
rep=$(printf '%s' "$txt" | tr -s ' ' '\n' | grep -c . || echo 0)
|
|
top=$(printf '%s' "$txt" | tr -s ' ' '\n' | sort | uniq -c | sort -rn | head -1 | awk '{print $1}')
|
|
: "${top:=0}"
|
|
if [ "$rep" -gt 8 ] && [ $(( top * 2 )) -gt "$rep" ]; then
|
|
verdict="BROKEN" # one token is more than half the output
|
|
elif [ "$rep" -le 1 ]; then
|
|
verdict="EMPTY" # nothing came back at all — also not healthy
|
|
else
|
|
verdict="OK"
|
|
fi
|
|
printf '%-6s prompt=%-6s %s %s\n' "$verdict" "${pt:-?}" "$(printf '%.50s' "$txt")" ""
|
|
[ "$verdict" != "OK" ] && { echo; echo "boundary is below ${pt:-?} prompt tokens"; echo "paperless needs ~5300 — $([ "${pt:-0}" -lt 6000 ] && echo 'DISQUALIFIED' || echo 'still above the bar')"; exit 0; }
|
|
done
|
|
echo
|
|
echo "coherent through the largest size probed — boundary is above that"
|