#!/bin/sh
# Where does this model's output stop being language?
#
#   ./probe-length.sh <label>
#
# THE TEST THAT SHOULD HAVE COME FIRST. On this Vulkan/Iris Xe stack every
# model has a prompt length past which llama.cpp stops producing language and
# starts repeating a single token — " de de de", " the the the", " phosphorus
# phosphorus", or plain whitespace. It is silent: HTTP 200, finish_reason
# "stop" or "length", nothing in the log.
#
# It is also STATEFUL. One request past the boundary leaves the slot broken for
# every request after it, until the pod restarts. So in a 50-document run the
# first bad document takes the rest with it, and "failed 3 of 3" is one
# measurement, not three.
#
# A real paperless request is ~5300 tokens once the document, its RAG
# neighbours and the taxonomy lists are in. So the only question that matters
# about a candidate is whether its boundary is above that — and answering it
# takes minutes, against ninety for a classification run that would have
# measured the same thing badly.
#
# Measured so far:
#   Qwen3.6-35B-A3B   coherent at 13588, broken at 15086
#   gemma-4-26B-A4B   coherent at ~350, whitespace at 2334
set -eu

label="${1:?usage: probe-length.sh <label>   (labels are in candidates.tsv)}"
here="$(dirname "$0")"
ns=paperless-eval
pod="lenprobe-$(echo "$label" | tr 'A-Z._' 'a-z--' | cut -c1-40)"

row="$(grep -v '^#' "$here/candidates.tsv" | awk -v l="$label" '$1==l')"
[ -n "$row" ] || { echo "unknown candidate '$label'" >&2; exit 1; }
repo="$(echo "$row"   | cut -f2)"
hffile="$(echo "$row" | cut -f3)"
memreq="$(echo "$row" | cut -f5)"
memlim="$(echo "$row" | cut -f6)"

cleanup() { kubectl -n "$ns" delete pod "$pod" --wait=false >/dev/null 2>&1 || true; }
trap cleanup EXIT INT TERM
cleanup; sleep 2

kubectl -n "$ns" apply -f - >/dev/null <<EOF
apiVersion: v1
kind: Pod
metadata: {name: $pod, namespace: $ns}
spec:
  restartPolicy: Never
  containers:
    - name: llama-server
      image: ghcr.io/ggml-org/llama.cpp:server-vulkan-b10731
      args: ["--hf-repo","$repo","--hf-file","$hffile","--alias","probe",
             "--n-gpu-layers","99","--threads","4","--ctx-size","16384",
             "--parallel","1","--ubatch-size","1024","--n-predict","200","--no-webui"]
      env: [{name: LLAMA_CACHE, value: /models}]
      volumeMounts: [{name: models, mountPath: /models}]
      resources:
        requests: {cpu: "1", memory: $memreq, gpu.intel.com/i915: "1"}
        limits: {memory: $memlim, gpu.intel.com/i915: "1"}
      securityContext: {allowPrivilegeEscalation: false, capabilities: {drop: ["ALL"]}}
  volumes:
    - name: models
      persistentVolumeClaim: {claimName: llama-models}
EOF

# Say which of the two it is doing. There are only two iGPUs, so a probe
# started while both are busy sits Pending indefinitely — and an earlier run
# spent an hour reporting nothing before giving up with "never became healthy",
# which reads as a broken model rather than a queue.
echo "waiting for $label"
i=0
until kubectl -n "$ns" exec "$pod" -- curl -sf --max-time 10 http://127.0.0.1:8080/health >/dev/null 2>&1; do
  i=$((i+1))
  if [ $(( i % 8 )) -eq 0 ]; then
    ph=$(kubectl -n "$ns" get pod "$pod" -o jsonpath='{.status.phase}' 2>&1)
    why=$(kubectl -n "$ns" get pod "$pod" -o jsonpath='{.status.conditions[?(@.type=="PodScheduled")].message}' 2>&1)
    echo "  ${ph}${why:+ — $why}"
    [ "$ph" = "Pending" ] && [ -n "$why" ] && { echo "FAIL: cannot schedule — free a GPU first"; exit 1; }
  fi
  [ "$i" -gt 240 ] && { echo "FAIL: never became healthy after 60 min"; exit 1; }
  sleep 15
done

# Fresh pod per size. The defect poisons the slot, so probing sizes in sequence
# against one server would measure the first bad request and nothing after it.
for reps in 4 20 40 70 110 160; do
  kubectl -n "$ns" delete pod "$pod" >/dev/null 2>&1 || true
  sleep 2
  kubectl -n "$ns" apply -f - >/dev/null <<EOF2
apiVersion: v1
kind: Pod
metadata: {name: $pod, namespace: $ns}
spec:
  restartPolicy: Never
  containers:
    - name: llama-server
      image: ghcr.io/ggml-org/llama.cpp:server-vulkan-b10731
      args: ["--hf-repo","$repo","--hf-file","$hffile","--alias","probe",
             "--n-gpu-layers","99","--threads","4","--ctx-size","16384",
             "--parallel","1","--ubatch-size","1024","--n-predict","200","--no-webui"]
      env: [{name: LLAMA_CACHE, value: /models}]
      volumeMounts: [{name: models, mountPath: /models}]
      resources:
        requests: {cpu: "1", memory: $memreq, gpu.intel.com/i915: "1"}
        limits: {memory: $memlim, gpu.intel.com/i915: "1"}
      securityContext: {allowPrivilegeEscalation: false, capabilities: {drop: ["ALL"]}}
  volumes:
    - name: models
      persistentVolumeClaim: {claimName: llama-models}
EOF2
  until kubectl -n "$ns" exec "$pod" -- curl -sf --max-time 10 http://127.0.0.1:8080/health >/dev/null 2>&1; do sleep 10; done

  filler=""; n=0
  while [ $n -lt $reps ]; do
    filler="${filler}Die Rechnung wurde am fuenften Maerz ausgestellt und umfasst mehrere Positionen aus dem laufenden Wartungsvertrag. "
    n=$((n+1))
  done
  printf '{"model":"probe","max_tokens":200,"messages":[{"role":"user","content":"%s Nenne die Hauptstadt von Frankreich."}]}' "$filler" > /tmp/lp.json
  kubectl -n "$ns" cp /tmp/lp.json "$pod":/tmp/lp.json >/dev/null 2>&1 || true
  r=$(kubectl -n "$ns" exec "$pod" -- curl -s --max-time 400 -X POST http://127.0.0.1:8080/v1/chat/completions \
        -H 'Content-Type: application/json' -d @/tmp/lp.json)
  pt=$(printf '%s' "$r" | grep -oE '"prompt_tokens":[0-9]+' | head -1 | cut -d: -f2)
  # Test for DEGENERATION, not for the right answer. Two earlier versions of
  # this check asked whether "Paris" came back and failed a working model
  # twice: once because a thinking model leaves `content` empty and puts its
  # answer in `reasoning_content`, then because 64 tokens is not enough for one
  # to think its way to the answer at all.
  #
  # What actually marks the defect is a single token repeated — " de de de",
  # " phosphorus phosphorus", whitespace. That signature survives truncation
  # and does not care which field the model writes to or whether it finished,
  # so it tests the thing itself instead of a proxy for it.
  txt=$(printf '%s' "$r" | grep -oE '"(reasoning_)?content":"[^"]*' | sed 's/"[a-z_]*content":"//' | tr '\n' ' ')
  rep=$(printf '%s' "$txt" | tr -s ' ' '\n' | grep -c . || echo 0)
  top=$(printf '%s' "$txt" | tr -s ' ' '\n' | sort | uniq -c | sort -rn | head -1 | awk '{print $1}')
  : "${top:=0}"
  if [ "$rep" -gt 8 ] && [ $(( top * 2 )) -gt "$rep" ]; then
    verdict="BROKEN"      # one token is more than half the output
  elif [ "$rep" -le 1 ]; then
    verdict="EMPTY"       # nothing came back at all — also not healthy
  else
    verdict="OK"
  fi
  printf '%-6s prompt=%-6s %s  %s\n' "$verdict" "${pt:-?}" "$(printf '%.50s' "$txt")" ""
  [ "$verdict" != "OK" ] && { echo; echo "boundary is below ${pt:-?} prompt tokens"; echo "paperless needs ~5300 — $([ "${pt:-0}" -lt 6000 ] && echo 'DISQUALIFIED' || echo 'still above the bar')"; exit 0; }
done
echo
echo "coherent through the largest size probed — boundary is above that"