#!/bin/sh
# Can this model make one valid tool call against paperless's real schema?
#
#   ./probe-toolcall.sh ggml-org/Ministral-3-8B-Instruct-2512-GGUF:Q8_0
#
# Two minutes, against ninety for a full run. Every candidate must pass this
# before it is worth measuring, because three in a row failed here and the
# remaining forty-nine documents of each run said nothing the first one had not.
#
# WHAT THIS CAN AND CANNOT TELL YOU
#
# A FAIL is conclusive: the model cannot make the call, do not spend a run.
# A PASS is not. This payload has been too easy twice — first with three fields
# instead of fourteen, then with a 1191-token prompt instead of production's
# ~5300 — and each time a pass sent a ninety-minute run after a model that
# failed on document one.
#
# It carries the real DocumentClassifierSchema and ~5000 tokens now, but it
# still cannot reproduce what actually inflates a paperless prompt: the RAG
# neighbours and the full taxonomy candidate lists, hundreds of names with IDs
# the model must copy exactly and align by position. Document 1970 is 1976
# characters and still produced a 5333-token prompt.
#
# So treat this as a cheap smoke test, not a gate. The real gate is
# ABORT_AFTER in eval.py, which stops a run after three failed documents —
# five minutes lost instead of ninety, using the genuine request.
set -eu

repo="${1:?usage: probe-toolcall.sh <hf-repo:quant> [extra llama-server args...]}"
shift || true
here="$(dirname "$0")"
ns=paperless-eval
pod="toolprobe-$(echo "$repo" | tr 'A-Z/:._' 'a-z----' | cut -c1-40 | sed 's/-*$//')"

cleanup() { kubectl -n "$ns" delete pod "$pod" --wait=false >/dev/null 2>&1 || true; }
trap cleanup EXIT INT TERM
cleanup
sleep 2

echo "probing $repo"
kubectl -n "$ns" apply -f - >/dev/null <<EOF
apiVersion: v1
kind: Pod
metadata:
  name: $pod
spec:
  restartPolicy: Never
  containers:
    - name: llama-server
      image: ghcr.io/ggml-org/llama.cpp:server-vulkan-b10731
      args: ["--hf-repo","$repo","--alias","probe","--n-gpu-layers","99",
             "--threads","4","--ctx-size","16384","--parallel","1",
             "--ubatch-size","1024","--n-predict","768","--reasoning","off",
             "--no-webui"$(for a in "$@"; do printf ',"%s"' "$a"; done)]
      env: [{name: LLAMA_CACHE, value: /models}]
      volumeMounts: [{name: models, mountPath: /models}]
      resources:
        requests: {cpu: "1", memory: 12Gi, gpu.intel.com/i915: "1"}
        limits: {memory: 20Gi, gpu.intel.com/i915: "1"}
      securityContext:
        allowPrivilegeEscalation: false
        capabilities: {drop: ["ALL"]}
  volumes:
    - name: models
      persistentVolumeClaim: {claimName: llama-models}
EOF

echo "waiting for the model to load (a first download takes minutes)"
i=0
until kubectl -n "$ns" exec "$pod" -- curl -sf --max-time 10 http://127.0.0.1:8080/health >/dev/null 2>&1; do
  i=$((i + 1))
  [ "$i" -gt 120 ] && { echo "FAIL: server never became healthy"; kubectl -n "$ns" logs "$pod" --tail=20; exit 1; }
  sleep 10
done

sed "s/\"MODEL\"/\"probe\"/" "$here/toolcall-probe.json" > /tmp/tcp.json
kubectl -n "$ns" cp /tmp/tcp.json "$pod":/tmp/probe.json >/dev/null 2>&1 || true
r=$(kubectl -n "$ns" exec "$pod" -- curl -s --max-time 300 \
      -X POST http://127.0.0.1:8080/v1/chat/completions \
      -H 'Content-Type: application/json' -d @/tmp/probe.json)

finish=$(printf '%s' "$r" | grep -oE '"finish_reason":"[a-z_]+"' | head -1 | cut -d'"' -f4)
echo
echo "finish_reason: ${finish:-<none>}"
printf '%s' "$r" | grep -oE '"arguments":"[^"]{0,300}' | head -1 || true
echo

# finish_reason "tool_calls" is the whole test. "length" means it never stopped,
# "stop" with no tool_calls means the server could not read the call back.
if [ "$finish" = "tool_calls" ]; then
  echo "PASS — usable, worth a full run"
else
  echo "FAIL — do not spend a run on this until the cause is understood"
  exit 1
fi