claude-botandClaude Fable 5.1
25f09e9b52
First live pick after #893 chose claude-sonnet-4-5 at 0.23 over claude-sonnet-4-6: same-family models split the probability for a mid-weight task, and "prefer newer" is a policy, not a judgment. New scripts/ci/choose-model.sh applies it to the distribution: among candidates within CHOOSE_MODEL_MARGIN (0.7) of the best probability the most recently released wins; a clear best stays; NONE yields nothing. The picker logs the top probabilities. Four policy tests added. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
124 lines
7.9 KiB
Bash
Executable File
124 lines
7.9 KiB
Bash
Executable File
#!/bin/sh
|
|
# Offline self-test for the model-selection validation gate (valid-model.sh).
|
|
# No network, no `claude`, no Woodpecker — just the string logic that decides
|
|
# whether an (untrusted) agent-supplied model id is allowed to reach a --model
|
|
# flag. This is the regression test for the flag-injection fix.
|
|
#
|
|
# Run: sh scripts/ci/test-model-selection.sh
|
|
set -eu
|
|
|
|
HERE="$(dirname "$0")"
|
|
CATALOG="$(printf '%s\n' \
|
|
'claude-opus-5 Claude Opus 5 released=2026-07-24' \
|
|
'claude-sonnet-5 Claude Sonnet 5 released=2026-06-29' \
|
|
'claude-haiku-4-5-20251001 Claude Haiku 4.5 released=2025-10-15')"
|
|
|
|
fails=0
|
|
check() { # description candidate expected-output
|
|
got="$(printf '%s\n' "$CATALOG" | sh "$HERE/valid-model.sh" "$2" || true)"
|
|
if [ "$got" = "$3" ]; then
|
|
printf 'ok %s\n' "$1"
|
|
else
|
|
printf 'FAIL %s: expected [%s] got [%s]\n' "$1" "$3" "$got"
|
|
fails=$((fails + 1))
|
|
fi
|
|
}
|
|
|
|
check "valid id passes" "claude-opus-5" "claude-opus-5"
|
|
check "valid dated id passes" "claude-haiku-4-5-20251001" "claude-haiku-4-5-20251001"
|
|
check "hallucinated id rejected" "claude-made-up-9" ""
|
|
check "empty rejected" "" ""
|
|
check "space injection rejected" "claude-opus-5 --dangerously-skip-permissions" ""
|
|
check "newline injection rejected" "$(printf 'claude-opus-5\n--dangerously-skip-permissions')" ""
|
|
check "leading-dash rejected" "--dangerously-skip-permissions" ""
|
|
check "path/quote chars rejected" 'claude-opus-5;rm -rf /' ""
|
|
|
|
# Empty catalog rejects everything (the fetch-failed / fallback case).
|
|
got="$(printf '' | sh "$HERE/valid-model.sh" "claude-opus-5" || true)"
|
|
if [ -z "$got" ]; then printf 'ok empty catalog rejects all\n'; else printf 'FAIL empty catalog: got [%s]\n' "$got"; fails=$((fails + 1)); fi
|
|
|
|
# Regression for the models.txt-writeback attack: pr-review must validate the
|
|
# triage's pick against the catalog snapshot captured BEFORE the (untrusted,
|
|
# unscoped-Write) triage agent runs — never a file the agent could overwrite.
|
|
# valid-model.sh takes the catalog on stdin precisely so the caller controls it;
|
|
# a token the attacker slipped into an on-disk file but that is NOT in the
|
|
# pristine snapshot passed here must be rejected.
|
|
pristine="$(printf '%s\n' 'claude-opus-5 Claude Opus 5' 'claude-sonnet-5 Claude Sonnet 5')"
|
|
got="$(printf '%s\n' "$pristine" | sh "$HERE/valid-model.sh" "claude-smuggled-weak" || true)"
|
|
if [ -z "$got" ]; then printf 'ok tampered id rejected by pristine-snapshot validation\n'; else printf 'FAIL writeback attack: got [%s]\n' "$got"; fails=$((fails + 1)); fi
|
|
|
|
# fetch-models.sh jq mapping — exercised over recorded /v1/models bodies via the
|
|
# FETCH_MODELS_BODY_FILE seam (no token, no network). Pins the response shape the
|
|
# selectors depend on, so an API change surfaces here instead of silently
|
|
# degrading both selectors to name+date.
|
|
FX="$(mktemp)"
|
|
trap 'rm -f "$FX"' EXIT
|
|
|
|
printf '%s' '{"data":[{"type":"model","id":"claude-opus-5","display_name":"Claude Opus 5","created_at":"2026-07-24T00:00:00Z","max_input_tokens":1000000,"max_tokens":128000,"capabilities":{"effort":{"supported":true},"thinking":{"supported":true},"code_execution":{"supported":true}}}],"has_more":false}' > "$FX"
|
|
row="$(FETCH_MODELS_BODY_FILE="$FX" sh "$HERE/fetch-models.sh" --table 2>/dev/null || true)"
|
|
case "$row" in
|
|
*"claude-opus-5"*"released=2026-07-24"*"context=1000000"*"max_output=128000"*"effort=true"*"thinking=true"*"code_exec=true"*)
|
|
printf 'ok well-formed /v1/models body maps to non-null capability columns\n' ;;
|
|
*) printf 'FAIL well-formed mapping: got [%s]\n' "$row"; fails=$((fails + 1)) ;;
|
|
esac
|
|
|
|
printf '%s' '{"data":[{"type":"model","id":"claude-x","display_name":"Claude X","created_at":"2026-01-01T00:00:00Z"}],"has_more":false}' > "$FX"
|
|
warn="$(FETCH_MODELS_BODY_FILE="$FX" sh "$HERE/fetch-models.sh" --table 2>&1 >/dev/null || true)"
|
|
case "$warn" in
|
|
*"no model reports a context window"*) printf 'ok missing capability fields warn loudly (schema-change guard)\n' ;;
|
|
*) printf 'FAIL degraded-schema warning: got [%s]\n' "$warn"; fails=$((fails + 1)) ;;
|
|
esac
|
|
|
|
# Recorded REAL /v1/models response (the shape verified live, 2026-07-25) mapped
|
|
# through the same seam — pins the actual field layout the selectors depend on,
|
|
# so a change to what fetch-models.sh extracts is caught here.
|
|
rows="$(FETCH_MODELS_BODY_FILE="$HERE/testdata/v1-models-sample.json" sh "$HERE/fetch-models.sh" --table 2>/dev/null || true)"
|
|
opusline="$(printf '%s\n' "$rows" | grep 'claude-opus-5' || true)"
|
|
haikuline="$(printf '%s\n' "$rows" | grep 'claude-haiku-4-5' || true)"
|
|
o=FAIL; h=FAIL
|
|
case "$opusline" in *"context=1000000"*"effort=true"*"thinking=true"*) o=ok ;; esac
|
|
case "$haikuline" in *"context=200000"*"effort=false"*) h=ok ;; esac
|
|
if [ "$o" = ok ] && [ "$h" = ok ]; then
|
|
printf 'ok recorded real /v1/models fixture maps correctly (opus effort=true, haiku effort=false)\n'
|
|
else
|
|
printf 'FAIL recorded fixture: opus=[%s] haiku=[%s]\n' "$opusline" "$haikuline"; fails=$((fails + 1))
|
|
fi
|
|
|
|
|
|
# TypeSafe path of pick-model.sh: the request offers exactly the catalog ids plus
|
|
# NONE as options, carries the task as state, and never sends anything else.
|
|
printf '%s' '{"data":[{"type":"model","id":"claude-opus-5","display_name":"Claude Opus 5","created_at":"2026-07-24T00:00:00Z","max_input_tokens":1000000,"max_tokens":128000},{"type":"model","id":"claude-haiku-4-5-20251001","display_name":"Claude Haiku 4.5","created_at":"2025-10-15T00:00:00Z","max_input_tokens":200000,"max_tokens":64000}],"has_more":false}' > "$FX"
|
|
req="$(FETCH_MODELS_BODY_FILE="$FX" TYPESAFE_API_KEY=dry PICK_MODEL_TYPESAFE_DRYRUN=1 sh "$HERE/pick-model.sh" "a task; with \"quotes\" and $ signs" 2>/dev/null || true)"
|
|
keys="$(printf '%s' "$req" | jq -r '.questions.model.criteria | keys | join(",")' 2>/dev/null || true)"
|
|
task="$(printf '%s' "$req" | jq -r '.state.task' 2>/dev/null || true)"
|
|
if [ "$keys" = "NONE,claude-haiku-4-5-20251001,claude-opus-5" ] && [ "$task" = 'a task; with "quotes" and $ signs' ]; then
|
|
printf 'ok typesafe request offers catalog ids + NONE and carries the task verbatim\n'
|
|
else
|
|
printf 'FAIL typesafe request: keys [%s] task [%s]\n' "$keys" "$task"; fails=$((fails + 1))
|
|
fi
|
|
|
|
# choose-model.sh: newest among comparable candidates, best otherwise, NONE -> nothing.
|
|
TB="$(mktemp)"
|
|
printf '%s\n' 'claude-sonnet-4-5 Claude Sonnet 4.5 released=2025-09-29 context=200000' \
|
|
'claude-sonnet-4-6 Claude Sonnet 4.6 released=2026-02-17 context=200000' \
|
|
'claude-opus-5 Claude Opus 5 released=2026-07-24 context=1000000' \
|
|
'claude-haiku-4-5 Claude Haiku 4.5 released=2025-10-15 context=200000' > "$TB"
|
|
choose() { printf '%s' "$1" | sh "$HERE/choose-model.sh" "$TB" || true; }
|
|
got="$(choose '{"answers":{"model":{"probabilities":{"claude-sonnet-4-5":0.30,"claude-sonnet-4-6":0.25,"claude-opus-5":0.10,"NONE":0.05}}}}')"
|
|
if [ "$got" = "claude-sonnet-4-6" ]; then printf 'ok newer sibling within margin wins\n'; else printf 'FAIL comparable: got [%s]\n' "$got"; fails=$((fails + 1)); fi
|
|
got="$(choose '{"answers":{"model":{"probabilities":{"claude-opus-5":0.90,"claude-haiku-4-5":0.05,"claude-sonnet-4-6":0.04,"NONE":0.01}}}}')"
|
|
if [ "$got" = "claude-opus-5" ]; then printf 'ok clear best is kept\n'; else printf 'FAIL clear best: got [%s]\n' "$got"; fails=$((fails + 1)); fi
|
|
got="$(choose '{"answers":{"model":{"probabilities":{"claude-haiku-4-5":0.60,"claude-sonnet-4-5":0.35,"NONE":0.05}}}}')"
|
|
if [ "$got" = "claude-haiku-4-5" ]; then printf 'ok older best outside margin is kept\n'; else printf 'FAIL margin: got [%s]\n' "$got"; fails=$((fails + 1)); fi
|
|
got="$(choose '{"answers":{"model":{"probabilities":{"NONE":1.0}}}}')"
|
|
if [ -z "$got" ]; then printf 'ok NONE yields nothing\n'; else printf 'FAIL NONE: got [%s]\n' "$got"; fails=$((fails + 1)); fi
|
|
rm -f "$TB"
|
|
|
|
if [ "$fails" -eq 0 ]; then
|
|
echo "All model-selection tests passed."
|
|
else
|
|
echo "$fails test(s) FAILED."
|
|
exit 1
|
|
fi
|
|
|