claude-botandClaude Opus 4.8
1992fabb12
Addresses the 14aa6415 review.
- pick-model.sh: the picker relied on `--allowedTools ""` to be tool-less, but an
empirical check showed an empty allowlist does NOT block tools (the Write tool
still ran) -- it is treated as "unset" and the default toolset stays live. Add
`--permission-mode plan`, which IS verified to block Write while still producing
the selection JSON. --allowedTools "" + the Read deny-list remain as additional
layers. Corrected the comments in pick-model.sh and the picker settings.json
that claimed the empty allowlist made it tool-less.
- .woodpecker/ci-scripts-test.yml: new pull_request-triggered pipeline (homelab
only) that runs scripts/ci/test-model-selection.sh, so a PR weakening the
injection gate fails checks immediately instead of only on the next scheduled
e2e cron (closes the coverage-timing gap the review flagged).
- Pin a recorded real /v1/models response (scripts/ci/testdata/v1-models-sample.json)
and assert fetch-models.sh maps it correctly (opus effort=true, haiku
effort=false), documenting the verified response shape.
Deferred with rationale: hard-failing a pipeline when the degraded-schema warning
fires -- the warning already reaches the job log (curl/jq stderr is no longer
suppressed), and hard-failing on a benign API schema addition would break all
model selection; graceful degradation to name+date is the deliberate choice.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
93 lines
5.2 KiB
Bash
Executable File
93 lines
5.2 KiB
Bash
Executable File
#!/bin/sh
|
|
# Offline self-test for the model-selection validation gate (valid-model.sh).
|
|
# No network, no `claude`, no Woodpecker — just the string logic that decides
|
|
# whether an (untrusted) agent-supplied model id is allowed to reach a --model
|
|
# flag. This is the regression test for the flag-injection fix.
|
|
#
|
|
# Run: sh scripts/ci/test-model-selection.sh
|
|
set -eu
|
|
|
|
HERE="$(dirname "$0")"
|
|
CATALOG="$(printf '%s\n' \
|
|
'claude-opus-5 Claude Opus 5 released=2026-07-24' \
|
|
'claude-sonnet-5 Claude Sonnet 5 released=2026-06-29' \
|
|
'claude-haiku-4-5-20251001 Claude Haiku 4.5 released=2025-10-15')"
|
|
|
|
fails=0
|
|
check() { # description candidate expected-output
|
|
got="$(printf '%s\n' "$CATALOG" | sh "$HERE/valid-model.sh" "$2" || true)"
|
|
if [ "$got" = "$3" ]; then
|
|
printf 'ok %s\n' "$1"
|
|
else
|
|
printf 'FAIL %s: expected [%s] got [%s]\n' "$1" "$3" "$got"
|
|
fails=$((fails + 1))
|
|
fi
|
|
}
|
|
|
|
check "valid id passes" "claude-opus-5" "claude-opus-5"
|
|
check "valid dated id passes" "claude-haiku-4-5-20251001" "claude-haiku-4-5-20251001"
|
|
check "hallucinated id rejected" "claude-made-up-9" ""
|
|
check "empty rejected" "" ""
|
|
check "space injection rejected" "claude-opus-5 --dangerously-skip-permissions" ""
|
|
check "newline injection rejected" "$(printf 'claude-opus-5\n--dangerously-skip-permissions')" ""
|
|
check "leading-dash rejected" "--dangerously-skip-permissions" ""
|
|
check "path/quote chars rejected" 'claude-opus-5;rm -rf /' ""
|
|
|
|
# Empty catalog rejects everything (the fetch-failed / fallback case).
|
|
got="$(printf '' | sh "$HERE/valid-model.sh" "claude-opus-5" || true)"
|
|
if [ -z "$got" ]; then printf 'ok empty catalog rejects all\n'; else printf 'FAIL empty catalog: got [%s]\n' "$got"; fails=$((fails + 1)); fi
|
|
|
|
# Regression for the models.txt-writeback attack: pr-review must validate the
|
|
# triage's pick against the catalog snapshot captured BEFORE the (untrusted,
|
|
# unscoped-Write) triage agent runs — never a file the agent could overwrite.
|
|
# valid-model.sh takes the catalog on stdin precisely so the caller controls it;
|
|
# a token the attacker slipped into an on-disk file but that is NOT in the
|
|
# pristine snapshot passed here must be rejected.
|
|
pristine="$(printf '%s\n' 'claude-opus-5 Claude Opus 5' 'claude-sonnet-5 Claude Sonnet 5')"
|
|
got="$(printf '%s\n' "$pristine" | sh "$HERE/valid-model.sh" "claude-smuggled-weak" || true)"
|
|
if [ -z "$got" ]; then printf 'ok tampered id rejected by pristine-snapshot validation\n'; else printf 'FAIL writeback attack: got [%s]\n' "$got"; fails=$((fails + 1)); fi
|
|
|
|
# fetch-models.sh jq mapping — exercised over recorded /v1/models bodies via the
|
|
# FETCH_MODELS_BODY_FILE seam (no token, no network). Pins the response shape the
|
|
# selectors depend on, so an API change surfaces here instead of silently
|
|
# degrading both selectors to name+date.
|
|
FX="$(mktemp)"
|
|
trap 'rm -f "$FX"' EXIT
|
|
|
|
printf '%s' '{"data":[{"type":"model","id":"claude-opus-5","display_name":"Claude Opus 5","created_at":"2026-07-24T00:00:00Z","max_input_tokens":1000000,"max_tokens":128000,"capabilities":{"effort":{"supported":true},"thinking":{"supported":true},"code_execution":{"supported":true}}}],"has_more":false}' > "$FX"
|
|
row="$(FETCH_MODELS_BODY_FILE="$FX" sh "$HERE/fetch-models.sh" --table 2>/dev/null || true)"
|
|
case "$row" in
|
|
*"claude-opus-5"*"released=2026-07-24"*"context=1000000"*"max_output=128000"*"effort=true"*"thinking=true"*"code_exec=true"*)
|
|
printf 'ok well-formed /v1/models body maps to non-null capability columns\n' ;;
|
|
*) printf 'FAIL well-formed mapping: got [%s]\n' "$row"; fails=$((fails + 1)) ;;
|
|
esac
|
|
|
|
printf '%s' '{"data":[{"type":"model","id":"claude-x","display_name":"Claude X","created_at":"2026-01-01T00:00:00Z"}],"has_more":false}' > "$FX"
|
|
warn="$(FETCH_MODELS_BODY_FILE="$FX" sh "$HERE/fetch-models.sh" --table 2>&1 >/dev/null || true)"
|
|
case "$warn" in
|
|
*"no model reports a context window"*) printf 'ok missing capability fields warn loudly (schema-change guard)\n' ;;
|
|
*) printf 'FAIL degraded-schema warning: got [%s]\n' "$warn"; fails=$((fails + 1)) ;;
|
|
esac
|
|
|
|
# Recorded REAL /v1/models response (the shape verified live, 2026-07-25) mapped
|
|
# through the same seam — pins the actual field layout the selectors depend on,
|
|
# so a change to what fetch-models.sh extracts is caught here.
|
|
rows="$(FETCH_MODELS_BODY_FILE="$HERE/testdata/v1-models-sample.json" sh "$HERE/fetch-models.sh" --table 2>/dev/null || true)"
|
|
opusline="$(printf '%s\n' "$rows" | grep 'claude-opus-5' || true)"
|
|
haikuline="$(printf '%s\n' "$rows" | grep 'claude-haiku-4-5' || true)"
|
|
o=FAIL; h=FAIL
|
|
case "$opusline" in *"context=1000000"*"effort=true"*"thinking=true"*) o=ok ;; esac
|
|
case "$haikuline" in *"context=200000"*"effort=false"*) h=ok ;; esac
|
|
if [ "$o" = ok ] && [ "$h" = ok ]; then
|
|
printf 'ok recorded real /v1/models fixture maps correctly (opus effort=true, haiku effort=false)\n'
|
|
else
|
|
printf 'FAIL recorded fixture: opus=[%s] haiku=[%s]\n' "$opusline" "$haikuline"; fails=$((fails + 1))
|
|
fi
|
|
|
|
if [ "$fails" -eq 0 ]; then
|
|
echo "All model-selection tests passed."
|
|
else
|
|
echo "$fails test(s) FAILED."
|
|
exit 1
|
|
fi
|