#!/bin/sh # Turn the TypeSafe picker's probability distribution into one model id. # # The judgment (how well each catalog model fits the task) comes from the # model; the policy lives here, in code: among the candidates whose probability # is within MARGIN of the best one, take the most recently released. Several # same-family models (Sonnet 4.5 / 4.6 / 5) split the probability for a # mid-weight task, and the newest of a comparable set is the one to run. # # Usage: choose-model.sh < # The table is fetch-models.sh --table output: id \t name \t released=YYYY-MM-DD ... # Prints the chosen id, or nothing when the answer is NONE / unusable. # Env: CHOOSE_MODEL_MARGIN (default 0.7): candidate if p >= MARGIN * p_best. set -eu TABLE_FILE="${1:?usage: choose-model.sh < response.json}" MARGIN="${CHOOSE_MODEL_MARGIN:-0.7}" jq -r --rawfile table "$TABLE_FILE" --argjson margin "$MARGIN" ' ($table | split("\n") | map(select(length > 0) | split("\t") | {key: .[0], value: ((.[2:] | map(select(startswith("released="))) | .[0] // "released=") | ltrimstr("released="))}) | from_entries) as $released | (.answers.model.probabilities // {}) | to_entries | map(select(.key != "NONE")) | sort_by(-.value) | if length == 0 then empty else (.[0].value) as $best | map(select(.value >= $margin * $best)) | sort_by([($released[.key] // ""), .value]) | .[-1].key end' 2>&1