homelab/scripts/ci/test-promtool-rules.sh

#!/bin/sh
# Offline self-test for the guards in promtool-rules.sh.
#
# Run:  sh scripts/ci/test-promtool-rules.sh
#
# No real promtool, no network, no cluster — a stub `promtool` on PATH plus small
# fixture trees. That is deliberate: what is being tested here is not whether
# promtool evaluates PromQL correctly (it does, and ci-manifests-test.yml checks
# the real rules with the real binary) but whether promtool-rules.sh REFUSES TO
# REPORT SUCCESS when it has actually checked nothing. Those are the cases a
# green pipeline cannot distinguish from real coverage, so they need their own
# test, and stubbing promtool is what lets this run on the ci-claude image next
# to the other script tests.
#
# Every case below is a failure mode that occurred or was one edit away:
#
#   empty-discovery       the floor guard the issue asked for
#   no-suites             same, for the test files
#   unresolved-rule-file  the one that actually shipped — promtool WARNs, loads
#                         zero rules, and passes every silence assertion
#   warning-only          the same hole reached by a route the path check cannot
#                         see; caught by scanning promtool's output
#   inline-rule-files     a shape the resolver cannot parse, rejected by name
#                         rather than skipped in silence
#   empty-rule-files      a suite that declares no rules at all
#   unstrippable-spec     a manifest the spec: strip does not turn into a rule
#                         file, which promtool would report as "0 rules found"
#   mutation-no-op        a control whose sed no longer matches anything
#   mutation-undetected   a control the suite does not catch
#   happy-path            all of the above satisfied: must PASS
set -eu

HERE="$(cd "$(dirname "$0")" && pwd)"
SUT="$HERE/promtool-rules.sh"
[ -f "$SUT" ] || { echo "FAIL: $SUT not found"; exit 1; }

TMP="$(mktemp -d)"
trap 'rm -rf "$TMP"' EXIT INT TERM
fails=0

# --- the stub -------------------------------------------------------------
# `check rules` always succeeds. `test rules` fails if the tree it is run from
# contains the sentinel a mutation introduces, which is what makes the mutation
# loop genuinely exercised rather than mocked: the baseline tree has no
# sentinel and passes, a mutated tree has one and fails.
#
# STUB_VERSION      what to report for --version
# STUB_ALWAYS_PASS  `test rules` always exits 0, even when mutated
# STUB_WARN         `test rules` emits promtool's unresolved-rule-file warning
#                   and still exits 0
mkdir -p "$TMP/bin"
cat > "$TMP/bin/promtool" <<'STUB'
#!/bin/sh
case "${1:-}" in
  --version) echo "promtool, version ${STUB_VERSION:-3.15.0} (branch: stub)"; exit 0 ;;
esac
case "${1:-} ${2:-}" in
  "check rules") echo "  SUCCESS: 1 rules found"; exit 0 ;;
  "test rules")
    [ -n "${STUB_WARN:-}" ] && echo "  WARNING: no file match pattern nowhere.yaml"
    [ -n "${STUB_ALWAYS_PASS:-}" ] && { echo "  SUCCESS"; exit 0; }
    if grep -rq 'MUTATED-SENTINEL' . 2>/dev/null; then
      echo "  FAILED: sentinel present"; exit 1
    fi
    echo "  SUCCESS"; exit 0 ;;
esac
echo "stub: unhandled [$*]" >&2; exit 2
STUB
chmod +x "$TMP/bin/promtool"

# --- fixtures -------------------------------------------------------------
# A whole fixture repo per case, so no case can leak into another. `rules` is
# the count of PrometheusRule manifests to emit; the suite references the first
# one via `../`, the same relative shape the real suites use.
mkfixture() { # <dir> [n-manifests]
  d="$1"; n="${2:-1}"
  mkdir -p "$d/kubernetes/apps/monitoring-rules/tests"
  i=1
  while [ "$i" -le "$n" ]; do
    cat > "$d/kubernetes/apps/monitoring-rules/prometheusrule-fix$i.yaml" <<YAML
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: fix$i
spec:
  groups:
    - name: fix$i.rules
      rules:
        - alert: Fixture$i
          expr: up == 0
          for: 8m
YAML
    i=$((i + 1))
  done
  cat > "$d/kubernetes/apps/monitoring-rules/tests/fixture_test.yaml" <<'YAML'
rule_files:
  - ../prometheusrule-fix1.yaml
tests:
  - interval: 1m
    input_series:
      - series: 'up'
        values: '1x10'
    alert_rule_test:
      - eval_time: 10m
        alertname: Fixture1
        exp_alerts: []
YAML
  # A control whose sed inserts the stub's sentinel, so the baseline passes and
  # the mutant fails — the shape a real control has.
  printf 'sentinel|kubernetes/apps/monitoring-rules/prometheusrule-fix1.yaml|s/for: 8m/for: 8m # MUTATED-SENTINEL/\n' \
    > "$d/mutations.txt"
}

# run <fixture-dir> [env assignments...] -> exit status of the SUT, output in
# $TMP/out. Trailing assignments are appended AFTER the defaults, and `env`
# applies them in order, so a case can override any default.
run() {
  d="$1"; shift
  set +e
  ( cd "$d" && env PATH="$TMP/bin:$PATH" \
      PROMTOOL_RULE_FLOOR=1 \
      PROMTOOL_MUTATIONS=mutations.txt \
      "$@" sh "$SUT" . ) > "$TMP/out" 2>&1
  rc=$?
  set -e
  return $rc
}

# expect_fail <name> <fixture> <substring the output must contain> [env...]
#
# The substring is not decoration. A test that only asserts "exited non-zero"
# passes when the script fails for an unrelated reason — a typo'd fixture, a
# floor left set by an earlier case — and that is how a guard test rots into a
# test of nothing, which is the same disease this whole script exists to treat.
expect_fail() {
  name="$1"; d="$2"; want="$3"; shift 3
  if run "$d" "$@"; then
    printf 'FAIL  %s: expected non-zero exit, got 0\n' "$name"
    sed 's/^/        /' "$TMP/out"
    fails=$((fails + 1))
  elif ! grep -qF "$want" "$TMP/out"; then
    printf 'FAIL  %s: rejected, but not for the expected reason\n' "$name"
    printf '        wanted output containing: %s\n' "$want"
    sed 's/^/        /' "$TMP/out"
    fails=$((fails + 1))
  else
    printf 'ok    %s\n' "$name"
  fi
}

expect_pass() { # <name> <fixture> [env...]
  name="$1"; d="$2"; shift 2
  if run "$d" "$@"; then
    printf 'ok    %s\n' "$name"
  else
    printf 'FAIL  %s: expected exit 0\n' "$name"
    sed 's/^/        /' "$TMP/out"
    fails=$((fails + 1))
  fi
}

# --- happy path, first: if this does not pass, nothing below means anything --
mkfixture "$TMP/happy" 1
expect_pass "happy-path" "$TMP/happy"

# A green run must actually say it ran the mutation. Guards against the whole
# phase being skipped while the script still exits 0.
if ! grep -q 'mutation \[sentinel\] detected' "$TMP/out"; then
  echo "FAIL  happy-path did not report the mutation as detected"
  sed 's/^/        /' "$TMP/out"
  fails=$((fails + 1))
else
  echo "ok    happy-path reports the mutation as detected"
fi

# --- the floors -----------------------------------------------------------
mkdir -p "$TMP/empty/kubernetes"
: > "$TMP/empty/mutations.txt"
expect_fail "empty-discovery (0 manifests)" "$TMP/empty" \
  "found only 0 PrometheusRule manifests"

mkfixture "$TMP/nosuites" 1
rm "$TMP/nosuites/kubernetes/apps/monitoring-rules/tests/fixture_test.yaml"
expect_fail "no-suites (0 test files)" "$TMP/nosuites" \
  "found only 0 *_test.yaml suites"

mkfixture "$TMP/floor" 1
expect_fail "rule floor not met (1 < 5)" "$TMP/floor" \
  "found only 1 PrometheusRule manifests; floor is 5" PROMTOOL_RULE_FLOOR=5

# --- the hole that shipped ------------------------------------------------
mkfixture "$TMP/unresolved" 1
sed 's|\.\./prometheusrule-fix1\.yaml|kubernetes/apps/monitoring-rules/prometheusrule-fix1.yaml|' \
  "$TMP/unresolved/kubernetes/apps/monitoring-rules/tests/fixture_test.yaml" > "$TMP/u.tmp"
mv "$TMP/u.tmp" "$TMP/unresolved/kubernetes/apps/monitoring-rules/tests/fixture_test.yaml"
# The expected substring must sit on ONE line: die() messages are wrapped in the
# source, so a phrase spanning the wrap will not match.
expect_fail "unresolved-rule-file (repo-root path, not suite-relative)" "$TMP/unresolved" \
  "references rule_files entry"

mkfixture "$TMP/warn" 1
expect_fail "warning-only (promtool WARNs but exits 0)" "$TMP/warn" \
  "do not satisfy their own unit suites" STUB_WARN=1

mkfixture "$TMP/inline" 1
printf 'rule_files: [../prometheusrule-fix1.yaml]\ntests: []\n' \
  > "$TMP/inline/kubernetes/apps/monitoring-rules/tests/fixture_test.yaml"
expect_fail "inline-rule-files" "$TMP/inline" "writes rule_files inline"

mkfixture "$TMP/norf" 1
printf 'tests: []\n' > "$TMP/norf/kubernetes/apps/monitoring-rules/tests/fixture_test.yaml"
expect_fail "empty-rule-files" "$TMP/norf" "declares no rule_files"

# --- a manifest the strip cannot turn into a rule file --------------------
mkfixture "$TMP/badspec" 2
cat > "$TMP/badspec/kubernetes/apps/monitoring-rules/prometheusrule-fix2.yaml" <<'YAML'
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: fix2
# No top-level spec: at all. promtool check rules on the empty strip would
# report "SUCCESS: 0 rules found" and exit 0.
YAML
expect_fail "unstrippable-spec (no groups: after strip)" "$TMP/badspec" \
  "expected [groups:]"

# --- the mutation phase's own guards -------------------------------------
mkfixture "$TMP/noop" 1
printf 'noop|kubernetes/apps/monitoring-rules/prometheusrule-fix1.yaml|s/for: 99m/for: 1m/\n' \
  > "$TMP/noop/mutations.txt"
expect_fail "mutation-no-op (sed matches nothing)" "$TMP/noop" "changed nothing in"

mkfixture "$TMP/undetected" 1
expect_fail "mutation-undetected (suite passes while mutated)" "$TMP/undetected" \
  "was NOT detected" STUB_ALWAYS_PASS=1

mkfixture "$TMP/nomut" 1
printf '# only a comment, no mutations\n' > "$TMP/nomut/mutations.txt"
expect_fail "mutation floor not met (0 mutations)" "$TMP/nomut" "ran only 0 mutations"

mkfixture "$TMP/mutmissing" 1
rm "$TMP/mutmissing/mutations.txt"
expect_fail "mutations file missing" "$TMP/mutmissing" "mutations.txt is missing"

mkfixture "$TMP/mutbadtarget" 1
printf 'bad|kubernetes/apps/monitoring-rules/does-not-exist.yaml|s/a/b/\n' \
  > "$TMP/mutbadtarget/mutations.txt"
expect_fail "mutation targets an undiscovered manifest" "$TMP/mutbadtarget" \
  "which is not one of"

# --- the version pin ------------------------------------------------------
mkfixture "$TMP/v2" 1
expect_fail "promtool 2.x" "$TMP/v2" "is not 3.x" STUB_VERSION=2.55.1

mkfixture "$TMP/vjunk" 1
expect_fail "unparseable promtool version" "$TMP/vjunk" \
  "cannot parse" STUB_VERSION=not-a-version

if [ "$fails" -eq 0 ]; then
  echo "PASS  all promtool-rules.sh guard tests"
  exit 0
fi
echo "FAIL  $fails promtool-rules.sh guard test(s)"
exit 1