homelab/scripts/ci/promtool-rules.sh

#!/bin/sh
# Machine-check the PrometheusRule manifests in this repo: parse every rule with
# `promtool check rules`, run the committed unit suites with `promtool test
# rules`, then damage the rules on purpose and require the suites to notice.
#
# Run:  sh scripts/ci/promtool-rules.sh [repo-root]
# Needs: promtool on PATH (3.x — see the version note below), sh, find, sed, awk.
# Offline: no cluster, no registry, no secrets, no network.
# Tests:  scripts/ci/test-promtool-rules.sh (runs the guards against fixture
#         trees with a stub promtool; no real promtool needed).
#
# WHY THE MIRROR. promtool reads Prometheus rule files, which are bare `groups:`
# documents. A PrometheusRule is a Kubernetes object carrying the same thing
# under `spec:`, so promtool cannot read one directly. This script copies every
# PrometheusRule into a temporary tree at the SAME relative path with `spec:`
# stripped and the body dedented, and runs promtool there. Nothing is written
# back into the checkout.
#
# WHY THE MUTATION PHASE. Two failure modes bit while the first suite was being
# written, and neither changed promtool's exit code:
#
#   * a `rule_files:` entry that did not resolve. promtool prints
#     "WARNING: no file match pattern ...", loads ZERO rules, and then every
#     `exp_alerts: []` assertion passes against an empty rule set. Ninety
#     silence assertions, green, testing nothing.
#   * traces generated against the wrong clock window, so every input series was
#     flat zero. Same outcome: green, testing nothing.
#
# A suite that passes tells you the rules match the assertions. It does not tell
# you the assertions reach the rules. The two guards in step 4 and step 5 cover
# the first failure mode by name; the mutation phase in step 6 is the backstop
# for the general case, and is the only part of this that would have caught the
# second.
#
# PROMTOOL VERSION. Must match the Prometheus the cluster actually runs, which is
# whatever kube-prometheus-stack >= 72.0.0 ships — 3.x. Alert `for` evaluation is
# version-sensitive, so a 2.x promtool would be checking a different evaluator
# than production and the margins asserted in the suites would be claims about
# the wrong program. The exact pin lives on the `image:` line in
# .woodpecker/ci-manifests-test.yml; this script asserts only the major version,
# so a patch bump there needs no change here.
set -eu

ROOT="${1:-.}"
cd "$ROOT"

# Floors against silent discovery failure. Both are LOOSE on purpose. A floor
# tight enough to catch the loss of one file sits inside normal churn, so it
# fails spuriously — and the fix a hurried person reaches for is lowering the
# floor, which is how a guard becomes decoration. These catch total failure and
# gross partial loss, which is the failure that actually happens: a `find`
# expression the image's BusyBox does not support, or a moved directory.
#
# Same reasoning, and the same asymmetry, as the floors in
# .woodpecker/ci-manifests-test.yml. 17 PrometheusRules and 1 suite exist today.
RULE_FLOOR="${PROMTOOL_RULE_FLOOR:-12}"
SUITE_FLOOR="${PROMTOOL_SUITE_FLOOR:-1}"
MUTATION_FLOOR="${PROMTOOL_MUTATION_FLOOR:-1}"

TESTS_DIR="${PROMTOOL_TESTS_DIR:-kubernetes/apps/monitoring-rules/tests}"
MUTATIONS="${PROMTOOL_MUTATIONS:-$TESTS_DIR/mutations.txt}"

WORK="$(mktemp -d)"
trap 'rm -rf "$WORK"' EXIT INT TERM
MIRROR="$WORK/mirror"   # stripped manifests + suites, at repo-relative paths
MUTANT="$WORK/mutant"   # a damaged copy of the above, one mutation at a time
mkdir -p "$MIRROR"

die() { echo "FAIL: $*" >&2; exit 1; }

command -v promtool >/dev/null 2>&1 || die "promtool is not on PATH."

# `promtool --version` writes to stderr on some builds, hence the 2>&1.
PROMTOOL_VERSION="$(promtool --version 2>&1 \
  | sed -n 's/^promtool, version \([0-9][0-9.]*\).*/\1/p' | head -1)"
[ -n "$PROMTOOL_VERSION" ] || die "cannot parse 'promtool --version' output."
case "$PROMTOOL_VERSION" in
  3.*) : ;;
  *) die "promtool $PROMTOOL_VERSION is not 3.x. The cluster runs Prometheus 3
        (kube-prometheus-stack >= 72.0.0) and alert 'for' evaluation is
        version-sensitive, so a 2.x promtool would check a different evaluator
        than production. Fix the image in .woodpecker/ci-manifests-test.yml." ;;
esac
echo "promtool $PROMTOOL_VERSION"

# ---------------------------------------------------------------------------
# 1. Discover PrometheusRule manifests.
# ---------------------------------------------------------------------------
# NOT `find -printf` and NOT `grep -r --include`: this runs on the BusyBox find
# and grep in prom/prometheus, which reject both. `find -print0 | xargs -0 grep`
# is the portable shape, and unlike a command substitution it does not run into
# ARG_MAX on a tree this wide. Anchoring `^kind:` at column 0 keeps the match to
# the top-level field and away from anything nested or quoted in a comment.
find kubernetes -name '*.yaml' -type f -print0 2>/dev/null \
  | xargs -0 grep -l '^kind: PrometheusRule' 2>/dev/null \
  | sort > "$WORK/rules" || true

rule_count=$(grep -c . "$WORK/rules" || true)
if [ "$rule_count" -lt "$RULE_FLOOR" ]; then
  echo "FAIL: found only $rule_count PrometheusRule manifests; floor is $RULE_FLOOR." >&2
  echo "      If the tree legitimately shrank, lower PROMTOOL_RULE_FLOOR in" >&2
  echo "      .woodpecker/ci-manifests-test.yml — do not delete the guard." >&2
  echo "      Otherwise discovery is broken, and a check that found nothing" >&2
  echo "      must not report success." >&2
  exit 1
fi
echo "discovered $rule_count PrometheusRule manifests"

# ---------------------------------------------------------------------------
# 2. Mirror each manifest as a bare rule file, and parse it.
# ---------------------------------------------------------------------------
# The awk strips the `spec:` line, dedents its block by two, and stops at the
# first line back at column 0 — so a trailing top-level key (`status:`) is
# dropped rather than emitted at the wrong depth. Blank lines pass through
# unchanged; they carry no indentation to dedent.
fail=0
while IFS= read -r f; do
  [ -n "$f" ] || continue
  mkdir -p "$MIRROR/$(dirname "$f")"
  awk '
    !ins && /^spec:[ \t]*$/ { ins = 1; next }
    ins {
      if ($0 == "") { print ""; next }
      if (substr($0, 1, 2) == "  ") { print substr($0, 3); next }
      exit
    }
  ' "$f" > "$MIRROR/$f"

  # `promtool check rules` reports "SUCCESS: 0 rules found" on an empty document
  # and exits 0, so a strip that produced nothing would pass. Require the result
  # to actually be a rule file.
  first="$(sed -n '/^[[:space:]]*$/d; p' "$MIRROR/$f" | head -1)"
  if [ "$first" != "groups:" ]; then
    echo "FAIL  $f" >&2
    echo "        stripping spec: yielded [$first], expected [groups:]." >&2
    echo "        This manifest's rules are NOT being checked. Either spec: is" >&2
    echo "        not a top-level key here, or its body is not a groups: list." >&2
    fail=1
    continue
  fi

  if out=$(promtool check rules "$MIRROR/$f" 2>&1); then
    echo "ok    $f — $(echo "$out" | sed -n 's/.*\(SUCCESS: .*\)/\1/p' | head -1)"
  else
    echo "FAIL  $f" >&2
    echo "$out" | sed 's/^/        /' >&2
    fail=1
  fi
done < "$WORK/rules"
[ "$fail" -eq 0 ] || exit 1

# ---------------------------------------------------------------------------
# 3. Discover the committed unit suites and copy them into the mirror.
# ---------------------------------------------------------------------------
find "$TESTS_DIR" -name '*_test.yaml' -type f 2>/dev/null | sort > "$WORK/suites" || true
suite_count=$(grep -c . "$WORK/suites" || true)
if [ "$suite_count" -lt "$SUITE_FLOOR" ]; then
  echo "FAIL: found only $suite_count *_test.yaml suites under $TESTS_DIR;" >&2
  echo "      floor is $SUITE_FLOOR. Discovery is broken, or the suites were" >&2
  echo "      deleted. Either way this step must not report success." >&2
  exit 1
fi
echo "discovered $suite_count unit suite(s)"

while IFS= read -r t; do
  [ -n "$t" ] || continue
  mkdir -p "$MIRROR/$(dirname "$t")"
  cp "$t" "$MIRROR/$t"
done < "$WORK/suites"

# ---------------------------------------------------------------------------
# 4. Resolve every rule_files entry BEFORE running anything.
# ---------------------------------------------------------------------------
# The guard for the first failure mode at the top of this file. promtool joins
# dirname(suite) onto each `rule_files` entry and, when the result matches
# nothing, prints a WARNING and carries on with zero rules loaded — so the suite
# passes having asserted nothing. There is no promtool flag that makes it fatal,
# so resolve the paths here and refuse.
#
# The extraction is deliberately narrow: `rule_files:` at column 0, then the
# `  - path` lines under it, stopping at the next column-0 key. The flow form
# (`rule_files: [a, b]`) is not parsed — so it is rejected by name below rather
# than skipped in silence, which would reopen the hole this closes.
while IFS= read -r t; do
  [ -n "$t" ] || continue
  if grep -q '^rule_files:[[:space:]]*[^[:space:]]' "$t"; then
    die "$t writes rule_files inline. Use the block form (rule_files: followed
        by '  - path' lines): this script does not resolve the inline form, and
        an unresolved path there would be checked by nobody."
  fi
  awk '
    /^rule_files:[[:space:]]*$/ { inrf = 1; next }
    inrf && /^[[:space:]]*-[[:space:]]*/ {
      sub(/^[[:space:]]*-[[:space:]]*/, "")
      sub(/[[:space:]]*(#.*)?$/, "")
      gsub(/^["'"'"']|["'"'"']$/, "")
      print
      next
    }
    inrf && /^[^[:space:]]/ { inrf = 0 }
  ' "$t" > "$WORK/rf"

  n=$(grep -c . "$WORK/rf" || true)
  [ "$n" -gt 0 ] || die "$t declares no rule_files. It would load zero rules and
        pass every exp_alerts: [] assertion against nothing."

  while IFS= read -r rf; do
    [ -n "$rf" ] || continue
    case "$rf" in
      /*) resolved="$rf" ;;
      *)  resolved="$MIRROR/$(dirname "$t")/$rf" ;;
    esac
    [ -f "$resolved" ] || die "$t references rule_files entry [$rf], which does
        not resolve. promtool would treat that as a WARNING, load zero rules and
        pass. Entries are relative to the SUITE's own directory, not the repo
        root: from $TESTS_DIR, a manifest in the parent is '../name.yaml'."
  done < "$WORK/rf"
  echo "ok    $t — $n rule_files entry/entries resolved"
done < "$WORK/suites"

# ---------------------------------------------------------------------------
# 5. Run the suites.
# ---------------------------------------------------------------------------
# run_suites <dir> -> 0 if every suite passed, 1 otherwise. Output goes to
# $WORK/out so the mutation phase can stay quiet when a mutation is correctly
# caught, and the caller can print it when the result is not what it expected.
run_suites() {
  _dir="$1"
  _rc=0
  : > "$WORK/out"
  while IFS= read -r _t; do
    [ -n "$_t" ] || continue
    if ! (cd "$_dir" && promtool test rules "$_t") >> "$WORK/out" 2>&1; then
      _rc=1
    fi
  done < "$WORK/suites"
  # Belt and braces on the same trap step 4 covers by path: if a future promtool
  # reaches a missing rule file by a route this script does not model, the
  # WARNING is still the tell. A warning here is a failure.
  if grep -q 'WARNING: no file match pattern' "$WORK/out"; then
    echo "  (promtool warned about an unresolved rule file — zero rules loaded)" \
      >> "$WORK/out"
    _rc=1
  fi
  return $_rc
}

if run_suites "$MIRROR"; then
  sed 's/^/        /' "$WORK/out"
  echo "ok    all $suite_count suite(s) pass against the rules as committed"
else
  echo "FAIL: the committed rules do not satisfy their own unit suites." >&2
  sed 's/^/        /' "$WORK/out" >&2
  exit 1
fi

# ---------------------------------------------------------------------------
# 6. Mutation phase: break the rules on purpose; the suites must notice.
# ---------------------------------------------------------------------------
[ -f "$MUTATIONS" ] || die "$MUTATIONS is missing. Without it the suites vouch
        for themselves — see WHY THE MUTATION PHASE at the top of this script."

mutation_count=0
while IFS='|' read -r name target program; do
  case "${name:-}" in ''|\#*) continue ;; esac
  [ -n "${target:-}" ] && [ -n "${program:-}" ] \
    || die "$MUTATIONS: malformed entry [$name]; expected name|path|sed-program."
  [ -f "$MIRROR/$target" ] \
    || die "$MUTATIONS: mutation [$name] targets $target, which is not one of
        the $rule_count discovered PrometheusRule manifests."

  rm -rf "$MUTANT"
  cp -R "$MIRROR" "$MUTANT"
  sed "$program" "$MIRROR/$target" > "$MUTANT/$target"

  # A sed program that matches nothing leaves the tree undamaged, the suites
  # pass, and the mutation reads as undetected — so it would already fail below.
  # It is checked separately because the two cases need different repairs: "the
  # suite got weaker" means fix the suite, "the rule was renamed or reworded"
  # means fix the sed program.
  if cmp -s "$MIRROR/$target" "$MUTANT/$target"; then
    die "mutation [$name] changed nothing in $target. Its sed program no longer
        matches — the rule was probably renamed or reworded. Update the program
        in $MUTATIONS; do not delete the control."
  fi

  if run_suites "$MUTANT"; then
    echo "FAIL: mutation [$name] was NOT detected — the suites still pass with" >&2
    echo "      $target damaged. Whatever this mutation breaks is uncovered." >&2
    echo "      Diff applied to the stripped manifest:" >&2
    diff "$MIRROR/$target" "$MUTANT/$target" | sed 's/^/        /' >&2 || true
    exit 1
  fi
  echo "ok    mutation [$name] detected"
  mutation_count=$((mutation_count + 1))
done < "$MUTATIONS"

if [ "$mutation_count" -lt "$MUTATION_FLOOR" ]; then
  echo "FAIL: ran only $mutation_count mutations; floor is $MUTATION_FLOOR." >&2
  echo "      An empty mutation phase leaves the suites vouching for" >&2
  echo "      themselves, which is the thing it exists to prevent." >&2
  exit 1
fi

echo "PASS  $rule_count manifests parsed, $suite_count suite(s) green," \
     "$mutation_count mutation(s) caught"