diff --git a/.woodpecker/alert-monitor.yml b/.woodpecker/alert-monitor.yml index 9b6f25f8..1d2f029e 100644 --- a/.woodpecker/alert-monitor.yml +++ b/.woodpecker/alert-monitor.yml @@ -61,6 +61,19 @@ steps: kubectl get --raw "$$AM/api/v2/alerts?active=true&silenced=false&inhibited=false" \ > .ci-alert-monitor/alerts-raw.json head -c 400 .ci-alert-monitor/alerts-raw.json; echo + - | + # Blip lookback: an alert that fires and resolves BETWEEN ticks is + # invisible to the live snapshot (a 17-minute KubeHpaMaxedOut escaped + # exactly this way on 2026-07-16). Scan Prometheus's ALERTS history + # over ~2 tick intervals + jitter; the overlap between runs is + # deduped downstream like everything else. + set -eu + PROM="/api/v1/namespaces/monitoring/services/kube-prometheus-stack-prometheus:9090/proxy" + kubectl get --raw "$$PROM/api/v1/query?query=max%20by%20(alertname,namespace,job,severity)%20(max_over_time(ALERTS%7Balertstate%3D%22firing%22,alertname!~%22Watchdog%7CInfoInhibitor%22%7D%5B35m%5D))" \ + > .ci-alert-monitor/blips-raw.json \ + || printf '{"data":{"result":[]}}' > .ci-alert-monitor/blips-raw.json + # (no jq in this image — counting happens in the agent step) + head -c 300 .ci-alert-monitor/blips-raw.json; echo - | # Cluster context for the triage agent — gathered here because the # agent itself gets NO cluster access (no Bash, no kubeconfig). Runs @@ -121,17 +134,41 @@ steps: .ci-alert-monitor/open-issues.json) FPS=$$(printf '%s' "$$NOISE_BODY" | grep -oE '\b[0-9a-f]{16}\b' \ | jq -R -s -c 'split("\n") | map(select(length > 0))') + # Synthesize pseudo-alerts for BLIPS: lookback series whose label set + # is not currently firing at all (checked against the PRE-filter raw + # snapshot, so live-but-null-routed alerts don't reappear as blips). + # Pseudo-fingerprint = sha256(alertname|scope)[:16] — STABLE across + # recurrences, so one noise verdict suppresses a flappy alert + # permanently via the noise log. + : > .ci-alert-monitor/blips-new.jsonl + jq -c '.data.result[]?.metric // empty' .ci-alert-monitor/blips-raw.json | while IFS= read -r M; do + NAME=$$(printf '%s' "$$M" | jq -r '.alertname') + SCOPE=$$(printf '%s' "$$M" | jq -r '.namespace // .job // "cluster"') + if jq -e --arg n "$$NAME" --arg s "$$SCOPE" \ + '[.[] | select(.labels.alertname == $$n and ((.labels.namespace // .labels.job // "cluster") == $$s))] | length > 0' \ + .ci-alert-monitor/alerts-raw.json > /dev/null; then + continue + fi + FP=$$(printf '%s|%s' "$$NAME" "$$SCOPE" | sha256sum | cut -c1-16) + jq -n --arg fp "$$FP" --argjson m "$$M" \ + '{labels: $$m, fingerprint: $$fp, blip: true, + annotations: {summary: "BLIP: fired within the lookback window (~35 min) and already resolved — caught by the ALERTS history scan, not currently active."}}' \ + >> .ci-alert-monitor/blips-new.jsonl + done + jq -s '.' .ci-alert-monitor/blips-new.jsonl > .ci-alert-monitor/blips-candidates.json # Title key: namespace, else job, else "cluster". The job fallback # exists because aggregate alerts like TargetDown carry no namespace # — with only "(cluster)" two simultaneous TargetDowns for different # jobs (pve and dns-servers, 2026-07-16) collapsed into ONE issue # and the second root cause was never triaged. + jq -s 'add' .ci-alert-monitor/alerts-active.json .ci-alert-monitor/blips-candidates.json \ + > .ci-alert-monitor/alerts-combined.json jq --argjson titles "$$TITLES" --argjson fps "$${FPS:-[]}" ' [ .[] | ("[alert] " + .labels.alertname + " (" + (.labels.namespace // .labels.job // "cluster") + ")") as $$t | select( ($$t | IN($$titles[]) | not) and (.fingerprint | IN($$fps[]) | not) ) | . + {issue_title: $$t} ]' \ - .ci-alert-monitor/alerts-active.json > .ci-alert-monitor/alerts-new.json + .ci-alert-monitor/alerts-combined.json > .ci-alert-monitor/alerts-new.json NEW=$$(jq length .ci-alert-monitor/alerts-new.json) jq -r '.[] | "\(.fingerprint)\t\(.issue_title)"' .ci-alert-monitor/alerts-new.json if [ "$$NEW" -eq 0 ]; then diff --git a/ci/agents/alert-monitor/prompt.md b/ci/agents/alert-monitor/prompt.md index 6a88171d..04c0db20 100644 --- a/ci/agents/alert-monitor/prompt.md +++ b/ci/agents/alert-monitor/prompt.md @@ -16,12 +16,26 @@ to verify before you classify. All inputs are in the **current working directory** (the project root, where `ansible/`, `kubernetes/`, `opentofu/` live): -- `.ci-alert-monitor/alerts-new.json` — array of Alertmanager v2 alert - objects, already filtered: only active, unsilenced, uninhibited alerts that - have no open issue and are not in the noise log. Each has `labels` - (including `alertname`, usually `namespace` and `severity`), `annotations` - (`summary`, `description`, often `runbook_url`), `startsAt`, `fingerprint`, - and an added `issue_title` field. +- `.ci-alert-monitor/alerts-new.json` — array of alert objects, already + filtered: no open issue, not in the noise log, unsilenced/uninhibited. + Two kinds appear: + - **live alerts** (Alertmanager v2 objects): `labels` (including + `alertname`, usually `namespace` and `severity`), `annotations` + (`summary`, `description`, often `runbook_url`), `startsAt`, + `fingerprint`, and an added `issue_title`. + - **blips** (`"blip": true`): alerts that fired within the last ~35 + minutes but already resolved — reconstructed from Prometheus's ALERTS + history, so they carry only identity labels, a synthetic stable + `fingerprint`, and a summary noting they are no longer active. For a + blip, judge whether the underlying cause needs action despite + self-resolving: recurring flaps and short outages of real services are + `problem`; benign transients are `noise` (the synthetic fingerprint is + stable, so one noise verdict suppresses that flap permanently). Blips + have no receiver info — check the Alertmanager route config in + `kubernetes/apps/monitoring/helmrelease-prometheus.yaml`: an alert the + owner routes to the `null` receiver is `noise` by prior decision. Use + `./.ci-diag/kubectl get --raw` with a Prometheus `query_range` on + `ALERTS{...}` to see the blip's exact firing window when timing matters. - `.ci-alert-monitor/context.txt` — cluster snapshot taken minutes ago: node status, non-running pods, recent Warning events, Flux kustomizations and helmreleases that are not ready.