.woodpecker/alert-monitor.yml

@@ -61,6 +61,19 @@ steps:
         kubectl get --raw "$$AM/api/v2/alerts?active=true&silenced=false&inhibited=false" \
           > .ci-alert-monitor/alerts-raw.json
         head -c 400 .ci-alert-monitor/alerts-raw.json; echo
+      - |
+        # Blip lookback: an alert that fires and resolves BETWEEN ticks is
+        # invisible to the live snapshot (a 17-minute KubeHpaMaxedOut escaped
+        # exactly this way on 2026-07-16). Scan Prometheus's ALERTS history
+        # over ~2 tick intervals + jitter; the overlap between runs is
+        # deduped downstream like everything else.
+        set -eu
+        PROM="/api/v1/namespaces/monitoring/services/kube-prometheus-stack-prometheus:9090/proxy"
+        kubectl get --raw "$$PROM/api/v1/query?query=max%20by%20(alertname,namespace,job,severity)%20(max_over_time(ALERTS%7Balertstate%3D%22firing%22,alertname!~%22Watchdog%7CInfoInhibitor%22%7D%5B35m%5D))" \
+          > .ci-alert-monitor/blips-raw.json \
+          || printf '{"data":{"result":[]}}' > .ci-alert-monitor/blips-raw.json
+        # (no jq in this image — counting happens in the agent step)
+        head -c 300 .ci-alert-monitor/blips-raw.json; echo
       - |
         # Cluster context for the triage agent — gathered here because the
         # agent itself gets NO cluster access (no Bash, no kubeconfig). Runs
@@ -121,17 +134,41 @@ steps:
           .ci-alert-monitor/open-issues.json)
         FPS=$$(printf '%s' "$$NOISE_BODY" | grep -oE '\b[0-9a-f]{16}\b' \
           | jq -R -s -c 'split("\n") | map(select(length > 0))')
+        # Synthesize pseudo-alerts for BLIPS: lookback series whose label set
+        # is not currently firing at all (checked against the PRE-filter raw
+        # snapshot, so live-but-null-routed alerts don't reappear as blips).
+        # Pseudo-fingerprint = sha256(alertname|scope)[:16] — STABLE across
+        # recurrences, so one noise verdict suppresses a flappy alert
+        # permanently via the noise log.
+        : > .ci-alert-monitor/blips-new.jsonl
+        jq -c '.data.result[]?.metric // empty' .ci-alert-monitor/blips-raw.json | while IFS= read -r M; do
+          NAME=$$(printf '%s' "$$M" | jq -r '.alertname')
+          SCOPE=$$(printf '%s' "$$M" | jq -r '.namespace // .job // "cluster"')
+          if jq -e --arg n "$$NAME" --arg s "$$SCOPE" \
+               '[.[] | select(.labels.alertname == $$n and ((.labels.namespace // .labels.job // "cluster") == $$s))] | length > 0' \
+               .ci-alert-monitor/alerts-raw.json > /dev/null; then
+            continue
+          fi
+          FP=$$(printf '%s|%s' "$$NAME" "$$SCOPE" | sha256sum | cut -c1-16)
+          jq -n --arg fp "$$FP" --argjson m "$$M" \
+            '{labels: $$m, fingerprint: $$fp, blip: true,
+              annotations: {summary: "BLIP: fired within the lookback window (~35 min) and already resolved — caught by the ALERTS history scan, not currently active."}}' \
+            >> .ci-alert-monitor/blips-new.jsonl
+        done
+        jq -s '.' .ci-alert-monitor/blips-new.jsonl > .ci-alert-monitor/blips-candidates.json
         # Title key: namespace, else job, else "cluster". The job fallback
         # exists because aggregate alerts like TargetDown carry no namespace
         # — with only "(cluster)" two simultaneous TargetDowns for different
         # jobs (pve and dns-servers, 2026-07-16) collapsed into ONE issue
         # and the second root cause was never triaged.
+        jq -s 'add' .ci-alert-monitor/alerts-active.json .ci-alert-monitor/blips-candidates.json \
+          > .ci-alert-monitor/alerts-combined.json
         jq --argjson titles "$$TITLES" --argjson fps "$${FPS:-[]}" '
           [ .[]
             | ("[alert] " + .labels.alertname + " (" + (.labels.namespace // .labels.job // "cluster") + ")") as $$t
             | select( ($$t | IN($$titles[]) | not) and (.fingerprint | IN($$fps[]) | not) )
             | . + {issue_title: $$t} ]' \
-          .ci-alert-monitor/alerts-active.json > .ci-alert-monitor/alerts-new.json
+          .ci-alert-monitor/alerts-combined.json > .ci-alert-monitor/alerts-new.json
         NEW=$$(jq length .ci-alert-monitor/alerts-new.json)
         jq -r '.[] | "\(.fingerprint)\t\(.issue_title)"' .ci-alert-monitor/alerts-new.json
         if [ "$$NEW" -eq 0 ]; then

ci/agents/alert-monitor/prompt.md

@@ -16,12 +16,26 @@ to verify before you classify.
 All inputs are in the **current working directory** (the project root, where
 `ansible/`, `kubernetes/`, `opentofu/` live):
 
-- `.ci-alert-monitor/alerts-new.json` — array of Alertmanager v2 alert
-  objects, already filtered: only active, unsilenced, uninhibited alerts that
-  have no open issue and are not in the noise log. Each has `labels`
-  (including `alertname`, usually `namespace` and `severity`), `annotations`
-  (`summary`, `description`, often `runbook_url`), `startsAt`, `fingerprint`,
-  and an added `issue_title` field.
+- `.ci-alert-monitor/alerts-new.json` — array of alert objects, already
+  filtered: no open issue, not in the noise log, unsilenced/uninhibited.
+  Two kinds appear:
+  - **live alerts** (Alertmanager v2 objects): `labels` (including
+    `alertname`, usually `namespace` and `severity`), `annotations`
+    (`summary`, `description`, often `runbook_url`), `startsAt`,
+    `fingerprint`, and an added `issue_title`.
+  - **blips** (`"blip": true`): alerts that fired within the last ~35
+    minutes but already resolved — reconstructed from Prometheus's ALERTS
+    history, so they carry only identity labels, a synthetic stable
+    `fingerprint`, and a summary noting they are no longer active. For a
+    blip, judge whether the underlying cause needs action despite
+    self-resolving: recurring flaps and short outages of real services are
+    `problem`; benign transients are `noise` (the synthetic fingerprint is
+    stable, so one noise verdict suppresses that flap permanently). Blips
+    have no receiver info — check the Alertmanager route config in
+    `kubernetes/apps/monitoring/helmrelease-prometheus.yaml`: an alert the
+    owner routes to the `null` receiver is `noise` by prior decision. Use
+    `./.ci-diag/kubectl get --raw` with a Prometheus `query_range` on
+    `ALERTS{...}` to see the blip's exact firing window when timing matters.
 - `.ci-alert-monitor/context.txt` — cluster snapshot taken minutes ago: node
   status, non-running pods, recent Warning events, Flux kustomizations and
   helmreleases that are not ready.