|
|
|
@@ -12,37 +12,37 @@ spec:
|
|
|
|
|
groups:
|
|
|
|
|
- name: woodpecker-rotation.rules
|
|
|
|
|
# Convergence safety net for the automated WOODPECKER_AGENT_SECRET rotation
|
|
|
|
|
# (kubernetes/apps/woodpecker/rotate-agent-secret.yaml). That rotation Job is
|
|
|
|
|
# (kubernetes/infrastructure/openbao/rotation/woodpecker.yaml). That rotation Job is
|
|
|
|
|
# fire-and-forget: it writes OpenBao and exits WITHOUT verifying ESO re-synced
|
|
|
|
|
# or that both consumers restarted healthy. These alerts close that gap. ESO
|
|
|
|
|
# metrics are not scraped in this cluster, so everything here is built on
|
|
|
|
|
# kube-state-metrics. The CronJob/Job series stay empty until the rotation
|
|
|
|
|
# feature (branch feat/woodpecker-secret-auto-rotation) is merged — harmless.
|
|
|
|
|
# kube-state-metrics. The rotation has since landed: the CronJob lives in the
|
|
|
|
|
# openbao namespace, not apps, which is where the last two rules look.
|
|
|
|
|
rules:
|
|
|
|
|
- alert: WoodpeckerServerNotReady
|
|
|
|
|
# 0 ready replicas well past the 10m ESO refresh + Reloader restart cycle.
|
|
|
|
|
expr: kube_statefulset_status_replicas_ready{namespace="apps",statefulset="woodpecker-server"} == 0
|
|
|
|
|
expr: kube_deployment_status_replicas_ready{namespace="woodpecker",deployment="woodpecker-server"} == 0
|
|
|
|
|
for: 20m
|
|
|
|
|
labels:
|
|
|
|
|
severity: critical
|
|
|
|
|
annotations:
|
|
|
|
|
summary: "Woodpecker CI server has no ready replica"
|
|
|
|
|
description: >-
|
|
|
|
|
StatefulSet woodpecker-server (ns apps) has had 0 ready replicas for 20m;
|
|
|
|
|
all CI is down (agents cannot reach woodpecker.apps.svc:9000). If this
|
|
|
|
|
followed a secret rotation, the server pod failed to return after its
|
|
|
|
|
Reloader-triggered restart — check `kubectl -n apps describe pod
|
|
|
|
|
woodpecker-server-0` (PVC attach / crashloop) and the woodpecker-secret
|
|
|
|
|
ExternalSecret sync status.
|
|
|
|
|
Deployment woodpecker-server (ns woodpecker) has had 0 ready replicas
|
|
|
|
|
for 20m; all CI is down (agents cannot reach
|
|
|
|
|
woodpecker.woodpecker.svc:9000). If this followed a secret rotation, the
|
|
|
|
|
server pod failed to return after its Reloader-triggered restart — check
|
|
|
|
|
`kubectl -n woodpecker describe deploy woodpecker-server` and the
|
|
|
|
|
woodpecker-secret ExternalSecret sync status.
|
|
|
|
|
- alert: WoodpeckerAgentNotReady
|
|
|
|
|
expr: kube_deployment_status_replicas_available{namespace="apps",deployment="woodpecker-agent"} == 0
|
|
|
|
|
expr: kube_deployment_status_replicas_available{namespace="woodpecker",deployment="woodpecker-agent"} == 0
|
|
|
|
|
for: 20m
|
|
|
|
|
labels:
|
|
|
|
|
severity: warning
|
|
|
|
|
annotations:
|
|
|
|
|
summary: "Woodpecker CI agent has no available replica"
|
|
|
|
|
description: >-
|
|
|
|
|
Deployment woodpecker-agent (ns apps) has had 0 available replicas for
|
|
|
|
|
Deployment woodpecker-agent (ns woodpecker) has had 0 available replicas for
|
|
|
|
|
20m; no pipelines will run. If this followed a secret rotation, the agent
|
|
|
|
|
did not return after its Reloader restart, or its WOODPECKER_AGENT_SECRET
|
|
|
|
|
no longer matches the server's — check the agent pod logs for a gRPC auth
|
|
|
|
@@ -50,29 +50,29 @@ spec:
|
|
|
|
|
- alert: WoodpeckerSecretRotationJobFailed
|
|
|
|
|
# Named, clearer signal than the generic KubeJobFailed for the OpenBao-write
|
|
|
|
|
# step. Fires while a failed rotation Job is retained (ttl 3d).
|
|
|
|
|
expr: kube_job_status_failed{namespace="apps",job_name=~"woodpecker-secret-rotate.*"} > 0
|
|
|
|
|
expr: kube_job_status_failed{namespace="openbao",job_name=~"woodpecker-secret-rotate.*"} > 0
|
|
|
|
|
for: 15m
|
|
|
|
|
labels:
|
|
|
|
|
severity: warning
|
|
|
|
|
annotations:
|
|
|
|
|
summary: "Woodpecker agent-secret rotation Job failed"
|
|
|
|
|
description: >-
|
|
|
|
|
A woodpecker-secret-rotate Job (ns apps) failed at the OpenBao-write step;
|
|
|
|
|
A woodpecker-secret-rotate Job (ns openbao) failed at the OpenBao-write step;
|
|
|
|
|
the agent secret was NOT rotated. Most common cause: the one-time OpenBao
|
|
|
|
|
`rotator` role/policy bootstrap was never applied (see
|
|
|
|
|
kubernetes/infrastructure/openbao/policies/rotator.hcl). Check
|
|
|
|
|
`kubectl -n apps logs job/<name>`.
|
|
|
|
|
`kubectl -n openbao logs job/<name>`.
|
|
|
|
|
- alert: WoodpeckerSecretRotationStale
|
|
|
|
|
# Quarterly cadence (~91d) + grace. The series is ABSENT until the first
|
|
|
|
|
# successful run, so this cannot false-fire before rotation has ever worked.
|
|
|
|
|
expr: (time() - kube_cronjob_status_last_successful_time{namespace="apps",cronjob="woodpecker-secret-rotate"}) > 100 * 24 * 3600
|
|
|
|
|
expr: (time() - kube_cronjob_status_last_successful_time{namespace="openbao",cronjob="woodpecker-secret-rotate"}) > 100 * 24 * 3600
|
|
|
|
|
for: 1h
|
|
|
|
|
labels:
|
|
|
|
|
severity: warning
|
|
|
|
|
annotations:
|
|
|
|
|
summary: "Woodpecker agent secret has not rotated in >100 days"
|
|
|
|
|
description: >-
|
|
|
|
|
The quarterly woodpecker-secret-rotate CronJob (ns apps) has not completed
|
|
|
|
|
The quarterly woodpecker-secret-rotate CronJob (ns openbao) has not completed
|
|
|
|
|
successfully in over 100 days — rotation has silently stopped (CronJob
|
|
|
|
|
suspended/deleted, or every run failing). Investigate the CronJob and its
|
|
|
|
|
recent Jobs.
|
|
|
|
|