diff --git a/kubernetes/apps/monitoring-rules/prometheusrule-woodpecker-rotation.yaml b/kubernetes/apps/monitoring-rules/prometheusrule-woodpecker-rotation.yaml index 0081f4ea..d5fef46e 100644 --- a/kubernetes/apps/monitoring-rules/prometheusrule-woodpecker-rotation.yaml +++ b/kubernetes/apps/monitoring-rules/prometheusrule-woodpecker-rotation.yaml @@ -12,37 +12,37 @@ spec: groups: - name: woodpecker-rotation.rules # Convergence safety net for the automated WOODPECKER_AGENT_SECRET rotation - # (kubernetes/apps/woodpecker/rotate-agent-secret.yaml). That rotation Job is + # (kubernetes/infrastructure/openbao/rotation/woodpecker.yaml). That rotation Job is # fire-and-forget: it writes OpenBao and exits WITHOUT verifying ESO re-synced # or that both consumers restarted healthy. These alerts close that gap. ESO # metrics are not scraped in this cluster, so everything here is built on - # kube-state-metrics. The CronJob/Job series stay empty until the rotation - # feature (branch feat/woodpecker-secret-auto-rotation) is merged — harmless. + # kube-state-metrics. The rotation has since landed: the CronJob lives in the + # openbao namespace, not apps, which is where the last two rules look. rules: - alert: WoodpeckerServerNotReady # 0 ready replicas well past the 10m ESO refresh + Reloader restart cycle. - expr: kube_statefulset_status_replicas_ready{namespace="apps",statefulset="woodpecker-server"} == 0 + expr: kube_deployment_status_replicas_ready{namespace="woodpecker",deployment="woodpecker-server"} == 0 for: 20m labels: severity: critical annotations: summary: "Woodpecker CI server has no ready replica" description: >- - StatefulSet woodpecker-server (ns apps) has had 0 ready replicas for 20m; - all CI is down (agents cannot reach woodpecker.apps.svc:9000). If this - followed a secret rotation, the server pod failed to return after its - Reloader-triggered restart — check `kubectl -n apps describe pod - woodpecker-server-0` (PVC attach / crashloop) and the woodpecker-secret - ExternalSecret sync status. + Deployment woodpecker-server (ns woodpecker) has had 0 ready replicas + for 20m; all CI is down (agents cannot reach + woodpecker.woodpecker.svc:9000). If this followed a secret rotation, the + server pod failed to return after its Reloader-triggered restart — check + `kubectl -n woodpecker describe deploy woodpecker-server` and the + woodpecker-secret ExternalSecret sync status. - alert: WoodpeckerAgentNotReady - expr: kube_deployment_status_replicas_available{namespace="apps",deployment="woodpecker-agent"} == 0 + expr: kube_deployment_status_replicas_available{namespace="woodpecker",deployment="woodpecker-agent"} == 0 for: 20m labels: severity: warning annotations: summary: "Woodpecker CI agent has no available replica" description: >- - Deployment woodpecker-agent (ns apps) has had 0 available replicas for + Deployment woodpecker-agent (ns woodpecker) has had 0 available replicas for 20m; no pipelines will run. If this followed a secret rotation, the agent did not return after its Reloader restart, or its WOODPECKER_AGENT_SECRET no longer matches the server's — check the agent pod logs for a gRPC auth @@ -50,29 +50,29 @@ spec: - alert: WoodpeckerSecretRotationJobFailed # Named, clearer signal than the generic KubeJobFailed for the OpenBao-write # step. Fires while a failed rotation Job is retained (ttl 3d). - expr: kube_job_status_failed{namespace="apps",job_name=~"woodpecker-secret-rotate.*"} > 0 + expr: kube_job_status_failed{namespace="openbao",job_name=~"woodpecker-secret-rotate.*"} > 0 for: 15m labels: severity: warning annotations: summary: "Woodpecker agent-secret rotation Job failed" description: >- - A woodpecker-secret-rotate Job (ns apps) failed at the OpenBao-write step; + A woodpecker-secret-rotate Job (ns openbao) failed at the OpenBao-write step; the agent secret was NOT rotated. Most common cause: the one-time OpenBao `rotator` role/policy bootstrap was never applied (see kubernetes/infrastructure/openbao/policies/rotator.hcl). Check - `kubectl -n apps logs job/`. + `kubectl -n openbao logs job/`. - alert: WoodpeckerSecretRotationStale # Quarterly cadence (~91d) + grace. The series is ABSENT until the first # successful run, so this cannot false-fire before rotation has ever worked. - expr: (time() - kube_cronjob_status_last_successful_time{namespace="apps",cronjob="woodpecker-secret-rotate"}) > 100 * 24 * 3600 + expr: (time() - kube_cronjob_status_last_successful_time{namespace="openbao",cronjob="woodpecker-secret-rotate"}) > 100 * 24 * 3600 for: 1h labels: severity: warning annotations: summary: "Woodpecker agent secret has not rotated in >100 days" description: >- - The quarterly woodpecker-secret-rotate CronJob (ns apps) has not completed + The quarterly woodpecker-secret-rotate CronJob (ns openbao) has not completed successfully in over 100 days — rotation has silently stopped (CronJob suspended/deleted, or every run failing). Investigate the CronJob and its recent Jobs. diff --git a/kubernetes/infrastructure/openbao/rotation/claude-oauth.yaml b/kubernetes/infrastructure/openbao/rotation/claude-oauth.yaml index e88d7c54..02176172 100644 --- a/kubernetes/infrastructure/openbao/rotation/claude-oauth.yaml +++ b/kubernetes/infrastructure/openbao/rotation/claude-oauth.yaml @@ -17,7 +17,7 @@ # woodpecker is reached in-cluster at woodpecker.woodpecker.svc:8000 (it moved # out of `apps` in the namespace migration — see # kubernetes/apps/woodpecker/MIGRATION-namespace.md). Needs the netpol -# halves woodpecker-allow-rotators (apps/*) + rotator-claude-oauth-egress +# halves woodpecker-allow-rotators (woodpecker ns) + rotator-claude-oauth-egress # (openbao-ns.yaml). OpenBao policy/role rotator-claude-oauth + the bootstraps # (woodpecker-bot token, the seeded OAuth value) are provisioned by hand — see # policies/rotator-claude-oauth.hcl. diff --git a/kubernetes/infrastructure/openbao/rotation/review-bot.yaml b/kubernetes/infrastructure/openbao/rotation/review-bot.yaml index 7dfcc7b1..37de79b9 100644 --- a/kubernetes/infrastructure/openbao/rotation/review-bot.yaml +++ b/kubernetes/infrastructure/openbao/rotation/review-bot.yaml @@ -28,7 +28,7 @@ # woodpecker.woodpecker.svc:8000 (it moved out of `apps` in the namespace # migration — see kubernetes/apps/woodpecker/MIGRATION-namespace.md). Needs # the netpol halves: gitea-allow-review-bot-rotator + woodpecker-allow-rotators -# (apps/*) and rotator-review-bot-egress (openbao-ns.yaml). +# (woodpecker ns) and rotator-review-bot-egress (openbao-ns.yaml). # # OpenBao policy/role rotator-review-bot + the bootstraps (review-bot password, # woodpecker-bot token, seed review-bot#token) are provisioned by hand — see