Compare commits

..
Author SHA1 Message Date
issue-fix-bot cf63aa61be fix: [alert] KubeJobFailed (openbao) (#554)
ci/woodpecker/pr/ci-manifests-test Pipeline was successful
ci/woodpecker/pr/ci-scripts-test Pipeline was successful
ci/woodpecker/pr/pr-review Pipeline was successful
2026-08-22 02:45:05 +00:00
security-bot 287cce7d1a chore(image-automation): bump ghcr.io/home-assistant/home-assistant:2026.8.3
ci/woodpecker/cron/update-all Pipeline failed
ci/woodpecker/cron/e2e Pipeline was successful
ci/woodpecker/cron/security-scan Pipeline was successful
ci/woodpecker/cron/issue-fix Pipeline was successful
ci/woodpecker/cron/alert-monitor Pipeline was successful
Updates managed by image-automation-controller.
2026-08-21 21:42:10 +00:00
4 changed files with 11 additions and 55 deletions
@@ -84,7 +84,7 @@ spec:
env:
- name: DB_URI
valueFrom:
secretKeyRef: { name: ha-pg-app, key: fqdn-uri }
secretKeyRef: {name: ha-pg-app, key: fqdn-uri}
volumeMounts:
- name: config
mountPath: /config
@@ -96,7 +96,7 @@ spec:
memory: 32Mi
containers:
- name: home-assistant
image: ghcr.io/home-assistant/home-assistant:2026.8.2 # {"$imagepolicy": "flux-system:home-assistant"}
image: ghcr.io/home-assistant/home-assistant:2026.8.3 # {"$imagepolicy": "flux-system:home-assistant"}
env:
- name: TZ
value: Europe/Berlin
@@ -8,7 +8,6 @@ resources:
- prometheusrule-cilium.yaml
- prometheusrule-hetzner-backup.yaml
- prometheusrule-loki.yaml
- prometheusrule-mariadb-backup.yaml
- prometheusrule-netbird-availability.yaml
- prometheusrule-netbird-backup.yaml
- prometheusrule-netbird-ingress.yaml
@@ -1,51 +0,0 @@
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
name: mariadb-backup
namespace: monitoring
labels:
# kube-prometheus-stack's Prometheus selects on this label.
release: kube-prometheus-stack
spec:
groups:
- name: mariadb-backup.rules
rules:
- alert: WordpressMariadbBackupStale
# The nightly 02:45 logical dump (kubernetes/apps/wordpress/mariadb.yaml)
# is the only transaction-consistent copy of www.renner.uno's database
# — Velero snapshots the PVC, which for a running DB is not a restore
# point you would trust.
#
# KubeJobFailed already covers a failed Job, but only while the Job
# object survives. On #550 it cleared when the failed Job was
# garbage-collected, not when a backup succeeded — so a failure that
# is GC'd before anyone looks leaves no signal at all.
# lastSuccessfulTime lives on the CronJob and outlives Job GC.
#
# 26h = one missed cycle plus 2h slack, matching NetbirdVpsBackupStale.
# It counts any Job owned by the CronJob, so a manual
# `kubectl create job --from=cronjob/...` re-run clears it too.
expr: |
time() - kube_cronjob_status_last_successful_time{namespace="wordpress", cronjob="wordpress-mariadb-backup"} > 26 * 3600
for: 30m
labels:
severity: warning
annotations:
summary: "WordPress MariaDB backup has not succeeded in 26h+"
description: "The wordpress-mariadb-backup CronJob's last successful run is over 26 hours old. Check the newest Job's pod logs in the wordpress namespace: `kubectl -n wordpress get jobs`. The 2026-08-19 failure was `The specified bucket does not exist.` against renner-uno-k8s-mariadb while the endpoint itself probed healthy, so a green HetznerBackupEndpointDown does not clear this."
- alert: WordpressMariadbBackupNeverRan
# The staleness expression above goes blind when the metric is
# absent, which is what happens once the operator recreates the
# CronJob. Without this guard "never backed up" is quieter than
# "backed up late".
expr: |
kube_cronjob_created{namespace="wordpress", cronjob="wordpress-mariadb-backup"}
unless
kube_cronjob_status_last_successful_time{namespace="wordpress", cronjob="wordpress-mariadb-backup"}
for: 6h
labels:
severity: warning
annotations:
summary: "WordPress MariaDB backup has no recorded successful run"
description: "The wordpress-mariadb-backup CronJob exists but has no lastSuccessfulTime: either it has not completed once since the Backup CR was recreated, or every run since has failed. Check `kubectl -n wordpress get jobs` and that the hetzner-s3-creds ExternalSecret is synced."
@@ -51,7 +51,15 @@ spec:
startingDeadlineSeconds: 3600
jobTemplate:
spec:
backoffLimit: 1
# 4 (not the sibling rotators' 1-2) because this is the only DAILY
# rotator — every other CronJob here is quarterly, so it takes ~90x as
# many runs and is correspondingly more exposed to per-run transients.
# backoffLimit: 1 gave two attempts ~11 s apart, and on 2026-08-19 a
# sub-minute resolver transient took both out (Job
# claude-oauth-secret-sync-29785100, BackoffLimitExceeded), costing a
# day's sync. 4 spreads the attempts over ~150 s (10/20/40/80 s backoff),
# still well inside activeDeadlineSeconds.
backoffLimit: 4
activeDeadlineSeconds: 300
ttlSecondsAfterFinished: 259200 # 3 d
template: