Compare commits

..
Author SHA1 Message Date
issue-fix-bot 08289947a1 fix: [alert] KubeJobFailed (wordpress) (#550)
ci/woodpecker/pr/ci-manifests-test Pipeline was successful
ci/woodpecker/pr/ci-scripts-test Pipeline was successful
ci/woodpecker/pr/pr-review Pipeline was successful
2026-08-21 21:47:22 +00:00
5 changed files with 55 additions and 22 deletions
@@ -84,7 +84,7 @@ spec:
env:
- name: DB_URI
valueFrom:
secretKeyRef: {name: ha-pg-app, key: fqdn-uri}
secretKeyRef: { name: ha-pg-app, key: fqdn-uri }
volumeMounts:
- name: config
mountPath: /config
@@ -96,7 +96,7 @@ spec:
memory: 32Mi
containers:
- name: home-assistant
image: ghcr.io/home-assistant/home-assistant:2026.8.3 # {"$imagepolicy": "flux-system:home-assistant"}
image: ghcr.io/home-assistant/home-assistant:2026.8.2 # {"$imagepolicy": "flux-system:home-assistant"}
env:
- name: TZ
value: Europe/Berlin
@@ -176,22 +176,3 @@ data:
friendly_name: Treppenhaus Haustür Schalter 1
'0x001788010b00d011':
friendly_name: Treppenhaus Haustür Schalter 2
# Ecowitt WS90 weather station. Joined after the ZHA-mirrored block above
# was written, hence listed apart from it.
#
# The station transmits every 8.8s, as a burst of seven per-cluster ZCL
# frames; Z2M publishes its full cumulative state once per frame. debounce
# merges a burst into one publish carrying the last value per attribute,
# and must stay below the transmit interval — a window that never sees
# silence never flushes.
#
# The firmware drops all readings to zero for a few seconds every few
# days. Merged, that surfaces as a single all-zero frame; unmerged, the
# zeros spread across the seven frames and none of them is individually
# distinguishable from a valid reading.
#
# Z2M reads this only at startup (requiresRestart), and this file is
# SEED-ONLY per the header — the running instance takes it from its PVC.
'0xc02cedfffe958f40':
friendly_name: Wetterstation
debounce: 2
@@ -158,7 +158,7 @@ spec:
# Pinned tag. No Flux image-automation marker yet (unlike mosquitto) —
# add an infrastructure/image-automation/zigbee2mqtt.yaml policy as a
# follow-up if desired; pinned-and-manual is fine for the initial cutover.
image: ghcr.io/koenkk/zigbee2mqtt:2.13.0
image: ghcr.io/koenkk/zigbee2mqtt:2.12.1
imagePullPolicy: IfNotPresent
env:
- name: TZ
@@ -8,6 +8,7 @@ resources:
- prometheusrule-cilium.yaml
- prometheusrule-hetzner-backup.yaml
- prometheusrule-loki.yaml
- prometheusrule-mariadb-backup.yaml
- prometheusrule-netbird-availability.yaml
- prometheusrule-netbird-backup.yaml
- prometheusrule-netbird-ingress.yaml
@@ -0,0 +1,51 @@
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
name: mariadb-backup
namespace: monitoring
labels:
# kube-prometheus-stack's Prometheus selects on this label.
release: kube-prometheus-stack
spec:
groups:
- name: mariadb-backup.rules
rules:
- alert: WordpressMariadbBackupStale
# The nightly 02:45 logical dump (kubernetes/apps/wordpress/mariadb.yaml)
# is the only transaction-consistent copy of www.renner.uno's database
# — Velero snapshots the PVC, which for a running DB is not a restore
# point you would trust.
#
# KubeJobFailed already covers a failed Job, but only while the Job
# object survives. On #550 it cleared when the failed Job was
# garbage-collected, not when a backup succeeded — so a failure that
# is GC'd before anyone looks leaves no signal at all.
# lastSuccessfulTime lives on the CronJob and outlives Job GC.
#
# 26h = one missed cycle plus 2h slack, matching NetbirdVpsBackupStale.
# It counts any Job owned by the CronJob, so a manual
# `kubectl create job --from=cronjob/...` re-run clears it too.
expr: |
time() - kube_cronjob_status_last_successful_time{namespace="wordpress", cronjob="wordpress-mariadb-backup"} > 26 * 3600
for: 30m
labels:
severity: warning
annotations:
summary: "WordPress MariaDB backup has not succeeded in 26h+"
description: "The wordpress-mariadb-backup CronJob's last successful run is over 26 hours old. Check the newest Job's pod logs in the wordpress namespace: `kubectl -n wordpress get jobs`. The 2026-08-19 failure was `The specified bucket does not exist.` against renner-uno-k8s-mariadb while the endpoint itself probed healthy, so a green HetznerBackupEndpointDown does not clear this."
- alert: WordpressMariadbBackupNeverRan
# The staleness expression above goes blind when the metric is
# absent, which is what happens once the operator recreates the
# CronJob. Without this guard "never backed up" is quieter than
# "backed up late".
expr: |
kube_cronjob_created{namespace="wordpress", cronjob="wordpress-mariadb-backup"}
unless
kube_cronjob_status_last_successful_time{namespace="wordpress", cronjob="wordpress-mariadb-backup"}
for: 6h
labels:
severity: warning
annotations:
summary: "WordPress MariaDB backup has no recorded successful run"
description: "The wordpress-mariadb-backup CronJob exists but has no lastSuccessfulTime: either it has not completed once since the Backup CR was recreated, or every run since has failed. Check `kubectl -n wordpress get jobs` and that the hetzner-s3-creds ExternalSecret is synced."