feat(monitoring): alert when k8s-gateway stops answering renner.uno Ingress names
From 2026-10-08 to 2026-10-16 k8s-gateway ran the chart's default Corefile (zone example.com, forward . /etc/resolv.conf). Pods stayed Ready, every renner.uno lookup was forwarded to the UDM and still answered (public names as CNAME proxy.renner.uno), and nothing alerted. probes-dns.yaml only asks the UDM. Two blackbox DNS modules ask the k8s-gateway Service (by name, not IP) for auth.renner.uno A over IPv4 (the path CoreDNS forwards on, 10.96.0.11) and AAAA over IPv6, and fail unless every answer RR is the Traefik VIP (10.2.99.86 / fd99::86). A forwarded answer — CNAME or any other address — fails. CoreDNS is bypassed on purpose: its renner.uno block serves stale for 1h. K8sGatewayIngressAnswerWrong (critical) fires when the 10m success average is under 0.8 for 5m: about 7m after a total failure starts, and also when one of the two replicas is broken (average ~0.5), where a `for` on the raw value would keep resetting. One or two failed probes stay silent. K8sGatewayProbesAbsent is the dead-man's switch. Verified: the module config was run with blackbox_exporter 0.29.0 against the live cluster resolver (which relays k8s-gateway's answer): A and AAAA succeed; a k8s-gateway-forwarded name (ha.renner.uno, 10.2.99.87) and a wildcard name (CNAME proxy.renner.uno) fail. promtool-rules.sh with promtool 3.15.0: 9 suites green, 41 mutations caught (4 new). Closes #1290
This commit is contained in:
@@ -41,6 +41,10 @@ resources:
|
||||
# UDM. Catches a cause external-dns cannot see, e.g. records removed there.
|
||||
- prometheusrule-dns-resolution.yaml
|
||||
- probes-dns.yaml
|
||||
# The cluster side: k8s-gateway itself answering an Ingress name with the
|
||||
# Traefik VIP. Its default Corefile forwarded renner.uno for 8 days unseen (#1290).
|
||||
- prometheusrule-k8s-gateway.yaml
|
||||
- probes-k8s-gateway.yaml
|
||||
- alloy-servicemonitor.yaml
|
||||
# Agent metrics. Ships the metrics Service as well as the ServiceMonitor:
|
||||
# the chart renders neither under our values, so this scraped nothing and
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
# k8s-gateway's own answer for an in-cluster Ingress name (issue #1290).
|
||||
#
|
||||
# Queried at the Service directly, not through CoreDNS: CoreDNS' renner.uno
|
||||
# block caches with `serve_stale 1h immediate` and would hide an outage. The
|
||||
# module rejects any answer that is not the Traefik VIP, so a k8s-gateway that
|
||||
# has lost the zone and forwards renner.uno to the UDM fails here.
|
||||
#
|
||||
# One Probe per module: the dns prober reads the query from the module, and
|
||||
# each module pins one address family. `release` is load-bearing.
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: Probe
|
||||
metadata:
|
||||
name: dns-k8s-gateway-auth-a
|
||||
namespace: monitoring
|
||||
labels:
|
||||
release: kube-prometheus-stack
|
||||
spec:
|
||||
jobName: blackbox-k8s-gateway
|
||||
prober:
|
||||
url: blackbox-exporter-prometheus-blackbox-exporter.monitoring.svc.cluster.local:9115
|
||||
module: dns_k8s_gateway_auth_a
|
||||
interval: 60s
|
||||
scrapeTimeout: 10s
|
||||
targets:
|
||||
staticConfig:
|
||||
static:
|
||||
- k8s-gateway.kube-system.svc.cluster.local:53
|
||||
labels:
|
||||
dns_name: auth.renner.uno
|
||||
record: A
|
||||
---
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: Probe
|
||||
metadata:
|
||||
name: dns-k8s-gateway-auth-aaaa
|
||||
namespace: monitoring
|
||||
labels:
|
||||
release: kube-prometheus-stack
|
||||
spec:
|
||||
jobName: blackbox-k8s-gateway
|
||||
prober:
|
||||
url: blackbox-exporter-prometheus-blackbox-exporter.monitoring.svc.cluster.local:9115
|
||||
module: dns_k8s_gateway_auth_aaaa
|
||||
interval: 60s
|
||||
scrapeTimeout: 10s
|
||||
targets:
|
||||
staticConfig:
|
||||
static:
|
||||
- k8s-gateway.kube-system.svc.cluster.local:53
|
||||
labels:
|
||||
dns_name: auth.renner.uno
|
||||
record: AAAA
|
||||
@@ -0,0 +1,53 @@
|
||||
# k8s-gateway answering for renner.uno (issue #1290).
|
||||
#
|
||||
# From 2026-10-08 to 2026-10-16 k8s-gateway ran the chart's default Corefile
|
||||
# (zone example.com) after a chart-schema change. Pods stayed Ready, every
|
||||
# renner.uno lookup was forwarded to the UDM and still answered, and nothing
|
||||
# alerted for eight days. The Probes in probes-k8s-gateway.yaml ask
|
||||
# k8s-gateway itself for an Ingress name and fail on anything but the VIP.
|
||||
#
|
||||
# Averaged over 10m rather than `== 0`: the ClusterIP balances over two
|
||||
# replicas, so one broken replica fails about half the probes and a plain
|
||||
# `for` on the raw value would keep resetting.
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PrometheusRule
|
||||
metadata:
|
||||
name: k8s-gateway
|
||||
namespace: monitoring
|
||||
labels:
|
||||
release: kube-prometheus-stack
|
||||
spec:
|
||||
groups:
|
||||
- name: k8s-gateway.rules
|
||||
rules:
|
||||
- alert: K8sGatewayIngressAnswerWrong
|
||||
expr: avg_over_time(probe_success{job="blackbox-k8s-gateway"}[10m]) < 0.8
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "k8s-gateway does not answer {{ $labels.dns_name }} {{ $labels.record }} with the Traefik VIP"
|
||||
description: >-
|
||||
{{ $value | humanizePercentage }} of probes in the last 10m got
|
||||
the Ingress address for {{ $labels.dns_name }} ({{ $labels.record }})
|
||||
from k8s-gateway. Pods then resolve Ingress names through the UDM
|
||||
and the public wildcard (CNAME proxy.renner.uno) and leave the
|
||||
cluster. Check the rendered Corefile in the k8s-gateway ConfigMap
|
||||
(kube-system) for the renner.uno zone, the k8s-gateway logs, and
|
||||
whether the Traefik VIP in infrastructure/traefik/helmrelease.yaml
|
||||
still matches the dns_k8s_gateway_auth_* modules in
|
||||
apps/monitoring/helmrelease-blackbox-exporter.yaml.
|
||||
|
||||
# Dead-man's-switch: without probe results the rule above cannot fire.
|
||||
- alert: K8sGatewayProbesAbsent
|
||||
expr: absent(probe_success{job="blackbox-k8s-gateway"})
|
||||
for: 15m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "No k8s-gateway DNS probe results are reaching Prometheus"
|
||||
description: >-
|
||||
The blackbox-k8s-gateway Probes stopped being discovered or the
|
||||
exporter rejects their modules (a 400 from /probe yields no
|
||||
probe_* series). Check that the dns_k8s_gateway_auth_* modules
|
||||
exist in the blackbox-exporter config.
|
||||
@@ -33,6 +33,7 @@ retired the depth rule in favour of one persistence rule (`> 5` for `20m`).
|
||||
| `disk-wear_test.yaml` | Hand-written suite for the EPHEMERAL exclusion in `prometheusrule-disk-wear.yaml` (REN-419). |
|
||||
| `gitea_test.yaml` | Hand-written suite for `GiteaDataVolumeLow` in `prometheusrule-gitea.yaml` (REN-583). |
|
||||
| `uptime-kuma_test.yaml` | Hand-written suite for `prometheusrule-uptime-kuma.yaml` and the kuma exclusion in `prometheusrule-blackbox.yaml`. |
|
||||
| `k8s-gateway_test.yaml` | Hand-written suite for `prometheusrule-k8s-gateway.yaml` (#1290). |
|
||||
| `mutations.txt` | Negative controls. Each must make the suite fail. |
|
||||
|
||||
## Adding a suite
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
# Suite for prometheusrule-k8s-gateway.yaml (issue #1290). Probes run every
|
||||
# 60s; one series per record type.
|
||||
rule_files:
|
||||
- ../prometheusrule-k8s-gateway.yaml
|
||||
|
||||
evaluation_interval: 1m
|
||||
|
||||
tests:
|
||||
# Total failure from minute 10: the 10m average drops under 0.8 at 12m
|
||||
# (three failures in the window), pending there, fires 5m later; by then
|
||||
# 2 of the 10 samples in the window are good.
|
||||
- interval: 1m
|
||||
input_series:
|
||||
- series: 'probe_success{job="blackbox-k8s-gateway",instance="k8s-gateway.kube-system.svc.cluster.local:53",dns_name="auth.renner.uno",record="A"}'
|
||||
values: '1+0x9 0+0x30'
|
||||
- series: 'probe_success{job="blackbox-k8s-gateway",instance="k8s-gateway.kube-system.svc.cluster.local:53",dns_name="auth.renner.uno",record="AAAA"}'
|
||||
values: '1+0x40'
|
||||
alert_rule_test:
|
||||
- eval_time: 16m
|
||||
alertname: K8sGatewayIngressAnswerWrong
|
||||
exp_alerts: []
|
||||
- eval_time: 17m
|
||||
alertname: K8sGatewayIngressAnswerWrong
|
||||
exp_alerts:
|
||||
- exp_labels:
|
||||
severity: critical
|
||||
job: blackbox-k8s-gateway
|
||||
instance: k8s-gateway.kube-system.svc.cluster.local:53
|
||||
dns_name: auth.renner.uno
|
||||
record: A
|
||||
exp_annotations:
|
||||
summary: "k8s-gateway does not answer auth.renner.uno A with the Traefik VIP"
|
||||
description: "20% of probes in the last 10m got the Ingress address for auth.renner.uno (A) from k8s-gateway. Pods then resolve Ingress names through the UDM and the public wildcard (CNAME proxy.renner.uno) and leave the cluster. Check the rendered Corefile in the k8s-gateway ConfigMap (kube-system) for the renner.uno zone, the k8s-gateway logs, and whether the Traefik VIP in infrastructure/traefik/helmrelease.yaml still matches the dns_k8s_gateway_auth_* modules in apps/monitoring/helmrelease-blackbox-exporter.yaml."
|
||||
- eval_time: 17m
|
||||
alertname: K8sGatewayProbesAbsent
|
||||
exp_alerts: []
|
||||
|
||||
# One of two replicas broken: probes alternate, the average sits at 0.5 and
|
||||
# the alert must not reset its `for` on every good sample.
|
||||
- interval: 1m
|
||||
input_series:
|
||||
- series: 'probe_success{job="blackbox-k8s-gateway",instance="k8s-gateway.kube-system.svc.cluster.local:53",dns_name="auth.renner.uno",record="AAAA"}'
|
||||
values: '1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1 0 1'
|
||||
alert_rule_test:
|
||||
- eval_time: 25m
|
||||
alertname: K8sGatewayIngressAnswerWrong
|
||||
exp_alerts:
|
||||
- exp_labels:
|
||||
severity: critical
|
||||
job: blackbox-k8s-gateway
|
||||
instance: k8s-gateway.kube-system.svc.cluster.local:53
|
||||
dns_name: auth.renner.uno
|
||||
record: AAAA
|
||||
exp_annotations:
|
||||
summary: "k8s-gateway does not answer auth.renner.uno AAAA with the Traefik VIP"
|
||||
description: "50% of probes in the last 10m got the Ingress address for auth.renner.uno (AAAA) from k8s-gateway. Pods then resolve Ingress names through the UDM and the public wildcard (CNAME proxy.renner.uno) and leave the cluster. Check the rendered Corefile in the k8s-gateway ConfigMap (kube-system) for the renner.uno zone, the k8s-gateway logs, and whether the Traefik VIP in infrastructure/traefik/helmrelease.yaml still matches the dns_k8s_gateway_auth_* modules in apps/monitoring/helmrelease-blackbox-exporter.yaml."
|
||||
|
||||
# Two consecutive failures in an otherwise healthy hour stay silent
|
||||
# (average exactly 0.8).
|
||||
- interval: 1m
|
||||
input_series:
|
||||
- series: 'probe_success{job="blackbox-k8s-gateway",instance="k8s-gateway.kube-system.svc.cluster.local:53",dns_name="auth.renner.uno",record="A"}'
|
||||
values: '1+0x19 0 0 1+0x30'
|
||||
alert_rule_test:
|
||||
- eval_time: 25m
|
||||
alertname: K8sGatewayIngressAnswerWrong
|
||||
exp_alerts: []
|
||||
- eval_time: 45m
|
||||
alertname: K8sGatewayIngressAnswerWrong
|
||||
exp_alerts: []
|
||||
|
||||
# Probe results for another job do not satisfy the dead-man's-switch.
|
||||
- interval: 1m
|
||||
input_series:
|
||||
- series: 'probe_success{job="blackbox-dns",instance="10.2.40.1:53",dns_name="ha.renner.uno"}'
|
||||
values: '1+0x30'
|
||||
alert_rule_test:
|
||||
- eval_time: 14m
|
||||
alertname: K8sGatewayProbesAbsent
|
||||
exp_alerts: []
|
||||
- eval_time: 16m
|
||||
alertname: K8sGatewayProbesAbsent
|
||||
exp_alerts:
|
||||
- exp_labels:
|
||||
severity: critical
|
||||
job: blackbox-k8s-gateway
|
||||
exp_annotations:
|
||||
summary: "No k8s-gateway DNS probe results are reaching Prometheus"
|
||||
description: "The blackbox-k8s-gateway Probes stopped being discovered or the exporter rejects their modules (a 400 from /probe yields no probe_* series). Check that the dns_k8s_gateway_auth_* modules exist in the blackbox-exporter config."
|
||||
- eval_time: 25m
|
||||
alertname: K8sGatewayIngressAnswerWrong
|
||||
exp_alerts: []
|
||||
@@ -181,3 +181,14 @@ kuma-slow-threshold-900|kubernetes/apps/monitoring-rules/prometheusrule-uptime-k
|
||||
kuma-slow-for-30m|kubernetes/apps/monitoring-rules/prometheusrule-uptime-kuma.yaml|/alert: UptimeKumaSlowResponse/,/for:/s/for: 15m/for: 30m/
|
||||
kuma-down-for-20m|kubernetes/apps/monitoring-rules/prometheusrule-uptime-kuma.yaml|/alert: UptimeKumaUnreachable/,/for:/s/for: 10m/for: 20m/
|
||||
kuma-blackbox-exclusion-dropped|kubernetes/apps/monitoring-rules/prometheusrule-blackbox.yaml|s/kuma\\\\\.renner/kumx\\\\.renner/
|
||||
|
||||
# prometheusrule-k8s-gateway.yaml (issue #1290). Detected by the cases in
|
||||
# k8s-gateway_test.yaml.
|
||||
# Raw value instead of the average: the one-broken-replica case resets `for`.
|
||||
k8s-gateway-no-average|kubernetes/apps/monitoring-rules/prometheusrule-k8s-gateway.yaml|s/avg_over_time(probe_success{job="blackbox-k8s-gateway"}\[10m\]) < 0.8/probe_success{job="blackbox-k8s-gateway"} < 0.8/
|
||||
# Threshold under the one-replica average. Detected by that case.
|
||||
k8s-gateway-threshold-0.4|kubernetes/apps/monitoring-rules/prometheusrule-k8s-gateway.yaml|s/\[10m\]) < 0.8$/[10m]) < 0.4/
|
||||
# Non-strict comparison pages on two consecutive failures.
|
||||
k8s-gateway-le-0.8|kubernetes/apps/monitoring-rules/prometheusrule-k8s-gateway.yaml|s/\[10m\]) < 0.8$/[10m]) <= 0.8/
|
||||
# The window. Detected by the 16m/17m pair.
|
||||
k8s-gateway-for-10m|kubernetes/apps/monitoring-rules/prometheusrule-k8s-gateway.yaml|/alert: K8sGatewayIngressAnswerWrong/,/for:/s/for: 5m/for: 10m/
|
||||
|
||||
@@ -143,3 +143,31 @@ spec:
|
||||
validate_answer_rrs:
|
||||
fail_if_not_matches_regexp: ['\sIN\s+A\s']
|
||||
fail_if_matches_regexp: ['\sIN\s+CNAME\s']
|
||||
# k8s-gateway's own answer for an Ingress name, used by
|
||||
# monitoring-rules/probes-k8s-gateway.yaml. A forwarded answer
|
||||
# (CNAME proxy.renner.uno, or any other address) fails: every RR must
|
||||
# be the Traefik LB VIP from infrastructure/traefik/helmrelease.yaml.
|
||||
# The A query goes over IPv4, the path CoreDNS uses (forward . 10.96.0.11);
|
||||
# the AAAA query over IPv6. No fallback, so each module tests one family.
|
||||
dns_k8s_gateway_auth_a:
|
||||
prober: dns
|
||||
timeout: 5s
|
||||
dns:
|
||||
preferred_ip_protocol: ip4
|
||||
ip_protocol_fallback: false
|
||||
query_name: auth.renner.uno
|
||||
query_type: A
|
||||
valid_rcodes: [NOERROR]
|
||||
validate_answer_rrs:
|
||||
fail_if_not_matches_regexp: ['^auth\.renner\.uno\.\s+\d+\s+IN\s+A\s+10\.2\.99\.86$']
|
||||
dns_k8s_gateway_auth_aaaa:
|
||||
prober: dns
|
||||
timeout: 5s
|
||||
dns:
|
||||
preferred_ip_protocol: ip6
|
||||
ip_protocol_fallback: false
|
||||
query_name: auth.renner.uno
|
||||
query_type: AAAA
|
||||
valid_rcodes: [NOERROR]
|
||||
validate_answer_rrs:
|
||||
fail_if_not_matches_regexp: ['^auth\.renner\.uno\.\s+\d+\s+IN\s+AAAA\s+fd99::86$']
|
||||
|
||||
Reference in New Issue
Block a user