Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
  claude-botandClaude Opus 5.5
|
72927293e2 | ||
|
claude-botandClaude Opus 5.5
|
a35bc6b91d |
@@ -18,6 +18,7 @@ resources:
|
||||
- prometheusrule-netbird-availability.yaml
|
||||
- prometheusrule-netbird-ingress.yaml
|
||||
- prometheusrule-openbao.yaml
|
||||
- prometheusrule-uptime-kuma.yaml
|
||||
# Prometheus's own retention. Until REN-107 `retention: 90d` and
|
||||
# `retentionSize: 30GB` were both set, the size cap bound first, and the
|
||||
# effective window was a 20.5-27.2 day saw-tooth with nothing reporting it.
|
||||
|
||||
@@ -44,6 +44,8 @@ spec:
|
||||
# traverses the WAN and detects a controller/DC outage (the
|
||||
# 2026-07-24 one ran ~20 h with no signal). Dashboard root, 200.
|
||||
- https://netbird.renner.uno
|
||||
# Dead-man probe for the off-site Uptime Kuma.
|
||||
- https://kuma.renner.uno
|
||||
# NetBird relay health endpoints on the netcup members (relay
|
||||
# binary serves :9000/health, verified on 0.73.2). Plain http,
|
||||
# ports outside the hairpin's 80/443 — real WAN path.
|
||||
|
||||
@@ -26,6 +26,7 @@ spec:
|
||||
# proxy-1/2:9000/health same file, relay redundancy
|
||||
# ci.renner.uno WoodpeckerDown
|
||||
# openbao .../sys/health OpenBaoUnavailable
|
||||
# kuma.renner.uno UptimeKumaUnreachable
|
||||
#
|
||||
# Warning, not critical, on purpose: this rule cannot know what a given
|
||||
# target means. A target whose loss deserves a critical page gets its
|
||||
@@ -33,7 +34,7 @@ spec:
|
||||
# exclusion list from being arbitrary.
|
||||
# anchor: blackbox-probe-failing
|
||||
- alert: BlackboxProbeFailing
|
||||
expr: probe_success{job="blackbox-http", instance!~"https://fsn1\\.your-objectstorage\\.com|https://netbird\\.renner\\.uno|http://proxy-[12]\\.renner\\.uno:9000/health|https://ci\\.renner\\.uno|https://openbao\\.renner\\.uno/v1/sys/health.*"} == 0
|
||||
expr: probe_success{job="blackbox-http", instance!~"https://fsn1\\.your-objectstorage\\.com|https://netbird\\.renner\\.uno|http://proxy-[12]\\.renner\\.uno:9000/health|https://ci\\.renner\\.uno|https://kuma\\.renner\\.uno|https://openbao\\.renner\\.uno/v1/sys/health.*"} == 0
|
||||
for: 10m
|
||||
annotations:
|
||||
summary: "Blackbox probe of {{ $labels.instance }} has been failing for 10 minutes"
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PrometheusRule
|
||||
metadata:
|
||||
name: uptime-kuma
|
||||
namespace: monitoring
|
||||
labels:
|
||||
# kube-prometheus-stack's Prometheus selects on this label.
|
||||
release: kube-prometheus-stack
|
||||
spec:
|
||||
groups:
|
||||
- name: uptime-kuma.rules
|
||||
rules:
|
||||
# Dead-man probe for the off-site Uptime Kuma.
|
||||
- alert: UptimeKumaUnreachable
|
||||
expr: probe_success{job="blackbox-http", instance="https://kuma.renner.uno"} == 0
|
||||
for: 10m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "Uptime Kuma (kuma.renner.uno) unreachable for 10m+"
|
||||
description: "The blackbox probe of the off-site Uptime Kuma has been failing for 10 minutes. The external endpoints it monitors are unwatched until it returns. Check the NetBird VPS (netbird.renner.uno), the uptime-kuma container and Traefik."
|
||||
|
||||
# monitor_response_time is ms; -1 when a beat has no ping (DOWN, group).
|
||||
# anchor: uptime-kuma-slow-response
|
||||
- alert: UptimeKumaSlowResponse
|
||||
expr: avg by (monitor_name) (avg_over_time((monitor_response_time > 0)[15m:1m])) > 1500
|
||||
for: 15m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Uptime Kuma monitor {{ $labels.monitor_name }} averaging {{ $value | humanize }} ms response time"
|
||||
description: >-
|
||||
The average response time of monitor {{ $labels.monitor_name }},
|
||||
as measured from the off-site Uptime Kuma, has stayed above
|
||||
1500 ms for 15 minutes.
|
||||
@@ -32,6 +32,7 @@ retired the depth rule in favour of one persistence rule (`> 5` for `20m`).
|
||||
| `ceph_test.yaml` | Hand-written suite for the `CephPoolLostRedundancyBlind` canary in `prometheusrule-ceph.yaml` (REN-93). |
|
||||
| `disk-wear_test.yaml` | Hand-written suite for the EPHEMERAL exclusion in `prometheusrule-disk-wear.yaml` (REN-419). |
|
||||
| `gitea_test.yaml` | Hand-written suite for `GiteaDataVolumeLow` in `prometheusrule-gitea.yaml` (REN-583). |
|
||||
| `uptime-kuma_test.yaml` | Hand-written suite for `prometheusrule-uptime-kuma.yaml` and the kuma exclusion in `prometheusrule-blackbox.yaml`. |
|
||||
| `mutations.txt` | Negative controls. Each must make the suite fail. |
|
||||
|
||||
## Adding a suite
|
||||
|
||||
@@ -172,3 +172,12 @@ disk-wear-ephemeral-keeps-partition|kubernetes/apps/monitoring-rules/prometheusr
|
||||
# exactly-25% case in gitea_test.yaml.
|
||||
gitea-for-10m|kubernetes/apps/monitoring-rules/prometheusrule-gitea.yaml|/alert: GiteaDataVolumeLow/,/for:/s/for: 15m/for: 10m/
|
||||
gitea-floor-le|kubernetes/apps/monitoring-rules/prometheusrule-gitea.yaml|s/^\([[:space:]]*\)< 0\.25$/\1<= 0.25/
|
||||
|
||||
# prometheusrule-uptime-kuma.yaml / blackbox kuma exclusion. Detected by the
|
||||
# cases in uptime-kuma_test.yaml.
|
||||
kuma-slow-no-positive-filter|kubernetes/apps/monitoring-rules/prometheusrule-uptime-kuma.yaml|s/(monitor_response_time > 0)/(monitor_response_time)/
|
||||
kuma-slow-threshold-2500|kubernetes/apps/monitoring-rules/prometheusrule-uptime-kuma.yaml|s/\[15m:1m\])) > 1500$/[15m:1m])) > 2500/
|
||||
kuma-slow-threshold-900|kubernetes/apps/monitoring-rules/prometheusrule-uptime-kuma.yaml|s/\[15m:1m\])) > 1500$/[15m:1m])) > 900/
|
||||
kuma-slow-for-30m|kubernetes/apps/monitoring-rules/prometheusrule-uptime-kuma.yaml|/alert: UptimeKumaSlowResponse/,/for:/s/for: 15m/for: 30m/
|
||||
kuma-down-for-20m|kubernetes/apps/monitoring-rules/prometheusrule-uptime-kuma.yaml|/alert: UptimeKumaUnreachable/,/for:/s/for: 10m/for: 20m/
|
||||
kuma-blackbox-exclusion-dropped|kubernetes/apps/monitoring-rules/prometheusrule-blackbox.yaml|s/kuma\\\\\.renner/kumx\\\\.renner/
|
||||
|
||||
@@ -0,0 +1,110 @@
|
||||
# Suite for prometheusrule-uptime-kuma.yaml and the kuma exclusion in
|
||||
# prometheusrule-blackbox.yaml.
|
||||
rule_files:
|
||||
- ../prometheusrule-uptime-kuma.yaml
|
||||
- ../prometheusrule-blackbox.yaml
|
||||
|
||||
evaluation_interval: 1m
|
||||
|
||||
tests:
|
||||
# 2000 ms sustained: pending at 0m, fires 15m later, not before.
|
||||
- interval: 1m
|
||||
input_series:
|
||||
- series: 'monitor_response_time{monitor_name="slow-mon"}'
|
||||
values: '2000+0x40'
|
||||
alert_rule_test:
|
||||
- eval_time: 14m
|
||||
alertname: UptimeKumaSlowResponse
|
||||
exp_alerts: []
|
||||
- eval_time: 16m
|
||||
alertname: UptimeKumaSlowResponse
|
||||
exp_alerts:
|
||||
- exp_labels:
|
||||
severity: warning
|
||||
monitor_name: slow-mon
|
||||
exp_annotations:
|
||||
summary: "Uptime Kuma monitor slow-mon averaging 2k ms response time"
|
||||
description: "The average response time of monitor slow-mon, as measured from the off-site Uptime Kuma, has stayed above 1500 ms for 15 minutes."
|
||||
|
||||
# 1000 ms stays silent; a monitor reporting only -1 (DOWN, group) stays silent.
|
||||
- interval: 1m
|
||||
input_series:
|
||||
- series: 'monitor_response_time{monitor_name="fast-mon"}'
|
||||
values: '1000+0x40'
|
||||
- series: 'monitor_response_time{monitor_name="down-mon"}'
|
||||
values: '-1+0x40'
|
||||
alert_rule_test:
|
||||
- eval_time: 35m
|
||||
alertname: UptimeKumaSlowResponse
|
||||
exp_alerts: []
|
||||
|
||||
# Alternating 1000 and -1: the -1 samples must not drag a slow average down
|
||||
# nor count as slow; positives average 1000, silent.
|
||||
- interval: 1m
|
||||
input_series:
|
||||
- series: 'monitor_response_time{monitor_name="flap-fast"}'
|
||||
values: '1000 -1 1000 -1 1000 -1 1000 -1 1000 -1 1000 -1 1000 -1 1000 -1 1000 -1 1000 -1 1000 -1 1000 -1 1000 -1 1000 -1 1000 -1 1000 -1 1000 -1 1000 -1 1000 -1 1000 -1'
|
||||
alert_rule_test:
|
||||
- eval_time: 35m
|
||||
alertname: UptimeKumaSlowResponse
|
||||
exp_alerts: []
|
||||
|
||||
# Alternating 2000 and -1: the -1 samples are excluded, so the average of the
|
||||
# real samples is 2000 and the alert fires. Unfiltered, the mean is ~500.
|
||||
- interval: 1m
|
||||
input_series:
|
||||
- series: 'monitor_response_time{monitor_name="flap-slow"}'
|
||||
values: '2000 -1 2000 -1 2000 -1 2000 -1 2000 -1 2000 -1 2000 -1 2000 -1 2000 -1 2000 -1 2000 -1 2000 -1 2000 -1 2000 -1 2000 -1 2000 -1 2000 -1 2000 -1 2000 -1 2000 -1'
|
||||
alert_rule_test:
|
||||
- eval_time: 35m
|
||||
alertname: UptimeKumaSlowResponse
|
||||
exp_alerts:
|
||||
- exp_labels:
|
||||
severity: warning
|
||||
monitor_name: flap-slow
|
||||
exp_annotations:
|
||||
summary: "Uptime Kuma monitor flap-slow averaging 2k ms response time"
|
||||
description: "The average response time of monitor flap-slow, as measured from the off-site Uptime Kuma, has stayed above 1500 ms for 15 minutes."
|
||||
|
||||
# Dead-man probe: critical after 10m of failure; the catch-all warning must
|
||||
# not also fire for kuma, but still fires for another failing instance.
|
||||
- interval: 1m
|
||||
input_series:
|
||||
- series: 'probe_success{job="blackbox-http",instance="https://kuma.renner.uno"}'
|
||||
values: '0+0x40'
|
||||
- series: 'probe_success{job="blackbox-http",instance="https://other.renner.uno"}'
|
||||
values: '0+0x40'
|
||||
alert_rule_test:
|
||||
- eval_time: 9m
|
||||
alertname: UptimeKumaUnreachable
|
||||
exp_alerts: []
|
||||
- eval_time: 11m
|
||||
alertname: UptimeKumaUnreachable
|
||||
exp_alerts:
|
||||
- exp_labels:
|
||||
severity: critical
|
||||
job: blackbox-http
|
||||
instance: https://kuma.renner.uno
|
||||
exp_annotations:
|
||||
summary: "Uptime Kuma (kuma.renner.uno) unreachable for 10m+"
|
||||
description: "The blackbox probe of the off-site Uptime Kuma has been failing for 10 minutes. The external endpoints it monitors are unwatched until it returns. Check the NetBird VPS (netbird.renner.uno), the uptime-kuma container and Traefik."
|
||||
- eval_time: 15m
|
||||
alertname: BlackboxProbeFailing
|
||||
exp_alerts:
|
||||
- exp_labels:
|
||||
severity: warning
|
||||
job: blackbox-http
|
||||
instance: https://other.renner.uno
|
||||
exp_annotations:
|
||||
summary: "Blackbox probe of https://other.renner.uno has been failing for 10 minutes"
|
||||
description: "The endpoint is not answering as expected. For the Authentik-gated hostnames this is a reachability check covering DNS, TLS, Traefik and the gate outpost — it says nothing about the app behind the gate, which the internal /-/healthy probes cover separately."
|
||||
|
||||
# Healthy probe stays silent.
|
||||
- interval: 1m
|
||||
input_series:
|
||||
- series: 'probe_success{job="blackbox-http",instance="https://kuma.renner.uno"}'
|
||||
values: '1+0x40'
|
||||
alert_rule_test:
|
||||
- eval_time: 30m
|
||||
alertname: UptimeKumaUnreachable
|
||||
exp_alerts: []
|
||||
@@ -97,6 +97,10 @@ spec:
|
||||
resources:
|
||||
requests:
|
||||
storage: 50Gi
|
||||
# Mounted at /etc/prometheus/secrets/<name>/; see
|
||||
# ./uptime-kuma-metrics-externalsecret.yaml.
|
||||
secrets:
|
||||
- uptime-kuma-metrics
|
||||
# Additional scrape targets not covered by ServiceMonitors.
|
||||
additionalScrapeConfigs:
|
||||
# Gitea exposes /metrics anonymously on its main HTTP port. The
|
||||
@@ -106,6 +110,16 @@ spec:
|
||||
static_configs:
|
||||
- targets: ["gitea.gitea.svc.cluster.local:3000"]
|
||||
metrics_path: /metrics
|
||||
# Off-site Uptime Kuma. /metrics takes the API key as the basic-auth
|
||||
# password with an empty username.
|
||||
- job_name: uptime-kuma
|
||||
scheme: https
|
||||
metrics_path: /metrics
|
||||
basic_auth:
|
||||
username: ""
|
||||
password_file: /etc/prometheus/secrets/uptime-kuma-metrics/api_key
|
||||
static_configs:
|
||||
- targets: ["kuma.renner.uno:443"]
|
||||
|
||||
# Grafana
|
||||
grafana:
|
||||
|
||||
@@ -3,6 +3,7 @@ kind: Kustomization
|
||||
resources:
|
||||
- grafana-oauth-externalsecret.yaml
|
||||
- alertmanager-smtp-externalsecret.yaml
|
||||
- uptime-kuma-metrics-externalsecret.yaml
|
||||
- helmrelease-prometheus.yaml
|
||||
- helmrelease-loki.yaml
|
||||
- helmrelease-alloy-syslog.yaml
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
---
|
||||
# Uptime Kuma /metrics API key for Prometheus; OpenBao homelab/infrastructure/uptime-kuma-metrics.
|
||||
apiVersion: external-secrets.io/v1
|
||||
kind: ExternalSecret
|
||||
metadata:
|
||||
name: uptime-kuma-metrics
|
||||
namespace: monitoring
|
||||
spec:
|
||||
refreshInterval: 1h
|
||||
secretStoreRef:
|
||||
kind: ClusterSecretStore
|
||||
name: openbao
|
||||
target:
|
||||
name: uptime-kuma-metrics
|
||||
data:
|
||||
- secretKey: api_key
|
||||
remoteRef:
|
||||
key: infrastructure/uptime-kuma-metrics
|
||||
property: api_key
|
||||
Reference in New Issue
Block a user