test(monitoring): promtool suite for Uptime Kuma rules, terse comments

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
feat(monitoring): watch off-site Uptime Kuma from Prometheus
2026-10-19 08:26:12 +02:00 · 2026-10-19 08:17:30 +02:00 · 2026-10-19 06:14:48 +00:00 · 2026-10-19 06:13:48 +00:00 · 2026-10-19 06:13:17 +00:00 · 2026-10-19 08:12:45 +02:00
15 changed files with 207 additions and 25 deletions
@@ -133,18 +133,19 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
 	var id int64
 	if res.mon == nil {
 		// The monitor this resource synced before may have lost its tag; a
-		// second one would then sit beside it.
+		// second one would then sit beside it. An ID now carrying another
+		// owner's key was reused by Kuma, so ours is gone.
 		if prev := km.Status.MonitorID; prev != 0 {
 			found, key, err := r.Kuma.Probe(ctx, prev)
 			switch {
 			case err != nil:
 				return r.fail(ctx, km, &failure{ReasonKumaUnavailable, msgKumaUnavailable, err})
-			case found && key != desired.OwnerKey:
+			case found && key == "":
 				log.FromContext(ctx).Error(nil, "previous monitor exists without our ownership tag, not creating another",
 					"ownerKey", desired.OwnerKey, "monitorID", prev, "tagValue", key)
 				err := r.setReady(ctx, km, metav1.ConditionFalse, ReasonNotOwned, msgNotOwned, nil)
 				return ctrl.Result{RequeueAfter: r.resync()}, err
-			case found:
+			case found && key == desired.OwnerKey:
 				return r.fail(ctx, km, &failure{ReasonKumaUnavailable, msgKumaUnavailable,
 					fmt.Errorf("monitor %d carries our key but is missing from the owned list", prev)})
 			}
@@ -428,7 +428,7 @@ func TestReconcileRefusesCreateBesideUntaggedPrevious(t *testing.T) {
 	}
 }

-func TestReconcileRefusesCreateBesideOtherOwnersMonitor(t *testing.T) {
+func TestReconcileCreatesWhenPreviousIDReusedByOtherOwner(t *testing.T) {
 	e := newEnv(t, withPreviousMonitor(1))
 	if _, err := e.kuma.Create(t.Context(), kumaclient.Monitor{Name: "x", OwnerKey: "default/other"}); err != nil {
 		t.Fatal(err)
@@ -436,12 +436,15 @@ func TestReconcileRefusesCreateBesideOtherOwnersMonitor(t *testing.T) {
 	if _, err := e.reconcile(t, "a"); err != nil {
 		t.Fatal(err)
 	}
-	if e.cnt.creates != 0 {
-		t.Errorf("creates = %d, want 0", e.cnt.creates)
+	if e.cnt.creates != 1 {
+		t.Errorf("creates = %d, want 1", e.cnt.creates)
 	}
-	if c := ready(t, e.get(t, "a")); c.Reason != ReasonNotOwned {
+	if c := ready(t, e.get(t, "a")); c.Reason != ReasonSynced {
 		t.Errorf("Ready = %+v", c)
 	}
+	if m := e.kuma.Get(1); m == nil || m.OwnerKey != "default/other" {
+		t.Errorf("other owner's monitor 1 = %+v, want untouched", m)
+	}
 }

 func TestReconcileCreatesWhenPreviousMonitorIsGone(t *testing.T) {
@@ -49,7 +49,7 @@ spec:
        # connects from the pod network. No env equivalent exists, so it is
        # set in config.yaml on every start (idempotent).
        - name: config
-          image: nousresearch/hermes-agent:v0.22.2 # {"$imagepolicy": "flux-system:hermes"}
+          image: nousresearch/hermes-agent:v2026.9.24 # {"$imagepolicy": "flux-system:hermes"}
          command:
            - /opt/hermes/.venv/bin/hermes
            - config
@@ -89,7 +89,7 @@ spec:
              mountPath: /tools
      containers:
        - name: hermes
-          image: nousresearch/hermes-agent:v0.22.2 # {"$imagepolicy": "flux-system:hermes"}
+          image: nousresearch/hermes-agent:v2026.9.24 # {"$imagepolicy": "flux-system:hermes"}
          args: ["gateway", "run"]
          env:
            - name: HERMES_DASHBOARD
@@ -5,4 +5,3 @@ resources:
  - externalsecret.yaml
  - rbac.yaml
  - deployment.yaml
-  - smoke-kumamonitor.yaml
@@ -1,14 +0,0 @@
---
-apiVersion: uptime.renner.uno/v1alpha1
-kind: KumaMonitor
-metadata:
-  name: www-renner-uno
-  namespace: kuma-sync
-spec:
-  type: keyword
-  displayName: www.renner.uno
-  http:
-    url: https://www.renner.uno/
-    expiryNotification: true
-  keyword:
-    keyword: Familie Renner
@@ -18,6 +18,7 @@ resources:
  - prometheusrule-netbird-availability.yaml
  - prometheusrule-netbird-ingress.yaml
  - prometheusrule-openbao.yaml
+  - prometheusrule-uptime-kuma.yaml
  # Prometheus's own retention. Until REN-107 `retention: 90d` and
  # `retentionSize: 30GB` were both set, the size cap bound first, and the
  # effective window was a 20.5-27.2 day saw-tooth with nothing reporting it.
@@ -44,6 +44,8 @@ spec:
        # traverses the WAN and detects a controller/DC outage (the
        # 2026-07-24 one ran ~20 h with no signal). Dashboard root, 200.
        - https://netbird.renner.uno
+        # Dead-man probe for the off-site Uptime Kuma.
+        - https://kuma.renner.uno
        # NetBird relay health endpoints on the netcup members (relay
        # binary serves :9000/health, verified on 0.73.2). Plain http,
        # ports outside the hairpin's 80/443 — real WAN path.
@@ -26,6 +26,7 @@ spec:
        #   proxy-1/2:9000/health         same file, relay redundancy
        #   ci.renner.uno                 WoodpeckerDown
        #   openbao .../sys/health        OpenBaoUnavailable
+        #   kuma.renner.uno               UptimeKumaUnreachable
        #
        # Warning, not critical, on purpose: this rule cannot know what a given
        # target means. A target whose loss deserves a critical page gets its
@@ -33,7 +34,7 @@ spec:
        # exclusion list from being arbitrary.
        # anchor: blackbox-probe-failing
        - alert: BlackboxProbeFailing
-          expr: probe_success{job="blackbox-http", instance!~"https://fsn1\\.your-objectstorage\\.com|https://netbird\\.renner\\.uno|http://proxy-[12]\\.renner\\.uno:9000/health|https://ci\\.renner\\.uno|https://openbao\\.renner\\.uno/v1/sys/health.*"} == 0
+          expr: probe_success{job="blackbox-http", instance!~"https://fsn1\\.your-objectstorage\\.com|https://netbird\\.renner\\.uno|http://proxy-[12]\\.renner\\.uno:9000/health|https://ci\\.renner\\.uno|https://kuma\\.renner\\.uno|https://openbao\\.renner\\.uno/v1/sys/health.*"} == 0
          for: 10m
          annotations:
            summary: "Blackbox probe of {{ $labels.instance }} has been failing for 10 minutes"
@@ -0,0 +1,35 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: uptime-kuma
+  namespace: monitoring
+  labels:
+    # kube-prometheus-stack's Prometheus selects on this label.
+    release: kube-prometheus-stack
+spec:
+  groups:
+    - name: uptime-kuma.rules
+      rules:
+        # Dead-man probe for the off-site Uptime Kuma.
+        - alert: UptimeKumaUnreachable
+          expr: probe_success{job="blackbox-http", instance="https://kuma.renner.uno"} == 0
+          for: 10m
+          labels:
+            severity: critical
+          annotations:
+            summary: "Uptime Kuma (kuma.renner.uno) unreachable for 10m+"
+            description: "The blackbox probe of the off-site Uptime Kuma has been failing for 10 minutes. The external endpoints it monitors are unwatched until it returns. Check the NetBird VPS (netbird.renner.uno), the uptime-kuma container and Traefik."
+
+        # monitor_response_time is ms; -1 when a beat has no ping (DOWN, group).
+        # anchor: uptime-kuma-slow-response
+        - alert: UptimeKumaSlowResponse
+          expr: avg by (monitor_name) (avg_over_time((monitor_response_time > 0)[15m:1m])) > 1500
+          for: 15m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Uptime Kuma monitor {{ $labels.monitor_name }} averaging {{ $value | humanize }} ms response time"
+            description: >-
+              The average response time of monitor {{ $labels.monitor_name }},
+              as measured from the off-site Uptime Kuma, has stayed above
+              1500 ms for 15 minutes.
@@ -32,6 +32,7 @@ retired the depth rule in favour of one persistence rule (`> 5` for `20m`).
 | `ceph_test.yaml` | Hand-written suite for the `CephPoolLostRedundancyBlind` canary in `prometheusrule-ceph.yaml` (REN-93). |
 | `disk-wear_test.yaml` | Hand-written suite for the EPHEMERAL exclusion in `prometheusrule-disk-wear.yaml` (REN-419). |
 | `gitea_test.yaml` | Hand-written suite for `GiteaDataVolumeLow` in `prometheusrule-gitea.yaml` (REN-583). |
+| `uptime-kuma_test.yaml` | Hand-written suite for `prometheusrule-uptime-kuma.yaml` and the kuma exclusion in `prometheusrule-blackbox.yaml`. |
 | `mutations.txt` | Negative controls. Each must make the suite fail. |

 ## Adding a suite
@@ -172,3 +172,12 @@ disk-wear-ephemeral-keeps-partition|kubernetes/apps/monitoring-rules/prometheusr
 # exactly-25% case in gitea_test.yaml.
 gitea-for-10m|kubernetes/apps/monitoring-rules/prometheusrule-gitea.yaml|/alert: GiteaDataVolumeLow/,/for:/s/for: 15m/for: 10m/
 gitea-floor-le|kubernetes/apps/monitoring-rules/prometheusrule-gitea.yaml|s/^\([[:space:]]*\)< 0\.25$/\1<= 0.25/
+
+# prometheusrule-uptime-kuma.yaml / blackbox kuma exclusion. Detected by the
+# cases in uptime-kuma_test.yaml.
+kuma-slow-no-positive-filter|kubernetes/apps/monitoring-rules/prometheusrule-uptime-kuma.yaml|s/(monitor_response_time > 0)/(monitor_response_time)/
+kuma-slow-threshold-2500|kubernetes/apps/monitoring-rules/prometheusrule-uptime-kuma.yaml|s/\[15m:1m\])) > 1500$/[15m:1m])) > 2500/
+kuma-slow-threshold-900|kubernetes/apps/monitoring-rules/prometheusrule-uptime-kuma.yaml|s/\[15m:1m\])) > 1500$/[15m:1m])) > 900/
+kuma-slow-for-30m|kubernetes/apps/monitoring-rules/prometheusrule-uptime-kuma.yaml|/alert: UptimeKumaSlowResponse/,/for:/s/for: 15m/for: 30m/
+kuma-down-for-20m|kubernetes/apps/monitoring-rules/prometheusrule-uptime-kuma.yaml|/alert: UptimeKumaUnreachable/,/for:/s/for: 10m/for: 20m/
+kuma-blackbox-exclusion-dropped|kubernetes/apps/monitoring-rules/prometheusrule-blackbox.yaml|s/kuma\\\\\.renner/kumx\\\\.renner/
@@ -0,0 +1,110 @@
+# Suite for prometheusrule-uptime-kuma.yaml and the kuma exclusion in
+# prometheusrule-blackbox.yaml.
+rule_files:
+  - ../prometheusrule-uptime-kuma.yaml
+  - ../prometheusrule-blackbox.yaml
+
+evaluation_interval: 1m
+
+tests:
+  # 2000 ms sustained: pending at 0m, fires 15m later, not before.
+  - interval: 1m
+    input_series:
+      - series: 'monitor_response_time{monitor_name="slow-mon"}'
+        values: '2000+0x40'
+    alert_rule_test:
+      - eval_time: 14m
+        alertname: UptimeKumaSlowResponse
+        exp_alerts: []
+      - eval_time: 16m
+        alertname: UptimeKumaSlowResponse
+        exp_alerts:
+          - exp_labels:
+              severity: warning
+              monitor_name: slow-mon
+            exp_annotations:
+              summary: "Uptime Kuma monitor slow-mon averaging 2k ms response time"
+              description: "The average response time of monitor slow-mon, as measured from the off-site Uptime Kuma, has stayed above 1500 ms for 15 minutes."
+
+  # 1000 ms stays silent; a monitor reporting only -1 (DOWN, group) stays silent.
+  - interval: 1m
+    input_series:
+      - series: 'monitor_response_time{monitor_name="fast-mon"}'
+        values: '1000+0x40'
+      - series: 'monitor_response_time{monitor_name="down-mon"}'
+        values: '-1+0x40'
+    alert_rule_test:
+      - eval_time: 35m
+        alertname: UptimeKumaSlowResponse
+        exp_alerts: []
+
+  # Alternating 1000 and -1: the -1 samples must not drag a slow average down
+  # nor count as slow; positives average 1000, silent.
+  - interval: 1m
+    input_series:
+      - series: 'monitor_response_time{monitor_name="flap-fast"}'
+        values: '1000 -1 1000 -1 1000 -1 1000 -1 1000 -1 1000 -1 1000 -1 1000 -1 1000 -1 1000 -1 1000 -1 1000 -1 1000 -1 1000 -1 1000 -1 1000 -1 1000 -1 1000 -1 1000 -1 1000 -1'
+    alert_rule_test:
+      - eval_time: 35m
+        alertname: UptimeKumaSlowResponse
+        exp_alerts: []
+
+  # Alternating 2000 and -1: the -1 samples are excluded, so the average of the
+  # real samples is 2000 and the alert fires. Unfiltered, the mean is ~500.
+  - interval: 1m
+    input_series:
+      - series: 'monitor_response_time{monitor_name="flap-slow"}'
+        values: '2000 -1 2000 -1 2000 -1 2000 -1 2000 -1 2000 -1 2000 -1 2000 -1 2000 -1 2000 -1 2000 -1 2000 -1 2000 -1 2000 -1 2000 -1 2000 -1 2000 -1 2000 -1 2000 -1 2000 -1'
+    alert_rule_test:
+      - eval_time: 35m
+        alertname: UptimeKumaSlowResponse
+        exp_alerts:
+          - exp_labels:
+              severity: warning
+              monitor_name: flap-slow
+            exp_annotations:
+              summary: "Uptime Kuma monitor flap-slow averaging 2k ms response time"
+              description: "The average response time of monitor flap-slow, as measured from the off-site Uptime Kuma, has stayed above 1500 ms for 15 minutes."
+
+  # Dead-man probe: critical after 10m of failure; the catch-all warning must
+  # not also fire for kuma, but still fires for another failing instance.
+  - interval: 1m
+    input_series:
+      - series: 'probe_success{job="blackbox-http",instance="https://kuma.renner.uno"}'
+        values: '0+0x40'
+      - series: 'probe_success{job="blackbox-http",instance="https://other.renner.uno"}'
+        values: '0+0x40'
+    alert_rule_test:
+      - eval_time: 9m
+        alertname: UptimeKumaUnreachable
+        exp_alerts: []
+      - eval_time: 11m
+        alertname: UptimeKumaUnreachable
+        exp_alerts:
+          - exp_labels:
+              severity: critical
+              job: blackbox-http
+              instance: https://kuma.renner.uno
+            exp_annotations:
+              summary: "Uptime Kuma (kuma.renner.uno) unreachable for 10m+"
+              description: "The blackbox probe of the off-site Uptime Kuma has been failing for 10 minutes. The external endpoints it monitors are unwatched until it returns. Check the NetBird VPS (netbird.renner.uno), the uptime-kuma container and Traefik."
+      - eval_time: 15m
+        alertname: BlackboxProbeFailing
+        exp_alerts:
+          - exp_labels:
+              severity: warning
+              job: blackbox-http
+              instance: https://other.renner.uno
+            exp_annotations:
+              summary: "Blackbox probe of https://other.renner.uno has been failing for 10 minutes"
+              description: "The endpoint is not answering as expected. For the Authentik-gated hostnames this is a reachability check covering DNS, TLS, Traefik and the gate outpost — it says nothing about the app behind the gate, which the internal /-/healthy probes cover separately."
+
+  # Healthy probe stays silent.
+  - interval: 1m
+    input_series:
+      - series: 'probe_success{job="blackbox-http",instance="https://kuma.renner.uno"}'
+        values: '1+0x40'
+    alert_rule_test:
+      - eval_time: 30m
+        alertname: UptimeKumaUnreachable
+        exp_alerts: []
@@ -97,6 +97,10 @@ spec:
              resources:
                requests:
                  storage: 50Gi
+        # Mounted at /etc/prometheus/secrets/<name>/; see
+        # ./uptime-kuma-metrics-externalsecret.yaml.
+        secrets:
+          - uptime-kuma-metrics
        # Additional scrape targets not covered by ServiceMonitors.
        additionalScrapeConfigs:
          # Gitea exposes /metrics anonymously on its main HTTP port. The
@@ -106,6 +110,16 @@ spec:
            static_configs:
              - targets: ["gitea.gitea.svc.cluster.local:3000"]
            metrics_path: /metrics
+          # Off-site Uptime Kuma. /metrics takes the API key as the basic-auth
+          # password with an empty username.
+          - job_name: uptime-kuma
+            scheme: https
+            metrics_path: /metrics
+            basic_auth:
+              username: ""
+              password_file: /etc/prometheus/secrets/uptime-kuma-metrics/api_key
+            static_configs:
+              - targets: ["kuma.renner.uno:443"]

    # Grafana
    grafana:
@@ -3,6 +3,7 @@ kind: Kustomization
 resources:
  - grafana-oauth-externalsecret.yaml
  - alertmanager-smtp-externalsecret.yaml
+  - uptime-kuma-metrics-externalsecret.yaml
  - helmrelease-prometheus.yaml
  - helmrelease-loki.yaml
  - helmrelease-alloy-syslog.yaml
@@ -0,0 +1,19 @@
+---
+# Uptime Kuma /metrics API key for Prometheus; OpenBao homelab/infrastructure/uptime-kuma-metrics.
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: uptime-kuma-metrics
+  namespace: monitoring
+spec:
+  refreshInterval: 1h
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: openbao
+  target:
+    name: uptime-kuma-metrics
+  data:
+    - secretKey: api_key
+      remoteRef:
+        key: infrastructure/uptime-kuma-metrics
+        property: api_key
Author	SHA1	Message	Date
claude-botandClaude Opus 5.5	72927293e2	test(monitoring): promtool suite for Uptime Kuma rules, terse comments ci/woodpecker/pr/ci-scripts-test Pipeline was successful Details review-bot skipped: draft Details ci/woodpecker/pr/ci-manifests-test Pipeline was successful Details Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>	2026-10-19 08:26:12 +02:00
claude-botandClaude Opus 5.5	a35bc6b91d	feat(monitoring): watch off-site Uptime Kuma from Prometheus Add a blackbox-http probe of kuma.renner.uno with a dedicated critical UptimeKumaUnreachable rule (excluded from BlackboxProbeFailing), scrape Kuma /metrics with the API key from OpenBao, and alert on sustained slow monitor response times. No egress policy exists in the monitoring namespace, so no CNP change is needed. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>	2026-10-19 08:17:30 +02:00
k.renner	e614c748b9	Merge pull request 'fix(kuma-sync): create when the previous monitor ID belongs to another owner' (#1354 ) from fix/kuma-sync-foreign-key-create into master ci/woodpecker/cron/alert-monitor Pipeline was successful Details ci/woodpecker/push/build-ci-images Pipeline was successful Details Reviewed-on: k.renner/homelab#1354	2026-10-19 06:14:48 +00:00
security-bot	30b0f43911	chore(image-automation): bump nousresearch/hermes-agent:v2026.9.24 Updates managed by image-automation-controller.	2026-10-19 06:13:48 +00:00
k.renner	10819fac62	Merge pull request 'feat(hermes): run Hermes Agent in the cluster' (#1352 ) from feat/hermes-agent into master Reviewed-on: k.renner/homelab#1352	2026-10-19 06:13:17 +00:00
claude-botandClaude Opus 5.5	286564a0d7	fix(kuma-sync): create when the previous monitor ID belongs to another owner ci/woodpecker/pr/ci-scripts-test Pipeline was successful Details review-bot reviewed: comment Details After a Kuma database loss or restore, IDs restart and a resource's status.monitorID can point at a monitor that now carries another resource's ownership tag. That proves our monitor is gone, so the controller creates a new one instead of parking at NotOwned forever. Only an untagged monitor at the previous ID still blocks the create (tag removed by hand), as before. Residual Important finding from the kuma-sync final review. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>	2026-10-19 08:12:45 +02:00
k.renner	1dc5ae7816	Merge pull request 'test(kuma-sync): remove smoke-test KumaMonitor' (#1353 ) from chore/kuma-sync-smoke-remove into master Reviewed-on: k.renner/homelab#1353	2026-10-19 06:04:39 +00:00
claude-botandClaude Opus 5.5	ada5b7ea4d	test(kuma-sync): remove smoke-test KumaMonitor ci/woodpecker/pr/ci-scripts-test Pipeline was successful Details ci/woodpecker/pr/ci-manifests-test Pipeline was successful Details review-bot skipped: closed Details Last smoke-test step: Flux prunes the KumaMonitor and the controller's finalizer deletes Kuma monitor 9. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>	2026-10-19 08:02:48 +02:00