test(monitoring): promtool suite for Uptime Kuma rules, terse comments

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

ansible/plays/install-uptime-kuma.yml

@@ -5,8 +5,6 @@
 # Prerequisites:
 # - netbird-server play applied (docker, Traefik on netbird_netbird)
 # - OpenBao homelab/infrastructure/uptime-kuma with keys username, password
-# - OpenBao homelab/infrastructure/uptime-kuma/smtp with keys server, port, username, token
-# - homelab/infrastructure/uptime-kuma-metrics (key api_key) is written by the role
 # - VAULT_TOKEN in the environment (policy: uptime-kuma-vps)
 # - DNS for kuma.renner.uno pointing at the VPS
 

ansible/roles/uptime-kuma/defaults/main.yml

@@ -1,20 +1,11 @@
 ---
 uptime_kuma_version: "2.5.6"
 uptime_kuma_host: kuma.renner.uno
-uptime_kuma_base_url: "https://{{ uptime_kuma_host }}"
-# Custom domain of the status page; not routed by Traefik yet.
+# Not routed yet; reserved for the public status page.
 uptime_kuma_status_host: uptime.renner.uno
-uptime_kuma_status_slug: renner
-uptime_kuma_status_title: Renner
 uptime_kuma_dir: /opt/uptime-kuma
 uptime_kuma_traefik_network: netbird_netbird
 
-uptime_kuma_smtp_from: uptime@renner.uno
-# Alertmanager email receiver in kubernetes/apps/monitoring/helmrelease-prometheus.yaml.
-uptime_kuma_smtp_to: admin@renner.uno
-
 uptime_kuma_openbao_url: https://openbao.renner.uno
 uptime_kuma_openbao_mount: homelab
 uptime_kuma_openbao_path: infrastructure/uptime-kuma
-uptime_kuma_openbao_smtp_path: infrastructure/uptime-kuma/smtp
-uptime_kuma_openbao_metrics_path: infrastructure/uptime-kuma-metrics

ansible/roles/uptime-kuma/files/kuma-setup.js

@@ -1,225 +1,46 @@
-// Bootstraps Uptime Kuma: admin user, general settings, SMTP notification, status page,
-// and (when KUMA_METRICS_ROTATE=true) a fresh Prometheus API key written to KUMA_METRICS_KEY_FILE.
-// All input comes from environment variables; secrets are never printed.
-// Output lines: "admin user created" / "already set up", then "changed: <step>" or "ok: <step>".
-const fs = require("fs");
+// Creates the Uptime Kuma admin user on first start; exits 0 if already set up.
+// Credentials come from KUMA_SETUP_USERNAME / KUMA_SETUP_PASSWORD only.
 const { io } = require("socket.io-client");
 
-const KUMA_URL = "http://127.0.0.1:3001";
-const EMIT_TIMEOUT_MS = 30000;
-const TOTAL_TIMEOUT_MS = 180000;
-// Defaults the Kuma settings page applies to unset keys before saving (src/pages/Settings.vue).
-const SETTINGS_DEFAULTS = {
-    checkUpdate: true,
-    searchEngineIndex: false,
-    entryPage: "dashboard",
-    nscd: true,
-    keepDataPeriodDays: 180,
-    tlsExpiryNotifyDays: [7, 14, 21],
-    domainExpiryNotifyDays: [7, 14, 21],
-    trustProxy: false,
-};
-
-const REQUIRED = [
-    "KUMA_SETUP_USERNAME",
-    "KUMA_SETUP_PASSWORD",
-    "KUMA_BASE_URL",
-    "KUMA_SMTP_HOST",
-    "KUMA_SMTP_PORT",
-    "KUMA_SMTP_USERNAME",
-    "KUMA_SMTP_PASSWORD",
-    "KUMA_SMTP_FROM",
-    "KUMA_SMTP_TO",
-    "KUMA_STATUS_SLUG",
-    "KUMA_STATUS_TITLE",
-    "KUMA_STATUS_DOMAIN",
-];
-const missing = REQUIRED.filter((name) => !process.env[name]);
-const rotateMetricsKey = process.env.KUMA_METRICS_ROTATE === "true";
-if (rotateMetricsKey && !process.env.KUMA_METRICS_KEY_FILE) {
-    missing.push("KUMA_METRICS_KEY_FILE");
-}
-if (missing.length > 0) {
-    console.error(`missing environment variables: ${missing.join(", ")}`);
+const username = process.env.KUMA_SETUP_USERNAME;
+const password = process.env.KUMA_SETUP_PASSWORD;
+if (!username || !password) {
+    console.error("KUMA_SETUP_USERNAME and KUMA_SETUP_PASSWORD must be set");
     process.exit(2);
 }
-const env = process.env;
 
-const socket = io(KUMA_URL, { transports: ["websocket"], reconnection: false });
-const lists = {};
-const LIST_EVENTS = ["notificationList", "apiKeyList", "statusPageList"];
-for (const event of LIST_EVENTS) {
-    socket.on(event, (data) => {
-        lists[event] = data;
-    });
-}
+const socket = io("http://127.0.0.1:3001", { transports: ["websocket"], reconnection: false });
+const timer = setTimeout(() => {
+    console.error("timeout: no response from Uptime Kuma within 30s");
+    process.exit(1);
+}, 30000);
 
 let finished = false;
 function done(code, msg) {
     if (finished) return;
     finished = true;
-    clearTimeout(watchdog);
-    if (msg) (code === 0 ? console.log : console.error)(msg);
+    clearTimeout(timer);
+    (code === 0 ? console.log : console.error)(msg);
     socket.close();
     process.exit(code);
 }
 
-const watchdog = setTimeout(() => done(1, `timeout: bootstrap did not finish within ${TOTAL_TIMEOUT_MS / 1000}s`), TOTAL_TIMEOUT_MS);
 socket.on("connect_error", (err) => done(1, `connect_error: ${err.message}`));
-socket.on("disconnect", (reason) => done(1, `disconnected before bootstrap finished: ${reason}`));
-
-function call(event, ...args) {
-    return new Promise((resolve, reject) => {
-        const timer = setTimeout(() => reject(new Error(`timeout waiting for ${event}`)), EMIT_TIMEOUT_MS);
-        socket.emit(event, ...args, (res) => {
-            clearTimeout(timer);
-            resolve(res);
-        });
-    });
-}
-
-async function callOk(event, ...args) {
-    const res = await call(event, ...args);
-    if (!res || !res.ok) {
-        throw new Error(`${event} failed: ${res && res.msg ? res.msg : JSON.stringify(res)}`);
-    }
-    return res;
-}
-
-function report(step, changed) {
-    console.log(`${changed ? "changed" : "ok"}: ${step}`);
-}
+socket.on("disconnect", (reason) => done(1, `disconnected before setup finished: ${reason}`));
 
 // Kuma registers socket handlers only after emitting info.
-function waitForInfo() {
-    return new Promise((resolve) => socket.once("info", resolve));
-}
-
-async function setupAdmin() {
-    const needSetup = await call("needSetup");
-    if (!needSetup) {
-        console.log("already set up");
-        return;
-    }
-    await callOk("setup", env.KUMA_SETUP_USERNAME, env.KUMA_SETUP_PASSWORD);
-    console.log("admin user created");
-}
-
-async function login() {
-    await callOk("login", { username: env.KUMA_SETUP_USERNAME, password: env.KUMA_SETUP_PASSWORD, token: "" });
-    // afterLogin sends the lists before the login ack but swallows send failures.
-    for (const event of LIST_EVENTS) {
-        if (lists[event] === undefined) {
-            throw new Error(`${event} not received after login`);
+socket.once("info", () => {
+    socket.emit("needSetup", (needSetup) => {
+        if (!needSetup) {
+            done(0, "already set up");
+            return;
         }
-    }
-}
-
-async function ensureSettings() {
-    const { data } = await callOk("getSettings");
-    if (data.trustProxy === true && data.primaryBaseURL === env.KUMA_BASE_URL) {
-        report("settings", false);
-        return;
-    }
-    const updated = { ...SETTINGS_DEFAULTS, ...data, trustProxy: true, primaryBaseURL: env.KUMA_BASE_URL };
-    await callOk("setSettings", updated, "");
-    report("settings", true);
-}
-
-async function ensureNotification() {
-    const port = Number(env.KUMA_SMTP_PORT);
-    const desired = {
-        name: "email",
-        type: "smtp",
-        isDefault: true,
-        smtpHost: env.KUMA_SMTP_HOST,
-        smtpPort: port,
-        // 465 is implicit TLS; any other port uses STARTTLS.
-        smtpSecure: port === 465,
-        smtpIgnoreSTARTTLS: false,
-        smtpUsername: env.KUMA_SMTP_USERNAME,
-        smtpPassword: env.KUMA_SMTP_PASSWORD,
-        smtpFrom: env.KUMA_SMTP_FROM,
-        smtpTo: env.KUMA_SMTP_TO,
-    };
-    const existing = (lists.notificationList || []).filter((n) => n.name === desired.name).sort((a, b) => a.id - b.id)[0];
-    const current = existing ? JSON.parse(existing.config || "{}") : {};
-    const upToDate =
-        existing && existing.isDefault === true && Object.keys(desired).every((key) => current[key] === desired[key]);
-    if (upToDate) {
-        report("notification", false);
-        return;
-    }
-    await callOk("addNotification", { ...current, ...desired, applyExisting: true }, existing ? existing.id : null);
-    report("notification", true);
-}
-
-async function fetchPublicGroupList(slug) {
-    const res = await fetch(`${KUMA_URL}/api/status-page/${encodeURIComponent(slug)}`);
-    if (!res.ok) {
-        throw new Error(`GET /api/status-page/${slug} returned ${res.status}`);
-    }
-    const body = await res.json();
-    return (body.publicGroupList || []).map((group) => ({
-        id: group.id,
-        name: group.name,
-        monitorList: (group.monitorList || []).map((monitor) => {
-            const entry = { id: monitor.id };
-            if (monitor.sendUrl !== undefined) entry.sendUrl = monitor.sendUrl;
-            if (monitor.url !== undefined) entry.url = monitor.url;
-            return entry;
-        }),
-    }));
-}
-
-async function ensureStatusPage() {
-    const slug = env.KUMA_STATUS_SLUG;
-    let changed = false;
-    const pages = Object.values(lists.statusPageList || {});
-    if (!pages.some((page) => page.slug === slug)) {
-        await callOk("addStatusPage", env.KUMA_STATUS_TITLE, slug);
-        changed = true;
-    }
-    const { config } = await callOk("getStatusPage", slug);
-    const domains = config.domainNameList || [];
-    if (config.title !== env.KUMA_STATUS_TITLE || !domains.includes(env.KUMA_STATUS_DOMAIN)) {
-        // saveStatusPage replaces all groups, so the current ones are sent back unchanged.
-        const publicGroupList = await fetchPublicGroupList(slug);
-        const updated = {
-            ...config,
-            title: env.KUMA_STATUS_TITLE,
-            domainNameList: domains.includes(env.KUMA_STATUS_DOMAIN) ? domains : [...domains, env.KUMA_STATUS_DOMAIN],
-        };
-        await callOk("saveStatusPage", slug, updated, config.icon, publicGroupList);
-        changed = true;
-    }
-    report("status page", changed);
-}
-
-async function rotateMetricsApiKey() {
-    if (!rotateMetricsKey) {
-        report("metrics api key", false);
-        return;
-    }
-    for (const key of (lists.apiKeyList || []).filter((k) => k.name === "prometheus")) {
-        await callOk("deleteAPIKey", key.id);
-    }
-    const res = await callOk("addAPIKey", { name: "prometheus", expires: null, active: 1 });
-    fs.writeFileSync(env.KUMA_METRICS_KEY_FILE, res.key, { mode: 0o600 });
-    report("metrics api key", true);
-}
-
-async function main() {
-    await waitForInfo();
-    await setupAdmin();
-    await login();
-    await ensureSettings();
-    await ensureNotification();
-    await ensureStatusPage();
-    await rotateMetricsApiKey();
-}
-
-main().then(
-    () => done(0),
-    (err) => done(1, err.message)
-);
+        socket.emit("setup", username, password, (res) => {
+            if (res && res.ok) {
+                done(0, "admin user created");
+            } else {
+                done(1, `setup failed: ${res && res.msg ? res.msg : JSON.stringify(res)}`);
+            }
+        });
+    });
+});

ansible/roles/uptime-kuma/tasks/main.yml

@@ -14,33 +14,6 @@
   register: _uptime_kuma_secret
   no_log: true
 
-- name: Read SMTP credentials from OpenBao
-  community.hashi_vault.vault_kv2_get:
-    url: "{{ uptime_kuma_openbao_url }}"
-    engine_mount_point: "{{ uptime_kuma_openbao_mount }}"
-    path: "{{ uptime_kuma_openbao_smtp_path }}"
-    auth_method: token
-  delegate_to: localhost
-  become: false
-  run_once: true
-  register: _uptime_kuma_smtp
-  no_log: true
-
-- name: Read metrics API key from OpenBao
-  community.hashi_vault.vault_kv2_get:
-    url: "{{ uptime_kuma_openbao_url }}"
-    engine_mount_point: "{{ uptime_kuma_openbao_mount }}"
-    path: "{{ uptime_kuma_openbao_metrics_path }}"
-    auth_method: token
-  delegate_to: localhost
-  become: false
-  run_once: true
-  register: _uptime_kuma_metrics_secret
-  failed_when:
-    - _uptime_kuma_metrics_secret.failed | default(false)
-    - "'Invalid or missing path' not in (_uptime_kuma_metrics_secret.msg | default(''))"
-  no_log: true
-
 - name: Ensure Uptime Kuma directories exist
   ansible.builtin.file:
     path: "{{ item }}"
@@ -49,12 +22,11 @@
   loop:
     - "{{ uptime_kuma_dir }}"
     - "{{ uptime_kuma_dir }}/data"
-    - "{{ uptime_kuma_dir }}/setup"
 
 - name: Deploy kuma-setup.js
   ansible.builtin.copy:
     src: kuma-setup.js
-    dest: "{{ uptime_kuma_dir }}/setup/kuma-setup.js"
+    dest: "{{ uptime_kuma_dir }}/kuma-setup.js"
     mode: "0644"
 
 - name: Check for setup marker
@@ -82,11 +54,6 @@
     chdir: "{{ uptime_kuma_dir }}"
   changed_when: false
 
-- name: Remove the script from its former single-file mount path
-  ansible.builtin.file:
-    path: "{{ uptime_kuma_dir }}/kuma-setup.js"
-    state: absent
-
 - name: Wait for Uptime Kuma on loopback
   ansible.builtin.uri:
     url: http://127.0.0.1:3001
@@ -96,80 +63,15 @@
   retries: 30
   delay: 5
 
-- name: Check the stored metrics API key against Uptime Kuma
-  ansible.builtin.uri:
-    url: http://127.0.0.1:3001/metrics
-    headers:
-      Authorization: "Basic {{ (':' ~ _uptime_kuma_metrics_key) | b64encode }}"
-    status_code: [200, 401]
-  vars:
-    _uptime_kuma_metrics_key: "{{ (_uptime_kuma_metrics_secret.secret | default({})).api_key | default('') }}"
-  when: _uptime_kuma_metrics_key | length > 0
-  register: _uptime_kuma_metrics_check
-  no_log: true
-
-- name: Remove a leftover metrics API key handoff file
-  ansible.builtin.file:
-    path: "{{ uptime_kuma_dir }}/data/.metrics-api-key"
-    state: absent
-
-- name: Bootstrap Uptime Kuma
-  ansible.builtin.command:
-    argv:
-      - docker
-      - compose
-      - exec
-      - -T
-      - -e
-      - KUMA_SETUP_USERNAME
-      - -e
-      - KUMA_SETUP_PASSWORD
-      - -e
-      - KUMA_BASE_URL
-      - -e
-      - KUMA_SMTP_HOST
-      - -e
-      - KUMA_SMTP_PORT
-      - -e
-      - KUMA_SMTP_USERNAME
-      - -e
-      - KUMA_SMTP_PASSWORD
-      - -e
-      - KUMA_SMTP_FROM
-      - -e
-      - KUMA_SMTP_TO
-      - -e
-      - KUMA_STATUS_SLUG
-      - -e
-      - KUMA_STATUS_TITLE
-      - -e
-      - KUMA_STATUS_DOMAIN
-      - -e
-      - KUMA_METRICS_ROTATE
-      - -e
-      - KUMA_METRICS_KEY_FILE
-      - uptime-kuma
-      - node
-      - /app/setup/kuma-setup.js
+- name: Create the admin user if Uptime Kuma is not set up
+  ansible.builtin.command: docker compose exec -T -e KUMA_SETUP_USERNAME -e KUMA_SETUP_PASSWORD uptime-kuma node /app/kuma-setup.js
   args:
     chdir: "{{ uptime_kuma_dir }}"
   environment:
     KUMA_SETUP_USERNAME: "{{ _uptime_kuma_secret.secret.username }}"
     KUMA_SETUP_PASSWORD: "{{ _uptime_kuma_secret.secret.password }}"
-    KUMA_BASE_URL: "{{ uptime_kuma_base_url }}"
-    KUMA_SMTP_HOST: "{{ _uptime_kuma_smtp.secret.server }}"
-    KUMA_SMTP_PORT: "{{ _uptime_kuma_smtp.secret.port }}"
-    KUMA_SMTP_USERNAME: "{{ _uptime_kuma_smtp.secret.username }}"
-    KUMA_SMTP_PASSWORD: "{{ _uptime_kuma_smtp.secret.token }}"
-    KUMA_SMTP_FROM: "{{ uptime_kuma_smtp_from }}"
-    KUMA_SMTP_TO: "{{ uptime_kuma_smtp_to }}"
-    KUMA_STATUS_SLUG: "{{ uptime_kuma_status_slug }}"
-    KUMA_STATUS_TITLE: "{{ uptime_kuma_status_title }}"
-    KUMA_STATUS_DOMAIN: "{{ uptime_kuma_status_host }}"
-    KUMA_METRICS_ROTATE: "{{ 'false' if (_uptime_kuma_metrics_check.status | default(0)) == 200 else 'true' }}"
-    KUMA_METRICS_KEY_FILE: /app/data/.metrics-api-key
   register: _uptime_kuma_setup
-  changed_when: "'admin user created' in _uptime_kuma_setup.stdout or 'changed:' in _uptime_kuma_setup.stdout"
+  changed_when: "'admin user created' in _uptime_kuma_setup.stdout"
   failed_when: false
   no_log: true
 
@@ -178,33 +80,6 @@
     msg: "kuma-setup.js failed: rc={{ _uptime_kuma_setup.rc | default('undefined') }} stdout={{ _uptime_kuma_setup.stdout | default('') }} stderr={{ _uptime_kuma_setup.stderr | default('') }}"
   when: (_uptime_kuma_setup.rc | default(1)) != 0
 
-- name: Store the new metrics API key in OpenBao
-  when: "'changed: metrics api key' in _uptime_kuma_setup.stdout"
-  block:
-    - name: Read the new metrics API key
-      ansible.builtin.slurp:
-        src: "{{ uptime_kuma_dir }}/data/.metrics-api-key"
-      register: _uptime_kuma_new_metrics_key
-      no_log: true
-
-    - name: Write the metrics API key to OpenBao
-      community.hashi_vault.vault_kv2_write:
-        url: "{{ uptime_kuma_openbao_url }}"
-        engine_mount_point: "{{ uptime_kuma_openbao_mount }}"
-        path: "{{ uptime_kuma_openbao_metrics_path }}"
-        auth_method: token
-        data:
-          api_key: "{{ _uptime_kuma_new_metrics_key.content | b64decode }}"
-      delegate_to: localhost
-      become: false
-      run_once: true
-      no_log: true
-  always:
-    - name: Remove the metrics API key handoff file
-      ansible.builtin.file:
-        path: "{{ uptime_kuma_dir }}/data/.metrics-api-key"
-        state: absent
-
 - name: Record setup completion
   ansible.builtin.copy:
     content: ""

ansible/roles/uptime-kuma/templates/docker-compose.yml.j2

@@ -9,8 +9,7 @@ services:
       - "127.0.0.1:3001:3001"
     volumes:
       - {{ uptime_kuma_dir }}/data:/app/data
-      # Directory mount: a single-file bind mount keeps the inode from container start.
-      - {{ uptime_kuma_dir }}/setup:/app/setup:ro
+      - {{ uptime_kuma_dir }}/kuma-setup.js:/app/kuma-setup.js:ro
     labels:
       - "traefik.enable={{ uptime_kuma_route_enabled | lower }}"
       - "traefik.http.routers.uptime-kuma.rule=Host(`{{ uptime_kuma_host }}`)"

ci/images/kuma-sync/internal/controller/e2e_integration_test.go

@@ -7,14 +7,12 @@ import (
 	"fmt"
 	"io"
 	"path/filepath"
-	"slices"
 	"sync"
 	"testing"
 	"time"
 
 	kuma "github.com/breml/go-uptime-kuma-client"
 	"github.com/breml/go-uptime-kuma-client/monitor"
-	"github.com/breml/go-uptime-kuma-client/notification"
 	"github.com/testcontainers/testcontainers-go"
 	"github.com/testcontainers/testcontainers-go/wait"
 	"k8s.io/apimachinery/pkg/api/meta"
@@ -397,60 +395,6 @@ func TestE2EDriftRepaired(t *testing.T) {
 	})
 }
 
-func (f *e2e) webhook(name string, isDefault bool) int64 {
-	f.t.Helper()
-	id, err := f.raw.CreateNotification(f.t.Context(), notification.Webhook{
-		Base: notification.Base{Name: name, IsActive: true, IsDefault: isDefault},
-		WebhookDetails: notification.WebhookDetails{
-			WebhookURL: "http://127.0.0.1:9/" + name, WebhookContentType: "json",
-		},
-	})
-	if err != nil {
-		f.t.Fatalf("create notification %s: %v", name, err)
-	}
-	return id
-}
-
-func TestE2EDefaultNotifications(t *testing.T) {
-	f, env := newE2E(t)
-	first := f.webhook("first", true)
-	f.webhook("opt-in", false)
-	f.start(env)
-	f.apply(group("g", ""), dns("probe", "g"))
-
-	carry := func(want ...int64) func() (bool, string) {
-		return func() (bool, string) {
-			m := byKey(f.owned())
-			for _, k := range []string{"default/g", "default/probe"} {
-				got, ok := m[k]
-				if !ok {
-					return false, k + " missing"
-				}
-				if !slices.Equal(got.NotificationIDs, want) {
-					return false, fmt.Sprintf("%s carries %v, want %v", k, got.NotificationIDs, want)
-				}
-			}
-			return true, ""
-		}
-	}
-	eventually(t, 30*time.Second, "monitors created with the default notification", carry(first))
-
-	second := f.webhook("second", true)
-	want := []int64{first, second}
-	slices.Sort(want)
-	eventually(t, 30*time.Second, "new default notification attached by a resync", carry(want...))
-
-	var live monitor.DNS
-	if err := f.raw.GetMonitorAs(t.Context(), byKey(f.owned())["default/probe"].ID, &live); err != nil {
-		t.Fatal(err)
-	}
-	live.NotificationIDs = nil
-	if err := f.raw.UpdateMonitor(t.Context(), &live); err != nil {
-		t.Fatal(err)
-	}
-	eventually(t, 30*time.Second, "notifications removed by hand restored by a resync", carry(want...))
-}
-
 // A monitor that lost its ownership tag in the UI must not get a twin when the
 // controller restarts: the resource reports NotOwned and nothing is created.
 func TestE2ETagRemovedNeverDuplicates(t *testing.T) {

ci/images/kuma-sync/internal/controller/kumamonitor_controller.go

@@ -129,11 +129,6 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
 	if fail != nil {
 		return r.fail(ctx, km, fail)
 	}
-	notifications, err := r.Kuma.DefaultNotifications(ctx)
-	if err != nil {
-		return r.fail(ctx, km, &failure{ReasonKumaUnavailable, msgKumaUnavailable, err})
-	}
-	desired.NotificationIDs = notifications
 
 	var id int64
 	if res.mon == nil {

ci/images/kuma-sync/internal/controller/kumamonitor_controller_test.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"slices"
 	"testing"
 	"time"
 
@@ -311,79 +310,6 @@ func TestReconcileListErrorNoCreate(t *testing.T) {
 	}
 }
 
-func TestReconcileCreateGetsDefaultNotifications(t *testing.T) {
-	group := &v1.KumaMonitor{
-		ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "grp"},
-		Spec:       v1.KumaMonitorSpec{Type: "group", DisplayName: "grp", Interval: 60, RetryInterval: 60},
-	}
-	e := newEnv(t, httpMonitor("a"), group)
-	e.kuma.SetDefaultNotifications(7, 3)
-	for _, name := range []string{"a", "grp"} {
-		if _, err := e.reconcile(t, name); err != nil {
-			t.Fatal(err)
-		}
-		got := e.kuma.Get(e.get(t, name).Status.MonitorID)
-		if got == nil || !slices.Equal(got.NotificationIDs, []int64{3, 7}) {
-			t.Errorf("%s: kuma monitor = %+v, want notifications [3 7]", name, got)
-		}
-	}
-}
-
-func TestReconcileRevertsNotificationDrift(t *testing.T) {
-	km := httpMonitor("a")
-	e := newEnv(t, km)
-	e.kuma.SetDefaultNotifications(1, 2)
-	drifted := Desired(km)
-	drifted.NotificationIDs = []int64{2, 9}
-	id, _ := e.kuma.Create(t.Context(), drifted)
-	if _, err := e.reconcile(t, "a"); err != nil {
-		t.Fatal(err)
-	}
-	if e.cnt.updates != 1 || e.cnt.creates != 0 {
-		t.Fatalf("updates=%d creates=%d", e.cnt.updates, e.cnt.creates)
-	}
-	if got := e.kuma.Get(id).NotificationIDs; !slices.Equal(got, []int64{1, 2}) {
-		t.Errorf("notifications = %v, want [1 2]", got)
-	}
-}
-
-func TestReconcileUnchangedNotificationsNoUpdate(t *testing.T) {
-	km := httpMonitor("a")
-	e := newEnv(t, km)
-	e.kuma.SetDefaultNotifications(4, 2)
-	existing := Desired(km)
-	existing.NotificationIDs = []int64{4, 2}
-	e.kuma.Create(t.Context(), existing)
-	for range 2 {
-		if _, err := e.reconcile(t, "a"); err != nil {
-			t.Fatal(err)
-		}
-	}
-	if e.cnt.updates != 0 || e.cnt.creates != 0 {
-		t.Fatalf("updates=%d creates=%d, want 0 and 0", e.cnt.updates, e.cnt.creates)
-	}
-}
-
-func TestReconcileNotificationListErrorNoWrite(t *testing.T) {
-	adopt := httpMonitor("b")
-	e := newEnv(t, httpMonitor("a"), adopt)
-	drifted := Desired(adopt)
-	drifted.Interval = 30
-	e.kuma.Create(t.Context(), drifted)
-	e.kuma.SetNotificationsErr(errors.New("boom"))
-	for _, name := range []string{"a", "b"} {
-		if _, err := e.reconcile(t, name); err == nil {
-			t.Fatalf("%s: expected error", name)
-		}
-		if c := ready(t, e.get(t, name)); c.Status != metav1.ConditionFalse || c.Reason != ReasonKumaUnavailable {
-			t.Errorf("%s: Ready = %+v", name, c)
-		}
-	}
-	if e.cnt.creates != 0 || e.cnt.updates != 0 {
-		t.Errorf("creates=%d updates=%d", e.cnt.creates, e.cnt.updates)
-	}
-}
-
 func TestReconcileOwnerTagMissing(t *testing.T) {
 	e := newEnv(t, httpMonitor("a"))
 	e.kuma.SetErrs(kumaclient.ErrOwnerTagMissing, nil, nil, nil)

ci/images/kuma-sync/internal/kumaclient/breml.go

@@ -13,7 +13,6 @@ import (
 
 	kuma "github.com/breml/go-uptime-kuma-client"
 	"github.com/breml/go-uptime-kuma-client/monitor"
-	"github.com/breml/go-uptime-kuma-client/notification"
 	"github.com/breml/go-uptime-kuma-client/tag"
 )
 
@@ -374,8 +373,8 @@ func (b *Breml) update(ctx context.Context, m Monitor) error {
 		return fmt.Errorf("update %d: monitor carries owner key %q, not %q: %w", m.ID, key, m.OwnerKey, ErrNotOwned)
 	}
 	// Start from the full current object, also across a type change, so
-	// fields we do not manage (proxy, resend interval, upside down, auth)
-	// survive the overwrite.
+	// fields we do not manage (notifications, proxy, resend interval, upside
+	// down, auth) survive the overwrite.
 	wire, err := toWire(m, &cur)
 	if err != nil {
 		return err
@@ -405,42 +404,18 @@ func (b *Breml) Delete(ctx context.Context, id int64) error {
 	return err
 }
 
-// DefaultNotifications reads the notification list cached by the session.
-// Kuma has no command to request it: it pushes the list at login and to every
-// session of the user after each change.
-func (b *Breml) DefaultNotifications(ctx context.Context) ([]int64, error) {
-	if err := ctx.Err(); err != nil {
-		return nil, err
-	}
-	b.mu.Lock()
-	defer b.mu.Unlock()
-	return defaultIDs(b.c.GetNotifications(ctx)), nil
-}
-
-func defaultIDs(list []notification.Base) []int64 {
-	var ids []int64
-	for _, n := range list {
-		if n.IsDefault {
-			ids = append(ids, n.ID)
-		}
-	}
-	slices.Sort(ids)
-	return ids
-}
-
 // toDomain converts a monitor from the Kuma monitor list.
 func toDomain(base monitor.Base, ownerKey string) (Monitor, error) {
 	m := Monitor{
-		ID:              base.ID,
-		Type:            base.Type(),
-		Name:            base.Name,
-		Parent:          base.Parent,
-		Active:          base.IsActive,
-		Interval:        int32(base.Interval),
-		RetryInterval:   int32(base.RetryInterval),
-		MaxRetries:      int32(base.MaxRetries),
-		NotificationIDs: slices.Clone(base.NotificationIDs),
-		OwnerKey:        ownerKey,
+		ID:            base.ID,
+		Type:          base.Type(),
+		Name:          base.Name,
+		Parent:        base.Parent,
+		Active:        base.IsActive,
+		Interval:      int32(base.Interval),
+		RetryInterval: int32(base.RetryInterval),
+		MaxRetries:    int32(base.MaxRetries),
+		OwnerKey:      ownerKey,
 	}
 	if base.Description != nil {
 		m.Description = *base.Description
@@ -590,7 +565,6 @@ func writeBase(b *monitor.Base, m Monitor) {
 	b.RetryInterval = int64(m.RetryInterval)
 	b.MaxRetries = int64(m.MaxRetries)
 	b.IsActive = m.Active
-	b.NotificationIDs = slices.Clone(m.NotificationIDs)
 }
 
 // writeHTTP expects m to be normalized.

ci/images/kuma-sync/internal/kumaclient/breml_integration_test.go

@@ -287,10 +287,12 @@ func TestUpdateKeepsUnchangedFields(t *testing.T) {
 	}
 }
 
-func createWebhook(t *testing.T, b *Breml, name string, isDefault bool) int64 {
+// setUnmanaged sets state through the raw library that Monitor does not model.
+func setUnmanaged(t *testing.T, b *Breml, id int64) (notificationID int64) {
 	t.Helper()
-	nid, err := b.c.CreateNotification(context.Background(), &notification.Webhook{
-		Base: notification.Base{Name: name, IsActive: true, IsDefault: isDefault},
+	ctx := context.Background()
+	nid, err := b.c.CreateNotification(ctx, &notification.Webhook{
+		Base: notification.Base{Name: "n", IsActive: true},
 		WebhookDetails: notification.WebhookDetails{
 			WebhookURL: "http://example.com/hook", WebhookContentType: "application/json",
 		},
@@ -298,15 +300,6 @@ func createWebhook(t *testing.T, b *Breml, name string, isDefault bool) int64 {
 	if err != nil {
 		t.Fatalf("create notification: %v", err)
 	}
-	return nid
-}
-
-// setUnmanaged sets state through the raw library that Monitor does not
-// model, and attaches a notification by hand.
-func setUnmanaged(t *testing.T, b *Breml, id int64) (notificationID int64) {
-	t.Helper()
-	ctx := context.Background()
-	nid := createWebhook(t, b, "n", false)
 	var kw monitor.HTTPKeyword
 	if err := b.c.GetMonitorAs(ctx, id, &kw); err != nil {
 		t.Fatal(err)
@@ -320,15 +313,16 @@ func setUnmanaged(t *testing.T, b *Breml, id int64) (notificationID int64) {
 	return nid
 }
 
-func assertUnmanaged(t *testing.T, c *Breml, id int64, when string) {
+func assertUnmanaged(t *testing.T, c *Breml, id, nid int64, when string) {
 	t.Helper()
 	var base monitor.Base
 	base, err := c.c.GetMonitor(t.Context(), id)
 	if err != nil {
 		t.Fatal(err)
 	}
-	if !base.UpsideDown || base.ResendInterval != 7 {
-		t.Errorf("%s: unmanaged state lost: upsideDown=%v resend=%d", when, base.UpsideDown, base.ResendInterval)
+	if !base.UpsideDown || base.ResendInterval != 7 || !reflect.DeepEqual(base.NotificationIDs, []int64{nid}) {
+		t.Errorf("%s: unmanaged state lost: upsideDown=%v resend=%d notifications=%v",
+			when, base.UpsideDown, base.ResendInterval, base.NotificationIDs)
 	}
 }
 
@@ -368,15 +362,14 @@ func TestUpdatePreservesUnmanagedAndClearsManaged(t *testing.T) {
 		t.Fatal(err)
 	}
 	m := byID(got)[id]
-	if m.Parent != nil || m.Headers != nil || m.Keyword != "" || m.Interval != 90 || m.NotificationIDs != nil {
+	if m.Parent != nil || m.Headers != nil || m.Keyword != "" || m.Interval != 90 {
 		t.Errorf("managed fields not cleared: %+v", m)
 	}
-	assertUnmanaged(t, fresh, id, "after plain update")
+	assertUnmanaged(t, fresh, id, nid, "after plain update")
 
 	// Type change must not wipe unmanaged state either.
 	changed := cleared
 	changed.Type = "http"
-	changed.NotificationIDs = []int64{nid}
 	if err := b.Update(ctx, changed); err != nil {
 		t.Fatal(err)
 	}
@@ -384,55 +377,10 @@ func TestUpdatePreservesUnmanagedAndClearsManaged(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if m := byID(got)[id]; m.Type != "http" || !reflect.DeepEqual(m.NotificationIDs, []int64{nid}) {
-		t.Errorf("type = %q, notifications = %v, want http and [%d]", m.Type, m.NotificationIDs, nid)
-	}
-	assertUnmanaged(t, fresh, id, "after type change")
-}
-
-func TestDefaultNotificationsAndWrite(t *testing.T) {
-	ctx := context.Background()
-	url := startKuma(t)
-	b := setupClient(t, url)
-	other := freshClient(t, url)
-
-	if ids, err := other.DefaultNotifications(ctx); err != nil || ids != nil {
-		t.Fatalf("no notifications: ids=%v err=%v", ids, err)
-	}
-	def := createWebhook(t, b, "default", true)
-	createWebhook(t, b, "opt-in", false)
-	// Kuma pushes the list to every session of the user, not only the writer.
-	deadline := time.Now().Add(10 * time.Second)
-	for {
-		ids, err := other.DefaultNotifications(ctx)
-		if err != nil {
-			t.Fatal(err)
-		}
-		if reflect.DeepEqual(ids, []int64{def}) {
-			break
-		}
-		if time.Now().After(deadline) {
-			t.Fatalf("other session sees defaults %v, want [%d]", ids, def)
-		}
-		time.Sleep(100 * time.Millisecond)
-	}
-
-	m := Monitor{
-		Type: "group", Name: "grp", Active: true, Interval: 60, RetryInterval: 60,
-		NotificationIDs: []int64{def}, OwnerKey: "default/grp",
-	}
-	id, err := b.Create(ctx, m)
-	if err != nil {
-		t.Fatal(err)
-	}
-	got, err := freshClient(t, url).ListOwned(ctx)
-	if err != nil {
-		t.Fatal(err)
-	}
-	m.ID = id
-	if !reflect.DeepEqual(byID(got)[id], m.Normalized()) {
-		t.Errorf("read-back differs:\n got  %+v\n want %+v", byID(got)[id], m.Normalized())
+	if m := byID(got)[id]; m.Type != "http" {
+		t.Errorf("type = %q, want http", m.Type)
 	}
+	assertUnmanaged(t, fresh, id, nid, "after type change")
 }
 
 func TestRestartAdoptsByTag(t *testing.T) {

ci/images/kuma-sync/internal/kumaclient/client.go

@@ -63,7 +63,6 @@ func Conflicts(err error) (conflicts []*ConflictError, only bool) {
 func (m Monitor) clone() Monitor {
 	m.Headers = maps.Clone(m.Headers)
 	m.AcceptedStatusCodes = slices.Clone(m.AcceptedStatusCodes)
-	m.NotificationIDs = slices.Clone(m.NotificationIDs)
 	if m.Parent != nil {
 		p := *m.Parent
 		m.Parent = &p
@@ -81,18 +80,12 @@ func isHTTPFamily(typ string) bool {
 // For http, keyword and json-query: an empty Method becomes GET, empty
 // AcceptedStatusCodes become ["200-299"], and Timeout 0 becomes
 // round(0.8*Interval). For all other types Timeout is stored as 0 and the
-// HTTP-only slices are nil. Empty Headers become nil. NotificationIDs are
-// sorted and deduplicated, empty becomes nil. It is idempotent.
+// HTTP-only slices are nil. Empty Headers become nil. It is idempotent.
 func (m Monitor) Normalized() Monitor {
 	m = m.clone()
 	if len(m.Headers) == 0 {
 		m.Headers = nil
 	}
-	slices.Sort(m.NotificationIDs)
-	m.NotificationIDs = slices.Compact(m.NotificationIDs)
-	if len(m.NotificationIDs) == 0 {
-		m.NotificationIDs = nil
-	}
 	if len(m.AcceptedStatusCodes) == 0 {
 		m.AcceptedStatusCodes = nil
 	}
@@ -150,9 +143,6 @@ type Monitor struct {
 	DNSResolveType   string
 	DNSResolveServer string
 
-	// NotificationIDs are the Kuma notifications attached to the monitor.
-	NotificationIDs []int64
-
 	// OwnerKey is the value of the kuma-sync tag; "" means not ours.
 	OwnerKey string
 }
@@ -180,7 +170,4 @@ type Client interface {
 	// Delete removes an owned monitor. It wraps ErrNotFound for an unknown ID
 	// and ErrNotOwned for a monitor without the kuma-sync tag.
 	Delete(ctx context.Context, id int64) error
-	// DefaultNotifications returns the IDs of all notifications marked
-	// "default enabled" in Kuma, sorted. An empty result is valid.
-	DefaultNotifications(ctx context.Context) ([]int64, error)
 }

ci/images/kuma-sync/internal/kumaclient/fake.go

@@ -3,7 +3,6 @@ package kumaclient
 import (
 	"context"
 	"fmt"
-	"slices"
 	"sort"
 	"sync"
 )
@@ -21,10 +20,6 @@ type Fake struct {
 	UpdateErr error
 	DeleteErr error
 	ProbeErr  error
-	// NotificationsErr is returned by DefaultNotifications.
-	NotificationsErr error
-	// Defaults are the IDs DefaultNotifications returns.
-	Defaults []int64
 	// ListPartial makes ListOwned return the stored monitors together with
 	// ListErr, as the real client does for owner conflicts.
 	ListPartial bool
@@ -95,32 +90,6 @@ func (f *Fake) Probe(_ context.Context, id int64) (bool, string, error) {
 	return ok, m.OwnerKey, nil
 }
 
-// SetDefaultNotifications sets the IDs DefaultNotifications returns.
-func (f *Fake) SetDefaultNotifications(ids ...int64) {
-	f.mu.Lock()
-	defer f.mu.Unlock()
-	f.Defaults = slices.Clone(ids)
-}
-
-// SetNotificationsErr makes DefaultNotifications return err.
-func (f *Fake) SetNotificationsErr(err error) {
-	f.mu.Lock()
-	defer f.mu.Unlock()
-	f.NotificationsErr = err
-}
-
-// DefaultNotifications returns Defaults, sorted.
-func (f *Fake) DefaultNotifications(_ context.Context) ([]int64, error) {
-	f.mu.Lock()
-	defer f.mu.Unlock()
-	if f.NotificationsErr != nil {
-		return nil, f.NotificationsErr
-	}
-	ids := slices.Clone(f.Defaults)
-	slices.Sort(ids)
-	return ids, nil
-}
-
 func (f *Fake) allocID() int64 {
 	id := f.nextID
 	f.nextID++

ci/images/kuma-sync/internal/kumaclient/fake_test.go

@@ -118,21 +118,6 @@ func TestFakeInjectedErrors(t *testing.T) {
 	}
 }
 
-func TestFakeDefaultNotifications(t *testing.T) {
-	f := NewFake()
-	if ids, err := f.DefaultNotifications(context.Background()); err != nil || ids != nil {
-		t.Fatalf("empty: %v %v", ids, err)
-	}
-	f.SetDefaultNotifications(4, 1)
-	if ids, err := f.DefaultNotifications(context.Background()); err != nil || len(ids) != 2 || ids[0] != 1 || ids[1] != 4 {
-		t.Fatalf("set: %v %v", ids, err)
-	}
-	f.SetNotificationsErr(errors.New("boom"))
-	if _, err := f.DefaultNotifications(context.Background()); err == nil {
-		t.Error("expected injected error")
-	}
-}
-
 func TestFakeUpdateRejectsOtherOwnerKey(t *testing.T) {
 	f := NewFake()
 	id, _ := f.Create(context.Background(), Monitor{Name: "a", OwnerKey: "ns/a"})

ci/images/kuma-sync/internal/kumaclient/logic_test.go

@@ -7,7 +7,6 @@ import (
 	"testing"
 
 	kuma "github.com/breml/go-uptime-kuma-client"
-	"github.com/breml/go-uptime-kuma-client/notification"
 	"github.com/breml/go-uptime-kuma-client/tag"
 )
 
@@ -168,29 +167,3 @@ func TestFakeStoresNormalized(t *testing.T) {
 		t.Fatalf("fake did not normalize: %+v", got)
 	}
 }
-
-func TestNormalizedNotificationIDs(t *testing.T) {
-	in := Monitor{Type: "group", NotificationIDs: []int64{10, 2, 10}}
-	got := in.Normalized()
-	if !reflect.DeepEqual(got.NotificationIDs, []int64{2, 10}) {
-		t.Errorf("notifications = %v, want [2 10]", got.NotificationIDs)
-	}
-	if !reflect.DeepEqual(in.NotificationIDs, []int64{10, 2, 10}) {
-		t.Errorf("Normalized changed its receiver: %v", in.NotificationIDs)
-	}
-	if e := (Monitor{Type: "group", NotificationIDs: []int64{}}).Normalized(); e.NotificationIDs != nil {
-		t.Errorf("empty notifications must be nil: %v", e.NotificationIDs)
-	}
-}
-
-func TestDefaultIDs(t *testing.T) {
-	got := defaultIDs([]notification.Base{
-		{ID: 5, IsDefault: true}, {ID: 1}, {ID: 3, IsDefault: true},
-	})
-	if !reflect.DeepEqual(got, []int64{3, 5}) {
-		t.Errorf("defaultIDs = %v, want [3 5]", got)
-	}
-	if got := defaultIDs(nil); got != nil {
-		t.Errorf("defaultIDs(nil) = %v, want nil", got)
-	}
-}

kubernetes/apps/hermes/hermes.yaml

@@ -49,7 +49,7 @@ spec:
         # connects from the pod network. No env equivalent exists, so it is
         # set in config.yaml on every start (idempotent).
         - name: config
-          image: nousresearch/hermes-agent:v0.22.2 # {"$imagepolicy": "flux-system:hermes"}
+          image: nousresearch/hermes-agent:v2026.9.24 # {"$imagepolicy": "flux-system:hermes"}
           command:
             - /opt/hermes/.venv/bin/hermes
             - config
@@ -89,7 +89,7 @@ spec:
               mountPath: /tools
       containers:
         - name: hermes
-          image: nousresearch/hermes-agent:v0.22.2 # {"$imagepolicy": "flux-system:hermes"}
+          image: nousresearch/hermes-agent:v2026.9.24 # {"$imagepolicy": "flux-system:hermes"}
           args: ["gateway", "run"]
           env:
             - name: HERMES_DASHBOARD

kubernetes/apps/kuma-sync/deployment.yaml

@@ -1,3 +1,4 @@
+---
 # kuma-sync: KumaMonitor -> Uptime Kuma monitors on kuma.renner.uno (ci/images/kuma-sync).
 apiVersion: apps/v1
 kind: Deployment
@@ -30,7 +31,7 @@ spec:
           type: RuntimeDefault
       containers:
         - name: controller
-          image: gitea.renner.uno/gitea_admin/kuma-sync:2026-10-19-0615 # {"$imagepolicy": "flux-system:kuma-sync"}
+          image: gitea.renner.uno/gitea_admin/kuma-sync:2026-10-19-0545 # {"$imagepolicy": "flux-system:kuma-sync"}
           env:
             - name: KUMA_URL
               value: https://kuma.renner.uno

kubernetes/apps/monitoring-rules/kustomization.yaml

@@ -18,6 +18,7 @@ resources:
   - prometheusrule-netbird-availability.yaml
   - prometheusrule-netbird-ingress.yaml
   - prometheusrule-openbao.yaml
+  - prometheusrule-uptime-kuma.yaml
   # Prometheus's own retention. Until REN-107 `retention: 90d` and
   # `retentionSize: 30GB` were both set, the size cap bound first, and the
   # effective window was a 20.5-27.2 day saw-tooth with nothing reporting it.

kubernetes/apps/monitoring-rules/probes-blackbox.yaml

@@ -44,6 +44,8 @@ spec:
         # traverses the WAN and detects a controller/DC outage (the
         # 2026-07-24 one ran ~20 h with no signal). Dashboard root, 200.
         - https://netbird.renner.uno
+        # Dead-man probe for the off-site Uptime Kuma.
+        - https://kuma.renner.uno
         # NetBird relay health endpoints on the netcup members (relay
         # binary serves :9000/health, verified on 0.73.2). Plain http,
         # ports outside the hairpin's 80/443 — real WAN path.

kubernetes/apps/monitoring-rules/prometheusrule-blackbox.yaml

@@ -26,6 +26,7 @@ spec:
         #   proxy-1/2:9000/health         same file, relay redundancy
         #   ci.renner.uno                 WoodpeckerDown
         #   openbao .../sys/health        OpenBaoUnavailable
+        #   kuma.renner.uno               UptimeKumaUnreachable
         #
         # Warning, not critical, on purpose: this rule cannot know what a given
         # target means. A target whose loss deserves a critical page gets its
@@ -33,7 +34,7 @@ spec:
         # exclusion list from being arbitrary.
         # anchor: blackbox-probe-failing
         - alert: BlackboxProbeFailing
-          expr: probe_success{job="blackbox-http", instance!~"https://fsn1\\.your-objectstorage\\.com|https://netbird\\.renner\\.uno|http://proxy-[12]\\.renner\\.uno:9000/health|https://ci\\.renner\\.uno|https://openbao\\.renner\\.uno/v1/sys/health.*"} == 0
+          expr: probe_success{job="blackbox-http", instance!~"https://fsn1\\.your-objectstorage\\.com|https://netbird\\.renner\\.uno|http://proxy-[12]\\.renner\\.uno:9000/health|https://ci\\.renner\\.uno|https://kuma\\.renner\\.uno|https://openbao\\.renner\\.uno/v1/sys/health.*"} == 0
           for: 10m
           annotations:
             summary: "Blackbox probe of {{ $labels.instance }} has been failing for 10 minutes"

kubernetes/apps/monitoring-rules/prometheusrule-uptime-kuma.yaml

@@ -0,0 +1,35 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: uptime-kuma
+  namespace: monitoring
+  labels:
+    # kube-prometheus-stack's Prometheus selects on this label.
+    release: kube-prometheus-stack
+spec:
+  groups:
+    - name: uptime-kuma.rules
+      rules:
+        # Dead-man probe for the off-site Uptime Kuma.
+        - alert: UptimeKumaUnreachable
+          expr: probe_success{job="blackbox-http", instance="https://kuma.renner.uno"} == 0
+          for: 10m
+          labels:
+            severity: critical
+          annotations:
+            summary: "Uptime Kuma (kuma.renner.uno) unreachable for 10m+"
+            description: "The blackbox probe of the off-site Uptime Kuma has been failing for 10 minutes. The external endpoints it monitors are unwatched until it returns. Check the NetBird VPS (netbird.renner.uno), the uptime-kuma container and Traefik."
+
+        # monitor_response_time is ms; -1 when a beat has no ping (DOWN, group).
+        # anchor: uptime-kuma-slow-response
+        - alert: UptimeKumaSlowResponse
+          expr: avg by (monitor_name) (avg_over_time((monitor_response_time > 0)[15m:1m])) > 1500
+          for: 15m
+          labels:
+            severity: warning
+          annotations:
+            summary: "Uptime Kuma monitor {{ $labels.monitor_name }} averaging {{ $value | humanize }} ms response time"
+            description: >-
+              The average response time of monitor {{ $labels.monitor_name }},
+              as measured from the off-site Uptime Kuma, has stayed above
+              1500 ms for 15 minutes.

kubernetes/apps/monitoring-rules/tests/README.md

@@ -32,6 +32,7 @@ retired the depth rule in favour of one persistence rule (`> 5` for `20m`).
 | `ceph_test.yaml` | Hand-written suite for the `CephPoolLostRedundancyBlind` canary in `prometheusrule-ceph.yaml` (REN-93). |
 | `disk-wear_test.yaml` | Hand-written suite for the EPHEMERAL exclusion in `prometheusrule-disk-wear.yaml` (REN-419). |
 | `gitea_test.yaml` | Hand-written suite for `GiteaDataVolumeLow` in `prometheusrule-gitea.yaml` (REN-583). |
+| `uptime-kuma_test.yaml` | Hand-written suite for `prometheusrule-uptime-kuma.yaml` and the kuma exclusion in `prometheusrule-blackbox.yaml`. |
 | `mutations.txt` | Negative controls. Each must make the suite fail. |
 
 ## Adding a suite

kubernetes/apps/monitoring-rules/tests/mutations.txt

@@ -172,3 +172,12 @@ disk-wear-ephemeral-keeps-partition|kubernetes/apps/monitoring-rules/prometheusr
 # exactly-25% case in gitea_test.yaml.
 gitea-for-10m|kubernetes/apps/monitoring-rules/prometheusrule-gitea.yaml|/alert: GiteaDataVolumeLow/,/for:/s/for: 15m/for: 10m/
 gitea-floor-le|kubernetes/apps/monitoring-rules/prometheusrule-gitea.yaml|s/^\([[:space:]]*\)< 0\.25$/\1<= 0.25/
+
+# prometheusrule-uptime-kuma.yaml / blackbox kuma exclusion. Detected by the
+# cases in uptime-kuma_test.yaml.
+kuma-slow-no-positive-filter|kubernetes/apps/monitoring-rules/prometheusrule-uptime-kuma.yaml|s/(monitor_response_time > 0)/(monitor_response_time)/
+kuma-slow-threshold-2500|kubernetes/apps/monitoring-rules/prometheusrule-uptime-kuma.yaml|s/\[15m:1m\])) > 1500$/[15m:1m])) > 2500/
+kuma-slow-threshold-900|kubernetes/apps/monitoring-rules/prometheusrule-uptime-kuma.yaml|s/\[15m:1m\])) > 1500$/[15m:1m])) > 900/
+kuma-slow-for-30m|kubernetes/apps/monitoring-rules/prometheusrule-uptime-kuma.yaml|/alert: UptimeKumaSlowResponse/,/for:/s/for: 15m/for: 30m/
+kuma-down-for-20m|kubernetes/apps/monitoring-rules/prometheusrule-uptime-kuma.yaml|/alert: UptimeKumaUnreachable/,/for:/s/for: 10m/for: 20m/
+kuma-blackbox-exclusion-dropped|kubernetes/apps/monitoring-rules/prometheusrule-blackbox.yaml|s/kuma\\\\\.renner/kumx\\\\.renner/

kubernetes/apps/monitoring-rules/tests/uptime-kuma_test.yaml

@@ -0,0 +1,110 @@
+# Suite for prometheusrule-uptime-kuma.yaml and the kuma exclusion in
+# prometheusrule-blackbox.yaml.
+rule_files:
+  - ../prometheusrule-uptime-kuma.yaml
+  - ../prometheusrule-blackbox.yaml
+
+evaluation_interval: 1m
+
+tests:
+  # 2000 ms sustained: pending at 0m, fires 15m later, not before.
+  - interval: 1m
+    input_series:
+      - series: 'monitor_response_time{monitor_name="slow-mon"}'
+        values: '2000+0x40'
+    alert_rule_test:
+      - eval_time: 14m
+        alertname: UptimeKumaSlowResponse
+        exp_alerts: []
+      - eval_time: 16m
+        alertname: UptimeKumaSlowResponse
+        exp_alerts:
+          - exp_labels:
+              severity: warning
+              monitor_name: slow-mon
+            exp_annotations:
+              summary: "Uptime Kuma monitor slow-mon averaging 2k ms response time"
+              description: "The average response time of monitor slow-mon, as measured from the off-site Uptime Kuma, has stayed above 1500 ms for 15 minutes."
+
+  # 1000 ms stays silent; a monitor reporting only -1 (DOWN, group) stays silent.
+  - interval: 1m
+    input_series:
+      - series: 'monitor_response_time{monitor_name="fast-mon"}'
+        values: '1000+0x40'
+      - series: 'monitor_response_time{monitor_name="down-mon"}'
+        values: '-1+0x40'
+    alert_rule_test:
+      - eval_time: 35m
+        alertname: UptimeKumaSlowResponse
+        exp_alerts: []
+
+  # Alternating 1000 and -1: the -1 samples must not drag a slow average down
+  # nor count as slow; positives average 1000, silent.
+  - interval: 1m
+    input_series:
+      - series: 'monitor_response_time{monitor_name="flap-fast"}'
+        values: '1000 -1 1000 -1 1000 -1 1000 -1 1000 -1 1000 -1 1000 -1 1000 -1 1000 -1 1000 -1 1000 -1 1000 -1 1000 -1 1000 -1 1000 -1 1000 -1 1000 -1 1000 -1 1000 -1 1000 -1'
+    alert_rule_test:
+      - eval_time: 35m
+        alertname: UptimeKumaSlowResponse
+        exp_alerts: []
+
+  # Alternating 2000 and -1: the -1 samples are excluded, so the average of the
+  # real samples is 2000 and the alert fires. Unfiltered, the mean is ~500.
+  - interval: 1m
+    input_series:
+      - series: 'monitor_response_time{monitor_name="flap-slow"}'
+        values: '2000 -1 2000 -1 2000 -1 2000 -1 2000 -1 2000 -1 2000 -1 2000 -1 2000 -1 2000 -1 2000 -1 2000 -1 2000 -1 2000 -1 2000 -1 2000 -1 2000 -1 2000 -1 2000 -1 2000 -1'
+    alert_rule_test:
+      - eval_time: 35m
+        alertname: UptimeKumaSlowResponse
+        exp_alerts:
+          - exp_labels:
+              severity: warning
+              monitor_name: flap-slow
+            exp_annotations:
+              summary: "Uptime Kuma monitor flap-slow averaging 2k ms response time"
+              description: "The average response time of monitor flap-slow, as measured from the off-site Uptime Kuma, has stayed above 1500 ms for 15 minutes."
+
+  # Dead-man probe: critical after 10m of failure; the catch-all warning must
+  # not also fire for kuma, but still fires for another failing instance.
+  - interval: 1m
+    input_series:
+      - series: 'probe_success{job="blackbox-http",instance="https://kuma.renner.uno"}'
+        values: '0+0x40'
+      - series: 'probe_success{job="blackbox-http",instance="https://other.renner.uno"}'
+        values: '0+0x40'
+    alert_rule_test:
+      - eval_time: 9m
+        alertname: UptimeKumaUnreachable
+        exp_alerts: []
+      - eval_time: 11m
+        alertname: UptimeKumaUnreachable
+        exp_alerts:
+          - exp_labels:
+              severity: critical
+              job: blackbox-http
+              instance: https://kuma.renner.uno
+            exp_annotations:
+              summary: "Uptime Kuma (kuma.renner.uno) unreachable for 10m+"
+              description: "The blackbox probe of the off-site Uptime Kuma has been failing for 10 minutes. The external endpoints it monitors are unwatched until it returns. Check the NetBird VPS (netbird.renner.uno), the uptime-kuma container and Traefik."
+      - eval_time: 15m
+        alertname: BlackboxProbeFailing
+        exp_alerts:
+          - exp_labels:
+              severity: warning
+              job: blackbox-http
+              instance: https://other.renner.uno
+            exp_annotations:
+              summary: "Blackbox probe of https://other.renner.uno has been failing for 10 minutes"
+              description: "The endpoint is not answering as expected. For the Authentik-gated hostnames this is a reachability check covering DNS, TLS, Traefik and the gate outpost — it says nothing about the app behind the gate, which the internal /-/healthy probes cover separately."
+
+  # Healthy probe stays silent.
+  - interval: 1m
+    input_series:
+      - series: 'probe_success{job="blackbox-http",instance="https://kuma.renner.uno"}'
+        values: '1+0x40'
+    alert_rule_test:
+      - eval_time: 30m
+        alertname: UptimeKumaUnreachable
+        exp_alerts: []

kubernetes/apps/monitoring/helmrelease-prometheus.yaml

@@ -97,6 +97,10 @@ spec:
               resources:
                 requests:
                   storage: 50Gi
+        # Mounted at /etc/prometheus/secrets/<name>/; see
+        # ./uptime-kuma-metrics-externalsecret.yaml.
+        secrets:
+          - uptime-kuma-metrics
         # Additional scrape targets not covered by ServiceMonitors.
         additionalScrapeConfigs:
           # Gitea exposes /metrics anonymously on its main HTTP port. The
@@ -106,6 +110,16 @@ spec:
             static_configs:
               - targets: ["gitea.gitea.svc.cluster.local:3000"]
             metrics_path: /metrics
+          # Off-site Uptime Kuma. /metrics takes the API key as the basic-auth
+          # password with an empty username.
+          - job_name: uptime-kuma
+            scheme: https
+            metrics_path: /metrics
+            basic_auth:
+              username: ""
+              password_file: /etc/prometheus/secrets/uptime-kuma-metrics/api_key
+            static_configs:
+              - targets: ["kuma.renner.uno:443"]
 
     # Grafana
     grafana:

kubernetes/apps/monitoring/kustomization.yaml

@@ -3,6 +3,7 @@ kind: Kustomization
 resources:
   - grafana-oauth-externalsecret.yaml
   - alertmanager-smtp-externalsecret.yaml
+  - uptime-kuma-metrics-externalsecret.yaml
   - helmrelease-prometheus.yaml
   - helmrelease-loki.yaml
   - helmrelease-alloy-syslog.yaml

kubernetes/apps/monitoring/uptime-kuma-metrics-externalsecret.yaml

@@ -0,0 +1,19 @@
+---
+# Uptime Kuma /metrics API key for Prometheus; OpenBao homelab/infrastructure/uptime-kuma-metrics.
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: uptime-kuma-metrics
+  namespace: monitoring
+spec:
+  refreshInterval: 1h
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: openbao
+  target:
+    name: uptime-kuma-metrics
+  data:
+    - secretKey: api_key
+      remoteRef:
+        key: infrastructure/uptime-kuma-metrics
+        property: api_key

kubernetes/infrastructure/image-automation/hermes.yaml

@@ -8,9 +8,6 @@ spec:
   interval: 1h
 ---
 # v-prefix tags (v0.22.2). The filter skips rc.*, -desktop and per-arch tags.
-# The repo also carries the retired date-versioned releases (v2026.4.3 ..
-# v2026.9.24, last one 2026-09-24); as semver they sort above 0.x, so the
-# range caps below 1.0.0 or the policy picks the old v2026.9.24.
 apiVersion: image.toolkit.fluxcd.io/v1
 kind: ImagePolicy
 metadata:
@@ -24,4 +21,4 @@ spec:
     extract: '$ver'
   policy:
     semver:
-      range: ">=0.22.2 <1.0.0"
+      range: ">=0.22.2"

kubernetes/infrastructure/openbao/policies/uptime-kuma-vps.hcl

@@ -1,18 +1,10 @@
 # OpenBao policy: uptime-kuma-vps
-# Read on the Uptime Kuma admin and SMTP credentials; read/write on the metrics API key; self-renewal for the CI token. NOT applied by Flux.
+# Read-only on the Uptime Kuma admin credentials; self-renewal for the CI token. NOT applied by Flux.
 
 path "homelab/data/infrastructure/uptime-kuma" {
   capabilities = ["read"]
 }
 
-path "homelab/data/infrastructure/uptime-kuma/smtp" {
-  capabilities = ["read"]
-}
-
-path "homelab/data/infrastructure/uptime-kuma-metrics" {
-  capabilities = ["read", "create", "update"]
-}
-
 path "auth/token/renew-self" {
   capabilities = ["update"]
 }