From 811178e321002c2d69b2a85449f54d25522cf556 Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Thu, 29 Oct 2026 16:33:16 +0100 Subject: [PATCH] feat(kuma-sync): one mail per outage via a root group Monitors without a parentRef are placed under the root group set by KUMA_ROOT_GROUP (kuma-sync/renner-uno). Only the root references the email notification, so a fleet-wide outage sends one DOWN and one UP mail listing the affected groups. resendInterval (consecutive DOWN heartbeats) is now a managed KumaMonitor field; the root resends hourly while down. Co-Authored-By: Claude Opus 5.5 --- .../api/v1alpha1/kumamonitor_types.go | 4 + .../kuma-sync/api/v1alpha1/validation_test.go | 9 + ci/images/kuma-sync/cmd/kuma-sync/config.go | 36 ++- .../kuma-sync/cmd/kuma-sync/config_test.go | 20 ++ ci/images/kuma-sync/cmd/kuma-sync/main.go | 1 + .../crd/uptime.renner.uno_kumamonitors.yaml | 7 + ci/images/kuma-sync/internal/app/manager.go | 8 +- .../controller/kumamonitor_controller.go | 126 +++++++++-- .../kuma-sync/internal/controller/mapping.go | 17 +- .../internal/controller/mapping_test.go | 5 + .../internal/controller/rootgroup_test.go | 209 ++++++++++++++++++ .../kuma-sync/internal/kumaclient/breml.go | 4 +- .../kumaclient/breml_integration_test.go | 12 +- .../kuma-sync/internal/kumaclient/client.go | 3 + .../internal/kumaclient/resend_test.go | 61 +++++ .../references/uptime-monitoring.md | 3 + kubernetes/apps/kuma-sync/deployment.yaml | 2 + .../apps/kuma-sync/kumamonitor-root.yaml | 14 ++ kubernetes/apps/kuma-sync/kustomization.yaml | 1 + kubernetes/clusters/homelab/kuma-sync.yaml | 8 +- .../kuma-sync-crds/crd-kumamonitors.yaml | 7 + 21 files changed, 519 insertions(+), 38 deletions(-) create mode 100644 ci/images/kuma-sync/internal/controller/rootgroup_test.go create mode 100644 ci/images/kuma-sync/internal/kumaclient/resend_test.go create mode 100644 kubernetes/apps/kuma-sync/kumamonitor-root.yaml diff --git a/ci/images/kuma-sync/api/v1alpha1/kumamonitor_types.go b/ci/images/kuma-sync/api/v1alpha1/kumamonitor_types.go index afed82d5..d9bb10a6 100644 --- a/ci/images/kuma-sync/api/v1alpha1/kumamonitor_types.go +++ b/ci/images/kuma-sync/api/v1alpha1/kumamonitor_types.go @@ -97,6 +97,10 @@ type KumaMonitorSpec struct { // +kubebuilder:validation:Minimum=0 // +kubebuilder:default=0 MaxRetries int32 `json:"maxRetries,omitempty"` + // Consecutive DOWN heartbeats after which Kuma sends the DOWN notification + // again, not minutes; 0 sends it once per outage. + // +kubebuilder:validation:Minimum=0 + ResendInterval int32 `json:"resendInterval,omitempty"` // Seconds. // +kubebuilder:validation:Minimum=1 Timeout *int32 `json:"timeout,omitempty"` diff --git a/ci/images/kuma-sync/api/v1alpha1/validation_test.go b/ci/images/kuma-sync/api/v1alpha1/validation_test.go index 437a1c0f..4e3449a8 100644 --- a/ci/images/kuma-sync/api/v1alpha1/validation_test.go +++ b/ci/images/kuma-sync/api/v1alpha1/validation_test.go @@ -131,6 +131,15 @@ func TestNegativeMaxRetriesRejected(t *testing.T) { expectInvalid(t, m, "spec.maxRetries") } +func TestNegativeResendIntervalRejected(t *testing.T) { + m := monitor("neg-resend", KumaMonitorSpec{ + Type: "group", + DisplayName: "g", + ResendInterval: -1, + }) + expectInvalid(t, m, "spec.resendInterval") +} + func TestJSONQueryBadOperator(t *testing.T) { m := monitor("jq-bad-op", KumaMonitorSpec{ Type: "json-query", diff --git a/ci/images/kuma-sync/cmd/kuma-sync/config.go b/ci/images/kuma-sync/cmd/kuma-sync/config.go index eb8b2433..aa65b09b 100644 --- a/ci/images/kuma-sync/cmd/kuma-sync/config.go +++ b/ci/images/kuma-sync/cmd/kuma-sync/config.go @@ -3,19 +3,25 @@ package main import ( "errors" "fmt" + "strings" + + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/validation" ) type config struct { URL string Username string Password string + // RootGroup is the zero value when KUMA_ROOT_GROUP is unset. + RootGroup types.NamespacedName } // loadConfig reads the Kuma connection settings from the environment. It -// names every missing variable. +// names every missing or invalid variable. func loadConfig(getenv func(string) string) (config, error) { var cfg config - var missing []error + var errs []error for _, v := range []struct { name string dst *string @@ -25,8 +31,30 @@ func loadConfig(getenv func(string) string) (config, error) { {"KUMA_PASSWORD", &cfg.Password}, } { if *v.dst = getenv(v.name); *v.dst == "" { - missing = append(missing, fmt.Errorf("missing required environment variable %s", v.name)) + errs = append(errs, fmt.Errorf("missing required environment variable %s", v.name)) } } - return cfg, errors.Join(missing...) + if raw := getenv("KUMA_ROOT_GROUP"); raw != "" { + root, err := parseRootGroup(raw) + if err != nil { + errs = append(errs, fmt.Errorf("KUMA_ROOT_GROUP: %w", err)) + } + cfg.RootGroup = root + } + return cfg, errors.Join(errs...) +} + +// parseRootGroup parses namespace/name of a KumaMonitor. +func parseRootGroup(raw string) (types.NamespacedName, error) { + ns, name, ok := strings.Cut(raw, "/") + if !ok || strings.Contains(name, "/") { + return types.NamespacedName{}, fmt.Errorf("%q is not namespace/name", raw) + } + if msgs := validation.IsDNS1123Label(ns); len(msgs) > 0 { + return types.NamespacedName{}, fmt.Errorf("namespace %q: %s", ns, strings.Join(msgs, "; ")) + } + if msgs := validation.IsDNS1123Subdomain(name); len(msgs) > 0 { + return types.NamespacedName{}, fmt.Errorf("name %q: %s", name, strings.Join(msgs, "; ")) + } + return types.NamespacedName{Namespace: ns, Name: name}, nil } diff --git a/ci/images/kuma-sync/cmd/kuma-sync/config_test.go b/ci/images/kuma-sync/cmd/kuma-sync/config_test.go index 1583bfc6..ee0b853a 100644 --- a/ci/images/kuma-sync/cmd/kuma-sync/config_test.go +++ b/ci/images/kuma-sync/cmd/kuma-sync/config_test.go @@ -3,6 +3,8 @@ package main import ( "strings" "testing" + + "k8s.io/apimachinery/pkg/types" ) func TestLoadConfig(t *testing.T) { @@ -36,3 +38,21 @@ func TestLoadConfig(t *testing.T) { } } } + +func TestLoadConfigRootGroup(t *testing.T) { + env := func(root string) func(string) string { + m := map[string]string{"KUMA_URL": "http://kuma", "KUMA_USERNAME": "u", "KUMA_PASSWORD": "p", "KUMA_ROOT_GROUP": root} + return func(k string) string { return m[k] } + } + + cfg, err := loadConfig(env("kuma-sync/renner-uno")) + if err != nil || cfg.RootGroup != (types.NamespacedName{Namespace: "kuma-sync", Name: "renner-uno"}) { + t.Fatalf("cfg=%+v err=%v", cfg, err) + } + + for _, bad := range []string{"renner-uno", "/renner-uno", "kuma-sync/", "a/b/c", "Kuma/x", "ns/UPPER", " ns/x"} { + if _, err := loadConfig(env(bad)); err == nil || !strings.Contains(err.Error(), "KUMA_ROOT_GROUP") { + t.Errorf("%q: err = %v, want KUMA_ROOT_GROUP rejected", bad, err) + } + } +} diff --git a/ci/images/kuma-sync/cmd/kuma-sync/main.go b/ci/images/kuma-sync/cmd/kuma-sync/main.go index 018028a2..33f88b8c 100644 --- a/ci/images/kuma-sync/cmd/kuma-sync/main.go +++ b/ci/images/kuma-sync/cmd/kuma-sync/main.go @@ -48,6 +48,7 @@ func run() error { GCInterval: 5 * time.Minute, OrphanGrace: time.Hour, Resync: 5 * time.Minute, + RootGroup: cfg.RootGroup, }) if err != nil { return err diff --git a/ci/images/kuma-sync/config/crd/uptime.renner.uno_kumamonitors.yaml b/ci/images/kuma-sync/config/crd/uptime.renner.uno_kumamonitors.yaml index 9143f70a..79f9ef18 100644 --- a/ci/images/kuma-sync/config/crd/uptime.renner.uno_kumamonitors.yaml +++ b/ci/images/kuma-sync/config/crd/uptime.renner.uno_kumamonitors.yaml @@ -222,6 +222,13 @@ spec: - hostname - port type: object + resendInterval: + description: |- + Consecutive DOWN heartbeats after which Kuma sends the DOWN notification + again, not minutes; 0 sends it once per outage. + format: int32 + minimum: 0 + type: integer retryInterval: default: 60 description: Seconds. diff --git a/ci/images/kuma-sync/internal/app/manager.go b/ci/images/kuma-sync/internal/app/manager.go index 4ac6f336..889820ce 100644 --- a/ci/images/kuma-sync/internal/app/manager.go +++ b/ci/images/kuma-sync/internal/app/manager.go @@ -10,6 +10,7 @@ import ( "time" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" clientgoscheme "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" "k8s.io/utils/ptr" @@ -41,6 +42,9 @@ type Options struct { // Resync is how often a healthy resource is reconciled again, which is // what repairs drift made in Kuma; 0 means 5 minutes. Resync time.Duration + // RootGroup is the group KumaMonitor that monitors without a parentRef are + // placed under; the zero value leaves them at the top level. + RootGroup types.NamespacedName // SkipNameValidation allows several managers in one process, as in tests // that restart the manager. @@ -72,7 +76,9 @@ func NewManager(o Options) (manager.Manager, error) { return nil, fmt.Errorf("create manager: %w", err) } - if err := (&controller.Reconciler{Client: mgr.GetClient(), Kuma: o.Kuma, Resync: o.Resync}).SetupWithManager(mgr); err != nil { + if err := (&controller.Reconciler{ + Client: mgr.GetClient(), Kuma: o.Kuma, Resync: o.Resync, RootGroup: o.RootGroup, + }).SetupWithManager(mgr); err != nil { return nil, fmt.Errorf("set up reconciler: %w", err) } if err := (&controller.StatusPageReconciler{Client: mgr.GetClient(), Kuma: o.Kuma, Resync: o.Resync}).SetupWithManager(mgr); err != nil { diff --git a/ci/images/kuma-sync/internal/controller/kumamonitor_controller.go b/ci/images/kuma-sync/internal/controller/kumamonitor_controller.go index 3636d27e..af8d58e1 100644 --- a/ci/images/kuma-sync/internal/controller/kumamonitor_controller.go +++ b/ci/images/kuma-sync/internal/controller/kumamonitor_controller.go @@ -58,6 +58,15 @@ type Reconciler struct { // Resync is how often a healthy resource is reconciled again, which repairs // drift made in Kuma. 0 means 5 minutes. Resync time.Duration + // RootGroup is the group KumaMonitor every monitor without a parentRef is + // placed under in Kuma; the zero value places them at the top level. + RootGroup types.NamespacedName +} + +func (r *Reconciler) hasRoot() bool { return r.RootGroup.Name != "" } + +func (r *Reconciler) isRoot(o client.Object) bool { + return r.hasRoot() && o.GetNamespace() == r.RootGroup.Namespace && o.GetName() == r.RootGroup.Name } func (r *Reconciler) resync() time.Duration { @@ -70,14 +79,18 @@ func (r *Reconciler) resync() time.Duration { // SetupWithManager ignores status-only updates, which keeps failure backoff // intact. Deletion still reconciles: setting deletionTimestamp bumps // metadata.generation. A KumaNotification whose ID changes enqueues the -// monitors referencing it. +// monitors referencing it; a change to the root group enqueues the monitors +// without a parentRef. func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { - return ctrl.NewControllerManagedBy(mgr). + b := ctrl.NewControllerManagedBy(mgr). For(&v1.KumaMonitor{}, builder.WithPredicates(predicate.GenerationChangedPredicate{})). Watches(&v1.KumaNotification{}, handler.EnqueueRequestsFromMapFunc(r.referencingMonitors), - builder.WithPredicates(predicate.Funcs{UpdateFunc: notificationIDChanged})). - Named("kumamonitor"). - Complete(r) + builder.WithPredicates(predicate.Funcs{UpdateFunc: notificationIDChanged})) + if r.hasRoot() { + b = b.Watches(&v1.KumaMonitor{}, handler.EnqueueRequestsFromMapFunc(r.rootChildren), + builder.WithPredicates(r.rootPredicate())) + } + return b.Named("kumamonitor").Complete(r) } // failure is a Ready=False outcome. The message is fixed per reason; the cause @@ -99,6 +112,9 @@ const ( msgParentNotGroup = "parent must be of type group" msgParentCycle = "parentRef forms a cycle" msgParentTooDeep = "parentRef chain is too deep" + msgRootPending = "root group has no monitor ID yet" + msgRootNotGroup = "root group must be of type group" + msgRootHasParent = "root group must not have a parentRef" msgNotificationPending = "a referenced notification has no notification ID yet" msgSynced = "monitor in sync" msgNotOwned = "previous monitor exists without our ownership tag; restore the tag or delete it in Kuma" @@ -119,17 +135,24 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu } desired := Desired(km) - if km.Spec.ParentRef != nil { - p, err := r.resolveParent(ctx, km) - if err != nil { + var p parentOutcome + var err error + switch { + case km.Spec.ParentRef != nil: + p, err = r.resolveParent(ctx, km) + case r.hasRoot() && !r.isRoot(km): + p, err = r.resolveRoot(ctx) + } + if err != nil { + return ctrl.Result{}, err + } + if p.reason != "" { + if err := r.setReady(ctx, km, metav1.ConditionFalse, p.reason, p.message, nil); err != nil { return ctrl.Result{}, err } - if p.reason != "" { - if err := r.setReady(ctx, km, metav1.ConditionFalse, p.reason, p.message, nil); err != nil { - return ctrl.Result{}, err - } - return ctrl.Result{RequeueAfter: p.requeue}, nil - } + return ctrl.Result{RequeueAfter: p.requeue}, nil + } + if p.id != 0 { desired.Parent = &p.id } @@ -323,8 +346,75 @@ type parentOutcome struct { requeue time.Duration } +// rootChanged reports changes to the root group that alter what the monitors +// without a parentRef resolve to. +func rootChanged(e event.UpdateEvent) bool { + o, ok1 := e.ObjectOld.(*v1.KumaMonitor) + n, ok2 := e.ObjectNew.(*v1.KumaMonitor) + if !ok1 || !ok2 { + return true + } + return o.Status.MonitorID != n.Status.MonitorID || + o.Spec.Type != n.Spec.Type || + (o.Spec.ParentRef == nil) != (n.Spec.ParentRef == nil) || + o.DeletionTimestamp.IsZero() != n.DeletionTimestamp.IsZero() +} + +// rootPredicate passes events of the root group only. +func (r *Reconciler) rootPredicate() predicate.Funcs { + return predicate.Funcs{ + CreateFunc: func(e event.CreateEvent) bool { return r.isRoot(e.Object) }, + DeleteFunc: func(e event.DeleteEvent) bool { return r.isRoot(e.Object) }, + GenericFunc: func(e event.GenericEvent) bool { return r.isRoot(e.Object) }, + UpdateFunc: func(e event.UpdateEvent) bool { return r.isRoot(e.ObjectNew) && rootChanged(e) }, + } +} + +// rootChildren enqueues the KumaMonitors of all namespaces that have no +// parentRef, except the root group itself. +func (r *Reconciler) rootChildren(ctx context.Context, _ client.Object) []reconcile.Request { + var list v1.KumaMonitorList + if err := r.Client.List(ctx, &list); err != nil { + log.FromContext(ctx).Error(err, "list KumaMonitors to enqueue") + return nil + } + var reqs []reconcile.Request + for i := range list.Items { + km := &list.Items[i] + if km.Spec.ParentRef == nil && !r.isRoot(km) { + reqs = append(reqs, reconcile.Request{NamespacedName: types.NamespacedName{Namespace: km.Namespace, Name: km.Name}}) + } + } + return reqs +} + +// resolveRoot returns the Kuma ID of the root group for a monitor without a +// parentRef. The root must be a group without a parentRef: under one of its +// own ancestors it would close a cycle in Kuma. +func (r *Reconciler) resolveRoot(ctx context.Context) (parentOutcome, error) { + pending := parentOutcome{reason: ReasonParentPending, message: msgRootPending, requeue: parentRetry} + root := &v1.KumaMonitor{} + err := r.Client.Get(ctx, r.RootGroup, root) + if apierrors.IsNotFound(err) { + return pending, nil + } + if err != nil { + return parentOutcome{}, err + } + switch { + case root.Spec.Type != "group": + return parentOutcome{reason: ReasonInvalidParent, message: msgRootNotGroup, requeue: r.resync()}, nil + case root.Spec.ParentRef != nil: + return parentOutcome{reason: ReasonInvalidParent, message: msgRootHasParent, requeue: r.resync()}, nil + case root.Status.MonitorID == 0 || !root.DeletionTimestamp.IsZero(): + return pending, nil + } + return parentOutcome{id: root.Status.MonitorID}, nil +} + // resolveParent checks the whole parentRef chain: every ancestor must exist -// and be a group, without cycles or more than maxParentDepth levels. +// and be a group, without cycles or more than maxParentDepth levels. The root +// group, which the topmost ancestor is placed under, counts as a level. func (r *Reconciler) resolveParent(ctx context.Context, km *v1.KumaMonitor) (parentOutcome, error) { invalid := func(msg string) (parentOutcome, error) { return parentOutcome{reason: ReasonInvalidParent, message: msg, requeue: r.resync()}, nil @@ -334,7 +424,8 @@ func (r *Reconciler) resolveParent(ctx context.Context, km *v1.KumaMonitor) (par visited := map[string]bool{km.Name: true} cur := km var directID int64 - for depth := 0; cur.Spec.ParentRef != nil; depth++ { + depth := 0 + for ; cur.Spec.ParentRef != nil; depth++ { if depth >= maxParentDepth { return invalid(msgParentTooDeep) } @@ -359,6 +450,9 @@ func (r *Reconciler) resolveParent(ctx context.Context, km *v1.KumaMonitor) (par } cur = parent } + if r.hasRoot() && !r.isRoot(cur) && depth >= maxParentDepth { + return invalid(msgParentTooDeep) + } if directID == 0 { return pending, nil } diff --git a/ci/images/kuma-sync/internal/controller/mapping.go b/ci/images/kuma-sync/internal/controller/mapping.go index b2b17a17..8b128a12 100644 --- a/ci/images/kuma-sync/internal/controller/mapping.go +++ b/ci/images/kuma-sync/internal/controller/mapping.go @@ -24,14 +24,15 @@ func OwnerKey(km *v1.KumaMonitor) string { func Desired(km *v1.KumaMonitor) kumaclient.Monitor { s := km.Spec m := kumaclient.Monitor{ - Type: s.Type, - Name: s.DisplayName, - Description: s.Description, - Active: !s.Paused, - Interval: s.Interval, - RetryInterval: s.RetryInterval, - MaxRetries: s.MaxRetries, - OwnerKey: OwnerKey(km), + Type: s.Type, + Name: s.DisplayName, + Description: s.Description, + Active: !s.Paused, + Interval: s.Interval, + RetryInterval: s.RetryInterval, + MaxRetries: s.MaxRetries, + ResendInterval: s.ResendInterval, + OwnerKey: OwnerKey(km), } if s.Timeout != nil { m.Timeout = *s.Timeout diff --git a/ci/images/kuma-sync/internal/controller/mapping_test.go b/ci/images/kuma-sync/internal/controller/mapping_test.go index 6b5245d0..e5f41df4 100644 --- a/ci/images/kuma-sync/internal/controller/mapping_test.go +++ b/ci/images/kuma-sync/internal/controller/mapping_test.go @@ -57,6 +57,8 @@ func TestMappingAllTypes(t *testing.T) { port.Port = &v1.PortSpec{Hostname: "p.com", Port: 443} paused := base("group") paused.Paused = true + resend := base("group") + resend.ResendInterval = 60 wantKW := withHTTP(common("keyword")) wantKW.Keyword, wantKW.InvertKeyword = "ok", true @@ -74,6 +76,8 @@ func TestMappingAllTypes(t *testing.T) { wantPort.Hostname, wantPort.Port = "p.com", 443 wantPaused := common("group") wantPaused.Active = false + wantResend := common("group") + wantResend.ResendInterval = 60 cases := []struct { name string @@ -88,6 +92,7 @@ func TestMappingAllTypes(t *testing.T) { {"dns with conditions", dnsCond, wantDNSCond}, {"port", port, wantPort}, {"paused maps to inactive", paused, wantPaused}, + {"resend interval", resend, wantResend}, } for _, c := range cases { got := Desired(mk(c.spec)) diff --git a/ci/images/kuma-sync/internal/controller/rootgroup_test.go b/ci/images/kuma-sync/internal/controller/rootgroup_test.go new file mode 100644 index 00000000..9579506e --- /dev/null +++ b/ci/images/kuma-sync/internal/controller/rootgroup_test.go @@ -0,0 +1,209 @@ +package controller + +import ( + "fmt" + "slices" + "testing" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + + v1 "gitea.renner.uno/konairius/homelab/kuma-sync/api/v1alpha1" +) + +const rootID = 7 + +var rootName = types.NamespacedName{Namespace: "default", Name: "root"} + +func rootGroup() *v1.KumaMonitor { + g := group("root", "") + g.Status.MonitorID = rootID + return g +} + +func newRootEnv(t *testing.T, objs ...client.Object) *env { + t.Helper() + e := newEnv(t, objs...) + e.r.RootGroup = rootName + return e +} + +func (e *env) parentOf(t *testing.T, name string) *int64 { + t.Helper() + got := e.kuma.Get(e.get(t, name).Status.MonitorID) + if got == nil { + t.Fatalf("%s: no Kuma monitor", name) + } + return got.Parent +} + +func TestRootGroupParentsParentless(t *testing.T) { + e := newRootEnv(t, rootGroup(), httpMonitor("a")) + if _, err := e.reconcile(t, "a"); err != nil { + t.Fatal(err) + } + if p := e.parentOf(t, "a"); p == nil || *p != rootID { + t.Errorf("parent = %v, want %d", p, rootID) + } +} + +func TestRootGroupKeepsParentRef(t *testing.T) { + grp := group("grp", "") + grp.Status.MonitorID = 42 + child := httpMonitor("child") + child.Spec.ParentRef = &v1.LocalRef{Name: "grp"} + e := newRootEnv(t, rootGroup(), grp, child) + if _, err := e.reconcile(t, "child"); err != nil { + t.Fatal(err) + } + if p := e.parentOf(t, "child"); p == nil || *p != 42 { + t.Errorf("parent = %v, want 42", p) + } +} + +func TestRootGroupItselfHasNoParent(t *testing.T) { + root := rootGroup() + root.Status.MonitorID = 0 + e := newRootEnv(t, root) + if _, err := e.reconcile(t, "root"); err != nil { + t.Fatal(err) + } + if p := e.parentOf(t, "root"); p != nil { + t.Errorf("parent = %d, want none", *p) + } +} + +func TestRootGroupUnsetLeavesTopLevel(t *testing.T) { + e := newEnv(t, rootGroup(), httpMonitor("a")) + if _, err := e.reconcile(t, "a"); err != nil { + t.Fatal(err) + } + if p := e.parentOf(t, "a"); p != nil { + t.Errorf("parent = %d, want none", *p) + } +} + +func TestRootGroupPending(t *testing.T) { + noID := rootGroup() + noID.Status.MonitorID = 0 + deletingRoot := deleting(rootGroup()) + cases := map[string][]client.Object{ + "missing": nil, + "no ID": {noID}, + "deleting": {deletingRoot}, + "not group": {func() *v1.KumaMonitor { m := httpMonitor("root"); m.Status.MonitorID = rootID; return m }()}, + "has parent": {func() *v1.KumaMonitor { + m := rootGroup() + m.Spec.ParentRef = &v1.LocalRef{Name: "x"} + return m + }()}, + } + want := map[string]struct { + reason string + requeue time.Duration + }{ + "missing": {ReasonParentPending, 10 * time.Second}, + "no ID": {ReasonParentPending, 10 * time.Second}, + "deleting": {ReasonParentPending, 10 * time.Second}, + "not group": {ReasonInvalidParent, 5 * time.Minute}, + "has parent": {ReasonInvalidParent, 5 * time.Minute}, + } + for name, objs := range cases { + e := newRootEnv(t, append(objs, httpMonitor("a"))...) + res, err := e.reconcile(t, "a") + if err != nil { + t.Fatalf("%s: %v", name, err) + } + if res.RequeueAfter != want[name].requeue { + t.Errorf("%s: RequeueAfter = %v, want %v", name, res.RequeueAfter, want[name].requeue) + } + if c := ready(t, e.get(t, "a")); c.Status != metav1.ConditionFalse || c.Reason != want[name].reason { + t.Errorf("%s: Ready = %+v", name, c) + } + if e.cnt.creates != 0 { + t.Errorf("%s: creates = %d", name, e.cnt.creates) + } + } +} + +// A chain of maxParentDepth groups is allowed on its own, but the root group +// adds one more level above it. +func TestRootGroupCountsTowardsDepth(t *testing.T) { + var objs []client.Object + for i := range maxParentDepth { + parent := "" + if i < maxParentDepth-1 { + parent = fmt.Sprintf("c%d", i+1) + } + objs = append(objs, group(fmt.Sprintf("c%d", i), parent)) + } + deep := httpMonitor("deep") + deep.Spec.ParentRef = &v1.LocalRef{Name: "c0"} + objs = append(objs, deep) + + if _, err := newEnv(t, objs...).reconcile(t, "deep"); err != nil { + t.Fatal(err) + } + + e := newRootEnv(t, append(objs, rootGroup())...) + if _, err := e.reconcile(t, "deep"); err != nil { + t.Fatal(err) + } + if c := ready(t, e.get(t, "deep")); c.Reason != ReasonInvalidParent || c.Message != msgParentTooDeep { + t.Errorf("Ready = %+v", c) + } + + // A chain ending in the root itself gets no extra level. + objs[maxParentDepth-1] = func() *v1.KumaMonitor { g := rootGroup(); g.Name = "c9"; return g }() + e = newEnv(t, objs...) + e.r.RootGroup = types.NamespacedName{Namespace: "default", Name: "c9"} + if _, err := e.reconcile(t, "deep"); err != nil { + t.Fatal(err) + } + if c := ready(t, e.get(t, "deep")); c.Status != metav1.ConditionTrue { + t.Errorf("chain ending in root: Ready = %+v", c) + } +} + +func TestRootChildrenEnqueuesParentless(t *testing.T) { + child := httpMonitor("child") + child.Spec.ParentRef = &v1.LocalRef{Name: "grp"} + other := httpMonitor("other") + other.Namespace = "elsewhere" + e := newRootEnv(t, rootGroup(), group("grp", ""), child, httpMonitor("a"), other) + + var got []string + for _, req := range e.r.rootChildren(t.Context(), rootGroup()) { + got = append(got, req.String()) + } + slices.Sort(got) + want := []string{"default/a", "default/grp", "elsewhere/other"} + if !slices.Equal(got, want) { + t.Errorf("enqueued %v, want %v", got, want) + } +} + +func TestRootPredicate(t *testing.T) { + r := &Reconciler{RootGroup: rootName} + p := r.rootPredicate() + old := rootGroup() + old.Status.MonitorID = 0 + withID := rootGroup() + other := group("other", "") + + if !p.Update(event.UpdateEvent{ObjectOld: old, ObjectNew: withID}) { + t.Error("root ID change must pass") + } + if p.Update(event.UpdateEvent{ObjectOld: withID, ObjectNew: withID.DeepCopy()}) { + t.Error("unchanged root must not pass") + } + if p.Update(event.UpdateEvent{ObjectOld: other, ObjectNew: other.DeepCopy()}) || p.Create(event.CreateEvent{Object: other}) { + t.Error("other monitors must not pass") + } + if !p.Create(event.CreateEvent{Object: withID}) || !p.Delete(event.DeleteEvent{Object: withID}) { + t.Error("root create and delete must pass") + } +} diff --git a/ci/images/kuma-sync/internal/kumaclient/breml.go b/ci/images/kuma-sync/internal/kumaclient/breml.go index 24dc4521..e8a960cc 100644 --- a/ci/images/kuma-sync/internal/kumaclient/breml.go +++ b/ci/images/kuma-sync/internal/kumaclient/breml.go @@ -378,7 +378,7 @@ func (b *Breml) update(ctx context.Context, m Monitor) error { return fmt.Errorf("update %d: monitor carries owner key %q, not %q: %w", m.ID, key, m.OwnerKey, ErrNotOwned) } // Start from the full current object, also across a type change, so - // fields we do not manage (proxy, resend interval, upside down, auth) + // fields we do not manage (proxy, upside down, auth) // survive the overwrite. wire, err := toWire(m, &cur) if err != nil { @@ -420,6 +420,7 @@ func toDomain(base monitor.Base, ownerKey string) (Monitor, error) { Interval: int32(base.Interval), RetryInterval: int32(base.RetryInterval), MaxRetries: int32(base.MaxRetries), + ResendInterval: int32(base.ResendInterval), NotificationIDs: slices.Clone(base.NotificationIDs), OwnerKey: ownerKey, } @@ -581,6 +582,7 @@ func writeBase(b *monitor.Base, m Monitor) { b.Interval = int64(m.Interval) b.RetryInterval = int64(m.RetryInterval) b.MaxRetries = int64(m.MaxRetries) + b.ResendInterval = int64(m.ResendInterval) b.IsActive = m.Active b.NotificationIDs = slices.Clone(m.NotificationIDs) } diff --git a/ci/images/kuma-sync/internal/kumaclient/breml_integration_test.go b/ci/images/kuma-sync/internal/kumaclient/breml_integration_test.go index aacfc401..b204f8ea 100644 --- a/ci/images/kuma-sync/internal/kumaclient/breml_integration_test.go +++ b/ci/images/kuma-sync/internal/kumaclient/breml_integration_test.go @@ -127,7 +127,7 @@ func TestCreateFieldsIntact(t *testing.T) { group := Monitor{ Type: "group", Name: "grp", Description: "a group", Active: true, - Interval: 60, RetryInterval: 60, MaxRetries: 0, OwnerKey: "default/grp", + Interval: 60, RetryInterval: 60, MaxRetries: 0, ResendInterval: 60, OwnerKey: "default/grp", } gid, err := b.Create(ctx, group) if err != nil { @@ -313,7 +313,6 @@ func setUnmanaged(t *testing.T, b *Breml, id int64) (notificationID int64) { t.Fatal(err) } kw.UpsideDown = true - kw.ResendInterval = 7 kw.NotificationIDs = []int64{nid} if err := b.c.UpdateMonitor(ctx, &kw); err != nil { t.Fatal(err) @@ -328,8 +327,8 @@ func assertUnmanaged(t *testing.T, c *Breml, id int64, when string) { if err != nil { t.Fatal(err) } - if !base.UpsideDown || base.ResendInterval != 7 { - t.Errorf("%s: unmanaged state lost: upsideDown=%v resend=%d", when, base.UpsideDown, base.ResendInterval) + if !base.UpsideDown { + t.Errorf("%s: unmanaged state lost: upsideDown=%v", when, base.UpsideDown) } } @@ -344,7 +343,7 @@ func TestUpdatePreservesUnmanagedAndClearsManaged(t *testing.T) { } orig := Monitor{ Type: "keyword", Name: "kw", Active: true, Parent: &gid, - Interval: 60, RetryInterval: 60, URL: "https://example.com", + Interval: 60, RetryInterval: 60, ResendInterval: 7, URL: "https://example.com", Headers: map[string]string{"A": "1"}, Keyword: "up", OwnerKey: "default/kw", } id, err := b.Create(ctx, orig) @@ -360,6 +359,7 @@ func TestUpdatePreservesUnmanagedAndClearsManaged(t *testing.T) { cleared.Headers = nil cleared.Keyword = "" cleared.Interval = 90 + cleared.ResendInterval = 0 if err := b.Update(ctx, cleared); err != nil { t.Fatal(err) } @@ -369,7 +369,7 @@ func TestUpdatePreservesUnmanagedAndClearsManaged(t *testing.T) { t.Fatal(err) } m := byID(got)[id] - if m.Parent != nil || m.Headers != nil || m.Keyword != "" || m.Interval != 90 || m.NotificationIDs != nil { + if m.Parent != nil || m.Headers != nil || m.Keyword != "" || m.Interval != 90 || m.ResendInterval != 0 || m.NotificationIDs != nil { t.Errorf("managed fields not cleared: %+v", m) } assertUnmanaged(t, fresh, id, "after plain update") diff --git a/ci/images/kuma-sync/internal/kumaclient/client.go b/ci/images/kuma-sync/internal/kumaclient/client.go index e9f57d13..e8cc8e5c 100644 --- a/ci/images/kuma-sync/internal/kumaclient/client.go +++ b/ci/images/kuma-sync/internal/kumaclient/client.go @@ -141,6 +141,9 @@ type Monitor struct { Interval int32 RetryInterval int32 MaxRetries int32 + // ResendInterval counts consecutive DOWN heartbeats between repeated DOWN + // notifications; 0 disables resending. + ResendInterval int32 // Timeout is in seconds and only applies to http, keyword and json-query. // 0 is written as round(0.8*Interval), because Kuma treats a stored 0 as a // huge request timeout. For other types it is ignored and reads back as 0. diff --git a/ci/images/kuma-sync/internal/kumaclient/resend_test.go b/ci/images/kuma-sync/internal/kumaclient/resend_test.go new file mode 100644 index 00000000..74094732 --- /dev/null +++ b/ci/images/kuma-sync/internal/kumaclient/resend_test.go @@ -0,0 +1,61 @@ +package kumaclient + +import ( + "encoding/json" + "testing" + + "github.com/breml/go-uptime-kuma-client/monitor" +) + +// resendOnWire writes m over existing (nil for a create) and returns the wire +// resendInterval and the value read back from it. +func resendOnWire(t *testing.T, m Monitor, existing *monitor.Base) (wire, back int64) { + t.Helper() + w, err := toWire(m, existing) + if err != nil { + t.Fatal(err) + } + raw, err := json.Marshal(w) + if err != nil { + t.Fatal(err) + } + var f struct { + ResendInterval int64 `json:"resendInterval"` + } + if err := json.Unmarshal(raw, &f); err != nil { + t.Fatal(err) + } + var base monitor.Base + if err := json.Unmarshal(raw, &base); err != nil { + t.Fatal(err) + } + got, err := toDomain(base, m.OwnerKey) + if err != nil { + t.Fatal(err) + } + return f.ResendInterval, int64(got.ResendInterval) +} + +func TestResendIntervalRoundTrip(t *testing.T) { + for _, m := range []Monitor{ + {ID: 1, Type: "group", Name: "g", Active: true, Interval: 60, RetryInterval: 60, ResendInterval: 60, OwnerKey: "ns/g"}, + {ID: 2, Type: "http", Name: "h", Active: true, Interval: 60, RetryInterval: 60, ResendInterval: 3, URL: "https://e.com", OwnerKey: "ns/h"}, + } { + wire, back := resendOnWire(t, m, nil) + if wire != int64(m.ResendInterval) || back != int64(m.ResendInterval) { + t.Errorf("%s: wire=%d read back=%d, want %d", m.Type, wire, back, m.ResendInterval) + } + } +} + +// 0 must overwrite a resend interval stored in Kuma, not keep it. +func TestResendIntervalClearedOnUpdate(t *testing.T) { + var existing monitor.Base + if err := json.Unmarshal([]byte(`{"id":1,"type":"group","name":"g","resendInterval":7}`), &existing); err != nil { + t.Fatal(err) + } + m := Monitor{ID: 1, Type: "group", Name: "g", Active: true, Interval: 60, RetryInterval: 60, OwnerKey: "ns/g"} + if wire, back := resendOnWire(t, m, &existing); wire != 0 || back != 0 { + t.Errorf("wire=%d read back=%d, want 0", wire, back) + } +} diff --git a/kubernetes/apps/hermes/skills/homelab-gitea-pr/references/uptime-monitoring.md b/kubernetes/apps/hermes/skills/homelab-gitea-pr/references/uptime-monitoring.md index d957c9a7..02577d69 100644 --- a/kubernetes/apps/hermes/skills/homelab-gitea-pr/references/uptime-monitoring.md +++ b/kubernetes/apps/hermes/skills/homelab-gitea-pr/references/uptime-monitoring.md @@ -14,6 +14,9 @@ monitor, and its history starts over. proxies, the Cloudflare-served apex `renner.uno` and third-party sites. - A monitor alerts only through its `notificationRefs` to cluster-scoped `KumaNotification` objects (`kubernetes/apps/kuma-sync/notification-*.yaml`). Without refs it sends nothing. +- Parentless monitors hang under the root group `kuma-sync/renner-uno` (controller env + `KUMA_ROOT_GROUP`). Only the root carries `notificationRefs`; don't add `notificationRefs` + to service monitors. - The CRD lives in `infrastructure/kuma-sync-crds`. Every consuming Flux Kustomization needs the KumaMonitor `healthCheckExprs` entry with `current: "true"` (see SKILL.md pitfalls). diff --git a/kubernetes/apps/kuma-sync/deployment.yaml b/kubernetes/apps/kuma-sync/deployment.yaml index 8d1f797e..509a6c4b 100644 --- a/kubernetes/apps/kuma-sync/deployment.yaml +++ b/kubernetes/apps/kuma-sync/deployment.yaml @@ -44,6 +44,8 @@ spec: secretKeyRef: name: kuma-sync-kuma key: password + - name: KUMA_ROOT_GROUP + value: kuma-sync/renner-uno ports: - name: metrics containerPort: 8080 diff --git a/kubernetes/apps/kuma-sync/kumamonitor-root.yaml b/kubernetes/apps/kuma-sync/kumamonitor-root.yaml new file mode 100644 index 00000000..dc7622fc --- /dev/null +++ b/kubernetes/apps/kuma-sync/kumamonitor-root.yaml @@ -0,0 +1,14 @@ +# Root group (KUMA_ROOT_GROUP): parent of every monitor without a parentRef; the only one with notificationRefs. +apiVersion: uptime.renner.uno/v1alpha1 +kind: KumaMonitor +metadata: + name: renner-uno + namespace: kuma-sync +spec: + type: group + displayName: renner.uno + interval: 60 + # heartbeats, not minutes: 60 x 60s = hourly reminder while down + resendInterval: 60 + notificationRefs: + - name: email diff --git a/kubernetes/apps/kuma-sync/kustomization.yaml b/kubernetes/apps/kuma-sync/kustomization.yaml index 3298feb2..33faccb6 100644 --- a/kubernetes/apps/kuma-sync/kustomization.yaml +++ b/kubernetes/apps/kuma-sync/kustomization.yaml @@ -6,3 +6,4 @@ resources: - deployment.yaml - statuspage-renner.yaml - notification-email.yaml + - kumamonitor-root.yaml diff --git a/kubernetes/clusters/homelab/kuma-sync.yaml b/kubernetes/clusters/homelab/kuma-sync.yaml index ebddddaa..3dc3dd62 100644 --- a/kubernetes/clusters/homelab/kuma-sync.yaml +++ b/kubernetes/clusters/homelab/kuma-sync.yaml @@ -1,5 +1,5 @@ -# kuma-sync controller: KumaMonitor -> Uptime Kuma monitors. Ships no KumaMonitor objects; -# they live with their app, the CRD in infrastructure/kuma-sync-crds. +# kuma-sync controller: KumaMonitor -> Uptime Kuma monitors. Ships only the root group KumaMonitor; +# service monitors live with their app, the CRD in infrastructure/kuma-sync-crds. # healthCheckExprs: Flux has no status reader for ExternalSecret; fail only on the non-retrying reasons. apiVersion: kustomize.toolkit.fluxcd.io/v1 kind: Kustomization @@ -21,6 +21,10 @@ spec: kind: ExternalSecret failed: "has(status.conditions) && status.conditions.exists(c, c.type == 'Ready' && c.status == 'False' && (c.reason == 'SecretOwnedByOther' || c.reason == 'SecretImmutable'))" current: "has(status.conditions) && status.conditions.exists(c, c.type == 'Ready' && c.status == 'True')" + # KumaMonitor Ready tracks Uptime Kuma on the VPS, not this apply. + - apiVersion: uptime.renner.uno/v1alpha1 + kind: KumaMonitor + current: "true" dependsOn: - name: infrastructure - name: infrastructure-crds diff --git a/kubernetes/infrastructure/kuma-sync-crds/crd-kumamonitors.yaml b/kubernetes/infrastructure/kuma-sync-crds/crd-kumamonitors.yaml index 3fd6745b..409227bc 100644 --- a/kubernetes/infrastructure/kuma-sync-crds/crd-kumamonitors.yaml +++ b/kubernetes/infrastructure/kuma-sync-crds/crd-kumamonitors.yaml @@ -223,6 +223,13 @@ spec: - hostname - port type: object + resendInterval: + description: |- + Consecutive DOWN heartbeats after which Kuma sends the DOWN notification + again, not minutes; 0 sends it once per outage. + format: int32 + minimum: 0 + type: integer retryInterval: default: 60 description: Seconds. -- 2.54.0