fix: [alert] KubeAggregatedAPIDown (default) (#843)

Reviewed-on: k.renner/homelab#842

kubernetes/apps/immich/machine-learning.yaml

@@ -32,7 +32,7 @@ spec:
           type: RuntimeDefault
       containers:
         - name: immich-machine-learning
-          image: ghcr.io/immich-app/immich-machine-learning:v3.2.1 # {"$imagepolicy": "flux-system:immich-machine-learning"}
+          image: ghcr.io/immich-app/immich-machine-learning:v3.2.2 # {"$imagepolicy": "flux-system:immich-machine-learning"}
           ports:
             - name: http
               containerPort: 3003

kubernetes/apps/immich/server.yaml

@@ -87,7 +87,7 @@ spec:
               mountPath: /ca
       containers:
         - name: immich-server
-          image: ghcr.io/immich-app/immich-server:v3.2.1 # {"$imagepolicy": "flux-system:immich-server"}
+          image: ghcr.io/immich-app/immich-server:v3.2.2 # {"$imagepolicy": "flux-system:immich-server"}
           ports:
             - name: http
               containerPort: 3001

kubernetes/infrastructure/metrics-server/helmrelease.yaml

@@ -37,3 +37,16 @@ spec:
     args:
       - --kubelet-insecure-tls
       - --kubelet-preferred-address-types=InternalIP
+    # v1beta1.metrics.k8s.io is a cluster-scoped API: with a single replica,
+    # any single-node disruption takes the aggregated API to 0 % availability
+    # (KubeAggregatedAPIDown) and breaks every HPA. Run two replicas pinned to
+    # different nodes, matching upstream's high-availability manifest.
+    replicas: 2
+    affinity:
+      podAntiAffinity:
+        requiredDuringSchedulingIgnoredDuringExecution:
+          - topologyKey: kubernetes.io/hostname
+            labelSelector:
+              matchLabels:
+                app.kubernetes.io/name: metrics-server
+                app.kubernetes.io/instance: metrics-server

kubernetes/talos/patches/k8s-1.yaml

@@ -182,8 +182,14 @@ addresses:
 # network's clients selectively, so the client goes rather than the record.
 # DHCP is off on VLAN 47 for the same reason.
 #
-# The metric loses to mgmt0's default 1024 deliberately: this pass only adds
-# the segment. The default route moves here in the change that flips it.
+# Both defaults live in table 47, not in main, so only the pod-source rules in
+# site-homelab.yaml reach them. mgmt0 keeps main's default and with it every
+# reply the node sends from its own addresses.
+#
+# The v6 gateway is the UDM's configured ULA on this segment, the same address
+# BGP peers with. The RA-learned link-local would be the alternative and is not
+# a value to write down. accept_ra_defrtr stays 0 for that reason: the RA
+# default would land in main, where it does not belong.
 apiVersion: v1alpha1
 kind: VLANConfig
 name: ingress0
@@ -193,8 +199,10 @@ up: true
 addresses:
   - address: 10.2.47.11/24
 routes:
+  - gateway: fd02:47::1
+    table: "47"
   - gateway: 10.2.47.1
-    metric: 2048
+    table: "47"
 ---
 # Dedicated Ceph cluster network on X710 port 1, VLAN 49, untagged on the
 # wire. OSD-to-OSD replication and nothing else. The udev rule in the machine

kubernetes/talos/patches/k8s-2.yaml

@@ -129,8 +129,10 @@ up: true
 addresses:
   - address: 10.2.47.12/24
 routes:
+  - gateway: fd02:47::1
+    table: "47"
   - gateway: 10.2.47.1
-    metric: 2048
+    table: "47"
 ---
 # Dedicated Ceph cluster network on X710 port 1, VLAN 49, untagged on the
 # wire. OSD-to-OSD replication only, not routed off its own segment.

kubernetes/talos/patches/k8s-3.yaml

@@ -116,8 +116,10 @@ up: true
 addresses:
   - address: 10.2.47.13/24
 routes:
+  - gateway: fd02:47::1
+    table: "47"
   - gateway: 10.2.47.1
-    metric: 2048
+    table: "47"
 ---
 # Dedicated Ceph cluster network — see k8s-1.yaml. The node holds NO address
 # here on purpose; the link is handed whole to the OSD pod (host-device).

kubernetes/talos/patches/site-homelab.yaml

@@ -20,3 +20,47 @@ machine:
       validSubnets:
         - 10.2.40.0/24
         - fd40::/64
+---
+# Pod egress leaves over ingress0. Matching on the pod source prefix is what
+# keeps the node's own traffic out of it: the node IP, the API VIP and apid all
+# answer on mgmt0, and a reply that left over ingress0 would carry an mgmt0
+# source onto VLAN 47.
+#
+# The routing decision runs before Cilium's masquerade hook, so the rule sees
+# the pod address and the masquerade then rewrites it to ingress0's.
+#
+# Table 47 holds nothing but the two default routes, so the destination rules
+# below have to run first: without them a pod's packet to a service or to a pod
+# on another node also matches the source rule, finds only the default, and
+# leaves for the UDM. Measured — in-cluster DNS stopped resolving.
+#
+# They send everything inside the cluster and the local networks back to main.
+# The prefixes are Cilium's ipv4/ipv6NativeRoutingCIDR, which already mark what
+# is not foreign traffic.
+#
+# Priorities are below main (32766) and clear of the rule Cilium installs at 9.
+# The two families are separate rule tables, so the priorities need not match;
+# the document name is the priority and must be unique.
+apiVersion: v1alpha1
+kind: RoutingRuleConfig
+name: "990"
+dst: fd00::/8
+table: main
+---
+apiVersion: v1alpha1
+kind: RoutingRuleConfig
+name: "991"
+dst: 10.0.0.0/8
+table: main
+---
+apiVersion: v1alpha1
+kind: RoutingRuleConfig
+name: "1000"
+src: fd10:244::/48
+table: "47"
+---
+apiVersion: v1alpha1
+kind: RoutingRuleConfig
+name: "1001"
+src: 10.244.0.0/16
+table: "47"