fix(netbird-proxy): watchdog restart cooldown 600 s -> 60 s

Every loss of the routing peer a proxy has pinned wedges its route
manager; the flood watchdog restarts the container within 2 s and the
public edge is back after one failed probe round. Inside the 600 s
cooldown the restart is suppressed and the URL watchdog three one-minute
rounds are the only recovery: 3.5 min of 000 on every public domain,
measured 2026-09-07 10:28 on proxy-2 ("restart suppressed, last restart
567s ago"). The morning node roll hit the same suppression on both
proxies at 06:58 and survived only because the routes recovered by luck.

Rolling reboots lose one peer every few minutes by design, so with 600 s
the second roll is always the unlucky one. A minute keeps the loop
protection (one restart per probe interval at most).

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

ansible/roles/netbird-proxy-watchdog/defaults/main.yml

@@ -32,8 +32,13 @@ netbird_watchdog_fail_threshold: 3
 
 # Minimum seconds between watchdog-issued restarts, so a genuinely broken
 # backend can't put the proxy into a restart loop. Shared by both the URL-probe
-# watchdog and the flood watchdog below.
-netbird_watchdog_cooldown_seconds: 600
+# watchdog and the flood watchdog below. One minute: the route-manager wedge
+# recurs on every routing-peer loss, and rolling node reboots lose one peer
+# every few minutes; a suppressed restart there leaves the wedge standing
+# until the URL probe's three rounds fire (3.5 min of 000 on every domain,
+# measured 2026-09-07). A minute still caps a loop at one restart per
+# probe interval.
+netbird_watchdog_cooldown_seconds: 60
 
 # Log fingerprint of the reverse-proxy route-manager wedge, matched as a plain
 # substring in the proxy's docker logs by the flood watchdog. On this build the

kubernetes/bootstrap.sh

@@ -1379,8 +1379,8 @@ EOF
             vm_addr="${NODE_ADDRS[$idx]}"
             vols=$(talosctl -n "$vm_addr" -e "$vm_addr" get discoveredvolumes 2>&1) \
                 || fail "could not read volumes from $vm_node ($vm_addr): ${vols//$'\n'/ | }"
-            awk '$NF == "r-ceph-osd" {found=1} END {exit !found}' <<<"$vols" \
-                || fail "$vm_node has no r-ceph-osd partition — RawVolumeConfig did not take"
+            awk '$NF == "r-osd-data" {found=1} END {exit !found}' <<<"$vols" \
+                || fail "$vm_node has no r-osd-data partition — RawVolumeConfig did not take"
             kubectl get node "$vm_node" \
                 -o jsonpath='{.metadata.labels.storage\.renner\.uno/ceph-osd}' \
                 | grep -q true \

kubernetes/infrastructure/post-crds/crowdsec-lapi-serverstransport.yaml

@@ -0,0 +1,11 @@
+# traefik.io CRD consumer for the public LAPI endpoint (crowdsec/lapi-ingress.yaml
+# holds the Ingress). The CA it names is the chart's self-signed LAPI CA.
+apiVersion: traefik.io/v1alpha1
+kind: ServersTransport
+metadata:
+  name: crowdsec-lapi
+  namespace: crowdsec
+spec:
+  serverName: crowdsec-service.crowdsec.svc.cluster.local
+  rootCAs:
+    - secret: crowdsec-root-secret

kubernetes/infrastructure/post-crds/kustomization.yaml

@@ -31,4 +31,7 @@ resources:
   - traefik-crowdsec-middleware.yaml
   - traefik-rate-limit-middleware.yaml
   - traefik-security-headers-middleware.yaml
+  # Phase 1 of a two-Kustomization move: also still in crowdsec/lapi-ingress.yaml
+  # until this layer has applied it once, so prune never runs ahead of apply.
+  - crowdsec-lapi-serverstransport.yaml
   - ../network-policies/

kubernetes/infrastructure/rook-ceph-cluster/cephcluster.yaml

@@ -95,12 +95,18 @@ spec:
   # Dedicated Ceph cluster network on VLAN 49, attached to the OSDs through
   # Multus. The attachment is san-network-attachment.yaml.
   #
-  # Only `cluster` is selected. Do not add `public`, and do not reach for
-  # `addressRanges` — that is honoured only under provider: host, which puts
-  # every daemon including the mons on host networking. A mon's address then
-  # becomes the node's DHCP lease, and when a lease moves the mon crash-loops
-  # on "Cannot assign requested address" with no node free to fail over to,
-  # because all three already hold the mon host ports.
+  # Only `cluster` is selected. Do not add `public`: under provider: host
+  # that puts every daemon including the mons on host networking. A mon's
+  # address then becomes the node's DHCP lease, and when a lease moves the mon
+  # crash-loops on "Cannot assign requested address" with no node free to
+  # fail over to, because all three already hold the mon host ports.
+  #
+  # addressRanges.cluster is stated, not discovered. Without it Rook runs a
+  # canary pod on an OSD node to read the CIDR off the attachment on EVERY
+  # reconcile; the attachment is host-device, so the OSD on that node already
+  # holds the NIC and the canary can never attach — the reconcile blocks for
+  # 15 minutes, fails, and repeats. The value must equal the whereabouts range
+  # in san-network-attachment.yaml.
   #
   # With `public` unset, client traffic stays on the pod network and VLAN 49
   # carries Ceph-internal traffic only — which is also why Rook's Multus
@@ -117,6 +123,9 @@ spec:
     provider: multus
     selectors:
       cluster: rook-ceph/ceph-san
+    addressRanges:
+      cluster:
+        - 10.99.0.0/29
   # Nodes listed explicitly, not useAllNodes: under useAllNodes the operator
   # re-provisions an OSD on any node whose device still carries a BlueStore
   # signature, so purging an OSD before pulling its drive is impossible — it
@@ -144,10 +153,12 @@ spec:
     useAllDevices: false
     # Two forms while the production drives are migrated:
     #
-    #   by-partlabel/r-ceph-osd  the target. Talos names the drive in the
-    #                            machine config (RawVolumeConfig ceph-osd), so
+    #   by-partlabel/r-osd-data  the target. Talos names the drive in the
+    #                            machine config (RawVolumeConfig osd-data), so
     #                            this file needs to know nothing about models,
-    #                            serials or hostnames.
+    #                            serials or hostnames. The label must not
+    #                            contain "ceph": ceph-volume rejects any such
+    #                            partition as a legacy ceph-disk member.
     #   by-id/nvme-SAMSUNG…      the three MS-01 drives as they are today:
     #                            bluestore written straight onto the raw disk,
     #                            with no partition table and therefore no
@@ -159,7 +170,7 @@ spec:
     #
     # The second half goes when those three are repartitioned, one node at a
     # time. Until then it is what keeps the running OSDs claimed.
-    devicePathFilter: "^/dev/disk/by-(partlabel/r-ceph-osd|id/nvme-SAMSUNG_MZQLB960HAJR)"
+    devicePathFilter: "^/dev/disk/by-(partlabel/r-osd-data|id/nvme-SAMSUNG_MZQLB960HAJR)"
     config:
       osdsPerDevice: "1"
   # Highest built-in class: kubelet evicts these last and the kernel gives

kubernetes/talos/patches/hcloud-1.yaml

@@ -57,7 +57,7 @@ up: true
 ---
 apiVersion: v1alpha1
 kind: RawVolumeConfig
-name: ceph-osd
+name: osd-data
 provisioning:
   diskSelector:
     # Two disks: the system disk and the attached volume. Matching on transport

kubernetes/talos/patches/hcloud-2.yaml

@@ -42,7 +42,7 @@ up: true
 ---
 apiVersion: v1alpha1
 kind: RawVolumeConfig
-name: ceph-osd
+name: osd-data
 provisioning:
   diskSelector:
     match: '!system_disk'

kubernetes/talos/patches/hcloud-3.yaml

@@ -42,7 +42,7 @@ up: true
 ---
 apiVersion: v1alpha1
 kind: RawVolumeConfig
-name: ceph-osd
+name: osd-data
 provisioning:
   diskSelector:
     match: '!system_disk'

kubernetes/talos/patches/k8s-1.yaml

@@ -14,6 +14,13 @@ machine:
     # Disabling v6 on a VLAN parent does not affect the VLAN child.
     net.ipv6.conf.enp2s0f0np0.disable_ipv6: "1"
     net.ipv6.conf.enp87s0.disable_ipv6: "1"
+    # That network also carries the mgmt VLAN's ARP. Answer ARP only on the
+    # link that owns the address, or the UDM learns the node IP against
+    # enp87s0's MAC and delivers IPv4 to a NIC Cilium is not attached to (#793).
+    net.ipv4.conf.all.arp_ignore: "1"
+    net.ipv4.conf.default.arp_ignore: "1"
+    net.ipv4.conf.all.arp_announce: "2"
+    net.ipv4.conf.default.arp_announce: "2"
   install:
     # Select disks by serial only. A broad selector such as
     # `disk.transport == 'nvme'` matches every drive in the machine, and the

kubernetes/talos/patches/k8s-2.yaml

@@ -12,6 +12,11 @@ machine:
     # cluster IPv6 egress.
     net.ipv6.conf.enp2s0f0np0.disable_ipv6: "1"
     net.ipv6.conf.enp87s0.disable_ipv6: "1"
+    # Answer ARP only on the link that owns the address — see k8s-1.yaml (#793).
+    net.ipv4.conf.all.arp_ignore: "1"
+    net.ipv4.conf.default.arp_ignore: "1"
+    net.ipv4.conf.all.arp_announce: "2"
+    net.ipv4.conf.default.arp_announce: "2"
   install:
     # Select the installer target by serial so it cannot land on the OSD drive
     # or on EPHEMERAL, whatever order the kernel enumerates them in.

kubernetes/talos/patches/k8s-3.yaml

@@ -8,6 +8,11 @@ machine:
     # k8s-1.yaml.
     net.ipv6.conf.enp2s0f0np0.disable_ipv6: "1"
     net.ipv6.conf.enp87s0.disable_ipv6: "1"
+    # Answer ARP only on the link that owns the address — see k8s-1.yaml (#793).
+    net.ipv4.conf.all.arp_ignore: "1"
+    net.ipv4.conf.default.arp_ignore: "1"
+    net.ipv4.conf.all.arp_announce: "2"
+    net.ipv4.conf.default.arp_announce: "2"
   install:
     # Select the installer target by serial so it cannot land on the OSD drive
     # or on EPHEMERAL, whatever order the kernel enumerates them in.

kubernetes/talos/patches/vm-2.yaml

@@ -21,7 +21,7 @@ auto: off
 hostname: vm-2
 ---
 # The OSD device, named by the machine that has it. Ceph then consumes
-# /dev/disk/by-partlabel/r-ceph-osd and needs to know nothing about models,
+# /dev/disk/by-partlabel/r-osd-data and needs to know nothing about models,
 # serials or hostnames — the same CephCluster works here and on the MS-01s.
 #
 # The selector is a CEL expression, so each machine says whatever identifies its
@@ -29,7 +29,7 @@ hostname: vm-2
 # real nodes it will name the Samsung.
 apiVersion: v1alpha1
 kind: RawVolumeConfig
-name: ceph-osd
+name: osd-data
 provisioning:
   diskSelector:
     match: disk.transport == "virtio" && !system_disk

kubernetes/talos/patches/vm-3.yaml

@@ -15,7 +15,7 @@ hostname: vm-3
 ---
 apiVersion: v1alpha1
 kind: RawVolumeConfig
-name: ceph-osd
+name: osd-data
 provisioning:
   diskSelector:
     match: disk.transport == "virtio" && !system_disk

kubernetes/talos/patches/vm-4.yaml

@@ -19,7 +19,7 @@ hostname: vm-4
 ---
 apiVersion: v1alpha1
 kind: RawVolumeConfig
-name: ceph-osd
+name: osd-data
 provisioning:
   diskSelector:
     match: disk.transport == "virtio" && !system_disk