infrastructure: add lb-ip-binder DaemonSet
Binds every LoadBalancer Service's external IP as a /32 (IPv4) or /128 (IPv6) on each node's dummy0 interface. Cilium's eBPF kpr intercepts TCP/UDP for these IPs at the TC ingress hook before the kernel's local-input decision, so local binding doesn't affect data flow — it only makes the kernel respond to ICMP echo locally. Why: Cilium L2 announcements and BGP both publish "I am this IP" to the network without binding the IP locally on any host. ICMP echo to LB IPs loops via the default route until TTL exhausts. kube-proxy IPVS used to bind LB IPs on kube-ipvs0; we run full kpr instead. Verified the binding+kpr composition works: bound 10.2.99.81 on k8s-edge's dummy0 manually, ping returned 0% loss, TCP to 1883/8883 still hit the mosquitto backend, Shelly MQTT session unaffected. Upstream Cilium PRs #37623/#40565 proposed an in-Cilium fix but neither merged.
This commit is contained in:
@@ -22,6 +22,7 @@ resources:
|
||||
- image-automation/
|
||||
- flux-notifications/
|
||||
- spegel/
|
||||
- lb-ip-binder/
|
||||
# cnpg-barman-plugin/ lives in its own Flux Kustomization
|
||||
# (`infrastructure-cnpg-barman`) so it can dependsOn infrastructure-crds
|
||||
# where the cert-manager Certificates create the TLS Secrets its
|
||||
|
||||
@@ -0,0 +1,160 @@
|
||||
# lb-ip-binder
|
||||
#
|
||||
# Binds every LoadBalancer Service's external IP as a /32 (IPv4) or /128
|
||||
# (IPv6) on each node's `dummy0` interface. Cilium's kube-proxy-
|
||||
# replacement intercepts TCP/UDP for these IPs at the TC ingress hook
|
||||
# (before the kernel's local-input decision), so this binding does not
|
||||
# affect data-plane traffic — it only makes the kernel respond to ICMP
|
||||
# echo for these IPs locally on every node.
|
||||
#
|
||||
# Why this exists: Cilium L2 announcements and BGP both publish "I am
|
||||
# this IP" to the network without binding the IP locally on any host.
|
||||
# That makes ICMP echo to LB IPs loop until TTL=0 (because the kernel
|
||||
# routes the packet back via the default gateway). kube-proxy in IPVS
|
||||
# mode used to add LB IPs to kube-ipvs0 for the same reason. We don't
|
||||
# run kube-proxy at all (`kube-proxy-replacement: true`), so we recreate
|
||||
# that behaviour ourselves.
|
||||
#
|
||||
# See https://github.com/cilium/cilium/issues/14118 for the upstream
|
||||
# discussion; PRs #37623 and #40565 proposed an in-Cilium fix but
|
||||
# neither merged.
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: lb-ip-binder
|
||||
namespace: kube-system
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRole
|
||||
metadata:
|
||||
name: lb-ip-binder
|
||||
rules:
|
||||
- apiGroups: [""]
|
||||
resources: ["services"]
|
||||
verbs: ["list", "watch"]
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRoleBinding
|
||||
metadata:
|
||||
name: lb-ip-binder
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: ClusterRole
|
||||
name: lb-ip-binder
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: lb-ip-binder
|
||||
namespace: kube-system
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: lb-ip-binder
|
||||
namespace: kube-system
|
||||
data:
|
||||
reconcile.sh: |
|
||||
#!/bin/sh
|
||||
set -u
|
||||
TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)
|
||||
CA=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt
|
||||
API=https://kubernetes.default.svc
|
||||
|
||||
# Ensure dummy0 exists and is up. Cilium creates it on startup but
|
||||
# we don't depend on that.
|
||||
ip link show dummy0 >/dev/null 2>&1 || ip link add dummy0 type dummy
|
||||
ip link set dummy0 up
|
||||
|
||||
while true; do
|
||||
# Desired: every assigned LoadBalancer ingress IP across the cluster
|
||||
# (both v4 and v6).
|
||||
desired=$(curl -sS --cacert "$CA" -H "Authorization: Bearer $TOKEN" \
|
||||
"$API/api/v1/services" 2>/dev/null \
|
||||
| jq -r '.items[]
|
||||
| select(.spec.type=="LoadBalancer")
|
||||
| .status.loadBalancer.ingress[]?
|
||||
| (.ip // .hostname // empty)' \
|
||||
| grep -E '^[0-9a-fA-F:.]+$' \
|
||||
| sort -u)
|
||||
|
||||
# Current: /32 and /128 addresses already on dummy0.
|
||||
current=$(ip -o addr show dev dummy0 \
|
||||
| awk '$3=="inet" && $4 ~ /\/32$/ { sub("/32", "", $4); print $4 }
|
||||
$3=="inet6" && $4 ~ /\/128$/ { sub("/128", "", $4); print $4 }' \
|
||||
| sort -u)
|
||||
|
||||
# Add missing.
|
||||
for ip in $desired; do
|
||||
if ! printf '%s\n' "$current" | grep -qx "$ip"; then
|
||||
case "$ip" in
|
||||
*:*) ip -6 addr add "$ip/128" dev dummy0 2>&1 ;;
|
||||
*) ip addr add "$ip/32" dev dummy0 2>&1 ;;
|
||||
esac
|
||||
fi
|
||||
done
|
||||
|
||||
# Remove extras (IPs we added previously that are no longer assigned).
|
||||
for ip in $current; do
|
||||
if ! printf '%s\n' "$desired" | grep -qx "$ip"; then
|
||||
case "$ip" in
|
||||
*:*) ip -6 addr del "$ip/128" dev dummy0 2>&1 ;;
|
||||
*) ip addr del "$ip/32" dev dummy0 2>&1 ;;
|
||||
esac
|
||||
fi
|
||||
done
|
||||
|
||||
sleep 30
|
||||
done
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: DaemonSet
|
||||
metadata:
|
||||
name: lb-ip-binder
|
||||
namespace: kube-system
|
||||
labels:
|
||||
app.kubernetes.io/name: lb-ip-binder
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: lb-ip-binder
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: lb-ip-binder
|
||||
spec:
|
||||
serviceAccountName: lb-ip-binder
|
||||
hostNetwork: true
|
||||
# Run on every node, including future control-plane nodes (this
|
||||
# cluster has none today but tolerations cost nothing).
|
||||
tolerations:
|
||||
- operator: Exists
|
||||
containers:
|
||||
- name: binder
|
||||
image: alpine:3.20
|
||||
command: ["/bin/sh", "-c"]
|
||||
# apk-add at startup so the manifest stays self-contained
|
||||
# without a custom image. ~10MB of additional packages.
|
||||
args:
|
||||
- |
|
||||
apk add --no-cache iproute2 curl jq >/dev/null
|
||||
exec /bin/sh /scripts/reconcile.sh
|
||||
securityContext:
|
||||
capabilities:
|
||||
add: ["NET_ADMIN"]
|
||||
allowPrivilegeEscalation: true
|
||||
readOnlyRootFilesystem: false
|
||||
resources:
|
||||
requests:
|
||||
cpu: 5m
|
||||
memory: 16Mi
|
||||
limits:
|
||||
memory: 64Mi
|
||||
volumeMounts:
|
||||
- name: scripts
|
||||
mountPath: /scripts
|
||||
readOnly: true
|
||||
volumes:
|
||||
- name: scripts
|
||||
configMap:
|
||||
name: lb-ip-binder
|
||||
defaultMode: 0755
|
||||
@@ -0,0 +1,4 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
resources:
|
||||
- daemonset.yaml
|
||||
Reference in New Issue
Block a user