From 4fc52db00b4d34eb50ebc399aed657467274e734 Mon Sep 17 00:00:00 2001 From: Konstantin Renner Date: Wed, 20 May 2026 11:18:44 +0200 Subject: [PATCH] =?UTF-8?q?network-policies:=20drop=20CCNP-based=20ICMP=20?= =?UTF-8?q?allow=20=E2=80=94=20was=20breaking=20ingress?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The CCNP `allow-icmp-default-deny-namespaces` (added in 66dbe9f, then shrunk in 7da87bf to ingress-only) was breaking *all* traefik->backend routing — even after the egress section was removed. Symptom: traefik logs filled with 504 / 499 / 30s upstream timeouts to every backend in the matched namespaces (authentik, home-assistant, audiobookshelf, music-assistant, monitoring/grafana, ...). Removing the CCNP from the cluster restored ingress immediately. I don't have a full root-cause yet for why an ingress-only CCNP with matchExpressions In on a list of namespaces causes Cilium to drop established traefik->pod TCP flows, but empirically: this shape of policy is incompatible with the existing per-namespace CNPs in this cluster. Falling back to per-namespace ICMP allows is the next attempt (in a separate commit, after the cluster has stabilised). Co-Authored-By: Claude Opus 4.7 (1M context) --- .../network-policies/allow-icmp.yaml | 133 ------------------ .../network-policies/kustomization.yaml | 1 - 2 files changed, 134 deletions(-) delete mode 100644 kubernetes/infrastructure/network-policies/allow-icmp.yaml diff --git a/kubernetes/infrastructure/network-policies/allow-icmp.yaml b/kubernetes/infrastructure/network-policies/allow-icmp.yaml deleted file mode 100644 index 2d39757d..00000000 --- a/kubernetes/infrastructure/network-policies/allow-icmp.yaml +++ /dev/null @@ -1,133 +0,0 @@ -# All ICMP types ingress into every namespace that has a default-deny. -# -# Scoped via `matchExpressions In` to the explicit list of namespaces -# that already opted into policy enforcement, so this CCNP only widens -# what's already enforced — it cannot transition kube-system or any -# other unmanaged namespace into policy-enforced mode (which is what -# broke metrics-server when I previously used `Exists`). -# -# INGRESS ONLY by design: a few of these namespaces (notably velero, -# monitoring, parts of rook-ceph) had per-namespace CNPs with an -# ingress section but NO egress section, so their pods were -# ingress-policy-enforced + egress-default-allow. Adding an egress -# section here transitions them into egress-policy-enforced mode too, -# and the CCNP would then deny everything except ICMP egress — which -# broke DNS, kube-apiserver, and world egress cluster-wide. -# -# The user-impacting drops were all ICMP errors landing on pods that -# had ingress-enforced policies; their reply ICMP egress already flows -# through the per-namespace egress rules (toEntities host/remote-node/ -# world/etc), or via default-allow where no egress section exists. -# -# Enumerates every assigned ICMPv4 type (0-30) and ICMPv6 type -# (1-4, 128-153). Cilium's policy schema requires an explicit `type` -# per ICMPField — there is no wildcard. -apiVersion: cilium.io/v2 -kind: CiliumClusterwideNetworkPolicy -metadata: - name: allow-icmp-default-deny-namespaces -spec: - endpointSelector: - matchExpressions: - - key: io.kubernetes.pod.namespace - operator: In - values: - - apps - - audiobookshelf - - authentik - - crowdsec - - database - - home-assistant - - immich - - influxdb - - ingress - - monitoring - - paperless - - rook-ceph - - spegel - - velero - ingress: - - fromEndpoints: - - matchExpressions: - - key: io.kubernetes.pod.namespace - operator: Exists - icmps: - - fields: - - type: 0 # Echo Reply - family: IPv4 - - type: 3 # Destination Unreachable - family: IPv4 - - type: 4 # Source Quench - family: IPv4 - - type: 5 # Redirect - family: IPv4 - - type: 8 # Echo Request - family: IPv4 - - type: 9 # Router Advertisement - family: IPv4 - - type: 10 # Router Solicitation - family: IPv4 - - type: 11 # Time Exceeded - family: IPv4 - - type: 12 # Parameter Problem - family: IPv4 - - type: 13 # Timestamp - family: IPv4 - - type: 14 # Timestamp Reply - family: IPv4 - - type: 15 # Information Request - family: IPv4 - - type: 16 # Information Reply - family: IPv4 - - type: 17 # Address Mask Request - family: IPv4 - - type: 18 # Address Mask Reply - family: IPv4 - - type: 30 # Traceroute - family: IPv4 - - type: 1 # Destination Unreachable - family: IPv6 - - type: 2 # Packet Too Big - family: IPv6 - - type: 3 # Time Exceeded - family: IPv6 - - type: 4 # Parameter Problem - family: IPv6 - - type: 128 # Echo Request - family: IPv6 - - type: 129 # Echo Reply - family: IPv6 - - type: 130 # Multicast Listener Query - family: IPv6 - - type: 131 # Multicast Listener Report - family: IPv6 - - type: 132 # Multicast Listener Done - family: IPv6 - - type: 133 # Router Solicitation - family: IPv6 - - type: 134 # Router Advertisement - family: IPv6 - - type: 135 # Neighbor Solicitation - family: IPv6 - - type: 136 # Neighbor Advertisement - family: IPv6 - - type: 137 # Redirect - family: IPv6 - - type: 138 # Router Renumbering - family: IPv6 - - type: 141 # Inverse Neighbor Discovery Solicitation - family: IPv6 - - type: 142 # Inverse Neighbor Discovery Advertisement - family: IPv6 - - type: 143 # MLDv2 Listener Report - family: IPv6 - - type: 148 # Certification Path Solicitation - family: IPv6 - - type: 149 # Certification Path Advertisement - family: IPv6 - - type: 151 # Multicast Router Advertisement - family: IPv6 - - type: 152 # Multicast Router Solicitation - family: IPv6 - - type: 153 # Multicast Router Termination - family: IPv6 diff --git a/kubernetes/infrastructure/network-policies/kustomization.yaml b/kubernetes/infrastructure/network-policies/kustomization.yaml index 4f37e8a3..2ec6185c 100644 --- a/kubernetes/infrastructure/network-policies/kustomization.yaml +++ b/kubernetes/infrastructure/network-policies/kustomization.yaml @@ -18,4 +18,3 @@ resources: - rook-ceph-ns.yaml - velero-ns.yaml - allow-cilium-health.yaml - - allow-icmp.yaml