From 413bfd5febd31a27d3c0f50694e1570b553d0c49 Mon Sep 17 00:00:00 2001 From: Konstantin Renner Date: Thu, 25 Jun 2026 19:33:04 +0200 Subject: [PATCH 1/2] feat(ingress): add NetBird mesh ingress (routing peer) alongside cloudflared MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a second, parallel ingress path over the NetBird overlay mesh (100.96.0.0/16), in addition to the existing Cloudflare path. A new in-cluster NetBird routing peer enrolls with the self-hosted management server (netbird.renner.uno) and advertises the Traefik VIP (10.2.99.86) into the mesh, so every Traefik-fronted service becomes reachable from mesh peers with no per-service work — mesh DNS (10.2.40.53) already resolves .renner.uno to that VIP; the only gap was that the bgp-pool CIDR wasn't routable over the overlay. Modeled on the cloudflared Deployment and held to the same in-cluster firewall discipline: a dedicated CiliumNetworkPolicy (netbird-ingress-egress) restricts egress to traefik + DNS + the NetBird control/data plane only. Unlike the upstream NetBird operator the pod is NOT privileged; the NET_ADMIN/SYS_ADMIN/SYS_RESOURCE caps are the irreducible minimum for an L3 WireGuard routing peer. - kubernetes/infrastructure/netbird-ingress/ (deployment, sops secret, kustomization) - image-automation policy pinned to bare-numeric semver (excludes -rootless/-rc) - netbird-ingress-egress CNP in network-policies/ingress-isolation.yaml Requires one-time NetBird-side setup (dashboard/API): create a reusable + ephemeral setup key (auto-group k8s-ingress-router), replace the placeholder in secret.sops.yaml, add a network route for 10.2.99.86/32 via that peer, and an access policy for the consuming mesh groups. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../image-automation/kustomization.yaml | 1 + .../image-automation/netbird-ingress.yaml | 26 ++++++ kubernetes/infrastructure/kustomization.yaml | 1 + .../netbird-ingress/deployment.yaml | 92 +++++++++++++++++++ .../netbird-ingress/kustomization.yaml | 5 + .../netbird-ingress/secret.sops.yaml | 26 ++++++ .../network-policies/ingress-isolation.yaml | 47 ++++++++++ 7 files changed, 198 insertions(+) create mode 100644 kubernetes/infrastructure/image-automation/netbird-ingress.yaml create mode 100644 kubernetes/infrastructure/netbird-ingress/deployment.yaml create mode 100644 kubernetes/infrastructure/netbird-ingress/kustomization.yaml create mode 100644 kubernetes/infrastructure/netbird-ingress/secret.sops.yaml diff --git a/kubernetes/infrastructure/image-automation/kustomization.yaml b/kubernetes/infrastructure/image-automation/kustomization.yaml index 5071e170..0c474b96 100644 --- a/kubernetes/infrastructure/image-automation/kustomization.yaml +++ b/kubernetes/infrastructure/image-automation/kustomization.yaml @@ -14,6 +14,7 @@ resources: - matter-server.yaml - mmwave-configurator.yaml - music-assistant.yaml + - netbird-ingress.yaml - paperless-ngx.yaml - protonmail-bridge.yaml - pve-exporter.yaml diff --git a/kubernetes/infrastructure/image-automation/netbird-ingress.yaml b/kubernetes/infrastructure/image-automation/netbird-ingress.yaml new file mode 100644 index 00000000..fd4afdb8 --- /dev/null +++ b/kubernetes/infrastructure/image-automation/netbird-ingress.yaml @@ -0,0 +1,26 @@ +apiVersion: image.toolkit.fluxcd.io/v1 +kind: ImageRepository +metadata: + name: netbird-ingress + namespace: flux-system +spec: + image: netbirdio/netbird + interval: 1h +--- +# Semver. filterTags restricts candidates to bare X.Y.Z release tags so +# pre-release (-rc.N) and the -rootless variant can't slip through; the +# -rootless image is userspace/netstack and can't act as a routing peer. +apiVersion: image.toolkit.fluxcd.io/v1 +kind: ImagePolicy +metadata: + name: netbird-ingress + namespace: flux-system +spec: + imageRepositoryRef: + name: netbird-ingress + filterTags: + pattern: '^(?P\d+\.\d+\.\d+)$' + extract: '$ver' + policy: + semver: + range: ">=0.73.0" diff --git a/kubernetes/infrastructure/kustomization.yaml b/kubernetes/infrastructure/kustomization.yaml index 5da22a72..697c5fc2 100644 --- a/kubernetes/infrastructure/kustomization.yaml +++ b/kubernetes/infrastructure/kustomization.yaml @@ -16,6 +16,7 @@ resources: - ceph-csi-drivers/ - cloudnative-pg/ - cloudflared/ + - netbird-ingress/ - coredns-config/ - k8s-gateway/ - external-dns-unifi/ diff --git a/kubernetes/infrastructure/netbird-ingress/deployment.yaml b/kubernetes/infrastructure/netbird-ingress/deployment.yaml new file mode 100644 index 00000000..2e11fe67 --- /dev/null +++ b/kubernetes/infrastructure/netbird-ingress/deployment.yaml @@ -0,0 +1,92 @@ +# NetBird mesh ingress — a second, parallel ingress path alongside +# cloudflared. This pod enrolls as a NetBird *routing peer* with the +# self-hosted management server (netbird.renner.uno) and advertises the +# Traefik VIP (10.2.99.86) into the mesh, so mesh peers can reach every +# Traefik-fronted service. Mesh DNS (10.2.40.53) already resolves +# .renner.uno → 10.2.99.86; the only missing piece was making that +# VIP routable over the overlay, which this peer provides. +# +# Flow: mesh peer (100.96.x.x) --wg--> this pod --(masq + fwd)--> 10.2.99.86 (Traefik) --> service +# +# Unlike the upstream NetBird k8s operator, this pod is NOT privileged. +# The NET_ADMIN/SYS_ADMIN/SYS_RESOURCE caps are the irreducible minimum +# for an L3 WireGuard routing peer (the netbird entrypoint mknods +# /dev/net/tun and programs the wg interface + masquerade itself). Its +# egress is locked down by the netbird-ingress-egress CiliumNetworkPolicy +# (network-policies/ingress-isolation.yaml), mirroring cloudflared. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: netbird-ingress + namespace: ingress + labels: + app: netbird-ingress +spec: + # Single routing peer = single advertised next-hop into the mesh. + # HA (multiple peers + distribution) is a later iteration. + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app: netbird-ingress + template: + metadata: + labels: + app: netbird-ingress + spec: + containers: + - name: netbird + image: netbirdio/netbird:0.73.2 # {"$imagepolicy": "flux-system:netbird-ingress"} + env: + - name: NB_SETUP_KEY + valueFrom: + secretKeyRef: + name: netbird-ingress-setup-key + key: setup-key + - name: NB_MANAGEMENT_URL + value: "https://netbird.renner.uno" + - name: NB_HOSTNAME + value: "netbird-ingress" + - name: NB_LOG_LEVEL + value: "info" + securityContext: + capabilities: + add: + - NET_ADMIN + - SYS_RESOURCE + - SYS_ADMIN + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + memory: 256Mi + livenessProbe: + exec: + command: ["netbird", "status", "--check", "live"] + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + readinessProbe: + exec: + command: ["netbird", "status", "--check", "ready"] + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 5 + failureThreshold: 3 + startupProbe: + exec: + command: ["netbird", "status", "--check", "startup"] + periodSeconds: 2 + timeoutSeconds: 10 + failureThreshold: 30 + volumeMounts: + - name: state + mountPath: /var/lib/netbird + volumes: + # Ephemeral state: the setup key is reusable, so a pod restart + # cleanly re-enrolls a fresh peer (old one is reaped as ephemeral). + - name: state + emptyDir: {} diff --git a/kubernetes/infrastructure/netbird-ingress/kustomization.yaml b/kubernetes/infrastructure/netbird-ingress/kustomization.yaml new file mode 100644 index 00000000..e65ce7aa --- /dev/null +++ b/kubernetes/infrastructure/netbird-ingress/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - secret.sops.yaml + - deployment.yaml diff --git a/kubernetes/infrastructure/netbird-ingress/secret.sops.yaml b/kubernetes/infrastructure/netbird-ingress/secret.sops.yaml new file mode 100644 index 00000000..cdcec3e0 --- /dev/null +++ b/kubernetes/infrastructure/netbird-ingress/secret.sops.yaml @@ -0,0 +1,26 @@ +apiVersion: ENC[AES256_GCM,data:n5g=,iv:AS2cmrgB/sOiONg/DoPRT6ZmPdLOPbWjaQqilYTJpUg=,tag:ulxs3Ez1IixOUIRDktDtew==,type:str] +kind: ENC[AES256_GCM,data:G2FirIAV,iv:xiJ8bkbXCRC8diE4UwYP3rnTBYG1BTszYOKcPTcMgCU=,tag:/vMwZg4Tu0NHwpzxX8RJ0g==,type:str] +metadata: + name: ENC[AES256_GCM,data:HTnb3axigMpnO7jTqK3XQ+V7TehSfDPH+Q==,iv:8TEl6Yvgpj54ZtY1aF1dJ6Hkb9M0bVjJR7fUhA0+RUk=,tag:u5QEgWAc4udwwwKMLB+F2g==,type:str] + namespace: ENC[AES256_GCM,data:BMF0U6V7GA==,iv:cImq31ZJLipyvkHOVcixDKyUakWKFTTQposOS6NDYqM=,tag:RGfZgiX9w4nEX2C9YXytPQ==,type:str] +type: ENC[AES256_GCM,data:EtCwuXbU,iv:jmjZOjGDXhK+GZ/kjd7cynRYfp+BxCMU6IuvS+2p5AU=,tag:Stc7GZZf6UjdR4svy2+M6Q==,type:str] +stringData: + #ENC[AES256_GCM,data:JuWSynK2Yry4tyzs79oXc+q8/unXFx1NlIpkib1HYd2J0T05EYBh7SOY6dAFe4CLfsPtwkpHE6Dogv9yPuVev7Qobncw,iv:crnpTQy8JKiq2HOJ2D7f1CWbr+Eu3SwqAXrpum3B3sg=,tag:skUxoGAE8zKrtiKfbNj8zQ==,type:comment] + #ENC[AES256_GCM,data:jc0NbiQ0u6Tx0TqmbqUgr+ZOO4pDt0Ob+P/5zXC6GRE3Mrq50TR3I6fybWvV40/6spamSL3cB3PZTRn9gcppkOwK6J/6cA0f,iv:Cva2Bj113/ksC0kgtAKPepjcjAxlw+hLQ5d7qZhqdVo=,tag:oyH5i0d68df4o4xcm95quw==,type:comment] + #ENC[AES256_GCM,data:2dz1u4eo1QjWS9Ofdo7/xubBRrv8kWOs84R/XZfhETzeEq6unFcl83QylYlkER7A49MY9Z47Dgg4+RdhMt7BbwVK5Pp6vi/knYkGRZ/W,iv:2GFlsEijXtv2ju2nZeQbZzm0pAnuJVJvEirScj6bXpk=,tag:rW7UTVq7Fu5OCKOsttJB8g==,type:comment] + setup-key: ENC[AES256_GCM,data:ODVU26ya8+SNQJt9m5DjXINnx6Y6bRc269NxyItqaLCJWWHrr2iUBOjmCviJvypd00fhkSnV9PF+HxqjAQ==,iv:v28LsZImEFFsOZa80dc8USWlsm1hafCr9oIn06e+Wg0=,tag:CnwyeR/KUSx9L5iNKa7yWw==,type:str] +sops: + age: + - recipient: age1yqjvqkwwwxdj7mr546zfs5ckzuwndp69ya2n2drd4vypwns8w4ys04qfy3 + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBvOWNLQXg3NUpiZFQrQmx5 + bUxkYlZ5U1ZuU0VGN1ZiSit1QjNtVTNxelRNCkRoYW1Kb0lDZlFXWmlOcjRETUw2 + SzV2OGlVZXNPUzlISkEwc3crcWdTNlEKLS0tIEVjeFFHakZWOUJBdVdQMGJXN1kr + c1gybVVEOVV5b1dwR2V6VEpqbUtPSlkKdrtsKN4ExpeRfT8tk+s0yUVlBmD20Tbw + zSGHaQvuFeAG4HJW6QBlFcJ9c2GGhlvYhAH2E31tHKzwPaqYLQYPww== + -----END AGE ENCRYPTED FILE----- + lastmodified: "2026-06-25T17:31:06Z" + mac: ENC[AES256_GCM,data:oA16m9dn7LOlqODlRSEJRiykvlVM+GiiRe4i2WN8+al9PM0Z5ibvV4I7skDYLu0xlDTW1yGlkLvH9M9xegl/90L3pwgtCCUgCw1PDozB58vnerqV2djDFPR0/rQg7+3/8L03dkp7glN3THMY6cJBcoEPHKGzB/+pPMj6k9eipQU=,iv:xFePiroxuo8A4RAefl6OjY4uqjVAth2YS5sSSZ2aHhE=,tag:MGI91+3Vu+O3uZ1kiPLSiA==,type:str] + unencrypted_suffix: _unencrypted + version: 3.10.2 diff --git a/kubernetes/infrastructure/network-policies/ingress-isolation.yaml b/kubernetes/infrastructure/network-policies/ingress-isolation.yaml index 56bfa555..1adc265f 100644 --- a/kubernetes/infrastructure/network-policies/ingress-isolation.yaml +++ b/kubernetes/infrastructure/network-policies/ingress-isolation.yaml @@ -42,6 +42,53 @@ spec: - port: "443" protocol: TCP --- +# NetBird mesh-ingress routing peer: same egress-lockdown discipline as +# cloudflared above. It may reach (a) traefik (by label selector — Cilium +# applies LB/DNAT before egress policy, so forwarded mesh→VIP traffic is +# enforced on traefik's identity, no static VIP needed); (b) DNS; (c) the +# NetBird control/data plane on netbird.renner.uno — mgmt/signal/relay +# over TCP/443, STUN/TURN over UDP/3478, and WireGuard over UDP/51820. +# Relay-over-443 is the guaranteed fallback when direct UDP P2P is blocked. +apiVersion: cilium.io/v2 +kind: CiliumNetworkPolicy +metadata: + name: netbird-ingress-egress + namespace: ingress +spec: + endpointSelector: + matchLabels: + app: netbird-ingress + egress: + - toEndpoints: + - matchLabels: + app.kubernetes.io/name: traefik + toPorts: + - ports: + - port: "8443" + protocol: TCP + - port: "8000" + protocol: TCP + # Allow DNS resolution + - toEndpoints: + - matchLabels: + "k8s:io.kubernetes.pod.namespace": kube-system + k8s-app: kube-dns + toPorts: + - ports: + - port: "53" + protocol: UDP + # NetBird control + data plane (netbird.renner.uno, external) + - toEntities: + - world + toPorts: + - ports: + - port: "443" + protocol: TCP + - port: "3478" + protocol: UDP + - port: "51820" + protocol: UDP +--- # Default deny ingress to ingress namespace from external sources only. # Any in-cluster pod may reach traefik (required for east-west OIDC # flows via k8s_gateway — audiobookshelf → Authentik etc.); we don't -- 2.54.0 From 87565c2aac99857d29a2ada40c3d6032fc6e1ebb Mon Sep 17 00:00:00 2001 From: Konstantin Renner Date: Thu, 25 Jun 2026 19:47:25 +0200 Subject: [PATCH 2/2] feat(ingress): set real NetBird setup key for mesh ingress peer Replaces the placeholder in secret.sops.yaml with the reusable+ephemeral setup key created on netbird.renner.uno (auto-group k8s-ingress-router). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../infrastructure/netbird-ingress/secret.sops.yaml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/kubernetes/infrastructure/netbird-ingress/secret.sops.yaml b/kubernetes/infrastructure/netbird-ingress/secret.sops.yaml index cdcec3e0..43137f5b 100644 --- a/kubernetes/infrastructure/netbird-ingress/secret.sops.yaml +++ b/kubernetes/infrastructure/netbird-ingress/secret.sops.yaml @@ -5,10 +5,7 @@ metadata: namespace: ENC[AES256_GCM,data:BMF0U6V7GA==,iv:cImq31ZJLipyvkHOVcixDKyUakWKFTTQposOS6NDYqM=,tag:RGfZgiX9w4nEX2C9YXytPQ==,type:str] type: ENC[AES256_GCM,data:EtCwuXbU,iv:jmjZOjGDXhK+GZ/kjd7cynRYfp+BxCMU6IuvS+2p5AU=,tag:Stc7GZZf6UjdR4svy2+M6Q==,type:str] stringData: - #ENC[AES256_GCM,data:JuWSynK2Yry4tyzs79oXc+q8/unXFx1NlIpkib1HYd2J0T05EYBh7SOY6dAFe4CLfsPtwkpHE6Dogv9yPuVev7Qobncw,iv:crnpTQy8JKiq2HOJ2D7f1CWbr+Eu3SwqAXrpum3B3sg=,tag:skUxoGAE8zKrtiKfbNj8zQ==,type:comment] - #ENC[AES256_GCM,data:jc0NbiQ0u6Tx0TqmbqUgr+ZOO4pDt0Ob+P/5zXC6GRE3Mrq50TR3I6fybWvV40/6spamSL3cB3PZTRn9gcppkOwK6J/6cA0f,iv:Cva2Bj113/ksC0kgtAKPepjcjAxlw+hLQ5d7qZhqdVo=,tag:oyH5i0d68df4o4xcm95quw==,type:comment] - #ENC[AES256_GCM,data:2dz1u4eo1QjWS9Ofdo7/xubBRrv8kWOs84R/XZfhETzeEq6unFcl83QylYlkER7A49MY9Z47Dgg4+RdhMt7BbwVK5Pp6vi/knYkGRZ/W,iv:2GFlsEijXtv2ju2nZeQbZzm0pAnuJVJvEirScj6bXpk=,tag:rW7UTVq7Fu5OCKOsttJB8g==,type:comment] - setup-key: ENC[AES256_GCM,data:ODVU26ya8+SNQJt9m5DjXINnx6Y6bRc269NxyItqaLCJWWHrr2iUBOjmCviJvypd00fhkSnV9PF+HxqjAQ==,iv:v28LsZImEFFsOZa80dc8USWlsm1hafCr9oIn06e+Wg0=,tag:CnwyeR/KUSx9L5iNKa7yWw==,type:str] + setup-key: ENC[AES256_GCM,data:+ivBRI8oBZaiKT8CYIIT4ECQk4tpX17BmCbncMJ3we3kmCq+,iv:7q51hlOukOAxUFEjWw3el0x6tTyJQdvelMXUxA0T5Ks=,tag:JoDZRAri8QO47Ouc5sJK0w==,type:str] sops: age: - recipient: age1yqjvqkwwwxdj7mr546zfs5ckzuwndp69ya2n2drd4vypwns8w4ys04qfy3 @@ -20,7 +17,7 @@ sops: c1gybVVEOVV5b1dwR2V6VEpqbUtPSlkKdrtsKN4ExpeRfT8tk+s0yUVlBmD20Tbw zSGHaQvuFeAG4HJW6QBlFcJ9c2GGhlvYhAH2E31tHKzwPaqYLQYPww== -----END AGE ENCRYPTED FILE----- - lastmodified: "2026-06-25T17:31:06Z" - mac: ENC[AES256_GCM,data:oA16m9dn7LOlqODlRSEJRiykvlVM+GiiRe4i2WN8+al9PM0Z5ibvV4I7skDYLu0xlDTW1yGlkLvH9M9xegl/90L3pwgtCCUgCw1PDozB58vnerqV2djDFPR0/rQg7+3/8L03dkp7glN3THMY6cJBcoEPHKGzB/+pPMj6k9eipQU=,iv:xFePiroxuo8A4RAefl6OjY4uqjVAth2YS5sSSZ2aHhE=,tag:MGI91+3Vu+O3uZ1kiPLSiA==,type:str] + lastmodified: "2026-06-25T17:46:34Z" + mac: ENC[AES256_GCM,data:gxxN4X4m3YTxL2nqMhG6eTrbOIhr2bMIhUdVZs1SeDldIgO+iXu8S8ComrEFHb7IeYh55OaCZjFz9CG9uR36JzAX7o2d7F7/HM7g3kzkoA7vV0PGX7BgGLWJsT5LxVuU7/300KEdaiqPXPubwAhHOEbVkVdyn8IXnrQRCHbkkzs=,iv:inS8+v2tbGYZAbD3zG3Bf5uVIckTTFMK8Tk0fPsVXXg=,tag:7JMmk6irsBCfM/za2CdkaQ==,type:str] unencrypted_suffix: _unencrypted version: 3.10.2 -- 2.54.0