From 2d6452650f56abdabd03508f80b3de5150c3d3c2 Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Sun, 19 Jul 2026 14:20:15 +0200 Subject: [PATCH] fix(pve): accept IPv6 RAs on forwarding nodes, converge vmbr0 MTU MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Live inspection of the PVE cluster on 2026-07-19 found two defects. 1. No IPv6 address on any external interface, on all three nodes. /etc/frr/frr.conf enables "ipv6 forwarding" for the OSPFv3 Thunderbolt mesh that carries corosync, which sets net.ipv6.conf.all.forwarding=1. The kernel default accept_ra=1 means "accept RAs only while NOT forwarding", so the nodes silently discarded the Router Advertisements the gateway sends on VLAN 40. Every external interface was left with only a link-local address; the sole non-link-local IPv6 anywhere on the nodes was the corosync mesh /128, which is static in frr.conf and never depended on RAs. Verified the gateway is not at fault: RAs arrive on vmbr1.40 from fe80::265a:4cff:fea6:fd3c every ~10-15s, carrying both the Telekom GUA prefix and a ULA, and non-forwarding hosts on the LAN autoconfigure normally. The PVE nodes are the only hosts running FRR. Fixed with a sysctl drop-in setting accept_ra=2, the value a forwarding host needs in order to still autoconfigure its own uplink. 2. saturn's vmbr0 is at MTU 1500 while its own uplink enp2s0f0np0 and both peer nodes' bridges are at 9000. Since the three bridges share one L2 domain, large frames between saturn's guests and the other nodes' guests blackhole rather than fragmenting. saturn's vmbr0 stanza is simply missing the mtu line that jupiter and neptune both have. The two fixes are separated by risk. The sysctl change is non-disruptive and applies on every run. The MTU change is staged as a pending PVE network change and applied only when pve_node_network_apply_pending is set, because applying it reloads networking on the node and briefly interrupts the Talos k8s VM and its Ceph OSD. The play uses serial: 1 so nodes converge one at a time. The MTU task reconstructs the full bridge definition from a pvesh GET before issuing the PUT. A PUT on /nodes//network/ replaces rather than merges, so passing only --mtu would silently drop bridge_ports, bridge_vlan_aware and bridge_vids, detaching the uplink and flattening every SDN VNet. Not applied yet — no run has been made against the cluster. Co-Authored-By: Claude Opus 4.8 (1M context) --- ansible/plays/configure-pve-node-network.yml | 53 +++++++ .../roles/pve-node-network/defaults/main.yml | 34 +++++ .../roles/pve-node-network/handlers/main.yml | 6 + ansible/roles/pve-node-network/tasks/main.yml | 141 ++++++++++++++++++ .../templates/70-pve-ipv6-ra.conf.j2 | 39 +++++ 5 files changed, 273 insertions(+) create mode 100644 ansible/plays/configure-pve-node-network.yml create mode 100644 ansible/roles/pve-node-network/defaults/main.yml create mode 100644 ansible/roles/pve-node-network/handlers/main.yml create mode 100644 ansible/roles/pve-node-network/tasks/main.yml create mode 100644 ansible/roles/pve-node-network/templates/70-pve-ipv6-ra.conf.j2 diff --git a/ansible/plays/configure-pve-node-network.yml b/ansible/plays/configure-pve-node-network.yml new file mode 100644 index 00000000..3e20de1e --- /dev/null +++ b/ansible/plays/configure-pve-node-network.yml @@ -0,0 +1,53 @@ +--- +# Configure Proxmox VE Node Host Networking +# +# Fixes two defects found by live inspection on 2026-07-19: +# +# 1. All three nodes had no IPv6 address on any external interface. FRR +# enables ipv6 forwarding for the OSPFv3 corosync mesh, and with the +# kernel default accept_ra=1 the nodes silently discarded the Router +# Advertisements the gateway sends on VLAN 40. Fixed with accept_ra=2. +# +# 2. saturn's vmbr0 sits at MTU 1500 while its own uplink and both peer +# nodes are at 9000, so large frames between saturn's guests and the +# other nodes' guests blackhole instead of fragmenting. +# +# The sysctl fix is non-disruptive and applies on every run. The MTU fix is +# staged as a pending PVE network change and only applied when explicitly +# requested, because applying it reloads networking on the node and briefly +# interrupts the Talos k8s VM and its Ceph OSD. +# +# Safe (converges IPv6, stages the MTU change without applying it): +# ansible-playbook -i inventory/ plays/configure-pve-node-network.yml +# +# Applying the staged MTU change, one node at a time: +# ansible-playbook -i inventory/ plays/configure-pve-node-network.yml \ +# --limit saturn -e pve_node_network_apply_pending=true +# +# See wiki: pve-ipv6-accept-ra-forwarding-defect, +# pve-saturn-vmbr0-mtu-asymmetry + +- name: Configure PVE node host networking + hosts: pve_cluster + become: yes + serial: 1 + + roles: + - role: pve-node-network + + post_tasks: + - name: Summarise node IPv6 state + ansible.builtin.command: ip -6 addr show dev vmbr1.40 scope global + register: pve_node_network_v6_summary + changed_when: false + failed_when: false + + - name: Display node IPv6 state + ansible.builtin.debug: + msg: >- + {{ inventory_hostname }}: + {{ pve_node_network_v6_summary.stdout_lines + | select('search', 'inet6') + | map('trim') + | list + | default(['no global IPv6'], true) }} diff --git a/ansible/roles/pve-node-network/defaults/main.yml b/ansible/roles/pve-node-network/defaults/main.yml new file mode 100644 index 00000000..32434dfa --- /dev/null +++ b/ansible/roles/pve-node-network/defaults/main.yml @@ -0,0 +1,34 @@ +--- +# PVE Node Network Role Defaults + +# --- IPv6 Router Advertisement acceptance ----------------------------------- +# The PVE nodes run FRR (OSPFv3 over the Thunderbolt mesh, carrying corosync), +# which enables net.ipv6.conf.all.forwarding. The kernel default accept_ra=1 +# means "accept RAs only while NOT forwarding", so the nodes discard the RAs +# the UniFi gateway sends. accept_ra=2 is the value a forwarding host needs in +# order to still autoconfigure its own uplink. +pve_node_network_accept_ra: 2 + +# Interfaces to set accept_ra on explicitly, in addition to all/default. +# Dots are translated to slashes for the sysctl key (vmbr1.40 -> vmbr1/40). +pve_node_network_ra_interfaces: + - vmbr1.40 + - vmbr0 + +# --- vmbr0 MTU -------------------------------------------------------------- +# jupiter and neptune already carry 9000; saturn's stanza is missing the mtu +# line and sits at 1500 while its own uplink is 9000. All three should match, +# since they share one L2 domain and mismatched MTUs blackhole large frames +# rather than fragmenting them. +pve_node_network_bridge: vmbr0 +pve_node_network_bridge_mtu: 9000 + +# Whether to manage the bridge MTU at all. +pve_node_network_manage_mtu: yes + +# Applying an MTU change requires a network reload (ifreload -a), which briefly +# interrupts guest connectivity on the node — including the Talos k8s VM and its +# Ceph OSD. Left false, the change is written to /etc/network/interfaces.new and +# shows as "pending" in the PVE UI, to be applied deliberately per node. +# Run with -e pve_node_network_apply_pending=true, ideally with --limit . +pve_node_network_apply_pending: no diff --git a/ansible/roles/pve-node-network/handlers/main.yml b/ansible/roles/pve-node-network/handlers/main.yml new file mode 100644 index 00000000..5e9623b2 --- /dev/null +++ b/ansible/roles/pve-node-network/handlers/main.yml @@ -0,0 +1,6 @@ +--- +# PVE Node Network Handlers + +- name: Reload sysctl + ansible.builtin.command: sysctl --system + changed_when: true diff --git a/ansible/roles/pve-node-network/tasks/main.yml b/ansible/roles/pve-node-network/tasks/main.yml new file mode 100644 index 00000000..80ca6e05 --- /dev/null +++ b/ansible/roles/pve-node-network/tasks/main.yml @@ -0,0 +1,141 @@ +--- +# PVE Node Network Role +# +# Two independent concerns, deliberately separated by risk: +# +# 1. IPv6 RA acceptance — a sysctl drop-in. Non-disruptive, reversible, +# applied unconditionally. +# 2. vmbr0 MTU convergence — requires a network reload that briefly drops +# guest connectivity. Staged as a pending change by default; applied only +# when pve_node_network_apply_pending is true. + +- name: Verify this is a Proxmox VE node + ansible.builtin.command: pveversion + register: pve_node_network_pveversion + changed_when: false + +# --- 1. IPv6 Router Advertisement acceptance -------------------------------- + +- name: Deploy IPv6 RA sysctl drop-in + ansible.builtin.template: + src: 70-pve-ipv6-ra.conf.j2 + dest: /etc/sysctl.d/70-pve-ipv6-ra.conf + owner: root + group: root + mode: '0644' + backup: yes + notify: Reload sysctl + +- name: Apply pending sysctl changes before verifying + ansible.builtin.meta: flush_handlers + +- name: Read back effective accept_ra on the management interface + ansible.builtin.command: sysctl -n net.ipv6.conf.vmbr1/40.accept_ra + register: pve_node_network_effective_accept_ra + changed_when: false + +- name: Fail if accept_ra did not take effect + ansible.builtin.assert: + that: + - pve_node_network_effective_accept_ra.stdout | int == pve_node_network_accept_ra | int + fail_msg: >- + accept_ra on vmbr1.40 is + {{ pve_node_network_effective_accept_ra.stdout }}, expected + {{ pve_node_network_accept_ra }}. The sysctl drop-in did not apply. + success_msg: "accept_ra on vmbr1.40 is {{ pve_node_network_accept_ra }}" + +# SLAAC needs a Router Advertisement to arrive before an address appears. The +# gateway advertises roughly every 10-15s, so a short wait is enough; this is +# reported rather than enforced, since a node may legitimately be on a segment +# without RAs. +- name: Wait for a global IPv6 address to be autoconfigured + ansible.builtin.shell: >- + ip -6 addr show dev vmbr1.40 scope global | grep -q 'inet6' + register: pve_node_network_slaac_check + retries: 6 + delay: 10 + until: pve_node_network_slaac_check.rc == 0 + failed_when: false + changed_when: false + +- name: Report IPv6 autoconfiguration result + ansible.builtin.debug: + msg: >- + {% if pve_node_network_slaac_check.rc == 0 %} + {{ inventory_hostname }}: global IPv6 address acquired on vmbr1.40 + {% else %} + {{ inventory_hostname }}: still no global IPv6 on vmbr1.40 after 60s. + Check that Router Advertisements reach this VLAN: + tcpdump -nni vmbr1.40 'icmp6 and ip6[40] == 134' + {% endif %} + +# --- 2. vmbr0 MTU ----------------------------------------------------------- + +- name: Query current bridge configuration + ansible.builtin.command: >- + pvesh get /nodes/{{ inventory_hostname }}/network/{{ pve_node_network_bridge }} + --output-format json + register: pve_node_network_bridge_raw + changed_when: false + when: pve_node_network_manage_mtu | bool + +- name: Parse current bridge configuration + ansible.builtin.set_fact: + pve_node_network_bridge_cfg: "{{ pve_node_network_bridge_raw.stdout | from_json }}" + when: pve_node_network_manage_mtu | bool + +# A PUT on /nodes//network/ REPLACES the interface definition +# rather than merging into it. Passing only --mtu would silently drop +# bridge_ports, bridge_vlan_aware and bridge_vids, detaching the uplink and +# flattening every VNet. Every existing attribute is therefore passed back. +- name: Set bridge MTU + ansible.builtin.command: >- + pvesh set /nodes/{{ inventory_hostname }}/network/{{ pve_node_network_bridge }} + --type bridge + --autostart {{ pve_node_network_bridge_cfg.autostart | default(1) }} + --bridge_ports "{{ pve_node_network_bridge_cfg.bridge_ports }}" + --bridge_stp {{ pve_node_network_bridge_cfg.bridge_stp | default('off') }} + --bridge_fd {{ pve_node_network_bridge_cfg.bridge_fd | default(0) }} + --bridge_vlan_aware {{ pve_node_network_bridge_cfg.bridge_vlan_aware | default(0) }} + --bridge_vids "{{ pve_node_network_bridge_cfg.bridge_vids | default('2-4094') }}" + --mtu {{ pve_node_network_bridge_mtu }} + when: + - pve_node_network_manage_mtu | bool + - pve_node_network_bridge_cfg.mtu | default(0) | int != pve_node_network_bridge_mtu | int + register: pve_node_network_mtu_staged + +- name: Report staged network change + ansible.builtin.debug: + msg: >- + {{ inventory_hostname }}: {{ pve_node_network_bridge }} MTU change to + {{ pve_node_network_bridge_mtu }} is staged in + /etc/network/interfaces.new and NOT yet live. Apply it deliberately, + one node at a time: + ansible-playbook -i inventory/ plays/configure-pve-node-network.yml + --limit {{ inventory_hostname }} -e pve_node_network_apply_pending=true + when: + - pve_node_network_mtu_staged is changed + - not (pve_node_network_apply_pending | bool) + +- name: Apply pending network configuration + ansible.builtin.command: pvesh set /nodes/{{ inventory_hostname }}/network + when: + - pve_node_network_apply_pending | bool + - pve_node_network_mtu_staged is changed + +- name: Verify bridge MTU is live + ansible.builtin.command: cat /sys/class/net/{{ pve_node_network_bridge }}/mtu + register: pve_node_network_live_mtu + changed_when: false + when: pve_node_network_apply_pending | bool + +- name: Fail if bridge MTU did not converge + ansible.builtin.assert: + that: + - pve_node_network_live_mtu.stdout | int == pve_node_network_bridge_mtu | int + fail_msg: >- + {{ pve_node_network_bridge }} MTU is {{ pve_node_network_live_mtu.stdout }}, + expected {{ pve_node_network_bridge_mtu }}. + success_msg: >- + {{ pve_node_network_bridge }} MTU is {{ pve_node_network_bridge_mtu }} + when: pve_node_network_apply_pending | bool diff --git a/ansible/roles/pve-node-network/templates/70-pve-ipv6-ra.conf.j2 b/ansible/roles/pve-node-network/templates/70-pve-ipv6-ra.conf.j2 new file mode 100644 index 00000000..070d5df4 --- /dev/null +++ b/ansible/roles/pve-node-network/templates/70-pve-ipv6-ra.conf.j2 @@ -0,0 +1,39 @@ +# +# Managed by Ansible (role: pve-node-network) — do not edit by hand. +# +# Why this file exists: +# +# /etc/frr/frr.conf enables "ipv6 forwarding" for the OSPFv3 Thunderbolt mesh +# that carries corosync. That sets net.ipv6.conf.all.forwarding=1. +# +# The kernel default is accept_ra=1, which means "accept Router Advertisements +# ONLY IF forwarding is disabled". So the moment FRR came up, these nodes began +# silently discarding the RAs the UniFi gateway sends on VLAN 40, leaving every +# external interface with nothing but a link-local fe80:: address. The only +# non-link-local IPv6 on the nodes was the corosync mesh /128, which is static +# in frr.conf and never depended on RAs. +# +# accept_ra=2 means "accept RAs even though I am forwarding" — the correct +# setting for a router that still needs to autoconfigure its own uplink. +# +# Verified 2026-07-19: the gateway does advertise on VLAN 40 (RAs from +# fe80::265a:4cff:fea6:fd3c roughly every 10-15s, carrying both the Telekom +# GUA prefix and a ULA). The nodes were receiving and discarding them. +# +# See wiki: pve-ipv6-accept-ra-forwarding-defect +# + +# 'default' covers interfaces created after boot; 'all' propagates to the +# interfaces that already exist when this is applied. +net.ipv6.conf.all.accept_ra = {{ pve_node_network_accept_ra }} +net.ipv6.conf.default.accept_ra = {{ pve_node_network_accept_ra }} + +# Autoconf is already on by default, but state it explicitly so the intent +# survives someone tightening the defaults later. +net.ipv6.conf.all.autoconf = 1 +net.ipv6.conf.default.autoconf = 1 +{% for iface in pve_node_network_ra_interfaces %} + +net.ipv6.conf.{{ iface | replace('.', '/') }}.accept_ra = {{ pve_node_network_accept_ra }} +net.ipv6.conf.{{ iface | replace('.', '/') }}.autoconf = 1 +{% endfor %} -- 2.54.0