diff --git a/ansible/plays/configure-pve-node-network.yml b/ansible/plays/configure-pve-node-network.yml new file mode 100644 index 00000000..3e20de1e --- /dev/null +++ b/ansible/plays/configure-pve-node-network.yml @@ -0,0 +1,53 @@ +--- +# Configure Proxmox VE Node Host Networking +# +# Fixes two defects found by live inspection on 2026-07-19: +# +# 1. All three nodes had no IPv6 address on any external interface. FRR +# enables ipv6 forwarding for the OSPFv3 corosync mesh, and with the +# kernel default accept_ra=1 the nodes silently discarded the Router +# Advertisements the gateway sends on VLAN 40. Fixed with accept_ra=2. +# +# 2. saturn's vmbr0 sits at MTU 1500 while its own uplink and both peer +# nodes are at 9000, so large frames between saturn's guests and the +# other nodes' guests blackhole instead of fragmenting. +# +# The sysctl fix is non-disruptive and applies on every run. The MTU fix is +# staged as a pending PVE network change and only applied when explicitly +# requested, because applying it reloads networking on the node and briefly +# interrupts the Talos k8s VM and its Ceph OSD. +# +# Safe (converges IPv6, stages the MTU change without applying it): +# ansible-playbook -i inventory/ plays/configure-pve-node-network.yml +# +# Applying the staged MTU change, one node at a time: +# ansible-playbook -i inventory/ plays/configure-pve-node-network.yml \ +# --limit saturn -e pve_node_network_apply_pending=true +# +# See wiki: pve-ipv6-accept-ra-forwarding-defect, +# pve-saturn-vmbr0-mtu-asymmetry + +- name: Configure PVE node host networking + hosts: pve_cluster + become: yes + serial: 1 + + roles: + - role: pve-node-network + + post_tasks: + - name: Summarise node IPv6 state + ansible.builtin.command: ip -6 addr show dev vmbr1.40 scope global + register: pve_node_network_v6_summary + changed_when: false + failed_when: false + + - name: Display node IPv6 state + ansible.builtin.debug: + msg: >- + {{ inventory_hostname }}: + {{ pve_node_network_v6_summary.stdout_lines + | select('search', 'inet6') + | map('trim') + | list + | default(['no global IPv6'], true) }} diff --git a/ansible/roles/pve-node-network/defaults/main.yml b/ansible/roles/pve-node-network/defaults/main.yml new file mode 100644 index 00000000..32434dfa --- /dev/null +++ b/ansible/roles/pve-node-network/defaults/main.yml @@ -0,0 +1,34 @@ +--- +# PVE Node Network Role Defaults + +# --- IPv6 Router Advertisement acceptance ----------------------------------- +# The PVE nodes run FRR (OSPFv3 over the Thunderbolt mesh, carrying corosync), +# which enables net.ipv6.conf.all.forwarding. The kernel default accept_ra=1 +# means "accept RAs only while NOT forwarding", so the nodes discard the RAs +# the UniFi gateway sends. accept_ra=2 is the value a forwarding host needs in +# order to still autoconfigure its own uplink. +pve_node_network_accept_ra: 2 + +# Interfaces to set accept_ra on explicitly, in addition to all/default. +# Dots are translated to slashes for the sysctl key (vmbr1.40 -> vmbr1/40). +pve_node_network_ra_interfaces: + - vmbr1.40 + - vmbr0 + +# --- vmbr0 MTU -------------------------------------------------------------- +# jupiter and neptune already carry 9000; saturn's stanza is missing the mtu +# line and sits at 1500 while its own uplink is 9000. All three should match, +# since they share one L2 domain and mismatched MTUs blackhole large frames +# rather than fragmenting them. +pve_node_network_bridge: vmbr0 +pve_node_network_bridge_mtu: 9000 + +# Whether to manage the bridge MTU at all. +pve_node_network_manage_mtu: yes + +# Applying an MTU change requires a network reload (ifreload -a), which briefly +# interrupts guest connectivity on the node — including the Talos k8s VM and its +# Ceph OSD. Left false, the change is written to /etc/network/interfaces.new and +# shows as "pending" in the PVE UI, to be applied deliberately per node. +# Run with -e pve_node_network_apply_pending=true, ideally with --limit . +pve_node_network_apply_pending: no diff --git a/ansible/roles/pve-node-network/handlers/main.yml b/ansible/roles/pve-node-network/handlers/main.yml new file mode 100644 index 00000000..5e9623b2 --- /dev/null +++ b/ansible/roles/pve-node-network/handlers/main.yml @@ -0,0 +1,6 @@ +--- +# PVE Node Network Handlers + +- name: Reload sysctl + ansible.builtin.command: sysctl --system + changed_when: true diff --git a/ansible/roles/pve-node-network/tasks/main.yml b/ansible/roles/pve-node-network/tasks/main.yml new file mode 100644 index 00000000..80ca6e05 --- /dev/null +++ b/ansible/roles/pve-node-network/tasks/main.yml @@ -0,0 +1,141 @@ +--- +# PVE Node Network Role +# +# Two independent concerns, deliberately separated by risk: +# +# 1. IPv6 RA acceptance — a sysctl drop-in. Non-disruptive, reversible, +# applied unconditionally. +# 2. vmbr0 MTU convergence — requires a network reload that briefly drops +# guest connectivity. Staged as a pending change by default; applied only +# when pve_node_network_apply_pending is true. + +- name: Verify this is a Proxmox VE node + ansible.builtin.command: pveversion + register: pve_node_network_pveversion + changed_when: false + +# --- 1. IPv6 Router Advertisement acceptance -------------------------------- + +- name: Deploy IPv6 RA sysctl drop-in + ansible.builtin.template: + src: 70-pve-ipv6-ra.conf.j2 + dest: /etc/sysctl.d/70-pve-ipv6-ra.conf + owner: root + group: root + mode: '0644' + backup: yes + notify: Reload sysctl + +- name: Apply pending sysctl changes before verifying + ansible.builtin.meta: flush_handlers + +- name: Read back effective accept_ra on the management interface + ansible.builtin.command: sysctl -n net.ipv6.conf.vmbr1/40.accept_ra + register: pve_node_network_effective_accept_ra + changed_when: false + +- name: Fail if accept_ra did not take effect + ansible.builtin.assert: + that: + - pve_node_network_effective_accept_ra.stdout | int == pve_node_network_accept_ra | int + fail_msg: >- + accept_ra on vmbr1.40 is + {{ pve_node_network_effective_accept_ra.stdout }}, expected + {{ pve_node_network_accept_ra }}. The sysctl drop-in did not apply. + success_msg: "accept_ra on vmbr1.40 is {{ pve_node_network_accept_ra }}" + +# SLAAC needs a Router Advertisement to arrive before an address appears. The +# gateway advertises roughly every 10-15s, so a short wait is enough; this is +# reported rather than enforced, since a node may legitimately be on a segment +# without RAs. +- name: Wait for a global IPv6 address to be autoconfigured + ansible.builtin.shell: >- + ip -6 addr show dev vmbr1.40 scope global | grep -q 'inet6' + register: pve_node_network_slaac_check + retries: 6 + delay: 10 + until: pve_node_network_slaac_check.rc == 0 + failed_when: false + changed_when: false + +- name: Report IPv6 autoconfiguration result + ansible.builtin.debug: + msg: >- + {% if pve_node_network_slaac_check.rc == 0 %} + {{ inventory_hostname }}: global IPv6 address acquired on vmbr1.40 + {% else %} + {{ inventory_hostname }}: still no global IPv6 on vmbr1.40 after 60s. + Check that Router Advertisements reach this VLAN: + tcpdump -nni vmbr1.40 'icmp6 and ip6[40] == 134' + {% endif %} + +# --- 2. vmbr0 MTU ----------------------------------------------------------- + +- name: Query current bridge configuration + ansible.builtin.command: >- + pvesh get /nodes/{{ inventory_hostname }}/network/{{ pve_node_network_bridge }} + --output-format json + register: pve_node_network_bridge_raw + changed_when: false + when: pve_node_network_manage_mtu | bool + +- name: Parse current bridge configuration + ansible.builtin.set_fact: + pve_node_network_bridge_cfg: "{{ pve_node_network_bridge_raw.stdout | from_json }}" + when: pve_node_network_manage_mtu | bool + +# A PUT on /nodes//network/ REPLACES the interface definition +# rather than merging into it. Passing only --mtu would silently drop +# bridge_ports, bridge_vlan_aware and bridge_vids, detaching the uplink and +# flattening every VNet. Every existing attribute is therefore passed back. +- name: Set bridge MTU + ansible.builtin.command: >- + pvesh set /nodes/{{ inventory_hostname }}/network/{{ pve_node_network_bridge }} + --type bridge + --autostart {{ pve_node_network_bridge_cfg.autostart | default(1) }} + --bridge_ports "{{ pve_node_network_bridge_cfg.bridge_ports }}" + --bridge_stp {{ pve_node_network_bridge_cfg.bridge_stp | default('off') }} + --bridge_fd {{ pve_node_network_bridge_cfg.bridge_fd | default(0) }} + --bridge_vlan_aware {{ pve_node_network_bridge_cfg.bridge_vlan_aware | default(0) }} + --bridge_vids "{{ pve_node_network_bridge_cfg.bridge_vids | default('2-4094') }}" + --mtu {{ pve_node_network_bridge_mtu }} + when: + - pve_node_network_manage_mtu | bool + - pve_node_network_bridge_cfg.mtu | default(0) | int != pve_node_network_bridge_mtu | int + register: pve_node_network_mtu_staged + +- name: Report staged network change + ansible.builtin.debug: + msg: >- + {{ inventory_hostname }}: {{ pve_node_network_bridge }} MTU change to + {{ pve_node_network_bridge_mtu }} is staged in + /etc/network/interfaces.new and NOT yet live. Apply it deliberately, + one node at a time: + ansible-playbook -i inventory/ plays/configure-pve-node-network.yml + --limit {{ inventory_hostname }} -e pve_node_network_apply_pending=true + when: + - pve_node_network_mtu_staged is changed + - not (pve_node_network_apply_pending | bool) + +- name: Apply pending network configuration + ansible.builtin.command: pvesh set /nodes/{{ inventory_hostname }}/network + when: + - pve_node_network_apply_pending | bool + - pve_node_network_mtu_staged is changed + +- name: Verify bridge MTU is live + ansible.builtin.command: cat /sys/class/net/{{ pve_node_network_bridge }}/mtu + register: pve_node_network_live_mtu + changed_when: false + when: pve_node_network_apply_pending | bool + +- name: Fail if bridge MTU did not converge + ansible.builtin.assert: + that: + - pve_node_network_live_mtu.stdout | int == pve_node_network_bridge_mtu | int + fail_msg: >- + {{ pve_node_network_bridge }} MTU is {{ pve_node_network_live_mtu.stdout }}, + expected {{ pve_node_network_bridge_mtu }}. + success_msg: >- + {{ pve_node_network_bridge }} MTU is {{ pve_node_network_bridge_mtu }} + when: pve_node_network_apply_pending | bool diff --git a/ansible/roles/pve-node-network/templates/70-pve-ipv6-ra.conf.j2 b/ansible/roles/pve-node-network/templates/70-pve-ipv6-ra.conf.j2 new file mode 100644 index 00000000..070d5df4 --- /dev/null +++ b/ansible/roles/pve-node-network/templates/70-pve-ipv6-ra.conf.j2 @@ -0,0 +1,39 @@ +# +# Managed by Ansible (role: pve-node-network) — do not edit by hand. +# +# Why this file exists: +# +# /etc/frr/frr.conf enables "ipv6 forwarding" for the OSPFv3 Thunderbolt mesh +# that carries corosync. That sets net.ipv6.conf.all.forwarding=1. +# +# The kernel default is accept_ra=1, which means "accept Router Advertisements +# ONLY IF forwarding is disabled". So the moment FRR came up, these nodes began +# silently discarding the RAs the UniFi gateway sends on VLAN 40, leaving every +# external interface with nothing but a link-local fe80:: address. The only +# non-link-local IPv6 on the nodes was the corosync mesh /128, which is static +# in frr.conf and never depended on RAs. +# +# accept_ra=2 means "accept RAs even though I am forwarding" — the correct +# setting for a router that still needs to autoconfigure its own uplink. +# +# Verified 2026-07-19: the gateway does advertise on VLAN 40 (RAs from +# fe80::265a:4cff:fea6:fd3c roughly every 10-15s, carrying both the Telekom +# GUA prefix and a ULA). The nodes were receiving and discarding them. +# +# See wiki: pve-ipv6-accept-ra-forwarding-defect +# + +# 'default' covers interfaces created after boot; 'all' propagates to the +# interfaces that already exist when this is applied. +net.ipv6.conf.all.accept_ra = {{ pve_node_network_accept_ra }} +net.ipv6.conf.default.accept_ra = {{ pve_node_network_accept_ra }} + +# Autoconf is already on by default, but state it explicitly so the intent +# survives someone tightening the defaults later. +net.ipv6.conf.all.autoconf = 1 +net.ipv6.conf.default.autoconf = 1 +{% for iface in pve_node_network_ra_interfaces %} + +net.ipv6.conf.{{ iface | replace('.', '/') }}.accept_ra = {{ pve_node_network_accept_ra }} +net.ipv6.conf.{{ iface | replace('.', '/') }}.autoconf = 1 +{% endfor %}