ansible/plays/configure-pve-node-network.yml

@@ -0,0 +1,53 @@
+---
+# Configure Proxmox VE Node Host Networking
+#
+# Fixes two defects found by live inspection on 2026-07-19:
+#
+#   1. All three nodes had no IPv6 address on any external interface. FRR
+#      enables ipv6 forwarding for the OSPFv3 corosync mesh, and with the
+#      kernel default accept_ra=1 the nodes silently discarded the Router
+#      Advertisements the gateway sends on VLAN 40. Fixed with accept_ra=2.
+#
+#   2. saturn's vmbr0 sits at MTU 1500 while its own uplink and both peer
+#      nodes are at 9000, so large frames between saturn's guests and the
+#      other nodes' guests blackhole instead of fragmenting.
+#
+# The sysctl fix is non-disruptive and applies on every run. The MTU fix is
+# staged as a pending PVE network change and only applied when explicitly
+# requested, because applying it reloads networking on the node and briefly
+# interrupts the Talos k8s VM and its Ceph OSD.
+#
+# Safe (converges IPv6, stages the MTU change without applying it):
+#   ansible-playbook -i inventory/ plays/configure-pve-node-network.yml
+#
+# Applying the staged MTU change, one node at a time:
+#   ansible-playbook -i inventory/ plays/configure-pve-node-network.yml \
+#     --limit saturn -e pve_node_network_apply_pending=true
+#
+# See wiki: pve-ipv6-accept-ra-forwarding-defect,
+#           pve-saturn-vmbr0-mtu-asymmetry
+
+- name: Configure PVE node host networking
+  hosts: pve_cluster
+  become: yes
+  serial: 1
+
+  roles:
+    - role: pve-node-network
+
+  post_tasks:
+    - name: Summarise node IPv6 state
+      ansible.builtin.command: ip -6 addr show dev vmbr1.40 scope global
+      register: pve_node_network_v6_summary
+      changed_when: false
+      failed_when: false
+
+    - name: Display node IPv6 state
+      ansible.builtin.debug:
+        msg: >-
+          {{ inventory_hostname }}:
+          {{ pve_node_network_v6_summary.stdout_lines
+             | select('search', 'inet6')
+             | map('trim')
+             | list
+             | default(['no global IPv6'], true) }}

ansible/roles/pve-node-network/defaults/main.yml

@@ -0,0 +1,34 @@
+---
+# PVE Node Network Role Defaults
+
+# --- IPv6 Router Advertisement acceptance -----------------------------------
+# The PVE nodes run FRR (OSPFv3 over the Thunderbolt mesh, carrying corosync),
+# which enables net.ipv6.conf.all.forwarding. The kernel default accept_ra=1
+# means "accept RAs only while NOT forwarding", so the nodes discard the RAs
+# the UniFi gateway sends. accept_ra=2 is the value a forwarding host needs in
+# order to still autoconfigure its own uplink.
+pve_node_network_accept_ra: 2
+
+# Interfaces to set accept_ra on explicitly, in addition to all/default.
+# Dots are translated to slashes for the sysctl key (vmbr1.40 -> vmbr1/40).
+pve_node_network_ra_interfaces:
+  - vmbr1.40
+  - vmbr0
+
+# --- vmbr0 MTU --------------------------------------------------------------
+# jupiter and neptune already carry 9000; saturn's stanza is missing the mtu
+# line and sits at 1500 while its own uplink is 9000. All three should match,
+# since they share one L2 domain and mismatched MTUs blackhole large frames
+# rather than fragmenting them.
+pve_node_network_bridge: vmbr0
+pve_node_network_bridge_mtu: 9000
+
+# Whether to manage the bridge MTU at all.
+pve_node_network_manage_mtu: yes
+
+# Applying an MTU change requires a network reload (ifreload -a), which briefly
+# interrupts guest connectivity on the node — including the Talos k8s VM and its
+# Ceph OSD. Left false, the change is written to /etc/network/interfaces.new and
+# shows as "pending" in the PVE UI, to be applied deliberately per node.
+# Run with -e pve_node_network_apply_pending=true, ideally with --limit <node>.
+pve_node_network_apply_pending: no

ansible/roles/pve-node-network/handlers/main.yml

@@ -0,0 +1,6 @@
+---
+# PVE Node Network Handlers
+
+- name: Reload sysctl
+  ansible.builtin.command: sysctl --system
+  changed_when: true

ansible/roles/pve-node-network/tasks/main.yml

@@ -0,0 +1,141 @@
+---
+# PVE Node Network Role
+#
+# Two independent concerns, deliberately separated by risk:
+#
+#   1. IPv6 RA acceptance — a sysctl drop-in. Non-disruptive, reversible,
+#      applied unconditionally.
+#   2. vmbr0 MTU convergence — requires a network reload that briefly drops
+#      guest connectivity. Staged as a pending change by default; applied only
+#      when pve_node_network_apply_pending is true.
+
+- name: Verify this is a Proxmox VE node
+  ansible.builtin.command: pveversion
+  register: pve_node_network_pveversion
+  changed_when: false
+
+# --- 1. IPv6 Router Advertisement acceptance --------------------------------
+
+- name: Deploy IPv6 RA sysctl drop-in
+  ansible.builtin.template:
+    src: 70-pve-ipv6-ra.conf.j2
+    dest: /etc/sysctl.d/70-pve-ipv6-ra.conf
+    owner: root
+    group: root
+    mode: '0644'
+    backup: yes
+  notify: Reload sysctl
+
+- name: Apply pending sysctl changes before verifying
+  ansible.builtin.meta: flush_handlers
+
+- name: Read back effective accept_ra on the management interface
+  ansible.builtin.command: sysctl -n net.ipv6.conf.vmbr1/40.accept_ra
+  register: pve_node_network_effective_accept_ra
+  changed_when: false
+
+- name: Fail if accept_ra did not take effect
+  ansible.builtin.assert:
+    that:
+      - pve_node_network_effective_accept_ra.stdout | int == pve_node_network_accept_ra | int
+    fail_msg: >-
+      accept_ra on vmbr1.40 is
+      {{ pve_node_network_effective_accept_ra.stdout }}, expected
+      {{ pve_node_network_accept_ra }}. The sysctl drop-in did not apply.
+    success_msg: "accept_ra on vmbr1.40 is {{ pve_node_network_accept_ra }}"
+
+# SLAAC needs a Router Advertisement to arrive before an address appears. The
+# gateway advertises roughly every 10-15s, so a short wait is enough; this is
+# reported rather than enforced, since a node may legitimately be on a segment
+# without RAs.
+- name: Wait for a global IPv6 address to be autoconfigured
+  ansible.builtin.shell: >-
+    ip -6 addr show dev vmbr1.40 scope global | grep -q 'inet6'
+  register: pve_node_network_slaac_check
+  retries: 6
+  delay: 10
+  until: pve_node_network_slaac_check.rc == 0
+  failed_when: false
+  changed_when: false
+
+- name: Report IPv6 autoconfiguration result
+  ansible.builtin.debug:
+    msg: >-
+      {% if pve_node_network_slaac_check.rc == 0 %}
+      {{ inventory_hostname }}: global IPv6 address acquired on vmbr1.40
+      {% else %}
+      {{ inventory_hostname }}: still no global IPv6 on vmbr1.40 after 60s.
+      Check that Router Advertisements reach this VLAN:
+      tcpdump -nni vmbr1.40 'icmp6 and ip6[40] == 134'
+      {% endif %}
+
+# --- 2. vmbr0 MTU -----------------------------------------------------------
+
+- name: Query current bridge configuration
+  ansible.builtin.command: >-
+    pvesh get /nodes/{{ inventory_hostname }}/network/{{ pve_node_network_bridge }}
+    --output-format json
+  register: pve_node_network_bridge_raw
+  changed_when: false
+  when: pve_node_network_manage_mtu | bool
+
+- name: Parse current bridge configuration
+  ansible.builtin.set_fact:
+    pve_node_network_bridge_cfg: "{{ pve_node_network_bridge_raw.stdout | from_json }}"
+  when: pve_node_network_manage_mtu | bool
+
+# A PUT on /nodes/<node>/network/<iface> REPLACES the interface definition
+# rather than merging into it. Passing only --mtu would silently drop
+# bridge_ports, bridge_vlan_aware and bridge_vids, detaching the uplink and
+# flattening every VNet. Every existing attribute is therefore passed back.
+- name: Set bridge MTU
+  ansible.builtin.command: >-
+    pvesh set /nodes/{{ inventory_hostname }}/network/{{ pve_node_network_bridge }}
+    --type bridge
+    --autostart {{ pve_node_network_bridge_cfg.autostart | default(1) }}
+    --bridge_ports "{{ pve_node_network_bridge_cfg.bridge_ports }}"
+    --bridge_stp {{ pve_node_network_bridge_cfg.bridge_stp | default('off') }}
+    --bridge_fd {{ pve_node_network_bridge_cfg.bridge_fd | default(0) }}
+    --bridge_vlan_aware {{ pve_node_network_bridge_cfg.bridge_vlan_aware | default(0) }}
+    --bridge_vids "{{ pve_node_network_bridge_cfg.bridge_vids | default('2-4094') }}"
+    --mtu {{ pve_node_network_bridge_mtu }}
+  when:
+    - pve_node_network_manage_mtu | bool
+    - pve_node_network_bridge_cfg.mtu | default(0) | int != pve_node_network_bridge_mtu | int
+  register: pve_node_network_mtu_staged
+
+- name: Report staged network change
+  ansible.builtin.debug:
+    msg: >-
+      {{ inventory_hostname }}: {{ pve_node_network_bridge }} MTU change to
+      {{ pve_node_network_bridge_mtu }} is staged in
+      /etc/network/interfaces.new and NOT yet live. Apply it deliberately,
+      one node at a time:
+      ansible-playbook -i inventory/ plays/configure-pve-node-network.yml
+      --limit {{ inventory_hostname }} -e pve_node_network_apply_pending=true
+  when:
+    - pve_node_network_mtu_staged is changed
+    - not (pve_node_network_apply_pending | bool)
+
+- name: Apply pending network configuration
+  ansible.builtin.command: pvesh set /nodes/{{ inventory_hostname }}/network
+  when:
+    - pve_node_network_apply_pending | bool
+    - pve_node_network_mtu_staged is changed
+
+- name: Verify bridge MTU is live
+  ansible.builtin.command: cat /sys/class/net/{{ pve_node_network_bridge }}/mtu
+  register: pve_node_network_live_mtu
+  changed_when: false
+  when: pve_node_network_apply_pending | bool
+
+- name: Fail if bridge MTU did not converge
+  ansible.builtin.assert:
+    that:
+      - pve_node_network_live_mtu.stdout | int == pve_node_network_bridge_mtu | int
+    fail_msg: >-
+      {{ pve_node_network_bridge }} MTU is {{ pve_node_network_live_mtu.stdout }},
+      expected {{ pve_node_network_bridge_mtu }}.
+    success_msg: >-
+      {{ pve_node_network_bridge }} MTU is {{ pve_node_network_bridge_mtu }}
+  when: pve_node_network_apply_pending | bool

ansible/roles/pve-node-network/templates/70-pve-ipv6-ra.conf.j2

@@ -0,0 +1,39 @@
+#
+# Managed by Ansible (role: pve-node-network) — do not edit by hand.
+#
+# Why this file exists:
+#
+# /etc/frr/frr.conf enables "ipv6 forwarding" for the OSPFv3 Thunderbolt mesh
+# that carries corosync. That sets net.ipv6.conf.all.forwarding=1.
+#
+# The kernel default is accept_ra=1, which means "accept Router Advertisements
+# ONLY IF forwarding is disabled". So the moment FRR came up, these nodes began
+# silently discarding the RAs the UniFi gateway sends on VLAN 40, leaving every
+# external interface with nothing but a link-local fe80:: address. The only
+# non-link-local IPv6 on the nodes was the corosync mesh /128, which is static
+# in frr.conf and never depended on RAs.
+#
+# accept_ra=2 means "accept RAs even though I am forwarding" — the correct
+# setting for a router that still needs to autoconfigure its own uplink.
+#
+# Verified 2026-07-19: the gateway does advertise on VLAN 40 (RAs from
+# fe80::265a:4cff:fea6:fd3c roughly every 10-15s, carrying both the Telekom
+# GUA prefix and a ULA). The nodes were receiving and discarding them.
+#
+# See wiki: pve-ipv6-accept-ra-forwarding-defect
+#
+
+# 'default' covers interfaces created after boot; 'all' propagates to the
+# interfaces that already exist when this is applied.
+net.ipv6.conf.all.accept_ra = {{ pve_node_network_accept_ra }}
+net.ipv6.conf.default.accept_ra = {{ pve_node_network_accept_ra }}
+
+# Autoconf is already on by default, but state it explicitly so the intent
+# survives someone tightening the defaults later.
+net.ipv6.conf.all.autoconf = 1
+net.ipv6.conf.default.autoconf = 1
+{% for iface in pve_node_network_ra_interfaces %}
+
+net.ipv6.conf.{{ iface | replace('.', '/') }}.accept_ra = {{ pve_node_network_accept_ra }}
+net.ipv6.conf.{{ iface | replace('.', '/') }}.autoconf = 1
+{% endfor %}