diff --git a/kubernetes/talos/patches/k8s-2.yaml b/kubernetes/talos/patches/k8s-2.yaml index 00a9c4ef..2c9e2e9c 100644 --- a/kubernetes/talos/patches/k8s-2.yaml +++ b/kubernetes/talos/patches/k8s-2.yaml @@ -1,31 +1,99 @@ +# k8s-2 runs Talos directly on saturn's MS-01 hardware. Structurally identical +# to k8s-1.yaml — see that file for the reasoning behind each choice; only the +# serials, MACs and addresses differ. k8s-3 is still a Proxmox VM and keeps the +# VM-era layout until neptune is converted. machine: network: hostname: k8s-2 - # Shared control-plane VIP — see k8s-1.yaml for the busPath rationale. - interfaces: - - deviceSelector: - busPath: "0000:06:12.0" - dhcp: true - vip: - ip: 10.2.40.60 - # v30iot: link only, no host addressing — see k8s-1.yaml. - - deviceSelector: - busPath: "0000:06:13.0" - dhcp: false - # See k8s-1.yaml for context on the dedicated Ceph cluster network. - - deviceSelector: - driver: i40e - addresses: - - 10.99.0.2/29 - mtu: 9000 sysctls: - # v30iot NIC carries no host addressing — see k8s-1.yaml. + # ens19 (VLAN 30, v30iot) is LINK ONLY — no host addressing. It is the + # macvlan parent for the v30iot pod attachments; host addresses on that L2 + # make node replies bypass the UDM and break every UDM-DNATed flow. net.ipv6.conf.ens19.disable_ipv6: "1" install: - disk: /dev/vda + # Three NVMe drives are visible on bare metal. Select the installer target + # by serial so it cannot land on the OSD drive or on EPHEMERAL, whatever + # order the kernel enumerates them in — Talos and Proxmox do not agree on + # that order. + diskSelector: + serial: 50026B7382DA5F6E certSANs: - k8s-2.srv.renner.uno nodeLabels: topology.kubernetes.io/zone: saturn cluster: allowSchedulingOnControlPlanes: true +--- +# EPHEMERAL (/var) goes on the retired Crucial P3, not the system disk. That +# drive is at 81% of its rated endurance and is finished as an OSD, but +# everything EPHEMERAL holds is reconstructible, so it is consumable in this +# role. It also gives /var ~931 GB instead of the 64 GB the VM-era zvol had. +# +# ONLY TAKES EFFECT ON FIRST PROVISION. Talos applies volume configuration +# solely when the volume does not yet exist; applied later it is silently +# ignored and relocating needs a disk wipe and a boot from install media. +# +# Matched by serial deliberately: a broad selector such as +# `disk.transport == 'nvme' && !system_disk` would match the PM983 as well. +apiVersion: v1alpha1 +kind: VolumeConfig +name: EPHEMERAL +provisioning: + diskSelector: + match: disk.serial == '2332E86B74BC' + minSize: 50GiB + grow: true +--- +# X710 port 0 carries the switch trunk and takes no addressing itself — the +# port's untagged VLAN is not one this node belongs on. Management and v30iot +# both ride it as tagged VLANs. +# +# Addressed by predictable kernel name, not by LinkAliasConfig. Aliases are +# valid VLAN parents, but a selector matching zero links is skipped silently +# and Talos disables default DHCP machine-wide as soon as any new-style network +# document exists — so a silent miss leaves the node with no address on any +# interface. See RUNBOOK-baremetal-conversion.md. +apiVersion: v1alpha1 +kind: LinkConfig +name: enp2s0f0np0 +up: true +--- +# Management, VLAN 40. Node IP, etcd, the API and Ceph's public network. +apiVersion: v1alpha1 +kind: VLANConfig +name: mgmt0 +vlanID: 40 +parent: enp2s0f0np0 +up: true +--- +apiVersion: v1alpha1 +kind: DHCPv4Config +name: mgmt0 +--- +# Shared control-plane VIP, etcd-elected. Exposed via UniFi DNS as +# k8s.renner.uno. +apiVersion: v1alpha1 +kind: Layer2VIPConfig +name: 10.2.40.60 +link: mgmt0 +--- +# Named ens19 on purpose: the four v30iot macvlan NADs hard-code +# `master: ens19`, and k8s-3 still presents the VLAN as a virtio NIC of that +# name. VLANConfig.name is free-form, so one set of NADs serves both. +apiVersion: v1alpha1 +kind: VLANConfig +name: ens19 +vlanID: 30 +parent: enp2s0f0np0 +up: true +--- +# Dedicated Ceph cluster network on X710 port 1, VLAN 49 / 10.99.0.0/29, +# untagged on the wire. OSD<->OSD replication only. Not routed off its own +# segment and deliberately has no default route. +apiVersion: v1alpha1 +kind: LinkConfig +name: enp2s0f1np1 +up: true +mtu: 9000 +addresses: + - address: 10.99.0.2/29