docs(talos): cut the narration out of the machine config patches #680
@@ -5,52 +5,24 @@ machine:
|
||||
- k8s-3.srv.renner.uno
|
||||
- k8s.renner.uno
|
||||
- 10.2.40.60
|
||||
# Install from the Image Factory metal installer, not the stock
|
||||
# ghcr.io/siderolabs/installer that `talosctl gen config` defaults to.
|
||||
# Extensions live in the installer image, so with the stock one the schematic
|
||||
# is only ever a property of the boot media and nothing lands on disk —
|
||||
# `talosctl get extensions` returns empty on a node installed that way.
|
||||
# Image Factory metal installer, NOT the stock ghcr.io/siderolabs/installer
|
||||
# that `talosctl gen config` defaults to. Extensions live in the installer
|
||||
# image; install with the stock one and nothing lands on disk —
|
||||
# `talosctl get extensions` comes back empty.
|
||||
#
|
||||
# What the schematic carries, and why each matters here:
|
||||
# The schematic carries intel-ucode, thunderbolt and i915, and sets
|
||||
# net.ifnames=1, which is what the per-node VLAN parents match against.
|
||||
# Changing which extensions are present means a new schematic hash and a
|
||||
# reinstall or upgrade, not an edit here.
|
||||
#
|
||||
# siderolabs/intel-ucode CPU microcode. On a VM the hypervisor supplied
|
||||
# it; on bare metal nothing does except the BIOS.
|
||||
# siderolabs/thunderbolt Inert until machine.kernel.modules asks for
|
||||
# thunderbolt/thunderbolt_net. Present so enabling
|
||||
# the MS-01 mesh later needs neither a new image
|
||||
# nor a reinstall — which is exactly the property
|
||||
# lost by installing without it.
|
||||
# siderolabs/i915 Intel GPU microcode and kernel modules for the
|
||||
# Alder Lake-P Iris Xe in the MS-01. No workload
|
||||
# requests /dev/dri today; it is here for the same
|
||||
# reason as thunderbolt, so that wanting hardware
|
||||
# transcoding later is a config change rather than
|
||||
# another rolling upgrade of every node.
|
||||
# Read only at install and upgrade time — editing this does nothing to a
|
||||
# running node. The tag must match the version being upgraded to, or the
|
||||
# upgrade is a downgrade. Talos only tests migration between adjacent minor
|
||||
# releases: take the latest patch of each intermediate minor rather than
|
||||
# skipping one.
|
||||
#
|
||||
# Not included, and why, so the absences are not mistaken for oversights:
|
||||
# the X710 (i40e) and I226 (igc) NICs need no firmware extension —
|
||||
# intel-ice-firmware covers E810/ice only. mei is a prerequisite for Intel
|
||||
# Arc discrete GPUs, of which there are none. iscsi-tools, nvme-cli and
|
||||
# util-linux-tools have no consumer: storage is Ceph RBD, and device health
|
||||
# is reported by Ceph itself rather than a node-level exporter.
|
||||
#
|
||||
# 0f6fb8bd… is the bare-metal schematic: same net.ifnames=1 as the Proxmox
|
||||
# one (the predictable interface names the per-node VLAN parents match
|
||||
# against), minus qemu-guest-agent. It supersedes a21cec8f…, which was
|
||||
# identical but without i915 — see RUNBOOK-cold-rebuild.md.
|
||||
#
|
||||
# Only read at install and upgrade time. Changing it here does nothing to a
|
||||
# running node until that node is upgraded. Bump the tag with the Talos
|
||||
# version — it must match the running version, or the upgrade is a downgrade.
|
||||
#
|
||||
# Upgrades follow the documented path: migration is only tested between
|
||||
# adjacent minor releases, so go to the latest patch of each intermediate
|
||||
# minor rather than skipping. 1.12.11 -> 1.13.9 satisfies that — one minor
|
||||
# step from the last 1.12 patch, no intermediate needed.
|
||||
#
|
||||
# Kubernetes is a separate decision and is not forced by this. Talos 1.13
|
||||
# supports Kubernetes 1.36, 1.35, 1.34, 1.33, 1.32 and 1.31, so the cluster
|
||||
# stays on 1.35.2 until `talosctl upgrade-k8s` moves it deliberately.
|
||||
# The Kubernetes version is a separate decision, moved by
|
||||
# `talosctl upgrade-k8s`.
|
||||
install:
|
||||
image: factory.talos.dev/metal-installer/0f6fb8bd8bb53013e9743b7c7b099d96fb9c32fd2464ced32693c2dc0ea5feb9:v1.13.9
|
||||
features:
|
||||
@@ -74,20 +46,13 @@ machine:
|
||||
nodeIP:
|
||||
validSubnets:
|
||||
- 10.2.40.0/24
|
||||
# Lower the image GC thresholds from the kubelet defaults (85/80) to
|
||||
# bound containerd's image cache, which is the largest single consumer
|
||||
# of /var. Spegel P2P pull-through softens the re-pull cost when GC'd
|
||||
# images are needed back.
|
||||
# Lower than the kubelet defaults (85/80) to bound containerd's image
|
||||
# cache, the largest single consumer of /var.
|
||||
#
|
||||
# These thresholds do NOT control whether Ceph's mon reports
|
||||
# MON_DISK_LOW, and must not be tuned for that. Measured on k8s-2:
|
||||
# image GC frees only the content store (~8.6 GB), while the bulk of
|
||||
# /var/lib/containerd is ~20 GB of overlayfs snapshots — unpacked
|
||||
# layers shared with running containers, which GC cannot reclaim.
|
||||
# Dropping high to 60 deleted 28 images and moved free space by 0.02
|
||||
# percentage points. The mon's own data is ~106 MB; its 30%-free
|
||||
# warning is measuring a filesystem it merely shares, so the lever for
|
||||
# it is elsewhere (mon volume, /var size, or mon_data_avail_warn).
|
||||
# Do NOT tune these to clear Ceph's MON_DISK_LOW. Image GC reclaims only
|
||||
# the content store; the bulk of /var/lib/containerd is overlayfs
|
||||
# snapshots shared with running containers, which GC cannot touch. The
|
||||
# lever for the mon warning is elsewhere.
|
||||
extraConfig:
|
||||
imageGCHighThresholdPercent: 70
|
||||
imageGCLowThresholdPercent: 60
|
||||
|
||||
@@ -1,55 +1,25 @@
|
||||
# k8s-1 runs Talos directly on jupiter's MS-01 hardware; k8s-2 and k8s-3 are
|
||||
# still Proxmox VMs. Every machine-specific selector below differs from theirs
|
||||
# for that reason: there are no virtio NICs, no PCI-passthrough layer, and all
|
||||
# four physical NICs and all three NVMe drives are visible to Talos at once.
|
||||
#
|
||||
# Networking uses the v1.12 multi-document config rather than the deprecated
|
||||
# machine.network.interfaces. That is deliberate: the docs do not define what
|
||||
# happens when the same link is configured by both schemas, so this file keeps
|
||||
# every link in one schema instead of straddling them.
|
||||
# Networking uses the multi-document config, not the deprecated
|
||||
# machine.network.interfaces. Keep every link in one schema: the docs do not
|
||||
# define what happens when both configure the same link.
|
||||
machine:
|
||||
network:
|
||||
hostname: k8s-1
|
||||
sysctls:
|
||||
# ens19 (VLAN 30, v30iot) is LINK ONLY — no host addressing. It exists as
|
||||
# the macvlan parent for the v30iot pod attachments
|
||||
# (matter/otbr/shelly-manager/music-assistant), whose children do their own
|
||||
# DHCP on that L2. Host addresses were never needed and are actively
|
||||
# harmful: a connected 10.2.30.0/24 route makes node replies to IoT-subnet
|
||||
# clients bypass the UDM, which breaks every UDM-DNATed flow (hairpin NAT:
|
||||
# un-translated SYN-ACK -> client kernel RST, verified live 2026-07-23).
|
||||
# This disables v6 entirely on that link, including link-local.
|
||||
# ens19 (VLAN 30, v30iot) is LINK ONLY — it is the macvlan parent for the
|
||||
# v30iot pod attachments and must never get host addressing. A connected
|
||||
# 10.2.30.0/24 route makes node replies bypass the UDM and breaks every
|
||||
# DNATed flow.
|
||||
net.ipv6.conf.ens19.disable_ipv6: "1"
|
||||
# Same argument, applied to the physical ports rather than a VLAN child.
|
||||
#
|
||||
# enp2s0f0np0 is nothing but the trunk parent for mgmt0 and ens19, and
|
||||
# enp87s0 is unused. Both sit on the untagged fallback network, where by
|
||||
# design nothing is supposed to run — and both were autoconfiguring
|
||||
# themselves from its router advertisements: an fd00::/64 ULA each, plus
|
||||
# an IPv6 default route each.
|
||||
#
|
||||
# The result was four IPv6 default routes per node at identical priority
|
||||
# 1024 — mgmt0, enp2s0f0np0, enp87s0 and lo — of which only mgmt0 has a
|
||||
# globally routable source address. With nothing to break the tie,
|
||||
# outbound IPv6 mostly left through an interface whose only usable source
|
||||
# was a ULA, so the cluster had no IPv6 egress at all while IPv4 worked
|
||||
# normally. It surfaced as two NetBird probes reporting the proxies down
|
||||
# for 30 hours; the proxies were fine and answered IPv6 in 13 ms from
|
||||
# outside.
|
||||
#
|
||||
# Verified on k8s-1 before rolling to the others: disabling v6 on a VLAN
|
||||
# parent leaves the VLAN child alone. mgmt0 kept its global address and
|
||||
# its fd40:: address, Ceph did not notice, and every failing probe
|
||||
# recovered on the next scrape.
|
||||
# These two sit on the untagged fallback network, where nothing may run.
|
||||
# Left enabled they autoconfigure a ULA and an IPv6 default route from its
|
||||
# RAs, which ties with mgmt0's and costs the cluster IPv6 egress.
|
||||
# Disabling v6 on a VLAN parent does not affect the VLAN child.
|
||||
net.ipv6.conf.enp2s0f0np0.disable_ipv6: "1"
|
||||
net.ipv6.conf.enp87s0.disable_ipv6: "1"
|
||||
install:
|
||||
# Three NVMe drives are ordinary block devices to Talos here: the Kingston
|
||||
# OM8PGP41024Q-A0 system disk, the Samsung PM983 that Rook claims as the
|
||||
# Ceph OSD, and the retired Crucial P3 that carries EPHEMERAL (see the
|
||||
# VolumeConfig below). Every one of them is selected by serial. Nothing in
|
||||
# this file may use a broad selector: `disk.transport == 'nvme'` matches all
|
||||
# three, and the failure mode is Talos taking the OSD drive.
|
||||
# Select disks by serial only. A broad selector such as
|
||||
# `disk.transport == 'nvme'` matches every drive in the machine, and the
|
||||
# failure mode is the installer taking the Ceph OSD drive.
|
||||
diskSelector:
|
||||
serial: 50026B7382B48B86
|
||||
certSANs:
|
||||
@@ -59,34 +29,10 @@ machine:
|
||||
cluster:
|
||||
allowSchedulingOnControlPlanes: true
|
||||
---
|
||||
# EPHEMERAL (/var) goes on the retired Crucial P3, not the system disk.
|
||||
#
|
||||
# That drive is at 87% of its rated endurance and was this node's Ceph OSD until
|
||||
# the PM983 replaced it. /var is exactly the right role for a disk in that
|
||||
# state: containerd's image and snapshot store, pod logs and the Ceph mon store
|
||||
# are all reconstructible, so the drive is consumable here in a way it is not as
|
||||
# an OSD. It also gives /var ~931 GB instead of the 64 GB the VM-era zvol had,
|
||||
# which is the constraint that makes k8s-2 and k8s-3 trip DiskPressure whenever
|
||||
# they absorb another node's pods.
|
||||
#
|
||||
# One caveat this node does NOT share with k8s-2 and k8s-3: their P3s report
|
||||
# zero media errors, this one reports 18 (smartctl_device_media_errors,
|
||||
# verified 2026-08-27 — the only non-zero counter in the fleet). Everything
|
||||
# else on the drive is clean, available spare 100% and critical-warning byte
|
||||
# 0, and NvmeDriveMediaErrors alerts on the increase rather than the total, so
|
||||
# a static count never surfaces there.
|
||||
#
|
||||
# Whether those 18 are old damage from its OSD career or ongoing is NOT
|
||||
# established. Prometheus only has a few hours of history for this drive, flat
|
||||
# at 18, because the exporter first scraped this node at its conversion — the
|
||||
# series being young is not evidence the count is stable. Settle it on the
|
||||
# node with `smartctl -a /dev/nvme1n1` and compare the error-log entries
|
||||
# against power-on hours.
|
||||
#
|
||||
# That question matters more here than the wear number does, because
|
||||
# /var/lib/etcd lives on EPHEMERAL: if the count is climbing, this node's etcd
|
||||
# member data is sitting on failing media and EPHEMERAL belongs on the system
|
||||
# disk instead — which means a wipe and install media, per the note below.
|
||||
# EPHEMERAL (/var) goes on the retired Ceph OSD drive, deliberately: its
|
||||
# contents are reconstructible, so a worn disk is consumable here in a way it
|
||||
# is not as an OSD. Note /var/lib/etcd lives here too, which is the one thing
|
||||
# on the volume that is not free to lose.
|
||||
#
|
||||
# THIS ONLY TAKES EFFECT ON FIRST PROVISION. Talos applies volume configuration
|
||||
# solely when the volume does not yet exist; a config applied after EPHEMERAL
|
||||
@@ -94,8 +40,8 @@ cluster:
|
||||
# requires wiping the disk and booting install media again.
|
||||
#
|
||||
# Matched by serial deliberately. `disk.transport == 'nvme' && !system_disk`
|
||||
# would match the PM983 as well, and losing that coin flip means Talos formats
|
||||
# the OSD drive.
|
||||
# still matches the OSD drive, and losing that coin flip means Talos formats
|
||||
# it.
|
||||
apiVersion: v1alpha1
|
||||
kind: VolumeConfig
|
||||
name: EPHEMERAL
|
||||
@@ -105,30 +51,25 @@ provisioning:
|
||||
minSize: 50GiB
|
||||
grow: true
|
||||
---
|
||||
# X710 port 0 carries the switch trunk. It takes no addressing itself — the
|
||||
# port's untagged VLAN is the 192.168.8.0/24 network, which this node has no
|
||||
# business being on — so management and v30iot both ride it as tagged VLANs.
|
||||
# X710 port 0 carries the switch trunk and takes no addressing itself — its
|
||||
# untagged VLAN is a network this node has no business being on. Management
|
||||
# and v30iot ride it as tagged VLANs.
|
||||
#
|
||||
# Links are addressed by predictable kernel name. Those derive from PCI
|
||||
# topology, not probe order — though not immune: adding the third NVMe
|
||||
# renumbered the buses and moved the unused second I226 from enp89s0 to
|
||||
# enp90s0. Neither name used here was affected.
|
||||
# Links are named by predictable kernel name, which derives from PCI topology
|
||||
# and can still shift when hardware is added.
|
||||
#
|
||||
# LinkAliasConfig would be the better way to pin these, and is worth
|
||||
# reintroducing — but only once it is proven. Aliasing both X710 ports by
|
||||
# permanent MAC and using the aliases as VLAN parents produced a node with no
|
||||
# address on any interface: the aliases were never created, so every document
|
||||
# referencing them did nothing, including the static address on port 1. Verify
|
||||
# any alias-based revision with `talosctl apply-config --mode=try`, which rolls
|
||||
# back on its own, before making it permanent.
|
||||
# LinkAliasConfig would pin these better, but aliasing the X710 ports by MAC
|
||||
# and using the aliases as VLAN parents left the node with no address on any
|
||||
# interface — the aliases were never created and every document referencing
|
||||
# them silently did nothing. Prove any alias-based revision with
|
||||
# `talosctl apply-config --mode=try` first; it rolls back on its own.
|
||||
apiVersion: v1alpha1
|
||||
kind: LinkConfig
|
||||
name: enp2s0f0np0
|
||||
up: true
|
||||
---
|
||||
# Management, VLAN 40. Carries the node IP, etcd, the API and Ceph's public
|
||||
# network. The k8s-1 VM reached 10.2.40.174 over this same physical port via
|
||||
# Proxmox's v40srv VNet (vmbr0.40), so the switch trunks VLAN 40 here.
|
||||
# network.
|
||||
apiVersion: v1alpha1
|
||||
kind: VLANConfig
|
||||
name: mgmt0
|
||||
@@ -136,30 +77,15 @@ vlanID: 40
|
||||
parent: enp2s0f0np0
|
||||
up: true
|
||||
---
|
||||
# Management addressing is DHCP and the lease is NOT reserved anywhere in this
|
||||
# repo — the k8s-{1,2,3}.srv.renner.uno reservations live on the UDM out of band
|
||||
# and opentofu/unifi.tf declares no live resources. The addresses do move under
|
||||
# a running node: k8s-3 went 10.2.40.168 -> .138 mid-flight on 2026-08-26
|
||||
# without its Node object ever re-registering (issue #635).
|
||||
# DHCP, and the lease is not reserved in this repo — reservations live on the
|
||||
# UDM out of band. Addresses do move under a running node.
|
||||
#
|
||||
# kube-apiserver does not follow when that happens. Talos starts it with
|
||||
# `--advertise-address=$(POD_IP)` and no `--endpoint-reconciler-type` override,
|
||||
# so the advertised address is fixed at pod start and the default lease
|
||||
# reconciler goes on renewing the masterlease under the OLD address. Nothing
|
||||
# expires it while the process lives, so `default/kubernetes` keeps handing out
|
||||
# a dead endpoint and the node's live address is never registered at all — 19h
|
||||
# of TargetDown on job="apiserver", a monitoring blind spot on one of three
|
||||
# apiservers, and it cleared only when the static pod restarted.
|
||||
#
|
||||
# So after any node re-IPs, check that
|
||||
# kube-apiserver does not follow a re-IP: it advertises $(POD_IP) fixed at pod
|
||||
# start and the lease reconciler keeps renewing the masterlease under the old
|
||||
# address, so `default/kubernetes` hands out a dead endpoint until the static
|
||||
# pod restarts (#635). After any node re-IPs, check that
|
||||
# `kubectl get endpoints kubernetes -n default` agrees with
|
||||
# `kubectl get nodes -o wide`, and restart the offending node
|
||||
# (`talosctl -n <node> reboot`) if it does not.
|
||||
#
|
||||
# Pinning mgmt0 with a static AddressConfig alongside this document would remove
|
||||
# the trigger, but it needs an address outside the UDM's DHCP pool — which is
|
||||
# not declared in this repo — and §6 of RUNBOOK-baremetal-conversion.md applies:
|
||||
# prove it with `talosctl apply-config --mode=try` before making it permanent.
|
||||
# `kubectl get nodes -o wide` and reboot the node if it does not.
|
||||
apiVersion: v1alpha1
|
||||
kind: DHCPv4Config
|
||||
name: mgmt0
|
||||
@@ -171,10 +97,8 @@ kind: Layer2VIPConfig
|
||||
name: 10.2.40.60
|
||||
link: mgmt0
|
||||
---
|
||||
# Named ens19 on purpose. The four v30iot macvlan NADs hard-code
|
||||
# `master: ens19`, and k8s-2/k8s-3 still present the VLAN as a virtio NIC of
|
||||
# that name. VLANConfig.name is free-form, so keeping it lets one set of NADs
|
||||
# serve both VM and bare-metal nodes for as long as the cluster is mixed.
|
||||
# Named ens19 on purpose: the v30iot macvlan NADs hard-code `master: ens19`.
|
||||
# VLANConfig.name is free-form, so renaming this link means editing them too.
|
||||
apiVersion: v1alpha1
|
||||
kind: VLANConfig
|
||||
name: ens19
|
||||
@@ -182,19 +106,13 @@ vlanID: 30
|
||||
parent: enp2s0f0np0
|
||||
up: true
|
||||
---
|
||||
# Dedicated Ceph cluster network on X710 port 1, VLAN 49 / 10.99.0.0/29,
|
||||
# untagged on the wire. Carries OSD<->OSD replication and nothing else.
|
||||
# Dedicated Ceph cluster network on X710 port 1, VLAN 49, untagged on the
|
||||
# wire. OSD-to-OSD replication and nothing else.
|
||||
#
|
||||
# The node deliberately holds NO address here. Rook attaches this link to the
|
||||
# OSD pods through a Multus macvlan (infrastructure/rook-ceph-cluster/
|
||||
# san-network-attachment.yaml), and a macvlan parent only has to be up — it
|
||||
# does not need an address of its own for its children to work.
|
||||
#
|
||||
# It used to carry 10.99.0.1/29, because reaching this link required
|
||||
# CephCluster.spec.network.provider: host, and that put every Ceph daemon on
|
||||
# host networking, mons included. Nothing needs it now, and by the rule this
|
||||
# segment is built on — Ceph-internal traffic and nothing else — the node has
|
||||
# no business on it either.
|
||||
# The node holds NO address here on purpose. Rook attaches the link to the OSD
|
||||
# pods through a Multus macvlan, and a macvlan parent only needs to be up.
|
||||
# Giving the node an address would put it on a segment reserved for
|
||||
# Ceph-internal traffic.
|
||||
apiVersion: v1alpha1
|
||||
kind: LinkConfig
|
||||
name: enp2s0f1np1
|
||||
|
||||
@@ -9,20 +9,14 @@ machine:
|
||||
# macvlan parent for the v30iot pod attachments; host addresses on that L2
|
||||
# make node replies bypass the UDM and break every UDM-DNATed flow.
|
||||
net.ipv6.conf.ens19.disable_ipv6: "1"
|
||||
# Same argument for the physical ports: enp2s0f0np0 is only the trunk
|
||||
# parent for mgmt0 and ens19, enp87s0 is unused, and both sit on the
|
||||
# untagged fallback network where nothing is supposed to run. Both were
|
||||
# taking an fd00::/64 and an IPv6 default route from its router
|
||||
# advertisements, leaving four equal-priority default routes per node of
|
||||
# which only mgmt0 has a globally routable source. See k8s-1.yaml for the
|
||||
# full reasoning and how it was verified.
|
||||
# Same for the physical ports on the untagged fallback network: left
|
||||
# enabled they take an IPv6 default route from its RAs and cost the
|
||||
# cluster IPv6 egress.
|
||||
net.ipv6.conf.enp2s0f0np0.disable_ipv6: "1"
|
||||
net.ipv6.conf.enp87s0.disable_ipv6: "1"
|
||||
install:
|
||||
# Three NVMe drives are visible on bare metal. Select the installer target
|
||||
# by serial so it cannot land on the OSD drive or on EPHEMERAL, whatever
|
||||
# order the kernel enumerates them in — Talos and Proxmox do not agree on
|
||||
# that order.
|
||||
# Select the installer target by serial so it cannot land on the OSD drive
|
||||
# or on EPHEMERAL, whatever order the kernel enumerates them in.
|
||||
diskSelector:
|
||||
serial: 50026B7382DA5F6E
|
||||
certSANs:
|
||||
@@ -32,17 +26,15 @@ machine:
|
||||
cluster:
|
||||
allowSchedulingOnControlPlanes: true
|
||||
---
|
||||
# EPHEMERAL (/var) goes on the retired Crucial P3, not the system disk. That
|
||||
# drive is at 81% of its rated endurance and is finished as an OSD, but
|
||||
# everything EPHEMERAL holds is reconstructible, so it is consumable in this
|
||||
# role. It also gives /var ~931 GB instead of the 64 GB the VM-era zvol had.
|
||||
# EPHEMERAL (/var) goes on the retired Ceph OSD drive, deliberately: its
|
||||
# contents are reconstructible, so a worn disk is consumable in this role.
|
||||
#
|
||||
# ONLY TAKES EFFECT ON FIRST PROVISION. Talos applies volume configuration
|
||||
# solely when the volume does not yet exist; applied later it is silently
|
||||
# ignored and relocating needs a disk wipe and a boot from install media.
|
||||
#
|
||||
# Matched by serial deliberately: a broad selector such as
|
||||
# `disk.transport == 'nvme' && !system_disk` would match the PM983 as well.
|
||||
# `disk.transport == 'nvme' && !system_disk` still matches the OSD drive.
|
||||
apiVersion: v1alpha1
|
||||
kind: VolumeConfig
|
||||
name: EPHEMERAL
|
||||
@@ -85,9 +77,8 @@ kind: Layer2VIPConfig
|
||||
name: 10.2.40.60
|
||||
link: mgmt0
|
||||
---
|
||||
# Named ens19 on purpose: the four v30iot macvlan NADs hard-code
|
||||
# `master: ens19`, and k8s-3 still presents the VLAN as a virtio NIC of that
|
||||
# name. VLANConfig.name is free-form, so one set of NADs serves both.
|
||||
# Named ens19 on purpose: the v30iot macvlan NADs hard-code `master: ens19`.
|
||||
# VLANConfig.name is free-form, so renaming this link means editing them too.
|
||||
apiVersion: v1alpha1
|
||||
kind: VLANConfig
|
||||
name: ens19
|
||||
@@ -95,18 +86,11 @@ vlanID: 30
|
||||
parent: enp2s0f0np0
|
||||
up: true
|
||||
---
|
||||
# Dedicated Ceph cluster network on X710 port 1, VLAN 49 / 10.99.0.0/29,
|
||||
# untagged on the wire. OSD<->OSD replication only. Not routed off its own
|
||||
# segment and deliberately has no default route.
|
||||
# Dedicated Ceph cluster network on X710 port 1, VLAN 49, untagged on the
|
||||
# wire. OSD-to-OSD replication only, not routed off its own segment.
|
||||
#
|
||||
# The node deliberately holds NO address here. Rook attaches this link to the
|
||||
# OSD pods through a Multus macvlan (infrastructure/rook-ceph-cluster/
|
||||
# san-network-attachment.yaml), and a macvlan parent only has to be up — it
|
||||
# does not need an address of its own for its children to work.
|
||||
#
|
||||
# It used to carry 10.99.0.2/29, because reaching this link required
|
||||
# CephCluster.spec.network.provider: host, which put every Ceph daemon on
|
||||
# host networking, mons included. Nothing needs it now.
|
||||
# The node holds NO address here on purpose. Rook attaches the link to the OSD
|
||||
# pods through a Multus macvlan, and a macvlan parent only needs to be up.
|
||||
apiVersion: v1alpha1
|
||||
kind: LinkConfig
|
||||
name: enp2s0f1np1
|
||||
|
||||
@@ -1,27 +1,18 @@
|
||||
# k8s-3 runs Talos directly on neptune's MS-01 hardware. Structurally identical
|
||||
# to k8s-1.yaml — see that file for the reasoning behind each choice; only the
|
||||
# serials and addresses differ. neptune was the last node converted, so all
|
||||
# three control-plane nodes now share this layout and no Proxmox remains.
|
||||
# Structurally identical to k8s-1.yaml — see that file for the reasoning
|
||||
# behind each choice; only the serials and addresses differ.
|
||||
machine:
|
||||
network:
|
||||
hostname: k8s-3
|
||||
sysctls:
|
||||
# v30iot NIC carries no host addressing — see k8s-1.yaml.
|
||||
net.ipv6.conf.ens19.disable_ipv6: "1"
|
||||
# Same argument for the physical ports: enp2s0f0np0 is only the trunk
|
||||
# parent for mgmt0 and ens19, enp87s0 is unused, and both sit on the
|
||||
# untagged fallback network where nothing is supposed to run. Both were
|
||||
# taking an fd00::/64 and an IPv6 default route from its router
|
||||
# advertisements, leaving four equal-priority default routes per node of
|
||||
# which only mgmt0 has a globally routable source. See k8s-1.yaml for the
|
||||
# full reasoning and how it was verified.
|
||||
# Same for the physical ports on the untagged fallback network — see
|
||||
# k8s-1.yaml.
|
||||
net.ipv6.conf.enp2s0f0np0.disable_ipv6: "1"
|
||||
net.ipv6.conf.enp87s0.disable_ipv6: "1"
|
||||
install:
|
||||
# Three NVMe drives are visible on bare metal. Select the installer target
|
||||
# by serial so it cannot land on the OSD drive or on EPHEMERAL, whatever
|
||||
# order the kernel enumerates them in — Talos and Proxmox do not agree on
|
||||
# that order.
|
||||
# Select the installer target by serial so it cannot land on the OSD drive
|
||||
# or on EPHEMERAL, whatever order the kernel enumerates them in.
|
||||
diskSelector:
|
||||
serial: 50026B7382B48B90
|
||||
certSANs:
|
||||
@@ -31,10 +22,8 @@ machine:
|
||||
cluster:
|
||||
allowSchedulingOnControlPlanes: true
|
||||
---
|
||||
# EPHEMERAL (/var) goes on the retired Crucial P3, not the system disk. That
|
||||
# drive is at 83% of its rated endurance and is finished as an OSD, but
|
||||
# everything EPHEMERAL holds is reconstructible, so it is consumable in this
|
||||
# role. Available spare is still 100% and it reports no media errors.
|
||||
# EPHEMERAL (/var) goes on the retired Ceph OSD drive, deliberately: its
|
||||
# contents are reconstructible, so a worn disk is consumable in this role.
|
||||
#
|
||||
# ONLY TAKES EFFECT ON FIRST PROVISION. Talos applies volume configuration
|
||||
# solely when the volume does not yet exist; applied later it is silently
|
||||
@@ -81,15 +70,8 @@ vlanID: 30
|
||||
parent: enp2s0f0np0
|
||||
up: true
|
||||
---
|
||||
# Dedicated Ceph cluster network — see k8s-1.yaml.
|
||||
# The node deliberately holds NO address here. Rook attaches this link to the
|
||||
# OSD pods through a Multus macvlan (infrastructure/rook-ceph-cluster/
|
||||
# san-network-attachment.yaml), and a macvlan parent only has to be up — it
|
||||
# does not need an address of its own for its children to work.
|
||||
#
|
||||
# It used to carry 10.99.0.3/29, because reaching this link required
|
||||
# CephCluster.spec.network.provider: host, which put every Ceph daemon on
|
||||
# host networking, mons included. Nothing needs it now.
|
||||
# Dedicated Ceph cluster network — see k8s-1.yaml. The node holds NO address
|
||||
# here on purpose; the link is only a macvlan parent for the OSD pods.
|
||||
apiVersion: v1alpha1
|
||||
kind: LinkConfig
|
||||
name: enp2s0f1np1
|
||||
|
||||
Reference in New Issue
Block a user