Compare commits
4
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
dd727769f6 | ||
|
|
562b2dcc32 | ||
|
|
ef6111bda3 | ||
|
|
269ccef5d2 |
@@ -131,18 +131,18 @@ steps:
|
||||
- |
|
||||
# Ensure Talos image exists on all Proxmox nodes via API.
|
||||
#
|
||||
# jupiter is NOT in this list: it runs Talos on bare metal and is no
|
||||
# longer a Proxmox node, so the storage API returns nothing for it and
|
||||
# the download-wait loop below would burn its full 60x10s before
|
||||
# giving up. saturn and neptune still host k8s-2/k8s-3 as VMs and
|
||||
# still need the nocloud image; bare-metal nodes boot the metal-amd64
|
||||
# image from their own installed disk instead.
|
||||
# Only nodes still running Proxmox belong in this list. jupiter and
|
||||
# saturn run Talos on bare metal now, so the storage API returns
|
||||
# nothing for them and the download-wait loop below would burn its full
|
||||
# 60x10s per node before giving up. Bare-metal nodes boot the
|
||||
# metal-amd64 image from their own installed disk instead, and the
|
||||
# whole step goes away once neptune is converted too.
|
||||
TALOS_VERSION="v1.12.6"
|
||||
TALOS_SCHEMATIC="88e3752db032a08082972bd0adf16a80849e3870c3f95cf5b669936c5699e5ad"
|
||||
TALOS_URL="https://factory.talos.dev/image/$TALOS_SCHEMATIC/$TALOS_VERSION/nocloud-amd64.raw"
|
||||
TALOS_FILENAME="talos-$TALOS_VERSION-$(echo "$TALOS_SCHEMATIC" | cut -c1-12)-nocloud-amd64.raw"
|
||||
API_TOKEN="$TF_VAR_proxmox_user!$TF_VAR_proxmox_token_id=$TF_VAR_proxmox_token_secret"
|
||||
for node in saturn neptune; do
|
||||
for node in neptune; do
|
||||
EXISTS=$(curl -sk "$TF_VAR_proxmox_api_url/api2/json/nodes/$node/storage/local/content" \
|
||||
-H "Authorization: PVEAPIToken=$API_TOKEN" 2>/dev/null | grep -c "$TALOS_FILENAME" || true)
|
||||
if [ "$EXISTS" -gt 0 ]; then
|
||||
|
||||
@@ -1,31 +1,99 @@
|
||||
# k8s-2 runs Talos directly on saturn's MS-01 hardware. Structurally identical
|
||||
# to k8s-1.yaml — see that file for the reasoning behind each choice; only the
|
||||
# serials, MACs and addresses differ. k8s-3 is still a Proxmox VM and keeps the
|
||||
# VM-era layout until neptune is converted.
|
||||
machine:
|
||||
network:
|
||||
hostname: k8s-2
|
||||
# Shared control-plane VIP — see k8s-1.yaml for the busPath rationale.
|
||||
interfaces:
|
||||
- deviceSelector:
|
||||
busPath: "0000:06:12.0"
|
||||
dhcp: true
|
||||
vip:
|
||||
ip: 10.2.40.60
|
||||
# v30iot: link only, no host addressing — see k8s-1.yaml.
|
||||
- deviceSelector:
|
||||
busPath: "0000:06:13.0"
|
||||
dhcp: false
|
||||
# See k8s-1.yaml for context on the dedicated Ceph cluster network.
|
||||
- deviceSelector:
|
||||
driver: i40e
|
||||
addresses:
|
||||
- 10.99.0.2/29
|
||||
mtu: 9000
|
||||
sysctls:
|
||||
# v30iot NIC carries no host addressing — see k8s-1.yaml.
|
||||
# ens19 (VLAN 30, v30iot) is LINK ONLY — no host addressing. It is the
|
||||
# macvlan parent for the v30iot pod attachments; host addresses on that L2
|
||||
# make node replies bypass the UDM and break every UDM-DNATed flow.
|
||||
net.ipv6.conf.ens19.disable_ipv6: "1"
|
||||
install:
|
||||
disk: /dev/vda
|
||||
# Three NVMe drives are visible on bare metal. Select the installer target
|
||||
# by serial so it cannot land on the OSD drive or on EPHEMERAL, whatever
|
||||
# order the kernel enumerates them in — Talos and Proxmox do not agree on
|
||||
# that order.
|
||||
diskSelector:
|
||||
serial: 50026B7382DA5F6E
|
||||
certSANs:
|
||||
- k8s-2.srv.renner.uno
|
||||
nodeLabels:
|
||||
topology.kubernetes.io/zone: saturn
|
||||
cluster:
|
||||
allowSchedulingOnControlPlanes: true
|
||||
---
|
||||
# EPHEMERAL (/var) goes on the retired Crucial P3, not the system disk. That
|
||||
# drive is at 81% of its rated endurance and is finished as an OSD, but
|
||||
# everything EPHEMERAL holds is reconstructible, so it is consumable in this
|
||||
# role. It also gives /var ~931 GB instead of the 64 GB the VM-era zvol had.
|
||||
#
|
||||
# ONLY TAKES EFFECT ON FIRST PROVISION. Talos applies volume configuration
|
||||
# solely when the volume does not yet exist; applied later it is silently
|
||||
# ignored and relocating needs a disk wipe and a boot from install media.
|
||||
#
|
||||
# Matched by serial deliberately: a broad selector such as
|
||||
# `disk.transport == 'nvme' && !system_disk` would match the PM983 as well.
|
||||
apiVersion: v1alpha1
|
||||
kind: VolumeConfig
|
||||
name: EPHEMERAL
|
||||
provisioning:
|
||||
diskSelector:
|
||||
match: disk.serial == '2332E86B74BC'
|
||||
minSize: 50GiB
|
||||
grow: true
|
||||
---
|
||||
# X710 port 0 carries the switch trunk and takes no addressing itself — the
|
||||
# port's untagged VLAN is not one this node belongs on. Management and v30iot
|
||||
# both ride it as tagged VLANs.
|
||||
#
|
||||
# Addressed by predictable kernel name, not by LinkAliasConfig. Aliases are
|
||||
# valid VLAN parents, but a selector matching zero links is skipped silently
|
||||
# and Talos disables default DHCP machine-wide as soon as any new-style network
|
||||
# document exists — so a silent miss leaves the node with no address on any
|
||||
# interface. See RUNBOOK-baremetal-conversion.md.
|
||||
apiVersion: v1alpha1
|
||||
kind: LinkConfig
|
||||
name: enp2s0f0np0
|
||||
up: true
|
||||
---
|
||||
# Management, VLAN 40. Node IP, etcd, the API and Ceph's public network.
|
||||
apiVersion: v1alpha1
|
||||
kind: VLANConfig
|
||||
name: mgmt0
|
||||
vlanID: 40
|
||||
parent: enp2s0f0np0
|
||||
up: true
|
||||
---
|
||||
apiVersion: v1alpha1
|
||||
kind: DHCPv4Config
|
||||
name: mgmt0
|
||||
---
|
||||
# Shared control-plane VIP, etcd-elected. Exposed via UniFi DNS as
|
||||
# k8s.renner.uno.
|
||||
apiVersion: v1alpha1
|
||||
kind: Layer2VIPConfig
|
||||
name: 10.2.40.60
|
||||
link: mgmt0
|
||||
---
|
||||
# Named ens19 on purpose: the four v30iot macvlan NADs hard-code
|
||||
# `master: ens19`, and k8s-3 still presents the VLAN as a virtio NIC of that
|
||||
# name. VLANConfig.name is free-form, so one set of NADs serves both.
|
||||
apiVersion: v1alpha1
|
||||
kind: VLANConfig
|
||||
name: ens19
|
||||
vlanID: 30
|
||||
parent: enp2s0f0np0
|
||||
up: true
|
||||
---
|
||||
# Dedicated Ceph cluster network on X710 port 1, VLAN 49 / 10.99.0.0/29,
|
||||
# untagged on the wire. OSD<->OSD replication only. Not routed off its own
|
||||
# segment and deliberately has no default route.
|
||||
apiVersion: v1alpha1
|
||||
kind: LinkConfig
|
||||
name: enp2s0f1np1
|
||||
up: true
|
||||
mtu: 9000
|
||||
addresses:
|
||||
- address: 10.99.0.2/29
|
||||
|
||||
@@ -1,50 +0,0 @@
|
||||
module "k8s_2" {
|
||||
source = "./modules/proxmox_vm"
|
||||
|
||||
name = "k8s-2.srv.renner.uno"
|
||||
target_node = "saturn"
|
||||
vmid = 151
|
||||
memory = 49152
|
||||
cores = 8
|
||||
|
||||
template_name = null
|
||||
cloud_init = false
|
||||
full_clone = false
|
||||
bios = "ovmf"
|
||||
machine = "q35"
|
||||
boot = "order=virtio0"
|
||||
agent_enabled = true
|
||||
|
||||
# Second NIC on v30iot so Cilium L2 announce can win ARP on the IoT
|
||||
# VLAN from this node (previously only k8s-edge had v30iot, so any
|
||||
# IoT-pool LB lease landing here was unreachable from devices).
|
||||
extra_network_bridges = ["v30iot"]
|
||||
|
||||
disks = [
|
||||
{
|
||||
storage = "local-zfs"
|
||||
size = "64G"
|
||||
format = "raw"
|
||||
slot = "virtio0"
|
||||
file_id = local.talos_image
|
||||
}
|
||||
]
|
||||
|
||||
hostpci = [
|
||||
{
|
||||
mapping = "osd-ssd"
|
||||
},
|
||||
{
|
||||
mapping = "x710-port1"
|
||||
}
|
||||
]
|
||||
|
||||
tags = "kubernetes;talos;controlplane"
|
||||
ssh_public_keys = local.common_ssh_keys
|
||||
|
||||
vm_state = "running"
|
||||
|
||||
# Rebuild pipeline issues `-replace`; skip graceful shutdown so a
|
||||
# non-responsive Talos guest can't hang the destroy for 30 minutes.
|
||||
stop_on_destroy = true
|
||||
}
|
||||
Reference in New Issue
Block a user