fix: [alert] NetbirdRelayMemberDown (monitoring) (#603)

Reviewed-on: k.renner/homelab#622

.woodpecker/deploy-infrastructure.yml

@@ -128,42 +128,6 @@ steps:
       - printf "Host *\n\tStrictHostKeyChecking no\n" > ~/.ssh/config
       - echo "$VAULT_PASS" > /tmp/.vault_pass && chmod 600 /tmp/.vault_pass
       - export ANSIBLE_VAULT_PASSWORD_FILE=/tmp/.vault_pass
-      - |
-        # Ensure Talos image exists on all Proxmox nodes via API.
-        #
-        # jupiter is NOT in this list: it runs Talos on bare metal and is no
-        # longer a Proxmox node, so the storage API returns nothing for it and
-        # the download-wait loop below would burn its full 60x10s before
-        # giving up. saturn and neptune still host k8s-2/k8s-3 as VMs and
-        # still need the nocloud image; bare-metal nodes boot the metal-amd64
-        # image from their own installed disk instead.
-        TALOS_VERSION="v1.12.6"
-        TALOS_SCHEMATIC="88e3752db032a08082972bd0adf16a80849e3870c3f95cf5b669936c5699e5ad"
-        TALOS_URL="https://factory.talos.dev/image/$TALOS_SCHEMATIC/$TALOS_VERSION/nocloud-amd64.raw"
-        TALOS_FILENAME="talos-$TALOS_VERSION-$(echo "$TALOS_SCHEMATIC" | cut -c1-12)-nocloud-amd64.raw"
-        API_TOKEN="$TF_VAR_proxmox_user!$TF_VAR_proxmox_token_id=$TF_VAR_proxmox_token_secret"
-        for node in saturn neptune; do
-          EXISTS=$(curl -sk "$TF_VAR_proxmox_api_url/api2/json/nodes/$node/storage/local/content" \
-            -H "Authorization: PVEAPIToken=$API_TOKEN" 2>/dev/null | grep -c "$TALOS_FILENAME" || true)
-          if [ "$EXISTS" -gt 0 ]; then
-            echo "Talos image already exists on $node"
-            continue
-          fi
-          echo "Downloading Talos image to $node..."
-          curl -sk "$TF_VAR_proxmox_api_url/api2/json/nodes/$node/storage/local/download-url" \
-            -H "Authorization: PVEAPIToken=$API_TOKEN" \
-            --data-urlencode "url=$TALOS_URL" \
-            --data-urlencode "content=import" \
-            --data-urlencode "filename=$TALOS_FILENAME" \
-            --max-time 600
-          echo "Waiting for download on $node..."
-          for i in $(seq 1 60); do
-            sleep 10
-            EXISTS=$(curl -sk "$TF_VAR_proxmox_api_url/api2/json/nodes/$node/storage/local/content" \
-              -H "Authorization: PVEAPIToken=$API_TOKEN" 2>/dev/null | grep -c "$TALOS_FILENAME" || true)
-            if [ "$EXISTS" -gt 0 ]; then echo "Download complete on $node"; break; fi
-          done
-        done
       - tofu init -upgrade -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY"
       - |
         if [ -n "$$TOFU_STATE_RM" ]; then

ansible/roles/gatus/templates/config.yaml.j2

@@ -253,22 +253,6 @@ endpoints:
 {% endif %}
 
   # Infrastructure
-  - name: Proxmox DNS (VIP)
-    group: infrastructure
-    url: 10.2.40.53
-    interval: 30s
-    dns:
-      query-name: k8s-1.srv.renner.uno
-      query-type: A
-    conditions:
-      - "[DNS_RCODE] == NOERROR"
-      - "[RESPONSE_TIME] < 500"
-    alerts:
-      - type: email
-{% if gatus_webhook_url %}
-      - type: custom
-{% endif %}
-
   - name: PostgreSQL HA (VIP)
     group: infrastructure
     url: tcp://10.2.40.135:5432

dns-server/config.yaml.example

@@ -1,31 +0,0 @@
-# Proxmox DNS Server Configuration
-
-# Proxmox API connection
-proxmox:
-  host: "pve.renner.uno"
-  user: "root@pam"
-
-  # Token-based authentication (recommended)
-  token_name: "dns"
-  token_value: "your-token-secret-here"
-
-  # OR password-based authentication
-  # password: "your-password-here"
-
-  verify_ssl: false
-
-# DNS server settings
-dns_host: "0.0.0.0"  # Listen on all interfaces
-dns_port: 53         # Standard DNS port (requires root or CAP_NET_BIND_SERVICE)
-
-# DNS suffix for VM/LXC hostnames
-dns_suffix: ".renner.uno"
-
-# TTL for DNS responses (in seconds)
-dns_ttl: 300
-
-# Cache settings
-cache_ttl: 60  # How often to refresh the cache (in seconds)
-
-# Upstream DNS server for non-Proxmox queries
-upstream_dns: "10.2.40.1"

dns-server/go.mod

@@ -1,15 +0,0 @@
-module proxmox-dns
-
-go 1.21
-
-require (
-	github.com/miekg/dns v1.1.58
-	gopkg.in/yaml.v3 v3.0.1
-)
-
-require (
-	golang.org/x/mod v0.14.0 // indirect
-	golang.org/x/net v0.20.0 // indirect
-	golang.org/x/sys v0.16.0 // indirect
-	golang.org/x/tools v0.17.0 // indirect
-)

dns-server/go.sum

@@ -1,16 +0,0 @@
-github.com/miekg/dns v1.1.58 h1:ca2Hdkz+cDg/7eNF6V56jjzuZ4aCAE+DbVkILdQWG/4=
-github.com/miekg/dns v1.1.58/go.mod h1:Ypv+3b/KadlvW9vJfXOTf300O4UqaHFzFCuHz+rPkBY=
-golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0=
-golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
-golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo=
-golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY=
-golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ=
-golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
-golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU=
-golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/tools v0.17.0 h1:FvmRgNOcs3kOa+T20R1uhfP9F6HgG2mfxDv1vrx1Htc=
-golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
-gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

dns-server/main.go

@@ -1,565 +0,0 @@
-package main
-
-import (
-	"crypto/tls"
-	"encoding/json"
-	"flag"
-	"fmt"
-	"io"
-	"log"
-	"net"
-	"net/http"
-	"os"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/miekg/dns"
-	"gopkg.in/yaml.v3"
-)
-
-type Config struct {
-	Proxmox struct {
-		Host       string `yaml:"host"`
-		User       string `yaml:"user"`
-		TokenName  string `yaml:"token_name"`
-		TokenValue string `yaml:"token_value"`
-		VerifySSL  bool   `yaml:"verify_ssl"`
-	} `yaml:"proxmox"`
-	DNSHost     string `yaml:"dns_host"`
-	DNSPort     int    `yaml:"dns_port"`
-	DNSSuffix   string `yaml:"dns_suffix"`
-	DNSTTL      int    `yaml:"dns_ttl"`
-	CacheTTL    int    `yaml:"cache_ttl"`
-	UpstreamDNS string `yaml:"upstream_dns"`
-}
-
-type DNSRecord struct {
-	IPv4      string
-	IPv6      string
-	Timestamp time.Time
-}
-
-type ProxmoxDNS struct {
-	config     Config
-	cache      map[string]*DNSRecord
-	cacheMux   sync.RWMutex
-	client     *http.Client
-	stopChan   chan struct{}
-	nodes      []string
-	nodesMux   sync.RWMutex
-}
-
-func NewProxmoxDNS(configPath string) (*ProxmoxDNS, error) {
-	data, err := os.ReadFile(configPath)
-	if err != nil {
-		return nil, fmt.Errorf("failed to read config: %w", err)
-	}
-
-	var config Config
-	if err := yaml.Unmarshal(data, &config); err != nil {
-		return nil, fmt.Errorf("failed to parse config: %w", err)
-	}
-
-	// Set defaults
-	if config.DNSPort == 0 {
-		config.DNSPort = 53
-	}
-	if config.CacheTTL == 0 {
-		config.CacheTTL = 60
-	}
-	if config.DNSTTL == 0 {
-		config.DNSTTL = 300
-	}
-	if config.UpstreamDNS == "" {
-		config.UpstreamDNS = "10.2.40.1:53"
-	}
-	if !strings.Contains(config.UpstreamDNS, ":") {
-		config.UpstreamDNS += ":53"
-	}
-
-	return &ProxmoxDNS{
-		config: config,
-		cache:  make(map[string]*DNSRecord),
-		nodes:  []string{},
-		client: &http.Client{
-			Timeout: 10 * time.Second,
-			Transport: &http.Transport{
-				TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
-			},
-		},
-		stopChan: make(chan struct{}),
-	}, nil
-}
-
-// isValidIPv6ForDNS checks if an IPv6 address is suitable for DNS records
-func isValidIPv6ForDNS(ip net.IP) bool {
-	if ip == nil || ip.To16() == nil {
-		return false
-	}
-
-	// Reject loopback
-	if ip.IsLoopback() {
-		return false
-	}
-
-	// Reject link-local (fe80::/10) - THIS IS THE KEY FIX
-	if ip.IsLinkLocalUnicast() {
-		return false
-	}
-
-	// Reject multicast
-	if ip.IsMulticast() {
-		return false
-	}
-
-	// Reject unspecified
-	if ip.IsUnspecified() {
-		return false
-	}
-
-	// Accept global unicast and ULA (fc00::/7)
-	return true
-}
-
-func (p *ProxmoxDNS) queryProxmoxAPI(endpoint string) ([]byte, error) {
-	// Remove port from host if present
-	host := p.config.Proxmox.Host
-	if !strings.Contains(host, ":") {
-		host += ":8006"
-	}
-
-	url := fmt.Sprintf("https://%s/api2/json/%s", host, endpoint)
-	req, err := http.NewRequest("GET", url, nil)
-	if err != nil {
-		return nil, err
-	}
-
-	auth := fmt.Sprintf("PVEAPIToken=%s!%s=%s",
-		p.config.Proxmox.User,
-		p.config.Proxmox.TokenName,
-		p.config.Proxmox.TokenValue)
-	req.Header.Set("Authorization", auth)
-
-	resp, err := p.client.Do(req)
-	if err != nil {
-		return nil, err
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode != 200 {
-		body, _ := io.ReadAll(resp.Body)
-		return nil, fmt.Errorf("API returned status %d: %s", resp.StatusCode, string(body))
-	}
-
-	return io.ReadAll(resp.Body)
-}
-
-func (p *ProxmoxDNS) discoverNodes() error {
-	data, err := p.queryProxmoxAPI("nodes")
-	if err != nil {
-		return fmt.Errorf("failed to discover nodes: %w", err)
-	}
-
-	var response struct {
-		Data []struct {
-			Node   string `json:"node"`
-			Status string `json:"status"`
-		} `json:"data"`
-	}
-
-	if err := json.Unmarshal(data, &response); err != nil {
-		return fmt.Errorf("failed to parse nodes: %w", err)
-	}
-
-	p.nodesMux.Lock()
-	defer p.nodesMux.Unlock()
-
-	p.nodes = []string{}
-	for _, node := range response.Data {
-		if node.Status == "online" {
-			p.nodes = append(p.nodes, node.Node)
-		}
-	}
-
-	log.Printf("Discovered %d online nodes: %v", len(p.nodes), p.nodes)
-	return nil
-}
-
-func (p *ProxmoxDNS) refreshCache() {
-	log.Println("Refreshing DNS cache...")
-
-	// Discover nodes first
-	if err := p.discoverNodes(); err != nil {
-		log.Printf("Failed to discover nodes: %v", err)
-		return
-	}
-
-	newCache := make(map[string]*DNSRecord)
-
-	p.nodesMux.RLock()
-	nodes := append([]string{}, p.nodes...)
-	p.nodesMux.RUnlock()
-
-	for _, node := range nodes {
-		// Query VMs
-		p.refreshVMs(node, newCache)
-		// Query LXC containers
-		p.refreshLXC(node, newCache)
-	}
-
-	p.cacheMux.Lock()
-	p.cache = newCache
-	p.cacheMux.Unlock()
-
-	log.Printf("Cache refreshed: %d entries", len(newCache))
-}
-
-func (p *ProxmoxDNS) refreshVMs(node string, cache map[string]*DNSRecord) {
-	data, err := p.queryProxmoxAPI(fmt.Sprintf("nodes/%s/qemu", node))
-	if err != nil {
-		log.Printf("Failed to query VMs on %s: %v", node, err)
-		return
-	}
-
-	var response struct {
-		Data []struct {
-			VMID   int    `json:"vmid"`
-			Name   string `json:"name"`
-			Status string `json:"status"`
-		} `json:"data"`
-	}
-
-	if err := json.Unmarshal(data, &response); err != nil {
-		log.Printf("Failed to parse VM data: %v", err)
-		return
-	}
-
-	for _, vm := range response.Data {
-		if vm.Status != "running" || vm.Name == "" {
-			continue
-		}
-
-		// Query agent network info
-		agentData, err := p.queryProxmoxAPI(fmt.Sprintf("nodes/%s/qemu/%d/agent/network-get-interfaces", node, vm.VMID))
-		if err != nil {
-			log.Printf("VM %s (ID %d): no guest agent data", vm.Name, vm.VMID)
-			continue
-		}
-
-		var agentResp struct {
-			Data struct {
-				Result []struct {
-					Name        string `json:"name"`
-					IPAddresses []struct {
-						IPAddress     string `json:"ip-address"`
-						IPAddressType string `json:"ip-address-type"`
-					} `json:"ip-addresses"`
-				} `json:"result"`
-			} `json:"data"`
-		}
-
-		if err := json.Unmarshal(agentData, &agentResp); err != nil {
-			continue
-		}
-
-		var ipv4, ipv6 string
-		for _, iface := range agentResp.Data.Result {
-			if iface.Name == "lo" {
-				continue
-			}
-
-			for _, ipAddr := range iface.IPAddresses {
-				if ipAddr.IPAddressType == "ipv4" && ipv4 == "" {
-					if !strings.HasPrefix(ipAddr.IPAddress, "127.") {
-						ipv4 = ipAddr.IPAddress
-					}
-				} else if ipAddr.IPAddressType == "ipv6" && ipv6 == "" {
-					ip := net.ParseIP(ipAddr.IPAddress)
-					if isValidIPv6ForDNS(ip) {
-						ipv6 = ipAddr.IPAddress
-						log.Printf("  VM %s: accepting IPv6 %s", vm.Name, ipv6)
-					} else if ip != nil {
-						log.Printf("  VM %s: rejecting IPv6 %s (link-local/loopback/multicast)", vm.Name, ipAddr.IPAddress)
-					}
-				}
-			}
-
-			// Stop after finding both on first non-lo interface
-			if ipv4 != "" && ipv6 != "" {
-				break
-			}
-		}
-
-		if ipv4 != "" || ipv6 != "" {
-			hostname := strings.ToLower(vm.Name)
-			// Strip DNS suffix from hostname if present
-			hostname = strings.TrimSuffix(hostname, p.config.DNSSuffix)
-			cache[hostname] = &DNSRecord{
-				IPv4:      ipv4,
-				IPv6:      ipv6,
-				Timestamp: time.Now(),
-			}
-			log.Printf("VM: %s -> IPv4=%s IPv6=%s", hostname, ipv4, ipv6)
-		}
-	}
-}
-
-func (p *ProxmoxDNS) refreshLXC(node string, cache map[string]*DNSRecord) {
-	data, err := p.queryProxmoxAPI(fmt.Sprintf("nodes/%s/lxc", node))
-	if err != nil {
-		log.Printf("Failed to query LXC on %s: %v", node, err)
-		return
-	}
-
-	var response struct {
-		Data []struct {
-			VMID   int    `json:"vmid"`
-			Name   string `json:"name"`
-			Status string `json:"status"`
-		} `json:"data"`
-	}
-
-	if err := json.Unmarshal(data, &response); err != nil {
-		log.Printf("Failed to parse LXC data: %v", err)
-		return
-	}
-
-	for _, lxc := range response.Data {
-		if lxc.Status != "running" || lxc.Name == "" {
-			continue
-		}
-
-		// Query LXC interfaces
-		ifaceData, err := p.queryProxmoxAPI(fmt.Sprintf("nodes/%s/lxc/%d/interfaces", node, lxc.VMID))
-		if err != nil {
-			log.Printf("LXC %s (ID %d): failed to get interfaces", lxc.Name, lxc.VMID)
-			continue
-		}
-
-		var ifaceResp struct {
-			Data []struct {
-				Name   string `json:"name"`
-				Inet   string `json:"inet"`
-				Inet6  string `json:"inet6"`
-			} `json:"data"`
-		}
-
-		if err := json.Unmarshal(ifaceData, &ifaceResp); err != nil {
-			continue
-		}
-
-		var ipv4, ipv6 string
-		for _, iface := range ifaceResp.Data {
-			if iface.Name == "lo" {
-				continue
-			}
-
-			if iface.Inet != "" && ipv4 == "" {
-				// Format is "IP/CIDR"
-				parts := strings.Split(iface.Inet, "/")
-				if len(parts) > 0 && !strings.HasPrefix(parts[0], "127.") {
-					ipv4 = parts[0]
-				}
-			}
-
-			if iface.Inet6 != "" && ipv6 == "" {
-				parts := strings.Split(iface.Inet6, "/")
-				if len(parts) > 0 {
-					ip := net.ParseIP(parts[0])
-					if isValidIPv6ForDNS(ip) {
-						ipv6 = parts[0]
-						log.Printf("  LXC %s: accepting IPv6 %s", lxc.Name, ipv6)
-					} else if ip != nil {
-						log.Printf("  LXC %s: rejecting IPv6 %s (link-local/loopback/multicast)", lxc.Name, parts[0])
-					}
-				}
-			}
-
-			// Stop after finding both on first non-lo interface
-			if ipv4 != "" && ipv6 != "" {
-				break
-			}
-		}
-
-		if ipv4 != "" || ipv6 != "" {
-			hostname := strings.ToLower(lxc.Name)
-			// Strip DNS suffix from hostname if present
-			hostname = strings.TrimSuffix(hostname, p.config.DNSSuffix)
-			cache[hostname] = &DNSRecord{
-				IPv4:      ipv4,
-				IPv6:      ipv6,
-				Timestamp: time.Now(),
-			}
-			log.Printf("LXC: %s -> IPv4=%s IPv6=%s", hostname, ipv4, ipv6)
-		}
-	}
-}
-
-func (p *ProxmoxDNS) handleDNSRequest(w dns.ResponseWriter, r *dns.Msg) {
-	m := new(dns.Msg)
-	m.SetReply(r)
-	m.Authoritative = true
-
-	for _, q := range r.Question {
-		name := strings.ToLower(strings.TrimSuffix(q.Name, "."))
-		qtype := dns.TypeToString[q.Qtype]
-
-		// Check if query is for our domain suffix
-		if !strings.HasSuffix(name, strings.TrimPrefix(p.config.DNSSuffix, ".")) {
-			// Forward to upstream
-			log.Printf("Query %s (%s) - forwarding to upstream", name, qtype)
-			p.forwardToUpstream(w, r)
-			return
-		}
-
-		// Extract hostname (remove suffix)
-		hostname := strings.TrimSuffix(name, p.config.DNSSuffix)
-
-		p.cacheMux.RLock()
-		record, found := p.cache[hostname]
-		p.cacheMux.RUnlock()
-
-		if !found {
-			// NXDOMAIN
-			m.SetRcode(r, dns.RcodeNameError)
-			log.Printf("Query %s (%s) - NXDOMAIN", name, qtype)
-			w.WriteMsg(m)
-			return
-		}
-
-		switch q.Qtype {
-		case dns.TypeA:
-			if record.IPv4 != "" {
-				rr := &dns.A{
-					Hdr: dns.RR_Header{
-						Name:   q.Name,
-						Rrtype: dns.TypeA,
-						Class:  dns.ClassINET,
-						Ttl:    uint32(p.config.DNSTTL),
-					},
-					A: net.ParseIP(record.IPv4),
-				}
-				m.Answer = append(m.Answer, rr)
-				log.Printf("Query %s (A) -> %s", name, record.IPv4)
-			} else {
-				// NODATA (host exists but no A record)
-				m.Ns = append(m.Ns, p.getSOARecord())
-				log.Printf("Query %s (A) - NODATA", name)
-			}
-
-		case dns.TypeAAAA:
-			if record.IPv6 != "" {
-				// Parse and validate IPv6 again before returning
-				ip := net.ParseIP(record.IPv6)
-				if ip != nil && isValidIPv6ForDNS(ip) {
-					rr := &dns.AAAA{
-						Hdr: dns.RR_Header{
-							Name:   q.Name,
-							Rrtype: dns.TypeAAAA,
-							Class:  dns.ClassINET,
-							Ttl:    uint32(p.config.DNSTTL),
-						},
-						AAAA: ip,
-					}
-					m.Answer = append(m.Answer, rr)
-					log.Printf("Query %s (AAAA) -> %s", name, record.IPv6)
-				} else {
-					// IPv6 is invalid (link-local etc)
-					m.Ns = append(m.Ns, p.getSOARecord())
-					log.Printf("Query %s (AAAA) - NODATA (invalid IPv6 in cache)", name)
-				}
-			} else {
-				// NODATA (host exists but no AAAA record)
-				m.Ns = append(m.Ns, p.getSOARecord())
-				log.Printf("Query %s (AAAA) - NODATA", name)
-			}
-		}
-	}
-
-	w.WriteMsg(m)
-}
-
-func (p *ProxmoxDNS) getSOARecord() dns.RR {
-	domain := strings.TrimPrefix(p.config.DNSSuffix, ".")
-	soa := &dns.SOA{
-		Hdr: dns.RR_Header{
-			Name:   domain + ".",
-			Rrtype: dns.TypeSOA,
-			Class:  dns.ClassINET,
-			Ttl:    300,
-		},
-		Ns:      "ns." + domain + ".",
-		Mbox:    "admin." + domain + ".",
-		Serial:  uint32(time.Now().Unix()),
-		Refresh: 3600,
-		Retry:   600,
-		Expire:  86400,
-		Minttl:  300,
-	}
-	return soa
-}
-
-func (p *ProxmoxDNS) forwardToUpstream(w dns.ResponseWriter, r *dns.Msg) {
-	c := new(dns.Client)
-	c.Timeout = 2 * time.Second
-
-	resp, _, err := c.Exchange(r, p.config.UpstreamDNS)
-	if err != nil {
-		log.Printf("Failed to forward to upstream: %v", err)
-		m := new(dns.Msg)
-		m.SetRcode(r, dns.RcodeServerFailure)
-		w.WriteMsg(m)
-		return
-	}
-
-	w.WriteMsg(resp)
-}
-
-func (p *ProxmoxDNS) startCacheRefresher() {
-	ticker := time.NewTicker(time.Duration(p.config.CacheTTL) * time.Second)
-	defer ticker.Stop()
-
-	// Initial refresh
-	p.refreshCache()
-
-	for {
-		select {
-		case <-ticker.C:
-			p.refreshCache()
-		case <-p.stopChan:
-			return
-		}
-	}
-}
-
-func (p *ProxmoxDNS) Start() error {
-	// Start cache refresher
-	go p.startCacheRefresher()
-
-	// Start DNS server
-	dns.HandleFunc(".", p.handleDNSRequest)
-
-	addr := fmt.Sprintf("%s:%d", p.config.DNSHost, p.config.DNSPort)
-	log.Printf("Starting Proxmox DNS server on %s", addr)
-	log.Printf("Base domain: %s", p.config.DNSSuffix)
-	log.Printf("Upstream DNS: %s", p.config.UpstreamDNS)
-
-	server := &dns.Server{Addr: addr, Net: "udp"}
-	return server.ListenAndServe()
-}
-
-func main() {
-	configPath := flag.String("c", "/etc/proxmox-dns/config.yaml", "Path to configuration file")
-	flag.Parse()
-
-	server, err := NewProxmoxDNS(*configPath)
-	if err != nil {
-		log.Fatalf("Failed to create DNS server: %v", err)
-	}
-
-	log.Printf("Proxmox DNS Server starting...")
-	if err := server.Start(); err != nil {
-		log.Fatalf("Failed to start DNS server: %v", err)
-	}
-}

dns-server/proxmox-dns.service

@@ -1,27 +0,0 @@
-[Unit]
-Description=Proxmox DNS Server
-After=network.target
-Wants=network-online.target
-
-[Service]
-Type=simple
-User=root
-Group=root
-WorkingDirectory=/opt/proxmox-dns
-ExecStart=/usr/bin/python3 /opt/proxmox-dns/proxmox_dns.py -c /etc/proxmox-dns/config.yaml
-Restart=always
-RestartSec=10
-StandardOutput=journal
-StandardError=journal
-
-# Security hardening
-NoNewPrivileges=true
-PrivateTmp=true
-ProtectSystem=strict
-ProtectHome=true
-ReadWritePaths=/var/log
-AmbientCapabilities=CAP_NET_BIND_SERVICE
-
-[Install]
-WantedBy=multi-user.target
-

kubernetes/RUNBOOK-baremetal-conversion.md

@@ -0,0 +1,446 @@
+# Runbook — converting a control-plane node from Proxmox VM to bare metal
+
+Converts one MS-01 from "Proxmox host running a Talos VM" to "Talos on bare
+metal", and swaps its Ceph OSD onto a Samsung PM983 at the same time. Written
+after doing jupiter on 2026-08-25; saturn and neptune remain.
+
+Everything here that reads like an over-specific warning is something that
+actually happened. The conversion itself is straightforward — the failures were
+all in state that outlived the VM.
+
+## What the node ends up as
+
+| | |
+|---|---|
+| System disk | Kingston OM8PGP41024Q-A0, selected by serial |
+| EPHEMERAL (`/var`) | the retired Crucial P3, selected by serial |
+| Ceph OSD | Samsung PM983, selected by `/dev/disk/by-id` |
+| Management | VLAN 40 tagged on X710 port 0 |
+| v30iot | VLAN 30 on the same port, named `ens19` |
+| Ceph cluster net | X710 port 1, `10.99.0.x/29`, MTU 9000 |
+
+## Before you start
+
+- [ ] All three OSDs up, all mons in quorum, PGs `active+clean`. Do not start
+      from a degraded cluster; the window itself removes one of three.
+- [ ] Cluster fits on two nodes. Check `kubectl describe node` allocation on the
+      two survivors. Take immich down (`kubernetes/apps/immich`, both
+      Deployments to `replicas: 0`) — it frees ~1000m against a shortfall of
+      roughly 200m and takes its image pulls off the remaining nodes.
+- [ ] `cephfs-metadata` to size 2 / min_size 1 for the duration. At size 3 on
+      two hosts every PG is undersized with min_size 2, so one further OSD loss
+      blocks IO on every CephFS mount cluster-wide.
+- [ ] Talos metal ISO on a USB stick. Schematic
+      `a21cec8f35ae32ab5e9a452e7b13500ca8776053930b8e182b9cb23eed186b64`
+      (`net.ifnames=1`, intel-ucode, thunderbolt; no qemu-guest-agent).
+      Verify by comparing partition offsets and sizes against the ISO with
+      `fdisk -l` — blkid labels lie after a rewrite.
+- [ ] Physical access. Secure Boot must be turned off (see below), and that is
+      the one step nothing can do remotely.
+
+## 1. Evacuate the OSD
+
+```sh
+ceph osd out <id>
+```
+
+Wait until no PGs are `backfilling` or `backfill_wait`. Do **not** wait for
+`ceph osd safe-to-destroy` — pools at size 3 keep the OSD in their acting set
+because three copies cannot be placed on two hosts, so it never returns clean.
+
+Then remove it. The node must already be out of `cephcluster.yaml`'s
+`storage.nodes` list, or the operator rebuilds the OSD within one reconcile and
+starts backfilling onto the drive you are about to pull:
+
+```sh
+kubectl -n rook-ceph scale deployment rook-ceph-osd-<id> --replicas=0
+ceph osd purge <id> --yes-i-really-mean-it
+kubectl -n rook-ceph delete deployment rook-ceph-osd-<id>
+```
+
+## 2. Drain, and leave the cluster
+
+```sh
+kubectl drain <node> --ignore-daemonsets --delete-emptydir-data --timeout=300s
+```
+
+Single-replica workloads with `minAvailable: 1` PDBs will refuse eviction — that
+is deliberate, they exist to surface the pod rather than let it be yanked. On
+this cluster that is `influxdb-influxdb2-0`, `woodpecker-server` and the
+CNPG primary. Move each one deliberately with `kubectl delete pod`; they
+reschedule onto the survivors and their RBD volumes follow.
+
+```sh
+talosctl -n <node> reset --graceful --wipe-mode system-disk
+kubectl delete node <node>
+```
+
+`--wipe-mode system-disk` and not the default `all`: the OSD drive is being
+pulled anyway and wiping a terabyte of it is wasted time. `--graceful` leaves
+etcd cleanly. Delete the Node object — it carries the `unschedulable` flag from
+the drain, which the rebuilt node would otherwise inherit.
+
+## 2a. Destroy the VM in OpenTofu — while the host is still a PVE node
+
+Remove `opentofu/vm_15N_k8s_N.tf`, then apply. This **must** happen before the
+host leaves the Proxmox cluster: afterwards the provider cannot reach the node,
+the plan errors, and the only escape is `tofu state rm`, which is off the table.
+
+Point `proxmox_api_url` at a host that will *survive* the conversion, not the one
+being converted.
+
+Manual triggers are currently broken by the Woodpecker v3.18.0 regression
+(issue #582). Until that is resolved, run it locally:
+
+```sh
+AWS_ACCESS_KEY_ID=… AWS_SECRET_ACCESS_KEY=… AWS_REGION=fsn1 \
+  tofu -chdir=opentofu init -backend-config=<file>
+… plan -target=module.k8s_N -var-file=<file> -out=<plan>
+… apply <plan>
+```
+
+The `AWS_*` environment variables are required *in addition to*
+`-backend-config`, or the S3 backend fails `HeadObject` with 403 even with
+credentials that can read the object fine via the aws CLI.
+
+## 2b. Proxmox quorum — matters from the second conversion onward
+
+Two nodes remain after jupiter, with `expected_votes: 2` and quorum 2. Removing
+one leaves the last node inquorate and `/etc/pve` read-only, which blocks the
+final VM destroy. On the surviving node:
+
+```sh
+pvecm expected 1
+```
+
+This is the official recovery. Proxmox documents it in the storage replication
+chapter (`/usr/share/pve-docs/chapter-pvesr.html`), verbatim:
+
+> If you have no quorum, we strongly advise to fix this first and make the node
+> operable again. Only if this is not possible at the moment, you may use the
+> following command to enforce quorum on the current node: `# pvecm expected 1`
+
+The caveat there is about restoring the missing node instead. Here the missing
+node is deliberately gone for good, so enforcing quorum is the correct branch
+rather than a workaround.
+
+Run it only once the node is actually alone — attempting it while both nodes are
+up fails with `CS_ERR_INVALID_PARAM`, since corosync will not set expected votes
+below the members it can currently see. Which is exactly why it becomes valid
+when one remains.
+
+Runtime only; it does not survive a reboot, so re-run it if the survivor
+restarts before its own conversion.
+
+## 3. Hardware
+
+Power the host off, then:
+
+1. Kingston boot drive out of slot 1, into slot 2.
+2. Crucial P3 out of slot 2 — it becomes EPHEMERAL, so put it in slot 3 rather
+   than setting it aside.
+3. Slot 1 switch to **U.2**. Set it before the drive goes in; there is nothing
+   to record afterwards, the installed drive type implies the position.
+4. PM983 onto the included 7mm M.2-to-U.2 adapter, into slot 1.
+
+Boot the host's existing PVE once before touching Talos and check the drive:
+
+```sh
+nvme smart-log /dev/nvmeXn1 | grep -iE 'percentage_used|media_errors|critical'
+nvme id-ns /dev/nvmeXn1 -H | grep -i 'in use'      # want 512b data, 0 metadata
+wipefs /dev/nvmeYn1                                 # Crucial: clear the old bluestore sig
+ip link set <ceph-port> up && ethtool <ceph-port> | grep -i 'link detected'
+```
+
+This is the last revert point. A bad drive here means the Crucial goes back and
+the VM is recreated from tofu with nothing lost. Once Talos writes to the
+Kingston, PVE on that host is gone.
+
+Relocating the boot drive is safe: `rpool` is referenced by NVMe EUI and the
+UEFI entries by GPT PARTUUID, neither of which encodes a slot.
+
+## 4. Secure Boot
+
+MS-01 ships with Secure Boot on, and Proxmox satisfies it with a
+Microsoft-signed shim. The Image Factory `metal-amd64.iso` is an unsigned UKI
+and the firmware rejects it outright. **Turn Secure Boot off in the BIOS.**
+
+A signed variant exists per schematic, but it is signed with SideroLabs' key
+which firmware does not trust by default, and the installed system would have to
+use the secureboot installer image too. That is a separate project.
+
+## 5. Install
+
+Boot the stick. Maintenance mode DHCPs untagged on every link; on these hosts
+that lands on `192.168.8.x`, which is routable from the workstation. It DHCPs on
+*every* link, so one machine answers on several addresses at once — they are the
+same host, not several hosts. `get disks` on each confirms it.
+
+Maintenance mode does serve `talosctl get`. `--insecure` is a flag on the
+subcommand rather than a global, so it must come *after* `get` — placed before
+it, cobra resolves to the legacy `talosctl disks` command and reports
+`unknown flag: --insecure`:
+
+```sh
+talosctl -n <maintenance-ip> get disks --insecure
+talosctl -n <maintenance-ip> get links --insecure
+```
+
+Verify serials and MACs here, on the machine itself, against the config about to
+be applied.
+
+Build the config from the **live cluster's secrets**, not a fresh
+`talosctl gen config`, or the node forms a new cluster instead of joining:
+
+```sh
+talosctl -n <surviving-node> get mc v1alpha1 -o jsonpath='{.spec}' > cp.yaml
+talosctl gen secrets --from-controlplane-config cp.yaml -o secrets.yaml
+talosctl gen config homelab https://k8s.renner.uno:6443 \
+  --with-secrets secrets.yaml --output-types controlplane \
+  --config-patch @talos/patches/cluster.yaml \
+  --config-patch @<cilium additive half> \
+  --config-patch @talos/patches/registry-mirror.yaml \
+  --config-patch @talos/patches/k8s-N.yaml -o install.yaml
+```
+
+Confirm `cluster.id` matches the running cluster before applying.
+
+Strip the generated `HostnameConfig` document — it collides with
+`machine.network.hostname` from the per-node patch. `bootstrap.sh` does the same.
+
+`cilium-cni.yaml` cannot be passed whole: its `$patch: delete` document fails
+against a base with no `inlineManifests` (issue #581). Use only the additive
+second document.
+
+Validate:
+
+```sh
+talosctl validate --config install.yaml --mode metal
+```
+
+### Install minimal, complete on the running node
+
+Do **not** apply the full per-node config in maintenance mode. A document that
+references an interface which does not exist yet hangs the install: the node
+sits at `STAGE: Installing` indefinitely with `KUBELET n/a`, `APISERVER n/a`,
+and the console scrolling
+
+```
+k8s.NodeApplyController: error getting node: nodes "k8s-N" not found
+```
+
+That message is a symptom of an install that never finished, not a Kubernetes or
+networking fault — the node holds a DHCP address and is reachable throughout. On
+saturn the offenders were the `ens19` VLAN document and the sysctl naming it,
+`net.ipv6.conf.ens19.disable_ipv6`. Three cold installs hung on it; the identical
+config applied cleanly once the node was up. Wiping disks makes no difference —
+it is not leftover state.
+
+**Stage 1** — the smallest config that installs and joins, plus `EPHEMERAL`,
+which is one-shot (section 7) and so cannot be deferred to stage 2:
+
+```
+machine.network.hostname, machine.install.diskSelector, certSANs, nodeLabels
+LinkConfig    <uplink>
+VLANConfig    mgmt0
+DHCPv4Config  mgmt0
+VolumeConfig  EPHEMERAL
+```
+
+```sh
+talosctl apply-config --insecure -n <maintenance-ip> --file install-stage1.yaml
+```
+
+**Stage 2** — once the node is `Ready` and `talosctl etcd members` lists it,
+apply the full config to the running node. `--mode=try` reverts on its own if the
+network config is wrong, so it cannot strand the node; re-apply with
+`--mode=auto` to persist.
+
+```sh
+talosctl -e <node-ip> -n <node-ip> apply-config --mode=try --timeout=3m --file install.yaml
+talosctl -e <node-ip> -n <node-ip> apply-config --mode=auto --file install.yaml
+```
+
+Both applied without a reboot on saturn. Confirm `ens19`, `mgmt0` and the Ceph
+link are up afterwards with `talosctl get links` and `get addresses`.
+
+## 6. The network config will strand the node if it is wrong
+
+Talos stops running default DHCP operators **for the entire machine** as soon as
+a single new-style network document exists — the gate is presence of a document,
+not success of resolution. If the configured path fails to materialise there is
+no fallback on any interface, and recovery needs a console and install media.
+
+Concretely: do not use `LinkAliasConfig` as a `VLANConfig` parent without
+verifying it. Aliases *are* supported (confirmed in v1.12.6 source and by a
+maintainer), but a selector matching zero links is skipped **silently**, and
+`permanent_addr` comes from an ethtool ioctl with no fallback — if the driver
+does not answer, the match is against an empty string and everything downstream
+that references the alias does nothing.
+
+Address links by predictable kernel name. To introduce aliases later, do it on a
+node that is already up, with `talosctl apply-config --mode=try --timeout=5m`,
+which rolls back on its own, and check `talosctl get linkaliasspec`.
+
+Interface names derive from PCI topology and are stable — but not immune.
+Adding a third NVMe to jupiter renumbered the buses and renamed the unused
+second I226 from `enp89s0` to `enp90s0`.
+
+## 7. EPHEMERAL placement is one-shot
+
+```yaml
+apiVersion: v1alpha1
+kind: VolumeConfig
+name: EPHEMERAL
+provisioning:
+  diskSelector:
+    match: disk.serial == '<crucial serial>'
+  minSize: 50GiB
+  grow: true
+```
+
+Talos applies volume configuration **only when the volume does not yet exist**.
+Applied afterwards it is silently ignored, and relocating EPHEMERAL then needs a
+disk wipe and a fresh boot from install media. It must be in the first
+`apply-config`.
+
+Match by serial. `disk.transport == 'nvme' && !system_disk` matches the PM983 as
+well, and losing that coin flip formats the OSD drive.
+
+Verify afterwards:
+
+```sh
+talosctl -n <node> get volumestatus EPHEMERAL -o yaml | grep -E 'location|phase'
+talosctl -n <node> get disks
+```
+
+## 8. The mon will not come back on its own
+
+**This is the step that cost the most time on jupiter.** The rebuilt node has a
+new MAC and therefore a new DHCP lease, but Ceph has its mon pinned to the old
+address in three places. The mon cannot bind an address the host no longer has,
+so it crashloops; the operator then tries to restart a *healthy* mon as its next
+step, Ceph refuses because that would drop quorum, and it retries every 60
+seconds forever without ever reaching the storage reconcile. Nothing else
+progresses behind it.
+
+Fix in this order — the order matters, because doing it the other way round has
+the operator faithfully recreate the mon from the stale ConfigMap:
+
+```sh
+# 1. ConfigMap first
+kubectl -n rook-ceph edit cm rook-ceph-mon-endpoints
+#    remove the dead mon from `data` and from `mapping`
+
+# 2. then the monmap
+ceph mon remove <id>
+
+# 3. then the deployment — check it really carries the old address
+kubectl -n rook-ceph get deploy rook-ceph-mon-<id> -o yaml | grep public-addr
+kubectl -n rook-ceph delete deploy rook-ceph-mon-<id>
+```
+
+Rook then creates a **new** mon with the next free ID (jupiter's `c` came back
+as `d`), on the correct address. It does not repair the old one.
+
+Note mons use host ports under `network.provider: host`, so a replacement mon
+cannot schedule while the broken one's deployment still holds 3300/6789 — the
+canary sits `Pending` with "didn't have free ports" until the old deployment is
+deleted.
+
+## 9. Re-add the OSD
+
+Add the node back to `cephcluster.yaml` under `storage.nodes`, naming the PM983
+by `/dev/disk/by-id`:
+
+```yaml
+- name: k8s-N
+  devices:
+    - fullpath: /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_<serial>
+```
+
+**Not** a bare device name. Talos enumerates these drives in a different order
+than Proxmox did — on jupiter `nvme0n1` is the Kingston *system* disk, and the
+PM983 is `nvme2n1`. Rook would probably decline a partitioned system disk, but
+that is luck rather than design.
+
+Get the path from `talosctl -n <node> get disk <dev> -o yaml`, under `symlinks`.
+
+## 10. Verify and restore
+
+```sh
+kubectl get nodes                                   # Ready
+talosctl -n <node> etcd members                     # 3 members
+ceph status                                         # 3 mons in quorum, 3 osds
+ceph osd tree                                       # OSD under the right zone/host
+ceph device ls                                      # PM983 backing the new OSD
+```
+
+Then undo the window changes: `cephfs-metadata` back to size 3, immich back to
+1 replica, and `ceph config set osd osd_mclock_profile balanced` if it was
+raised to `high_recovery_ops` to speed the evacuation.
+
+The retired Crucial leaves a stale entry in `ceph device ls`. Remove it with
+`ceph config-key rm device/<devid>`; the mgr's devicehealth view may lag.
+
+## Known-stale things that will bite
+
+- `bootstrap.sh` cannot generate a config at all right now (issue #581), so a
+  cold rebuild is broken independently of this work.
+- Manual Woodpecker pipelines are broken (issue #582), so anything gated on
+  `PIPELINE == "..."` must be run another way.
+- Cilium's eBPF datapath drops VLAN-tagged traffic unless `bpf.vlanBypass` is
+  set. **This happened** — every v30iot macvlan workload on a converted node
+  stayed in sandbox creation until `vlanBypass: [30]` was added (issue #592).
+
+  Only `ens19` is affected, and the reason is worth knowing: Cilium attaches to
+  devices that carry an address. `cilium-dbg status --verbose` lists `mgmt0`
+  among its devices but not `ens19`, because `ens19` is deliberately
+  address-less. Cilium therefore owns `mgmt0` and sees VLAN 40 frames already
+  untagged by the 8021q layer, while VLAN 30 frames only ever meet the filter
+  on the shared parent `enp2s0f0np0`, still tagged. Any future address-less
+  VLAN carrying pod traffic will need the same treatment.
+
+  The symptom does not look like a VLAN problem. A tc-layer drop reaches an
+  AF_PACKET sender as ENOBUFS, so the CNI dhcp plugin reports `no buffer space
+  available` and `network is down` rather than a DHCP timeout, and the pod sits
+  in `ContainerCreating` behind `plugin type="macvlan" failed (add): error
+  calling DHCP.Allocate: no more tries`. Confirm it with
+  `cilium_drop_count_total{reason=~"VLAN.*"}` per node rather than by reading
+  the CNI logs, and check `node_network_receive_packets_total{device="ens19"}`
+  — it reads 0 on an affected node.
+
+- The conversion drops the node's global IPv6 on `mgmt0`, and nothing in the
+  config says so. **This happened** (issue #603). Pods get only a ULA
+  (`fd10:244::/48`) and reach global IPv6 by being masqueraded to the node's
+  global address, so a node without one silently loses outbound IPv6 for every
+  pod scheduled onto it. The only in-cluster symptom is one line at Cilium
+  agent startup — `k8s-1` and `k8s-2` log it, `k8s-3` does not:
+
+  ```
+  level=warn msg="IPv6 is enabled, but Cilium cannot find the IPv6 address
+  for this node" module=agent.controlplane.local-node-sync
+  ```
+
+  It surfaced as `NetbirdRelayMemberDown` plus both v6 `blackbox-edge-wan`
+  probes going red the moment the blackbox exporter was rescheduled onto a
+  converted node. The relays themselves were healthy throughout.
+
+  Settle where the address comes from before converting the last VM. It
+  decides the fix and is not visible through kubectl — nodes publish only a
+  v4 `InternalIP`:
+
+  ```sh
+  talosctl -n <unconverted> get addresses    # while a VM is still up
+  talosctl -n <converted>   get addresses
+  ```
+
+  DHCPv6 means the converted nodes need a `DHCPv6Config` document for `mgmt0`
+  beside `DHCPv4Config`; SLAAC means they need RA acceptance instead and
+  `DHCPv6Config` alone would not help. Legacy `dhcp: true` requests DHCPv4
+  only, so SLAAC is the likelier source. Validate either with
+  `talosctl apply-config --mode=try`, per section 6.
+
+  Converting the last VM before this is settled leaves no IPv6 path at all:
+  `proxy-1` is the v6-only edge/relay member and goes from unmonitorable to
+  genuinely unreachable.

kubernetes/apps/gatus/configmap.yaml

@@ -19,7 +19,7 @@ metadata:
 # Opaque-label map (maintainer reference only — NOT served publicly):
 #   services group                   infrastructure group
 #     Web Service 1 = Authentik        Infrastructure 1 = PBS 2 (pbs2.renner.uno)
-#     Web Service 2 = Gitea            Infrastructure 2 = Proxmox VE (pve.renner.uno)
+#     Web Service 2 = Gitea            Infrastructure 2 = retired (was Proxmox VE)
 #     Web Service 3 = Immich           Infrastructure 3 = UDM DNS (10.2.40.1)
 #     Web Service 4 = Woodpecker CI    Infrastructure 4 = Mosquitto MQTT broker
 #     Web Service 5 = Grafana
@@ -206,21 +206,6 @@ data:
         conditions:
           - "[STATUS] == any(200, 301, 302)"
 
-      - name: Infrastructure 2
-        group: infrastructure
-        url: https://pve.renner.uno
-        interval: 60s
-        client:
-          insecure-skip-verify: true
-        ui:
-          hide-hostname: true
-          hide-url: true
-          hide-port: true
-          hide-conditions: true
-          hide-errors: true
-        conditions:
-          - "[STATUS] == any(200, 301, 302)"
-
       - name: Infrastructure 3
         group: infrastructure
         url: 10.2.40.1

kubernetes/apps/immich/machine-learning.yaml

@@ -6,6 +6,13 @@ metadata:
   labels:
     app.kubernetes.io/name: immich-machine-learning
 spec:
+  # Scale this and immich-server to 0 for the duration of any single-node
+  # maintenance — cluster CPU requests exceed what two nodes can hold, and this
+  # pair frees ~1000m against a shortfall of roughly 200m. Stopping them is the
+  # right lever rather than lowering their requests: 500m reserved against a
+  # 2m idle draw looks like waste, but inference bursts hard and the request is
+  # honest about what it needs when someone uploads photos.
+  # See RUNBOOK-baremetal-conversion.md.
   replicas: 1
   strategy:
     type: RollingUpdate

kubernetes/apps/immich/server.yaml

@@ -36,6 +36,9 @@ metadata:
   labels:
     app.kubernetes.io/name: immich-server
 spec:
+  # Scaled to 0 alongside immich-machine-learning during single-node
+  # maintenance; see the note there. The CNPG Postgres cluster stays running,
+  # so recovery is just these two replica counts.
   replicas: 1
   strategy:
     type: Recreate

kubernetes/apps/insecure-backend-serverstransport.yaml

@@ -2,7 +2,6 @@
 # backend serving HTTPS with a self-signed / hostname-mismatched cert:
 #
 #   apps/file       — NAS web UI on a self-signed cert
-#   apps/pve        — PVE on :8006 with per-node self-signed cert
 #
 # Replaces the nginx `backend-protocol: HTTPS` + `proxy-ssl-verify: off`
 # annotation pair. Each Ingress that wants this behaviour sets

kubernetes/apps/kustomization.yaml

@@ -7,7 +7,6 @@ resources:
   - file/
   - gitea/
   - ical-proxy/
-  - pve/
   # woodpecker/ moved to its own dedicated `woodpecker` Kustomization
   # (clusters/homelab/woodpecker.yaml) so its wp-pg CNPG cluster's barman
   # dependency + Cluster health-gating don't contaminate this shared apps KS.

kubernetes/apps/monitoring/dashboards/kustomization.yaml

@@ -83,9 +83,6 @@ configMapGenerator:
   - name: dashboard-power-flow
     files:
       - power-flow.json
-  - name: dashboard-proxmox-cluster
-    files:
-      - proxmox-cluster.json
   - name: dashboard-pv-system-roi
     files:
       - pv-system-roi.json

kubernetes/apps/monitoring/dashboards/proxmox-cluster.json

@@ -1,239 +0,0 @@
-{
-  "annotations": {"list": []},
-  "description": "Proxmox VE Cluster Overview",
-  "editable": true,
-  "fiscalYearStartMonth": 0,
-  "graphTooltip": 0,
-  "id": null,
-  "links": [],
-  "panels": [
-    {
-      "datasource": {"type": "prometheus", "uid": "prometheus"},
-      "fieldConfig": {
-        "defaults": {"color": {"mode": "thresholds"}, "mappings": [], "thresholds": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "green", "value": 3}]}},
-        "overrides": []
-      },
-      "gridPos": {"h": 4, "w": 6, "x": 0, "y": 0},
-      "id": 1,
-      "options": {"colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto"},
-      "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "count(count by (id) (pve_up{id=~\"node/.*\"} == 1))", "refId": "A"}],
-      "title": "Online Nodes",
-      "type": "stat"
-    },
-    {
-      "datasource": {"type": "prometheus", "uid": "prometheus"},
-      "fieldConfig": {
-        "defaults": {"color": {"mode": "thresholds"}, "mappings": [], "thresholds": {"mode": "absolute", "steps": [{"color": "blue", "value": null}]}},
-        "overrides": []
-      },
-      "gridPos": {"h": 4, "w": 6, "x": 6, "y": 0},
-      "id": 2,
-      "options": {"colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto"},
-      "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "count(count by (id) (pve_guest_info{type=\"qemu\", template=\"0\"}))", "refId": "A"}],
-      "title": "Total VMs",
-      "type": "stat"
-    },
-    {
-      "datasource": {"type": "prometheus", "uid": "prometheus"},
-      "fieldConfig": {
-        "defaults": {"color": {"mode": "thresholds"}, "mappings": [], "thresholds": {"mode": "absolute", "steps": [{"color": "purple", "value": null}]}},
-        "overrides": []
-      },
-      "gridPos": {"h": 4, "w": 6, "x": 12, "y": 0},
-      "id": 3,
-      "options": {"colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto"},
-      "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "count(count by (id) (pve_guest_info{type=\"lxc\"}))", "refId": "A"}],
-      "title": "Total LXC Containers",
-      "type": "stat"
-    },
-    {
-      "datasource": {"type": "prometheus", "uid": "prometheus"},
-      "fieldConfig": {
-        "defaults": {"color": {"mode": "thresholds"}, "mappings": [], "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}]}},
-        "overrides": []
-      },
-      "gridPos": {"h": 4, "w": 5, "x": 15, "y": 0},
-      "id": 4,
-      "options": {"colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto"},
-      "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "count(count by (id) (pve_up{id=~\"(qemu|lxc)/.*\"} == 1))", "refId": "A"}],
-      "title": "Running Guests",
-      "type": "stat"
-    },
-    {
-      "datasource": {"type": "prometheus", "uid": "prometheus"},
-      "fieldConfig": {
-        "defaults": {"color": {"mode": "thresholds"}, "mappings": [{"options": {"0": {"color": "green", "index": 0, "text": "All Backed Up"}}, "type": "value"}], "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "yellow", "value": 1}, {"color": "red", "value": 5}]}},
-        "overrides": []
-      },
-      "gridPos": {"h": 4, "w": 4, "x": 20, "y": 0},
-      "id": 12,
-      "options": {"colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto"},
-      "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "max(pve_not_backed_up_total)", "refId": "A"}],
-      "title": "Not Backed Up",
-      "type": "stat"
-    },
-    {
-      "datasource": {"type": "prometheus", "uid": "prometheus"},
-      "fieldConfig": {
-        "defaults": {
-          "color": {"mode": "thresholds"},
-          "mappings": [],
-          "max": 100,
-          "min": 0,
-          "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "yellow", "value": 60}, {"color": "red", "value": 80}]},
-          "unit": "percent"
-        },
-        "overrides": []
-      },
-      "gridPos": {"h": 6, "w": 8, "x": 0, "y": 4},
-      "id": 5,
-      "options": {"displayMode": "lcd", "minVizHeight": 10, "minVizWidth": 0, "orientation": "horizontal", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "showUnfilled": true},
-      "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "max by (id) (pve_cpu_usage_ratio{id=~\"node/.*\"}) * 100", "legendFormat": "{{id}}", "refId": "A"}],
-      "title": "Node CPU Usage",
-      "type": "bargauge"
-    },
-    {
-      "datasource": {"type": "prometheus", "uid": "prometheus"},
-      "fieldConfig": {
-        "defaults": {
-          "color": {"mode": "thresholds"},
-          "mappings": [],
-          "max": 100,
-          "min": 0,
-          "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "yellow", "value": 70}, {"color": "red", "value": 85}]},
-          "unit": "percent"
-        },
-        "overrides": []
-      },
-      "gridPos": {"h": 6, "w": 8, "x": 8, "y": 4},
-      "id": 6,
-      "options": {"displayMode": "lcd", "minVizHeight": 10, "minVizWidth": 0, "orientation": "horizontal", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "showUnfilled": true},
-      "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "max by (id) (pve_memory_usage_bytes{id=~\"node/.*\"}) / max by (id) (pve_memory_size_bytes{id=~\"node/.*\"}) * 100", "legendFormat": "{{id}}", "refId": "A"}],
-      "title": "Node Memory Usage",
-      "type": "bargauge"
-    },
-    {
-      "datasource": {"type": "prometheus", "uid": "prometheus"},
-      "fieldConfig": {
-        "defaults": {
-          "color": {"mode": "thresholds"},
-          "mappings": [],
-          "max": 100,
-          "min": 0,
-          "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "yellow", "value": 70}, {"color": "red", "value": 85}]},
-          "unit": "percent"
-        },
-        "overrides": []
-      },
-      "gridPos": {"h": 6, "w": 8, "x": 16, "y": 4},
-      "id": 7,
-      "options": {"displayMode": "lcd", "minVizHeight": 10, "minVizWidth": 0, "orientation": "horizontal", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "showUnfilled": true},
-      "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "max by (id) (pve_disk_usage_bytes{id=~\"storage/.*\"}) / max by (id) (pve_disk_size_bytes{id=~\"storage/.*\"}) * 100", "legendFormat": "{{id}}", "refId": "A"}],
-      "title": "Storage Usage",
-      "type": "bargauge"
-    },
-    {
-      "datasource": {"type": "prometheus", "uid": "prometheus"},
-      "fieldConfig": {
-        "defaults": {
-          "color": {"mode": "palette-classic"},
-          "custom": {"axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": {"legend": false, "tooltip": false, "viz": false}, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": {"type": "linear"}, "showPoints": "never", "spanNulls": false, "stacking": {"group": "A", "mode": "none"}, "thresholdsStyle": {"mode": "off"}},
-          "mappings": [],
-          "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}]},
-          "unit": "percent"
-        },
-        "overrides": []
-      },
-      "gridPos": {"h": 8, "w": 12, "x": 0, "y": 10},
-      "id": 8,
-      "options": {"legend": {"calcs": ["mean", "max"], "displayMode": "table", "placement": "right", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
-      "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "max by (id) (pve_cpu_usage_ratio{id=~\"node/.*\"}) * 100", "legendFormat": "{{id}}", "refId": "A"}],
-      "title": "Node CPU Over Time",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {"type": "prometheus", "uid": "prometheus"},
-      "fieldConfig": {
-        "defaults": {
-          "color": {"mode": "palette-classic"},
-          "custom": {"axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": {"legend": false, "tooltip": false, "viz": false}, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": {"type": "linear"}, "showPoints": "never", "spanNulls": false, "stacking": {"group": "A", "mode": "none"}, "thresholdsStyle": {"mode": "off"}},
-          "mappings": [],
-          "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}]},
-          "unit": "bytes"
-        },
-        "overrides": []
-      },
-      "gridPos": {"h": 8, "w": 12, "x": 12, "y": 10},
-      "id": 9,
-      "options": {"legend": {"calcs": ["mean", "max"], "displayMode": "table", "placement": "right", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
-      "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "max by (id) (pve_memory_usage_bytes{id=~\"node/.*\"})", "legendFormat": "{{id}}", "refId": "A"}],
-      "title": "Node Memory Usage Over Time",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {"type": "prometheus", "uid": "prometheus"},
-      "fieldConfig": {
-        "defaults": {
-          "color": {"mode": "thresholds"},
-          "custom": {"align": "auto", "cellOptions": {"type": "auto"}, "inspect": false},
-          "mappings": [],
-          "thresholds": {"mode": "absolute", "steps": [{"color": "text", "value": null}]}
-        },
-        "overrides": [
-          {"matcher": {"id": "byName", "options": "Status"}, "properties": [{"id": "custom.cellOptions", "value": {"type": "color-background"}}, {"id": "mappings", "value": [{"options": {"0": {"color": "red", "index": 1, "text": "Stopped"}, "1": {"color": "green", "index": 0, "text": "Running"}}, "type": "value"}]}]},
-          {"matcher": {"id": "byName", "options": "CPU %"}, "properties": [{"id": "unit", "value": "percent"}, {"id": "custom.cellOptions", "value": {"mode": "gradient", "type": "gauge"}}, {"id": "min", "value": 0}, {"id": "max", "value": 100}]},
-          {"matcher": {"id": "byName", "options": "Memory %"}, "properties": [{"id": "unit", "value": "percent"}, {"id": "custom.cellOptions", "value": {"mode": "gradient", "type": "gauge"}}, {"id": "min", "value": 0}, {"id": "max", "value": 100}]}
-        ]
-      },
-      "gridPos": {"h": 10, "w": 24, "x": 0, "y": 18},
-      "id": 10,
-      "options": {"footer": {"fields": "", "reducer": ["sum"], "show": false}, "showHeader": true, "sortBy": [{"desc": false, "displayName": "Name"}]},
-      "targets": [
-        {"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "max by (id, name, node, type) (pve_guest_info)", "format": "table", "instant": true, "refId": "A"},
-        {"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "max by (id) (pve_cpu_usage_ratio{id=~\"(qemu|lxc)/.*\"}) * 100", "format": "table", "instant": true, "refId": "B"},
-        {"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "max by (id) (pve_memory_usage_bytes{id=~\"(qemu|lxc)/.*\"}) / max by (id) (pve_memory_size_bytes{id=~\"(qemu|lxc)/.*\"}) * 100", "format": "table", "instant": true, "refId": "C"},
-        {"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "max by (id) (pve_up{id=~\"(qemu|lxc)/.*\"})", "format": "table", "instant": true, "refId": "D"}
-      ],
-      "title": "Guest Status",
-      "transformations": [
-        {"id": "seriesToColumns", "options": {"byField": "id"}},
-        {"id": "organize", "options": {"excludeByName": {"Time": true, "Time 1": true, "Time 2": true, "Time 3": true, "Time 4": true, "Value #A": true}, "indexByName": {"id": 0, "name": 1, "node": 2, "type": 3, "Value #D": 4, "Value #B": 5, "Value #C": 6}, "renameByName": {"Value #B": "CPU %", "Value #C": "Memory %", "Value #D": "Status", "id": "ID", "name": "Name", "node": "Node", "type": "Type"}}}
-      ],
-      "type": "table"
-    },
-    {
-      "datasource": {"type": "prometheus", "uid": "prometheus"},
-      "fieldConfig": {
-        "defaults": {
-          "color": {"mode": "palette-classic"},
-          "custom": {"axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": {"legend": false, "tooltip": false, "viz": false}, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": {"type": "linear"}, "showPoints": "never", "spanNulls": false, "stacking": {"group": "A", "mode": "none"}, "thresholdsStyle": {"mode": "off"}},
-          "mappings": [],
-          "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}]},
-          "unit": "Bps"
-        },
-        "overrides": []
-      },
-      "gridPos": {"h": 8, "w": 24, "x": 0, "y": 28},
-      "id": 11,
-      "options": {"legend": {"calcs": ["mean", "max"], "displayMode": "table", "placement": "right", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
-      "targets": [
-        {"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "max by (id) (rate(pve_disk_read_bytes_total{id=~\"(qemu|lxc)/.*\"}[5m]))", "legendFormat": "{{id}} read", "refId": "A"},
-        {"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "max by (id) (rate(pve_disk_written_bytes_total{id=~\"(qemu|lxc)/.*\"}[5m]))", "legendFormat": "{{id}} write", "refId": "B"}
-      ],
-      "title": "Guest Disk I/O",
-      "type": "timeseries"
-    }
-  ],
-  "refresh": "30s",
-  "schemaVersion": 38,
-  "style": "dark",
-  "tags": ["homelab", "proxmox", "virtualization"],
-  "templating": {"list": []},
-  "time": {"from": "now-1h", "to": "now"},
-  "timepicker": {},
-  "timezone": "",
-  "title": "Proxmox VE Cluster",
-  "uid": "proxmox-cluster",
-  "version": 1,
-  "weekStart": ""
-}

kubernetes/apps/monitoring/helmrelease-prometheus.yaml

@@ -93,68 +93,8 @@ spec:
               resources:
                 requests:
                   storage: 50Gi
-        # Scrape existing Proxmox infrastructure targets
+        # Additional scrape targets not covered by ServiceMonitors.
         additionalScrapeConfigs:
-          # pve-exporter is a blackbox-style exporter: Prometheus passes
-          # ?target=<host> via relabel, and one in-cluster exporter
-          # instance queries each PVE node's API on the user's behalf.
-          # The targets list is the PVE node addresses; the actual
-          # scrape goes to pve-exporter.monitoring.svc:9221.
-          #
-          # Cluster collectors (pve_up, pve_guest_info, …) return the SAME
-          # cluster-wide resource set no matter which node answers, so they
-          # are scraped from a single node only. Scraping them from all three
-          # (the pre-2026-07 config) ingested every node/guest series 3x —
-          # identical `id`, differing `instance` — a real TSDB cardinality
-          # cost that dashboards had to paper over with max by (id).
-          #
-          # Target the pve Service (apps ns) rather than a hardcoded node IP:
-          # its EndpointSlice load-balances the PVE API across all three nodes,
-          # so cluster metrics keep flowing as long as ANY node is up, instead
-          # of pausing whenever .201 (jupiter) specifically is down. The
-          # exporter reaches the API on 8006 (verify_ssl:false, so the LB'd
-          # cert hostname mismatch is a non-issue).
-          - job_name: pve
-            static_configs:
-              - targets:
-                  - "pve.apps.svc.cluster.local"
-            metrics_path: /pve
-            params:
-              cluster: ["1"]
-              node: ["0"]
-            relabel_configs:
-              - source_labels: [__address__]
-                target_label: __param_target
-              - source_labels: [__param_target]
-                target_label: instance
-              - target_label: __address__
-                replacement: pve-exporter.monitoring.svc.cluster.local:9221
-
-          # Node collectors (pve_version_info, replication status) are
-          # node-local and genuinely distinct per target, so every node is
-          # scraped — this is the split recommended by prometheus-pve-exporter
-          # upstream to avoid the cluster-collector duplication above.
-          # Targets are the node FQDNs, not raw IPs: the instance label then
-          # reads jupiter/neptune/saturn.renner.uno, and a renumbered node is
-          # picked up from DNS without editing this file.
-          - job_name: pve-node
-            static_configs:
-              - targets:
-                  - "jupiter.renner.uno"
-                  - "neptune.renner.uno"
-                  - "saturn.renner.uno"
-            metrics_path: /pve
-            params:
-              cluster: ["0"]
-              node: ["1"]
-            relabel_configs:
-              - source_labels: [__address__]
-                target_label: __param_target
-              - source_labels: [__param_target]
-                target_label: instance
-              - target_label: __address__
-                replacement: pve-exporter.monitoring.svc.cluster.local:9221
-
           # Gitea exposes /metrics anonymously on its main HTTP port. The
           # `gitea_*` metrics power the gitea-overview dashboard. Gitea now runs
           # in-cluster (StatefulSet in apps ns; LXC 114 retired), so scrape the

kubernetes/apps/monitoring/kustomization.yaml

@@ -3,8 +3,6 @@ kind: Kustomization
 resources:
   - grafana-oauth-externalsecret.yaml
   - alertmanager-smtp-externalsecret.yaml
-  - pve-exporter-config-externalsecret.yaml
-  - pve-exporter-deployment.yaml
   - helmrelease-prometheus.yaml
   - helmrelease-loki.yaml
   - helmrelease-alloy-syslog.yaml

kubernetes/apps/monitoring/pve-exporter-config-externalsecret.yaml

@@ -1,17 +0,0 @@
-apiVersion: external-secrets.io/v1
-kind: ExternalSecret
-metadata:
-  name: pve-exporter-config
-  namespace: monitoring
-spec:
-  # 10m (was 1h) so an automated token rotation propagates within ~10m; Reloader
-  # then restarts pve-exporter. See kubernetes/infrastructure/openbao/rotation/pve-exporter.yaml.
-  refreshInterval: 10m
-  secretStoreRef:
-    kind: ClusterSecretStore
-    name: openbao
-  target:
-    name: pve-exporter-config # keeps the old Secret's name — consumers unchanged
-  dataFrom:
-    - extract:
-        key: apps/monitoring/pve-exporter-config

kubernetes/apps/monitoring/pve-exporter-deployment.yaml

@@ -1,84 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: pve-exporter
-  namespace: monitoring
-  annotations:
-    # Reloader restarts pve-exporter when pve-exporter-config changes (token
-    # rotation) so it reloads pve.yml. See
-    # kubernetes/infrastructure/openbao/rotation/pve-exporter.yaml.
-    secret.reloader.stakater.com/reload: pve-exporter-config
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app.kubernetes.io/name: pve-exporter
-  template:
-    metadata:
-      labels:
-        app.kubernetes.io/name: pve-exporter
-    spec:
-      securityContext:
-        runAsNonRoot: true
-        runAsUser: 1000
-        runAsGroup: 1000
-        fsGroup: 1000
-        seccompProfile:
-          type: RuntimeDefault
-      containers:
-        - name: pve-exporter
-          image: prompve/prometheus-pve-exporter:3.10.0 # {"$imagepolicy": "flux-system:pve-exporter"}
-          # Image ENTRYPOINT is already `pve_exporter`; pass flags only.
-          args:
-            - --config.file=/etc/pve-exporter/pve.yml
-            - --web.listen-address=0.0.0.0:9221
-          ports:
-            - name: http
-              containerPort: 9221
-          readinessProbe:
-            httpGet:
-              path: /
-              port: http
-            initialDelaySeconds: 5
-            periodSeconds: 30
-          resources:
-            requests:
-              cpu: 10m
-              memory: 64Mi
-            limits:
-              memory: 256Mi
-          securityContext:
-            allowPrivilegeEscalation: false
-            readOnlyRootFilesystem: true
-            capabilities:
-              drop: ["ALL"]
-          volumeMounts:
-            - name: config
-              mountPath: /etc/pve-exporter
-              readOnly: true
-            # Python needs a writable temp dir; the rootfs is read-only.
-            - name: tmp
-              mountPath: /tmp
-      volumes:
-        - name: config
-          secret:
-            secretName: pve-exporter-config
-        - name: tmp
-          emptyDir:
-            medium: Memory
-            sizeLimit: 16Mi
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: pve-exporter
-  namespace: monitoring
-  labels:
-    app.kubernetes.io/name: pve-exporter
-spec:
-  selector:
-    app.kubernetes.io/name: pve-exporter
-  ports:
-    - name: http
-      port: 9221
-      targetPort: http

kubernetes/apps/music-assistant/deployment.yaml

@@ -25,8 +25,11 @@ spec:
         # replaces the former hostNetwork attachment (see
         # network-attachment.yaml) so the NODE no longer holds an address
         # on the compromised IoT VLAN; MA gets its own leased IoT IP on
-        # net1 instead. master: ens19 is the v30iot NIC on every k8s-{1,2,3}/edge
-        # worker.
+        # net1 instead. master: ens19 resolves on every k8s-{1,2,3}/edge
+        # worker, though it is no longer the same kind of device everywhere:
+        # on the bare-metal control-plane nodes it is a VLAN 30 sub-interface
+        # of enp2s0f0np0 rather than a NIC, which is why the datapath needs
+        # bpf.vlanBypass to carry it (issue #592).
         k8s.v1.cni.cncf.io/networks: music-assistant-iot
     spec:
       enableServiceLinks: false

kubernetes/apps/pve/endpointslice.yaml

@@ -1,33 +0,0 @@
-# PVE node IPs are hardcoded here despite the general "IPs aren't static"
-# rule for the homelab: corosync requires stable cluster member identity,
-# so jupiter/neptune/saturn keep .201/.202/.203 across reboots and PVE
-# upgrades. If a node is permanently renumbered, edit this file.
-apiVersion: discovery.k8s.io/v1
-kind: EndpointSlice
-metadata:
-  name: pve-nodes
-  namespace: apps
-  labels:
-    kubernetes.io/service-name: pve
-    app.kubernetes.io/name: pve
-addressType: IPv4
-ports:
-  - name: https
-    port: 8006
-    protocol: TCP
-endpoints:
-  - addresses:
-      - 10.2.40.201
-    hostname: jupiter
-    conditions:
-      ready: true
-  - addresses:
-      - 10.2.40.202
-    hostname: neptune
-    conditions:
-      ready: true
-  - addresses:
-      - 10.2.40.203
-    hostname: saturn
-    conditions:
-      ready: true

kubernetes/apps/pve/ingressroute.yaml

@@ -1,48 +0,0 @@
-# PVE (Proxmox cluster UI) on Traefik. Replaced the standard Ingress
-# pattern because PVE's backend is per-node HTTPS on :8006 with a
-# self-signed cert that has no IP SAN — Traefik's kubernetesIngress
-# provider's service.serverstransport annotation doesn't resolve the
-# apps/insecure-backend ServersTransport CRD, so TLS verification ran
-# and rejected. IngressRoute's native services[].serversTransport field
-# binds directly with no provider-qualifier guessing.
-apiVersion: cert-manager.io/v1
-kind: Certificate
-metadata:
-  name: pve-tls
-  namespace: apps
-  labels:
-    app.kubernetes.io/name: pve
-spec:
-  secretName: pve-tls
-  issuerRef:
-    name: letsencrypt-cluster
-    kind: ClusterIssuer
-  dnsNames:
-    - pve.renner.uno
----
-apiVersion: traefik.io/v1alpha1
-kind: IngressRoute
-metadata:
-  name: pve
-  namespace: apps
-  labels:
-    app.kubernetes.io/name: pve
-  annotations:
-    # Traefik does not populate IngressRoute.status.loadBalancer, so
-    # external-dns-unifi (traefik-proxy source) has no implicit target.
-    # Pin to the traefik Service VIP (kubernetes/infrastructure/traefik
-    # service.annotations io.cilium/lb-ipam-ips).
-    external-dns.alpha.kubernetes.io/target: "10.2.99.86,fd99::86"
-spec:
-  entryPoints:
-    - websecure
-  routes:
-    - match: Host(`pve.renner.uno`)
-      kind: Rule
-      services:
-        - name: pve
-          port: 8006
-          scheme: https
-          serversTransport: insecure-backend
-  tls:
-    secretName: pve-tls

kubernetes/apps/pve/kustomization.yaml

@@ -1,6 +0,0 @@
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-resources:
-  - service.yaml
-  - endpointslice.yaml
-  - ingressroute.yaml

kubernetes/apps/pve/service.yaml

@@ -1,15 +0,0 @@
-apiVersion: v1
-kind: Service
-metadata:
-  name: pve
-  namespace: apps
-  labels:
-    app.kubernetes.io/name: pve
-spec:
-  # No selector: endpoints managed manually via the sibling EndpointSlice
-  # so ingress-nginx fans out across the three Proxmox cluster members.
-  ports:
-    - name: https
-      port: 8006
-      targetPort: 8006
-      protocol: TCP

kubernetes/infrastructure/cilium/helmrelease.yaml

@@ -33,6 +33,24 @@ spec:
     remediation:
       retries: 3
   values:
+    # Roll the agent DaemonSet when cilium-config changes. Cilium reads
+    # datapath options once, at agent start, and the chart puts no config
+    # checksum on the pod template — so a changed ConfigMap leaves every agent
+    # running the previous datapath until something restarts it. Nothing did.
+    #
+    # This is not hypothetical. vlan-bpf-bypass reached cilium-config and every
+    # v30iot macvlan workload stayed broken until the agents were deleted by
+    # hand, one node at a time (issue #592). policyMapMax has the same
+    # exposure, and its comment below says "takes effect on agent restart" —
+    # documentation standing in for automation.
+    #
+    # The trade: any cilium-config change now rolls the CNI cluster-wide,
+    # node by node. That is disruptive-ish, but it is what a datapath change
+    # already required — the difference is that it now happens on its own
+    # instead of being forgotten. Reloader is already deployed for exactly
+    # this pattern; named per-ConfigMap rather than blanket, as elsewhere.
+    annotations:
+      configmap.reloader.stakater.com/reload: cilium-config
     # Which pod labels feed security-identity computation. Patterns are
     # start-anchored regexes; a leading "!" excludes while keeping the default
     # set. Woodpecker stamps every step pod with woodpecker-ci.org/task-uuid
@@ -73,6 +91,36 @@ spec:
       # kubernetes/talos/cilium.yaml carries a third copy of the same key.
       # Nothing enforces any of this; grep bpf-policy-map-max before changing it.
       policyMapMax: 65536
+      # VLAN ids the eBPF datapath is allowed to pass. Without this it drops
+      # VLAN-tagged frames, accounted as "VLAN traffic disallowed by VLAN
+      # filter" in cilium_drop_count_total.
+      #
+      # 30 is v30iot, the macvlan parent for the shelly-manager, matter-server,
+      # otbr and music-assistant attachments. On bare metal it is a VLAN
+      # sub-interface of enp2s0f0np0; inside the former Proxmox guests it was
+      # an untagged virtio NIC, which is why this was not needed before.
+      #
+      # VLAN 40 (mgmt0) is deliberately NOT listed, and the asymmetry is the
+      # interesting part. Cilium attaches to devices that carry an address —
+      # `cilium-dbg status --verbose` lists enp87s0, enp2s0f0np0, enp2s0f1np1
+      # and mgmt0, but NOT ens19. mgmt0 holds the node IP, so Cilium owns it as
+      # a device and sees frames the 8021q layer has already untagged; nothing
+      # for a VLAN filter to match. ens19 is deliberately address-less (see
+      # talos/patches/k8s-1.yaml — host addresses on that L2 make node replies
+      # bypass the UDM), so Cilium never attaches to it and VLAN 30 frames only
+      # meet the filter on the shared parent, still tagged.
+      #
+      # So the fix belongs on 30 alone. Listing 40 would change nothing, and
+      # would touch the datapath carrying etcd and the API for no reason.
+      #
+      # A tc-layer drop surfaces to an AF_PACKET sender as ENOBUFS, so the
+      # symptom is the CNI dhcp plugin reporting "no buffer space available"
+      # rather than a DHCP timeout. See issue #592.
+      #
+      # Same three-copy problem as policyMapMax above: vlan-bpf-bypass in
+      # kubernetes/talos/cilium.yaml and kubernetes/talos/patches/cilium-cni.yaml
+      # must move with this list. Takes effect on agent restart.
+      vlanBypass: [30]
       # Let pods reach the cluster's own LoadBalancer external IPs; Cilium
       # redirects the socket directly to the backend pod instead of sending
       # traffic out to the VIP and back (which fails with L2 announcements).

kubernetes/infrastructure/crowdsec/helmrelease.yaml

@@ -150,16 +150,3 @@ spec:
       env:
         - name: COLLECTIONS
           value: "crowdsecurity/traefik crowdsecurity/http-cve crowdsecurity/appsec-virtual-patching crowdsecurity/appsec-generic-rules"
-
-    config:
-      parsers:
-        s02-enrich:
-          pve-ui-polling-whitelist.yaml: |
-            name: krenner/pve-ui-polling-whitelist
-            description: "Suppress CrowdSec scenarios for the Proxmox UI's legitimate API polling on pve.renner.uno"
-            filter: "evt.Meta.log_type == 'http_access-log' && evt.Meta.target_fqdn == 'pve.renner.uno'"
-            whitelist:
-              reason: "Proxmox web UI polls /api2/json/cluster/* and /api2/json/nodes/<n>/{lxc,qemu}/<id>/status/current every ~2s; http-probing flagged it as scanning and banned the residential IPv6."
-              expression:
-                - "evt.Meta.http_path startsWith '/api2/json/cluster/'"
-                - "evt.Meta.http_path matches '^/api2/json/nodes/[^/]+/(lxc|qemu)/[0-9]+/status/current$'"

kubernetes/infrastructure/image-automation/kustomization.yaml

@@ -15,7 +15,6 @@ resources:
   - netbird-ingress.yaml
   - paperless-ngx.yaml
   - protonmail-bridge.yaml
-  - pve-exporter.yaml
   - mosquitto.yaml
   - redis.yaml
   - shelly-manager.yaml

kubernetes/infrastructure/image-automation/pve-exporter.yaml

@@ -1,23 +0,0 @@
-apiVersion: image.toolkit.fluxcd.io/v1
-kind: ImageRepository
-metadata:
-  name: pve-exporter
-  namespace: flux-system
-spec:
-  image: prompve/prometheus-pve-exporter
-  interval: 1h
----
-apiVersion: image.toolkit.fluxcd.io/v1
-kind: ImagePolicy
-metadata:
-  name: pve-exporter
-  namespace: flux-system
-spec:
-  imageRepositoryRef:
-    name: pve-exporter
-  filterTags:
-    pattern: '^(?P<ver>\d+\.\d+\.\d+)$'
-    extract: '$ver'
-  policy:
-    semver:
-      range: "3.x"

kubernetes/infrastructure/network-policies/monitoring-ns.yaml

@@ -7,7 +7,7 @@ spec:
   endpointSelector: {}
   ingress:
     # Intra-namespace — Prometheus ↔ Alertmanager ↔ Grafana ↔ Loki ↔
-    # alloy ↔ kube-state-metrics ↔ blackbox-exporter ↔ pve-exporter.
+    # alloy ↔ kube-state-metrics ↔ blackbox-exporter.
     - fromEndpoints:
         - matchLabels:
             "k8s:io.kubernetes.pod.namespace": monitoring

kubernetes/infrastructure/openbao/policies/rotator-proxmox.hcl

@@ -1,53 +0,0 @@
-# OpenBao policy: rotator-proxmox
-#
-# Class-B rotation for the pve-exporter Proxmox API token. Grants the
-# proxmox-secret-rotator CronJob READ on its bootstrap credential (a PRIVILEGED
-# Proxmox token allowed to delete/create tokens) and PATCH on the target
-# pve-exporter-config path. Per-secret isolation (#213/#202).
-#
-# NOT applied by Flux. Apply ONCE with an admin token:
-#
-#   bao policy write rotator-proxmox kubernetes/infrastructure/openbao/policies/rotator-proxmox.hcl
-#   bao write auth/kubernetes/role/rotator-proxmox \
-#     bound_service_account_names=proxmox-secret-rotator \
-#     bound_service_account_namespaces=openbao \
-#     policies=rotator-proxmox ttl=15m
-#
-# BOOTSTRAP CREDENTIAL (PROVISIONED — recorded here for rebuild/DR).
-#
-# A SEPARATE least-privilege Proxmox identity, NOT root@pam. PVE has no
-# per-user ACL path (/access/users/<id> is rejected — see AccessControl.pm
-# check_path), and the token endpoints authorise via a `userid-group` check.
-# So the tightest possible scope is a group containing only the target user:
-#
-#   pveum role add SecretRotator -privs "User.Modify"
-#   pveum user add secret-rotator@pve
-#   pveum group add secret-rotated
-#   pveum user modify pve-exporter@pve --groups secret-rotated
-#   pveum acl modify /access/groups/secret-rotated \
-#     --users secret-rotator@pve --roles SecretRotator
-#   pveum user token add secret-rotator@pve rotate --privsep 0
-#
-# This grants exactly one thing: managing API tokens of users in the
-# secret-rotated group. Verified: no VM/storage visibility, and minting a
-# root@pam token returns 403.
-#
-#   bao kv put homelab/infrastructure/secret-rotators-bootstrap/proxmox \
-#     api-url=https://pve.apps.svc.cluster.local:8006 \
-#     auth='secret-rotator@pve!rotate=<secret>'
-#
-# api-url MUST be the in-cluster Service (backed by all three PVE nodes).
-# The external name pve.renner.uno does NOT work from a pod — pods cannot
-# hairpin to their own cluster's ingress LB and the Job hangs.
-#
-# Do NOT set token_no_default_policy=true (kv-v2 preflight needs the default policy).
-
-# Bootstrap: the privileged Proxmox token + API URL.
-path "homelab/data/infrastructure/secret-rotators-bootstrap/proxmox" {
-  capabilities = ["read"]
-}
-
-# Target: the pve-exporter config (holds the pve.yml with the token_value).
-path "homelab/data/apps/monitoring/pve-exporter-config" {
-  capabilities = ["read", "patch"]
-}

kubernetes/infrastructure/openbao/rotation/kustomization.yaml

@@ -52,7 +52,6 @@ resources:
   - crowdsec-bouncer-key.yaml
   - netbird-ingress.yaml
   - cloudflare.yaml
-  - pve-exporter.yaml
   - external-dns.yaml
   - flux-git.yaml
   - home-assistant.yaml

kubernetes/infrastructure/openbao/rotation/pve-exporter.yaml

@@ -1,91 +0,0 @@
----
-# Class-B rotation of the pve-exporter Proxmox API token via the secret-rotator
-# binary (proxmox-token mode). The RISKIEST rotator: Proxmox has no in-place
-# token roll, so the binary DELETEs and recreates pve-exporter@pve!monitoring
-# (same id, privsep=0) and patches OpenBao with a rendered pve.yml. That leaves
-# a brief window with NO valid token — pve-exporter metrics gap until ESO(10m)+
-# Reloader restart. Quarterly, off-hours: acceptable.
-#
-# privsep=0 makes the recreated token inherit pve-exporter@pve's own perms
-# (PVEAuditor ACL is on the USER, not the token — verified), so no ACL re-grant.
-# The bootstrap 'auth' is the least-privilege secret-rotator@pve!rotate identity
-# (User.Modify scoped to the secret-rotated group), NOT root.
-#
-# api-url is the in-cluster Service https://pve.apps.svc.cluster.local:8006 (all
-# three PVE nodes back it → node-failover). TLS verification is disabled for the
-# PVE calls: the nodes present valid Let's Encrypt certs that fail only on
-# hostname mismatch (Service DNS vs node FQDN); ureq has no skip-hostname-only
-# option, and this matches the previous python's CERT_NONE. Do NOT use
-# pve.renner.uno (pod hairpin off the ingress LB) or a single node FQDN (SPOF).
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: proxmox-secret-rotator
-  namespace: openbao
-  labels:
-    app.kubernetes.io/name: pve-exporter
-    app.kubernetes.io/component: secret-rotator
----
-apiVersion: batch/v1
-kind: CronJob
-metadata:
-  name: proxmox-secret-rotate
-  namespace: openbao
-  labels:
-    app.kubernetes.io/name: pve-exporter
-    app.kubernetes.io/component: secret-rotator
-spec:
-  suspend: false
-  schedule: "0 4 1 1,4,7,10 *" # quarterly, Europe/Berlin
-  timeZone: Europe/Berlin
-  concurrencyPolicy: Forbid
-  successfulJobsHistoryLimit: 2
-  failedJobsHistoryLimit: 5
-  startingDeadlineSeconds: 3600
-  jobTemplate:
-    spec:
-      backoffLimit: 1
-      activeDeadlineSeconds: 300
-      ttlSecondsAfterFinished: 259200 # 3 d
-      template:
-        metadata:
-          labels:
-            app.kubernetes.io/name: pve-exporter
-            app.kubernetes.io/component: secret-rotator
-        spec:
-          serviceAccountName: proxmox-secret-rotator
-          restartPolicy: Never
-          securityContext:
-            runAsNonRoot: true
-            runAsUser: 65534
-            runAsGroup: 65534
-            seccompProfile:
-              type: RuntimeDefault
-          containers:
-            - name: rotate
-              image: gitea.renner.uno/gitea_admin/secret-rotator:2026-07-30
-              args: ["proxmox-token"]
-              securityContext:
-                allowPrivilegeEscalation: false
-                readOnlyRootFilesystem: true
-                capabilities:
-                  drop: ["ALL"]
-              env:
-                - name: BAO_ADDR
-                  value: http://openbao.openbao.svc.cluster.local:8200
-                - name: BAO_ROLE
-                  value: rotator-proxmox
-                - name: BOOTSTRAP_PATH
-                  value: homelab/infrastructure/secret-rotators-bootstrap/proxmox
-                - name: BAO_PATH
-                  value: homelab/apps/monitoring/pve-exporter-config
-                - name: PVE_USER
-                  value: pve-exporter@pve
-                - name: PVE_TOKENID
-                  value: monitoring
-              resources:
-                requests:
-                  cpu: 25m
-                  memory: 16Mi
-                limits:
-                  memory: 48Mi

kubernetes/infrastructure/rook-ceph-cluster/cephcluster.yaml

@@ -160,19 +160,51 @@ spec:
         - "10.2.40.0/24"
       cluster:
         - "10.99.0.0/29"
+  # Nodes are listed explicitly rather than useAllNodes, so that a node can be
+  # taken out of the storage fleet by removing it here. Under useAllNodes the
+  # operator re-provisions an OSD on any node whose device still carries a
+  # BlueStore signature, which makes purging an OSD ahead of pulling its drive
+  # impossible: the OSD is recreated within a reconcile and starts backfilling
+  # onto the drive being replaced.
+  #
+  # All three nodes are bare metal and see three NVMe drives each, so they name
+  # their OSDs by /dev/disk/by-id path. A bare name would be actively wrong:
+  # Talos enumerates these machines' drives in a different order than Proxmox
+  # did, and the kernel name lands on the Kingston SYSTEM disk or on EPHEMERAL
+  # depending on the boot. Rook would probably decline the system disk for
+  # carrying partitions, but that is luck rather than intent. The by-id path is
+  # derived from model and serial and does not depend on enumeration at all.
+  #
+  # k8s-3 demonstrated why: it enumerated the Kingston as nvme0n1 on one boot
+  # and the Crucial as nvme0n1 on the very next one.
+  #
+  #   k8s-1 (jupiter)
+  #   50026B7382B48B86  Kingston  — Talos system disk
+  #   2332E86B749A      Crucial   — Talos EPHEMERAL (/var)
+  #   S437NX0M400426    PM983     — this OSD
+  #
+  #   k8s-2 (saturn)
+  #   50026B7382DA5F6E  Kingston  — Talos system disk
+  #   2332E86B74BC      Crucial   — Talos EPHEMERAL (/var)
+  #   S437NX0M403161    PM983     — this OSD
+  #
+  #   k8s-3 (neptune)
+  #   50026B7382B48B90  Kingston  — Talos system disk
+  #   2326E6E8E67D      Crucial   — Talos EPHEMERAL (/var)
+  #   S437NF0M501176    PM983     — this OSD
   storage:
-    useAllNodes: true
+    useAllNodes: false
     useAllDevices: false
-    devices:
-      # On k8s-2/k8s-3 this is the NVMe drive PCI-passed-through from saturn
-      # and neptune, and it is the only block device the guest can see. k8s-1
-      # is bare metal on jupiter and sees two: the OSD drive in MS-01 slot 1
-      # (PCI 01:00.0) and the Talos system disk in slot 2 (58:00.0). Slot 1
-      # enumerates first, so nvme0n1 is still the OSD drive there — but that
-      # is enumeration order, not a guarantee. The system disk is protected
-      # from the other direction instead: talos/patches/k8s-1.yaml pins the
-      # installer to it by serial, and Rook skips a disk carrying partitions.
-      - name: nvme0n1
+    nodes:
+      - name: k8s-1
+        devices:
+          - fullpath: /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NX0M400426
+      - name: k8s-2
+        devices:
+          - fullpath: /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NX0M403161
+      - name: k8s-3
+        devices:
+          - fullpath: /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NF0M501176
     config:
       osdsPerDevice: "1"
   # Storage layer must never be evicted by kubelet under node pressure

kubernetes/infrastructure/rook-ceph-cluster/storageclass.yaml

@@ -5,31 +5,24 @@ metadata:
   namespace: rook-ceph
 spec:
   failureDomain: host
-  # size 2, NOT 3. With 3 OSDs on 3 hosts, size=3 meant every write landed on
-  # every drive in the cluster: per-drive write volume equalled total cluster
-  # write volume, and all three drives wore at an identical rate and would have
-  # reached end-of-rated-life together (measured 2026-08-19: 80/82/87% endurance
-  # used, ~0.13 points/day, all three exhausted between Nov 2026 and Jan 2027 —
-  # see #434). Replication does not protect against that failure mode, because
-  # wear-out here is correlated by construction rather than independent.
+  # size 3. This pool sat at 2 while the OSDs were Crucial P3 QLC drives: with
+  # 3 OSDs on 3 hosts, size=3 put every write on every drive, so per-drive write
+  # volume equalled total cluster write volume and all three wore at an
+  # identical rate (measured 2026-08-19: 80/82/87% endurance used, ~0.13
+  # points/day, all three projected to exhaust between Nov 2026 and Jan 2027 —
+  # see #434). Replication does not protect against that, because the wear was
+  # correlated by construction rather than independent.
   #
-  # size 2 cuts per-drive write volume by a third and desynchronises the wear.
+  # The OSDs are Samsung PM983 enterprise drives now, at 2-3% of rated
+  # endurance with a far higher write budget, so wear-out is no longer the
+  # binding constraint and the correlation argument no longer applies.
   #
-  # min_size 1 is REQUIRED and deliberate. Ceph does not lower min_size when
-  # size is reduced — it was 2 here, and leaving it would make the pool 2/2,
-  # where any single OSD down (every rolling node reboot, every OSD restart)
+  # min_size is left at the size-3 default of 2. The explicit min_size 1
+  # override existed only because Ceph does not lower min_size when size is
+  # reduced, which would have left the pool 2/2 — where any single OSD down
   # blocks IO for its PGs and hangs every RBD volume in the cluster.
-  #
-  # The trade this accepts: while the pool is degraded (one OSD down), a second
-  # drive failing loses data, and with all three drives at 80%+ endurance that
-  # window is not hypothetical. Mitigations that already exist: CNPG barman and
-  # Velero both ship offsite to Hetzner (opentofu/object_storage.tf), so the
-  # exposure is the RPO of those backups, not total loss. Keep node reboots
-  # serialised and wait for HEALTH_OK between them.
   replicated:
-    size: 2
-  parameters:
-    min_size: "1"
+    size: 3
 ---
 apiVersion: storage.k8s.io/v1
 kind: StorageClass
@@ -87,35 +80,26 @@ metadata:
   name: cephfs
   namespace: rook-ceph
 spec:
-  # TEMPORARY — REVERT TO size 3 / no min_size OVERRIDE ONCE k8s-1 IS BACK.
+  # size 3 with no min_size override. This pool holds ~253 MiB against the data
+  # pool's 280 GiB, and losing it would destroy the whole filesystem rather
+  # than just the objects on the lost OSD, so it is the last pool that should
+  # ever run thin.
   #
-  # The standing trade for this pool is size 3: it holds ~253 MiB against the
-  # data pool's 280 GiB, so it contributes nothing measurable to drive wear —
-  # the entire reason data0 dropped to 2 — while losing it would destroy the
-  # whole filesystem, not just the objects on the lost OSD. Paying 3x on a
-  # quarter of a gigabyte to keep that intact is the right trade in steady
-  # state.
-  #
-  # It does not hold while one of three hosts is out for the jupiter bare-metal
-  # conversion. size 3 on two remaining hosts leaves every PG undersized at two
-  # copies with min_size 2, so a single further OSD loss drops the pool below
-  # min_size and blocks IO on every CephFS mount cluster-wide. At 2/1 the same
-  # two copies are active+clean instead, and a further loss degrades rather
-  # than blocks — availability bought at the cost of accepting writes onto a
-  # single copy while degraded.
+  # It drops to 2/1 whenever one of the three hosts is out for longer than a
+  # reboot. size 3 on two remaining hosts leaves every PG undersized at two
+  # copies with min_size 2, so a single further OSD loss falls below min_size
+  # and blocks IO on every CephFS mount cluster-wide. At 2/1 the same two
+  # copies are active+clean instead, and a further loss degrades rather than
+  # blocks. See RUNBOOK-baremetal-conversion.md.
   metadataPool:
     replicated:
-      size: 2
-    parameters:
-      min_size: "1"
+      size: 3
   dataPools:
     - name: data0
-      # size 2 / min_size 1 for the same reasons as replicapool above — see
-      # the comment there for the full rationale and the accepted trade.
+      # size 3 for the same reasons as replicapool above — see the comment
+      # there for why this pool sat at 2 while the Crucials backed the OSDs.
       replicated:
-        size: 2
-      parameters:
-        min_size: "1"
+        size: 3
   metadataServer:
     activeCount: 1
     activeStandby: true

kubernetes/talos/cilium.yaml

@@ -310,6 +310,13 @@ data:
   # policy map (per endpoint). Keep in lockstep with patches/cilium-cni.yaml and
   # values.bpf.policyMapMax in kubernetes/infrastructure/cilium/helmrelease.yaml.
   bpf-policy-map-max: "65536"
+  # VLAN ids the eBPF datapath may pass; without this it drops VLAN-tagged
+  # frames. 30 is v30iot, the macvlan parent for the IoT workloads, and a VLAN
+  # sub-interface of enp2s0f0np0 on bare metal. VLAN 40 (mgmt0) rides the same
+  # parent but passes without an entry, so it is deliberately not listed.
+  # Keep in lockstep with patches/cilium-cni.yaml and values.bpf.vlanBypass in
+  # kubernetes/infrastructure/cilium/helmrelease.yaml. See issue #592.
+  vlan-bpf-bypass: "30"
   # bpf-policy-stats-map-max specifies the maximum number of entries in global
   # policy stats map
   bpf-policy-stats-map-max: "65536"

kubernetes/talos/patches/cilium-cni.yaml

@@ -331,6 +331,16 @@ cluster:
           # inline manifest, so a lower value here means a cold rebuild comes up with
           # a map that overflows until Flux reconciles.
           bpf-policy-map-max: "65536"
+          # VLAN ids the eBPF datapath may pass; without this it drops
+          # VLAN-tagged frames. 30 is v30iot, the macvlan parent for the IoT
+          # workloads, and a VLAN sub-interface of enp2s0f0np0 on bare metal.
+          # VLAN 40 (mgmt0) rides the same parent but passes without an entry,
+          # so it is deliberately not listed. Keep in lockstep with
+          # values.bpf.vlanBypass in kubernetes/infrastructure/cilium/helmrelease.yaml
+          # and the third copy in kubernetes/talos/cilium.yaml — bootstrap installs
+          # from this inline manifest, so omitting it here breaks the IoT macvlan
+          # workloads until Flux reconciles. See issue #592.
+          vlan-bpf-bypass: "30"
           # bpf-policy-stats-map-max specifies the maximum number of entries in global
           # policy stats map
           bpf-policy-stats-map-max: "65536"

kubernetes/talos/patches/cluster.yaml

@@ -5,6 +5,31 @@ machine:
     - k8s-3.srv.renner.uno
     - k8s.renner.uno
     - 10.2.40.60
+  # Install from the Image Factory metal installer, not the stock
+  # ghcr.io/siderolabs/installer that `talosctl gen config` defaults to.
+  # Extensions live in the installer image, so with the stock one the schematic
+  # is only ever a property of the boot media and nothing lands on disk —
+  # `talosctl get extensions` returns empty on a node installed that way.
+  #
+  # What the schematic carries, and why each matters here:
+  #
+  #   siderolabs/intel-ucode   CPU microcode. On a VM the hypervisor supplied
+  #                            it; on bare metal nothing does except the BIOS.
+  #   siderolabs/thunderbolt   Inert until machine.kernel.modules asks for
+  #                            thunderbolt/thunderbolt_net. Present so enabling
+  #                            the MS-01 mesh later needs neither a new image
+  #                            nor a reinstall — which is exactly the property
+  #                            lost by installing without it.
+  #
+  # a21cec8f… is the bare-metal schematic described in RUNBOOK-cold-rebuild.md:
+  # same net.ifnames=1 as the Proxmox one (the predictable interface names the
+  # per-node VLAN parents match against), minus qemu-guest-agent.
+  #
+  # Only read at install and upgrade time. Changing it here does nothing to a
+  # running node until that node is upgraded. Bump the tag with the Talos
+  # version — it must match the running version, or the upgrade is a downgrade.
+  install:
+    image: factory.talos.dev/metal-installer/a21cec8f35ae32ab5e9a452e7b13500ca8776053930b8e182b9cb23eed186b64:v1.12.6
   features:
     rbac: true
     stableHostname: true

kubernetes/talos/patches/k8s-1.yaml

@@ -1,7 +1,7 @@
 # k8s-1 runs Talos directly on jupiter's MS-01 hardware; k8s-2 and k8s-3 are
 # still Proxmox VMs. Every machine-specific selector below differs from theirs
-# for that reason: there are no virtio NICs, no PCI-passthrough layer, and both
-# NVMe drives are visible to Talos at once.
+# for that reason: there are no virtio NICs, no PCI-passthrough layer, and all
+# four physical NICs and all three NVMe drives are visible to Talos at once.
 #
 # Networking uses the v1.12 multi-document config rather than the deprecated
 # machine.network.interfaces. That is deliberate: the docs do not define what
@@ -9,9 +9,6 @@
 # every link in one schema instead of straddling them.
 machine:
   network:
-    # Shared control-plane VIP 10.2.40.60 is configured as a Layer2VIPConfig
-    # below rather than machine.network.interfaces[].vip. Exposed via UniFi DNS
-    # as k8s.renner.uno.
     hostname: k8s-1
   sysctls:
     # ens19 (VLAN 30, v30iot) is LINK ONLY — no host addressing. It exists as
@@ -24,10 +21,12 @@ machine:
     # This disables v6 entirely on that link, including link-local.
     net.ipv6.conf.ens19.disable_ipv6: "1"
   install:
-    # Both NVMe drives are now ordinary block devices to Talos: the Kingston
-    # OM8PGP41024Q-A0 system disk and the Samsung PM983 that Rook claims as the
-    # Ceph OSD. Selecting the installer target by serial means it cannot land
-    # on the OSD drive regardless of which one the kernel enumerates first.
+    # Three NVMe drives are ordinary block devices to Talos here: the Kingston
+    # OM8PGP41024Q-A0 system disk, the Samsung PM983 that Rook claims as the
+    # Ceph OSD, and the retired Crucial P3 that carries EPHEMERAL (see the
+    # VolumeConfig below). Every one of them is selected by serial. Nothing in
+    # this file may use a broad selector: `disk.transport == 'nvme'` matches all
+    # three, and the failure mode is Talos taking the OSD drive.
     diskSelector:
       serial: 50026B7382B48B86
   certSANs:
@@ -37,40 +36,70 @@ machine:
 cluster:
   allowSchedulingOnControlPlanes: true
 ---
-# Physical NICs are aliased by permanent MAC. Both X710 ports bind i40e, so the
-# driver selector k8s-2/k8s-3 use for their single passed-through port would
-# match two links here. MAC matching is unambiguous and survives udev renaming.
+# EPHEMERAL (/var) goes on the retired Crucial P3, not the system disk.
+#
+# That drive is at 87% of its rated endurance and was this node's Ceph OSD until
+# the PM983 replaced it. /var is exactly the right role for a disk in that
+# state: containerd's image and snapshot store, pod logs and the Ceph mon store
+# are all reconstructible, so the drive is consumable here in a way it is not as
+# an OSD. It also gives /var ~931 GB instead of the 64 GB the VM-era zvol had,
+# which is the constraint that makes k8s-2 and k8s-3 trip DiskPressure whenever
+# they absorb another node's pods.
+#
+# THIS ONLY TAKES EFFECT ON FIRST PROVISION. Talos applies volume configuration
+# solely when the volume does not yet exist; a config applied after EPHEMERAL
+# has been created elsewhere is silently ignored, and relocating it then
+# requires wiping the disk and booting install media again.
+#
+# Matched by serial deliberately. `disk.transport == 'nvme' && !system_disk`
+# would match the PM983 as well, and losing that coin flip means Talos formats
+# the OSD drive.
 apiVersion: v1alpha1
-kind: LinkAliasConfig
-name: lan0
-selector:
-  match: glob("58:47:ca:74:70:27", mac(link.permanent_addr))
+kind: VolumeConfig
+name: EPHEMERAL
+provisioning:
+  diskSelector:
+    match: disk.serial == '2332E86B749A'
+  minSize: 50GiB
+  grow: true
 ---
-apiVersion: v1alpha1
-kind: LinkAliasConfig
-name: ceph0
-selector:
-  match: glob("58:47:ca:74:70:28", mac(link.permanent_addr))
----
-# lan0 is the switch trunk (X710 port 0). It carries no addressing of its own;
-# VLAN 40 and VLAN 30 ride on top. Proxmox's vlan-aware vmbr0 did this tagging
-# before, which is why the VM saw two already-untagged virtio NICs instead.
+# X710 port 0 carries the switch trunk. It takes no addressing itself — the
+# port's untagged VLAN is the 192.168.8.0/24 network, which this node has no
+# business being on — so management and v30iot both ride it as tagged VLANs.
+#
+# Links are addressed by predictable kernel name. Those derive from PCI
+# topology, not probe order — though not immune: adding the third NVMe
+# renumbered the buses and moved the unused second I226 from enp89s0 to
+# enp90s0. Neither name used here was affected.
+#
+# LinkAliasConfig would be the better way to pin these, and is worth
+# reintroducing — but only once it is proven. Aliasing both X710 ports by
+# permanent MAC and using the aliases as VLAN parents produced a node with no
+# address on any interface: the aliases were never created, so every document
+# referencing them did nothing, including the static address on port 1. Verify
+# any alias-based revision with `talosctl apply-config --mode=try`, which rolls
+# back on its own, before making it permanent.
 apiVersion: v1alpha1
 kind: LinkConfig
-name: lan0
+name: enp2s0f0np0
 up: true
 ---
+# Management, VLAN 40. Carries the node IP, etcd, the API and Ceph's public
+# network. The k8s-1 VM reached 10.2.40.174 over this same physical port via
+# Proxmox's v40srv VNet (vmbr0.40), so the switch trunks VLAN 40 here.
 apiVersion: v1alpha1
 kind: VLANConfig
 name: mgmt0
 vlanID: 40
-parent: lan0
+parent: enp2s0f0np0
 up: true
 ---
 apiVersion: v1alpha1
 kind: DHCPv4Config
 name: mgmt0
 ---
+# Shared control-plane VIP, etcd-elected. Exposed via UniFi DNS as
+# k8s.renner.uno.
 apiVersion: v1alpha1
 kind: Layer2VIPConfig
 name: 10.2.40.60
@@ -80,21 +109,20 @@ link: mgmt0
 # `master: ens19`, and k8s-2/k8s-3 still present the VLAN as a virtio NIC of
 # that name. VLANConfig.name is free-form, so keeping it lets one set of NADs
 # serve both VM and bare-metal nodes for as long as the cluster is mixed.
-# Nothing else on this machine is called ens19: the physical NICs are
-# enp2s0f0np0, enp2s0f1np1, enp87s0 and enp89s0.
 apiVersion: v1alpha1
 kind: VLANConfig
 name: ens19
 vlanID: 30
-parent: lan0
+parent: enp2s0f0np0
 up: true
 ---
-# Dedicated Ceph cluster network, VLAN 49 / 10.99.0.0/29, untagged on the wire
-# on X710 port 1. Used by Rook for OSD<->OSD replication via
-# CephCluster.spec.network.addressRanges.cluster. No default route.
+# Dedicated Ceph cluster network on X710 port 1, VLAN 49 / 10.99.0.0/29,
+# untagged on the wire. Used by Rook for OSD<->OSD replication via
+# CephCluster.spec.network.addressRanges.cluster. Not routed off its own
+# segment, and deliberately has no default route.
 apiVersion: v1alpha1
 kind: LinkConfig
-name: ceph0
+name: enp2s0f1np1
 up: true
 mtu: 9000
 addresses:

kubernetes/talos/patches/k8s-2.yaml

@@ -1,31 +1,98 @@
+# k8s-2 runs Talos directly on saturn's MS-01 hardware. Structurally identical
+# to k8s-1.yaml — see that file for the reasoning behind each choice; only the
+# serials, MACs and addresses differ.
 machine:
   network:
     hostname: k8s-2
-    # Shared control-plane VIP — see k8s-1.yaml for the busPath rationale.
-    interfaces:
-      - deviceSelector:
-          busPath: "0000:06:12.0"
-        dhcp: true
-        vip:
-          ip: 10.2.40.60
-      # v30iot: link only, no host addressing — see k8s-1.yaml.
-      - deviceSelector:
-          busPath: "0000:06:13.0"
-        dhcp: false
-      # See k8s-1.yaml for context on the dedicated Ceph cluster network.
-      - deviceSelector:
-          driver: i40e
-        addresses:
-          - 10.99.0.2/29
-        mtu: 9000
   sysctls:
-    # v30iot NIC carries no host addressing — see k8s-1.yaml.
+    # ens19 (VLAN 30, v30iot) is LINK ONLY — no host addressing. It is the
+    # macvlan parent for the v30iot pod attachments; host addresses on that L2
+    # make node replies bypass the UDM and break every UDM-DNATed flow.
     net.ipv6.conf.ens19.disable_ipv6: "1"
   install:
-    disk: /dev/vda
+    # Three NVMe drives are visible on bare metal. Select the installer target
+    # by serial so it cannot land on the OSD drive or on EPHEMERAL, whatever
+    # order the kernel enumerates them in — Talos and Proxmox do not agree on
+    # that order.
+    diskSelector:
+      serial: 50026B7382DA5F6E
   certSANs:
     - k8s-2.srv.renner.uno
   nodeLabels:
     topology.kubernetes.io/zone: saturn
 cluster:
   allowSchedulingOnControlPlanes: true
+---
+# EPHEMERAL (/var) goes on the retired Crucial P3, not the system disk. That
+# drive is at 81% of its rated endurance and is finished as an OSD, but
+# everything EPHEMERAL holds is reconstructible, so it is consumable in this
+# role. It also gives /var ~931 GB instead of the 64 GB the VM-era zvol had.
+#
+# ONLY TAKES EFFECT ON FIRST PROVISION. Talos applies volume configuration
+# solely when the volume does not yet exist; applied later it is silently
+# ignored and relocating needs a disk wipe and a boot from install media.
+#
+# Matched by serial deliberately: a broad selector such as
+# `disk.transport == 'nvme' && !system_disk` would match the PM983 as well.
+apiVersion: v1alpha1
+kind: VolumeConfig
+name: EPHEMERAL
+provisioning:
+  diskSelector:
+    match: disk.serial == '2332E86B74BC'
+  minSize: 50GiB
+  grow: true
+---
+# X710 port 0 carries the switch trunk and takes no addressing itself — the
+# port's untagged VLAN is not one this node belongs on. Management and v30iot
+# both ride it as tagged VLANs.
+#
+# Addressed by predictable kernel name, not by LinkAliasConfig. Aliases are
+# valid VLAN parents, but a selector matching zero links is skipped silently
+# and Talos disables default DHCP machine-wide as soon as any new-style network
+# document exists — so a silent miss leaves the node with no address on any
+# interface. See RUNBOOK-baremetal-conversion.md.
+apiVersion: v1alpha1
+kind: LinkConfig
+name: enp2s0f0np0
+up: true
+---
+# Management, VLAN 40. Node IP, etcd, the API and Ceph's public network.
+apiVersion: v1alpha1
+kind: VLANConfig
+name: mgmt0
+vlanID: 40
+parent: enp2s0f0np0
+up: true
+---
+apiVersion: v1alpha1
+kind: DHCPv4Config
+name: mgmt0
+---
+# Shared control-plane VIP, etcd-elected. Exposed via UniFi DNS as
+# k8s.renner.uno.
+apiVersion: v1alpha1
+kind: Layer2VIPConfig
+name: 10.2.40.60
+link: mgmt0
+---
+# Named ens19 on purpose: the four v30iot macvlan NADs hard-code
+# `master: ens19`, and k8s-3 still presents the VLAN as a virtio NIC of that
+# name. VLANConfig.name is free-form, so one set of NADs serves both.
+apiVersion: v1alpha1
+kind: VLANConfig
+name: ens19
+vlanID: 30
+parent: enp2s0f0np0
+up: true
+---
+# Dedicated Ceph cluster network on X710 port 1, VLAN 49 / 10.99.0.0/29,
+# untagged on the wire. OSD<->OSD replication only. Not routed off its own
+# segment and deliberately has no default route.
+apiVersion: v1alpha1
+kind: LinkConfig
+name: enp2s0f1np1
+up: true
+mtu: 9000
+addresses:
+  - address: 10.99.0.2/29

kubernetes/talos/patches/k8s-3.yaml

@@ -1,31 +1,82 @@
+# k8s-3 runs Talos directly on neptune's MS-01 hardware. Structurally identical
+# to k8s-1.yaml — see that file for the reasoning behind each choice; only the
+# serials and addresses differ. neptune was the last node converted, so all
+# three control-plane nodes now share this layout and no Proxmox remains.
 machine:
   network:
     hostname: k8s-3
-    # Shared control-plane VIP — see k8s-1.yaml for the busPath rationale.
-    interfaces:
-      - deviceSelector:
-          busPath: "0000:06:12.0"
-        dhcp: true
-        vip:
-          ip: 10.2.40.60
-      # v30iot: link only, no host addressing — see k8s-1.yaml.
-      - deviceSelector:
-          busPath: "0000:06:13.0"
-        dhcp: false
-      # See k8s-1.yaml for context on the dedicated Ceph cluster network.
-      - deviceSelector:
-          driver: i40e
-        addresses:
-          - 10.99.0.3/29
-        mtu: 9000
   sysctls:
     # v30iot NIC carries no host addressing — see k8s-1.yaml.
     net.ipv6.conf.ens19.disable_ipv6: "1"
   install:
-    disk: /dev/vda
+    # Three NVMe drives are visible on bare metal. Select the installer target
+    # by serial so it cannot land on the OSD drive or on EPHEMERAL, whatever
+    # order the kernel enumerates them in — Talos and Proxmox do not agree on
+    # that order.
+    diskSelector:
+      serial: 50026B7382B48B90
   certSANs:
     - k8s-3.srv.renner.uno
   nodeLabels:
     topology.kubernetes.io/zone: neptune
 cluster:
   allowSchedulingOnControlPlanes: true
+---
+# EPHEMERAL (/var) goes on the retired Crucial P3, not the system disk. That
+# drive is at 83% of its rated endurance and is finished as an OSD, but
+# everything EPHEMERAL holds is reconstructible, so it is consumable in this
+# role. Available spare is still 100% and it reports no media errors.
+#
+# ONLY TAKES EFFECT ON FIRST PROVISION. Talos applies volume configuration
+# solely when the volume does not yet exist; applied later it is silently
+# ignored and relocating needs a disk wipe and a boot from install media.
+#
+# Matched by serial deliberately: a broad selector such as
+# `disk.transport == 'nvme' && !system_disk` would match the PM983 as well.
+apiVersion: v1alpha1
+kind: VolumeConfig
+name: EPHEMERAL
+provisioning:
+  diskSelector:
+    match: disk.serial == '2326E6E8E67D'
+  minSize: 50GiB
+  grow: true
+---
+apiVersion: v1alpha1
+kind: LinkConfig
+name: enp2s0f0np0
+up: true
+---
+apiVersion: v1alpha1
+kind: VLANConfig
+name: mgmt0
+vlanID: 40
+parent: enp2s0f0np0
+up: true
+---
+apiVersion: v1alpha1
+kind: DHCPv4Config
+name: mgmt0
+---
+# Shared control-plane VIP, held by whichever node wins the etcd campaign.
+apiVersion: v1alpha1
+kind: Layer2VIPConfig
+name: 10.2.40.60
+link: mgmt0
+---
+# v30iot: link only, no host addressing — see k8s-1.yaml.
+apiVersion: v1alpha1
+kind: VLANConfig
+name: ens19
+vlanID: 30
+parent: enp2s0f0np0
+up: true
+---
+# Dedicated Ceph cluster network — see k8s-1.yaml.
+apiVersion: v1alpha1
+kind: LinkConfig
+name: enp2s0f1np1
+up: true
+mtu: 9000
+addresses:
+  - address: 10.99.0.3/29

opentofu/modules/proxmox_vm/variables.tf

@@ -47,8 +47,13 @@ variable "tags" {
 }
 
 variable "template_name" {
-  type    = string
-  default = "ubuntu-24-04-template"
+  type = string
+  # No default clone source. The ubuntu-24.04-template it used to name was
+  # deleted with saturn's conversion to bare metal, and Proxmox is being
+  # retired, so a default pointing at a template that does not exist would
+  # only fail at apply time. Set it explicitly if a clone source is ever
+  # wanted again.
+  default = null
 }
 
 variable "full_clone" {

opentofu/removed_k8s_3.tf

@@ -0,0 +1,27 @@
+# k8s-3 was the last Proxmox VM. neptune now runs Talos on bare metal, and its
+# disks were wiped during the conversion, so the VM this module described does
+# not exist in any form.
+#
+# `removed` rather than simply deleting vm_152_k8s_3.tf: dropping the resource
+# from configuration alone makes OpenTofu plan a DESTROY, and planning a destroy
+# refreshes the resource through the bpg/proxmox provider — which now has no
+# endpoint to reach. That is exactly what has been failing every infrastructure
+# plan since the conversion:
+#
+#   Error: failed to get resources list of type ("vm") for cluster:
+#   dial tcp 10.2.40.201:8006: connect: no route to host
+#     with module.k8s_3.proxmox_virtual_environment_vm.this
+#
+# `lifecycle { destroy = false }` forgets the state entry without attempting to
+# contact Proxmox. Nothing is orphaned by doing so: the guest, its disks and the
+# hypervisor underneath it are all already gone.
+#
+# This block can be deleted once one plan has run cleanly with it — it is a
+# one-shot instruction, not standing configuration.
+removed {
+  from = module.k8s_3
+
+  lifecycle {
+    destroy = false
+  }
+}

opentofu/scripts/remove_unifi_clients.sh

@@ -1,52 +0,0 @@
-#!/bin/bash
-# Script to help with UniFi client migration
-#
-# NOTE: This script is NO LONGER RECOMMENDED
-# Instead, follow the migration guide in MIGRATION_QUICKSTART.md
-#
-# This script removes unifi_client resources from state, but the recommended
-# approach is to let 'tofu apply' handle the removal automatically.
-
-echo "⚠️  WARNING: This script is deprecated"
-echo ""
-echo "Please use the recommended migration path instead:"
-echo "  1. Deploy DNS server: tofu apply -target=module.proxmox_dns"
-echo "  2. Configure UniFi DNS to use 10.2.40.53"
-echo "  3. Run full apply: tofu apply"
-echo ""
-echo "See MIGRATION_QUICKSTART.md for details."
-echo ""
-read -p "Do you still want to remove resources from state? (y/N) " -n 1 -r
-echo
-if [[ ! $REPLY =~ ^[Yy]$ ]]; then
-    echo "Cancelled. See MIGRATION_QUICKSTART.md for recommended approach."
-    exit 0
-fi
-
-set -e
-
-echo "Removing unifi_client resources from OpenTofu state..."
-echo "Note: VMs will keep their current network configuration"
-echo ""
-
-# List of all unifi_client resources to remove
-RESOURCES=(
-  "module.cdns_srv_renner_uno.unifi_client.this[0]"
-  "module.fortress_renner_uno.unifi_client.this[0]"
-  "module.grafana_srv_renner_uno.unifi_client.this[0]"
-  "module.influxdb_srv_renner_uno.unifi_client.this[0]"
-  "module.kasm_srv_renner_uno.unifi_client.this[0]"
-  "module.mosquitto_srv_renner_uno.unifi_client.this[0]"
-  "module.netbox_srv_renner_uno.unifi_client.this[0]"
-  "module.ntp_srv_renner_uno.unifi_client.this[0]"
-  "module.paperless_srv_renner_uno.unifi_client.this[0]"
-  "module.uptime_kuma_srv_renner_uno.unifi_client.this[0]"
-)
-
-for resource in "${RESOURCES[@]}"; do
-  echo "Removing: $resource"
-  tofu state rm "$resource" || echo "  (already removed or not found)"
-done
-
-echo ""
-echo "Done! Run 'tofu plan' to verify."

opentofu/talos_image.tf

@@ -1,13 +0,0 @@
-locals {
-  talos_version = "v1.12.6"
-  # Image Factory schematic (predictable naming: net.ifnames=1). Keep in
-  # sync with TALOS_SCHEMATIC in .woodpecker/deploy-infrastructure.yml —
-  # that pipeline downloads the image, this local names it for VM import,
-  # and both MUST agree on the filename. The schematic hash is embedded in
-  # the filename so a schematic bump changes the name, forcing a fresh
-  # download instead of silently reusing a stale cached image (which would
-  # reintroduce net.ifnames=0 on the next VM (re)provision).
-  talos_schematic = "88e3752db032a08082972bd0adf16a80849e3870c3f95cf5b669936c5699e5ad"
-  talos_filename  = "talos-${local.talos_version}-${substr(local.talos_schematic, 0, 12)}-nocloud-amd64.raw"
-  talos_image     = "local:import/${local.talos_filename}"
-}

opentofu/vm_151_k8s_2.tf

@@ -1,50 +0,0 @@
-module "k8s_2" {
-  source = "./modules/proxmox_vm"
-
-  name        = "k8s-2.srv.renner.uno"
-  target_node = "saturn"
-  vmid        = 151
-  memory      = 49152
-  cores       = 8
-
-  template_name = null
-  cloud_init    = false
-  full_clone    = false
-  bios          = "ovmf"
-  machine       = "q35"
-  boot          = "order=virtio0"
-  agent_enabled = true
-
-  # Second NIC on v30iot so Cilium L2 announce can win ARP on the IoT
-  # VLAN from this node (previously only k8s-edge had v30iot, so any
-  # IoT-pool LB lease landing here was unreachable from devices).
-  extra_network_bridges = ["v30iot"]
-
-  disks = [
-    {
-      storage = "local-zfs"
-      size    = "64G"
-      format  = "raw"
-      slot    = "virtio0"
-      file_id = local.talos_image
-    }
-  ]
-
-  hostpci = [
-    {
-      mapping = "osd-ssd"
-    },
-    {
-      mapping = "x710-port1"
-    }
-  ]
-
-  tags            = "kubernetes;talos;controlplane"
-  ssh_public_keys = local.common_ssh_keys
-
-  vm_state = "running"
-
-  # Rebuild pipeline issues `-replace`; skip graceful shutdown so a
-  # non-responsive Talos guest can't hang the destroy for 30 minutes.
-  stop_on_destroy = true
-}

opentofu/vm_152_k8s_3.tf

@@ -1,50 +0,0 @@
-module "k8s_3" {
-  source = "./modules/proxmox_vm"
-
-  name        = "k8s-3.srv.renner.uno"
-  target_node = "neptune"
-  vmid        = 152
-  memory      = 49152
-  cores       = 8
-
-  template_name = null
-  cloud_init    = false
-  full_clone    = false
-  bios          = "ovmf"
-  machine       = "q35"
-  boot          = "order=virtio0"
-  agent_enabled = true
-
-  # Second NIC on v30iot so Cilium L2 announce can win ARP on the IoT
-  # VLAN from this node (previously only k8s-edge had v30iot, so any
-  # IoT-pool LB lease landing here was unreachable from devices).
-  extra_network_bridges = ["v30iot"]
-
-  disks = [
-    {
-      storage = "local-zfs"
-      size    = "64G"
-      format  = "raw"
-      slot    = "virtio0"
-      file_id = local.talos_image
-    }
-  ]
-
-  hostpci = [
-    {
-      mapping = "osd-ssd"
-    },
-    {
-      mapping = "x710-port1"
-    }
-  ]
-
-  tags            = "kubernetes;talos;controlplane"
-  ssh_public_keys = local.common_ssh_keys
-
-  vm_state = "running"
-
-  # Rebuild pipeline issues `-replace`; skip graceful shutdown so a
-  # non-responsive Talos guest can't hang the destroy for 30 minutes.
-  stop_on_destroy = true
-}

opentofu/vm_605_template.tf

@@ -1,71 +0,0 @@
-resource "proxmox_virtual_environment_vm" "ubuntu_24_04_template" {
-  name      = "ubuntu-24.04-template"
-  node_name = "saturn"
-  vm_id     = 605
-
-  started = false
-  template = true
-  on_boot = false
-
-  keyboard_layout = "en-us"
-
-  bios = "seabios"
-
-  agent {
-    enabled = true
-  }
-
-  cpu {
-    cores = 2
-    type  = "x86-64-v3"
-  }
-
-  memory {
-    dedicated = 2048
-  }
-
-  initialization {
-    datastore_id = "local-zfs"
-
-    ip_config {
-      ipv4 {
-        address = "dhcp"
-      }
-    }
-
-    user_account {
-      username = "ladmin"
-      keys = [
-        "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPzFNBP3bq2m/6JqDqod9w+FP/uxXxHMolqhq3Ojflr1 krenner@fortress"
-      ]
-    }
-  }
-
-  disk {
-    datastore_id = "local-zfs"
-    interface    = "scsi0"
-    size         = 32
-    file_format  = "raw"
-    iothread     = false
-  }
-
-  network_device {
-    bridge = "v40srv"
-    model  = "virtio"
-  }
-
-  serial_device {}
-
-  scsi_hardware = "virtio-scsi-pci"
-
-  lifecycle {
-    ignore_changes = [
-      node_name,
-      started,
-      initialization,
-      disk,
-      cpu,
-      memory
-    ]
-  }
-}