chore(opentofu): drop the spent removed block for module.k8s_3

The removal has been applied:

  Apply complete! Resources: 0 added, 0 changed, 0 destroyed, 1 forgotten.

`tofu state list` now returns no Proxmox resources at all, and a full plan with
refresh enabled produces no Proxmox errors — the "no route to host" against
10.2.40.201:8006 that failed every infrastructure plan since the conversion is
gone.

The block was always one-shot. Left in place it refers to a state entry that no
longer exists, which is harmless but misleading: it reads as pending work.

Worth recording for the next time this shape comes up, because it cost a failed
merge to learn. A `removed` block alone does not fix an unreachable provider:
OpenTofu still refreshes the resource before it can forget it, and that refresh
is exactly what cannot succeed. It takes one run with `-refresh=false` to
process. `-target` does not help — targeting does not reach `removed` blocks, so
a targeted plan reports "No changes" while the state entry sits there untouched.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

.woodpecker/alert-monitor.yml

@@ -135,8 +135,38 @@ steps:
         TITLES=$$(jq -c '[.[].title]' .ci-alert-monitor/open-issues.json)
         NOISE_BODY=$$(jq -r 'map(select(.title == "Alert monitor: noise log")) | .[0].body // ""' \
           .ci-alert-monitor/open-issues.json)
-        FPS=$$(printf '%s' "$$NOISE_BODY" | grep -oE '\b[0-9a-f]{16}\b' \
+        # Suppression entries are exactly the structured lines the write-back
+        # step emits: "- <fp> <ALERTNAME> — <YYYY-MM-DD>: <reason>". Parse THAT
+        # shape rather than grepping the whole body for 16-hex tokens. A reason
+        # routinely cites another alert's fingerprint in prose ("see the issue
+        # filed for fingerprint <fp>"), and the body-wide grep turned every such
+        # citation into a silent, permanent suppression of an alert that was
+        # never classified as noise — verified against the live noise log, which
+        # yielded 22 fingerprints for 21 real entries, the extra one being
+        # 2ddc41edb689fc33, an alert that HAD been filed as a genuine issue.
+        #
+        # Entries also expire after NOISE_TTL_DAYS. A noise verdict says "this
+        # occurrence was benign", not "this alert can never be interesting
+        # again" — but the fingerprint is sha256(alertname|scope) and therefore
+        # stable across recurrences, so without an expiry a single blip mutes
+        # that alertname+scope pair forever. Re-triaging a recurrence weeks
+        # later is cheap; missing a real outage because a July blip muted
+        # openbao's KubePodNotReady is not.
+        #
+        # Lines that fail to parse are simply dropped, i.e. they stop
+        # suppressing — this fails toward MORE alerting, never less.
+        #
+        # date is BusyBox here, not GNU: `date -d '30 days ago'` errors out, so
+        # the cutoff goes through epoch arithmetic and `date -d @<epoch>`, which
+        # this image does support.
+        NOISE_TTL_DAYS=$${NOISE_TTL_DAYS:-30}
+        CUTOFF=$$(date -u -d @$$(( $$(date -u +%s) - NOISE_TTL_DAYS * 86400 )) +%F)
+        FPS=$$(printf '%s\n' "$$NOISE_BODY" \
+          | sed -nE 's/^- ([0-9a-f]{16}) [^ ]+ — ([0-9]{4}-[0-9]{2}-[0-9]{2}):.*/\2 \1/p' \
+          | awk -v c="$$CUTOFF" 'BEGIN{gsub(/-/,"",c)} { d=$$1; gsub(/-/,"",d); if (d+0 >= c+0) print $$2 }' \
           | jq -R -s -c 'split("\n") | map(select(length > 0))')
+        echo "Noise suppressions in effect: $$(printf '%s' "$${FPS:-[]}" | jq length)" \
+             "(TTL $${NOISE_TTL_DAYS}d, entries dated >= $$CUTOFF)"
         # Synthesize pseudo-alerts for BLIPS: lookback series whose label set
         # is not currently firing at all (checked against the PRE-filter raw
         # snapshot, so live-but-null-routed alerts don't reappear as blips).
@@ -260,7 +290,7 @@ steps:
               '{body: ($$old + "\n" + $$add)}' > /tmp/gitea-patch.json
             gitea_write PATCH "$$API/issues/$$NOISE_NUM" /tmp/gitea-patch.json
           else
-            printf '%s\n\n' "Fingerprints the alert-monitor triage agent classified as noise. The dedup step skips any alert whose fingerprint appears below — delete a line to make that alert eligible for triage again. Keep this issue open; it is the suppression state." > /tmp/noise-header.txt
+            printf '%s\n\n' "Fingerprints the alert-monitor triage agent classified as noise. The dedup step skips any alert whose fingerprint appears below — delete a line to make that alert eligible for triage again. Keep this issue open; it is the suppression state. Entries EXPIRE after NOISE_TTL_DAYS (default 30) counted from the date in the line, so a one-off blip stops suppressing its alertname+scope pair once the window passes; re-add happens automatically if it is triaged as noise again. Only lines in the exact '- <fingerprint> <ALERTNAME> — <YYYY-MM-DD>: <reason>' form suppress anything — a fingerprint mentioned in prose does not." > /tmp/noise-header.txt
             jq -n --rawfile hdr /tmp/noise-header.txt --rawfile add /tmp/noise-lines.txt \
               '{title: "Alert monitor: noise log", body: ($$hdr + $$add)}' > /tmp/gitea-create.json
             gitea_write POST "$$API/issues" /tmp/gitea-create.json

.woodpecker/deploy-infrastructure.yml

@@ -13,7 +13,7 @@ when:
 
 steps:
   - name: validate-secrets
-    image: alpine:3.21
+    image: alpine:3.24.1
     environment:
       TF_VAR_proxmox_api_url:
         from_secret: proxmox_api_url
@@ -128,35 +128,6 @@ steps:
       - printf "Host *\n\tStrictHostKeyChecking no\n" > ~/.ssh/config
       - echo "$VAULT_PASS" > /tmp/.vault_pass && chmod 600 /tmp/.vault_pass
       - export ANSIBLE_VAULT_PASSWORD_FILE=/tmp/.vault_pass
-      - |
-        # Ensure Talos image exists on all Proxmox nodes via API
-        TALOS_VERSION="v1.12.6"
-        TALOS_SCHEMATIC="88e3752db032a08082972bd0adf16a80849e3870c3f95cf5b669936c5699e5ad"
-        TALOS_URL="https://factory.talos.dev/image/$TALOS_SCHEMATIC/$TALOS_VERSION/nocloud-amd64.raw"
-        TALOS_FILENAME="talos-$TALOS_VERSION-$(echo "$TALOS_SCHEMATIC" | cut -c1-12)-nocloud-amd64.raw"
-        API_TOKEN="$TF_VAR_proxmox_user!$TF_VAR_proxmox_token_id=$TF_VAR_proxmox_token_secret"
-        for node in jupiter saturn neptune; do
-          EXISTS=$(curl -sk "$TF_VAR_proxmox_api_url/api2/json/nodes/$node/storage/local/content" \
-            -H "Authorization: PVEAPIToken=$API_TOKEN" 2>/dev/null | grep -c "$TALOS_FILENAME" || true)
-          if [ "$EXISTS" -gt 0 ]; then
-            echo "Talos image already exists on $node"
-            continue
-          fi
-          echo "Downloading Talos image to $node..."
-          curl -sk "$TF_VAR_proxmox_api_url/api2/json/nodes/$node/storage/local/download-url" \
-            -H "Authorization: PVEAPIToken=$API_TOKEN" \
-            --data-urlencode "url=$TALOS_URL" \
-            --data-urlencode "content=import" \
-            --data-urlencode "filename=$TALOS_FILENAME" \
-            --max-time 600
-          echo "Waiting for download on $node..."
-          for i in $(seq 1 60); do
-            sleep 10
-            EXISTS=$(curl -sk "$TF_VAR_proxmox_api_url/api2/json/nodes/$node/storage/local/content" \
-              -H "Authorization: PVEAPIToken=$API_TOKEN" 2>/dev/null | grep -c "$TALOS_FILENAME" || true)
-            if [ "$EXISTS" -gt 0 ]; then echo "Download complete on $node"; break; fi
-          done
-        done
       - tofu init -upgrade -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY"
       - |
         if [ -n "$$TOFU_STATE_RM" ]; then

ansible/roles/gatus/templates/config.yaml.j2

@@ -253,22 +253,6 @@ endpoints:
 {% endif %}
 
   # Infrastructure
-  - name: Proxmox DNS (VIP)
-    group: infrastructure
-    url: 10.2.40.53
-    interval: 30s
-    dns:
-      query-name: k8s-1.srv.renner.uno
-      query-type: A
-    conditions:
-      - "[DNS_RCODE] == NOERROR"
-      - "[RESPONSE_TIME] < 500"
-    alerts:
-      - type: email
-{% if gatus_webhook_url %}
-      - type: custom
-{% endif %}
-
   - name: PostgreSQL HA (VIP)
     group: infrastructure
     url: tcp://10.2.40.135:5432

dns-server/config.yaml.example

@@ -1,31 +0,0 @@
-# Proxmox DNS Server Configuration
-
-# Proxmox API connection
-proxmox:
-  host: "pve.renner.uno"
-  user: "root@pam"
-
-  # Token-based authentication (recommended)
-  token_name: "dns"
-  token_value: "your-token-secret-here"
-
-  # OR password-based authentication
-  # password: "your-password-here"
-
-  verify_ssl: false
-
-# DNS server settings
-dns_host: "0.0.0.0"  # Listen on all interfaces
-dns_port: 53         # Standard DNS port (requires root or CAP_NET_BIND_SERVICE)
-
-# DNS suffix for VM/LXC hostnames
-dns_suffix: ".renner.uno"
-
-# TTL for DNS responses (in seconds)
-dns_ttl: 300
-
-# Cache settings
-cache_ttl: 60  # How often to refresh the cache (in seconds)
-
-# Upstream DNS server for non-Proxmox queries
-upstream_dns: "10.2.40.1"

dns-server/go.mod

@@ -1,15 +0,0 @@
-module proxmox-dns
-
-go 1.21
-
-require (
-	github.com/miekg/dns v1.1.58
-	gopkg.in/yaml.v3 v3.0.1
-)
-
-require (
-	golang.org/x/mod v0.14.0 // indirect
-	golang.org/x/net v0.20.0 // indirect
-	golang.org/x/sys v0.16.0 // indirect
-	golang.org/x/tools v0.17.0 // indirect
-)

dns-server/go.sum

@@ -1,16 +0,0 @@
-github.com/miekg/dns v1.1.58 h1:ca2Hdkz+cDg/7eNF6V56jjzuZ4aCAE+DbVkILdQWG/4=
-github.com/miekg/dns v1.1.58/go.mod h1:Ypv+3b/KadlvW9vJfXOTf300O4UqaHFzFCuHz+rPkBY=
-golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0=
-golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
-golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo=
-golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY=
-golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ=
-golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
-golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU=
-golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/tools v0.17.0 h1:FvmRgNOcs3kOa+T20R1uhfP9F6HgG2mfxDv1vrx1Htc=
-golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
-gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

dns-server/main.go

@@ -1,565 +0,0 @@
-package main
-
-import (
-	"crypto/tls"
-	"encoding/json"
-	"flag"
-	"fmt"
-	"io"
-	"log"
-	"net"
-	"net/http"
-	"os"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/miekg/dns"
-	"gopkg.in/yaml.v3"
-)
-
-type Config struct {
-	Proxmox struct {
-		Host       string `yaml:"host"`
-		User       string `yaml:"user"`
-		TokenName  string `yaml:"token_name"`
-		TokenValue string `yaml:"token_value"`
-		VerifySSL  bool   `yaml:"verify_ssl"`
-	} `yaml:"proxmox"`
-	DNSHost     string `yaml:"dns_host"`
-	DNSPort     int    `yaml:"dns_port"`
-	DNSSuffix   string `yaml:"dns_suffix"`
-	DNSTTL      int    `yaml:"dns_ttl"`
-	CacheTTL    int    `yaml:"cache_ttl"`
-	UpstreamDNS string `yaml:"upstream_dns"`
-}
-
-type DNSRecord struct {
-	IPv4      string
-	IPv6      string
-	Timestamp time.Time
-}
-
-type ProxmoxDNS struct {
-	config     Config
-	cache      map[string]*DNSRecord
-	cacheMux   sync.RWMutex
-	client     *http.Client
-	stopChan   chan struct{}
-	nodes      []string
-	nodesMux   sync.RWMutex
-}
-
-func NewProxmoxDNS(configPath string) (*ProxmoxDNS, error) {
-	data, err := os.ReadFile(configPath)
-	if err != nil {
-		return nil, fmt.Errorf("failed to read config: %w", err)
-	}
-
-	var config Config
-	if err := yaml.Unmarshal(data, &config); err != nil {
-		return nil, fmt.Errorf("failed to parse config: %w", err)
-	}
-
-	// Set defaults
-	if config.DNSPort == 0 {
-		config.DNSPort = 53
-	}
-	if config.CacheTTL == 0 {
-		config.CacheTTL = 60
-	}
-	if config.DNSTTL == 0 {
-		config.DNSTTL = 300
-	}
-	if config.UpstreamDNS == "" {
-		config.UpstreamDNS = "10.2.40.1:53"
-	}
-	if !strings.Contains(config.UpstreamDNS, ":") {
-		config.UpstreamDNS += ":53"
-	}
-
-	return &ProxmoxDNS{
-		config: config,
-		cache:  make(map[string]*DNSRecord),
-		nodes:  []string{},
-		client: &http.Client{
-			Timeout: 10 * time.Second,
-			Transport: &http.Transport{
-				TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
-			},
-		},
-		stopChan: make(chan struct{}),
-	}, nil
-}
-
-// isValidIPv6ForDNS checks if an IPv6 address is suitable for DNS records
-func isValidIPv6ForDNS(ip net.IP) bool {
-	if ip == nil || ip.To16() == nil {
-		return false
-	}
-
-	// Reject loopback
-	if ip.IsLoopback() {
-		return false
-	}
-
-	// Reject link-local (fe80::/10) - THIS IS THE KEY FIX
-	if ip.IsLinkLocalUnicast() {
-		return false
-	}
-
-	// Reject multicast
-	if ip.IsMulticast() {
-		return false
-	}
-
-	// Reject unspecified
-	if ip.IsUnspecified() {
-		return false
-	}
-
-	// Accept global unicast and ULA (fc00::/7)
-	return true
-}
-
-func (p *ProxmoxDNS) queryProxmoxAPI(endpoint string) ([]byte, error) {
-	// Remove port from host if present
-	host := p.config.Proxmox.Host
-	if !strings.Contains(host, ":") {
-		host += ":8006"
-	}
-
-	url := fmt.Sprintf("https://%s/api2/json/%s", host, endpoint)
-	req, err := http.NewRequest("GET", url, nil)
-	if err != nil {
-		return nil, err
-	}
-
-	auth := fmt.Sprintf("PVEAPIToken=%s!%s=%s",
-		p.config.Proxmox.User,
-		p.config.Proxmox.TokenName,
-		p.config.Proxmox.TokenValue)
-	req.Header.Set("Authorization", auth)
-
-	resp, err := p.client.Do(req)
-	if err != nil {
-		return nil, err
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode != 200 {
-		body, _ := io.ReadAll(resp.Body)
-		return nil, fmt.Errorf("API returned status %d: %s", resp.StatusCode, string(body))
-	}
-
-	return io.ReadAll(resp.Body)
-}
-
-func (p *ProxmoxDNS) discoverNodes() error {
-	data, err := p.queryProxmoxAPI("nodes")
-	if err != nil {
-		return fmt.Errorf("failed to discover nodes: %w", err)
-	}
-
-	var response struct {
-		Data []struct {
-			Node   string `json:"node"`
-			Status string `json:"status"`
-		} `json:"data"`
-	}
-
-	if err := json.Unmarshal(data, &response); err != nil {
-		return fmt.Errorf("failed to parse nodes: %w", err)
-	}
-
-	p.nodesMux.Lock()
-	defer p.nodesMux.Unlock()
-
-	p.nodes = []string{}
-	for _, node := range response.Data {
-		if node.Status == "online" {
-			p.nodes = append(p.nodes, node.Node)
-		}
-	}
-
-	log.Printf("Discovered %d online nodes: %v", len(p.nodes), p.nodes)
-	return nil
-}
-
-func (p *ProxmoxDNS) refreshCache() {
-	log.Println("Refreshing DNS cache...")
-
-	// Discover nodes first
-	if err := p.discoverNodes(); err != nil {
-		log.Printf("Failed to discover nodes: %v", err)
-		return
-	}
-
-	newCache := make(map[string]*DNSRecord)
-
-	p.nodesMux.RLock()
-	nodes := append([]string{}, p.nodes...)
-	p.nodesMux.RUnlock()
-
-	for _, node := range nodes {
-		// Query VMs
-		p.refreshVMs(node, newCache)
-		// Query LXC containers
-		p.refreshLXC(node, newCache)
-	}
-
-	p.cacheMux.Lock()
-	p.cache = newCache
-	p.cacheMux.Unlock()
-
-	log.Printf("Cache refreshed: %d entries", len(newCache))
-}
-
-func (p *ProxmoxDNS) refreshVMs(node string, cache map[string]*DNSRecord) {
-	data, err := p.queryProxmoxAPI(fmt.Sprintf("nodes/%s/qemu", node))
-	if err != nil {
-		log.Printf("Failed to query VMs on %s: %v", node, err)
-		return
-	}
-
-	var response struct {
-		Data []struct {
-			VMID   int    `json:"vmid"`
-			Name   string `json:"name"`
-			Status string `json:"status"`
-		} `json:"data"`
-	}
-
-	if err := json.Unmarshal(data, &response); err != nil {
-		log.Printf("Failed to parse VM data: %v", err)
-		return
-	}
-
-	for _, vm := range response.Data {
-		if vm.Status != "running" || vm.Name == "" {
-			continue
-		}
-
-		// Query agent network info
-		agentData, err := p.queryProxmoxAPI(fmt.Sprintf("nodes/%s/qemu/%d/agent/network-get-interfaces", node, vm.VMID))
-		if err != nil {
-			log.Printf("VM %s (ID %d): no guest agent data", vm.Name, vm.VMID)
-			continue
-		}
-
-		var agentResp struct {
-			Data struct {
-				Result []struct {
-					Name        string `json:"name"`
-					IPAddresses []struct {
-						IPAddress     string `json:"ip-address"`
-						IPAddressType string `json:"ip-address-type"`
-					} `json:"ip-addresses"`
-				} `json:"result"`
-			} `json:"data"`
-		}
-
-		if err := json.Unmarshal(agentData, &agentResp); err != nil {
-			continue
-		}
-
-		var ipv4, ipv6 string
-		for _, iface := range agentResp.Data.Result {
-			if iface.Name == "lo" {
-				continue
-			}
-
-			for _, ipAddr := range iface.IPAddresses {
-				if ipAddr.IPAddressType == "ipv4" && ipv4 == "" {
-					if !strings.HasPrefix(ipAddr.IPAddress, "127.") {
-						ipv4 = ipAddr.IPAddress
-					}
-				} else if ipAddr.IPAddressType == "ipv6" && ipv6 == "" {
-					ip := net.ParseIP(ipAddr.IPAddress)
-					if isValidIPv6ForDNS(ip) {
-						ipv6 = ipAddr.IPAddress
-						log.Printf("  VM %s: accepting IPv6 %s", vm.Name, ipv6)
-					} else if ip != nil {
-						log.Printf("  VM %s: rejecting IPv6 %s (link-local/loopback/multicast)", vm.Name, ipAddr.IPAddress)
-					}
-				}
-			}
-
-			// Stop after finding both on first non-lo interface
-			if ipv4 != "" && ipv6 != "" {
-				break
-			}
-		}
-
-		if ipv4 != "" || ipv6 != "" {
-			hostname := strings.ToLower(vm.Name)
-			// Strip DNS suffix from hostname if present
-			hostname = strings.TrimSuffix(hostname, p.config.DNSSuffix)
-			cache[hostname] = &DNSRecord{
-				IPv4:      ipv4,
-				IPv6:      ipv6,
-				Timestamp: time.Now(),
-			}
-			log.Printf("VM: %s -> IPv4=%s IPv6=%s", hostname, ipv4, ipv6)
-		}
-	}
-}
-
-func (p *ProxmoxDNS) refreshLXC(node string, cache map[string]*DNSRecord) {
-	data, err := p.queryProxmoxAPI(fmt.Sprintf("nodes/%s/lxc", node))
-	if err != nil {
-		log.Printf("Failed to query LXC on %s: %v", node, err)
-		return
-	}
-
-	var response struct {
-		Data []struct {
-			VMID   int    `json:"vmid"`
-			Name   string `json:"name"`
-			Status string `json:"status"`
-		} `json:"data"`
-	}
-
-	if err := json.Unmarshal(data, &response); err != nil {
-		log.Printf("Failed to parse LXC data: %v", err)
-		return
-	}
-
-	for _, lxc := range response.Data {
-		if lxc.Status != "running" || lxc.Name == "" {
-			continue
-		}
-
-		// Query LXC interfaces
-		ifaceData, err := p.queryProxmoxAPI(fmt.Sprintf("nodes/%s/lxc/%d/interfaces", node, lxc.VMID))
-		if err != nil {
-			log.Printf("LXC %s (ID %d): failed to get interfaces", lxc.Name, lxc.VMID)
-			continue
-		}
-
-		var ifaceResp struct {
-			Data []struct {
-				Name   string `json:"name"`
-				Inet   string `json:"inet"`
-				Inet6  string `json:"inet6"`
-			} `json:"data"`
-		}
-
-		if err := json.Unmarshal(ifaceData, &ifaceResp); err != nil {
-			continue
-		}
-
-		var ipv4, ipv6 string
-		for _, iface := range ifaceResp.Data {
-			if iface.Name == "lo" {
-				continue
-			}
-
-			if iface.Inet != "" && ipv4 == "" {
-				// Format is "IP/CIDR"
-				parts := strings.Split(iface.Inet, "/")
-				if len(parts) > 0 && !strings.HasPrefix(parts[0], "127.") {
-					ipv4 = parts[0]
-				}
-			}
-
-			if iface.Inet6 != "" && ipv6 == "" {
-				parts := strings.Split(iface.Inet6, "/")
-				if len(parts) > 0 {
-					ip := net.ParseIP(parts[0])
-					if isValidIPv6ForDNS(ip) {
-						ipv6 = parts[0]
-						log.Printf("  LXC %s: accepting IPv6 %s", lxc.Name, ipv6)
-					} else if ip != nil {
-						log.Printf("  LXC %s: rejecting IPv6 %s (link-local/loopback/multicast)", lxc.Name, parts[0])
-					}
-				}
-			}
-
-			// Stop after finding both on first non-lo interface
-			if ipv4 != "" && ipv6 != "" {
-				break
-			}
-		}
-
-		if ipv4 != "" || ipv6 != "" {
-			hostname := strings.ToLower(lxc.Name)
-			// Strip DNS suffix from hostname if present
-			hostname = strings.TrimSuffix(hostname, p.config.DNSSuffix)
-			cache[hostname] = &DNSRecord{
-				IPv4:      ipv4,
-				IPv6:      ipv6,
-				Timestamp: time.Now(),
-			}
-			log.Printf("LXC: %s -> IPv4=%s IPv6=%s", hostname, ipv4, ipv6)
-		}
-	}
-}
-
-func (p *ProxmoxDNS) handleDNSRequest(w dns.ResponseWriter, r *dns.Msg) {
-	m := new(dns.Msg)
-	m.SetReply(r)
-	m.Authoritative = true
-
-	for _, q := range r.Question {
-		name := strings.ToLower(strings.TrimSuffix(q.Name, "."))
-		qtype := dns.TypeToString[q.Qtype]
-
-		// Check if query is for our domain suffix
-		if !strings.HasSuffix(name, strings.TrimPrefix(p.config.DNSSuffix, ".")) {
-			// Forward to upstream
-			log.Printf("Query %s (%s) - forwarding to upstream", name, qtype)
-			p.forwardToUpstream(w, r)
-			return
-		}
-
-		// Extract hostname (remove suffix)
-		hostname := strings.TrimSuffix(name, p.config.DNSSuffix)
-
-		p.cacheMux.RLock()
-		record, found := p.cache[hostname]
-		p.cacheMux.RUnlock()
-
-		if !found {
-			// NXDOMAIN
-			m.SetRcode(r, dns.RcodeNameError)
-			log.Printf("Query %s (%s) - NXDOMAIN", name, qtype)
-			w.WriteMsg(m)
-			return
-		}
-
-		switch q.Qtype {
-		case dns.TypeA:
-			if record.IPv4 != "" {
-				rr := &dns.A{
-					Hdr: dns.RR_Header{
-						Name:   q.Name,
-						Rrtype: dns.TypeA,
-						Class:  dns.ClassINET,
-						Ttl:    uint32(p.config.DNSTTL),
-					},
-					A: net.ParseIP(record.IPv4),
-				}
-				m.Answer = append(m.Answer, rr)
-				log.Printf("Query %s (A) -> %s", name, record.IPv4)
-			} else {
-				// NODATA (host exists but no A record)
-				m.Ns = append(m.Ns, p.getSOARecord())
-				log.Printf("Query %s (A) - NODATA", name)
-			}
-
-		case dns.TypeAAAA:
-			if record.IPv6 != "" {
-				// Parse and validate IPv6 again before returning
-				ip := net.ParseIP(record.IPv6)
-				if ip != nil && isValidIPv6ForDNS(ip) {
-					rr := &dns.AAAA{
-						Hdr: dns.RR_Header{
-							Name:   q.Name,
-							Rrtype: dns.TypeAAAA,
-							Class:  dns.ClassINET,
-							Ttl:    uint32(p.config.DNSTTL),
-						},
-						AAAA: ip,
-					}
-					m.Answer = append(m.Answer, rr)
-					log.Printf("Query %s (AAAA) -> %s", name, record.IPv6)
-				} else {
-					// IPv6 is invalid (link-local etc)
-					m.Ns = append(m.Ns, p.getSOARecord())
-					log.Printf("Query %s (AAAA) - NODATA (invalid IPv6 in cache)", name)
-				}
-			} else {
-				// NODATA (host exists but no AAAA record)
-				m.Ns = append(m.Ns, p.getSOARecord())
-				log.Printf("Query %s (AAAA) - NODATA", name)
-			}
-		}
-	}
-
-	w.WriteMsg(m)
-}
-
-func (p *ProxmoxDNS) getSOARecord() dns.RR {
-	domain := strings.TrimPrefix(p.config.DNSSuffix, ".")
-	soa := &dns.SOA{
-		Hdr: dns.RR_Header{
-			Name:   domain + ".",
-			Rrtype: dns.TypeSOA,
-			Class:  dns.ClassINET,
-			Ttl:    300,
-		},
-		Ns:      "ns." + domain + ".",
-		Mbox:    "admin." + domain + ".",
-		Serial:  uint32(time.Now().Unix()),
-		Refresh: 3600,
-		Retry:   600,
-		Expire:  86400,
-		Minttl:  300,
-	}
-	return soa
-}
-
-func (p *ProxmoxDNS) forwardToUpstream(w dns.ResponseWriter, r *dns.Msg) {
-	c := new(dns.Client)
-	c.Timeout = 2 * time.Second
-
-	resp, _, err := c.Exchange(r, p.config.UpstreamDNS)
-	if err != nil {
-		log.Printf("Failed to forward to upstream: %v", err)
-		m := new(dns.Msg)
-		m.SetRcode(r, dns.RcodeServerFailure)
-		w.WriteMsg(m)
-		return
-	}
-
-	w.WriteMsg(resp)
-}
-
-func (p *ProxmoxDNS) startCacheRefresher() {
-	ticker := time.NewTicker(time.Duration(p.config.CacheTTL) * time.Second)
-	defer ticker.Stop()
-
-	// Initial refresh
-	p.refreshCache()
-
-	for {
-		select {
-		case <-ticker.C:
-			p.refreshCache()
-		case <-p.stopChan:
-			return
-		}
-	}
-}
-
-func (p *ProxmoxDNS) Start() error {
-	// Start cache refresher
-	go p.startCacheRefresher()
-
-	// Start DNS server
-	dns.HandleFunc(".", p.handleDNSRequest)
-
-	addr := fmt.Sprintf("%s:%d", p.config.DNSHost, p.config.DNSPort)
-	log.Printf("Starting Proxmox DNS server on %s", addr)
-	log.Printf("Base domain: %s", p.config.DNSSuffix)
-	log.Printf("Upstream DNS: %s", p.config.UpstreamDNS)
-
-	server := &dns.Server{Addr: addr, Net: "udp"}
-	return server.ListenAndServe()
-}
-
-func main() {
-	configPath := flag.String("c", "/etc/proxmox-dns/config.yaml", "Path to configuration file")
-	flag.Parse()
-
-	server, err := NewProxmoxDNS(*configPath)
-	if err != nil {
-		log.Fatalf("Failed to create DNS server: %v", err)
-	}
-
-	log.Printf("Proxmox DNS Server starting...")
-	if err := server.Start(); err != nil {
-		log.Fatalf("Failed to start DNS server: %v", err)
-	}
-}

dns-server/proxmox-dns.service

@@ -1,27 +0,0 @@
-[Unit]
-Description=Proxmox DNS Server
-After=network.target
-Wants=network-online.target
-
-[Service]
-Type=simple
-User=root
-Group=root
-WorkingDirectory=/opt/proxmox-dns
-ExecStart=/usr/bin/python3 /opt/proxmox-dns/proxmox_dns.py -c /etc/proxmox-dns/config.yaml
-Restart=always
-RestartSec=10
-StandardOutput=journal
-StandardError=journal
-
-# Security hardening
-NoNewPrivileges=true
-PrivateTmp=true
-ProtectSystem=strict
-ProtectHome=true
-ReadWritePaths=/var/log
-AmbientCapabilities=CAP_NET_BIND_SERVICE
-
-[Install]
-WantedBy=multi-user.target
-

kubernetes/RECOVERY.md

@@ -167,7 +167,7 @@ permanent recovery.
 
 ## CNPG restore-test (verify backups are restorable — non-destructive)
 
-The weekly Velero restore-test (`kubernetes/infrastructure/velero/`) exercises
+The monthly Velero restore-test (`kubernetes/infrastructure/velero/`) exercises
 PVC restores but **explicitly does not touch CNPG** (barman-cloud is a separate
 code path). Run this drill periodically to prove the CNPG backups are actually
 restorable, **without touching the live clusters**. It restores into a throwaway

kubernetes/RUNBOOK-baremetal-conversion.md

@@ -0,0 +1,411 @@
+# Runbook — converting a control-plane node from Proxmox VM to bare metal
+
+Converts one MS-01 from "Proxmox host running a Talos VM" to "Talos on bare
+metal", and swaps its Ceph OSD onto a Samsung PM983 at the same time. Written
+after doing jupiter on 2026-08-25; saturn and neptune remain.
+
+Everything here that reads like an over-specific warning is something that
+actually happened. The conversion itself is straightforward — the failures were
+all in state that outlived the VM.
+
+## What the node ends up as
+
+| | |
+|---|---|
+| System disk | Kingston OM8PGP41024Q-A0, selected by serial |
+| EPHEMERAL (`/var`) | the retired Crucial P3, selected by serial |
+| Ceph OSD | Samsung PM983, selected by `/dev/disk/by-id` |
+| Management | VLAN 40 tagged on X710 port 0 |
+| v30iot | VLAN 30 on the same port, named `ens19` |
+| Ceph cluster net | X710 port 1, `10.99.0.x/29`, MTU 9000 |
+
+## Before you start
+
+- [ ] All three OSDs up, all mons in quorum, PGs `active+clean`. Do not start
+      from a degraded cluster; the window itself removes one of three.
+- [ ] Cluster fits on two nodes. Check `kubectl describe node` allocation on the
+      two survivors. Take immich down (`kubernetes/apps/immich`, both
+      Deployments to `replicas: 0`) — it frees ~1000m against a shortfall of
+      roughly 200m and takes its image pulls off the remaining nodes.
+- [ ] `cephfs-metadata` to size 2 / min_size 1 for the duration. At size 3 on
+      two hosts every PG is undersized with min_size 2, so one further OSD loss
+      blocks IO on every CephFS mount cluster-wide.
+- [ ] Talos metal ISO on a USB stick. Schematic
+      `a21cec8f35ae32ab5e9a452e7b13500ca8776053930b8e182b9cb23eed186b64`
+      (`net.ifnames=1`, intel-ucode, thunderbolt; no qemu-guest-agent).
+      Verify by comparing partition offsets and sizes against the ISO with
+      `fdisk -l` — blkid labels lie after a rewrite.
+- [ ] Physical access. Secure Boot must be turned off (see below), and that is
+      the one step nothing can do remotely.
+
+## 1. Evacuate the OSD
+
+```sh
+ceph osd out <id>
+```
+
+Wait until no PGs are `backfilling` or `backfill_wait`. Do **not** wait for
+`ceph osd safe-to-destroy` — pools at size 3 keep the OSD in their acting set
+because three copies cannot be placed on two hosts, so it never returns clean.
+
+Then remove it. The node must already be out of `cephcluster.yaml`'s
+`storage.nodes` list, or the operator rebuilds the OSD within one reconcile and
+starts backfilling onto the drive you are about to pull:
+
+```sh
+kubectl -n rook-ceph scale deployment rook-ceph-osd-<id> --replicas=0
+ceph osd purge <id> --yes-i-really-mean-it
+kubectl -n rook-ceph delete deployment rook-ceph-osd-<id>
+```
+
+## 2. Drain, and leave the cluster
+
+```sh
+kubectl drain <node> --ignore-daemonsets --delete-emptydir-data --timeout=300s
+```
+
+Single-replica workloads with `minAvailable: 1` PDBs will refuse eviction — that
+is deliberate, they exist to surface the pod rather than let it be yanked. On
+this cluster that is `influxdb-influxdb2-0`, `woodpecker-server` and the
+CNPG primary. Move each one deliberately with `kubectl delete pod`; they
+reschedule onto the survivors and their RBD volumes follow.
+
+```sh
+talosctl -n <node> reset --graceful --wipe-mode system-disk
+kubectl delete node <node>
+```
+
+`--wipe-mode system-disk` and not the default `all`: the OSD drive is being
+pulled anyway and wiping a terabyte of it is wasted time. `--graceful` leaves
+etcd cleanly. Delete the Node object — it carries the `unschedulable` flag from
+the drain, which the rebuilt node would otherwise inherit.
+
+## 2a. Destroy the VM in OpenTofu — while the host is still a PVE node
+
+Remove `opentofu/vm_15N_k8s_N.tf`, then apply. This **must** happen before the
+host leaves the Proxmox cluster: afterwards the provider cannot reach the node,
+the plan errors, and the only escape is `tofu state rm`, which is off the table.
+
+Point `proxmox_api_url` at a host that will *survive* the conversion, not the one
+being converted.
+
+Manual triggers are currently broken by the Woodpecker v3.18.0 regression
+(issue #582). Until that is resolved, run it locally:
+
+```sh
+AWS_ACCESS_KEY_ID=… AWS_SECRET_ACCESS_KEY=… AWS_REGION=fsn1 \
+  tofu -chdir=opentofu init -backend-config=<file>
+… plan -target=module.k8s_N -var-file=<file> -out=<plan>
+… apply <plan>
+```
+
+The `AWS_*` environment variables are required *in addition to*
+`-backend-config`, or the S3 backend fails `HeadObject` with 403 even with
+credentials that can read the object fine via the aws CLI.
+
+## 2b. Proxmox quorum — matters from the second conversion onward
+
+Two nodes remain after jupiter, with `expected_votes: 2` and quorum 2. Removing
+one leaves the last node inquorate and `/etc/pve` read-only, which blocks the
+final VM destroy. On the surviving node:
+
+```sh
+pvecm expected 1
+```
+
+This is the official recovery. Proxmox documents it in the storage replication
+chapter (`/usr/share/pve-docs/chapter-pvesr.html`), verbatim:
+
+> If you have no quorum, we strongly advise to fix this first and make the node
+> operable again. Only if this is not possible at the moment, you may use the
+> following command to enforce quorum on the current node: `# pvecm expected 1`
+
+The caveat there is about restoring the missing node instead. Here the missing
+node is deliberately gone for good, so enforcing quorum is the correct branch
+rather than a workaround.
+
+Run it only once the node is actually alone — attempting it while both nodes are
+up fails with `CS_ERR_INVALID_PARAM`, since corosync will not set expected votes
+below the members it can currently see. Which is exactly why it becomes valid
+when one remains.
+
+Runtime only; it does not survive a reboot, so re-run it if the survivor
+restarts before its own conversion.
+
+## 3. Hardware
+
+Power the host off, then:
+
+1. Kingston boot drive out of slot 1, into slot 2.
+2. Crucial P3 out of slot 2 — it becomes EPHEMERAL, so put it in slot 3 rather
+   than setting it aside.
+3. Slot 1 switch to **U.2**. Set it before the drive goes in; there is nothing
+   to record afterwards, the installed drive type implies the position.
+4. PM983 onto the included 7mm M.2-to-U.2 adapter, into slot 1.
+
+Boot the host's existing PVE once before touching Talos and check the drive:
+
+```sh
+nvme smart-log /dev/nvmeXn1 | grep -iE 'percentage_used|media_errors|critical'
+nvme id-ns /dev/nvmeXn1 -H | grep -i 'in use'      # want 512b data, 0 metadata
+wipefs /dev/nvmeYn1                                 # Crucial: clear the old bluestore sig
+ip link set <ceph-port> up && ethtool <ceph-port> | grep -i 'link detected'
+```
+
+This is the last revert point. A bad drive here means the Crucial goes back and
+the VM is recreated from tofu with nothing lost. Once Talos writes to the
+Kingston, PVE on that host is gone.
+
+Relocating the boot drive is safe: `rpool` is referenced by NVMe EUI and the
+UEFI entries by GPT PARTUUID, neither of which encodes a slot.
+
+## 4. Secure Boot
+
+MS-01 ships with Secure Boot on, and Proxmox satisfies it with a
+Microsoft-signed shim. The Image Factory `metal-amd64.iso` is an unsigned UKI
+and the firmware rejects it outright. **Turn Secure Boot off in the BIOS.**
+
+A signed variant exists per schematic, but it is signed with SideroLabs' key
+which firmware does not trust by default, and the installed system would have to
+use the secureboot installer image too. That is a separate project.
+
+## 5. Install
+
+Boot the stick. Maintenance mode DHCPs untagged on every link; on these hosts
+that lands on `192.168.8.x`, which is routable from the workstation. It DHCPs on
+*every* link, so one machine answers on several addresses at once — they are the
+same host, not several hosts. `get disks` on each confirms it.
+
+Maintenance mode does serve `talosctl get`. `--insecure` is a flag on the
+subcommand rather than a global, so it must come *after* `get` — placed before
+it, cobra resolves to the legacy `talosctl disks` command and reports
+`unknown flag: --insecure`:
+
+```sh
+talosctl -n <maintenance-ip> get disks --insecure
+talosctl -n <maintenance-ip> get links --insecure
+```
+
+Verify serials and MACs here, on the machine itself, against the config about to
+be applied.
+
+Build the config from the **live cluster's secrets**, not a fresh
+`talosctl gen config`, or the node forms a new cluster instead of joining:
+
+```sh
+talosctl -n <surviving-node> get mc v1alpha1 -o jsonpath='{.spec}' > cp.yaml
+talosctl gen secrets --from-controlplane-config cp.yaml -o secrets.yaml
+talosctl gen config homelab https://k8s.renner.uno:6443 \
+  --with-secrets secrets.yaml --output-types controlplane \
+  --config-patch @talos/patches/cluster.yaml \
+  --config-patch @<cilium additive half> \
+  --config-patch @talos/patches/registry-mirror.yaml \
+  --config-patch @talos/patches/k8s-N.yaml -o install.yaml
+```
+
+Confirm `cluster.id` matches the running cluster before applying.
+
+Strip the generated `HostnameConfig` document — it collides with
+`machine.network.hostname` from the per-node patch. `bootstrap.sh` does the same.
+
+`cilium-cni.yaml` cannot be passed whole: its `$patch: delete` document fails
+against a base with no `inlineManifests` (issue #581). Use only the additive
+second document.
+
+Validate:
+
+```sh
+talosctl validate --config install.yaml --mode metal
+```
+
+### Install minimal, complete on the running node
+
+Do **not** apply the full per-node config in maintenance mode. A document that
+references an interface which does not exist yet hangs the install: the node
+sits at `STAGE: Installing` indefinitely with `KUBELET n/a`, `APISERVER n/a`,
+and the console scrolling
+
+```
+k8s.NodeApplyController: error getting node: nodes "k8s-N" not found
+```
+
+That message is a symptom of an install that never finished, not a Kubernetes or
+networking fault — the node holds a DHCP address and is reachable throughout. On
+saturn the offenders were the `ens19` VLAN document and the sysctl naming it,
+`net.ipv6.conf.ens19.disable_ipv6`. Three cold installs hung on it; the identical
+config applied cleanly once the node was up. Wiping disks makes no difference —
+it is not leftover state.
+
+**Stage 1** — the smallest config that installs and joins, plus `EPHEMERAL`,
+which is one-shot (section 7) and so cannot be deferred to stage 2:
+
+```
+machine.network.hostname, machine.install.diskSelector, certSANs, nodeLabels
+LinkConfig    <uplink>
+VLANConfig    mgmt0
+DHCPv4Config  mgmt0
+VolumeConfig  EPHEMERAL
+```
+
+```sh
+talosctl apply-config --insecure -n <maintenance-ip> --file install-stage1.yaml
+```
+
+**Stage 2** — once the node is `Ready` and `talosctl etcd members` lists it,
+apply the full config to the running node. `--mode=try` reverts on its own if the
+network config is wrong, so it cannot strand the node; re-apply with
+`--mode=auto` to persist.
+
+```sh
+talosctl -e <node-ip> -n <node-ip> apply-config --mode=try --timeout=3m --file install.yaml
+talosctl -e <node-ip> -n <node-ip> apply-config --mode=auto --file install.yaml
+```
+
+Both applied without a reboot on saturn. Confirm `ens19`, `mgmt0` and the Ceph
+link are up afterwards with `talosctl get links` and `get addresses`.
+
+## 6. The network config will strand the node if it is wrong
+
+Talos stops running default DHCP operators **for the entire machine** as soon as
+a single new-style network document exists — the gate is presence of a document,
+not success of resolution. If the configured path fails to materialise there is
+no fallback on any interface, and recovery needs a console and install media.
+
+Concretely: do not use `LinkAliasConfig` as a `VLANConfig` parent without
+verifying it. Aliases *are* supported (confirmed in v1.12.6 source and by a
+maintainer), but a selector matching zero links is skipped **silently**, and
+`permanent_addr` comes from an ethtool ioctl with no fallback — if the driver
+does not answer, the match is against an empty string and everything downstream
+that references the alias does nothing.
+
+Address links by predictable kernel name. To introduce aliases later, do it on a
+node that is already up, with `talosctl apply-config --mode=try --timeout=5m`,
+which rolls back on its own, and check `talosctl get linkaliasspec`.
+
+Interface names derive from PCI topology and are stable — but not immune.
+Adding a third NVMe to jupiter renumbered the buses and renamed the unused
+second I226 from `enp89s0` to `enp90s0`.
+
+## 7. EPHEMERAL placement is one-shot
+
+```yaml
+apiVersion: v1alpha1
+kind: VolumeConfig
+name: EPHEMERAL
+provisioning:
+  diskSelector:
+    match: disk.serial == '<crucial serial>'
+  minSize: 50GiB
+  grow: true
+```
+
+Talos applies volume configuration **only when the volume does not yet exist**.
+Applied afterwards it is silently ignored, and relocating EPHEMERAL then needs a
+disk wipe and a fresh boot from install media. It must be in the first
+`apply-config`.
+
+Match by serial. `disk.transport == 'nvme' && !system_disk` matches the PM983 as
+well, and losing that coin flip formats the OSD drive.
+
+Verify afterwards:
+
+```sh
+talosctl -n <node> get volumestatus EPHEMERAL -o yaml | grep -E 'location|phase'
+talosctl -n <node> get disks
+```
+
+## 8. The mon will not come back on its own
+
+**This is the step that cost the most time on jupiter.** The rebuilt node has a
+new MAC and therefore a new DHCP lease, but Ceph has its mon pinned to the old
+address in three places. The mon cannot bind an address the host no longer has,
+so it crashloops; the operator then tries to restart a *healthy* mon as its next
+step, Ceph refuses because that would drop quorum, and it retries every 60
+seconds forever without ever reaching the storage reconcile. Nothing else
+progresses behind it.
+
+Fix in this order — the order matters, because doing it the other way round has
+the operator faithfully recreate the mon from the stale ConfigMap:
+
+```sh
+# 1. ConfigMap first
+kubectl -n rook-ceph edit cm rook-ceph-mon-endpoints
+#    remove the dead mon from `data` and from `mapping`
+
+# 2. then the monmap
+ceph mon remove <id>
+
+# 3. then the deployment — check it really carries the old address
+kubectl -n rook-ceph get deploy rook-ceph-mon-<id> -o yaml | grep public-addr
+kubectl -n rook-ceph delete deploy rook-ceph-mon-<id>
+```
+
+Rook then creates a **new** mon with the next free ID (jupiter's `c` came back
+as `d`), on the correct address. It does not repair the old one.
+
+Note mons use host ports under `network.provider: host`, so a replacement mon
+cannot schedule while the broken one's deployment still holds 3300/6789 — the
+canary sits `Pending` with "didn't have free ports" until the old deployment is
+deleted.
+
+## 9. Re-add the OSD
+
+Add the node back to `cephcluster.yaml` under `storage.nodes`, naming the PM983
+by `/dev/disk/by-id`:
+
+```yaml
+- name: k8s-N
+  devices:
+    - fullpath: /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_<serial>
+```
+
+**Not** a bare device name. Talos enumerates these drives in a different order
+than Proxmox did — on jupiter `nvme0n1` is the Kingston *system* disk, and the
+PM983 is `nvme2n1`. Rook would probably decline a partitioned system disk, but
+that is luck rather than design.
+
+Get the path from `talosctl -n <node> get disk <dev> -o yaml`, under `symlinks`.
+
+## 10. Verify and restore
+
+```sh
+kubectl get nodes                                   # Ready
+talosctl -n <node> etcd members                     # 3 members
+ceph status                                         # 3 mons in quorum, 3 osds
+ceph osd tree                                       # OSD under the right zone/host
+ceph device ls                                      # PM983 backing the new OSD
+```
+
+Then undo the window changes: `cephfs-metadata` back to size 3, immich back to
+1 replica, and `ceph config set osd osd_mclock_profile balanced` if it was
+raised to `high_recovery_ops` to speed the evacuation.
+
+The retired Crucial leaves a stale entry in `ceph device ls`. Remove it with
+`ceph config-key rm device/<devid>`; the mgr's devicehealth view may lag.
+
+## Known-stale things that will bite
+
+- `bootstrap.sh` cannot generate a config at all right now (issue #581), so a
+  cold rebuild is broken independently of this work.
+- Manual Woodpecker pipelines are broken (issue #582), so anything gated on
+  `PIPELINE == "..."` must be run another way.
+- Cilium's eBPF datapath drops VLAN-tagged traffic unless `bpf.vlanBypass` is
+  set. **This happened** — every v30iot macvlan workload on a converted node
+  stayed in sandbox creation until `vlanBypass: [30]` was added (issue #592).
+
+  Only `ens19` is affected, and the reason is worth knowing: Cilium attaches to
+  devices that carry an address. `cilium-dbg status --verbose` lists `mgmt0`
+  among its devices but not `ens19`, because `ens19` is deliberately
+  address-less. Cilium therefore owns `mgmt0` and sees VLAN 40 frames already
+  untagged by the 8021q layer, while VLAN 30 frames only ever meet the filter
+  on the shared parent `enp2s0f0np0`, still tagged. Any future address-less
+  VLAN carrying pod traffic will need the same treatment.
+
+  The symptom does not look like a VLAN problem. A tc-layer drop reaches an
+  AF_PACKET sender as ENOBUFS, so the CNI dhcp plugin reports `no buffer space
+  available` and `network is down` rather than a DHCP timeout, and the pod sits
+  in `ContainerCreating` behind `plugin type="macvlan" failed (add): error
+  calling DHCP.Allocate: no more tries`. Confirm it with
+  `cilium_drop_count_total{reason=~"VLAN.*"}` per node rather than by reading
+  the CNI logs, and check `node_network_receive_packets_total{device="ens19"}`
+  — it reads 0 on an affected node.

kubernetes/RUNBOOK-cold-rebuild.md

@@ -79,6 +79,30 @@ which also pre-seeds the Talos image onto every Proxmox node. Wait until the
 node FQDNs (`k8s-{1,2,3}.srv.renner.uno`) resolve and the VMs are up before
 Step 2 — `bootstrap.sh` polls DNS but will not create the records.
 
+**k8s-1 is not a VM and OpenTofu does not create it.** jupiter runs Talos on
+bare metal, so this step only provisions k8s-2 (saturn) and k8s-3 (neptune).
+Before Step 2, boot jupiter from the bare-metal image below so it comes up in
+maintenance mode and answers on `k8s-1.srv.renner.uno` — `bootstrap.sh` treats
+all three nodes identically from there, since `apply-config --insecure` does
+not care whether the target is a VM or bare metal.
+
+```
+https://factory.talos.dev/image/a21cec8f35ae32ab5e9a452e7b13500ca8776053930b8e182b9cb23eed186b64/v1.12.6/metal-amd64.iso
+```
+
+That schematic is deliberately not the one `.woodpecker/deploy-infrastructure.yml`
+seeds onto the Proxmox nodes. It keeps the same `net.ifnames=1` kernel argument
+— without it the predictable interface names that `talos/patches/k8s-1.yaml`
+matches its VLAN parents against do not exist — and differs in three ways:
+
+- drops `siderolabs/qemu-guest-agent`, which has nothing to talk to on bare metal
+- adds `siderolabs/intel-ucode`, since no hypervisor is supplying microcode now
+- adds `siderolabs/thunderbolt`, unused so far and inert until
+  `machine.kernel.modules` requests `thunderbolt`/`thunderbolt_net`. It is
+  present only so enabling the MS-01 Thunderbolt mesh later does not require a
+  new image and a reinstall. Note its udev rule auto-authorises any Thunderbolt
+  device attached to the machine once those modules are loaded.
+
 ## Step 2 — talos: form the cluster + get kubeconfig
 
 ```sh

kubernetes/apps/audiobookshelf/deployment.yaml

@@ -121,7 +121,9 @@ spec:
           persistentVolumeClaim:
             claimName: audiobookshelf-books
         - name: le-ca
-          emptyDir: {}
+          emptyDir:
+            medium: Memory
+            sizeLimit: 16Mi
 ---
 apiVersion: v1
 kind: Service

kubernetes/apps/authentik/cluster.yaml

@@ -43,6 +43,44 @@ spec:
     parameters:
       shared_buffers: "256MB"
       max_connections: "400"
+      # Write-reduction tuning. These databases are largely idle, and the
+      # Postgres defaults favour throughput over storage that wears out.
+      #
+      # checkpoint_timeout is the lever. The 5min default pushes every dirty
+      # page to disk 288 times a day, so a small database whose same few pages
+      # are re-dirtied constantly (sessions, tokens, job state) gets rewritten
+      # over and over. Measured on ak-pg: 25.8M checkpoint buffers flushed
+      # (~206 GB) against a 1046 MB database -- the whole thing roughly twice a
+      # day. At 15min those pages coalesce and are written a third as often.
+      #
+      # max_wal_size must rise with it, or checkpoints trigger on size instead
+      # and the timeout change does nothing. It also bounds crash recovery to
+      # at most 2GB of WAL replay.
+      #
+      # NOT changed: synchronous_commit stays at the default `on`. It is off for
+      # ha-pg because a state recorder is telemetry; these hold credentials,
+      # documents and job state, where losing the last few hundred ms of commits
+      # on an unclean shutdown is not an acceptable trade.
+      checkpoint_timeout: "15min"
+      max_wal_size: "2GB"
+      # zstd, not "on": the boolean spelling resolves to pglz (2016-era, weaker
+      # and slower than the lz4/zstd this build also accepts). Compresses the
+      # full-page images that dominate WAL volume after each checkpoint.
+      wal_compression: "zstd"
+      # A WAL segment is a fixed 16 MiB file, and archive_timeout forces a switch
+      # on a timer -- the segment is then written and archived at its full size
+      # no matter how little real WAL it holds. CNPG defaults to 5min, i.e. up
+      # to 288 segments/day = 3.2 GB/day here, most of it padding.
+      #
+      # 15min, NOT the 30min used elsewhere in this rollout. This cluster holds
+      # credentials, sessions and enrolment state, so its RPO is worth more than
+      # the difference: 30min would save a further 0.3 GB/day, which does not
+      # justify doubling the worst-case loss window on this data.
+      #
+      # It does carry a synchronous standby (instances: 2), so the realistic
+      # exposure is losing both instances, not one -- but the conservative value
+      # is cheap here and the aggressive one buys almost nothing.
+      archive_timeout: "15min"
     # Async replication makes CNPG switchover slow because the standby has
     # to chase missing WAL via barman-cloud and replay queued records
     # before it can promote. With one synchronous standby, the standby is

kubernetes/apps/gatus/configmap.yaml

@@ -19,7 +19,7 @@ metadata:
 # Opaque-label map (maintainer reference only — NOT served publicly):
 #   services group                   infrastructure group
 #     Web Service 1 = Authentik        Infrastructure 1 = PBS 2 (pbs2.renner.uno)
-#     Web Service 2 = Gitea            Infrastructure 2 = Proxmox VE (pve.renner.uno)
+#     Web Service 2 = Gitea            Infrastructure 2 = retired (was Proxmox VE)
 #     Web Service 3 = Immich           Infrastructure 3 = UDM DNS (10.2.40.1)
 #     Web Service 4 = Woodpecker CI    Infrastructure 4 = Mosquitto MQTT broker
 #     Web Service 5 = Grafana
@@ -206,21 +206,6 @@ data:
         conditions:
           - "[STATUS] == any(200, 301, 302)"
 
-      - name: Infrastructure 2
-        group: infrastructure
-        url: https://pve.renner.uno
-        interval: 60s
-        client:
-          insecure-skip-verify: true
-        ui:
-          hide-hostname: true
-          hide-url: true
-          hide-port: true
-          hide-conditions: true
-          hide-errors: true
-        conditions:
-          - "[STATUS] == any(200, 301, 302)"
-
       - name: Infrastructure 3
         group: infrastructure
         url: 10.2.40.1

kubernetes/apps/grafana-db/postgres.yaml

@@ -29,6 +29,52 @@ spec:
     parameters:
       shared_buffers: "128MB"
       max_connections: "50"
+      # Write-reduction tuning. These databases are largely idle, and the
+      # Postgres defaults favour throughput over storage that wears out.
+      #
+      # checkpoint_timeout is the lever. The 5min default pushes every dirty
+      # page to disk 288 times a day, so a small database whose same few pages
+      # are re-dirtied constantly (sessions, tokens, job state) gets rewritten
+      # over and over. Measured on ak-pg: 25.8M checkpoint buffers flushed
+      # (~206 GB) against a 1046 MB database -- the whole thing roughly twice a
+      # day. At 15min those pages coalesce and are written a third as often.
+      #
+      # max_wal_size must rise with it, or checkpoints trigger on size instead
+      # and the timeout change does nothing. It also bounds crash recovery to
+      # at most 2GB of WAL replay.
+      #
+      # NOT changed: synchronous_commit stays at the default `on`. It is off for
+      # ha-pg because a state recorder is telemetry; these hold credentials,
+      # documents and job state, where losing the last few hundred ms of commits
+      # on an unclean shutdown is not an acceptable trade.
+      checkpoint_timeout: "15min"
+      max_wal_size: "2GB"
+      # zstd, not "on": the boolean spelling resolves to pglz (2016-era, weaker
+      # and slower than the lz4/zstd this build also accepts). Compresses the
+      # full-page images that dominate WAL volume after each checkpoint.
+      wal_compression: "zstd"
+      # THE dominant write source on this cluster, by three orders of magnitude.
+      #
+      # A WAL segment is a fixed 16 MiB file. archive_timeout forces a segment
+      # switch on a timer so the archive never lags reality by more than that
+      # interval -- and the switched-out segment is written and archived at its
+      # full 16 MiB regardless of how few bytes of real WAL it contains.
+      #
+      # CNPG defaults this to 5min, i.e. up to 288 segments/day = 4.6 GB/day.
+      # Measured here: this database is 21 MB and generates 4.5 MB/day of actual
+      # WAL, while its RBD volume takes 211 MB/hour. Essentially every byte
+      # written is padding -- roughly 1100x amplification.
+      #
+      # 30min cuts that to 48 segments/day. The cost is RPO: on total loss of
+      # the cluster, up to 30min of Grafana state is unrecoverable rather than
+      # 5min. Grafana's state here is dashboards, annotations and preferences,
+      # and the write rate says it changes ~4.5 MB/day -- so that window is
+      # nearly always empty anyway.
+      #
+      # Applied to gf-pg FIRST, alone, as a staged rollout: it is the lowest-risk
+      # cluster and has the cleanest signal, since its real WAL is small enough
+      # that any change in volume is unambiguously this parameter.
+      archive_timeout: "30min"
   resources:
     requests:
       cpu: 100m

kubernetes/apps/home-assistant/MIGRATION-recorder-postgres.md

@@ -0,0 +1,179 @@
+# Home Assistant recorder: SQLite → Postgres
+
+## Why
+
+The recorder's SQLite database is the single largest client writer in the
+cluster. Measured 2026-08-19 (cgroup `io.stat` on the home-assistant container,
+corroborated by kernel diskstats and Ceph pool counters):
+
+| | |
+|---|---|
+| HA writes to its RBD volume | **40 GB/day** |
+| share of all client writes | **~26%** |
+| actual row data underneath | **~5 GB/day** |
+
+The 8× gap is the write *pattern*, not the volume:
+
+```
+~30 state changes/sec  (107k rows/hour, 2.57M/day)
+  ×2   SQLite WAL + checkpoint — every dirtied page written twice
+  ×4   ext4 journal + commit, because synchronous=FULL fsyncs every commit
+       (measured: 819 KB of 4 KiB writes → 3.28 MB at the block device)
+  = ~15 KB written per recorded state change
+```
+
+Postgres replaces scattered fsync'd page rewrites with sequential batched WAL.
+This attacks the multiplier rather than trimming what gets recorded — excluding
+entities would hide the problem, not fix it.
+
+**Context:** this is one of three write-reduction changes. The other two are
+merged and verified — `size 3→2` on the Ceph pools (#519) and
+`mon_compact_on_trim=false` (#520, which alone removed ~228 GB/day). See #434.
+
+## What is at risk
+
+`states` is disposable — `purge_keep_days: 7`, so it regenerates in a week.
+
+**`statistics` is not.** As of 2026-08-19:
+
+| table | rows | note |
+|---|---|---|
+| `statistics` | 10,793,712 | **spans 2024-03-13 → today, 2.4 years** |
+| `statistics_short_term` | 1,902,624 | 5-minute buckets |
+| `statistics_meta` | 1,448 | entity ↔ id mapping, required by both |
+| `states` | 18,289,190 | disposable, 7-day retention |
+| `events` | 483,989 | disposable |
+
+**Home Assistant does not migrate data between database engines.** Repointing
+`db_url` at an empty Postgres silently starts from zero and the energy dashboard
+loses its entire history. The long-term statistics are irreplaceable — they are
+not in InfluxDB, which HA does not read for the energy dashboard.
+
+## Phase 1 — deploy the cluster (this PR)
+
+`ha-pg` comes up **empty** and nothing consumes it. HA stays on SQLite, so a
+failure here is inert and there is nothing to roll back.
+
+Verify before going further:
+
+```sh
+kubectl -n home-assistant get cluster ha-pg          # Cluster in healthy state, 1/1
+kubectl -n home-assistant get backup                 # ha-pg-daily Completed (immediate: true)
+kubectl -n home-assistant logs ha-pg-1 -c postgres | grep -i archive   # WAL archiving, no errors
+```
+
+Confirm the backup actually landed in the bucket rather than trusting the CR
+status — the WordPress backup failed nightly for three days against a bucket
+that was declared but never created (see `opentofu/object_storage.tf`).
+
+## Phase 2b — bulk statistics copy: DONE 2026-08-19
+
+Completed with **HA running** — the SQLite file was read through a read-only
+mount, and SQLite's WAL mode gives readers a consistent snapshot, so no downtime
+was needed for the bulk copy.
+
+```
+statistics_meta:        sqlite=1449       pg=1449       OK
+statistics:             sqlite=10799904   pg=10799904   OK   (153.9s)
+statistics_short_term:  sqlite=1969804    pg=1969804    OK   ( 27.0s)
+
+statistics span in Postgres: 2024-03-13 -> 2026-08-19
+entities with long-term history: 1449
+rows carrying sum (energy dashboard): 2627859
+database size: 2252 MB   (PVC is 20Gi)
+```
+
+`ANALYZE` was run on all three tables. That is not housekeeping: after a bulk
+`COPY` the planner has no statistics, and HA queries these tables constantly for
+history and energy views, so without it every view seq-scans 10.8M rows.
+
+Two schema details that cost a failed first attempt, recorded so the top-up does
+not repeat them:
+
+- `has_mean` / `has_sum` are INTEGER in SQLite and BOOLEAN in Postgres. Postgres
+  will not coerce; convert explicitly and preserve NULL.
+- The legacy `created` / `start` / `last_reset` DATETIME columns are 100% NULL in
+  both bulk tables (superseded by the `_ts` variants), which is what makes a
+  tab-separated `COPY` safe — nothing in those tables needs escaping.
+
+The schema itself was created with HA's own models
+(`Base.metadata.create_all`), **not** hand-written DDL, and `schema_changes` was
+left EMPTY on purpose. HA's `_initialize_database` detects a new database by
+looking for an index on `events.time_fired_ts`, then writes `SCHEMA_VERSION`
+itself. Pre-stamping that row would skip HA's own initialisation path.
+
+### High-water marks for the cutover top-up
+
+Rows written after the bulk copy are not yet in Postgres. Resume from:
+
+```
+statistics             max_id=13323904    max_start_ts=1787162400.0
+statistics_short_term  max_id=159205762   max_start_ts=1787166000.0
+```
+
+## Phase 2c — cutover (REMAINING)
+
+Do not start until phase 1 is verified.
+
+**Migrate statistics only.** Copying all 18M `states` rows is slow, needs a long
+HA outage, and buys a table that is discarded within 7 days anyway.
+
+1. **Snapshot first.** Velero backup of the HA PVC, plus a `.backup` copy of
+   `home-assistant_v2.db`. This is the rollback.
+2. **Stop the recorder** — scale `deploy/home-assistant` to 0. The database must
+   be quiescent; copying a live SQLite file gives a torn read.
+3. **Create the schema.** Start HA once against the empty Postgres so it runs
+   its own migrations. Never hand-create the tables — the schema version must
+   match the HA version exactly or HA will refuse to start or silently migrate.
+4. **Copy** `statistics_meta` → `statistics` → `statistics_short_term`, in that
+   order (the other two carry `metadata_id` foreign keys into the first).
+   Sequences must be reset afterwards, or the first insert collides on a
+   duplicate primary key:
+   ```sql
+   SELECT setval('statistics_id_seq', (SELECT MAX(id) FROM statistics));
+   SELECT setval('statistics_meta_id_seq', (SELECT MAX(id) FROM statistics_meta));
+   SELECT setval('statistics_short_term_id_seq', (SELECT MAX(id) FROM statistics_short_term));
+   ```
+5. **Repoint** `recorder.db_url` in the `homeassistant` repo — that change lives
+   there, not here. The `recorder_db_url` secret it references is already
+   rendered into `secrets.yaml` by the `render-db-url` init container in
+   `deployment.yaml` (phase 2a), so this order is mandatory: **the init
+   container must be deployed first**, or HA refuses to start on a missing
+   secret.
+
+   The init container percent-encodes the credentials and validates the
+   resulting YAML rather than counting lines. That is not incidental — the CNPG
+   password is random, and a naive `"postgresql://user:pass@host/db"` string
+   makes `secrets.yaml` unparseable the moment the password contains a quote,
+   which takes out *every* secret HA loads, not just this one. Verified against
+   a deliberately hostile password before merge.
+   ```yaml
+   recorder:
+     db_url: !secret recorder_db_url    # postgresql://homeassistant:<pw>@ha-pg-rw.home-assistant.svc.cluster.local/homeassistant
+   ```
+   The password is in the CNPG-generated `ha-pg-app` Secret. It must not be
+   committed — render it into `secrets.yaml` the same way the Z2M MQTT password
+   is handled in `zigbee2mqtt-deployment.yaml`.
+6. **Verify** the energy dashboard renders history back to 2024, not just today.
+   That is the check that proves the statistics migration worked; "HA starts and
+   records" does not.
+
+### Rollback
+
+Revert `db_url` to the SQLite path and restore the `.backup` copy. Keep the
+SQLite file until the energy dashboard has been confirmed over a full day —
+deleting it early removes the only rollback that preserves the 2.4 years.
+
+## After the migration
+
+- The HA config PVC shrinks from 5.5 GB to a few hundred MB. Do not shrink the
+  PVC itself: `ceph-block` allows expansion, not reduction.
+- Re-measure. `kubectl -n home-assistant exec deploy/home-assistant -c
+  home-assistant -- cat /sys/fs/cgroup/io.stat` was 40 GB/day before. If it does
+  not drop substantially, the migration did not achieve its purpose and that
+  should be recorded rather than assumed.
+- `synchronous_commit: off` in `ha-pg-cluster.yaml` is the single biggest
+  contributor to the reduction. It is safe for telemetry (Postgres stays crash
+  consistent; at most a few hundred ms of commits are lost on unclean shutdown)
+  but it is a deliberate trade and should not be "corrected" later without
+  understanding why it is set.

kubernetes/apps/home-assistant/deployment.yaml

@@ -36,9 +36,67 @@ spec:
       # (sml2mqtt bridge owns the USB device on edge), so it no longer needs
       # the edge USB device and schedules on the general cluster pool. This
       # gets the heavy HA pod (8Gi) off the soon-to-be-tiny edge box.
+      initContainers:
+        # Puts CNPG's connection URI into /config/secrets.yaml as
+        # `recorder_db_url`, for the recorder's move off SQLite
+        # (MIGRATION-recorder-postgres.md).
+        #
+        # HA has no way to read a db_url from the environment — `!env_var` was
+        # removed from its YAML loader — so `!secret` is the only mechanism, and
+        # secrets.yaml is PVC state that config-sync deliberately does not
+        # manage. Hence a small init container rather than a env: entry.
+        #
+        # The URI comes STRAIGHT from the ha-pg-app Secret. CNPG builds and
+        # percent-encodes it, so there is nothing to escape here: the value is
+        # restricted to [A-Za-z0-9:/@._%-], verified against the live secret.
+        # An earlier version of this assembled the URI from user/password/dbname
+        # and needed percent-encoding plus YAML round-trip validation to be
+        # safe. That was solving a problem CNPG had already solved.
+        #
+        # Re-rendered on every pod start, so a CNPG password rotation takes
+        # effect rather than silently leaving a stale URI behind.
+        #
+        # INERT until the recorder is repointed: nothing reads this key until
+        # `db_url: !secret recorder_db_url` lands in configuration.yaml in the
+        # homeassistant repo. That ordering is required — HA refuses to start
+        # when a referenced secret is missing.
+        - name: render-db-url
+          image: busybox:1.36
+          command:
+            - sh
+            - -c
+            - |
+              set -eu
+              S=/config/secrets.yaml
+              T=/config/.secrets.render.tmp
+              [ -f "$S" ] || : > "$S"
+              # Filter-then-append, never rewrite in place: this file holds
+              # every other HA secret (influxdb_token and friends).
+              grep -v '^recorder_db_url:' "$S" > "$T" || true
+              printf 'recorder_db_url: "%s"\n' "$DB_URI" >> "$T"
+              # Refuse to shrink the file. Cheap guard against the one failure
+              # that would actually hurt: losing the other keys.
+              [ "$(wc -l < "$T")" -ge "$(wc -l < "$S")" ] || {
+                echo "FATAL: rewrite would shrink secrets.yaml; aborting" >&2; exit 1; }
+              chmod 600 "$T"
+              mv "$T" "$S"
+              echo "recorder_db_url rendered"
+          env:
+            - name: DB_URI
+              valueFrom:
+                secretKeyRef: {name: ha-pg-app, key: fqdn-uri}
+          volumeMounts:
+            - name: config
+              mountPath: /config
+          resources:
+            requests:
+              cpu: 10m
+              memory: 16Mi
+            limits:
+              memory: 32Mi
       containers:
         - name: home-assistant
-          image: ghcr.io/home-assistant/home-assistant:2026.8.2 # {"$imagepolicy": "flux-system:home-assistant"}
+          image: ghcr.io/home-assistant/home-assistant:2026.8.3 # {"$imagepolicy": "flux-system:home-assistant"}
           env:
             - name: TZ
               value: Europe/Berlin
@@ -221,9 +279,13 @@ spec:
         # Writable emptyDir for the nginx entrypoint's envsubst output
         # (/etc/nginx/conf.d/default.conf rendered from the .template).
         - name: proxy-conf-d
-          emptyDir: {}
+          emptyDir:
+            medium: Memory
+            sizeLimit: 16Mi
         # code-server's settings/extensions cache. Not on the config PVC
         # so installed extensions don't pollute HA's /config listing.
         # Trade-off: extensions reset across pod restarts.
         - name: vscode-state
-          emptyDir: {}
+          emptyDir:
+            medium: Memory
+            sizeLimit: 16Mi

kubernetes/apps/home-assistant/ha-pg-cluster.yaml

@@ -0,0 +1,162 @@
+---
+# Postgres backend for the Home Assistant recorder.
+#
+# WHY THIS EXISTS
+# The recorder's SQLite database is the single largest client writer in the
+# cluster. Measured 2026-08-19 (cgroup io.stat on the home-assistant container,
+# corroborated by kernel diskstats): 40 GB/day to its RBD volume, ~26% of all
+# client writes, against only ~5 GB/day of actual row data. The gap is the write
+# PATTERN, not the volume:
+#
+#   ~30 state changes/sec (107k rows/hour, 2.57M/day)
+#     x2  SQLite WAL + checkpoint — every dirtied page is written twice
+#     x4  ext4 journal + commit, because synchronous=FULL fsyncs every commit
+#         (measured: 819 KB of 4 KiB writes -> 3.28 MB at the block device)
+#     = ~15 KB written per recorded state change
+#
+# Postgres replaces scattered fsync'd page rewrites with sequential, batched WAL.
+# That is the point of the move — it attacks the multiplier rather than trimming
+# what gets recorded, which would only hide the problem.
+#
+# STAGING
+# Phase 1 (this manifest): the cluster comes up EMPTY and HA keeps using SQLite.
+# Nothing depends on it, so a failure here is inert. Verify Ready, WAL archiving
+# to Hetzner, and a successful base backup before going further.
+# Phase 2: migrate the statistics tables and repoint recorder.db_url.
+# See MIGRATION-recorder-postgres.md for the sequence and the rollback.
+apiVersion: postgresql.cnpg.io/v1
+kind: Cluster
+metadata:
+  name: ha-pg
+  namespace: home-assistant
+spec:
+  instances: 1
+  imageName: ghcr.io/cloudnative-pg/postgresql:16
+  storage:
+    storageClass: ceph-block
+    # Sized for the migrated statistics plus 7 days of states at the current
+    # rate: statistics 10.8M rows + statistics_short_term 1.9M (~1.8 GB with
+    # indexes in Postgres) and ~18M states rows for the purge_keep_days:7 window
+    # (~4 GB). 20Gi leaves room for autovacuum bloat and index churn.
+    # ceph-block has allowVolumeExpansion: true, so this can grow in place.
+    size: 20Gi
+  plugins:
+    - name: barman-cloud.cloudnative-pg.io
+      isWALArchiver: true
+      parameters:
+        barmanObjectName: ha-pg-store
+  bootstrap:
+    initdb:
+      # Explicit, not left to CNPG's `app` default: the generated ha-pg-app
+      # Secret and the -app Service point at whatever is named here, and the
+      # recorder db_url in phase 2 must match. Same trap that made the wp-pg
+      # namespace migration point credentials at an empty database (#433).
+      database: homeassistant
+      owner: homeassistant
+  postgresql:
+    parameters:
+      # 512MB, not the 256MB this shipped with. Measured after 15h on Postgres:
+      # buffers_backend was 629,704 of 939,025 total page writes — 67% written
+      # by backend processes rather than the checkpointer or bgwriter. That
+      # ratio means the working set does not fit in shared_buffers, so backends
+      # evict dirty pages themselves and the same pages are written out
+      # repeatedly instead of coalescing in memory.
+      #
+      # The recorder touches a wide set of pages continuously (~30 state
+      # changes/sec across states, state_attributes and five indexes), which is
+      # exactly the workload that punishes a small buffer pool.
+      #
+      # Memory headroom checked before raising: the instance was using 383Mi of
+      # a 1Gi limit with 256MB of that being shared_buffers, so the limit goes
+      # to 1.5Gi to keep the same margin.
+      shared_buffers: "512MB"
+      max_connections: "50"
+      # The rest of this block is tuned to MINIMISE WRITE VOLUME, which is the
+      # entire reason for the migration. Defaults would give a correct database
+      # that writes far more than it needs to on replicated Ceph storage.
+      #
+      # synchronous_commit off: the recorder is telemetry, not transactional
+      # record-keeping. This does NOT risk corruption — Postgres stays crash
+      # consistent; it only means the last few hundred ms of commits can be lost
+      # on an unclean shutdown, which for state history is a non-event. In
+      # exchange every commit stops forcing an fsync, which is precisely the
+      # cost that makes SQLite expensive here.
+      synchronous_commit: "off"
+      # Compress full-page images in WAL. Recorder writes are many small row
+      # inserts scattered across pages, so full-page writes after each
+      # checkpoint dominate WAL volume; compression cuts that directly.
+      #
+      # Full-page images really are ~62% of WAL here: 61.9% measured as a
+      # steady-state delta over one checkpoint cycle, against 62.4% lifetime.
+      #
+      # But switching pglz -> zstd changed NOTHING. Measured by
+      # pg_current_wal_lsn advance over 20min windows: 4911 and 4976 MB/day
+      # before, 5002 MB/day after.
+      #
+      # The reason is that `wal_compression: "on"` was never "no compression" --
+      # it is pglz, which was already compressing these page images. zstd only
+      # changes the ratio, and on recorder pages that margin is too small to
+      # measure. The original framing of this change (that the clusters "were
+      # not getting compression") was wrong; they were.
+      #
+      # Note the 62% figure is an upper bound: it is computed as
+      # wal_fpi * 8192 / wal_bytes, which assumes uncompressed 8 KiB pages. With
+      # compression active the true byte share is lower, which is consistent
+      # with there being little headroom left for a better algorithm.
+      #
+      # Kept because zstd is not worse than pglz and costs nothing. It is simply
+      # not a lever. The real one on this cluster was archive_timeout, below.
+      #
+      # zstd, not "on": the boolean spelling resolves to pglz, the original
+      # 2016-era algorithm, which is both slower and weaker than the two
+      # alternatives PostgreSQL 16 accepts here (lz4, zstd). zstd gives the
+      # best ratio on 8 KiB page images, and the CPU cost lands on a cluster
+      # that is IO-bound on QLC NVMe, not CPU-bound.
+      wal_compression: "zstd"
+      # A WAL segment is a fixed 16 MiB file, and archive_timeout forces a switch
+      # on a timer -- the segment is then written and archived at its full size
+      # no matter how little real WAL it holds. CNPG defaults to 5min, i.e. up
+      # to 288 segments/day = 4.6 GB/day that is largely padding.
+      #
+      # Verified on gf-pg first (the staged rollout): 211 -> 41 MB/hour, an 81%
+      # cut, with archived segments falling from ~12/hour to ~1.5/hour in step.
+      # Two independent signals moving together is what confirms the volume was
+      # padding rather than data.
+      #
+      # The cost is RPO: up to 30min of WAL is unrecoverable if this cluster and
+      # its volume are lost outright. This is a single-instance cluster, so the
+      # archive is the only off-node copy.
+      archive_timeout: "30min"
+      # Fewer checkpoints => fewer full-page writes, since a page is only
+      # fully-logged on its first modification after a checkpoint.
+      #
+      # That reasoning did not show up in WAL volume: 15min -> 30min moved the
+      # WAL rate not at all (4911/4976 -> 5002 MB/day). The likely reason is
+      # that this workload is append-mostly -- a page is dirtied while it fills
+      # and then not touched again, so its full-page image is paid once
+      # regardless of how often checkpoints run. Interval only helps pages that
+      # are re-dirtied across many cycles, which is not what a recorder does.
+      #
+      # Kept, but NOT because WAL improved. checkpoint_timeout also governs
+      # data-file writes at checkpoint, and those are unmeasured here: this
+      # volume takes ~11 GB/day against 4.9 GB/day of WAL, so most of its
+      # traffic is outside what the WAL measurement covers. Revisit with a
+      # data-file measurement before treating this value as justified.
+      #
+      # Recovery cost is bounded by how much WAL a single interval produces,
+      # not by the interval itself: at the measured rate 30min is well under
+      # 200MB of replay, i.e. seconds. max_wal_size stays 2GB precisely so the
+      # timeout stays the binding trigger -- if size were the binding one, the
+      # interval change would do nothing.
+      checkpoint_timeout: "30min"
+      max_wal_size: "2GB"
+      min_wal_size: "512MB"
+      # full_page_writes is deliberately LEFT ON (the default). RBD can tear a
+      # page across a 4 MiB object boundary, and turning this off would trade
+      # write volume for the risk of an unrecoverable database. Not worth it.
+  resources:
+    requests:
+      cpu: 200m
+      memory: 512Mi
+    limits:
+      memory: 1536Mi

kubernetes/apps/home-assistant/ha-pg-hetzner-creds-externalsecret.yaml

@@ -0,0 +1,38 @@
+---
+# Barman S3 credentials for ha-pg.
+#
+# FOLLOW-UP (#397): sourced from seed/paperless/hetzner-creds, which is not where
+# they belong. woodpecker/hetzner-creds-externalsecret.yaml has the identical
+# problem and the same note. The values are the same today, so this is correct
+# but coupled: rotating paperless's Hetzner credentials would silently change
+# the backup credentials for Home Assistant and Woodpecker too.
+#
+# The fix is one dedicated path per consumer (seed/home-assistant/hetzner-creds)
+# and a one-line change to the `key:` below. Creating it needs an OpenBao write:
+#
+#   bao kv put homelab/seed/home-assistant/hetzner-creds \
+#     @<(bao kv get -format=json homelab/seed/paperless/hetzner-creds | jq '.data.data')
+#
+# Note the mount is `homelab`, so the path is homelab/seed/... — #397 records a
+# 403 on `bao kv list seed`, which was a wrong-path error rather than a
+# permissions problem.
+#
+# The namespace allowlist on the openbao ClusterSecretStore
+# (infrastructure/post-crds/openbao-clustersecretstore.yaml) must include this
+# namespace or ESO silently never syncs (#505). home-assistant is already
+# allowlisted — it has several ExternalSecrets in this directory already.
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: hetzner-s3-creds
+  namespace: home-assistant
+spec:
+  refreshInterval: 1h
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: openbao
+  target:
+    name: hetzner-s3-creds
+  dataFrom:
+    - extract:
+        key: seed/paperless/hetzner-creds

kubernetes/apps/home-assistant/ha-pg-objectstore.yaml

@@ -0,0 +1,35 @@
+---
+# Barman object store for ha-pg: base backups + continuous WAL archiving to
+# Hetzner Object Storage, same target bucket as the other CNPG clusters.
+#
+# Its own prefix. Two clusters archiving WAL to one destinationPath interleave
+# their streams and corrupt both — see the note in
+# kubernetes/apps/woodpecker/objectstore.yaml, which learned that the hard way.
+apiVersion: barmancloud.cnpg.io/v1
+kind: ObjectStore
+metadata:
+  name: ha-pg-store
+  namespace: home-assistant
+spec:
+  configuration:
+    destinationPath: s3://renner-uno-k8s-cnpg/ha-pg
+    endpointURL: https://fsn1.your-objectstorage.com
+    s3Credentials:
+      accessKeyId:
+        name: hetzner-s3-creds
+        key: ACCESS_KEY_ID
+      secretAccessKey:
+        name: hetzner-s3-creds
+        key: ACCESS_SECRET_KEY
+      region:
+        name: hetzner-s3-creds
+        key: REGION
+    wal:
+      compression: gzip
+    data:
+      compression: gzip
+  # Matches the 30d retention the other CNPG clusters use and the
+  # renner-uno-k8s-cnpg bucket's noncurrent-version expiry in
+  # opentofu/object_storage.tf, so old S3 versions don't outlive the backups
+  # that superseded them.
+  retentionPolicy: "30d"

kubernetes/apps/home-assistant/ha-pg-scheduled-backup.yaml

@@ -0,0 +1,23 @@
+---
+# Daily base backups for ha-pg. Continuous WAL archiving runs separately
+# (ha-pg-cluster.yaml spec.plugins[].isWALArchiver); without periodic base
+# backups the prefix would have no recovery point of its own, and
+# ha-pg-store's retentionPolicy: 30d would have nothing to prune against, so the
+# WAL chain would grow without bound.
+#
+# 02:50 UTC, offset from the other CNPG clusters (02:30, wp-pg at 02:40) so they
+# do not all hit the Hetzner bucket at once.
+apiVersion: postgresql.cnpg.io/v1
+kind: ScheduledBackup
+metadata:
+  name: ha-pg-daily
+  namespace: home-assistant
+spec:
+  schedule: "0 50 2 * * *"
+  immediate: true
+  backupOwnerReference: self
+  cluster:
+    name: ha-pg
+  method: plugin
+  pluginConfiguration:
+    name: barman-cloud.cloudnative-pg.io

kubernetes/apps/home-assistant/kustomization.yaml

@@ -52,6 +52,14 @@ resources:
   # CNP ingress rule.
   - zigbee2mqtt-gate-service.yaml
   - zigbee2mqtt-ingress.yaml
+  # Postgres backend for the recorder (MIGRATION-recorder-postgres.md).
+  # PHASE 1: this cluster comes up EMPTY and nothing consumes it — HA is still
+  # on SQLite. Deploying it is therefore inert; phase 2 migrates the statistics
+  # tables and repoints recorder.db_url in the homeassistant repo.
+  - ha-pg-hetzner-creds-externalsecret.yaml
+  - ha-pg-objectstore.yaml
+  - ha-pg-cluster.yaml
+  - ha-pg-scheduled-backup.yaml
 configMapGenerator:
   # nginx config for the reverse-proxy sidecar. Hash suffix disabled so
   # the cm name stays stable; editing proxy.conf does not auto-roll the
@@ -79,5 +87,13 @@ configMapGenerator:
     namespace: home-assistant
     files:
       - watch.sh=zigbee2mqtt-interview-watch.sh
+  # Z2M discovery guard sidecar script (gitea issue #381). Hash suffix disabled
+  # (name stays stable), so editing zigbee2mqtt-discovery-guard.sh does not
+  # auto-roll the pod — restart manually with
+  # `kubectl rollout restart deploy/zigbee2mqtt -n home-assistant`.
+  - name: zigbee2mqtt-discovery-guard-script
+    namespace: home-assistant
+    files:
+      - guard.sh=zigbee2mqtt-discovery-guard.sh
 generatorOptions:
   disableNameSuffixHash: true

kubernetes/apps/home-assistant/mosquitto-bootstrap.yaml

@@ -373,8 +373,9 @@ spec:
 # Broker user `zigbee2mqtt` — the Zigbee2MQTT bridge (ZHA replacement). Scoped
 # to its own `zigbee2mqtt/#` subtree plus `write homeassistant/#` (so it can
 # publish HA MQTT-discovery configs, exactly like the `shellies` user) and
-# `read homeassistant/status` (Z2M re-publishes discovery when HA restarts —
-# ACL in mosquitto-deployment.yaml). Own credential so its lifecycle is
+# `read homeassistant/status` (HA's birth/will topic; on `online` Z2M
+# republishes cached device states, NOT discovery configs — ACL and the full
+# reasoning in mosquitto-deployment.yaml). Own credential so its lifecycle is
 # independent of the others. The Z2M init container reads this password and
 # renders it into /app/data/secret.yaml.
 apiVersion: batch/v1

kubernetes/apps/home-assistant/mosquitto-config.yaml

@@ -5,13 +5,31 @@ metadata:
   namespace: home-assistant
 data:
   mosquitto.conf: |
-    # No on-disk persistence. Retained messages + sessions are re-populated by
-    # clients on reconnect (evcc republishes state, Shellies re-send their
-    # announce/online, HA re-publishes discovery), so the broker keeps no
-    # durable state. This drops the Ceph PVC dependency (the broker starts even
-    # if Ceph is degraded) and lets it reschedule anywhere in seconds — no RWO
-    # volume to fence/detach/reattach. Trade-off: a broker restart briefly
-    # empties retained state until clients reconnect (self-healing).
+    # No on-disk persistence, and no PVC of any kind for this broker. The
+    # broker keeps no durable state, so it starts even if Ceph is degraded and
+    # reschedules anywhere in seconds — no RWO volume to fence/detach/reattach.
+    #
+    # Most clients re-populate their retained state on reconnect: evcc
+    # republishes state, Shellies re-send their announce/online, HA re-publishes
+    # its own discovery.
+    #
+    # Zigbee2MQTT is the EXCEPTION and does not self-heal. Its HA discovery
+    # configs are published retained, and retention is the only mechanism it
+    # relies on to survive a broker restart — verified against Z2M 2.12.1
+    # source: discover() runs only from start(), the MQTT reconnect path
+    # (lib/mqtt.ts onConnect) republishes bridge/state and nothing else, and the
+    # homeassistant/status "online" handler (lib/extension/homeassistant.ts
+    # onMQTTMessage) republishes bridge/state + cached device states but never
+    # the discovery configs. There is no bridge/request endpoint that forces a
+    # rediscovery. Upstream has declined to add one (Koenkk/zigbee2mqtt #27458,
+    # #22977, #16950, all closed NOT_PLANNED).
+    #
+    # Consequence: a mosquitto restart that is not followed by a Zigbee2MQTT
+    # restart leaves every Z2M entity unavailable in HA until Z2M is restarted
+    # (`kubectl rollout restart deploy/zigbee2mqtt -n home-assistant`). This is
+    # a live operational hazard, tracked in gitea issue #381 — it caused a ~14 h
+    # outage of every Z2M entity on 2026-07-23. Enabling persistence is NOT the
+    # answer: this broker deliberately has no PVC (see above).
     persistence false
     log_dest stdout
     log_type error

kubernetes/apps/home-assistant/mosquitto-deployment.yaml

@@ -25,7 +25,13 @@ spec:
     # a broker serving throughout an image bump / reschedule (clients on the old
     # pod just reconnect to the new one); maxSurge:1 permits the brief two-pod
     # overlap — two independent stateless brokers behind the LB, fine since there
-    # is no shared state and discovery self-heals.
+    # is no shared state.
+    #
+    # NOTE: connectivity survives the roll, but RETAINED STATE DOES NOT — the
+    # new pod starts with an empty retained store, so a roll is equivalent to a
+    # broker restart for every client that depends on retention. Zigbee2MQTT
+    # does not recover from that on its own; see mosquitto-config.yaml and
+    # gitea issue #381.
     type: RollingUpdate
     rollingUpdate:
       maxUnavailable: 0
@@ -64,7 +70,9 @@ spec:
           # Pinned tag, auto-bumped within the 2.x line by Flux image-automation
           # (infrastructure/image-automation/mosquitto.yaml). The marker tells
           # Flux where to write; patch/minor updates land as git commits and
-          # roll the broker (brief restart; discovery self-heals via HA).
+          # roll the broker. HA re-publishes its own discovery afterwards, but
+          # Zigbee2MQTT does not — an image bump therefore needs the same
+          # follow-up as any other broker restart (see mosquitto-config.yaml).
           image: eclipse-mosquitto:2.0.22 # {"$imagepolicy": "flux-system:mosquitto"}
           # The shim runs as root (image sets no USER) and prepares the
           # auth files for the broker, which drops to user mosquitto
@@ -148,8 +156,16 @@ spec:
               # Owns its own `zigbee2mqtt/#` subtree; may WRITE `homeassistant/#`
               # to publish HA MQTT-discovery configs (write-only there, like the
               # shellies user, to deny discovery-tree recon from a compromised
-              # bridge); and READs `homeassistant/status` so it re-publishes
-              # discovery when HA restarts (birth/will).
+              # bridge); and READs `homeassistant/status`, HA's birth/will topic.
+              #
+              # That read grant does NOT buy a discovery re-publish: on
+              # `online`, Z2M 2.12.1 republishes bridge/state and cached device
+              # states only (lib/extension/homeassistant.ts onMQTTMessage) —
+              # discover() runs solely from start(). The grant is still correct,
+              # because the cached-state republish is what makes HA show current
+              # values right after an HA restart. Restoring discovery after a
+              # BROKER restart is handled by the discovery-guard sidecar in
+              # zigbee2mqtt-deployment.yaml.
               user zigbee2mqtt
               topic readwrite zigbee2mqtt/#
               topic write homeassistant/#
@@ -297,7 +313,9 @@ spec:
           configMap:
             name: mosquitto-config
         - name: auth
-          emptyDir: {}
+          emptyDir:
+            medium: Memory
+            sizeLimit: 16Mi
         - name: tls
           secret:
             secretName: mosquitto-tls

kubernetes/apps/home-assistant/zigbee2mqtt-config.yaml

@@ -176,3 +176,38 @@ data:
         friendly_name: Treppenhaus Haustür Schalter 1
       '0x001788010b00d011':
         friendly_name: Treppenhaus Haustür Schalter 2
+      # Ecowitt WS90 weather station. Joined after the ZHA-mirrored block above
+      # was written, hence listed apart from it.
+      #
+      # This device also carries a non-default ZCL reporting configuration
+      # (per-attribute min interval and reportable-change deadbands). That is
+      # device-side state, not expressible here — Z2M has no settings field for
+      # it — so it is documented, with restore commands, in
+      # zigbee2mqtt-ws90-reporting.md. It is lost whenever Z2M re-runs
+      # `configure` on the device, and a fresh PVC will not have it.
+      #
+      # The station transmits every 8.8s, as a burst of seven per-cluster ZCL
+      # frames; Z2M publishes its full cumulative state once per frame. debounce
+      # merges a burst into one publish carrying the last value per attribute,
+      # and must stay below the transmit interval — a window that never sees
+      # silence never flushes.
+      #
+      # Every few days the firmware zeroes its readings for ~8s, then skips a
+      # transmit cycle and recovers on the next. The zeros propagate attribute
+      # by attribute through a burst rather than arriving together, and they
+      # start inside an otherwise normal cycle — the last valid reading and the
+      # first zero are ~20ms apart.
+      #
+      # Merging cuts an episode from six published bad frames to one. It does
+      # NOT make that frame recognisable: it carries whatever subset had zeroed
+      # when the burst ended, which may be every field or only temperature, and
+      # a frame reading {"temperature":0,"humidity":92,"pressure":96.9} is not
+      # separable from a valid winter reading. Since the merge is last-wins and
+      # the zeros arrive at the tail, it also discards the valid reading that
+      # led the burst. This bounds the corruption; it does not filter it.
+      #
+      # Z2M reads this only at startup (requiresRestart), and this file is
+      # SEED-ONLY per the header — the running instance takes it from its PVC.
+      '0xc02cedfffe958f40':
+        friendly_name: Wetterstation
+        debounce: 2

kubernetes/apps/home-assistant/zigbee2mqtt-deployment.yaml

@@ -158,7 +158,7 @@ spec:
           # Pinned tag. No Flux image-automation marker yet (unlike mosquitto) —
           # add an infrastructure/image-automation/zigbee2mqtt.yaml policy as a
           # follow-up if desired; pinned-and-manual is fine for the initial cutover.
-          image: ghcr.io/koenkk/zigbee2mqtt:2.12.1
+          image: ghcr.io/koenkk/zigbee2mqtt:2.13.0
           imagePullPolicy: IfNotPresent
           env:
             - name: TZ
@@ -272,6 +272,64 @@ spec:
               memory: 16Mi
             limits:
               memory: 64Mi
+        # Re-runs Zigbee2MQTT's HA discovery after the broker loses its retained
+        # store. mosquitto runs `persistence false` with no PVC, so every broker
+        # restart wipes the retained discovery configs, and nothing on the Z2M
+        # side restores them — not reconnect, not HA's birth message, and there
+        # is no rediscovery endpoint. Without this, a broker restart leaves every
+        # Z2M entity unavailable in HA until someone restarts Z2M by hand (gitea
+        # issue #381; ~14 h outage on 2026-07-23). The full reasoning, including
+        # why bridge/groups is the sentinel and why the guard is tuned for
+        # sensitivity, is in the script header.
+        - name: discovery-guard
+          # Same image and rationale as interview-watch above: reused purely for
+          # its bundled mosquitto_sub/mosquitto_pub, nothing installed at
+          # runtime. Verified present in this exact image: sh, mosquitto_sub
+          # (incl. -C and -W), mosquitto_pub, date -u +%s, mkdir, cat.
+          # Deliberately NOT carrying a Flux $imagepolicy marker, so it does not
+          # drag the broker's image-automation policy along with it.
+          image: eclipse-mosquitto:2.0.22
+          imagePullPolicy: IfNotPresent
+          command: [sh, /script/guard.sh]
+          env:
+            # Reuses the Z2M broker credential: its ACL is `readwrite
+            # zigbee2mqtt/#`, which covers reading bridge/state/bridge/groups and
+            # writing bridge/request/restart. No new broker user needed.
+            - name: MQTT_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: mosquitto-zigbee2mqtt-credentials
+                  key: password
+            # Must track zigbee2mqtt-config.yaml, same as interview-watch.
+            - name: MQTT_HOST
+              value: mosquitto
+            - name: MQTT_USER
+              value: zigbee2mqtt
+            - name: BASE_TOPIC
+              value: zigbee2mqtt
+            - name: STATEDIR
+              value: /state
+          securityContext:
+            runAsNonRoot: true
+            runAsUser: 1883
+            runAsGroup: 1883
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: [ALL]
+          volumeMounts:
+            - name: discovery-guard-script
+              mountPath: /script
+              readOnly: true
+            # Restart cooldown stamp. Its own emptyDir rather than the image's
+            # /tmp, so it stays writable as non-root.
+            - name: discovery-guard-state
+              mountPath: /state
+          resources:
+            requests:
+              cpu: 10m
+              memory: 16Mi
+            limits:
+              memory: 64Mi
       volumes:
         - name: data
           persistentVolumeClaim:
@@ -286,4 +344,13 @@ spec:
           configMap:
             name: zigbee2mqtt-interview-watch-script
         - name: interview-watch-state
-          emptyDir: {}
+          emptyDir:
+            medium: Memory
+            sizeLimit: 16Mi
+        - name: discovery-guard-script
+          configMap:
+            name: zigbee2mqtt-discovery-guard-script
+        - name: discovery-guard-state
+          emptyDir:
+            medium: Memory
+            sizeLimit: 16Mi

kubernetes/apps/home-assistant/zigbee2mqtt-discovery-guard.sh

@@ -0,0 +1,162 @@
+#!/bin/sh
+# Restores Home Assistant MQTT discovery after the broker loses its retained
+# store, which mosquitto does on every restart because it runs `persistence
+# false` and has no PVC (a deliberate, settled decision — see
+# mosquitto-config.yaml).
+#
+# WHY THIS EXISTS
+# Zigbee2MQTT publishes its HA discovery configs retained and relies on broker
+# retention alone to survive a broker restart. Verified against Z2M 2.12.1
+# source, nothing on the Z2M side restores them:
+#   - discover() runs only from HomeAssistant.start().
+#   - lib/mqtt.ts onConnect() republishes bridge/state and resubscribes; the
+#     republishRetainedTimer is armed only inside connect() and never re-armed
+#     on reconnect.
+#   - The homeassistant/status "online" handler republishes bridge/state plus
+#     cached device states, never the discovery configs.
+#   - There is no bridge/request endpoint that forces a rediscovery; upstream
+#     declined to add one (Koenkk/zigbee2mqtt #27458, #22977, #16950).
+# So a broker restart not followed by a Z2M restart leaves every Z2M entity
+# unavailable in HA indefinitely. That cost ~14 h of dead entities on
+# 2026-07-23 (gitea issue #381).
+#
+# DETECTION — why bridge/groups
+# bridge/state and the four bridge/{info,devices,groups,definitions} topics are
+# all retained, but only bridge/state is republished on reconnect (mqtt.ts
+# onConnect). So "bridge/state present + one of the others absent" means: Z2M is
+# talking to this broker right now, but nothing has re-run start() since the
+# retained store was wiped. That is exactly the broken state.
+#
+# The absent-half partner must not be re-publishable by ordinary runtime
+# events, or it self-heals and the guard goes blind while HA is still broken:
+#   bridge/devices     — republished on announce/join/leave/interview. Worst.
+#   bridge/definitions — republished on devicesChanged and OTA.
+#   bridge/info        — republished on devicesChanged, which reaches inbound
+#                        radio traffic via the deviceExposesChanged callback
+#                        handed to every fromZigbee converter (receive.ts).
+#   bridge/health      — self-heals on its own 10 min timer. Useless here.
+#   bridge/groups      — only re-published by deliberate admin actions: group
+#                        membership edits, group add/remove/rename, and scene
+#                        changes (which come solely from an explicit
+#                        <device>/set scene command, never an inbound report).
+# Hence bridge/groups. It is also safe with no groups defined: publishGroups()
+# always publishes, emitting `[]`, which is still a non-empty retained payload.
+#
+# DELIBERATELY NOT AND-ed with bridge/info being absent. That would cut false
+# positives but introduce false negatives, and the costs are wildly asymmetric:
+# a false positive is one ~60 s Z2M restart, a false negative is every Zigbee
+# entity dead until a human notices. This guard is tuned for sensitivity.
+#
+# Two known ways to get a spurious positive, both handled by CONFIRM_DELAY:
+#   - Startup race. bridge/state is published at mqtt.connect() but
+#     bridge/groups only at the end of BridgeExtension.start(), so the signature
+#     is legitimately true for a few seconds during every normal Z2M boot.
+#   - QoS 0. The bridge/* topics are published with no qos option, so they go
+#     out at-most-once and a publish can in principle be dropped.
+#
+# ACTION
+# Publishes bridge/request/restart, which is a full in-process restart: index.js
+# restart() -> stop(true) tears down every extension, then start() builds a new
+# Controller and re-runs HomeAssistant.start(), hence discover(). It is NOT a
+# herdsman-only restart. discover() republishes only configs that are missing or
+# whose payload changed, which after a wipe is all of them.
+#
+# Credentials: reuses the existing zigbee2mqtt broker user, whose ACL is
+# `readwrite zigbee2mqtt/#` — enough to read bridge/state and bridge/groups and
+# to write bridge/request/restart. No new broker user, and no read grant on
+# homeassistant/# (checking for the discovery configs directly would have needed
+# one, which the ACL denies on purpose).
+set -eu
+
+MQTT_HOST="${MQTT_HOST:-mosquitto}"
+MQTT_PORT="${MQTT_PORT:-1883}"
+MQTT_USER="${MQTT_USER:-zigbee2mqtt}"
+BASE_TOPIC="${BASE_TOPIC:-zigbee2mqtt}"
+# Poll cadence. Worst-case time-to-repair is CHECK_INTERVAL + CONFIRM_DELAY.
+CHECK_INTERVAL="${CHECK_INTERVAL:-300}"
+# Re-check gap before acting, to ride out a normal Z2M boot and a dropped QoS 0
+# publish. Must comfortably exceed a Z2M cold start.
+CONFIRM_DELAY="${CONFIRM_DELAY:-90}"
+# Floor between two restarts, so a Z2M that will not come back healthy is not
+# restarted in a loop.
+COOLDOWN="${COOLDOWN:-1800}"
+# How long to wait for a retained message to land before calling it absent.
+READ_TIMEOUT="${READ_TIMEOUT:-5}"
+# Hard ceiling on a single client invocation, as a backstop to the protocol-level
+# waits below. Every MQTT call here is bounded with `timeout`, because neither
+# client bounds a TCP connect that HANGS rather than being refused — and a stalled
+# connect is most likely precisely when this guard runs, around a broker restart.
+# Without it the loop would block forever and stop logging, which is exactly the
+# silent-failure mode the guard exists to prevent. Same reasoning and the same
+# PUB_TIMEOUT default as zigbee2mqtt-interview-watch.sh.
+#
+# NOT `mosquitto_pub -W`: that flag does not exist for mosquitto_pub (it is a
+# mosquitto_sub option) and makes every publish fail on a usage error.
+# mosquitto_sub keeps its -W, which bounds the wait for a message once CONNECTED;
+# the `timeout` wrapper is what covers the connect phase, so it is set above -W.
+PUB_TIMEOUT="${PUB_TIMEOUT:-10}"
+STATEDIR="${STATEDIR:-/state}"
+STAMP="$STATEDIR/last-restart"
+
+log() { echo "[$(date -u '+%Y-%m-%dT%H:%M:%SZ')] discovery-guard: $*"; }
+
+# Echoes the retained payload of $1, or nothing if the topic holds none.
+# A retained message is delivered immediately on subscribe, so the wait is only
+# ever paid when the topic is genuinely empty.
+retained() {
+  timeout "$((READ_TIMEOUT + 5))" mosquitto_sub -h "$MQTT_HOST" -p "$MQTT_PORT" \
+    -u "$MQTT_USER" -P "$MQTT_PASSWORD" \
+    -t "$1" -C 1 -W "$READ_TIMEOUT" 2>/dev/null || true
+}
+
+# True when Z2M is connected to this broker but has not re-run start() since the
+# retained store was last wiped.
+discovery_lost() {
+  case "$(retained "$BASE_TOPIC/bridge/state")" in
+    *online*) ;;
+    *) return 1 ;;
+  esac
+  [ -z "$(retained "$BASE_TOPIC/bridge/groups")" ]
+}
+
+in_cooldown() {
+  [ -f "$STAMP" ] || return 1
+  last=$(cat "$STAMP" 2>/dev/null || echo 0)
+  now=$(date -u +%s)
+  [ $((now - last)) -lt "$COOLDOWN" ]
+}
+
+mkdir -p "$STATEDIR"
+log "watching $BASE_TOPIC/bridge/groups on $MQTT_HOST:$MQTT_PORT" \
+    "(interval ${CHECK_INTERVAL}s, confirm ${CONFIRM_DELAY}s, cooldown ${COOLDOWN}s)"
+
+while true; do
+  sleep "$CHECK_INTERVAL"
+
+  discovery_lost || continue
+
+  log "bridge/groups missing while Z2M is online — retained store looks wiped;" \
+      "re-checking in ${CONFIRM_DELAY}s before acting"
+  sleep "$CONFIRM_DELAY"
+
+  if ! discovery_lost; then
+    log "recovered on its own, no action taken"
+    continue
+  fi
+
+  if in_cooldown; then
+    log "confirmed, but a restart was requested less than ${COOLDOWN}s ago —" \
+        "skipping. Z2M may be failing to come up; check its logs."
+    continue
+  fi
+
+  log "confirmed — requesting a Zigbee2MQTT restart to re-run discovery"
+  if timeout "$PUB_TIMEOUT" mosquitto_pub -h "$MQTT_HOST" -p "$MQTT_PORT" \
+       -u "$MQTT_USER" -P "$MQTT_PASSWORD" \
+       -t "$BASE_TOPIC/bridge/request/restart" -m '' 2>/dev/null; then
+    date -u +%s > "$STAMP"
+    log "restart requested"
+  else
+    log "failed to publish the restart request; will retry next cycle"
+  fi
+done

kubernetes/apps/home-assistant/zigbee2mqtt-ws90-reporting.md

@@ -0,0 +1,149 @@
+# WS90 Zigbee reporting configuration
+
+Records the ZCL attribute-reporting configuration applied to the Wetterstation
+(Ecowitt WS90 powered by Shelly, IEEE `0xc02cedfffe958f40`) and how to restore
+it, because **this configuration lives on the device and in Z2M's
+`database.db`, not in git**. Zigbee2MQTT has no settings-schema field for
+reporting configuration, so there is nothing declarative to reconcile — Flux
+cannot restore it and a fresh PVC will not have it.
+
+## Applied configuration
+
+Endpoint 1, all intervals in seconds. `change` is in the attribute's raw ZCL
+units; the physical deadband is `change ÷ scale`.
+
+| attribute | cluster | min | max | change | deadband | converter default |
+| --- | --- | --- | --- | --- | --- | --- |
+| temperature | `msTemperatureMeasurement` (1026) | 5 | 3600 | 10 | 0.1 °C | 10 / 3600 / 100 |
+| humidity | `msRelativeHumidity` (1029) | 5 | 3600 | 50 | 0.5 % | 10 / 3600 / 100 |
+| pressure | `msPressureMeasurement` (1027) | 5 | 3600 | 1 | 1 hPa | 10 / 3600 / 50 |
+| illuminance | `msIlluminanceMeasurement` (1024) | 5 | 3600 | 5 | 5 lx | 10 / 3600 / 5 |
+| windSpeed | `shellyWS90Wind` (0xfc01/0) | 5 | 3600 | 1 | — | 10 / 3600 / 1 |
+| windDirection | `shellyWS90Wind` (0xfc01/4) | 5 | 3600 | 1 | — | 10 / 3600 / 1 |
+| gustSpeed | `shellyWS90Wind` (0xfc01/7) | 5 | 3600 | 1 | — | 10 / 3600 / 1 |
+| uvIndex | `shellyWS90UV` (0xfc02/0) | 5 | 3600 | 1 | — | 10 / 3600 / 1 |
+| rainStatus | `shellyWS90Rain` (0xfc03/0) | 5 | 3600 | 1 | — | 10 / 3600 / 1 |
+| precipitation | `shellyWS90Rain` (0xfc03/1) | 5 | 3600 | 1 | — | 10 / 3600 / 1 |
+
+`genPowerCfg.batteryVoltage` (cluster 1, attribute 33) is deliberately left at
+its hourly default.
+
+The three `shellyWS90*` clusters are manufacturer-specific and require
+`"options": {"manufacturerCode": 5264}` on every reconfiguration request.
+
+## Why min is 5 and not 10
+
+The station transmits every 8.8 s. `min` is a hold-off: a change-triggered
+report is held until at least `min` seconds after the previous one. With
+`min: 10` every report was therefore held past a transmit cycle and landed on
+the next, so attributes reported at best every 17.6 s and drifted out of phase
+with each other — the published stream showed no 8.8 s periodicity at all,
+just a smear across 2–8 s.
+
+At `min: 5` each attribute reports on its own transmit. The MQTT stream then
+carries one publish per transmit cycle, measured at a mean gap of 8.69 s.
+Total publish volume *fell* (~631/h to ~445/h) because each publish now
+carries a whole cycle instead of a fragment of one.
+
+Any value below 8.8 s achieves this. 5 leaves margin without approaching the
+cycle time.
+
+## Why the deadbands differ from the converter defaults
+
+The defaults come from the shared `modernExtend` helpers in
+zigbee-herdsman-converters (`m.temperature()`, `m.pressure()`, `m.humidity()`,
+`m.illuminance()`), which the WS90 definition in `shelly.ts` composes without
+overrides. They are anti-chatter values sized for battery-powered indoor
+sensors, where report volume costs coin-cell life and mesh airtime. This
+station is mains-and-solar and exists to resolve small changes, so the
+trade-off runs the other way.
+
+Each value above is matched to the resolution the device actually reports:
+temperature and pressure arrive in 0.1-unit steps, humidity in whole percent.
+
+Pressure is worth calling out. The converter default of `change: 50` with
+`scale: 10` is a 5 kPa (50 hPa) deadband, which real barometric pressure never
+traverses — so pressure only ever updated on the hourly `max` heartbeat, and
+`pressure_trend` and `weather_condition` were derived from that. The same
+library's older `reporting.ts` helper configures the identical ZCL attribute
+with `change: 5`, a 5 hPa deadband, so the two code paths disagree by 10× and
+the modernExtend value is the outlier.
+
+## When this is lost
+
+Anything that makes Z2M re-run `configure` on the device overwrites all of it
+with the converter defaults:
+
+- re-pairing or re-joining the device
+- an explicit `zigbee2mqtt/bridge/request/device/configure`
+- a converter update that changes the definition and triggers reconfiguration
+
+It also will not exist on a fresh PVC. Re-apply with the commands below.
+
+## Re-applying
+
+Published from the `discovery-guard` sidecar, which already holds the
+`zigbee2mqtt` broker credential (ACL `readwrite zigbee2mqtt/#`, which covers
+`bridge/request/#`) so no credential has to leave the cluster:
+
+Ten separate requests, one per attribute — written out literally rather than
+generated by a shell helper, because the nested quoting through `kubectl exec
+-- sh -c` is where this goes wrong. Each line below is in the exact form that
+was executed and verified.
+
+```sh
+POD=$(kubectl get pod -n home-assistant -l app.kubernetes.io/name=zigbee2mqtt -o jsonpath='{.items[0].metadata.name}')
+```
+
+```sh
+kubectl exec -n home-assistant "$POD" -c discovery-guard -- sh -c 'mosquitto_pub -h "$MQTT_HOST" -p 1883 -u "$MQTT_USER" -P "$MQTT_PASSWORD" -t "$BASE_TOPIC/bridge/request/device/configure_reporting" -m "{\"id\":\"0xc02cedfffe958f40\",\"endpoint\":1,\"cluster\":\"msTemperatureMeasurement\",\"attribute\":\"measuredValue\",\"minimum_report_interval\":5,\"maximum_report_interval\":3600,\"reportable_change\":10}"'
+
+kubectl exec -n home-assistant "$POD" -c discovery-guard -- sh -c 'mosquitto_pub -h "$MQTT_HOST" -p 1883 -u "$MQTT_USER" -P "$MQTT_PASSWORD" -t "$BASE_TOPIC/bridge/request/device/configure_reporting" -m "{\"id\":\"0xc02cedfffe958f40\",\"endpoint\":1,\"cluster\":\"msRelativeHumidity\",\"attribute\":\"measuredValue\",\"minimum_report_interval\":5,\"maximum_report_interval\":3600,\"reportable_change\":50}"'
+
+kubectl exec -n home-assistant "$POD" -c discovery-guard -- sh -c 'mosquitto_pub -h "$MQTT_HOST" -p 1883 -u "$MQTT_USER" -P "$MQTT_PASSWORD" -t "$BASE_TOPIC/bridge/request/device/configure_reporting" -m "{\"id\":\"0xc02cedfffe958f40\",\"endpoint\":1,\"cluster\":\"msPressureMeasurement\",\"attribute\":\"measuredValue\",\"minimum_report_interval\":5,\"maximum_report_interval\":3600,\"reportable_change\":1}"'
+
+kubectl exec -n home-assistant "$POD" -c discovery-guard -- sh -c 'mosquitto_pub -h "$MQTT_HOST" -p 1883 -u "$MQTT_USER" -P "$MQTT_PASSWORD" -t "$BASE_TOPIC/bridge/request/device/configure_reporting" -m "{\"id\":\"0xc02cedfffe958f40\",\"endpoint\":1,\"cluster\":\"msIlluminanceMeasurement\",\"attribute\":\"measuredValue\",\"minimum_report_interval\":5,\"maximum_report_interval\":3600,\"reportable_change\":5}"'
+
+kubectl exec -n home-assistant "$POD" -c discovery-guard -- sh -c 'mosquitto_pub -h "$MQTT_HOST" -p 1883 -u "$MQTT_USER" -P "$MQTT_PASSWORD" -t "$BASE_TOPIC/bridge/request/device/configure_reporting" -m "{\"id\":\"0xc02cedfffe958f40\",\"endpoint\":1,\"cluster\":\"shellyWS90Wind\",\"attribute\":\"windSpeed\",\"minimum_report_interval\":5,\"maximum_report_interval\":3600,\"reportable_change\":1,\"options\":{\"manufacturerCode\":5264}}"'
+
+kubectl exec -n home-assistant "$POD" -c discovery-guard -- sh -c 'mosquitto_pub -h "$MQTT_HOST" -p 1883 -u "$MQTT_USER" -P "$MQTT_PASSWORD" -t "$BASE_TOPIC/bridge/request/device/configure_reporting" -m "{\"id\":\"0xc02cedfffe958f40\",\"endpoint\":1,\"cluster\":\"shellyWS90Wind\",\"attribute\":\"windDirection\",\"minimum_report_interval\":5,\"maximum_report_interval\":3600,\"reportable_change\":1,\"options\":{\"manufacturerCode\":5264}}"'
+
+kubectl exec -n home-assistant "$POD" -c discovery-guard -- sh -c 'mosquitto_pub -h "$MQTT_HOST" -p 1883 -u "$MQTT_USER" -P "$MQTT_PASSWORD" -t "$BASE_TOPIC/bridge/request/device/configure_reporting" -m "{\"id\":\"0xc02cedfffe958f40\",\"endpoint\":1,\"cluster\":\"shellyWS90Wind\",\"attribute\":\"gustSpeed\",\"minimum_report_interval\":5,\"maximum_report_interval\":3600,\"reportable_change\":1,\"options\":{\"manufacturerCode\":5264}}"'
+
+kubectl exec -n home-assistant "$POD" -c discovery-guard -- sh -c 'mosquitto_pub -h "$MQTT_HOST" -p 1883 -u "$MQTT_USER" -P "$MQTT_PASSWORD" -t "$BASE_TOPIC/bridge/request/device/configure_reporting" -m "{\"id\":\"0xc02cedfffe958f40\",\"endpoint\":1,\"cluster\":\"shellyWS90UV\",\"attribute\":\"uvIndex\",\"minimum_report_interval\":5,\"maximum_report_interval\":3600,\"reportable_change\":1,\"options\":{\"manufacturerCode\":5264}}"'
+
+kubectl exec -n home-assistant "$POD" -c discovery-guard -- sh -c 'mosquitto_pub -h "$MQTT_HOST" -p 1883 -u "$MQTT_USER" -P "$MQTT_PASSWORD" -t "$BASE_TOPIC/bridge/request/device/configure_reporting" -m "{\"id\":\"0xc02cedfffe958f40\",\"endpoint\":1,\"cluster\":\"shellyWS90Rain\",\"attribute\":\"rainStatus\",\"minimum_report_interval\":5,\"maximum_report_interval\":3600,\"reportable_change\":1,\"options\":{\"manufacturerCode\":5264}}"'
+
+kubectl exec -n home-assistant "$POD" -c discovery-guard -- sh -c 'mosquitto_pub -h "$MQTT_HOST" -p 1883 -u "$MQTT_USER" -P "$MQTT_PASSWORD" -t "$BASE_TOPIC/bridge/request/device/configure_reporting" -m "{\"id\":\"0xc02cedfffe958f40\",\"endpoint\":1,\"cluster\":\"shellyWS90Rain\",\"attribute\":\"precipitation\",\"minimum_report_interval\":5,\"maximum_report_interval\":3600,\"reportable_change\":1,\"options\":{\"manufacturerCode\":5264}}"'
+```
+
+These are idempotent — re-sending a request that matches the current
+configuration is a no-op on the device.
+
+The device must be awake and reachable; each request performs a real
+`ConfigureReporting` exchange and fails if it is not.
+
+## Verifying
+
+Read back what the device actually accepted — Z2M records the negotiated
+result, so this reflects the device, not the request:
+
+```sh
+kubectl exec -n home-assistant "$POD" -c zigbee2mqtt -- \
+  grep 0xc02cedfffe958f40 /app/data/database.db |
+  jq -r '.endpoints | to_entries[] | .value.configuredReportings[]?
+         | "cluster=\(.cluster) attr=\(.attrId) min=\(.minRepIntval) max=\(.maxRepIntval) change=\(.repChange)"'
+```
+
+All ten rows above should read `min=5 max=3600`. On the wire, the publish
+cadence on `zigbee2mqtt/Wetterstation` should sit at ~8.7 s and temperature
+should move in 0.1 °C steps; 1 °C steps mean the defaults are back.
+
+## Making it durable upstream
+
+The lasting fix is for the WS90 definition to pass explicit `reporting`
+overrides rather than inherit indoor-sensor defaults, which would fix every
+WS90 rather than this one. `m.temperature()` and the other helpers already
+accept a `reporting` argument. The pressure divergence between
+`modernExtend.ts` and `reporting.ts` is a separate upstream bug worth
+reporting on its own.

kubernetes/apps/immich/cluster.yaml

@@ -42,7 +42,43 @@ spec:
     parameters:
       shared_buffers: "512MB"
       max_wal_size: "2GB"
-      wal_compression: "on"
+      # zstd, not "on": the boolean spelling resolves to pglz (2016-era, weaker
+      # and slower than the lz4/zstd this build also accepts). Compresses the
+      # full-page images that dominate WAL volume after each checkpoint.
+      wal_compression: "zstd"
+      # A WAL segment is a fixed 16 MiB file, and archive_timeout forces a switch
+      # on a timer -- the segment is then written and archived at its full size
+      # no matter how little real WAL it holds. CNPG defaults to 5min, i.e. up
+      # to 288 segments/day = 4.6 GB/day that is largely padding.
+      #
+      # Verified on gf-pg first (the staged rollout): 211 -> 41 MB/hour, an 81%
+      # cut, with archived segments falling from ~12/hour to ~1.5/hour in step.
+      # Two independent signals moving together is what confirms the volume was
+      # padding rather than data.
+      #
+      # The cost is RPO: up to 30min of WAL is unrecoverable if this cluster and
+      # its volume are lost outright. This is a single-instance cluster, so the
+      # archive is the only off-node copy.
+      archive_timeout: "30min"
+      # Write-reduction tuning. These databases are largely idle, and the
+      # Postgres defaults favour throughput over storage that wears out.
+      #
+      # checkpoint_timeout is the lever. The 5min default pushes every dirty
+      # page to disk 288 times a day, so a small database whose same few pages
+      # are re-dirtied constantly (sessions, tokens, job state) gets rewritten
+      # over and over. Measured on ak-pg: 25.8M checkpoint buffers flushed
+      # (~206 GB) against a 1046 MB database -- the whole thing roughly twice a
+      # day. At 15min those pages coalesce and are written a third as often.
+      #
+      # max_wal_size must rise with it, or checkpoints trigger on size instead
+      # and the timeout change does nothing. It also bounds crash recovery to
+      # at most 2GB of WAL replay.
+      #
+      # NOT changed: synchronous_commit stays at the default `on`. It is off for
+      # ha-pg because a state recorder is telemetry; these hold credentials,
+      # documents and job state, where losing the last few hundred ms of commits
+      # on an unclean shutdown is not an acceptable trade.
+      checkpoint_timeout: "15min"
   resources:
     requests:
       cpu: 500m

kubernetes/apps/immich/machine-learning.yaml

@@ -6,6 +6,13 @@ metadata:
   labels:
     app.kubernetes.io/name: immich-machine-learning
 spec:
+  # Scale this and immich-server to 0 for the duration of any single-node
+  # maintenance — cluster CPU requests exceed what two nodes can hold, and this
+  # pair frees ~1000m against a shortfall of roughly 200m. Stopping them is the
+  # right lever rather than lowering their requests: 500m reserved against a
+  # 2m idle draw looks like waste, but inference bursts hard and the request is
+  # honest about what it needs when someone uploads photos.
+  # See RUNBOOK-baremetal-conversion.md.
   replicas: 1
   strategy:
     type: RollingUpdate

kubernetes/apps/immich/redis.yaml

@@ -27,7 +27,7 @@ spec:
         # Pinned to 7.4-alpine to match the LXC 225 compose exactly.
         # Any redis major-version bump is a separate migration.
         - name: redis
-          image: redis:7.4.10-alpine # {"$imagepolicy": "flux-system:redis-immich"}
+          image: redis:7.4.11-alpine # {"$imagepolicy": "flux-system:redis-immich"}
           args:
             - redis-server
             - --save

kubernetes/apps/immich/server.yaml

@@ -36,6 +36,9 @@ metadata:
   labels:
     app.kubernetes.io/name: immich-server
 spec:
+  # Scaled to 0 alongside immich-machine-learning during single-node
+  # maintenance; see the note there. The CNPG Postgres cluster stays running,
+  # so recovery is just these two replica counts.
   replicas: 1
   strategy:
     type: Recreate
@@ -147,7 +150,9 @@ spec:
               - key: immich-config.json
                 path: immich-config.json
         - name: le-ca
-          emptyDir: {}
+          emptyDir:
+            medium: Memory
+            sizeLimit: 16Mi
 ---
 apiVersion: v1
 kind: Service

kubernetes/apps/insecure-backend-serverstransport.yaml

@@ -2,7 +2,6 @@
 # backend serving HTTPS with a self-signed / hostname-mismatched cert:
 #
 #   apps/file       — NAS web UI on a self-signed cert
-#   apps/pve        — PVE on :8006 with per-node self-signed cert
 #
 # Replaces the nginx `backend-protocol: HTTPS` + `proxy-ssl-verify: off`
 # annotation pair. Each Ingress that wants this behaviour sets

kubernetes/apps/kustomization.yaml

@@ -7,7 +7,6 @@ resources:
   - file/
   - gitea/
   - ical-proxy/
-  - pve/
   # woodpecker/ moved to its own dedicated `woodpecker` Kustomization
   # (clusters/homelab/woodpecker.yaml) so its wp-pg CNPG cluster's barman
   # dependency + Cluster health-gating don't contaminate this shared apps KS.

kubernetes/apps/monitoring-rules/cnpg-podmonitors.yaml

@@ -99,3 +99,26 @@ spec:
       interval: 30s
   podTargetLabels:
     - role
+---
+# ha-pg — Postgres backend for the Home Assistant recorder
+# (kubernetes/apps/home-assistant/ha-pg-cluster.yaml). Added with the cluster
+# rather than after it, so the migration has metrics from the first moment
+# instead of discovering the gap later, which is how wp-pg's scrape hole was
+# found (#433).
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+  name: cnpg-ha-pg
+  namespace: home-assistant
+  labels:
+    app.kubernetes.io/name: cloudnative-pg
+    release: kube-prometheus-stack
+spec:
+  selector:
+    matchLabels:
+      cnpg.io/cluster: ha-pg
+  podMetricsEndpoints:
+    - port: metrics
+      interval: 30s
+  podTargetLabels:
+    - role

kubernetes/apps/monitoring-rules/kustomization.yaml

@@ -8,11 +8,16 @@ resources:
   - prometheusrule-cilium.yaml
   - prometheusrule-hetzner-backup.yaml
   - prometheusrule-loki.yaml
+  - prometheusrule-mariadb-backup.yaml
   - prometheusrule-netbird-availability.yaml
   - prometheusrule-netbird-backup.yaml
   - prometheusrule-netbird-ingress.yaml
   - prometheusrule-velero.yaml
   - prometheusrule-woodpecker-rotation.yaml
+  # CNPG backup health. Seven Postgres clusters ship to Hetzner and nothing
+  # alerted if that stopped — HetznerBackupEndpointDown only probes the S3
+  # endpoint, which proves reachability, not that anything was written.
+  - prometheusrule-cnpg-backup.yaml
   - alloy-servicemonitor.yaml
   - cnpg-podmonitors.yaml
   - home-assistant-servicemonitor.yaml

kubernetes/apps/monitoring-rules/prometheusrule-ceph.yaml

@@ -12,18 +12,59 @@ spec:
     - name: ceph.rules
       rules:
         - alert: CephHealthWarn
-          # ceph_health_status: 0 = HEALTH_OK, 1 = HEALTH_WARN, 2 = HEALTH_ERR.
-          # 6h is deliberately long. Ceph warns for self-clearing conditions —
-          # a slow BlueStore op during the Sunday restore sweep, a mon briefly
-          # below its disk threshold — and paging on those trains people to
-          # ignore the alert. What matters is a warning that does not clear.
-          expr: ceph_health_status == 1
+          # Per-check rather than ceph_health_status, so one permanently-active
+          # check cannot mask every other Ceph warning.
+          #
+          # AUTH_INSECURE_* is excluded: it cannot clear until the nodes run a
+          # kernel supporting aes256k over the CSI mount path (#527).
+          #
+          # POOL_NO_REDUNDANCY is in scope. An unreplicated pool makes
+          # `ceph osd ok-to-stop` refuse for every OSD, so it reports blocked
+          # maintenance and not just data risk — do not exclude it for a pool
+          # that is unreplicated deliberately.
+          #
+          # 6h: Ceph warns for self-clearing conditions. Only a warning that
+          # does not clear is worth paging on.
+          #
+          # BLUESTORE_SLOW_OP_ALERT breaks that premise and is still NOT
+          # excluded (#557). Ceph holds the check for
+          # bluestore_slow_ops_warn_lifetime (24h) after the LAST slow op, so
+          # one multi-second stall outlives the 6h `for` and always pages,
+          # with latency back to normal long before anyone looks. Measured:
+          # both episodes in the 30d to 2026-08-23 ran ~26h and cleared
+          # untouched, the first ~24h after the last stall.
+          #
+          # Both began inside the Velero restore-test sweep — a ~5h sustained
+          # restore that outruns the SLC cache on these QLC NVMe. The sweep
+          # runs monthly, 08:00-14:00Z on the 1st.
+          #
+          # Expected for the hardware, but not excludable:
+          # this is also the earliest signal of a genuinely failing drive, and
+          # all three are at 80-87% endurance used (see cephcluster.yaml).
+          expr: ceph_health_detail{name!~"AUTH_INSECURE_.+"} == 1
           for: 6h
           labels:
             severity: warning
           annotations:
-            summary: "Ceph has been HEALTH_WARN for 6h"
-            description: "The Ceph cluster has held HEALTH_WARN for six hours, so it is not a self-clearing blip. Get the detail with `kubectl -n rook-ceph exec deploy/rook-ceph-tools -- ceph health detail`."
+            summary: "Ceph health check {{ $labels.name }} active for 6h"
+            description: "Ceph health check {{ $labels.name }} has been active for six hours. Only AUTH_INSECURE_* is excluded (kernel-blocked, see #527). BLUESTORE_SLOW_OP_ALERT is sticky for 24h after the last stall and clears itself: check whether the stall fell inside the monthly Velero restore-test window (08:00-14:00Z on the 1st), which is expected on these QLC drives — one outside that window, or a repeat on the same OSD, means check that drive (#557). Detail: `kubectl -n rook-ceph exec deploy/rook-ceph-tools -- ceph health detail`."
+
+        - alert: CephPoolLostRedundancy
+          # Names the pool, which POOL_NO_REDUNDANCY does not.
+          #
+          # There is no pool-size metric, but bytes_used/stored IS the
+          # replication factor. The >1e6 floor drops pools too small to measure
+          # meaningfully, empty ones included.
+          #
+          # No pool is exempt.
+          expr: |
+            ceph_pool_bytes_used / ceph_pool_stored < 1.5 and ceph_pool_stored > 1e6
+          for: 30m
+          labels:
+            severity: critical
+          annotations:
+            summary: "Ceph pool {{ $labels.pool_id }} is running unreplicated"
+            description: "Pool {{ $labels.pool_id }} has a replication factor below 1.5, meaning a single OSD loss destroys data. No pool is meant to be unreplicated — an unreplicated pool also blocks ALL OSD maintenance cluster-wide, since `ceph osd ok-to-stop` refuses while its PGs would go offline. Check with `kubectl -n rook-ceph exec deploy/rook-ceph-tools -- ceph osd pool ls detail`."
 
         - alert: CephHealthError
           # HEALTH_ERR means data availability or redundancy is at risk.

kubernetes/apps/monitoring-rules/prometheusrule-cnpg-backup.yaml

@@ -0,0 +1,78 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: cnpg-backup
+  namespace: monitoring
+  labels:
+    # kube-prometheus-stack's Prometheus resource selects on this label;
+    # without it the rule is created but never loaded into Prometheus.
+    release: kube-prometheus-stack
+spec:
+  groups:
+    - name: cnpg-backup.rules
+      rules:
+        # WHY THIS FILE EXISTS
+        # Seven Postgres clusters back up to Hetzner and NOTHING alerted if that
+        # stopped. The only related rule was HetznerBackupEndpointDown, which
+        # probes the S3 endpoint — it proves the bucket is reachable, not that
+        # anything was written to it. Same failure shape as the WordPress backup
+        # that failed nightly for three days against a bucket that was declared
+        # in tofu but never applied: the status surface looked fine because
+        # nothing checked the thing that mattered.
+
+        - alert: CNPGWalArchiveBacklog
+          # `ready` counts WAL segments sitting in pg_wal/archive_status waiting
+          # to be archived. Steady state is 0 across every cluster (verified
+          # 2026-08-19). A sustained backlog means the archiver is failing:
+          # point-in-time recovery is silently broken from that moment, and
+          # pg_wal keeps growing until it fills the PVC and takes Postgres down.
+          #
+          # This is the load-bearing alert of the two. It depends only on the
+          # instance-level collector, so unlike the backup-age metrics below it
+          # is unaffected by the missing Backup CRs (#522).
+          #
+          # Threshold: >5 segments sustained for 30m. Brief non-zero values are
+          # normal under write bursts — a segment is 16 MiB, so 5 is roughly
+          # 80 MiB of un-archived WAL, well below anything that threatens the
+          # volume but far above idle noise.
+          expr: cnpg_collector_pg_wal_archive_status{value="ready"} > 5
+          for: 30m
+          labels:
+            severity: warning
+          annotations:
+            summary: "CNPG WAL archiving is falling behind on {{ $labels.namespace }}/{{ $labels.pod }}"
+            description: "{{ $value }} WAL segments have been waiting to archive for over 30 minutes. Point-in-time recovery is broken for anything after the last archived segment, and pg_wal will grow until the PVC fills. Check the barman-cloud sidecar logs and the ObjectStore credentials: `kubectl -n {{ $labels.namespace }} logs {{ $labels.pod }} -c plugin-barman-cloud`."
+
+        - alert: CNPGBackupFailed
+          # cnpg_collector_last_failed_backup_timestamp is a unix timestamp, 0
+          # when no failure is recorded. Guard on > 0 so the epoch-zero default
+          # is not read as "failed in 1970".
+          expr: |
+            cnpg_collector_last_failed_backup_timestamp > 0
+            and
+            (time() - cnpg_collector_last_failed_backup_timestamp) < 86400
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: "CNPG backup failed in the last 24h on {{ $labels.namespace }}/{{ $labels.pod }}"
+            description: "A base backup for {{ $labels.namespace }}/{{ $labels.pod }} failed within the last 24 hours. Verify what is actually in the bucket rather than trusting CR status — see the barman-cloud-backup-list recipe in kubernetes/apps/home-assistant/MIGRATION-recorder-postgres.md."
+
+        # DELIBERATELY ABSENT: a backup-age alert, which is the one you would
+        # most want here.
+        #
+        # The natural expression is
+        #   (time() - cnpg_collector_last_available_backup_timestamp) > 48*3600
+        # and it CANNOT be used today: that metric reads exactly 0 on every
+        # instance in the cluster (verified 2026-08-19 across gf-pg, ak-pg-1/2,
+        # pprl-pg-2/3, im-pg, wp-pg, ha-pg). CNPG has no record of any
+        # successful backup anywhere, because the Backup CRs it derives that
+        # from are being deleted within a minute of completing — see #522.
+        #
+        # Backups themselves DO work; that was verified independently by
+        # listing the bucket with barman-cloud-backup-list and restoring a WAL
+        # segment back out of S3. The gap is in the record, not the data.
+        #
+        # Adding the rule now would fire permanently for all seven clusters and
+        # be silenced within a day, which is worse than not having it. Add it as
+        # part of resolving #522, once the metric reflects reality.

kubernetes/apps/monitoring-rules/prometheusrule-mariadb-backup.yaml

@@ -0,0 +1,51 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: mariadb-backup
+  namespace: monitoring
+  labels:
+    # kube-prometheus-stack's Prometheus selects on this label.
+    release: kube-prometheus-stack
+spec:
+  groups:
+    - name: mariadb-backup.rules
+      rules:
+        - alert: WordpressMariadbBackupStale
+          # The nightly 02:45 logical dump (kubernetes/apps/wordpress/mariadb.yaml)
+          # is the only transaction-consistent copy of www.renner.uno's database
+          # — Velero snapshots the PVC, which for a running DB is not a restore
+          # point you would trust.
+          #
+          # KubeJobFailed already covers a failed Job, but only while the Job
+          # object survives. On #550 it cleared when the failed Job was
+          # garbage-collected, not when a backup succeeded — so a failure that
+          # is GC'd before anyone looks leaves no signal at all.
+          # lastSuccessfulTime lives on the CronJob and outlives Job GC.
+          #
+          # 26h = one missed cycle plus 2h slack, matching NetbirdVpsBackupStale.
+          # It counts any Job owned by the CronJob, so a manual
+          # `kubectl create job --from=cronjob/...` re-run clears it too.
+          expr: |
+            time() - kube_cronjob_status_last_successful_time{namespace="wordpress", cronjob="wordpress-mariadb-backup"} > 26 * 3600
+          for: 30m
+          labels:
+            severity: warning
+          annotations:
+            summary: "WordPress MariaDB backup has not succeeded in 26h+"
+            description: "The wordpress-mariadb-backup CronJob's last successful run is over 26 hours old. Check the newest Job's pod logs in the wordpress namespace: `kubectl -n wordpress get jobs`. The 2026-08-19 failure was `The specified bucket does not exist.` against renner-uno-k8s-mariadb while the endpoint itself probed healthy, so a green HetznerBackupEndpointDown does not clear this."
+
+        - alert: WordpressMariadbBackupNeverRan
+          # The staleness expression above goes blind when the metric is
+          # absent, which is what happens once the operator recreates the
+          # CronJob. Without this guard "never backed up" is quieter than
+          # "backed up late".
+          expr: |
+            kube_cronjob_created{namespace="wordpress", cronjob="wordpress-mariadb-backup"}
+            unless
+            kube_cronjob_status_last_successful_time{namespace="wordpress", cronjob="wordpress-mariadb-backup"}
+          for: 6h
+          labels:
+            severity: warning
+          annotations:
+            summary: "WordPress MariaDB backup has no recorded successful run"
+            description: "The wordpress-mariadb-backup CronJob exists but has no lastSuccessfulTime: either it has not completed once since the Backup CR was recreated, or every run since has failed. Check `kubectl -n wordpress get jobs` and that the hetzner-s3-creds ExternalSecret is synced."

kubernetes/apps/monitoring/dashboards/kustomization.yaml

@@ -83,9 +83,6 @@ configMapGenerator:
   - name: dashboard-power-flow
     files:
       - power-flow.json
-  - name: dashboard-proxmox-cluster
-    files:
-      - proxmox-cluster.json
   - name: dashboard-pv-system-roi
     files:
       - pv-system-roi.json

kubernetes/apps/monitoring/dashboards/proxmox-cluster.json

@@ -1,239 +0,0 @@
-{
-  "annotations": {"list": []},
-  "description": "Proxmox VE Cluster Overview",
-  "editable": true,
-  "fiscalYearStartMonth": 0,
-  "graphTooltip": 0,
-  "id": null,
-  "links": [],
-  "panels": [
-    {
-      "datasource": {"type": "prometheus", "uid": "prometheus"},
-      "fieldConfig": {
-        "defaults": {"color": {"mode": "thresholds"}, "mappings": [], "thresholds": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "green", "value": 3}]}},
-        "overrides": []
-      },
-      "gridPos": {"h": 4, "w": 6, "x": 0, "y": 0},
-      "id": 1,
-      "options": {"colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto"},
-      "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "count(count by (id) (pve_up{id=~\"node/.*\"} == 1))", "refId": "A"}],
-      "title": "Online Nodes",
-      "type": "stat"
-    },
-    {
-      "datasource": {"type": "prometheus", "uid": "prometheus"},
-      "fieldConfig": {
-        "defaults": {"color": {"mode": "thresholds"}, "mappings": [], "thresholds": {"mode": "absolute", "steps": [{"color": "blue", "value": null}]}},
-        "overrides": []
-      },
-      "gridPos": {"h": 4, "w": 6, "x": 6, "y": 0},
-      "id": 2,
-      "options": {"colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto"},
-      "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "count(count by (id) (pve_guest_info{type=\"qemu\", template=\"0\"}))", "refId": "A"}],
-      "title": "Total VMs",
-      "type": "stat"
-    },
-    {
-      "datasource": {"type": "prometheus", "uid": "prometheus"},
-      "fieldConfig": {
-        "defaults": {"color": {"mode": "thresholds"}, "mappings": [], "thresholds": {"mode": "absolute", "steps": [{"color": "purple", "value": null}]}},
-        "overrides": []
-      },
-      "gridPos": {"h": 4, "w": 6, "x": 12, "y": 0},
-      "id": 3,
-      "options": {"colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto"},
-      "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "count(count by (id) (pve_guest_info{type=\"lxc\"}))", "refId": "A"}],
-      "title": "Total LXC Containers",
-      "type": "stat"
-    },
-    {
-      "datasource": {"type": "prometheus", "uid": "prometheus"},
-      "fieldConfig": {
-        "defaults": {"color": {"mode": "thresholds"}, "mappings": [], "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}]}},
-        "overrides": []
-      },
-      "gridPos": {"h": 4, "w": 5, "x": 15, "y": 0},
-      "id": 4,
-      "options": {"colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto"},
-      "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "count(count by (id) (pve_up{id=~\"(qemu|lxc)/.*\"} == 1))", "refId": "A"}],
-      "title": "Running Guests",
-      "type": "stat"
-    },
-    {
-      "datasource": {"type": "prometheus", "uid": "prometheus"},
-      "fieldConfig": {
-        "defaults": {"color": {"mode": "thresholds"}, "mappings": [{"options": {"0": {"color": "green", "index": 0, "text": "All Backed Up"}}, "type": "value"}], "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "yellow", "value": 1}, {"color": "red", "value": 5}]}},
-        "overrides": []
-      },
-      "gridPos": {"h": 4, "w": 4, "x": 20, "y": 0},
-      "id": 12,
-      "options": {"colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto"},
-      "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "max(pve_not_backed_up_total)", "refId": "A"}],
-      "title": "Not Backed Up",
-      "type": "stat"
-    },
-    {
-      "datasource": {"type": "prometheus", "uid": "prometheus"},
-      "fieldConfig": {
-        "defaults": {
-          "color": {"mode": "thresholds"},
-          "mappings": [],
-          "max": 100,
-          "min": 0,
-          "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "yellow", "value": 60}, {"color": "red", "value": 80}]},
-          "unit": "percent"
-        },
-        "overrides": []
-      },
-      "gridPos": {"h": 6, "w": 8, "x": 0, "y": 4},
-      "id": 5,
-      "options": {"displayMode": "lcd", "minVizHeight": 10, "minVizWidth": 0, "orientation": "horizontal", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "showUnfilled": true},
-      "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "max by (id) (pve_cpu_usage_ratio{id=~\"node/.*\"}) * 100", "legendFormat": "{{id}}", "refId": "A"}],
-      "title": "Node CPU Usage",
-      "type": "bargauge"
-    },
-    {
-      "datasource": {"type": "prometheus", "uid": "prometheus"},
-      "fieldConfig": {
-        "defaults": {
-          "color": {"mode": "thresholds"},
-          "mappings": [],
-          "max": 100,
-          "min": 0,
-          "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "yellow", "value": 70}, {"color": "red", "value": 85}]},
-          "unit": "percent"
-        },
-        "overrides": []
-      },
-      "gridPos": {"h": 6, "w": 8, "x": 8, "y": 4},
-      "id": 6,
-      "options": {"displayMode": "lcd", "minVizHeight": 10, "minVizWidth": 0, "orientation": "horizontal", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "showUnfilled": true},
-      "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "max by (id) (pve_memory_usage_bytes{id=~\"node/.*\"}) / max by (id) (pve_memory_size_bytes{id=~\"node/.*\"}) * 100", "legendFormat": "{{id}}", "refId": "A"}],
-      "title": "Node Memory Usage",
-      "type": "bargauge"
-    },
-    {
-      "datasource": {"type": "prometheus", "uid": "prometheus"},
-      "fieldConfig": {
-        "defaults": {
-          "color": {"mode": "thresholds"},
-          "mappings": [],
-          "max": 100,
-          "min": 0,
-          "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}, {"color": "yellow", "value": 70}, {"color": "red", "value": 85}]},
-          "unit": "percent"
-        },
-        "overrides": []
-      },
-      "gridPos": {"h": 6, "w": 8, "x": 16, "y": 4},
-      "id": 7,
-      "options": {"displayMode": "lcd", "minVizHeight": 10, "minVizWidth": 0, "orientation": "horizontal", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "showUnfilled": true},
-      "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "max by (id) (pve_disk_usage_bytes{id=~\"storage/.*\"}) / max by (id) (pve_disk_size_bytes{id=~\"storage/.*\"}) * 100", "legendFormat": "{{id}}", "refId": "A"}],
-      "title": "Storage Usage",
-      "type": "bargauge"
-    },
-    {
-      "datasource": {"type": "prometheus", "uid": "prometheus"},
-      "fieldConfig": {
-        "defaults": {
-          "color": {"mode": "palette-classic"},
-          "custom": {"axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": {"legend": false, "tooltip": false, "viz": false}, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": {"type": "linear"}, "showPoints": "never", "spanNulls": false, "stacking": {"group": "A", "mode": "none"}, "thresholdsStyle": {"mode": "off"}},
-          "mappings": [],
-          "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}]},
-          "unit": "percent"
-        },
-        "overrides": []
-      },
-      "gridPos": {"h": 8, "w": 12, "x": 0, "y": 10},
-      "id": 8,
-      "options": {"legend": {"calcs": ["mean", "max"], "displayMode": "table", "placement": "right", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
-      "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "max by (id) (pve_cpu_usage_ratio{id=~\"node/.*\"}) * 100", "legendFormat": "{{id}}", "refId": "A"}],
-      "title": "Node CPU Over Time",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {"type": "prometheus", "uid": "prometheus"},
-      "fieldConfig": {
-        "defaults": {
-          "color": {"mode": "palette-classic"},
-          "custom": {"axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": {"legend": false, "tooltip": false, "viz": false}, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": {"type": "linear"}, "showPoints": "never", "spanNulls": false, "stacking": {"group": "A", "mode": "none"}, "thresholdsStyle": {"mode": "off"}},
-          "mappings": [],
-          "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}]},
-          "unit": "bytes"
-        },
-        "overrides": []
-      },
-      "gridPos": {"h": 8, "w": 12, "x": 12, "y": 10},
-      "id": 9,
-      "options": {"legend": {"calcs": ["mean", "max"], "displayMode": "table", "placement": "right", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
-      "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "max by (id) (pve_memory_usage_bytes{id=~\"node/.*\"})", "legendFormat": "{{id}}", "refId": "A"}],
-      "title": "Node Memory Usage Over Time",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {"type": "prometheus", "uid": "prometheus"},
-      "fieldConfig": {
-        "defaults": {
-          "color": {"mode": "thresholds"},
-          "custom": {"align": "auto", "cellOptions": {"type": "auto"}, "inspect": false},
-          "mappings": [],
-          "thresholds": {"mode": "absolute", "steps": [{"color": "text", "value": null}]}
-        },
-        "overrides": [
-          {"matcher": {"id": "byName", "options": "Status"}, "properties": [{"id": "custom.cellOptions", "value": {"type": "color-background"}}, {"id": "mappings", "value": [{"options": {"0": {"color": "red", "index": 1, "text": "Stopped"}, "1": {"color": "green", "index": 0, "text": "Running"}}, "type": "value"}]}]},
-          {"matcher": {"id": "byName", "options": "CPU %"}, "properties": [{"id": "unit", "value": "percent"}, {"id": "custom.cellOptions", "value": {"mode": "gradient", "type": "gauge"}}, {"id": "min", "value": 0}, {"id": "max", "value": 100}]},
-          {"matcher": {"id": "byName", "options": "Memory %"}, "properties": [{"id": "unit", "value": "percent"}, {"id": "custom.cellOptions", "value": {"mode": "gradient", "type": "gauge"}}, {"id": "min", "value": 0}, {"id": "max", "value": 100}]}
-        ]
-      },
-      "gridPos": {"h": 10, "w": 24, "x": 0, "y": 18},
-      "id": 10,
-      "options": {"footer": {"fields": "", "reducer": ["sum"], "show": false}, "showHeader": true, "sortBy": [{"desc": false, "displayName": "Name"}]},
-      "targets": [
-        {"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "max by (id, name, node, type) (pve_guest_info)", "format": "table", "instant": true, "refId": "A"},
-        {"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "max by (id) (pve_cpu_usage_ratio{id=~\"(qemu|lxc)/.*\"}) * 100", "format": "table", "instant": true, "refId": "B"},
-        {"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "max by (id) (pve_memory_usage_bytes{id=~\"(qemu|lxc)/.*\"}) / max by (id) (pve_memory_size_bytes{id=~\"(qemu|lxc)/.*\"}) * 100", "format": "table", "instant": true, "refId": "C"},
-        {"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "max by (id) (pve_up{id=~\"(qemu|lxc)/.*\"})", "format": "table", "instant": true, "refId": "D"}
-      ],
-      "title": "Guest Status",
-      "transformations": [
-        {"id": "seriesToColumns", "options": {"byField": "id"}},
-        {"id": "organize", "options": {"excludeByName": {"Time": true, "Time 1": true, "Time 2": true, "Time 3": true, "Time 4": true, "Value #A": true}, "indexByName": {"id": 0, "name": 1, "node": 2, "type": 3, "Value #D": 4, "Value #B": 5, "Value #C": 6}, "renameByName": {"Value #B": "CPU %", "Value #C": "Memory %", "Value #D": "Status", "id": "ID", "name": "Name", "node": "Node", "type": "Type"}}}
-      ],
-      "type": "table"
-    },
-    {
-      "datasource": {"type": "prometheus", "uid": "prometheus"},
-      "fieldConfig": {
-        "defaults": {
-          "color": {"mode": "palette-classic"},
-          "custom": {"axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": {"legend": false, "tooltip": false, "viz": false}, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": {"type": "linear"}, "showPoints": "never", "spanNulls": false, "stacking": {"group": "A", "mode": "none"}, "thresholdsStyle": {"mode": "off"}},
-          "mappings": [],
-          "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}]},
-          "unit": "Bps"
-        },
-        "overrides": []
-      },
-      "gridPos": {"h": 8, "w": 24, "x": 0, "y": 28},
-      "id": 11,
-      "options": {"legend": {"calcs": ["mean", "max"], "displayMode": "table", "placement": "right", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
-      "targets": [
-        {"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "max by (id) (rate(pve_disk_read_bytes_total{id=~\"(qemu|lxc)/.*\"}[5m]))", "legendFormat": "{{id}} read", "refId": "A"},
-        {"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "max by (id) (rate(pve_disk_written_bytes_total{id=~\"(qemu|lxc)/.*\"}[5m]))", "legendFormat": "{{id}} write", "refId": "B"}
-      ],
-      "title": "Guest Disk I/O",
-      "type": "timeseries"
-    }
-  ],
-  "refresh": "30s",
-  "schemaVersion": 38,
-  "style": "dark",
-  "tags": ["homelab", "proxmox", "virtualization"],
-  "templating": {"list": []},
-  "time": {"from": "now-1h", "to": "now"},
-  "timepicker": {},
-  "timezone": "",
-  "title": "Proxmox VE Cluster",
-  "uid": "proxmox-cluster",
-  "version": 1,
-  "weekStart": ""
-}

kubernetes/apps/monitoring/helmrelease-prometheus.yaml

@@ -93,68 +93,8 @@ spec:
               resources:
                 requests:
                   storage: 50Gi
-        # Scrape existing Proxmox infrastructure targets
+        # Additional scrape targets not covered by ServiceMonitors.
         additionalScrapeConfigs:
-          # pve-exporter is a blackbox-style exporter: Prometheus passes
-          # ?target=<host> via relabel, and one in-cluster exporter
-          # instance queries each PVE node's API on the user's behalf.
-          # The targets list is the PVE node addresses; the actual
-          # scrape goes to pve-exporter.monitoring.svc:9221.
-          #
-          # Cluster collectors (pve_up, pve_guest_info, …) return the SAME
-          # cluster-wide resource set no matter which node answers, so they
-          # are scraped from a single node only. Scraping them from all three
-          # (the pre-2026-07 config) ingested every node/guest series 3x —
-          # identical `id`, differing `instance` — a real TSDB cardinality
-          # cost that dashboards had to paper over with max by (id).
-          #
-          # Target the pve Service (apps ns) rather than a hardcoded node IP:
-          # its EndpointSlice load-balances the PVE API across all three nodes,
-          # so cluster metrics keep flowing as long as ANY node is up, instead
-          # of pausing whenever .201 (jupiter) specifically is down. The
-          # exporter reaches the API on 8006 (verify_ssl:false, so the LB'd
-          # cert hostname mismatch is a non-issue).
-          - job_name: pve
-            static_configs:
-              - targets:
-                  - "pve.apps.svc.cluster.local"
-            metrics_path: /pve
-            params:
-              cluster: ["1"]
-              node: ["0"]
-            relabel_configs:
-              - source_labels: [__address__]
-                target_label: __param_target
-              - source_labels: [__param_target]
-                target_label: instance
-              - target_label: __address__
-                replacement: pve-exporter.monitoring.svc.cluster.local:9221
-
-          # Node collectors (pve_version_info, replication status) are
-          # node-local and genuinely distinct per target, so every node is
-          # scraped — this is the split recommended by prometheus-pve-exporter
-          # upstream to avoid the cluster-collector duplication above.
-          # Targets are the node FQDNs, not raw IPs: the instance label then
-          # reads jupiter/neptune/saturn.renner.uno, and a renumbered node is
-          # picked up from DNS without editing this file.
-          - job_name: pve-node
-            static_configs:
-              - targets:
-                  - "jupiter.renner.uno"
-                  - "neptune.renner.uno"
-                  - "saturn.renner.uno"
-            metrics_path: /pve
-            params:
-              cluster: ["0"]
-              node: ["1"]
-            relabel_configs:
-              - source_labels: [__address__]
-                target_label: __param_target
-              - source_labels: [__param_target]
-                target_label: instance
-              - target_label: __address__
-                replacement: pve-exporter.monitoring.svc.cluster.local:9221
-
           # Gitea exposes /metrics anonymously on its main HTTP port. The
           # `gitea_*` metrics power the gitea-overview dashboard. Gitea now runs
           # in-cluster (StatefulSet in apps ns; LXC 114 retired), so scrape the
@@ -402,6 +342,52 @@ spec:
         limits:
           memory: 256Mi
 
+    # Drop the apiserver's high-cardinality latency histograms.
+    #
+    # Measured 2026-08-20: Prometheus held 298,236 active series, of which
+    # ~108,000 — over a third — were apiserver/etcd `_bucket` histograms. Each
+    # histogram fans out to one series per bucket x verb x resource x scope, so
+    # a single metric like apiserver_request_duration_seconds_bucket was 23,810
+    # series on its own. For comparison, every Home Assistant metric combined is
+    # ~4,100.
+    #
+    # That cardinality is the dominant driver of Prometheus write volume
+    # (~11 GB/day of TSDB + WAL, doubled again by Ceph replication), and none of
+    # it is observable value here: per-bucket apiserver latency quantiles are for
+    # clusters with SLAs and dozens of nodes, not three.
+    #
+    # ONLY metrics with ZERO rule dependencies are dropped. Verified against the
+    # live rule set before writing this — each metric checked individually for
+    # references in alert and recording rules:
+    #
+    #   apiserver_request_duration_seconds_bucket        rules=0   23,810 series
+    #   apiserver_request_body_size_bytes_bucket         rules=0   23,200
+    #   etcd_request_duration_seconds_bucket             rules=0   15,840
+    #   apiserver_watch_list_duration_seconds_bucket     rules=0   11,538
+    #   apiserver_watch_cache_read_wait_seconds_bucket   rules=0    8,232
+    #   apiserver_response_sizes_bucket                  rules=0    7,592
+    #   apiserver_watch_events_sizes_bucket              rules=0    5,220
+    #
+    # apiserver_request_sli_duration_seconds_bucket is DELIBERATELY KEPT: 23
+    # recording rules depend on it, including the apiserver availability SLO and
+    # the KubeAPIErrorBudgetBurn burn-rate series. Dropping it would silently
+    # break that alerting, which is the failure mode this comment exists to
+    # prevent if someone later "tidies up" this regex.
+    #
+    # Only the `_bucket` series are dropped. The matching `_count` and `_sum`
+    # remain, so request rates and mean latencies still work — what is lost is
+    # per-bucket quantiles.
+    #
+    # etcd_* appears here even though kubeEtcd is disabled below: these are the
+    # apiserver's own etcd-client metrics, exposed on the apiserver endpoint, not
+    # a separate etcd scrape.
+    kubeApiServer:
+      serviceMonitor:
+        metricRelabelings:
+          - action: drop
+            sourceLabels: [__name__]
+            regex: (apiserver_request_duration_seconds_bucket|apiserver_request_body_size_bytes_bucket|etcd_request_duration_seconds_bucket|apiserver_watch_list_duration_seconds_bucket|apiserver_watch_cache_read_wait_seconds_bucket|apiserver_response_sizes_bucket|apiserver_watch_events_sizes_bucket)
+
     # Disable etcd/scheduler/controller-manager scraping (Talos doesn't expose them by default)
     kubeEtcd:
       enabled: false

kubernetes/apps/monitoring/kustomization.yaml

@@ -3,8 +3,6 @@ kind: Kustomization
 resources:
   - grafana-oauth-externalsecret.yaml
   - alertmanager-smtp-externalsecret.yaml
-  - pve-exporter-config-externalsecret.yaml
-  - pve-exporter-deployment.yaml
   - helmrelease-prometheus.yaml
   - helmrelease-loki.yaml
   - helmrelease-alloy-syslog.yaml

kubernetes/apps/monitoring/pve-exporter-config-externalsecret.yaml

@@ -1,17 +0,0 @@
-apiVersion: external-secrets.io/v1
-kind: ExternalSecret
-metadata:
-  name: pve-exporter-config
-  namespace: monitoring
-spec:
-  # 10m (was 1h) so an automated token rotation propagates within ~10m; Reloader
-  # then restarts pve-exporter. See kubernetes/infrastructure/openbao/rotation/pve-exporter.yaml.
-  refreshInterval: 10m
-  secretStoreRef:
-    kind: ClusterSecretStore
-    name: openbao
-  target:
-    name: pve-exporter-config # keeps the old Secret's name — consumers unchanged
-  dataFrom:
-    - extract:
-        key: apps/monitoring/pve-exporter-config

kubernetes/apps/monitoring/pve-exporter-deployment.yaml

@@ -1,82 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: pve-exporter
-  namespace: monitoring
-  annotations:
-    # Reloader restarts pve-exporter when pve-exporter-config changes (token
-    # rotation) so it reloads pve.yml. See
-    # kubernetes/infrastructure/openbao/rotation/pve-exporter.yaml.
-    secret.reloader.stakater.com/reload: pve-exporter-config
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app.kubernetes.io/name: pve-exporter
-  template:
-    metadata:
-      labels:
-        app.kubernetes.io/name: pve-exporter
-    spec:
-      securityContext:
-        runAsNonRoot: true
-        runAsUser: 1000
-        runAsGroup: 1000
-        fsGroup: 1000
-        seccompProfile:
-          type: RuntimeDefault
-      containers:
-        - name: pve-exporter
-          image: prompve/prometheus-pve-exporter:3.10.0 # {"$imagepolicy": "flux-system:pve-exporter"}
-          # Image ENTRYPOINT is already `pve_exporter`; pass flags only.
-          args:
-            - --config.file=/etc/pve-exporter/pve.yml
-            - --web.listen-address=0.0.0.0:9221
-          ports:
-            - name: http
-              containerPort: 9221
-          readinessProbe:
-            httpGet:
-              path: /
-              port: http
-            initialDelaySeconds: 5
-            periodSeconds: 30
-          resources:
-            requests:
-              cpu: 10m
-              memory: 64Mi
-            limits:
-              memory: 256Mi
-          securityContext:
-            allowPrivilegeEscalation: false
-            readOnlyRootFilesystem: true
-            capabilities:
-              drop: ["ALL"]
-          volumeMounts:
-            - name: config
-              mountPath: /etc/pve-exporter
-              readOnly: true
-            # Python needs a writable temp dir; the rootfs is read-only.
-            - name: tmp
-              mountPath: /tmp
-      volumes:
-        - name: config
-          secret:
-            secretName: pve-exporter-config
-        - name: tmp
-          emptyDir: {}
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: pve-exporter
-  namespace: monitoring
-  labels:
-    app.kubernetes.io/name: pve-exporter
-spec:
-  selector:
-    app.kubernetes.io/name: pve-exporter
-  ports:
-    - name: http
-      port: 9221
-      targetPort: http

kubernetes/apps/music-assistant/deployment.yaml

@@ -25,8 +25,11 @@ spec:
         # replaces the former hostNetwork attachment (see
         # network-attachment.yaml) so the NODE no longer holds an address
         # on the compromised IoT VLAN; MA gets its own leased IoT IP on
-        # net1 instead. master: ens19 is the v30iot NIC on every k8s-{1,2,3}/edge
-        # worker.
+        # net1 instead. master: ens19 resolves on every k8s-{1,2,3}/edge
+        # worker, though it is no longer the same kind of device everywhere:
+        # on the bare-metal control-plane nodes it is a VLAN 30 sub-interface
+        # of enp2s0f0np0 rather than a NIC, which is why the datapath needs
+        # bpf.vlanBypass to carry it (issue #592).
         k8s.v1.cni.cncf.io/networks: music-assistant-iot
     spec:
       enableServiceLinks: false

kubernetes/apps/paperless/MIGRATION-data-to-block.md

@@ -0,0 +1,97 @@
+# paperless-data: CephFS → RBD
+
+## Why
+
+`paperless-data` is `ReadWriteMany` on `ceph-filesystem`, but **only the paperless
+Deployment mounts it**. Verified 2026-08-20:
+
+```
+$ pods mounting paperless-data
+paperless-68cb8b8744-7hjb8      <- one, and only one
+
+paperless-data     ReadWriteMany  ceph-filesystem
+paperless-media    ReadWriteMany  ceph-filesystem   <- genuinely shared (ftp-scanner)
+paperless-consume  ReadWriteMany  ceph-filesystem   <- genuinely shared (ftp-scanner)
+```
+
+Its contents are single-writer state — the 50 MB Whoosh index,
+`classification_model.pickle`, `celerybeat-schedule.db`, `log/`,
+`migration_lock`, `protonmail.pem`. 75 files. Nothing else reads or writes them.
+
+It is on CephFS by default, not by requirement.
+
+## What it costs
+
+Every access goes through the MDS. Measured with **zero CI pipelines running**:
+
+```
+paperless-data client   request_load_avg 22   (all other sessions 0-1)
+MDS over 180s           4,052 client requests  = 22.5/s
+                        2,718 journal events   = 15/s
+                            2 dir commits      <- almost none: not file writes
+```
+
+So it is cap/inode churn from repeated access, not data being written. That is
+most of the constant **7-8 GB/day** written to `cephfs-metadata`, and since that
+pool is `size: 3`, roughly **21-24 GB/day** reaches the drives.
+
+`cephfs-metadata` deliberately stays at size 3. Losing it destroys the entire
+filesystem rather than the objects on one OSD, and the 275 GiB in `cephfs-data0`
+depends on it. Cutting the writes at source is correct; cutting the replication
+is not.
+
+## Sequence
+
+Staged deliberately: the PVC lands first and empty, so merging it changes
+nothing. The copy and the repoint are separate.
+
+### Phase 1 — provision (this PR)
+
+`paperless-data-block` (RWO, `ceph-block`, 10Gi) is created and mounted by
+nothing. Inert.
+
+Verify: `kubectl -n paperless get pvc paperless-data-block` → Bound.
+
+### Phase 2 — copy
+
+The Whoosh index must come across intact. Paperless *can* rebuild it
+(`document_index reindex`), but that reprocesses every document, so copying is
+both faster and less disruptive.
+
+1. Scale paperless to 0. The source must be quiescent — `celerybeat-schedule.db`
+   and the Whoosh index are both live-written, and copying them under an active
+   writer gives a torn read.
+2. Run a one-off pod that mounts BOTH PVCs and copies:
+   ```
+   cp -a /src/. /dst/
+   ```
+   The RWO target pins the pod to one node; the RWX source will follow.
+3. Verify file count and index size match:
+   ```
+   find /src -type f | wc -l   ==   find /dst -type f | wc -l
+   du -sh /src/index           ==   du -sh /dst/index
+   ```
+
+### Phase 3 — repoint
+
+Change the Deployment's `data` volume to `claimName: paperless-data-block`,
+merge, let it roll.
+
+Verify afterwards:
+- paperless starts, search returns results (proves the Whoosh index survived)
+- `ceph tell mds.cephfs-b session ls` no longer shows a busy `paperless-data`
+  client
+- `rate(ceph_pool_wr_bytes{pool_id="3"})` drops from its 7-8 GB/day baseline
+
+**Do not delete `paperless-data` until search has been confirmed working.** It is
+the rollback, and it is `prune: disabled` so Flux will not remove it.
+
+## Expected effect
+
+~7-8 GB/day off `cephfs-metadata`, so ~21-24 GB/day off the drives once
+replication is counted. Against ~432 GB/day cluster-wide that is roughly 5% —
+worth doing, but it is a correctness fix (a single-writer volume does not belong
+on a shared filesystem) more than a large win.
+
+It also removes 22 req/sec of MDS load, which benefits the workloads that
+genuinely need CephFS — Woodpecker CI workspaces share the same MDS.

kubernetes/apps/paperless/cluster.yaml

@@ -34,6 +34,42 @@ spec:
     parameters:
       shared_buffers: "256MB"
       max_connections: "200"
+      # Write-reduction tuning. These databases are largely idle, and the
+      # Postgres defaults favour throughput over storage that wears out.
+      #
+      # checkpoint_timeout is the lever. The 5min default pushes every dirty
+      # page to disk 288 times a day, so a small database whose same few pages
+      # are re-dirtied constantly (sessions, tokens, job state) gets rewritten
+      # over and over. Measured on ak-pg: 25.8M checkpoint buffers flushed
+      # (~206 GB) against a 1046 MB database -- the whole thing roughly twice a
+      # day. At 15min those pages coalesce and are written a third as often.
+      #
+      # max_wal_size must rise with it, or checkpoints trigger on size instead
+      # and the timeout change does nothing. It also bounds crash recovery to
+      # at most 2GB of WAL replay.
+      #
+      # NOT changed: synchronous_commit stays at the default `on`. It is off for
+      # ha-pg because a state recorder is telemetry; these hold credentials,
+      # documents and job state, where losing the last few hundred ms of commits
+      # on an unclean shutdown is not an acceptable trade.
+      checkpoint_timeout: "15min"
+      max_wal_size: "2GB"
+      # zstd, not "on": the boolean spelling resolves to pglz (2016-era, weaker
+      # and slower than the lz4/zstd this build also accepts). Compresses the
+      # full-page images that dominate WAL volume after each checkpoint.
+      wal_compression: "zstd"
+      # A WAL segment is a fixed 16 MiB file, and archive_timeout forces a switch
+      # on a timer -- the segment is then written and archived at its full size
+      # no matter how little real WAL it holds. CNPG defaults to 5min, i.e. up
+      # to 288 segments/day = 4.1 GB/day that is largely padding.
+      #
+      # Verified on gf-pg first (the staged rollout): 211 -> 41 MB/hour, an 81%
+      # cut, with archived segments falling from ~12/hour to ~1.5/hour in step.
+      #
+      # The cost is RPO: up to 30min of WAL is unrecoverable if both instances
+      # are lost. The documents themselves live on the media PVC, not in
+      # Postgres -- this database holds their metadata.
+      archive_timeout: "30min"
     # See ak-pg cluster.yaml for the rationale on synchronous replication.
     synchronous:
       method: any

kubernetes/apps/paperless/paperless.yaml

@@ -208,9 +208,25 @@ spec:
             seccompProfile:
               type: RuntimeDefault
       volumes:
+        # paperless-data-block (RWO, ceph-block), NOT paperless-data
+        # (RWX, ceph-filesystem). Only this Deployment ever mounted the volume,
+        # and its contents are single-writer state — the Whoosh index,
+        # classification model, celerybeat schedule, logs. Being on CephFS meant
+        # every access went through the MDS: 22 requests/sec sustained with zero
+        # CI running, ~15 journal events/sec against 2 directory commits per
+        # 180s, i.e. cap/inode churn rather than file writes. That was most of
+        # the constant 7-8 GB/day on cephfs-metadata, and that pool is size 3.
+        #
+        # `consume` and `media` below STAY on CephFS: the ftp-scanner writes to
+        # those from another node, so they genuinely need RWX.
+        #
+        # paperless-data is retained (prune: disabled) as the rollback. Do not
+        # delete it until search has been confirmed working — that is what
+        # proves the Whoosh index survived the copy, not merely that paperless
+        # starts. Migration record: MIGRATION-data-to-block.md.
         - name: data
           persistentVolumeClaim:
-            claimName: paperless-data
+            claimName: paperless-data-block
         - name: media
           persistentVolumeClaim:
             claimName: paperless-media
@@ -218,7 +234,9 @@ spec:
           persistentVolumeClaim:
             claimName: paperless-consume
         - name: bridge-cert
-          emptyDir: {}
+          emptyDir:
+            medium: Memory
+            sizeLimit: 16Mi
 ---
 apiVersion: v1
 kind: Service

kubernetes/apps/paperless/pvcs.yaml

@@ -18,6 +18,45 @@ spec:
     requests:
       storage: 10Gi
 ---
+# Replacement for paperless-data, on RBD instead of CephFS.
+#
+# WHY: paperless-data is declared ReadWriteMany on ceph-filesystem, but only ONE
+# pod ever mounts it — the paperless Deployment itself. Its contents are all
+# single-writer state: the 50 MB Whoosh index, classification_model.pickle,
+# celerybeat-schedule.db, logs, migration_lock. Nothing shares it. Only `consume`
+# and `media` genuinely need RWX, because the FTP scanner writes to those from a
+# different node.
+#
+# COST OF LEAVING IT: every access goes through the MDS. Measured 2026-08-20 with
+# zero CI running, this volume's client was doing 22 MDS requests/sec sustained,
+# and the MDS logged ~15 journal events/sec against only 2 directory commits per
+# 180s — i.e. cap and inode churn from repeated access, not file writes. That is
+# most of the constant 7-8 GB/day written to cephfs-metadata, which is a size 3
+# pool, so ~21-24 GB/day actually hits the drives.
+#
+# cephfs-metadata deliberately STAYS at size 3 (see storageclass.yaml): losing
+# the metadata pool destroys the whole filesystem, not just the objects on one
+# OSD, so the 275 GiB in cephfs-data0 rides on it. Reducing the writes at source
+# is the right fix; reducing the replication is not.
+#
+# MIGRATION: this PVC is created EMPTY and nothing mounts it yet. The data copy
+# and the deployment repoint are a separate, staged step — see
+# MIGRATION-data-to-block.md. Merging this alone is inert.
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: paperless-data-block
+  namespace: paperless
+  annotations:
+    kustomize.toolkit.fluxcd.io/prune: disabled
+spec:
+  accessModes:
+    - ReadWriteOnce
+  storageClassName: ceph-block
+  resources:
+    requests:
+      storage: 10Gi
+---
 apiVersion: v1
 kind: PersistentVolumeClaim
 metadata:

kubernetes/apps/paperless/redis.yaml

@@ -25,7 +25,7 @@ spec:
           type: RuntimeDefault
       containers:
         - name: redis
-          image: redis:8.10.0-alpine # {"$imagepolicy": "flux-system:redis-paperless"}
+          image: redis:8.10.1-alpine # {"$imagepolicy": "flux-system:redis-paperless"}
           args:
             - redis-server
             - --save

kubernetes/apps/pve/endpointslice.yaml

@@ -1,33 +0,0 @@
-# PVE node IPs are hardcoded here despite the general "IPs aren't static"
-# rule for the homelab: corosync requires stable cluster member identity,
-# so jupiter/neptune/saturn keep .201/.202/.203 across reboots and PVE
-# upgrades. If a node is permanently renumbered, edit this file.
-apiVersion: discovery.k8s.io/v1
-kind: EndpointSlice
-metadata:
-  name: pve-nodes
-  namespace: apps
-  labels:
-    kubernetes.io/service-name: pve
-    app.kubernetes.io/name: pve
-addressType: IPv4
-ports:
-  - name: https
-    port: 8006
-    protocol: TCP
-endpoints:
-  - addresses:
-      - 10.2.40.201
-    hostname: jupiter
-    conditions:
-      ready: true
-  - addresses:
-      - 10.2.40.202
-    hostname: neptune
-    conditions:
-      ready: true
-  - addresses:
-      - 10.2.40.203
-    hostname: saturn
-    conditions:
-      ready: true

kubernetes/apps/pve/ingressroute.yaml

@@ -1,48 +0,0 @@
-# PVE (Proxmox cluster UI) on Traefik. Replaced the standard Ingress
-# pattern because PVE's backend is per-node HTTPS on :8006 with a
-# self-signed cert that has no IP SAN — Traefik's kubernetesIngress
-# provider's service.serverstransport annotation doesn't resolve the
-# apps/insecure-backend ServersTransport CRD, so TLS verification ran
-# and rejected. IngressRoute's native services[].serversTransport field
-# binds directly with no provider-qualifier guessing.
-apiVersion: cert-manager.io/v1
-kind: Certificate
-metadata:
-  name: pve-tls
-  namespace: apps
-  labels:
-    app.kubernetes.io/name: pve
-spec:
-  secretName: pve-tls
-  issuerRef:
-    name: letsencrypt-cluster
-    kind: ClusterIssuer
-  dnsNames:
-    - pve.renner.uno
----
-apiVersion: traefik.io/v1alpha1
-kind: IngressRoute
-metadata:
-  name: pve
-  namespace: apps
-  labels:
-    app.kubernetes.io/name: pve
-  annotations:
-    # Traefik does not populate IngressRoute.status.loadBalancer, so
-    # external-dns-unifi (traefik-proxy source) has no implicit target.
-    # Pin to the traefik Service VIP (kubernetes/infrastructure/traefik
-    # service.annotations io.cilium/lb-ipam-ips).
-    external-dns.alpha.kubernetes.io/target: "10.2.99.86,fd99::86"
-spec:
-  entryPoints:
-    - websecure
-  routes:
-    - match: Host(`pve.renner.uno`)
-      kind: Rule
-      services:
-        - name: pve
-          port: 8006
-          scheme: https
-          serversTransport: insecure-backend
-  tls:
-    secretName: pve-tls

kubernetes/apps/pve/kustomization.yaml

@@ -1,6 +0,0 @@
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-resources:
-  - service.yaml
-  - endpointslice.yaml
-  - ingressroute.yaml

kubernetes/apps/pve/service.yaml

@@ -1,15 +0,0 @@
-apiVersion: v1
-kind: Service
-metadata:
-  name: pve
-  namespace: apps
-  labels:
-    app.kubernetes.io/name: pve
-spec:
-  # No selector: endpoints managed manually via the sibling EndpointSlice
-  # so ingress-nginx fans out across the three Proxmox cluster members.
-  ports:
-    - name: https
-      port: 8006
-      targetPort: 8006
-      protocol: TCP

kubernetes/apps/trmnl/cluster.yaml

@@ -32,7 +32,44 @@ spec:
     parameters:
       shared_buffers: "128MB"
       max_connections: "50"
-      wal_compression: "on"
+      # zstd, not "on": the boolean spelling resolves to pglz (2016-era, weaker
+      # and slower than the lz4/zstd this build also accepts). Compresses the
+      # full-page images that dominate WAL volume after each checkpoint.
+      wal_compression: "zstd"
+      # A WAL segment is a fixed 16 MiB file, and archive_timeout forces a switch
+      # on a timer -- the segment is then written and archived at its full size
+      # no matter how little real WAL it holds. CNPG defaults to 5min, i.e. up
+      # to 288 segments/day = 4.6 GB/day that is largely padding.
+      #
+      # Verified on gf-pg first (the staged rollout): 211 -> 41 MB/hour, an 81%
+      # cut, with archived segments falling from ~12/hour to ~1.5/hour in step.
+      # Two independent signals moving together is what confirms the volume was
+      # padding rather than data.
+      #
+      # The cost is RPO: up to 30min of WAL is unrecoverable if this cluster and
+      # its volume are lost outright. This is a single-instance cluster, so the
+      # archive is the only off-node copy.
+      archive_timeout: "30min"
+      # Write-reduction tuning. These databases are largely idle, and the
+      # Postgres defaults favour throughput over storage that wears out.
+      #
+      # checkpoint_timeout is the lever. The 5min default pushes every dirty
+      # page to disk 288 times a day, so a small database whose same few pages
+      # are re-dirtied constantly (sessions, tokens, job state) gets rewritten
+      # over and over. Measured on ak-pg: 25.8M checkpoint buffers flushed
+      # (~206 GB) against a 1046 MB database -- the whole thing roughly twice a
+      # day. At 15min those pages coalesce and are written a third as often.
+      #
+      # max_wal_size must rise with it, or checkpoints trigger on size instead
+      # and the timeout change does nothing. It also bounds crash recovery to
+      # at most 2GB of WAL replay.
+      #
+      # NOT changed: synchronous_commit stays at the default `on`. It is off for
+      # ha-pg because a state recorder is telemetry; these hold credentials,
+      # documents and job state, where losing the last few hundred ms of commits
+      # on an unclean shutdown is not an acceptable trade.
+      checkpoint_timeout: "15min"
+      max_wal_size: "2GB"
   resources:
     requests:
       cpu: 100m

kubernetes/apps/trmnl/renderer.yaml

@@ -362,6 +362,22 @@ spec:
             name: trmnl-renderer-schedule
         - name: tmp
           emptyDir:
+            # RAM-backed, NOT disk. Without `medium: Memory` an emptyDir lands
+            # on the node's filesystem, and this one was the single largest
+            # writer in the cluster: 249 GB over 7.3 days = ~34 GB/day onto
+            # /dev/vda5, more than every Ceph OSD and every database combined
+            # (measured 2026-08-20 from the container's cgroup io.stat).
+            #
+            # The cause is cadence, not volume. schedules.template.json renders
+            # every 60 s, so Chrome is launched 1440 times a day and each launch
+            # creates, fills and deletes a /tmp/puppeteer_dev_chrome_profile-*
+            # directory. At rest /tmp holds under 8 MB; it is pure churn for
+            # data that never outlives the render.
+            #
+            # tmpfs pages count against the container's memory limit, which is
+            # why the cap stays at 512Mi: measured usage is 129Mi against a 2Gi
+            # limit, so even the full cap leaves ample headroom.
+            medium: Memory
             sizeLimit: 512Mi
 ---
 apiVersion: v1

kubernetes/apps/woodpecker/agent.yaml

@@ -34,7 +34,7 @@ spec:
           # Image automation, range 3.x (see image-automation/woodpecker.yaml).
           # MUST stay version-matched with woodpecker-server (gRPC parity) — both
           # track the same range/filter so they bump together on each release.
-          image: woodpeckerci/woodpecker-agent:v3.17.0 # {"$imagepolicy": "flux-system:woodpecker-agent"}
+          image: woodpeckerci/woodpecker-agent:v3.18.0 # {"$imagepolicy": "flux-system:woodpecker-agent"}
           env:
             - name: WOODPECKER_SERVER
               value: woodpecker.woodpecker.svc.cluster.local:9000 # in-cluster gRPC, replaces LXC :9000
@@ -50,12 +50,39 @@ spec:
             # clone + subsequent steps share the workspace PVC and the k8s backend
             # can schedule them on different nodes -> an RWO volume then fails to
             # mount on the second node. CephFS (RWX) lets the workspace follow
-            # the pipeline across nodes. The -ephemeral variant (reclaim Delete,
-            # rook-ceph-cluster/storageclass.yaml) is REQUIRED here: workspaces
-            # are throwaway, and the Retain class leaked one Released PV per
-            # finished pipeline (issue #262 follow-up).
+            # the pipeline across nodes. Reclaim MUST be Delete: workspaces are
+            # throwaway, and a Retain class leaked one Released PV per finished
+            # pipeline (issue #262 follow-up).
+            #
+            # ceph-filesystem-ci, not ceph-filesystem-ephemeral: a dedicated
+            # CephFS whose pools are BOTH size 1 (infrastructure/
+            # rook-ceph-cluster/cephfs-ci.yaml). The shared filesystem
+            # replicates data0 2x and metadata 3x, which is the correct trade
+            # for real data and pure waste for a git clone that is deleted
+            # minutes later.
+            #
+            # Measured 2026-08-21, polling both pools every 15s (a 5-minute
+            # interval misses an entire pipeline -- PVC created, used and
+            # deleted inside ~2 minutes):
+            #   CI active  1.5min: data0 143.2 MB, metadata 17.2 MB
+            #   CI idle   16.0min: data0   0.0 MB, metadata 11.0 MB
+            # data0 write volume is 100% CI. Per pipeline this moves the drive
+            # cost from 71.6x2 + 8.2x3 = ~168 MB to ~80 MB.
+            #
+            # ceph-filesystem-ephemeral is NOT repointed, and the reason is a
+            # technical constraint rather than a safety one. Its other consumer
+            # is velero/restore-test.sh, whose probe PVCs are snapshot CLONES:
+            # `ceph fs subvolume snapshot clone` operates within one filesystem,
+            # so a snapshot taken in `cephfs` cannot be cloned into `cephfs-ci`.
+            #
+            # Those probes are just as throwaway as a CI workspace -- the
+            # authoritative copy is the backup on Hetzner, and a lost probe just
+            # means re-running the test -- so they are equally undeserving of
+            # replication. Landing them unreplicated would mean a second size-1
+            # data pool inside `cephfs`, and restore-test runs MONTHLY against
+            # CI's ~96 pipelines/day, so it is not worth the complexity.
             - name: WOODPECKER_BACKEND_K8S_STORAGE_CLASS
-              value: ceph-filesystem-ephemeral
+              value: ceph-filesystem-ci
             - name: WOODPECKER_BACKEND_K8S_STORAGE_RWX
               value: "true"
             # MUST be a JSON object literal — the backend json.Unmarshal's this

kubernetes/apps/woodpecker/cluster.yaml

@@ -56,6 +56,44 @@ spec:
     parameters:
       shared_buffers: "128MB"
       max_connections: "50"
+      # Write-reduction tuning. These databases are largely idle, and the
+      # Postgres defaults favour throughput over storage that wears out.
+      #
+      # checkpoint_timeout is the lever. The 5min default pushes every dirty
+      # page to disk 288 times a day, so a small database whose same few pages
+      # are re-dirtied constantly (sessions, tokens, job state) gets rewritten
+      # over and over. Measured on ak-pg: 25.8M checkpoint buffers flushed
+      # (~206 GB) against a 1046 MB database -- the whole thing roughly twice a
+      # day. At 15min those pages coalesce and are written a third as often.
+      #
+      # max_wal_size must rise with it, or checkpoints trigger on size instead
+      # and the timeout change does nothing. It also bounds crash recovery to
+      # at most 2GB of WAL replay.
+      #
+      # NOT changed: synchronous_commit stays at the default `on`. It is off for
+      # ha-pg because a state recorder is telemetry; these hold credentials,
+      # documents and job state, where losing the last few hundred ms of commits
+      # on an unclean shutdown is not an acceptable trade.
+      checkpoint_timeout: "15min"
+      max_wal_size: "2GB"
+      # zstd, not "on": the boolean spelling resolves to pglz (2016-era, weaker
+      # and slower than the lz4/zstd this build also accepts). Compresses the
+      # full-page images that dominate WAL volume after each checkpoint.
+      wal_compression: "zstd"
+      # A WAL segment is a fixed 16 MiB file, and archive_timeout forces a switch
+      # on a timer -- the segment is then written and archived at its full size
+      # no matter how little real WAL it holds. CNPG defaults to 5min, i.e. up
+      # to 288 segments/day = 4.6 GB/day that is largely padding.
+      #
+      # Verified on gf-pg first (the staged rollout): 211 -> 41 MB/hour, an 81%
+      # cut, with archived segments falling from ~12/hour to ~1.5/hour in step.
+      # Two independent signals moving together is what confirms the volume was
+      # padding rather than data.
+      #
+      # The cost is RPO: up to 30min of WAL is unrecoverable if this cluster and
+      # its volume are lost outright. This is a single-instance cluster, so the
+      # archive is the only off-node copy.
+      archive_timeout: "30min"
   resources:
     requests:
       cpu: 100m

kubernetes/apps/woodpecker/server.yaml

@@ -40,7 +40,7 @@ spec:
           # minor + patch auto-update within 3.x; only the 4.0 major is held for
           # a reviewed bump. Safe now the DB is postgres (the old sqlite-surprise
           # -bump hazard is gone). Stays in lockstep with the agent.
-          image: woodpeckerci/woodpecker-server:v3.17.0 # {"$imagepolicy": "flux-system:woodpecker-server"}
+          image: woodpeckerci/woodpecker-server:v3.18.0 # {"$imagepolicy": "flux-system:woodpecker-server"}
           env:
             - name: WOODPECKER_HOST
               value: https://ci.renner.uno

kubernetes/e2e-test.sh

@@ -268,7 +268,7 @@ EOF
 
 # 120x15s = 30 min, not the old 60x5s = 5 min. node-agent has no
 # loadConcurrency override, so Velero's default of one data path per node
-# applies: while the Sunday restore-test sweep holds the slot, even this 1 GiB
+# applies: while the restore-test sweep holds the slot, even this 1 GiB
 # DataUpload sits in phase Prepared for well over 5 minutes. The old 300 s wait
 # then let the teardown below delete the source namespace and the Backup
 # mid-upload ("Cancel du under phase Prepared because it is being deleted"),

kubernetes/infrastructure/cilium/helmrelease.yaml

@@ -33,6 +33,24 @@ spec:
     remediation:
       retries: 3
   values:
+    # Roll the agent DaemonSet when cilium-config changes. Cilium reads
+    # datapath options once, at agent start, and the chart puts no config
+    # checksum on the pod template — so a changed ConfigMap leaves every agent
+    # running the previous datapath until something restarts it. Nothing did.
+    #
+    # This is not hypothetical. vlan-bpf-bypass reached cilium-config and every
+    # v30iot macvlan workload stayed broken until the agents were deleted by
+    # hand, one node at a time (issue #592). policyMapMax has the same
+    # exposure, and its comment below says "takes effect on agent restart" —
+    # documentation standing in for automation.
+    #
+    # The trade: any cilium-config change now rolls the CNI cluster-wide,
+    # node by node. That is disruptive-ish, but it is what a datapath change
+    # already required — the difference is that it now happens on its own
+    # instead of being forgotten. Reloader is already deployed for exactly
+    # this pattern; named per-ConfigMap rather than blanket, as elsewhere.
+    annotations:
+      configmap.reloader.stakater.com/reload: cilium-config
     # Which pod labels feed security-identity computation. Patterns are
     # start-anchored regexes; a leading "!" excludes while keeping the default
     # set. Woodpecker stamps every step pod with woodpecker-ci.org/task-uuid
@@ -73,6 +91,36 @@ spec:
       # kubernetes/talos/cilium.yaml carries a third copy of the same key.
       # Nothing enforces any of this; grep bpf-policy-map-max before changing it.
       policyMapMax: 65536
+      # VLAN ids the eBPF datapath is allowed to pass. Without this it drops
+      # VLAN-tagged frames, accounted as "VLAN traffic disallowed by VLAN
+      # filter" in cilium_drop_count_total.
+      #
+      # 30 is v30iot, the macvlan parent for the shelly-manager, matter-server,
+      # otbr and music-assistant attachments. On bare metal it is a VLAN
+      # sub-interface of enp2s0f0np0; inside the former Proxmox guests it was
+      # an untagged virtio NIC, which is why this was not needed before.
+      #
+      # VLAN 40 (mgmt0) is deliberately NOT listed, and the asymmetry is the
+      # interesting part. Cilium attaches to devices that carry an address —
+      # `cilium-dbg status --verbose` lists enp87s0, enp2s0f0np0, enp2s0f1np1
+      # and mgmt0, but NOT ens19. mgmt0 holds the node IP, so Cilium owns it as
+      # a device and sees frames the 8021q layer has already untagged; nothing
+      # for a VLAN filter to match. ens19 is deliberately address-less (see
+      # talos/patches/k8s-1.yaml — host addresses on that L2 make node replies
+      # bypass the UDM), so Cilium never attaches to it and VLAN 30 frames only
+      # meet the filter on the shared parent, still tagged.
+      #
+      # So the fix belongs on 30 alone. Listing 40 would change nothing, and
+      # would touch the datapath carrying etcd and the API for no reason.
+      #
+      # A tc-layer drop surfaces to an AF_PACKET sender as ENOBUFS, so the
+      # symptom is the CNI dhcp plugin reporting "no buffer space available"
+      # rather than a DHCP timeout. See issue #592.
+      #
+      # Same three-copy problem as policyMapMax above: vlan-bpf-bypass in
+      # kubernetes/talos/cilium.yaml and kubernetes/talos/patches/cilium-cni.yaml
+      # must move with this list. Takes effect on agent restart.
+      vlanBypass: [30]
       # Let pods reach the cluster's own LoadBalancer external IPs; Cilium
       # redirects the socket directly to the backend pod instead of sending
       # traffic out to the VIP and back (which fails with L2 announcements).

kubernetes/infrastructure/cnpg-barman-plugin/manifest.yaml

@@ -917,11 +917,27 @@ subjects:
   name: plugin-barman-cloud
   namespace: database
 ---
+# stringData, NOT base64 `data`. An image reference is not a secret, and
+# encoding it here had two real costs: no reviewer sees a version change in a
+# diff, and no $imagepolicy marker can be attached to a base64 blob. The result
+# was that automation advanced the controller to v0.14.0 while this sidecar sat
+# at v0.12.0 — a two-minor skew in the component that performs every base backup
+# and WAL archive, invisible until someone read the pod spec (#535).
+#
+# The Secret KIND is kept because the controller reads it via secretKeyRef
+# (deployment env SIDECAR_IMAGE); changing it to a ConfigMap would need a matching
+# edit to that reference and diverge further from upstream. stringData is
+# accepted identically by the API server and serialised into `data` on write.
+#
+# The name carries a kustomize hash suffix from upstream's secretGenerator. It is
+# static here, so editing the value does NOT change the name and the deployment's
+# secretKeyRef stays valid.
+#
+# IF YOU RE-IMPORT THE UPSTREAM MANIFEST: upstream ships this as base64 `data`
+# with no marker. Re-apply this block, or the skew silently returns.
 apiVersion: v1
-data:
-  SIDECAR_IMAGE: |
-    Z2hjci5pby9jbG91ZG5hdGl2ZS1wZy9wbHVnaW4tYmFybWFuLWNsb3VkLXNpZGVjYXI6dj
-    AuMTIuMA==
+stringData:
+  SIDECAR_IMAGE: ghcr.io/cloudnative-pg/plugin-barman-cloud-sidecar:v0.14.0 # {"$imagepolicy": "flux-system:cnpg-barman-plugin-sidecar"}
 kind: Secret
 metadata:
   name: plugin-barman-cloud-kg5c2795f7

kubernetes/infrastructure/crowdsec/helmrelease.yaml

@@ -87,6 +87,16 @@ spec:
 
     lapi:
       replicas: 1
+      # The chart defaults request 500m. Measured steady-state draw is 2m, and
+      # the request is what the scheduler reserves, so the default parks half a
+      # core of unusable capacity per LAPI. That is affordable on three nodes
+      # and is not on two: with one node out for maintenance the remaining pair
+      # sit at 100% of requested CPU while actually using ~22%, and nothing new
+      # can be placed. The limit stays at the chart default so a log flood can
+      # still burst; only the reservation shrinks.
+      resources:
+        requests:
+          cpu: 50m
       podAnnotations:
         # Reloader restarts the LAPI when crowdsec-bouncer-key changes (it's an
         # env-var secret, so a restart is required to pick up a rotated key). See
@@ -122,6 +132,12 @@ spec:
               key: key
 
     agent:
+      # Same reasoning as lapi.resources above, multiplied by three: this is a
+      # DaemonSet, so the chart's 500m default reserves half a core on every
+      # node to run a process that measures 1-2m.
+      resources:
+        requests:
+          cpu: 50m
       # The agent parses the ingress controller's access logs (read from
       # the pod's stdout) to detect attacks. Repointed from the retired
       # ingress-nginx (decommissioned 2026-05-17) to Traefik. The
@@ -134,16 +150,3 @@ spec:
       env:
         - name: COLLECTIONS
           value: "crowdsecurity/traefik crowdsecurity/http-cve crowdsecurity/appsec-virtual-patching crowdsecurity/appsec-generic-rules"
-
-    config:
-      parsers:
-        s02-enrich:
-          pve-ui-polling-whitelist.yaml: |
-            name: krenner/pve-ui-polling-whitelist
-            description: "Suppress CrowdSec scenarios for the Proxmox UI's legitimate API polling on pve.renner.uno"
-            filter: "evt.Meta.log_type == 'http_access-log' && evt.Meta.target_fqdn == 'pve.renner.uno'"
-            whitelist:
-              reason: "Proxmox web UI polls /api2/json/cluster/* and /api2/json/nodes/<n>/{lxc,qemu}/<id>/status/current every ~2s; http-probing flagged it as scanning and banned the residential IPv6."
-              expression:
-                - "evt.Meta.http_path startsWith '/api2/json/cluster/'"
-                - "evt.Meta.http_path matches '^/api2/json/nodes/[^/]+/(lxc|qemu)/[0-9]+/status/current$'"

kubernetes/infrastructure/image-automation/cnpg-barman-plugin.yaml

@@ -28,3 +28,40 @@ spec:
   policy:
     semver:
       range: "0.x"
+---
+# The SIDECAR image, tracked separately from the controller above.
+#
+# These are two DIFFERENT images that must move together:
+#   plugin-barman-cloud          the controller, in the database namespace
+#   plugin-barman-cloud-sidecar  injected into every Postgres pod
+#
+# Only the controller was tracked, so automation walked it from v0.12.0 to
+# v0.14.0 while the sidecar stayed at v0.12.0 — a two-minor skew in the
+# component that performs every base backup and WAL archive (#535). The skew was
+# invisible because the sidecar tag was base64-encoded inside a Secret, where no
+# $imagepolicy marker can reach it and no reviewer would see it in a diff.
+apiVersion: image.toolkit.fluxcd.io/v1
+kind: ImageRepository
+metadata:
+  name: cnpg-barman-plugin-sidecar
+  namespace: flux-system
+spec:
+  image: ghcr.io/cloudnative-pg/plugin-barman-cloud-sidecar
+  interval: 1h
+---
+# Same filter and range as the controller policy deliberately: the two images
+# are released together and must not be allowed to drift apart again.
+apiVersion: image.toolkit.fluxcd.io/v1
+kind: ImagePolicy
+metadata:
+  name: cnpg-barman-plugin-sidecar
+  namespace: flux-system
+spec:
+  imageRepositoryRef:
+    name: cnpg-barman-plugin-sidecar
+  filterTags:
+    pattern: '^v(?P<ver>\d+\.\d+\.\d+)$'
+    extract: '$ver'
+  policy:
+    semver:
+      range: "0.x"

kubernetes/infrastructure/image-automation/kustomization.yaml

@@ -15,7 +15,6 @@ resources:
   - netbird-ingress.yaml
   - paperless-ngx.yaml
   - protonmail-bridge.yaml
-  - pve-exporter.yaml
   - mosquitto.yaml
   - redis.yaml
   - shelly-manager.yaml

kubernetes/infrastructure/image-automation/pve-exporter.yaml

@@ -1,23 +0,0 @@
-apiVersion: image.toolkit.fluxcd.io/v1
-kind: ImageRepository
-metadata:
-  name: pve-exporter
-  namespace: flux-system
-spec:
-  image: prompve/prometheus-pve-exporter
-  interval: 1h
----
-apiVersion: image.toolkit.fluxcd.io/v1
-kind: ImagePolicy
-metadata:
-  name: pve-exporter
-  namespace: flux-system
-spec:
-  imageRepositoryRef:
-    name: pve-exporter
-  filterTags:
-    pattern: '^(?P<ver>\d+\.\d+\.\d+)$'
-    extract: '$ver'
-  policy:
-    semver:
-      range: "3.x"

kubernetes/infrastructure/netbird-ingress/daemonset.yaml

@@ -156,4 +156,6 @@ spec:
         # each pod enrolls its own peer and a restart re-enrolls cleanly
         # (the old peer is reaped).
         - name: state
-          emptyDir: {}
+          emptyDir:
+            medium: Memory
+            sizeLimit: 16Mi

kubernetes/infrastructure/network-policies/home-assistant-ns.yaml

@@ -358,3 +358,67 @@ spec:
         - ports:
             - port: "8123"
               protocol: TCP
+---
+# ha-pg (CNPG) — instance-manager access for the CloudNativePG operator.
+#
+# DEDICATED policy with its own endpointSelector, NOT another rule on the
+# namespace-wide default-deny above. That policy has endpointSelector:{}, so any
+# rule added there applies to EVERY pod in the namespace — and :8000 is served
+# both by the CNPG instance manager and by other workloads, so a namespace-wide
+# :8000 grant would expose far more than intended. Same reasoning as
+# wp-pg-allow-cnpg-operator in woodpecker-ns.yaml, which was corrected during
+# review for exactly this (#433).
+apiVersion: cilium.io/v2
+kind: CiliumNetworkPolicy
+metadata:
+  name: ha-pg-allow-cnpg-operator
+  namespace: home-assistant
+spec:
+  endpointSelector:
+    matchLabels:
+      cnpg.io/cluster: ha-pg
+  ingress:
+    - fromEndpoints:
+        - matchLabels:
+            "k8s:io.kubernetes.pod.namespace": database
+            app.kubernetes.io/name: cloudnative-pg
+      toPorts:
+        - ports:
+            - port: "8000"
+              protocol: TCP
+    # ICMP to the ha-pg pods. REQUIRED because the L4 rules in this policy
+    # select those endpoints, and Cilium drops ICMP to an endpoint selected by
+    # any layer 4 policy unless the traffic is CT-related. Without this, PMTUD
+    # Destination Unreachable / Packet Too Big toward ha-pg are dropped
+    # silently, and the symptom is stalled transfers rather than a policy
+    # denial — the failure mode documented in woodpecker-ns.yaml.
+    - fromEntities:
+        - cluster
+      icmps:
+        - fields:
+            - { type: 3,   family: IPv4 }   # Destination Unreachable — PMTUD
+            - { type: 1,   family: IPv6 }   # Destination Unreachable
+            - { type: 2,   family: IPv6 }   # Packet Too Big — PMTUD, mandatory
+---
+# ha-pg — Postgres access for the Home Assistant recorder.
+#
+# Scoped to the HA pod on both ends: only home-assistant reaches ha-pg, and only
+# on 5432. Nothing else in this namespace has any business in the recorder
+# database.
+apiVersion: cilium.io/v2
+kind: CiliumNetworkPolicy
+metadata:
+  name: ha-pg-allow-home-assistant
+  namespace: home-assistant
+spec:
+  endpointSelector:
+    matchLabels:
+      cnpg.io/cluster: ha-pg
+  ingress:
+    - fromEndpoints:
+        - matchLabels:
+            app.kubernetes.io/name: home-assistant
+      toPorts:
+        - ports:
+            - port: "5432"
+              protocol: TCP

kubernetes/infrastructure/network-policies/monitoring-ns.yaml

@@ -7,7 +7,7 @@ spec:
   endpointSelector: {}
   ingress:
     # Intra-namespace — Prometheus ↔ Alertmanager ↔ Grafana ↔ Loki ↔
-    # alloy ↔ kube-state-metrics ↔ blackbox-exporter ↔ pve-exporter.
+    # alloy ↔ kube-state-metrics ↔ blackbox-exporter.
     - fromEndpoints:
         - matchLabels:
             "k8s:io.kubernetes.pod.namespace": monitoring

kubernetes/infrastructure/openbao/policies/rotator-proxmox.hcl

@@ -1,53 +0,0 @@
-# OpenBao policy: rotator-proxmox
-#
-# Class-B rotation for the pve-exporter Proxmox API token. Grants the
-# proxmox-secret-rotator CronJob READ on its bootstrap credential (a PRIVILEGED
-# Proxmox token allowed to delete/create tokens) and PATCH on the target
-# pve-exporter-config path. Per-secret isolation (#213/#202).
-#
-# NOT applied by Flux. Apply ONCE with an admin token:
-#
-#   bao policy write rotator-proxmox kubernetes/infrastructure/openbao/policies/rotator-proxmox.hcl
-#   bao write auth/kubernetes/role/rotator-proxmox \
-#     bound_service_account_names=proxmox-secret-rotator \
-#     bound_service_account_namespaces=openbao \
-#     policies=rotator-proxmox ttl=15m
-#
-# BOOTSTRAP CREDENTIAL (PROVISIONED — recorded here for rebuild/DR).
-#
-# A SEPARATE least-privilege Proxmox identity, NOT root@pam. PVE has no
-# per-user ACL path (/access/users/<id> is rejected — see AccessControl.pm
-# check_path), and the token endpoints authorise via a `userid-group` check.
-# So the tightest possible scope is a group containing only the target user:
-#
-#   pveum role add SecretRotator -privs "User.Modify"
-#   pveum user add secret-rotator@pve
-#   pveum group add secret-rotated
-#   pveum user modify pve-exporter@pve --groups secret-rotated
-#   pveum acl modify /access/groups/secret-rotated \
-#     --users secret-rotator@pve --roles SecretRotator
-#   pveum user token add secret-rotator@pve rotate --privsep 0
-#
-# This grants exactly one thing: managing API tokens of users in the
-# secret-rotated group. Verified: no VM/storage visibility, and minting a
-# root@pam token returns 403.
-#
-#   bao kv put homelab/infrastructure/secret-rotators-bootstrap/proxmox \
-#     api-url=https://pve.apps.svc.cluster.local:8006 \
-#     auth='secret-rotator@pve!rotate=<secret>'
-#
-# api-url MUST be the in-cluster Service (backed by all three PVE nodes).
-# The external name pve.renner.uno does NOT work from a pod — pods cannot
-# hairpin to their own cluster's ingress LB and the Job hangs.
-#
-# Do NOT set token_no_default_policy=true (kv-v2 preflight needs the default policy).
-
-# Bootstrap: the privileged Proxmox token + API URL.
-path "homelab/data/infrastructure/secret-rotators-bootstrap/proxmox" {
-  capabilities = ["read"]
-}
-
-# Target: the pve-exporter config (holds the pve.yml with the token_value).
-path "homelab/data/apps/monitoring/pve-exporter-config" {
-  capabilities = ["read", "patch"]
-}

kubernetes/infrastructure/openbao/rotation/claude-oauth.yaml

@@ -51,7 +51,15 @@ spec:
   startingDeadlineSeconds: 3600
   jobTemplate:
     spec:
-      backoffLimit: 1
+      # 4 (not the sibling rotators' 1-2) because this is the only DAILY
+      # rotator — every other CronJob here is quarterly, so it takes ~90x as
+      # many runs and is correspondingly more exposed to per-run transients.
+      # backoffLimit: 1 gave two attempts ~11 s apart, and on 2026-08-19 a
+      # sub-minute resolver transient took both out (Job
+      # claude-oauth-secret-sync-29785100, BackoffLimitExceeded), costing a
+      # day's sync. 4 spreads the attempts over ~150 s (10/20/40/80 s backoff),
+      # still well inside activeDeadlineSeconds.
+      backoffLimit: 4
       activeDeadlineSeconds: 300
       ttlSecondsAfterFinished: 259200 # 3 d
       template:

kubernetes/infrastructure/openbao/rotation/kustomization.yaml

@@ -52,7 +52,6 @@ resources:
   - crowdsec-bouncer-key.yaml
   - netbird-ingress.yaml
   - cloudflare.yaml
-  - pve-exporter.yaml
   - external-dns.yaml
   - flux-git.yaml
   - home-assistant.yaml

kubernetes/infrastructure/openbao/rotation/pve-exporter.yaml

@@ -1,91 +0,0 @@
----
-# Class-B rotation of the pve-exporter Proxmox API token via the secret-rotator
-# binary (proxmox-token mode). The RISKIEST rotator: Proxmox has no in-place
-# token roll, so the binary DELETEs and recreates pve-exporter@pve!monitoring
-# (same id, privsep=0) and patches OpenBao with a rendered pve.yml. That leaves
-# a brief window with NO valid token — pve-exporter metrics gap until ESO(10m)+
-# Reloader restart. Quarterly, off-hours: acceptable.
-#
-# privsep=0 makes the recreated token inherit pve-exporter@pve's own perms
-# (PVEAuditor ACL is on the USER, not the token — verified), so no ACL re-grant.
-# The bootstrap 'auth' is the least-privilege secret-rotator@pve!rotate identity
-# (User.Modify scoped to the secret-rotated group), NOT root.
-#
-# api-url is the in-cluster Service https://pve.apps.svc.cluster.local:8006 (all
-# three PVE nodes back it → node-failover). TLS verification is disabled for the
-# PVE calls: the nodes present valid Let's Encrypt certs that fail only on
-# hostname mismatch (Service DNS vs node FQDN); ureq has no skip-hostname-only
-# option, and this matches the previous python's CERT_NONE. Do NOT use
-# pve.renner.uno (pod hairpin off the ingress LB) or a single node FQDN (SPOF).
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: proxmox-secret-rotator
-  namespace: openbao
-  labels:
-    app.kubernetes.io/name: pve-exporter
-    app.kubernetes.io/component: secret-rotator
----
-apiVersion: batch/v1
-kind: CronJob
-metadata:
-  name: proxmox-secret-rotate
-  namespace: openbao
-  labels:
-    app.kubernetes.io/name: pve-exporter
-    app.kubernetes.io/component: secret-rotator
-spec:
-  suspend: false
-  schedule: "0 4 1 1,4,7,10 *" # quarterly, Europe/Berlin
-  timeZone: Europe/Berlin
-  concurrencyPolicy: Forbid
-  successfulJobsHistoryLimit: 2
-  failedJobsHistoryLimit: 5
-  startingDeadlineSeconds: 3600
-  jobTemplate:
-    spec:
-      backoffLimit: 1
-      activeDeadlineSeconds: 300
-      ttlSecondsAfterFinished: 259200 # 3 d
-      template:
-        metadata:
-          labels:
-            app.kubernetes.io/name: pve-exporter
-            app.kubernetes.io/component: secret-rotator
-        spec:
-          serviceAccountName: proxmox-secret-rotator
-          restartPolicy: Never
-          securityContext:
-            runAsNonRoot: true
-            runAsUser: 65534
-            runAsGroup: 65534
-            seccompProfile:
-              type: RuntimeDefault
-          containers:
-            - name: rotate
-              image: gitea.renner.uno/gitea_admin/secret-rotator:2026-07-30
-              args: ["proxmox-token"]
-              securityContext:
-                allowPrivilegeEscalation: false
-                readOnlyRootFilesystem: true
-                capabilities:
-                  drop: ["ALL"]
-              env:
-                - name: BAO_ADDR
-                  value: http://openbao.openbao.svc.cluster.local:8200
-                - name: BAO_ROLE
-                  value: rotator-proxmox
-                - name: BOOTSTRAP_PATH
-                  value: homelab/infrastructure/secret-rotators-bootstrap/proxmox
-                - name: BAO_PATH
-                  value: homelab/apps/monitoring/pve-exporter-config
-                - name: PVE_USER
-                  value: pve-exporter@pve
-                - name: PVE_TOKENID
-                  value: monitoring
-              resources:
-                requests:
-                  cpu: 25m
-                  memory: 16Mi
-                limits:
-                  memory: 48Mi

kubernetes/infrastructure/rook-ceph-cluster/cephcluster.yaml

@@ -5,7 +5,7 @@ metadata:
   namespace: rook-ceph
 spec:
   cephVersion:
-    image: quay.io/ceph/ceph:v19.2.5 # {"$imagepolicy": "flux-system:ceph"}
+    image: quay.io/ceph/ceph:v19.2.6 # {"$imagepolicy": "flux-system:ceph"}
     allowUnsupported: false
   dataDirHostPath: /var/lib/rook
   mon:
@@ -17,6 +17,17 @@ spec:
     modules:
       - name: pg_autoscaler
         enabled: true
+  # Rotates daemon keys to aes256k for CVE-2025-30156 (#527). Opt-in; disabled
+  # by default. Increment keyGeneration to rotate again.
+  #
+  # Daemon keys only — aes256k over the CSI mount path needs kernel 7.0+ and
+  # these nodes run 6.18, so CSI stays on aes and the client-key checks stay
+  # raised. keyType is unset so Rook picks it.
+  security:
+    cephx:
+      daemon:
+        keyRotationPolicy: KeyGeneration
+        keyGeneration: 2
   # Bump CephFS subvolume cloner pool from 4 → 16. Velero CSI Snapshot
   # Data Movement provisions a backupPVC per source PVC by cloning the
   # VolumeSnapshot, and a 4-thread default stalls migrate-immich's
@@ -45,18 +56,92 @@ spec:
       # would be circular, and Rook only applies mon.volumeClaimTemplate to
       # newly created mons, never migrating existing ones.
       mon_data_avail_warn: "20"
+      # Stop compacting the whole RocksDB prefix on every paxos trim.
+      #
+      # Upstream default is true — "Compact a certain prefix (including paxos)
+      # when we trim its old states" — which is fine at low paxos rates and
+      # pathological here. Measured 2026-08-19:
+      #
+      #   paxos transactions      ~99,000/day  (1.15/s, sampled over 60 s)
+      #   paxos logical data       811 MB/day  (21.9 GB over 27 d uptime)
+      #   ACTUAL mon disk writes    80 GB/day  per mon, 240 GB/day for three
+      #                                        (cgroup io.stat: 2.15 TB/27 d)
+      #   => ~98x write amplification, for a ~106 MB database
+      #
+      # That was ~62% of all OS-disk writes on these nodes and roughly a
+      # quarter of everything the cluster writes — more than every client
+      # workload combined. With paxos_trim_min at 250, ~99k transactions/day
+      # means hundreds of forced range compactions per day, each rewriting the
+      # store.
+      #
+      # The churn itself is 99% cluster log and mgr stats, not real cluster
+      # activity: logm holds 6.7M paxos versions and mgrstat 4.4M, against
+      # osdmap's 9.6k. Every Rook-operator poll (mgr stat / versions / status /
+      # report / osd dump — 23 audit entries per minute per mon) becomes a
+      # cluster-log entry, hence a paxos transaction, replicated to all three
+      # mons. Reducing that polling is the complementary fix and is tracked
+      # separately; this setting removes the amplification regardless of rate.
+      #
+      # This does NOT disable compaction. RocksDB's own levelled compaction
+      # still runs and still reclaims trimmed keys; what stops is the extra
+      # forced compact_range on every trim. Store growth is therefore bounded
+      # by normal compaction, and there are two backstops if that assumption is
+      # wrong: mon_data_size_warn (15 GB default) raises HEALTH_WARN, which
+      # CephHealthWarn in monitoring-rules already alerts on, and
+      # mon_compact_on_start below compacts fully on every mon restart.
+      #
+      # Watch after applying: `du -sh /var/lib/rook/mon-*/store.db` on a node,
+      # or the mon store size in `ceph status`. If it grows past ~1 GB and does
+      # not come back down, revisit rather than leaving it.
+      mon_compact_on_trim: "false"
+      # Compensating control for the above: a full compaction on mon start.
+      # Upstream default is false. Mons restart on upgrades, node reboots and
+      # reschedules, so this gives a periodic full reclaim without paying for
+      # one on every trim. Cheap here — the store is ~106 MB.
+      mon_compact_on_start: "true"
     mgr:
       mgr/volumes/max_concurrent_clones: "16"
-    # QLC tune for the Crucial P2 NVMe per host. Baseline 2026-05-05:
-    # 71.6 MB/s avg / 0.89 s avg latency on a 5-min replicated 4 MiB
-    # write bench (post-SLC-cache). bdev_*_discard need an OSD restart
-    # to take effect (init-time); the bluestore_* params are runtime.
-    # See ceph-speed-baseline-2026-05-05 in the wiki.
+    # Discard/TRIM for the QLC NVMe per host (CT1000P3PSSD8, Crucial P3 Plus —
+    # earlier comments here said "P2", which is a different drive). These need
+    # an OSD restart to take effect (init-time). Baseline 2026-05-05:
+    # 71.6 MB/s avg / 0.89 s avg latency on a 5-min replicated 4 MiB write
+    # bench (post-SLC-cache). See ceph-speed-baseline-2026-05-05 in the wiki.
+    #
+    # bluestore_prefer_deferred_size_ssd / bluestore_deferred_batch_ops_ssd are
+    # DELIBERATELY LEFT AT THE CEPH DEFAULTS (0 and 16) and must not be raised
+    # again without re-reading this.
+    #
+    # They were previously set to the HDD-style 65536 / 64 as a throughput tune.
+    # prefer_deferred_size defers every write SMALLER than the threshold: the
+    # write goes first into RocksDB's WAL and is applied to the object store
+    # afterwards. That is a double write, and at 65536 it caught essentially
+    # every write this cluster makes — SQLite pages are 4 KiB, Postgres pages
+    # 8 KiB. Ceph disables deferral on SSD (default 0) precisely because solid
+    # state does not need the random-write batching that helps spinning rust.
+    #
+    # Two reasons it was the wrong trade here:
+    #   - The benchmark that justified it cannot have exercised it. It used
+    #     4 MiB writes, 64x ABOVE the 64 KiB cutoff, so not one of those writes
+    #     was ever deferred. The setting was validated by a workload it does
+    #     not affect.
+    #   - It doubles writes onto QLC, which has the lowest endurance of any
+    #     NAND type. Measured 2026-08-19: all three OSD drives at 80/82/87%
+    #     endurance used, advancing ~0.13 points/day, i.e. rated life exhausted
+    #     between Nov 2026 and Jan 2027 — and simultaneously, since size=3 on
+    #     3 OSDs means every drive absorbs 100% of cluster writes (see #434).
+    #
+    # This cluster is write-light (~117 GB/day at the RBD client layer), so
+    # throughput was never the binding constraint. Endurance is. If small-write
+    # latency ever becomes the real problem, fix it with more OSDs rather than
+    # by paying a 2x write tax on every 4 KiB page.
     osd:
       bdev_enable_discard: "true"
       bdev_async_discard: "true"
-      bluestore_prefer_deferred_size_ssd: "65536"
-      bluestore_deferred_batch_ops_ssd: "64"
+      # Ceph defaults, set explicitly so the previous override is actively
+      # overwritten rather than merely removed — `ceph config` retains a value
+      # that is only deleted from this file.
+      bluestore_prefer_deferred_size_ssd: "0"
+      bluestore_deferred_batch_ops_ssd: "16"
   dashboard:
     enabled: true
     ssl: false
@@ -75,11 +160,51 @@ spec:
         - "10.2.40.0/24"
       cluster:
         - "10.99.0.0/29"
+  # Nodes are listed explicitly rather than useAllNodes, so that a node can be
+  # taken out of the storage fleet by removing it here. Under useAllNodes the
+  # operator re-provisions an OSD on any node whose device still carries a
+  # BlueStore signature, which makes purging an OSD ahead of pulling its drive
+  # impossible: the OSD is recreated within a reconcile and starts backfilling
+  # onto the drive being replaced.
+  #
+  # All three nodes are bare metal and see three NVMe drives each, so they name
+  # their OSDs by /dev/disk/by-id path. A bare name would be actively wrong:
+  # Talos enumerates these machines' drives in a different order than Proxmox
+  # did, and the kernel name lands on the Kingston SYSTEM disk or on EPHEMERAL
+  # depending on the boot. Rook would probably decline the system disk for
+  # carrying partitions, but that is luck rather than intent. The by-id path is
+  # derived from model and serial and does not depend on enumeration at all.
+  #
+  # k8s-3 demonstrated why: it enumerated the Kingston as nvme0n1 on one boot
+  # and the Crucial as nvme0n1 on the very next one.
+  #
+  #   k8s-1 (jupiter)
+  #   50026B7382B48B86  Kingston  — Talos system disk
+  #   2332E86B749A      Crucial   — Talos EPHEMERAL (/var)
+  #   S437NX0M400426    PM983     — this OSD
+  #
+  #   k8s-2 (saturn)
+  #   50026B7382DA5F6E  Kingston  — Talos system disk
+  #   2332E86B74BC      Crucial   — Talos EPHEMERAL (/var)
+  #   S437NX0M403161    PM983     — this OSD
+  #
+  #   k8s-3 (neptune)
+  #   50026B7382B48B90  Kingston  — Talos system disk
+  #   2326E6E8E67D      Crucial   — Talos EPHEMERAL (/var)
+  #   S437NF0M501176    PM983     — this OSD
   storage:
-    useAllNodes: true
+    useAllNodes: false
     useAllDevices: false
-    devices:
-      - name: nvme0n1 # NVMe drive PCI-passed-through from each PVE node
+    nodes:
+      - name: k8s-1
+        devices:
+          - fullpath: /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NX0M400426
+      - name: k8s-2
+        devices:
+          - fullpath: /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NX0M403161
+      - name: k8s-3
+        devices:
+          - fullpath: /dev/disk/by-id/nvme-SAMSUNG_MZQLB960HAJR-00007_S437NF0M501176
     config:
       osdsPerDevice: "1"
   # Storage layer must never be evicted by kubelet under node pressure

kubernetes/infrastructure/rook-ceph-cluster/cephfs-ci.yaml

@@ -0,0 +1,64 @@
+---
+# A SEPARATE CephFS whose only purpose is Woodpecker CI workspaces. Each
+# pipeline gets a PVC that is created, filled and deleted within ~2 minutes;
+# keeping that churn off `cephfs` spares its size-3 metadataPool.
+#
+# size 2, not 1: an unreplicated pool makes `ceph osd ok-to-stop` refuse for
+# every OSD, blocking all OSD maintenance cluster-wide. size 1 is viable only
+# once these pools have OSDs of their own.
+apiVersion: ceph.rook.io/v1
+kind: CephFilesystem
+metadata:
+  name: cephfs-ci
+  namespace: rook-ceph
+spec:
+  metadataPool:
+    replicated:
+      size: 2
+  dataPools:
+    - name: data0
+      replicated:
+        size: 2
+  metadataServer:
+    activeCount: 1
+    # No standby-replay, unlike `cephfs`. A standby exists there so a failed
+    # MDS does not hang every CephFS mount cluster-wide; here the only casualty
+    # is a running pipeline, and a second daemon to protect throwaway work is
+    # not worth its memory.
+    activeStandby: false
+    resources:
+      # Deliberately NOT the 4Gi that `cephfs` needs. That figure is driven by
+      # immich's millions of cached inodes; a CI workspace is a few thousand
+      # short-lived files that vanish with the pipeline.
+      requests:
+        cpu: 100m
+        memory: 512Mi
+      limits:
+        memory: 1Gi
+---
+# CI-only class. `ceph-filesystem-ephemeral` is deliberately left pointing at
+# the main filesystem: velero/restore-test.sh uses it to restore REAL data for
+# verification, which must not land on unreplicated pools.
+#
+# A new class rather than repointing the old one — StorageClass.parameters are
+# immutable, so `fsName` cannot be edited in place.
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: ceph-filesystem-ci
+provisioner: rook-ceph.cephfs.csi.ceph.com
+parameters:
+  clusterID: rook-ceph
+  fsName: cephfs-ci
+  pool: cephfs-ci-data0
+  csi.storage.k8s.io/provisioner-secret-name: rook-csi-cephfs-provisioner
+  csi.storage.k8s.io/provisioner-secret-namespace: rook-ceph
+  csi.storage.k8s.io/controller-expand-secret-name: rook-csi-cephfs-provisioner
+  csi.storage.k8s.io/controller-expand-secret-namespace: rook-ceph
+  csi.storage.k8s.io/node-stage-secret-name: rook-csi-cephfs-node
+  csi.storage.k8s.io/node-stage-secret-namespace: rook-ceph
+# Delete, for the same reason ceph-filesystem-ephemeral uses it: with Retain,
+# every finished pipeline leaks a Released PV pinning a subvolume (191 of them
+# by 2026-07-16, issue #262 follow-up).
+reclaimPolicy: Delete
+allowVolumeExpansion: true

kubernetes/infrastructure/rook-ceph-cluster/kustomization.yaml

@@ -4,5 +4,6 @@ resources:
   - cephcluster.yaml
   - toolbox.yaml
   - storageclass.yaml
+  - cephfs-ci.yaml
   - volumesnapshotclass.yaml
   - objectstore.yaml

kubernetes/infrastructure/rook-ceph-cluster/storageclass.yaml

@@ -5,6 +5,22 @@ metadata:
   namespace: rook-ceph
 spec:
   failureDomain: host
+  # size 3. This pool sat at 2 while the OSDs were Crucial P3 QLC drives: with
+  # 3 OSDs on 3 hosts, size=3 put every write on every drive, so per-drive write
+  # volume equalled total cluster write volume and all three wore at an
+  # identical rate (measured 2026-08-19: 80/82/87% endurance used, ~0.13
+  # points/day, all three projected to exhaust between Nov 2026 and Jan 2027 —
+  # see #434). Replication does not protect against that, because the wear was
+  # correlated by construction rather than independent.
+  #
+  # The OSDs are Samsung PM983 enterprise drives now, at 2-3% of rated
+  # endurance with a far higher write budget, so wear-out is no longer the
+  # binding constraint and the correlation argument no longer applies.
+  #
+  # min_size is left at the size-3 default of 2. The explicit min_size 1
+  # override existed only because Ceph does not lower min_size when size is
+  # reduced, which would have left the pool 2/2 — where any single OSD down
+  # blocks IO for its PGs and hangs every RBD volume in the cluster.
   replicated:
     size: 3
 ---
@@ -64,11 +80,24 @@ metadata:
   name: cephfs
   namespace: rook-ceph
 spec:
+  # size 3 with no min_size override. This pool holds ~253 MiB against the data
+  # pool's 280 GiB, and losing it would destroy the whole filesystem rather
+  # than just the objects on the lost OSD, so it is the last pool that should
+  # ever run thin.
+  #
+  # It drops to 2/1 whenever one of the three hosts is out for longer than a
+  # reboot. size 3 on two remaining hosts leaves every PG undersized at two
+  # copies with min_size 2, so a single further OSD loss falls below min_size
+  # and blocks IO on every CephFS mount cluster-wide. At 2/1 the same two
+  # copies are active+clean instead, and a further loss degrades rather than
+  # blocks. See RUNBOOK-baremetal-conversion.md.
   metadataPool:
     replicated:
       size: 3
   dataPools:
     - name: data0
+      # size 3 for the same reasons as replicapool above — see the comment
+      # there for why this pool sat at 2 while the Crucials backed the OSDs.
       replicated:
         size: 3
   metadataServer:

kubernetes/infrastructure/rook-ceph-cluster/toolbox.yaml

@@ -38,7 +38,7 @@ spec:
       serviceAccountName: rook-ceph-default
       containers:
         - name: rook-ceph-tools
-          image: quay.io/ceph/ceph:v19.2.5 # {"$imagepolicy": "flux-system:ceph"}
+          image: quay.io/ceph/ceph:v19.2.6 # {"$imagepolicy": "flux-system:ceph"}
           command:
             - /bin/bash
             - -c
@@ -164,7 +164,9 @@ spec:
             name: rook-config-override
             optional: true
         - name: ceph-config
-          emptyDir: {}
+          emptyDir:
+            medium: Memory
+            sizeLimit: 16Mi
       tolerations:
         - key: "node.kubernetes.io/unreachable"
           operator: "Exists"

kubernetes/infrastructure/velero/restore-test-cronjob.yaml

@@ -1,19 +1,30 @@
 ---
-# Weekly Velero restore-test CronJob.
+# Monthly Velero restore-test CronJob — 08:00 UTC on the 1st.
 #
-# Sunday 08:00 UTC — *after* Sunday's own daily backups have finished
-# (paperless 03:15 through apps 07:15, each normally ~1 min). The previous
-# 02:00 slot was chosen as "well before Monday's backups", which missed
-# that the daily schedules also fire on Sunday: a sweep starting at 02:00
-# and running ~5 h sat on top of the entire 03:15-07:15 backup window and
+# The sweep restores every discovered schedule, immich's ~180 GiB library
+# included. At `size: 2` that is ~360 GiB raw, ~120 GiB per OSD, against a
+# ~65 GB/day weekday baseline on three QLC NVMe at 81-87% endurance used —
+# ~47% of each drive's weekly write budget. Monthly keeps every restore path
+# exercised and cuts that share ~4x.
+#
+# 08:00 — *after* that day's own daily backups have finished (paperless
+# 03:15 through apps 07:15, each normally ~1 min). The previous 02:00 slot
+# was chosen as "well before the next day's backups", which missed that the
+# daily schedules fire on the same day: a sweep starting at 02:00 and
+# running ~5 h sat on top of the entire 03:15-07:15 backup window and
 # starved it of node-agent DataUpload concurrency. Observed effect on
 # 2026-07-19 and 2026-07-26 — audiobookshelf-daily took 2 h 18 m and
 # 2 h 33 m respectively against a ~1 min baseline, with the books
-# DataUpload not starting until 06:18. Both are Sundays; no other day
-# shows it.
+# DataUpload not starting until 06:18. Both are Sundays, the only day the
+# sweep ran then; no other day shows it.
 #
-# 08:00 + the 6 h activeDeadlineSeconds cap still lands well clear of
-# Monday's 03:15.
+# 08:00 + the 6 h activeDeadlineSeconds cap still lands well clear of the
+# next day's 03:15.
+#
+# Day-of-month rather than a fixed weekday: Kubernetes parses cron with
+# robfig/cron's standard parser, which has no `#` nth-weekday operator, and
+# restricting both day fields (`0 8 1-7 * 0`) ORs them — every Sunday plus
+# the 1st through 7th.
 #
 # Concurrency policy Forbid: the immich restore (~180 GiB, millions of
 # small files) alone can run multiple hours, so the full sweep is sized
@@ -29,7 +40,7 @@ metadata:
   name: restore-test
   namespace: velero
 spec:
-  schedule: "0 8 * * 0"
+  schedule: "0 8 1 * *"
   timeZone: Etc/UTC
   concurrencyPolicy: Forbid
   successfulJobsHistoryLimit: 2
@@ -39,7 +50,7 @@ spec:
     spec:
       backoffLimit: 0
       activeDeadlineSeconds: 21600  # 6 h hard cap (immich alone can take ~3 h)
-      ttlSecondsAfterFinished: 1209600  # 14 d — enough to inspect after the next weekly run
+      ttlSecondsAfterFinished: 3456000  # 40 d — enough to inspect after the next monthly run
       template:
         spec:
           serviceAccountName: restore-test
@@ -85,4 +96,6 @@ spec:
                 name: restore-test-script
                 defaultMode: 0755
             - name: tmp
-              emptyDir: {}
+              emptyDir:
+                medium: Memory
+                sizeLimit: 16Mi

kubernetes/infrastructure/velero/restore-test-rbac.yaml

@@ -1,5 +1,5 @@
 ---
-# RBAC for the weekly restore-test CronJob.
+# RBAC for the monthly restore-test CronJob.
 #
 # Permissions required by restore-test.sh:
 #   - velero.io: read schedules/backups/BSL, create+delete restores

kubernetes/infrastructure/velero/restore-test.sh

@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# Weekly verification that Velero PVC backups are actually restorable.
+# Monthly verification that Velero PVC backups are actually restorable.
 #
 # What this exercises (per service):
 #   1. Velero Restore reaches phase=Completed against yesterday's
@@ -53,7 +53,7 @@ check() {
 # and ceph-filesystem StorageClasses ship with reclaimPolicy: Retain so that
 # production PVC deletions can be undone manually. The restore-test has the
 # opposite need: every PVC it creates is throwaway and MUST not survive the
-# iteration. Without this, each weekly run leaks O(100 GiB) of CephFS
+# iteration. Without this, each run leaks O(100 GiB) of CephFS
 # subvolumes and the cluster eventually goes OSD_FULL (incident 2026-05-17).
 #
 # Why this still exists after ceph-*-ephemeral landed. Three categories of

kubernetes/talos/cilium.yaml

@@ -310,6 +310,13 @@ data:
   # policy map (per endpoint). Keep in lockstep with patches/cilium-cni.yaml and
   # values.bpf.policyMapMax in kubernetes/infrastructure/cilium/helmrelease.yaml.
   bpf-policy-map-max: "65536"
+  # VLAN ids the eBPF datapath may pass; without this it drops VLAN-tagged
+  # frames. 30 is v30iot, the macvlan parent for the IoT workloads, and a VLAN
+  # sub-interface of enp2s0f0np0 on bare metal. VLAN 40 (mgmt0) rides the same
+  # parent but passes without an entry, so it is deliberately not listed.
+  # Keep in lockstep with patches/cilium-cni.yaml and values.bpf.vlanBypass in
+  # kubernetes/infrastructure/cilium/helmrelease.yaml. See issue #592.
+  vlan-bpf-bypass: "30"
   # bpf-policy-stats-map-max specifies the maximum number of entries in global
   # policy stats map
   bpf-policy-stats-map-max: "65536"

kubernetes/talos/patches/cilium-cni.yaml

@@ -331,6 +331,16 @@ cluster:
           # inline manifest, so a lower value here means a cold rebuild comes up with
           # a map that overflows until Flux reconciles.
           bpf-policy-map-max: "65536"
+          # VLAN ids the eBPF datapath may pass; without this it drops
+          # VLAN-tagged frames. 30 is v30iot, the macvlan parent for the IoT
+          # workloads, and a VLAN sub-interface of enp2s0f0np0 on bare metal.
+          # VLAN 40 (mgmt0) rides the same parent but passes without an entry,
+          # so it is deliberately not listed. Keep in lockstep with
+          # values.bpf.vlanBypass in kubernetes/infrastructure/cilium/helmrelease.yaml
+          # and the third copy in kubernetes/talos/cilium.yaml — bootstrap installs
+          # from this inline manifest, so omitting it here breaks the IoT macvlan
+          # workloads until Flux reconciles. See issue #592.
+          vlan-bpf-bypass: "30"
           # bpf-policy-stats-map-max specifies the maximum number of entries in global
           # policy stats map
           bpf-policy-stats-map-max: "65536"

kubernetes/talos/patches/cluster.yaml

@@ -5,6 +5,31 @@ machine:
     - k8s-3.srv.renner.uno
     - k8s.renner.uno
     - 10.2.40.60
+  # Install from the Image Factory metal installer, not the stock
+  # ghcr.io/siderolabs/installer that `talosctl gen config` defaults to.
+  # Extensions live in the installer image, so with the stock one the schematic
+  # is only ever a property of the boot media and nothing lands on disk —
+  # `talosctl get extensions` returns empty on a node installed that way.
+  #
+  # What the schematic carries, and why each matters here:
+  #
+  #   siderolabs/intel-ucode   CPU microcode. On a VM the hypervisor supplied
+  #                            it; on bare metal nothing does except the BIOS.
+  #   siderolabs/thunderbolt   Inert until machine.kernel.modules asks for
+  #                            thunderbolt/thunderbolt_net. Present so enabling
+  #                            the MS-01 mesh later needs neither a new image
+  #                            nor a reinstall — which is exactly the property
+  #                            lost by installing without it.
+  #
+  # a21cec8f… is the bare-metal schematic described in RUNBOOK-cold-rebuild.md:
+  # same net.ifnames=1 as the Proxmox one (the predictable interface names the
+  # per-node VLAN parents match against), minus qemu-guest-agent.
+  #
+  # Only read at install and upgrade time. Changing it here does nothing to a
+  # running node until that node is upgraded. Bump the tag with the Talos
+  # version — it must match the running version, or the upgrade is a downgrade.
+  install:
+    image: factory.talos.dev/metal-installer/a21cec8f35ae32ab5e9a452e7b13500ca8776053930b8e182b9cb23eed186b64:v1.12.6
   features:
     rbac: true
     stableHostname: true

kubernetes/talos/patches/k8s-1.yaml

@@ -1,62 +1,129 @@
+# k8s-1 runs Talos directly on jupiter's MS-01 hardware; k8s-2 and k8s-3 are
+# still Proxmox VMs. Every machine-specific selector below differs from theirs
+# for that reason: there are no virtio NICs, no PCI-passthrough layer, and all
+# four physical NICs and all three NVMe drives are visible to Talos at once.
+#
+# Networking uses the v1.12 multi-document config rather than the deprecated
+# machine.network.interfaces. That is deliberate: the docs do not define what
+# happens when the same link is configured by both schemas, so this file keeps
+# every link in one schema instead of straddling them.
 machine:
   network:
     hostname: k8s-1
-    # Shared control-plane VIP — etcd-elected leader hosts 10.2.40.60 as a
-    # secondary IP. Exposed via UniFi DNS as k8s.renner.uno.
-    #
-    # Two virtio NICs (mgmt + v30iot). `driver: virtio_net` would match
-    # both, so each is pinned by its PCI bus path:
-    #   - 0000:06:12.0 — original mgmt NIC (the only virtio before)
-    #   - 0000:06:13.0 — extra_network_bridges = ["v30iot"] in tofu
-    # Bus paths are consistent across k8s-{1,2,3}; verified live 2026-05-13.
-    # The X710 selector stays driver-based (i40e is unambiguous) and
-    # tolerates kernel renaming.
-    interfaces:
-      - deviceSelector:
-          busPath: "0000:06:12.0"
-        dhcp: true
-        vip:
-          ip: 10.2.40.60
-      # v30iot: LINK ONLY — no host addressing. The NIC is the macvlan
-      # parent for the v30iot pod attachments (matter/otbr/shelly-manager),
-      # whose children do their own DHCP on the L2; host addresses were
-      # never needed for that. Host DHCP (formerly dhcp + routeMetric 4096)
-      # installed a connected 10.2.30.0/24 route, so node replies to
-      # IoT-subnet clients bypassed the UDM and broke every UDM-DNATed
-      # flow (hairpin NAT: un-translated SYN-ACK -> client kernel RST,
-      # verified live 2026-07-23). The purpose the old comment claimed —
-      # Cilium L2 announce for an IoT LB pool — does not exist (no
-      # L2AnnouncementPolicy, no such pool). v6 RA addressing on this NIC
-      # is disabled via machine.sysctls for the same reason.
-      - deviceSelector:
-          busPath: "0000:06:13.0"
-        dhcp: false
-      # Dedicated Ceph cluster network on the X710 SFP+ port (PCI passthrough,
-      # mapping x710-port1 in opentofu/vm_150_k8s_1.tf). Separate /29 on VLAN 49,
-      # MTU 9000, no default route — used by Rook for OSD↔OSD replication via
-      # CephCluster.spec.network.addressRanges.cluster.
-      - deviceSelector:
-          driver: i40e
-        addresses:
-          - 10.99.0.1/29
-        mtu: 9000
   sysctls:
-    # v30iot NIC carries no host addressing (see interfaces above); this
-    # disables v6 entirely (incl. link-local) so the host is unreachable on that L2.
-    #
-    # Keyed on the STABLE predictable name `ens19` (busPath 0000:06:13.0 =
-    # v30iot). Predictable naming is re-enabled via the Image Factory
-    # schematic (net.ifnames=1; see .woodpecker/deploy-infrastructure.yml),
-    # so `ens19` is topology-derived and does NOT shuffle across reboots —
-    # unlike the former `eth0/1/2` (driver-probe order) which renamed on a
-    # k8s-3 reboot and silently mis-targeted this sysctl onto mgmt. The four
-    # v30iot macvlan NADs key on the same `ens19`.
+    # ens19 (VLAN 30, v30iot) is LINK ONLY — no host addressing. It exists as
+    # the macvlan parent for the v30iot pod attachments
+    # (matter/otbr/shelly-manager/music-assistant), whose children do their own
+    # DHCP on that L2. Host addresses were never needed and are actively
+    # harmful: a connected 10.2.30.0/24 route makes node replies to IoT-subnet
+    # clients bypass the UDM, which breaks every UDM-DNATed flow (hairpin NAT:
+    # un-translated SYN-ACK -> client kernel RST, verified live 2026-07-23).
+    # This disables v6 entirely on that link, including link-local.
     net.ipv6.conf.ens19.disable_ipv6: "1"
   install:
-    disk: /dev/vda
+    # Three NVMe drives are ordinary block devices to Talos here: the Kingston
+    # OM8PGP41024Q-A0 system disk, the Samsung PM983 that Rook claims as the
+    # Ceph OSD, and the retired Crucial P3 that carries EPHEMERAL (see the
+    # VolumeConfig below). Every one of them is selected by serial. Nothing in
+    # this file may use a broad selector: `disk.transport == 'nvme'` matches all
+    # three, and the failure mode is Talos taking the OSD drive.
+    diskSelector:
+      serial: 50026B7382B48B86
   certSANs:
     - k8s-1.srv.renner.uno
   nodeLabels:
     topology.kubernetes.io/zone: jupiter
 cluster:
   allowSchedulingOnControlPlanes: true
+---
+# EPHEMERAL (/var) goes on the retired Crucial P3, not the system disk.
+#
+# That drive is at 87% of its rated endurance and was this node's Ceph OSD until
+# the PM983 replaced it. /var is exactly the right role for a disk in that
+# state: containerd's image and snapshot store, pod logs and the Ceph mon store
+# are all reconstructible, so the drive is consumable here in a way it is not as
+# an OSD. It also gives /var ~931 GB instead of the 64 GB the VM-era zvol had,
+# which is the constraint that makes k8s-2 and k8s-3 trip DiskPressure whenever
+# they absorb another node's pods.
+#
+# THIS ONLY TAKES EFFECT ON FIRST PROVISION. Talos applies volume configuration
+# solely when the volume does not yet exist; a config applied after EPHEMERAL
+# has been created elsewhere is silently ignored, and relocating it then
+# requires wiping the disk and booting install media again.
+#
+# Matched by serial deliberately. `disk.transport == 'nvme' && !system_disk`
+# would match the PM983 as well, and losing that coin flip means Talos formats
+# the OSD drive.
+apiVersion: v1alpha1
+kind: VolumeConfig
+name: EPHEMERAL
+provisioning:
+  diskSelector:
+    match: disk.serial == '2332E86B749A'
+  minSize: 50GiB
+  grow: true
+---
+# X710 port 0 carries the switch trunk. It takes no addressing itself — the
+# port's untagged VLAN is the 192.168.8.0/24 network, which this node has no
+# business being on — so management and v30iot both ride it as tagged VLANs.
+#
+# Links are addressed by predictable kernel name. Those derive from PCI
+# topology, not probe order — though not immune: adding the third NVMe
+# renumbered the buses and moved the unused second I226 from enp89s0 to
+# enp90s0. Neither name used here was affected.
+#
+# LinkAliasConfig would be the better way to pin these, and is worth
+# reintroducing — but only once it is proven. Aliasing both X710 ports by
+# permanent MAC and using the aliases as VLAN parents produced a node with no
+# address on any interface: the aliases were never created, so every document
+# referencing them did nothing, including the static address on port 1. Verify
+# any alias-based revision with `talosctl apply-config --mode=try`, which rolls
+# back on its own, before making it permanent.
+apiVersion: v1alpha1
+kind: LinkConfig
+name: enp2s0f0np0
+up: true
+---
+# Management, VLAN 40. Carries the node IP, etcd, the API and Ceph's public
+# network. The k8s-1 VM reached 10.2.40.174 over this same physical port via
+# Proxmox's v40srv VNet (vmbr0.40), so the switch trunks VLAN 40 here.
+apiVersion: v1alpha1
+kind: VLANConfig
+name: mgmt0
+vlanID: 40
+parent: enp2s0f0np0
+up: true
+---
+apiVersion: v1alpha1
+kind: DHCPv4Config
+name: mgmt0
+---
+# Shared control-plane VIP, etcd-elected. Exposed via UniFi DNS as
+# k8s.renner.uno.
+apiVersion: v1alpha1
+kind: Layer2VIPConfig
+name: 10.2.40.60
+link: mgmt0
+---
+# Named ens19 on purpose. The four v30iot macvlan NADs hard-code
+# `master: ens19`, and k8s-2/k8s-3 still present the VLAN as a virtio NIC of
+# that name. VLANConfig.name is free-form, so keeping it lets one set of NADs
+# serve both VM and bare-metal nodes for as long as the cluster is mixed.
+apiVersion: v1alpha1
+kind: VLANConfig
+name: ens19
+vlanID: 30
+parent: enp2s0f0np0
+up: true
+---
+# Dedicated Ceph cluster network on X710 port 1, VLAN 49 / 10.99.0.0/29,
+# untagged on the wire. Used by Rook for OSD<->OSD replication via
+# CephCluster.spec.network.addressRanges.cluster. Not routed off its own
+# segment, and deliberately has no default route.
+apiVersion: v1alpha1
+kind: LinkConfig
+name: enp2s0f1np1
+up: true
+mtu: 9000
+addresses:
+  - address: 10.99.0.1/29

kubernetes/talos/patches/k8s-2.yaml

@@ -1,31 +1,98 @@
+# k8s-2 runs Talos directly on saturn's MS-01 hardware. Structurally identical
+# to k8s-1.yaml — see that file for the reasoning behind each choice; only the
+# serials, MACs and addresses differ.
 machine:
   network:
     hostname: k8s-2
-    # Shared control-plane VIP — see k8s-1.yaml for the busPath rationale.
-    interfaces:
-      - deviceSelector:
-          busPath: "0000:06:12.0"
-        dhcp: true
-        vip:
-          ip: 10.2.40.60
-      # v30iot: link only, no host addressing — see k8s-1.yaml.
-      - deviceSelector:
-          busPath: "0000:06:13.0"
-        dhcp: false
-      # See k8s-1.yaml for context on the dedicated Ceph cluster network.
-      - deviceSelector:
-          driver: i40e
-        addresses:
-          - 10.99.0.2/29
-        mtu: 9000
   sysctls:
-    # v30iot NIC carries no host addressing — see k8s-1.yaml.
+    # ens19 (VLAN 30, v30iot) is LINK ONLY — no host addressing. It is the
+    # macvlan parent for the v30iot pod attachments; host addresses on that L2
+    # make node replies bypass the UDM and break every UDM-DNATed flow.
     net.ipv6.conf.ens19.disable_ipv6: "1"
   install:
-    disk: /dev/vda
+    # Three NVMe drives are visible on bare metal. Select the installer target
+    # by serial so it cannot land on the OSD drive or on EPHEMERAL, whatever
+    # order the kernel enumerates them in — Talos and Proxmox do not agree on
+    # that order.
+    diskSelector:
+      serial: 50026B7382DA5F6E
   certSANs:
     - k8s-2.srv.renner.uno
   nodeLabels:
     topology.kubernetes.io/zone: saturn
 cluster:
   allowSchedulingOnControlPlanes: true
+---
+# EPHEMERAL (/var) goes on the retired Crucial P3, not the system disk. That
+# drive is at 81% of its rated endurance and is finished as an OSD, but
+# everything EPHEMERAL holds is reconstructible, so it is consumable in this
+# role. It also gives /var ~931 GB instead of the 64 GB the VM-era zvol had.
+#
+# ONLY TAKES EFFECT ON FIRST PROVISION. Talos applies volume configuration
+# solely when the volume does not yet exist; applied later it is silently
+# ignored and relocating needs a disk wipe and a boot from install media.
+#
+# Matched by serial deliberately: a broad selector such as
+# `disk.transport == 'nvme' && !system_disk` would match the PM983 as well.
+apiVersion: v1alpha1
+kind: VolumeConfig
+name: EPHEMERAL
+provisioning:
+  diskSelector:
+    match: disk.serial == '2332E86B74BC'
+  minSize: 50GiB
+  grow: true
+---
+# X710 port 0 carries the switch trunk and takes no addressing itself — the
+# port's untagged VLAN is not one this node belongs on. Management and v30iot
+# both ride it as tagged VLANs.
+#
+# Addressed by predictable kernel name, not by LinkAliasConfig. Aliases are
+# valid VLAN parents, but a selector matching zero links is skipped silently
+# and Talos disables default DHCP machine-wide as soon as any new-style network
+# document exists — so a silent miss leaves the node with no address on any
+# interface. See RUNBOOK-baremetal-conversion.md.
+apiVersion: v1alpha1
+kind: LinkConfig
+name: enp2s0f0np0
+up: true
+---
+# Management, VLAN 40. Node IP, etcd, the API and Ceph's public network.
+apiVersion: v1alpha1
+kind: VLANConfig
+name: mgmt0
+vlanID: 40
+parent: enp2s0f0np0
+up: true
+---
+apiVersion: v1alpha1
+kind: DHCPv4Config
+name: mgmt0
+---
+# Shared control-plane VIP, etcd-elected. Exposed via UniFi DNS as
+# k8s.renner.uno.
+apiVersion: v1alpha1
+kind: Layer2VIPConfig
+name: 10.2.40.60
+link: mgmt0
+---
+# Named ens19 on purpose: the four v30iot macvlan NADs hard-code
+# `master: ens19`, and k8s-3 still presents the VLAN as a virtio NIC of that
+# name. VLANConfig.name is free-form, so one set of NADs serves both.
+apiVersion: v1alpha1
+kind: VLANConfig
+name: ens19
+vlanID: 30
+parent: enp2s0f0np0
+up: true
+---
+# Dedicated Ceph cluster network on X710 port 1, VLAN 49 / 10.99.0.0/29,
+# untagged on the wire. OSD<->OSD replication only. Not routed off its own
+# segment and deliberately has no default route.
+apiVersion: v1alpha1
+kind: LinkConfig
+name: enp2s0f1np1
+up: true
+mtu: 9000
+addresses:
+  - address: 10.99.0.2/29

kubernetes/talos/patches/k8s-3.yaml

@@ -1,31 +1,82 @@
+# k8s-3 runs Talos directly on neptune's MS-01 hardware. Structurally identical
+# to k8s-1.yaml — see that file for the reasoning behind each choice; only the
+# serials and addresses differ. neptune was the last node converted, so all
+# three control-plane nodes now share this layout and no Proxmox remains.
 machine:
   network:
     hostname: k8s-3
-    # Shared control-plane VIP — see k8s-1.yaml for the busPath rationale.
-    interfaces:
-      - deviceSelector:
-          busPath: "0000:06:12.0"
-        dhcp: true
-        vip:
-          ip: 10.2.40.60
-      # v30iot: link only, no host addressing — see k8s-1.yaml.
-      - deviceSelector:
-          busPath: "0000:06:13.0"
-        dhcp: false
-      # See k8s-1.yaml for context on the dedicated Ceph cluster network.
-      - deviceSelector:
-          driver: i40e
-        addresses:
-          - 10.99.0.3/29
-        mtu: 9000
   sysctls:
     # v30iot NIC carries no host addressing — see k8s-1.yaml.
     net.ipv6.conf.ens19.disable_ipv6: "1"
   install:
-    disk: /dev/vda
+    # Three NVMe drives are visible on bare metal. Select the installer target
+    # by serial so it cannot land on the OSD drive or on EPHEMERAL, whatever
+    # order the kernel enumerates them in — Talos and Proxmox do not agree on
+    # that order.
+    diskSelector:
+      serial: 50026B7382B48B90
   certSANs:
     - k8s-3.srv.renner.uno
   nodeLabels:
     topology.kubernetes.io/zone: neptune
 cluster:
   allowSchedulingOnControlPlanes: true
+---
+# EPHEMERAL (/var) goes on the retired Crucial P3, not the system disk. That
+# drive is at 83% of its rated endurance and is finished as an OSD, but
+# everything EPHEMERAL holds is reconstructible, so it is consumable in this
+# role. Available spare is still 100% and it reports no media errors.
+#
+# ONLY TAKES EFFECT ON FIRST PROVISION. Talos applies volume configuration
+# solely when the volume does not yet exist; applied later it is silently
+# ignored and relocating needs a disk wipe and a boot from install media.
+#
+# Matched by serial deliberately: a broad selector such as
+# `disk.transport == 'nvme' && !system_disk` would match the PM983 as well.
+apiVersion: v1alpha1
+kind: VolumeConfig
+name: EPHEMERAL
+provisioning:
+  diskSelector:
+    match: disk.serial == '2326E6E8E67D'
+  minSize: 50GiB
+  grow: true
+---
+apiVersion: v1alpha1
+kind: LinkConfig
+name: enp2s0f0np0
+up: true
+---
+apiVersion: v1alpha1
+kind: VLANConfig
+name: mgmt0
+vlanID: 40
+parent: enp2s0f0np0
+up: true
+---
+apiVersion: v1alpha1
+kind: DHCPv4Config
+name: mgmt0
+---
+# Shared control-plane VIP, held by whichever node wins the etcd campaign.
+apiVersion: v1alpha1
+kind: Layer2VIPConfig
+name: 10.2.40.60
+link: mgmt0
+---
+# v30iot: link only, no host addressing — see k8s-1.yaml.
+apiVersion: v1alpha1
+kind: VLANConfig
+name: ens19
+vlanID: 30
+parent: enp2s0f0np0
+up: true
+---
+# Dedicated Ceph cluster network — see k8s-1.yaml.
+apiVersion: v1alpha1
+kind: LinkConfig
+name: enp2s0f1np1
+up: true
+mtu: 9000
+addresses:
+  - address: 10.99.0.3/29

opentofu/lxc_800_rustfs_decommission.tf

@@ -1,122 +0,0 @@
-# TEMPORARY shims v3 — rustfs LXCs 800/801/802, decommission in progress.
-#
-# Stage 2 (#270) failed: both destroy attempts died instantly with
-# "context deadline exceeded" on the pre-destroy status GET, while the PVE
-# API answers the same call in <1s. Cause: the bpg provider reads
-# `timeout_delete` from STATE at destroy time, and importing a resource
-# whose config declares no timeouts stored 0 — a zero-second context.
-# (The import/plan read path uses its own timeout, which is why stage 1
-# planned clean.)
-#
-# Fix: declare the timeouts explicitly, apply ONCE so they land in state
-# (in-place no-API update), then delete this file again and the next apply
-# destroys the containers.
-#
-# v3: v2 declared ONLY the ids, `started`, `unprivileged` and the timeouts,
-# claiming "the other attributes mirror the real containers so the plan stays
-# in-place". They did not. Everything absent from config planned as a removal,
-# and two of those removals — `mount_point` and `operating_system` — force
-# replacement, so the plan came back `must be replaced` for all three. Applying
-# that would have destroyed and recreated containers tagged `critical`,
-# `state-backend`, `terraform-backend` instead of nudging a timeout into state.
-#
-# So the blocks below are not decoration: each one exists to keep an attribute
-# out of the diff. They mirror the live containers, read from the plan's own
-# state values. `tags` is declared for the same reason — without it the apply
-# strips the tags off the real containers.
-#
-#   1. merge; run PIPELINE=deploy-infrastructure (plan MUST show in-place
-#      update only, no replace), then TOFU_APPLY=true
-#   2. follow-up PR deletes this file; plan shows 3 destroys; apply.
-
-locals {
-  # Per-container values mirrored from live state. hostname and mac_address
-  # differ per container; declaring them wrong would itself force a diff.
-  rustfs_decommission = {
-    "800" = { node = "saturn", hostname = "rustfs-1.srv.renner.uno", mac = "BC:24:11:0B:E5:7B" }
-    "801" = { node = "jupiter", hostname = "rustfs-2.srv.renner.uno", mac = "BC:24:11:EA:1D:38" }
-    "802" = { node = "neptune", hostname = "rustfs-3.srv.renner.uno", mac = "BC:24:11:8F:F4:67" }
-  }
-}
-
-resource "proxmox_virtual_environment_container" "rustfs_decommission" {
-  for_each = local.rustfs_decommission
-
-  node_name     = each.value.node
-  vm_id         = tonumber(each.key)
-  started       = false
-  unprivileged  = true
-  start_on_boot = true
-
-  tags = [
-    "critical",
-    "rustfs",
-    "state-backend",
-    "terraform-backend",
-  ]
-
-  timeout_create = 1800
-  timeout_clone  = 1800
-  timeout_update = 1800
-  timeout_delete = 1800
-  timeout_start  = 300
-
-  cpu {
-    architecture = "amd64"
-    cores        = 1
-    units        = 1024
-  }
-
-  memory {
-    dedicated = 1024
-    swap      = 0
-  }
-
-  disk {
-    datastore_id = "local"
-    size         = 10
-  }
-
-  # forces replacement if absent — this is the block that broke v2.
-  mount_point {
-    path      = "/mnt/data"
-    size      = "50G"
-    volume    = "local:${each.key}/vm-${each.key}-disk-1.raw"
-    backup    = true
-    replicate = true
-  }
-
-  network_interface {
-    name        = "eth0"
-    bridge      = "v40srv"
-    enabled     = true
-    firewall    = false
-    mac_address = each.value.mac
-  }
-
-  initialization {
-    hostname = each.value.hostname
-
-    ip_config {
-      ipv4 {
-        address = "dhcp"
-      }
-    }
-  }
-
-  # operating_system is NOT declared, and must not be. The provider requires
-  # template_file_id inside it, which is a create-time-only value: PVE does not
-  # report the original template for an existing container, so it is absent
-  # from state and cannot be mirrored. Declaring the block without it fails
-  # validation ("Missing required argument"); declaring it with a guessed
-  # template would be a lie that only surfaces if something ever recreates
-  # these.
-  #
-  # Left undeclared it would plan as a removal, and removing operating_system
-  # forces replacement — the exact failure this file exists to avoid. So it is
-  # ignored instead. The timeouts above are deliberately NOT ignored: they are
-  # the one thing this shim needs to actually apply.
-  lifecycle {
-    ignore_changes = [operating_system]
-  }
-}

opentofu/modules/proxmox_vm/variables.tf

@@ -47,8 +47,13 @@ variable "tags" {
 }
 
 variable "template_name" {
-  type    = string
-  default = "ubuntu-24-04-template"
+  type = string
+  # No default clone source. The ubuntu-24.04-template it used to name was
+  # deleted with saturn's conversion to bare metal, and Proxmox is being
+  # retired, so a default pointing at a template that does not exist would
+  # only fail at apply time. Set it explicitly if a clone source is ever
+  # wanted again.
+  default = null
 }
 
 variable "full_clone" {

opentofu/providers.tf

@@ -59,8 +59,25 @@ provider "unifi" {
 
 # UniFi Network v2 API (policy-table NAT rules). Same controller and API key
 # as the unifi provider above; the v2 endpoints are simply not covered by it.
-# UniFi PUT quirk: updates may require _id in the body, which restapi does
-# not send — if an in-place update 400s, taint the object so CI recreates it.
+# UniFi PUT quirk — DISPROVEN 2026-08-19, left here because it was wrong for
+# long enough to do damage. The claim was that a UniFi PUT may require _id in
+# the body, which restapi does not send, and that a 400 should be answered by
+# tainting the object so CI recreates it. Tested directly against a live rule
+# with exactly the body tofu sends:
+#
+#   PUT /proxy/network/v2/api/site/default/nat/6a61c7d7bbd7b2ca90981f51
+#   body: the object, _id removed          -> HTTP 200, object returned intact
+#   rule re-read and diffed before/after   -> byte-identical
+#   rule count                             -> 60 before, 60 after
+#
+# Confirmed again at scale by apply 5675: all 60 rules "Modifications complete
+# after 0s", no 400s, ci.renner.uno and immich.renner.uno both still 200.
+#
+# This mattered. A warning that applying might break firewall rules is a
+# warning nobody argues with, so the apply did not get run — and the
+# renner-uno-k8s-mariadb bucket added in b976a67b sat uncreated for three days
+# while the WordPress backup failed nightly with "The specified bucket does not
+# exist". Applying these NAT rules is safe. Do not taint them.
 #
 # create_returns_object, NOT write_returns_object. Both make the provider store
 # the API response into `data`; write_returns_object does it for PUT as well as

opentofu/scripts/remove_unifi_clients.sh

@@ -1,52 +0,0 @@
-#!/bin/bash
-# Script to help with UniFi client migration
-#
-# NOTE: This script is NO LONGER RECOMMENDED
-# Instead, follow the migration guide in MIGRATION_QUICKSTART.md
-#
-# This script removes unifi_client resources from state, but the recommended
-# approach is to let 'tofu apply' handle the removal automatically.
-
-echo "⚠️  WARNING: This script is deprecated"
-echo ""
-echo "Please use the recommended migration path instead:"
-echo "  1. Deploy DNS server: tofu apply -target=module.proxmox_dns"
-echo "  2. Configure UniFi DNS to use 10.2.40.53"
-echo "  3. Run full apply: tofu apply"
-echo ""
-echo "See MIGRATION_QUICKSTART.md for details."
-echo ""
-read -p "Do you still want to remove resources from state? (y/N) " -n 1 -r
-echo
-if [[ ! $REPLY =~ ^[Yy]$ ]]; then
-    echo "Cancelled. See MIGRATION_QUICKSTART.md for recommended approach."
-    exit 0
-fi
-
-set -e
-
-echo "Removing unifi_client resources from OpenTofu state..."
-echo "Note: VMs will keep their current network configuration"
-echo ""
-
-# List of all unifi_client resources to remove
-RESOURCES=(
-  "module.cdns_srv_renner_uno.unifi_client.this[0]"
-  "module.fortress_renner_uno.unifi_client.this[0]"
-  "module.grafana_srv_renner_uno.unifi_client.this[0]"
-  "module.influxdb_srv_renner_uno.unifi_client.this[0]"
-  "module.kasm_srv_renner_uno.unifi_client.this[0]"
-  "module.mosquitto_srv_renner_uno.unifi_client.this[0]"
-  "module.netbox_srv_renner_uno.unifi_client.this[0]"
-  "module.ntp_srv_renner_uno.unifi_client.this[0]"
-  "module.paperless_srv_renner_uno.unifi_client.this[0]"
-  "module.uptime_kuma_srv_renner_uno.unifi_client.this[0]"
-)
-
-for resource in "${RESOURCES[@]}"; do
-  echo "Removing: $resource"
-  tofu state rm "$resource" || echo "  (already removed or not found)"
-done
-
-echo ""
-echo "Done! Run 'tofu plan' to verify."

opentofu/talos_image.tf

@@ -1,13 +0,0 @@
-locals {
-  talos_version = "v1.12.6"
-  # Image Factory schematic (predictable naming: net.ifnames=1). Keep in
-  # sync with TALOS_SCHEMATIC in .woodpecker/deploy-infrastructure.yml —
-  # that pipeline downloads the image, this local names it for VM import,
-  # and both MUST agree on the filename. The schematic hash is embedded in
-  # the filename so a schematic bump changes the name, forcing a fresh
-  # download instead of silently reusing a stale cached image (which would
-  # reintroduce net.ifnames=0 on the next VM (re)provision).
-  talos_schematic = "88e3752db032a08082972bd0adf16a80849e3870c3f95cf5b669936c5699e5ad"
-  talos_filename  = "talos-${local.talos_version}-${substr(local.talos_schematic, 0, 12)}-nocloud-amd64.raw"
-  talos_image     = "local:import/${local.talos_filename}"
-}

opentofu/unifi_nat_hairpin.tf

@@ -107,17 +107,51 @@ resource "restapi_object" "hairpin" {
   for_each = local.hairpin_rules
 
   path = "/nat"
+
+  # KNOWN PERMANENT DIFF — cosmetic. Do not try to fix it with a provider
+  # setting; four have been tried and none work. Recorded here so the next
+  # person does not spend a day rediscovering that.
+  #
+  # Every plan shows all 60 of these as "will be updated in-place" with the
+  # entire diff being one line:
+  #
+  #   ~ data = jsonencode( ~ { - _id = "6a61c7d7bbd7b2ca90981f51"
+  #                              # (15 unchanged attributes hidden) } )
+  #
+  # The UDM assigns _id and returns it; the provider puts the response into
+  # `data`; the `data` below never sets it. So state carries a field the config
+  # does not. It survives an apply: 5675 rewrote all 60, and the very next plan
+  # (5677) showed the same 60 again.
+  #
+  # Tried, with the plan that disproved each — do not repeat these:
+  #
+  #   ignore_changes_to = ["_id"]                    5661  60 remained
+  #   ignore_all_server_changes = true               5662  60 remained
+  #   restapi 3.x + ignore_server_additions = true   5667  60 remained, and
+  #                                                        added an api_data diff
+  #   write_returns_object -> create_returns_object  5677  60 remained
+  #
+  # The first three cannot work in principle: `data` is a configured attribute,
+  # so Terraform core diffs config against state before any provider-side ignore
+  # logic runs. The fourth was the better theory — that PUT responses were
+  # re-polluting `data` — and it is wrong too, which means the response is
+  # written into `data` on the READ path. Settling it properly needs the
+  # provider source, not more guesses at the schema.
+  #
+  # It is safe to apply through. Verified twice: a single PUT without _id
+  # returns 200 and leaves the rule byte-identical (see providers.tf), and
+  # apply 5675 modified all 60 with no errors and no service impact.
   data = jsonencode({
-    description  = "Hairpin ${each.key}"
-    type         = "DNAT"
-    enabled      = true
-    exclude      = false
-    in_interface = each.value.network_id
-    ip_address   = each.value.endpoint.target
-    ip_version   = each.value.endpoint.ip_version
-    protocol     = "tcp_udp"
-    rule_index   = each.value.rule_index
-    logging      = false
+    description              = "Hairpin ${each.key}"
+    type                     = "DNAT"
+    enabled                  = true
+    exclude                  = false
+    in_interface             = each.value.network_id
+    ip_address               = each.value.endpoint.target
+    ip_version               = each.value.endpoint.ip_version
+    protocol                 = "tcp_udp"
+    rule_index               = each.value.rule_index
+    logging                  = false
     is_predefined            = false
     pppoe_use_base_interface = false
     setting_preference       = "manual"

opentofu/vm_150_k8s_1.tf

@@ -1,51 +0,0 @@
-module "k8s_1" {
-  source = "./modules/proxmox_vm"
-
-  name        = "k8s-1.srv.renner.uno"
-  target_node = "jupiter"
-  vmid        = 150
-  memory      = 49152
-  cores       = 8
-
-  template_name = null
-  cloud_init    = false
-  full_clone    = false
-  bios          = "ovmf"
-  machine       = "q35"
-  boot          = "order=virtio0"
-  agent_enabled = true
-
-  # Second NIC on v30iot so Cilium L2 announce can win ARP on the IoT
-  # VLAN from this node (previously only k8s-edge had v30iot, so any
-  # IoT-pool LB lease landing here was unreachable from devices).
-  extra_network_bridges = ["v30iot"]
-
-  disks = [
-    {
-      storage = "local-zfs"
-      size    = "64G"
-      format  = "raw"
-      slot    = "virtio0"
-      file_id = local.talos_image
-    }
-  ]
-
-  hostpci = [
-    {
-      mapping = "osd-ssd"
-    },
-    {
-      mapping = "x710-port1"
-    }
-  ]
-
-  tags            = "kubernetes;talos;controlplane"
-  ssh_public_keys = local.common_ssh_keys
-
-  vm_state = "running"
-
-  # Rebuild pipeline issues `-replace`; skip graceful shutdown so a
-  # non-responsive Talos guest can't hang the destroy for 30 minutes.
-  stop_on_destroy = true
-}
-

opentofu/vm_151_k8s_2.tf

@@ -1,50 +0,0 @@
-module "k8s_2" {
-  source = "./modules/proxmox_vm"
-
-  name        = "k8s-2.srv.renner.uno"
-  target_node = "saturn"
-  vmid        = 151
-  memory      = 49152
-  cores       = 8
-
-  template_name = null
-  cloud_init    = false
-  full_clone    = false
-  bios          = "ovmf"
-  machine       = "q35"
-  boot          = "order=virtio0"
-  agent_enabled = true
-
-  # Second NIC on v30iot so Cilium L2 announce can win ARP on the IoT
-  # VLAN from this node (previously only k8s-edge had v30iot, so any
-  # IoT-pool LB lease landing here was unreachable from devices).
-  extra_network_bridges = ["v30iot"]
-
-  disks = [
-    {
-      storage = "local-zfs"
-      size    = "64G"
-      format  = "raw"
-      slot    = "virtio0"
-      file_id = local.talos_image
-    }
-  ]
-
-  hostpci = [
-    {
-      mapping = "osd-ssd"
-    },
-    {
-      mapping = "x710-port1"
-    }
-  ]
-
-  tags            = "kubernetes;talos;controlplane"
-  ssh_public_keys = local.common_ssh_keys
-
-  vm_state = "running"
-
-  # Rebuild pipeline issues `-replace`; skip graceful shutdown so a
-  # non-responsive Talos guest can't hang the destroy for 30 minutes.
-  stop_on_destroy = true
-}

opentofu/vm_152_k8s_3.tf

@@ -1,50 +0,0 @@
-module "k8s_3" {
-  source = "./modules/proxmox_vm"
-
-  name        = "k8s-3.srv.renner.uno"
-  target_node = "neptune"
-  vmid        = 152
-  memory      = 49152
-  cores       = 8
-
-  template_name = null
-  cloud_init    = false
-  full_clone    = false
-  bios          = "ovmf"
-  machine       = "q35"
-  boot          = "order=virtio0"
-  agent_enabled = true
-
-  # Second NIC on v30iot so Cilium L2 announce can win ARP on the IoT
-  # VLAN from this node (previously only k8s-edge had v30iot, so any
-  # IoT-pool LB lease landing here was unreachable from devices).
-  extra_network_bridges = ["v30iot"]
-
-  disks = [
-    {
-      storage = "local-zfs"
-      size    = "64G"
-      format  = "raw"
-      slot    = "virtio0"
-      file_id = local.talos_image
-    }
-  ]
-
-  hostpci = [
-    {
-      mapping = "osd-ssd"
-    },
-    {
-      mapping = "x710-port1"
-    }
-  ]
-
-  tags            = "kubernetes;talos;controlplane"
-  ssh_public_keys = local.common_ssh_keys
-
-  vm_state = "running"
-
-  # Rebuild pipeline issues `-replace`; skip graceful shutdown so a
-  # non-responsive Talos guest can't hang the destroy for 30 minutes.
-  stop_on_destroy = true
-}

opentofu/vm_605_template.tf

@@ -1,71 +0,0 @@
-resource "proxmox_virtual_environment_vm" "ubuntu_24_04_template" {
-  name      = "ubuntu-24.04-template"
-  node_name = "saturn"
-  vm_id     = 605
-
-  started = false
-  template = true
-  on_boot = false
-
-  keyboard_layout = "en-us"
-
-  bios = "seabios"
-
-  agent {
-    enabled = true
-  }
-
-  cpu {
-    cores = 2
-    type  = "x86-64-v3"
-  }
-
-  memory {
-    dedicated = 2048
-  }
-
-  initialization {
-    datastore_id = "local-zfs"
-
-    ip_config {
-      ipv4 {
-        address = "dhcp"
-      }
-    }
-
-    user_account {
-      username = "ladmin"
-      keys = [
-        "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPzFNBP3bq2m/6JqDqod9w+FP/uxXxHMolqhq3Ojflr1 krenner@fortress"
-      ]
-    }
-  }
-
-  disk {
-    datastore_id = "local-zfs"
-    interface    = "scsi0"
-    size         = 32
-    file_format  = "raw"
-    iothread     = false
-  }
-
-  network_device {
-    bridge = "v40srv"
-    model  = "virtio"
-  }
-
-  serial_device {}
-
-  scsi_hardware = "virtio-scsi-pci"
-
-  lifecycle {
-    ignore_changes = [
-      node_name,
-      started,
-      initialization,
-      disk,
-      cpu,
-      memory
-    ]
-  }
-}