chore(decommission): sweep all references to destroyed PVE guests #265

Merged
k.renner merged 1 commits from chore/decommission-sweep into master 2026-07-16 15:46:38 +00:00
17 changed files with 0 additions and 726 deletions
-5
View File
@@ -15,11 +15,6 @@ authentik_app
authentik-app-1.srv.renner.uno ansible_user=root
authentik-app-2.srv.renner.uno ansible_user=root
[proxmox_dns_ha]
proxmox-dns-1.srv.renner.uno ansible_host=10.2.40.54 ansible_user=root
proxmox-dns-2.srv.renner.uno ansible_host=10.2.40.55 ansible_user=root
proxmox-dns-3.srv.renner.uno ansible_host=10.2.40.56 ansible_user=root
[netbird_server]
netbird.renner.uno ansible_user=root
-200
View File
@@ -1,200 +0,0 @@
---
# Chaos Testing Playbook
# Tests failover capabilities of HA components
#
# Usage:
# # Test all components
# ansible-playbook -i inventory/ plays/chaos-testing.yml
#
# # Test specific component
# ansible-playbook -i inventory/ plays/chaos-testing.yml --tags dns
# ansible-playbook -i inventory/ plays/chaos-testing.yml --tags postgres
#
# WARNING: This will temporarily disrupt services. Run during maintenance windows.
- name: Chaos Testing - Infrastructure Failover Validation
hosts: localhost
gather_facts: no
vars:
dns_vip: "10.2.40.53"
dns_primary: "10.2.40.54"
dns_backup1: "10.2.40.55"
dns_backup2: "10.2.40.56"
postgres_vip: "10.2.40.135"
postgres_primary: "authentik-db-1.srv.renner.uno"
postgres_backup: "authentik-db-2.srv.renner.uno"
test_results: []
tasks:
# =========================================================================
# DNS CLUSTER FAILOVER TEST
# =========================================================================
- name: "DNS Test - Record initial state"
tags: [dns]
block:
- name: Check which node holds DNS VIP
shell: |
for ip in {{ dns_primary }} {{ dns_backup1 }} {{ dns_backup2 }}; do
if ssh -o ConnectTimeout=2 root@$ip 'ip addr show | grep -q {{ dns_vip }}' 2>/dev/null; then
echo $ip
exit 0
fi
done
echo "NONE"
register: dns_vip_holder
changed_when: false
- name: Record DNS VIP holder
set_fact:
dns_master_before: "{{ dns_vip_holder.stdout }}"
- name: Verify DNS resolution works before test
shell: dig @{{ dns_vip }} k8s-1.srv.renner.uno +short +time=2
register: dns_before
failed_when: dns_before.rc != 0
- name: "DNS CHAOS - Stop DNS service on master ({{ dns_master_before }})"
shell: ssh root@{{ dns_master_before }} 'systemctl stop proxmox-dns'
when: dns_master_before != "NONE"
- name: Wait for VIP migration (max 10 seconds)
pause:
seconds: 10
- name: Verify DNS still resolves via VIP
shell: dig @{{ dns_vip }} k8s-1.srv.renner.uno +short +time=5
register: dns_after
failed_when: dns_after.rc != 0
ignore_errors: yes
- name: Check which node now holds VIP
shell: |
for ip in {{ dns_primary }} {{ dns_backup1 }} {{ dns_backup2 }}; do
if ssh -o ConnectTimeout=2 root@$ip 'ip addr show | grep -q {{ dns_vip }}' 2>/dev/null; then
echo $ip
exit 0
fi
done
echo "NONE"
register: dns_vip_holder_after
changed_when: false
- name: Record DNS failover result
set_fact:
test_results: "{{ test_results + [{'component': 'DNS Cluster', 'status': 'PASS' if dns_after.rc == 0 else 'FAIL', 'master_before': dns_master_before, 'master_after': dns_vip_holder_after.stdout, 'failover_time': '< 10 seconds'}] }}"
- name: "DNS RESTORE - Restart DNS service on original master"
shell: ssh root@{{ dns_master_before }} 'systemctl start proxmox-dns'
when: dns_master_before != "NONE"
- name: Wait for service to stabilize
pause:
seconds: 5
# =========================================================================
# POSTGRESQL HA FAILOVER TEST
# =========================================================================
- name: "PostgreSQL Test - Record initial state"
tags: [postgres]
block:
- name: Check PostgreSQL replication status on primary
shell: |
ssh root@{{ postgres_primary }} 'sudo -u postgres psql -t -c "SELECT pg_is_in_recovery();"' 2>/dev/null | tr -d ' '
register: pg_primary_status
changed_when: false
ignore_errors: yes
- name: Check which node holds PostgreSQL VIP
shell: |
if ssh -o ConnectTimeout=2 root@{{ postgres_primary }} 'ip addr show | grep -q {{ postgres_vip }}' 2>/dev/null; then
echo "{{ postgres_primary }}"
elif ssh -o ConnectTimeout=2 root@{{ postgres_backup }} 'ip addr show | grep -q {{ postgres_vip }}' 2>/dev/null; then
echo "{{ postgres_backup }}"
else
echo "NONE"
fi
register: pg_vip_holder
changed_when: false
- name: Record PostgreSQL VIP holder
set_fact:
pg_master_before: "{{ pg_vip_holder.stdout }}"
- name: Verify PostgreSQL connection via VIP
shell: |
ssh root@{{ postgres_primary }} 'PGPASSWORD=test psql -h {{ postgres_vip }} -U postgres -c "SELECT 1;" 2>&1' || echo "Connection test skipped"
register: pg_before
changed_when: false
ignore_errors: yes
- name: "PostgreSQL CHAOS - Stop keepalived on VIP holder"
shell: ssh root@{{ pg_master_before }} 'systemctl stop keepalived'
when: pg_master_before != "NONE"
- name: Wait for VIP migration (max 10 seconds)
pause:
seconds: 10
- name: Check which node now holds PostgreSQL VIP
shell: |
if ssh -o ConnectTimeout=2 root@{{ postgres_primary }} 'ip addr show | grep -q {{ postgres_vip }}' 2>/dev/null; then
echo "{{ postgres_primary }}"
elif ssh -o ConnectTimeout=2 root@{{ postgres_backup }} 'ip addr show | grep -q {{ postgres_vip }}' 2>/dev/null; then
echo "{{ postgres_backup }}"
else
echo "NONE"
fi
register: pg_vip_holder_after
changed_when: false
- name: Record PostgreSQL failover result
set_fact:
test_results: "{{ test_results + [{'component': 'PostgreSQL HA', 'status': 'PASS' if pg_vip_holder_after.stdout != 'NONE' and pg_vip_holder_after.stdout != pg_master_before else 'FAIL', 'master_before': pg_master_before, 'master_after': pg_vip_holder_after.stdout, 'failover_time': '< 10 seconds'}] }}"
- name: "PostgreSQL RESTORE - Restart keepalived on original master"
shell: ssh root@{{ pg_master_before }} 'systemctl start keepalived'
when: pg_master_before != "NONE"
- name: Wait for service to stabilize
pause:
seconds: 5
# =========================================================================
# TEST SUMMARY
# =========================================================================
- name: Display test results
tags: [always]
debug:
msg: |
╔══════════════════════════════════════════════════════════════╗
║ CHAOS TESTING RESULTS - {{ ansible_date_time.iso8601 | default('NOW') }}
╠══════════════════════════════════════════════════════════════╣
{% for result in test_results %}
║ {{ result.component | ljust(20) }} │ {{ result.status | ljust(6) }} │ {{ result.get('failover_time', result.get('details', 'N/A')) }}
{% endfor %}
╚══════════════════════════════════════════════════════════════╝
{% if test_results | selectattr('status', 'equalto', 'FAIL') | list | length > 0 %}
⚠️ SOME TESTS FAILED - Review results above
{% else %}
✅ ALL TESTS PASSED - HA failover working correctly
{% endif %}
- name: Save test results to file
tags: [always]
copy:
content: |
# Chaos Test Results
# Date: {{ ansible_date_time.iso8601 | default('unknown') }}
{% for result in test_results %}
## {{ result.component }}
- Status: {{ result.status }}
- Master Before: {{ result.get('master_before', 'N/A') }}
- Master After: {{ result.get('master_after', 'N/A') }}
- Failover Time: {{ result.get('failover_time', 'N/A') }}
- Details: {{ result.get('details', 'N/A') }}
{% endfor %}
dest: "/tmp/chaos-test-results-{{ ansible_date_time.date | default('latest') }}.md"
delegate_to: localhost
-72
View File
@@ -1,72 +0,0 @@
---
# Install Proxmox DNS Server in High Availability Configuration
# This playbook installs the DNS servers + keepalived for VIP failover
- name: Install Proxmox DNS Server (HA)
hosts: proxmox_dns_ha
become: yes
vars:
dns_vip: "10.2.40.53"
dns_servers:
proxmox-dns-1.srv.renner.uno:
priority: 100
state: "MASTER"
proxmox-dns-2.srv.renner.uno:
priority: 90
state: "BACKUP"
proxmox-dns-3.srv.renner.uno:
priority: 80
state: "BACKUP"
roles:
- role: proxmox-dns
- role: keepalived-vip
vars:
keepalived_vip: "{{ dns_vip }}"
keepalived_state: "{{ dns_servers[inventory_hostname].state }}"
keepalived_priority: "{{ dns_servers[inventory_hostname].priority }}"
post_tasks:
- name: Wait for DNS server to start
wait_for:
port: 53
timeout: 30
- name: Wait for keepalived to start
wait_for:
timeout: 10
- name: Check if this server has the VIP
shell: ip addr show | grep {{ dns_vip }}
register: vip_check
failed_when: false
changed_when: false
- name: Display VIP status
debug:
msg: >
{% if vip_check.rc == 0 %}
This server ({{ inventory_hostname }}) is MASTER and holds the VIP {{ dns_vip }}
{% else %}
This server ({{ inventory_hostname }}) is BACKUP
{% endif %}
- name: Test DNS resolution locally
shell: |
dig @localhost k8s-1.srv.renner.uno +short
register: dns_test
failed_when: false
changed_when: false
- name: Display DNS test result
debug:
msg: "DNS test result: {{ dns_test.stdout if dns_test.stdout else 'No result (may be expected if host does not exist yet)' }}"
- name: Install notification script
copy:
src: "{{ playbook_dir }}/../ansible/roles/keepalived-vip/files/keepalived-notify.sh"
dest: /usr/local/bin/keepalived-notify.sh
owner: root
group: root
mode: '0755'
-28
View File
@@ -1,28 +0,0 @@
---
# Install Proxmox DNS Server
# This playbook installs the custom DNS server that resolves VM/LXC hostnames
# by querying the Proxmox API
- name: Install Proxmox DNS Server
hosts: proxmox_dns
become: yes
roles:
- proxmox-dns
post_tasks:
- name: Wait for DNS server to start
wait_for:
port: 53
timeout: 30
- name: Test DNS resolution
shell: |
dig @localhost k8s-1.srv.renner.uno +short
register: dns_test
failed_when: false
changed_when: false
- name: Display DNS test result
debug:
msg: "DNS test result: {{ dns_test.stdout }}"
-8
View File
@@ -1,8 +0,0 @@
---
# Proxmox DNS Server Secrets
# Encrypt this file with: ansible-vault encrypt proxmox_dns_vault.yml
# Proxmox API token for DNS server
# Create in Proxmox: Datacenter > Permissions > API Tokens
# Format: <token-id>!<token-name> = <secret>
proxmox_dns_api_token_value: "CHANGEME-your-proxmox-api-token-here"
-7
View File
@@ -1,7 +0,0 @@
vault_rustfs_secret_key: !vault |
$ANSIBLE_VAULT;1.1;AES256
32363232626635353039653538623036386161393539353265366433306533346666616362373036
3633316536626339386236333839636538643438616531650a316539373264636438363262333633
63653737363433643833393561343138303564623635653437636164306238303539343934646361
3666343466333439630a323432666430386637663165323165363536633838303938323239313762
62396631366438303265653566366462363137353135663839303439313537666531
@@ -71,16 +71,6 @@ scrape_configs:
- target_label: type
replacement: hypervisor
- job_name: 'dns-servers'
static_configs:
- targets:
- 'proxmox-dns-1.srv.renner.uno:12345'
- 'proxmox-dns-2.srv.renner.uno:12345'
- 'proxmox-dns-3.srv.renner.uno:12345'
labels:
group: 'dns'
type: 'infrastructure'
# RustFS (S3-compatible storage)
# NOTE: RustFS uses OpenTelemetry push-model via --obs-endpoint, not Prometheus pull.
# Unlike MinIO, it doesn't expose /minio/v2/metrics/cluster endpoint.
@@ -1,22 +0,0 @@
---
# Proxmox DNS Server Role Defaults
proxmox_dns_install_dir: "/opt/proxmox-dns"
proxmox_dns_config_dir: "/etc/proxmox-dns"
# Proxmox API connection (uses existing vault variables)
proxmox_dns_api_host: "{{ pve_api_url | regex_replace('https://|http://|/api2/json', '') }}"
proxmox_dns_api_user: "{{ pve_user }}"
proxmox_dns_api_token_name: "{{ pve_token_id }}"
proxmox_dns_api_token_value: "{{ pve_token_secret }}"
# DNS server configuration
proxmox_dns_host: "0.0.0.0" # Listen on all interfaces (required for HA with VIP)
proxmox_dns_port: 53
proxmox_dns_suffix: ".srv.renner.uno"
proxmox_dns_ttl: 300
proxmox_dns_cache_ttl: 60
proxmox_dns_upstream: "10.2.40.1"
# Go binary path (for building on localhost)
go_binary_path: "/nix/store/zzvsjgylnphvhms3lgr2qlwdxmc68z66-go-1.25.5/bin/go"
@@ -1,11 +0,0 @@
---
# Handlers for Proxmox DNS Server
- name: reload systemd
systemd:
daemon_reload: yes
- name: restart proxmox-dns
systemd:
name: proxmox-dns
state: restarted
-65
View File
@@ -1,65 +0,0 @@
---
# Proxmox DNS Server Installation (Go version)
- name: Create installation directory
file:
path: "{{ proxmox_dns_install_dir }}"
state: directory
owner: root
group: root
mode: '0755'
- name: Create configuration directory
file:
path: "{{ proxmox_dns_config_dir }}"
state: directory
owner: root
group: root
mode: '0750'
- name: Build Go DNS server binary locally
delegate_to: localhost
become: no
shell: |
cd {{ playbook_dir }}/../../dns-server
export CGO_ENABLED=0
{{ go_binary_path }} build -o proxmox-dns-go -ldflags="-s -w" main.go
args:
creates: "{{ playbook_dir }}/../../dns-server/proxmox-dns-go"
run_once: true
- name: Copy DNS server binary
copy:
src: "{{ playbook_dir }}/../../dns-server/proxmox-dns-go"
dest: "{{ proxmox_dns_install_dir }}/proxmox-dns-go"
owner: root
group: root
mode: '0755'
notify: restart proxmox-dns
- name: Create configuration file
template:
src: config.yaml.j2
dest: "{{ proxmox_dns_config_dir }}/config.yaml"
owner: root
group: root
mode: '0640'
notify: restart proxmox-dns
- name: Create systemd service file
template:
src: proxmox-dns.service.j2
dest: /etc/systemd/system/proxmox-dns.service
owner: root
group: root
mode: '0644'
notify:
- reload systemd
- restart proxmox-dns
- name: Enable and start proxmox-dns service
systemd:
name: proxmox-dns
enabled: yes
state: started
daemon_reload: yes
@@ -1,19 +0,0 @@
# Proxmox DNS Server Configuration
# Managed by Ansible - Do not edit manually
proxmox:
host: "{{ proxmox_dns_api_host }}"
user: "{{ proxmox_dns_api_user }}"
token_name: "{{ proxmox_dns_api_token_name }}"
token_value: "{{ proxmox_dns_api_token_value }}"
verify_ssl: false
dns_host: "{{ proxmox_dns_host }}"
dns_port: {{ proxmox_dns_port }}
dns_suffix: "{{ proxmox_dns_suffix }}"
dns_ttl: {{ proxmox_dns_ttl }}
cache_ttl: {{ proxmox_dns_cache_ttl }}
upstream_dns: "{{ proxmox_dns_upstream }}"
@@ -1,26 +0,0 @@
[Unit]
Description=Proxmox DNS Server
After=network.target
Wants=network-online.target
[Service]
Type=simple
User=root
Group=root
WorkingDirectory={{ proxmox_dns_install_dir }}
ExecStart={{ proxmox_dns_install_dir }}/proxmox-dns-go -c {{ proxmox_dns_config_dir }}/config.yaml
Restart=always
RestartSec=10
StandardOutput=journal
StandardError=journal
# Security hardening
NoNewPrivileges=true
PrivateTmp=true
ProtectSystem=strict
ProtectHome=true
ReadWritePaths=/var/log
AmbientCapabilities=CAP_NET_BIND_SERVICE
[Install]
WantedBy=multi-user.target
@@ -92,13 +92,6 @@ spec:
- target_label: __address__
replacement: pve-exporter.monitoring.svc.cluster.local:9221
- job_name: dns-servers
static_configs:
- targets:
- proxmox-dns-1.srv.renner.uno:12345
- proxmox-dns-2.srv.renner.uno:12345
- proxmox-dns-3.srv.renner.uno:12345
# Gitea exposes /metrics anonymously on its main HTTP port. The
# `gitea_*` metrics power the gitea-overview dashboard. Gitea now runs
# in-cluster (StatefulSet in apps ns; LXC 114 retired), so scrape the
-61
View File
@@ -1,61 +0,0 @@
# Proxmox cluster-wide vzdump backup job.
#
# Historically managed via the PVE UI. Imported here on 2026-04-23 after a
# backup-induced I/O freeze cascaded into a full Rook-Ceph / K8s outage.
# Future edits happen in this file, not the UI.
#
# The bpg/proxmox `proxmox_backup_job` resource does not expose vzdump's
# `exclude` option, so the K8s cluster VMs (150-153) are omitted by using an
# explicit `vmid` include list instead of `all = true`. When adding or
# removing any backed-up VM/LXC, update `local.backed_up_vmids` below.
#
# K8s-side data protection is handled separately via Velero and CNPG backing
# up to Garage on nas-1; Flux rebuilds cluster state from git.
locals {
# Authoritative list of VMIDs covered by the nightly vzdump job.
# Excludes: 150-153 (K8s cluster), 605 (template — PVE skips templates
# automatically, listing it here would error).
backed_up_vmids = concat(
# VMs
[
"102", # fortress
"111", # paperless
],
# LXCs
[
"118", # proxmox-dns-1 (jupiter)
"218", # proxmox-dns-2 (saturn)
"318", # proxmox-dns-3 (neptune)
],
)
}
import {
to = proxmox_backup_job.daily_pbs2
id = "backup-51b113b6-676a"
}
resource "proxmox_backup_job" "daily_pbs2" {
id = "backup-51b113b6-676a"
schedule = "10:15"
storage = "pbs2"
vmid = local.backed_up_vmids
mode = "snapshot"
compress = "zstd"
enabled = true
mailnotification = "always"
mailto = ["konstantin@renner.uno"]
notes_template = "{{guestname}}"
fleecing = {
enabled = false
}
prune_backups = {
"keep-all" = "1"
}
}
-84
View File
@@ -1,84 +0,0 @@
# Proxmox DNS Server - High Availability Setup
# Custom DNS servers that resolve VM/LXC hostnames by querying Proxmox API
# Uses keepalived for VIP (10.2.40.53) and automatic failover
locals {
dns_servers = [
{
vmid = 118
hostname = "proxmox-dns-1.srv.renner.uno"
node = "jupiter"
ip = "10.2.40.54"
priority = 100 # MASTER
},
{
vmid = 218
hostname = "proxmox-dns-2.srv.renner.uno"
node = "saturn"
ip = "10.2.40.55"
priority = 90 # BACKUP
},
{
vmid = 318
hostname = "proxmox-dns-3.srv.renner.uno"
node = "neptune"
ip = "10.2.40.56"
priority = 80 # BACKUP
}
]
dns_vip = "10.2.40.53" # Virtual IP managed by keepalived
}
module "proxmox_dns" {
source = "./modules/proxmox_lxc"
for_each = { for idx, server in local.dns_servers : tostring(server.vmid) => server }
hostname = each.value.hostname
vmid = each.value.vmid
target_node = each.value.node
memory = 512
cores = 1
unprivileged = true
network_bridge = "v40srv"
ip_address = each.value.ip # Each server has its own IP
storage = "local"
disk_size = "4G"
ostemplate = "local:vztmpl/debian-12-standard_12.12-1_amd64.tar.zst"
ssh_public_keys = local.common_ssh_keys
tags = "dns;infrastructure;ha"
onboot = true
start = true
features = {
nesting = false
fuse = false
keyctl = false
mknod = false
}
# No CNAMEs needed - these ARE the DNS servers
ingress_cnames = []
cnames = []
}
output "proxmox_dns_vip" {
value = local.dns_vip
description = "Virtual IP address for Proxmox DNS (keepalived VIP)"
}
output "proxmox_dns_servers" {
value = {
for vmid, server in module.proxmox_dns : vmid => {
hostname = server.id
ip = local.dns_servers[index(local.dns_servers.*.vmid, tonumber(vmid))].ip
}
}
description = "Individual DNS server instances"
}
-45
View File
@@ -1,45 +0,0 @@
module "fortress_renner_uno" {
source = "./modules/proxmox_vm"
name = "fortress.priv.renner.uno"
target_node = "neptune"
vmid = 102
memory = 8192
cores = 4
mac_address = "BC:24:11:33:6D:DA"
ip_address = "10.2.50.153"
gateway = "10.2.50.1"
network_bridge = "v50pri"
disks = [
{
storage = "local-zfs"
size = "64G"
slot = "scsi0"
discard = false
}
]
tags = "noansible"
hastate = "stopped"
vm_state = "stopped"
onboot = false
cloud_init = false
full_clone = false
template_name = null
cnames = ["fortress.renner.uno"]
ssh_public_keys = <<-EOT
ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEArD1...
EOT
}
moved {
from = proxmox_vm_qemu.fortress_renner_uno
to = module.fortress_renner_uno.proxmox_vm_qemu.this
}
moved {
from = unifi_client.fortress
to = module.fortress_renner_uno.unifi_client.this
}
-56
View File
@@ -1,56 +0,0 @@
moved {
from = proxmox_vm_qemu.paperless_srv_renner_uno
to = module.paperless_srv_renner_uno.proxmox_vm_qemu.this
}
moved {
from = unifi_client.paperless
to = module.paperless_srv_renner_uno.unifi_client.this
}
module "paperless_srv_renner_uno" {
source = "./modules/proxmox_vm"
name = "paperless.srv.renner.uno"
target_node = "saturn"
vmid = 111
memory = 8192
cores = 8
mac_address = "bc:24:11:b3:2f:97"
ip_address = "10.2.40.19"
cloud_init_slot = "ide2"
disks = [
{
storage = "local-zfs"
size = "63692M"
format = "raw"
slot = "virtio0"
},
{
storage = "local-zfs"
size = "100G"
format = "raw"
slot = "virtio1"
}
]
tags = "docker"
ssh_public_keys = local.common_ssh_keys
# CNAMEs retired: K8s apps/paperless Ingress owns both hostnames.
ingress_cnames = []
# Match existing state
template_name = null
full_clone = false
bios = "ovmf"
boot = " "
automatic_reboot = true
tablet = true
vm_state = "stopped"
hastate = "stopped"
onboot = false
}