feat(edge-host-iso): trust the SSH user CA

The build fetches the CA public key from the ssh-ca role URL and renders the
role drop-in, revocation list and root principal into the live system;
edge-install copies them to the installed system and checks sshd -T uses
the CA. Owner certificates log in as root from the first boot, with the
break-glass key as fallback.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

.woodpecker/deploy-claude-workstation.yml

@@ -1,33 +0,0 @@
-when:
-  # Deploy only on main (merge) + manual — never auto-deploy from a branch.
-  - event: push
-    branch: main
-    path:
-      - "ansible/roles/claude-workstation/**"
-      - "ansible/plays/install-claude-workstation.yml"
-      - ".woodpecker/deploy-claude-workstation.yml"
-  - event: deployment
-    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "deploy-claude-workstation"'
-
-timeout: 30
-
-steps:
-  - name: deploy-claude-workstation
-    image: gitea.renner.uno/gitea_admin/ci-ansible:2026-04-29
-    environment:
-      ANSIBLE_FORCE_COLOR: "true"
-      ANSIBLE_HOST_KEY_CHECKING: "false"
-      ANSIBLE_ROLES_PATH: "../roles:./roles"
-      SSH_PRIVATE_KEY:
-        from_secret: ssh_private_key
-      VAULT_PASS:
-        from_secret: vault_pass
-    commands:
-      - mkdir -p ~/.ssh
-      - echo "$SSH_PRIVATE_KEY" | base64 -d > ~/.ssh/id_ed25519
-      - chmod 600 ~/.ssh/id_ed25519
-      - printf "Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile /dev/null\n" > ~/.ssh/config
-      - cd ansible
-      - echo "$VAULT_PASS" > .vault_pass && chmod 600 .vault_pass
-      - export ANSIBLE_VAULT_PASSWORD_FILE=.vault_pass
-      - ansible-playbook -i inventory/ plays/install-claude-workstation.yml -v

.woodpecker/deploy-edge-mqtt-bridge.yml

@@ -1,36 +0,0 @@
-when:
-  # Deploy only on main (merge) + manual — never auto-deploy from a branch.
-  - event: push
-    branch: main
-    path:
-      - "ansible/roles/mqtt-bridge/**"
-      - "ansible/plays/edge-mqtt-bridge.yml"
-      - "ansible/inventory/group_vars/edge_host/**"
-      - ".woodpecker/deploy-edge-mqtt-bridge.yml"
-  - event: deployment
-    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "deploy-edge-mqtt-bridge"'
-
-timeout: 30
-
-steps:
-  - name: deploy-edge-mqtt-bridge
-    image: gitea.renner.uno/gitea_admin/ci-ansible:2026-04-29
-    environment:
-      ANSIBLE_FORCE_COLOR: "true"
-      ANSIBLE_HOST_KEY_CHECKING: "false"
-      ANSIBLE_ROLES_PATH: "../roles:./roles"
-      SSH_PRIVATE_KEY:
-        from_secret: ssh_private_key
-      VAULT_PASS:
-        from_secret: vault_pass
-    commands:
-      - mkdir -p ~/.ssh
-      - echo "$SSH_PRIVATE_KEY" | base64 -d > ~/.ssh/id_ed25519
-      - chmod 600 ~/.ssh/id_ed25519
-      - printf "Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile /dev/null\n" > ~/.ssh/config
-      - cd ansible
-      - echo "$VAULT_PASS" > .vault_pass && chmod 600 .vault_pass
-      - export ANSIBLE_VAULT_PASSWORD_FILE=.vault_pass
-      # Local MQTT bridge on the edge hosts only (REN-1307); restarts only the
-      # local mosquitto on edge-1, never the cluster broker.
-      - ansible-playbook -i inventory/ plays/edge-mqtt-bridge.yml -v

.woodpecker/deploy-infrastructure.yml

@@ -109,8 +109,6 @@ steps:
         from_secret: hetzner_s3_access_key
       TF_VAR_hetzner_s3_secret_key:
         from_secret: hetzner_s3_secret_key
-      SSH_PRIVATE_KEY:
-        from_secret: ssh_private_key
       PROXMOX_TOKEN_ID:
         from_secret: proxmox_token_id
       PROXMOX_TOKEN_SECRET:
@@ -121,11 +119,6 @@ steps:
       ANSIBLE_ROLES_PATH: "../roles:./roles"
       ANSIBLE_FORCE_COLOR: "true"
     commands:
-      - mkdir -p ~/.ssh
-      - echo "$SSH_PRIVATE_KEY" | base64 -d > ~/.ssh/id_ed25519
-      - chmod 600 ~/.ssh/id_ed25519
-      - export TF_VAR_ssh_private_key="$(cat ~/.ssh/id_ed25519)"
-      - printf "Host *\n\tStrictHostKeyChecking no\n" > ~/.ssh/config
       - echo "$VAULT_PASS" > /tmp/.vault_pass && chmod 600 /tmp/.vault_pass
       - export ANSIBLE_VAULT_PASSWORD_FILE=/tmp/.vault_pass
       - tofu init -upgrade -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY"

.woodpecker/deploy-netbird-client.yml

@@ -1,35 +0,0 @@
-when:
-  # Deploy only on main (merge) + manual — never auto-deploy from a branch.
-  - event: push
-    branch: main
-    path:
-      - "ansible/roles/netbird-client/**"
-      - "ansible/plays/install-netbird-client.yml"
-      - "ansible/inventory/group_vars/claude_workstation/**"
-      - ".woodpecker/deploy-netbird-client.yml"
-  - event: deployment
-    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "deploy-netbird-client"'
-
-timeout: 30
-
-steps:
-  - name: deploy-netbird-client
-    image: gitea.renner.uno/gitea_admin/ci-ansible:2026-04-29
-    environment:
-      ANSIBLE_FORCE_COLOR: "true"
-      ANSIBLE_HOST_KEY_CHECKING: "false"
-      ANSIBLE_ROLES_PATH: "../roles:./roles"
-      SSH_PRIVATE_KEY:
-        from_secret: ssh_private_key
-      VAULT_PASS:
-        from_secret: vault_pass
-    commands:
-      - mkdir -p ~/.ssh
-      - echo "$SSH_PRIVATE_KEY" | base64 -d > ~/.ssh/id_ed25519
-      - chmod 600 ~/.ssh/id_ed25519
-      - printf "Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile /dev/null\n" > ~/.ssh/config
-      - cd ansible
-      - echo "$VAULT_PASS" > .vault_pass && chmod 600 .vault_pass
-      - export ANSIBLE_VAULT_PASSWORD_FILE=.vault_pass
-      # Enrol only the off-site Claude workstation into the Netbird mesh.
-      - ansible-playbook -i inventory/ plays/install-netbird-client.yml --limit claude_workstation -v

.woodpecker/deploy-netbird-proxy.yml

@@ -1,54 +0,0 @@
-when:
-  # Deploy only on main (merge) + manual — never auto-deploy from a branch.
-  - event: push
-    branch: main
-    path:
-      - "ansible/roles/netbird-proxy/**"
-      # install-netbird-proxy.yml also applies the netbird-proxy-watchdog role,
-      # so changes to it must redeploy this host too.
-      - "ansible/roles/netbird-proxy-watchdog/**"
-      # install-netbird-proxy.yml also applies the edge bouncer.
-      - "ansible/roles/crowdsec-edge-bouncer/**"
-      - "ansible/plays/install-netbird-proxy.yml"
-      # The variables that steer those roles, not just the roles themselves.
-      # Without these two, a change that only touches inventory never deploys
-      # and nothing says so: arming the bouncer lives in host_vars, and its
-      # LAPI key lives in group_vars, so both would sit in git looking applied
-      # while the hosts kept running the previous configuration.
-      #
-      # pve_hosts.yml is deliberately NOT listed. It is the single inventory
-      # file for the whole fleet, so listing it would redeploy the proxies for
-      # any unrelated host edit. The cost is that a change to [netbird_proxy]
-      # group membership needs a manual deployment run.
-      - "ansible/inventory/host_vars/proxy-*.yml"
-      - "ansible/inventory/group_vars/netbird_proxy/**"
-      - ".woodpecker/deploy-netbird-proxy.yml"
-  - event: deployment
-    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "deploy-netbird-proxy"'
-
-timeout: 30
-
-steps:
-  - name: deploy-netbird-proxy
-    image: gitea.renner.uno/gitea_admin/ci-ansible:2026-04-29
-    environment:
-      ANSIBLE_FORCE_COLOR: "true"
-      ANSIBLE_HOST_KEY_CHECKING: "false"
-      ANSIBLE_ROLES_PATH: "../roles:./roles"
-      SSH_PRIVATE_KEY:
-        from_secret: ssh_private_key
-      VAULT_PASS:
-        from_secret: vault_pass
-      PROXMOX_TOKEN_ID:
-        from_secret: proxmox_token_id
-      PROXMOX_TOKEN_SECRET:
-        from_secret: proxmox_token_secret
-    commands:
-      - mkdir -p ~/.ssh
-      - echo "$SSH_PRIVATE_KEY" | base64 -d > ~/.ssh/id_ed25519
-      - chmod 600 ~/.ssh/id_ed25519
-      - printf "Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile /dev/null\n" > ~/.ssh/config
-      - cd ansible
-      - echo "$VAULT_PASS" > .vault_pass && chmod 600 .vault_pass
-      - export ANSIBLE_VAULT_PASSWORD_FILE=.vault_pass
-      - ansible-playbook -i inventory/ plays/install-netbird-proxy.yml -v

.woodpecker/deploy-netbird-server.yml

@@ -1,37 +0,0 @@
-when:
-  # Deploy only on main (merge) + manual — never auto-deploy from a branch.
-  - event: push
-    branch: main
-    path:
-      - "ansible/roles/netbird-server/**"
-      - "ansible/plays/install-netbird-server.yml"
-      - ".woodpecker/deploy-netbird-server.yml"
-  - event: deployment
-    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "deploy-netbird-server"'
-
-timeout: 30
-
-steps:
-  - name: deploy-netbird-server
-    image: gitea.renner.uno/gitea_admin/ci-ansible:2026-04-29
-    environment:
-      ANSIBLE_FORCE_COLOR: "true"
-      ANSIBLE_HOST_KEY_CHECKING: "false"
-      ANSIBLE_ROLES_PATH: "../roles:./roles"
-      SSH_PRIVATE_KEY:
-        from_secret: ssh_private_key
-      VAULT_PASS:
-        from_secret: vault_pass
-      PROXMOX_TOKEN_ID:
-        from_secret: proxmox_token_id
-      PROXMOX_TOKEN_SECRET:
-        from_secret: proxmox_token_secret
-    commands:
-      - mkdir -p ~/.ssh
-      - echo "$SSH_PRIVATE_KEY" | base64 -d > ~/.ssh/id_ed25519
-      - chmod 600 ~/.ssh/id_ed25519
-      - printf "Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile /dev/null\n" > ~/.ssh/config
-      - cd ansible
-      - echo "$VAULT_PASS" > .vault_pass && chmod 600 .vault_pass
-      - export ANSIBLE_VAULT_PASSWORD_FILE=.vault_pass
-      - ansible-playbook -i inventory/ plays/install-netbird-server.yml -v

.woodpecker/deploy-ssh-ca.yml

@@ -1,37 +0,0 @@
-when:
-  # Deploy only on main (merge) + manual — never auto-deploy from a branch.
-  - event: push
-    branch: main
-    path:
-      - "ansible/roles/ssh-ca/**"
-      - "ansible/plays/configure-ssh-ca.yml"
-      - "ansible/inventory/pve_hosts.yml"
-      - "ansible/inventory/group_vars/all/**"
-      - "ansible/inventory/group_vars/netbird_proxy/vars.yml"
-      - "ansible/inventory/host_vars/netbird.renner.uno.yml"
-      - ".woodpecker/deploy-ssh-ca.yml"
-  - event: deployment
-    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "deploy-ssh-ca"'
-
-timeout: 30
-
-steps:
-  - name: deploy-ssh-ca
-    image: gitea.renner.uno/gitea_admin/ci-ansible:2026-04-29
-    environment:
-      ANSIBLE_FORCE_COLOR: "true"
-      ANSIBLE_HOST_KEY_CHECKING: "false"
-      ANSIBLE_ROLES_PATH: "../roles:./roles"
-      SSH_PRIVATE_KEY:
-        from_secret: ssh_private_key
-      VAULT_PASS:
-        from_secret: vault_pass
-    commands:
-      - mkdir -p ~/.ssh
-      - echo "$SSH_PRIVATE_KEY" | base64 -d > ~/.ssh/id_ed25519
-      - chmod 600 ~/.ssh/id_ed25519
-      - printf "Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile /dev/null\n" > ~/.ssh/config
-      - cd ansible
-      - echo "$VAULT_PASS" > .vault_pass && chmod 600 .vault_pass
-      - export ANSIBLE_VAULT_PASSWORD_FILE=.vault_pass
-      - ansible-playbook -i inventory/ plays/configure-ssh-ca.yml -v

.woodpecker/deploy-uptime-kuma.yml

@@ -1,13 +1,4 @@
 when:
-  # Deploy only on main (merge) + manual — never auto-deploy from a branch.
-  - event: push
-    branch: main
-    path:
-      - "ansible/roles/uptime-kuma/**"
-      - "ansible/plays/install-uptime-kuma.yml"
-      - ".woodpecker/deploy-uptime-kuma.yml"
-  - event: deployment
-    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "deploy-uptime-kuma"'
   - event: cron
     cron: uptime-kuma-token-renew
 
@@ -21,27 +12,3 @@ steps:
         from_secret: openbao_uptime_kuma_token
     commands:
       - 'curl -sSf -X POST -H "X-Vault-Token: $VAULT_TOKEN" https://openbao.renner.uno/v1/auth/token/renew-self -o /dev/null'
-
-  - name: deploy-uptime-kuma
-    image: gitea.renner.uno/gitea_admin/ci-ansible:2026-10-18
-    when:
-      - event: [push, deployment]
-    environment:
-      ANSIBLE_FORCE_COLOR: "true"
-      ANSIBLE_HOST_KEY_CHECKING: "false"
-      ANSIBLE_ROLES_PATH: "../roles:./roles"
-      SSH_PRIVATE_KEY:
-        from_secret: ssh_private_key
-      VAULT_PASS:
-        from_secret: vault_pass
-      VAULT_TOKEN:
-        from_secret: openbao_uptime_kuma_token
-    commands:
-      - mkdir -p ~/.ssh
-      - echo "$SSH_PRIVATE_KEY" | base64 -d > ~/.ssh/id_ed25519
-      - chmod 600 ~/.ssh/id_ed25519
-      - printf "Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile /dev/null\n" > ~/.ssh/config
-      - cd ansible
-      - echo "$VAULT_PASS" > .vault_pass && chmod 600 .vault_pass
-      - export ANSIBLE_VAULT_PASSWORD_FILE=.vault_pass
-      - ansible-playbook -i inventory/ plays/install-uptime-kuma.yml -v

.woodpecker/update-os.yml

@@ -1,28 +0,0 @@
-when:
-  # Manual only: reboots edge-1 (Z-Wave, heating CAN, sole NTP server) and the
-  # NetBird control plane. Run inside the 02:00-05:00 Europe/Berlin window.
-  - event: deployment
-    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "update-os"'
-
-timeout: 120
-
-steps:
-  - name: update-os
-    image: gitea.renner.uno/gitea_admin/ci-ansible:2026-04-29
-    environment:
-      ANSIBLE_FORCE_COLOR: "true"
-      ANSIBLE_HOST_KEY_CHECKING: "false"
-      ANSIBLE_ROLES_PATH: "../roles:./roles"
-      SSH_PRIVATE_KEY:
-        from_secret: ssh_private_key
-      VAULT_PASS:
-        from_secret: vault_pass
-    commands:
-      - mkdir -p ~/.ssh
-      - echo "$SSH_PRIVATE_KEY" | base64 -d > ~/.ssh/id_ed25519
-      - chmod 600 ~/.ssh/id_ed25519
-      - printf "Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile /dev/null\n" > ~/.ssh/config
-      - cd ansible
-      - echo "$VAULT_PASS" > .vault_pass && chmod 600 .vault_pass
-      - export ANSIBLE_VAULT_PASSWORD_FILE=.vault_pass
-      - ansible-playbook -i inventory/ plays/update-os.yml -v

ansible/ansible.cfg

@@ -206,7 +206,7 @@ inventory=~/source/homelab/ansible/inventory
 ;poll_interval=15
 
 # (path) Option for connections using a certificate or key file to authenticate, rather than an agent or passwords, you can set the default value here to avoid re-specifying --private-key with every invocation.
-private_key_file=~/.ssh/id_ed25519
+;private_key_file=
 
 # (boolean) By default, imported roles publish their variables to the play and other roles, this setting can avoid that.
 # This was introduced as a way to reset role variables to default values if a role is used more than once in a playbook.

ansible/docs/edge-host.md

@@ -245,8 +245,8 @@ its own server key, nothing else. On a push it writes
 `/etc/mosquitto/bridge.d/edge-1.conf`, deletes `bridge.d/legacy.conf` and
 restarts the local mosquitto. The controller sets a new password whenever the
 bridge reports "not connected" (also after a reinstall), so no play run is
-needed for a rotation. `.woodpecker/deploy-edge-mqtt-bridge.yml` runs
-`plays/edge-mqtt-bridge.yml` on merge. Diagnosis: `journalctl -u
+needed for a rotation. `plays/edge-mqtt-bridge.yml` is run by hand from a
+workstation. Diagnosis: `journalctl -u
 edge-mqtt-cred`, and `mosquitto_sub -t '$SYS/broker/connection/edge-1/state'`
 (retained `1` = connected).
 

ansible/inventory/group_vars/all/vars.yml

@@ -5,37 +5,10 @@ pve_token_id: "ansible"
 ansible_user: "ladmin"
 ansible_python_interpreter: /usr/bin/python3
 
-# The only plain keys the admin logins accept on [ssh_keys_managed] hosts
+# The only plain key the admin logins accept on [ssh_keys_managed] hosts
 # (ssh-ca role, tasks/authorized_keys.yml). People log in with OpenBao SSH CA
-# certificates; these two are for when that path is unavailable.
+# certificates; this one is for when that path is unavailable.
 #   breakglass-neo: PIV key (slot 9a, PIN) on the offline YubiKey NEO.
 #     ssh -I /usr/lib/opensc-pkcs11.so root@<host>
-#   woodpecker-ci: the CI deploy key, from ssh_ci_key_from where set.
 ssh_admin_authorized_keys:
   - key: "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBKvSl9LUP8u3S2Sq5YTWpW0Jxbue8cJhD7/LMFQbEsa3xo145HVR7pBzRxW71MUy+jgrgmpSlDhIb606Hswhvoo= breakglass-neo-6532348"
-  - key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICyNnX1I4W9jyDvCJ1DJauHSmbo1ZXSyBxVhSAwMWgen woodpecker-ci@homelab"
-    options: "{{ ('from=\"' ~ ssh_ci_key_from ~ '\"') if ssh_ci_key_from else '' }}"
-
-# Source the CI key is accepted from. LAN hosts see the pod address; hosts on
-# the internet see the home WAN address (NAT, dynamic) and set this empty in
-# their group/host vars.
-ssh_ci_key_from: "{{ ci_ssh_from_cidrs }}"
-
-# Network-of-origin restriction for the woodpecker-ci@homelab CI key
-# (issue #184). Used as the authorized_keys `from=` option so the CI key only
-# authenticates from within the cluster's pod network — NOT from any
-# host-pinned IP.
-#
-# The in-cluster Woodpecker step pods SSH to fleet hosts using their POD IP as
-# source, not a node IP: Cilium runs `ipv4-native-routing-cidr=10.0.0.0/8` +
-# `ipv6-native-routing-cidr=fd00::/8` with masquerade enabled, so pod egress to
-# any 10.x / fd.. LAN destination is NOT SNAT'd, and each node's PodCIDR is
-# advertised to the UDM over BGP (kubernetes/infrastructure/post-crds/
-# cilium-bgp.yaml) so return traffic L3-routes back to the pod. Hosts therefore
-# see the real pod source IP.
-#
-# The cluster pod subnets (kubernetes/talos/patches/cluster.yaml podSubnets)
-# are the stable boundary — they survive node DHCP-lease churn and pod
-# rescheduling across k8s-1/2/3, unlike pinning any node's current IP.
-# v6 is included because some SSH targets (edge IoT hosts) are IPv6-only.
-ci_ssh_from_cidrs: '10.244.0.0/16,fd10:244::/48'

ansible/inventory/group_vars/netbird_proxy/vars.yml

@@ -1,4 +1,4 @@
 ---
-# CI reaches these internet hosts NATed from the home WAN address, never from
-# the pod network, so the CI key carries no from= restriction here.
-ssh_ci_key_from: ""
+# The host NetBird client (wt0) must not take over DNS: resolv.conf is pinned to
+# the systemd-resolved stub by the netbird-proxy role.
+netbird_disable_dns: true

ansible/inventory/host_vars/netbird.renner.uno.yml

@@ -2,7 +2,3 @@
 # NetBird server - VPS not managed by Proxmox
 # Enable Docker monitoring (cAdvisor) for container metrics
 docker_monitoring_enabled: true
-
-# CI reaches this internet host NATed from the home WAN address, never from
-# the pod network, so the CI key carries no from= restriction here.
-ssh_ci_key_from: ""

ansible/inventory/pve_hosts.yml

@@ -13,11 +13,11 @@
 # Their hosts no longer resolve, except the three that resolve to Talos nodes.
 
 [netbird_server]
-netbird.renner.uno ansible_user=root
+netbird.renner.uno ansible_host=netbird.nb.renner.uno ansible_user=root
 
 [netbird_proxy]
-proxy-1.renner.uno ansible_user=root
-proxy-2.renner.uno ansible_user=root
+proxy-1.renner.uno ansible_host=proxy-1.nb.renner.uno ansible_user=root
+proxy-2.renner.uno ansible_host=proxy-2.nb.renner.uno ansible_user=root
 
 # Every bare-metal edge thin client. Gets the host-generic roles only:
 # flash-wear handling, the log shipper, the MQTT bridge, the VLAN

ansible/plays/configure-ssh-ca.yml

@@ -15,7 +15,7 @@
   roles:
     - ssh-ca
 
-- name: Reduce authorized_keys to break-glass and CI
+- name: Reduce authorized_keys to the break-glass key
   hosts: ssh_keys_managed
   become: true
   gather_facts: false

ansible/plays/edge-mqtt-bridge.yml

@@ -1,8 +1,6 @@
 ---
-# Local MQTT bridge and its credential receiver only (REN-1307). Run by
-# .woodpecker/deploy-edge-mqtt-bridge.yml on merge, so a change to this role
-# reaches edge-1 without a play run by hand. The full base play
-# (install-edge-host.yml) carries the same role.
+# Local MQTT bridge and its credential receiver only (REN-1307). Run from a
+# workstation. The full base play (install-edge-host.yml) carries the same role.
 - name: Converge the edge MQTT bridge
   hosts: edge_host
   become: true

ansible/plays/install-netbird-client.yml

@@ -15,17 +15,20 @@
 - name: Install NetBird client on hosts with setup key defined
   hosts: all
   become: true
-  gather_facts: true
+  # One host at a time: from a workstation every new SSH session asks for the
+  # YubiKey PIN and a touch, and parallel hosts would prompt at once.
+  serial: 1
+  # Facts only after the skip: gathering them first opens an SSH session (a
+  # YubiKey touch from a workstation) to every host, including skipped ones.
+  gather_facts: false
 
   pre_tasks:
-    - name: Check if netbird_setup_key is defined
-      ansible.builtin.debug:
-        msg: "NetBird setup key is defined for {{ inventory_hostname }}"
-      when: netbird_setup_key is defined
-
     - name: Skip hosts without netbird_setup_key
       ansible.builtin.meta: end_host
       when: netbird_setup_key is not defined
 
+    - name: Gather facts
+      ansible.builtin.setup:
+
   roles:
     - netbird-client

ansible/roles/netbird-client/defaults/main.yml

@@ -6,3 +6,13 @@ netbird_management_url: "https://netbird.renner.uno"
 
 # Setup key - should be defined in group_vars or host_vars
 # netbird_setup_key: ""
+
+# Release installed on hosts without a NetBird package repository (Arch).
+# Debian/Ubuntu follow the apt repository instead.
+netbird_client_version: "0.75.0"
+netbird_release_url: "https://github.com/netbirdio/netbird/releases/download/v{{ netbird_client_version }}"
+# On the controller; the archive is copied from there to the hosts.
+netbird_download_dir: "{{ lookup('ansible.builtin.env', 'HOME') }}/.cache/ansible/netbird"
+
+# Leave the host's DNS configuration alone (hosts with a pinned resolv.conf).
+netbird_disable_dns: false

ansible/roles/netbird-client/tasks/main.yml

@@ -2,32 +2,82 @@
 # NetBird Client Installation Role
 # Installs and configures NetBird client on hosts with netbird_setup_key defined
 
-- name: Install required packages
-  ansible.builtin.apt:
-    name:
-      - curl
-      - ca-certificates
-      - gnupg
-    state: present
-    update_cache: true
+- name: Install NetBird from the NetBird apt repository
+  when: ansible_facts['os_family'] == 'Debian'
+  block:
+    - name: Install required packages
+      ansible.builtin.apt:
+        name:
+          - curl
+          - ca-certificates
+          - gnupg
+        state: present
+        update_cache: true
 
-- name: Add NetBird repository key
-  ansible.builtin.shell:
-    cmd: |
-      curl -sSL https://pkgs.netbird.io/debian/public.key | gpg --dearmor -o /usr/share/keyrings/netbird-archive-keyring.gpg
-    creates: /usr/share/keyrings/netbird-archive-keyring.gpg
+    - name: Add NetBird repository key
+      ansible.builtin.shell:
+        cmd: |
+          curl -sSL https://pkgs.netbird.io/debian/public.key | gpg --dearmor -o /usr/share/keyrings/netbird-archive-keyring.gpg
+        creates: /usr/share/keyrings/netbird-archive-keyring.gpg
 
-- name: Add NetBird repository
-  ansible.builtin.apt_repository:
-    repo: "deb [signed-by=/usr/share/keyrings/netbird-archive-keyring.gpg] https://pkgs.netbird.io/debian stable main"
-    state: present
-    filename: netbird
+    - name: Add NetBird repository
+      ansible.builtin.apt_repository:
+        repo: "deb [signed-by=/usr/share/keyrings/netbird-archive-keyring.gpg] https://pkgs.netbird.io/debian stable main"
+        state: present
+        filename: netbird
 
-- name: Install NetBird client
-  ansible.builtin.apt:
+    - name: Install NetBird client
+      ansible.builtin.apt:
+        name: netbird
+        state: present
+        update_cache: true
+
+# Arch has no netbird package in its repositories; the release binary is
+# verified against the release's checksums file. It is downloaded on the
+# controller and copied over: github.com has no IPv6 address, and proxy-1 is
+# IPv6-only.
+- name: Install NetBird from the release tarball
+  when: ansible_facts['os_family'] == 'Archlinux'
+  block:
+    - name: Create the controller download directory
+      ansible.builtin.file:
+        path: "{{ netbird_download_dir }}"
+        state: directory
+        mode: "0755"
+      delegate_to: localhost
+      become: false
+      run_once: true
+
+    - name: Download the NetBird release on the controller
+      ansible.builtin.get_url:
+        url: "{{ netbird_release_url }}/netbird_{{ netbird_client_version }}_linux_amd64.tar.gz"
+        dest: "{{ netbird_download_dir }}/netbird_{{ netbird_client_version }}_linux_amd64.tar.gz"
+        checksum: "sha256:{{ netbird_release_url }}/netbird_{{ netbird_client_version }}_checksums.txt"
+        mode: "0644"
+      delegate_to: localhost
+      become: false
+      run_once: true
+
+    - name: Install the NetBird binary
+      ansible.builtin.unarchive:
+        src: "{{ netbird_download_dir }}/netbird_{{ netbird_client_version }}_linux_amd64.tar.gz"
+        dest: /usr/local/bin
+        include:
+          - netbird
+        owner: root
+        group: root
+        mode: "0755"
+
+    - name: Install the NetBird service
+      ansible.builtin.command: netbird service install
+      args:
+        creates: /etc/systemd/system/netbird.service
+
+- name: Enable NetBird service
+  ansible.builtin.service:
     name: netbird
-    state: present
-    update_cache: true
+    state: started
+    enabled: true
 
 - name: Check if NetBird is already connected
   ansible.builtin.command: netbird status
@@ -35,20 +85,17 @@
   changed_when: false
   failed_when: false
 
+# "Disconnected" contains "Connected", so match the management line.
 - name: Connect NetBird to management server
   ansible.builtin.command: >
     netbird up
     --management-url {{ netbird_management_url }}
     --setup-key {{ netbird_setup_key }}
-  when: "'Connected' not in netbird_status.stdout"
+    {{ '--disable-dns' if netbird_disable_dns else '' }}
+  when: "'Management: Connected' not in netbird_status.stdout"
   register: netbird_connect
   changed_when: netbird_connect.rc == 0
-
-- name: Enable NetBird service
-  ansible.builtin.service:
-    name: netbird
-    state: started
-    enabled: true
+  no_log: true
 
 - name: Get NetBird status
   ansible.builtin.command: netbird status

ansible/roles/netbird-server/tasks/main.yml

@@ -136,23 +136,13 @@
     - /opt/netbird/proxy
     - /opt/netbird/proxy-state
 
-# The management store has no cluster-side backup. Pulling one put a root
-# key on this host that the cluster held, and this server's own state is
-# its own business to protect rather than the cluster's. Dropping the two
-# tasks that placed them would only stop ansible managing the key and the
-# script — the pair below revokes them instead, and can go once the role
-# has run against every host that ever carried them.
+# The management store has no cluster-side backup; the cluster's backup
+# key is gone with the managed authorized_keys (ssh-ca role).
 - name: Remove the backup dump script
   ansible.builtin.file:
     path: /usr/local/bin/netbird-backup-dump
     state: absent
 
-- name: Revoke the cluster backup key
-  ansible.posix.authorized_key:
-    user: root
-    key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFeW/L9bHMTSa5/Qb0ClppCf4Z0hL39JwzI6tBmKd2FG netbird-backup@homelab-cluster"
-    state: absent
-
 - name: Wait for management container to be ready
   ansible.builtin.pause:
     seconds: 10

images/edge-host-iso/README.md

@@ -12,7 +12,10 @@ The image is the official archiso **releng** profile plus:
 - `edge-install` in the live environment — partitions the NVMe
   (ESP + ext4 root, systemd-boot), pacstraps the base system incl.
   docker/python, configures DHCP via systemd-networkd, enables sshd and
-  drops the common homelab SSH keys for root (key-only, no root password)
+  sets up root login (key-only, no root password) with owner certificates
+  from the OpenBao SSH CA, fetched at build time from the `ssh-ca` role's
+  URL, and the break-glass key (YubiKey NEO, PIV) as fallback:
+  `ssh -I /usr/lib/opensc-pkcs11.so root@<host>`
 - the extra disk tools the installer needs
 
 Division of labour: the ISO produces an **ansible-ready base system**;
@@ -30,7 +33,8 @@ sudo ./build.sh          # ISO lands in ./out/
 
 The profile is assembled at build time from `/usr/share/archiso/configs/releng`
 plus the overlay in `airootfs/` — the repo carries only the delta, so
-archiso updates flow in automatically.
+archiso updates flow in automatically. The build needs openbao.renner.uno
+for the CA key; rebuild the ISO after a CA rotation.
 
 ## Install
 
@@ -38,8 +42,7 @@ archiso updates flow in automatically.
 2. Boot the target from it (UEFI, Secure Boot off; one-time F12/setup
    visit if the box doesn't already boot USB first)
 3. Either on the console (auto-login as root) or headless —
-   `ssh root@<dhcp-ip>` works out of the box, the live env carries the
-   homelab keys — run `edge-install`. It auto-detects the internal disk
+   `ssh root@<dhcp-ip>` with an owner certificate — run `edge-install`. It auto-detects the internal disk
    (NVMe, else eMMC — covers the Wyse 3040 — else first non-USB disk),
    shows the target and requires typing the hostname before wiping
    anything. Override with `edge-install /dev/sdY`.

images/edge-host-iso/airootfs/root/.ssh/authorized_keys

@@ -1,4 +1 @@
-ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPzFNBP3bq2m/6JqDqod9w+FP/uxXxHMolqhq3Ojflr1 krenner@fortress
-ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGfe3HJqFR+jIGrboweseAO7XjIVE24sje/b1HybgU9C ansible-runner@ci-new
-ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICyNnX1I4W9jyDvCJ1DJauHSmbo1ZXSyBxVhSAwMWgen woodpecker-ci@homelab
-ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBB1OFjh6Q5xeioI6Cwfuv8Xka27veomYy9tKBbpMo8mcrsB4yIN4FX8HovBLtogetBhwBUefBar8nhA9GbRh6HQ= krenner@krenner-XPS-13-9310
+ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBKvSl9LUP8u3S2Sq5YTWpW0Jxbue8cJhD7/LMFQbEsa3xo145HVR7pBzRxW71MUy+jgrgmpSlDhIb606Hswhvoo= breakglass-neo-6532348

images/edge-host-iso/airootfs/root/edge-authorized_keys

@@ -1,4 +1 @@
-ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPzFNBP3bq2m/6JqDqod9w+FP/uxXxHMolqhq3Ojflr1 krenner@fortress
-ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGfe3HJqFR+jIGrboweseAO7XjIVE24sje/b1HybgU9C ansible-runner@ci-new
-ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICyNnX1I4W9jyDvCJ1DJauHSmbo1ZXSyBxVhSAwMWgen woodpecker-ci@homelab
-ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBB1OFjh6Q5xeioI6Cwfuv8Xka27veomYy9tKBbpMo8mcrsB4yIN4FX8HovBLtogetBhwBUefBar8nhA9GbRh6HQ= krenner@krenner-XPS-13-9310
+ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBKvSl9LUP8u3S2Sq5YTWpW0Jxbue8cJhD7/LMFQbEsa3xo145HVR7pBzRxW71MUy+jgrgmpSlDhIb606Hswhvoo= breakglass-neo-6532348

images/edge-host-iso/airootfs/usr/local/bin/edge-install

@@ -81,6 +81,17 @@ ln -sf /run/systemd/resolve/stub-resolv.conf /mnt/etc/resolv.conf
 install -d -m700 /mnt/root/.ssh
 install -m600 /root/edge-authorized_keys /mnt/root/.ssh/authorized_keys
 
+# SSH user CA trust as baked into this ISO (owner certificates for root).
+install -m644 -t /mnt/etc/ssh /etc/ssh/trusted_user_ca_keys /etc/ssh/revoked_keys
+install -d -m755 /mnt/etc/ssh/auth_principals /mnt/etc/ssh/sshd_config.d
+install -m644 /etc/ssh/auth_principals/root /mnt/etc/ssh/auth_principals/root
+install -m644 /etc/ssh/sshd_config.d/50-ssh-ca.conf /mnt/etc/ssh/sshd_config.d/
+# sshd -t/-T need host keys; sshdgenkeys would create them on first boot.
+ssh-keygen -A -f /mnt
+arch-chroot /mnt sshd -t
+arch-chroot /mnt sshd -T | grep -qix 'trustedusercakeys /etc/ssh/trusted_user_ca_keys' \
+  || { echo "installed sshd does not use the SSH CA"; exit 1; }
+
 arch-chroot /mnt systemctl enable \
   systemd-networkd systemd-resolved systemd-timesyncd sshd docker
 

images/edge-host-iso/build-inner.sh

@@ -10,6 +10,23 @@ profile=/tmp/profile
 cp -r /usr/share/archiso/configs/releng/. "$profile"
 cp -r /src/airootfs/. "$profile/airootfs/"
 
+# SSH user CA trust from the ssh-ca role (mounted at /ssh-ca): the live system
+# and the installed one accept owner certificates for root, like the managed
+# hosts. The CA public key is fetched from OpenBao at build time.
+ca_url=$(sed -n 's/^ssh_ca_public_key_url: "\(.*\)"$/\1/p' /ssh-ca/defaults/main.yml)
+principal=$(sed -n 's/^ssh_ca_owner_principal: \(.*\)$/\1/p' /ssh-ca/defaults/main.yml)
+[ -n "$ca_url" ] && [ -n "$principal" ] || { echo "ssh-ca defaults not readable"; exit 1; }
+ca_key=$(curl -sSf "$ca_url")
+[[ "$ca_key" == "ssh-ed25519 AAAA"* ]] || { echo "unexpected CA key from $ca_url"; exit 1; }
+sshd_conf=$(grep -vx '# {{ ansible_managed }}' /ssh-ca/templates/ssh-ca.conf.j2)
+! grep -q '{[{%]' <<<"$sshd_conf" || { echo "ssh-ca.conf.j2 has template logic"; exit 1; }
+ssh_dir="$profile/airootfs/etc/ssh"
+install -d "$ssh_dir/sshd_config.d" "$ssh_dir/auth_principals"
+printf '%s\n' "$ca_key" > "$ssh_dir/trusted_user_ca_keys"
+cp /ssh-ca/files/revoked_keys "$ssh_dir/revoked_keys"
+printf '%s\n' "$principal" > "$ssh_dir/auth_principals/root"
+printf '%s\n' "$sshd_conf" > "$ssh_dir/sshd_config.d/50-ssh-ca.conf"
+
 # Tools the installer script needs in the live environment (idempotent —
 # skip entries releng already ships).
 for pkg in gptfdisk dosfstools e2fsprogs; do

images/edge-host-iso/build.sh

@@ -13,6 +13,7 @@ mkdir -p "$out"
 
 docker run --rm --privileged \
   -v "$here:/src:ro" \
+  -v "$here/../../ansible/roles/ssh-ca:/ssh-ca:ro" \
   -v "$out:/out" \
   archlinux:latest /src/build-inner.sh
 

kubernetes/MIGRATION-sops-openbao.md

@@ -52,9 +52,9 @@ kubectl -n openbao exec -it openbao-0 -- bao operator unseal   # paste key
 
 **Unseal is automatic since 2026-09-08** (static seal, PR #825, issue #805):
 the seal key lives in the `openbao-seal-key` Secret (created by `bootstrap.sh`
-from `pass show homelab/openbao/seal-key`) and every restarted replica unseals
-itself. The shamir key above became the **recovery key** in the migration; it
-is kept in `pass` under its old name, `homelab/openbao/unseal-key`. The server
+from `~/.config/homelab-secrets/seal-key.age`) and every restarted replica
+unseals itself. The shamir key above became the **recovery key** in the
+migration; it is `~/.config/homelab-secrets/openbao-recovery-key.age`. The server
 runs three raft peers (see `infrastructure/openbao/helmrelease.yaml`); a sealed
 one is neither a voter nor a standby, so a replica that stays sealed after a
 restart — wrong or missing seal-key Secret — costs redundancy, two cost
@@ -237,19 +237,21 @@ one full reconcile + one OpenBao pod restart/unseal cycle:
 - **Backups:** ✅ nightly raft-snapshot CronJob (03:00, keeps 7 on a
   dedicated PVC; see infrastructure/openbao/snapshot-cronjob.yaml) + velero
   `openbao-daily` Schedule (06:15) ships that PVC offsite, 30 d retention.
-  A raft snapshot restore needs the matching seal key — which lives in
-  `pass`, outside the cluster (GPG key backed up externally 2026-07-14).
+  A raft snapshot restore needs the matching seal key — which lives outside
+  the cluster in `~/.config/homelab-secrets/seal-key.age` (age, encrypted to
+  both YubiKeys and the offline paper key; copy on nas-2).
   A failed snapshot Job is the sealed-after-restart alarm. Still owed:
   one tested restore before phase 4.
-- **Cluster rebuild order:** seal-key Secret from `pass` → Flux brings up
+- **Cluster rebuild order:** seal-key Secret from `seal-key.age` → Flux brings up
   openbao (empty) → init → restore raft snapshot → replicas unseal themselves
   → ClusterSecretStore Ready → ESO materializes Secrets → apps start
   (`bootstrap.sh flux` does all of it; RUNBOOK-cold-rebuild.md Step 4 by
   hand). Until then, secret-consuming apps simply stay Pending on missing
   Secrets — same as any missing dependency.
-- **Break-glass:** `pass` holds seal key, recovery key + root token (GPG,
-  offline-capable). With those, everything is recoverable from a raft
-  snapshot alone.
+- **Break-glass:** `~/.config/homelab-secrets` holds the seal key and
+  recovery key (YubiKeys + paper key, copy on nas-2);
+  `~/.config/openbao/root-token.age` holds the root token (YubiKeys). With
+  those, everything is recoverable from a raft snapshot alone.
 - **Ansible Vault password:** `ansible/vault-password.sh` reads
   `ANSIBLE_VAULT_PASSWORD` from the environment with no offline backup as of
   2026-07-14. It gates decryption of several Ansible-Vault files (PBS

kubernetes/RUNBOOK-cold-rebuild.md

@@ -67,17 +67,24 @@ These must exist outside the cluster or the rebuild cannot complete:
       `flux` phase creates the `sops-age` Secret, and any not-yet-migrated sops
       files decrypt with it. (Post-OpenBao-phase-4 this may be retired — see
       `MIGRATION-sops-openbao.md`.)
-- [ ] **OpenBao seal key, recovery key + root token** — in the GPG `pass`
-      store (`homelab/openbao/seal-key`, `homelab/openbao/unseal-key` — the
-      former shamir key, now the recovery key — and `homelab/openbao/root-token`),
-      and the GPG private key backed up off-machine. The seal key is the one
-      the snapshots are encrypted under: without it, no OpenBao snapshot can be
-      restored and no secret-consuming app can start.
+- [ ] **OpenBao seal key, recovery key, restore + pull credentials** — age
+      files in `~/.config/homelab-secrets/` (`seal-key.age`,
+      `openbao-recovery-key.age` — the former shamir key —, `s3-access.age`,
+      `s3-secret.age`, `ghcr-pull.age`, `hetzner-cloud-api-secret.age`),
+      encrypted to both YubiKeys and the
+      offline paper key in the safe; a copy is on nas-2 in
+      `~ladmin/homelab-secrets-backup` (`~/.ssh/ssh-keys-rotate dr-copy`). The
+      root token is `~/.config/openbao/root-token.age` (YubiKeys only). Needed
+      to decrypt: one YubiKey with its PIV PIN, or the paper key. The seal key
+      is the one the snapshots are encrypted under: without it, no OpenBao
+      snapshot can be restored and no secret-consuming app can start.
 - [ ] **OpenBao raft snapshot** reachable — nightly PVC snapshots
       (`infrastructure/openbao/snapshot-cronjob.yaml`) plus the Velero
       `openbao-daily` offsite copy. In a total-loss rebuild the offsite Velero
       copy is the one that survives.
-- [ ] **Tooling:** `talosctl`, `kubectl`, `flux`, `tofu`, `bao`, plus the
+- [ ] **Tooling:** `talosctl`, `kubectl`, `flux`, `tofu`, `bao`, `age` +
+      `age-plugin-yubikey` (the plugin is only needed for the YubiKeys; the
+      paper key needs `age` alone), plus the
       OpenTofu backend + UniFi credentials the `opentofu/` stack needs. No
       Proxmox credentials — there is no Proxmox any more.
 - [ ] **Install media** written and to hand: the Image Factory ISO named in
@@ -299,14 +306,16 @@ output** — a green step over a still-forming Ceph is a known trap.
 ## Step 4 — openbao: restore the snapshot
 
 `bootstrap.sh flux` does this step itself (`restore_openbao`): it creates the
-`openbao-seal-key` Secret from `pass` before Flux brings OpenBao up, initialises
+`openbao-seal-key` Secret from `~/.config/homelab-secrets/seal-key.age` before Flux brings OpenBao up, initialises
 the empty server, restores the newest raft snapshot straight out of the Velero
 kopia repository, and waits until every replica has unsealed itself from the
 static seal. What follows is the same procedure by hand, for when the script
 is not an option.
 
 OpenBao unseals itself from the static seal (`openbao-seal-key` Secret, key
-`key`, the 32-byte base64 key from `pass show homelab/openbao/seal-key`). The
+`key`, the 32-byte base64 key in `~/.config/homelab-secrets/seal-key.age`,
+age-encrypted to the owner's YubiKeys and the offline paper key; a copy sits on
+nas-2 in `~ladmin/homelab-secrets-backup`). The
 snapshots are encrypted under that key, so it must be the same key the backed
 up cluster used — a different one restores nothing. Until the snapshot is
 restored, the `ClusterSecretStore` stays NotReady, ESO cannot materialize
@@ -316,7 +325,8 @@ Secrets, and every secret-consuming app stays Pending.
 # 1. The seal key, before the pods start (a pod without it never starts).
 kubectl create namespace openbao --dry-run=client -o yaml | kubectl apply -f -
 kubectl -n openbao create secret generic openbao-seal-key \
-  --from-literal=key="$(pass show homelab/openbao/seal-key)"
+  --from-literal=key="$(age -d -i ~/.config/age/yubikeys.txt ~/.config/homelab-secrets/seal-key.age)"
+# Both YubiKeys lost: -i <file holding the paper key> instead.
 
 # 2. Wait for openbao-0 (uninitialised: 0/1 Ready is expected), then
 #    initialise it. With an auto-seal, init hands out recovery keys and the
@@ -337,7 +347,8 @@ kubectl -n openbao exec -it openbao-0 -- \
 
 # 4. The server seals against the snapshot's keyring and unseals itself from
 #    the static seal; openbao-1 and openbao-2 follow (OrderedReady). Nothing
-#    to type. Afterwards the tokens and recovery key from pass apply again.
+#    to type. Afterwards the operator's root token
+#    (~/.config/openbao/root-token.age) and recovery key apply again.
 kubectl -n openbao get pods -w
 ```
 
@@ -352,7 +363,7 @@ kubectl -n <ns> get externalsecret                # want: SecretSynced
 > A replica that stays **sealed** after a restart could not use the static
 > seal — the Secret is missing or does not match the key the data was
 > written under. Fix the Secret and restart the pod; `bao operator unseal`
-> with the recovery key (`pass show homelab/openbao/unseal-key`) does not
+> with the recovery key (`~/.config/homelab-secrets/openbao-recovery-key.age`) does not
 > unseal an auto-sealed server and is only for recovery-mode operations.
 > Details + rationale: `MIGRATION-sops-openbao.md`.
 
@@ -434,7 +445,7 @@ it: repo → **Settings → Repository → Mirror Settings** in Gitea, or
   Step 2 no longer uses it. Its `flux` and `wait-apps` phases are still used in
   Steps 3, 5 and 6 and work; do not read this as the whole script being gone.
   Its `flux` phase carries the OpenBao restore (Step 4); the seal key and the
-  other secrets are read from `pass` or streamed in at run time and never live
+  other secrets are decrypted from `~/.config/homelab-secrets` or streamed in at run time and never live
   in CI.
 - **OpenBao unseals itself, but only under the seal key the snapshot was
   written with.** Three raft replicas mean the service survives losing one; a

kubernetes/apps/monitoring-rules/prometheusrule-openbao.yaml

@@ -49,7 +49,7 @@ spec:
               failure would cost quorum. Check `kubectl -n openbao get pods`,
               the pod log and `bao status` in the not-ready pod. If it is
               sealed, the static seal failed: verify the openbao-seal-key
-              Secret matches `pass show homelab/openbao/seal-key`.
+              Secret matches ~/.config/homelab-secrets/seal-key.age (age, YubiKey).
 
         - alert: OpenBaoQuorumLost
           # Raft quorum of a three-node cluster is two. At one ready replica
@@ -76,7 +76,7 @@ spec:
               surface only when something rotates or a new secret is needed.
               Replicas unseal themselves from the static seal; one that stays
               sealed could not. Check its log and the openbao-seal-key Secret
-              against `pass show homelab/openbao/seal-key`, fix the Secret and
+              against ~/.config/homelab-secrets/seal-key.age, fix the Secret and
               restart the pod. The recovery key cannot unseal an auto-sealed
               server. Procedure: kubernetes/RUNBOOK-cold-rebuild.md, Step 4.
 

kubernetes/bootstrap.sh

@@ -35,7 +35,8 @@ set -euo pipefail
 # The mirror is kept current server-side by Gitea's native push-mirror. Full
 # cold-rebuild procedure: kubernetes/RUNBOOK-cold-rebuild.md.
 #
-# Requires: talosctl, kubectl, flux, sops age key at ~/.config/sops/age/keys.txt
+# Requires: talosctl, kubectl, flux, sops age key at ~/.config/sops/age/keys.txt,
+# age + age-plugin-yubikey (the DR secrets in ~/.config/homelab-secrets)
 
 TALOS_DIR="$(cd "$(dirname "$0")/talos" && pwd)"
 REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
@@ -190,13 +191,16 @@ if [[ "$FLUX_SOURCE" == "restore-test" && "$PHASE" != "flux" && "$PHASE" != "wai
 fi
 OPENBAO_NS="openbao"
 
-# Where the secrets come from. On a workstation, the operator's pass store. In
-# the in-cluster Job there is no pass and no GPG key — and there should not be:
-# the pass store opens everything, and it stays on the operator's machine. Set
-# OPENBAO_SECRETS_DIR and the run reads them from files instead, which the
+# Where the secrets come from. On a workstation, age files in
+# HOMELAB_SECRETS_DIR, encrypted to the operator's YubiKeys and an offline paper
+# key (a copy sits on nas-2). HOMELAB_AGE_IDENTITY picks what decrypts them: the
+# YubiKey identity stubs by default (touch per file), the paper key's file when
+# both sticks are gone. In the in-cluster Job there is neither — and there
+# should not be: those keys open everything, and they stay with the operator.
+# Set OPENBAO_SECRETS_DIR and the run reads them from files instead, which the
 # operator streams in when the run asks for them:
 #
-#     pass show homelab/openbao/seal-key \
+#     age -d -i ~/.config/age/yubikeys.txt ~/.config/homelab-secrets/seal-key.age \
 #       | kubectl -n vmtest exec -i <pod> -- sh -c 'cat > /secrets/seal-key'
 #
 # They exist in the pod's filesystem for the length of the run and are never
@@ -240,6 +244,15 @@ git_token_for_source() {
     fi
 }
 
+# Literal ~ on purpose: the commands are eval'd on the workstation, which
+# expands it there, and the in-cluster Job prints them for the operator to run
+# on that workstation, where the Job's own HOME (/work) would be wrong.
+# shellcheck disable=SC2088
+[[ -n "${HOMELAB_SECRETS_DIR:-}" ]] || HOMELAB_SECRETS_DIR='~/.config/homelab-secrets'
+# shellcheck disable=SC2088
+[[ -n "${HOMELAB_AGE_IDENTITY:-}" ]] || HOMELAB_AGE_IDENTITY='~/.config/age/yubikeys.txt'
+AGE_SECRET="age -d -i $HOMELAB_AGE_IDENTITY $HOMELAB_SECRETS_DIR"
+
 # Format: <file in SECRETS_DIR>|<what it is>|<how the operator produces it>
 OPENBAO_REQUIRED_SECRETS=(
     "sops-age.key|SOPS age key|cat ~/.config/sops/age/keys.txt"
@@ -255,18 +268,18 @@ OPENBAO_REQUIRED_SECRETS=(
     # Emptiness is not tested here on purpose — require_secrets rejects an empty
     # file and names what is missing.
     "git-token|token for flux bootstrap, for whichever forge FLUX_SOURCE names|git_token_for_source"
-    "s3-access|Hetzner restore access key|pass show homelab/hetztner/k8s-restore-access"
-    "s3-secret|Hetzner restore secret key|pass show homelab/hetztner/k8s-restore-secret"
+    "s3-access|Hetzner restore access key|$AGE_SECRET/s3-access.age"
+    "s3-secret|Hetzner restore secret key|$AGE_SECRET/s3-secret.age"
     # The static seal key. The snapshots are encrypted under it, so it has to
     # be the one the backed-up cluster ran with — a fresh key restores nothing.
-    "seal-key|OpenBao seal key|pass show homelab/openbao/seal-key"
+    "seal-key|OpenBao seal key|$AGE_SECRET/seal-key.age"
     # Read by the node registry auth (see registry_auth_patch): the images an app
     # needs while the cluster is still being built are mirrored to a private
     # ghcr.io namespace, and a node has to be able to pull them before anything
     # in the cluster exists to hold a credential. Demanded up front like the
     # rest, because a rebuild that discovers it missing has already wiped the
     # disks.
-    "ghcr-pull|GHCR pull token for the image mirror|pass show homelab/github/k8s-package-pull"
+    "ghcr-pull|GHCR pull token for the image mirror|$AGE_SECRET/ghcr-pull.age"
 )
 # Asked for by name only, never by a bare require_secrets: a rebuild of the
 # real cluster cannot depend on the OpenBao it is rebuilding. Scoped to the
@@ -281,8 +294,8 @@ HCLOUD_REQUIRED_SECRETS=(
 # during a real recovery or in the in-cluster Job.
 #
 # The only difference between those two is where the material comes from, and it
-# is confined to this function: on a laptop the commands in the list produce it
-# from pass; in the Job the operator streams it in. Everything downstream cannot
+# is confined to this function: on a laptop the commands in the list decrypt
+# it from HOMELAB_SECRETS_DIR; in the Job the operator streams it in. Everything downstream cannot
 # tell the difference — which is the point, because the recovery path must not be
 # a path that only gets exercised during a recovery.
 # require_secrets [name...]
@@ -311,7 +324,7 @@ require_secrets() {
         wanted=("${OPENBAO_REQUIRED_SECRETS[@]}")
     fi
 
-    # Which branch: fetch from pass, or wait for someone to stream files in.
+    # Which branch: decrypt locally, or wait for someone to stream files in.
     #
     # The condition cannot be "is OPENBAO_SECRETS_DIR set", because THIS
     # FUNCTION sets it. Once a subset call had run — hcloudtest asks for
@@ -322,7 +335,7 @@ require_secrets() {
     # maintenance.
     #
     # SECRETS_SELF_FETCH records which world we are in, and it is decided once:
-    # an empty OPENBAO_SECRETS_DIR on entry means pass, an inherited one means
+    # an empty OPENBAO_SECRETS_DIR on entry means local age decryption, an inherited one means
     # the Job. Later calls follow that decision instead of re-deriving it from
     # state this function has since changed.
     if [[ -z "$OPENBAO_SECRETS_DIR" || "${SECRETS_SELF_FETCH:-0}" == "1" ]]; then
@@ -342,12 +355,14 @@ require_secrets() {
             name=${entry%%|*}; what=${entry#*|}; how=${what#*|}; what=${what%%|*}
             # Already collected by an earlier call — a subset call fetches only
             # what it asked for, so the full call that follows must fill the
-            # gaps rather than re-run every command. `pass show` on a locked
-            # store prompts, and prompting twice for the same key is how an
+            # gaps rather than re-run every command. Each decryption waits for
+            # a YubiKey touch, and asking twice for the same key is how an
             # unattended run turns into a hung one.
             [[ -s "$OPENBAO_SECRETS_DIR/$name" ]] && continue
-            if ! eval "$how" > "$OPENBAO_SECRETS_DIR/$name" 2>"$OPENBAO_SECRETS_DIR/.err"; then
-                echo; cat "$OPENBAO_SECRETS_DIR/.err" >&2
+            # stderr stays on the terminal: age-plugin-yubikey asks for the
+            # PIN and the touch there, and a captured prompt is a silent hang.
+            echo
+            if ! eval "$how" > "$OPENBAO_SECRETS_DIR/$name"; then
                 fail "could not obtain the $what with: $how"
             fi
             [[ -s "$OPENBAO_SECRETS_DIR/$name" ]] \
@@ -382,7 +397,7 @@ OPENBAO_NS_HINT=vmtest
 
 # Reads one secret by its name in the list above. In the Job that is the file the
 # operator streamed in; on a workstation it runs the command the list names,
-# which for these is `pass show <path>`.
+# which for these is `age -d` of a file in HOMELAB_SECRETS_DIR.
 #
 # The list is the only place the paths appear. They used to sit here as
 # constants as well, which is how the same value ends up defined twice and one
@@ -1486,7 +1501,8 @@ fi
 
 # Before anything destructive, and after the vmtest block has decided which
 # phases will run. A rebuild that will need these has no business wiping disks
-# first, only to discover an hour later that pass is locked or a token is unset —
+# first, only to discover an hour later that no YubiKey is plugged in or a
+# token is unset —
 # by then the old cluster is gone and the new one cannot be finished.
 #
 # In the Job this is where the file-waiting branch blocks, which is the right
@@ -1994,7 +2010,8 @@ bao_exec() {
 # have been doing this all along.
 #
 # Reading them needs three things, all of which live OUTSIDE the cluster, which
-# is the point: bucket credentials from pass (their own key, not Velero's),
+# is the point: bucket credentials from HOMELAB_SECRETS_DIR (their own key, not
+# Velero's),
 # kopia, and the repository password. The last is not a secret — Velero
 # hardcodes `static-passw0rd` in its source. Worth knowing for what it means:
 # the offsite backup's confidentiality rests on bucket access alone.
@@ -2071,7 +2088,8 @@ restore_openbao() {
     # rather than an unseal key and the server unseals itself; the only thing
     # used from its output is the root token, for the next few seconds, and
     # then it stops existing: restoring replaces the keyring with the
-    # snapshot's, after which the root token and recovery key from pass apply.
+    # snapshot's, after which the operator's root token
+    # (~/.config/openbao/root-token.age) and recovery key apply.
     #
     # No -force. Without it OpenBao checks that the snapshot was written under
     # the seal key this server runs with, and refuses a mismatch by name
@@ -2355,7 +2373,7 @@ if phase_runs flux; then
     # Secret would leave every replica sealed for good at its next restart.
     # That is a migration, not a bootstrap step.
     #
-    # --from-literal, not --from-file: `pass show` ends its output with a
+    # --from-literal, not --from-file: the stored value ends with a
     # newline, the substitution strips it, and OpenBao rejects a key that is
     # not exactly 32 bytes of base64. The key sits in this process's argument
     # list for the length of the call, as it did in the pod's before.

kubernetes/infrastructure/crowdsec/helmrelease.yaml

@@ -80,6 +80,25 @@ spec:
           # community blocklists.
           - 162.55.50.97/32
           - 2a01:4f8:c2c:4579::1/128
+      postoverflows:
+        s01-whitelist:
+          # http-crawl-non_statics counts requests regardless of outcome, so a
+          # signed-in user of any image-heavy UI (Immich timeline, Music
+          # Assistant covers) trips it with a burst of 200s. Scanners trip it
+          # with 404s for paths that do not exist. Drop the overflow unless at
+          # least half of its requests were client errors (the alert carries
+          # the scenario's last 5 events, cache_size: 5). Only this scenario
+          # is affected; probing/sensitive-files/cve still fire on their own.
+          crawl-non-statics-success-whitelist.yaml: |
+            name: krenner/crawl-non-statics-success-whitelist
+            description: "Ignore http-crawl-non_statics bursts that mostly succeeded"
+            whitelist:
+              reason: "crawl burst of successful requests is a UI, not a scanner"
+              expression:
+                - >-
+                  evt.Overflow.Alert.GetScenario() == 'crowdsecurity/http-crawl-non_statics' &&
+                  count(evt.Overflow.Alert.Events, {.GetMeta('http_status') startsWith '4'}) * 2
+                  < len(evt.Overflow.Alert.Events)
 
     # mTLS between agents and the LAPI. Do not switch back to password auth:
     # the LAPI runs without persistence, so its machine-registration DB is

opentofu/providers.tf

@@ -43,12 +43,6 @@ provider "proxmox" {
   endpoint  = var.proxmox_api_url
   api_token = "${var.proxmox_user}!${var.proxmox_token_id}=${var.proxmox_token_secret}"
   insecure  = var.proxmox_tls_insecure
-
-  ssh {
-    agent       = false
-    username    = "root"
-    private_key = var.ssh_private_key
-  }
 }
 
 provider "unifi" {

opentofu/variables.tf

@@ -28,16 +28,4 @@ variable "proxmox_tls_insecure" {
   default     = false
 }
 
-variable "ssh_public_key" {
-  description = "Public SSH key to inject into containers"
-  type        = string
-  default     = null
-}
-
-variable "ssh_private_key" {
-  description = "SSH private key for Proxmox provider to import disk images"
-  type        = string
-  sensitive   = true
-  default     = ""
-}