fix(ansible): install-netbird-client runs one host at a time

From the workstation each new SSH session asks for the YubiKey PIN and
a touch; with the default forks both proxies prompted at once. serial: 1
like the other plays that touch the proxies.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

.woodpecker/deploy-ssh-ca.yml

@@ -6,6 +6,9 @@ when:
       - "ansible/roles/ssh-ca/**"
       - "ansible/plays/configure-ssh-ca.yml"
       - "ansible/inventory/pve_hosts.yml"
+      - "ansible/inventory/group_vars/all/**"
+      - "ansible/inventory/group_vars/netbird_proxy/vars.yml"
+      - "ansible/inventory/host_vars/netbird.renner.uno.yml"
       - ".woodpecker/deploy-ssh-ca.yml"
   - event: deployment
     evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "deploy-ssh-ca"'

ansible/ansible.cfg

@@ -206,7 +206,7 @@ inventory=~/source/homelab/ansible/inventory
 ;poll_interval=15
 
 # (path) Option for connections using a certificate or key file to authenticate, rather than an agent or passwords, you can set the default value here to avoid re-specifying --private-key with every invocation.
-private_key_file=~/.ssh/id_ed25519
+;private_key_file=
 
 # (boolean) By default, imported roles publish their variables to the play and other roles, this setting can avoid that.
 # This was introduced as a way to reset role variables to default values if a role is used more than once in a playbook.

ansible/inventory/group_vars/all/vars.yml

@@ -5,16 +5,26 @@ pve_token_id: "ansible"
 ansible_user: "ladmin"
 ansible_python_interpreter: /usr/bin/python3
 
-# SSH Public Keys for Cloud-Init
-# ansible-runner@ci-new removed (issue #184): dead key, no such host exists.
-common_ssh_public_keys: |
-  ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPzFNBP3bq2m/6JqDqod9w+FP/uxXxHMolqhq3Ojflr1 krenner@fortress
-  ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICyNnX1I4W9jyDvCJ1DJauHSmbo1ZXSyBxVhSAwMWgen woodpecker-ci@homelab
+# The only plain keys the admin logins accept on [ssh_keys_managed] hosts
+# (ssh-ca role, tasks/authorized_keys.yml). People log in with OpenBao SSH CA
+# certificates; these two are for when that path is unavailable.
+#   breakglass-neo: PIV key (slot 9a, PIN) on the offline YubiKey NEO.
+#     ssh -I /usr/lib/opensc-pkcs11.so root@<host>
+#   woodpecker-ci: the CI deploy key, from ssh_ci_key_from where set.
+ssh_admin_authorized_keys:
+  - key: "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBKvSl9LUP8u3S2Sq5YTWpW0Jxbue8cJhD7/LMFQbEsa3xo145HVR7pBzRxW71MUy+jgrgmpSlDhIb606Hswhvoo= breakglass-neo-6532348"
+  - key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICyNnX1I4W9jyDvCJ1DJauHSmbo1ZXSyBxVhSAwMWgen woodpecker-ci@homelab"
+    options: "{{ ('from=\"' ~ ssh_ci_key_from ~ '\"') if ssh_ci_key_from else '' }}"
+
+# Source the CI key is accepted from. LAN hosts see the pod address; hosts on
+# the internet see the home WAN address (NAT, dynamic) and set this empty in
+# their group/host vars.
+ssh_ci_key_from: "{{ ci_ssh_from_cidrs }}"
 
 # Network-of-origin restriction for the woodpecker-ci@homelab CI key
-# (issue #184). Consumed by plays/configure-ssh-keys.yml as the authorized_key
-# `from=` option so the CI key only authenticates from within the cluster's
-# pod network — NOT from any host-pinned IP.
+# (issue #184). Used as the authorized_keys `from=` option so the CI key only
+# authenticates from within the cluster's pod network — NOT from any
+# host-pinned IP.
 #
 # The in-cluster Woodpecker step pods SSH to fleet hosts using their POD IP as
 # source, not a node IP: Cilium runs `ipv4-native-routing-cidr=10.0.0.0/8` +

ansible/inventory/group_vars/netbird_proxy/vars.yml

@@ -0,0 +1,8 @@
+---
+# CI reaches these internet hosts NATed from the home WAN address, never from
+# the pod network, so the CI key carries no from= restriction here.
+ssh_ci_key_from: ""
+
+# The host NetBird client (wt0) must not take over DNS: resolv.conf is pinned to
+# the systemd-resolved stub by the netbird-proxy role.
+netbird_disable_dns: true

ansible/inventory/host_vars/netbird.renner.uno.yml

@@ -2,3 +2,7 @@
 # NetBird server - VPS not managed by Proxmox
 # Enable Docker monitoring (cAdvisor) for container metrics
 docker_monitoring_enabled: true
+
+# CI reaches this internet host NATed from the home WAN address, never from
+# the pod network, so the CI key carries no from= restriction here.
+ssh_ci_key_from: ""

ansible/inventory/pve_hosts.yml

@@ -77,3 +77,13 @@ claude-ws ansible_host=10.3.40.78 ansible_user=root
 proxy-1.renner.uno
 proxy-2.renner.uno
 netbird.renner.uno
+edge-1.iot.renner.uno
+
+# Hosts whose admin logins accept only ssh_admin_authorized_keys (break-glass
+# + CI); everything else needs a CA certificate. Members must be in [ssh_ca].
+# Added one host at a time after the [ssh_ca] report was reviewed.
+[ssh_keys_managed]
+proxy-1.renner.uno
+proxy-2.renner.uno
+netbird.renner.uno
+edge-1.iot.renner.uno

ansible/plays/configure-ssh-ca.yml

@@ -2,7 +2,8 @@
 # Trust the OpenBao SSH user CA (ssh-client-signer) on the hosts in [ssh_ca]:
 # owner certificates (principal k.renner) for root/ladmin, Claude's 1h
 # certificates (principal claude-bot) for its own claude-bot account with
-# limited sudo. Existing authorized_keys stay untouched.
+# limited sudo. The first play only reports the admin logins' authorized_keys;
+# the second reduces them to ssh_admin_authorized_keys on [ssh_keys_managed].
 #
 # serial: 1 so a host whose sshd rejects the change stops the run before the
 # next host is touched.
@@ -13,3 +14,14 @@
   serial: 1
   roles:
     - ssh-ca
+
+- name: Reduce authorized_keys to break-glass and CI
+  hosts: ssh_keys_managed
+  become: true
+  gather_facts: false
+  serial: 1
+  tasks:
+    - name: Set the managed authorized_keys
+      ansible.builtin.include_role:
+        name: ssh-ca
+        tasks_from: authorized_keys

ansible/plays/configure-ssh-keys.yml

@@ -1,48 +0,0 @@
----
-- name: Deploy authorized SSH keys to all hosts
-  hosts: all:!localhost
-  become: true
-  gather_facts: false
-  # Authorized keys deployed to root and ladmin on every managed host.
-  #
-  # - krenner@fortress (human): unrestricted.
-  # - woodpecker-ci@homelab (CI): restricted with from= to the cluster's pod
-  #   network (ci_ssh_from_cidrs in group_vars/all/vars.yml). The in-cluster
-  #   Woodpecker step pods SSH to fleet hosts presenting their POD IP as source
-  #   (Cilium native-routing-cidr covers the LAN, so no SNAT; PodCIDRs are
-  #   BGP-advertised). This is a network-of-origin restriction, NOT host-pinned,
-  #   so it survives node DHCP churn and pod rescheduling. See vars.yml for the
-  #   full derivation.
-  # - ansible-runner@ci-new: removed — dead key, no such host exists.
-  vars:
-    authorized_keys:
-      - key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPzFNBP3bq2m/6JqDqod9w+FP/uxXxHMolqhq3Ojflr1 krenner@fortress"
-      - key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICyNnX1I4W9jyDvCJ1DJauHSmbo1ZXSyBxVhSAwMWgen woodpecker-ci@homelab"
-        options: 'from="{{ ci_ssh_from_cidrs }}"'
-  tasks:
-    - name: Check SSH connectivity
-      ansible.builtin.wait_for_connection:
-        timeout: 5
-      ignore_errors: true
-      register: ssh_check
-
-    - name: Skip unreachable hosts
-      ansible.builtin.meta: end_host
-      when: ssh_check is failed
-
-    - name: Deploy authorized keys to root
-      ansible.builtin.authorized_key:
-        user: root
-        key: "{{ item.key }}"
-        key_options: "{{ item.options | default(omit) }}"
-        state: present
-      loop: "{{ authorized_keys }}"
-
-    - name: Deploy authorized keys to ladmin
-      ansible.builtin.authorized_key:
-        user: ladmin
-        key: "{{ item.key }}"
-        key_options: "{{ item.options | default(omit) }}"
-        state: present
-      loop: "{{ authorized_keys }}"
-      ignore_errors: true

ansible/plays/install-netbird-client.yml

@@ -15,17 +15,20 @@
 - name: Install NetBird client on hosts with setup key defined
   hosts: all
   become: true
-  gather_facts: true
+  # One host at a time: from a workstation every new SSH session asks for the
+  # YubiKey PIN and a touch, and parallel hosts would prompt at once.
+  serial: 1
+  # Facts only after the skip: gathering them first opens an SSH session (a
+  # YubiKey touch from a workstation) to every host, including skipped ones.
+  gather_facts: false
 
   pre_tasks:
-    - name: Check if netbird_setup_key is defined
-      ansible.builtin.debug:
-        msg: "NetBird setup key is defined for {{ inventory_hostname }}"
-      when: netbird_setup_key is defined
-
     - name: Skip hosts without netbird_setup_key
       ansible.builtin.meta: end_host
       when: netbird_setup_key is not defined
 
+    - name: Gather facts
+      ansible.builtin.setup:
+
   roles:
     - netbird-client

ansible/roles/common/tasks/main.yml

@@ -51,10 +51,8 @@
 # -------------------------------------------------------
 # 4. SSH Configuration
 # -------------------------------------------------------
-# Authorized keys (human + woodpecker-ci, the latter from=-restricted) are
-# managed by the dedicated ansible/plays/configure-ssh-keys.yml play. The old
-# "Add Ansible Runner SSH Key" task deployed ansible-runner@ci-new, a dead key
-# for a host that no longer exists — removed (issue #184).
+# SSH access (CA trust, admin authorized_keys) is managed by the ssh-ca role
+# via plays/configure-ssh-ca.yml.
 
 # -------------------------------------------------------
 # 5. Disable NTP on VMs/LXC (use host time)

ansible/roles/netbird-client/defaults/main.yml

@@ -6,3 +6,11 @@ netbird_management_url: "https://netbird.renner.uno"
 
 # Setup key - should be defined in group_vars or host_vars
 # netbird_setup_key: ""
+
+# Release installed on hosts without a NetBird package repository (Arch).
+# Debian/Ubuntu follow the apt repository instead.
+netbird_client_version: "0.75.0"
+netbird_release_url: "https://github.com/netbirdio/netbird/releases/download/v{{ netbird_client_version }}"
+
+# Leave the host's DNS configuration alone (hosts with a pinned resolv.conf).
+netbird_disable_dns: false

ansible/roles/netbird-client/tasks/main.yml

@@ -2,32 +2,69 @@
 # NetBird Client Installation Role
 # Installs and configures NetBird client on hosts with netbird_setup_key defined
 
-- name: Install required packages
-  ansible.builtin.apt:
-    name:
-      - curl
-      - ca-certificates
-      - gnupg
-    state: present
-    update_cache: true
+- name: Install NetBird from the NetBird apt repository
+  when: ansible_facts['os_family'] == 'Debian'
+  block:
+    - name: Install required packages
+      ansible.builtin.apt:
+        name:
+          - curl
+          - ca-certificates
+          - gnupg
+        state: present
+        update_cache: true
 
-- name: Add NetBird repository key
-  ansible.builtin.shell:
-    cmd: |
-      curl -sSL https://pkgs.netbird.io/debian/public.key | gpg --dearmor -o /usr/share/keyrings/netbird-archive-keyring.gpg
-    creates: /usr/share/keyrings/netbird-archive-keyring.gpg
+    - name: Add NetBird repository key
+      ansible.builtin.shell:
+        cmd: |
+          curl -sSL https://pkgs.netbird.io/debian/public.key | gpg --dearmor -o /usr/share/keyrings/netbird-archive-keyring.gpg
+        creates: /usr/share/keyrings/netbird-archive-keyring.gpg
 
-- name: Add NetBird repository
-  ansible.builtin.apt_repository:
-    repo: "deb [signed-by=/usr/share/keyrings/netbird-archive-keyring.gpg] https://pkgs.netbird.io/debian stable main"
-    state: present
-    filename: netbird
+    - name: Add NetBird repository
+      ansible.builtin.apt_repository:
+        repo: "deb [signed-by=/usr/share/keyrings/netbird-archive-keyring.gpg] https://pkgs.netbird.io/debian stable main"
+        state: present
+        filename: netbird
 
-- name: Install NetBird client
-  ansible.builtin.apt:
+    - name: Install NetBird client
+      ansible.builtin.apt:
+        name: netbird
+        state: present
+        update_cache: true
+
+# Arch has no netbird package in its repositories; the release binary is
+# verified against the release's checksums file.
+- name: Install NetBird from the release tarball
+  when: ansible_facts['os_family'] == 'Archlinux'
+  block:
+    - name: Download the NetBird release
+      ansible.builtin.get_url:
+        url: "{{ netbird_release_url }}/netbird_{{ netbird_client_version }}_linux_amd64.tar.gz"
+        dest: "/tmp/netbird_{{ netbird_client_version }}_linux_amd64.tar.gz"
+        checksum: "sha256:{{ netbird_release_url }}/netbird_{{ netbird_client_version }}_checksums.txt"
+        mode: "0644"
+
+    - name: Install the NetBird binary
+      ansible.builtin.unarchive:
+        src: "/tmp/netbird_{{ netbird_client_version }}_linux_amd64.tar.gz"
+        dest: /usr/local/bin
+        remote_src: true
+        include:
+          - netbird
+        owner: root
+        group: root
+        mode: "0755"
+
+    - name: Install the NetBird service
+      ansible.builtin.command: netbird service install
+      args:
+        creates: /etc/systemd/system/netbird.service
+
+- name: Enable NetBird service
+  ansible.builtin.service:
     name: netbird
-    state: present
-    update_cache: true
+    state: started
+    enabled: true
 
 - name: Check if NetBird is already connected
   ansible.builtin.command: netbird status
@@ -35,20 +72,17 @@
   changed_when: false
   failed_when: false
 
+# "Disconnected" contains "Connected", so match the management line.
 - name: Connect NetBird to management server
   ansible.builtin.command: >
     netbird up
     --management-url {{ netbird_management_url }}
     --setup-key {{ netbird_setup_key }}
-  when: "'Connected' not in netbird_status.stdout"
+    {{ '--disable-dns' if netbird_disable_dns else '' }}
+  when: "'Management: Connected' not in netbird_status.stdout"
   register: netbird_connect
   changed_when: netbird_connect.rc == 0
-
-- name: Enable NetBird service
-  ansible.builtin.service:
-    name: netbird
-    state: started
-    enabled: true
+  no_log: true
 
 - name: Get NetBird status
   ansible.builtin.command: netbird status

ansible/roles/ssh-ca/tasks/authorized_keys.yml

@@ -0,0 +1,23 @@
+---
+# Replaces the admin logins' authorized_keys with ssh_admin_authorized_keys
+# (break-glass + CI). Everything else logs in with a CA certificate.
+- name: Look up the admin logins
+  ansible.builtin.getent:
+    database: passwd
+    key: "{{ item }}"
+    fail_key: false
+  loop: "{{ ssh_ca_owner_users }}"
+  register: ssh_ca_admin_lookup
+
+- name: Set the admin logins' authorized_keys to the managed list
+  ansible.posix.authorized_key:
+    user: "{{ item.item }}"
+    key: |
+      {% for k in ssh_admin_authorized_keys %}
+      {{ (k.options ~ ' ') if (k.options | default('')) else '' }}{{ k.key }}
+      {% endfor %}
+    exclusive: true
+  loop: "{{ ssh_ca_admin_lookup.results }}"
+  loop_control:
+    label: "{{ item.item }}"
+  when: (item.ansible_facts.getent_passwd | default({})).get(item.item) is not none

ansible/roles/ssh-ca/tasks/main.yml

@@ -127,13 +127,27 @@
       - "'revokedkeys /etc/ssh/revoked_keys' in ssh_ca_effective_lines"
     fail_msg: "sshd -T does not show the SSH CA settings; another config line takes precedence"
 
-# The unit is sshd.service on Arch and RedHat, ssh.service on Debian. sshd -T
-# only parses the files, so the running daemon is reloaded on every run; a
-# reload keeps open sessions.
+# The unit is sshd.service on Arch and RedHat, ssh.service on Debian/Ubuntu.
+# Ubuntu also lists sshd.service as a not-found alias, so the running one is
+# picked. sshd -T only parses the files, so the running daemon is reloaded on
+# every run; a reload keeps open sessions.
 - name: Read the service list
   ansible.builtin.service_facts:
 
-- name: Reload sshd so the running daemon uses the CA config
-  ansible.builtin.systemd:
-    name: "{{ 'sshd.service' if 'sshd.service' in ansible_facts.services else 'ssh.service' }}"
-    state: reloaded
+- name: Reload the running sshd unit
+  vars:
+    ssh_ca_sshd_units: "{{ ansible_facts.services | dict2items | selectattr('key', 'in', ['sshd.service', 'ssh.service']) | selectattr('value.state', 'equalto', 'running') | map(attribute='key') | list }}"
+  block:
+    - name: Check exactly one sshd unit is running
+      ansible.builtin.assert:
+        that:
+          - ssh_ca_sshd_units | length == 1
+        fail_msg: "Expected one running sshd.service or ssh.service, found {{ ssh_ca_sshd_units }}"
+
+    - name: Reload sshd so the running daemon uses the CA config
+      ansible.builtin.systemd:
+        name: "{{ ssh_ca_sshd_units | first }}"
+        state: reloaded
+
+- name: Report the admin logins' authorized keys
+  ansible.builtin.include_tasks: report.yml

ansible/roles/ssh-ca/tasks/report.yml

@@ -0,0 +1,26 @@
+---
+# Read-only: what the admin logins accept besides certificates. Hosts in
+# [ssh_keys_managed] have this reduced to ssh_admin_authorized_keys.
+- name: Find the admin logins' authorized_keys
+  ansible.builtin.stat:
+    path: "{{ item.ansible_facts.getent_passwd[item.item][4] }}/.ssh/authorized_keys"
+  loop: "{{ ssh_ca_owner_lookup.results }}"
+  loop_control:
+    label: "{{ item.item }}"
+  when: (item.ansible_facts.getent_passwd | default({})).get(item.item) is not none
+  register: ssh_ca_keyfiles
+
+- name: Fingerprint the admin logins' authorized keys
+  ansible.builtin.command: ssh-keygen -lf {{ item.stat.path }}
+  loop: "{{ ssh_ca_keyfiles.results | selectattr('stat', 'defined') | selectattr('stat.exists') | list }}"
+  loop_control:
+    label: "{{ item.stat.path }}"
+  changed_when: false
+  register: ssh_ca_fingerprints
+
+- name: Report the admin logins' authorized keys
+  ansible.builtin.debug:
+    msg: "{{ item.item.stat.path }}: {{ item.stdout_lines }}"
+  loop: "{{ ssh_ca_fingerprints.results }}"
+  loop_control:
+    label: "{{ item.item.stat.path }}"

kubernetes/MIGRATION-sops-openbao.md

@@ -52,9 +52,9 @@ kubectl -n openbao exec -it openbao-0 -- bao operator unseal   # paste key
 
 **Unseal is automatic since 2026-09-08** (static seal, PR #825, issue #805):
 the seal key lives in the `openbao-seal-key` Secret (created by `bootstrap.sh`
-from `pass show homelab/openbao/seal-key`) and every restarted replica unseals
-itself. The shamir key above became the **recovery key** in the migration; it
-is kept in `pass` under its old name, `homelab/openbao/unseal-key`. The server
+from `~/.config/homelab-secrets/seal-key.age`) and every restarted replica
+unseals itself. The shamir key above became the **recovery key** in the
+migration; it is `~/.config/homelab-secrets/openbao-recovery-key.age`. The server
 runs three raft peers (see `infrastructure/openbao/helmrelease.yaml`); a sealed
 one is neither a voter nor a standby, so a replica that stays sealed after a
 restart — wrong or missing seal-key Secret — costs redundancy, two cost
@@ -237,19 +237,21 @@ one full reconcile + one OpenBao pod restart/unseal cycle:
 - **Backups:** ✅ nightly raft-snapshot CronJob (03:00, keeps 7 on a
   dedicated PVC; see infrastructure/openbao/snapshot-cronjob.yaml) + velero
   `openbao-daily` Schedule (06:15) ships that PVC offsite, 30 d retention.
-  A raft snapshot restore needs the matching seal key — which lives in
-  `pass`, outside the cluster (GPG key backed up externally 2026-07-14).
+  A raft snapshot restore needs the matching seal key — which lives outside
+  the cluster in `~/.config/homelab-secrets/seal-key.age` (age, encrypted to
+  both YubiKeys and the offline paper key; copy on nas-2).
   A failed snapshot Job is the sealed-after-restart alarm. Still owed:
   one tested restore before phase 4.
-- **Cluster rebuild order:** seal-key Secret from `pass` → Flux brings up
+- **Cluster rebuild order:** seal-key Secret from `seal-key.age` → Flux brings up
   openbao (empty) → init → restore raft snapshot → replicas unseal themselves
   → ClusterSecretStore Ready → ESO materializes Secrets → apps start
   (`bootstrap.sh flux` does all of it; RUNBOOK-cold-rebuild.md Step 4 by
   hand). Until then, secret-consuming apps simply stay Pending on missing
   Secrets — same as any missing dependency.
-- **Break-glass:** `pass` holds seal key, recovery key + root token (GPG,
-  offline-capable). With those, everything is recoverable from a raft
-  snapshot alone.
+- **Break-glass:** `~/.config/homelab-secrets` holds the seal key and
+  recovery key (YubiKeys + paper key, copy on nas-2);
+  `~/.config/openbao/root-token.age` holds the root token (YubiKeys). With
+  those, everything is recoverable from a raft snapshot alone.
 - **Ansible Vault password:** `ansible/vault-password.sh` reads
   `ANSIBLE_VAULT_PASSWORD` from the environment with no offline backup as of
   2026-07-14. It gates decryption of several Ansible-Vault files (PBS

kubernetes/RUNBOOK-cold-rebuild.md

@@ -67,17 +67,24 @@ These must exist outside the cluster or the rebuild cannot complete:
       `flux` phase creates the `sops-age` Secret, and any not-yet-migrated sops
       files decrypt with it. (Post-OpenBao-phase-4 this may be retired — see
       `MIGRATION-sops-openbao.md`.)
-- [ ] **OpenBao seal key, recovery key + root token** — in the GPG `pass`
-      store (`homelab/openbao/seal-key`, `homelab/openbao/unseal-key` — the
-      former shamir key, now the recovery key — and `homelab/openbao/root-token`),
-      and the GPG private key backed up off-machine. The seal key is the one
-      the snapshots are encrypted under: without it, no OpenBao snapshot can be
-      restored and no secret-consuming app can start.
+- [ ] **OpenBao seal key, recovery key, restore + pull credentials** — age
+      files in `~/.config/homelab-secrets/` (`seal-key.age`,
+      `openbao-recovery-key.age` — the former shamir key —, `s3-access.age`,
+      `s3-secret.age`, `ghcr-pull.age`, `hetzner-cloud-api-secret.age`),
+      encrypted to both YubiKeys and the
+      offline paper key in the safe; a copy is on nas-2 in
+      `~ladmin/homelab-secrets-backup` (`~/.ssh/ssh-keys-rotate dr-copy`). The
+      root token is `~/.config/openbao/root-token.age` (YubiKeys only). Needed
+      to decrypt: one YubiKey with its PIV PIN, or the paper key. The seal key
+      is the one the snapshots are encrypted under: without it, no OpenBao
+      snapshot can be restored and no secret-consuming app can start.
 - [ ] **OpenBao raft snapshot** reachable — nightly PVC snapshots
       (`infrastructure/openbao/snapshot-cronjob.yaml`) plus the Velero
       `openbao-daily` offsite copy. In a total-loss rebuild the offsite Velero
       copy is the one that survives.
-- [ ] **Tooling:** `talosctl`, `kubectl`, `flux`, `tofu`, `bao`, plus the
+- [ ] **Tooling:** `talosctl`, `kubectl`, `flux`, `tofu`, `bao`, `age` +
+      `age-plugin-yubikey` (the plugin is only needed for the YubiKeys; the
+      paper key needs `age` alone), plus the
       OpenTofu backend + UniFi credentials the `opentofu/` stack needs. No
       Proxmox credentials — there is no Proxmox any more.
 - [ ] **Install media** written and to hand: the Image Factory ISO named in
@@ -299,14 +306,16 @@ output** — a green step over a still-forming Ceph is a known trap.
 ## Step 4 — openbao: restore the snapshot
 
 `bootstrap.sh flux` does this step itself (`restore_openbao`): it creates the
-`openbao-seal-key` Secret from `pass` before Flux brings OpenBao up, initialises
+`openbao-seal-key` Secret from `~/.config/homelab-secrets/seal-key.age` before Flux brings OpenBao up, initialises
 the empty server, restores the newest raft snapshot straight out of the Velero
 kopia repository, and waits until every replica has unsealed itself from the
 static seal. What follows is the same procedure by hand, for when the script
 is not an option.
 
 OpenBao unseals itself from the static seal (`openbao-seal-key` Secret, key
-`key`, the 32-byte base64 key from `pass show homelab/openbao/seal-key`). The
+`key`, the 32-byte base64 key in `~/.config/homelab-secrets/seal-key.age`,
+age-encrypted to the owner's YubiKeys and the offline paper key; a copy sits on
+nas-2 in `~ladmin/homelab-secrets-backup`). The
 snapshots are encrypted under that key, so it must be the same key the backed
 up cluster used — a different one restores nothing. Until the snapshot is
 restored, the `ClusterSecretStore` stays NotReady, ESO cannot materialize
@@ -316,7 +325,8 @@ Secrets, and every secret-consuming app stays Pending.
 # 1. The seal key, before the pods start (a pod without it never starts).
 kubectl create namespace openbao --dry-run=client -o yaml | kubectl apply -f -
 kubectl -n openbao create secret generic openbao-seal-key \
-  --from-literal=key="$(pass show homelab/openbao/seal-key)"
+  --from-literal=key="$(age -d -i ~/.config/age/yubikeys.txt ~/.config/homelab-secrets/seal-key.age)"
+# Both YubiKeys lost: -i <file holding the paper key> instead.
 
 # 2. Wait for openbao-0 (uninitialised: 0/1 Ready is expected), then
 #    initialise it. With an auto-seal, init hands out recovery keys and the
@@ -337,7 +347,8 @@ kubectl -n openbao exec -it openbao-0 -- \
 
 # 4. The server seals against the snapshot's keyring and unseals itself from
 #    the static seal; openbao-1 and openbao-2 follow (OrderedReady). Nothing
-#    to type. Afterwards the tokens and recovery key from pass apply again.
+#    to type. Afterwards the operator's root token
+#    (~/.config/openbao/root-token.age) and recovery key apply again.
 kubectl -n openbao get pods -w
 ```
 
@@ -352,7 +363,7 @@ kubectl -n <ns> get externalsecret                # want: SecretSynced
 > A replica that stays **sealed** after a restart could not use the static
 > seal — the Secret is missing or does not match the key the data was
 > written under. Fix the Secret and restart the pod; `bao operator unseal`
-> with the recovery key (`pass show homelab/openbao/unseal-key`) does not
+> with the recovery key (`~/.config/homelab-secrets/openbao-recovery-key.age`) does not
 > unseal an auto-sealed server and is only for recovery-mode operations.
 > Details + rationale: `MIGRATION-sops-openbao.md`.
 
@@ -434,7 +445,7 @@ it: repo → **Settings → Repository → Mirror Settings** in Gitea, or
   Step 2 no longer uses it. Its `flux` and `wait-apps` phases are still used in
   Steps 3, 5 and 6 and work; do not read this as the whole script being gone.
   Its `flux` phase carries the OpenBao restore (Step 4); the seal key and the
-  other secrets are read from `pass` or streamed in at run time and never live
+  other secrets are decrypted from `~/.config/homelab-secrets` or streamed in at run time and never live
   in CI.
 - **OpenBao unseals itself, but only under the seal key the snapshot was
   written with.** Three raft replicas mean the service survives losing one; a

kubernetes/apps/gitea/kustomization.yaml

@@ -16,6 +16,8 @@ resources:
   - networkpolicy.yaml
   # Nightly prune of old CI image builds (REN-1203).
   - package-retention.yaml
+  # OpenBao SSH CA public key for certificate logins.
+  - ssh-ca.yaml
   - kumamonitors.yaml
 configMapGenerator:
   - name: gitea-package-retention

kubernetes/apps/gitea/ssh-ca.yaml

@@ -0,0 +1,37 @@
+# Public key of the OpenBao SSH user CA (ssh-client-signer), for Gitea's
+# built-in SSH server to accept owner and claude-bot certificates. OpenBao
+# serves it unauthenticated; ESO reads it from the in-cluster Service, so the
+# key has one source. Gitea reads it from env at start; Reloader restarts the
+# StatefulSet when the Secret changes.
+apiVersion: external-secrets.io/v1
+kind: SecretStore
+metadata:
+  name: openbao-ssh-ca
+  namespace: gitea
+spec:
+  provider:
+    webhook:
+      url: http://openbao.openbao.svc.cluster.local:8200/v1/ssh-client-signer/public_key
+      method: GET
+---
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: gitea-ssh-ca
+  namespace: gitea
+spec:
+  refreshInterval: 1h
+  secretStoreRef:
+    kind: SecretStore
+    name: openbao-ssh-ca
+  target:
+    name: gitea-ssh-ca
+    # OpenBao ends the body with a newline; it must not reach app.ini.
+    template:
+      engineVersion: v2
+      data:
+        trusted-user-ca-keys: "{{ .ca | trim }}"
+  data:
+    - secretKey: ca
+      remoteRef:
+        key: public_key

kubernetes/apps/gitea/statefulset.yaml

@@ -5,6 +5,9 @@ metadata:
   namespace: gitea
   labels:
     app.kubernetes.io/name: gitea
+  annotations:
+    # A CA rotation changes gitea-ssh-ca; Gitea reads it only at start.
+    secret.reloader.stakater.com/reload: gitea-ssh-ca
 spec:
   serviceName: gitea
   replicas: 1 # SQLite => single writer. Do NOT scale up without moving to Postgres first.
@@ -85,6 +88,16 @@ spec:
             # HAS CHANGED". Paths are relative to APP_DATA_PATH (/var/lib/gitea/data).
             - name: GITEA__server__SSH_SERVER_HOST_KEYS
               value: "ssh/ssh_host_ed25519_key,ssh/ssh_host_rsa_key"
+            # OpenBao SSH CA (ssh-ca.yaml): certificates log in as the user whose
+            # name is a principal of the cert. Each user registers that
+            # principal once (Settings > SSH / GPG Keys > Manage principals).
+            - name: GITEA__server__SSH_TRUSTED_USER_CA_KEYS
+              valueFrom:
+                secretKeyRef:
+                  name: gitea-ssh-ca
+                  key: trusted-user-ca-keys
+            - name: GITEA__server__SSH_AUTHORIZED_PRINCIPALS_ALLOW
+              value: username
             # Gitea Actions was decommissioned — keep it off.
             - name: GITEA__actions__ENABLED
               value: "false"

kubernetes/apps/monitoring-rules/prometheusrule-openbao.yaml

@@ -49,7 +49,7 @@ spec:
               failure would cost quorum. Check `kubectl -n openbao get pods`,
               the pod log and `bao status` in the not-ready pod. If it is
               sealed, the static seal failed: verify the openbao-seal-key
-              Secret matches `pass show homelab/openbao/seal-key`.
+              Secret matches ~/.config/homelab-secrets/seal-key.age (age, YubiKey).
 
         - alert: OpenBaoQuorumLost
           # Raft quorum of a three-node cluster is two. At one ready replica
@@ -76,7 +76,7 @@ spec:
               surface only when something rotates or a new secret is needed.
               Replicas unseal themselves from the static seal; one that stays
               sealed could not. Check its log and the openbao-seal-key Secret
-              against `pass show homelab/openbao/seal-key`, fix the Secret and
+              against ~/.config/homelab-secrets/seal-key.age, fix the Secret and
               restart the pod. The recovery key cannot unseal an auto-sealed
               server. Procedure: kubernetes/RUNBOOK-cold-rebuild.md, Step 4.
 

kubernetes/apps/wiki/deployment.yaml

@@ -53,7 +53,7 @@ spec:
           type: RuntimeDefault
       containers:
         - name: wiki
-          image: gitea.renner.uno/konairius/wiki:build.1793086240 # {"$imagepolicy": "flux-system:wiki"}
+          image: gitea.renner.uno/konairius/wiki:build.1793094262 # {"$imagepolicy": "flux-system:wiki"}
           securityContext:
             allowPrivilegeEscalation: false
             readOnlyRootFilesystem: true

kubernetes/bootstrap.sh

@@ -35,7 +35,8 @@ set -euo pipefail
 # The mirror is kept current server-side by Gitea's native push-mirror. Full
 # cold-rebuild procedure: kubernetes/RUNBOOK-cold-rebuild.md.
 #
-# Requires: talosctl, kubectl, flux, sops age key at ~/.config/sops/age/keys.txt
+# Requires: talosctl, kubectl, flux, sops age key at ~/.config/sops/age/keys.txt,
+# age + age-plugin-yubikey (the DR secrets in ~/.config/homelab-secrets)
 
 TALOS_DIR="$(cd "$(dirname "$0")/talos" && pwd)"
 REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
@@ -190,13 +191,16 @@ if [[ "$FLUX_SOURCE" == "restore-test" && "$PHASE" != "flux" && "$PHASE" != "wai
 fi
 OPENBAO_NS="openbao"
 
-# Where the secrets come from. On a workstation, the operator's pass store. In
-# the in-cluster Job there is no pass and no GPG key — and there should not be:
-# the pass store opens everything, and it stays on the operator's machine. Set
-# OPENBAO_SECRETS_DIR and the run reads them from files instead, which the
+# Where the secrets come from. On a workstation, age files in
+# HOMELAB_SECRETS_DIR, encrypted to the operator's YubiKeys and an offline paper
+# key (a copy sits on nas-2). HOMELAB_AGE_IDENTITY picks what decrypts them: the
+# YubiKey identity stubs by default (touch per file), the paper key's file when
+# both sticks are gone. In the in-cluster Job there is neither — and there
+# should not be: those keys open everything, and they stay with the operator.
+# Set OPENBAO_SECRETS_DIR and the run reads them from files instead, which the
 # operator streams in when the run asks for them:
 #
-#     pass show homelab/openbao/seal-key \
+#     age -d -i ~/.config/age/yubikeys.txt ~/.config/homelab-secrets/seal-key.age \
 #       | kubectl -n vmtest exec -i <pod> -- sh -c 'cat > /secrets/seal-key'
 #
 # They exist in the pod's filesystem for the length of the run and are never
@@ -240,6 +244,15 @@ git_token_for_source() {
     fi
 }
 
+# Literal ~ on purpose: the commands are eval'd on the workstation, which
+# expands it there, and the in-cluster Job prints them for the operator to run
+# on that workstation, where the Job's own HOME (/work) would be wrong.
+# shellcheck disable=SC2088
+[[ -n "${HOMELAB_SECRETS_DIR:-}" ]] || HOMELAB_SECRETS_DIR='~/.config/homelab-secrets'
+# shellcheck disable=SC2088
+[[ -n "${HOMELAB_AGE_IDENTITY:-}" ]] || HOMELAB_AGE_IDENTITY='~/.config/age/yubikeys.txt'
+AGE_SECRET="age -d -i $HOMELAB_AGE_IDENTITY $HOMELAB_SECRETS_DIR"
+
 # Format: <file in SECRETS_DIR>|<what it is>|<how the operator produces it>
 OPENBAO_REQUIRED_SECRETS=(
     "sops-age.key|SOPS age key|cat ~/.config/sops/age/keys.txt"
@@ -255,18 +268,18 @@ OPENBAO_REQUIRED_SECRETS=(
     # Emptiness is not tested here on purpose — require_secrets rejects an empty
     # file and names what is missing.
     "git-token|token for flux bootstrap, for whichever forge FLUX_SOURCE names|git_token_for_source"
-    "s3-access|Hetzner restore access key|pass show homelab/hetztner/k8s-restore-access"
-    "s3-secret|Hetzner restore secret key|pass show homelab/hetztner/k8s-restore-secret"
+    "s3-access|Hetzner restore access key|$AGE_SECRET/s3-access.age"
+    "s3-secret|Hetzner restore secret key|$AGE_SECRET/s3-secret.age"
     # The static seal key. The snapshots are encrypted under it, so it has to
     # be the one the backed-up cluster ran with — a fresh key restores nothing.
-    "seal-key|OpenBao seal key|pass show homelab/openbao/seal-key"
+    "seal-key|OpenBao seal key|$AGE_SECRET/seal-key.age"
     # Read by the node registry auth (see registry_auth_patch): the images an app
     # needs while the cluster is still being built are mirrored to a private
     # ghcr.io namespace, and a node has to be able to pull them before anything
     # in the cluster exists to hold a credential. Demanded up front like the
     # rest, because a rebuild that discovers it missing has already wiped the
     # disks.
-    "ghcr-pull|GHCR pull token for the image mirror|pass show homelab/github/k8s-package-pull"
+    "ghcr-pull|GHCR pull token for the image mirror|$AGE_SECRET/ghcr-pull.age"
 )
 # Asked for by name only, never by a bare require_secrets: a rebuild of the
 # real cluster cannot depend on the OpenBao it is rebuilding. Scoped to the
@@ -281,8 +294,8 @@ HCLOUD_REQUIRED_SECRETS=(
 # during a real recovery or in the in-cluster Job.
 #
 # The only difference between those two is where the material comes from, and it
-# is confined to this function: on a laptop the commands in the list produce it
-# from pass; in the Job the operator streams it in. Everything downstream cannot
+# is confined to this function: on a laptop the commands in the list decrypt
+# it from HOMELAB_SECRETS_DIR; in the Job the operator streams it in. Everything downstream cannot
 # tell the difference — which is the point, because the recovery path must not be
 # a path that only gets exercised during a recovery.
 # require_secrets [name...]
@@ -311,7 +324,7 @@ require_secrets() {
         wanted=("${OPENBAO_REQUIRED_SECRETS[@]}")
     fi
 
-    # Which branch: fetch from pass, or wait for someone to stream files in.
+    # Which branch: decrypt locally, or wait for someone to stream files in.
     #
     # The condition cannot be "is OPENBAO_SECRETS_DIR set", because THIS
     # FUNCTION sets it. Once a subset call had run — hcloudtest asks for
@@ -322,7 +335,7 @@ require_secrets() {
     # maintenance.
     #
     # SECRETS_SELF_FETCH records which world we are in, and it is decided once:
-    # an empty OPENBAO_SECRETS_DIR on entry means pass, an inherited one means
+    # an empty OPENBAO_SECRETS_DIR on entry means local age decryption, an inherited one means
     # the Job. Later calls follow that decision instead of re-deriving it from
     # state this function has since changed.
     if [[ -z "$OPENBAO_SECRETS_DIR" || "${SECRETS_SELF_FETCH:-0}" == "1" ]]; then
@@ -342,12 +355,14 @@ require_secrets() {
             name=${entry%%|*}; what=${entry#*|}; how=${what#*|}; what=${what%%|*}
             # Already collected by an earlier call — a subset call fetches only
             # what it asked for, so the full call that follows must fill the
-            # gaps rather than re-run every command. `pass show` on a locked
-            # store prompts, and prompting twice for the same key is how an
+            # gaps rather than re-run every command. Each decryption waits for
+            # a YubiKey touch, and asking twice for the same key is how an
             # unattended run turns into a hung one.
             [[ -s "$OPENBAO_SECRETS_DIR/$name" ]] && continue
-            if ! eval "$how" > "$OPENBAO_SECRETS_DIR/$name" 2>"$OPENBAO_SECRETS_DIR/.err"; then
-                echo; cat "$OPENBAO_SECRETS_DIR/.err" >&2
+            # stderr stays on the terminal: age-plugin-yubikey asks for the
+            # PIN and the touch there, and a captured prompt is a silent hang.
+            echo
+            if ! eval "$how" > "$OPENBAO_SECRETS_DIR/$name"; then
                 fail "could not obtain the $what with: $how"
             fi
             [[ -s "$OPENBAO_SECRETS_DIR/$name" ]] \
@@ -382,7 +397,7 @@ OPENBAO_NS_HINT=vmtest
 
 # Reads one secret by its name in the list above. In the Job that is the file the
 # operator streamed in; on a workstation it runs the command the list names,
-# which for these is `pass show <path>`.
+# which for these is `age -d` of a file in HOMELAB_SECRETS_DIR.
 #
 # The list is the only place the paths appear. They used to sit here as
 # constants as well, which is how the same value ends up defined twice and one
@@ -1486,7 +1501,8 @@ fi
 
 # Before anything destructive, and after the vmtest block has decided which
 # phases will run. A rebuild that will need these has no business wiping disks
-# first, only to discover an hour later that pass is locked or a token is unset —
+# first, only to discover an hour later that no YubiKey is plugged in or a
+# token is unset —
 # by then the old cluster is gone and the new one cannot be finished.
 #
 # In the Job this is where the file-waiting branch blocks, which is the right
@@ -1994,7 +2010,8 @@ bao_exec() {
 # have been doing this all along.
 #
 # Reading them needs three things, all of which live OUTSIDE the cluster, which
-# is the point: bucket credentials from pass (their own key, not Velero's),
+# is the point: bucket credentials from HOMELAB_SECRETS_DIR (their own key, not
+# Velero's),
 # kopia, and the repository password. The last is not a secret — Velero
 # hardcodes `static-passw0rd` in its source. Worth knowing for what it means:
 # the offsite backup's confidentiality rests on bucket access alone.
@@ -2071,7 +2088,8 @@ restore_openbao() {
     # rather than an unseal key and the server unseals itself; the only thing
     # used from its output is the root token, for the next few seconds, and
     # then it stops existing: restoring replaces the keyring with the
-    # snapshot's, after which the root token and recovery key from pass apply.
+    # snapshot's, after which the operator's root token
+    # (~/.config/openbao/root-token.age) and recovery key apply.
     #
     # No -force. Without it OpenBao checks that the snapshot was written under
     # the seal key this server runs with, and refuses a mismatch by name
@@ -2355,7 +2373,7 @@ if phase_runs flux; then
     # Secret would leave every replica sealed for good at its next restart.
     # That is a migration, not a bootstrap step.
     #
-    # --from-literal, not --from-file: `pass show` ends its output with a
+    # --from-literal, not --from-file: the stored value ends with a
     # newline, the substitution strips it, and OpenBao rejects a key that is
     # not exactly 32 bytes of base64. The key sits in this process's argument
     # list for the length of the call, as it did in the pod's before.

kubernetes/infrastructure/crowdsec/helmrelease.yaml

@@ -80,6 +80,25 @@ spec:
           # community blocklists.
           - 162.55.50.97/32
           - 2a01:4f8:c2c:4579::1/128
+      postoverflows:
+        s01-whitelist:
+          # http-crawl-non_statics counts requests regardless of outcome, so a
+          # signed-in user of any image-heavy UI (Immich timeline, Music
+          # Assistant covers) trips it with a burst of 200s. Scanners trip it
+          # with 404s for paths that do not exist. Drop the overflow unless at
+          # least half of its requests were client errors (the alert carries
+          # the scenario's last 5 events, cache_size: 5). Only this scenario
+          # is affected; probing/sensitive-files/cve still fire on their own.
+          crawl-non-statics-success-whitelist.yaml: |
+            name: krenner/crawl-non-statics-success-whitelist
+            description: "Ignore http-crawl-non_statics bursts that mostly succeeded"
+            whitelist:
+              reason: "crawl burst of successful requests is a UI, not a scanner"
+              expression:
+                - >-
+                  evt.Overflow.Alert.GetScenario() == 'crowdsecurity/http-crawl-non_statics' &&
+                  count(evt.Overflow.Alert.Events, {.GetMeta('http_status') startsWith '4'}) * 2
+                  < len(evt.Overflow.Alert.Events)
 
     # mTLS between agents and the LAPI. Do not switch back to password auth:
     # the LAPI runs without persistence, so its machine-registration DB is

opentofu/locals.tf

@@ -1,9 +0,0 @@
-locals {
-  common_ssh_keys = join("\n", [
-    "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPzFNBP3bq2m/6JqDqod9w+FP/uxXxHMolqhq3Ojflr1 krenner@fortress",
-    "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGfe3HJqFR+jIGrboweseAO7XjIVE24sje/b1HybgU9C ansible-runner@ci-new",
-    "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICyNnX1I4W9jyDvCJ1DJauHSmbo1ZXSyBxVhSAwMWgen woodpecker-ci@homelab",
-    "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBB1OFjh6Q5xeioI6Cwfuv8Xka27veomYy9tKBbpMo8mcrsB4yIN4FX8HovBLtogetBhwBUefBar8nhA9GbRh6HQ= krenner@krenner-XPS-13-9310"
-  ])
-
-}