Merge pull request 'fix(ansible): drop node-exporter from the state' (#1497) from drop-node-exporter into main

Reviewed-on: #1497

.woodpecker/deploy-claude-workstation.yml

@@ -1,33 +0,0 @@
-when:
-  # Deploy only on main (merge) + manual — never auto-deploy from a branch.
-  - event: push
-    branch: main
-    path:
-      - "ansible/roles/claude-workstation/**"
-      - "ansible/plays/install-claude-workstation.yml"
-      - ".woodpecker/deploy-claude-workstation.yml"
-  - event: deployment
-    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "deploy-claude-workstation"'
-
-timeout: 30
-
-steps:
-  - name: deploy-claude-workstation
-    image: gitea.renner.uno/gitea_admin/ci-ansible:2026-04-29
-    environment:
-      ANSIBLE_FORCE_COLOR: "true"
-      ANSIBLE_HOST_KEY_CHECKING: "false"
-      ANSIBLE_ROLES_PATH: "../roles:./roles"
-      SSH_PRIVATE_KEY:
-        from_secret: ssh_private_key
-      VAULT_PASS:
-        from_secret: vault_pass
-    commands:
-      - mkdir -p ~/.ssh
-      - echo "$SSH_PRIVATE_KEY" | base64 -d > ~/.ssh/id_ed25519
-      - chmod 600 ~/.ssh/id_ed25519
-      - printf "Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile /dev/null\n" > ~/.ssh/config
-      - cd ansible
-      - echo "$VAULT_PASS" > .vault_pass && chmod 600 .vault_pass
-      - export ANSIBLE_VAULT_PASSWORD_FILE=.vault_pass
-      - ansible-playbook -i inventory/ plays/install-claude-workstation.yml -v

.woodpecker/deploy-edge-mqtt-bridge.yml

@@ -1,36 +0,0 @@
-when:
-  # Deploy only on main (merge) + manual — never auto-deploy from a branch.
-  - event: push
-    branch: main
-    path:
-      - "ansible/roles/mqtt-bridge/**"
-      - "ansible/plays/edge-mqtt-bridge.yml"
-      - "ansible/inventory/group_vars/edge_host/**"
-      - ".woodpecker/deploy-edge-mqtt-bridge.yml"
-  - event: deployment
-    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "deploy-edge-mqtt-bridge"'
-
-timeout: 30
-
-steps:
-  - name: deploy-edge-mqtt-bridge
-    image: gitea.renner.uno/gitea_admin/ci-ansible:2026-04-29
-    environment:
-      ANSIBLE_FORCE_COLOR: "true"
-      ANSIBLE_HOST_KEY_CHECKING: "false"
-      ANSIBLE_ROLES_PATH: "../roles:./roles"
-      SSH_PRIVATE_KEY:
-        from_secret: ssh_private_key
-      VAULT_PASS:
-        from_secret: vault_pass
-    commands:
-      - mkdir -p ~/.ssh
-      - echo "$SSH_PRIVATE_KEY" | base64 -d > ~/.ssh/id_ed25519
-      - chmod 600 ~/.ssh/id_ed25519
-      - printf "Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile /dev/null\n" > ~/.ssh/config
-      - cd ansible
-      - echo "$VAULT_PASS" > .vault_pass && chmod 600 .vault_pass
-      - export ANSIBLE_VAULT_PASSWORD_FILE=.vault_pass
-      # Local MQTT bridge on the edge hosts only (REN-1307); restarts only the
-      # local mosquitto on edge-1, never the cluster broker.
-      - ansible-playbook -i inventory/ plays/edge-mqtt-bridge.yml -v

.woodpecker/deploy-infrastructure.yml

@@ -109,8 +109,6 @@ steps:
         from_secret: hetzner_s3_access_key
       TF_VAR_hetzner_s3_secret_key:
         from_secret: hetzner_s3_secret_key
-      SSH_PRIVATE_KEY:
-        from_secret: ssh_private_key
       PROXMOX_TOKEN_ID:
         from_secret: proxmox_token_id
       PROXMOX_TOKEN_SECRET:
@@ -121,11 +119,6 @@ steps:
       ANSIBLE_ROLES_PATH: "../roles:./roles"
       ANSIBLE_FORCE_COLOR: "true"
     commands:
-      - mkdir -p ~/.ssh
-      - echo "$SSH_PRIVATE_KEY" | base64 -d > ~/.ssh/id_ed25519
-      - chmod 600 ~/.ssh/id_ed25519
-      - export TF_VAR_ssh_private_key="$(cat ~/.ssh/id_ed25519)"
-      - printf "Host *\n\tStrictHostKeyChecking no\n" > ~/.ssh/config
       - echo "$VAULT_PASS" > /tmp/.vault_pass && chmod 600 /tmp/.vault_pass
       - export ANSIBLE_VAULT_PASSWORD_FILE=/tmp/.vault_pass
       - tofu init -upgrade -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY"

.woodpecker/deploy-netbird-client.yml

@@ -1,35 +0,0 @@
-when:
-  # Deploy only on main (merge) + manual — never auto-deploy from a branch.
-  - event: push
-    branch: main
-    path:
-      - "ansible/roles/netbird-client/**"
-      - "ansible/plays/install-netbird-client.yml"
-      - "ansible/inventory/group_vars/claude_workstation/**"
-      - ".woodpecker/deploy-netbird-client.yml"
-  - event: deployment
-    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "deploy-netbird-client"'
-
-timeout: 30
-
-steps:
-  - name: deploy-netbird-client
-    image: gitea.renner.uno/gitea_admin/ci-ansible:2026-04-29
-    environment:
-      ANSIBLE_FORCE_COLOR: "true"
-      ANSIBLE_HOST_KEY_CHECKING: "false"
-      ANSIBLE_ROLES_PATH: "../roles:./roles"
-      SSH_PRIVATE_KEY:
-        from_secret: ssh_private_key
-      VAULT_PASS:
-        from_secret: vault_pass
-    commands:
-      - mkdir -p ~/.ssh
-      - echo "$SSH_PRIVATE_KEY" | base64 -d > ~/.ssh/id_ed25519
-      - chmod 600 ~/.ssh/id_ed25519
-      - printf "Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile /dev/null\n" > ~/.ssh/config
-      - cd ansible
-      - echo "$VAULT_PASS" > .vault_pass && chmod 600 .vault_pass
-      - export ANSIBLE_VAULT_PASSWORD_FILE=.vault_pass
-      # Enrol only the off-site Claude workstation into the Netbird mesh.
-      - ansible-playbook -i inventory/ plays/install-netbird-client.yml --limit claude_workstation -v

.woodpecker/deploy-netbird-proxy.yml

@@ -1,54 +0,0 @@
-when:
-  # Deploy only on main (merge) + manual — never auto-deploy from a branch.
-  - event: push
-    branch: main
-    path:
-      - "ansible/roles/netbird-proxy/**"
-      # install-netbird-proxy.yml also applies the netbird-proxy-watchdog role,
-      # so changes to it must redeploy this host too.
-      - "ansible/roles/netbird-proxy-watchdog/**"
-      # install-netbird-proxy.yml also applies the edge bouncer.
-      - "ansible/roles/crowdsec-edge-bouncer/**"
-      - "ansible/plays/install-netbird-proxy.yml"
-      # The variables that steer those roles, not just the roles themselves.
-      # Without these two, a change that only touches inventory never deploys
-      # and nothing says so: arming the bouncer lives in host_vars, and its
-      # LAPI key lives in group_vars, so both would sit in git looking applied
-      # while the hosts kept running the previous configuration.
-      #
-      # pve_hosts.yml is deliberately NOT listed. It is the single inventory
-      # file for the whole fleet, so listing it would redeploy the proxies for
-      # any unrelated host edit. The cost is that a change to [netbird_proxy]
-      # group membership needs a manual deployment run.
-      - "ansible/inventory/host_vars/proxy-*.yml"
-      - "ansible/inventory/group_vars/netbird_proxy/**"
-      - ".woodpecker/deploy-netbird-proxy.yml"
-  - event: deployment
-    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "deploy-netbird-proxy"'
-
-timeout: 30
-
-steps:
-  - name: deploy-netbird-proxy
-    image: gitea.renner.uno/gitea_admin/ci-ansible:2026-04-29
-    environment:
-      ANSIBLE_FORCE_COLOR: "true"
-      ANSIBLE_HOST_KEY_CHECKING: "false"
-      ANSIBLE_ROLES_PATH: "../roles:./roles"
-      SSH_PRIVATE_KEY:
-        from_secret: ssh_private_key
-      VAULT_PASS:
-        from_secret: vault_pass
-      PROXMOX_TOKEN_ID:
-        from_secret: proxmox_token_id
-      PROXMOX_TOKEN_SECRET:
-        from_secret: proxmox_token_secret
-    commands:
-      - mkdir -p ~/.ssh
-      - echo "$SSH_PRIVATE_KEY" | base64 -d > ~/.ssh/id_ed25519
-      - chmod 600 ~/.ssh/id_ed25519
-      - printf "Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile /dev/null\n" > ~/.ssh/config
-      - cd ansible
-      - echo "$VAULT_PASS" > .vault_pass && chmod 600 .vault_pass
-      - export ANSIBLE_VAULT_PASSWORD_FILE=.vault_pass
-      - ansible-playbook -i inventory/ plays/install-netbird-proxy.yml -v

.woodpecker/deploy-netbird-server.yml

@@ -1,37 +0,0 @@
-when:
-  # Deploy only on main (merge) + manual — never auto-deploy from a branch.
-  - event: push
-    branch: main
-    path:
-      - "ansible/roles/netbird-server/**"
-      - "ansible/plays/install-netbird-server.yml"
-      - ".woodpecker/deploy-netbird-server.yml"
-  - event: deployment
-    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "deploy-netbird-server"'
-
-timeout: 30
-
-steps:
-  - name: deploy-netbird-server
-    image: gitea.renner.uno/gitea_admin/ci-ansible:2026-04-29
-    environment:
-      ANSIBLE_FORCE_COLOR: "true"
-      ANSIBLE_HOST_KEY_CHECKING: "false"
-      ANSIBLE_ROLES_PATH: "../roles:./roles"
-      SSH_PRIVATE_KEY:
-        from_secret: ssh_private_key
-      VAULT_PASS:
-        from_secret: vault_pass
-      PROXMOX_TOKEN_ID:
-        from_secret: proxmox_token_id
-      PROXMOX_TOKEN_SECRET:
-        from_secret: proxmox_token_secret
-    commands:
-      - mkdir -p ~/.ssh
-      - echo "$SSH_PRIVATE_KEY" | base64 -d > ~/.ssh/id_ed25519
-      - chmod 600 ~/.ssh/id_ed25519
-      - printf "Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile /dev/null\n" > ~/.ssh/config
-      - cd ansible
-      - echo "$VAULT_PASS" > .vault_pass && chmod 600 .vault_pass
-      - export ANSIBLE_VAULT_PASSWORD_FILE=.vault_pass
-      - ansible-playbook -i inventory/ plays/install-netbird-server.yml -v

.woodpecker/deploy-ssh-ca.yml

@@ -1,37 +0,0 @@
-when:
-  # Deploy only on main (merge) + manual — never auto-deploy from a branch.
-  - event: push
-    branch: main
-    path:
-      - "ansible/roles/ssh-ca/**"
-      - "ansible/plays/configure-ssh-ca.yml"
-      - "ansible/inventory/pve_hosts.yml"
-      - "ansible/inventory/group_vars/all/**"
-      - "ansible/inventory/group_vars/netbird_proxy/vars.yml"
-      - "ansible/inventory/host_vars/netbird.renner.uno.yml"
-      - ".woodpecker/deploy-ssh-ca.yml"
-  - event: deployment
-    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "deploy-ssh-ca"'
-
-timeout: 30
-
-steps:
-  - name: deploy-ssh-ca
-    image: gitea.renner.uno/gitea_admin/ci-ansible:2026-04-29
-    environment:
-      ANSIBLE_FORCE_COLOR: "true"
-      ANSIBLE_HOST_KEY_CHECKING: "false"
-      ANSIBLE_ROLES_PATH: "../roles:./roles"
-      SSH_PRIVATE_KEY:
-        from_secret: ssh_private_key
-      VAULT_PASS:
-        from_secret: vault_pass
-    commands:
-      - mkdir -p ~/.ssh
-      - echo "$SSH_PRIVATE_KEY" | base64 -d > ~/.ssh/id_ed25519
-      - chmod 600 ~/.ssh/id_ed25519
-      - printf "Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile /dev/null\n" > ~/.ssh/config
-      - cd ansible
-      - echo "$VAULT_PASS" > .vault_pass && chmod 600 .vault_pass
-      - export ANSIBLE_VAULT_PASSWORD_FILE=.vault_pass
-      - ansible-playbook -i inventory/ plays/configure-ssh-ca.yml -v

.woodpecker/deploy-uptime-kuma.yml

@@ -1,13 +1,4 @@
 when:
-  # Deploy only on main (merge) + manual — never auto-deploy from a branch.
-  - event: push
-    branch: main
-    path:
-      - "ansible/roles/uptime-kuma/**"
-      - "ansible/plays/install-uptime-kuma.yml"
-      - ".woodpecker/deploy-uptime-kuma.yml"
-  - event: deployment
-    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "deploy-uptime-kuma"'
   - event: cron
     cron: uptime-kuma-token-renew
 
@@ -21,27 +12,3 @@ steps:
         from_secret: openbao_uptime_kuma_token
     commands:
       - 'curl -sSf -X POST -H "X-Vault-Token: $VAULT_TOKEN" https://openbao.renner.uno/v1/auth/token/renew-self -o /dev/null'
-
-  - name: deploy-uptime-kuma
-    image: gitea.renner.uno/gitea_admin/ci-ansible:2026-10-18
-    when:
-      - event: [push, deployment]
-    environment:
-      ANSIBLE_FORCE_COLOR: "true"
-      ANSIBLE_HOST_KEY_CHECKING: "false"
-      ANSIBLE_ROLES_PATH: "../roles:./roles"
-      SSH_PRIVATE_KEY:
-        from_secret: ssh_private_key
-      VAULT_PASS:
-        from_secret: vault_pass
-      VAULT_TOKEN:
-        from_secret: openbao_uptime_kuma_token
-    commands:
-      - mkdir -p ~/.ssh
-      - echo "$SSH_PRIVATE_KEY" | base64 -d > ~/.ssh/id_ed25519
-      - chmod 600 ~/.ssh/id_ed25519
-      - printf "Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile /dev/null\n" > ~/.ssh/config
-      - cd ansible
-      - echo "$VAULT_PASS" > .vault_pass && chmod 600 .vault_pass
-      - export ANSIBLE_VAULT_PASSWORD_FILE=.vault_pass
-      - ansible-playbook -i inventory/ plays/install-uptime-kuma.yml -v

.woodpecker/update-os.yml

@@ -1,28 +0,0 @@
-when:
-  # Manual only: reboots edge-1 (Z-Wave, heating CAN, sole NTP server) and the
-  # NetBird control plane. Run inside the 02:00-05:00 Europe/Berlin window.
-  - event: deployment
-    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "update-os"'
-
-timeout: 120
-
-steps:
-  - name: update-os
-    image: gitea.renner.uno/gitea_admin/ci-ansible:2026-04-29
-    environment:
-      ANSIBLE_FORCE_COLOR: "true"
-      ANSIBLE_HOST_KEY_CHECKING: "false"
-      ANSIBLE_ROLES_PATH: "../roles:./roles"
-      SSH_PRIVATE_KEY:
-        from_secret: ssh_private_key
-      VAULT_PASS:
-        from_secret: vault_pass
-    commands:
-      - mkdir -p ~/.ssh
-      - echo "$SSH_PRIVATE_KEY" | base64 -d > ~/.ssh/id_ed25519
-      - chmod 600 ~/.ssh/id_ed25519
-      - printf "Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile /dev/null\n" > ~/.ssh/config
-      - cd ansible
-      - echo "$VAULT_PASS" > .vault_pass && chmod 600 .vault_pass
-      - export ANSIBLE_VAULT_PASSWORD_FILE=.vault_pass
-      - ansible-playbook -i inventory/ plays/update-os.yml -v

ansible/ansible.cfg

@@ -112,7 +112,7 @@
 
 # (string) This setting controls the default policy of fact gathering (facts discovered about remote systems).
 # This option can be useful for those wishing to save fact gathering time. Both 'smart' and 'explicit' will use the cache plugin.
-;gathering=implicit
+gathering=smart
 
 # (list) Set the `gather_subset` option for the :ref:`ansible_collections.ansible.builtin.setup_module` task in the implicit fact gathering. See the module documentation for specifics.
 # It does **not** apply to user defined ``ansible.builtin.setup`` tasks.
@@ -134,7 +134,7 @@
 ;hash_behaviour=replace
 
 # (pathlist) Comma separated list of Ansible inventory sources
-inventory=~/source/homelab/ansible/inventory
+inventory=inventory
 
 # (pathspec) Colon separated paths in which Ansible will search for HttpApi Plugins.
 ;httpapi_plugins={{ ANSIBLE_HOME ~ "/plugins/httpapi:/usr/share/ansible/plugins/httpapi" }}
@@ -206,7 +206,7 @@ inventory=~/source/homelab/ansible/inventory
 ;poll_interval=15
 
 # (path) Option for connections using a certificate or key file to authenticate, rather than an agent or passwords, you can set the default value here to avoid re-specifying --private-key with every invocation.
-private_key_file=~/.ssh/id_ed25519
+;private_key_file=
 
 # (boolean) By default, imported roles publish their variables to the play and other roles, this setting can avoid that.
 # This was introduced as a way to reset role variables to default values if a role is used more than once in a playbook.
@@ -274,7 +274,7 @@ roles_path=../roles:./roles
 
 # (path) The vault password file to use. Equivalent to --vault-password-file or --vault-id
 # If executable, it will be run and the resulting stdout will be used as the password.
-vault_password_file=~/source/homelab/ansible/vault-password.sh
+vault_password_file=vault-password.sh
 
 # (integer) Sets the default verbosity, equivalent to the number of ``-v`` passed in the command line.
 ;verbosity=0
@@ -484,7 +484,7 @@ vault_password_file=~/source/homelab/ansible/vault-password.sh
 # It can result in a very significant performance improvement when enabled.
 # However this conflicts with privilege escalation (become). For example, when using 'sudo:' operations you must first disable 'requiretty' in /etc/sudoers on all managed hosts, which is why it is disabled by default.
 # This setting will be disabled if ``ANSIBLE_KEEP_REMOTE_FILES`` is enabled.
-;pipelining=False
+pipelining=True
 
 
 [colors]
@@ -694,3 +694,8 @@ vault_password_file=~/source/homelab/ansible/vault-password.sh
 # (list) default list of tags to skip in your plays, has precedence over Run Tags
 ;skip=
 
+
+[ssh_connection]
+# Connections stay open across the plays of a run, so each host costs one
+# YubiKey PIN; plays/close-connections.yml closes them at the end of a run.
+ssh_args=-C -o ControlMaster=auto -o ControlPersist=30m

ansible/docs/edge-host.md

@@ -41,13 +41,11 @@ Build and boot the dedicated installer image — see
 4. Add the host to `[edge_host]` in `inventory/pve_hosts.yml`, by name.
    That group is the base: every edge thin client belongs to it, and it
    assumes no peripherals.
-5. From the workstation:
+5. From the workstation, in `ansible/`:
    ```
-   ansible-playbook -i inventory/pve_hosts.yml plays/install-edge-host.yml
+   ansible-playbook site.yml
    ```
-   One unrestricted run over the whole group. The play is self-contained --
-   it carries `certificates` and `common` itself, so there is no second
-   command and no `--limit`. Running it converges every edge host, which is
+   One unrestricted run, no `--limit`. It converges every host, which is
    the point: a run that touches only the new box tells you nothing about
    the others.
 
@@ -245,8 +243,7 @@ its own server key, nothing else. On a push it writes
 `/etc/mosquitto/bridge.d/edge-1.conf`, deletes `bridge.d/legacy.conf` and
 restarts the local mosquitto. The controller sets a new password whenever the
 bridge reports "not connected" (also after a reinstall), so no play run is
-needed for a rotation. `.woodpecker/deploy-edge-mqtt-bridge.yml` runs
-`plays/edge-mqtt-bridge.yml` on merge. Diagnosis: `journalctl -u
+needed for a rotation. Diagnosis: `journalctl -u
 edge-mqtt-cred`, and `mosquitto_sub -t '$SYS/broker/connection/edge-1/state'`
 (retained `1` = connected).
 

ansible/inventory/group_vars/all/vars.yml

@@ -5,37 +5,10 @@ pve_token_id: "ansible"
 ansible_user: "ladmin"
 ansible_python_interpreter: /usr/bin/python3
 
-# The only plain keys the admin logins accept on [ssh_keys_managed] hosts
+# The only plain key the admin logins accept on [ssh_keys_managed] hosts
 # (ssh-ca role, tasks/authorized_keys.yml). People log in with OpenBao SSH CA
-# certificates; these two are for when that path is unavailable.
+# certificates; this one is for when that path is unavailable.
 #   breakglass-neo: PIV key (slot 9a, PIN) on the offline YubiKey NEO.
 #     ssh -I /usr/lib/opensc-pkcs11.so root@<host>
-#   woodpecker-ci: the CI deploy key, from ssh_ci_key_from where set.
 ssh_admin_authorized_keys:
   - key: "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBKvSl9LUP8u3S2Sq5YTWpW0Jxbue8cJhD7/LMFQbEsa3xo145HVR7pBzRxW71MUy+jgrgmpSlDhIb606Hswhvoo= breakglass-neo-6532348"
-  - key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICyNnX1I4W9jyDvCJ1DJauHSmbo1ZXSyBxVhSAwMWgen woodpecker-ci@homelab"
-    options: "{{ ('from=\"' ~ ssh_ci_key_from ~ '\"') if ssh_ci_key_from else '' }}"
-
-# Source the CI key is accepted from. LAN hosts see the pod address; hosts on
-# the internet see the home WAN address (NAT, dynamic) and set this empty in
-# their group/host vars.
-ssh_ci_key_from: "{{ ci_ssh_from_cidrs }}"
-
-# Network-of-origin restriction for the woodpecker-ci@homelab CI key
-# (issue #184). Used as the authorized_keys `from=` option so the CI key only
-# authenticates from within the cluster's pod network — NOT from any
-# host-pinned IP.
-#
-# The in-cluster Woodpecker step pods SSH to fleet hosts using their POD IP as
-# source, not a node IP: Cilium runs `ipv4-native-routing-cidr=10.0.0.0/8` +
-# `ipv6-native-routing-cidr=fd00::/8` with masquerade enabled, so pod egress to
-# any 10.x / fd.. LAN destination is NOT SNAT'd, and each node's PodCIDR is
-# advertised to the UDM over BGP (kubernetes/infrastructure/post-crds/
-# cilium-bgp.yaml) so return traffic L3-routes back to the pod. Hosts therefore
-# see the real pod source IP.
-#
-# The cluster pod subnets (kubernetes/talos/patches/cluster.yaml podSubnets)
-# are the stable boundary — they survive node DHCP-lease churn and pod
-# rescheduling across k8s-1/2/3, unlike pinning any node's current IP.
-# v6 is included because some SSH targets (edge IoT hosts) are IPv6-only.
-ci_ssh_from_cidrs: '10.244.0.0/16,fd10:244::/48'

ansible/inventory/group_vars/claude_workstation/vault.yml

@@ -1,16 +0,0 @@
-$ANSIBLE_VAULT;1.1;AES256
-35646538363164363061366433343963623166336630376233333033646637336266343337343230
-3233356135323762636439336466663430366238363338650a633762623366643137303333383634
-62623739363132303335636261626263333332623066303161633466306431623930666166323833
-6439663964353338350a656435626664646666363831363164393335386666356637613037323838
-63653461656366613331353064656161346263356131346437666632386166623037373363333534
-37323435623366643534633333303534313434613135653237303966656162633266623433363934
-36343133393834393765353338383764303664363431653865633764306138376366623832346262
-36303766623331666632613639366139333661636665363631326565653938343637323861653064
-37393465383263376337376639613162306563653365623036376662303030613264653534353332
-66363030356363373439306164386561363734353631363738656236633164633062613437663366
-62386430396262366336316166353366383233316333333234666538643962336361363630363762
-31653963303032373531386566336632336130383034343430303964396538663237346434613737
-39643765623766373639306132316635376434643232336163303066393736313130323065666232
-39393038616162666666383837326137376233613437636361636238663566336365653863643933
-353338646134383038323134323034653564

ansible/inventory/group_vars/edge_host/vars.yml

@@ -50,3 +50,11 @@ edge_mqtt_host: 127.0.0.1
 edge_mqtt_port: 1883
 edge_open3e_mqtt_user: open3e
 edge_open3e_mqtt_topic: open3e
+
+# Units claude-bot may restart (ssh-ca role); the containers go through docker.
+ssh_ca_claude_sudo_units:
+  - mosquitto
+  - edge-mqtt-cred
+  - edge-alloy
+  - keepalived
+  - chronyd

ansible/inventory/group_vars/netbird_proxy/vars.yml

@@ -1,4 +1,9 @@
 ---
-# CI reaches these internet hosts NATed from the home WAN address, never from
-# the pod network, so the CI key carries no from= restriction here.
-ssh_ci_key_from: ""
+# The host NetBird client (wt0) must not take over DNS: resolv.conf is pinned to
+# the systemd-resolved stub by the netbird-proxy role.
+netbird_disable_dns: true
+
+ssh_ca_claude_sudo_units:
+  - netbird
+  - netbird-proxy-flood-watchdog
+  - crowdsec-firewall-bouncer

ansible/inventory/host_vars/netbird.renner.uno.yml

@@ -3,6 +3,5 @@
 # Enable Docker monitoring (cAdvisor) for container metrics
 docker_monitoring_enabled: true
 
-# CI reaches this internet host NATed from the home WAN address, never from
-# the pod network, so the CI key carries no from= restriction here.
-ssh_ci_key_from: ""
+ssh_ca_claude_sudo_units:
+  - netbird

ansible/inventory/pve_hosts.yml

@@ -13,11 +13,11 @@
 # Their hosts no longer resolve, except the three that resolve to Talos nodes.
 
 [netbird_server]
-netbird.renner.uno ansible_user=root
+netbird.renner.uno ansible_host=netbird.nb.renner.uno ansible_user=root
 
 [netbird_proxy]
-proxy-1.renner.uno ansible_user=root
-proxy-2.renner.uno ansible_user=root
+proxy-1.renner.uno ansible_host=proxy-1.nb.renner.uno ansible_user=root
+proxy-2.renner.uno ansible_host=proxy-2.nb.renner.uno ansible_user=root
 
 # Every bare-metal edge thin client. Gets the host-generic roles only:
 # flash-wear handling, the log shipper, the MQTT bridge, the VLAN
@@ -62,14 +62,6 @@ heating_can
 [ntp_cluster:children]
 edge_host
 
-# Off-site Debian VM (Synology VMM on nas-2, 10.3.40.0/24) running Claude Code.
-# Reached over the site-to-site tunnel at its current DHCP lease. Switch
-# ansible_host to the stable Netbird mesh IP once the host is enrolled — the
-# obg.renner.uno FQDN is split-horizon and resolves to the public proxy from
-# home, so it must NOT be used as ansible_host.
-[claude_workstation]
-claude-ws ansible_host=10.3.40.78 ansible_user=root
-
 # Hosts that trust the OpenBao SSH user CA (plays/configure-ssh-ca.yml).
 # Rolled out one host at a time: add the next host after the previous one
 # was verified.
@@ -79,11 +71,19 @@ proxy-2.renner.uno
 netbird.renner.uno
 edge-1.iot.renner.uno
 
-# Hosts whose admin logins accept only ssh_admin_authorized_keys (break-glass
-# + CI); everything else needs a CA certificate. Members must be in [ssh_ca].
+# Hosts whose admin logins accept only ssh_admin_authorized_keys (break-glass);
+# everything else needs a CA certificate. Members must be in [ssh_ca].
 # Added one host at a time after the [ssh_ca] report was reviewed.
 [ssh_keys_managed]
 proxy-1.renner.uno
 proxy-2.renner.uno
 netbird.renner.uno
 edge-1.iot.renner.uno
+
+# Internet hosts whose sshd listens on the NetBird interface only
+# (plays/configure-ssh-mesh-only.yml). Members need the host NetBird client.
+# Added one host at a time after the previous one was verified.
+[ssh_mesh_only]
+proxy-1.renner.uno
+proxy-2.renner.uno
+netbird.renner.uno

ansible/plays/close-connections.yml

@@ -0,0 +1,11 @@
+---
+# Last play of every playbook that is run directly (site.yml and the one-off
+# actions). SSH connections stay open for 30 minutes (ansible.cfg) so a run
+# costs one YubiKey PIN per host; closing them at the end leaves nothing that
+# another process of the same user could reuse.
+- name: Close the SSH connections
+  hosts: all
+  gather_facts: false
+  tasks:
+    - name: Close the connection
+      ansible.builtin.meta: reset_connection

ansible/plays/common.yml

@@ -13,4 +13,5 @@
 
   roles:
     - common
-    - qemu-guest-agent
+    - role: qemu-guest-agent
+      when: ansible_facts['virtualization_role'] == 'guest'

ansible/plays/configure-ssh-ca.yml

@@ -2,11 +2,12 @@
 # Trust the OpenBao SSH user CA (ssh-client-signer) on the hosts in [ssh_ca]:
 # owner certificates (principal k.renner) for root/ladmin, Claude's 1h
 # certificates (principal claude-bot) for its own claude-bot account with
-# limited sudo. The first play only reports the admin logins' authorized_keys;
-# the second reduces them to ssh_admin_authorized_keys on [ssh_keys_managed].
+# limited sudo. The role reports the admin logins' authorized_keys; on
+# [ssh_keys_managed] they are then reduced to ssh_admin_authorized_keys.
 #
 # serial: 1 so a host whose sshd rejects the change stops the run before the
-# next host is touched.
+# next host is touched. One play, so each host needs one login (one YubiKey
+# PIN from a workstation).
 - name: Configure the SSH user CA
   hosts: ssh_ca
   become: true
@@ -14,14 +15,9 @@
   serial: 1
   roles:
     - ssh-ca
-
-- name: Reduce authorized_keys to break-glass and CI
-  hosts: ssh_keys_managed
-  become: true
-  gather_facts: false
-  serial: 1
   tasks:
-    - name: Set the managed authorized_keys
+    - name: Reduce authorized_keys to the break-glass key
       ansible.builtin.include_role:
         name: ssh-ca
         tasks_from: authorized_keys
+      when: inventory_hostname in groups['ssh_keys_managed']

ansible/plays/configure-ssh-mesh-only.yml

@@ -0,0 +1,15 @@
+---
+# sshd on the hosts in [ssh_mesh_only] accepts connections only on the NetBird
+# interface; nothing listens on their public addresses. Run from a workstation
+# in the mesh. A switchover reverts by itself unless the play logs in again
+# through the new listener.
+#
+# serial: 1 so a host that does not come back stops the run before the next
+# host is touched.
+- name: Restrict sshd to the mesh interface
+  hosts: ssh_mesh_only
+  become: true
+  gather_facts: true
+  serial: 1
+  roles:
+    - ssh-mesh-only

ansible/plays/edge-mqtt-bridge.yml

@@ -1,33 +0,0 @@
----
-# Local MQTT bridge and its credential receiver only (REN-1307). Run by
-# .woodpecker/deploy-edge-mqtt-bridge.yml on merge, so a change to this role
-# reaches edge-1 without a play run by hand. The full base play
-# (install-edge-host.yml) carries the same role.
-- name: Converge the edge MQTT bridge
-  hosts: edge_host
-  become: true
-
-  # The SML power-meter reader (sml2mqtt) is retired for good: owner decision
-  # 2026-09-19, grid data comes from the E380 on the CAN bus. Its role and
-  # play are gone; this removes the leftover compose project from edge-1 on
-  # the next pipeline run. Idempotent: a no-op once the directory is gone.
-  pre_tasks:
-    - name: Check for the retired sml2mqtt compose project
-      ansible.builtin.stat:
-        path: /opt/edge-host/sml2mqtt/docker-compose.yml
-      register: edge_sml2mqtt_compose
-
-    - name: Stop the retired sml2mqtt container
-      ansible.builtin.command:
-        cmd: docker compose down --remove-orphans
-        chdir: /opt/edge-host/sml2mqtt
-      when: edge_sml2mqtt_compose.stat.exists
-      changed_when: true
-
-    - name: Remove the retired sml2mqtt directory
-      ansible.builtin.file:
-        path: /opt/edge-host/sml2mqtt
-        state: absent
-
-  roles:
-    - mqtt-bridge

ansible/plays/install-claude-workstation.yml

@@ -1,20 +0,0 @@
----
-# Off-site Debian VM running Claude Code — reattachable interactive sessions
-# plus an unattended notes-distill timer. Host creation (Synology VMM), the
-# Netbird dashboard route and secret material are manual prerequisites; see
-# roles/claude-workstation/README.md and docs/claude-workstation.md.
-#
-# CI reaches this off-site host over the site-to-site tunnel (home -> off-site
-# works), same as it reaches every other inventory host.
-
-- name: Deploy the Claude Code workstation
-  hosts: claude_workstation
-  become: true
-  # qemu-guest-agent is applied
-  # to every inventory host by the umbrella plays/common.yml, same as for the
-  # netbird-proxy host — run that against this host too (see RUNBOOK Step 5).
-  roles:
-    - common
-    - role: netbird-client       # mesh enrolment (needs netbird_setup_key in vault)
-      when: netbird_setup_key is defined
-    - claude-workstation

ansible/plays/install-netbird-client.yml

@@ -6,26 +6,25 @@
 #   - host_vars/<hostname>/vars.yml - per-host key (overrides group)
 #   - group_vars/all/vault.yml - encrypted keys
 #
-# Usage:
-#   ansible-playbook -i inventory/ plays/setup-netbird-client.yml
-#
-# Limit to specific hosts:
-#   ansible-playbook -i inventory/ plays/setup-netbird-client.yml --limit caddy_servers
+# Applied by site.yml.
 
 - name: Install NetBird client on hosts with setup key defined
   hosts: all
   become: true
-  gather_facts: true
+  # One host at a time: from a workstation every new SSH session asks for the
+  # YubiKey PIN and a touch, and parallel hosts would prompt at once.
+  serial: 1
+  # Facts only after the skip: gathering them first opens an SSH session (a
+  # YubiKey touch from a workstation) to every host, including skipped ones.
+  gather_facts: false
 
   pre_tasks:
-    - name: Check if netbird_setup_key is defined
-      ansible.builtin.debug:
-        msg: "NetBird setup key is defined for {{ inventory_hostname }}"
-      when: netbird_setup_key is defined
-
     - name: Skip hosts without netbird_setup_key
       ansible.builtin.meta: end_host
       when: netbird_setup_key is not defined
 
+    - name: Gather facts
+      ansible.builtin.setup:
+
   roles:
     - netbird-client

ansible/plays/install-netbird-proxy.yml

@@ -1,6 +1,6 @@
 ---
 # Deploy the NetBird reverse-proxy HA cluster members (the public edge).
-# Usage: ansible-playbook -i inventory/ plays/install-netbird-proxy.yml
+# Applied by site.yml.
 #
 # Prerequisites:
 # - Proxy VPS provisioned (proxy-1 is Arch Linux), root SSH reachable, and

ansible/plays/install-netbird-server.yml

@@ -1,6 +1,6 @@
 ---
 # Deploy NetBird Server to VPS
-# Usage: ansible-playbook -i inventory/ plays/install-netbird-server.yml
+# Applied by site.yml.
 #
 # Prerequisites:
 # - VPS provisioned with Ubuntu 22.04/24.04

ansible/plays/install-node-exporter.yml

@@ -1,22 +0,0 @@
----
-# Install Node Exporter on all managed hosts
-# Usage: ansible-playbook -i inventory/ plays/install-node-exporter.yml
-
-- name: Install Node Exporter
-  hosts: all
-  become: yes
-  gather_facts: yes
-
-  roles:
-    - node-exporter
-
-  post_tasks:
-    - name: Verify node-exporter is running
-      uri:
-        url: "http://127.0.0.1:9100/metrics"
-        status_code: 200
-      register: exporter_health
-      retries: 3
-      delay: 5
-      until: exporter_health.status == 200
-      ignore_errors: yes

ansible/plays/install-uptime-kuma.yml

@@ -1,6 +1,6 @@
 ---
 # Deploy Uptime Kuma to the NetBird VPS
-# Usage: ansible-playbook -i inventory/ plays/install-uptime-kuma.yml
+# Applied by site.yml.
 #
 # Prerequisites:
 # - netbird-server play applied (docker, Traefik on netbird_netbird)

ansible/plays/rotate-netbird-secrets.yml

@@ -248,3 +248,6 @@
           - "Commit inventory/group_vars/all/netbird-rotatable.yml — it is encrypted."
           - "NOT rotated: datastore_key (needs dump/restore), auth_client_secret (Authentik side)."
           - "OpenBao is unaffected by design — its NetBird credentials are separate ones."
+
+- name: Close the SSH connections
+  ansible.builtin.import_playbook: close-connections.yml

ansible/plays/update-dist-upgrade.yml

@@ -111,3 +111,6 @@
       when: 
         - (ansible_distribution == 'Ubuntu' and reboot_required.stat.exists) or
           (ansible_distribution == 'Debian')
+
+- name: Close the SSH connections
+  ansible.builtin.import_playbook: close-connections.yml

ansible/plays/update-os.yml

@@ -1,7 +1,7 @@
 ---
 # Package upgrade + reboot for netbird_proxy, edge_host and netbird_server,
 # one host at a time.
-# Usage: ansible-playbook -i inventory/ plays/update-os.yml
+# Usage (from ansible/): ansible-playbook plays/update-os.yml
 #
 # Plays run in order; a failed host stops the whole run (max_fail_percentage: 0).
 
@@ -28,3 +28,6 @@
   become: true
   roles:
     - os-update
+
+- name: Close the SSH connections
+  ansible.builtin.import_playbook: close-connections.yml

ansible/roles/claude-workstation/README.md

@@ -1,57 +0,0 @@
-# claude-workstation
-
-An always-on, **off-site** headless Claude Code host (Debian 12 VM under Synology
-VMM on nas-2). It runs (a) reattachable interactive `tmux` sessions and (b) an
-unattended notes-distill timer. It deliberately holds the **least** privilege
-that still does the job.
-
-## What lives here vs. what's manual
-
-This role configures everything **inside** the VM. Three things are **manual**
-(no IaC path) and precede/accompany it — see [`RUNBOOK.md`](RUNBOOK.md) for the
-full step-by-step:
-
-1. **VM creation** in Synology VMM (`ladmin` has no sudo; the DSM API is
-   root-only), with a cloud-init drive that installs the CI SSH key.
-2. **Netbird dashboard route** advertising the k8s API to the mesh, and the
-   **site-to-site return route** on the off-site gateway (backup path).
-3. **Secret material**: `claude setup-token` (subscription OAuth), the claude-bot
-   creds, and the read-only SA kubeconfig — encrypted into the group vault.
-
-## Credential isolation
-
-Two Unix users with separate credential stores so an unattended job cannot read
-the interactive session's secrets:
-
-| User | Holds | Runs |
-|------|-------|------|
-| `claude` | OAuth token, claude-bot creds, **read-only** SA kubeconfig | interactive tmux |
-| `claude-job` | its **own** OAuth token, claude-bot creds (no kubeconfig) | notes-distill timer |
-
-**No talosconfig, no SOPS age key** ever land here (Fable review #1). The
-kubeconfig is a read-only `view` ServiceAccount (`kubernetes/apps/claude-workstation/`).
-
-## Egress lockdown (staged)
-
-`claude_ws_egress_enforce` ships **`false`** so a fresh host is never bricked.
-The deny-default nftables ruleset, the DNS-resolver, and its refresh timer are
-installed regardless. Flip to `true` (in host/group vars) only after confirming
-`claude_ws_egress_allow_fqdns` / `claude_ws_egress_allow_cidrs*` cover real
-traffic — watch the `claude-egress-dropped` counter first.
-
-## Vault keys (encrypt into `inventory/group_vars/claude_workstation/vault.yml`)
-
-```
-netbird_setup_key                  # from the Netbird dashboard
-claude_ws_oauth_token_interactive  # `claude setup-token` (interactive user)
-claude_ws_oauth_token_job          # `claude setup-token` (job user, separate)
-claude_ws_gitea_token              # claude-bot PAT
-claude_ws_gitea_git_credentials    # https://claude-bot:<pat>@gitea.renner.uno
-claude_ws_kubeconfig               # read-only SA kubeconfig (literal YAML)
-```
-
-## Kill switch
-
-Everything unattended hangs off systemd timers; `systemctl disable --now
-claude-notes-distill.timer` (and the egress refresh) stops it. Every credential
-is independently revocable (SA token, claude-bot PAT, OAuth token, Netbird peer).

ansible/roles/claude-workstation/RUNBOOK.md

@@ -1,221 +0,0 @@
-# Runbook — off-site Claude Code workstation (nas-2)
-
-Operator guide for the manual steps around the `claude-workstation` role. The
-role automates everything **inside** the VM; this covers what it can't:
-VM creation, networking, secret material, egress cutover, rotation and DR.
-
-## Architecture (scope)
-
-An always-on **Debian 13 VM under Synology VMM on the off-site nas-2**
-(DS1522+, 10.3.40.0/24, reached over the site-to-site VPN, ~55 ms). It runs:
-
-- **Reattachable interactive sessions** (`tmux` over SSH as user `claude`).
-- **Unattended notes-distill timer** (as user `claude-job`).
-- **A thin is-home-up signal** (the existing external Gatus on the Netbird VPS
-  already covers this — don't duplicate it).
-
-**Out of scope (runs in-cluster instead):** scheduled repo-automation and
-cluster-reports go in as Woodpecker cron / CronJobs with an in-cluster
-ServiceAccount — *not* on this VM, so the VM holds no cluster-write creds.
-
-Credentials on the VM are minimal: a **read-only** `view` SA kubeconfig,
-claude-bot (non-admin) gitea creds, and a Claude subscription OAuth token.
-**No talosconfig, no SOPS age key.**
-
----
-
-## Step 1 — Create + install the VM (Synology VMM)
-
-`ladmin` has no sudo and the DSM API is root-only, so VMM is driven through the
-DSM **web UI**. Two hard facts dictate the method (all confirmed 2026-07-16, as
-built): **VMM has no cloud-init**, and its ISO-install wizard only creates blank
-disks. So the OS is installed from a **remastered Debian netinst ISO that
-auto-boots a preseed** — no console interaction, no boot-menu race.
-
-VMM gotchas that shape this (see the wiki build-record):
-- VMM refuses to change/unmount a VM's ISO while it is **running** ("Failed to
-  replace ISO files"). Do all ISO/boot-device edits with the VM **off**.
-- A **guest reboot does not re-read** VMM's boot-device/ISO config — only a full
-  VMM power-cycle applies it. So an auto-booting install ISO **loops** (reboot →
-  CD → reinstall, wiping the disk). Break it by preseeding the installer to
-  **power off** at the end: `d-i debian-installer/exit/poweroff boolean true`.
-- Attach the NIC to the **untagged `Default VM Network`** — the VLAN-40
-  `40_Server` network is isolated (no DHCP, no network). The LAN has DHCP.
-
-Procedure (build the ISO on a host with `xorriso`):
-1. Download the current Debian **netinst** (`.../debian-cd/current/amd64/iso-cd/`).
-2. Remaster it — add `/preseed.cfg` and replace `/isolinux/isolinux.cfg` with a
-   minimal one (`default preseed`, `timeout 1`, one label:
-   `kernel /install.amd/vmlinuz` /
-   `append auto=true priority=critical preseed/file=/cdrom/preseed.cfg vga=788 initrd=/install.amd/initrd.gz --- quiet`):
-   `xorriso -indev in.iso -outdev out.iso -boot_image any replay -map preseed.cfg /preseed.cfg -map isolinux.cfg /isolinux/isolinux.cfg`.
-   The preseed uses **DHCP** (`netcfg/dhcp_timeout 120` — the virtual-switch port
-   is slow to link), disk **`/dev/sda`** (VirtIO SCSI), **no user** + a locked
-   random root hash (key-only via `PermitRootLogin prohibit-password`), packages
-   `openssh-server sudo qemu-guest-agent python3`, and a `late_command` that
-   injects the SSH keys (`krenner@fortress`, `woodpecker-ci@homelab`, your
-   workstation key) into `/root/.ssh/authorized_keys`, plus the `poweroff` line.
-3. Upload to nas-2 via `ssh 'cat > /volume1/Install/<iso>' < <iso>` — **scp /
-   sftp / rsync are blocked on DSM**. Then VMM → Image → Add → From Synology NAS
-   → import (VMM **copies** it; re-import if you change the source file).
-4. Create VM: 2 vCPU, 4 GB, 50 GB on `/volume1`, **Default VM Network**, Legacy
-   BIOS, **Autostart: Yes**; attach the ISO, **Boot from DVD/CD-ROM**, power on.
-5. It installs unattended and **powers off**. Then (VM off) set **ISO =
-   Unmounted**, **Boot from = Virtual Disk**, power on → boots the installed
-   Debian. `qemu-guest-agent` reports the DHCP lease in VMM's IP column.
-6. **Pin a DSM update window** (Control Panel → Update & Restore) so an
-   auto-reboot doesn't kill tmux/jobs.
-
-> As built 2026-07-16: Debian 13 (trixie), got DHCP `10.3.40.78`, all three keys
-> injected, packages present. Freeing RAM required powering off the pre-existing
-> `pbs1` PBS VM (kept its 3.4 TB backup disk).
-
-> **At-rest encryption (Fable #6).** Off-site physical custody is weaker than
-> home. For true at-rest protection of the VM's secrets *with* unattended
-> autostart, use **network-bound decryption** (clevis + a Tang server at home):
-> a stolen disk off-site can't unlock because it can't reach Tang. Full-disk
-> LUKS with a manual passphrase would break autostart. Until Tang is stood up,
-> secrets rely on 0600 perms + VM access control — documented gap, not a fix.
-
----
-
-## Step 2 — Networking (two paths)
-
-**Netbird mesh (done via CI):** mint a setup key in the dashboard
-(`https://netbird.renner.uno`), vault it as `netbird_setup_key` under
-`group_vars/claude_workstation/`, and run `.woodpecker/deploy-netbird-client.yml`
-(`install-netbird-client.yml --limit claude_workstation`). As built 2026-07-16
-the VM enrolled as `claude-ws-190-112.nb.renner.uno` / `100.96.190.112` (+ IPv6).
-
-> **Mesh reachability caveat.** The mesh IP (`100.96.x`) is only reachable from
-> **other Netbird peers**. The home workstation and the Woodpecker CI agent are
-> **not** peers — they reach the VM only via the site-to-site tunnel at its DHCP
-> lease. So a *stable* mesh-based `ansible_host` needs CI (or a home host) on the
-> mesh first, or a Netbird routing peer bridging tunnel↔mesh. Until then the
-> inventory targets the tunnel DHCP IP.
-
-> **DNS trap.** `claude-ws.obg.renner.uno` resolves to the VM's lease **only at
-> the off-site DNS** (`@10.3.40.1`). From home this FQDN does not resolve (it
-> exists only at the off-site DNS @10.3.40.1). Never use it as `ansible_host`.
-
-**Optional — mesh route to the k8s API (redundant read path):** to reach the API
-over the mesh too, add a Netbird dashboard **network route** advertising the
-apiserver's ClusterIP to the `netbird-ingress` peer group, and add an egress
-rule to the `netbird-ingress-egress` CiliumNetworkPolicy
-(`kubernetes/infrastructure/network-policies/ingress-isolation.yaml`) permitting
-`app: netbird-ingress` pods to reach `kube-apiserver:443`. Deferred from this PR
-because it edits a security-critical CNP that can't be validated offline — do it
-as a follow-up when you want the redundant API path.
-
----
-
-## Step 3 — Read-only SA kubeconfig
-
-After the `kubernetes/apps/claude-workstation/` PR merges and Flux reconciles:
-
-```sh
-NS=apps
-TOKEN=$(kubectl -n $NS get secret claude-workstation-token -o jsonpath='{.data.token}' | base64 -d)
-kubectl -n $NS get secret claude-workstation-token -o jsonpath='{.data.ca\.crt}' | base64 -d > /tmp/ca.crt
-# server = the cluster's stable API endpoint (kubernetes/talos/patches/cluster.yaml).
-# Use the name, not an address: it is what the API server certificate carries as a
-# SAN, so an IP here fails TLS verification even when it is reachable. It must
-# resolve from the workstation over the site-to-site route.
-kubectl config set-cluster homelab --server=https://k8s.renner.uno:6443 \
-  --certificate-authority=/tmp/ca.crt --embed-certs=true --kubeconfig=/tmp/claude-ws.kubeconfig
-kubectl config set-credentials claude-workstation --token="$TOKEN" --kubeconfig=/tmp/claude-ws.kubeconfig
-kubectl config set-context homelab --cluster=homelab --user=claude-workstation --kubeconfig=/tmp/claude-ws.kubeconfig
-kubectl config use-context homelab --kubeconfig=/tmp/claude-ws.kubeconfig
-# verify read-only:
-KUBECONFIG=/tmp/claude-ws.kubeconfig kubectl get nodes        # works
-KUBECONFIG=/tmp/claude-ws.kubeconfig kubectl get secrets -A   # Forbidden (view excludes secrets)
-```
-
-The resulting file becomes the `claude_ws_kubeconfig` vault value.
-
----
-
-## Step 4 — Secret material → vault
-
-```sh
-# Two independent OAuth tokens so job jobs can't impersonate the interactive user:
-claude setup-token   # -> claude_ws_oauth_token_interactive
-claude setup-token   # -> claude_ws_oauth_token_job  (run again / separate session)
-```
-
-Encrypt into `ansible/inventory/group_vars/claude_workstation/vault.yml`
-(ansible-vault; same VAULT_PASS the CI uses):
-
-```yaml
-netbird_setup_key: "..."
-claude_ws_oauth_token_interactive: "..."
-claude_ws_oauth_token_job: "..."
-claude_ws_gitea_token: "..."                # claude-bot PAT
-claude_ws_gitea_git_credentials: "https://claude-bot:<PAT>@gitea.renner.uno"
-claude_ws_kubeconfig: |
-  apiVersion: v1
-  ...                                        # the file from Step 3
-```
-
-> **Never run `claude` with `--bare`** here or on the VM — bare mode silently
-> ignores `CLAUDE_CODE_OAUTH_TOKEN` and the jobs will fail auth.
-
----
-
-## Step 5 — Deploy
-
-Merge the PR, then trigger the pipeline (Woodpecker → Run → `PIPELINE ==
-"deploy-claude-workstation"`), or push a change under the role's paths. CI SSHes
-to `10.3.40.60` over the tunnel and applies `common`, `netbird-client`, and this
-role. Also run the umbrella `plays/common.yml` against the new host once, so it
-picks up `qemu-guest-agent` like every other inventory host.
-
----
-
-## Step 6 — Egress cutover (staged)
-
-The nftables allowlist ships **disabled**. To enforce:
-
-1. Leave it off for a few days; watch what the box actually talks to.
-2. Confirm `claude_ws_egress_allow_fqdns` + `claude_ws_egress_allow_cidrs*`
-   (home Prometheus/Loki/API) cover it.
-3. Set `claude_ws_egress_enforce: true` in host/group vars, redeploy.
-4. Watch the `claude-egress-dropped` nft counter (`nft list ruleset`) and Loki
-   for anything legitimate being dropped; widen the allowlist as needed.
-
----
-
-## Step 7 — Notes (Syncthing pairing)
-
-The role installs Syncthing for `claude-job` and starts
-`syncthing@claude-job`. Pair it once (manual): from the VM's GUI (SSH tunnel to
-`127.0.0.1:8384`) add your existing vault's device ID and accept the shared
-`Notes` folder pointed at `{{ claude_ws_notes_dir }}`. The distill timer runs
-daily at 04:30 and only ever **creates** new atomic notes.
-
----
-
-## Rotation
-
-- **OAuth token** (expires/can be revoked): re-run `claude setup-token` on your
-  laptop, update the two vault keys, redeploy. Headless auth failures surface in
-  Loki (the wrapper logs `result=failure`) — don't let them go quiet.
-- **claude-bot PAT**: rotate in Gitea, update `claude_ws_gitea_*`, redeploy.
-- **SA token**: `kubectl -n apps delete secret claude-workstation-token` (Flux
-  recreates it), re-assemble the kubeconfig (Step 3), update vault, redeploy.
-
-## Kill switch
-
-```sh
-systemctl disable --now claude-notes-distill.timer claude-egress-refresh.timer
-```
-
-Every credential is independently revocable: delete the SA, revoke the PAT,
-revoke the OAuth token, or remove the Netbird peer — no shared blast radius.
-
-## Disaster recovery (cattle, not pet)
-
-The VM holds no unique state — the only real state is the Notes replica, which
-re-syncs from Syncthing. To rebuild: redo **Step 1** (VM + cloud-init), then
-**Step 5** (pipeline). No VMM snapshots needed (they'd bloat the 86%-full
-`/volume1`).

ansible/roles/claude-workstation/defaults/main.yml

@@ -1,68 +0,0 @@
----
-# claude-workstation role defaults
-#
-# An always-on headless host that runs Claude Code for (a) reattachable
-# interactive sessions and (b) unattended notes/wiki timers. It deliberately
-# holds the LEAST privilege that still lets it do its job: a read-only k8s
-# ServiceAccount kubeconfig, claude-bot (non-admin) gitea creds, and a Claude
-# subscription OAuth token — NO talosconfig and NO SOPS age key. See README.
-
-# --- Users -----------------------------------------------------------------
-# Two Unix users with separate credential stores so an unattended (timer) job
-# cannot read the interactive session's OAuth token / kube / gitea creds.
-claude_ws_interactive_user: claude          # human tmux sessions
-claude_ws_job_user: claude-job              # unattended systemd-timer jobs
-
-claude_ws_base_dir: /opt/claude-workstation
-
-# --- Toolchain -------------------------------------------------------------
-claude_ws_node_major: 22                     # matches the ci-claude image
-claude_ws_claude_code_pkg: "@anthropic-ai/claude-code"
-claude_ws_base_packages:
-  - git
-  - tmux
-  - jq
-  - curl
-  - ca-certificates
-  - ripgrep
-  - unzip
-  - nftables
-  - python3
-  - python3-venv
-
-# --- Egress allowlist ------------------------------------------------------
-# Deny-default OUTPUT is the target end-state, but it can brick a fresh host if
-# an allowed destination is missed, so it ships DISABLED and is tightened after
-# observing real egress (staged rollout). Flip claude_ws_egress_enforce to true
-# once the allowlist below is confirmed complete. See templates/nftables.conf.j2.
-claude_ws_egress_enforce: false
-# Static destinations the host must always reach. CDN-backed hosts (Anthropic,
-# npm, apt) are matched by name at apply time via the fqdn set; home services
-# are matched by IP. Tune in host_vars.
-claude_ws_egress_allow_fqdns:
-  - api.anthropic.com
-  - statsig.anthropic.com
-  - gitea.renner.uno
-  - fsn1.your-objectstorage.com          # Hetzner Object Storage (SOPS bundle)
-  - deb.debian.org
-  - security.debian.org
-  - deb.nodesource.com
-  - pkgs.netbird.io
-  - registry.npmjs.org
-claude_ws_egress_allow_cidrs: []          # home Prometheus/Loki/API (v4), per-host
-claude_ws_egress_allow_cidrs6: []         # same (v6) — IPv6-first, prefer these
-
-# --- Notes / wiki pipeline -------------------------------------------------
-# The Obsidian vault syncs onto this host via Syncthing (matches the user's
-# existing multi-device setup). Unattended pipeline jobs must CREATE new atomic
-# files rather than rewrite existing ones, to keep Syncthing conflicts rare.
-claude_ws_notes_enabled: true
-claude_ws_notes_dir: "/home/{{ claude_ws_job_user }}/Notes"
-claude_ws_syncthing_gui_bind: "127.0.0.1:8384"
-
-# --- Secrets (populated from the ansible-vault'd group_vars — see README) ---
-# claude_ws_oauth_token_interactive : CLAUDE_CODE_OAUTH_TOKEN for the human user
-# claude_ws_oauth_token_job         : CLAUDE_CODE_OAUTH_TOKEN for timer jobs
-# claude_ws_gitea_token             : claude-bot PAT
-# claude_ws_gitea_git_credentials   : https://claude-bot:<pat>@gitea.renner.uno line
-# claude_ws_kubeconfig              : read-only SA kubeconfig (base64 or literal)

ansible/roles/claude-workstation/files/claude-notes-distill-prompt.md

@@ -1,22 +0,0 @@
-You are running unattended as a scheduled job on the claude-workstation host.
-Your job is a conservative distillation pass over the Obsidian wiki vault.
-
-Scope and rules (follow exactly — you have only Read, Write, Glob, Grep):
-
-1. Look at `~/Notes/raw/` (ignore `~/Notes/raw/_processed/` and `~/Notes/_archive/`
-   entirely) for source material.
-2. For each raw source, check whether `~/Notes/wiki/` already contains a note
-   grounded in it (Grep the wiki for the source filename / its `sources:` slug).
-   If a note already exists, SKIP that source — do not rewrite or duplicate it.
-3. For raw sources with NO corresponding wiki note yet, create new atomic
-   concept notes in `~/Notes/wiki/`: one concept per file, self-contained,
-   tagged, cross-linked with `[[wikilinks]]`, every claim grounded in a
-   `sources:` entry. Never invent facts not present in the source.
-4. CREATE new files only. Never rewrite, move, delete, or truncate an existing
-   file in `~/Notes/` — appending a missing cross-link to a note you created
-   THIS run is fine; touching pre-existing notes is not.
-5. If there is nothing new to distill, do nothing and stop. Do not manufacture
-   work.
-
-Preserve disagreement: if sources conflict, surface the conflict in the note
-rather than silently picking a winner. Keep each note short and single-purpose.

ansible/roles/claude-workstation/files/tmux.conf

@@ -1,8 +0,0 @@
-# claude-workstation shared tmux config — reattachable long-running sessions.
-set -g history-limit 100000
-set -g mouse on
-set -g base-index 1
-setw -g pane-base-index 1
-set -g status-right "#H  %Y-%m-%d %H:%M"
-# Keep sessions alive across client disconnects (the whole point of this host).
-set -g destroy-unattached off

ansible/roles/claude-workstation/handlers/main.yml

@@ -1,10 +0,0 @@
----
-- name: Reload systemd
-  ansible.builtin.systemd:
-    daemon_reload: true
-
-- name: Reload nftables
-  ansible.builtin.systemd:
-    name: nftables.service
-    state: reloaded
-  when: claude_ws_egress_enforce | bool

ansible/roles/claude-workstation/tasks/firewall.yml

@@ -1,42 +0,0 @@
----
-# Deny-default egress allowlist. Ships DISABLED (claude_ws_egress_enforce:
-# false) so a fresh host is never bricked by a missed destination; the ruleset,
-# the FQDN-resolver, and its refresh timer are installed regardless, and the
-# operator flips the toggle to true once egress is confirmed complete against
-# real traffic (staged rollout). CDN-backed hosts are matched via nft named
-# sets that the refresh unit repopulates from DNS; static home services are
-# matched by claude_ws_egress_allow_cidrs.
-
-- name: Deploy the egress-refresh resolver
-  ansible.builtin.template:
-    src: claude-egress-refresh.sh.j2
-    dest: "{{ claude_ws_base_dir }}/claude-egress-refresh.sh"
-    mode: "0755"
-
-- name: Deploy the egress-refresh service + timer
-  ansible.builtin.template:
-    src: "{{ item }}.j2"
-    dest: "/etc/systemd/system/{{ item }}"
-    mode: "0644"
-  loop:
-    - claude-egress-refresh.service
-    - claude-egress-refresh.timer
-  notify: Reload systemd
-
-- name: Deploy the nftables ruleset
-  ansible.builtin.template:
-    src: nftables.conf.j2
-    dest: /etc/nftables.conf
-    mode: "0644"
-    validate: "nft -c -f %s"
-  notify: Reload nftables
-
-- name: Enforce or relax egress lockdown
-  ansible.builtin.systemd:
-    name: "{{ item }}"
-    enabled: "{{ claude_ws_egress_enforce | bool }}"
-    state: "{{ 'started' if (claude_ws_egress_enforce | bool) else 'stopped' }}"
-    daemon_reload: true
-  loop:
-    - nftables.service
-    - claude-egress-refresh.timer

ansible/roles/claude-workstation/tasks/main.yml

@@ -1,183 +0,0 @@
----
-# claude-workstation: an always-on headless Claude Code host.
-# Debian 12 VM on the off-site nas-2 (Synology VMM). See README for the manual
-# bootstrap (VM creation, Netbird route, secret material) that precedes this.
-#
-# common.yml already applies `common` and `qemu-guest-agent`
-# to every inventory host, so this role owns only the
-# Claude-specific surface: toolchain, the two credential-isolated users, the
-# read-only creds, notes sync, the unattended timers, and egress lockdown.
-
-- name: Assert we are on Debian (role is apt-only)
-  ansible.builtin.assert:
-    that:
-      - ansible_facts['os_family'] == 'Debian'
-    fail_msg: "claude-workstation targets Debian; got {{ ansible_facts['os_family'] }}."
-
-# --- Base packages ---------------------------------------------------------
-- name: Install base packages
-  ansible.builtin.apt:
-    name: "{{ claude_ws_base_packages }}"
-    state: present
-    update_cache: true
-
-# --- Node.js 22 + Claude Code (matches the ci-claude image toolchain) -------
-- name: Add NodeSource signing key
-  ansible.builtin.get_url:
-    url: https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key
-    dest: /usr/share/keyrings/nodesource.gpg
-    mode: "0644"
-    force: false
-
-- name: Add NodeSource apt repository
-  ansible.builtin.apt_repository:
-    repo: "deb [signed-by=/usr/share/keyrings/nodesource.gpg] https://deb.nodesource.com/node_{{ claude_ws_node_major }}.x nodistro main"
-    state: present
-    filename: nodesource
-
-- name: Install Node.js
-  ansible.builtin.apt:
-    name: nodejs
-    state: present
-    update_cache: true
-
-# NodeSource installs Node under /usr, so global npm modules land in
-# /usr/lib/node_modules and `claude` is on PATH for every user/timer. `creates`
-# keeps it idempotent (re-run `npm install -g` manually to bump the version).
-- name: Install Claude Code globally
-  ansible.builtin.command:
-    cmd: "npm install -g {{ claude_ws_claude_code_pkg }}"
-    creates: /usr/lib/node_modules/@anthropic-ai/claude-code
-
-# --- Credential-isolated users --------------------------------------------
-- name: Create Claude users
-  ansible.builtin.user:
-    name: "{{ item }}"
-    shell: /bin/bash
-    create_home: true
-  loop:
-    - "{{ claude_ws_interactive_user }}"
-    - "{{ claude_ws_job_user }}"
-
-- name: Create the role base directory (scripts, prompts)
-  ansible.builtin.file:
-    path: "{{ claude_ws_base_dir }}"
-    state: directory
-    mode: "0755"
-
-# Let the interactive user's tmux server survive SSH logout / reboot.
-- name: Enable lingering for the interactive user
-  ansible.builtin.command: "loginctl enable-linger {{ claude_ws_interactive_user }}"
-  args:
-    creates: "/var/lib/systemd/linger/{{ claude_ws_interactive_user }}"
-
-- name: Deploy shared tmux config
-  ansible.builtin.copy:
-    src: tmux.conf
-    dest: "/home/{{ item }}/.tmux.conf"
-    owner: "{{ item }}"
-    group: "{{ item }}"
-    mode: "0644"
-  loop:
-    - "{{ claude_ws_interactive_user }}"
-    - "{{ claude_ws_job_user }}"
-
-# --- Secrets: each user gets only what its role needs ----------------------
-# The interactive user gets gitea + OAuth + read-only kubeconfig. The job user
-# gets its OWN OAuth token + gitea (to open PRs) but NO kubeconfig.
-- name: Create ~/.config/claude-bot for the interactive user
-  ansible.builtin.file:
-    path: "/home/{{ item }}/.config/claude-bot"
-    state: directory
-    owner: "{{ item }}"
-    group: "{{ item }}"
-    mode: "0700"
-  loop:
-    - "{{ claude_ws_interactive_user }}"
-    - "{{ claude_ws_job_user }}"
-
-- name: Deploy claude-bot gitea token
-  ansible.builtin.copy:
-    content: "{{ claude_ws_gitea_token }}\n"
-    dest: "/home/{{ item }}/.config/claude-bot/gitea-token"
-    owner: "{{ item }}"
-    group: "{{ item }}"
-    mode: "0600"
-  loop:
-    - "{{ claude_ws_interactive_user }}"
-    - "{{ claude_ws_job_user }}"
-  no_log: true
-
-- name: Deploy claude-bot git-credentials
-  ansible.builtin.copy:
-    content: "{{ claude_ws_gitea_git_credentials }}\n"
-    dest: "/home/{{ item }}/.config/claude-bot/git-credentials"
-    owner: "{{ item }}"
-    group: "{{ item }}"
-    mode: "0600"
-  loop:
-    - "{{ claude_ws_interactive_user }}"
-    - "{{ claude_ws_job_user }}"
-  no_log: true
-
-# CLAUDE_CODE_OAUTH_TOKEN is delivered as an EnvironmentFile so both interactive
-# shells and timer units can source it without it landing in shell history.
-# NEVER run claude with --bare here: bare mode silently ignores this token.
-- name: Deploy per-user Claude OAuth environment file
-  ansible.builtin.copy:
-    content: "CLAUDE_CODE_OAUTH_TOKEN={{ item.token }}\n"
-    dest: "/home/{{ item.user }}/.config/claude-code.env"
-    owner: "{{ item.user }}"
-    group: "{{ item.user }}"
-    mode: "0600"
-  loop:
-    - { user: "{{ claude_ws_interactive_user }}", token: "{{ claude_ws_oauth_token_interactive }}" }
-    - { user: "{{ claude_ws_job_user }}", token: "{{ claude_ws_oauth_token_job }}" }
-  loop_control:
-    label: "{{ item.user }}"
-  no_log: true
-
-- name: Create ~/.kube for the interactive user
-  ansible.builtin.file:
-    path: "/home/{{ claude_ws_interactive_user }}/.kube"
-    state: directory
-    owner: "{{ claude_ws_interactive_user }}"
-    group: "{{ claude_ws_interactive_user }}"
-    mode: "0700"
-
-- name: Deploy read-only SA kubeconfig (interactive user only)
-  ansible.builtin.copy:
-    content: "{{ claude_ws_kubeconfig }}"
-    dest: "/home/{{ claude_ws_interactive_user }}/.kube/config"
-    owner: "{{ claude_ws_interactive_user }}"
-    group: "{{ claude_ws_interactive_user }}"
-    mode: "0600"
-  no_log: true
-
-# This VM's only git identity IS claude-bot (unlike the shared human
-# workstation, where a global credential.helper is forbidden), so a per-user
-# ~/.gitconfig pointing at each user's own claude-bot store is appropriate.
-- name: Configure git identity + credential store for the Claude users
-  ansible.builtin.copy:
-    dest: "/home/{{ item }}/.gitconfig"
-    owner: "{{ item }}"
-    group: "{{ item }}"
-    mode: "0644"
-    content: |
-      [user]
-      	name = Claude Bot
-      	email = claude-bot@renner.uno
-      [credential]
-      	helper = store --file=/home/{{ item }}/.config/claude-bot/git-credentials
-  loop:
-    - "{{ claude_ws_interactive_user }}"
-    - "{{ claude_ws_job_user }}"
-
-# --- Notes / wiki pipeline (Syncthing + timers) ----------------------------
-- name: Notes pipeline
-  ansible.builtin.include_tasks: notes.yml
-  when: claude_ws_notes_enabled | bool
-
-# --- Egress lockdown -------------------------------------------------------
-- name: Egress firewall
-  ansible.builtin.include_tasks: firewall.yml

ansible/roles/claude-workstation/tasks/notes.yml

@@ -1,48 +0,0 @@
----
-# Notes/wiki pipeline: the Obsidian vault syncs onto this host via Syncthing
-# (matching the user's existing multi-device setup), and a hardened, bounded
-# systemd timer runs an unattended `claude -p` distill pass over it as the
-# credential-isolated job user. Device pairing is a one-time manual step (see
-# README) — this only installs Syncthing and the timer.
-
-- name: Install Syncthing
-  ansible.builtin.apt:
-    name: syncthing
-    state: present
-    update_cache: true
-
-- name: Enable Syncthing for the job user
-  ansible.builtin.systemd:
-    name: "syncthing@{{ claude_ws_job_user }}.service"
-    enabled: true
-    state: started
-    daemon_reload: true
-
-- name: Deploy the notes-distill prompt
-  ansible.builtin.copy:
-    src: claude-notes-distill-prompt.md
-    dest: "{{ claude_ws_base_dir }}/claude-notes-distill-prompt.md"
-    mode: "0644"
-
-- name: Deploy the notes-distill wrapper
-  ansible.builtin.template:
-    src: claude-notes-distill.sh.j2
-    dest: "{{ claude_ws_base_dir }}/claude-notes-distill.sh"
-    mode: "0755"
-
-- name: Deploy the notes-distill service + timer
-  ansible.builtin.template:
-    src: "{{ item }}.j2"
-    dest: "/etc/systemd/system/{{ item }}"
-    mode: "0644"
-  loop:
-    - claude-notes-distill.service
-    - claude-notes-distill.timer
-  notify: Reload systemd
-
-- name: Enable the notes-distill timer
-  ansible.builtin.systemd:
-    name: claude-notes-distill.timer
-    enabled: true
-    state: started
-    daemon_reload: true

ansible/roles/claude-workstation/templates/claude-egress-refresh.service.j2

@@ -1,10 +0,0 @@
-[Unit]
-# {{ ansible_managed }}
-Description=Refresh claude-workstation egress allowlist from DNS
-After=nftables.service network-online.target
-Wants=network-online.target
-Requires=nftables.service
-
-[Service]
-Type=oneshot
-ExecStart={{ claude_ws_base_dir }}/claude-egress-refresh.sh

ansible/roles/claude-workstation/templates/claude-egress-refresh.sh.j2

@@ -1,27 +0,0 @@
-#!/usr/bin/env bash
-# {{ ansible_managed }}
-# Resolve the egress FQDN allowlist to current IPs and load them into the
-# nftables @allow4/@allow6 sets. CDN IPs rotate, so this runs on a timer.
-# No-op unless the claude_egress table is loaded (i.e. enforcement is on).
-set -euo pipefail
-
-TABLE="inet claude_egress"
-FQDNS=(
-{% for fqdn in claude_ws_egress_allow_fqdns %}
-    "{{ fqdn }}"
-{% endfor %}
-)
-
-nft list table $TABLE >/dev/null 2>&1 || { echo "claude_egress table not loaded; enforcement off"; exit 0; }
-
-v4=(); v6=()
-for host in "${FQDNS[@]}"; do
-    while read -r ip; do [ -n "$ip" ] && v4+=("$ip"); done < <(getent ahostsv4 "$host" | awk '{print $1}' | sort -u)
-    while read -r ip; do [ -n "$ip" ] && v6+=("$ip"); done < <(getent ahostsv6 "$host" | awk '{print $1}' | sort -u)
-done
-
-nft flush set $TABLE allow4
-nft flush set $TABLE allow6
-[ ${#v4[@]} -gt 0 ] && nft add element $TABLE allow4 "{ $(IFS=,; echo "${v4[*]}") }"
-[ ${#v6[@]} -gt 0 ] && nft add element $TABLE allow6 "{ $(IFS=,; echo "${v6[*]}") }"
-echo "claude-egress-refresh: ${#v4[@]} v4 + ${#v6[@]} v6 addresses loaded"

ansible/roles/claude-workstation/templates/claude-egress-refresh.timer.j2

@@ -1,10 +0,0 @@
-[Unit]
-# {{ ansible_managed }}
-Description=Periodically refresh the claude-workstation egress allowlist
-
-[Timer]
-OnBootSec=30s
-OnUnitActiveSec=15min
-
-[Install]
-WantedBy=timers.target

ansible/roles/claude-workstation/templates/claude-notes-distill.service.j2

@@ -1,32 +0,0 @@
-[Unit]
-# {{ ansible_managed }}
-Description=Claude Code unattended notes-distill pass
-After=network-online.target syncthing@{{ claude_ws_job_user }}.service
-Wants=network-online.target
-
-[Service]
-Type=oneshot
-User={{ claude_ws_job_user }}
-Group={{ claude_ws_job_user }}
-EnvironmentFile=/home/{{ claude_ws_job_user }}/.config/claude-code.env
-ExecStart={{ claude_ws_base_dir }}/claude-notes-distill.sh
-# Hard wall-clock kill — the runaway/kill switch for this unattended agent.
-TimeoutStartSec=1800
-
-# --- Hardening: an unattended LLM job on internet-influenced input ----------
-NoNewPrivileges=true
-PrivateTmp=true
-ProtectSystem=strict
-ReadWritePaths=/home/{{ claude_ws_job_user }}
-# Credential isolation: cannot read the interactive session's tokens/kubeconfig.
-InaccessiblePaths=/home/{{ claude_ws_interactive_user }}
-ProtectKernelTunables=true
-ProtectKernelModules=true
-ProtectControlGroups=true
-RestrictSUIDSGID=true
-RestrictNamespaces=true
-LockPersonality=true
-MemoryDenyWriteExecute=false
-
-[Install]
-WantedBy=multi-user.target

ansible/roles/claude-workstation/templates/claude-notes-distill.sh.j2

@@ -1,32 +0,0 @@
-#!/usr/bin/env bash
-# {{ ansible_managed }}
-# Unattended notes-distill pass. Prompt via stdin (never shell-substituted),
-# tool-allowlisted, no --bare (bare silently drops CLAUDE_CODE_OAUTH_TOKEN).
-# Bounded by the unit's TimeoutStartSec, not a CLI flag (--max-turns is absent
-# in the installed Claude Code). Emits a terminal state line to the journal on
-# EVERY outcome so a Loki alert can catch silent failures (dead-man switch).
-set -euo pipefail
-
-export HOME="/home/{{ claude_ws_job_user }}"
-NOTES="{{ claude_ws_notes_dir }}"
-PROMPT="{{ claude_ws_base_dir }}/claude-notes-distill-prompt.md"
-
-log() { logger -t claude-notes-distill -- "$*"; echo "claude-notes-distill: $*"; }
-
-if [ ! -d "$NOTES/raw" ]; then
-  log "result=skipped reason=no-vault path=$NOTES"
-  exit 0
-fi
-
-log "result=start"
-if claude -p \
-     --add-dir "$NOTES" \
-     --allowedTools "Read,Write,Glob,Grep" \
-     --model sonnet \
-     < "$PROMPT"; then
-  log "result=success"
-else
-  rc=$?
-  log "result=failure rc=$rc"
-  exit "$rc"
-fi

ansible/roles/claude-workstation/templates/claude-notes-distill.timer.j2

@@ -1,11 +0,0 @@
-[Unit]
-# {{ ansible_managed }}
-Description=Run the Claude Code notes-distill pass daily
-
-[Timer]
-OnCalendar=*-*-* 04:30:00
-Persistent=true
-RandomizedDelaySec=900
-
-[Install]
-WantedBy=timers.target

ansible/roles/claude-workstation/templates/nftables.conf.j2

@@ -1,55 +0,0 @@
-#!/usr/sbin/nft -f
-# {{ ansible_managed }}
-# Deny-default egress for the claude-workstation. CDN-backed destinations are
-# matched via the @allow4/@allow6 named sets, repopulated from DNS by
-# claude-egress-refresh. Static home services are matched by the @static* sets.
-# Only loaded when claude_ws_egress_enforce is true.
-flush ruleset
-
-table inet claude_egress {
-    set allow4 { type ipv4_addr; flags interval; }
-    set allow6 { type ipv6_addr; flags interval; }
-
-    set static4 {
-        type ipv4_addr
-        flags interval
-{% if (claude_ws_egress_allow_cidrs | default([])) | length > 0 %}
-        elements = { {{ claude_ws_egress_allow_cidrs | join(', ') }} }
-{% endif %}
-    }
-    set static6 {
-        type ipv6_addr
-        flags interval
-{% if (claude_ws_egress_allow_cidrs6 | default([])) | length > 0 %}
-        elements = { {{ claude_ws_egress_allow_cidrs6 | join(', ') }} }
-{% endif %}
-    }
-
-    chain output {
-        type filter hook output priority 0; policy drop;
-
-        oif "lo" accept
-        ct state established,related accept
-
-        # DNS — required to resolve the allowlist itself.
-        udp dport 53 accept
-        tcp dport 53 accept
-
-        # Diagnostics + DHCP lease renewal on the bridged interface.
-        ip protocol icmp accept
-        ip6 nexthdr icmpv6 accept
-        udp dport { 67, 68 } accept
-
-        # NetBird / WireGuard control + data plane (initial handshakes).
-        udp dport { 3478, 51820 } accept
-
-        # Allowed destinations.
-        ip  daddr @allow4  accept
-        ip6 daddr @allow6  accept
-        ip  daddr @static4 accept
-        ip6 daddr @static6 accept
-
-        # Everything else is dropped and counted for inspection.
-        counter comment "claude-egress-dropped"
-    }
-}

ansible/roles/keepalived-ntp/tasks/main.yml

@@ -140,9 +140,8 @@
       Handing the address to keepalived while networkd also assigns it means
       it would NOT be withdrawn when check_ntp fails -- the host would keep
       serving time on an address VRRP believes it has released.
-      Re-apply the edge-vlan role first, whose template no longer carries the
-      [Address] block:
-      ansible-playbook -i inventory/pve_hosts.yml plays/install-edge-host.yml
+      Re-apply the edge-vlan role first (site.yml runs it before this play),
+      whose template no longer carries the [Address] block.
     success_msg: >-
       No static networkd assignment of {{ keepalived_ntp_vip6 }}; safe for
       keepalived to own it.

ansible/roles/netbird-client/defaults/main.yml

@@ -6,3 +6,13 @@ netbird_management_url: "https://netbird.renner.uno"
 
 # Setup key - should be defined in group_vars or host_vars
 # netbird_setup_key: ""
+
+# Release installed on hosts without a NetBird package repository (Arch).
+# Debian/Ubuntu follow the apt repository instead.
+netbird_client_version: "0.75.0"
+netbird_release_url: "https://github.com/netbirdio/netbird/releases/download/v{{ netbird_client_version }}"
+# On the controller; the archive is copied from there to the hosts.
+netbird_download_dir: "{{ lookup('ansible.builtin.env', 'HOME') }}/.cache/ansible/netbird"
+
+# Leave the host's DNS configuration alone (hosts with a pinned resolv.conf).
+netbird_disable_dns: false

ansible/roles/netbird-client/tasks/main.yml

@@ -2,32 +2,82 @@
 # NetBird Client Installation Role
 # Installs and configures NetBird client on hosts with netbird_setup_key defined
 
-- name: Install required packages
-  ansible.builtin.apt:
-    name:
-      - curl
-      - ca-certificates
-      - gnupg
-    state: present
-    update_cache: true
+- name: Install NetBird from the NetBird apt repository
+  when: ansible_facts['os_family'] == 'Debian'
+  block:
+    - name: Install required packages
+      ansible.builtin.apt:
+        name:
+          - curl
+          - ca-certificates
+          - gnupg
+        state: present
+        update_cache: true
 
-- name: Add NetBird repository key
-  ansible.builtin.shell:
-    cmd: |
-      curl -sSL https://pkgs.netbird.io/debian/public.key | gpg --dearmor -o /usr/share/keyrings/netbird-archive-keyring.gpg
-    creates: /usr/share/keyrings/netbird-archive-keyring.gpg
+    - name: Add NetBird repository key
+      ansible.builtin.shell:
+        cmd: |
+          curl -sSL https://pkgs.netbird.io/debian/public.key | gpg --dearmor -o /usr/share/keyrings/netbird-archive-keyring.gpg
+        creates: /usr/share/keyrings/netbird-archive-keyring.gpg
 
-- name: Add NetBird repository
-  ansible.builtin.apt_repository:
-    repo: "deb [signed-by=/usr/share/keyrings/netbird-archive-keyring.gpg] https://pkgs.netbird.io/debian stable main"
-    state: present
-    filename: netbird
+    - name: Add NetBird repository
+      ansible.builtin.apt_repository:
+        repo: "deb [signed-by=/usr/share/keyrings/netbird-archive-keyring.gpg] https://pkgs.netbird.io/debian stable main"
+        state: present
+        filename: netbird
 
-- name: Install NetBird client
-  ansible.builtin.apt:
+    - name: Install NetBird client
+      ansible.builtin.apt:
+        name: netbird
+        state: present
+        update_cache: true
+
+# Arch has no netbird package in its repositories; the release binary is
+# verified against the release's checksums file. It is downloaded on the
+# controller and copied over: github.com has no IPv6 address, and proxy-1 is
+# IPv6-only.
+- name: Install NetBird from the release tarball
+  when: ansible_facts['os_family'] == 'Archlinux'
+  block:
+    - name: Create the controller download directory
+      ansible.builtin.file:
+        path: "{{ netbird_download_dir }}"
+        state: directory
+        mode: "0755"
+      delegate_to: localhost
+      become: false
+      run_once: true
+
+    - name: Download the NetBird release on the controller
+      ansible.builtin.get_url:
+        url: "{{ netbird_release_url }}/netbird_{{ netbird_client_version }}_linux_amd64.tar.gz"
+        dest: "{{ netbird_download_dir }}/netbird_{{ netbird_client_version }}_linux_amd64.tar.gz"
+        checksum: "sha256:{{ netbird_release_url }}/netbird_{{ netbird_client_version }}_checksums.txt"
+        mode: "0644"
+      delegate_to: localhost
+      become: false
+      run_once: true
+
+    - name: Install the NetBird binary
+      ansible.builtin.unarchive:
+        src: "{{ netbird_download_dir }}/netbird_{{ netbird_client_version }}_linux_amd64.tar.gz"
+        dest: /usr/local/bin
+        include:
+          - netbird
+        owner: root
+        group: root
+        mode: "0755"
+
+    - name: Install the NetBird service
+      ansible.builtin.command: netbird service install
+      args:
+        creates: /etc/systemd/system/netbird.service
+
+- name: Enable NetBird service
+  ansible.builtin.service:
     name: netbird
-    state: present
-    update_cache: true
+    state: started
+    enabled: true
 
 - name: Check if NetBird is already connected
   ansible.builtin.command: netbird status
@@ -35,20 +85,17 @@
   changed_when: false
   failed_when: false
 
+# "Disconnected" contains "Connected", so match the management line.
 - name: Connect NetBird to management server
   ansible.builtin.command: >
     netbird up
     --management-url {{ netbird_management_url }}
     --setup-key {{ netbird_setup_key }}
-  when: "'Connected' not in netbird_status.stdout"
+    {{ '--disable-dns' if netbird_disable_dns else '' }}
+  when: "'Management: Connected' not in netbird_status.stdout"
   register: netbird_connect
   changed_when: netbird_connect.rc == 0
-
-- name: Enable NetBird service
-  ansible.builtin.service:
-    name: netbird
-    state: started
-    enabled: true
+  no_log: true
 
 - name: Get NetBird status
   ansible.builtin.command: netbird status

ansible/roles/netbird-server/tasks/main.yml

@@ -136,23 +136,13 @@
     - /opt/netbird/proxy
     - /opt/netbird/proxy-state
 
-# The management store has no cluster-side backup. Pulling one put a root
-# key on this host that the cluster held, and this server's own state is
-# its own business to protect rather than the cluster's. Dropping the two
-# tasks that placed them would only stop ansible managing the key and the
-# script — the pair below revokes them instead, and can go once the role
-# has run against every host that ever carried them.
+# The management store has no cluster-side backup; the cluster's backup
+# key is gone with the managed authorized_keys (ssh-ca role).
 - name: Remove the backup dump script
   ansible.builtin.file:
     path: /usr/local/bin/netbird-backup-dump
     state: absent
 
-- name: Revoke the cluster backup key
-  ansible.posix.authorized_key:
-    user: root
-    key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFeW/L9bHMTSa5/Qb0ClppCf4Z0hL39JwzI6tBmKd2FG netbird-backup@homelab-cluster"
-    state: absent
-
 - name: Wait for management container to be ready
   ansible.builtin.pause:
     seconds: 10

ansible/roles/node-exporter/defaults/main.yml

@@ -1,20 +0,0 @@
----
-# Node Exporter defaults
-
-node_exporter_version: "1.8.2"
-node_exporter_port: 9100
-node_exporter_user: node-exporter
-node_exporter_group: node-exporter
-
-# Installation paths
-node_exporter_install_dir: /opt/node-exporter
-node_exporter_binary: "{{ node_exporter_install_dir }}/node_exporter"
-
-# Collectors to enable (defaults are usually sufficient)
-node_exporter_enabled_collectors:
-  - systemd
-  - processes
-  - timex
-
-# Collectors to disable
-node_exporter_disabled_collectors: []

ansible/roles/node-exporter/files/chrony-metrics.sh

@@ -1,62 +0,0 @@
-#!/bin/bash
-# Generate Prometheus metrics from chronyc tracking output
-# Runs as a cron job and outputs to textfile collector directory
-
-OUTPUT_FILE="${1:-/var/lib/node-exporter/chrony.prom}"
-
-# Get chrony tracking info
-tracking=$(chronyc tracking 2>/dev/null)
-
-if [ -z "$tracking" ]; then
-    echo "chrony_up 0" > "$OUTPUT_FILE.tmp"
-    mv "$OUTPUT_FILE.tmp" "$OUTPUT_FILE"
-    exit 0
-fi
-
-# Parse values
-stratum=$(echo "$tracking" | grep "^Stratum" | awk '{print $3}')
-system_time=$(echo "$tracking" | grep "^System time" | awk '{print $4}')
-root_delay=$(echo "$tracking" | grep "^Root delay" | awk '{print $4}')
-root_dispersion=$(echo "$tracking" | grep "^Root dispersion" | awk '{print $4}')
-last_offset=$(echo "$tracking" | grep "^Last offset" | awk '{print $4}')
-rms_offset=$(echo "$tracking" | grep "^RMS offset" | awk '{print $4}')
-frequency=$(echo "$tracking" | grep "^Frequency" | awk '{print $3}')
-ref_id=$(echo "$tracking" | grep "^Reference ID" | awk '{print $4}' | tr -d '()')
-
-# Handle negative offsets (remove leading + or keep -)
-system_time=$(echo "$system_time" | sed 's/^+//')
-last_offset=$(echo "$last_offset" | sed 's/^+//')
-rms_offset=$(echo "$rms_offset" | sed 's/^+//')
-
-# Output Prometheus metrics
-cat > "$OUTPUT_FILE.tmp" << EOF
-# HELP chrony_stratum Current stratum of the NTP server
-# TYPE chrony_stratum gauge
-chrony_stratum ${stratum:-0}
-# HELP chrony_system_time_offset_seconds System time offset from NTP time
-# TYPE chrony_system_time_offset_seconds gauge
-chrony_system_time_offset_seconds ${system_time:-0}
-# HELP chrony_root_delay_seconds Total round-trip delay to the primary reference clock
-# TYPE chrony_root_delay_seconds gauge
-chrony_root_delay_seconds ${root_delay:-0}
-# HELP chrony_root_dispersion_seconds Total dispersion to the primary reference clock
-# TYPE chrony_root_dispersion_seconds gauge
-chrony_root_dispersion_seconds ${root_dispersion:-0}
-# HELP chrony_last_offset_seconds Estimated offset of the last update
-# TYPE chrony_last_offset_seconds gauge
-chrony_last_offset_seconds ${last_offset:-0}
-# HELP chrony_rms_offset_seconds RMS offset
-# TYPE chrony_rms_offset_seconds gauge
-chrony_rms_offset_seconds ${rms_offset:-0}
-# HELP chrony_frequency_ppm Frequency offset in parts per million
-# TYPE chrony_frequency_ppm gauge
-chrony_frequency_ppm ${frequency:-0}
-# HELP chrony_up Whether chrony is running and synchronized
-# TYPE chrony_up gauge
-chrony_up 1
-# HELP chrony_reference_info Reference clock information
-# TYPE chrony_reference_info gauge
-chrony_reference_info{reference="${ref_id:-unknown}"} 1
-EOF
-
-mv "$OUTPUT_FILE.tmp" "$OUTPUT_FILE"

ansible/roles/node-exporter/handlers/main.yml

@@ -1,9 +0,0 @@
----
-- name: Reload systemd
-  systemd:
-    daemon_reload: yes
-
-- name: Restart node-exporter
-  systemd:
-    name: node-exporter
-    state: restarted

ansible/roles/node-exporter/tasks/main.yml

@@ -1,88 +0,0 @@
----
-# Install and configure Node Exporter
-
-- name: Create node-exporter system group
-  group:
-    name: "{{ node_exporter_group }}"
-    system: yes
-    state: present
-
-- name: Create node-exporter system user
-  user:
-    name: "{{ node_exporter_user }}"
-    group: "{{ node_exporter_group }}"
-    system: yes
-    shell: /usr/sbin/nologin
-    home: "{{ node_exporter_install_dir }}"
-    createhome: no
-
-- name: Create installation directory
-  file:
-    path: "{{ node_exporter_install_dir }}"
-    state: directory
-    owner: "{{ node_exporter_user }}"
-    group: "{{ node_exporter_group }}"
-    mode: '0755'
-
-- name: Check if node_exporter is already installed
-  stat:
-    path: "{{ node_exporter_binary }}"
-  register: node_exporter_installed
-
-- name: Get installed version
-  command: "{{ node_exporter_binary }} --version"
-  register: installed_version
-  when: node_exporter_installed.stat.exists
-  changed_when: false
-  failed_when: false
-
-- name: Download node_exporter
-  get_url:
-    url: "https://github.com/prometheus/node_exporter/releases/download/v{{ node_exporter_version }}/node_exporter-{{ node_exporter_version }}.linux-amd64.tar.gz"
-    dest: /tmp/node_exporter.tar.gz
-    mode: '0644'
-  when: not node_exporter_installed.stat.exists or (installed_version.stdout is defined and node_exporter_version not in installed_version.stdout)
-
-- name: Extract node_exporter
-  unarchive:
-    src: /tmp/node_exporter.tar.gz
-    dest: /tmp
-    remote_src: yes
-  when: not node_exporter_installed.stat.exists or (installed_version.stdout is defined and node_exporter_version not in installed_version.stdout)
-
-- name: Install node_exporter binary
-  copy:
-    src: "/tmp/node_exporter-{{ node_exporter_version }}.linux-amd64/node_exporter"
-    dest: "{{ node_exporter_binary }}"
-    owner: "{{ node_exporter_user }}"
-    group: "{{ node_exporter_group }}"
-    mode: '0755'
-    remote_src: yes
-  when: not node_exporter_installed.stat.exists or (installed_version.stdout is defined and node_exporter_version not in installed_version.stdout)
-  notify: Restart node-exporter
-
-- name: Create systemd service
-  template:
-    src: node_exporter.service.j2
-    dest: /etc/systemd/system/node-exporter.service
-    owner: root
-    group: root
-    mode: '0644'
-  notify:
-    - Reload systemd
-    - Restart node-exporter
-
-- name: Enable and start node-exporter service
-  systemd:
-    name: node-exporter
-    enabled: yes
-    state: started
-    daemon_reload: yes
-
-- name: Clean up downloaded files
-  file:
-    path: "{{ item }}"
-    state: absent
-  loop:
-    - /tmp/node_exporter.tar.gz
-    - "/tmp/node_exporter-{{ node_exporter_version }}.linux-amd64"

ansible/roles/node-exporter/templates/node_exporter.service.j2

@@ -1,38 +0,0 @@
-[Unit]
-Description=Prometheus Node Exporter
-Documentation=https://github.com/prometheus/node_exporter
-After=network-online.target
-Wants=network-online.target
-
-[Service]
-Type=simple
-User={{ node_exporter_user }}
-Group={{ node_exporter_group }}
-ExecStart={{ node_exporter_binary }} \
-  --web.listen-address=:{{ node_exporter_port }} \
-{% for collector in node_exporter_enabled_collectors %}
-  --collector.{{ collector }} \
-{% endfor %}
-{% for collector in node_exporter_disabled_collectors %}
-  --no-collector.{{ collector }} \
-{% endfor %}
-{% if node_exporter_textfile_dir is defined %}
-  --collector.textfile \
-  --collector.textfile.directory={{ node_exporter_textfile_dir }} \
-{% endif %}
-  --collector.filesystem.mount-points-exclude='^/(sys|proc|dev|host|etc)($$|/)'
-Restart=on-failure
-RestartSec=10s
-
-# Security hardening
-NoNewPrivileges=yes
-ProtectSystem=strict
-ProtectHome=yes
-PrivateTmp=yes
-ReadOnlyPaths=/
-{% if node_exporter_textfile_dir is defined %}
-ReadWritePaths={{ node_exporter_textfile_dir }}
-{% endif %}
-
-[Install]
-WantedBy=multi-user.target

ansible/roles/ssh-ca/tasks/main.yml

@@ -33,12 +33,15 @@
     group: root
     mode: "0644"
 
+# systemd-journal: read every journal without sudo (see sudoers-claude.j2).
 - name: Create the Claude account
   ansible.builtin.user:
     name: "{{ ssh_ca_claude_user }}"
     shell: /bin/bash
     create_home: true
     password: "!"
+    groups: systemd-journal
+    append: true
 
 - name: Look up the owner admin logins
   ansible.builtin.getent:
@@ -125,29 +128,36 @@
       - "'trustedusercakeys /etc/ssh/trusted_user_ca_keys' in ssh_ca_effective_lines"
       - "'authorizedprincipalsfile /etc/ssh/auth_principals/%u' in ssh_ca_effective_lines"
       - "'revokedkeys /etc/ssh/revoked_keys' in ssh_ca_effective_lines"
+      - "'passwordauthentication no' in ssh_ca_effective_lines"
+      - "'kbdinteractiveauthentication no' in ssh_ca_effective_lines"
     fail_msg: "sshd -T does not show the SSH CA settings; another config line takes precedence"
 
-# The unit is sshd.service on Arch and RedHat, ssh.service on Debian/Ubuntu.
-# Ubuntu also lists sshd.service as a not-found alias, so the running one is
-# picked. sshd -T only parses the files, so the running daemon is reloaded on
-# every run; a reload keeps open sessions.
-- name: Read the service list
-  ansible.builtin.service_facts:
+# The unit is sshd.service on Arch and RedHat, ssh.service on Debian/Ubuntu,
+# where sshd.service is an alias that resolves to it. sshd -T only parses the
+# files, so the running daemon is reloaded on every run; a reload keeps open
+# sessions. None runs when sshd is socket activated and stopped, or per
+# connection (ssh-mesh-only role); the next start reads the config.
+# systemctl directly: service_facts took 3-12 s per host.
+- name: Find the running sshd unit
+  ansible.builtin.command: systemctl list-units --state=running --plain --no-legend --type=service sshd.service ssh.service
+  changed_when: false
+  register: ssh_ca_sshd_running
 
 - name: Reload the running sshd unit
   vars:
-    ssh_ca_sshd_units: "{{ ansible_facts.services | dict2items | selectattr('key', 'in', ['sshd.service', 'ssh.service']) | selectattr('value.state', 'equalto', 'running') | map(attribute='key') | list }}"
+    ssh_ca_sshd_units: "{{ ssh_ca_sshd_running.stdout_lines | map('split') | map('first') | unique | list }}"
   block:
-    - name: Check exactly one sshd unit is running
+    - name: Check at most one sshd unit is running
       ansible.builtin.assert:
         that:
-          - ssh_ca_sshd_units | length == 1
-        fail_msg: "Expected one running sshd.service or ssh.service, found {{ ssh_ca_sshd_units }}"
+          - ssh_ca_sshd_units | length <= 1
+        fail_msg: "Expected at most one running sshd.service or ssh.service, found {{ ssh_ca_sshd_units }}"
 
     - name: Reload sshd so the running daemon uses the CA config
       ansible.builtin.systemd:
         name: "{{ ssh_ca_sshd_units | first }}"
         state: reloaded
+      when: ssh_ca_sshd_units | length == 1
 
 - name: Report the admin logins' authorized keys
   ansible.builtin.include_tasks: report.yml

ansible/roles/ssh-ca/templates/ssh-ca.conf.j2

@@ -2,3 +2,5 @@
 TrustedUserCAKeys /etc/ssh/trusted_user_ca_keys
 AuthorizedPrincipalsFile /etc/ssh/auth_principals/%u
 RevokedKeys /etc/ssh/revoked_keys
+PasswordAuthentication no
+KbdInteractiveAuthentication no

ansible/roles/ssh-ca/templates/sudoers-claude.j2

@@ -1,6 +1,12 @@
 # {{ ansible_managed }}
-# NOEXEC: journalctl/systemctl open a pager, which must not spawn a shell.
-{{ ssh_ca_claude_user }} ALL=(root) NOPASSWD:NOEXEC: /usr/bin/journalctl, /usr/bin/journalctl *, /usr/bin/systemctl status, /usr/bin/systemctl status *, /usr/bin/systemctl list-units, /usr/bin/systemctl list-units *, /usr/bin/ss -tulpn, /usr/sbin/sshd -T
+# Logs and unit status come from the systemd-journal group, not sudo:
+# journalctl as root would also allow --vacuum-*/--rotate.
+# Commands are listed for both the Arch and the Debian/Ubuntu path.
+# nft only with fixed arguments: it joins argv into one command line, so a
+# wildcard would admit "; flush ruleset". No docker start: -a/-i attach to a
+# container's stdin; docker restart also starts stopped containers.
+{{ ssh_ca_claude_user }} ALL=(root) NOPASSWD:NOEXEC: /usr/bin/ss -tulpn, /usr/sbin/sshd -T, /usr/bin/nft list ruleset, /usr/sbin/nft list ruleset, /usr/bin/netbird status, /usr/bin/netbird status *, /usr/local/bin/netbird status, /usr/local/bin/netbird status *
+{{ ssh_ca_claude_user }} ALL=(root) NOPASSWD:NOEXEC: /usr/bin/docker ps, /usr/bin/docker ps *, /usr/bin/docker logs *, /usr/bin/docker stats --no-stream, /usr/bin/docker restart *
 {% for unit in ssh_ca_claude_sudo_units %}
 {{ ssh_ca_claude_user }} ALL=(root) NOPASSWD:NOEXEC: /usr/bin/systemctl restart {{ unit }}, /usr/bin/systemctl reload {{ unit }}
 {% endfor %}

ansible/roles/ssh-mesh-only/defaults/main.yml

@@ -0,0 +1,9 @@
+---
+# The only interface sshd accepts connections on.
+ssh_mesh_interface: wt0
+
+# Minutes until a switchover reverts to the distro listener, unless a new
+# login through the mesh-only listener confirms it first.
+ssh_mesh_revert_minutes: 5
+
+ssh_mesh_udev_rule: /etc/udev/rules.d/90-ssh-mesh-only.rules

ansible/roles/ssh-mesh-only/tasks/main.yml

@@ -0,0 +1,113 @@
+---
+- name: Load the distro's sshd unit names
+  ansible.builtin.include_vars: "{{ ansible_facts['os_family'] }}.yml"
+
+- name: Look for the mesh interface
+  ansible.builtin.stat:
+    path: "/sys/class/net/{{ ssh_mesh_interface }}"
+  register: ssh_mesh_iface
+
+- name: Check the mesh interface exists
+  ansible.builtin.assert:
+    that:
+      - ssh_mesh_iface.stat.exists
+    fail_msg: "No {{ ssh_mesh_interface }}: sshd would be unreachable. Install the NetBird client first."
+
+- name: Create the socket unit directory
+  ansible.builtin.file:
+    path: "{{ ssh_mesh_socket_file | dirname }}"
+    state: directory
+    owner: root
+    group: root
+    mode: "0755"
+
+- name: Install the mesh-only socket
+  ansible.builtin.template:
+    src: "{{ ssh_mesh_socket_template }}"
+    dest: "{{ ssh_mesh_socket_file }}"
+    owner: root
+    group: root
+    mode: "0644"
+
+- name: Bind the sshd service to the socket
+  when: ssh_mesh_service_dropin | length > 0
+  block:
+    - name: Create the service drop-in directory
+      ansible.builtin.file:
+        path: "{{ ssh_mesh_service_dropin | dirname }}"
+        state: directory
+        owner: root
+        group: root
+        mode: "0755"
+
+    - name: Install the service drop-in
+      ansible.builtin.template:
+        src: ssh.service-dropin.conf.j2
+        dest: "{{ ssh_mesh_service_dropin }}"
+        owner: root
+        group: root
+        mode: "0644"
+
+- name: Install the udev rule that starts the socket with the interface
+  ansible.builtin.template:
+    src: udev.rules.j2
+    dest: "{{ ssh_mesh_udev_rule }}"
+    owner: root
+    group: root
+    mode: "0644"
+  register: ssh_mesh_udev
+
+- name: Reload the udev rules
+  ansible.builtin.command: udevadm control --reload
+  when: ssh_mesh_udev is changed
+  changed_when: true
+
+- name: Install the switch and revert scripts
+  ansible.builtin.template:
+    src: "{{ item }}.j2"
+    dest: "/usr/local/sbin/{{ item }}"
+    owner: root
+    group: root
+    mode: "0755"
+  loop:
+    - ssh-mesh-switch
+    - ssh-mesh-revert
+
+- name: Reload systemd units
+  ansible.builtin.systemd:
+    daemon_reload: true
+
+- name: Read the sshd listeners
+  ansible.builtin.command: ss -Htln 'sport = :22'
+  changed_when: false
+  register: ssh_mesh_listeners
+
+# Each run that finds a listener outside the interface switches over. An
+# include, because meta reset_connection ignores when: and a reconnect costs a
+# YubiKey PIN from a workstation.
+- name: Switch sshd to the mesh-only socket
+  ansible.builtin.include_tasks: switch.yml
+  when: >-
+    ssh_mesh_listeners.stdout_lines | length == 0 or
+    ssh_mesh_listeners.stdout_lines | reject('search', '%' ~ ssh_mesh_interface ~ ':22\s') | list | length > 0
+
+- name: Check sshd listens on the mesh interface only
+  ansible.builtin.assert:
+    that:
+      - ssh_mesh_listeners.stdout_lines | length > 0
+      - ssh_mesh_listeners.stdout_lines | reject('search', '%' ~ ssh_mesh_interface ~ ':22\s') | list | length == 0
+    fail_msg: "Listeners outside {{ ssh_mesh_interface }}: {{ ssh_mesh_listeners.stdout_lines }}"
+
+- name: Cancel the revert timer
+  ansible.builtin.command: systemctl stop ssh-mesh-revert.timer
+  changed_when: false
+  failed_when: false
+
+- name: Check the public name refuses SSH
+  ansible.builtin.wait_for:
+    host: "{{ inventory_hostname }}"
+    port: 22
+    state: stopped
+    timeout: 15
+  delegate_to: localhost
+  become: false

ansible/roles/ssh-mesh-only/tasks/switch.yml

@@ -0,0 +1,27 @@
+---
+# The revert timer runs in its own unit, so a switch that locks the play out is
+# undone after ssh_mesh_revert_minutes. main.yml cancels it after checking the
+# listeners read through the new one.
+- name: Clear a revert timer left by an earlier run
+  ansible.builtin.command: systemctl stop ssh-mesh-revert.timer ssh-mesh-revert.service
+  changed_when: false
+  failed_when: false
+
+- name: Arm the revert timer
+  ansible.builtin.command: >-
+    systemd-run --unit=ssh-mesh-revert --collect
+    --on-active={{ ssh_mesh_revert_minutes }}min
+    /usr/local/sbin/ssh-mesh-revert
+  changed_when: true
+
+- name: Switch the listener
+  ansible.builtin.command: systemd-run --unit=ssh-mesh-switch --collect --wait /usr/local/sbin/ssh-mesh-switch
+  changed_when: true
+
+- name: Reconnect through the new listener
+  ansible.builtin.meta: reset_connection
+
+- name: Read the sshd listeners after reconnecting
+  ansible.builtin.command: ss -Htln 'sport = :22'
+  changed_when: false
+  register: ssh_mesh_listeners

ansible/roles/ssh-mesh-only/templates/ssh-mesh-revert.j2

@@ -0,0 +1,9 @@
+#!/bin/sh
+# {{ ansible_managed }}
+# Restores the distro listener on all addresses. Armed as a timer by every
+# switchover; the play cancels it after a login through the new listener.
+systemctl stop {{ ssh_mesh_service }} {{ ssh_mesh_socket }}
+rm -f {{ ssh_mesh_udev_rule }} {{ ssh_mesh_socket_file }}{{ (' ' ~ ssh_mesh_service_dropin) if ssh_mesh_service_dropin else '' }}
+udevadm control --reload
+systemctl daemon-reload
+systemctl enable --now {{ ssh_mesh_distro_listener }}

ansible/roles/ssh-mesh-only/templates/ssh-mesh-switch.j2

@@ -0,0 +1,11 @@
+#!/bin/sh
+# {{ ansible_managed }}
+# Replaces the distro listener with the {{ ssh_mesh_interface }}-only socket.
+# Open sessions survive: they run in logind scopes and the sshd units use
+# KillMode=process. Any failure restores the distro listener at once.
+set -eu
+trap '/usr/local/sbin/ssh-mesh-revert' EXIT
+systemctl disable {{ ssh_mesh_disable_units }}
+systemctl stop {{ ssh_mesh_service }} {{ ssh_mesh_socket }}
+systemctl start {{ ssh_mesh_socket }}
+trap - EXIT

ansible/roles/ssh-mesh-only/templates/ssh.service-dropin.conf.j2

@@ -0,0 +1,7 @@
+# {{ ansible_managed }}
+# sshd holds the socket's listener; it stops with the socket, so it never keeps
+# a listener on a removed {{ ssh_mesh_interface }}. Replaces the Requires= that
+# enabling {{ ssh_mesh_socket }} would add.
+[Unit]
+BindsTo={{ ssh_mesh_socket }}
+After={{ ssh_mesh_socket }}

ansible/roles/ssh-mesh-only/templates/ssh.socket-addresses.conf.j2

@@ -0,0 +1,8 @@
+# {{ ansible_managed }}
+# Listen on {{ ssh_mesh_interface }} only. The unit is disabled; the udev rule
+# starts it whenever {{ ssh_mesh_interface }} appears.
+[Socket]
+ListenStream=
+ListenStream=22
+BindToDevice={{ ssh_mesh_interface }}
+BindIPv6Only=both

ansible/roles/ssh-mesh-only/templates/sshd.socket.j2

@@ -0,0 +1,12 @@
+# {{ ansible_managed }}
+# Started by the udev rule whenever {{ ssh_mesh_interface }} appears; BindToDevice
+# stops it when the interface goes away. No [Install]: waiting for the device
+# from sockets.target would hold up boot until NetBird is up.
+[Unit]
+Description=OpenSSH on {{ ssh_mesh_interface }} only
+
+[Socket]
+ListenStream=22
+BindToDevice={{ ssh_mesh_interface }}
+BindIPv6Only=both
+Accept=yes

ansible/roles/ssh-mesh-only/templates/udev.rules.j2

@@ -0,0 +1,3 @@
+# {{ ansible_managed }}
+# NetBird deletes and recreates {{ ssh_mesh_interface }} on every engine restart.
+SUBSYSTEM=="net", KERNEL=="{{ ssh_mesh_interface }}", ENV{SYSTEMD_WANTS}+="{{ ssh_mesh_socket }}"

ansible/roles/ssh-mesh-only/vars/Archlinux.yml

@@ -0,0 +1,10 @@
+---
+# No packaged socket; sshd.socket (Accept=yes) pairs with the packaged
+# per-connection sshd@.service, since sshd -D cannot take a listening fd.
+ssh_mesh_socket: sshd.socket
+ssh_mesh_service: sshd.service
+ssh_mesh_distro_listener: sshd.service
+ssh_mesh_disable_units: sshd.service
+ssh_mesh_socket_file: /etc/systemd/system/sshd.socket
+ssh_mesh_socket_template: sshd.socket.j2
+ssh_mesh_service_dropin: ""

ansible/roles/ssh-mesh-only/vars/Debian.yml

@@ -0,0 +1,10 @@
+---
+# Ubuntu's ssh.socket (Accept=no) hands its listener to ssh.service. The
+# drop-in is named like sshd-socket-generator's output so it masks it.
+ssh_mesh_socket: ssh.socket
+ssh_mesh_service: ssh.service
+ssh_mesh_distro_listener: ssh.socket
+ssh_mesh_disable_units: ssh.service ssh.socket
+ssh_mesh_socket_file: /etc/systemd/system/ssh.socket.d/addresses.conf
+ssh_mesh_socket_template: ssh.socket-addresses.conf.j2
+ssh_mesh_service_dropin: /etc/systemd/system/ssh.service.d/ssh-mesh-only.conf

ansible/site.yml

@@ -0,0 +1,41 @@
+---
+# The whole defined state of every Ansible-managed host, in dependency order.
+# Run from ansible/ on a mesh-connected workstation; the inventory comes from
+# ansible.cfg:
+#
+#   ansible-playbook site.yml
+#
+# One-off actions are not state and stay separate: plays/update-os.yml,
+# plays/update-dist-upgrade.yml, plays/rotate-netbird-secrets.yml,
+# plays/configure-shelly-ntp.yml (devices outside the inventory).
+
+- name: Base system
+  ansible.builtin.import_playbook: plays/common.yml
+- name: VM time sync
+  ansible.builtin.import_playbook: plays/configure-vm-time-sync.yml
+
+- name: SSH user CA
+  ansible.builtin.import_playbook: plays/configure-ssh-ca.yml
+- name: NetBird host clients
+  ansible.builtin.import_playbook: plays/install-netbird-client.yml
+- name: sshd on the mesh interface only
+  ansible.builtin.import_playbook: plays/configure-ssh-mesh-only.yml
+
+- name: NetBird server
+  ansible.builtin.import_playbook: plays/install-netbird-server.yml
+- name: Uptime Kuma
+  ansible.builtin.import_playbook: plays/install-uptime-kuma.yml
+- name: NetBird proxies
+  ansible.builtin.import_playbook: plays/install-netbird-proxy.yml
+
+- name: Edge host base
+  ansible.builtin.import_playbook: plays/install-edge-host.yml
+- name: Z-Wave gateway
+  ansible.builtin.import_playbook: plays/install-zwave-gateway.yml
+- name: Heating CAN
+  ansible.builtin.import_playbook: plays/install-heating-can.yml
+- name: NTP service
+  ansible.builtin.import_playbook: plays/install-ntp-ha.yml
+
+- name: Close the SSH connections
+  ansible.builtin.import_playbook: plays/close-connections.yml

images/edge-host-iso/README.md

@@ -12,7 +12,10 @@ The image is the official archiso **releng** profile plus:
 - `edge-install` in the live environment — partitions the NVMe
   (ESP + ext4 root, systemd-boot), pacstraps the base system incl.
   docker/python, configures DHCP via systemd-networkd, enables sshd and
-  drops the common homelab SSH keys for root (key-only, no root password)
+  sets up root login (key-only, no root password) with owner certificates
+  from the OpenBao SSH CA, fetched at build time from the `ssh-ca` role's
+  URL, and the break-glass key (YubiKey NEO, PIV) as fallback:
+  `ssh -I /usr/lib/opensc-pkcs11.so root@<host>`
 - the extra disk tools the installer needs
 
 Division of labour: the ISO produces an **ansible-ready base system**;
@@ -30,7 +33,8 @@ sudo ./build.sh          # ISO lands in ./out/
 
 The profile is assembled at build time from `/usr/share/archiso/configs/releng`
 plus the overlay in `airootfs/` — the repo carries only the delta, so
-archiso updates flow in automatically.
+archiso updates flow in automatically. The build needs openbao.renner.uno
+for the CA key; rebuild the ISO after a CA rotation.
 
 ## Install
 
@@ -38,8 +42,7 @@ archiso updates flow in automatically.
 2. Boot the target from it (UEFI, Secure Boot off; one-time F12/setup
    visit if the box doesn't already boot USB first)
 3. Either on the console (auto-login as root) or headless —
-   `ssh root@<dhcp-ip>` works out of the box, the live env carries the
-   homelab keys — run `edge-install`. It auto-detects the internal disk
+   `ssh root@<dhcp-ip>` with an owner certificate — run `edge-install`. It auto-detects the internal disk
    (NVMe, else eMMC — covers the Wyse 3040 — else first non-USB disk),
    shows the target and requires typing the hostname before wiping
    anything. Override with `edge-install /dev/sdY`.
@@ -49,10 +52,8 @@ archiso updates flow in automatically.
    `ansible/inventory/pve_hosts.yml` and, from the workstation:
    ```
    cd ansible
-   ansible-playbook -i inventory/pve_hosts.yml plays/install-edge-host.yml
+   ansible-playbook site.yml
    ```
-   One run over the whole group, no `--limit`: the play carries
-   `certificates` and `common` itself, so there is no separate `common.yml`
-   step. Running it converges every edge host, which is the point — a run
-   that touches only the new box tells you nothing about the others.
+   One run, no `--limit`. It converges every host, which is the point — a
+   run that touches only the new box tells you nothing about the others.
 5. Continue with the cutover checklist in `ansible/docs/edge-host.md`.

images/edge-host-iso/airootfs/root/.ssh/authorized_keys

@@ -1,4 +1 @@
-ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPzFNBP3bq2m/6JqDqod9w+FP/uxXxHMolqhq3Ojflr1 krenner@fortress
-ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGfe3HJqFR+jIGrboweseAO7XjIVE24sje/b1HybgU9C ansible-runner@ci-new
-ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICyNnX1I4W9jyDvCJ1DJauHSmbo1ZXSyBxVhSAwMWgen woodpecker-ci@homelab
-ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBB1OFjh6Q5xeioI6Cwfuv8Xka27veomYy9tKBbpMo8mcrsB4yIN4FX8HovBLtogetBhwBUefBar8nhA9GbRh6HQ= krenner@krenner-XPS-13-9310
+ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBKvSl9LUP8u3S2Sq5YTWpW0Jxbue8cJhD7/LMFQbEsa3xo145HVR7pBzRxW71MUy+jgrgmpSlDhIb606Hswhvoo= breakglass-neo-6532348

images/edge-host-iso/airootfs/root/edge-authorized_keys

@@ -1,4 +1 @@
-ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPzFNBP3bq2m/6JqDqod9w+FP/uxXxHMolqhq3Ojflr1 krenner@fortress
-ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGfe3HJqFR+jIGrboweseAO7XjIVE24sje/b1HybgU9C ansible-runner@ci-new
-ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICyNnX1I4W9jyDvCJ1DJauHSmbo1ZXSyBxVhSAwMWgen woodpecker-ci@homelab
-ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBB1OFjh6Q5xeioI6Cwfuv8Xka27veomYy9tKBbpMo8mcrsB4yIN4FX8HovBLtogetBhwBUefBar8nhA9GbRh6HQ= krenner@krenner-XPS-13-9310
+ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBKvSl9LUP8u3S2Sq5YTWpW0Jxbue8cJhD7/LMFQbEsa3xo145HVR7pBzRxW71MUy+jgrgmpSlDhIb606Hswhvoo= breakglass-neo-6532348

images/edge-host-iso/airootfs/usr/local/bin/edge-install

@@ -81,6 +81,17 @@ ln -sf /run/systemd/resolve/stub-resolv.conf /mnt/etc/resolv.conf
 install -d -m700 /mnt/root/.ssh
 install -m600 /root/edge-authorized_keys /mnt/root/.ssh/authorized_keys
 
+# SSH user CA trust as baked into this ISO (owner certificates for root).
+install -m644 -t /mnt/etc/ssh /etc/ssh/trusted_user_ca_keys /etc/ssh/revoked_keys
+install -d -m755 /mnt/etc/ssh/auth_principals /mnt/etc/ssh/sshd_config.d
+install -m644 /etc/ssh/auth_principals/root /mnt/etc/ssh/auth_principals/root
+install -m644 /etc/ssh/sshd_config.d/50-ssh-ca.conf /mnt/etc/ssh/sshd_config.d/
+# sshd -t/-T need host keys; sshdgenkeys would create them on first boot.
+ssh-keygen -A -f /mnt
+arch-chroot /mnt sshd -t
+arch-chroot /mnt sshd -T | grep -qix 'trustedusercakeys /etc/ssh/trusted_user_ca_keys' \
+  || { echo "installed sshd does not use the SSH CA"; exit 1; }
+
 arch-chroot /mnt systemctl enable \
   systemd-networkd systemd-resolved systemd-timesyncd sshd docker
 
@@ -104,8 +115,6 @@ echo
 echo "Then, from the workstation:"
 echo "  1. confirm 'ssh root@${HOST}' works (it must be reachable by name)"
 echo "  2. add ${HOST} to [edge_host] in ansible/inventory/pve_hosts.yml"
-echo "  3. cd ansible && ansible-playbook -i inventory/pve_hosts.yml \\"
-echo "       plays/install-edge-host.yml"
+echo "  3. cd ansible && ansible-playbook site.yml"
 echo
-echo "One run, whole group, no --limit: the play carries certificates and"
-echo "common itself, so there is no separate common.yml step."
+echo "One run, all hosts, no --limit."

images/edge-host-iso/build-inner.sh

@@ -10,6 +10,23 @@ profile=/tmp/profile
 cp -r /usr/share/archiso/configs/releng/. "$profile"
 cp -r /src/airootfs/. "$profile/airootfs/"
 
+# SSH user CA trust from the ssh-ca role (mounted at /ssh-ca): the live system
+# and the installed one accept owner certificates for root, like the managed
+# hosts. The CA public key is fetched from OpenBao at build time.
+ca_url=$(sed -n 's/^ssh_ca_public_key_url: "\(.*\)"$/\1/p' /ssh-ca/defaults/main.yml)
+principal=$(sed -n 's/^ssh_ca_owner_principal: \(.*\)$/\1/p' /ssh-ca/defaults/main.yml)
+[ -n "$ca_url" ] && [ -n "$principal" ] || { echo "ssh-ca defaults not readable"; exit 1; }
+ca_key=$(curl -sSf "$ca_url")
+[[ "$ca_key" == "ssh-ed25519 AAAA"* ]] || { echo "unexpected CA key from $ca_url"; exit 1; }
+sshd_conf=$(grep -vx '# {{ ansible_managed }}' /ssh-ca/templates/ssh-ca.conf.j2)
+! grep -q '{[{%]' <<<"$sshd_conf" || { echo "ssh-ca.conf.j2 has template logic"; exit 1; }
+ssh_dir="$profile/airootfs/etc/ssh"
+install -d "$ssh_dir/sshd_config.d" "$ssh_dir/auth_principals"
+printf '%s\n' "$ca_key" > "$ssh_dir/trusted_user_ca_keys"
+cp /ssh-ca/files/revoked_keys "$ssh_dir/revoked_keys"
+printf '%s\n' "$principal" > "$ssh_dir/auth_principals/root"
+printf '%s\n' "$sshd_conf" > "$ssh_dir/sshd_config.d/50-ssh-ca.conf"
+
 # Tools the installer script needs in the live environment (idempotent —
 # skip entries releng already ships).
 for pkg in gptfdisk dosfstools e2fsprogs; do

images/edge-host-iso/build.sh

@@ -13,6 +13,7 @@ mkdir -p "$out"
 
 docker run --rm --privileged \
   -v "$here:/src:ro" \
+  -v "$here/../../ansible/roles/ssh-ca:/ssh-ca:ro" \
   -v "$out:/out" \
   archlinux:latest /src/build-inner.sh
 

kubernetes/MIGRATION-sops-openbao.md

@@ -52,9 +52,9 @@ kubectl -n openbao exec -it openbao-0 -- bao operator unseal   # paste key
 
 **Unseal is automatic since 2026-09-08** (static seal, PR #825, issue #805):
 the seal key lives in the `openbao-seal-key` Secret (created by `bootstrap.sh`
-from `pass show homelab/openbao/seal-key`) and every restarted replica unseals
-itself. The shamir key above became the **recovery key** in the migration; it
-is kept in `pass` under its old name, `homelab/openbao/unseal-key`. The server
+from `~/.config/homelab-secrets/seal-key.age`) and every restarted replica
+unseals itself. The shamir key above became the **recovery key** in the
+migration; it is `~/.config/homelab-secrets/openbao-recovery-key.age`. The server
 runs three raft peers (see `infrastructure/openbao/helmrelease.yaml`); a sealed
 one is neither a voter nor a standby, so a replica that stays sealed after a
 restart — wrong or missing seal-key Secret — costs redundancy, two cost
@@ -237,19 +237,21 @@ one full reconcile + one OpenBao pod restart/unseal cycle:
 - **Backups:** ✅ nightly raft-snapshot CronJob (03:00, keeps 7 on a
   dedicated PVC; see infrastructure/openbao/snapshot-cronjob.yaml) + velero
   `openbao-daily` Schedule (06:15) ships that PVC offsite, 30 d retention.
-  A raft snapshot restore needs the matching seal key — which lives in
-  `pass`, outside the cluster (GPG key backed up externally 2026-07-14).
+  A raft snapshot restore needs the matching seal key — which lives outside
+  the cluster in `~/.config/homelab-secrets/seal-key.age` (age, encrypted to
+  both YubiKeys and the offline paper key; copy on nas-2).
   A failed snapshot Job is the sealed-after-restart alarm. Still owed:
   one tested restore before phase 4.
-- **Cluster rebuild order:** seal-key Secret from `pass` → Flux brings up
+- **Cluster rebuild order:** seal-key Secret from `seal-key.age` → Flux brings up
   openbao (empty) → init → restore raft snapshot → replicas unseal themselves
   → ClusterSecretStore Ready → ESO materializes Secrets → apps start
   (`bootstrap.sh flux` does all of it; RUNBOOK-cold-rebuild.md Step 4 by
   hand). Until then, secret-consuming apps simply stay Pending on missing
   Secrets — same as any missing dependency.
-- **Break-glass:** `pass` holds seal key, recovery key + root token (GPG,
-  offline-capable). With those, everything is recoverable from a raft
-  snapshot alone.
+- **Break-glass:** `~/.config/homelab-secrets` holds the seal key and
+  recovery key (YubiKeys + paper key, copy on nas-2);
+  `~/.config/openbao/root-token.age` holds the root token (YubiKeys). With
+  those, everything is recoverable from a raft snapshot alone.
 - **Ansible Vault password:** `ansible/vault-password.sh` reads
   `ANSIBLE_VAULT_PASSWORD` from the environment with no offline backup as of
   2026-07-14. It gates decryption of several Ansible-Vault files (PBS

kubernetes/RUNBOOK-cold-rebuild.md

@@ -67,17 +67,24 @@ These must exist outside the cluster or the rebuild cannot complete:
       `flux` phase creates the `sops-age` Secret, and any not-yet-migrated sops
       files decrypt with it. (Post-OpenBao-phase-4 this may be retired — see
       `MIGRATION-sops-openbao.md`.)
-- [ ] **OpenBao seal key, recovery key + root token** — in the GPG `pass`
-      store (`homelab/openbao/seal-key`, `homelab/openbao/unseal-key` — the
-      former shamir key, now the recovery key — and `homelab/openbao/root-token`),
-      and the GPG private key backed up off-machine. The seal key is the one
-      the snapshots are encrypted under: without it, no OpenBao snapshot can be
-      restored and no secret-consuming app can start.
+- [ ] **OpenBao seal key, recovery key, restore + pull credentials** — age
+      files in `~/.config/homelab-secrets/` (`seal-key.age`,
+      `openbao-recovery-key.age` — the former shamir key —, `s3-access.age`,
+      `s3-secret.age`, `ghcr-pull.age`, `hetzner-cloud-api-secret.age`),
+      encrypted to both YubiKeys and the
+      offline paper key in the safe; a copy is on nas-2 in
+      `~ladmin/homelab-secrets-backup` (`~/.ssh/ssh-keys-rotate dr-copy`). The
+      root token is `~/.config/openbao/root-token.age` (YubiKeys only). Needed
+      to decrypt: one YubiKey with its PIV PIN, or the paper key. The seal key
+      is the one the snapshots are encrypted under: without it, no OpenBao
+      snapshot can be restored and no secret-consuming app can start.
 - [ ] **OpenBao raft snapshot** reachable — nightly PVC snapshots
       (`infrastructure/openbao/snapshot-cronjob.yaml`) plus the Velero
       `openbao-daily` offsite copy. In a total-loss rebuild the offsite Velero
       copy is the one that survives.
-- [ ] **Tooling:** `talosctl`, `kubectl`, `flux`, `tofu`, `bao`, plus the
+- [ ] **Tooling:** `talosctl`, `kubectl`, `flux`, `tofu`, `bao`, `age` +
+      `age-plugin-yubikey` (the plugin is only needed for the YubiKeys; the
+      paper key needs `age` alone), plus the
       OpenTofu backend + UniFi credentials the `opentofu/` stack needs. No
       Proxmox credentials — there is no Proxmox any more.
 - [ ] **Install media** written and to hand: the Image Factory ISO named in
@@ -299,14 +306,16 @@ output** — a green step over a still-forming Ceph is a known trap.
 ## Step 4 — openbao: restore the snapshot
 
 `bootstrap.sh flux` does this step itself (`restore_openbao`): it creates the
-`openbao-seal-key` Secret from `pass` before Flux brings OpenBao up, initialises
+`openbao-seal-key` Secret from `~/.config/homelab-secrets/seal-key.age` before Flux brings OpenBao up, initialises
 the empty server, restores the newest raft snapshot straight out of the Velero
 kopia repository, and waits until every replica has unsealed itself from the
 static seal. What follows is the same procedure by hand, for when the script
 is not an option.
 
 OpenBao unseals itself from the static seal (`openbao-seal-key` Secret, key
-`key`, the 32-byte base64 key from `pass show homelab/openbao/seal-key`). The
+`key`, the 32-byte base64 key in `~/.config/homelab-secrets/seal-key.age`,
+age-encrypted to the owner's YubiKeys and the offline paper key; a copy sits on
+nas-2 in `~ladmin/homelab-secrets-backup`). The
 snapshots are encrypted under that key, so it must be the same key the backed
 up cluster used — a different one restores nothing. Until the snapshot is
 restored, the `ClusterSecretStore` stays NotReady, ESO cannot materialize
@@ -316,7 +325,8 @@ Secrets, and every secret-consuming app stays Pending.
 # 1. The seal key, before the pods start (a pod without it never starts).
 kubectl create namespace openbao --dry-run=client -o yaml | kubectl apply -f -
 kubectl -n openbao create secret generic openbao-seal-key \
-  --from-literal=key="$(pass show homelab/openbao/seal-key)"
+  --from-literal=key="$(age -d -i ~/.config/age/yubikeys.txt ~/.config/homelab-secrets/seal-key.age)"
+# Both YubiKeys lost: -i <file holding the paper key> instead.
 
 # 2. Wait for openbao-0 (uninitialised: 0/1 Ready is expected), then
 #    initialise it. With an auto-seal, init hands out recovery keys and the
@@ -337,7 +347,8 @@ kubectl -n openbao exec -it openbao-0 -- \
 
 # 4. The server seals against the snapshot's keyring and unseals itself from
 #    the static seal; openbao-1 and openbao-2 follow (OrderedReady). Nothing
-#    to type. Afterwards the tokens and recovery key from pass apply again.
+#    to type. Afterwards the operator's root token
+#    (~/.config/openbao/root-token.age) and recovery key apply again.
 kubectl -n openbao get pods -w
 ```
 
@@ -352,7 +363,7 @@ kubectl -n <ns> get externalsecret                # want: SecretSynced
 > A replica that stays **sealed** after a restart could not use the static
 > seal — the Secret is missing or does not match the key the data was
 > written under. Fix the Secret and restart the pod; `bao operator unseal`
-> with the recovery key (`pass show homelab/openbao/unseal-key`) does not
+> with the recovery key (`~/.config/homelab-secrets/openbao-recovery-key.age`) does not
 > unseal an auto-sealed server and is only for recovery-mode operations.
 > Details + rationale: `MIGRATION-sops-openbao.md`.
 
@@ -434,7 +445,7 @@ it: repo → **Settings → Repository → Mirror Settings** in Gitea, or
   Step 2 no longer uses it. Its `flux` and `wait-apps` phases are still used in
   Steps 3, 5 and 6 and work; do not read this as the whole script being gone.
   Its `flux` phase carries the OpenBao restore (Step 4); the seal key and the
-  other secrets are read from `pass` or streamed in at run time and never live
+  other secrets are decrypted from `~/.config/homelab-secrets` or streamed in at run time and never live
   in CI.
 - **OpenBao unseals itself, but only under the seal key the snapshot was
   written with.** Three raft replicas mean the service survives losing one; a

kubernetes/apps/monitoring-rules/prometheusrule-openbao.yaml

@@ -49,7 +49,7 @@ spec:
               failure would cost quorum. Check `kubectl -n openbao get pods`,
               the pod log and `bao status` in the not-ready pod. If it is
               sealed, the static seal failed: verify the openbao-seal-key
-              Secret matches `pass show homelab/openbao/seal-key`.
+              Secret matches ~/.config/homelab-secrets/seal-key.age (age, YubiKey).
 
         - alert: OpenBaoQuorumLost
           # Raft quorum of a three-node cluster is two. At one ready replica
@@ -76,7 +76,7 @@ spec:
               surface only when something rotates or a new secret is needed.
               Replicas unseal themselves from the static seal; one that stays
               sealed could not. Check its log and the openbao-seal-key Secret
-              against `pass show homelab/openbao/seal-key`, fix the Secret and
+              against ~/.config/homelab-secrets/seal-key.age, fix the Secret and
               restart the pod. The recovery key cannot unseal an auto-sealed
               server. Procedure: kubernetes/RUNBOOK-cold-rebuild.md, Step 4.
 

kubernetes/bootstrap.sh

@@ -35,7 +35,8 @@ set -euo pipefail
 # The mirror is kept current server-side by Gitea's native push-mirror. Full
 # cold-rebuild procedure: kubernetes/RUNBOOK-cold-rebuild.md.
 #
-# Requires: talosctl, kubectl, flux, sops age key at ~/.config/sops/age/keys.txt
+# Requires: talosctl, kubectl, flux, sops age key at ~/.config/sops/age/keys.txt,
+# age + age-plugin-yubikey (the DR secrets in ~/.config/homelab-secrets)
 
 TALOS_DIR="$(cd "$(dirname "$0")/talos" && pwd)"
 REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
@@ -190,13 +191,16 @@ if [[ "$FLUX_SOURCE" == "restore-test" && "$PHASE" != "flux" && "$PHASE" != "wai
 fi
 OPENBAO_NS="openbao"
 
-# Where the secrets come from. On a workstation, the operator's pass store. In
-# the in-cluster Job there is no pass and no GPG key — and there should not be:
-# the pass store opens everything, and it stays on the operator's machine. Set
-# OPENBAO_SECRETS_DIR and the run reads them from files instead, which the
+# Where the secrets come from. On a workstation, age files in
+# HOMELAB_SECRETS_DIR, encrypted to the operator's YubiKeys and an offline paper
+# key (a copy sits on nas-2). HOMELAB_AGE_IDENTITY picks what decrypts them: the
+# YubiKey identity stubs by default (touch per file), the paper key's file when
+# both sticks are gone. In the in-cluster Job there is neither — and there
+# should not be: those keys open everything, and they stay with the operator.
+# Set OPENBAO_SECRETS_DIR and the run reads them from files instead, which the
 # operator streams in when the run asks for them:
 #
-#     pass show homelab/openbao/seal-key \
+#     age -d -i ~/.config/age/yubikeys.txt ~/.config/homelab-secrets/seal-key.age \
 #       | kubectl -n vmtest exec -i <pod> -- sh -c 'cat > /secrets/seal-key'
 #
 # They exist in the pod's filesystem for the length of the run and are never
@@ -240,6 +244,15 @@ git_token_for_source() {
     fi
 }
 
+# Literal ~ on purpose: the commands are eval'd on the workstation, which
+# expands it there, and the in-cluster Job prints them for the operator to run
+# on that workstation, where the Job's own HOME (/work) would be wrong.
+# shellcheck disable=SC2088
+[[ -n "${HOMELAB_SECRETS_DIR:-}" ]] || HOMELAB_SECRETS_DIR='~/.config/homelab-secrets'
+# shellcheck disable=SC2088
+[[ -n "${HOMELAB_AGE_IDENTITY:-}" ]] || HOMELAB_AGE_IDENTITY='~/.config/age/yubikeys.txt'
+AGE_SECRET="age -d -i $HOMELAB_AGE_IDENTITY $HOMELAB_SECRETS_DIR"
+
 # Format: <file in SECRETS_DIR>|<what it is>|<how the operator produces it>
 OPENBAO_REQUIRED_SECRETS=(
     "sops-age.key|SOPS age key|cat ~/.config/sops/age/keys.txt"
@@ -255,18 +268,18 @@ OPENBAO_REQUIRED_SECRETS=(
     # Emptiness is not tested here on purpose — require_secrets rejects an empty
     # file and names what is missing.
     "git-token|token for flux bootstrap, for whichever forge FLUX_SOURCE names|git_token_for_source"
-    "s3-access|Hetzner restore access key|pass show homelab/hetztner/k8s-restore-access"
-    "s3-secret|Hetzner restore secret key|pass show homelab/hetztner/k8s-restore-secret"
+    "s3-access|Hetzner restore access key|$AGE_SECRET/s3-access.age"
+    "s3-secret|Hetzner restore secret key|$AGE_SECRET/s3-secret.age"
     # The static seal key. The snapshots are encrypted under it, so it has to
     # be the one the backed-up cluster ran with — a fresh key restores nothing.
-    "seal-key|OpenBao seal key|pass show homelab/openbao/seal-key"
+    "seal-key|OpenBao seal key|$AGE_SECRET/seal-key.age"
     # Read by the node registry auth (see registry_auth_patch): the images an app
     # needs while the cluster is still being built are mirrored to a private
     # ghcr.io namespace, and a node has to be able to pull them before anything
     # in the cluster exists to hold a credential. Demanded up front like the
     # rest, because a rebuild that discovers it missing has already wiped the
     # disks.
-    "ghcr-pull|GHCR pull token for the image mirror|pass show homelab/github/k8s-package-pull"
+    "ghcr-pull|GHCR pull token for the image mirror|$AGE_SECRET/ghcr-pull.age"
 )
 # Asked for by name only, never by a bare require_secrets: a rebuild of the
 # real cluster cannot depend on the OpenBao it is rebuilding. Scoped to the
@@ -281,8 +294,8 @@ HCLOUD_REQUIRED_SECRETS=(
 # during a real recovery or in the in-cluster Job.
 #
 # The only difference between those two is where the material comes from, and it
-# is confined to this function: on a laptop the commands in the list produce it
-# from pass; in the Job the operator streams it in. Everything downstream cannot
+# is confined to this function: on a laptop the commands in the list decrypt
+# it from HOMELAB_SECRETS_DIR; in the Job the operator streams it in. Everything downstream cannot
 # tell the difference — which is the point, because the recovery path must not be
 # a path that only gets exercised during a recovery.
 # require_secrets [name...]
@@ -311,7 +324,7 @@ require_secrets() {
         wanted=("${OPENBAO_REQUIRED_SECRETS[@]}")
     fi
 
-    # Which branch: fetch from pass, or wait for someone to stream files in.
+    # Which branch: decrypt locally, or wait for someone to stream files in.
     #
     # The condition cannot be "is OPENBAO_SECRETS_DIR set", because THIS
     # FUNCTION sets it. Once a subset call had run — hcloudtest asks for
@@ -322,7 +335,7 @@ require_secrets() {
     # maintenance.
     #
     # SECRETS_SELF_FETCH records which world we are in, and it is decided once:
-    # an empty OPENBAO_SECRETS_DIR on entry means pass, an inherited one means
+    # an empty OPENBAO_SECRETS_DIR on entry means local age decryption, an inherited one means
     # the Job. Later calls follow that decision instead of re-deriving it from
     # state this function has since changed.
     if [[ -z "$OPENBAO_SECRETS_DIR" || "${SECRETS_SELF_FETCH:-0}" == "1" ]]; then
@@ -342,12 +355,14 @@ require_secrets() {
             name=${entry%%|*}; what=${entry#*|}; how=${what#*|}; what=${what%%|*}
             # Already collected by an earlier call — a subset call fetches only
             # what it asked for, so the full call that follows must fill the
-            # gaps rather than re-run every command. `pass show` on a locked
-            # store prompts, and prompting twice for the same key is how an
+            # gaps rather than re-run every command. Each decryption waits for
+            # a YubiKey touch, and asking twice for the same key is how an
             # unattended run turns into a hung one.
             [[ -s "$OPENBAO_SECRETS_DIR/$name" ]] && continue
-            if ! eval "$how" > "$OPENBAO_SECRETS_DIR/$name" 2>"$OPENBAO_SECRETS_DIR/.err"; then
-                echo; cat "$OPENBAO_SECRETS_DIR/.err" >&2
+            # stderr stays on the terminal: age-plugin-yubikey asks for the
+            # PIN and the touch there, and a captured prompt is a silent hang.
+            echo
+            if ! eval "$how" > "$OPENBAO_SECRETS_DIR/$name"; then
                 fail "could not obtain the $what with: $how"
             fi
             [[ -s "$OPENBAO_SECRETS_DIR/$name" ]] \
@@ -382,7 +397,7 @@ OPENBAO_NS_HINT=vmtest
 
 # Reads one secret by its name in the list above. In the Job that is the file the
 # operator streamed in; on a workstation it runs the command the list names,
-# which for these is `pass show <path>`.
+# which for these is `age -d` of a file in HOMELAB_SECRETS_DIR.
 #
 # The list is the only place the paths appear. They used to sit here as
 # constants as well, which is how the same value ends up defined twice and one
@@ -1486,7 +1501,8 @@ fi
 
 # Before anything destructive, and after the vmtest block has decided which
 # phases will run. A rebuild that will need these has no business wiping disks
-# first, only to discover an hour later that pass is locked or a token is unset —
+# first, only to discover an hour later that no YubiKey is plugged in or a
+# token is unset —
 # by then the old cluster is gone and the new one cannot be finished.
 #
 # In the Job this is where the file-waiting branch blocks, which is the right
@@ -1994,7 +2010,8 @@ bao_exec() {
 # have been doing this all along.
 #
 # Reading them needs three things, all of which live OUTSIDE the cluster, which
-# is the point: bucket credentials from pass (their own key, not Velero's),
+# is the point: bucket credentials from HOMELAB_SECRETS_DIR (their own key, not
+# Velero's),
 # kopia, and the repository password. The last is not a secret — Velero
 # hardcodes `static-passw0rd` in its source. Worth knowing for what it means:
 # the offsite backup's confidentiality rests on bucket access alone.
@@ -2071,7 +2088,8 @@ restore_openbao() {
     # rather than an unseal key and the server unseals itself; the only thing
     # used from its output is the root token, for the next few seconds, and
     # then it stops existing: restoring replaces the keyring with the
-    # snapshot's, after which the root token and recovery key from pass apply.
+    # snapshot's, after which the operator's root token
+    # (~/.config/openbao/root-token.age) and recovery key apply.
     #
     # No -force. Without it OpenBao checks that the snapshot was written under
     # the seal key this server runs with, and refuses a mismatch by name
@@ -2355,7 +2373,7 @@ if phase_runs flux; then
     # Secret would leave every replica sealed for good at its next restart.
     # That is a migration, not a bootstrap step.
     #
-    # --from-literal, not --from-file: `pass show` ends its output with a
+    # --from-literal, not --from-file: the stored value ends with a
     # newline, the substitution strips it, and OpenBao rejects a key that is
     # not exactly 32 bytes of base64. The key sits in this process's argument
     # list for the length of the call, as it did in the pod's before.

kubernetes/infrastructure/crowdsec/helmrelease.yaml

@@ -80,6 +80,25 @@ spec:
           # community blocklists.
           - 162.55.50.97/32
           - 2a01:4f8:c2c:4579::1/128
+      postoverflows:
+        s01-whitelist:
+          # http-crawl-non_statics counts requests regardless of outcome, so a
+          # signed-in user of any image-heavy UI (Immich timeline, Music
+          # Assistant covers) trips it with a burst of 200s. Scanners trip it
+          # with 404s for paths that do not exist. Drop the overflow unless at
+          # least half of its requests were client errors (the alert carries
+          # the scenario's last 5 events, cache_size: 5). Only this scenario
+          # is affected; probing/sensitive-files/cve still fire on their own.
+          crawl-non-statics-success-whitelist.yaml: |
+            name: krenner/crawl-non-statics-success-whitelist
+            description: "Ignore http-crawl-non_statics bursts that mostly succeeded"
+            whitelist:
+              reason: "crawl burst of successful requests is a UI, not a scanner"
+              expression:
+                - >-
+                  evt.Overflow.Alert.GetScenario() == 'crowdsecurity/http-crawl-non_statics' &&
+                  count(evt.Overflow.Alert.Events, {.GetMeta('http_status') startsWith '4'}) * 2
+                  < len(evt.Overflow.Alert.Events)
 
     # mTLS between agents and the LAPI. Do not switch back to password auth:
     # the LAPI runs without persistence, so its machine-registration DB is

opentofu/providers.tf

@@ -43,12 +43,6 @@ provider "proxmox" {
   endpoint  = var.proxmox_api_url
   api_token = "${var.proxmox_user}!${var.proxmox_token_id}=${var.proxmox_token_secret}"
   insecure  = var.proxmox_tls_insecure
-
-  ssh {
-    agent       = false
-    username    = "root"
-    private_key = var.ssh_private_key
-  }
 }
 
 provider "unifi" {

opentofu/variables.tf

@@ -28,16 +28,4 @@ variable "proxmox_tls_insecure" {
   default     = false
 }
 
-variable "ssh_public_key" {
-  description = "Public SSH key to inject into containers"
-  type        = string
-  default     = null
-}
-
-variable "ssh_private_key" {
-  description = "SSH private key for Proxmox provider to import disk images"
-  type        = string
-  sensitive   = true
-  default     = ""
-}