feat(bootstrap): DR secrets from YubiKey/paper-key age files, not pass

The cold-rebuild secrets (seal key, Hetzner restore keys, GHCR pull
token) came from the pass store: one gpg key file on the owner's laptop,
readable by any process of that user while gpg-agent caches the
passphrase, and with no second copy.

bootstrap.sh now decrypts ~/.config/homelab-secrets/<name>.age, files
encrypted to both YubiKeys (touch) and an offline paper key; a copy is
on nas-2. HOMELAB_AGE_IDENTITY switches to the paper key when both
sticks are gone. The in-cluster Job path is unchanged: the printed
streaming command is the age one. Runbook and the two OpenBao alert
texts point at the new files.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

.woodpecker/deploy-ssh-ca.yml

@@ -6,6 +6,9 @@ when:
       - "ansible/roles/ssh-ca/**"
       - "ansible/plays/configure-ssh-ca.yml"
       - "ansible/inventory/pve_hosts.yml"
+      - "ansible/inventory/group_vars/all/**"
+      - "ansible/inventory/group_vars/netbird_proxy/vars.yml"
+      - "ansible/inventory/host_vars/netbird.renner.uno.yml"
       - ".woodpecker/deploy-ssh-ca.yml"
   - event: deployment
     evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "deploy-ssh-ca"'

ansible/inventory/group_vars/all/vars.yml

@@ -5,16 +5,26 @@ pve_token_id: "ansible"
 ansible_user: "ladmin"
 ansible_python_interpreter: /usr/bin/python3
 
-# SSH Public Keys for Cloud-Init
-# ansible-runner@ci-new removed (issue #184): dead key, no such host exists.
-common_ssh_public_keys: |
-  ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPzFNBP3bq2m/6JqDqod9w+FP/uxXxHMolqhq3Ojflr1 krenner@fortress
-  ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICyNnX1I4W9jyDvCJ1DJauHSmbo1ZXSyBxVhSAwMWgen woodpecker-ci@homelab
+# The only plain keys the admin logins accept on [ssh_keys_managed] hosts
+# (ssh-ca role, tasks/authorized_keys.yml). People log in with OpenBao SSH CA
+# certificates; these two are for when that path is unavailable.
+#   breakglass-neo: PIV key (slot 9a, PIN) on the offline YubiKey NEO.
+#     ssh -I /usr/lib/opensc-pkcs11.so root@<host>
+#   woodpecker-ci: the CI deploy key, from ssh_ci_key_from where set.
+ssh_admin_authorized_keys:
+  - key: "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBKvSl9LUP8u3S2Sq5YTWpW0Jxbue8cJhD7/LMFQbEsa3xo145HVR7pBzRxW71MUy+jgrgmpSlDhIb606Hswhvoo= breakglass-neo-6532348"
+  - key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICyNnX1I4W9jyDvCJ1DJauHSmbo1ZXSyBxVhSAwMWgen woodpecker-ci@homelab"
+    options: "{{ ('from=\"' ~ ssh_ci_key_from ~ '\"') if ssh_ci_key_from else '' }}"
+
+# Source the CI key is accepted from. LAN hosts see the pod address; hosts on
+# the internet see the home WAN address (NAT, dynamic) and set this empty in
+# their group/host vars.
+ssh_ci_key_from: "{{ ci_ssh_from_cidrs }}"
 
 # Network-of-origin restriction for the woodpecker-ci@homelab CI key
-# (issue #184). Consumed by plays/configure-ssh-keys.yml as the authorized_key
-# `from=` option so the CI key only authenticates from within the cluster's
-# pod network — NOT from any host-pinned IP.
+# (issue #184). Used as the authorized_keys `from=` option so the CI key only
+# authenticates from within the cluster's pod network — NOT from any
+# host-pinned IP.
 #
 # The in-cluster Woodpecker step pods SSH to fleet hosts using their POD IP as
 # source, not a node IP: Cilium runs `ipv4-native-routing-cidr=10.0.0.0/8` +

ansible/inventory/group_vars/netbird_proxy/vars.yml

@@ -0,0 +1,4 @@
+---
+# CI reaches these internet hosts NATed from the home WAN address, never from
+# the pod network, so the CI key carries no from= restriction here.
+ssh_ci_key_from: ""

ansible/inventory/host_vars/netbird.renner.uno.yml

@@ -2,3 +2,7 @@
 # NetBird server - VPS not managed by Proxmox
 # Enable Docker monitoring (cAdvisor) for container metrics
 docker_monitoring_enabled: true
+
+# CI reaches this internet host NATed from the home WAN address, never from
+# the pod network, so the CI key carries no from= restriction here.
+ssh_ci_key_from: ""

ansible/inventory/pve_hosts.yml

@@ -77,3 +77,13 @@ claude-ws ansible_host=10.3.40.78 ansible_user=root
 proxy-1.renner.uno
 proxy-2.renner.uno
 netbird.renner.uno
+edge-1.iot.renner.uno
+
+# Hosts whose admin logins accept only ssh_admin_authorized_keys (break-glass
+# + CI); everything else needs a CA certificate. Members must be in [ssh_ca].
+# Added one host at a time after the [ssh_ca] report was reviewed.
+[ssh_keys_managed]
+proxy-1.renner.uno
+proxy-2.renner.uno
+netbird.renner.uno
+edge-1.iot.renner.uno

ansible/plays/configure-ssh-ca.yml

@@ -2,7 +2,8 @@
 # Trust the OpenBao SSH user CA (ssh-client-signer) on the hosts in [ssh_ca]:
 # owner certificates (principal k.renner) for root/ladmin, Claude's 1h
 # certificates (principal claude-bot) for its own claude-bot account with
-# limited sudo. Existing authorized_keys stay untouched.
+# limited sudo. The first play only reports the admin logins' authorized_keys;
+# the second reduces them to ssh_admin_authorized_keys on [ssh_keys_managed].
 #
 # serial: 1 so a host whose sshd rejects the change stops the run before the
 # next host is touched.
@@ -13,3 +14,14 @@
   serial: 1
   roles:
     - ssh-ca
+
+- name: Reduce authorized_keys to break-glass and CI
+  hosts: ssh_keys_managed
+  become: true
+  gather_facts: false
+  serial: 1
+  tasks:
+    - name: Set the managed authorized_keys
+      ansible.builtin.include_role:
+        name: ssh-ca
+        tasks_from: authorized_keys

ansible/plays/configure-ssh-keys.yml

@@ -1,48 +0,0 @@
----
-- name: Deploy authorized SSH keys to all hosts
-  hosts: all:!localhost
-  become: true
-  gather_facts: false
-  # Authorized keys deployed to root and ladmin on every managed host.
-  #
-  # - krenner@fortress (human): unrestricted.
-  # - woodpecker-ci@homelab (CI): restricted with from= to the cluster's pod
-  #   network (ci_ssh_from_cidrs in group_vars/all/vars.yml). The in-cluster
-  #   Woodpecker step pods SSH to fleet hosts presenting their POD IP as source
-  #   (Cilium native-routing-cidr covers the LAN, so no SNAT; PodCIDRs are
-  #   BGP-advertised). This is a network-of-origin restriction, NOT host-pinned,
-  #   so it survives node DHCP churn and pod rescheduling. See vars.yml for the
-  #   full derivation.
-  # - ansible-runner@ci-new: removed — dead key, no such host exists.
-  vars:
-    authorized_keys:
-      - key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPzFNBP3bq2m/6JqDqod9w+FP/uxXxHMolqhq3Ojflr1 krenner@fortress"
-      - key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICyNnX1I4W9jyDvCJ1DJauHSmbo1ZXSyBxVhSAwMWgen woodpecker-ci@homelab"
-        options: 'from="{{ ci_ssh_from_cidrs }}"'
-  tasks:
-    - name: Check SSH connectivity
-      ansible.builtin.wait_for_connection:
-        timeout: 5
-      ignore_errors: true
-      register: ssh_check
-
-    - name: Skip unreachable hosts
-      ansible.builtin.meta: end_host
-      when: ssh_check is failed
-
-    - name: Deploy authorized keys to root
-      ansible.builtin.authorized_key:
-        user: root
-        key: "{{ item.key }}"
-        key_options: "{{ item.options | default(omit) }}"
-        state: present
-      loop: "{{ authorized_keys }}"
-
-    - name: Deploy authorized keys to ladmin
-      ansible.builtin.authorized_key:
-        user: ladmin
-        key: "{{ item.key }}"
-        key_options: "{{ item.options | default(omit) }}"
-        state: present
-      loop: "{{ authorized_keys }}"
-      ignore_errors: true

ansible/roles/common/tasks/main.yml

@@ -51,10 +51,8 @@
 # -------------------------------------------------------
 # 4. SSH Configuration
 # -------------------------------------------------------
-# Authorized keys (human + woodpecker-ci, the latter from=-restricted) are
-# managed by the dedicated ansible/plays/configure-ssh-keys.yml play. The old
-# "Add Ansible Runner SSH Key" task deployed ansible-runner@ci-new, a dead key
-# for a host that no longer exists — removed (issue #184).
+# SSH access (CA trust, admin authorized_keys) is managed by the ssh-ca role
+# via plays/configure-ssh-ca.yml.
 
 # -------------------------------------------------------
 # 5. Disable NTP on VMs/LXC (use host time)

ansible/roles/ssh-ca/tasks/authorized_keys.yml

@@ -0,0 +1,23 @@
+---
+# Replaces the admin logins' authorized_keys with ssh_admin_authorized_keys
+# (break-glass + CI). Everything else logs in with a CA certificate.
+- name: Look up the admin logins
+  ansible.builtin.getent:
+    database: passwd
+    key: "{{ item }}"
+    fail_key: false
+  loop: "{{ ssh_ca_owner_users }}"
+  register: ssh_ca_admin_lookup
+
+- name: Set the admin logins' authorized_keys to the managed list
+  ansible.posix.authorized_key:
+    user: "{{ item.item }}"
+    key: |
+      {% for k in ssh_admin_authorized_keys %}
+      {{ (k.options ~ ' ') if (k.options | default('')) else '' }}{{ k.key }}
+      {% endfor %}
+    exclusive: true
+  loop: "{{ ssh_ca_admin_lookup.results }}"
+  loop_control:
+    label: "{{ item.item }}"
+  when: (item.ansible_facts.getent_passwd | default({})).get(item.item) is not none

ansible/roles/ssh-ca/tasks/main.yml

@@ -148,3 +148,6 @@
       ansible.builtin.systemd:
         name: "{{ ssh_ca_sshd_units | first }}"
         state: reloaded
+
+- name: Report the admin logins' authorized keys
+  ansible.builtin.include_tasks: report.yml

ansible/roles/ssh-ca/tasks/report.yml

@@ -0,0 +1,26 @@
+---
+# Read-only: what the admin logins accept besides certificates. Hosts in
+# [ssh_keys_managed] have this reduced to ssh_admin_authorized_keys.
+- name: Find the admin logins' authorized_keys
+  ansible.builtin.stat:
+    path: "{{ item.ansible_facts.getent_passwd[item.item][4] }}/.ssh/authorized_keys"
+  loop: "{{ ssh_ca_owner_lookup.results }}"
+  loop_control:
+    label: "{{ item.item }}"
+  when: (item.ansible_facts.getent_passwd | default({})).get(item.item) is not none
+  register: ssh_ca_keyfiles
+
+- name: Fingerprint the admin logins' authorized keys
+  ansible.builtin.command: ssh-keygen -lf {{ item.stat.path }}
+  loop: "{{ ssh_ca_keyfiles.results | selectattr('stat', 'defined') | selectattr('stat.exists') | list }}"
+  loop_control:
+    label: "{{ item.stat.path }}"
+  changed_when: false
+  register: ssh_ca_fingerprints
+
+- name: Report the admin logins' authorized keys
+  ansible.builtin.debug:
+    msg: "{{ item.item.stat.path }}: {{ item.stdout_lines }}"
+  loop: "{{ ssh_ca_fingerprints.results }}"
+  loop_control:
+    label: "{{ item.item.stat.path }}"

kubernetes/RUNBOOK-cold-rebuild.md

@@ -67,12 +67,16 @@ These must exist outside the cluster or the rebuild cannot complete:
       `flux` phase creates the `sops-age` Secret, and any not-yet-migrated sops
       files decrypt with it. (Post-OpenBao-phase-4 this may be retired — see
       `MIGRATION-sops-openbao.md`.)
-- [ ] **OpenBao seal key, recovery key + root token** — in the GPG `pass`
-      store (`homelab/openbao/seal-key`, `homelab/openbao/unseal-key` — the
-      former shamir key, now the recovery key — and `homelab/openbao/root-token`),
-      and the GPG private key backed up off-machine. The seal key is the one
-      the snapshots are encrypted under: without it, no OpenBao snapshot can be
-      restored and no secret-consuming app can start.
+- [ ] **OpenBao seal key, recovery key, restore + pull credentials** — age
+      files in `~/.config/homelab-secrets/` (`seal-key.age`,
+      `openbao-recovery-key.age` — the former shamir key —, `s3-access.age`,
+      `s3-secret.age`, `ghcr-pull.age`), encrypted to both YubiKeys and the
+      offline paper key in the safe; a copy is on nas-2 in
+      `~ladmin/homelab-secrets-backup` (`~/.ssh/ssh-keys-rotate dr-copy`). The
+      root token is `~/.config/openbao/root-token.age` (YubiKeys only). Needed
+      to decrypt: one YubiKey with its PIV PIN, or the paper key. The seal key
+      is the one the snapshots are encrypted under: without it, no OpenBao
+      snapshot can be restored and no secret-consuming app can start.
 - [ ] **OpenBao raft snapshot** reachable — nightly PVC snapshots
       (`infrastructure/openbao/snapshot-cronjob.yaml`) plus the Velero
       `openbao-daily` offsite copy. In a total-loss rebuild the offsite Velero
@@ -306,7 +310,9 @@ static seal. What follows is the same procedure by hand, for when the script
 is not an option.
 
 OpenBao unseals itself from the static seal (`openbao-seal-key` Secret, key
-`key`, the 32-byte base64 key from `pass show homelab/openbao/seal-key`). The
+`key`, the 32-byte base64 key in `~/.config/homelab-secrets/seal-key.age`,
+age-encrypted to the owner's YubiKeys and the offline paper key; a copy sits on
+nas-2 in `~ladmin/homelab-secrets-backup`). The
 snapshots are encrypted under that key, so it must be the same key the backed
 up cluster used — a different one restores nothing. Until the snapshot is
 restored, the `ClusterSecretStore` stays NotReady, ESO cannot materialize
@@ -316,7 +322,8 @@ Secrets, and every secret-consuming app stays Pending.
 # 1. The seal key, before the pods start (a pod without it never starts).
 kubectl create namespace openbao --dry-run=client -o yaml | kubectl apply -f -
 kubectl -n openbao create secret generic openbao-seal-key \
-  --from-literal=key="$(pass show homelab/openbao/seal-key)"
+  --from-literal=key="$(age -d -i ~/.config/age/yubikeys.txt ~/.config/homelab-secrets/seal-key.age)"
+# Both YubiKeys lost: -i <file holding the paper key> instead.
 
 # 2. Wait for openbao-0 (uninitialised: 0/1 Ready is expected), then
 #    initialise it. With an auto-seal, init hands out recovery keys and the
@@ -337,7 +344,8 @@ kubectl -n openbao exec -it openbao-0 -- \
 
 # 4. The server seals against the snapshot's keyring and unseals itself from
 #    the static seal; openbao-1 and openbao-2 follow (OrderedReady). Nothing
-#    to type. Afterwards the tokens and recovery key from pass apply again.
+#    to type. Afterwards the operator's root token
+#    (~/.config/openbao/root-token.age) and recovery key apply again.
 kubectl -n openbao get pods -w
 ```
 
@@ -352,7 +360,7 @@ kubectl -n <ns> get externalsecret                # want: SecretSynced
 > A replica that stays **sealed** after a restart could not use the static
 > seal — the Secret is missing or does not match the key the data was
 > written under. Fix the Secret and restart the pod; `bao operator unseal`
-> with the recovery key (`pass show homelab/openbao/unseal-key`) does not
+> with the recovery key (`~/.config/homelab-secrets/openbao-recovery-key.age`) does not
 > unseal an auto-sealed server and is only for recovery-mode operations.
 > Details + rationale: `MIGRATION-sops-openbao.md`.
 

kubernetes/apps/gitea/kustomization.yaml

@@ -16,6 +16,8 @@ resources:
   - networkpolicy.yaml
   # Nightly prune of old CI image builds (REN-1203).
   - package-retention.yaml
+  # OpenBao SSH CA public key for certificate logins.
+  - ssh-ca.yaml
   - kumamonitors.yaml
 configMapGenerator:
   - name: gitea-package-retention

kubernetes/apps/gitea/ssh-ca.yaml

@@ -0,0 +1,37 @@
+# Public key of the OpenBao SSH user CA (ssh-client-signer), for Gitea's
+# built-in SSH server to accept owner and claude-bot certificates. OpenBao
+# serves it unauthenticated; ESO reads it from the in-cluster Service, so the
+# key has one source. Gitea reads it from env at start; Reloader restarts the
+# StatefulSet when the Secret changes.
+apiVersion: external-secrets.io/v1
+kind: SecretStore
+metadata:
+  name: openbao-ssh-ca
+  namespace: gitea
+spec:
+  provider:
+    webhook:
+      url: http://openbao.openbao.svc.cluster.local:8200/v1/ssh-client-signer/public_key
+      method: GET
+---
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: gitea-ssh-ca
+  namespace: gitea
+spec:
+  refreshInterval: 1h
+  secretStoreRef:
+    kind: SecretStore
+    name: openbao-ssh-ca
+  target:
+    name: gitea-ssh-ca
+    # OpenBao ends the body with a newline; it must not reach app.ini.
+    template:
+      engineVersion: v2
+      data:
+        trusted-user-ca-keys: "{{ .ca | trim }}"
+  data:
+    - secretKey: ca
+      remoteRef:
+        key: public_key

kubernetes/apps/gitea/statefulset.yaml

@@ -5,6 +5,9 @@ metadata:
   namespace: gitea
   labels:
     app.kubernetes.io/name: gitea
+  annotations:
+    # A CA rotation changes gitea-ssh-ca; Gitea reads it only at start.
+    secret.reloader.stakater.com/reload: gitea-ssh-ca
 spec:
   serviceName: gitea
   replicas: 1 # SQLite => single writer. Do NOT scale up without moving to Postgres first.
@@ -85,6 +88,16 @@ spec:
             # HAS CHANGED". Paths are relative to APP_DATA_PATH (/var/lib/gitea/data).
             - name: GITEA__server__SSH_SERVER_HOST_KEYS
               value: "ssh/ssh_host_ed25519_key,ssh/ssh_host_rsa_key"
+            # OpenBao SSH CA (ssh-ca.yaml): certificates log in as the user whose
+            # name is a principal of the cert. Each user registers that
+            # principal once (Settings > SSH / GPG Keys > Manage principals).
+            - name: GITEA__server__SSH_TRUSTED_USER_CA_KEYS
+              valueFrom:
+                secretKeyRef:
+                  name: gitea-ssh-ca
+                  key: trusted-user-ca-keys
+            - name: GITEA__server__SSH_AUTHORIZED_PRINCIPALS_ALLOW
+              value: username
             # Gitea Actions was decommissioned — keep it off.
             - name: GITEA__actions__ENABLED
               value: "false"

kubernetes/apps/monitoring-rules/prometheusrule-openbao.yaml

@@ -49,7 +49,7 @@ spec:
               failure would cost quorum. Check `kubectl -n openbao get pods`,
               the pod log and `bao status` in the not-ready pod. If it is
               sealed, the static seal failed: verify the openbao-seal-key
-              Secret matches `pass show homelab/openbao/seal-key`.
+              Secret matches ~/.config/homelab-secrets/seal-key.age (age, YubiKey).
 
         - alert: OpenBaoQuorumLost
           # Raft quorum of a three-node cluster is two. At one ready replica
@@ -76,7 +76,7 @@ spec:
               surface only when something rotates or a new secret is needed.
               Replicas unseal themselves from the static seal; one that stays
               sealed could not. Check its log and the openbao-seal-key Secret
-              against `pass show homelab/openbao/seal-key`, fix the Secret and
+              against ~/.config/homelab-secrets/seal-key.age, fix the Secret and
               restart the pod. The recovery key cannot unseal an auto-sealed
               server. Procedure: kubernetes/RUNBOOK-cold-rebuild.md, Step 4.
 

kubernetes/apps/wiki/deployment.yaml

@@ -53,7 +53,7 @@ spec:
           type: RuntimeDefault
       containers:
         - name: wiki
-          image: gitea.renner.uno/konairius/wiki:build.1793086240 # {"$imagepolicy": "flux-system:wiki"}
+          image: gitea.renner.uno/konairius/wiki:build.1793094262 # {"$imagepolicy": "flux-system:wiki"}
           securityContext:
             allowPrivilegeEscalation: false
             readOnlyRootFilesystem: true

kubernetes/bootstrap.sh

@@ -190,13 +190,16 @@ if [[ "$FLUX_SOURCE" == "restore-test" && "$PHASE" != "flux" && "$PHASE" != "wai
 fi
 OPENBAO_NS="openbao"
 
-# Where the secrets come from. On a workstation, the operator's pass store. In
-# the in-cluster Job there is no pass and no GPG key — and there should not be:
-# the pass store opens everything, and it stays on the operator's machine. Set
-# OPENBAO_SECRETS_DIR and the run reads them from files instead, which the
+# Where the secrets come from. On a workstation, age files in
+# HOMELAB_SECRETS_DIR, encrypted to the operator's YubiKeys and an offline paper
+# key (a copy sits on nas-2). HOMELAB_AGE_IDENTITY picks what decrypts them: the
+# YubiKey identity stubs by default (touch per file), the paper key's file when
+# both sticks are gone. In the in-cluster Job there is neither — and there
+# should not be: those keys open everything, and they stay with the operator.
+# Set OPENBAO_SECRETS_DIR and the run reads them from files instead, which the
 # operator streams in when the run asks for them:
 #
-#     pass show homelab/openbao/seal-key \
+#     age -d -i ~/.config/age/yubikeys.txt ~/.config/homelab-secrets/seal-key.age \
 #       | kubectl -n vmtest exec -i <pod> -- sh -c 'cat > /secrets/seal-key'
 #
 # They exist in the pod's filesystem for the length of the run and are never
@@ -240,6 +243,10 @@ git_token_for_source() {
     fi
 }
 
+HOMELAB_SECRETS_DIR="${HOMELAB_SECRETS_DIR:-$HOME/.config/homelab-secrets}"
+HOMELAB_AGE_IDENTITY="${HOMELAB_AGE_IDENTITY:-$HOME/.config/age/yubikeys.txt}"
+AGE_SECRET="age -d -i $HOMELAB_AGE_IDENTITY $HOMELAB_SECRETS_DIR"
+
 # Format: <file in SECRETS_DIR>|<what it is>|<how the operator produces it>
 OPENBAO_REQUIRED_SECRETS=(
     "sops-age.key|SOPS age key|cat ~/.config/sops/age/keys.txt"
@@ -255,18 +262,18 @@ OPENBAO_REQUIRED_SECRETS=(
     # Emptiness is not tested here on purpose — require_secrets rejects an empty
     # file and names what is missing.
     "git-token|token for flux bootstrap, for whichever forge FLUX_SOURCE names|git_token_for_source"
-    "s3-access|Hetzner restore access key|pass show homelab/hetztner/k8s-restore-access"
-    "s3-secret|Hetzner restore secret key|pass show homelab/hetztner/k8s-restore-secret"
+    "s3-access|Hetzner restore access key|$AGE_SECRET/s3-access.age"
+    "s3-secret|Hetzner restore secret key|$AGE_SECRET/s3-secret.age"
     # The static seal key. The snapshots are encrypted under it, so it has to
     # be the one the backed-up cluster ran with — a fresh key restores nothing.
-    "seal-key|OpenBao seal key|pass show homelab/openbao/seal-key"
+    "seal-key|OpenBao seal key|$AGE_SECRET/seal-key.age"
     # Read by the node registry auth (see registry_auth_patch): the images an app
     # needs while the cluster is still being built are mirrored to a private
     # ghcr.io namespace, and a node has to be able to pull them before anything
     # in the cluster exists to hold a credential. Demanded up front like the
     # rest, because a rebuild that discovers it missing has already wiped the
     # disks.
-    "ghcr-pull|GHCR pull token for the image mirror|pass show homelab/github/k8s-package-pull"
+    "ghcr-pull|GHCR pull token for the image mirror|$AGE_SECRET/ghcr-pull.age"
 )
 # Asked for by name only, never by a bare require_secrets: a rebuild of the
 # real cluster cannot depend on the OpenBao it is rebuilding. Scoped to the
@@ -281,8 +288,8 @@ HCLOUD_REQUIRED_SECRETS=(
 # during a real recovery or in the in-cluster Job.
 #
 # The only difference between those two is where the material comes from, and it
-# is confined to this function: on a laptop the commands in the list produce it
-# from pass; in the Job the operator streams it in. Everything downstream cannot
+# is confined to this function: on a laptop the commands in the list decrypt
+# it from HOMELAB_SECRETS_DIR; in the Job the operator streams it in. Everything downstream cannot
 # tell the difference — which is the point, because the recovery path must not be
 # a path that only gets exercised during a recovery.
 # require_secrets [name...]
@@ -311,7 +318,7 @@ require_secrets() {
         wanted=("${OPENBAO_REQUIRED_SECRETS[@]}")
     fi
 
-    # Which branch: fetch from pass, or wait for someone to stream files in.
+    # Which branch: decrypt locally, or wait for someone to stream files in.
     #
     # The condition cannot be "is OPENBAO_SECRETS_DIR set", because THIS
     # FUNCTION sets it. Once a subset call had run — hcloudtest asks for
@@ -342,8 +349,8 @@ require_secrets() {
             name=${entry%%|*}; what=${entry#*|}; how=${what#*|}; what=${what%%|*}
             # Already collected by an earlier call — a subset call fetches only
             # what it asked for, so the full call that follows must fill the
-            # gaps rather than re-run every command. `pass show` on a locked
-            # store prompts, and prompting twice for the same key is how an
+            # gaps rather than re-run every command. Each decryption waits for
+            # a YubiKey touch, and asking twice for the same key is how an
             # unattended run turns into a hung one.
             [[ -s "$OPENBAO_SECRETS_DIR/$name" ]] && continue
             if ! eval "$how" > "$OPENBAO_SECRETS_DIR/$name" 2>"$OPENBAO_SECRETS_DIR/.err"; then
@@ -382,7 +389,7 @@ OPENBAO_NS_HINT=vmtest
 
 # Reads one secret by its name in the list above. In the Job that is the file the
 # operator streamed in; on a workstation it runs the command the list names,
-# which for these is `pass show <path>`.
+# which for these is `age -d` of a file in HOMELAB_SECRETS_DIR.
 #
 # The list is the only place the paths appear. They used to sit here as
 # constants as well, which is how the same value ends up defined twice and one
@@ -1486,7 +1493,8 @@ fi
 
 # Before anything destructive, and after the vmtest block has decided which
 # phases will run. A rebuild that will need these has no business wiping disks
-# first, only to discover an hour later that pass is locked or a token is unset —
+# first, only to discover an hour later that no YubiKey is plugged in or a
+# token is unset —
 # by then the old cluster is gone and the new one cannot be finished.
 #
 # In the Job this is where the file-waiting branch blocks, which is the right
@@ -1994,7 +2002,8 @@ bao_exec() {
 # have been doing this all along.
 #
 # Reading them needs three things, all of which live OUTSIDE the cluster, which
-# is the point: bucket credentials from pass (their own key, not Velero's),
+# is the point: bucket credentials from HOMELAB_SECRETS_DIR (their own key, not
+# Velero's),
 # kopia, and the repository password. The last is not a secret — Velero
 # hardcodes `static-passw0rd` in its source. Worth knowing for what it means:
 # the offsite backup's confidentiality rests on bucket access alone.
@@ -2071,7 +2080,8 @@ restore_openbao() {
     # rather than an unseal key and the server unseals itself; the only thing
     # used from its output is the root token, for the next few seconds, and
     # then it stops existing: restoring replaces the keyring with the
-    # snapshot's, after which the root token and recovery key from pass apply.
+    # snapshot's, after which the operator's root token
+    # (~/.config/openbao/root-token.age) and recovery key apply.
     #
     # No -force. Without it OpenBao checks that the snapshot was written under
     # the seal key this server runs with, and refuses a mismatch by name
@@ -2355,7 +2365,7 @@ if phase_runs flux; then
     # Secret would leave every replica sealed for good at its next restart.
     # That is a migration, not a bootstrap step.
     #
-    # --from-literal, not --from-file: `pass show` ends its output with a
+    # --from-literal, not --from-file: the stored value ends with a
     # newline, the substitution strips it, and OpenBao rejects a key that is
     # not exactly 32 bytes of base64. The key sits in this process's argument
     # list for the length of the call, as it did in the pod's before.

kubernetes/infrastructure/crowdsec/helmrelease.yaml

@@ -80,6 +80,25 @@ spec:
           # community blocklists.
           - 162.55.50.97/32
           - 2a01:4f8:c2c:4579::1/128
+      postoverflows:
+        s01-whitelist:
+          # http-crawl-non_statics counts requests regardless of outcome, so a
+          # signed-in user of any image-heavy UI (Immich timeline, Music
+          # Assistant covers) trips it with a burst of 200s. Scanners trip it
+          # with 404s for paths that do not exist. Drop the overflow unless at
+          # least half of its requests were client errors (the alert carries
+          # the scenario's last 5 events, cache_size: 5). Only this scenario
+          # is affected; probing/sensitive-files/cve still fire on their own.
+          crawl-non-statics-success-whitelist.yaml: |
+            name: krenner/crawl-non-statics-success-whitelist
+            description: "Ignore http-crawl-non_statics bursts that mostly succeeded"
+            whitelist:
+              reason: "crawl burst of successful requests is a UI, not a scanner"
+              expression:
+                - >-
+                  evt.Overflow.Alert.GetScenario() == 'crowdsecurity/http-crawl-non_statics' &&
+                  count(evt.Overflow.Alert.Events, {.GetMeta('http_status') startsWith '4'}) * 2
+                  < len(evt.Overflow.Alert.Events)
 
     # mTLS between agents and the LAPI. Do not switch back to password auth:
     # the LAPI runs without persistence, so its machine-registration DB is

opentofu/locals.tf

@@ -1,9 +0,0 @@
-locals {
-  common_ssh_keys = join("\n", [
-    "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPzFNBP3bq2m/6JqDqod9w+FP/uxXxHMolqhq3Ojflr1 krenner@fortress",
-    "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGfe3HJqFR+jIGrboweseAO7XjIVE24sje/b1HybgU9C ansible-runner@ci-new",
-    "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICyNnX1I4W9jyDvCJ1DJauHSmbo1ZXSyBxVhSAwMWgen woodpecker-ci@homelab",
-    "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBB1OFjh6Q5xeioI6Cwfuv8Xka27veomYy9tKBbpMo8mcrsB4yIN4FX8HovBLtogetBhwBUefBar8nhA9GbRh6HQ= krenner@krenner-XPS-13-9310"
-  ])
-
-}