fix(ansible): ssh-ca reloads the sshd unit that is running

The netbird run (pipeline 14795) failed at the reload: 'Could not find
the requested service sshd.service'. Ubuntu's service_facts list
sshd.service as a not-found alias next to the real ssh.service
(running; disabled because ssh.socket activates it), and the role only
checked that the name exists.

The unit is now the one of sshd.service/ssh.service whose state is
running, asserted to be exactly one.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

.woodpecker/ci-scripts-test.yml

@@ -35,25 +35,25 @@ when:
 steps:
   - name: model-selection-tests
     depends_on: []
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.310
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.311
     commands:
       - sh scripts/ci/test-model-selection.sh
 
   - name: pr-review-mistral-tests
     depends_on: []
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.310
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.311
     commands:
       - sh scripts/ci/test-pr-review-mistral.sh
 
   - name: promtool-rules-guard-tests
     depends_on: []
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.310
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.311
     commands:
       - sh scripts/ci/test-promtool-rules.sh
 
   - name: yaml-citations-guard-tests
     depends_on: []
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.310
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.311
     commands:
       - sh scripts/ci/test-yaml-citations.sh
 

.woodpecker/deploy-ssh-ca.yml

@@ -0,0 +1,34 @@
+when:
+  # Deploy only on main (merge) + manual — never auto-deploy from a branch.
+  - event: push
+    branch: main
+    path:
+      - "ansible/roles/ssh-ca/**"
+      - "ansible/plays/configure-ssh-ca.yml"
+      - "ansible/inventory/pve_hosts.yml"
+      - ".woodpecker/deploy-ssh-ca.yml"
+  - event: deployment
+    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "deploy-ssh-ca"'
+
+timeout: 30
+
+steps:
+  - name: deploy-ssh-ca
+    image: gitea.renner.uno/gitea_admin/ci-ansible:2026-04-29
+    environment:
+      ANSIBLE_FORCE_COLOR: "true"
+      ANSIBLE_HOST_KEY_CHECKING: "false"
+      ANSIBLE_ROLES_PATH: "../roles:./roles"
+      SSH_PRIVATE_KEY:
+        from_secret: ssh_private_key
+      VAULT_PASS:
+        from_secret: vault_pass
+    commands:
+      - mkdir -p ~/.ssh
+      - echo "$SSH_PRIVATE_KEY" | base64 -d > ~/.ssh/id_ed25519
+      - chmod 600 ~/.ssh/id_ed25519
+      - printf "Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile /dev/null\n" > ~/.ssh/config
+      - cd ansible
+      - echo "$VAULT_PASS" > .vault_pass && chmod 600 .vault_pass
+      - export ANSIBLE_VAULT_PASSWORD_FILE=.vault_pass
+      - ansible-playbook -i inventory/ plays/configure-ssh-ca.yml -v

.woodpecker/e2e.yml

@@ -26,7 +26,7 @@ steps:
     # fetch-models.sh response mapping. No cluster, no secrets, no network — the
     # ci-claude image just provides sh + jq. Keeps the flag-injection regression
     # test from rotting (it exercises the exact injection shapes).
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.310
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.311
     commands:
       - sh scripts/ci/test-model-selection.sh
 

.woodpecker/security-scan.yml

@@ -70,7 +70,7 @@ steps:
     # this step is skipped — otherwise we'd fork-bomb PRs.
     when:
       - branch: main
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.310
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.311
     environment:
       CLAUDE_CODE_OAUTH_TOKEN:
         from_secret: claude_code_oauth_token

.woodpecker/update-audit.yml

@@ -30,7 +30,7 @@ when:
 
 steps:
   - name: update-audit-agent
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.310
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.311
     environment:
       CLAUDE_CODE_OAUTH_TOKEN:
         from_secret: claude_code_oauth_token

ansible/inventory/pve_hosts.yml

@@ -69,3 +69,11 @@ edge_host
 # home, so it must NOT be used as ansible_host.
 [claude_workstation]
 claude-ws ansible_host=10.3.40.78 ansible_user=root
+
+# Hosts that trust the OpenBao SSH user CA (plays/configure-ssh-ca.yml).
+# Rolled out one host at a time: add the next host after the previous one
+# was verified.
+[ssh_ca]
+proxy-1.renner.uno
+proxy-2.renner.uno
+netbird.renner.uno

ansible/plays/configure-ssh-ca.yml

@@ -0,0 +1,15 @@
+---
+# Trust the OpenBao SSH user CA (ssh-client-signer) on the hosts in [ssh_ca]:
+# owner certificates (principal k.renner) for root/ladmin, Claude's 1h
+# certificates (principal claude-bot) for its own claude-bot account with
+# limited sudo. Existing authorized_keys stay untouched.
+#
+# serial: 1 so a host whose sshd rejects the change stops the run before the
+# next host is touched.
+- name: Configure the SSH user CA
+  hosts: ssh_ca
+  become: true
+  gather_facts: true
+  serial: 1
+  roles:
+    - ssh-ca

ansible/roles/ssh-ca/defaults/main.yml

@@ -0,0 +1,18 @@
+---
+# SSH user CA (OpenBao ssh-client-signer). Hosts trust certificates signed by
+# it; the public key is fetched from OpenBao on the controller at run time.
+ssh_ca_public_key_url: "https://openbao.renner.uno/v1/ssh-client-signer/public_key"
+
+# Owner certificates (role owner) carry this principal. It is accepted for
+# whichever admin login the distro has; users that do not exist are skipped.
+ssh_ca_owner_principal: k.renner
+ssh_ca_owner_users:
+  - root
+  - ladmin
+
+# Claude's own account. Certificates (role claude, 1h) carry the principal.
+ssh_ca_claude_user: claude-bot
+ssh_ca_claude_principal: claude-bot
+
+# Units claude-bot may restart/reload via sudo on this host, e.g. [netbird].
+ssh_ca_claude_sudo_units: []

ansible/roles/ssh-ca/files/revoked_keys

@@ -0,0 +1,2 @@
+# Public keys whose certificates sshd must reject (one per line, OpenSSH format).
+# Revoking a key also revokes every certificate issued for it.

ansible/roles/ssh-ca/tasks/main.yml

@@ -0,0 +1,150 @@
+---
+- name: Fetch the SSH CA public key from OpenBao
+  ansible.builtin.uri:
+    url: "{{ ssh_ca_public_key_url }}"
+    return_content: true
+  delegate_to: localhost
+  become: false
+  run_once: true
+  register: ssh_ca_key
+
+- name: Check the fetched CA key is an SSH public key
+  ansible.builtin.assert:
+    that:
+      - ssh_ca_key.content is match('^ssh-ed25519 AAAA')
+    fail_msg: "Unexpected CA key from {{ ssh_ca_public_key_url }}"
+  run_once: true
+
+- name: Install the trusted user CA key
+  ansible.builtin.copy:
+    content: "{{ ssh_ca_key.content | trim }}\n"
+    dest: /etc/ssh/trusted_user_ca_keys
+    owner: root
+    group: root
+    mode: "0644"
+
+# sshd refuses all public key logins when RevokedKeys is set but unreadable,
+# so the file is always deployed, even when empty.
+- name: Install the revoked keys list
+  ansible.builtin.copy:
+    src: revoked_keys
+    dest: /etc/ssh/revoked_keys
+    owner: root
+    group: root
+    mode: "0644"
+
+- name: Create the Claude account
+  ansible.builtin.user:
+    name: "{{ ssh_ca_claude_user }}"
+    shell: /bin/bash
+    create_home: true
+    password: "!"
+
+- name: Look up the owner admin logins
+  ansible.builtin.getent:
+    database: passwd
+    key: "{{ item }}"
+    fail_key: false
+  loop: "{{ ssh_ca_owner_users }}"
+  register: ssh_ca_owner_lookup
+
+- name: Create the principals directory
+  ansible.builtin.file:
+    path: /etc/ssh/auth_principals
+    state: directory
+    owner: root
+    group: root
+    mode: "0755"
+
+- name: Accept the owner principal for the existing admin logins
+  ansible.builtin.copy:
+    content: "{{ ssh_ca_owner_principal }}\n"
+    dest: "/etc/ssh/auth_principals/{{ item.item }}"
+    owner: root
+    group: root
+    mode: "0644"
+  loop: "{{ ssh_ca_owner_lookup.results }}"
+  loop_control:
+    label: "{{ item.item }}"
+  when: (item.ansible_facts.getent_passwd | default({})).get(item.item) is not none
+
+- name: Accept the Claude principal for the Claude account
+  ansible.builtin.copy:
+    content: "{{ ssh_ca_claude_principal }}\n"
+    dest: "/etc/ssh/auth_principals/{{ ssh_ca_claude_user }}"
+    owner: root
+    group: root
+    mode: "0644"
+
+- name: Install the Claude sudo rules
+  ansible.builtin.template:
+    src: sudoers-claude.j2
+    dest: "/etc/sudoers.d/{{ ssh_ca_claude_user }}"
+    owner: root
+    group: root
+    mode: "0440"
+    validate: visudo -cf %s
+
+# A broken drop-in only bites on the next sshd restart, so it is tested here
+# and removed again if the full config does not parse.
+- name: Configure sshd for the user CA
+  block:
+    - name: Install the sshd drop-in
+      ansible.builtin.template:
+        src: ssh-ca.conf.j2
+        dest: /etc/ssh/sshd_config.d/50-ssh-ca.conf
+        owner: root
+        group: root
+        mode: "0644"
+
+    - name: Test the full sshd config
+      ansible.builtin.command: sshd -t
+      changed_when: false
+  rescue:
+    - name: Remove the drop-in that broke sshd -t
+      ansible.builtin.file:
+        path: /etc/ssh/sshd_config.d/50-ssh-ca.conf
+        state: absent
+
+    - name: Fail the host
+      ansible.builtin.fail:
+        msg: "sshd -t failed with the SSH CA drop-in; it was removed again"
+
+- name: Read the effective sshd settings
+  ansible.builtin.command: sshd -T
+  changed_when: false
+  register: ssh_ca_effective
+
+# Keyword case in sshd -T output differs between OpenSSH versions (mixed case
+# on 10.6), so the lines are compared lowercased.
+- name: Check sshd uses the CA, principals and revocation list
+  vars:
+    ssh_ca_effective_lines: "{{ ssh_ca_effective.stdout_lines | map('lower') | list }}"
+  ansible.builtin.assert:
+    that:
+      - "'trustedusercakeys /etc/ssh/trusted_user_ca_keys' in ssh_ca_effective_lines"
+      - "'authorizedprincipalsfile /etc/ssh/auth_principals/%u' in ssh_ca_effective_lines"
+      - "'revokedkeys /etc/ssh/revoked_keys' in ssh_ca_effective_lines"
+    fail_msg: "sshd -T does not show the SSH CA settings; another config line takes precedence"
+
+# The unit is sshd.service on Arch and RedHat, ssh.service on Debian/Ubuntu.
+# Ubuntu also lists sshd.service as a not-found alias, so the running one is
+# picked. sshd -T only parses the files, so the running daemon is reloaded on
+# every run; a reload keeps open sessions.
+- name: Read the service list
+  ansible.builtin.service_facts:
+
+- name: Reload the running sshd unit
+  vars:
+    ssh_ca_sshd_units: "{{ ansible_facts.services | dict2items | selectattr('key', 'in', ['sshd.service', 'ssh.service']) | selectattr('value.state', 'equalto', 'running') | map(attribute='key') | list }}"
+  block:
+    - name: Check exactly one sshd unit is running
+      ansible.builtin.assert:
+        that:
+          - ssh_ca_sshd_units | length == 1
+        fail_msg: "Expected one running sshd.service or ssh.service, found {{ ssh_ca_sshd_units }}"
+
+    - name: Reload sshd so the running daemon uses the CA config
+      ansible.builtin.systemd:
+        name: "{{ ssh_ca_sshd_units | first }}"
+        state: reloaded

ansible/roles/ssh-ca/templates/ssh-ca.conf.j2

@@ -0,0 +1,4 @@
+# {{ ansible_managed }}
+TrustedUserCAKeys /etc/ssh/trusted_user_ca_keys
+AuthorizedPrincipalsFile /etc/ssh/auth_principals/%u
+RevokedKeys /etc/ssh/revoked_keys

ansible/roles/ssh-ca/templates/sudoers-claude.j2

@@ -0,0 +1,6 @@
+# {{ ansible_managed }}
+# NOEXEC: journalctl/systemctl open a pager, which must not spawn a shell.
+{{ ssh_ca_claude_user }} ALL=(root) NOPASSWD:NOEXEC: /usr/bin/journalctl, /usr/bin/journalctl *, /usr/bin/systemctl status, /usr/bin/systemctl status *, /usr/bin/systemctl list-units, /usr/bin/systemctl list-units *, /usr/bin/ss -tulpn, /usr/sbin/sshd -T
+{% for unit in ssh_ca_claude_sudo_units %}
+{{ ssh_ca_claude_user }} ALL=(root) NOPASSWD:NOEXEC: /usr/bin/systemctl restart {{ unit }}, /usr/bin/systemctl reload {{ unit }}
+{% endfor %}

kubernetes/apps/paperless/gotenberg.yaml

@@ -22,7 +22,7 @@ spec:
           type: RuntimeDefault
       containers:
         - name: gotenberg
-          image: gotenberg/gotenberg:8.38.1 # {"$imagepolicy": "flux-system:gotenberg"}
+          image: gotenberg/gotenberg:8.39.0 # {"$imagepolicy": "flux-system:gotenberg"}
           args:
             - "gotenberg"
             - "--chromium-disable-javascript=true"

kubernetes/infrastructure/openbao/policies/ssh-sign-owner.hcl

@@ -1,6 +1,9 @@
 # OpenBao policy: ssh-sign-owner
 # Sign the owner's SSH public keys on ssh-client-signer role owner. Bound to auth/oidc role owner. NOT applied by Flux.
 #
+#   bao secrets enable -path=ssh-client-signer -max-lease-ttl=4380h ssh   # 768h default would cap the 6-month certs
+#   bao write ssh-client-signer/config/ca generate_signing_key=true key_type=ed25519
+#   bao write ssh-client-signer/roles/owner @kubernetes/infrastructure/openbao/ssh-ca/role-owner.json
 #   bao policy write ssh-sign-owner kubernetes/infrastructure/openbao/policies/ssh-sign-owner.hcl
 #   bao auth enable oidc
 #   bao write auth/oidc/config oidc_discovery_url=https://auth.renner.uno/application/o/openbao/ \