Merge pull request 'fix(ansible): no from= on the CI key for internet hosts' (#1479) from ssh-ci-key-from-per-host into main

Reviewed-on: #1479

.woodpecker/deploy-ssh-ca.yml

@@ -6,6 +6,9 @@ when:
       - "ansible/roles/ssh-ca/**"
       - "ansible/plays/configure-ssh-ca.yml"
       - "ansible/inventory/pve_hosts.yml"
+      - "ansible/inventory/group_vars/all/**"
+      - "ansible/inventory/group_vars/netbird_proxy/vars.yml"
+      - "ansible/inventory/host_vars/netbird.renner.uno.yml"
       - ".woodpecker/deploy-ssh-ca.yml"
   - event: deployment
     evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "deploy-ssh-ca"'

ansible/inventory/group_vars/all/vars.yml

@@ -5,16 +5,26 @@ pve_token_id: "ansible"
 ansible_user: "ladmin"
 ansible_python_interpreter: /usr/bin/python3
 
-# SSH Public Keys for Cloud-Init
-# ansible-runner@ci-new removed (issue #184): dead key, no such host exists.
-common_ssh_public_keys: |
-  ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPzFNBP3bq2m/6JqDqod9w+FP/uxXxHMolqhq3Ojflr1 krenner@fortress
-  ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICyNnX1I4W9jyDvCJ1DJauHSmbo1ZXSyBxVhSAwMWgen woodpecker-ci@homelab
+# The only plain keys the admin logins accept on [ssh_keys_managed] hosts
+# (ssh-ca role, tasks/authorized_keys.yml). People log in with OpenBao SSH CA
+# certificates; these two are for when that path is unavailable.
+#   breakglass-neo: PIV key (slot 9a, PIN) on the offline YubiKey NEO.
+#     ssh -I /usr/lib/opensc-pkcs11.so root@<host>
+#   woodpecker-ci: the CI deploy key, from ssh_ci_key_from where set.
+ssh_admin_authorized_keys:
+  - key: "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBKvSl9LUP8u3S2Sq5YTWpW0Jxbue8cJhD7/LMFQbEsa3xo145HVR7pBzRxW71MUy+jgrgmpSlDhIb606Hswhvoo= breakglass-neo-6532348"
+  - key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICyNnX1I4W9jyDvCJ1DJauHSmbo1ZXSyBxVhSAwMWgen woodpecker-ci@homelab"
+    options: "{{ ('from=\"' ~ ssh_ci_key_from ~ '\"') if ssh_ci_key_from else '' }}"
+
+# Source the CI key is accepted from. LAN hosts see the pod address; hosts on
+# the internet see the home WAN address (NAT, dynamic) and set this empty in
+# their group/host vars.
+ssh_ci_key_from: "{{ ci_ssh_from_cidrs }}"
 
 # Network-of-origin restriction for the woodpecker-ci@homelab CI key
-# (issue #184). Consumed by plays/configure-ssh-keys.yml as the authorized_key
-# `from=` option so the CI key only authenticates from within the cluster's
-# pod network — NOT from any host-pinned IP.
+# (issue #184). Used as the authorized_keys `from=` option so the CI key only
+# authenticates from within the cluster's pod network — NOT from any
+# host-pinned IP.
 #
 # The in-cluster Woodpecker step pods SSH to fleet hosts using their POD IP as
 # source, not a node IP: Cilium runs `ipv4-native-routing-cidr=10.0.0.0/8` +

ansible/inventory/group_vars/netbird_proxy/vars.yml

@@ -0,0 +1,4 @@
+---
+# CI reaches these internet hosts NATed from the home WAN address, never from
+# the pod network, so the CI key carries no from= restriction here.
+ssh_ci_key_from: ""

ansible/inventory/host_vars/netbird.renner.uno.yml

@@ -2,3 +2,7 @@
 # NetBird server - VPS not managed by Proxmox
 # Enable Docker monitoring (cAdvisor) for container metrics
 docker_monitoring_enabled: true
+
+# CI reaches this internet host NATed from the home WAN address, never from
+# the pod network, so the CI key carries no from= restriction here.
+ssh_ci_key_from: ""

ansible/inventory/pve_hosts.yml

@@ -75,3 +75,12 @@ claude-ws ansible_host=10.3.40.78 ansible_user=root
 # was verified.
 [ssh_ca]
 proxy-1.renner.uno
+proxy-2.renner.uno
+netbird.renner.uno
+edge-1.iot.renner.uno
+
+# Hosts whose admin logins accept only ssh_admin_authorized_keys (break-glass
+# + CI); everything else needs a CA certificate. Members must be in [ssh_ca].
+# Added one host at a time after the [ssh_ca] report was reviewed.
+[ssh_keys_managed]
+proxy-1.renner.uno

ansible/plays/configure-ssh-ca.yml

@@ -2,7 +2,8 @@
 # Trust the OpenBao SSH user CA (ssh-client-signer) on the hosts in [ssh_ca]:
 # owner certificates (principal k.renner) for root/ladmin, Claude's 1h
 # certificates (principal claude-bot) for its own claude-bot account with
-# limited sudo. Existing authorized_keys stay untouched.
+# limited sudo. The first play only reports the admin logins' authorized_keys;
+# the second reduces them to ssh_admin_authorized_keys on [ssh_keys_managed].
 #
 # serial: 1 so a host whose sshd rejects the change stops the run before the
 # next host is touched.
@@ -13,3 +14,14 @@
   serial: 1
   roles:
     - ssh-ca
+
+- name: Reduce authorized_keys to break-glass and CI
+  hosts: ssh_keys_managed
+  become: true
+  gather_facts: false
+  serial: 1
+  tasks:
+    - name: Set the managed authorized_keys
+      ansible.builtin.include_role:
+        name: ssh-ca
+        tasks_from: authorized_keys

ansible/plays/configure-ssh-keys.yml

@@ -1,48 +0,0 @@
----
-- name: Deploy authorized SSH keys to all hosts
-  hosts: all:!localhost
-  become: true
-  gather_facts: false
-  # Authorized keys deployed to root and ladmin on every managed host.
-  #
-  # - krenner@fortress (human): unrestricted.
-  # - woodpecker-ci@homelab (CI): restricted with from= to the cluster's pod
-  #   network (ci_ssh_from_cidrs in group_vars/all/vars.yml). The in-cluster
-  #   Woodpecker step pods SSH to fleet hosts presenting their POD IP as source
-  #   (Cilium native-routing-cidr covers the LAN, so no SNAT; PodCIDRs are
-  #   BGP-advertised). This is a network-of-origin restriction, NOT host-pinned,
-  #   so it survives node DHCP churn and pod rescheduling. See vars.yml for the
-  #   full derivation.
-  # - ansible-runner@ci-new: removed — dead key, no such host exists.
-  vars:
-    authorized_keys:
-      - key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPzFNBP3bq2m/6JqDqod9w+FP/uxXxHMolqhq3Ojflr1 krenner@fortress"
-      - key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICyNnX1I4W9jyDvCJ1DJauHSmbo1ZXSyBxVhSAwMWgen woodpecker-ci@homelab"
-        options: 'from="{{ ci_ssh_from_cidrs }}"'
-  tasks:
-    - name: Check SSH connectivity
-      ansible.builtin.wait_for_connection:
-        timeout: 5
-      ignore_errors: true
-      register: ssh_check
-
-    - name: Skip unreachable hosts
-      ansible.builtin.meta: end_host
-      when: ssh_check is failed
-
-    - name: Deploy authorized keys to root
-      ansible.builtin.authorized_key:
-        user: root
-        key: "{{ item.key }}"
-        key_options: "{{ item.options | default(omit) }}"
-        state: present
-      loop: "{{ authorized_keys }}"
-
-    - name: Deploy authorized keys to ladmin
-      ansible.builtin.authorized_key:
-        user: ladmin
-        key: "{{ item.key }}"
-        key_options: "{{ item.options | default(omit) }}"
-        state: present
-      loop: "{{ authorized_keys }}"
-      ignore_errors: true

ansible/roles/common/tasks/main.yml

@@ -51,10 +51,8 @@
 # -------------------------------------------------------
 # 4. SSH Configuration
 # -------------------------------------------------------
-# Authorized keys (human + woodpecker-ci, the latter from=-restricted) are
-# managed by the dedicated ansible/plays/configure-ssh-keys.yml play. The old
-# "Add Ansible Runner SSH Key" task deployed ansible-runner@ci-new, a dead key
-# for a host that no longer exists — removed (issue #184).
+# SSH access (CA trust, admin authorized_keys) is managed by the ssh-ca role
+# via plays/configure-ssh-ca.yml.
 
 # -------------------------------------------------------
 # 5. Disable NTP on VMs/LXC (use host time)

ansible/roles/ssh-ca/tasks/authorized_keys.yml

@@ -0,0 +1,23 @@
+---
+# Replaces the admin logins' authorized_keys with ssh_admin_authorized_keys
+# (break-glass + CI). Everything else logs in with a CA certificate.
+- name: Look up the admin logins
+  ansible.builtin.getent:
+    database: passwd
+    key: "{{ item }}"
+    fail_key: false
+  loop: "{{ ssh_ca_owner_users }}"
+  register: ssh_ca_admin_lookup
+
+- name: Set the admin logins' authorized_keys to the managed list
+  ansible.posix.authorized_key:
+    user: "{{ item.item }}"
+    key: |
+      {% for k in ssh_admin_authorized_keys %}
+      {{ (k.options ~ ' ') if (k.options | default('')) else '' }}{{ k.key }}
+      {% endfor %}
+    exclusive: true
+  loop: "{{ ssh_ca_admin_lookup.results }}"
+  loop_control:
+    label: "{{ item.item }}"
+  when: (item.ansible_facts.getent_passwd | default({})).get(item.item) is not none

ansible/roles/ssh-ca/tasks/main.yml

@@ -115,21 +115,39 @@
   changed_when: false
   register: ssh_ca_effective
 
+# Keyword case in sshd -T output differs between OpenSSH versions (mixed case
+# on 10.6), so the lines are compared lowercased.
 - name: Check sshd uses the CA, principals and revocation list
+  vars:
+    ssh_ca_effective_lines: "{{ ssh_ca_effective.stdout_lines | map('lower') | list }}"
   ansible.builtin.assert:
     that:
-      - "'trustedusercakeys /etc/ssh/trusted_user_ca_keys' in ssh_ca_effective.stdout_lines"
-      - "'authorizedprincipalsfile /etc/ssh/auth_principals/%u' in ssh_ca_effective.stdout_lines"
-      - "'revokedkeys /etc/ssh/revoked_keys' in ssh_ca_effective.stdout_lines"
+      - "'trustedusercakeys /etc/ssh/trusted_user_ca_keys' in ssh_ca_effective_lines"
+      - "'authorizedprincipalsfile /etc/ssh/auth_principals/%u' in ssh_ca_effective_lines"
+      - "'revokedkeys /etc/ssh/revoked_keys' in ssh_ca_effective_lines"
     fail_msg: "sshd -T does not show the SSH CA settings; another config line takes precedence"
 
-# The unit is sshd.service on Arch and RedHat, ssh.service on Debian. sshd -T
-# only parses the files, so the running daemon is reloaded on every run; a
-# reload keeps open sessions.
+# The unit is sshd.service on Arch and RedHat, ssh.service on Debian/Ubuntu.
+# Ubuntu also lists sshd.service as a not-found alias, so the running one is
+# picked. sshd -T only parses the files, so the running daemon is reloaded on
+# every run; a reload keeps open sessions.
 - name: Read the service list
   ansible.builtin.service_facts:
 
-- name: Reload sshd so the running daemon uses the CA config
-  ansible.builtin.systemd:
-    name: "{{ 'sshd.service' if 'sshd.service' in ansible_facts.services else 'ssh.service' }}"
-    state: reloaded
+- name: Reload the running sshd unit
+  vars:
+    ssh_ca_sshd_units: "{{ ansible_facts.services | dict2items | selectattr('key', 'in', ['sshd.service', 'ssh.service']) | selectattr('value.state', 'equalto', 'running') | map(attribute='key') | list }}"
+  block:
+    - name: Check exactly one sshd unit is running
+      ansible.builtin.assert:
+        that:
+          - ssh_ca_sshd_units | length == 1
+        fail_msg: "Expected one running sshd.service or ssh.service, found {{ ssh_ca_sshd_units }}"
+
+    - name: Reload sshd so the running daemon uses the CA config
+      ansible.builtin.systemd:
+        name: "{{ ssh_ca_sshd_units | first }}"
+        state: reloaded
+
+- name: Report the admin logins' authorized keys
+  ansible.builtin.include_tasks: report.yml

ansible/roles/ssh-ca/tasks/report.yml

@@ -0,0 +1,26 @@
+---
+# Read-only: what the admin logins accept besides certificates. Hosts in
+# [ssh_keys_managed] have this reduced to ssh_admin_authorized_keys.
+- name: Find the admin logins' authorized_keys
+  ansible.builtin.stat:
+    path: "{{ item.ansible_facts.getent_passwd[item.item][4] }}/.ssh/authorized_keys"
+  loop: "{{ ssh_ca_owner_lookup.results }}"
+  loop_control:
+    label: "{{ item.item }}"
+  when: (item.ansible_facts.getent_passwd | default({})).get(item.item) is not none
+  register: ssh_ca_keyfiles
+
+- name: Fingerprint the admin logins' authorized keys
+  ansible.builtin.command: ssh-keygen -lf {{ item.stat.path }}
+  loop: "{{ ssh_ca_keyfiles.results | selectattr('stat', 'defined') | selectattr('stat.exists') | list }}"
+  loop_control:
+    label: "{{ item.stat.path }}"
+  changed_when: false
+  register: ssh_ca_fingerprints
+
+- name: Report the admin logins' authorized keys
+  ansible.builtin.debug:
+    msg: "{{ item.item.stat.path }}: {{ item.stdout_lines }}"
+  loop: "{{ ssh_ca_fingerprints.results }}"
+  loop_control:
+    label: "{{ item.item.stat.path }}"

kubernetes/apps/gitea/kustomization.yaml

@@ -16,6 +16,8 @@ resources:
   - networkpolicy.yaml
   # Nightly prune of old CI image builds (REN-1203).
   - package-retention.yaml
+  # OpenBao SSH CA public key for certificate logins.
+  - ssh-ca.yaml
   - kumamonitors.yaml
 configMapGenerator:
   - name: gitea-package-retention

kubernetes/apps/gitea/ssh-ca.yaml

@@ -0,0 +1,37 @@
+# Public key of the OpenBao SSH user CA (ssh-client-signer), for Gitea's
+# built-in SSH server to accept owner and claude-bot certificates. OpenBao
+# serves it unauthenticated; ESO reads it from the in-cluster Service, so the
+# key has one source. Gitea reads it from env at start; Reloader restarts the
+# StatefulSet when the Secret changes.
+apiVersion: external-secrets.io/v1
+kind: SecretStore
+metadata:
+  name: openbao-ssh-ca
+  namespace: gitea
+spec:
+  provider:
+    webhook:
+      url: http://openbao.openbao.svc.cluster.local:8200/v1/ssh-client-signer/public_key
+      method: GET
+---
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: gitea-ssh-ca
+  namespace: gitea
+spec:
+  refreshInterval: 1h
+  secretStoreRef:
+    kind: SecretStore
+    name: openbao-ssh-ca
+  target:
+    name: gitea-ssh-ca
+    # OpenBao ends the body with a newline; it must not reach app.ini.
+    template:
+      engineVersion: v2
+      data:
+        trusted-user-ca-keys: "{{ .ca | trim }}"
+  data:
+    - secretKey: ca
+      remoteRef:
+        key: public_key

kubernetes/apps/gitea/statefulset.yaml

@@ -5,6 +5,9 @@ metadata:
   namespace: gitea
   labels:
     app.kubernetes.io/name: gitea
+  annotations:
+    # A CA rotation changes gitea-ssh-ca; Gitea reads it only at start.
+    secret.reloader.stakater.com/reload: gitea-ssh-ca
 spec:
   serviceName: gitea
   replicas: 1 # SQLite => single writer. Do NOT scale up without moving to Postgres first.
@@ -85,6 +88,16 @@ spec:
             # HAS CHANGED". Paths are relative to APP_DATA_PATH (/var/lib/gitea/data).
             - name: GITEA__server__SSH_SERVER_HOST_KEYS
               value: "ssh/ssh_host_ed25519_key,ssh/ssh_host_rsa_key"
+            # OpenBao SSH CA (ssh-ca.yaml): certificates log in as the user whose
+            # name is a principal of the cert. Each user registers that
+            # principal once (Settings > SSH / GPG Keys > Manage principals).
+            - name: GITEA__server__SSH_TRUSTED_USER_CA_KEYS
+              valueFrom:
+                secretKeyRef:
+                  name: gitea-ssh-ca
+                  key: trusted-user-ca-keys
+            - name: GITEA__server__SSH_AUTHORIZED_PRINCIPALS_ALLOW
+              value: username
             # Gitea Actions was decommissioned — keep it off.
             - name: GITEA__actions__ENABLED
               value: "false"

kubernetes/apps/wiki/deployment.yaml

@@ -53,7 +53,7 @@ spec:
           type: RuntimeDefault
       containers:
         - name: wiki
-          image: gitea.renner.uno/konairius/wiki:build.1793086240 # {"$imagepolicy": "flux-system:wiki"}
+          image: gitea.renner.uno/konairius/wiki:build.1793094262 # {"$imagepolicy": "flux-system:wiki"}
           securityContext:
             allowPrivilegeEscalation: false
             readOnlyRootFilesystem: true

opentofu/locals.tf

@@ -1,9 +0,0 @@
-locals {
-  common_ssh_keys = join("\n", [
-    "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPzFNBP3bq2m/6JqDqod9w+FP/uxXxHMolqhq3Ojflr1 krenner@fortress",
-    "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGfe3HJqFR+jIGrboweseAO7XjIVE24sje/b1HybgU9C ansible-runner@ci-new",
-    "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICyNnX1I4W9jyDvCJ1DJauHSmbo1ZXSyBxVhSAwMWgen woodpecker-ci@homelab",
-    "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBB1OFjh6Q5xeioI6Cwfuv8Xka27veomYy9tKBbpMo8mcrsB4yIN4FX8HovBLtogetBhwBUefBar8nhA9GbRh6HQ= krenner@krenner-XPS-13-9310"
-  ])
-
-}