Merge pull request 'perf(ssh-ca): find the sshd unit without service_facts' (#1495) from ssh-ca-no-service-facts into main

Reviewed-on: #1495

ansible/ansible.cfg

@@ -484,7 +484,7 @@ vault_password_file=~/source/homelab/ansible/vault-password.sh
 # It can result in a very significant performance improvement when enabled.
 # However this conflicts with privilege escalation (become). For example, when using 'sudo:' operations you must first disable 'requiretty' in /etc/sudoers on all managed hosts, which is why it is disabled by default.
 # This setting will be disabled if ``ANSIBLE_KEEP_REMOTE_FILES`` is enabled.
-;pipelining=False
+pipelining=True
 
 
 [colors]

ansible/inventory/group_vars/edge_host/vars.yml

@@ -50,3 +50,11 @@ edge_mqtt_host: 127.0.0.1
 edge_mqtt_port: 1883
 edge_open3e_mqtt_user: open3e
 edge_open3e_mqtt_topic: open3e
+
+# Units claude-bot may restart (ssh-ca role); the containers go through docker.
+ssh_ca_claude_sudo_units:
+  - mosquitto
+  - edge-mqtt-cred
+  - edge-alloy
+  - keepalived
+  - chronyd

ansible/inventory/group_vars/netbird_proxy/vars.yml

@@ -2,3 +2,8 @@
 # The host NetBird client (wt0) must not take over DNS: resolv.conf is pinned to
 # the systemd-resolved stub by the netbird-proxy role.
 netbird_disable_dns: true
+
+ssh_ca_claude_sudo_units:
+  - netbird
+  - netbird-proxy-flood-watchdog
+  - crowdsec-firewall-bouncer

ansible/inventory/host_vars/netbird.renner.uno.yml

@@ -2,3 +2,6 @@
 # NetBird server - VPS not managed by Proxmox
 # Enable Docker monitoring (cAdvisor) for container metrics
 docker_monitoring_enabled: true
+
+ssh_ca_claude_sudo_units:
+  - netbird

ansible/inventory/pve_hosts.yml

@@ -79,11 +79,19 @@ proxy-2.renner.uno
 netbird.renner.uno
 edge-1.iot.renner.uno
 
-# Hosts whose admin logins accept only ssh_admin_authorized_keys (break-glass
-# + CI); everything else needs a CA certificate. Members must be in [ssh_ca].
+# Hosts whose admin logins accept only ssh_admin_authorized_keys (break-glass);
+# everything else needs a CA certificate. Members must be in [ssh_ca].
 # Added one host at a time after the [ssh_ca] report was reviewed.
 [ssh_keys_managed]
 proxy-1.renner.uno
 proxy-2.renner.uno
 netbird.renner.uno
 edge-1.iot.renner.uno
+
+# Internet hosts whose sshd listens on the NetBird interface only
+# (plays/configure-ssh-mesh-only.yml). Members need the host NetBird client.
+# Added one host at a time after the previous one was verified.
+[ssh_mesh_only]
+proxy-1.renner.uno
+proxy-2.renner.uno
+netbird.renner.uno

ansible/plays/configure-ssh-ca.yml

@@ -2,11 +2,12 @@
 # Trust the OpenBao SSH user CA (ssh-client-signer) on the hosts in [ssh_ca]:
 # owner certificates (principal k.renner) for root/ladmin, Claude's 1h
 # certificates (principal claude-bot) for its own claude-bot account with
-# limited sudo. The first play only reports the admin logins' authorized_keys;
-# the second reduces them to ssh_admin_authorized_keys on [ssh_keys_managed].
+# limited sudo. The role reports the admin logins' authorized_keys; on
+# [ssh_keys_managed] they are then reduced to ssh_admin_authorized_keys.
 #
 # serial: 1 so a host whose sshd rejects the change stops the run before the
-# next host is touched.
+# next host is touched. One play, so each host needs one login (one YubiKey
+# PIN from a workstation).
 - name: Configure the SSH user CA
   hosts: ssh_ca
   become: true
@@ -14,14 +15,9 @@
   serial: 1
   roles:
     - ssh-ca
-
-- name: Reduce authorized_keys to the break-glass key
-  hosts: ssh_keys_managed
-  become: true
-  gather_facts: false
-  serial: 1
   tasks:
-    - name: Set the managed authorized_keys
+    - name: Reduce authorized_keys to the break-glass key
       ansible.builtin.include_role:
         name: ssh-ca
         tasks_from: authorized_keys
+      when: inventory_hostname in groups['ssh_keys_managed']

ansible/plays/configure-ssh-mesh-only.yml

@@ -0,0 +1,15 @@
+---
+# sshd on the hosts in [ssh_mesh_only] accepts connections only on the NetBird
+# interface; nothing listens on their public addresses. Run from a workstation
+# in the mesh. A switchover reverts by itself unless the play logs in again
+# through the new listener.
+#
+# serial: 1 so a host that does not come back stops the run before the next
+# host is touched.
+- name: Restrict sshd to the mesh interface
+  hosts: ssh_mesh_only
+  become: true
+  gather_facts: true
+  serial: 1
+  roles:
+    - ssh-mesh-only

ansible/roles/ssh-ca/tasks/main.yml

@@ -33,12 +33,15 @@
     group: root
     mode: "0644"
 
+# systemd-journal: read every journal without sudo (see sudoers-claude.j2).
 - name: Create the Claude account
   ansible.builtin.user:
     name: "{{ ssh_ca_claude_user }}"
     shell: /bin/bash
     create_home: true
     password: "!"
+    groups: systemd-journal
+    append: true
 
 - name: Look up the owner admin logins
   ansible.builtin.getent:
@@ -125,29 +128,36 @@
       - "'trustedusercakeys /etc/ssh/trusted_user_ca_keys' in ssh_ca_effective_lines"
       - "'authorizedprincipalsfile /etc/ssh/auth_principals/%u' in ssh_ca_effective_lines"
       - "'revokedkeys /etc/ssh/revoked_keys' in ssh_ca_effective_lines"
+      - "'passwordauthentication no' in ssh_ca_effective_lines"
+      - "'kbdinteractiveauthentication no' in ssh_ca_effective_lines"
     fail_msg: "sshd -T does not show the SSH CA settings; another config line takes precedence"
 
-# The unit is sshd.service on Arch and RedHat, ssh.service on Debian/Ubuntu.
-# Ubuntu also lists sshd.service as a not-found alias, so the running one is
-# picked. sshd -T only parses the files, so the running daemon is reloaded on
-# every run; a reload keeps open sessions.
-- name: Read the service list
-  ansible.builtin.service_facts:
+# The unit is sshd.service on Arch and RedHat, ssh.service on Debian/Ubuntu,
+# where sshd.service is an alias that resolves to it. sshd -T only parses the
+# files, so the running daemon is reloaded on every run; a reload keeps open
+# sessions. None runs when sshd is socket activated and stopped, or per
+# connection (ssh-mesh-only role); the next start reads the config.
+# systemctl directly: service_facts took 3-12 s per host.
+- name: Find the running sshd unit
+  ansible.builtin.command: systemctl list-units --state=running --plain --no-legend --type=service sshd.service ssh.service
+  changed_when: false
+  register: ssh_ca_sshd_running
 
 - name: Reload the running sshd unit
   vars:
-    ssh_ca_sshd_units: "{{ ansible_facts.services | dict2items | selectattr('key', 'in', ['sshd.service', 'ssh.service']) | selectattr('value.state', 'equalto', 'running') | map(attribute='key') | list }}"
+    ssh_ca_sshd_units: "{{ ssh_ca_sshd_running.stdout_lines | map('split') | map('first') | unique | list }}"
   block:
-    - name: Check exactly one sshd unit is running
+    - name: Check at most one sshd unit is running
       ansible.builtin.assert:
         that:
-          - ssh_ca_sshd_units | length == 1
-        fail_msg: "Expected one running sshd.service or ssh.service, found {{ ssh_ca_sshd_units }}"
+          - ssh_ca_sshd_units | length <= 1
+        fail_msg: "Expected at most one running sshd.service or ssh.service, found {{ ssh_ca_sshd_units }}"
 
     - name: Reload sshd so the running daemon uses the CA config
       ansible.builtin.systemd:
         name: "{{ ssh_ca_sshd_units | first }}"
         state: reloaded
+      when: ssh_ca_sshd_units | length == 1
 
 - name: Report the admin logins' authorized keys
   ansible.builtin.include_tasks: report.yml

ansible/roles/ssh-ca/templates/ssh-ca.conf.j2

@@ -2,3 +2,5 @@
 TrustedUserCAKeys /etc/ssh/trusted_user_ca_keys
 AuthorizedPrincipalsFile /etc/ssh/auth_principals/%u
 RevokedKeys /etc/ssh/revoked_keys
+PasswordAuthentication no
+KbdInteractiveAuthentication no

ansible/roles/ssh-ca/templates/sudoers-claude.j2

@@ -1,6 +1,12 @@
 # {{ ansible_managed }}
-# NOEXEC: journalctl/systemctl open a pager, which must not spawn a shell.
-{{ ssh_ca_claude_user }} ALL=(root) NOPASSWD:NOEXEC: /usr/bin/journalctl, /usr/bin/journalctl *, /usr/bin/systemctl status, /usr/bin/systemctl status *, /usr/bin/systemctl list-units, /usr/bin/systemctl list-units *, /usr/bin/ss -tulpn, /usr/sbin/sshd -T
+# Logs and unit status come from the systemd-journal group, not sudo:
+# journalctl as root would also allow --vacuum-*/--rotate.
+# Commands are listed for both the Arch and the Debian/Ubuntu path.
+# nft only with fixed arguments: it joins argv into one command line, so a
+# wildcard would admit "; flush ruleset". No docker start: -a/-i attach to a
+# container's stdin; docker restart also starts stopped containers.
+{{ ssh_ca_claude_user }} ALL=(root) NOPASSWD:NOEXEC: /usr/bin/ss -tulpn, /usr/sbin/sshd -T, /usr/bin/nft list ruleset, /usr/sbin/nft list ruleset, /usr/bin/netbird status, /usr/bin/netbird status *, /usr/local/bin/netbird status, /usr/local/bin/netbird status *
+{{ ssh_ca_claude_user }} ALL=(root) NOPASSWD:NOEXEC: /usr/bin/docker ps, /usr/bin/docker ps *, /usr/bin/docker logs *, /usr/bin/docker stats --no-stream, /usr/bin/docker restart *
 {% for unit in ssh_ca_claude_sudo_units %}
 {{ ssh_ca_claude_user }} ALL=(root) NOPASSWD:NOEXEC: /usr/bin/systemctl restart {{ unit }}, /usr/bin/systemctl reload {{ unit }}
 {% endfor %}

ansible/roles/ssh-mesh-only/defaults/main.yml

@@ -0,0 +1,9 @@
+---
+# The only interface sshd accepts connections on.
+ssh_mesh_interface: wt0
+
+# Minutes until a switchover reverts to the distro listener, unless a new
+# login through the mesh-only listener confirms it first.
+ssh_mesh_revert_minutes: 5
+
+ssh_mesh_udev_rule: /etc/udev/rules.d/90-ssh-mesh-only.rules

ansible/roles/ssh-mesh-only/tasks/main.yml

@@ -0,0 +1,113 @@
+---
+- name: Load the distro's sshd unit names
+  ansible.builtin.include_vars: "{{ ansible_facts['os_family'] }}.yml"
+
+- name: Look for the mesh interface
+  ansible.builtin.stat:
+    path: "/sys/class/net/{{ ssh_mesh_interface }}"
+  register: ssh_mesh_iface
+
+- name: Check the mesh interface exists
+  ansible.builtin.assert:
+    that:
+      - ssh_mesh_iface.stat.exists
+    fail_msg: "No {{ ssh_mesh_interface }}: sshd would be unreachable. Install the NetBird client first."
+
+- name: Create the socket unit directory
+  ansible.builtin.file:
+    path: "{{ ssh_mesh_socket_file | dirname }}"
+    state: directory
+    owner: root
+    group: root
+    mode: "0755"
+
+- name: Install the mesh-only socket
+  ansible.builtin.template:
+    src: "{{ ssh_mesh_socket_template }}"
+    dest: "{{ ssh_mesh_socket_file }}"
+    owner: root
+    group: root
+    mode: "0644"
+
+- name: Bind the sshd service to the socket
+  when: ssh_mesh_service_dropin | length > 0
+  block:
+    - name: Create the service drop-in directory
+      ansible.builtin.file:
+        path: "{{ ssh_mesh_service_dropin | dirname }}"
+        state: directory
+        owner: root
+        group: root
+        mode: "0755"
+
+    - name: Install the service drop-in
+      ansible.builtin.template:
+        src: ssh.service-dropin.conf.j2
+        dest: "{{ ssh_mesh_service_dropin }}"
+        owner: root
+        group: root
+        mode: "0644"
+
+- name: Install the udev rule that starts the socket with the interface
+  ansible.builtin.template:
+    src: udev.rules.j2
+    dest: "{{ ssh_mesh_udev_rule }}"
+    owner: root
+    group: root
+    mode: "0644"
+  register: ssh_mesh_udev
+
+- name: Reload the udev rules
+  ansible.builtin.command: udevadm control --reload
+  when: ssh_mesh_udev is changed
+  changed_when: true
+
+- name: Install the switch and revert scripts
+  ansible.builtin.template:
+    src: "{{ item }}.j2"
+    dest: "/usr/local/sbin/{{ item }}"
+    owner: root
+    group: root
+    mode: "0755"
+  loop:
+    - ssh-mesh-switch
+    - ssh-mesh-revert
+
+- name: Reload systemd units
+  ansible.builtin.systemd:
+    daemon_reload: true
+
+- name: Read the sshd listeners
+  ansible.builtin.command: ss -Htln 'sport = :22'
+  changed_when: false
+  register: ssh_mesh_listeners
+
+# Each run that finds a listener outside the interface switches over. An
+# include, because meta reset_connection ignores when: and a reconnect costs a
+# YubiKey PIN from a workstation.
+- name: Switch sshd to the mesh-only socket
+  ansible.builtin.include_tasks: switch.yml
+  when: >-
+    ssh_mesh_listeners.stdout_lines | length == 0 or
+    ssh_mesh_listeners.stdout_lines | reject('search', '%' ~ ssh_mesh_interface ~ ':22\s') | list | length > 0
+
+- name: Check sshd listens on the mesh interface only
+  ansible.builtin.assert:
+    that:
+      - ssh_mesh_listeners.stdout_lines | length > 0
+      - ssh_mesh_listeners.stdout_lines | reject('search', '%' ~ ssh_mesh_interface ~ ':22\s') | list | length == 0
+    fail_msg: "Listeners outside {{ ssh_mesh_interface }}: {{ ssh_mesh_listeners.stdout_lines }}"
+
+- name: Cancel the revert timer
+  ansible.builtin.command: systemctl stop ssh-mesh-revert.timer
+  changed_when: false
+  failed_when: false
+
+- name: Check the public name refuses SSH
+  ansible.builtin.wait_for:
+    host: "{{ inventory_hostname }}"
+    port: 22
+    state: stopped
+    timeout: 15
+  delegate_to: localhost
+  become: false

ansible/roles/ssh-mesh-only/tasks/switch.yml

@@ -0,0 +1,27 @@
+---
+# The revert timer runs in its own unit, so a switch that locks the play out is
+# undone after ssh_mesh_revert_minutes. main.yml cancels it after checking the
+# listeners read through the new one.
+- name: Clear a revert timer left by an earlier run
+  ansible.builtin.command: systemctl stop ssh-mesh-revert.timer ssh-mesh-revert.service
+  changed_when: false
+  failed_when: false
+
+- name: Arm the revert timer
+  ansible.builtin.command: >-
+    systemd-run --unit=ssh-mesh-revert --collect
+    --on-active={{ ssh_mesh_revert_minutes }}min
+    /usr/local/sbin/ssh-mesh-revert
+  changed_when: true
+
+- name: Switch the listener
+  ansible.builtin.command: systemd-run --unit=ssh-mesh-switch --collect --wait /usr/local/sbin/ssh-mesh-switch
+  changed_when: true
+
+- name: Reconnect through the new listener
+  ansible.builtin.meta: reset_connection
+
+- name: Read the sshd listeners after reconnecting
+  ansible.builtin.command: ss -Htln 'sport = :22'
+  changed_when: false
+  register: ssh_mesh_listeners

ansible/roles/ssh-mesh-only/templates/ssh-mesh-revert.j2

@@ -0,0 +1,9 @@
+#!/bin/sh
+# {{ ansible_managed }}
+# Restores the distro listener on all addresses. Armed as a timer by every
+# switchover; the play cancels it after a login through the new listener.
+systemctl stop {{ ssh_mesh_service }} {{ ssh_mesh_socket }}
+rm -f {{ ssh_mesh_udev_rule }} {{ ssh_mesh_socket_file }}{{ (' ' ~ ssh_mesh_service_dropin) if ssh_mesh_service_dropin else '' }}
+udevadm control --reload
+systemctl daemon-reload
+systemctl enable --now {{ ssh_mesh_distro_listener }}

ansible/roles/ssh-mesh-only/templates/ssh-mesh-switch.j2

@@ -0,0 +1,11 @@
+#!/bin/sh
+# {{ ansible_managed }}
+# Replaces the distro listener with the {{ ssh_mesh_interface }}-only socket.
+# Open sessions survive: they run in logind scopes and the sshd units use
+# KillMode=process. Any failure restores the distro listener at once.
+set -eu
+trap '/usr/local/sbin/ssh-mesh-revert' EXIT
+systemctl disable {{ ssh_mesh_disable_units }}
+systemctl stop {{ ssh_mesh_service }} {{ ssh_mesh_socket }}
+systemctl start {{ ssh_mesh_socket }}
+trap - EXIT

ansible/roles/ssh-mesh-only/templates/ssh.service-dropin.conf.j2

@@ -0,0 +1,7 @@
+# {{ ansible_managed }}
+# sshd holds the socket's listener; it stops with the socket, so it never keeps
+# a listener on a removed {{ ssh_mesh_interface }}. Replaces the Requires= that
+# enabling {{ ssh_mesh_socket }} would add.
+[Unit]
+BindsTo={{ ssh_mesh_socket }}
+After={{ ssh_mesh_socket }}

ansible/roles/ssh-mesh-only/templates/ssh.socket-addresses.conf.j2

@@ -0,0 +1,8 @@
+# {{ ansible_managed }}
+# Listen on {{ ssh_mesh_interface }} only. The unit is disabled; the udev rule
+# starts it whenever {{ ssh_mesh_interface }} appears.
+[Socket]
+ListenStream=
+ListenStream=22
+BindToDevice={{ ssh_mesh_interface }}
+BindIPv6Only=both

ansible/roles/ssh-mesh-only/templates/sshd.socket.j2

@@ -0,0 +1,12 @@
+# {{ ansible_managed }}
+# Started by the udev rule whenever {{ ssh_mesh_interface }} appears; BindToDevice
+# stops it when the interface goes away. No [Install]: waiting for the device
+# from sockets.target would hold up boot until NetBird is up.
+[Unit]
+Description=OpenSSH on {{ ssh_mesh_interface }} only
+
+[Socket]
+ListenStream=22
+BindToDevice={{ ssh_mesh_interface }}
+BindIPv6Only=both
+Accept=yes

ansible/roles/ssh-mesh-only/templates/udev.rules.j2

@@ -0,0 +1,3 @@
+# {{ ansible_managed }}
+# NetBird deletes and recreates {{ ssh_mesh_interface }} on every engine restart.
+SUBSYSTEM=="net", KERNEL=="{{ ssh_mesh_interface }}", ENV{SYSTEMD_WANTS}+="{{ ssh_mesh_socket }}"

ansible/roles/ssh-mesh-only/vars/Archlinux.yml

@@ -0,0 +1,10 @@
+---
+# No packaged socket; sshd.socket (Accept=yes) pairs with the packaged
+# per-connection sshd@.service, since sshd -D cannot take a listening fd.
+ssh_mesh_socket: sshd.socket
+ssh_mesh_service: sshd.service
+ssh_mesh_distro_listener: sshd.service
+ssh_mesh_disable_units: sshd.service
+ssh_mesh_socket_file: /etc/systemd/system/sshd.socket
+ssh_mesh_socket_template: sshd.socket.j2
+ssh_mesh_service_dropin: ""

ansible/roles/ssh-mesh-only/vars/Debian.yml

@@ -0,0 +1,10 @@
+---
+# Ubuntu's ssh.socket (Accept=no) hands its listener to ssh.service. The
+# drop-in is named like sshd-socket-generator's output so it masks it.
+ssh_mesh_socket: ssh.socket
+ssh_mesh_service: ssh.service
+ssh_mesh_distro_listener: ssh.socket
+ssh_mesh_disable_units: ssh.service ssh.socket
+ssh_mesh_socket_file: /etc/systemd/system/ssh.socket.d/addresses.conf
+ssh_mesh_socket_template: ssh.socket-addresses.conf.j2
+ssh_mesh_service_dropin: /etc/systemd/system/ssh.service.d/ssh-mesh-only.conf