fix(os-update): reboot when the booted kernel is no longer installed

Review #1811: on Arch the reboot only fired when this run's pacman task
changed something, while the post-reboot check looks at host state. A
host whose newer kernel was installed by an interrupted earlier run (or
by hand) skipped the reboot, failed the kernel check and stopped the
whole run. The play now also reboots when /usr/lib/modules/$(uname -r)
is gone before the reboot step.

Also from the review:
- assert >= 1 GiB free on / before upgrading (edge-1 runs from a
  6.6 GiB eMMC root, 2.1 GiB free on 2026-10-22)
- play comment names the groups it covers instead of 'every host'
- pipeline comment points at the README 02:00-05:00 window

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

.woodpecker/update-os.yml

@@ -0,0 +1,28 @@
+when:
+  # Manual only: reboots edge-1 (Z-Wave, heating CAN, sole NTP server) and the
+  # NetBird control plane. Run inside the 02:00-05:00 Europe/Berlin window.
+  - event: deployment
+    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "update-os"'
+
+timeout: 120
+
+steps:
+  - name: update-os
+    image: gitea.renner.uno/gitea_admin/ci-ansible:2026-04-29
+    environment:
+      ANSIBLE_FORCE_COLOR: "true"
+      ANSIBLE_HOST_KEY_CHECKING: "false"
+      ANSIBLE_ROLES_PATH: "../roles:./roles"
+      SSH_PRIVATE_KEY:
+        from_secret: ssh_private_key
+      VAULT_PASS:
+        from_secret: vault_pass
+    commands:
+      - mkdir -p ~/.ssh
+      - echo "$SSH_PRIVATE_KEY" | base64 -d > ~/.ssh/id_ed25519
+      - chmod 600 ~/.ssh/id_ed25519
+      - printf "Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile /dev/null\n" > ~/.ssh/config
+      - cd ansible
+      - echo "$VAULT_PASS" > .vault_pass && chmod 600 .vault_pass
+      - export ANSIBLE_VAULT_PASSWORD_FILE=.vault_pass
+      - ansible-playbook -i inventory/ plays/update-os.yml -v

ansible/plays/update-os.yml

@@ -0,0 +1,30 @@
+---
+# Package upgrade + reboot for netbird_proxy, edge_host and netbird_server,
+# one host at a time.
+# Usage: ansible-playbook -i inventory/ plays/update-os.yml
+#
+# Plays run in order; a failed host stops the whole run (max_fail_percentage: 0).
+
+- name: Update NetBird proxies
+  hosts: netbird_proxy
+  serial: 1
+  max_fail_percentage: 0
+  become: true
+  roles:
+    - os-update
+
+- name: Update edge hosts
+  hosts: edge_host
+  serial: 1
+  max_fail_percentage: 0
+  become: true
+  roles:
+    - os-update
+
+- name: Update NetBird server
+  hosts: netbird_server
+  serial: 1
+  max_fail_percentage: 0
+  become: true
+  roles:
+    - os-update

ansible/roles/os-update/tasks/main.yml

@@ -0,0 +1,142 @@
+---
+- name: Fail on unsupported distribution
+  ansible.builtin.fail:
+    msg: >-
+      os-update supports Debian and Archlinux only;
+      this host reports os_family={{ ansible_facts['os_family'] }}
+  when: ansible_facts['os_family'] not in ['Debian', 'Archlinux']
+
+# --- Baseline: what is healthy before the upgrade must be healthy after ---
+- name: Record failed units before upgrade
+  ansible.builtin.command: systemctl list-units --state=failed --plain --no-legend --no-pager
+  register: os_update_failed_before
+  changed_when: false
+  check_mode: false
+
+- name: Check for docker
+  ansible.builtin.stat:
+    path: /usr/bin/docker
+  register: os_update_docker
+
+- name: Record running containers before upgrade
+  ansible.builtin.command: docker ps --format "{{ '{{' }}.Names{{ '}}' }}"
+  register: os_update_containers_before
+  changed_when: false
+  check_mode: false
+  when: os_update_docker.stat.exists
+
+- name: Assert at least 1 GiB free on /
+  ansible.builtin.assert:
+    that: >-
+      (ansible_facts['mounts'] | selectattr('mount', 'equalto', '/') | first)['size_available']
+      >= 1073741824
+    fail_msg: "Less than 1 GiB free on / -- free space before upgrading"
+
+# --- Arch Linux ---
+- name: Upgrade keyring first (Arch)
+  community.general.pacman:
+    name: archlinux-keyring
+    state: latest
+    update_cache: true
+  when: ansible_facts['os_family'] == 'Archlinux'
+
+- name: Upgrade all packages (Arch)
+  community.general.pacman:
+    upgrade: true
+  register: os_update_pacman
+  when: ansible_facts['os_family'] == 'Archlinux'
+
+# --- Debian/Ubuntu ---
+- name: Upgrade all packages (Debian)
+  ansible.builtin.apt:
+    update_cache: true
+    upgrade: full
+    autoremove: true
+  environment:
+    DEBIAN_FRONTEND: noninteractive
+  when: ansible_facts['os_family'] == 'Debian'
+
+- name: Check reboot-required flag (Debian)
+  ansible.builtin.stat:
+    path: /var/run/reboot-required
+  register: os_update_reboot_flag
+  when: ansible_facts['os_family'] == 'Debian'
+
+# --- Reboot ---
+- name: Read running kernel before reboot (Arch)
+  ansible.builtin.command: uname -r
+  register: os_update_kernel_before
+  changed_when: false
+  check_mode: false
+  when: ansible_facts['os_family'] == 'Archlinux'
+
+# Missing modules dir: a newer kernel is installed but not booted, including
+# from an earlier interrupted run where pacman now reports no change.
+- name: Check running kernel is still installed (Arch)
+  ansible.builtin.stat:
+    path: "/usr/lib/modules/{{ os_update_kernel_before.stdout }}/vmlinuz"
+  register: os_update_kernel_present
+  when: ansible_facts['os_family'] == 'Archlinux'
+
+- name: Reboot
+  ansible.builtin.reboot:
+    msg: "Rebooting after package upgrade"
+    reboot_timeout: 900
+    post_reboot_delay: 30
+  when: >-
+    (ansible_facts['os_family'] == 'Archlinux' and
+      (os_update_pacman is changed or not os_update_kernel_present.stat.exists)) or
+    (ansible_facts['os_family'] == 'Debian' and os_update_reboot_flag.stat.exists)
+
+# --- Verify ---
+- name: Read running kernel
+  ansible.builtin.command: uname -r
+  register: os_update_kernel
+  changed_when: false
+  check_mode: false
+
+# Arch keeps only the installed kernel's modules; the directory existing means
+# the host booted the kernel pacman installed.
+- name: Assert running kernel is the installed one (Arch)
+  ansible.builtin.stat:
+    path: "/usr/lib/modules/{{ os_update_kernel.stdout }}/vmlinuz"
+  register: os_update_kernel_installed
+  failed_when: not os_update_kernel_installed.stat.exists
+  when: ansible_facts['os_family'] == 'Archlinux'
+
+- name: Assert no reboot pending (Debian)
+  ansible.builtin.stat:
+    path: /var/run/reboot-required
+  register: os_update_reboot_flag_after
+  failed_when: os_update_reboot_flag_after.stat.exists
+  when: ansible_facts['os_family'] == 'Debian'
+
+- name: Wait for previously running containers
+  ansible.builtin.command: docker ps --format "{{ '{{' }}.Names{{ '}}' }}"
+  register: os_update_containers_after
+  changed_when: false
+  check_mode: false
+  until: >-
+    os_update_containers_before.stdout_lines
+    | difference(os_update_containers_after.stdout_lines)
+    | length == 0
+  retries: 20
+  delay: 15
+  when: os_update_docker.stat.exists
+
+- name: Wait for no new failed units
+  ansible.builtin.command: systemctl list-units --state=failed --plain --no-legend --no-pager
+  register: os_update_failed_after
+  changed_when: false
+  check_mode: false
+  until: >-
+    os_update_failed_after.stdout_lines
+    | map('split') | map('first')
+    | difference(os_update_failed_before.stdout_lines | map('split') | map('first'))
+    | length == 0
+  retries: 10
+  delay: 15
+
+- name: Report kernel
+  ansible.builtin.debug:
+    msg: "{{ inventory_hostname }} runs {{ os_update_kernel.stdout }}"