From 92615c0402e3d101d33381b0de5b13382c877c75 Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Thu, 22 Oct 2026 10:04:28 +0200 Subject: [PATCH 1/2] feat(ansible): add update-os play and pipeline for host package upgrades Since 52899ecb (#625) removed plays/update.yml and the update-all pipeline together with the Proxmox fleet, nothing upgrades the remaining Ansible hosts. proxy-1/2 and edge-1 (Arch) and netbird (Ubuntu) all run kernels affected by CVE-2026-99999. os-update role, per host: - records failed systemd units and running docker containers - Arch: archlinux-keyring first, then pacman -Syu - Debian/Ubuntu: apt full-upgrade + autoremove - reboots when pacman changed something / reboot-required is set - asserts the running kernel is the installed one (Arch) or no reboot is pending (Debian) - waits until every previously running container is back and no new failed units appear update-os.yml runs netbird_proxy, then edge_host, then netbird_server, serial: 1 with max_fail_percentage: 0, so the first failing host stops the whole run before the next host is touched. The pipeline is deployment-only (target update-os): edge-1 is the sole NTP server and carries Z-Wave and the heating CAN bus, and netbird is the control plane; neither may reboot on a cron. Co-Authored-By: Claude Opus 5.5 --- .woodpecker/update-os.yml | 28 ++++++ ansible/plays/update-os.yml | 29 ++++++ ansible/roles/os-update/tasks/main.yml | 119 +++++++++++++++++++++++++ 3 files changed, 176 insertions(+) create mode 100644 .woodpecker/update-os.yml create mode 100644 ansible/plays/update-os.yml create mode 100644 ansible/roles/os-update/tasks/main.yml diff --git a/.woodpecker/update-os.yml b/.woodpecker/update-os.yml new file mode 100644 index 00000000..9c0a8970 --- /dev/null +++ b/.woodpecker/update-os.yml @@ -0,0 +1,28 @@ +when: + # Manual only: reboots edge-1 (Z-Wave, heating CAN, sole NTP server) and the + # NetBird control plane. + - event: deployment + evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "update-os"' + +timeout: 120 + +steps: + - name: update-os + image: gitea.renner.uno/gitea_admin/ci-ansible:2026-04-29 + environment: + ANSIBLE_FORCE_COLOR: "true" + ANSIBLE_HOST_KEY_CHECKING: "false" + ANSIBLE_ROLES_PATH: "../roles:./roles" + SSH_PRIVATE_KEY: + from_secret: ssh_private_key + VAULT_PASS: + from_secret: vault_pass + commands: + - mkdir -p ~/.ssh + - echo "$SSH_PRIVATE_KEY" | base64 -d > ~/.ssh/id_ed25519 + - chmod 600 ~/.ssh/id_ed25519 + - printf "Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile /dev/null\n" > ~/.ssh/config + - cd ansible + - echo "$VAULT_PASS" > .vault_pass && chmod 600 .vault_pass + - export ANSIBLE_VAULT_PASSWORD_FILE=.vault_pass + - ansible-playbook -i inventory/ plays/update-os.yml -v diff --git a/ansible/plays/update-os.yml b/ansible/plays/update-os.yml new file mode 100644 index 00000000..a1f719e8 --- /dev/null +++ b/ansible/plays/update-os.yml @@ -0,0 +1,29 @@ +--- +# Package upgrade + reboot for every Ansible-managed host, one host at a time. +# Usage: ansible-playbook -i inventory/ plays/update-os.yml +# +# Plays run in order; a failed host stops the whole run (max_fail_percentage: 0). + +- name: Update NetBird proxies + hosts: netbird_proxy + serial: 1 + max_fail_percentage: 0 + become: true + roles: + - os-update + +- name: Update edge hosts + hosts: edge_host + serial: 1 + max_fail_percentage: 0 + become: true + roles: + - os-update + +- name: Update NetBird server + hosts: netbird_server + serial: 1 + max_fail_percentage: 0 + become: true + roles: + - os-update diff --git a/ansible/roles/os-update/tasks/main.yml b/ansible/roles/os-update/tasks/main.yml new file mode 100644 index 00000000..d0c0cf20 --- /dev/null +++ b/ansible/roles/os-update/tasks/main.yml @@ -0,0 +1,119 @@ +--- +- name: Fail on unsupported distribution + ansible.builtin.fail: + msg: >- + os-update supports Debian and Archlinux only; + this host reports os_family={{ ansible_facts['os_family'] }} + when: ansible_facts['os_family'] not in ['Debian', 'Archlinux'] + +# --- Baseline: what is healthy before the upgrade must be healthy after --- +- name: Record failed units before upgrade + ansible.builtin.command: systemctl list-units --state=failed --plain --no-legend --no-pager + register: os_update_failed_before + changed_when: false + check_mode: false + +- name: Check for docker + ansible.builtin.stat: + path: /usr/bin/docker + register: os_update_docker + +- name: Record running containers before upgrade + ansible.builtin.command: docker ps --format "{{ '{{' }}.Names{{ '}}' }}" + register: os_update_containers_before + changed_when: false + check_mode: false + when: os_update_docker.stat.exists + +# --- Arch Linux --- +- name: Upgrade keyring first (Arch) + community.general.pacman: + name: archlinux-keyring + state: latest + update_cache: true + when: ansible_facts['os_family'] == 'Archlinux' + +- name: Upgrade all packages (Arch) + community.general.pacman: + upgrade: true + register: os_update_pacman + when: ansible_facts['os_family'] == 'Archlinux' + +# --- Debian/Ubuntu --- +- name: Upgrade all packages (Debian) + ansible.builtin.apt: + update_cache: true + upgrade: full + autoremove: true + environment: + DEBIAN_FRONTEND: noninteractive + when: ansible_facts['os_family'] == 'Debian' + +- name: Check reboot-required flag (Debian) + ansible.builtin.stat: + path: /var/run/reboot-required + register: os_update_reboot_flag + when: ansible_facts['os_family'] == 'Debian' + +# --- Reboot --- +- name: Reboot + ansible.builtin.reboot: + msg: "Rebooting after package upgrade" + reboot_timeout: 900 + post_reboot_delay: 30 + when: >- + (ansible_facts['os_family'] == 'Archlinux' and os_update_pacman is changed) or + (ansible_facts['os_family'] == 'Debian' and os_update_reboot_flag.stat.exists) + +# --- Verify --- +- name: Read running kernel + ansible.builtin.command: uname -r + register: os_update_kernel + changed_when: false + check_mode: false + +# Arch keeps only the installed kernel's modules; the directory existing means +# the host booted the kernel pacman installed. +- name: Assert running kernel is the installed one (Arch) + ansible.builtin.stat: + path: "/usr/lib/modules/{{ os_update_kernel.stdout }}/vmlinuz" + register: os_update_kernel_installed + failed_when: not os_update_kernel_installed.stat.exists + when: ansible_facts['os_family'] == 'Archlinux' + +- name: Assert no reboot pending (Debian) + ansible.builtin.stat: + path: /var/run/reboot-required + register: os_update_reboot_flag_after + failed_when: os_update_reboot_flag_after.stat.exists + when: ansible_facts['os_family'] == 'Debian' + +- name: Wait for previously running containers + ansible.builtin.command: docker ps --format "{{ '{{' }}.Names{{ '}}' }}" + register: os_update_containers_after + changed_when: false + check_mode: false + until: >- + os_update_containers_before.stdout_lines + | difference(os_update_containers_after.stdout_lines) + | length == 0 + retries: 20 + delay: 15 + when: os_update_docker.stat.exists + +- name: Wait for no new failed units + ansible.builtin.command: systemctl list-units --state=failed --plain --no-legend --no-pager + register: os_update_failed_after + changed_when: false + check_mode: false + until: >- + os_update_failed_after.stdout_lines + | map('split') | map('first') + | difference(os_update_failed_before.stdout_lines | map('split') | map('first')) + | length == 0 + retries: 10 + delay: 15 + +- name: Report kernel + ansible.builtin.debug: + msg: "{{ inventory_hostname }} runs {{ os_update_kernel.stdout }}" -- 2.54.0 From 9439e0d58cf7904408d762fa5fbee532ed1cc7ab Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Thu, 22 Oct 2026 11:55:18 +0200 Subject: [PATCH 2/2] fix(os-update): reboot when the booted kernel is no longer installed Review #1811: on Arch the reboot only fired when this run's pacman task changed something, while the post-reboot check looks at host state. A host whose newer kernel was installed by an interrupted earlier run (or by hand) skipped the reboot, failed the kernel check and stopped the whole run. The play now also reboots when /usr/lib/modules/$(uname -r) is gone before the reboot step. Also from the review: - assert >= 1 GiB free on / before upgrading (edge-1 runs from a 6.6 GiB eMMC root, 2.1 GiB free on 2026-10-22) - play comment names the groups it covers instead of 'every host' - pipeline comment points at the README 02:00-05:00 window Co-Authored-By: Claude Opus 5.5 --- .woodpecker/update-os.yml | 2 +- ansible/plays/update-os.yml | 3 ++- ansible/roles/os-update/tasks/main.yml | 25 ++++++++++++++++++++++++- 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/.woodpecker/update-os.yml b/.woodpecker/update-os.yml index 9c0a8970..b44f3b79 100644 --- a/.woodpecker/update-os.yml +++ b/.woodpecker/update-os.yml @@ -1,6 +1,6 @@ when: # Manual only: reboots edge-1 (Z-Wave, heating CAN, sole NTP server) and the - # NetBird control plane. + # NetBird control plane. Run inside the 02:00-05:00 Europe/Berlin window. - event: deployment evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "update-os"' diff --git a/ansible/plays/update-os.yml b/ansible/plays/update-os.yml index a1f719e8..229ed9ec 100644 --- a/ansible/plays/update-os.yml +++ b/ansible/plays/update-os.yml @@ -1,5 +1,6 @@ --- -# Package upgrade + reboot for every Ansible-managed host, one host at a time. +# Package upgrade + reboot for netbird_proxy, edge_host and netbird_server, +# one host at a time. # Usage: ansible-playbook -i inventory/ plays/update-os.yml # # Plays run in order; a failed host stops the whole run (max_fail_percentage: 0). diff --git a/ansible/roles/os-update/tasks/main.yml b/ansible/roles/os-update/tasks/main.yml index d0c0cf20..d7963f05 100644 --- a/ansible/roles/os-update/tasks/main.yml +++ b/ansible/roles/os-update/tasks/main.yml @@ -25,6 +25,13 @@ check_mode: false when: os_update_docker.stat.exists +- name: Assert at least 1 GiB free on / + ansible.builtin.assert: + that: >- + (ansible_facts['mounts'] | selectattr('mount', 'equalto', '/') | first)['size_available'] + >= 1073741824 + fail_msg: "Less than 1 GiB free on / -- free space before upgrading" + # --- Arch Linux --- - name: Upgrade keyring first (Arch) community.general.pacman: @@ -56,13 +63,29 @@ when: ansible_facts['os_family'] == 'Debian' # --- Reboot --- +- name: Read running kernel before reboot (Arch) + ansible.builtin.command: uname -r + register: os_update_kernel_before + changed_when: false + check_mode: false + when: ansible_facts['os_family'] == 'Archlinux' + +# Missing modules dir: a newer kernel is installed but not booted, including +# from an earlier interrupted run where pacman now reports no change. +- name: Check running kernel is still installed (Arch) + ansible.builtin.stat: + path: "/usr/lib/modules/{{ os_update_kernel_before.stdout }}/vmlinuz" + register: os_update_kernel_present + when: ansible_facts['os_family'] == 'Archlinux' + - name: Reboot ansible.builtin.reboot: msg: "Rebooting after package upgrade" reboot_timeout: 900 post_reboot_delay: 30 when: >- - (ansible_facts['os_family'] == 'Archlinux' and os_update_pacman is changed) or + (ansible_facts['os_family'] == 'Archlinux' and + (os_update_pacman is changed or not os_update_kernel_present.stat.exists)) or (ansible_facts['os_family'] == 'Debian' and os_update_reboot_flag.stat.exists) # --- Verify --- -- 2.54.0