diff --git a/.woodpecker/update-os.yml b/.woodpecker/update-os.yml new file mode 100644 index 00000000..b44f3b79 --- /dev/null +++ b/.woodpecker/update-os.yml @@ -0,0 +1,28 @@ +when: + # Manual only: reboots edge-1 (Z-Wave, heating CAN, sole NTP server) and the + # NetBird control plane. Run inside the 02:00-05:00 Europe/Berlin window. + - event: deployment + evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "update-os"' + +timeout: 120 + +steps: + - name: update-os + image: gitea.renner.uno/gitea_admin/ci-ansible:2026-04-29 + environment: + ANSIBLE_FORCE_COLOR: "true" + ANSIBLE_HOST_KEY_CHECKING: "false" + ANSIBLE_ROLES_PATH: "../roles:./roles" + SSH_PRIVATE_KEY: + from_secret: ssh_private_key + VAULT_PASS: + from_secret: vault_pass + commands: + - mkdir -p ~/.ssh + - echo "$SSH_PRIVATE_KEY" | base64 -d > ~/.ssh/id_ed25519 + - chmod 600 ~/.ssh/id_ed25519 + - printf "Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile /dev/null\n" > ~/.ssh/config + - cd ansible + - echo "$VAULT_PASS" > .vault_pass && chmod 600 .vault_pass + - export ANSIBLE_VAULT_PASSWORD_FILE=.vault_pass + - ansible-playbook -i inventory/ plays/update-os.yml -v diff --git a/ansible/plays/update-os.yml b/ansible/plays/update-os.yml new file mode 100644 index 00000000..229ed9ec --- /dev/null +++ b/ansible/plays/update-os.yml @@ -0,0 +1,30 @@ +--- +# Package upgrade + reboot for netbird_proxy, edge_host and netbird_server, +# one host at a time. +# Usage: ansible-playbook -i inventory/ plays/update-os.yml +# +# Plays run in order; a failed host stops the whole run (max_fail_percentage: 0). + +- name: Update NetBird proxies + hosts: netbird_proxy + serial: 1 + max_fail_percentage: 0 + become: true + roles: + - os-update + +- name: Update edge hosts + hosts: edge_host + serial: 1 + max_fail_percentage: 0 + become: true + roles: + - os-update + +- name: Update NetBird server + hosts: netbird_server + serial: 1 + max_fail_percentage: 0 + become: true + roles: + - os-update diff --git a/ansible/roles/os-update/tasks/main.yml b/ansible/roles/os-update/tasks/main.yml new file mode 100644 index 00000000..d7963f05 --- /dev/null +++ b/ansible/roles/os-update/tasks/main.yml @@ -0,0 +1,142 @@ +--- +- name: Fail on unsupported distribution + ansible.builtin.fail: + msg: >- + os-update supports Debian and Archlinux only; + this host reports os_family={{ ansible_facts['os_family'] }} + when: ansible_facts['os_family'] not in ['Debian', 'Archlinux'] + +# --- Baseline: what is healthy before the upgrade must be healthy after --- +- name: Record failed units before upgrade + ansible.builtin.command: systemctl list-units --state=failed --plain --no-legend --no-pager + register: os_update_failed_before + changed_when: false + check_mode: false + +- name: Check for docker + ansible.builtin.stat: + path: /usr/bin/docker + register: os_update_docker + +- name: Record running containers before upgrade + ansible.builtin.command: docker ps --format "{{ '{{' }}.Names{{ '}}' }}" + register: os_update_containers_before + changed_when: false + check_mode: false + when: os_update_docker.stat.exists + +- name: Assert at least 1 GiB free on / + ansible.builtin.assert: + that: >- + (ansible_facts['mounts'] | selectattr('mount', 'equalto', '/') | first)['size_available'] + >= 1073741824 + fail_msg: "Less than 1 GiB free on / -- free space before upgrading" + +# --- Arch Linux --- +- name: Upgrade keyring first (Arch) + community.general.pacman: + name: archlinux-keyring + state: latest + update_cache: true + when: ansible_facts['os_family'] == 'Archlinux' + +- name: Upgrade all packages (Arch) + community.general.pacman: + upgrade: true + register: os_update_pacman + when: ansible_facts['os_family'] == 'Archlinux' + +# --- Debian/Ubuntu --- +- name: Upgrade all packages (Debian) + ansible.builtin.apt: + update_cache: true + upgrade: full + autoremove: true + environment: + DEBIAN_FRONTEND: noninteractive + when: ansible_facts['os_family'] == 'Debian' + +- name: Check reboot-required flag (Debian) + ansible.builtin.stat: + path: /var/run/reboot-required + register: os_update_reboot_flag + when: ansible_facts['os_family'] == 'Debian' + +# --- Reboot --- +- name: Read running kernel before reboot (Arch) + ansible.builtin.command: uname -r + register: os_update_kernel_before + changed_when: false + check_mode: false + when: ansible_facts['os_family'] == 'Archlinux' + +# Missing modules dir: a newer kernel is installed but not booted, including +# from an earlier interrupted run where pacman now reports no change. +- name: Check running kernel is still installed (Arch) + ansible.builtin.stat: + path: "/usr/lib/modules/{{ os_update_kernel_before.stdout }}/vmlinuz" + register: os_update_kernel_present + when: ansible_facts['os_family'] == 'Archlinux' + +- name: Reboot + ansible.builtin.reboot: + msg: "Rebooting after package upgrade" + reboot_timeout: 900 + post_reboot_delay: 30 + when: >- + (ansible_facts['os_family'] == 'Archlinux' and + (os_update_pacman is changed or not os_update_kernel_present.stat.exists)) or + (ansible_facts['os_family'] == 'Debian' and os_update_reboot_flag.stat.exists) + +# --- Verify --- +- name: Read running kernel + ansible.builtin.command: uname -r + register: os_update_kernel + changed_when: false + check_mode: false + +# Arch keeps only the installed kernel's modules; the directory existing means +# the host booted the kernel pacman installed. +- name: Assert running kernel is the installed one (Arch) + ansible.builtin.stat: + path: "/usr/lib/modules/{{ os_update_kernel.stdout }}/vmlinuz" + register: os_update_kernel_installed + failed_when: not os_update_kernel_installed.stat.exists + when: ansible_facts['os_family'] == 'Archlinux' + +- name: Assert no reboot pending (Debian) + ansible.builtin.stat: + path: /var/run/reboot-required + register: os_update_reboot_flag_after + failed_when: os_update_reboot_flag_after.stat.exists + when: ansible_facts['os_family'] == 'Debian' + +- name: Wait for previously running containers + ansible.builtin.command: docker ps --format "{{ '{{' }}.Names{{ '}}' }}" + register: os_update_containers_after + changed_when: false + check_mode: false + until: >- + os_update_containers_before.stdout_lines + | difference(os_update_containers_after.stdout_lines) + | length == 0 + retries: 20 + delay: 15 + when: os_update_docker.stat.exists + +- name: Wait for no new failed units + ansible.builtin.command: systemctl list-units --state=failed --plain --no-legend --no-pager + register: os_update_failed_after + changed_when: false + check_mode: false + until: >- + os_update_failed_after.stdout_lines + | map('split') | map('first') + | difference(os_update_failed_before.stdout_lines | map('split') | map('first')) + | length == 0 + retries: 10 + delay: 15 + +- name: Report kernel + ansible.builtin.debug: + msg: "{{ inventory_hostname }} runs {{ os_update_kernel.stdout }}"