From 1b7e3dc1000a1aa9525aec3f3e03489e45935bc0 Mon Sep 17 00:00:00 2001 From: Claude Bot Date: Mon, 10 Aug 2026 21:45:47 +0200 Subject: [PATCH] fix(heating-can): stop the native services before removing their units The migration did not stop anything. It ran stop + disable + mask in one call with failed_when: false, then deleted the unit files. Masking a unit whose file still exists conflicts, so the call failed, failed_when hid the failure, nothing was stopped -- and the next task deleted the unit files out from under two running processes. That left systemctl is-enabled reporting "not-found", which looks clean, while is-active still said "active". I reported the cutover successful on the strength of the former. On 2026-08-10 the result was two open3e instances issuing UDS requests on the same CAN bus from 19:28 to 19:37, taking over each other's open3e-edge MQTT session -- the exact failure the block's own comment said it existed to prevent. Fixed in three parts: - Stop and disable FIRST, and do not swallow the result. Only units that actually exist are touched (stat first), so a fresh host does not need an error hidden to get through. - No masking. It writes a symlink at the same path the next task removes, so the two cancel out. With the unit files gone there is nothing to start. - Assert the invariant that actually matters -- no native process on the CAN bus -- rather than inferring it from systemd reporting success. pgrep, then fail loudly with the surviving command lines and why it matters. Verified against the live host: ok=15 changed=1 failed=0, the assert passing with "No native CAN processes remain; safe to start the container", and the run now idempotent (the single change is the compose up, not the teardown). Co-Authored-By: Claude Bot Co-Authored-By: Claude Opus 5 (1M context) --- ansible/roles/heating-can/tasks/main.yml | 67 ++++++++++++++++++++---- 1 file changed, 56 insertions(+), 11 deletions(-) diff --git a/ansible/roles/heating-can/tasks/main.yml b/ansible/roles/heating-can/tasks/main.yml index 37b03005..290e5262 100644 --- a/ansible/roles/heating-can/tasks/main.yml +++ b/ansible/roles/heating-can/tasks/main.yml @@ -34,17 +34,42 @@ # stopped and masked rather than merely disabled -- masked so a stray # `systemctl start` or a leftover dependency cannot bring them back alongside # the containers. -- name: Stop and mask the native CAN services - ansible.builtin.systemd: - name: "{{ item }}" - state: stopped - enabled: false - masked: true +- name: Find which native CAN units still exist + ansible.builtin.stat: + path: "/etc/systemd/system/{{ item }}.service" loop: - open3e - e3oncan - failed_when: false + register: heating_can_native_units +# Stop and disable BEFORE anything else, and do NOT swallow the result. +# +# The first version did stop + disable + mask in one call with +# failed_when: false, then deleted the unit files. Masking a unit whose file +# still exists conflicts, so the call failed, failed_when hid it, nothing was +# stopped -- and the next task deleted the unit files out from under two +# running processes. systemctl is-enabled then reported "not-found" and looked +# clean while is-active still said "active". +# +# The result on 2026-08-10 was two open3e instances issuing UDS requests on the +# same CAN bus for nine minutes, fighting over the open3e-edge client ID and +# kicking each other off the broker. +# +# Only units that actually exist are touched, so a fresh host does not need an +# error swallowed to get through this. +- name: Stop and disable the native CAN services + ansible.builtin.systemd: + name: "{{ item.item }}" + state: stopped + enabled: false + loop: "{{ heating_can_native_units.results }}" + loop_control: + label: "{{ item.item }}" + when: item.stat.exists + +# Masking is deliberately NOT used. It writes a symlink at the same path the +# next task removes, so the two cancel out -- which is half of how the original +# failed. With the unit files gone there is nothing left to start. - name: Remove the native unit files ansible.builtin.file: path: "/etc/systemd/system/{{ item }}.service" @@ -54,10 +79,30 @@ - e3oncan notify: Reload systemd -# The venvs are the point of this change -- leaving them would leave the thing -# that churned. Their configuration is NOT removed: devices.json, the overlay -# and the env file are still rendered below and bind-mounted into the -# containers. +# The invariant that actually matters is "no native process is on the CAN bus", +# and systemd reporting success is not the same claim. Two open3e instances +# polling one bus is the failure this whole block exists to prevent, so it is +# asserted directly rather than inferred from the calls above. +- name: Check for surviving native CAN processes + ansible.builtin.command: + argv: [pgrep, -af, "edge-host/(open3e|e3oncan)"] + register: heating_can_survivors + changed_when: false + failed_when: false + +- name: Assert no native CAN process survived + ansible.builtin.assert: + that: + - heating_can_survivors.stdout | default('') | length == 0 + fail_msg: >- + A native open3e/e3oncan process is still running after the units were + stopped and removed: + {{ heating_can_survivors.stdout_lines | default([]) | join('; ') }}. + Starting the container now would put two readers on the same CAN bus -- + two open3e instances issue competing UDS requests and take over each + other's MQTT session. Kill the survivors, then re-run. + success_msg: No native CAN processes remain; safe to start the container. + - name: Remove the native venvs ansible.builtin.file: path: "{{ item }}" -- 2.54.0