ansible/roles/edge-host/README.md

@@ -110,7 +110,9 @@ Afterwards:
 
 - Image bumps: edit `edge_zwavejs_image` / `edge_sml2mqtt_image` in
   defaults and re-run the play (no flux image-automation out here —
-  deliberate for sml2mqtt, acceptable for zwave-js).
+  deliberate for sml2mqtt, acceptable for zwave-js). If a zwave-js bump
+  changes its Alpine major, also update `edge_zwave_alpine_base` and refresh
+  the preloaded `.so` (see "Logging & eMMC" → zwave-js).
 - open3e updates: re-run the play; the pip task tracks the git default
   branch (pin `edge_open3e_pkg` to a tag if that ever bites).
 - Ad-hoc CAN scans: `systemctl stop open3e` first (single-instance rule),
@@ -140,9 +142,41 @@ role keeps logs off local flash and monitors wear:
   with basic-auth (`edge_loki_username` / `vault_edge_loki_password`).
   **Delivery only works once the `edge-1` htpasswd line has reconciled into
   the cluster (Flux) — before that Alloy gets 401 and buffers in RAM.**
-- **Flash longevity:** `noatime` on root, weekly `fstrim.timer`, and a weekly
-  `edge-emmc-health.timer` that logs the JEDEC life-time / pre-EOL estimators
-  and warns (journal priority) once the flash leaves the healthy band. Check
-  by hand any time with `/usr/local/bin/edge-emmc-health` or
+- **Flash longevity:** `noatime` + `lazytime` on root, weekly `fstrim.timer`,
+  and a weekly `edge-emmc-health.timer` that logs the JEDEC life-time / pre-EOL
+  estimators and warns (journal priority) once the flash leaves the healthy
+  band. Check by hand any time with `/usr/local/bin/edge-emmc-health` or
   `cat /sys/block/mmcblk0/device/life_time` (`0x01 0x01` = <10 % consumed).
   `mmc extcsd read /dev/mmcblk0` (from `mmc-utils`) gives the full detail.
+
+### zwave-js value-DB writes (the dominant residual writer)
+
+With logs off flash, the biggest remaining eMMC writer is **zwave-js-ui**: its
+value database (`store/*.jsonl`) is `fsync`'d on *every* value change
+(jsonl-db, and zwave-js-ui doesn't expose the driver's `throttle`). On a busy
+network that fsync storm churned ~2.6 GiB/day for ~0.1 KiB/s of actual new
+data — near-pure write amplification (each fsync forces a block write + ext4
+journal commit).
+
+- **Fix:** the `zwave-js` container preloads **`libeatmydata`**
+  (`LD_PRELOAD=/usr/lib/libeatmydata.so`, mounted read-only from
+  `zwave-js/libeatmydata.so`). This makes `fsync`/`fdatasync` no-ops *for that
+  one process*, so the kernel batches the writes via normal writeback (~30 s)
+  instead of forcing each one. The WS/MQTT value stream to Home Assistant and
+  the on-disk cache are **unchanged** — full data resolution, no re-interview
+  on restart. No mount option can do this: `fsync` is a durability contract
+  the filesystem must honor (`commit=`/`data=writeback`/`nobarrier` don't
+  batch it); the only mount-level alternative is putting the cache on tmpfs,
+  which makes the *whole* store volatile → re-interview on every restart.
+- **The `.so` is musl** (the image is Alpine-based), extracted from
+  `edge_zwave_alpine_base` — keep that var in sync with the image's Alpine
+  version; to refresh after a major bump, `rm zwave-js/libeatmydata.so` and
+  re-run.
+- **Durability guard:** because `fsync` is neutralized, a hard power-loss
+  during the (rare) settings write could truncate `settings.json`, which holds
+  the **Z-Wave network keys**. `edge-zwave-settings-backup.timer` keeps a
+  validated hourly rolling backup (`settings.json.bak`) on the eMMC so the keys
+  stay recoverable. Value-cache loss on crash is harmless (devices re-report;
+  jsonl-db tolerates a partial trailing line).
+- Verify the drop with the eMMC-write sampling described above; roll back by
+  removing the `LD_PRELOAD` env + `.so` mount from the compose file.

ansible/roles/edge-host/defaults/main.yml

@@ -39,6 +39,11 @@ edge_stromzaehler_device: /dev/serial/by-id/usb-FTDI_FT230X_Basic_UART_DK0E1GP5-
 # excluded from image-automation in k8s, keep that property here).
 edge_zwavejs_image: ghcr.io/zwave-js/zwave-js-ui:11.21.1
 edge_sml2mqtt_image: spacemanspiff2007/sml2mqtt:3.6.0
+# Alpine version the zwave-js-ui image is built on — the source for the musl
+# libeatmydata.so preloaded into that container (see tasks "Extract musl
+# libeatmydata.so"). Keep in sync with the base if edge_zwavejs_image bumps
+# its Alpine major (check: docker exec zwave-js cat /etc/os-release).
+edge_zwave_alpine_base: alpine:3.22
 
 # --- CAN / open3e (Viessmann E3 external bus) ---
 edge_can_interface: can0

ansible/roles/edge-host/files/edge-zwave-settings-backup.service

@@ -0,0 +1,6 @@
+[Unit]
+Description=Back up zwave-js settings.json (Z-Wave keys) — libeatmydata durability guard
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/bin/edge-zwave-settings-backup

ansible/roles/edge-host/files/edge-zwave-settings-backup.timer

@@ -0,0 +1,10 @@
+[Unit]
+Description=Hourly zwave-js settings.json backup
+
+[Timer]
+OnCalendar=hourly
+Persistent=true
+RandomizedDelaySec=5m
+
+[Install]
+WantedBy=timers.target

ansible/roles/edge-host/tasks/main.yml

@@ -49,12 +49,15 @@
       - edge_root_fs.uuid | default('') | length > 0
     fail_msg: "Could not determine root UUID from facts; not rewriting fstab."
 
-- name: Mount root with noatime (persisted in fstab)
+# lazytime batches inode-timestamp writeback (e.g. the zwave-js lockfile
+# heartbeat that otherwise dirties the journal every couple seconds); noatime
+# drops access-time writes. Neither weakens data durability.
+- name: Mount root with noatime + lazytime (persisted in fstab)
   ansible.posix.mount:
     path: /
     src: "UUID={{ edge_root_fs.uuid }}"
     fstype: "{{ edge_root_fs.fstype }}"
-    opts: "{{ ((edge_root_fs.options.split(',') | reject('in', ['atime', 'noatime', 'relatime', 'diratime']) | list) + ['noatime']) | unique | join(',') }}"
+    opts: "{{ ((edge_root_fs.options.split(',') | reject('in', ['atime', 'noatime', 'relatime', 'diratime', 'lazytime', 'nolazytime']) | list) + ['noatime', 'lazytime']) | unique | join(',') }}"
     dump: "0"
     passno: "1"   # root fs — keep boot-time fsck ordering
     backup: true
@@ -179,6 +182,55 @@
     - "{{ edge_base_dir }}/zwave-js/store"
     - "{{ edge_base_dir }}/sml2mqtt"
 
+# --- zwave-js eMMC write reduction: batch fsync via libeatmydata ---------
+# zwave-js persists its value DB with an fsync on *every* value change
+# (jsonl-db, throttleFS unset by zwave-js-ui). On a chatty network that
+# fsync storm churns the eMMC (~2.6 GiB/day for ~0.1 KiB/s of real data).
+# Preloading libeatmydata makes fsync a no-op so the kernel batches the
+# writes via normal writeback — full HA data resolution and cache
+# persistence are untouched (see docker-compose.yml.j2 for the mount + env).
+# The .so must match the zwave-js-ui image's musl/Alpine base, so extract it
+# from that exact Alpine version (bridge networking is broken on this host,
+# hence --network=host). Refresh by deleting the .so if the base ever bumps
+# its Alpine major version.
+- name: Extract musl libeatmydata.so (matches the zwave-js-ui Alpine base)
+  ansible.builtin.shell:
+    cmd: |
+      set -euo pipefail
+      tmp="$(mktemp)"
+      trap 'rm -f "$tmp"' EXIT
+      docker run --rm --network=host {{ edge_zwave_alpine_base }} \
+        sh -c 'apk add --no-cache libeatmydata >/dev/null 2>&1 && cat /usr/lib/libeatmydata.so' > "$tmp"
+      test "$(stat -c%s "$tmp")" -ge 8000   # sanity: real .so is ~14 KB, guards against a failed/empty pull
+      install -m 0644 "$tmp" {{ edge_base_dir }}/zwave-js/libeatmydata.so
+    creates: "{{ edge_base_dir }}/zwave-js/libeatmydata.so"
+  notify: Restart edge compose services
+
+# Because fsync is neutralized, a hard power-loss during the (rare) settings
+# write could truncate settings.json — which holds the Z-Wave network keys.
+# Keep a validated rolling backup on the eMMC so the keys stay recoverable.
+- name: Install zwave-js settings.json backup script
+  ansible.builtin.template:
+    src: edge-zwave-settings-backup.j2
+    dest: /usr/local/bin/edge-zwave-settings-backup
+    mode: "0755"
+
+- name: Install zwave-js settings backup service + timer
+  ansible.builtin.copy:
+    src: "{{ item }}"
+    dest: "/etc/systemd/system/{{ item }}"
+    mode: "0644"
+  loop:
+    - edge-zwave-settings-backup.service
+    - edge-zwave-settings-backup.timer
+
+- name: Enable zwave-js settings backup timer
+  ansible.builtin.systemd:
+    name: edge-zwave-settings-backup.timer
+    enabled: true
+    state: started
+    daemon_reload: true
+
 # --- Compose services: zwave-js-ui + sml2mqtt ---
 - name: Deploy docker-compose.yml
   ansible.builtin.template:

ansible/roles/edge-host/templates/docker-compose.yml.j2

@@ -17,11 +17,17 @@ services:
     tty: true
     environment:
       - TZ={{ edge_tz }}
+      # Batch the value-DB fsync storm to spare the eMMC (see role tasks /
+      # README "Logging & eMMC"). Neutralizes fsync only for this container;
+      # the WS/MQTT value stream to HA and cache persistence are unchanged.
+      - LD_PRELOAD=/usr/lib/libeatmydata.so
     env_file:
       - zwavejs.env
     devices:
       - "{{ edge_zwave_device }}:/dev/zwave"
     volumes:
+      # musl libeatmydata.so extracted from the image's Alpine base (ro)
+      - ./zwave-js/libeatmydata.so:/usr/lib/libeatmydata.so:ro
       # Persistent store: settings.json, device DB, network keys. Seeded
       # from the k8s PVC `zwavejs-store` at cutover (see README). Since
       # this host is stateful there is no init-container merge step —

ansible/roles/edge-host/templates/edge-zwave-settings-backup.j2

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+# Rolling backup of zwave-js settings.json (holds the Z-Wave network keys).
+# Managed by roles/edge-host. edge-1 runs zwave-js under libeatmydata (fsync
+# neutralized to spare the eMMC), so a hard power-loss during the rare
+# settings write could truncate settings.json. Keep a validated backup on the
+# eMMC so the keys are always recoverable. Runs hourly via a systemd timer.
+set -euo pipefail
+
+src="{{ edge_base_dir }}/zwave-js/store/settings.json"
+bak="$src.bak"
+
+# Only back up a plausibly-intact file: non-empty and JSON-object-terminated.
+# This avoids overwriting a good backup with a truncated (mid-write) file.
+[ -s "$src" ] || { echo "settings.json missing/empty — skip"; exit 0; }
+[ "$(tail -c1 "$src")" = "}" ] || { echo "settings.json not }-terminated (mid-write?) — skip"; exit 0; }
+
+# Skip if unchanged since the last backup.
+if [ -f "$bak" ] && cmp -s "$src" "$bak"; then
+  exit 0
+fi
+
+# Keep one generation of history, then atomically replace the current backup.
+[ -f "$bak" ] && cp -a "$bak" "$bak.1"
+cp -a "$src" "$bak.tmp" && mv "$bak.tmp" "$bak"
+echo "settings.json backed up ($(stat -c%s "$src") bytes)"