From 3ae01e9f6dfead382ea2039358fa85cb3874143e Mon Sep 17 00:00:00 2001 From: Konstantin Renner Date: Tue, 7 Jul 2026 08:49:07 +0200 Subject: [PATCH] edge-1: keep logs off the eMMC (volatile journal + Alloy->Loki) + flash tuning MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit edge-1 is a Dell Wyse 3040 on a 7.3 GB internal eMMC (not the N100/NVMe the README assumed). Unbounded docker json logs (a single zwave-js log had grown to 214 MB) plus a persistent journald were churning the flash. Move all logging off local flash and ship it to Loki instead: - journald Storage=volatile (RAM), size-capped; purge the on-disk store - docker json-file -> journald log driver (no container json logs on disk) - Grafana Alloy (Arch pkg) tails the RAM journal -> Loki, WAL on /run tmpfs Loki endpoint: the unauthenticated srv-network Loki is firewalled off the IoT VLAN, so Alloy pushes to the authenticated cluster ingress (https://loki.renner.uno) with a dedicated 'edge-1' basic-auth user — its htpasswd line is added to the loki-basic-auth cluster secret, plaintext in the edge_host ansible vault. Delivery activates once Flux reconciles that secret; until then Alloy 401s and buffers in RAM. Flash longevity: noatime on root, weekly fstrim.timer, and a weekly edge-emmc-health check of the JEDEC life-time / pre-EOL estimators. The Alloy config is purpose-built in the edge-host role rather than reusing the shared monitoring-agent role: that role is Debian-oriented (apt + GitHub binary), force-enables a docker log source that would double-ship against the journald driver, and always configures a Prometheus remote-write whose endpoint is unreachable from the IoT VLAN. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../inventory/group_vars/edge_host/vars.yml | 4 + .../inventory/group_vars/edge_host/vault.yml | 41 ++--- ansible/roles/edge-host/README.md | 35 +++++ ansible/roles/edge-host/defaults/main.yml | 20 +++ .../roles/edge-host/files/edge-emmc-health | 56 +++++++ .../edge-host/files/edge-emmc-health.service | 7 + .../edge-host/files/edge-emmc-health.timer | 10 ++ ansible/roles/edge-host/handlers/main.yml | 25 ++++ ansible/roles/edge-host/tasks/main.yml | 140 +++++++++++++++++- .../edge-host/templates/docker-daemon.json.j2 | 3 + .../edge-host/templates/edge-alloy.alloy.j2 | 56 +++++++ .../edge-host/templates/edge-alloy.service.j2 | 32 ++++ .../edge-host/templates/journald-edge.conf.j2 | 7 + .../apps/monitoring/loki-basic-auth.sops.yaml | 6 +- 14 files changed, 420 insertions(+), 22 deletions(-) create mode 100644 ansible/roles/edge-host/files/edge-emmc-health create mode 100644 ansible/roles/edge-host/files/edge-emmc-health.service create mode 100644 ansible/roles/edge-host/files/edge-emmc-health.timer create mode 100644 ansible/roles/edge-host/templates/docker-daemon.json.j2 create mode 100644 ansible/roles/edge-host/templates/edge-alloy.alloy.j2 create mode 100644 ansible/roles/edge-host/templates/edge-alloy.service.j2 create mode 100644 ansible/roles/edge-host/templates/journald-edge.conf.j2 diff --git a/ansible/inventory/group_vars/edge_host/vars.yml b/ansible/inventory/group_vars/edge_host/vars.yml index c8d429e8..c1cbca40 100644 --- a/ansible/inventory/group_vars/edge_host/vars.yml +++ b/ansible/inventory/group_vars/edge_host/vars.yml @@ -5,3 +5,7 @@ edge_sml2mqtt_mqtt_password: "{{ vault_edge_sml2mqtt_mqtt_password }}" edge_open3e_mqtt_password: "{{ vault_edge_open3e_mqtt_password }}" edge_zwavejs_session_secret: "{{ vault_edge_zwavejs_session_secret }}" +# Loki push credential (dedicated 'edge-1' user, Traefik basic-auth on the +# loki.renner.uno ingress; the matching htpasswd line lives in the cluster +# secret kubernetes/apps/monitoring/loki-basic-auth.sops.yaml). +edge_loki_password: "{{ vault_edge_loki_password }}" diff --git a/ansible/inventory/group_vars/edge_host/vault.yml b/ansible/inventory/group_vars/edge_host/vault.yml index a94e2b6f..9a971363 100644 --- a/ansible/inventory/group_vars/edge_host/vault.yml +++ b/ansible/inventory/group_vars/edge_host/vault.yml @@ -1,18 +1,25 @@ $ANSIBLE_VAULT;1.1;AES256 -66366335386261303463626230383532323132366232373839643563643531666662383366313336 -6562666333613539396234363135306238323662343132380a636631303364356564656536633164 -62373531663635633139373231343434393035646263373833623636613133653561353266383064 -3261663765316431620a343665653835363433373637616538396163376366353235653561663364 -30353532373061633139623032373263376339616334623931636537633263663237663466636461 -39646165343961316364613431393563356562646438346466653133313264366562393164666633 -64616164333235653734313562303864633534303037306332613934386238313764656465333465 -63363936653566326534333434633631316536336537656334656661663637353830333237356532 -66323566356138633161383334363334663035313762636561626466613165333065623136663537 -62623863373465316636643933323539303232653263636462646363383861353932366138353233 -37303363313362306364303062663435653335343561373934366433306439616431323365636365 -64363837353466396130643962343134376361343431383631336565633866623037613065346364 -61373664653634656135373466373038626533376361373862316562623035306632333533333231 -37353665623738623534353965303634663331613433623661343338666530313863313731316337 -37616364633261356361383937376132333636663464326336343439626666313636373436633338 -63663730323134376536663562633034323339643537393937336362633333373434643037663331 -3634 +36303532663231326533323264316664636264343531656330353530653630663564383231366130 +3833653663663739303466343366353238643064363063630a366561626238643065373138663335 +36383830636432623933613232343464383264356364616331356662313064396262626165663136 +3838653564653737350a633838653437613837336163333330323137376333363361316532653736 +65326438313232383034316661656338333736323531633337306435353561393735376239353736 +35613237623332613863646362333537653036383335386263323032643563643064396238393138 +64343561303638356535663431663035663935386263656366326166356139303138373530306630 +61353437333538353562373735643939653437393363653932653765626531313831313737643839 +66616230323965616561646539613763373261396438333732386163303934383539396166386465 +64656361636563613366326566326336336164353438343932373764656233386138633036316436 +33313935303865323735623233343739326534653136336432643831633830623566323836393031 +33643439656162336363663862623236633836616234396565656637656163303064636161376361 +62623663323765343531343639336638306161613164613163313331636164666363393433323831 +34396165353035333533373833633132633463633937666162386635636234623730653135373066 +39363561336338373365663765363932313263643163323863376439613961346366313162333266 +31363031356335393461666238393035633466333030623036333134393337383932656139393762 +31323438353539656235616263356666383933396231616338643631373336326639303765353166 +37643162306164326239393738303861323739306539646163663339326134326139353732356265 +64313861613834333436383032623839393961646630646332303439343033333964656133663734 +65376630393431653830636364663337343665323937303531646361313932363561636535346233 +33376661326664326230636230653930323261653564633234373138646131663430333039313766 +66363063333563653332366331616531333332376433346465646163326633613866356337626535 +62343438373766316438303032623631316162323536653436386266386362646239666663326162 +33356633613462633338 diff --git a/ansible/roles/edge-host/README.md b/ansible/roles/edge-host/README.md index a7fc4f63..6c1c2478 100644 --- a/ansible/roles/edge-host/README.md +++ b/ansible/roles/edge-host/README.md @@ -54,8 +54,15 @@ kubectl -n home-assistant get secret zwave-js-session -o jsonpath='{.data.sessio vault_edge_sml2mqtt_mqtt_password: "..." vault_edge_open3e_mqtt_password: "..." vault_edge_zwavejs_session_secret: "..." +vault_edge_loki_password: "..." # dedicated edge-1 Loki push cred (see "Logging & eMMC") ``` +`vault_edge_loki_password` is a freshly-minted secret, not copied from the +cluster: a dedicated `edge-1` htpasswd user whose hash lives in the cluster +secret `kubernetes/apps/monitoring/loki-basic-auth.sops.yaml`. To rotate, +regenerate `openssl rand -hex 24`, update both the htpasswd line in that +sops secret and this vault key, then re-run the play. + The Z-Wave network keys do NOT need separate handling — they live in `settings.json` inside the store that gets migrated below. @@ -105,3 +112,31 @@ Afterwards: - can-utils (`candump`) is AUR-only on Arch — install manually via `pamac build can-utils` if needed for debugging; the role deliberately doesn't manage AUR packages. + +## Logging & eMMC + +edge-1 boots off a small **internal eMMC** (Dell Wyse 3040, ~7.3 GB), so the +role keeps logs off local flash and monitors wear: + +- **journald is volatile** (`Storage=volatile`, `journald-edge.conf`) — the + journal lives in `/run/log/journal` (RAM), capped at `edge_journald_runtime_max`. + The old persistent store `/var/log/journal` is purged on first apply. +- **Docker logs to journald** (`docker-daemon.json`, `log-driver: journald`) + instead of unbounded json files under `/var/lib/docker/containers`. Both + compose services inherit it; `docker logs` still works. +- **Grafana Alloy ships the journal to Loki** (`edge-alloy.service`). Its + WAL/positions sit on `/run/alloy` (tmpfs), so the agent writes nothing to + the eMMC either. Net effect: no log writes hit local flash — the trade-off + is that logs are RAM-only and lost on reboot unless already shipped. +- **Loki endpoint:** the unauthenticated srv-network Loki + (`loki.srv.renner.uno:3100`) is firewalled off the IoT VLAN, so Alloy pushes + to the authenticated cluster ingress `https://loki.renner.uno/loki/api/v1/push` + with basic-auth (`edge_loki_username` / `vault_edge_loki_password`). + **Delivery only works once the `edge-1` htpasswd line has reconciled into + the cluster (Flux) — before that Alloy gets 401 and buffers in RAM.** +- **Flash longevity:** `noatime` on root, weekly `fstrim.timer`, and a weekly + `edge-emmc-health.timer` that logs the JEDEC life-time / pre-EOL estimators + and warns (journal priority) once the flash leaves the healthy band. Check + by hand any time with `/usr/local/bin/edge-emmc-health` or + `cat /sys/block/mmcblk0/device/life_time` (`0x01 0x01` = <10 % consumed). + `mmc extcsd read /dev/mmcblk0` (from `mmc-utils`) gives the full detail. diff --git a/ansible/roles/edge-host/defaults/main.yml b/ansible/roles/edge-host/defaults/main.yml index c7a37863..9614956d 100644 --- a/ansible/roles/edge-host/defaults/main.yml +++ b/ansible/roles/edge-host/defaults/main.yml @@ -2,6 +2,26 @@ edge_base_dir: /opt/edge-host edge_tz: Europe/Berlin +# --- Logging & eMMC wear mitigation ------------------------------------ +# edge-1 runs on a small internal eMMC (Dell Wyse 3040, 7.3 GB), so logs are +# kept off local flash: containers + native services log to a RAM-only +# journal and Grafana Alloy ships it to Loki. TRIM, noatime and a weekly +# wear check round out the flash-longevity measures. See README "Logging & +# eMMC". +edge_journald_runtime_max: "128M" # cap the volatile (RAM) journal +edge_docker_log_driver: journald # container stdout -> journal, no json logs on eMMC + +# Grafana Alloy log shipper. The srv-network Loki (loki.srv.renner.uno:3100, +# no auth) is firewalled off the IoT VLAN; the only endpoint edge-1 can reach +# is the authenticated cluster ingress (Traefik basic-auth). +edge_alloy_config_file: /etc/alloy/edge.alloy +edge_alloy_http_port: 12345 +edge_loki_push_url: "https://loki.renner.uno/loki/api/v1/push" +edge_loki_username: edge-1 +edge_alloy_labels: + environment: production + datacenter: homelab + # --- MQTT: all services publish to the central in-cluster Mosquitto via # its LoadBalancer hostname (the edge host is outside the cluster). edge_mqtt_host: mqtt.renner.uno diff --git a/ansible/roles/edge-host/files/edge-emmc-health b/ansible/roles/edge-host/files/edge-emmc-health new file mode 100644 index 00000000..7ad8701f --- /dev/null +++ b/ansible/roles/edge-host/files/edge-emmc-health @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# edge-1 eMMC wear/health check. Managed by roles/edge-host. +# +# Reads the JEDEC eMMC 5.x life-time and pre-EOL estimators exposed via sysfs +# and emits a warning-priority journal message when the flash leaves the +# healthy band. Run weekly by edge-emmc-health.timer. No arguments; it +# auto-detects the eMMC backing the root filesystem. +set -euo pipefail + +root_src="$(findmnt -no SOURCE /)" +disk="$(lsblk -no PKNAME "$root_src" 2>/dev/null || true)" +[ -z "$disk" ] && disk="$(basename "$root_src")" +sys="/sys/block/${disk}/device" + +if [ ! -e "$sys/life_time" ]; then + echo "no eMMC life_time attribute for /dev/$disk (not an eMMC?) — nothing to check" + exit 0 +fi + +read -r lt_a lt_b < "$sys/life_time" +pre_eol="$(cat "$sys/pre_eol_info" 2>/dev/null || echo 0x00)" + +# JEDEC DEVICE_LIFE_TIME_EST_TYP_A/B: 0x01=0-10% consumed ... 0x0A=90-100%, +# 0x0B=exceeded. pre_eol: 0x01=normal, 0x02=warning (80% reserve used), +# 0x03=urgent (90% reserve used). +band() { + case "${1,,}" in + 0x01) echo "0-10%";; 0x02) echo "10-20%";; 0x03) echo "20-30%";; + 0x04) echo "30-40%";; 0x05) echo "40-50%";; 0x06) echo "50-60%";; + 0x07) echo "60-70%";; 0x08) echo "70-80%";; 0x09) echo "80-90%";; + 0x0a) echo "90-100%";; 0x0b) echo ">100% (exceeded)";; + *) echo "unknown ($1)";; + esac +} +eol_txt() { + case "${1,,}" in + 0x00) echo "not-defined";; + 0x01) echo "normal";; + 0x02) echo "WARNING (80% reserved blocks consumed)";; + 0x03) echo "URGENT (90% reserved blocks consumed)";; + *) echo "unknown ($1)";; + esac +} + +msg="eMMC /dev/$disk: life_time A=$(band "$lt_a") B=$(band "$lt_b"), pre_eol=$(eol_txt "$pre_eol")" +echo "$msg" + +# Healthy = both estimators in the first band and pre-EOL normal/undefined. +worn=0 +case "${lt_a,,}" in 0x01) ;; *) worn=1;; esac +case "${lt_b,,}" in 0x01) ;; *) worn=1;; esac +case "${pre_eol,,}" in 0x00|0x01) ;; *) worn=1;; esac + +if [ "$worn" -ne 0 ]; then + echo "$msg" | systemd-cat -t edge-emmc-health -p warning +fi diff --git a/ansible/roles/edge-host/files/edge-emmc-health.service b/ansible/roles/edge-host/files/edge-emmc-health.service new file mode 100644 index 00000000..c5bad0bb --- /dev/null +++ b/ansible/roles/edge-host/files/edge-emmc-health.service @@ -0,0 +1,7 @@ +[Unit] +Description=edge-1 eMMC wear/health check +Documentation=man:mmc(1) + +[Service] +Type=oneshot +ExecStart=/usr/local/bin/edge-emmc-health diff --git a/ansible/roles/edge-host/files/edge-emmc-health.timer b/ansible/roles/edge-host/files/edge-emmc-health.timer new file mode 100644 index 00000000..450db984 --- /dev/null +++ b/ansible/roles/edge-host/files/edge-emmc-health.timer @@ -0,0 +1,10 @@ +[Unit] +Description=Weekly edge-1 eMMC wear/health check + +[Timer] +OnCalendar=weekly +Persistent=true +RandomizedDelaySec=1h + +[Install] +WantedBy=timers.target diff --git a/ansible/roles/edge-host/handlers/main.yml b/ansible/roles/edge-host/handlers/main.yml index 228c7c46..7d79fb2a 100644 --- a/ansible/roles/edge-host/handlers/main.yml +++ b/ansible/roles/edge-host/handlers/main.yml @@ -6,6 +6,31 @@ cmd: networkctl reload && networkctl reconfigure {{ edge_can_interface }} changed_when: true +# Ordered before 'Restart edge compose services': the daemon must pick up the +# new default log driver before the containers are recreated onto it. +- name: Restart docker + ansible.builtin.service: + name: docker + state: restarted + +- name: Restart journald + ansible.builtin.service: + name: systemd-journald + state: restarted + +# Handlers run in definition order, so this fires after 'Restart journald': +# once the journal is volatile, the on-disk store is dead weight on the eMMC. +- name: Purge persistent journal store + ansible.builtin.file: + path: /var/log/journal + state: absent + +- name: Restart alloy + ansible.builtin.systemd: + name: edge-alloy + state: restarted + daemon_reload: true + - name: Restart edge compose services ansible.builtin.command: cmd: docker compose up -d --force-recreate diff --git a/ansible/roles/edge-host/tasks/main.yml b/ansible/roles/edge-host/tasks/main.yml index 81c19096..551a9b96 100644 --- a/ansible/roles/edge-host/tasks/main.yml +++ b/ansible/roles/edge-host/tasks/main.yml @@ -10,8 +10,133 @@ - python - python-packaging # ansible's pip module imports it on the target - git + - mmc-utils # eMMC lifetime/EOL introspection (mmc extcsd read) + - grafana-alloy # log shipper -> Loki (see Alloy section below) state: present +# ====================================================================== +# eMMC wear mitigation + log shipping. edge-1 (Dell Wyse 3040) runs on a +# small internal eMMC, so logs must not churn local flash. Strategy: +# - journald -> volatile (RAM), size-capped (journald-edge.conf) +# - docker -> journald log driver, no json files (docker-daemon.json) +# - Grafana Alloy tails the RAM journal -> Loki, WAL on tmpfs +# plus TRIM, noatime and a weekly wear check for flash longevity. +# See roles/edge-host/README.md "Logging & eMMC". +# ====================================================================== + +- name: Ensure journald drop-in directory exists + ansible.builtin.file: + path: /etc/systemd/journald.conf.d + state: directory + mode: "0755" + +- name: Switch the systemd journal to volatile (RAM) storage + ansible.builtin.template: + src: journald-edge.conf.j2 + dest: /etc/systemd/journald.conf.d/00-edge-emmc.conf + mode: "0644" + notify: + - Restart journald + - Purge persistent journal store + +- name: Resolve root filesystem facts (for noatime) + ansible.builtin.set_fact: + edge_root_fs: "{{ ansible_facts.mounts | selectattr('mount', 'equalto', '/') | first }}" + +- name: Refuse to touch fstab without a stable root UUID + ansible.builtin.assert: + that: + - edge_root_fs.uuid | default('') | length > 0 + fail_msg: "Could not determine root UUID from facts; not rewriting fstab." + +- name: Mount root with noatime (persisted in fstab) + ansible.posix.mount: + path: / + src: "UUID={{ edge_root_fs.uuid }}" + fstype: "{{ edge_root_fs.fstype }}" + opts: "{{ ((edge_root_fs.options.split(',') | reject('in', ['atime', 'noatime', 'relatime', 'diratime']) | list) + ['noatime']) | unique | join(',') }}" + dump: "0" + passno: "1" # root fs — keep boot-time fsck ordering + backup: true + state: mounted + +- name: Enable periodic TRIM (helps eMMC wear-leveling) + ansible.builtin.systemd: + name: fstrim.timer + enabled: true + state: started + +- name: Install eMMC health-check script + ansible.builtin.copy: + src: edge-emmc-health + dest: /usr/local/bin/edge-emmc-health + mode: "0755" + +- name: Install eMMC health-check service + timer + ansible.builtin.copy: + src: "{{ item }}" + dest: "/etc/systemd/system/{{ item }}" + mode: "0644" + loop: + - edge-emmc-health.service + - edge-emmc-health.timer + +- name: Enable eMMC health-check timer + ansible.builtin.systemd: + name: edge-emmc-health.timer + enabled: true + state: started + daemon_reload: true + +# --- Grafana Alloy: ship the volatile journal to Loki. Container logs reach +# the journal via docker's journald driver (configured below), native +# services log to the journal directly, so a single journal source covers +# everything. Storage/WAL sits on /run (tmpfs) — see edge-alloy.service. +- name: Create alloy system group + ansible.builtin.group: + name: alloy + system: true + +- name: Create alloy system user + ansible.builtin.user: + name: alloy + group: alloy + groups: systemd-journal + system: true + create_home: false + shell: /usr/bin/nologin + +- name: Create alloy config directory + ansible.builtin.file: + path: /etc/alloy + state: directory + owner: alloy + group: alloy + mode: "0750" + +- name: Deploy alloy config (journal -> Loki) + ansible.builtin.template: + src: edge-alloy.alloy.j2 + dest: "{{ edge_alloy_config_file }}" + owner: alloy + group: alloy + mode: "0600" # carries the Loki basic-auth password + notify: Restart alloy + +- name: Deploy alloy systemd unit + ansible.builtin.template: + src: edge-alloy.service.j2 + dest: /etc/systemd/system/edge-alloy.service + mode: "0644" + notify: Restart alloy + +- name: Enable edge-alloy + ansible.builtin.systemd: + name: edge-alloy + enabled: true + state: started + daemon_reload: true + # --- SocketCAN: systemd-networkd owns can0. gs_usb is an in-tree module # and autoloads when the Innomaker adapter enumerates; networkd then # matches the interface and applies bitrate + up. RestartSec recovers the @@ -29,8 +154,19 @@ state: started enabled: true -# No docker daemon.json needed: both services run with network_mode: host -# (real host addresses, v4+v6, no bridge/NAT) — see docker-compose.yml.j2. +# Docker networking needs no daemon.json (both services use network_mode: +# host — real addresses, v4+v6, no bridge/NAT). The daemon.json below exists +# only to route container stdout into the journald log driver so json logs +# never touch the eMMC; the journal itself is volatile (RAM) and shipped to +# Loki by edge-alloy. Both compose services inherit this default driver. +- name: Configure docker daemon (journald log driver, eMMC-friendly) + ansible.builtin.template: + src: docker-daemon.json.j2 + dest: /etc/docker/daemon.json + mode: "0644" + notify: + - Restart docker + - Restart edge compose services # --- Directory layout --- - name: Create service directories diff --git a/ansible/roles/edge-host/templates/docker-daemon.json.j2 b/ansible/roles/edge-host/templates/docker-daemon.json.j2 new file mode 100644 index 00000000..17fb574e --- /dev/null +++ b/ansible/roles/edge-host/templates/docker-daemon.json.j2 @@ -0,0 +1,3 @@ +{ + "log-driver": "{{ edge_docker_log_driver }}" +} diff --git a/ansible/roles/edge-host/templates/edge-alloy.alloy.j2 b/ansible/roles/edge-host/templates/edge-alloy.alloy.j2 new file mode 100644 index 00000000..ad3ea764 --- /dev/null +++ b/ansible/roles/edge-host/templates/edge-alloy.alloy.j2 @@ -0,0 +1,56 @@ +// Grafana Alloy — edge-1 log shipper. Managed by roles/edge-host. +// +// edge-1 runs on internal eMMC, so logs are kept out of local flash: the +// systemd journal is volatile (RAM, see journald-edge.conf) and container +// stdout goes to that same journal via the docker `journald` log driver +// (see docker-daemon.json). Alloy tails the journal and pushes straight to +// Loki. Its own WAL/positions live on /run (tmpfs, see edge-alloy.service), +// so this agent writes nothing to the eMMC either. +// +// The srv-network Loki (loki.srv.renner.uno:3100, unauthenticated) is not +// reachable from the IoT VLAN; the only reachable endpoint is the cluster +// ingress, which enforces Traefik basic-auth — hence the credential below. + +loki.relabel "journal" { + forward_to = [] + + rule { + source_labels = ["__journal__systemd_unit"] + target_label = "unit" + } + rule { + source_labels = ["__journal_priority_keyword"] + target_label = "level" + } + // docker's journald driver stamps CONTAINER_NAME on each entry, so + // container stdout stays attributable once it lands in the journal. + rule { + source_labels = ["__journal_container_name"] + target_label = "container" + } +} + +loki.source.journal "journal" { + forward_to = [loki.write.default.receiver] + relabel_rules = loki.relabel.journal.rules + labels = { + job = "systemd-journal", + host = "{{ inventory_hostname }}", + } +} + +loki.write "default" { + endpoint { + url = "{{ edge_loki_push_url }}" + basic_auth { + username = "{{ edge_loki_username }}" + password = "{{ edge_loki_password }}" + } + } + external_labels = { + host = "{{ inventory_hostname }}", +{% for k, v in edge_alloy_labels.items() %} + {{ k }} = "{{ v }}", +{% endfor %} + } +} diff --git a/ansible/roles/edge-host/templates/edge-alloy.service.j2 b/ansible/roles/edge-host/templates/edge-alloy.service.j2 new file mode 100644 index 00000000..830925b2 --- /dev/null +++ b/ansible/roles/edge-host/templates/edge-alloy.service.j2 @@ -0,0 +1,32 @@ +[Unit] +Description=Grafana Alloy (edge-1 log shipper -> Loki) +Documentation=https://grafana.com/docs/alloy/ +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=alloy +Group=alloy +# Read the (volatile) systemd journal. +SupplementaryGroups=systemd-journal +# WAL + positions on tmpfs so the agent never writes to the eMMC. %t is the +# runtime dir root (/run); RuntimeDirectory creates /run/alloy owned by us. +RuntimeDirectory=alloy +ExecStart=/usr/bin/alloy run \ + --server.http.listen-addr=127.0.0.1:{{ edge_alloy_http_port }} \ + --storage.path=%t/alloy \ + {{ edge_alloy_config_file }} +ExecReload=/bin/kill -HUP $MAINPID +Restart=on-failure +RestartSec=10s +LimitNOFILE=65536 + +# Hardening (bare metal, no LXC namespace limits here). +NoNewPrivileges=yes +ProtectSystem=strict +ProtectHome=yes +PrivateTmp=yes + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/edge-host/templates/journald-edge.conf.j2 b/ansible/roles/edge-host/templates/journald-edge.conf.j2 new file mode 100644 index 00000000..1a4e1edb --- /dev/null +++ b/ansible/roles/edge-host/templates/journald-edge.conf.j2 @@ -0,0 +1,7 @@ +# Managed by roles/edge-host. edge-1 boots off a small internal eMMC, so the +# journal must not persist to flash: keep it in RAM (/run/log/journal) and cap +# its size. Grafana Alloy (edge-alloy.service) ships this journal to Loki +# before it rotates out of RAM — nothing durable lives here. +[Journal] +Storage=volatile +RuntimeMaxUse={{ edge_journald_runtime_max }} diff --git a/kubernetes/apps/monitoring/loki-basic-auth.sops.yaml b/kubernetes/apps/monitoring/loki-basic-auth.sops.yaml index 85a0993d..cee0a7b7 100644 --- a/kubernetes/apps/monitoring/loki-basic-auth.sops.yaml +++ b/kubernetes/apps/monitoring/loki-basic-auth.sops.yaml @@ -5,7 +5,7 @@ metadata: namespace: monitoring type: Opaque stringData: - users: ENC[AES256_GCM,data:8vhe/wad6X9U58c9TZuYgxlS2ZXCEnXo5S3G7m0x6wO2DWpQi/syeqQ3G9eEBUErsQMEVX7q1YzVP6Mlxdozf0/IIGA=,iv:CigjvAZR3cUkjY+bRwaPTR4+4KeFtGYPYnbvHvRmbHU=,tag:9G3Ceucs9FjaVpRpCXepVQ==,type:str] + users: ENC[AES256_GCM,data:LFh7/+lhYoyAwELQaG2EfMBCipoJrG9/QCrhUGM+k5+mpS2L2DjHBNE7mei80+sqVgmPZoVd2tQgzI9icwZdQ94UiJttGeT1cRqv1AyJxGqgVZ7wrKET11nxCeaNtymLj8kfI3oaSiNZ3BEvCPFqN6ZtPAYVD6uVPkkRect+aQQGvmTLAUKY+A==,iv:XWuHPXyOsNpBjaLC2OZxYeNBPkr4hOMEZMMAgBgBDq0=,tag:ih7dS4ifa5OQQzJfLQGQIg==,type:str] sops: age: - recipient: age1yqjvqkwwwxdj7mr546zfs5ckzuwndp69ya2n2drd4vypwns8w4ys04qfy3 @@ -17,7 +17,7 @@ sops: WlhndVNoeFJkMGdhY05RbjFuY2J5Z1EKCzi11fwIkATwPUHvAjuvF0LN/jZDHXGA jGIVhngGk/61OKRm3+zuKoCsEHkdEuXRV+En15raeeWvUJIq9+Pv6g== -----END AGE ENCRYPTED FILE----- - lastmodified: "2026-05-17T14:59:41Z" - mac: ENC[AES256_GCM,data:FYz85BwOzBJYGwKKrG8aCqn4RUYwmmyaxlcml/Evlzr0LfGnQfjrMmG6L5lhM7UlS0bMmnwsDW7f/TDMmPEzdmG+/FBQ/ysvctD4JZy07Nd+maM6dS5MtLrgLM+4sRxXmyVJyWeZJQ7GspqNldKv09WfkOLuVPWkxDI/3qsTMSE=,iv:N9nJ/+kPK0IxbpUZf7Bnb+JF/vTGhybs+yVWlClil/k=,tag:SSIZWGLd2jBOHEgdRCHKsQ==,type:str] + lastmodified: "2026-07-07T06:40:10Z" + mac: ENC[AES256_GCM,data:n21Pljk9aWDoXtVFrWuLnNr6wiPWwfpx5VBPYIoyHnmTFMAd0kK8ztNK+yef9yULRChef1f0mfFK9pevR/oJ6W3RXlXd5Q4NhhvIOTCjpIrYSzdRg0Q8x9trXSLQPuVhh4MiDrOeRE1/ZnAgYqxHWQyXrpSEJm+4/TCy1u/xuFI=,iv:7GjRK1kJqpDtrBgLZH57DvdlhG0iixp+GN2OPZRpEu4=,tag:loWJMVcecEb4vxwBGLCGlQ==,type:str] encrypted_regex: ^(data|stringData)$ version: 3.10.2 -- 2.54.0