feat(cilium): export policy drops to the agent log

Hubble's static exporter writes every POLICY_DENIED drop event to the
cilium-agent stdout, which Alloy already ships to Loki (1 year
retention). That makes denied traffic reviewable after the fact instead
of only for the minutes the Hubble ring buffer holds.

Filter and field mask were checked against the running 1.20.4 agent
with hubble observe --allowlist / --field-mask. The cilium-config change
rolls the agents one at a time via the reloader annotation.

Refs S28-30

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

ansible/docs/edge-host.md

@@ -164,7 +164,9 @@ role keeps logs off local flash and monitors wear:
 - **Docker logs to journald** (`docker-daemon.json`, `log-driver: journald`)
   instead of unbounded json files under `/var/lib/docker/containers`. Both
   compose services inherit it; `docker logs` still works.
-- **Grafana Alloy ships the journal to Loki** (`edge-alloy.service`). Its
+- **Grafana Alloy ships the journal to Loki** (`edge-alloy.service`, which
+  runs the upstream `grafana/alloy` image with `/run/log/journal` mounted
+  read-only; Arch's `grafana-alloy` package has no `loki.source.journal`). Its
   WAL/positions sit on `/run/alloy` (tmpfs), so the agent writes nothing to
   the eMMC either. Net effect: no log writes hit local flash — the trade-off
   is that logs are RAM-only and lost on reboot unless already shipped.

ansible/inventory/pve_hosts.yml

@@ -49,20 +49,11 @@ edge-1.iot.renner.uno
 zwave_gateway
 heating_can
 
-# NTP servers. edge-1 was added here so the role would survive the PVE hosts
-# being retired for bare-metal Talos, which cannot run chrony or keepalived
-# itself. That retirement has now happened, so edge-1 is the ONLY member.
-#
-# THIS IS NO LONGER HIGHLY AVAILABLE. The group was jupiter (MASTER, prio 100),
-# saturn (90), neptune (80) and edge-1 (BACKUP, 70); the first three are gone.
-# A single VRRP member takes the address regardless of its configured state, so
-# the service address is still served, but by one thin client with
-# no failover behind it. install-ntp-ha.yml still carries the four-host
-# ntp_servers priority map, which now describes hosts that do not exist.
-#
-# Restoring redundancy needs a second chrony host that is not a Talos node.
-# See plays/install-ntp-ha.yml for why keepalived is scoped more narrowly than
-# chrony within this group.
+# NTP servers: every edge host (edge-1 BACKUP prio 70, edge-2 BACKUP prio 60;
+# see the ntp_servers map in plays/install-ntp-ha.yml). Talos nodes cannot run
+# chrony or keepalived. The jupiter/saturn/neptune entries in that map
+# describe retired PVE hosts. See plays/install-ntp-ha.yml for why keepalived
+# is scoped more narrowly than chrony within this group.
 [ntp_cluster:children]
 edge_host
 

ansible/plays/install-edge-host.yml

@@ -34,6 +34,7 @@
   roles:
     - certificates
     - common
+    - pacman-cache
     - edge-flash-wear
     - docker-install
     - edge-log-shipper

ansible/roles/edge-log-shipper/defaults/main.yml

@@ -7,7 +7,9 @@
 
 edge_alloy_config_file: /etc/alloy/edge.alloy
 edge_alloy_http_port: 12345
-edge_alloy_bin: /usr/bin/grafana-alloy   # Arch `grafana-alloy` pkg names the binary this, not `alloy`
+# Upstream image: Arch's grafana-alloy 1.21 is built without
+# loki.source.journal. Renovate-tracked (customManagers in renovate.json).
+edge_alloy_image: grafana/alloy:v1.21.3@sha256:8ab871206bd2d8f89277f53abffee5f452a293980788d52f85fa183a87a505a4
 edge_loki_push_url: "https://loki.renner.uno/loki/api/v1/push"
 edge_loki_username: edge-1
 edge_alloy_labels:

ansible/roles/edge-log-shipper/tasks/main.yml

@@ -6,15 +6,18 @@
 # directly, so one journal source covers everything. Alloy s storage/WAL sits
 # on /run (tmpfs) so it does not write to flash either.
 
-- name: Install grafana-alloy
+# Alloy runs from the upstream image (see edge-alloy.service); the Arch package
+# lacks journal support.
+- name: Remove the grafana-alloy package
   ansible.builtin.package:
     name: grafana-alloy
-    state: present
+    state: absent
 
 - name: Create alloy system group
   ansible.builtin.group:
     name: alloy
     system: true
+  register: edge_alloy_group
 
 - name: Create alloy system user
   ansible.builtin.user:
@@ -24,6 +27,25 @@
     system: true
     create_home: false
     shell: /usr/bin/nologin
+  register: edge_alloy_user
+
+- name: Look up the systemd-journal group
+  ansible.builtin.getent:
+    database: group
+    key: systemd-journal
+
+- name: Set the container identity
+  ansible.builtin.set_fact:
+    edge_alloy_uid: "{{ edge_alloy_user.uid }}"
+    edge_alloy_gid: "{{ edge_alloy_group.gid }}"
+    edge_alloy_journal_gid: "{{ ansible_facts.getent_group['systemd-journal'][1] }}"
+
+- name: Pull the alloy image
+  ansible.builtin.command:
+    argv: [docker, pull, "{{ edge_alloy_image }}"]
+  register: edge_alloy_pull
+  changed_when: "'Downloaded newer image' in edge_alloy_pull.stdout"
+  notify: Restart alloy
 
 - name: Create alloy config directory
   ansible.builtin.file:

ansible/roles/edge-log-shipper/templates/edge-alloy.service.j2

@@ -1,32 +1,35 @@
 [Unit]
-Description=Grafana Alloy (edge-1 log shipper -> Loki)
+Description=Grafana Alloy (journal -> Loki)
 Documentation=https://grafana.com/docs/alloy/
-After=network-online.target
+After=docker.service network-online.target
+Requires=docker.service
 Wants=network-online.target
 
 [Service]
 Type=simple
-User=alloy
-Group=alloy
-# Read the (volatile) systemd journal.
-SupplementaryGroups=systemd-journal
-# WAL + positions on tmpfs so the agent never writes to the eMMC. %t is the
-# runtime dir root (/run); RuntimeDirectory creates /run/alloy owned by us.
-RuntimeDirectory=alloy
-ExecStart={{ edge_alloy_bin }} run \
+# WAL + positions on tmpfs so the agent never writes to the eMMC.
+ExecStartPre=/usr/bin/install -d -o alloy -g alloy -m 0750 /run/alloy
+ExecStartPre=-/usr/bin/docker rm -f edge-alloy
+# Attached run with --log-driver none: the container's output reaches the
+# journal once, through this unit.
+ExecStart=/usr/bin/docker run --rm --name edge-alloy \
+  --log-driver none \
+  --network host \
+  --user {{ edge_alloy_uid }}:{{ edge_alloy_gid }} \
+  --group-add {{ edge_alloy_journal_gid }} \
+  --read-only --tmpfs /tmp \
+  --cap-drop ALL --security-opt no-new-privileges \
+  -v /run/log/journal:/run/log/journal:ro \
+  -v /etc/machine-id:/etc/machine-id:ro \
+  -v {{ edge_alloy_config_file }}:/etc/alloy/config.alloy:ro \
+  -v /run/alloy:/var/lib/alloy \
+  {{ edge_alloy_image }} run \
   --server.http.listen-addr=127.0.0.1:{{ edge_alloy_http_port }} \
-  --storage.path=%t/alloy \
-  {{ edge_alloy_config_file }}
-ExecReload=/bin/kill -HUP $MAINPID
-Restart=on-failure
+  --storage.path=/var/lib/alloy \
+  /etc/alloy/config.alloy
+ExecStop=/usr/bin/docker stop edge-alloy
+Restart=always
 RestartSec=10s
-LimitNOFILE=65536
-
-# Hardening (bare metal, no LXC namespace limits here).
-NoNewPrivileges=yes
-ProtectSystem=strict
-ProtectHome=yes
-PrivateTmp=yes
 
 [Install]
 WantedBy=multi-user.target

ansible/roles/pacman-cache/files/pacman-cache.hook

@@ -0,0 +1,14 @@
+# Managed by ansible (roles/pacman-cache).
+# Downloaded packages are not kept; older versions come from the Arch Linux
+# Archive.
+[Trigger]
+Operation = Install
+Operation = Upgrade
+Operation = Remove
+Type = Package
+Target = *
+
+[Action]
+Description = Removing downloaded packages from the cache...
+When = PostTransaction
+Exec = /usr/bin/find /var/cache/pacman/pkg -mindepth 1 -delete

ansible/roles/pacman-cache/tasks/main.yml

@@ -0,0 +1,21 @@
+---
+# Keep the pacman package cache empty. pacman keeps every downloaded package;
+# on a 7 GB eMMC root the cache grows to gigabytes. Arch hosts only.
+
+- name: Ensure the pacman hook directory exists
+  ansible.builtin.file:
+    path: /etc/pacman.d/hooks
+    state: directory
+    mode: "0755"
+
+- name: Empty the package cache after every transaction
+  ansible.builtin.copy:
+    src: pacman-cache.hook
+    dest: /etc/pacman.d/hooks/pacman-cache.hook
+    mode: "0644"
+
+- name: Empty the package cache
+  ansible.builtin.command:
+    argv: [find, /var/cache/pacman/pkg, -mindepth, "1", -delete, -print]
+  register: pacman_cache_clean
+  changed_when: pacman_cache_clean.stdout | length > 0

kubernetes/infrastructure/cilium/helmrelease.yaml

@@ -156,13 +156,39 @@ spec:
     k8sServicePort: 7445
     # Hubble itself is on in every agent (chart default); the relay aggregates
     # the three nodes' flow buffers behind one gRPC endpoint so `hubble observe`
-    # works without exec into kube-system (REN-487). Renders no change to
-    # cilium-config or the agent DaemonSet, so the reloader annotation above
-    # does not roll the CNI. The relay has no client auth; who may reach it is
-    # network-policies/kube-system-hubble-relay.yaml.
+    # works without exec into kube-system (REN-487). The relay has no client
+    # auth; who may reach it is network-policies/kube-system-hubble-relay.yaml.
     hubble:
       relay:
         enabled: true
+      # Every policy drop goes to the agent's stdout and from there to Loki
+      # ({namespace="kube-system", container="cilium-agent"} |= "POLICY_DENIED"),
+      # so denied traffic can be reviewed after the fact. Changes cilium-config,
+      # so the reloader rolls the agents.
+      export:
+        static:
+          enabled: true
+          filePath: stdout
+          allowList:
+            - '{"verdict":["DROPPED"],"drop_reason_desc":["POLICY_DENIED"],"event_type":[{"type":1}]}'
+          fieldMask:
+            - time
+            - node_name
+            - verdict
+            - drop_reason_desc
+            - traffic_direction
+            - IP
+            - l4
+            - source.namespace
+            - source.pod_name
+            - source.identity
+            - source.workloads
+            - source_names
+            - destination.namespace
+            - destination.pod_name
+            - destination.identity
+            - destination.workloads
+            - destination_names
     l2announcements:
       enabled: true
     externalIPs:

kubernetes/infrastructure/network-policies/cidr-groups.yaml

@@ -0,0 +1,150 @@
+# Client networks as Cilium CIDR groups, referenced from CNPs via
+# fromCIDRSet.cidrGroupRef. ULA + IPv4 only: the ISP-delegated GUA prefixes
+# change and are not listed.
+---
+apiVersion: cilium.io/v2
+kind: CiliumCIDRGroup
+metadata:
+  name: net-default
+spec:
+  externalCIDRs:
+    - fd00::/64
+    - 192.168.8.0/24
+---
+apiVersion: cilium.io/v2
+kind: CiliumCIDRGroup
+metadata:
+  name: net-management
+spec:
+  externalCIDRs:
+    - fd02:10::/64
+    - 10.2.10.0/24
+---
+apiVersion: cilium.io/v2
+kind: CiliumCIDRGroup
+metadata:
+  name: net-radius
+spec:
+  externalCIDRs:
+    - fd02:11::/64
+    - 10.2.11.0/24
+---
+apiVersion: cilium.io/v2
+kind: CiliumCIDRGroup
+metadata:
+  name: net-video
+spec:
+  externalCIDRs:
+    - 10.2.20.0/24
+---
+apiVersion: cilium.io/v2
+kind: CiliumCIDRGroup
+metadata:
+  name: net-iot
+spec:
+  externalCIDRs:
+    - fd02:30::/64
+    - 10.2.30.0/24
+---
+apiVersion: cilium.io/v2
+kind: CiliumCIDRGroup
+metadata:
+  name: net-server
+spec:
+  externalCIDRs:
+    - fd02:40::/64
+    - fd40::/64
+    - 10.2.40.0/24
+---
+apiVersion: cilium.io/v2
+kind: CiliumCIDRGroup
+metadata:
+  name: net-k8s-node
+spec:
+  externalCIDRs:
+    - fd02:46::/64
+    - 10.2.46.0/24
+---
+apiVersion: cilium.io/v2
+kind: CiliumCIDRGroup
+metadata:
+  name: net-k8s-ingress
+spec:
+  externalCIDRs:
+    - fd02:47::/64
+    - 10.2.47.0/24
+---
+apiVersion: cilium.io/v2
+kind: CiliumCIDRGroup
+metadata:
+  name: net-k8s-pod
+spec:
+  externalCIDRs:
+    - fd02:48::/64
+    - 10.2.48.0/24
+---
+apiVersion: cilium.io/v2
+kind: CiliumCIDRGroup
+metadata:
+  name: net-privat
+spec:
+  externalCIDRs:
+    - fd02:50::/64
+    - 10.2.50.0/24
+---
+apiVersion: cilium.io/v2
+kind: CiliumCIDRGroup
+metadata:
+  name: net-phone
+spec:
+  externalCIDRs:
+    - 10.2.60.0/24
+---
+apiVersion: cilium.io/v2
+kind: CiliumCIDRGroup
+metadata:
+  name: net-access
+spec:
+  externalCIDRs:
+    - 10.2.70.0/24
+---
+apiVersion: cilium.io/v2
+kind: CiliumCIDRGroup
+metadata:
+  name: net-vpn
+spec:
+  externalCIDRs:
+    - 10.2.80.0/24
+    - 192.168.32.0/24
+---
+apiVersion: cilium.io/v2
+kind: CiliumCIDRGroup
+metadata:
+  name: net-guest
+spec:
+  externalCIDRs:
+    - 10.2.90.0/24
+---
+apiVersion: cilium.io/v2
+kind: CiliumCIDRGroup
+metadata:
+  name: net-multi-transfer
+spec:
+  externalCIDRs:
+    - 10.2.122.0/24
+---
+apiVersion: cilium.io/v2
+kind: CiliumCIDRGroup
+metadata:
+  name: net-multi-lan
+spec:
+  externalCIDRs:
+    - 10.2.123.0/24
+---
+apiVersion: cilium.io/v2
+kind: CiliumCIDRGroup
+metadata:
+  name: net-mutti
+spec:
+  externalCIDRs:
+    - 10.2.200.0/24

kubernetes/infrastructure/network-policies/kustomization.yaml

@@ -1,6 +1,7 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
+  - cidr-groups.yaml
   - default-deny.yaml
   - allow-ingress.yaml
   - ingress-isolation.yaml

kubernetes/infrastructure/network-policies/trmnl-ns.yaml

@@ -62,9 +62,10 @@ spec:
 # CA, so this bypasses Traefik entirely and the namespace default-deny has
 # to be opened for it — same shape as paperless's allow-ftp-scanner.
 #
-# Scoped to the terminus pods and port 2345 only. Requests still have to
-# carry a known device MAC in the `ID` header plus its api_key; this rule
-# opens the port, not the application.
+# Scoped to the terminus pods, port 2345 and the IoT network (the panel's
+# network; the LB is ETP=Local, so the client address reaches the pod).
+# Requests still have to carry a known device MAC in the `ID` header plus
+# its api_key; this rule opens the port, not the application.
 apiVersion: cilium.io/v2
 kind: CiliumNetworkPolicy
 metadata:
@@ -75,8 +76,13 @@ spec:
     matchLabels:
       app.kubernetes.io/name: terminus
   ingress:
+    - fromCIDRSet:
+        - cidrGroupRef: net-iot
+      toPorts:
+        - ports:
+            - port: "2345"
+              protocol: TCP
     - fromEntities:
-        - world
         - host
         - remote-node
       toPorts:

opentofu/unifi_nat_hairpin.tf

@@ -68,13 +68,14 @@ locals {
   # idx is a static per-network slot for rule_index derivation — never
   # reuse a freed idx for a new network.
   #
-  # Deliberately absent: (40) Server idx=4, (42) Ingress idx=5, and
-  # (49) k8s Ceph idx=6. The cluster nodes have interfaces on those VLANs,
-  # so a DNATed flow's reply would go directly over the shared L2 and
-  # bypass the UDM's conntrack (asymmetric, connection drops) — and
-  # clients there are the nodes themselves, whose DNS chain (CoreDNS ->
-  # k8s_gateway) already resolves the public names in-cluster. Their idx
-  # slots stay reserved.
+  # Deliberately absent: (40) Server idx=4, (49) k8s Ceph idx=6,
+  # (46) k8s Node, (47) k8s Ingress and (48) k8s Pod. The cluster nodes
+  # have interfaces on those VLANs (mgmt0, node0, ingress0, pod0), so a
+  # DNATed flow's reply would go directly over the shared L2 and bypass
+  # the UDM's conntrack (asymmetric, connection drops) — and clients there
+  # are the nodes themselves, whose DNS chain (CoreDNS -> k8s_gateway)
+  # already resolves the public names in-cluster. idx=5 stays reserved
+  # (its network, (42) Ingress, no longer exists).
   hairpin_networks = {
     "Default"              = { id = "5d4fba77ce63ee614841d53d", idx = 0 }
     "(10) Management"      = { id = "600334abf9ee0c059da6be8f", idx = 1 }