fix(time-sync): keep the internet VPS hosts on NTP

common and plays/configure-vm-time-sync.yml masked systemd-timesyncd on
every KVM guest, assuming a Proxmox host supplies the time. The Proxmox
VMs are gone; the only KVM guests left are the Hetzner/netcup VPS hosts,
which have no such host. netbird.renner.uno ran with no NTP client since
provisioning, drifted behind authentik, and NetBird management rejected
SSO tokens with "token used before issued".

Drop both masking paths and add a timesyncd role to the netbird server
and proxy plays; it unmasks, enables and starts systemd-timesyncd and
waits for NTPSynchronized=yes.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

ansible/inventory/pve_hosts.yml

@@ -49,20 +49,11 @@ edge-1.iot.renner.uno
 zwave_gateway
 heating_can
 
-# NTP servers. edge-1 was added here so the role would survive the PVE hosts
-# being retired for bare-metal Talos, which cannot run chrony or keepalived
-# itself. That retirement has now happened, so edge-1 is the ONLY member.
-#
-# THIS IS NO LONGER HIGHLY AVAILABLE. The group was jupiter (MASTER, prio 100),
-# saturn (90), neptune (80) and edge-1 (BACKUP, 70); the first three are gone.
-# A single VRRP member takes the address regardless of its configured state, so
-# the service address is still served, but by one thin client with
-# no failover behind it. install-ntp-ha.yml still carries the four-host
-# ntp_servers priority map, which now describes hosts that do not exist.
-#
-# Restoring redundancy needs a second chrony host that is not a Talos node.
-# See plays/install-ntp-ha.yml for why keepalived is scoped more narrowly than
-# chrony within this group.
+# NTP servers: every edge host (edge-1 BACKUP prio 70, edge-2 BACKUP prio 60;
+# see the ntp_servers map in plays/install-ntp-ha.yml). Talos nodes cannot run
+# chrony or keepalived. The jupiter/saturn/neptune entries in that map
+# describe retired PVE hosts. See plays/install-ntp-ha.yml for why keepalived
+# is scoped more narrowly than chrony within this group.
 [ntp_cluster:children]
 edge_host
 

ansible/plays/configure-vm-time-sync.yml

@@ -1,70 +0,0 @@
----
-# Configure VMs to use host time sync via kvm-clock
-# Disables systemd-timesyncd to prevent NTP conflicts
-#
-# VMs get time from Proxmox host via kvm-clock paravirtualized clock
-# This is more accurate and reliable than network NTP for VMs
-
-- name: Configure VM time synchronization from host
-  hosts: all
-  become: yes
-  gather_facts: yes
-
-  tasks:
-    - name: Check if running as VM (not LXC)
-      stat:
-        path: /sys/devices/system/clocksource/clocksource0/current_clocksource
-      register: clocksource_file
-
-    - name: Get current clock source
-      slurp:
-        src: /sys/devices/system/clocksource/clocksource0/current_clocksource
-      register: clocksource
-      when: clocksource_file.stat.exists
-
-    - name: Display clock source
-      debug:
-        msg: "Clock source: {{ (clocksource.content | b64decode | trim) if clocksource_file.stat.exists else 'N/A (LXC)' }}"
-
-    - name: Check if systemd-timesyncd is installed
-      command: systemctl list-unit-files systemd-timesyncd.service
-      register: timesyncd_installed
-      check_mode: false
-      failed_when: false
-      changed_when: false
-
-    - name: Stop and disable systemd-timesyncd on VMs using kvm-clock
-      systemd:
-        name: systemd-timesyncd
-        state: stopped
-        enabled: no
-      when:
-        - clocksource_file.stat.exists
-        - clocksource.content is defined
-        - "'kvm-clock' in (clocksource.content | b64decode)"
-        - "'systemd-timesyncd.service' in timesyncd_installed.stdout"
-
-    - name: Mask systemd-timesyncd to prevent re-enabling
-      systemd:
-        name: systemd-timesyncd
-        masked: yes
-      when:
-        - clocksource_file.stat.exists
-        - clocksource.content is defined
-        - "'kvm-clock' in (clocksource.content | b64decode)"
-        - "'systemd-timesyncd.service' in timesyncd_installed.stdout"
-
-    - name: Verify time sync status
-      command: timedatectl show --property=NTPSynchronized
-      register: timesync_status
-      check_mode: false
-      changed_when: false
-      failed_when: false
-
-    - name: Display final status
-      debug:
-        msg: |
-          Host: {{ inventory_hostname }}
-          Type: {{ 'VM (kvm-clock)' if (clocksource_file.stat.exists and 'kvm-clock' in (clocksource.content | b64decode | default(''))) else 'LXC/Other' }}
-          Time sync: {{ timesync_status.stdout | default('unknown') }}
-          Note: VMs sync time from Proxmox host, LXCs inherit host time automatically

ansible/plays/install-edge-host.yml

@@ -34,6 +34,7 @@
   roles:
     - certificates
     - common
+    - pacman-cache
     - edge-flash-wear
     - docker-install
     - edge-log-shipper

ansible/plays/install-netbird-proxy.yml

@@ -22,6 +22,7 @@
   become: true
   roles:
     - common
+    - timesyncd
     - docker-install
     - netbird-proxy
     - netbird-proxy-watchdog

ansible/plays/install-netbird-server.yml

@@ -16,5 +16,6 @@
   become: true
   roles:
     - common
+    - timesyncd
     - docker-install
     - netbird-server

ansible/roles/common/tasks/main.yml

@@ -54,18 +54,9 @@
 # via plays/configure-ssh-ca.yml.
 
 # -------------------------------------------------------
-# 5. Disable NTP on VMs/LXC (use host time)
+# 5. Time sync
 # -------------------------------------------------------
-- name: Stop and disable systemd-timesyncd on VMs/LXC
-  systemd:
-    name: systemd-timesyncd
-    state: stopped
-    enabled: no
-    masked: yes
-  when:
-    - ansible_facts['virtualization_role'] == 'guest'
-    - ansible_facts['virtualization_type'] in ['kvm', 'lxc']
-  ignore_errors: yes
+# Moved to separate roles: timesyncd (internet VPS), chrony-ntp-server (edge).
 
 # -------------------------------------------------------
 # 6. Certificates

ansible/roles/pacman-cache/files/pacman-cache.hook

@@ -0,0 +1,14 @@
+# Managed by ansible (roles/pacman-cache).
+# Downloaded packages are not kept; older versions come from the Arch Linux
+# Archive.
+[Trigger]
+Operation = Install
+Operation = Upgrade
+Operation = Remove
+Type = Package
+Target = *
+
+[Action]
+Description = Removing downloaded packages from the cache...
+When = PostTransaction
+Exec = /usr/bin/find /var/cache/pacman/pkg -mindepth 1 -delete

ansible/roles/pacman-cache/tasks/main.yml

@@ -0,0 +1,21 @@
+---
+# Keep the pacman package cache empty. pacman keeps every downloaded package;
+# on a 7 GB eMMC root the cache grows to gigabytes. Arch hosts only.
+
+- name: Ensure the pacman hook directory exists
+  ansible.builtin.file:
+    path: /etc/pacman.d/hooks
+    state: directory
+    mode: "0755"
+
+- name: Empty the package cache after every transaction
+  ansible.builtin.copy:
+    src: pacman-cache.hook
+    dest: /etc/pacman.d/hooks/pacman-cache.hook
+    mode: "0644"
+
+- name: Empty the package cache
+  ansible.builtin.command:
+    argv: [find, /var/cache/pacman/pkg, -mindepth, "1", -delete, -print]
+  register: pacman_cache_clean
+  changed_when: pacman_cache_clean.stdout | length > 0

ansible/roles/timesyncd/tasks/main.yml

@@ -0,0 +1,26 @@
+---
+# Keep the system clock on NTP with systemd-timesyncd, on hosts that run no
+# chrony. A KVM guest's clocksource does not steer wall-clock time; without an
+# NTP client the clock drifts until JWT iat/exp checks fail.
+
+- name: Install systemd-timesyncd
+  ansible.builtin.apt:
+    name: systemd-timesyncd
+    state: present
+  when: ansible_facts['os_family'] == 'Debian'
+
+- name: Run systemd-timesyncd
+  ansible.builtin.systemd_service:
+    name: systemd-timesyncd
+    masked: false
+    enabled: true
+    state: started
+
+- name: Wait for the clock to synchronize
+  ansible.builtin.command:
+    argv: [timedatectl, show, --property=NTPSynchronized, --value]
+  register: timesyncd_synchronized
+  changed_when: false
+  until: timesyncd_synchronized.stdout == "yes"
+  retries: 12
+  delay: 5

ansible/site.yml

@@ -12,8 +12,6 @@
   ansible.builtin.import_playbook: plays/configure-synology-python.yml
 - name: Base system
   ansible.builtin.import_playbook: plays/common.yml
-- name: VM time sync
-  ansible.builtin.import_playbook: plays/configure-vm-time-sync.yml
 
 - name: SSH user CA
   ansible.builtin.import_playbook: plays/configure-ssh-ca.yml

kubernetes/infrastructure/cilium/helmrelease.yaml

@@ -156,13 +156,39 @@ spec:
     k8sServicePort: 7445
     # Hubble itself is on in every agent (chart default); the relay aggregates
     # the three nodes' flow buffers behind one gRPC endpoint so `hubble observe`
-    # works without exec into kube-system (REN-487). Renders no change to
-    # cilium-config or the agent DaemonSet, so the reloader annotation above
-    # does not roll the CNI. The relay has no client auth; who may reach it is
-    # network-policies/kube-system-hubble-relay.yaml.
+    # works without exec into kube-system (REN-487). The relay has no client
+    # auth; who may reach it is network-policies/kube-system-hubble-relay.yaml.
     hubble:
       relay:
         enabled: true
+      # Every policy drop goes to the agent's stdout and from there to Loki
+      # ({namespace="kube-system", container="cilium-agent"} |= "POLICY_DENIED"),
+      # so denied traffic can be reviewed after the fact. Changes cilium-config,
+      # so the reloader rolls the agents.
+      export:
+        static:
+          enabled: true
+          filePath: stdout
+          allowList:
+            - '{"verdict":["DROPPED"],"drop_reason_desc":["POLICY_DENIED"],"event_type":[{"type":1}]}'
+          fieldMask:
+            - time
+            - node_name
+            - verdict
+            - drop_reason_desc
+            - traffic_direction
+            - IP
+            - l4
+            - source.namespace
+            - source.pod_name
+            - source.identity
+            - source.workloads
+            - source_names
+            - destination.namespace
+            - destination.pod_name
+            - destination.identity
+            - destination.workloads
+            - destination_names
     l2announcements:
       enabled: true
     externalIPs:

kubernetes/infrastructure/network-policies/cidr-groups.yaml

@@ -0,0 +1,150 @@
+# Client networks as Cilium CIDR groups, referenced from CNPs via
+# fromCIDRSet.cidrGroupRef. ULA + IPv4 only: the ISP-delegated GUA prefixes
+# change and are not listed.
+---
+apiVersion: cilium.io/v2
+kind: CiliumCIDRGroup
+metadata:
+  name: net-default
+spec:
+  externalCIDRs:
+    - fd00::/64
+    - 192.168.8.0/24
+---
+apiVersion: cilium.io/v2
+kind: CiliumCIDRGroup
+metadata:
+  name: net-management
+spec:
+  externalCIDRs:
+    - fd02:10::/64
+    - 10.2.10.0/24
+---
+apiVersion: cilium.io/v2
+kind: CiliumCIDRGroup
+metadata:
+  name: net-radius
+spec:
+  externalCIDRs:
+    - fd02:11::/64
+    - 10.2.11.0/24
+---
+apiVersion: cilium.io/v2
+kind: CiliumCIDRGroup
+metadata:
+  name: net-video
+spec:
+  externalCIDRs:
+    - 10.2.20.0/24
+---
+apiVersion: cilium.io/v2
+kind: CiliumCIDRGroup
+metadata:
+  name: net-iot
+spec:
+  externalCIDRs:
+    - fd02:30::/64
+    - 10.2.30.0/24
+---
+apiVersion: cilium.io/v2
+kind: CiliumCIDRGroup
+metadata:
+  name: net-server
+spec:
+  externalCIDRs:
+    - fd02:40::/64
+    - fd40::/64
+    - 10.2.40.0/24
+---
+apiVersion: cilium.io/v2
+kind: CiliumCIDRGroup
+metadata:
+  name: net-k8s-node
+spec:
+  externalCIDRs:
+    - fd02:46::/64
+    - 10.2.46.0/24
+---
+apiVersion: cilium.io/v2
+kind: CiliumCIDRGroup
+metadata:
+  name: net-k8s-ingress
+spec:
+  externalCIDRs:
+    - fd02:47::/64
+    - 10.2.47.0/24
+---
+apiVersion: cilium.io/v2
+kind: CiliumCIDRGroup
+metadata:
+  name: net-k8s-pod
+spec:
+  externalCIDRs:
+    - fd02:48::/64
+    - 10.2.48.0/24
+---
+apiVersion: cilium.io/v2
+kind: CiliumCIDRGroup
+metadata:
+  name: net-privat
+spec:
+  externalCIDRs:
+    - fd02:50::/64
+    - 10.2.50.0/24
+---
+apiVersion: cilium.io/v2
+kind: CiliumCIDRGroup
+metadata:
+  name: net-phone
+spec:
+  externalCIDRs:
+    - 10.2.60.0/24
+---
+apiVersion: cilium.io/v2
+kind: CiliumCIDRGroup
+metadata:
+  name: net-access
+spec:
+  externalCIDRs:
+    - 10.2.70.0/24
+---
+apiVersion: cilium.io/v2
+kind: CiliumCIDRGroup
+metadata:
+  name: net-vpn
+spec:
+  externalCIDRs:
+    - 10.2.80.0/24
+    - 192.168.32.0/24
+---
+apiVersion: cilium.io/v2
+kind: CiliumCIDRGroup
+metadata:
+  name: net-guest
+spec:
+  externalCIDRs:
+    - 10.2.90.0/24
+---
+apiVersion: cilium.io/v2
+kind: CiliumCIDRGroup
+metadata:
+  name: net-multi-transfer
+spec:
+  externalCIDRs:
+    - 10.2.122.0/24
+---
+apiVersion: cilium.io/v2
+kind: CiliumCIDRGroup
+metadata:
+  name: net-multi-lan
+spec:
+  externalCIDRs:
+    - 10.2.123.0/24
+---
+apiVersion: cilium.io/v2
+kind: CiliumCIDRGroup
+metadata:
+  name: net-mutti
+spec:
+  externalCIDRs:
+    - 10.2.200.0/24

kubernetes/infrastructure/network-policies/home-assistant-ns.yaml

@@ -186,8 +186,8 @@ spec:
     matchLabels:
       app.kubernetes.io/name: mosquitto
   ingress:
-    - fromEntities:
-        - world
+    - fromCIDRSet:
+        - cidrGroupRef: net-iot
       toPorts:
         - ports:
             - port: "1883"
@@ -205,8 +205,11 @@ spec:
     matchLabels:
       app.kubernetes.io/name: home-assistant
   ingress:
-    - fromEntities:
-        - world
+    - fromCIDRSet:
+        - cidrGroupRef: net-privat
+        - cidrGroupRef: net-vpn
+        - cidrGroupRef: net-iot
+        - cidrGroupRef: net-server
       toPorts:
         - ports:
             - port: "8123"
@@ -225,8 +228,8 @@ spec:
     matchLabels:
       app.kubernetes.io/name: shelly-manager
   ingress:
-    - fromEntities:
-        - world
+    - fromCIDRSet:
+        - cidrGroupRef: net-iot
       toPorts:
         - ports:
             - port: "8082"

kubernetes/infrastructure/network-policies/kube-system-coredns.yaml

@@ -0,0 +1,40 @@
+# CoreDNS is reachable from the LAN through the kube-dns-lb LoadBalancer
+# (ETP=Local, client address reaches the pod). Selecting the pods makes
+# them default-deny for ingress, so the whole cluster (pods, nodes, kubelet
+# probes, Prometheus) is allowed on every port; LAN clients only on 53.
+apiVersion: cilium.io/v2
+kind: CiliumNetworkPolicy
+metadata:
+  name: coredns-ingress
+  namespace: kube-system
+spec:
+  endpointSelector:
+    matchLabels:
+      k8s-app: kube-dns
+  ingress:
+    - fromEntities:
+        - cluster
+    - fromCIDRSet:
+        - cidrGroupRef: net-default
+        - cidrGroupRef: net-management
+        - cidrGroupRef: net-radius
+        - cidrGroupRef: net-video
+        - cidrGroupRef: net-iot
+        - cidrGroupRef: net-server
+        - cidrGroupRef: net-k8s-node
+        - cidrGroupRef: net-k8s-ingress
+        - cidrGroupRef: net-k8s-pod
+        - cidrGroupRef: net-privat
+        - cidrGroupRef: net-phone
+        - cidrGroupRef: net-access
+        - cidrGroupRef: net-vpn
+        - cidrGroupRef: net-guest
+        - cidrGroupRef: net-multi-transfer
+        - cidrGroupRef: net-multi-lan
+        - cidrGroupRef: net-mutti
+      toPorts:
+        - ports:
+            - port: "53"
+              protocol: UDP
+            - port: "53"
+              protocol: TCP

kubernetes/infrastructure/network-policies/kustomization.yaml

@@ -1,6 +1,7 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
+  - cidr-groups.yaml
   - default-deny.yaml
   - allow-ingress.yaml
   - ingress-isolation.yaml
@@ -40,3 +41,4 @@ resources:
   - allow-cilium-health.yaml
   - cnpg-barman-s3.yaml
   - kube-system-hubble-relay.yaml
+  - kube-system-coredns.yaml

kubernetes/infrastructure/network-policies/monitoring-alloy-syslog.yaml

@@ -1,13 +1,6 @@
-# Targeted carve-out: alloy-syslog needs world ingress on the syslog
-# ports specifically. The Service is ETP=Cluster, so external syslog
-# senders (Proxmox, LXC hosts, network gear) hit the LB IP, get DNAT'd
-# to an alloy pod, and get SNAT'd to the receiving node's cilium_host
-# address. Cilium then sees the source as `world-ipv4` identity (the
-# cilium_host IP isn't a known endpoint), NOT `host` — so the broad
-# monitoring CNP's `fromEntities: host` rule doesn't catch it.
-#
-# Scoped via endpointSelector so this only widens ingress on the
-# alloy-syslog pod, not the whole namespace.
+# Syslog senders reaching the alloy-syslog LB (ETP=Local, client address
+# reaches the pod): network gear on Management, the SLZB coordinators on
+# IoT, and the UDM itself, which sources from its k8s-ingress address.
 apiVersion: cilium.io/v2
 kind: CiliumNetworkPolicy
 metadata:
@@ -19,8 +12,10 @@ spec:
       app.kubernetes.io/instance: alloy-syslog
       app.kubernetes.io/name: alloy
   ingress:
-    - fromEntities:
-        - world
+    - fromCIDRSet:
+        - cidrGroupRef: net-management
+        - cidrGroupRef: net-iot
+        - cidrGroupRef: net-k8s-ingress
       toPorts:
         - ports:
             - port: "514"

kubernetes/infrastructure/network-policies/paperless-ns.yaml

@@ -112,7 +112,7 @@ spec:
             - { type: 136, family: IPv6 }
             - { type: 143, family: IPv6 }
 ---
-# Allow FTP access (from scanners on the LAN) to the ftp-scanner pod.
+# FTP from the scanners (IoT and Privat) to the ftp-scanner pod.
 apiVersion: cilium.io/v2
 kind: CiliumNetworkPolicy
 metadata:
@@ -123,8 +123,24 @@ spec:
     matchLabels:
       app.kubernetes.io/name: ftp-scanner
   ingress:
+    - fromCIDRSet:
+        - cidrGroupRef: net-iot
+        - cidrGroupRef: net-privat
+      toPorts:
+        - ports:
+            - port: "21"
+              protocol: TCP
+            - port: "21100"
+              protocol: TCP
+            - port: "21101"
+              protocol: TCP
+            - port: "21102"
+              protocol: TCP
+            - port: "21103"
+              protocol: TCP
+            - port: "21104"
+              protocol: TCP
     - fromEntities:
-        - world
         - host
         - remote-node
       toPorts:
@@ -142,7 +158,7 @@ spec:
             - port: "21104"
               protocol: TCP
 ---
-# Allow Syncthing peer-to-peer sync from the LAN/world.
+# Allow Syncthing peer-to-peer sync from Privat and VPN.
 # - 22000 TCP+QUIC: actual sync traffic between peers.
 # - 21027 UDP: local-discovery broadcast packets from peers on the LAN.
 # UI (8384) is intentionally NOT listed — authentik-gate-sync generates the
@@ -157,8 +173,18 @@ spec:
     matchLabels:
       app.kubernetes.io/name: syncthing
   ingress:
+    - fromCIDRSet:
+        - cidrGroupRef: net-privat
+        - cidrGroupRef: net-vpn
+      toPorts:
+        - ports:
+            - port: "22000"
+              protocol: TCP
+            - port: "22000"
+              protocol: UDP
+            - port: "21027"
+              protocol: UDP
     - fromEntities:
-        - world
         - host
         - remote-node
       toPorts:

kubernetes/infrastructure/network-policies/trmnl-ns.yaml

@@ -62,9 +62,10 @@ spec:
 # CA, so this bypasses Traefik entirely and the namespace default-deny has
 # to be opened for it — same shape as paperless's allow-ftp-scanner.
 #
-# Scoped to the terminus pods and port 2345 only. Requests still have to
-# carry a known device MAC in the `ID` header plus its api_key; this rule
-# opens the port, not the application.
+# Scoped to the terminus pods, port 2345 and the IoT network (the panel's
+# network; the LB is ETP=Local, so the client address reaches the pod).
+# Requests still have to carry a known device MAC in the `ID` header plus
+# its api_key; this rule opens the port, not the application.
 apiVersion: cilium.io/v2
 kind: CiliumNetworkPolicy
 metadata:
@@ -75,8 +76,13 @@ spec:
     matchLabels:
       app.kubernetes.io/name: terminus
   ingress:
+    - fromCIDRSet:
+        - cidrGroupRef: net-iot
+      toPorts:
+        - ports:
+            - port: "2345"
+              protocol: TCP
     - fromEntities:
-        - world
         - host
         - remote-node
       toPorts:

opentofu/unifi_nat_hairpin.tf

@@ -68,13 +68,14 @@ locals {
   # idx is a static per-network slot for rule_index derivation — never
   # reuse a freed idx for a new network.
   #
-  # Deliberately absent: (40) Server idx=4, (42) Ingress idx=5, and
-  # (49) k8s Ceph idx=6. The cluster nodes have interfaces on those VLANs,
-  # so a DNATed flow's reply would go directly over the shared L2 and
-  # bypass the UDM's conntrack (asymmetric, connection drops) — and
-  # clients there are the nodes themselves, whose DNS chain (CoreDNS ->
-  # k8s_gateway) already resolves the public names in-cluster. Their idx
-  # slots stay reserved.
+  # Deliberately absent: (40) Server idx=4, (49) k8s Ceph idx=6,
+  # (46) k8s Node, (47) k8s Ingress and (48) k8s Pod. The cluster nodes
+  # have interfaces on those VLANs (mgmt0, node0, ingress0, pod0), so a
+  # DNATed flow's reply would go directly over the shared L2 and bypass
+  # the UDM's conntrack (asymmetric, connection drops) — and clients there
+  # are the nodes themselves, whose DNS chain (CoreDNS -> k8s_gateway)
+  # already resolves the public names in-cluster. idx=5 stays reserved
+  # (its network, (42) Ingress, no longer exists).
   hairpin_networks = {
     "Default"              = { id = "5d4fba77ce63ee614841d53d", idx = 0 }
     "(10) Management"      = { id = "600334abf9ee0c059da6be8f", idx = 1 }