From 32a7799e2b91b5f8b5b91c392d02a3bf11fa30e2 Mon Sep 17 00:00:00 2001 From: Konstantin Renner Date: Sat, 2 May 2026 12:56:15 +0200 Subject: [PATCH] =?UTF-8?q?feat(monitoring):=20add=20K8s=20syslog=E2=86=92?= =?UTF-8?q?Loki=20receiver=20via=20Alloy?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stand up a dedicated alloy-syslog Deployment in monitoring that listens on UDP/514, UDP/1514 (legacy parity), TCP/514 and TCP/6514+TLS and writes into the in-cluster Loki SingleBinary. Exposed as a Cilium L2 LoadBalancer pinned to 10.2.40.84 and published as syslog.k.renner.uno via external-dns; that meant adding `service` to external-dns-unifi sources (was ingress-only). TLS cert issued by letsencrypt-prod through cert-manager. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../monitoring/helmrelease-alloy-syslog.yaml | 192 ++++++++++++++++++ kubernetes/apps/monitoring/kustomization.yaml | 1 + .../external-dns-unifi/helmrelease.yaml | 1 + 3 files changed, 194 insertions(+) create mode 100644 kubernetes/apps/monitoring/helmrelease-alloy-syslog.yaml diff --git a/kubernetes/apps/monitoring/helmrelease-alloy-syslog.yaml b/kubernetes/apps/monitoring/helmrelease-alloy-syslog.yaml new file mode 100644 index 00000000..147f3a5d --- /dev/null +++ b/kubernetes/apps/monitoring/helmrelease-alloy-syslog.yaml @@ -0,0 +1,192 @@ +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: syslog-tls + namespace: monitoring +spec: + secretName: syslog-tls + issuerRef: + name: letsencrypt-prod + kind: ClusterIssuer + dnsNames: + - syslog.k.renner.uno + privateKey: + rotationPolicy: Always +--- +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: alloy-syslog + namespace: monitoring +spec: + interval: 30m + chart: + spec: + chart: alloy + version: ">=0.12.0" + sourceRef: + kind: HelmRepository + name: grafana + namespace: monitoring + install: + remediation: + retries: 3 + upgrade: + remediation: + retries: 3 + values: + fullnameOverride: alloy-syslog + controller: + type: deployment + replicas: 1 + volumes: + extra: + - name: tls + secret: + secretName: syslog-tls + service: + # We provide our own LoadBalancer Service below; suppress the chart's + # internal ClusterIP so the in-cluster name `alloy-syslog` resolves to + # ours, not the chart's. + enabled: false + alloy: + mounts: + extra: + - name: tls + mountPath: /etc/alloy-tls + readOnly: true + extraPorts: + - name: syslog-udp-514 + port: 514 + targetPort: 514 + protocol: UDP + - name: syslog-udp-1514 + port: 1514 + targetPort: 1514 + protocol: UDP + - name: syslog-tcp-514 + port: 514 + targetPort: 514 + protocol: TCP + - name: syslog-tcp-6514 + port: 6514 + targetPort: 6514 + protocol: TCP + configMap: + content: | + // Promote useful syslog fields to Loki labels before write. + loki.relabel "syslog" { + forward_to = [loki.write.default.receiver] + + rule { + source_labels = ["__syslog_message_hostname"] + target_label = "host" + } + rule { + source_labels = ["__syslog_message_severity"] + target_label = "severity" + } + rule { + source_labels = ["__syslog_message_facility"] + target_label = "facility" + } + rule { + source_labels = ["__syslog_message_app_name"] + target_label = "app" + } + rule { + source_labels = ["__syslog_connection_ip_address"] + target_label = "source_ip" + } + } + + // RFC 3164 over UDP/514 — UniFi UDM, switches, APs, SLZB devices. + loki.source.syslog "udp_514" { + listener { + address = "0.0.0.0:514" + protocol = "udp" + syslog_format = "rfc3164" + label_structured_data = true + labels = { job = "syslog", listener = "udp-514" } + } + forward_to = [loki.relabel.syslog.receiver] + } + + // RFC 3164 over UDP/1514 — legacy parity with the LXC Alloy receiver + // (Synology DSM ships here today). + loki.source.syslog "udp_1514" { + listener { + address = "0.0.0.0:1514" + protocol = "udp" + syslog_format = "rfc3164" + label_structured_data = true + labels = { job = "syslog", listener = "udp-1514" } + } + forward_to = [loki.relabel.syslog.receiver] + } + + // RFC 5424 over TCP/514 — reliable plaintext for senders that prefer it. + loki.source.syslog "tcp_514" { + listener { + address = "0.0.0.0:514" + protocol = "tcp" + syslog_format = "rfc5424" + label_structured_data = true + labels = { job = "syslog", listener = "tcp-514" } + } + forward_to = [loki.relabel.syslog.receiver] + } + + // RFC 5424 over TCP/6514 with TLS — for senders requiring encryption. + loki.source.syslog "tcp_6514_tls" { + listener { + address = "0.0.0.0:6514" + protocol = "tcp" + syslog_format = "rfc5424" + label_structured_data = true + labels = { job = "syslog", listener = "tcp-6514-tls" } + tls_config { + cert_file = "/etc/alloy-tls/tls.crt" + key_file = "/etc/alloy-tls/tls.key" + } + } + forward_to = [loki.relabel.syslog.receiver] + } + + loki.write "default" { + endpoint { + url = "http://loki.monitoring.svc.cluster.local:3100/loki/api/v1/push" + } + } +--- +apiVersion: v1 +kind: Service +metadata: + name: alloy-syslog + namespace: monitoring + annotations: + io.cilium/lb-ipam-ips: "10.2.40.84" + external-dns.alpha.kubernetes.io/hostname: syslog.k.renner.uno +spec: + type: LoadBalancer + externalTrafficPolicy: Local + selector: + app.kubernetes.io/name: alloy + app.kubernetes.io/instance: alloy-syslog + ports: + - name: syslog-udp-514 + port: 514 + targetPort: 514 + protocol: UDP + - name: syslog-udp-1514 + port: 1514 + targetPort: 1514 + protocol: UDP + - name: syslog-tcp-514 + port: 514 + targetPort: 514 + protocol: TCP + - name: syslog-tcp-6514 + port: 6514 + targetPort: 6514 + protocol: TCP diff --git a/kubernetes/apps/monitoring/kustomization.yaml b/kubernetes/apps/monitoring/kustomization.yaml index 80584ddf..69e782d8 100644 --- a/kubernetes/apps/monitoring/kustomization.yaml +++ b/kubernetes/apps/monitoring/kustomization.yaml @@ -6,6 +6,7 @@ resources: - garage-metrics-token.sops.yaml - helmrelease-prometheus.yaml - helmrelease-loki.yaml + - helmrelease-alloy-syslog.yaml - dashboards/ # prometheusrule-garage.yaml lives in ./monitoring-rules/ and is applied by a # separate Flux Kustomization that dependsOn `monitoring` — otherwise the diff --git a/kubernetes/infrastructure/external-dns-unifi/helmrelease.yaml b/kubernetes/infrastructure/external-dns-unifi/helmrelease.yaml index e1cced5b..d908d7c6 100644 --- a/kubernetes/infrastructure/external-dns-unifi/helmrelease.yaml +++ b/kubernetes/infrastructure/external-dns-unifi/helmrelease.yaml @@ -67,5 +67,6 @@ spec: - k.renner.uno sources: - ingress + - service txtOwnerId: k8s-homelab policy: sync