monitoring(grafana): repoint InfluxDB datasource at in-cluster server

VM 108 (the legacy InfluxDB at influxdb.srv.renner.uno:10.2.40.16) is unreachable — no ICMP, no SSH, no port 8086. K8s Grafana's InfluxDB datasource has been silently broken since the in-cluster InfluxDB came online (commit 1daaa6f, 2026-05-05). The heatpump-roi dashboard and any historical HA panel that queries InfluxDB are dead until this repoint lands. Datasource URL: http://influxdb-influxdb2.influxdb.svc.cluster.local:8086. The legacy SOPS-stored token (`grafana-influxdb-token.sops.yaml`) is useless against the in-cluster bolt store. Replaced with a seed Job that mirrors the existing `influxdb-ha-token` pattern: ServiceAccount in influxdb ns + cross-namespace Role on the `grafana-influxdb-token` Secret in monitoring ns + Job that creates a read-only auth on the suedstr28 bucket and writes the token to that Secret. The key name inside the Secret stays `INFLUXDB_TOKEN` so the existing `envFromSecrets` reference in the helmrelease keeps working. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-12 10:55:28 +02:00
parent 8e429175fa
commit eb6399fae8
5 changed files with 153 additions and 29 deletions
@@ -1,23 +0,0 @@
-apiVersion: ENC[AES256_GCM,data:UJo=,iv:dWPZY9IPOwP/wz7NNHWgNLqcemZ7SF5vTLGkLAIVtMw=,tag:/mUiJ+1bSvM1Q+1Z2MAoAQ==,type:str]
-kind: ENC[AES256_GCM,data:2evJBSao,iv:GV5uMMGXkj4unsrX6eSDSnXztVJvO3PH7KysfZVZf9Y=,tag:jL1ouzE3T6utlUgWe5xRGA==,type:str]
-metadata:
-    name: ENC[AES256_GCM,data:yBHEPANSWoXEFtd5/hJaUhPOtGNtUg==,iv:IJVco8HWDAEpkS0R7Lw1YDI374cfG+5FIcr81jg6JUA=,tag:cmz68dU++5sqoPoHsTfBJQ==,type:str]
-    namespace: ENC[AES256_GCM,data:a31jcEaz5JIKSg==,iv:qiyyEy3CNgEbTIrjMIt5wurQWNsXp2Y32TIZ13pNFWM=,tag:hqtHHq+vdxOs8evHkZs2Fw==,type:str]
-type: ENC[AES256_GCM,data:VE7pZ34m,iv:I22j9ZxbieeHiQwrXiS9xpHNzp49Yu2g+kbLk8CM5HU=,tag:WyaAjY6fz3RJK+ScAlkS8A==,type:str]
-stringData:
-    INFLUXDB_TOKEN: ENC[AES256_GCM,data:MhpHqSIk3Nea7msIxzQmvEuFjChNIDfnRpHge4WNVVV3YWeecslMk7cMvRSUBSb0zQ9YYyCKWlnoM7Z+45mfpDDiYB/cnOx1DGXDPGNA8nb6wqJZkbRciA==,iv:xdzC52H9ajkCC2kyXsrlRoBBIYfZ9qD41CrPCntxPTA=,tag:e1Kdix1oX90vVhAp0eq7yQ==,type:str]
-sops:
-    age:
-        - recipient: age1yqjvqkwwwxdj7mr546zfs5ckzuwndp69ya2n2drd4vypwns8w4ys04qfy3
-          enc: |
-            -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBQZXBHaVpLUVJtOWJCaTJG
-            cDg3SUR1QUhDb1BhaGtIWG9tdGlFRit4L3hRCi9LNHdNdS95RHFmQTNNYk43OEJV
-            WGtNd1RuL0N6QTFvZVMvdHAzbmx1b2cKLS0tIHpEeStVWVI5MFlENDE5VXV4RE80
-            Y2hNRHpzeTVEZmhPUjhzbklvdVR4UGcKBfngE1xQY14LM03nu1q905JrLkG3SVrX
-            qyGIEe5xjxTLY4WCu5jfKAEkLmjTwFTGJLg+0F0J+S2MmXyyigBNiQ==
-            -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2026-04-23T12:57:29Z"
-    mac: ENC[AES256_GCM,data:LsMf8qJh519Ylg62jmNmMLt98db8b5S6tvJnxZgpLo9es8jux5g0f75loWTIqzsXL6T7nglWuTp2ouwX4YCgiqTjHPSjJ94ACLpeiFEmjY23gGHtZP1PY1dXZ69OwRJNNmJbEYtopw/rxO/J54MhYi28IL1MDRuwdU/ZttSlqH8=,iv:tgRMH6F4HCHctmAa8UAPnlGUojl+GN3vbDflPbhwxuI=,tag:Ol9YUiEbXSJL+49Klhh/lg==,type:str]
-    unencrypted_suffix: _unencrypted
-    version: 3.10.2
@@ -183,14 +183,16 @@ spec:
          isDefault: false
          jsonData:
            httpMethod: POST
-        # InfluxDB on VM 108 — Home Assistant is the sole writer. K8s
-        # Grafana consumes the live VM 108 endpoint until HA migrates
-        # (writer cut-over + datasource flip to cluster-internal deferred
-        # to the HA migration plan).
+        # In-cluster InfluxDB (chart `influxdb-influxdb2`). Both HA (write)
+        # and Grafana (read) hit this directly via Service DNS. The
+        # `grafana-influxdb-token` Secret is created by the seed Job
+        # `influxdb-grafana-token` (kubernetes/seed/influxdb/grafana-
+        # token-job.yaml), which mints a read-only auth on the
+        # suedstr28 bucket every fresh deploy.
        - name: InfluxDB
          uid: influxdb
          type: influxdb
-          url: http://influxdb.srv.renner.uno:8086
+          url: http://influxdb-influxdb2.influxdb.svc.cluster.local:8086
          access: proxy
          isDefault: false
          jsonData:
@@ -2,7 +2,6 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
  - grafana-oauth.sops.yaml
-  - grafana-influxdb-token.sops.yaml
  - garage-metrics-token.sops.yaml
  - alertmanager-smtp.sops.yaml
  - pve-exporter-config.sops.yaml
@@ -0,0 +1,145 @@
+---
+# Mirror of ha-token-job.yaml but creates a read-only auth for K8s
+# Grafana on the suedstr28 bucket, and stashes the token as the
+# `grafana-influxdb-token` Secret in the monitoring namespace. Replaces
+# the previously SOPS-stored token that targeted the (now-retired) VM
+# 108 InfluxDB and no longer matches the in-cluster server's bolt store.
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: influxdb-grafana-token
+  namespace: influxdb
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: grafana-influxdb-token-writer
+  namespace: monitoring
+rules:
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["create"]
+  - apiGroups: [""]
+    resources: ["secrets"]
+    resourceNames: ["grafana-influxdb-token"]
+    verbs: ["get", "update", "patch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: grafana-influxdb-token-writer
+  namespace: monitoring
+subjects:
+  - kind: ServiceAccount
+    name: influxdb-grafana-token
+    namespace: influxdb
+roleRef:
+  kind: Role
+  name: grafana-influxdb-token-writer
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: influxdb-grafana-token
+  namespace: influxdb
+  annotations:
+    kustomize.toolkit.fluxcd.io/force: enabled
+spec:
+  backoffLimit: 6
+  ttlSecondsAfterFinished: 86400
+  template:
+    spec:
+      restartPolicy: OnFailure
+      serviceAccountName: influxdb-grafana-token
+      securityContext:
+        runAsUser: 0
+        runAsNonRoot: false
+      initContainers:
+        - name: wait-for-influxd
+          image: influxdb:2.7.4
+          env:
+            - { name: INFLUX_HOST, value: http://influxdb-influxdb2.influxdb.svc.cluster.local:8086 }
+            - { name: INFLUX_ORG, value: renner }
+            - name: INFLUX_TOKEN
+              valueFrom:
+                secretKeyRef: { name: influxdb-bootstrap, key: admin-token }
+          command:
+            - sh
+            - -c
+            - |
+              set -eu
+              until influx ping >/dev/null 2>&1; do
+                echo "[$(date -u +%H:%M:%S)] waiting for $INFLUX_HOST..."
+                sleep 5
+              done
+              until influx bucket list >/dev/null 2>&1; do
+                echo "[$(date -u +%H:%M:%S)] API up; waiting for bootstrap admin token..."
+                sleep 5
+              done
+              echo "[$(date -u +%H:%M:%S)] $INFLUX_HOST ready and authenticated"
+      containers:
+        - name: create
+          image: influxdb:2.7.4
+          env:
+            - { name: INFLUX_HOST, value: http://influxdb-influxdb2.influxdb.svc.cluster.local:8086 }
+            - { name: INFLUX_ORG, value: renner }
+            - { name: INFLUX_BUCKET, value: suedstr28 }
+            - { name: TOKEN_DESC, value: grafana read }
+            - { name: TARGET_NS, value: monitoring }
+            - { name: TARGET_SECRET, value: grafana-influxdb-token }
+            - name: INFLUX_TOKEN
+              valueFrom:
+                secretKeyRef: { name: influxdb-bootstrap, key: admin-token }
+          command:
+            - bash
+            - -c
+            - |
+              set -euo pipefail
+              log() { printf '[%s] %s\n' "$(date -u +%H:%M:%S)" "$*"; }
+
+              apt-get update -qq
+              apt-get install -y --no-install-recommends curl jq ca-certificates >/dev/null
+              KV=$(curl -fsSL https://dl.k8s.io/release/stable.txt)
+              curl -fsSL "https://dl.k8s.io/release/$KV/bin/linux/amd64/kubectl" -o /usr/local/bin/kubectl
+              chmod +x /usr/local/bin/kubectl
+
+              BUCKET_ID=$(influx bucket list --name "$INFLUX_BUCKET" --json | jq -r '.[0].id')
+              [ -n "$BUCKET_ID" ] && [ "$BUCKET_ID" != "null" ] || {
+                log "ERROR: bucket $INFLUX_BUCKET not found"
+                exit 1
+              }
+              log "bucket $INFLUX_BUCKET id=$BUCKET_ID"
+
+              EXISTING=$(influx auth list --json \
+                | jq -r --arg d "$TOKEN_DESC" '.[] | select(.description == $d) | .token' \
+                | head -1)
+              if [ -n "$EXISTING" ]; then
+                TOKEN="$EXISTING"
+                log "reusing existing auth ($TOKEN_DESC)"
+              else
+                # Read-only on the bucket. Grafana queries via Flux which
+                # only reads.
+                TOKEN=$(influx auth create \
+                  --org "$INFLUX_ORG" \
+                  --description "$TOKEN_DESC" \
+                  --read-bucket "$BUCKET_ID" \
+                  --json | jq -r '.token')
+                [ -n "$TOKEN" ] && [ "$TOKEN" != "null" ] || {
+                  log "ERROR: influx auth create produced empty token"
+                  exit 1
+                }
+                log "created new auth ($TOKEN_DESC)"
+              fi
+
+              # K8s Grafana's envFromSecret reads the key as
+              # INFLUXDB_TOKEN (set in the chart values); use the same
+              # key name so a single Secret-rotate refreshes the
+              # template var.
+              log "writing $TARGET_NS/$TARGET_SECRET via apply"
+              kubectl -n "$TARGET_NS" create secret generic "$TARGET_SECRET" \
+                --from-literal=INFLUXDB_TOKEN="$TOKEN" \
+                --dry-run=client -o yaml \
+                | kubectl apply -f -
+
+              log "Done."
@@ -4,3 +4,4 @@ resources:
  - bootstrap-secret.sops.yaml
  - helmrelease.yaml
  - ha-token-job.yaml
+  - grafana-token-job.yaml