diff --git a/kubernetes/seed/influxdb/seed-job.yaml b/kubernetes/seed/influxdb/seed-job.yaml index cc685a94..aa855380 100644 --- a/kubernetes/seed/influxdb/seed-job.yaml +++ b/kubernetes/seed/influxdb/seed-job.yaml @@ -1,33 +1,37 @@ --- -# InfluxDB seed: query legacy → write local, both via v2 HTTP API. +# InfluxDB seed: backup → staging PVC → restore. # # Mirrors the CNPG pg-seed pattern (ak-pg-seed, im-pg-seed, pprl-pg-seed): -# wait for the database's API to be reachable, then read from the -# source via its native query API and write into the freshly- -# bootstrapped chart instance. +# wait for the database's API to be reachable, then create a backup of +# the source instance via its native tooling, transport it into the +# cluster, and restore via the API into the freshly-bootstrapped chart +# instance. # # Both source (legacy VM 108) and target (in-cluster chart) are v2.7.4 -# with v2 HTTP APIs. The seed Job runs `influx query --host -# --raw 'from(bucket: ...) |> range(...)'` to stream the source bucket's -# data as InfluxData-annotated CSV (org-scoped read:buckets is enough -# — no operator token required), pipes it into `influx write --host -# --format csv` which writes the same annotated CSV directly -# into the chart-bootstrapped bucket. No `influx backup`/`influx -# restore` (which would need a system-wide operator token); no SSH; -# no rsync. +# with v2 HTTP APIs. The seed Job runs `influx backup --host ` +# from inside the cluster — the backup directory is written directly +# into the staging PVC by the influx CLI (no SSH, no rsync, no shell +# on the legacy host). Then `influx restore --host ` imports +# from the staging PVC through the chart's API. +# +# `influx backup` requires an operator-scope token on the source +# (the system-wide bolt + per-bucket TSM snapshot endpoints both gate +# on system-wide read:authorizations). The `influxdb-legacy-token` +# Secret carries that token. # # The chart's adminUser bootstrap (DOCKER_INFLUXDB_INIT_*, sourced from -# influxdb-bootstrap) creates the admin user / token / org / target -# bucket on first start. This Job uses that admin token to write into -# the target bucket; the target bucket already exists from the -# bootstrap, which is fine — `influx write` happily appends. +# influxdb-bootstrap) creates the admin user / token / org / placeholder +# bucket on first start. This Job uses that same admin token to call +# `influx restore`, so the bootstrap-secret token stays the live admin +# token (no clobbering, no auth divergence). `--full` is intentionally +# NOT used — single-bucket restore preserves auth. # -# Trade-off vs `influx backup`/`influx restore`: bucket retention -# policy / shard-group duration etc. are NOT carried over from legacy. -# The chart's adminUser block sets `retention_policy: "0s"` (forever) -# explicitly, which matches the legacy bucket settings — so this trade- -# off is a no-op for the suedstr28 bucket today. If a future bucket -# needs different metadata, set it in the chart values. +# Lossless: backup is a bit-for-bit copy of the bucket's TSM shards +# plus its metadata snapshot; restore replays both. No type filtering, +# no string-mangling, no per-row CSV<->LP conversion. The earlier +# query-pivot-write streaming attempt routinely lost string-valued +# points to `influx write --format csv`'s line-protocol escaping bugs, +# which is unacceptable for a migration. apiVersion: batch/v1 kind: Job metadata: @@ -44,6 +48,12 @@ spec: template: spec: restartPolicy: OnFailure + # Defensive: influxdb:2.7.4's upstream Dockerfile has no USER + # directive (verified) so the container already runs as root, but + # explicit runAsUser:0 covers slim-variant changes upstream. + securityContext: + runAsUser: 0 + runAsNonRoot: false initContainers: # Wait until the chart's API is reachable AND the bootstrap- # secret admin token is live. Two-stage check mirrors the @@ -87,106 +97,38 @@ spec: valueFrom: secretKeyRef: { name: influxdb-bootstrap, key: admin-token } command: - # bash, not sh — `set -o pipefail` is bash-only; debian's - # /bin/sh is dash and rejects the option (`Illegal option`). - bash - -c - | set -euo pipefail log() { printf '[%s] %s\n' "$(date -u +%H:%M:%S)" "$*"; } - # Stream legacy bucket → local bucket through the v2 HTTP - # API in per-month chunks. Annotated-CSV is the wire format - # on both ends, but `influx query` emits LONG format (one - # row per _time/_measurement/_field/_value tuple) while - # `influx write --format csv` requires WIDE format (one row - # per _time/_measurement, each field as its own column). - # The `pivot()` step collapses _field+_value into wide - # columns; `drop()` strips the _start/_stop range columns - # that `influx write` rejects. - # - # Why chunked, and why daily: pivoting in-memory in - # legacy means the whole result set has to fit before it - # flushes. Per-year EOF'd on the current year; per-month - # also EOF'd on dense months (HA-telemetry from 2024+ is - # too big at one-month granularity). Per-day windows keep - # each pivot tiny regardless of write rate, so legacy - # never OOMs. Empty days are no-ops (~1 s each); the - # outer loop is ~3300 iterations across 9 years but most - # complete in <1 s. - # - # No script-side idempotency: InfluxDB writes are upserts - # on (timestamp,measurement,tagset,field), so a retry - # after partial-failure just re-asserts the same points. - # The K8s Job's `Complete` state is the post-success guard - # — Flux reconcile sees Complete and leaves the Job alone. - # - # `pipefail` (bash-only, hence the `command: bash` above): - # an `influx query` EOF inside the pipe propagates a non- - # zero exit, the loop aborts, the Job retries. - # Resume from the last completed day if a sentinel exists in - # the staging PVC. Without this, kubelet's OnFailure restart - # would replay from 2018 every time — 30+ min of retraced - # empty days per retry — and the same bad chunk would burn - # the backoffLimit before reaching new ground. - SENTINEL=/staging/last-completed-day - START="2018-01-01T00:00:00Z" - END=$(date -u -d "tomorrow" +%Y-%m-%dT00:00:00Z) - if [[ -f "$SENTINEL" ]]; then - CUR=$(cat "$SENTINEL") - log "Resuming from $CUR (sentinel)" - else - CUR="$START" - log "Starting fresh from $CUR" - fi + # Stage 1: backup the legacy bucket via its v2 HTTP API + # directly into the staging PVC. The CLI streams the + # bolt+TSM bytes; no SSH, no shell on legacy. + log "influx backup --host $LEGACY_HOST --bucket $LEGACY_BUCKET → /staging/ ..." + rm -rf /staging/* /staging/.[!.]* 2>/dev/null || true + influx backup \ + --host "$LEGACY_HOST" \ + --token "$LEGACY_TOKEN" \ + --bucket "$LEGACY_BUCKET" \ + /staging/ + log "Backup staged. Size: $(du -sh /staging/ | cut -f1)" - while [[ "$CUR" < "$END" ]]; do - NEXT=$(date -u -d "$CUR +1 day" +%Y-%m-%dT00:00:00Z) - log "Streaming $CUR → $NEXT ..." - # Filters in front of pivot, in order of broadness: - # - # 1. `_field == "source"`: HA writes `source` as both a - # tag (integration emitter, e.g. "Home Assistant") AND - # a field (some entity attributes). pivot() can't - # materialise a field-derived column with the same - # name as an existing tag column. Drop the field, keep - # the tag. - # - # 2. `typeOf(_value) != "string"`: drops every string- - # valued point. The CSV→line-protocol conversion in - # `influx write --format csv` doesn't escape special - # chars (commas, newlines, embedded quotes) inside - # string values, and HA's string-valued points - # routinely contain all three (release notes, - # geocoded locations like - # `Location_str="[49.48..., 11.76...]"`, multi-line - # markdown). Each new entity exposes a new flavour of - # parser break; whitelisting numeric values is the - # only stable filter. Numeric metrics — temperature, - # humidity, counters, on/off-as-int — pass through - # unchanged. HA's `state` text mirrors and entity- - # attribute strings are lost; they were already - # awkward to query in Grafana. - influx query --raw \ - --host "$LEGACY_HOST" \ - --token "$LEGACY_TOKEN" \ - --org "$LEGACY_ORG" \ - "from(bucket: \"$LEGACY_BUCKET\") - |> range(start: $CUR, stop: $NEXT) - |> filter(fn: (r) => r._field != \"source\" and typeOf(v: r._value) != \"string\") - |> pivot(rowKey: [\"_time\"], columnKey: [\"_field\"], valueColumn: \"_value\") - |> drop(columns: [\"_start\", \"_stop\"])" \ - | influx write \ - --host "$INFLUX_HOST" \ - --token "$INFLUX_TOKEN" \ - --org "$INFLUX_ORG" \ - --bucket "$INFLUX_BUCKET" \ - --format csv - # Day complete — checkpoint so a kubelet OnFailure restart - # picks up here instead of replaying from 2018. - echo "$NEXT" > "$SENTINEL" - CUR="$NEXT" - done + # Stage 2: drop the chart-bootstrap placeholder bucket so + # `influx restore` (which cannot restore into an existing + # bucket) has a clean slot. Tolerate "already absent" + # for retries after a partial run. + log "Deleting placeholder $INFLUX_BUCKET bucket from chart bootstrap ..." + influx bucket delete --name "$INFLUX_BUCKET" --org "$INFLUX_ORG" || \ + log " (bucket already absent — ok)" + + # Stage 3: restore the staged backup into the chart's + # bootstrapped instance via its v2 API. Selective restore + # (no --full) leaves the bootstrap-secret admin token + # untouched. + log "influx restore --bucket $INFLUX_BUCKET /staging/ → $INFLUX_HOST ..." + influx restore --bucket "$INFLUX_BUCKET" /staging/ log "Seed complete." volumeMounts: