diff --git a/kubernetes/RECOVERY.md b/kubernetes/RECOVERY.md index 5cdd0240..7bfa57ef 100644 --- a/kubernetes/RECOVERY.md +++ b/kubernetes/RECOVERY.md @@ -6,18 +6,24 @@ below (like `wp-pg`) keeps `bootstrap.recovery` in git: a cold rebuild restores each from its backup with no manual step. This document covers the precondition, watching and verifying that restore, and the reasoning behind it. -| Cluster | Namespace | ObjectStore | barman path (`destinationPath`) | app DB / owner | -|-----------|----------------|-----------------|--------------------------------------|---------------------------------| -| `ak-pg` | authentik | `ak-pg-store` | `s3://renner-uno-k8s-cnpg/ak-pg` | `authentik`/`authentik` | -| `im-pg` | immich | `im-pg-store` | `s3://renner-uno-k8s-cnpg/im-pg` | `immich`/`immich` | -| `pprl-pg` | paperless | `pprl-pg-store` | `s3://renner-uno-k8s-cnpg/pprl-pg` | `paperless`/`paperless` | -| `ha-pg` | home-assistant | `ha-pg-store` | `s3://renner-uno-k8s-cnpg/ha-pg` | `homeassistant`/`homeassistant` | -| `mx-pg` | matrix | `mx-pg-store` | `s3://renner-uno-k8s-cnpg/mx-pg` | `synapse`/`synapse` | -| `tr-pg` | trmnl | `tr-pg-store` | `s3://renner-uno-k8s-cnpg/tr-pg` | `terminus`/`terminus` | +| Cluster | Namespace | app DB / owner | +|-----------|----------------|---------------------------------| +| `ak-pg` | authentik | `authentik`/`authentik` | +| `im-pg` | immich | `immich`/`immich` | +| `pprl-pg` | paperless | `paperless`/`paperless` | +| `ha-pg` | home-assistant | `homeassistant`/`homeassistant` | +| `mx-pg` | matrix | `synapse`/`synapse` | +| `tr-pg` | trmnl | `terminus`/`terminus` | +| `wp-pg` | woodpecker | `woodpecker`/`woodpecker` | -The barman **`serverName`** is the last path segment of `destinationPath`, i.e. -it equals the cluster name. `gf-pg` (monitoring) has no ObjectStore and stays -on `initdb`. +Each cluster `` has two ObjectStores in `kubernetes/cnpg-object-stores/`: +`-store`, where it archives WAL and takes base backups, and +`-store-source`, which `bootstrap.recovery` reads. Their paths come from the +`flux-system/cnpg-paths` ConfigMap, which `bootstrap.sh` writes once per +cluster incarnation: the source is the prefix in `s3://renner-uno-k8s-cnpg` +whose `/base/` holds the newest base backup, the archive a new prefix +`-g`. The barman **`serverName`** is the cluster name in both. +`gf-pg` (monitoring) has no ObjectStore and stays on `initdb`. --- @@ -129,7 +135,7 @@ cluster in the *same* namespace (so it can reference the existing namespaced restore, but never archiving, so it only ever *reads* the backup path. ```sh -NS=authentik ; SRC=ak-pg ; STORE=ak-pg-store ; DB=authentik +NS=authentik ; SRC=ak-pg ; STORE=ak-pg-store-source ; DB=authentik kubectl apply -f - <authentik-KS handoff (backup target). - kustomize.toolkit.fluxcd.io/prune: disabled -spec: - configuration: - # Offsite Hetzner Object Storage (fsn1) — replaces Garage on nas-1. - # No SSE: Hetzner supports only SSE-C, unusable by barman-cloud - # (plugin #646), so at-rest encryption is unavailable; transit is TLS. - destinationPath: s3://renner-uno-k8s-cnpg/ak-pg - endpointURL: https://fsn1.your-objectstorage.com - s3Credentials: - accessKeyId: - name: hetzner-s3-creds - key: ACCESS_KEY_ID - secretAccessKey: - name: hetzner-s3-creds - key: ACCESS_SECRET_KEY - # Region is required (REGION=fsn1) — without it boto3's HeadBucket - # (barman-cloud-check-wal-archive during bootstrap.recovery) 400s. - region: - name: hetzner-s3-creds - key: REGION - wal: - compression: gzip - data: - compression: gzip - retentionPolicy: "30d" diff --git a/kubernetes/apps/gitea/pvc.yaml b/kubernetes/apps/gitea/pvc.yaml index bbc49a1e..cab84949 100644 --- a/kubernetes/apps/gitea/pvc.yaml +++ b/kubernetes/apps/gitea/pvc.yaml @@ -10,10 +10,8 @@ # - Name, namespace, accessModes, storageClassName and volumeMode must match # the live claim exactly: they are immutable, a mismatch fails the whole # gitea Kustomization (no change to the live claim). -# - volumeName must stay: kustomize-controller has owned spec.volumeName -# since the namespace move pre-created this claim (eb23d3ac, #878). -# Without it, server-side apply tries to clear the field and the API server -# rejects the whole claim as an immutable-spec change. +# - No volumeName: the gitea Kustomization ignores /spec/volumeName on this +# claim, so the bound volume is kept and a fresh cluster binds a new one. # - Grow only. A smaller value is rejected by the API server. # - prune: disabled, so removing this file from git never deletes the claim # and its data. To hand ownership back, delete the file; the claim stays. @@ -33,7 +31,6 @@ spec: accessModes: ["ReadWriteOnce"] storageClassName: ceph-block volumeMode: Filesystem - volumeName: pvc-e7c56db2-481d-40cb-b94a-b264b7b4b5be resources: requests: storage: 60Gi diff --git a/kubernetes/apps/home-assistant-iot/kustomization.yaml b/kubernetes/apps/home-assistant-iot/kustomization.yaml new file mode 100644 index 00000000..ea19f14c --- /dev/null +++ b/kubernetes/apps/home-assistant-iot/kustomization.yaml @@ -0,0 +1,35 @@ +# The home-assistant workloads attached to the IoT VLAN (v30iot). Separate from +# ../home-assistant so that HA and what depends on it do not wait on a VLAN +# the node may not have; the pods run only where it does +# (net.renner.uno/iot-vlan). +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - shelly-manager-pvc.yaml + - shelly-manager-network-attachment.yaml + - shelly-manager-firmware-rbac.yaml + - shelly-ble-proxy-key-externalsecret.yaml + # shelly-manager-auth comes from ../home-assistant: mqtt-account-controller + # mounts it too, and that Kustomization comes first. + - shelly-manager-llat-externalsecret.yaml + - shelly-manager-deployment.yaml + - shelly-manager-service.yaml + - shelly-manager-ingress.yaml + # mDNS relay — carries IoT-VLAN announcements into HA's zeroconf listener as + # unicast, which is the only direction the CNI will move them. Without it HA + # never learns that a device's DHCP lease moved. See the header comment in + # mdns-relay-deployment.yaml. + - mdns-relay-network-attachment.yaml + - mdns-relay-service.yaml + - mdns-relay-deployment.yaml +configMapGenerator: + # shelly-manager api-proxy sidecar config (REN-935). Hash suffix disabled + # (name stays stable), so editing shelly-manager-api-proxy.conf does not + # auto-roll the pod — restart manually with + # `kubectl rollout restart deploy/shelly-manager -n home-assistant`. + - name: shelly-manager-api-proxy + namespace: home-assistant + files: + - api.conf.template=shelly-manager-api-proxy.conf +generatorOptions: + disableNameSuffixHash: true diff --git a/kubernetes/apps/home-assistant/mdns-relay-deployment.yaml b/kubernetes/apps/home-assistant-iot/mdns-relay-deployment.yaml similarity index 98% rename from kubernetes/apps/home-assistant/mdns-relay-deployment.yaml rename to kubernetes/apps/home-assistant-iot/mdns-relay-deployment.yaml index cceabccb..a981372a 100644 --- a/kubernetes/apps/home-assistant/mdns-relay-deployment.yaml +++ b/kubernetes/apps/home-assistant-iot/mdns-relay-deployment.yaml @@ -47,6 +47,9 @@ spec: homelab.renner.uno/restarted-for: "2026-10-06 cni-plugins v1.9.1 attach check" k8s.v1.cni.cncf.io/networks: mdns-relay-iot spec: + # Runs where the node patch declares the IoT VLAN (ens19). + nodeSelector: + net.renner.uno/iot-vlan: "true" enableServiceLinks: false securityContext: runAsNonRoot: true diff --git a/kubernetes/apps/home-assistant/mdns-relay-network-attachment.yaml b/kubernetes/apps/home-assistant-iot/mdns-relay-network-attachment.yaml similarity index 100% rename from kubernetes/apps/home-assistant/mdns-relay-network-attachment.yaml rename to kubernetes/apps/home-assistant-iot/mdns-relay-network-attachment.yaml diff --git a/kubernetes/apps/home-assistant/mdns-relay-service.yaml b/kubernetes/apps/home-assistant-iot/mdns-relay-service.yaml similarity index 100% rename from kubernetes/apps/home-assistant/mdns-relay-service.yaml rename to kubernetes/apps/home-assistant-iot/mdns-relay-service.yaml diff --git a/kubernetes/apps/home-assistant/shelly-ble-proxy-key-externalsecret.yaml b/kubernetes/apps/home-assistant-iot/shelly-ble-proxy-key-externalsecret.yaml similarity index 100% rename from kubernetes/apps/home-assistant/shelly-ble-proxy-key-externalsecret.yaml rename to kubernetes/apps/home-assistant-iot/shelly-ble-proxy-key-externalsecret.yaml diff --git a/kubernetes/apps/home-assistant/shelly-manager-api-proxy.conf b/kubernetes/apps/home-assistant-iot/shelly-manager-api-proxy.conf similarity index 100% rename from kubernetes/apps/home-assistant/shelly-manager-api-proxy.conf rename to kubernetes/apps/home-assistant-iot/shelly-manager-api-proxy.conf diff --git a/kubernetes/apps/home-assistant/shelly-manager-deployment.yaml b/kubernetes/apps/home-assistant-iot/shelly-manager-deployment.yaml similarity index 97% rename from kubernetes/apps/home-assistant/shelly-manager-deployment.yaml rename to kubernetes/apps/home-assistant-iot/shelly-manager-deployment.yaml index 1ef51599..a3335472 100644 --- a/kubernetes/apps/home-assistant/shelly-manager-deployment.yaml +++ b/kubernetes/apps/home-assistant-iot/shelly-manager-deployment.yaml @@ -47,10 +47,10 @@ spec: spec: # eth0 (Cilium-managed pod network) is still the primary — it # carries the ingress, mosquitto LB return path, etc. The macvlan - # above is additive. No explicit edge exclusion is needed: the - # `workload-class=pinned:NoSchedule` taint on k8s-edge already - # keeps this pod off there (we never tolerate it), and the NAD's - # `master: ens19` (stable predictable name for the v30iot NIC) is present on every k8s-{1,2,3} worker. + # above is additive. + # Runs where the node patch declares the IoT VLAN (ens19). + nodeSelector: + net.renner.uno/iot-vlan: "true" enableServiceLinks: false # Grants shelly-manager the in-cluster identity used by the firmware # pipeline to create esphome flash Jobs and stream their logs. RBAC is diff --git a/kubernetes/apps/home-assistant/shelly-manager-firmware-rbac.yaml b/kubernetes/apps/home-assistant-iot/shelly-manager-firmware-rbac.yaml similarity index 100% rename from kubernetes/apps/home-assistant/shelly-manager-firmware-rbac.yaml rename to kubernetes/apps/home-assistant-iot/shelly-manager-firmware-rbac.yaml diff --git a/kubernetes/apps/home-assistant/shelly-manager-ingress.yaml b/kubernetes/apps/home-assistant-iot/shelly-manager-ingress.yaml similarity index 100% rename from kubernetes/apps/home-assistant/shelly-manager-ingress.yaml rename to kubernetes/apps/home-assistant-iot/shelly-manager-ingress.yaml diff --git a/kubernetes/apps/home-assistant/shelly-manager-llat-externalsecret.yaml b/kubernetes/apps/home-assistant-iot/shelly-manager-llat-externalsecret.yaml similarity index 100% rename from kubernetes/apps/home-assistant/shelly-manager-llat-externalsecret.yaml rename to kubernetes/apps/home-assistant-iot/shelly-manager-llat-externalsecret.yaml diff --git a/kubernetes/apps/home-assistant/shelly-manager-network-attachment.yaml b/kubernetes/apps/home-assistant-iot/shelly-manager-network-attachment.yaml similarity index 100% rename from kubernetes/apps/home-assistant/shelly-manager-network-attachment.yaml rename to kubernetes/apps/home-assistant-iot/shelly-manager-network-attachment.yaml diff --git a/kubernetes/apps/home-assistant/shelly-manager-pvc.yaml b/kubernetes/apps/home-assistant-iot/shelly-manager-pvc.yaml similarity index 100% rename from kubernetes/apps/home-assistant/shelly-manager-pvc.yaml rename to kubernetes/apps/home-assistant-iot/shelly-manager-pvc.yaml diff --git a/kubernetes/apps/home-assistant/shelly-manager-service.yaml b/kubernetes/apps/home-assistant-iot/shelly-manager-service.yaml similarity index 100% rename from kubernetes/apps/home-assistant/shelly-manager-service.yaml rename to kubernetes/apps/home-assistant-iot/shelly-manager-service.yaml diff --git a/kubernetes/apps/home-assistant/ha-pg-cluster.yaml b/kubernetes/apps/home-assistant/ha-pg-cluster.yaml index f1190645..8a0f57f2 100644 --- a/kubernetes/apps/home-assistant/ha-pg-cluster.yaml +++ b/kubernetes/apps/home-assistant/ha-pg-cluster.yaml @@ -116,7 +116,7 @@ spec: plugin: name: barman-cloud.cloudnative-pg.io parameters: - barmanObjectName: ha-pg-store + barmanObjectName: ha-pg-store-source serverName: ha-pg postgresql: parameters: diff --git a/kubernetes/apps/home-assistant/ha-pg-objectstore.yaml b/kubernetes/apps/home-assistant/ha-pg-objectstore.yaml deleted file mode 100644 index 444ec01a..00000000 --- a/kubernetes/apps/home-assistant/ha-pg-objectstore.yaml +++ /dev/null @@ -1,35 +0,0 @@ ---- -# Barman object store for ha-pg: base backups + continuous WAL archiving to -# Hetzner Object Storage, same target bucket as the other CNPG clusters. -# -# Its own prefix. Two clusters archiving WAL to one destinationPath interleave -# their streams and corrupt both — see the note in -# kubernetes/apps/woodpecker/objectstore.yaml, which learned that the hard way. -apiVersion: barmancloud.cnpg.io/v1 -kind: ObjectStore -metadata: - name: ha-pg-store - namespace: home-assistant -spec: - configuration: - destinationPath: s3://renner-uno-k8s-cnpg/ha-pg - endpointURL: https://fsn1.your-objectstorage.com - s3Credentials: - accessKeyId: - name: hetzner-s3-creds - key: ACCESS_KEY_ID - secretAccessKey: - name: hetzner-s3-creds - key: ACCESS_SECRET_KEY - region: - name: hetzner-s3-creds - key: REGION - wal: - compression: gzip - data: - compression: gzip - # Matches the 30d retention the other CNPG clusters use and the - # renner-uno-k8s-cnpg bucket's noncurrent-version expiry in - # opentofu/object_storage.tf, so old S3 versions don't outlive the backups - # that superseded them. - retentionPolicy: "30d" diff --git a/kubernetes/apps/home-assistant/kustomization.yaml b/kubernetes/apps/home-assistant/kustomization.yaml index 6ffb8cd1..bb727887 100644 --- a/kubernetes/apps/home-assistant/kustomization.yaml +++ b/kubernetes/apps/home-assistant/kustomization.yaml @@ -21,26 +21,11 @@ resources: - mosquitto-iot-service.yaml - mosquitto-deployment.yaml - mosquitto-evcc-credentials-externalsecret.yaml - - shelly-manager-pvc.yaml - - shelly-manager-network-attachment.yaml - - shelly-manager-firmware-rbac.yaml - - shelly-ble-proxy-key-externalsecret.yaml - - shelly-manager-auth-externalsecret.yaml - - shelly-manager-llat-externalsecret.yaml - - shelly-manager-deployment.yaml - - shelly-manager-service.yaml - - shelly-manager-ingress.yaml + # shelly-manager and the mDNS relay sit on the IoT VLAN: ../home-assistant-iot. - esphome-pvc.yaml - esphome-deployment.yaml - esphome-service.yaml - esphome-ingress.yaml - # mDNS relay — carries IoT-VLAN announcements into HA's zeroconf listener as - # unicast, which is the only direction the CNI will move them. Without it HA - # never learns that a device's DHCP lease moved. See the header comment in - # mdns-relay-deployment.yaml. - - mdns-relay-network-attachment.yaml - - mdns-relay-service.yaml - - mdns-relay-deployment.yaml # Zigbee2MQTT — ZHA replacement bridge. Ships scaled to 0; the cutover # runbook (zigbee2mqtt-migration-runbook.md) scales it up. The MQTT # `zigbee2mqtt` broker user is provisioned by mosquitto-bootstrap.yaml @@ -57,7 +42,6 @@ resources: # Live: recorder.db_url in the homeassistant repo points here, and the # long-term statistics carry their full history from 2024. - ha-pg-hetzner-creds-externalsecret.yaml - - ha-pg-objectstore.yaml - ha-pg-cluster.yaml - ha-pg-scheduled-backup.yaml # MQTT credential management (REN-802): the account controller manages the @@ -66,6 +50,8 @@ resources: # Register: mqtt-accounts.yaml (configMapGenerator). - mosquitto-dynsec-test.yaml - mqtt-account-controller.yaml + # Also mounted by shelly-manager (../home-assistant-iot), which runs after this. + - shelly-manager-auth-externalsecret.yaml configMapGenerator: # nginx config for the reverse-proxy sidecar. Hash suffix disabled so # the cm name stays stable; editing proxy.conf does not auto-roll the @@ -116,13 +102,5 @@ configMapGenerator: namespace: home-assistant files: - mqtt-accounts.yaml - # shelly-manager api-proxy sidecar config (REN-935). Hash suffix disabled - # (name stays stable), so editing shelly-manager-api-proxy.conf does not - # auto-roll the pod — restart manually with - # `kubectl rollout restart deploy/shelly-manager -n home-assistant`. - - name: shelly-manager-api-proxy - namespace: home-assistant - files: - - api.conf.template=shelly-manager-api-proxy.conf generatorOptions: disableNameSuffixHash: true diff --git a/kubernetes/apps/home-assistant/mqtt-account-controller.yaml b/kubernetes/apps/home-assistant/mqtt-account-controller.yaml index 145dc29e..a1be2ac8 100644 --- a/kubernetes/apps/home-assistant/mqtt-account-controller.yaml +++ b/kubernetes/apps/home-assistant/mqtt-account-controller.yaml @@ -305,10 +305,13 @@ spec: - key: api-key path: api-key # Client certificate HA gets (cert-manager renews it in place; not a - # subPath mount, so the next loop sees the new file). + # subPath mount, so the next loop sees the new file). Optional: this + # controller creates its Certificate, so on a new cluster the Secret + # appears only after the controller has started. - name: ha-mqtt-tls secret: secretName: mqtt-home-assistant-tls + optional: true # HA long-lived token of the service user `MQTT Cert Sync` # (system-admin; config flows need admin). REN-1268. - name: ha-token diff --git a/kubernetes/apps/immich/cluster.yaml b/kubernetes/apps/immich/cluster.yaml index f16049ee..8cd16413 100644 --- a/kubernetes/apps/immich/cluster.yaml +++ b/kubernetes/apps/immich/cluster.yaml @@ -43,7 +43,7 @@ spec: plugin: name: barman-cloud.cloudnative-pg.io parameters: - barmanObjectName: im-pg-store + barmanObjectName: im-pg-store-source serverName: im-pg postgresql: # vchord only. The image also ships pgvecto.rs (vectors.so), but its diff --git a/kubernetes/apps/immich/kustomization.yaml b/kubernetes/apps/immich/kustomization.yaml index 258d32c7..c4010e65 100644 --- a/kubernetes/apps/immich/kustomization.yaml +++ b/kubernetes/apps/immich/kustomization.yaml @@ -11,7 +11,6 @@ resources: # a teardown). This KS gains the infrastructure-cnpg-barman dep + CNPG Cluster # healthCheckExprs (see clusters/homelab/immich.yaml). - cluster.yaml - - objectstore.yaml - scheduled-backup.yaml - hetzner-creds-externalsecret.yaml - pvcs.yaml diff --git a/kubernetes/apps/immich/objectstore.yaml b/kubernetes/apps/immich/objectstore.yaml deleted file mode 100644 index 1e66103a..00000000 --- a/kubernetes/apps/immich/objectstore.yaml +++ /dev/null @@ -1,33 +0,0 @@ ---- -apiVersion: barmancloud.cnpg.io/v1 -kind: ObjectStore -metadata: - name: im-pg-store - namespace: immich - annotations: - # Prune-disabled — a standing guard kept from the seed->immich-KS handoff (backup target). - kustomize.toolkit.fluxcd.io/prune: disabled -spec: - configuration: - # Offsite Hetzner Object Storage (fsn1) — replaces Garage on nas-1. - # No SSE: Hetzner supports only SSE-C, unusable by barman-cloud - # (plugin #646), so at-rest encryption is unavailable; transit is TLS. - destinationPath: s3://renner-uno-k8s-cnpg/im-pg - endpointURL: https://fsn1.your-objectstorage.com - s3Credentials: - accessKeyId: - name: hetzner-s3-creds - key: ACCESS_KEY_ID - secretAccessKey: - name: hetzner-s3-creds - key: ACCESS_SECRET_KEY - # Region is required (REGION=fsn1) — without it boto3's HeadBucket - # (barman-cloud-check-wal-archive during bootstrap.recovery) 400s. - region: - name: hetzner-s3-creds - key: REGION - wal: - compression: gzip - data: - compression: gzip - retentionPolicy: "30d" diff --git a/kubernetes/apps/llama-server/deployment.yaml b/kubernetes/apps/llama-server/deployment.yaml index 57460f58..051729c0 100644 --- a/kubernetes/apps/llama-server/deployment.yaml +++ b/kubernetes/apps/llama-server/deployment.yaml @@ -44,6 +44,9 @@ spec: labels: app.kubernetes.io/name: llama-server spec: + # Runs only where the node patch declares the Intel iGPU. + nodeSelector: + hardware.renner.uno/intel-gpu: "true" securityContext: runAsNonRoot: true runAsUser: 1000 diff --git a/kubernetes/apps/llama-server/embed.yaml b/kubernetes/apps/llama-server/embed.yaml index 5382f32d..9dd13a49 100644 --- a/kubernetes/apps/llama-server/embed.yaml +++ b/kubernetes/apps/llama-server/embed.yaml @@ -34,6 +34,9 @@ spec: labels: app.kubernetes.io/name: llama-embed spec: + # Runs only where the node patch declares the Intel iGPU. + nodeSelector: + hardware.renner.uno/intel-gpu: "true" securityContext: runAsNonRoot: true runAsUser: 1000 diff --git a/kubernetes/apps/llama-server/wiki-embed.yaml b/kubernetes/apps/llama-server/wiki-embed.yaml index ba80bf11..405eb1cc 100644 --- a/kubernetes/apps/llama-server/wiki-embed.yaml +++ b/kubernetes/apps/llama-server/wiki-embed.yaml @@ -23,6 +23,9 @@ spec: labels: app.kubernetes.io/name: wiki-embed spec: + # Runs only where the node patch declares the Intel iGPU. + nodeSelector: + hardware.renner.uno/intel-gpu: "true" securityContext: runAsNonRoot: true runAsUser: 1000 diff --git a/kubernetes/apps/matrix/cluster.yaml b/kubernetes/apps/matrix/cluster.yaml index eae9dc88..b7d4a4f0 100644 --- a/kubernetes/apps/matrix/cluster.yaml +++ b/kubernetes/apps/matrix/cluster.yaml @@ -31,7 +31,7 @@ spec: plugin: name: barman-cloud.cloudnative-pg.io parameters: - barmanObjectName: mx-pg-store + barmanObjectName: mx-pg-store-source serverName: mx-pg postgresql: parameters: diff --git a/kubernetes/apps/matrix/kustomization.yaml b/kubernetes/apps/matrix/kustomization.yaml index 94a5282b..7453c856 100644 --- a/kubernetes/apps/matrix/kustomization.yaml +++ b/kubernetes/apps/matrix/kustomization.yaml @@ -9,6 +9,5 @@ resources: # mx-pg CNPG Postgres + offsite Barman backup. The media store and signing # key are on synapse-data, backed up by Velero (matrix-daily). - cluster.yaml - - objectstore.yaml - scheduled-backup.yaml - hetzner-creds-externalsecret.yaml diff --git a/kubernetes/apps/matrix/objectstore.yaml b/kubernetes/apps/matrix/objectstore.yaml deleted file mode 100644 index 808a04c1..00000000 --- a/kubernetes/apps/matrix/objectstore.yaml +++ /dev/null @@ -1,30 +0,0 @@ ---- -apiVersion: barmancloud.cnpg.io/v1 -kind: ObjectStore -metadata: - name: mx-pg-store - namespace: matrix -spec: - configuration: - # Offsite Hetzner Object Storage (fsn1), same bucket as the other CNPG - # clusters. No SSE: Hetzner supports only SSE-C, unusable by barman-cloud - # (plugin #646), so at-rest encryption is unavailable; transit is TLS. - destinationPath: s3://renner-uno-k8s-cnpg/mx-pg - endpointURL: https://fsn1.your-objectstorage.com - s3Credentials: - accessKeyId: - name: hetzner-s3-creds - key: ACCESS_KEY_ID - secretAccessKey: - name: hetzner-s3-creds - key: ACCESS_SECRET_KEY - # Region is required (REGION=fsn1) — without it boto3's HeadBucket - # (barman-cloud-check-wal-archive during bootstrap.recovery) 400s. - region: - name: hetzner-s3-creds - key: REGION - wal: - compression: gzip - data: - compression: gzip - retentionPolicy: "30d" diff --git a/kubernetes/apps/matter-server/deployment.yaml b/kubernetes/apps/matter-server/deployment.yaml index 9621177d..2587aec9 100644 --- a/kubernetes/apps/matter-server/deployment.yaml +++ b/kubernetes/apps/matter-server/deployment.yaml @@ -32,11 +32,9 @@ spec: # OTBR via OTBR's ClusterIP Service). The macvlan attachment # above is additive, not a replacement. # - # No nodeSelector/toleration: this pod only needs v30iot (present - # on all nodes), not the edge USB devices. Dropping the - # workload-class=pinned toleration lets the edge taint keep it off - # the (tiny) edge box, reserving that node for the USB-bound - # home-assistant + zwave-js pods. + # Runs where the node patch declares the IoT VLAN (ens19). + nodeSelector: + net.renner.uno/iot-vlan: "true" containers: - name: matter-server # Use the HAOS-published image from `ghcr.io/matter-js/...`, diff --git a/kubernetes/apps/monitoring-rules/authentik-outpost-servicemonitors.yaml b/kubernetes/apps/monitoring-rules/authentik-outpost-servicemonitors.yaml deleted file mode 100644 index 92371c68..00000000 --- a/kubernetes/apps/monitoring-rules/authentik-outpost-servicemonitors.yaml +++ /dev/null @@ -1,63 +0,0 @@ ---- -# Only radius needs a hand-written ServiceMonitor. The goauthentik.io operator -# creates one per outpost and Prometheus discovers those directly, because -# serviceMonitorSelector here is empty (see monitoring/helmrelease-prometheus.yaml). -# Standalone copies for ldap and paperless-gate used to live here as well; they -# duplicated the operator's and were removed. -# -# Radius requires extra plumbing. The radius binary listens on :9300 just -# like ldap/proxy (hardcoded in authentik/lib/default.yml), but authentik's -# Python operator (`providers/radius/controllers/kubernetes.py`) omits the -# `DeploymentPort(9300, "http-metrics", "tcp", 9300)` entry from its -# `deployment_ports` list — that's an upstream bug; the ldap controller has -# it, radius does not. Result: the operator-managed `ak-outpost-local-radius` -# Service exposes only UDP 1812. Add a sibling Service here that selects the -# same pod labels and exposes 9300/TCP, then a ServiceMonitor for it. -# -# The operator's own `ak-outpost-local-radius` ServiceMonitor cannot cover -# this: it selects on `app.kubernetes.io/managed-by: goauthentik.io` together -# with `goauthentik.io/service-type: metrics`, and no Service carries both — -# the operator's has the first, the one below has the second. -apiVersion: v1 -kind: Service -metadata: - name: ak-outpost-local-radius-metrics - namespace: authentik - labels: - app.kubernetes.io/managed-by: homelab - goauthentik.io/outpost-name: local-radius - goauthentik.io/outpost-type: radius -spec: - type: ClusterIP - clusterIP: None - selector: - goauthentik.io/outpost-name: local-radius - goauthentik.io/outpost-type: radius - ports: - - name: http-metrics - port: 9300 - targetPort: 9300 - protocol: TCP ---- -apiVersion: monitoring.coreos.com/v1 -kind: ServiceMonitor -metadata: - name: authentik-outpost-radius - namespace: authentik - labels: - release: kube-prometheus-stack -spec: - namespaceSelector: - matchNames: - - authentik - selector: - # Pinned to the Service above. Without managed-by this would also match the - # operator's Service and scrape the same endpoint twice. - matchLabels: - app.kubernetes.io/managed-by: homelab - goauthentik.io/outpost-name: local-radius - goauthentik.io/outpost-type: radius - endpoints: - - port: http-metrics - path: /metrics - interval: 30s diff --git a/kubernetes/apps/monitoring-rules/authentik-servicemonitor.yaml b/kubernetes/apps/monitoring-rules/authentik-servicemonitor.yaml new file mode 100644 index 00000000..2f49d266 --- /dev/null +++ b/kubernetes/apps/monitoring-rules/authentik-servicemonitor.yaml @@ -0,0 +1,24 @@ +--- +# authentik server metrics. The chart's metrics Service stays; only its +# ServiceMonitor is declared here (see apps/authentik/helmrelease.yaml). +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: authentik-server-metrics + namespace: authentik + labels: + release: kube-prometheus-stack +spec: + namespaceSelector: + matchNames: + - authentik + selector: + matchLabels: + app.kubernetes.io/name: authentik + app.kubernetes.io/instance: authentik + app.kubernetes.io/component: server-metrics + endpoints: + - port: metrics + path: /metrics + interval: 30s + scrapeTimeout: 3s diff --git a/kubernetes/apps/monitoring-rules/kustomization.yaml b/kubernetes/apps/monitoring-rules/kustomization.yaml index 43e3ed03..97f14961 100644 --- a/kubernetes/apps/monitoring-rules/kustomization.yaml +++ b/kubernetes/apps/monitoring-rules/kustomization.yaml @@ -49,7 +49,7 @@ resources: # cannot live beside the Velero HelmRelease. - velero-podmonitor.yaml - velero-servicemonitor.yaml - - authentik-outpost-servicemonitors.yaml + - authentik-servicemonitor.yaml # Rook's exporter/mgr monitors: the operator aborts its node-daemon reconcile # when it cannot create these, which held the CephCluster in Progressing and # deadlocked the whole apps layer. See the file for the cycle. diff --git a/kubernetes/apps/monitoring-rules/probes-blackbox.yaml b/kubernetes/apps/monitoring-rules/probes-blackbox.yaml index f2009161..83df9a47 100644 --- a/kubernetes/apps/monitoring-rules/probes-blackbox.yaml +++ b/kubernetes/apps/monitoring-rules/probes-blackbox.yaml @@ -269,9 +269,9 @@ spec: # per the app's own test (shelly-manager repo, # AuthenticationTests.Health_IsUnauthenticated); it is also the pod's # kubelet liveness/readiness path - # (../home-assistant/shelly-manager-deployment.yaml). AllowedHosts is + # (../home-assistant-iot/shelly-manager-deployment.yaml). AllowedHosts is # "*", so the cluster-local Host header is accepted. Service :80 -> - # container :8080 (../home-assistant/shelly-manager-service.yaml). + # container :8080 (../home-assistant-iot/shelly-manager-service.yaml). # # NO CiliumNetworkPolicy change: the home-assistant default-deny # admits the whole monitoring namespace on every port diff --git a/kubernetes/apps/monitoring/intel-gpu-exporter.yaml b/kubernetes/apps/monitoring/intel-gpu-exporter.yaml index 2ab049e1..5df5d89c 100644 --- a/kubernetes/apps/monitoring/intel-gpu-exporter.yaml +++ b/kubernetes/apps/monitoring/intel-gpu-exporter.yaml @@ -25,6 +25,8 @@ spec: - key: node-role.kubernetes.io/control-plane operator: Exists effect: NoSchedule + nodeSelector: + hardware.renner.uno/intel-gpu: "true" securityContext: seccompProfile: type: RuntimeDefault diff --git a/kubernetes/apps/music-assistant/deployment.yaml b/kubernetes/apps/music-assistant/deployment.yaml index 4dcecdb3..828ddd4e 100644 --- a/kubernetes/apps/music-assistant/deployment.yaml +++ b/kubernetes/apps/music-assistant/deployment.yaml @@ -33,12 +33,9 @@ spec: k8s.v1.cni.cncf.io/networks: music-assistant-iot spec: enableServiceLinks: false - # No nodeSelector/toleration: MA only needs a node on v30iot (all - # of them) for audio-sync L2, not the edge USB devices. It's the - # heaviest of the formerly edge-pinned pods (2Gi limit), so - # dropping the workload-class=pinned toleration — which lets the - # edge taint keep it off the tiny edge box — is the main capacity - # win for this step. + # Runs where the node patch declares the IoT VLAN (ens19). + nodeSelector: + net.renner.uno/iot-vlan: "true" initContainers: # MA advertises a stored publish_ip to players (Sendspin / Snapcast / # SlimProto) and binds its stream + Sendspin servers to a stored diff --git a/kubernetes/apps/otbr/deployment.yaml b/kubernetes/apps/otbr/deployment.yaml index d260664f..25381a25 100644 --- a/kubernetes/apps/otbr/deployment.yaml +++ b/kubernetes/apps/otbr/deployment.yaml @@ -30,11 +30,10 @@ spec: # the IoT VLAN for mDNS/SRP advertisements and RA listening. k8s.v1.cni.cncf.io/networks: otbr-iot spec: - # No nodeSelector/toleration: OTBR only needs v30iot L2 presence - # (macvlan master: ens19, present on every node), not the edge USB - # devices, and its Thread RCP is reached over the network - # (slzb-thread.renner.uno). Dropping the workload-class=pinned - # toleration lets the edge taint keep it off the tiny edge box. + # Runs where the node patch declares the IoT VLAN (ens19). Its Thread + # RCP is reached over the network (slzb-thread.renner.uno). + nodeSelector: + net.renner.uno/iot-vlan: "true" containers: # The Thread RCP is a network-attached radio (Serial-over-IP), so the # spinel transport needs a TCP-to-PTY bridge. Two obvious approaches diff --git a/kubernetes/apps/paperless/cluster.yaml b/kubernetes/apps/paperless/cluster.yaml index e50fbb8d..0cc852cf 100644 --- a/kubernetes/apps/paperless/cluster.yaml +++ b/kubernetes/apps/paperless/cluster.yaml @@ -39,7 +39,7 @@ spec: plugin: name: barman-cloud.cloudnative-pg.io parameters: - barmanObjectName: pprl-pg-store + barmanObjectName: pprl-pg-store-source serverName: pprl-pg postgresql: parameters: diff --git a/kubernetes/apps/paperless/kustomization.yaml b/kubernetes/apps/paperless/kustomization.yaml index 48b5e086..20ee427f 100644 --- a/kubernetes/apps/paperless/kustomization.yaml +++ b/kubernetes/apps/paperless/kustomization.yaml @@ -17,7 +17,6 @@ resources: # transferred without a teardown). This KS gains the infrastructure-cnpg-barman # dep + CNPG Cluster healthCheckExprs (see clusters/homelab/paperless.yaml). - cluster.yaml - - objectstore.yaml - scheduled-backup.yaml - hetzner-creds-externalsecret.yaml - pvcs.yaml diff --git a/kubernetes/apps/paperless/objectstore.yaml b/kubernetes/apps/paperless/objectstore.yaml deleted file mode 100644 index 03d006bf..00000000 --- a/kubernetes/apps/paperless/objectstore.yaml +++ /dev/null @@ -1,35 +0,0 @@ ---- -apiVersion: barmancloud.cnpg.io/v1 -kind: ObjectStore -metadata: - name: pprl-pg-store - namespace: paperless - annotations: - # Prune-disabled — a standing guard kept from the seed->paperless-KS handoff (backup target). - kustomize.toolkit.fluxcd.io/prune: disabled -spec: - configuration: - # Offsite Hetzner Object Storage (fsn1) — replaces local Garage on - # nas-1 so paperless backups survive nas-1 being powered off. barman - # uses boto3 default (auto) addressing; Hetzner (Ceph RGW) accepts it. - destinationPath: s3://renner-uno-k8s-cnpg/pprl-pg - endpointURL: https://fsn1.your-objectstorage.com - s3Credentials: - accessKeyId: - name: hetzner-s3-creds - key: ACCESS_KEY_ID - secretAccessKey: - name: hetzner-s3-creds - key: ACCESS_SECRET_KEY - region: - name: hetzner-s3-creds - key: REGION - # No `encryption: AES256` (SSE-S3): Hetzner Object Storage supports - # only SSE-C, which the barman-cloud plugin can't use (cnpg - # plugin-barman-cloud #646). At-rest encryption is therefore not - # available here; transit is TLS and the bucket is private. - wal: - compression: gzip - data: - compression: gzip - retentionPolicy: "30d" diff --git a/kubernetes/apps/trmnl/cluster.yaml b/kubernetes/apps/trmnl/cluster.yaml index 187c2050..e3b5554c 100644 --- a/kubernetes/apps/trmnl/cluster.yaml +++ b/kubernetes/apps/trmnl/cluster.yaml @@ -36,7 +36,7 @@ spec: plugin: name: barman-cloud.cloudnative-pg.io parameters: - barmanObjectName: tr-pg-store + barmanObjectName: tr-pg-store-source serverName: tr-pg postgresql: parameters: diff --git a/kubernetes/apps/trmnl/kustomization.yaml b/kubernetes/apps/trmnl/kustomization.yaml index 20b26fb7..011ba273 100644 --- a/kubernetes/apps/trmnl/kustomization.yaml +++ b/kubernetes/apps/trmnl/kustomization.yaml @@ -18,6 +18,5 @@ resources: # tr-pg CNPG Postgres + offsite Barman backup. Note this cluster runs # PostgreSQL 18, not the repo-standard 16 — see the comment in cluster.yaml. - cluster.yaml - - objectstore.yaml - scheduled-backup.yaml - hetzner-creds-externalsecret.yaml diff --git a/kubernetes/apps/trmnl/objectstore.yaml b/kubernetes/apps/trmnl/objectstore.yaml deleted file mode 100644 index 61b0bc3b..00000000 --- a/kubernetes/apps/trmnl/objectstore.yaml +++ /dev/null @@ -1,30 +0,0 @@ ---- -apiVersion: barmancloud.cnpg.io/v1 -kind: ObjectStore -metadata: - name: tr-pg-store - namespace: trmnl -spec: - configuration: - # Offsite Hetzner Object Storage (fsn1), same bucket as the other CNPG - # clusters. No SSE: Hetzner supports only SSE-C, unusable by barman-cloud - # (plugin #646), so at-rest encryption is unavailable; transit is TLS. - destinationPath: s3://renner-uno-k8s-cnpg/tr-pg - endpointURL: https://fsn1.your-objectstorage.com - s3Credentials: - accessKeyId: - name: hetzner-s3-creds - key: ACCESS_KEY_ID - secretAccessKey: - name: hetzner-s3-creds - key: ACCESS_SECRET_KEY - # Region is required (REGION=fsn1) — without it boto3's HeadBucket - # (barman-cloud-check-wal-archive during bootstrap.recovery) 400s. - region: - name: hetzner-s3-creds - key: REGION - wal: - compression: gzip - data: - compression: gzip - retentionPolicy: "30d" diff --git a/kubernetes/apps/wiki/deployment.yaml b/kubernetes/apps/wiki/deployment.yaml index 5c12bf59..970c51a2 100644 --- a/kubernetes/apps/wiki/deployment.yaml +++ b/kubernetes/apps/wiki/deployment.yaml @@ -39,6 +39,10 @@ spec: labels: app.kubernetes.io/name: wiki spec: + # Its embedding backend (wiki-embed) needs the Intel iGPU; without it + # the wiki has nothing to serve, so it runs where the iGPU is declared. + nodeSelector: + hardware.renner.uno/intel-gpu: "true" automountServiceAccountToken: false securityContext: runAsNonRoot: true diff --git a/kubernetes/apps/woodpecker/kustomization.yaml b/kubernetes/apps/woodpecker/kustomization.yaml index b820f6b3..16709f57 100644 --- a/kubernetes/apps/woodpecker/kustomization.yaml +++ b/kubernetes/apps/woodpecker/kustomization.yaml @@ -34,6 +34,5 @@ resources: # This KS depends on infrastructure-cnpg-barman for the barman plugin behind # the ObjectStore and ScheduledBackup. - hetzner-creds-externalsecret.yaml - - objectstore.yaml - cluster.yaml - scheduled-backup.yaml diff --git a/kubernetes/bootstrap.sh b/kubernetes/bootstrap.sh index f2872850..386e08ce 100755 --- a/kubernetes/bootstrap.sh +++ b/kubernetes/bootstrap.sh @@ -15,10 +15,17 @@ set -euo pipefail # # the only place the registry fallback can be # # tested: gitea.renner.uno has no external # # ingress, so a node there genuinely cannot -# # reach it. BILLS BY THE HOUR — teardown runs -# # from the EXIT trap and is by label. -# ./bootstrap.sh hcloudclean # that teardown on its own, for a run that was -# # killed before its trap could fire. +# # reach it. BILLS BY THE HOUR — the machines +# # stay up until hcloudclean. Access files land in +# # ~/.local/state/homelab-hcloudtest/. +# ./bootstrap.sh hcloudclean # deletes everything hcloudtest created, by +# # label, and the restore test's deploy keys. +# +# Restore test on the hcloudtest cluster (production tree, read-only source, +# kubernetes/restore-test/flux-root.yaml): +# KUBECONFIG=~/.local/state/homelab-hcloudtest/kubeconfig \ +# TALOSCONFIG=~/.local/state/homelab-hcloudtest/talosconfig \ +# FLUX_SOURCE=restore-test ./bootstrap.sh flux # # Flux git source (break-glass): the in-cluster Gitea that Flux normally pulls # from is itself Flux-managed, so a cold rebuild cannot pull from it until the @@ -39,6 +46,7 @@ REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)" # needs is read out of this file at run time so there is no second copy of a # version or a value to keep in step. See PROPOSAL-cilium-single-owner.md. CILIUM_HR="$REPO_ROOT/kubernetes/infrastructure/cilium/helmrelease.yaml" +K8SGW_HR="$REPO_ROOT/kubernetes/infrastructure/k8s-gateway/helmrelease.yaml" ACCESS_STAGING="$REPO_ROOT/kubernetes/access-staging" NODES=(k8s-1 k8s-2 k8s-3) # No dedicated workers since the k8s-edge VM was retired (2026-07-02) — @@ -176,6 +184,10 @@ resolve_node_ip() { phase_runs() { [[ "$PHASE" == "all" || "$PHASE" == "$1" ]] } +# The restore test targets the cluster bootstrap.sh hcloudtest builds. +if [[ "$FLUX_SOURCE" == "restore-test" && "$PHASE" != "flux" && "$PHASE" != "wait-apps" ]]; then + fail "FLUX_SOURCE=restore-test runs the flux and wait-apps phases only" +fi OPENBAO_NS="openbao" # Where the secrets come from. On a workstation, the operator's pass store. In @@ -217,7 +229,7 @@ OPENBAO_SECRETS_DIR="${OPENBAO_SECRETS_DIR:-}" # that cannot be told apart from a fault. Its non-zero exit propagates, which is # what makes require_secrets print the reason instead. git_token_for_source() { - if [[ "$FLUX_SOURCE" == "github" ]]; then + if [[ "$FLUX_SOURCE" == "github" || "$FLUX_SOURCE" == "restore-test" ]]; then if [[ -n "${GITHUB_TOKEN:-}" ]]; then printf '%s' "$GITHUB_TOKEN" else @@ -255,13 +267,13 @@ OPENBAO_REQUIRED_SECRETS=( # rest, because a rebuild that discovers it missing has already wiped the # disks. "ghcr-pull|GHCR pull token for the image mirror|pass show homelab/github/k8s-package-pull" - # Only the hcloudtest and hcloudclean phases ask for this one, by name, so a - # rebuild of the real cluster never demands it. - # - # Note the path: homelab/hetzner/, spelled correctly, while the two restore - # keys above live under homelab/hetztner/. Two folders, one typo, both real - # — do not "fix" one of these paths without moving the entry first. - "hcloud-token|Hetzner Cloud API token for the throwaway test cluster|pass show homelab/hetzner/cloud-api-secret" +) +# Asked for by name only, never by a bare require_secrets: a rebuild of the +# real cluster cannot depend on the OpenBao it is rebuilding. Scoped to the +# Hetzner project homelab-backup-verify, which holds nothing but throwaway +# machines. +HCLOUD_REQUIRED_SECRETS=( + "hcloud-token|Hetzner Cloud API token for the throwaway test cluster|bao kv get -mount=homelab -field=token infrastructure/hcloud-backup-verify" ) # Collects all of them up front, into one directory, so everything after this @@ -288,7 +300,7 @@ require_secrets() { local entry name what how local -a wanted=() if [[ $# -gt 0 ]]; then - for entry in "${OPENBAO_REQUIRED_SECRETS[@]}"; do + for entry in "${OPENBAO_REQUIRED_SECRETS[@]}" "${HCLOUD_REQUIRED_SECRETS[@]}"; do for name in "$@"; do [[ "${entry%%|*}" == "$name" ]] && wanted+=("$entry") done @@ -382,6 +394,155 @@ read_secret() { cat "$OPENBAO_SECRETS_DIR/$name" } +# Hetzner Object Storage, where the offsite backups live. Read with the restore +# key (s3-access, s3-secret). +S3_ENDPOINT="fsn1.your-objectstorage.com" +S3_REGION="fsn1" + +# One request, signed by curl with the restore key. The body goes to stdout; +# anything but a 2xx fails, with S3's error document on stderr. +s3_request() { + local method=$1 path=$2 body code + body=$(mktemp) + if ! code=$(curl -sS -o "$body" -w '%{http_code}' -X "$method" \ + --aws-sigv4 "aws:amz:$S3_REGION:s3" \ + --user "$(read_secret s3-access):$(read_secret s3-secret)" \ + "https://$S3_ENDPOINT/$path"); then + rm -f "$body" + return 1 + fi + if [[ "$code" != 2?? ]]; then + echo "$method $path: HTTP $code" >&2 + cat "$body" >&2 + echo >&2 + rm -f "$body" + return 1 + fi + cat "$body" + rm -f "$body" +} + +# The "directories" directly under PREFIX in BUCKET, one per line, without +# PREFIX and the trailing slash. +s3_dirs() { + local bucket=$1 prefix=$2 xml + xml=$(s3_request GET "$bucket?list-type=2&delimiter=%2F&prefix=${prefix//\//%2F}") || return 1 + [[ "$xml" == *"false"* ]] \ + || { echo "the listing of $bucket/$prefix is truncated or not a listing" >&2; return 1; } + # awk, not grep: no match is not an error. + awk -v p="$prefix" '{ + while (match($0, /[^<]*<\/Prefix>/)) { + d = substr($0, RSTART + 24, RLENGTH - 33) + $0 = substr($0, RSTART + RLENGTH) + sub(/\/$/, "", d) + print substr(d, length(p) + 1) + } + }' <<<"$xml" +} + +# The bucket every CNPG cluster recovers from (cnpg-object-stores/). The +# restore test archives to CNPG_TEST_BUCKET instead: a test incarnation with a +# base backup in the production bucket would be the newest there, and the next +# real recovery would read the test's data. +CNPG_BUCKET="renner-uno-k8s-cnpg" +CNPG_TEST_BUCKET="renner-uno-k8s-cnpg-rtest" +CNPG_STORES_DIR="$REPO_ROOT/kubernetes/cnpg-object-stores" + +# cnpg-paths, the ConfigMap cnpg-object-stores takes its paths from. Written +# once per cluster incarnation: an existing one belongs to clusters already +# archiving under its paths and is kept. +# +# For each source ObjectStore in cnpg-object-stores/, cluster : +# CNPG__SOURCE the candidate prefix whose /base/ holds the newest +# base backup. Candidates: , -g<14 digits> and the +# store's cnpg.renner.uno/source-prefixes. +# CNPG__ARCHIVE -g in ARCHIVE_BUCKET: new, so empty, so the +# plugin's empty-archive check passes rather than being +# skipped. +write_cnpg_paths() { + local archive_bucket=$1 + local gen dirs name extra var p listing newest best best_at + local -a literals=() + echo "Writing the CNPG backup paths (cnpg-paths)..." + if kubectl -n flux-system get configmap cnpg-paths -o name >/dev/null 2>"$OPENBAO_SECRETS_DIR/.err"; then + echo " already present, kept" + return 0 + fi + grep -q NotFound "$OPENBAO_SECRETS_DIR/.err" \ + || { cat "$OPENBAO_SECRETS_DIR/.err" >&2; fail "could not read the cnpg-paths ConfigMap"; } + + gen=g$(date -u +%Y%m%d%H%M%S) + dirs=$(s3_dirs "$CNPG_BUCKET" "") || fail "could not list $CNPG_BUCKET" + while IFS=$'\t' read -r name extra; do + best='' + best_at='' + for p in $dirs; do + [[ "$p" == "$name" || "$p" =~ ^${name}-g[0-9]{14}$ || " $extra " == *" $p "* ]] || continue + listing=$(s3_dirs "$CNPG_BUCKET" "$p/$name/base/") \ + || fail "could not list $CNPG_BUCKET/$p/$name/base/" + newest=$(sort <<<"$listing" | tail -1) + if [[ -n "$newest" && "$newest" > "$best_at" ]]; then + best=$p + best_at=$newest + fi + done + [[ -n "$best" ]] \ + || fail "no base backup of $name in $CNPG_BUCKET (looked under $name, $name-g${extra:+, $extra})" + var=CNPG_$(tr 'a-z-' 'A-Z_' <<<"$name") + literals+=("--from-literal=${var}_SOURCE=s3://$CNPG_BUCKET/$best" + "--from-literal=${var}_ARCHIVE=s3://$archive_bucket/$name-$gen") + echo " $name: recovers from $CNPG_BUCKET/$best (base backup $best_at), archives to $archive_bucket/$name-$gen" + done < <(yq -r 'select(.kind == "ObjectStore" and .metadata.annotations["cnpg.renner.uno/server-name"] != null) + | [.metadata.annotations["cnpg.renner.uno/server-name"], + (.metadata.annotations["cnpg.renner.uno/source-prefixes"] // "")] | @tsv' \ + "$CNPG_STORES_DIR"/*.yaml) + [[ ${#literals[@]} -gt 0 ]] \ + || fail "no source ObjectStore (cnpg.renner.uno/server-name) in $CNPG_STORES_DIR" + kubectl -n flux-system create configmap cnpg-paths "${literals[@]}" >/dev/null \ + || fail "could not create the cnpg-paths ConfigMap" + ok +} + +# The restore test's archive bucket. Created when missing; on a rerun it +# already exists, which S3 reports as an error. +create_cnpg_test_bucket() { + local out + out=$(s3_request PUT "$CNPG_TEST_BUCKET" 2>&1) && return 0 + [[ "$out" == *BucketAlreadyOwnedByYou* ]] && return 0 + echo "$out" >&2 + fail "could not create $CNPG_TEST_BUCKET" +} + +# Empties and deletes the restore test's archive bucket. Each pass deletes the +# first page of keys and lists again, so no continuation token is needed. +delete_cnpg_test_bucket() { + local xml key + while :; do + if ! xml=$(s3_request GET "$CNPG_TEST_BUCKET?list-type=2" 2>&1); then + [[ "$xml" == *NoSuchBucket* ]] && return 0 + echo "$xml" >&2 + echo " WARNING: $CNPG_TEST_BUCKET not emptied — check the console" >&2 + return 0 + fi + while read -r key; do + [[ -n "$key" ]] || continue + s3_request DELETE "$CNPG_TEST_BUCKET/$key" >/dev/null \ + || { echo " WARNING: $CNPG_TEST_BUCKET/$key not deleted — check the console" >&2; return 0; } + done < <(awk '{ + while (match($0, /[^<]*<\/Key>/)) { + print substr($0, RSTART + 5, RLENGTH - 11) + $0 = substr($0, RSTART + RLENGTH) + } + }' <<<"$xml") + [[ "$xml" == *"true"* ]] || break + done + if s3_request DELETE "$CNPG_TEST_BUCKET" >/dev/null; then + echo " deleted bucket $CNPG_TEST_BUCKET" + else + echo " WARNING: bucket $CNPG_TEST_BUCKET not deleted — check the console" >&2 + fi +} + # Provision a throwaway cluster on Hetzner Cloud, then run the Talos phase # against it unchanged — same shape as the vmtest phase below, and for the same # reason: a rehearsal that runs different code proves less than one that does @@ -405,15 +566,18 @@ read_secret() { # which does nothing but the sweep. Anything this phase creates carries # HCLOUD_LABEL so that sweep can find it without being told. # -# WHAT IT DOES NOT COVER. +# WHAT IT IS. # -# Everything downstream of Multus. The provider is not the limit — a cloud -# server takes up to 16 volumes and 3 private networks — but ceph-san binds to -# the kernel link name enp2s0f1np1, and that name cannot be manufactured here -# without assuming a Hetzner private network passes VLAN tags. See -# talos/patches/site-hcloud.yaml. Nor Flux and everything after it: that phase -# needs the OpenBao seal key, which deliberately never reaches CI, and this -# test is meant to run unattended. +# The production config on new hardware: cluster.yaml, site-hcloud.yaml (what +# the Hetzner site is), containerd-layer-retention.yaml and hcloud-N.yaml (each +# machine). Nothing stands in for the home network. Flux is a separate run, +# FLUX_SOURCE=restore-test bootstrap.sh flux, because it needs the OpenBao seal +# key. Ceph replicates over the pod network there: Hetzner networks are layer 3 +# (restore-test/flux-root.yaml). +# +# Containment: an outbound Hetzner firewall on every node admits udp and icmp +# anywhere and tcp anywhere except the addresses of proxy.renner.uno, the public +# edge in front of production. # Talos version and Image Factory schematic, read out of the installer image in # cluster.yaml. Both throwaway-cluster phases need them and neither may state # them: a second copy is a copy that goes stale, and the day it did the test @@ -433,7 +597,7 @@ HCLOUD_LABEL="managed-by=bootstrap-hcloudtest" # The disks a machine has are stated in its Talos patch and read from there, so # the provisioner carries no second copy. Emits one size in whole GB per raw -# volume the patch declares, in order. +# volume the patch declares off the system disk, in order. hcloud_volumes_for_node() { local patch="$TALOS_DIR/patches/$1.yaml" size gib [[ -f "$patch" ]] || fail "$patch not found — the machine config is the list" @@ -444,22 +608,260 @@ hcloud_volumes_for_node() { gib=${size%GiB} [[ "$gib" =~ ^[0-9]+$ ]] || fail "$patch: cannot read volume size '$size' — state it as GiB" echo $(( (gib * 1074 + 999) / 1000 )) - done < <(yq -r 'select(.kind == "RawVolumeConfig") | .provisioning.minSize' "$patch") + done < <(yq -r 'select(.kind == "RawVolumeConfig" and .provisioning.diskSelector.match != "system_disk") | .provisioning.minSize' "$patch") } # Deletes everything this phase created, and NOTHING else. # -# That distinction is not theoretical: the project this runs in also holds -# netbird.renner.uno, which has been up for months and is the VPN way in. The -# label is the only thing separating a throwaway server from it — and -# `hcloud server list -l ""` lists EVERY server, so an unset HCLOUD_LABEL would -# turn this function into a project wipe. Hence the guard: the label is checked -# before it is used, and every candidate must also carry the name prefix this -# phase assigns. Two independent conditions, because one of them is a variable. -# -# A separate Hetzner project, with a token scoped to it, would make this -# structural rather than defensive. Worth doing before this runs unattended. +# The project (homelab-backup-verify) also runs the backup-verify CronJob's +# machines. `hcloud server list -l ""` lists EVERY server, so an unset +# HCLOUD_LABEL would turn this function into a project wipe: the label is +# checked before it is used, and every candidate must also carry the name +# prefix this phase assigns. HCLOUD_NAME_PREFIX="hcloud" +# Outlives the run: the test cluster's talosconfig and kubeconfig. +HCLOUD_STATE_DIR="${XDG_STATE_HOME:-$HOME/.local/state}/homelab-hcloudtest" +# The public edge in front of production. The test cluster must not reach it. +HCLOUD_PROXY_HOST="proxy.renner.uno" +# Hetzner's limit on effective rules per firewall and direction. +HCLOUD_FW_RULE_LIMIT=500 +# Hetzner's limit on prefixes in one rule (not documented; 101 is rejected). +HCLOUD_FW_RULE_PREFIXES=100 + +# The Talos snapshot this cluster installs from, into HCLOUD_IMAGE. Built when +# missing: a throwaway server in rescue mode writes the Image Factory hcloud +# image to its disk, which is then snapshotted. Matched by version and +# schematic. Called directly, not in $(...), so that fail stops the run. +hcloud_talos_snapshot() { + local labels="os=talos,talos-version=$VM_VERSION,talos-schematic=${VM_SCHEMATIC:0:16}" + local builder="$HCLOUD_NAME_PREFIX-imgbuild" key_dir addr ids listing + listing=$(hcloud image list --type snapshot -l "$labels" -o noheader -o columns=id) \ + || fail "could not list the Talos snapshots" + mapfile -t ids <<<"$listing" + if [[ -n "${ids[0]:-}" ]]; then + HCLOUD_IMAGE=${ids[0]} + return 0 + fi + echo " no snapshot for $VM_VERSION/${VM_SCHEMATIC:0:16}, building one" + key_dir=$(mktemp -d) + ssh-keygen -q -t ed25519 -N '' -C "$builder" -f "$key_dir/key" + if hcloud ssh-key list -l "$HCLOUD_LABEL" -o noheader -o columns=name | grep -qx "$builder"; then + hcloud ssh-key delete "$builder" >/dev/null || fail "could not replace the stale image builder key" + fi + hcloud ssh-key create --name "$builder" --label "$HCLOUD_LABEL" \ + --public-key-from-file "$key_dir/key.pub" >/dev/null || fail "could not upload the image builder key" + hcloud server create --name "$builder" --type cx23 --image debian-12 \ + --location "${HCLOUD_LOCATION:-fsn1}" --ssh-key "$builder" --label "$HCLOUD_LABEL" >/dev/null \ + || fail "could not create the image builder" + # From here a failure leaves a billed server: remove it before failing. + builder_fail() { + hcloud server delete "$builder" >/dev/null || echo " WARNING: $builder not deleted" >&2 + hcloud ssh-key delete "$builder" >/dev/null || echo " WARNING: ssh key $builder not deleted" >&2 + rm -rf "$key_dir" + fail "$1" + } + hcloud server enable-rescue "$builder" --ssh-key "$builder" >/dev/null || builder_fail "could not enable rescue on $builder" + hcloud server reset "$builder" >/dev/null || builder_fail "could not reset $builder into rescue" + addr=$(hcloud server ip "$builder") || builder_fail "could not read the address of $builder" + for i in $(seq 1 30); do + ssh -i "$key_dir/key" -o BatchMode=yes -o ConnectTimeout=10 \ + -o UserKnownHostsFile="$key_dir/known_hosts" -o StrictHostKeyChecking=accept-new \ + "root@$addr" 'grep -q rescue /etc/hostname || hostname | grep -q rescue' && break + sleep 10 + [[ $i -eq 30 ]] && builder_fail "$builder never came up in rescue" + done + ssh -i "$key_dir/key" -o BatchMode=yes -o UserKnownHostsFile="$key_dir/known_hosts" "root@$addr" \ + "set -eo pipefail; curl -fsSL https://factory.talos.dev/image/$VM_SCHEMATIC/$VM_VERSION/hcloud-amd64.raw.xz | xz -d | dd of=/dev/sda bs=4M conv=fsync" \ + || builder_fail "writing the Talos image on $builder failed" + hcloud server poweroff "$builder" >/dev/null || builder_fail "could not power off $builder" + hcloud server create-image "$builder" --type snapshot \ + --description "talos $VM_VERSION ${VM_SCHEMATIC:0:16}" --label "$labels" \ + || builder_fail "could not snapshot $builder" + listing=$(hcloud image list --type snapshot -l "$labels" -o noheader -o columns=id) \ + || builder_fail "could not list the Talos snapshots" + mapfile -t ids <<<"$listing" + [[ -n "${ids[0]:-}" ]] || builder_fail "the snapshot of $builder is not listed under $labels" + HCLOUD_IMAGE=${ids[0]} + hcloud server delete "$builder" >/dev/null || echo " WARNING: $builder not deleted" >&2 + hcloud ssh-key delete "$builder" >/dev/null || echo " WARNING: ssh key $builder not deleted" >&2 + rm -rf "$key_dir" +} + +# One address as a string of bits: 32 for IPv4, 128 for IPv6. Returns 1, with +# the reason on stderr, for anything else. +ip_to_bits() { + local addr=$1 bits="" part v i head tail + local -a parts=() hparts=() tparts=() + if [[ "$addr" == *:* ]]; then + if [[ "$addr" == *::* ]]; then + head=${addr%%::*}; tail=${addr#*::} + [[ -n "$head" ]] && IFS=: read -r -a hparts <<<"$head" + [[ -n "$tail" ]] && IFS=: read -r -a tparts <<<"$tail" + parts=("${hparts[@]}") + for (( i = ${#hparts[@]} + ${#tparts[@]}; i < 8; i++ )); do parts+=(0); done + parts+=("${tparts[@]}") + else + IFS=: read -r -a parts <<<"$addr" + fi + [[ ${#parts[@]} -eq 8 ]] || { echo "cannot read IPv6 address '$addr'" >&2; return 1; } + for part in "${parts[@]}"; do + [[ "$part" =~ ^[0-9a-fA-F]{1,4}$ ]] || { echo "cannot read IPv6 address '$addr'" >&2; return 1; } + v=$(( 16#$part )) + for (( i = 15; i >= 0; i-- )); do bits+=$(( (v >> i) & 1 )); done + done + else + IFS=. read -r -a parts <<<"$addr" + [[ ${#parts[@]} -eq 4 ]] || { echo "cannot read IPv4 address '$addr'" >&2; return 1; } + for part in "${parts[@]}"; do + [[ "$part" =~ ^[0-9]{1,3}$ ]] || { echo "cannot read IPv4 address '$addr'" >&2; return 1; } + v=$(( 10#$part )) + (( v <= 255 )) || { echo "cannot read IPv4 address '$addr'" >&2; return 1; } + for (( i = 7; i >= 0; i-- )); do bits+=$(( (v >> i) & 1 )); done + done + fi + printf '%s' "$bits" +} + +# A prefix given as bits, in CIDR notation. $2 is the address length (32|128). +bits_to_cidr() { + local p=$1 width=$2 full out="" i + full=$p + while (( ${#full} < width )); do full+="0"; done + if (( width == 32 )); then + for (( i = 0; i < 32; i += 8 )); do out+="${out:+.}$(( 2#${full:i:8} ))"; done + else + for (( i = 0; i < 128; i += 16 )); do out+="${out:+:}$(printf '%x' $(( 2#${full:i:16} )))"; done + fi + printf '%s/%s\n' "$out" "${#p}" +} + +# cidr_complement
...: the prefixes that, with the given addresses +# of one family, cover 0.0.0.0/0 or ::/0 exactly once. Returns 1, with the +# reason on stderr, on a malformed or mixed-family list. +cidr_complement() { + local width=32 a bits + local -a excl=() + [[ "$1" == *:* ]] && width=128 + for a in "$@"; do + [[ ( "$a" == *:* && $width -eq 128 ) || ( "$a" != *:* && $width -eq 32 ) ]] \ + || { echo "cidr_complement: '$a' is not of the same family as '$1'" >&2; return 1; } + bits=$(ip_to_bits "$a") || return 1 + excl+=("$bits") + done + _cidr_walk() { + local p=$1 x hit=0 + for x in "${excl[@]}"; do + [[ "$x" == "$p"* ]] && { hit=1; break; } + done + if (( hit == 0 )); then + bits_to_cidr "$p" "$width" + return 0 + fi + (( ${#p} == width )) && return 0 + _cidr_walk "${p}0" + _cidr_walk "${p}1" + } + _cidr_walk "" +} + +# Addresses of HCLOUD_PROXY_HOST for one record type (A|AAAA), over DoH: the +# operator's LAN resolver intercepts plain DNS. Returns 1, with the reason on +# stderr, when the lookup fails. +hcloud_proxy_addrs() { + local type=$1 num answer + case "$type" in + A) num=1 ;; + AAAA) num=28 ;; + *) echo "hcloud_proxy_addrs: unknown type $type" >&2; return 1 ;; + esac + answer=$(curl -sS --max-time 10 -H 'accept: application/dns-json' \ + "https://cloudflare-dns.com/dns-query?name=$HCLOUD_PROXY_HOST&type=$type") \ + || { echo "could not resolve $HCLOUD_PROXY_HOST $type over DoH" >&2; return 1; } + jq -e '.Status == 0' >/dev/null <<<"$answer" \ + || { echo "DoH lookup of $HCLOUD_PROXY_HOST $type failed: $answer" >&2; return 1; } + jq -r --argjson t "$num" '.Answer[]? | select(.type == $t) | .data' <<<"$answer" | sort -u +} + +# Outbound firewall for the nodes: udp and icmp anywhere, tcp anywhere except +# HCLOUD_PROXY_HOST. Attached before an earlier run's egress firewall is +# removed, so no node is ever without one. +hcloud_apply_egress() { + local fw="hcloudtest-node-egress-$$" v4 v6 c4 c6 n count fw_id fw_name srv_id + local -a a4=() a6=() tcp_dst=() + v4=$(hcloud_proxy_addrs A) || fail "could not resolve $HCLOUD_PROXY_HOST (A)" + v6=$(hcloud_proxy_addrs AAAA) || fail "could not resolve $HCLOUD_PROXY_HOST (AAAA)" + [[ -n "$v4" ]] && mapfile -t a4 <<<"$v4" + [[ -n "$v6" ]] && mapfile -t a6 <<<"$v6" + (( ${#a4[@]} + ${#a6[@]} > 0 )) || fail "$HCLOUD_PROXY_HOST resolved to no address — refusing to build the egress firewall" + echo " blocking tcp to $HCLOUD_PROXY_HOST: ${a4[*]} ${a6[*]}" + if (( ${#a4[@]} > 0 )); then + c4=$(cidr_complement "${a4[@]}") || fail "could not compute the IPv4 complement of ${a4[*]}" + else + c4=0.0.0.0/0 + fi + if (( ${#a6[@]} > 0 )); then + c6=$(cidr_complement "${a6[@]}") || fail "could not compute the IPv6 complement of ${a6[*]}" + else + c6=::/0 + fi + mapfile -t tcp_dst <<<"$c4"$'\n'"$c6" + # Effective rules are counted per destination prefix; 0.0.0.0/0 and ::/0 + # carry udp and icmp. + count=$(printf '%s\n' 0.0.0.0/0 ::/0 "${tcp_dst[@]}" | sort -u | grep -c .) + (( count < HCLOUD_FW_RULE_LIMIT )) \ + || fail "the egress firewall needs $count effective rules, Hetzner allows $HCLOUD_FW_RULE_LIMIT per firewall" + hcloud firewall create --name "$fw" --label "$HCLOUD_LABEL" >/dev/null || fail "could not create $fw" + hcloud firewall add-rule "$fw" --direction out --protocol udp --port any \ + --destination-ips 0.0.0.0/0,::/0 --description "udp" >/dev/null || fail "could not add udp to $fw" + hcloud firewall add-rule "$fw" --direction out --protocol icmp \ + --destination-ips 0.0.0.0/0,::/0 --description "icmp" >/dev/null || fail "could not add icmp to $fw" + for (( n = 0; n < ${#tcp_dst[@]}; n += HCLOUD_FW_RULE_PREFIXES )); do + hcloud firewall add-rule "$fw" --direction out --protocol tcp --port any \ + --destination-ips "$(IFS=,; printf '%s' "${tcp_dst[*]:n:HCLOUD_FW_RULE_PREFIXES}")" \ + --description "tcp except $HCLOUD_PROXY_HOST" >/dev/null || fail "could not add tcp to $fw" + done + for n in "${HCLOUD_NODES[@]}"; do + hcloud firewall apply-to-resource "$fw" --type server --server "$n" >/dev/null \ + || fail "could not attach $fw to $n" + done + while read -r fw_id fw_name; do + [[ -n "$fw_id" && "$fw_name" == hcloudtest-node-egress-* && "$fw_name" != "$fw" ]] || continue + while read -r srv_id; do + [[ -n "$srv_id" ]] || continue + hcloud firewall remove-from-resource "$fw_id" --type server --server "$srv_id" >/dev/null \ + || fail "could not detach $fw_name from server $srv_id" + done < <(hcloud firewall describe "$fw_id" -o json | jq -r '.applied_to[] | select(.type == "server") | .server.id') + hcloud firewall delete "$fw_id" >/dev/null || fail "could not delete the previous firewall $fw_name" + echo " removed $fw_name" + done < <(hcloud firewall list -l "$HCLOUD_LABEL" -o noheader -o columns=id,name) + echo " $fw: $count effective rules" +} + +# Runs restore-test/egress-check.yaml in the cluster KUBECONFIG names and fails +# unless every expectation holds. +hcloud_egress_check() { + local addrs phase="" + step 0 "Checking egress containment from a pod..." + addrs=$( { hcloud_proxy_addrs A; hcloud_proxy_addrs AAAA; } | paste -sd' ' -) \ + || fail "could not resolve $HCLOUD_PROXY_HOST" + [[ -n "$addrs" ]] || fail "$HCLOUD_PROXY_HOST resolved to no address" + kubectl delete pod egress-check --ignore-not-found --wait=true >/dev/null + sed "s|PROXY_ADDRS_PLACEHOLDER|$addrs|" "$REPO_ROOT/kubernetes/restore-test/egress-check.yaml" \ + | kubectl apply -f - >/dev/null || fail "could not start the egress check" + for i in $(seq 1 48); do + phase=$(kubectl get pod egress-check -o jsonpath='{.status.phase}') || phase="" + [[ "$phase" == Succeeded || "$phase" == Failed ]] && break + sleep 5 + done + if [[ "$phase" != Succeeded && "$phase" != Failed ]]; then + kubectl describe pod egress-check | sed 's/^/ /' || true + fail "the egress check never finished (phase '${phase:-none}') — do not start Flux" + fi + kubectl logs egress-check | sed 's/^/ /' || true + [[ "$phase" == "Succeeded" ]] || fail "egress containment does not hold — do not start Flux" + kubectl delete pod egress-check --wait=false >/dev/null + ok +} cleanup_hcloudtest() { command -v hcloud >/dev/null || return 0 @@ -490,6 +892,11 @@ cleanup_hcloudtest() { hcloud volume delete "$id" >/dev/null \ || echo " WARNING: volume $id not deleted — check the console" >&2 done + for id in $(hcloud ssh-key list -l "$HCLOUD_LABEL" -o noheader -o columns=id); do + echo " deleting ssh key $id" + hcloud ssh-key delete "$id" >/dev/null \ + || echo " WARNING: ssh key $id not deleted — check the console" >&2 + done # Last: a network with a server still attached cannot be deleted, and a # leftover one takes its name with it, so the next run cannot recreate it. for id in $(hcloud network list -l "$HCLOUD_LABEL" -o noheader -o columns=id); do @@ -499,6 +906,17 @@ cleanup_hcloudtest() { done } +# The restore test's source, as restore-test/flux-root.yaml names it: sets +# restore_url and restore_repo (/). +restore_test_repo() { + command -v yq >/dev/null || fail "yq not found (reads restore-test/flux-root.yaml)" + restore_url=$(yq -r 'select(.kind == "GitRepository" and .metadata.name == "restore-test") | .spec.url' \ + "$REPO_ROOT/kubernetes/restore-test/flux-root.yaml") + [[ "$restore_url" =~ ^ssh://git@github\.com/([^/]+)/([^/]+)\.git$ ]] \ + || fail "restore-test/flux-root.yaml names '$restore_url', not ssh://git@github.com//.git" + restore_repo="${BASH_REMATCH[1]}/${BASH_REMATCH[2]}" +} + # Sweep only. For the case where a previous run was killed hard enough that its # own trap never ran, and for a cron job to call before it starts. if [[ "$PHASE" == "hcloudclean" ]]; then @@ -507,6 +925,20 @@ if [[ "$PHASE" == "hcloudclean" ]]; then require_secrets hcloud-token HCLOUD_TOKEN=$(read_secret hcloud-token); export HCLOUD_TOKEN cleanup_hcloudtest + require_secrets s3-access s3-secret + delete_cnpg_test_bucket + # The restore test's read-only deploy keys on the GitHub mirror. + if command -v gh >/dev/null; then + restore_test_repo + for key_id in $(gh api "repos/$restore_repo/keys" \ + --jq '.[] | select(.title | startswith("restore-test ")) | .id'); do + echo " deleting deploy key $key_id" + gh api -X DELETE "repos/$restore_repo/keys/$key_id" \ + || echo " WARNING: deploy key $key_id not deleted — check the repo settings" >&2 + done + else + echo " WARNING: gh not found — restore-test deploy keys on the GitHub mirror not checked" >&2 + fi ok exit 0 fi @@ -518,6 +950,8 @@ if [[ "$PHASE" == "hcloudtest" ]]; then command -v yq >/dev/null || fail "yq not found" command -v jq >/dev/null || fail "jq not found (needed to read the SAN pool and server attachments)" command -v curl >/dev/null || fail "curl not found (needed to learn this host's public address)" + command -v bao >/dev/null || fail "bao not found (the Hetzner token is read from OpenBao)" + command -v kubectl >/dev/null || fail "kubectl not found (runs the egress check)" # Three control planes, all carrying workloads — the production shape. One # machine is a node, not a cluster: etcd has no quorum to lose and nothing @@ -536,18 +970,11 @@ if [[ "$PHASE" == "hcloudtest" ]]; then # this one, adopted below. The names are fixed, so duplicates cannot # accumulate. `hcloudclean` is what deletes. - # The Talos snapshot is a prerequisite, not something this builds: making it - # means booting a server into rescue and writing the raw image, which is a - # one-off and needs a different set of permissions. Fail with the recipe - # rather than silently booting Debian and timing out on the Talos API. step 0 "Looking up the Talos snapshot..." talos_image_coords - HCLOUD_IMAGE=$(hcloud image list --type snapshot -l os=talos -o noheader -o columns=id | head -1) - [[ -n "$HCLOUD_IMAGE" ]] || fail "no snapshot labelled os=talos. Create one once — boot any server into rescue, then: - wget -O /tmp/talos.raw.xz https://factory.talos.dev/image/$VM_SCHEMATIC/$VM_VERSION/hcloud-amd64.raw.xz - xz -d -c /tmp/talos.raw.xz | dd of=/dev/sda && sync && shutdown -h now - then snapshot that server in the console and label the snapshot os=talos. - The schematic and version above are this cluster's, read from cluster.yaml." + HCLOUD_IMAGE="" + hcloud_talos_snapshot + [[ -n "$HCLOUD_IMAGE" ]] || fail "no Talos snapshot" echo " snapshot $HCLOUD_IMAGE" # The Talos API is unauthenticated in maintenance mode, which is how @@ -590,26 +1017,30 @@ if [[ "$PHASE" == "hcloudtest" ]]; then --source-ips "$fw_src" --description "Kubernetes API, this host only" >/dev/null \ || fail "could not add the Kubernetes API rule" echo " open to $fw_src only" + # Servers held from an earlier run get this firewall first; only then are + # the earlier runs' inbound firewalls detached and deleted, so no server is + # ever without one. Egress firewalls are replaced by hcloud_apply_egress. + while read -r srv; do + [[ -n "$srv" ]] || continue + hcloud firewall apply-to-resource "hcloudtest-$$" --type server --server "$srv" >/dev/null \ + || fail "could not attach this run's firewall to $srv" + done < <(hcloud server list -l "$HCLOUD_LABEL" -o noheader -o columns=name) + while read -r fw_id fw_name; do + [[ -n "$fw_id" && "$fw_name" != "hcloudtest-$$" && "$fw_name" != hcloudtest-node-egress-* ]] || continue + while read -r srv_id; do + [[ -n "$srv_id" ]] || continue + hcloud firewall remove-from-resource "$fw_id" --type server --server "$srv_id" >/dev/null \ + || fail "could not detach $fw_name from server $srv_id" + done < <(hcloud firewall describe "$fw_id" -o json | jq -r '.applied_to[] | select(.type == "server") | .server.id') + hcloud firewall delete "$fw_id" >/dev/null || fail "could not delete the previous firewall $fw_name" + echo " removed $fw_name" + done < <(hcloud firewall list -l "$HCLOUD_LABEL" -o noheader -o columns=id,name) ok - # cx43 — 16 GB, 8 vCPU. Sized from what the workload needs, not from the - # Talos guide, which names cx22 as a minimal example (and that type no - # longer exists; `server create` rejects it outright). + # Default type cpx52 (12 vCPU, 24 GB, 480 GB local disk): the hcloud-N + # patches put a 345GiB OSD partition beside a 60GiB EPHEMERAL on the system + # disk. cax* are ARM and do not match the amd64 schematic. # - # cx23 was tried first, on the reasoning that it is cx22's successor. It has - # 4 GB, 3.2 GB of it allocatable, and the three nodes sat at 59-78% of that - # with the infrastructure skeleton alone — no Ceph OSDs, no OpenBao running, - # not one app. Limits were already at 134-207% of allocatable and both - # kube-controller-manager and kube-scheduler had restarted twice. - # - # The local rehearsal is the reference: it runs a 16GiB control plane and - # three 8GiB workers, 40GiB in total, and needed every step of the way up - # from 6GiB to get there. Three nodes here carry the same workload plus a - # control plane each, so 48GiB total is the comparable figure. - # - # Constraints on the choice: cpx11/cpx21 are US-only and cax* are ARM, which - # would not match the amd64 schematic this cluster installs. That leaves the - # cx*/cpx*2 families in the German locations. # Two segments only, unlike the homelab's split in # NETWORK-traffic-classes.md: here the node address carries management, # cluster-internal and external alike, and Ceph replication has its own. @@ -629,7 +1060,7 @@ if [[ "$PHASE" == "hcloudtest" ]]; then seg_range() { case "$1" in cluster) - yq -r '.machine.kubelet.nodeIP.validSubnets[0]' "$TALOS_DIR/patches/site-hcloud.yaml" ;; + yq -r 'select(.machine) | .machine.kubelet.nodeIP.validSubnets[0]' "$TALOS_DIR/patches/site-hcloud.yaml" ;; storage) local pool pool=$(yq -r '.spec.config' "$REPO_ROOT/kubernetes/infrastructure/rook-ceph-cluster/san-network-attachment.yaml" | jq -r '.ipam.range') @@ -691,12 +1122,7 @@ if [[ "$PHASE" == "hcloudtest" ]]; then echo " $n: reusing the server we already hold" hcloud server rebuild "$n" --image "$HCLOUD_IMAGE" >/dev/null \ || fail "could not rebuild $n back into maintenance mode" - # The firewall is named per run (hcloudtest-$$), so an adopted - # server is still attached to the previous run's. Attach this run's - # too — without it the Talos API is unreachable once the old one is - # cleaned up, and the addresses in it are last run's anyway. - hcloud firewall apply-to-resource "hcloudtest-$$" --type server --server "$n" >/dev/null \ - || fail "could not attach this run's firewall to $n" + # This run's firewall was attached above, before the old ones went. else # Retried, because "error during placement (resource_unavailable)" # is the provider being momentarily out of that type in that @@ -708,13 +1134,13 @@ if [[ "$PHASE" == "hcloudtest" ]]; then create_err="" for attempt in $(seq 1 30); do create_err=$(hcloud server create --name "$n" --image "$HCLOUD_IMAGE" \ - --type "${HCLOUD_TYPE:-cx43}" --location "${HCLOUD_LOCATION:-fsn1}" \ + --type "${HCLOUD_TYPE:-cpx52}" --location "${HCLOUD_LOCATION:-fsn1}" \ --firewall "hcloudtest-$$" --label "$HCLOUD_LABEL" 2>&1) && break grep -qi 'resource_unavailable\|placement' <<<"$create_err" \ || fail "could not create $n: ${create_err//$'\n'/ | }" [[ $attempt -eq 1 ]] && echo " $n: no capacity yet, retrying" sleep 20 - [[ $attempt -eq 30 ]] && fail "no capacity for ${HCLOUD_TYPE:-cx43} in ${HCLOUD_LOCATION:-fsn1} after 10 minutes" + [[ $attempt -eq 30 ]] && fail "no capacity for ${HCLOUD_TYPE:-cpx52} in ${HCLOUD_LOCATION:-fsn1} after 10 minutes" done fi addr=$(hcloud server ip "$n") || fail "could not read the address of $n" @@ -757,13 +1183,18 @@ if [[ "$PHASE" == "hcloudtest" ]]; then hcloud volume delete "$n-osd" >/dev/null \ || fail "could not delete the previous OSD volume for $n" fi + yq -e 'select(.kind == "RawVolumeConfig" and .name == "osd-data")' "$TALOS_DIR/patches/$n.yaml" >/dev/null \ + || fail "$n declares no osd-data RawVolumeConfig — a node with no OSD device is not this cluster's shape" + # A raw volume on the system disk needs no Hetzner volume. vol_size=$(hcloud_volumes_for_node "$n") - [[ -n "$vol_size" ]] || fail "$n declares no RawVolumeConfig — a node with no OSD device is not this cluster's shape" - hcloud volume create --name "$n-osd" --size "$vol_size" --server "$n" \ - --label "$HCLOUD_LABEL" >/dev/null \ - || fail "could not create the OSD volume for $n" - - echo " $n at $addr, with a ${vol_size}GB OSD volume" + if [[ -n "$vol_size" ]]; then + hcloud volume create --name "$n-osd" --size "$vol_size" --server "$n" \ + --label "$HCLOUD_LABEL" >/dev/null \ + || fail "could not create the OSD volume for $n" + echo " $n at $addr, with a ${vol_size}GB OSD volume" + else + echo " $n at $addr, OSD on the system disk" + fi NODE_ADDRS+=("$addr") done ok @@ -792,20 +1223,26 @@ if [[ "$PHASE" == "hcloudtest" ]]; then echo " $node_src" ok - # Handover, and this list is the whole of it — the same four variables the - # vmtest phase sets. If it grows, the test is drifting away from production. + # Before any node is configured. + step 0 "Closing tcp egress to $HCLOUD_PROXY_HOST (Hetzner firewall)..." + hcloud_apply_egress + ok + + # Handover: the variables the vmtest phase sets, and HCLOUDTEST. SITE=hcloud NODES=("${HCLOUD_NODES[@]}") WORKERS=() ALL_NODES=("${NODES[@]}") K8S_API_HOST="${NODE_ADDRS[0]}" + HCLOUDTEST=1 # Isolation from the production cluster. Both halves are required: the # _PATH variables are derived before any phase runs, and the talos phase # writes to those. - HCLOUD_DIR=$(mktemp -d) - export TALOSCONFIG="$HCLOUD_DIR/talosconfig" - export KUBECONFIG="$HCLOUD_DIR/kubeconfig" + mkdir -p "$HCLOUD_STATE_DIR" + chmod 700 "$HCLOUD_STATE_DIR" + export TALOSCONFIG="$HCLOUD_STATE_DIR/talosconfig" + export KUBECONFIG="$HCLOUD_STATE_DIR/kubeconfig" TALOSCONFIG_PATH="$TALOSCONFIG" KUBECONFIG_PATH="$KUBECONFIG" @@ -1320,16 +1757,93 @@ EOF yq 'select(.kind == "HelmRelease") | .spec.values' "$CILIUM_HR" > "$CILIUM_VALUES" grep -q 'policyMapMax' "$CILIUM_VALUES" \ || fail "extracted Cilium values look wrong (no policyMapMax) — check $CILIUM_HR" + # The site's part, from the cilium-site ConfigMap in the site patch: the + # HelmRelease reads it through valuesFrom, under its own values. + CILIUM_SITE_VALUES=$(mktemp) + yq -r 'select(.cluster) | .cluster.inlineManifests[] | select(.name == "cilium-site") | .contents' \ + "$TALOS_DIR/patches/site-${SITE}.yaml" \ + | yq -r '.data["values.yaml"]' > "$CILIUM_SITE_VALUES" + grep -q 'routingMode' "$CILIUM_SITE_VALUES" \ + || fail "no cilium-site values (routingMode) in talos/patches/site-${SITE}.yaml" + + # Cilium's cni.confPath is a subdirectory of /etc/cni/net.d that Multus + # creates (#832). /etc/cni is read-only to the kubelet on Talos, so the + # agent's DirectoryOrCreate mount cannot make it, and on an empty cluster + # Multus is not there yet. A hostNetwork pod per node creates it; there is + # no pod network before Cilium. + cni_conf_dir=$(yq -r '.cni.confPath' "$CILIUM_VALUES") + if [[ "$cni_conf_dir" == /etc/cni/net.d/* ]]; then + step 0 "Creating $cni_conf_dir on every node..." + # kube-controller-manager creates kube-system/default shortly after the API answers. + for i in $(seq 1 30); do + kubectl -n kube-system get serviceaccount default >/dev/null && break + sleep 5 + [[ $i -eq 30 ]] && fail "kube-system/default ServiceAccount never appeared" + done + # A pod bound to a node that has not registered yet is garbage collected. + for node in "${ALL_NODES[@]}"; do + for i in $(seq 1 60); do + kubectl get node "$node" >/dev/null && break + sleep 5 + [[ $i -eq 60 ]] && fail "$node never registered with the API server" + done + done + for node in "${ALL_NODES[@]}"; do + kubectl apply -f - >/dev/null </dev/null \ + || fail "creating $cni_conf_dir on $node did not succeed (kubectl -n kube-system describe pod cni-confdir-$node)" + kubectl -n kube-system delete pod "cni-confdir-$node" >/dev/null + done + ok + fi helm repo add cilium "$(yq -r 'select(.kind == "HelmRepository") | .spec.url' "$CILIUM_HR")" >/dev/null helm repo update cilium >/dev/null helm upgrade --install cilium cilium/cilium \ --version "$(yq -r 'select(.kind == "HelmRelease") | .spec.chart.spec.version' "$CILIUM_HR")" \ --namespace kube-system \ + -f "$CILIUM_SITE_VALUES" \ -f "$CILIUM_VALUES" \ --set prometheus.serviceMonitor.enabled=false \ --wait --timeout 10m - rm -f "$CILIUM_VALUES" + rm -f "$CILIUM_VALUES" "$CILIUM_SITE_VALUES" + ok + + # Multus is the only writer of /etc/cni/net.d (#832); without it the kubelet + # reports "cni plugin not initialized" and nothing schedules, Flux included. + # Applied from the manifests Flux owns, which adopts them. + step 0 "Installing Multus (the CNI entry point)..." + kubectl apply -k "$REPO_ROOT/kubernetes/infrastructure/multus" >/dev/null \ + || fail "could not apply kubernetes/infrastructure/multus" + kubectl -n kube-system rollout status daemonset/kube-multus-ds-subdir --timeout=10m >/dev/null \ + || fail "Multus never became ready (kubectl -n kube-system get pods -l app=multus)" ok step 0 "Waiting for K8s nodes..." @@ -1368,10 +1882,40 @@ EOF fi done + # The nodes' resolver asks k8s-gateway first (ResolverConfig in the site + # patch), and CoreDNS forwards external names to the node. Until k8s-gateway + # answers, every lookup waits out the node's 2 s read timeout and CoreDNS + # gives up first: pods resolve nothing, Flux included. So it is installed + # here, from the values Flux will reconcile (the HelmRelease, under the + # site's k8s-gateway-site values, as valuesFrom merges them), and Flux + # adopts the release. + step 0 "Installing k8s-gateway (the nodes' first resolver)..." + k8sgw_values=$(mktemp) + k8sgw_site_values=$(mktemp) + yq -y 'select(.kind == "HelmRelease") | .spec.values' "$K8SGW_HR" > "$k8sgw_values" + yq -r 'select(.cluster) | .cluster.inlineManifests[] | select(.name == "k8s-gateway-site") | .contents' \ + "$TALOS_DIR/patches/site-${SITE}.yaml" \ + | yq -r '.data["values.yaml"]' > "$k8sgw_site_values" + grep -q 'dnsPolicy' "$k8sgw_site_values" \ + || fail "no k8s-gateway-site values (dnsPolicy) in talos/patches/site-${SITE}.yaml" + helm upgrade --install k8s-gateway \ + "$(yq -r 'select(.kind == "OCIRepository") | .spec.url' "$K8SGW_HR")" \ + --namespace kube-system \ + -f "$k8sgw_site_values" \ + -f "$k8sgw_values" \ + --wait --timeout 5m >/dev/null \ + || fail "k8s-gateway did not come up (kubectl -n kube-system get pods -l app.kubernetes.io/name=k8s-gateway)" + rm -f "$k8sgw_values" "$k8sgw_site_values" + ok + # Stash configs for subsequent phases / export-access step. Skipped under - # vmtest: those are a throwaway cluster's credentials, and writing them here - # would leave the flux phase pointed at VMs that no longer exist. - if [[ "${VMTEST:-0}" == "1" ]]; then + # vmtest and hcloudtest: those are a throwaway cluster's credentials, and a + # later production `bootstrap.sh flux` copies access-staging/ over the + # operator's kubeconfig and talosconfig. + if [[ "${HCLOUDTEST:-0}" == "1" ]]; then + hcloud_egress_check + echo -e "${GREEN}Talos phase complete.${NC} Access files in $HCLOUD_STATE_DIR/ (not staged)" + elif [[ "${VMTEST:-0}" == "1" ]]; then # Nodes being Ready says nothing about storage. The CephCluster picks # OSD nodes by a label out of the machine config and their device by the # partition label RawVolumeConfig creates, so assert both here — running @@ -1441,8 +1985,8 @@ bao_exec() { # OpenBao — so Velero cannot be the thing that restores OpenBao. OPENBAO_KOPIA_BUCKET="renner-uno-k8s-velero" OPENBAO_KOPIA_PREFIX="kopia/openbao/" -OPENBAO_S3_ENDPOINT="fsn1.your-objectstorage.com" -OPENBAO_S3_REGION="fsn1" +OPENBAO_S3_ENDPOINT="$S3_ENDPOINT" +OPENBAO_S3_REGION="$S3_REGION" # Velero's hardcoded backup repository password (pkg/repository/keys/keys.go). OPENBAO_KOPIA_PASSWORD="static-passw0rd" @@ -1620,7 +2164,9 @@ bring_up_openbao() { # entered by mistake. Wait for the container to be running first. echo -n " waiting for ${pods[0]} to answer" local status="" status_err="" - for i in $(seq 1 90); do + # Bounded at 30 minutes: on a fresh cluster the pod waits for Ceph to + # provision its volume. + for i in $(seq 1 180); do if status=$(bao_exec "${pods[0]}" bao status -format=json 2>"$OPENBAO_SECRETS_DIR/.baoerr"); then echo; break fi @@ -1638,7 +2184,7 @@ bring_up_openbao() { if jq -e 'has("initialized")' >/dev/null 2>&1 <<<"$status"; then echo; break; fi printf '.' sleep 10 - if [[ $i -eq 90 ]]; then + if [[ $i -eq 180 ]]; then echo echo " last error: $status_err" >&2 fail "${pods[0]} never answered bao status — refusing to guess whether it is sealed or uninitialised" @@ -1747,9 +2293,33 @@ if phase_runs flux; then case "$FLUX_SOURCE" in gitea) GITEA_TOKEN=$(read_secret git-token); export GITEA_TOKEN ;; github) GITHUB_TOKEN=$(read_secret git-token); export GITHUB_TOKEN ;; - *) fail "Unknown FLUX_SOURCE '$FLUX_SOURCE' (want gitea|github)" ;; + restore-test) + GITHUB_TOKEN=$(read_secret git-token); export GITHUB_TOKEN + command -v gh >/dev/null || fail "gh not found (registers the read-only deploy key)" + command -v jq >/dev/null || fail "jq not found (reads the DoH answer for the egress check)" + # The throwaway cluster carries production secrets, and this phase + # copies access files over KUBECONFIG_PATH and TALOSCONFIG_PATH: + # neither may be the operator's production file. + [[ "$KUBECONFIG_PATH" != "$HOME/.kube/config" && "$TALOSCONFIG_PATH" != "$HOME/.talos/config" ]] \ + || fail "FLUX_SOURCE=restore-test needs KUBECONFIG and TALOSCONFIG pointed at the test cluster ($HCLOUD_STATE_DIR)" + ;; + *) fail "Unknown FLUX_SOURCE '$FLUX_SOURCE' (want gitea|github|restore-test)" ;; esac - [[ "$PHASE" == "flux" ]] && load_staged_access + # The restore test reads its access from KUBECONFIG/TALOSCONFIG, which + # hcloudtest writes; access-staging/ may hold an older run's files. + [[ "$PHASE" == "flux" && "$FLUX_SOURCE" != "restore-test" ]] && load_staged_access + if [[ "$FLUX_SOURCE" == "restore-test" ]]; then + # Whatever the paths say, the cluster itself must be the test cluster, + # before anything below writes a secret to it. + node_names=$(kubectl get nodes -o jsonpath='{.items[*].metadata.name}') \ + || fail "cannot list the nodes of the target cluster" + [[ -n "$node_names" ]] || fail "the target cluster has no nodes" + for n in $node_names; do + [[ "$n" == "$HCLOUD_NAME_PREFIX"-* ]] \ + || fail "node '$n' is not a $HCLOUD_NAME_PREFIX-* test node — refusing the restore test against this cluster" + done + hcloud_egress_check + fi echo -n "Creating SOPS age secret... " kubectl create namespace flux-system --dry-run=client -o yaml | kubectl apply -f - @@ -1789,7 +2359,45 @@ if phase_runs flux; then fi unset seal_key existing - if [[ "$FLUX_SOURCE" == "github" ]]; then + # Before Flux: cnpg-object-stores fails without it rather than render empty + # paths, and the paths must exist before any Cluster bootstraps. + if [[ "$FLUX_SOURCE" == "restore-test" ]]; then + create_cnpg_test_bucket + write_cnpg_paths "$CNPG_TEST_BUCKET" + else + write_cnpg_paths "$CNPG_BUCKET" + fi + + if [[ "$FLUX_SOURCE" == "restore-test" ]]; then + # The production tree through kubernetes/restore-test/flux-root.yaml, + # whose patches are the test's whole deviation from production. Nothing + # is committed anywhere: the source is the GitHub mirror, read through a + # deploy key GitHub itself holds to read-only, so the test cluster + # cannot push (image automation) even with production's manifests. + step 0 "Bootstrapping FluxCD for the restore test (restore-test/flux-root.yaml)..." + flux install --components-extra=image-reflector-controller,image-automation-controller \ + || fail "flux install failed" + restore_test_repo + deploy_key="$OPENBAO_SECRETS_DIR/restore-test-git" + rm -f "$deploy_key" "$deploy_key.pub" + ssh-keygen -q -t ed25519 -N '' -C "restore-test" -f "$deploy_key" + gh api -X POST "repos/$restore_repo/keys" \ + -f title="restore-test $(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + -f key="$(cat "$deploy_key.pub")" -F read_only=true >/dev/null \ + || fail "could not register the read-only deploy key on $restore_repo" + flux create secret git restore-test-git \ + --url="$restore_url" \ + --private-key-file="$deploy_key" >/dev/null \ + || fail "could not create the restore-test-git Secret" + # On FLUX_BRANCH, so a branch can be tried here before it is merged. + # shellcheck disable=SC2016 # $b is a jq variable + yq -y --arg b "$FLUX_BRANCH" \ + 'if .kind == "GitRepository" and .metadata.name == "restore-test" then .spec.ref.branch = $b else . end' \ + "$REPO_ROOT/kubernetes/restore-test/flux-root.yaml" | kubectl apply -f - >/dev/null \ + || fail "could not apply restore-test/flux-root.yaml" + echo " source: $restore_url, branch $FLUX_BRANCH" + ok + elif [[ "$FLUX_SOURCE" == "github" ]]; then step 0 "Bootstrapping FluxCD from the GitHub break-glass mirror..." echo -e "${YELLOW}BREAK-GLASS: Flux source is github.com/${GITHUB_OWNER}/${GITHUB_REPO}, NOT in-cluster Gitea.${NC}" echo -e "${YELLOW}Switch the source back to Gitea once it is serving again (see RUNBOOK-cold-rebuild.md).${NC}" @@ -1893,7 +2501,8 @@ if phase_runs wait-apps; then echo -e "${YELLOW}=== Phase: Wait for apps ===${NC}" command -v kubectl >/dev/null || fail "kubectl not found" command -v flux >/dev/null || fail "flux not found" - [[ "$PHASE" == "wait-apps" ]] && load_staged_access + # The restore test reads its access from KUBECONFIG/TALOSCONFIG. + [[ "$PHASE" == "wait-apps" && "$FLUX_SOURCE" != "restore-test" ]] && load_staged_access # Wait for a Flux Kustomization to become Ready=True. Prints a status line # whenever the Ready/Message pair changes, plus a heartbeat every 60 s so a @@ -1940,8 +2549,8 @@ if phase_runs wait-apps; then # Each app KS now owns its CNPG database (+ any data PVCs) directly — the # per-app `seed-` KSes that used to bootstrap them were folded into the # app KSes when the seed layer was retired. Each wait below gates on the app - # KS, whose readiness includes its CNPG cluster coming up (initdb on a fresh - # cluster; a cold-rebuild data restore is the separate RECOVERY.md runbook). + # KS, whose readiness includes its CNPG cluster coming up (bootstrap.recovery) + # and, through the restore KS it depends on, its volumes restored by Velero. # Timeouts stay generous to cover first-time cluster bootstrap on fresh Ceph. # apps waits on infrastructure-ceph-cluster (Ceph mon/mgr/OSD bring-up # is genuinely 5–10 min on a fresh OSD-wiped cluster). diff --git a/kubernetes/clusters/homelab/apps.yaml b/kubernetes/clusters/homelab/apps.yaml index e5965cfe..88ff04e4 100644 --- a/kubernetes/clusters/homelab/apps.yaml +++ b/kubernetes/clusters/homelab/apps.yaml @@ -17,6 +17,7 @@ spec: # woodpecker migrated) — secrets come from OpenBao via ESO # (MIGRATION-sops-openbao.md). dependsOn: + - name: restore-apps - name: infrastructure - name: infrastructure-crds - - name: infrastructure-ceph-cluster + - name: infrastructure-ceph-cluster \ No newline at end of file diff --git a/kubernetes/clusters/homelab/audiobookshelf.yaml b/kubernetes/clusters/homelab/audiobookshelf.yaml index 6e936ae4..764dde33 100644 --- a/kubernetes/clusters/homelab/audiobookshelf.yaml +++ b/kubernetes/clusters/homelab/audiobookshelf.yaml @@ -22,6 +22,7 @@ spec: secretRef: name: sops-age dependsOn: + - name: restore-audiobookshelf - name: infrastructure - name: infrastructure-crds - name: infrastructure-ceph-cluster diff --git a/kubernetes/clusters/homelab/authentik.yaml b/kubernetes/clusters/homelab/authentik.yaml index 6eb5e39e..571c8a6b 100644 --- a/kubernetes/clusters/homelab/authentik.yaml +++ b/kubernetes/clusters/homelab/authentik.yaml @@ -20,10 +20,12 @@ spec: # from OpenBao via ESO (MIGRATION-sops-openbao.md). This was the LAST # sops consumer; phase 4 (age-key retirement) may proceed after soak. dependsOn: + - name: restore-authentik - name: infrastructure - name: infrastructure-crds - name: infrastructure-ceph-cluster - - name: infrastructure-cnpg-barman + - name: cnpg-object-stores # the ObjectStores its Cluster names + - name: gitea # server, LDAP outpost and gate-sync images are in Gitea's registry healthCheckExprs: - apiVersion: postgresql.cnpg.io/v1 kind: Cluster diff --git a/kubernetes/clusters/homelab/cnpg-object-stores.yaml b/kubernetes/clusters/homelab/cnpg-object-stores.yaml new file mode 100644 index 00000000..29c6df13 --- /dev/null +++ b/kubernetes/clusters/homelab/cnpg-object-stores.yaml @@ -0,0 +1,27 @@ +# The CNPG ObjectStores. Their paths come from the cnpg-paths ConfigMap, which +# bootstrap.sh writes once per cluster incarnation and never replaces: +# CNPG__SOURCE the bucket prefix holding 's newest base backup +# CNPG__ARCHIVE a new, empty prefix -g +# Not optional: without it every path renders empty. +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: cnpg-object-stores + namespace: flux-system +spec: + interval: 10m + retryInterval: 1m + timeout: 5m + sourceRef: + kind: GitRepository + name: flux-system + path: ./kubernetes/cnpg-object-stores + prune: true + wait: true + dependsOn: + - name: infrastructure + - name: infrastructure-cnpg-barman + postBuild: + substituteFrom: + - kind: ConfigMap + name: cnpg-paths diff --git a/kubernetes/clusters/homelab/evcc.yaml b/kubernetes/clusters/homelab/evcc.yaml index 1240d82f..cb3eb3b0 100644 --- a/kubernetes/clusters/homelab/evcc.yaml +++ b/kubernetes/clusters/homelab/evcc.yaml @@ -19,3 +19,5 @@ spec: - name: infrastructure - name: infrastructure-crds - name: infrastructure-ceph-cluster + # mqtt-account-controller there delivers mqtt-evcc-client-tls; mosquitto is the broker. + - name: home-assistant diff --git a/kubernetes/clusters/homelab/gitea-mcp.yaml b/kubernetes/clusters/homelab/gitea-mcp.yaml index 6c70ff97..34fb1e08 100644 --- a/kubernetes/clusters/homelab/gitea-mcp.yaml +++ b/kubernetes/clusters/homelab/gitea-mcp.yaml @@ -16,3 +16,4 @@ spec: dependsOn: - name: infrastructure - name: infrastructure-crds + - name: gitea # serves Gitea's API diff --git a/kubernetes/clusters/homelab/gitea.yaml b/kubernetes/clusters/homelab/gitea.yaml index ff9bff09..ba6fc93a 100644 --- a/kubernetes/clusters/homelab/gitea.yaml +++ b/kubernetes/clusters/homelab/gitea.yaml @@ -14,7 +14,14 @@ spec: path: ./kubernetes/apps/gitea prune: true wait: true + # The claim's bound volume differs per cluster (apps/gitea/pvc.yaml). + ignore: + - paths: ["/spec/volumeName"] + target: + kind: PersistentVolumeClaim + name: data-gitea-0 dependsOn: + - name: restore-gitea - name: infrastructure - name: infrastructure-crds - name: infrastructure-ceph-cluster diff --git a/kubernetes/clusters/homelab/home-assistant.yaml b/kubernetes/clusters/homelab/home-assistant.yaml index 4bdfff33..62def0d2 100644 --- a/kubernetes/clusters/homelab/home-assistant.yaml +++ b/kubernetes/clusters/homelab/home-assistant.yaml @@ -4,43 +4,28 @@ metadata: name: home-assistant namespace: flux-system spec: - # Seed Job collapsed into the HA pod's `seed` initContainer so the - # backup tarball + .HA_RESTORE manifest staging happens in the same - # pod as HA Core (no Multi-Attach race against the RWO Ceph-RBD - # PVC if production grows to multiple edge nodes). HA Core still - # does the actual restore from the staged manifest on its boot. - # Timeout has to absorb the worst-case 3h wait for the HAOS backup - # to appear in Garage plus the in-pod download + HA's native restore. + # The slowest part on a cold start is ha-pg recovering from barman (base + # backup ~1.7 GiB compressed, then WAL replay); the config PVC is restored + # beforehand by restore-home-assistant. interval: 10m retryInterval: 1m - timeout: 4h + timeout: 30m sourceRef: kind: GitRepository name: flux-system path: ./kubernetes/apps/home-assistant prune: true - # Same TBD-USB caveat for the stromzaehler hostPath. Manifests apply; - # HA pod may CrashLoop until the USB is present — acceptable during canary. - wait: false + wait: true # decryption block dropped 2026-07-13: path is sops-free — secrets come # from OpenBao via ESO (MIGRATION-sops-openbao.md). dependsOn: + - name: restore-home-assistant - name: infrastructure - name: infrastructure-crds - name: infrastructure-ceph-cluster - # ha-pg-objectstore.yaml declares an ObjectStore, whose CRD comes from the - # CNPG Barman plugin in infrastructure-cnpg-barman. Without this edge the - # kind is unknown when this path is first applied, and a server-side - # dry-run rejects the ENTIRE set over the one object it cannot recognise: - # - # ObjectStore/home-assistant/ha-pg-store dry-run failed: - # no matches for kind "ObjectStore" in version ... - # - # Every other ObjectStore user — authentik, immich, paperless, trmnl, - # woodpecker — already declares it. This was the only one that did not, and - # it took speech-to-phrase, zone-configurator and matter-server down with - # it, since all three wait on home-assistant. - - name: infrastructure-cnpg-barman + - name: cnpg-object-stores # the ObjectStores its Cluster names + - name: gitea # mqtt-account-controller's image is in Gitea's registry + - name: influxdb # HA's InfluxDB integration writes there --- apiVersion: kustomize.toolkit.fluxcd.io/v1 kind: Kustomization @@ -58,10 +43,35 @@ spec: prune: true wait: true dependsOn: + - name: restore-home-assistant - name: infrastructure - name: infrastructure-crds - name: infrastructure-ceph-cluster --- +# shelly-manager and the mDNS relay: home-assistant's IoT-VLAN workloads. +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: home-assistant-iot + namespace: flux-system +spec: + interval: 10m + retryInterval: 1m + timeout: 10m + sourceRef: + kind: GitRepository + name: flux-system + path: ./kubernetes/apps/home-assistant-iot + prune: true + wait: true + dependsOn: + - name: restore-home-assistant # shelly-manager's PVC + - name: infrastructure + - name: infrastructure-crds + - name: infrastructure-ceph-cluster + - name: gitea # shelly-manager's image is in Gitea's registry + - name: home-assistant # both serve HA: its API, its zeroconf listener +--- apiVersion: kustomize.toolkit.fluxcd.io/v1 kind: Kustomization metadata: @@ -76,10 +86,13 @@ spec: name: flux-system path: ./kubernetes/apps/otbr prune: true - wait: false + # Not in use. + suspend: true + wait: true # decryption block dropped 2026-07-13: path is sops-free — secrets come # from OpenBao via ESO (MIGRATION-sops-openbao.md). dependsOn: + - name: restore-home-assistant - name: infrastructure - name: infrastructure-crds --- @@ -120,6 +133,7 @@ spec: # First boot downloads the Whisper model (a few hundred MB) — give it room. wait: true dependsOn: + - name: restore-home-assistant - name: infrastructure - name: infrastructure-crds - name: infrastructure-ceph-cluster diff --git a/kubernetes/clusters/homelab/immich.yaml b/kubernetes/clusters/homelab/immich.yaml index 01d0f5c5..4afdbcf2 100644 --- a/kubernetes/clusters/homelab/immich.yaml +++ b/kubernetes/clusters/homelab/immich.yaml @@ -20,10 +20,11 @@ spec: # decryption block dropped 2026-07-13: path is sops-free — secrets come # from OpenBao via ESO (MIGRATION-sops-openbao.md). dependsOn: + - name: restore-immich - name: infrastructure - name: infrastructure-crds - name: infrastructure-ceph-cluster - - name: infrastructure-cnpg-barman + - name: cnpg-object-stores # the ObjectStores its Cluster names healthCheckExprs: - apiVersion: postgresql.cnpg.io/v1 kind: Cluster diff --git a/kubernetes/clusters/homelab/influxdb.yaml b/kubernetes/clusters/homelab/influxdb.yaml index f9947bde..e4bc17a7 100644 --- a/kubernetes/clusters/homelab/influxdb.yaml +++ b/kubernetes/clusters/homelab/influxdb.yaml @@ -25,6 +25,7 @@ spec: # boilerplate from the monolithic seed KS — InfluxDB has no CNPG cluster, only # a chart-provisioned ceph-block PVC, so ceph is the only storage dep needed. dependsOn: + - name: restore-influxdb - name: infrastructure - name: infrastructure-crds - name: infrastructure-ceph-cluster diff --git a/kubernetes/clusters/homelab/infrastructure-inhouse.yaml b/kubernetes/clusters/homelab/infrastructure-inhouse.yaml new file mode 100644 index 00000000..a2a10814 --- /dev/null +++ b/kubernetes/clusters/homelab/infrastructure-inhouse.yaml @@ -0,0 +1,19 @@ +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: infrastructure-inhouse + namespace: flux-system +spec: + interval: 10m + retryInterval: 2m + timeout: 10m + sourceRef: + kind: GitRepository + name: flux-system + path: ./kubernetes/infrastructure-inhouse + prune: true + wait: true + dependsOn: + - name: infrastructure + - name: infrastructure-crds + - name: gitea # its images are in Gitea's registry diff --git a/kubernetes/clusters/homelab/infrastructure-services.yaml b/kubernetes/clusters/homelab/infrastructure-services.yaml new file mode 100644 index 00000000..a4035bd8 --- /dev/null +++ b/kubernetes/clusters/homelab/infrastructure-services.yaml @@ -0,0 +1,98 @@ +# Infrastructure services that consume Secrets from OpenBao. Each carries its +# ExternalSecrets and depends on infrastructure-crds, where the openbao +# ClusterSecretStore turns Ready only once OpenBao is unsealed, so the +# HelmRelease never installs before its Secret exists. One Kustomization each: +# a broken service holds back nothing but itself (velero: the restore chain). +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: crowdsec + namespace: flux-system +spec: + interval: 10m + retryInterval: 1m + timeout: 10m + sourceRef: + kind: GitRepository + name: flux-system + path: ./kubernetes/infrastructure/crowdsec + prune: true + wait: true + dependsOn: + - name: infrastructure + - name: infrastructure-crds +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: external-dns-unifi + namespace: flux-system +spec: + interval: 10m + retryInterval: 1m + timeout: 5m + sourceRef: + kind: GitRepository + name: flux-system + path: ./kubernetes/infrastructure/external-dns-unifi + prune: true + wait: true + dependsOn: + - name: infrastructure + - name: infrastructure-crds +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: netbird-ingress + namespace: flux-system +spec: + interval: 10m + retryInterval: 1m + timeout: 5m + sourceRef: + kind: GitRepository + name: flux-system + path: ./kubernetes/infrastructure/netbird-ingress + prune: true + wait: true + dependsOn: + - name: infrastructure + - name: infrastructure-crds +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: velero + namespace: flux-system +spec: + interval: 10m + retryInterval: 1m + timeout: 15m + sourceRef: + kind: GitRepository + name: flux-system + path: ./kubernetes/infrastructure/velero + prune: true + wait: true + dependsOn: + - name: infrastructure + - name: infrastructure-crds +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: velero-schedules + namespace: flux-system +spec: + interval: 10m + retryInterval: 1m + timeout: 5m + sourceRef: + kind: GitRepository + name: flux-system + path: ./kubernetes/infrastructure/velero-schedules + prune: true + wait: true + dependsOn: + - name: velero # the velero.io CRDs diff --git a/kubernetes/clusters/homelab/infrastructure.yaml b/kubernetes/clusters/homelab/infrastructure.yaml index 5f5a4a82..1892cf91 100644 --- a/kubernetes/clusters/homelab/infrastructure.yaml +++ b/kubernetes/clusters/homelab/infrastructure.yaml @@ -40,6 +40,13 @@ spec: kind: HelmRelease name: openbao # the ESO backend ExternalSecrets resolve against namespace: openbao + # Ready only when every replica is unsealed (readiness probe: bao status). + # The HelmRelease is Ready as soon as the chart is installed; the + # ClusterSecretStore in infrastructure-crds must not come up before. + - apiVersion: apps/v1 + kind: StatefulSet + name: openbao + namespace: openbao - apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease name: rook-ceph-operator # storage — apps mount PVCs diff --git a/kubernetes/clusters/homelab/matrix.yaml b/kubernetes/clusters/homelab/matrix.yaml index 1ab1a467..ff54b15d 100644 --- a/kubernetes/clusters/homelab/matrix.yaml +++ b/kubernetes/clusters/homelab/matrix.yaml @@ -17,10 +17,11 @@ spec: prune: true wait: true dependsOn: + - name: restore-matrix - name: infrastructure - name: infrastructure-crds - name: infrastructure-ceph-cluster - - name: infrastructure-cnpg-barman + - name: cnpg-object-stores # the ObjectStores its Cluster names healthCheckExprs: - apiVersion: postgresql.cnpg.io/v1 kind: Cluster diff --git a/kubernetes/clusters/homelab/monitoring.yaml b/kubernetes/clusters/homelab/monitoring.yaml index 30547bd4..38ea4190 100644 --- a/kubernetes/clusters/homelab/monitoring.yaml +++ b/kubernetes/clusters/homelab/monitoring.yaml @@ -46,6 +46,7 @@ spec: # from OpenBao via ESO (MIGRATION-sops-openbao.md). dependsOn: - name: grafana-db + - name: influxdb # its Job writes the grafana-influxdb-token Secret Grafana mounts --- apiVersion: kustomize.toolkit.fluxcd.io/v1 kind: Kustomization diff --git a/kubernetes/clusters/homelab/music-assistant.yaml b/kubernetes/clusters/homelab/music-assistant.yaml index b1697caa..dbd995c7 100644 --- a/kubernetes/clusters/homelab/music-assistant.yaml +++ b/kubernetes/clusters/homelab/music-assistant.yaml @@ -22,6 +22,7 @@ spec: secretRef: name: sops-age dependsOn: + - name: restore-music-assistant - name: infrastructure - name: infrastructure-crds - name: infrastructure-ceph-cluster diff --git a/kubernetes/clusters/homelab/navidrome.yaml b/kubernetes/clusters/homelab/navidrome.yaml index ee88da95..782fd751 100644 --- a/kubernetes/clusters/homelab/navidrome.yaml +++ b/kubernetes/clusters/homelab/navidrome.yaml @@ -14,6 +14,7 @@ spec: prune: true wait: true dependsOn: + - name: restore-navidrome - name: infrastructure - name: infrastructure-crds - name: infrastructure-ceph-cluster diff --git a/kubernetes/clusters/homelab/paperless.yaml b/kubernetes/clusters/homelab/paperless.yaml index 7171042d..c0ad9b7e 100644 --- a/kubernetes/clusters/homelab/paperless.yaml +++ b/kubernetes/clusters/homelab/paperless.yaml @@ -20,10 +20,12 @@ spec: # decryption block dropped 2026-07-13: path is sops-free — secrets come # from OpenBao via ESO (MIGRATION-sops-openbao.md). dependsOn: + - name: restore-paperless - name: infrastructure - name: infrastructure-crds - name: infrastructure-ceph-cluster - - name: infrastructure-cnpg-barman + - name: cnpg-object-stores # the ObjectStores its Cluster names + - name: gitea # ftp-scanner (vsftpd) image is in Gitea's registry healthCheckExprs: - apiVersion: postgresql.cnpg.io/v1 kind: Cluster diff --git a/kubernetes/clusters/homelab/renovate.yaml b/kubernetes/clusters/homelab/renovate.yaml index 6e0173fe..a3c5e283 100644 --- a/kubernetes/clusters/homelab/renovate.yaml +++ b/kubernetes/clusters/homelab/renovate.yaml @@ -35,3 +35,4 @@ spec: # The `openbao` ClusterSecretStore, including the `renovate` entry in its # conditions.namespaces allowlist. - name: infrastructure-crds + - name: gitea # the platform it runs against (git.renner.uno) diff --git a/kubernetes/clusters/homelab/restore.yaml b/kubernetes/clusters/homelab/restore.yaml new file mode 100644 index 00000000..33f6d1ee --- /dev/null +++ b/kubernetes/clusters/homelab/restore.yaml @@ -0,0 +1,396 @@ +# Ready once Velero has completed a sync of the backups in the bucket. A +# Restore created before that finds no backup and fails for good. +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: velero-backups-synced + namespace: flux-system +spec: + interval: 10m + timeout: 30m + sourceRef: + kind: GitRepository + name: flux-system + path: ./kubernetes/restore/gate + prune: true + dependsOn: + - name: velero # the BackupStorageLocation and its credentials + healthChecks: + - apiVersion: velero.io/v1 + kind: BackupStorageLocation + name: default + namespace: velero + healthCheckExprs: + - apiVersion: velero.io/v1 + kind: BackupStorageLocation + current: has(status.lastSyncedTime) +--- +# One Kustomization per Velero Schedule (infrastructure/velero-schedules/schedules.yaml), +# each restoring that schedule's newest backup. An app depends on its own, so +# it waits for its data and for nothing else. Gitea's data goes first: it is +# the registry for the in-house images; the others depend on restore-gitea. +# openbao-daily has none: bootstrap.sh restores OpenBao from its raft snapshot. +# Every one carries the same health check: a schedule without any backup has +# nothing to restore (FailedValidation "No backups found for schedule"). +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: restore-gitea + namespace: flux-system +spec: + interval: 10m + timeout: 1h + sourceRef: + kind: GitRepository + name: flux-system + path: ./kubernetes/restore/gitea + prune: true + wait: true + dependsOn: + - name: velero-backups-synced + - name: infrastructure-ceph-cluster + healthCheckExprs: + - apiVersion: velero.io/v1 + kind: Restore + failed: >- + has(status.phase) && (status.phase in ['Failed', 'PartiallyFailed'] || + (status.phase == 'FailedValidation' && !(has(status.validationErrors) && + status.validationErrors.exists(e, e.startsWith('No backups found for schedule'))))) + current: >- + has(status.phase) && (status.phase == 'Completed' || status.phase == 'FailedValidation') +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: restore-apps + namespace: flux-system +spec: + interval: 10m + timeout: 1h + sourceRef: + kind: GitRepository + name: flux-system + path: ./kubernetes/restore/apps + prune: true + wait: true + dependsOn: + - name: restore-gitea + - name: velero-backups-synced + - name: infrastructure-ceph-cluster + healthCheckExprs: + - apiVersion: velero.io/v1 + kind: Restore + failed: >- + has(status.phase) && (status.phase in ['Failed', 'PartiallyFailed'] || + (status.phase == 'FailedValidation' && !(has(status.validationErrors) && + status.validationErrors.exists(e, e.startsWith('No backups found for schedule'))))) + current: >- + has(status.phase) && (status.phase == 'Completed' || status.phase == 'FailedValidation') +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: restore-audiobookshelf + namespace: flux-system +spec: + interval: 10m + timeout: 3h + sourceRef: + kind: GitRepository + name: flux-system + path: ./kubernetes/restore/audiobookshelf + prune: true + wait: true + dependsOn: + - name: restore-gitea + - name: velero-backups-synced + - name: infrastructure-ceph-cluster + healthCheckExprs: + - apiVersion: velero.io/v1 + kind: Restore + failed: >- + has(status.phase) && (status.phase in ['Failed', 'PartiallyFailed'] || + (status.phase == 'FailedValidation' && !(has(status.validationErrors) && + status.validationErrors.exists(e, e.startsWith('No backups found for schedule'))))) + current: >- + has(status.phase) && (status.phase == 'Completed' || status.phase == 'FailedValidation') +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: restore-authentik + namespace: flux-system +spec: + interval: 10m + timeout: 1h + sourceRef: + kind: GitRepository + name: flux-system + path: ./kubernetes/restore/authentik + prune: true + wait: true + dependsOn: + - name: restore-gitea + - name: velero-backups-synced + - name: infrastructure-ceph-cluster + healthCheckExprs: + - apiVersion: velero.io/v1 + kind: Restore + failed: >- + has(status.phase) && (status.phase in ['Failed', 'PartiallyFailed'] || + (status.phase == 'FailedValidation' && !(has(status.validationErrors) && + status.validationErrors.exists(e, e.startsWith('No backups found for schedule'))))) + current: >- + has(status.phase) && (status.phase == 'Completed' || status.phase == 'FailedValidation') +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: restore-home-assistant + namespace: flux-system +spec: + interval: 10m + timeout: 1h + sourceRef: + kind: GitRepository + name: flux-system + path: ./kubernetes/restore/home-assistant + prune: true + wait: true + dependsOn: + - name: restore-gitea + - name: velero-backups-synced + - name: infrastructure-ceph-cluster + healthCheckExprs: + - apiVersion: velero.io/v1 + kind: Restore + failed: >- + has(status.phase) && (status.phase in ['Failed', 'PartiallyFailed'] || + (status.phase == 'FailedValidation' && !(has(status.validationErrors) && + status.validationErrors.exists(e, e.startsWith('No backups found for schedule'))))) + current: >- + has(status.phase) && (status.phase == 'Completed' || status.phase == 'FailedValidation') +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: restore-immich + namespace: flux-system +spec: + interval: 10m + timeout: 6h + sourceRef: + kind: GitRepository + name: flux-system + path: ./kubernetes/restore/immich + prune: true + wait: true + dependsOn: + - name: restore-gitea + - name: velero-backups-synced + - name: infrastructure-ceph-cluster + healthCheckExprs: + - apiVersion: velero.io/v1 + kind: Restore + failed: >- + has(status.phase) && (status.phase in ['Failed', 'PartiallyFailed'] || + (status.phase == 'FailedValidation' && !(has(status.validationErrors) && + status.validationErrors.exists(e, e.startsWith('No backups found for schedule'))))) + current: >- + has(status.phase) && (status.phase == 'Completed' || status.phase == 'FailedValidation') +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: restore-influxdb + namespace: flux-system +spec: + interval: 10m + timeout: 1h + sourceRef: + kind: GitRepository + name: flux-system + path: ./kubernetes/restore/influxdb + prune: true + wait: true + dependsOn: + - name: restore-gitea + - name: velero-backups-synced + - name: infrastructure-ceph-cluster + healthCheckExprs: + - apiVersion: velero.io/v1 + kind: Restore + failed: >- + has(status.phase) && (status.phase in ['Failed', 'PartiallyFailed'] || + (status.phase == 'FailedValidation' && !(has(status.validationErrors) && + status.validationErrors.exists(e, e.startsWith('No backups found for schedule'))))) + current: >- + has(status.phase) && (status.phase == 'Completed' || status.phase == 'FailedValidation') +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: restore-matrix + namespace: flux-system +spec: + interval: 10m + timeout: 1h + sourceRef: + kind: GitRepository + name: flux-system + path: ./kubernetes/restore/matrix + prune: true + wait: true + dependsOn: + - name: restore-gitea + - name: velero-backups-synced + - name: infrastructure-ceph-cluster + healthCheckExprs: + - apiVersion: velero.io/v1 + kind: Restore + failed: >- + has(status.phase) && (status.phase in ['Failed', 'PartiallyFailed'] || + (status.phase == 'FailedValidation' && !(has(status.validationErrors) && + status.validationErrors.exists(e, e.startsWith('No backups found for schedule'))))) + current: >- + has(status.phase) && (status.phase == 'Completed' || status.phase == 'FailedValidation') +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: restore-music-assistant + namespace: flux-system +spec: + interval: 10m + timeout: 1h + sourceRef: + kind: GitRepository + name: flux-system + path: ./kubernetes/restore/music-assistant + prune: true + wait: true + dependsOn: + - name: restore-gitea + - name: velero-backups-synced + - name: infrastructure-ceph-cluster + healthCheckExprs: + - apiVersion: velero.io/v1 + kind: Restore + failed: >- + has(status.phase) && (status.phase in ['Failed', 'PartiallyFailed'] || + (status.phase == 'FailedValidation' && !(has(status.validationErrors) && + status.validationErrors.exists(e, e.startsWith('No backups found for schedule'))))) + current: >- + has(status.phase) && (status.phase == 'Completed' || status.phase == 'FailedValidation') +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: restore-navidrome + namespace: flux-system +spec: + interval: 10m + timeout: 1h + sourceRef: + kind: GitRepository + name: flux-system + path: ./kubernetes/restore/navidrome + prune: true + wait: true + dependsOn: + - name: restore-gitea + - name: velero-backups-synced + - name: infrastructure-ceph-cluster + healthCheckExprs: + - apiVersion: velero.io/v1 + kind: Restore + failed: >- + has(status.phase) && (status.phase in ['Failed', 'PartiallyFailed'] || + (status.phase == 'FailedValidation' && !(has(status.validationErrors) && + status.validationErrors.exists(e, e.startsWith('No backups found for schedule'))))) + current: >- + has(status.phase) && (status.phase == 'Completed' || status.phase == 'FailedValidation') +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: restore-paperless + namespace: flux-system +spec: + interval: 10m + timeout: 1h + sourceRef: + kind: GitRepository + name: flux-system + path: ./kubernetes/restore/paperless + prune: true + wait: true + dependsOn: + - name: restore-gitea + - name: velero-backups-synced + - name: infrastructure-ceph-cluster + healthCheckExprs: + - apiVersion: velero.io/v1 + kind: Restore + failed: >- + has(status.phase) && (status.phase in ['Failed', 'PartiallyFailed'] || + (status.phase == 'FailedValidation' && !(has(status.validationErrors) && + status.validationErrors.exists(e, e.startsWith('No backups found for schedule'))))) + current: >- + has(status.phase) && (status.phase == 'Completed' || status.phase == 'FailedValidation') +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: restore-trmnl + namespace: flux-system +spec: + interval: 10m + timeout: 1h + sourceRef: + kind: GitRepository + name: flux-system + path: ./kubernetes/restore/trmnl + prune: true + wait: true + dependsOn: + - name: restore-gitea + - name: velero-backups-synced + - name: infrastructure-ceph-cluster + healthCheckExprs: + - apiVersion: velero.io/v1 + kind: Restore + failed: >- + has(status.phase) && (status.phase in ['Failed', 'PartiallyFailed'] || + (status.phase == 'FailedValidation' && !(has(status.validationErrors) && + status.validationErrors.exists(e, e.startsWith('No backups found for schedule'))))) + current: >- + has(status.phase) && (status.phase == 'Completed' || status.phase == 'FailedValidation') +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: restore-wordpress + namespace: flux-system +spec: + interval: 10m + timeout: 1h + sourceRef: + kind: GitRepository + name: flux-system + path: ./kubernetes/restore/wordpress + prune: true + wait: true + dependsOn: + - name: restore-gitea + - name: velero-backups-synced + - name: infrastructure-ceph-cluster + healthCheckExprs: + - apiVersion: velero.io/v1 + kind: Restore + failed: >- + has(status.phase) && (status.phase in ['Failed', 'PartiallyFailed'] || + (status.phase == 'FailedValidation' && !(has(status.validationErrors) && + status.validationErrors.exists(e, e.startsWith('No backups found for schedule'))))) + current: >- + has(status.phase) && (status.phase == 'Completed' || status.phase == 'FailedValidation') diff --git a/kubernetes/clusters/homelab/review-bot.yaml b/kubernetes/clusters/homelab/review-bot.yaml index d5f7264d..083ead44 100644 --- a/kubernetes/clusters/homelab/review-bot.yaml +++ b/kubernetes/clusters/homelab/review-bot.yaml @@ -16,3 +16,4 @@ spec: dependsOn: - name: infrastructure - name: infrastructure-crds + - name: gitea # its image is in Gitea's registry diff --git a/kubernetes/clusters/homelab/trmnl.yaml b/kubernetes/clusters/homelab/trmnl.yaml index 328ab405..eeb2d3bb 100644 --- a/kubernetes/clusters/homelab/trmnl.yaml +++ b/kubernetes/clusters/homelab/trmnl.yaml @@ -17,10 +17,12 @@ spec: prune: true wait: true dependsOn: + - name: restore-trmnl - name: infrastructure - name: infrastructure-crds - name: infrastructure-ceph-cluster - - name: infrastructure-cnpg-barman + - name: cnpg-object-stores # the ObjectStores its Cluster names + - name: home-assistant # the renderer reads HA's API healthCheckExprs: - apiVersion: postgresql.cnpg.io/v1 kind: Cluster diff --git a/kubernetes/clusters/homelab/wiki.yaml b/kubernetes/clusters/homelab/wiki.yaml index 55303de8..2e6c52cd 100644 --- a/kubernetes/clusters/homelab/wiki.yaml +++ b/kubernetes/clusters/homelab/wiki.yaml @@ -20,3 +20,4 @@ spec: - name: infrastructure-ceph-cluster # wiki-embed (the embedding backend) lives in the llama-server app. - name: llama-server + - name: gitea # its image is in Gitea's registry diff --git a/kubernetes/clusters/homelab/woodpecker.yaml b/kubernetes/clusters/homelab/woodpecker.yaml index 5eb61493..4c68137b 100644 --- a/kubernetes/clusters/homelab/woodpecker.yaml +++ b/kubernetes/clusters/homelab/woodpecker.yaml @@ -24,7 +24,8 @@ spec: - name: infrastructure - name: infrastructure-crds - name: infrastructure-ceph-cluster - - name: infrastructure-cnpg-barman + - name: cnpg-object-stores # the ObjectStores its Cluster names + - name: gitea # its forge and login provider healthCheckExprs: - apiVersion: postgresql.cnpg.io/v1 kind: Cluster diff --git a/kubernetes/clusters/homelab/wordpress.yaml b/kubernetes/clusters/homelab/wordpress.yaml index 054b7345..00f0607c 100644 --- a/kubernetes/clusters/homelab/wordpress.yaml +++ b/kubernetes/clusters/homelab/wordpress.yaml @@ -12,7 +12,9 @@ spec: name: flux-system path: ./kubernetes/apps/wordpress prune: true + wait: true dependsOn: + - name: restore-wordpress # infrastructure gates on the mariadb-operator HelmRelease being Ready, # so the MariaDB CRD exists and has a controller before this applies. - name: infrastructure diff --git a/kubernetes/cnpg-object-stores/ak-pg.yaml b/kubernetes/cnpg-object-stores/ak-pg.yaml new file mode 100644 index 00000000..b466a9b7 --- /dev/null +++ b/kubernetes/cnpg-object-stores/ak-pg.yaml @@ -0,0 +1,54 @@ +--- +apiVersion: barmancloud.cnpg.io/v1 +kind: ObjectStore +metadata: + name: ak-pg-store + namespace: authentik + annotations: + kustomize.toolkit.fluxcd.io/prune: disabled +spec: + configuration: + destinationPath: ${CNPG_AK_PG_ARCHIVE} + endpointURL: https://fsn1.your-objectstorage.com + s3Credentials: + accessKeyId: + name: hetzner-s3-creds + key: ACCESS_KEY_ID + secretAccessKey: + name: hetzner-s3-creds + key: ACCESS_SECRET_KEY + region: + name: hetzner-s3-creds + key: REGION + wal: + compression: gzip + data: + compression: gzip + retentionPolicy: "30d" +--- +apiVersion: barmancloud.cnpg.io/v1 +kind: ObjectStore +metadata: + name: ak-pg-store-source + namespace: authentik + annotations: + kustomize.toolkit.fluxcd.io/prune: disabled + cnpg.renner.uno/server-name: ak-pg +spec: + configuration: + destinationPath: ${CNPG_AK_PG_SOURCE} + endpointURL: https://fsn1.your-objectstorage.com + s3Credentials: + accessKeyId: + name: hetzner-s3-creds + key: ACCESS_KEY_ID + secretAccessKey: + name: hetzner-s3-creds + key: ACCESS_SECRET_KEY + region: + name: hetzner-s3-creds + key: REGION + wal: + compression: gzip + data: + compression: gzip diff --git a/kubernetes/cnpg-object-stores/ha-pg.yaml b/kubernetes/cnpg-object-stores/ha-pg.yaml new file mode 100644 index 00000000..c32f4cdc --- /dev/null +++ b/kubernetes/cnpg-object-stores/ha-pg.yaml @@ -0,0 +1,54 @@ +--- +apiVersion: barmancloud.cnpg.io/v1 +kind: ObjectStore +metadata: + name: ha-pg-store + namespace: home-assistant + annotations: + kustomize.toolkit.fluxcd.io/prune: disabled +spec: + configuration: + destinationPath: ${CNPG_HA_PG_ARCHIVE} + endpointURL: https://fsn1.your-objectstorage.com + s3Credentials: + accessKeyId: + name: hetzner-s3-creds + key: ACCESS_KEY_ID + secretAccessKey: + name: hetzner-s3-creds + key: ACCESS_SECRET_KEY + region: + name: hetzner-s3-creds + key: REGION + wal: + compression: gzip + data: + compression: gzip + retentionPolicy: "30d" +--- +apiVersion: barmancloud.cnpg.io/v1 +kind: ObjectStore +metadata: + name: ha-pg-store-source + namespace: home-assistant + annotations: + kustomize.toolkit.fluxcd.io/prune: disabled + cnpg.renner.uno/server-name: ha-pg +spec: + configuration: + destinationPath: ${CNPG_HA_PG_SOURCE} + endpointURL: https://fsn1.your-objectstorage.com + s3Credentials: + accessKeyId: + name: hetzner-s3-creds + key: ACCESS_KEY_ID + secretAccessKey: + name: hetzner-s3-creds + key: ACCESS_SECRET_KEY + region: + name: hetzner-s3-creds + key: REGION + wal: + compression: gzip + data: + compression: gzip diff --git a/kubernetes/cnpg-object-stores/im-pg.yaml b/kubernetes/cnpg-object-stores/im-pg.yaml new file mode 100644 index 00000000..a4e4025e --- /dev/null +++ b/kubernetes/cnpg-object-stores/im-pg.yaml @@ -0,0 +1,54 @@ +--- +apiVersion: barmancloud.cnpg.io/v1 +kind: ObjectStore +metadata: + name: im-pg-store + namespace: immich + annotations: + kustomize.toolkit.fluxcd.io/prune: disabled +spec: + configuration: + destinationPath: ${CNPG_IM_PG_ARCHIVE} + endpointURL: https://fsn1.your-objectstorage.com + s3Credentials: + accessKeyId: + name: hetzner-s3-creds + key: ACCESS_KEY_ID + secretAccessKey: + name: hetzner-s3-creds + key: ACCESS_SECRET_KEY + region: + name: hetzner-s3-creds + key: REGION + wal: + compression: gzip + data: + compression: gzip + retentionPolicy: "30d" +--- +apiVersion: barmancloud.cnpg.io/v1 +kind: ObjectStore +metadata: + name: im-pg-store-source + namespace: immich + annotations: + kustomize.toolkit.fluxcd.io/prune: disabled + cnpg.renner.uno/server-name: im-pg +spec: + configuration: + destinationPath: ${CNPG_IM_PG_SOURCE} + endpointURL: https://fsn1.your-objectstorage.com + s3Credentials: + accessKeyId: + name: hetzner-s3-creds + key: ACCESS_KEY_ID + secretAccessKey: + name: hetzner-s3-creds + key: ACCESS_SECRET_KEY + region: + name: hetzner-s3-creds + key: REGION + wal: + compression: gzip + data: + compression: gzip diff --git a/kubernetes/cnpg-object-stores/kustomization.yaml b/kubernetes/cnpg-object-stores/kustomization.yaml new file mode 100644 index 00000000..585621c7 --- /dev/null +++ b/kubernetes/cnpg-object-stores/kustomization.yaml @@ -0,0 +1,24 @@ +# Every CNPG cluster's two ObjectStores: -store, where it archives WAL and +# takes base backups, and -store-source, which its bootstrap.recovery reads. +# +# Their destinationPaths are Flux variables from the cnpg-paths ConfigMap +# (clusters/homelab/cnpg-object-stores.yaml). Nothing else may live in this +# directory: postBuild substitution blanks every ${...} it renders. +# +# The source store carries cnpg.renner.uno/server-name, the cluster whose +# backups bootstrap.sh looks for, and optionally +# cnpg.renner.uno/source-prefixes, bucket prefixes outside the and +# -g pattern that may still hold them. +# +# No SSE: Hetzner supports only SSE-C, unusable by barman-cloud (plugin #646), +# so at-rest encryption is unavailable; transit is TLS. +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ak-pg.yaml + - ha-pg.yaml + - im-pg.yaml + - mx-pg.yaml + - pprl-pg.yaml + - tr-pg.yaml + - wp-pg.yaml diff --git a/kubernetes/cnpg-object-stores/mx-pg.yaml b/kubernetes/cnpg-object-stores/mx-pg.yaml new file mode 100644 index 00000000..21b16b73 --- /dev/null +++ b/kubernetes/cnpg-object-stores/mx-pg.yaml @@ -0,0 +1,54 @@ +--- +apiVersion: barmancloud.cnpg.io/v1 +kind: ObjectStore +metadata: + name: mx-pg-store + namespace: matrix + annotations: + kustomize.toolkit.fluxcd.io/prune: disabled +spec: + configuration: + destinationPath: ${CNPG_MX_PG_ARCHIVE} + endpointURL: https://fsn1.your-objectstorage.com + s3Credentials: + accessKeyId: + name: hetzner-s3-creds + key: ACCESS_KEY_ID + secretAccessKey: + name: hetzner-s3-creds + key: ACCESS_SECRET_KEY + region: + name: hetzner-s3-creds + key: REGION + wal: + compression: gzip + data: + compression: gzip + retentionPolicy: "30d" +--- +apiVersion: barmancloud.cnpg.io/v1 +kind: ObjectStore +metadata: + name: mx-pg-store-source + namespace: matrix + annotations: + kustomize.toolkit.fluxcd.io/prune: disabled + cnpg.renner.uno/server-name: mx-pg +spec: + configuration: + destinationPath: ${CNPG_MX_PG_SOURCE} + endpointURL: https://fsn1.your-objectstorage.com + s3Credentials: + accessKeyId: + name: hetzner-s3-creds + key: ACCESS_KEY_ID + secretAccessKey: + name: hetzner-s3-creds + key: ACCESS_SECRET_KEY + region: + name: hetzner-s3-creds + key: REGION + wal: + compression: gzip + data: + compression: gzip diff --git a/kubernetes/cnpg-object-stores/pprl-pg.yaml b/kubernetes/cnpg-object-stores/pprl-pg.yaml new file mode 100644 index 00000000..aae6cb6f --- /dev/null +++ b/kubernetes/cnpg-object-stores/pprl-pg.yaml @@ -0,0 +1,54 @@ +--- +apiVersion: barmancloud.cnpg.io/v1 +kind: ObjectStore +metadata: + name: pprl-pg-store + namespace: paperless + annotations: + kustomize.toolkit.fluxcd.io/prune: disabled +spec: + configuration: + destinationPath: ${CNPG_PPRL_PG_ARCHIVE} + endpointURL: https://fsn1.your-objectstorage.com + s3Credentials: + accessKeyId: + name: hetzner-s3-creds + key: ACCESS_KEY_ID + secretAccessKey: + name: hetzner-s3-creds + key: ACCESS_SECRET_KEY + region: + name: hetzner-s3-creds + key: REGION + wal: + compression: gzip + data: + compression: gzip + retentionPolicy: "30d" +--- +apiVersion: barmancloud.cnpg.io/v1 +kind: ObjectStore +metadata: + name: pprl-pg-store-source + namespace: paperless + annotations: + kustomize.toolkit.fluxcd.io/prune: disabled + cnpg.renner.uno/server-name: pprl-pg +spec: + configuration: + destinationPath: ${CNPG_PPRL_PG_SOURCE} + endpointURL: https://fsn1.your-objectstorage.com + s3Credentials: + accessKeyId: + name: hetzner-s3-creds + key: ACCESS_KEY_ID + secretAccessKey: + name: hetzner-s3-creds + key: ACCESS_SECRET_KEY + region: + name: hetzner-s3-creds + key: REGION + wal: + compression: gzip + data: + compression: gzip diff --git a/kubernetes/cnpg-object-stores/tr-pg.yaml b/kubernetes/cnpg-object-stores/tr-pg.yaml new file mode 100644 index 00000000..d8b93654 --- /dev/null +++ b/kubernetes/cnpg-object-stores/tr-pg.yaml @@ -0,0 +1,54 @@ +--- +apiVersion: barmancloud.cnpg.io/v1 +kind: ObjectStore +metadata: + name: tr-pg-store + namespace: trmnl + annotations: + kustomize.toolkit.fluxcd.io/prune: disabled +spec: + configuration: + destinationPath: ${CNPG_TR_PG_ARCHIVE} + endpointURL: https://fsn1.your-objectstorage.com + s3Credentials: + accessKeyId: + name: hetzner-s3-creds + key: ACCESS_KEY_ID + secretAccessKey: + name: hetzner-s3-creds + key: ACCESS_SECRET_KEY + region: + name: hetzner-s3-creds + key: REGION + wal: + compression: gzip + data: + compression: gzip + retentionPolicy: "30d" +--- +apiVersion: barmancloud.cnpg.io/v1 +kind: ObjectStore +metadata: + name: tr-pg-store-source + namespace: trmnl + annotations: + kustomize.toolkit.fluxcd.io/prune: disabled + cnpg.renner.uno/server-name: tr-pg +spec: + configuration: + destinationPath: ${CNPG_TR_PG_SOURCE} + endpointURL: https://fsn1.your-objectstorage.com + s3Credentials: + accessKeyId: + name: hetzner-s3-creds + key: ACCESS_KEY_ID + secretAccessKey: + name: hetzner-s3-creds + key: ACCESS_SECRET_KEY + region: + name: hetzner-s3-creds + key: REGION + wal: + compression: gzip + data: + compression: gzip diff --git a/kubernetes/apps/woodpecker/objectstore.yaml b/kubernetes/cnpg-object-stores/wp-pg.yaml similarity index 63% rename from kubernetes/apps/woodpecker/objectstore.yaml rename to kubernetes/cnpg-object-stores/wp-pg.yaml index ef132eb7..e4f7f6e6 100644 --- a/kubernetes/apps/woodpecker/objectstore.yaml +++ b/kubernetes/cnpg-object-stores/wp-pg.yaml @@ -1,46 +1,14 @@ --- -# RECOVERY SOURCE — read-only. The prefix wp-pg archives to (wp-pg-store below). -# -# serverName on the consuming externalCluster must be `wp-pg`, the name of the -# cluster that produced these backups. -apiVersion: barmancloud.cnpg.io/v1 -kind: ObjectStore -metadata: - name: wp-pg-store-source - namespace: woodpecker -spec: - configuration: - destinationPath: s3://renner-uno-k8s-cnpg/wp-pg-woodpecker - endpointURL: https://fsn1.your-objectstorage.com - s3Credentials: - accessKeyId: - name: hetzner-s3-creds - key: ACCESS_KEY_ID - secretAccessKey: - name: hetzner-s3-creds - key: ACCESS_SECRET_KEY - region: - name: hetzner-s3-creds - key: REGION - wal: - compression: gzip - data: - compression: gzip - # No retentionPolicy: wp-pg-store owns the lifecycle of this prefix. ---- -# ARCHIVING TARGET. -# -# Not s3://renner-uno-k8s-cnpg/wp-pg: the retired `apps/wp-pg` archived there, -# and two clusters archiving WAL to one destinationPath interleave their streams -# and corrupt both. apiVersion: barmancloud.cnpg.io/v1 kind: ObjectStore metadata: name: wp-pg-store namespace: woodpecker + annotations: + kustomize.toolkit.fluxcd.io/prune: disabled spec: configuration: - destinationPath: s3://renner-uno-k8s-cnpg/wp-pg-woodpecker + destinationPath: ${CNPG_WP_PG_ARCHIVE} endpointURL: https://fsn1.your-objectstorage.com s3Credentials: accessKeyId: @@ -57,3 +25,32 @@ spec: data: compression: gzip retentionPolicy: "30d" +--- +apiVersion: barmancloud.cnpg.io/v1 +kind: ObjectStore +metadata: + name: wp-pg-store-source + namespace: woodpecker + annotations: + kustomize.toolkit.fluxcd.io/prune: disabled + cnpg.renner.uno/server-name: wp-pg + # wp-pg's archive since its move out of `apps`. + cnpg.renner.uno/source-prefixes: wp-pg-woodpecker +spec: + configuration: + destinationPath: ${CNPG_WP_PG_SOURCE} + endpointURL: https://fsn1.your-objectstorage.com + s3Credentials: + accessKeyId: + name: hetzner-s3-creds + key: ACCESS_KEY_ID + secretAccessKey: + name: hetzner-s3-creds + key: ACCESS_SECRET_KEY + region: + name: hetzner-s3-creds + key: REGION + wal: + compression: gzip + data: + compression: gzip diff --git a/kubernetes/infrastructure/bird/bird.conf b/kubernetes/infrastructure-inhouse/bird/bird.conf similarity index 100% rename from kubernetes/infrastructure/bird/bird.conf rename to kubernetes/infrastructure-inhouse/bird/bird.conf diff --git a/kubernetes/infrastructure/bird/daemonset.yaml b/kubernetes/infrastructure-inhouse/bird/daemonset.yaml similarity index 94% rename from kubernetes/infrastructure/bird/daemonset.yaml rename to kubernetes/infrastructure-inhouse/bird/daemonset.yaml index 0a5da93e..c5aea481 100644 --- a/kubernetes/infrastructure/bird/daemonset.yaml +++ b/kubernetes/infrastructure-inhouse/bird/daemonset.yaml @@ -51,6 +51,10 @@ spec: # Owns table 47, where all pod traffic is routed. Without it the table # stays empty and the rules fall through to main, unseparated. priorityClassName: system-node-critical + # Table 47 exists only where the site patch routes pod traffic into it + # (RoutingRuleConfig); the same patch labels those nodes. + nodeSelector: + net.renner.uno/pod-table: "47" # Writes host routes and peers on host addresses. hostNetwork: true serviceAccountName: bird diff --git a/kubernetes/infrastructure/bird/defines.tmpl b/kubernetes/infrastructure-inhouse/bird/defines.tmpl similarity index 100% rename from kubernetes/infrastructure/bird/defines.tmpl rename to kubernetes/infrastructure-inhouse/bird/defines.tmpl diff --git a/kubernetes/infrastructure/bird/kustomization.yaml b/kubernetes/infrastructure-inhouse/bird/kustomization.yaml similarity index 100% rename from kubernetes/infrastructure/bird/kustomization.yaml rename to kubernetes/infrastructure-inhouse/bird/kustomization.yaml diff --git a/kubernetes/infrastructure/bird/peers.tmpl b/kubernetes/infrastructure-inhouse/bird/peers.tmpl similarity index 100% rename from kubernetes/infrastructure/bird/peers.tmpl rename to kubernetes/infrastructure-inhouse/bird/peers.tmpl diff --git a/kubernetes/infrastructure-inhouse/kustomization.yaml b/kubernetes/infrastructure-inhouse/kustomization.yaml new file mode 100644 index 00000000..30d1a476 --- /dev/null +++ b/kubernetes/infrastructure-inhouse/kustomization.yaml @@ -0,0 +1,7 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +# Infrastructure that runs images from Gitea's registry with no copy on +# ghcr.io, so it cannot start before Gitea serves them. +resources: + - bird/ + - netbird-service-sync/ diff --git a/kubernetes/infrastructure/netbird-service-sync/deployment.yaml b/kubernetes/infrastructure-inhouse/netbird-service-sync/deployment.yaml similarity index 100% rename from kubernetes/infrastructure/netbird-service-sync/deployment.yaml rename to kubernetes/infrastructure-inhouse/netbird-service-sync/deployment.yaml diff --git a/kubernetes/infrastructure/netbird-service-sync/kustomization.yaml b/kubernetes/infrastructure-inhouse/netbird-service-sync/kustomization.yaml similarity index 100% rename from kubernetes/infrastructure/netbird-service-sync/kustomization.yaml rename to kubernetes/infrastructure-inhouse/netbird-service-sync/kustomization.yaml diff --git a/kubernetes/infrastructure/ceph-csi-drivers/helmrelease.yaml b/kubernetes/infrastructure/ceph-csi-drivers/helmrelease.yaml new file mode 100644 index 00000000..8c548ce3 --- /dev/null +++ b/kubernetes/infrastructure/ceph-csi-drivers/helmrelease.yaml @@ -0,0 +1,79 @@ +apiVersion: source.toolkit.fluxcd.io/v1 +kind: HelmRepository +metadata: + name: ceph-csi-operator + namespace: rook-ceph +spec: + interval: 24h + url: https://ceph.github.io/ceph-csi-operator +--- +# The CSI Driver and OperatorConfig resources Rook requires +# (rook/Documentation/Helm-Charts/csi-drivers-chart.md). Values are Rook's +# recommended ones (rook/deploy/charts/ceph-csi-drivers/values.yaml) plus the +# shape this cluster runs with. +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: ceph-csi-drivers + namespace: rook-ceph +spec: + interval: 30m + dependsOn: + - name: rook-ceph-operator + chart: + spec: + chart: ceph-csi-drivers + sourceRef: + kind: HelmRepository + name: ceph-csi-operator + namespace: rook-ceph + install: + remediation: + retries: 3 + upgrade: + remediation: + retries: 3 + values: + operatorConfig: + namespace: rook-ceph + driverSpecDefaults: + imageSet: + name: rook-csi-operator-image-set-configmap + nodePlugin: + priorityClassName: system-node-critical + controllerPlugin: + priorityClassName: system-cluster-critical + drivers: + rbd: + enabled: true + name: rook-ceph.rbd.csi.ceph.com + # Velero snapshots ceph-block claims through CSI. + snapshotPolicy: volumeSnapshot + nodePlugin: + tolerations: + - key: workload-class + operator: Equal + value: pinned + effect: NoSchedule + controllerPlugin: + replicas: 2 + deploymentStrategy: + type: Recreate + cephfs: + enabled: true + name: rook-ceph.cephfs.csi.ceph.com + snapshotPolicy: volumeGroupSnapshot + nodePlugin: + tolerations: + - key: workload-class + operator: Equal + value: pinned + effect: NoSchedule + controllerPlugin: + replicas: 2 + deploymentStrategy: + type: Recreate + nfs: + enabled: false + nvmeof: + enabled: false diff --git a/kubernetes/infrastructure/ceph-csi-drivers/kustomization.yaml b/kubernetes/infrastructure/ceph-csi-drivers/kustomization.yaml index df7a9f93..a53f6ad0 100644 --- a/kubernetes/infrastructure/ceph-csi-drivers/kustomization.yaml +++ b/kubernetes/infrastructure/ceph-csi-drivers/kustomization.yaml @@ -1,5 +1,6 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: + - helmrelease.yaml - serviceaccounts.yaml - rbac.yaml diff --git a/kubernetes/infrastructure/cilium/helmrelease.yaml b/kubernetes/infrastructure/cilium/helmrelease.yaml index cae0738c..ec50484e 100644 --- a/kubernetes/infrastructure/cilium/helmrelease.yaml +++ b/kubernetes/infrastructure/cilium/helmrelease.yaml @@ -184,26 +184,13 @@ spec: # Cilium's, the sessions re-establish. bgpControlPlane: enabled: true - # Masquerading exception, not a routing statement: pod traffic to these - # ranges leaves unmasqueraded, so LAN services see real pod IPs. That only - # works because BGP advertises the pod CIDRs, which is what gives the UDM a - # route back. The two settings are one decision and have to move together. - # - # Pod CIDRs are 10.244.0.0/16 and fd10:244::/48 (talos/patches/cluster.yaml); - # the supernets here cover the LAN as well, which is the point. - ipv4NativeRoutingCIDR: 10.0.0.0/8 - ipv6NativeRoutingCIDR: fd00::/8 - # Pod east-west leaves the node on pod0 (VLAN 48) rather than the - # management VLAN. Native routing hands forwarding to the kernel; - # autoDirectNodeRoutes installs the remote PodCIDR routes, which is - # allowed because all three nodes share pod0's L2 segment. - # - # directRoutingDevice is explicit on purpose: with several addressed - # devices Cilium falls back to the one carrying the Kubernetes InternalIP, - # which is mgmt0. - # - # Encapsulation is gone, so the datapath MTU goes 1450 -> 1500. - routingMode: native - autoDirectNodeRoutes: true - nodePort: - directRoutingDevice: pod0 + # What the site's network can carry decides how pod traffic crosses between + # nodes and which destinations see real pod IPs: routingMode, + # autoDirectNodeRoutes, the direct-routing device and the native-routing + # (masquerade-exempt) CIDRs come from the cilium-site ConfigMap that each + # Talos site patch carries (talos/patches/site-*.yaml). They must not be set + # in values above, which would override it. + valuesFrom: + - kind: ConfigMap + name: cilium-site + valuesKey: values.yaml diff --git a/kubernetes/infrastructure/post-crds/crowdsec-bouncer-key-externalsecret.yaml b/kubernetes/infrastructure/crowdsec/crowdsec-bouncer-key-externalsecret.yaml similarity index 100% rename from kubernetes/infrastructure/post-crds/crowdsec-bouncer-key-externalsecret.yaml rename to kubernetes/infrastructure/crowdsec/crowdsec-bouncer-key-externalsecret.yaml diff --git a/kubernetes/infrastructure/post-crds/crowdsec-edge-bouncer-key-externalsecret.yaml b/kubernetes/infrastructure/crowdsec/crowdsec-edge-bouncer-key-externalsecret.yaml similarity index 100% rename from kubernetes/infrastructure/post-crds/crowdsec-edge-bouncer-key-externalsecret.yaml rename to kubernetes/infrastructure/crowdsec/crowdsec-edge-bouncer-key-externalsecret.yaml diff --git a/kubernetes/infrastructure/crowdsec/kustomization.yaml b/kubernetes/infrastructure/crowdsec/kustomization.yaml index 23d63af4..6e4ac540 100644 --- a/kubernetes/infrastructure/crowdsec/kustomization.yaml +++ b/kubernetes/infrastructure/crowdsec/kustomization.yaml @@ -3,3 +3,5 @@ kind: Kustomization resources: - lapi-ingress.yaml - helmrelease.yaml + - crowdsec-bouncer-key-externalsecret.yaml + - crowdsec-edge-bouncer-key-externalsecret.yaml diff --git a/kubernetes/infrastructure/post-crds/external-dns-unifi-externalsecret.yaml b/kubernetes/infrastructure/external-dns-unifi/externalsecret.yaml similarity index 100% rename from kubernetes/infrastructure/post-crds/external-dns-unifi-externalsecret.yaml rename to kubernetes/infrastructure/external-dns-unifi/externalsecret.yaml diff --git a/kubernetes/infrastructure/external-dns-unifi/kustomization.yaml b/kubernetes/infrastructure/external-dns-unifi/kustomization.yaml index 4fd939d4..bc610851 100644 --- a/kubernetes/infrastructure/external-dns-unifi/kustomization.yaml +++ b/kubernetes/infrastructure/external-dns-unifi/kustomization.yaml @@ -2,3 +2,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - helmrelease.yaml + - externalsecret.yaml diff --git a/kubernetes/infrastructure/intel-gpu-plugin/daemonset.yaml b/kubernetes/infrastructure/intel-gpu-plugin/daemonset.yaml index 556dfe76..fb9cde14 100644 --- a/kubernetes/infrastructure/intel-gpu-plugin/daemonset.yaml +++ b/kubernetes/infrastructure/intel-gpu-plugin/daemonset.yaml @@ -119,3 +119,4 @@ spec: type: DirectoryOrCreate nodeSelector: kubernetes.io/arch: amd64 + hardware.renner.uno/intel-gpu: "true" diff --git a/kubernetes/infrastructure/k8s-gateway/helmrelease.yaml b/kubernetes/infrastructure/k8s-gateway/helmrelease.yaml index 20c99f49..903853ee 100644 --- a/kubernetes/infrastructure/k8s-gateway/helmrelease.yaml +++ b/kubernetes/infrastructure/k8s-gateway/helmrelease.yaml @@ -19,13 +19,10 @@ spec: # `auth.renner.uno`, `audiobooks.renner.uno`, etc. stay inside the # cluster instead of hairpinning through the legacy Caddy VIP. # -# Names not backed by an in-cluster Ingress fall through to the UDM -# (10.2.40.1 / fd00::1), which serves the split-horizon renner.uno zone — -# preserving resolution of LXC/VM-hosted services (gitea, immich, …) and -# physical devices (udm.renner.uno, …) without manual enumeration. -# NOTE: the old Proxmox DNS HA VIP (10.2.40.53) is DECOMMISSIONED; pointing -# here dead-ended non-Ingress resolution and broke ~1374 HA entities on a -# pod restart (2026-07-14). Do not reintroduce it. +# Names not backed by an in-cluster Ingress fall through to the site's +# resolver: the pod's own resolv.conf, which the site patch sets through the +# k8s-gateway-site ConfigMap (deployment.dnsPolicy/dnsConfig). Never the node's +# resolver: the nodes ask k8s-gateway first, which would loop. apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: @@ -95,7 +92,7 @@ spec: - name: loop - name: reload - name: forward - parameters: ". 10.2.40.1 fd00::1" + parameters: ". /etc/resolv.conf" - name: prometheus parameters: "0.0.0.0:9153" serviceType: ClusterIP @@ -106,3 +103,10 @@ spec: # populated end of the service CIDR (kube-dns is 10.96.0.10). clusterIP: 10.96.0.11 ipFamilyPolicy: PreferDualStack + # The site's upstream resolvers (deployment.dnsPolicy/dnsConfig), from the + # k8s-gateway-site ConfigMap each Talos site patch carries. Not set in values + # above, which would override it. + valuesFrom: + - kind: ConfigMap + name: k8s-gateway-site + valuesKey: values.yaml diff --git a/kubernetes/infrastructure/kustomization.yaml b/kubernetes/infrastructure/kustomization.yaml index 577755c8..79656783 100644 --- a/kubernetes/infrastructure/kustomization.yaml +++ b/kubernetes/infrastructure/kustomization.yaml @@ -7,23 +7,17 @@ resources: - cluster-settings/ - cilium/ - multus/ - - bird/ - cert-manager/ - snapshot-controller/ - metrics-server/ - intel-gpu-plugin/ - - crowdsec/ - traefik/ - rook-ceph/ - ceph-csi-drivers/ - cloudnative-pg/ - mariadb-operator/ - - netbird-ingress/ - - netbird-service-sync/ - coredns-config/ - k8s-gateway/ - - external-dns-unifi/ - - velero/ - image-automation/ - flux-notifications/ - spegel/ @@ -39,3 +33,9 @@ resources: # Deployment mounts. Including it here causes the Deployment to block # on missing Secrets while infrastructure itself cannot finish without # the Deployment becoming Ready — circular. +# bird/ and netbird-service-sync/ run images from Gitea's registry and live in +# infrastructure-inhouse/, which waits for Gitea. +# crowdsec/, external-dns-unifi/, netbird-ingress/ and velero/ consume Secrets +# from OpenBao and have Kustomizations of their own after infrastructure-crds +# (clusters/homelab/infrastructure-services.yaml); velero-schedules/ follows +# velero. diff --git a/kubernetes/infrastructure/post-crds/netbird-ingress-externalsecret.yaml b/kubernetes/infrastructure/netbird-ingress/externalsecret.yaml similarity index 100% rename from kubernetes/infrastructure/post-crds/netbird-ingress-externalsecret.yaml rename to kubernetes/infrastructure/netbird-ingress/externalsecret.yaml diff --git a/kubernetes/infrastructure/netbird-ingress/kustomization.yaml b/kubernetes/infrastructure/netbird-ingress/kustomization.yaml index 48f1d3d7..43bb9c6c 100644 --- a/kubernetes/infrastructure/netbird-ingress/kustomization.yaml +++ b/kubernetes/infrastructure/netbird-ingress/kustomization.yaml @@ -2,3 +2,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - daemonset.yaml + - externalsecret.yaml diff --git a/kubernetes/infrastructure/post-crds/kustomization.yaml b/kubernetes/infrastructure/post-crds/kustomization.yaml index ddc88e40..de063631 100644 --- a/kubernetes/infrastructure/post-crds/kustomization.yaml +++ b/kubernetes/infrastructure/post-crds/kustomization.yaml @@ -7,7 +7,6 @@ resources: - cnpg-barman-certs.yaml # Private CA for MQTT client/broker certificates (REN-802). - mqtt-ca.yaml - - velero-schedules.yaml # Only safe here because OpenBao is already initialized + k8s-auth is # configured (2026-07-13): a store that can't turn Ready would block this # KS's wait gate and freeze the dependsOn chain. @@ -18,14 +17,11 @@ resources: # the ESO HelmRelease that would have installed the CRDs. They sat next to # their consumers until a cold-rebuild test hit that deadlock (2026-09-01); # nothing had ever applied this layer against an empty API server before. - - crowdsec-bouncer-key-externalsecret.yaml - - crowdsec-edge-bouncer-key-externalsecret.yaml - - external-dns-unifi-externalsecret.yaml + # ExternalSecrets whose consumer is a workload live beside it, in that + # workload's own Kustomization after this one (crowdsec, external-dns-unifi, + # netbird-ingress, velero): applied here, the consumer would start before + # its Secret exists. - flux-git-externalsecret.yaml - - netbird-ingress-externalsecret.yaml - - velero-externalsecret.yaml - - velero-hetzner-creds-externalsecret.yaml - - hcloud-backup-verify-externalsecret.yaml # traefik.io CRDs, same reasoning — installed by the Traefik HelmRelease in # infrastructure. Traefik is deliberately absent from that Kustomization's # healthChecks, so this KS may dry-run before the chart lands; it retries on diff --git a/kubernetes/infrastructure/velero-schedules/kustomization.yaml b/kubernetes/infrastructure/velero-schedules/kustomization.yaml new file mode 100644 index 00000000..bcaf1271 --- /dev/null +++ b/kubernetes/infrastructure/velero-schedules/kustomization.yaml @@ -0,0 +1,6 @@ +# Velero's Schedules: their velero.io CRDs come from the Velero HelmRelease +# (../velero), so they apply after it. +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - schedules.yaml diff --git a/kubernetes/infrastructure/post-crds/velero-schedules.yaml b/kubernetes/infrastructure/velero-schedules/schedules.yaml similarity index 98% rename from kubernetes/infrastructure/post-crds/velero-schedules.yaml rename to kubernetes/infrastructure/velero-schedules/schedules.yaml index 7a25784d..ccea8bc6 100644 --- a/kubernetes/infrastructure/post-crds/velero-schedules.yaml +++ b/kubernetes/infrastructure/velero-schedules/schedules.yaml @@ -9,7 +9,9 @@ # snapshotMoveData uploads PVC contents through Kopia and the node-agent, # independently of the K8s API surface. # -# Naming: -daily. Restore is manual — see RUNBOOK-cold-rebuild.md. +# Naming: -daily. Each has a Restore in kubernetes/restore// and a +# restore- Kustomization (clusters/homelab/restore.yaml), which a fresh +# cluster runs before the app starts. # Retention matches the CNPG Barman policy. apiVersion: velero.io/v1 @@ -199,7 +201,6 @@ spec: # This is the ONLY restore path for home-assistant-config (.storage, # secrets.yaml, the config git checkout). The HAOS-backup `seed` # initContainer that used to repopulate it was removed in 21a0a43f. - # Restore procedure and test: REN-249 runbook section C.2. schedule: "45 6 * * *" useOwnerReferencesInBackup: false template: diff --git a/kubernetes/infrastructure/post-crds/hcloud-backup-verify-externalsecret.yaml b/kubernetes/infrastructure/velero/hcloud-backup-verify-externalsecret.yaml similarity index 100% rename from kubernetes/infrastructure/post-crds/hcloud-backup-verify-externalsecret.yaml rename to kubernetes/infrastructure/velero/hcloud-backup-verify-externalsecret.yaml diff --git a/kubernetes/infrastructure/velero/kustomization.yaml b/kubernetes/infrastructure/velero/kustomization.yaml index 83984f0b..f929d884 100644 --- a/kubernetes/infrastructure/velero/kustomization.yaml +++ b/kubernetes/infrastructure/velero/kustomization.yaml @@ -6,9 +6,11 @@ resources: - helmrelease.yaml - backup-verify-rbac.yaml - backup-verify-cronjob.yaml -# Velero Schedule CRs live in kubernetes/infrastructure/post-crds/ -# (velero-schedules.yaml) so they reconcile after the Velero HelmRelease -# finishes installing velero.io CRDs. + - velero-externalsecret.yaml + - velero-hetzner-creds-externalsecret.yaml + - hcloud-backup-verify-externalsecret.yaml +# Velero Schedule CRs live in ../velero-schedules so they reconcile after the +# Velero HelmRelease installs the velero.io CRDs. # Bake the backup-verify scripts into a ConfigMap mounted by the CronJob. # Hash suffix is disabled because Kustomize's NameReference transformer diff --git a/kubernetes/infrastructure/post-crds/velero-externalsecret.yaml b/kubernetes/infrastructure/velero/velero-externalsecret.yaml similarity index 100% rename from kubernetes/infrastructure/post-crds/velero-externalsecret.yaml rename to kubernetes/infrastructure/velero/velero-externalsecret.yaml diff --git a/kubernetes/infrastructure/post-crds/velero-hetzner-creds-externalsecret.yaml b/kubernetes/infrastructure/velero/velero-hetzner-creds-externalsecret.yaml similarity index 100% rename from kubernetes/infrastructure/post-crds/velero-hetzner-creds-externalsecret.yaml rename to kubernetes/infrastructure/velero/velero-hetzner-creds-externalsecret.yaml diff --git a/kubernetes/restore-test/egress-check.yaml b/kubernetes/restore-test/egress-check.yaml new file mode 100644 index 00000000..d49be521 --- /dev/null +++ b/kubernetes/restore-test/egress-check.yaml @@ -0,0 +1,36 @@ +# Asserts the restore test's containment from a pod, before Flux restores +# production's secrets. Applied by bootstrap.sh (hcloudtest, and +# FLUX_SOURCE=restore-test flux), which substitutes PROXY_ADDRS; exits non-zero +# if any expectation does not hold. +apiVersion: v1 +kind: Pod +metadata: + name: egress-check + namespace: default +spec: + restartPolicy: Never + containers: + - name: check + image: docker.io/nicolaka/netshoot:v0.13 + command: + - /bin/bash + - -c + - | + rc=0 + expect() { # + if [[ "$3" == $2 ]]; then echo "PASS $1: $3"; else echo "FAIL $1: got '$3', want '$2'"; rc=1; fi + } + code() { curl -sS -m 8 -o /dev/null -w '%{http_code} via %{remote_ip}' "$1" 2>&1 || true; } + port() { local out; if out=$(nc -z -w 5 "$1" "$2" 2>&1); then echo open; else echo "closed ${out}"; fi; } + expect "public DNS resolves" "[0-9]*.[0-9]*.[0-9]*.[0-9]*" "$(dig +short example.com A 2>&1 | head -1)" + expect "github.com https reachable" "[23]?? via *" "$(code https://github.com)" + expect "github.com ssh reachable" "open" "$(port github.com 22)" + expect "ghcr.io reachable" "401 via *" "$(code https://ghcr.io/v2/)" + expect "object storage reachable" "[1-4]?? via *" "$(code https://fsn1.your-objectstorage.com)" + for a in $PROXY_ADDRS; do + expect "proxy $a:443 blocked" "closed*" "$(port "$a" 443)" + done + exit $rc + env: + - name: PROXY_ADDRS + value: PROXY_ADDRS_PLACEHOLDER diff --git a/kubernetes/restore-test/flux-root.yaml b/kubernetes/restore-test/flux-root.yaml new file mode 100644 index 00000000..7aacc935 --- /dev/null +++ b/kubernetes/restore-test/flux-root.yaml @@ -0,0 +1,144 @@ +# Flux entry point of the off-site restore test (bootstrap.sh hcloudtest, then +# FLUX_SOURCE=restore-test bootstrap.sh flux). Not reconciled by production. +# +# Applies the production tree unchanged. spec.patches below is the complete +# list of differences between the test cluster and production; each is marked +# source, hardware or safety. +apiVersion: source.toolkit.fluxcd.io/v1 +kind: GitRepository +metadata: + name: restore-test + namespace: flux-system +spec: + interval: 10m + url: ssh://git@github.com/konairius/Homelab.git + ref: + branch: master + # Read-only deploy key, created per run by bootstrap.sh. + secretRef: + name: restore-test-git +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: restore-test + namespace: flux-system +spec: + interval: 10m + path: ./kubernetes/clusters/homelab + prune: true + sourceRef: + kind: GitRepository + name: restore-test + patches: + # 1. Source: the GitHub mirror, read-only. In-cluster Gitea is restored by + # this test, not available to it. + - target: + kind: GitRepository + name: flux-system + namespace: flux-system + patch: | + - op: replace + path: /spec/url + value: ssh://git@github.com/konairius/Homelab.git + - op: replace + path: /spec/secretRef/name + value: restore-test-git + # 2. Source: the tree's own root sync is suspended; this Kustomization + # replaces it. + - target: + kind: Kustomization + name: flux-system + namespace: flux-system + patch: | + - op: add + path: /spec/suspend + value: true + # 3. Source: every Kustomization of the tree reads restore-test (the branch + # bootstrap.sh sets) and carries the patches below; a target a + # Kustomization does not contain matches nothing. + - target: + kind: Kustomization + patch: | + - op: replace + path: /spec/sourceRef/name + value: restore-test + - op: add + path: /spec/patches + value: + # 3a. Hardware: no Ceph cluster network. Hetzner networks are + # layer 3 and a macvlan child does not cross servers; Ceph + # replicates over the public (pod) network. + - target: + kind: CephCluster + name: rook-ceph + patch: | + - op: remove + path: /spec/network + # 3b. Safety: Velero ReadOnly blocks backups, deletion, GC and + # repository maintenance; restores and backup sync still work. + - target: + kind: HelmRelease + name: velero + patch: | + - op: add + path: /spec/values/configuration/backupStorageLocation/0/accessMode + value: ReadOnly + # CNPG needs no patch: bootstrap.sh points every cluster's archive + # at the test bucket (cnpg-paths), and the source stores carry no + # retentionPolicy, so WAL, base backups and retention all act on + # the test bucket only. + # 3f. Safety: no MariaDB logical dumps. + - target: + group: k8s.mariadb.com + kind: Backup + patch: | + - op: replace + path: /spec/schedule/suspend + value: true + # 3g. Safety: credential rotators would roll production's tokens + # at their issuers. + - target: + group: batch + kind: CronJob + labelSelector: app.kubernetes.io/component=secret-rotator + patch: | + - op: add + path: /spec/suspend + value: true + # 3h. Safety: Gitea gets no egress beyond the cluster. Its restored + # push mirror force-pushes (--mirror) over the GitHub mirror. + - target: + kind: CiliumNetworkPolicy + name: default-deny + namespace: gitea + patch: | + - op: test + path: /spec/egress/5/toEntities + value: [host, remote-node, world] + - op: replace + path: /spec/egress/5/toEntities + value: [host, remote-node] + # 3i. Safety: no writes to production NetBird. netbird-service-sync + # rewrites the public proxy services at netbird.renner.uno to + # this cluster's Traefik, fighting production's own controller. + - target: + kind: Deployment + name: netbird-service-sync + namespace: netbird-service-sync + patch: | + - op: replace + path: /spec/replicas + value: 0 + # 3j. Safety: no routing peers in the production mesh. They would + # announce this cluster's service and LoadBalancer ranges, the + # same ranges production announces. + - target: + kind: DaemonSet + name: netbird-ingress + namespace: ingress + patch: | + - op: add + path: /spec/template/spec/nodeSelector + value: + restore-test.renner.uno/never: "true" diff --git a/kubernetes/restore/apps/kustomization.yaml b/kubernetes/restore/apps/kustomization.yaml new file mode 100644 index 00000000..bc08b2d5 --- /dev/null +++ b/kubernetes/restore/apps/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - restore.yaml diff --git a/kubernetes/restore/apps/restore.yaml b/kubernetes/restore/apps/restore.yaml new file mode 100644 index 00000000..9bb5a23b --- /dev/null +++ b/kubernetes/restore/apps/restore.yaml @@ -0,0 +1,7 @@ +apiVersion: velero.io/v1 +kind: Restore +metadata: + name: apps + namespace: velero +spec: + scheduleName: apps-daily diff --git a/kubernetes/restore/audiobookshelf/kustomization.yaml b/kubernetes/restore/audiobookshelf/kustomization.yaml new file mode 100644 index 00000000..bc08b2d5 --- /dev/null +++ b/kubernetes/restore/audiobookshelf/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - restore.yaml diff --git a/kubernetes/restore/audiobookshelf/restore.yaml b/kubernetes/restore/audiobookshelf/restore.yaml new file mode 100644 index 00000000..15b6b741 --- /dev/null +++ b/kubernetes/restore/audiobookshelf/restore.yaml @@ -0,0 +1,7 @@ +apiVersion: velero.io/v1 +kind: Restore +metadata: + name: audiobookshelf + namespace: velero +spec: + scheduleName: audiobookshelf-daily diff --git a/kubernetes/restore/authentik/kustomization.yaml b/kubernetes/restore/authentik/kustomization.yaml new file mode 100644 index 00000000..bc08b2d5 --- /dev/null +++ b/kubernetes/restore/authentik/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - restore.yaml diff --git a/kubernetes/restore/authentik/restore.yaml b/kubernetes/restore/authentik/restore.yaml new file mode 100644 index 00000000..88bb44d6 --- /dev/null +++ b/kubernetes/restore/authentik/restore.yaml @@ -0,0 +1,7 @@ +apiVersion: velero.io/v1 +kind: Restore +metadata: + name: authentik + namespace: velero +spec: + scheduleName: authentik-daily diff --git a/kubernetes/restore/gate/kustomization.yaml b/kubernetes/restore/gate/kustomization.yaml new file mode 100644 index 00000000..5191cdb5 --- /dev/null +++ b/kubernetes/restore/gate/kustomization.yaml @@ -0,0 +1,5 @@ +# Applies nothing: its Kustomization exists for its health check on the +# BackupStorageLocation (clusters/homelab/restore.yaml). +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: [] diff --git a/kubernetes/restore/gitea/kustomization.yaml b/kubernetes/restore/gitea/kustomization.yaml new file mode 100644 index 00000000..bc08b2d5 --- /dev/null +++ b/kubernetes/restore/gitea/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - restore.yaml diff --git a/kubernetes/restore/gitea/restore.yaml b/kubernetes/restore/gitea/restore.yaml new file mode 100644 index 00000000..f68caa0e --- /dev/null +++ b/kubernetes/restore/gitea/restore.yaml @@ -0,0 +1,8 @@ +# Gitea first: it is the registry for the in-house images. +apiVersion: velero.io/v1 +kind: Restore +metadata: + name: gitea + namespace: velero +spec: + scheduleName: gitea-daily diff --git a/kubernetes/restore/home-assistant/kustomization.yaml b/kubernetes/restore/home-assistant/kustomization.yaml new file mode 100644 index 00000000..bc08b2d5 --- /dev/null +++ b/kubernetes/restore/home-assistant/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - restore.yaml diff --git a/kubernetes/restore/home-assistant/restore.yaml b/kubernetes/restore/home-assistant/restore.yaml new file mode 100644 index 00000000..72f4e6b0 --- /dev/null +++ b/kubernetes/restore/home-assistant/restore.yaml @@ -0,0 +1,7 @@ +apiVersion: velero.io/v1 +kind: Restore +metadata: + name: home-assistant + namespace: velero +spec: + scheduleName: home-assistant-daily diff --git a/kubernetes/restore/immich/kustomization.yaml b/kubernetes/restore/immich/kustomization.yaml new file mode 100644 index 00000000..bc08b2d5 --- /dev/null +++ b/kubernetes/restore/immich/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - restore.yaml diff --git a/kubernetes/restore/immich/restore.yaml b/kubernetes/restore/immich/restore.yaml new file mode 100644 index 00000000..1c31c184 --- /dev/null +++ b/kubernetes/restore/immich/restore.yaml @@ -0,0 +1,7 @@ +apiVersion: velero.io/v1 +kind: Restore +metadata: + name: immich + namespace: velero +spec: + scheduleName: immich-daily diff --git a/kubernetes/restore/influxdb/kustomization.yaml b/kubernetes/restore/influxdb/kustomization.yaml new file mode 100644 index 00000000..bc08b2d5 --- /dev/null +++ b/kubernetes/restore/influxdb/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - restore.yaml diff --git a/kubernetes/restore/influxdb/restore.yaml b/kubernetes/restore/influxdb/restore.yaml new file mode 100644 index 00000000..9a9b2d0b --- /dev/null +++ b/kubernetes/restore/influxdb/restore.yaml @@ -0,0 +1,7 @@ +apiVersion: velero.io/v1 +kind: Restore +metadata: + name: influxdb + namespace: velero +spec: + scheduleName: influxdb-daily diff --git a/kubernetes/restore/matrix/kustomization.yaml b/kubernetes/restore/matrix/kustomization.yaml new file mode 100644 index 00000000..bc08b2d5 --- /dev/null +++ b/kubernetes/restore/matrix/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - restore.yaml diff --git a/kubernetes/restore/matrix/restore.yaml b/kubernetes/restore/matrix/restore.yaml new file mode 100644 index 00000000..83b5b34f --- /dev/null +++ b/kubernetes/restore/matrix/restore.yaml @@ -0,0 +1,7 @@ +apiVersion: velero.io/v1 +kind: Restore +metadata: + name: matrix + namespace: velero +spec: + scheduleName: matrix-daily diff --git a/kubernetes/restore/music-assistant/kustomization.yaml b/kubernetes/restore/music-assistant/kustomization.yaml new file mode 100644 index 00000000..bc08b2d5 --- /dev/null +++ b/kubernetes/restore/music-assistant/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - restore.yaml diff --git a/kubernetes/restore/music-assistant/restore.yaml b/kubernetes/restore/music-assistant/restore.yaml new file mode 100644 index 00000000..4c5b2643 --- /dev/null +++ b/kubernetes/restore/music-assistant/restore.yaml @@ -0,0 +1,7 @@ +apiVersion: velero.io/v1 +kind: Restore +metadata: + name: music-assistant + namespace: velero +spec: + scheduleName: music-assistant-daily diff --git a/kubernetes/restore/navidrome/kustomization.yaml b/kubernetes/restore/navidrome/kustomization.yaml new file mode 100644 index 00000000..bc08b2d5 --- /dev/null +++ b/kubernetes/restore/navidrome/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - restore.yaml diff --git a/kubernetes/restore/navidrome/restore.yaml b/kubernetes/restore/navidrome/restore.yaml new file mode 100644 index 00000000..4b8a9e1f --- /dev/null +++ b/kubernetes/restore/navidrome/restore.yaml @@ -0,0 +1,7 @@ +apiVersion: velero.io/v1 +kind: Restore +metadata: + name: navidrome + namespace: velero +spec: + scheduleName: navidrome-daily diff --git a/kubernetes/restore/paperless/kustomization.yaml b/kubernetes/restore/paperless/kustomization.yaml new file mode 100644 index 00000000..bc08b2d5 --- /dev/null +++ b/kubernetes/restore/paperless/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - restore.yaml diff --git a/kubernetes/restore/paperless/restore.yaml b/kubernetes/restore/paperless/restore.yaml new file mode 100644 index 00000000..9daaaf0e --- /dev/null +++ b/kubernetes/restore/paperless/restore.yaml @@ -0,0 +1,7 @@ +apiVersion: velero.io/v1 +kind: Restore +metadata: + name: paperless + namespace: velero +spec: + scheduleName: paperless-daily diff --git a/kubernetes/restore/trmnl/kustomization.yaml b/kubernetes/restore/trmnl/kustomization.yaml new file mode 100644 index 00000000..bc08b2d5 --- /dev/null +++ b/kubernetes/restore/trmnl/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - restore.yaml diff --git a/kubernetes/restore/trmnl/restore.yaml b/kubernetes/restore/trmnl/restore.yaml new file mode 100644 index 00000000..dda59430 --- /dev/null +++ b/kubernetes/restore/trmnl/restore.yaml @@ -0,0 +1,7 @@ +apiVersion: velero.io/v1 +kind: Restore +metadata: + name: trmnl + namespace: velero +spec: + scheduleName: trmnl-daily diff --git a/kubernetes/restore/wordpress/kustomization.yaml b/kubernetes/restore/wordpress/kustomization.yaml new file mode 100644 index 00000000..bc08b2d5 --- /dev/null +++ b/kubernetes/restore/wordpress/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - restore.yaml diff --git a/kubernetes/restore/wordpress/restore.yaml b/kubernetes/restore/wordpress/restore.yaml new file mode 100644 index 00000000..b1725f6e --- /dev/null +++ b/kubernetes/restore/wordpress/restore.yaml @@ -0,0 +1,7 @@ +apiVersion: velero.io/v1 +kind: Restore +metadata: + name: wordpress + namespace: velero +spec: + scheduleName: wordpress-daily diff --git a/kubernetes/talos/patches/cluster.yaml b/kubernetes/talos/patches/cluster.yaml index 6541a69b..e0e5d41e 100644 --- a/kubernetes/talos/patches/cluster.yaml +++ b/kubernetes/talos/patches/cluster.yaml @@ -58,12 +58,7 @@ machine: imageGCHighThresholdPercent: 70 imageGCLowThresholdPercent: 60 cluster: - # etcd peers on node0, VLAN 46 (#576). Pinned: Talos never updates an - # existing member's peer URLs, so changing this subnet means leave + rejoin - # per member (talosctl etcd leave, reboot). - etcd: - advertisedSubnets: - - 10.2.46.0/24 + # etcd.advertisedSubnets is in the site patch: it names the site's node segment. # controlPlane.endpoint is NOT here: `talosctl gen config` takes it as its # second argument, and bootstrap.sh already passes https://k8s.renner.uno:6443 # there. Setting it again in this patch restated the same value in a second diff --git a/kubernetes/talos/patches/hcloud-1.yaml b/kubernetes/talos/patches/hcloud-1.yaml index 298a0251..13a7f6d6 100644 --- a/kubernetes/talos/patches/hcloud-1.yaml +++ b/kubernetes/talos/patches/hcloud-1.yaml @@ -55,15 +55,21 @@ kind: LinkConfig name: ceph-san up: true --- +# Capped so the OSD partition fits beside it on cpx52's 480 GB local disk. +apiVersion: v1alpha1 +kind: VolumeConfig +name: EPHEMERAL +provisioning: + maxSize: 60GiB + grow: false +--- +# The OSD is a partition on the local system disk; bootstrap.sh creates no +# Hetzner volume for a raw volume matching system_disk. apiVersion: v1alpha1 kind: RawVolumeConfig name: osd-data provisioning: diskSelector: - # Two disks: the system disk and the attached volume. Matching on transport - # as the qemu patches do selects nothing here — a Hetzner volume arrives - # over virtio-scsi, and a selector matching zero disks is skipped silently. - match: '!system_disk' - # bootstrap.sh reads this size and creates the volume to match. - minSize: 50GiB + match: system_disk + minSize: 345GiB grow: true diff --git a/kubernetes/talos/patches/hcloud-2.yaml b/kubernetes/talos/patches/hcloud-2.yaml index a32341e7..32d3b3d0 100644 --- a/kubernetes/talos/patches/hcloud-2.yaml +++ b/kubernetes/talos/patches/hcloud-2.yaml @@ -40,11 +40,19 @@ kind: LinkConfig name: ceph-san up: true --- +# See hcloud-1.yaml. +apiVersion: v1alpha1 +kind: VolumeConfig +name: EPHEMERAL +provisioning: + maxSize: 60GiB + grow: false +--- apiVersion: v1alpha1 kind: RawVolumeConfig name: osd-data provisioning: diskSelector: - match: '!system_disk' - minSize: 50GiB + match: system_disk + minSize: 345GiB grow: true diff --git a/kubernetes/talos/patches/hcloud-3.yaml b/kubernetes/talos/patches/hcloud-3.yaml index 05a424b3..cdcceb03 100644 --- a/kubernetes/talos/patches/hcloud-3.yaml +++ b/kubernetes/talos/patches/hcloud-3.yaml @@ -40,11 +40,19 @@ kind: LinkConfig name: ceph-san up: true --- +# See hcloud-1.yaml. +apiVersion: v1alpha1 +kind: VolumeConfig +name: EPHEMERAL +provisioning: + maxSize: 60GiB + grow: false +--- apiVersion: v1alpha1 kind: RawVolumeConfig name: osd-data provisioning: diskSelector: - match: '!system_disk' - minSize: 50GiB + match: system_disk + minSize: 345GiB grow: true diff --git a/kubernetes/talos/patches/k8s-1.yaml b/kubernetes/talos/patches/k8s-1.yaml index 1c3fead3..3f7aa0de 100644 --- a/kubernetes/talos/patches/k8s-1.yaml +++ b/kubernetes/talos/patches/k8s-1.yaml @@ -54,6 +54,12 @@ machine: # this label instead of listing hostnames, so which boxes carry storage is # stated once, here, by the machine that knows. storage.renner.uno/ceph-osd: "true" + # This machine has the Intel iGPU. The GPU plugin and everything that needs + # gpu.intel.com run only on nodes with this label. + hardware.renner.uno/intel-gpu: "true" + # This machine is on the IoT VLAN: ens19, declared below in its VLANConfig. + # The macvlan workloads on v30iot run only on nodes with this label. + net.renner.uno/iot-vlan: "true" # Publishes this node's pod0 addresses, declared below in its VLANConfig, so # the bird DaemonSet reads its peers from the node list rather than from a # copy. Annotations, not labels: a label value cannot contain a colon. diff --git a/kubernetes/talos/patches/k8s-2.yaml b/kubernetes/talos/patches/k8s-2.yaml index a5ceabf1..32b16c79 100644 --- a/kubernetes/talos/patches/k8s-2.yaml +++ b/kubernetes/talos/patches/k8s-2.yaml @@ -34,6 +34,10 @@ machine: topology.kubernetes.io/zone: saturn # Holds a Ceph OSD drive — see k8s-1.yaml. storage.renner.uno/ceph-osd: "true" + # Has the Intel iGPU — see k8s-1.yaml. + hardware.renner.uno/intel-gpu: "true" + # See k8s-1.yaml. + net.renner.uno/iot-vlan: "true" # pod0 addresses for the bird peer list — see k8s-1.yaml. nodeAnnotations: net.renner.uno/pod0-v6: fd02:48::12 diff --git a/kubernetes/talos/patches/k8s-3.yaml b/kubernetes/talos/patches/k8s-3.yaml index a26b57e5..20012676 100644 --- a/kubernetes/talos/patches/k8s-3.yaml +++ b/kubernetes/talos/patches/k8s-3.yaml @@ -30,6 +30,10 @@ machine: topology.kubernetes.io/zone: neptune # Holds a Ceph OSD drive — see k8s-1.yaml. storage.renner.uno/ceph-osd: "true" + # Has the Intel iGPU — see k8s-1.yaml. + hardware.renner.uno/intel-gpu: "true" + # See k8s-1.yaml. + net.renner.uno/iot-vlan: "true" # pod0 addresses for the bird peer list — see k8s-1.yaml. nodeAnnotations: net.renner.uno/pod0-v6: fd02:48::13 diff --git a/kubernetes/talos/patches/site-hcloud.yaml b/kubernetes/talos/patches/site-hcloud.yaml index e2ce3856..fad5a8f0 100644 --- a/kubernetes/talos/patches/site-hcloud.yaml +++ b/kubernetes/talos/patches/site-hcloud.yaml @@ -19,7 +19,64 @@ machine: validSubnets: - 10.90.0.0/16 cluster: + # etcd peers on the node segment. Hetzner firewalls do not filter private + # networks; the public addresses admit only this host and the nodes. + etcd: + advertisedSubnets: + - 10.90.0.0/16 # Three control planes that also carry workloads, which is the production # shape. The qemu site cannot mirror it: that provisioner gives a second disk # to workers only, so a Ceph OSD forces 1 control plane and 3 workers there. allowSchedulingOnControlPlanes: true + # The site's part of Cilium's values (see site-homelab.yaml). Hetzner + # networks route only server addresses and only IPv4, so pod traffic between + # nodes is tunnelled (VXLAN) over the node segment. Their gateway drops any + # source address not assigned to the sending server (uRPF), so only the pod + # CIDRs are exempt from masquerading. + inlineManifests: + - name: cilium-site + contents: | + apiVersion: v1 + kind: ConfigMap + metadata: + name: cilium-site + namespace: kube-system + labels: + reconcile.fluxcd.io/watch: Enabled + data: + values.yaml: | + routingMode: tunnel + tunnelProtocol: vxlan + autoDirectNodeRoutes: false + ipv4NativeRoutingCIDR: 10.244.0.0/16 + ipv6NativeRoutingCIDR: fd10:244::/48 + # The site's upstream for k8s-gateway (its pod resolv.conf): the + # resolvers Hetzner's DHCP hands out. + - name: k8s-gateway-site + contents: | + apiVersion: v1 + kind: ConfigMap + metadata: + name: k8s-gateway-site + namespace: kube-system + labels: + reconcile.fluxcd.io/watch: Enabled + data: + values.yaml: | + deployment: + dnsPolicy: None + dnsConfig: + nameservers: + - 185.12.64.1 + - 185.12.64.2 +--- +# Host DNS asks k8s-gateway first, so containerd resolves in-cluster names +# (gitea.renner.uno) in the cluster; bootstrap.sh installs k8s-gateway right +# after the CNI. Hetzner's resolvers after it. This list replaces the +# nameservers from DHCP. +apiVersion: v1alpha1 +kind: ResolverConfig +nameservers: + - address: 10.96.0.11 + - address: 185.12.64.1 + - address: 185.12.64.2 diff --git a/kubernetes/talos/patches/site-homelab.yaml b/kubernetes/talos/patches/site-homelab.yaml index 4f790a2d..32df7d7d 100644 --- a/kubernetes/talos/patches/site-homelab.yaml +++ b/kubernetes/talos/patches/site-homelab.yaml @@ -10,18 +10,77 @@ machine: # The node IP lives on node0, VLAN 46: the closed node↔node network (#576). # Both families are listed so the Node object carries an IPv6 InternalIP at # all; without one Cilium detects the node's IPv6 itself and picks a SLAAC - # address from the dynamic provider prefix (#789). Talos picks one address - # per family, IPv4 first, whatever the order here. + # address from the dynamic provider prefix (#789). + # + # Order is load-bearing: Talos keeps this order, the first entry is the + # primary node IP, and kube-apiserver requires its family to match the + # first service subnet (IPv4, cluster.yaml). nodeIP: validSubnets: - - fd02:46::/64 - 10.2.46.0/24 + - fd02:46::/64 # The UDM on the ingress segment. Published here rather than as a route # because bird owns table 47 and originates the default from it; this is the # one place the address is stated. nodeAnnotations: net.renner.uno/egress-gw-v6: fd02:47::1 net.renner.uno/egress-gw-v4: 10.2.47.1 + # Pod traffic is routed into table 47 here (RoutingRuleConfig below); bird + # fills that table and runs only on nodes with this label. + nodeLabels: + net.renner.uno/pod-table: "47" +cluster: + # etcd peers on node0, VLAN 46 (#576). Pinned: Talos never updates an + # existing member's peer URLs, so changing this subnet means leave + rejoin + # per member (talosctl etcd leave, reboot). + etcd: + advertisedSubnets: + - 10.2.46.0/24 + # The site's part of Cilium's values, read by the cilium HelmRelease + # (valuesFrom) and by bootstrap.sh. Pod east-west leaves on pod0 (VLAN 48): + # native routing, and autoDirectNodeRoutes installs the remote PodCIDR routes + # because all nodes share pod0's L2 segment. directRoutingDevice is explicit: + # otherwise Cilium takes the InternalIP device, mgmt0. The native-routing + # CIDRs are a masquerading exception: pod traffic to the LAN keeps its pod + # IP, which works because BGP advertises the pod CIDRs to the UDM — the two + # settings move together. Talos only creates this ConfigMap; a changed one + # is synced with talosctl upgrade-k8s. + inlineManifests: + - name: cilium-site + contents: | + apiVersion: v1 + kind: ConfigMap + metadata: + name: cilium-site + namespace: kube-system + labels: + reconcile.fluxcd.io/watch: Enabled + data: + values.yaml: | + routingMode: native + autoDirectNodeRoutes: true + nodePort: + directRoutingDevice: pod0 + ipv4NativeRoutingCIDR: 10.0.0.0/8 + ipv6NativeRoutingCIDR: fd00::/8 + # The site's upstream for k8s-gateway (its pod resolv.conf): the UDM. + - name: k8s-gateway-site + contents: | + apiVersion: v1 + kind: ConfigMap + metadata: + name: k8s-gateway-site + namespace: kube-system + labels: + reconcile.fluxcd.io/watch: Enabled + data: + values.yaml: | + deployment: + dnsPolicy: None + dnsConfig: + nameservers: + - 10.2.40.1 + - fd00::1 --- # All pod traffic goes to table 47, and the table decides — longest prefix # match, no exceptions to carve out. bird fills it with the node's attached @@ -53,9 +112,9 @@ src: 10.244.0.0/16 table: "47" --- # Host DNS asks k8s-gateway first, so containerd resolves in-cluster names -# (gitea.renner.uno) in the cluster. The UDM answers while k8s-gateway is not -# reachable yet; Talos moves on after a 2 s read timeout. This list replaces -# the nameservers from DHCP. +# (gitea.renner.uno) in the cluster; bootstrap.sh installs k8s-gateway right +# after the CNI, before anything else needs DNS. The UDM is the site's +# resolver after it. This list replaces the nameservers from DHCP. apiVersion: v1alpha1 kind: ResolverConfig nameservers: diff --git a/kubernetes/talos/patches/site-qemu.yaml b/kubernetes/talos/patches/site-qemu.yaml index fd22df2f..077d9954 100644 --- a/kubernetes/talos/patches/site-qemu.yaml +++ b/kubernetes/talos/patches/site-qemu.yaml @@ -13,3 +13,37 @@ machine: nodeIP: validSubnets: - 10.5.0.0/24 +cluster: + # The site's part of Cilium's values (see site-homelab.yaml). The nodes share + # the provisioner's bridge, one L2 segment; Cilium picks its only NIC. + inlineManifests: + - name: cilium-site + contents: | + apiVersion: v1 + kind: ConfigMap + metadata: + name: cilium-site + namespace: kube-system + labels: + reconcile.fluxcd.io/watch: Enabled + data: + values.yaml: | + routingMode: native + autoDirectNodeRoutes: true + ipv4NativeRoutingCIDR: 10.0.0.0/8 + ipv6NativeRoutingCIDR: fd00::/8 + # The nodes keep the provisioner's resolver (no ResolverConfig here), so + # k8s-gateway can use the node's. + - name: k8s-gateway-site + contents: | + apiVersion: v1 + kind: ConfigMap + metadata: + name: k8s-gateway-site + namespace: kube-system + labels: + reconcile.fluxcd.io/watch: Enabled + data: + values.yaml: | + deployment: + dnsPolicy: Default diff --git a/scripts/flux-graph.sh b/scripts/flux-graph.sh new file mode 100755 index 00000000..a5d86243 --- /dev/null +++ b/scripts/flux-graph.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Draws the Flux Kustomization dependency graph from the dependsOn fields in +# kubernetes/clusters/homelab. Edges point from a dependency to its dependent. +# +# flux-graph.sh DOT on stdout, every edge +# flux-graph.sh out.dot the same, written to out.dot +# flux-graph.sh out.svg rendered; out.png likewise +# +# Rendered output is the transitive reduction (tred): an edge implied by a +# longer path is left out, the order is the same. +# +# Green: infrastructure. Blue: the Velero restore chain. Dashed: nothing is +# health-checked. Grey fill: suspended. +# +# Needs jq, either yq (mikefarah's or the Python wrapper) and, to render, +# graphviz. + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +CLUSTER_DIR="${REPO_ROOT}/kubernetes/clusters/homelab" +OUT="${1:-}" + +command -v jq >/dev/null || { echo "jq not found" >&2; exit 1; } +command -v yq >/dev/null || { echo "yq not found" >&2; exit 1; } + +# One JSON document per YAML document. +yaml_to_json() { + if [[ "$(yq --version)" == *mikefarah* ]]; then + yq eval-all -o=json -I=0 '.' "$@" + else + yq -c '.' "$@" + fi +} + +to_dot() { + yaml_to_json "${CLUSTER_DIR}"/*.yaml | jq -r -s ' + [ .[] + | select(.kind == "Kustomization" + and .apiVersion == "kustomize.toolkit.fluxcd.io/v1") ] as $ks + | "digraph flux {", + " rankdir=LR;", + " node [shape=box, style=\"rounded\", fontname=\"Helvetica\", fontsize=10];", + " edge [color=\"#888888\"];", + ( $ks[] + | .metadata.name as $n + # healthChecks are ignored under wait: true; without either, nothing + # is checked, healthCheckExprs included. + | (if .spec.wait == true then "wait" + elif ((.spec.healthChecks // []) | length) > 0 then "health checks" + else "wait: false" end) as $wait + | (if .spec.suspend == true then ", style=\"rounded,filled\", fillcolor=\"#dddddd\"" + elif $wait == "wait: false" then ", style=\"rounded,dashed\"" + else "" end) as $dash + | (if .spec.suspend == true then "\\nsuspended" else "" end) as $suspended + | (if ($n | startswith("restore-")) or $n == "velero-backups-synced" + then ", color=\"#1f77b4\", fontcolor=\"#1f77b4\"" + elif ($n | startswith("infrastructure")) + then ", color=\"#2ca02c\", fontcolor=\"#2ca02c\"" + else "" end) as $colour + | " \"\($n)\" [label=\"\($n)\\n\($wait) · \(.spec.timeout // "-")\($suspended)\"\($dash)\($colour)];" ), + ( $ks[] + | .metadata.name as $n + | (.spec.dependsOn // [])[] + | " \"\(.name)\" -> \"\($n)\";" ), + "}"' +} + +case "${OUT}" in + "") + to_dot + ;; + *.dot) + to_dot > "${OUT}" + ;; + *.svg|*.png) + command -v dot >/dev/null || { echo "graphviz (dot) not found" >&2; exit 1; } + to_dot | tred | dot -T"${OUT##*.}" -Gdpi=110 -o "${OUT}" + ;; + *) + echo "usage: $0 [out.dot|out.svg|out.png]" >&2 + exit 1 + ;; +esac