perf(grafana-db): raise archive_timeout to 30min on gf-pg

A WAL segment is a fixed 16 MiB file. archive_timeout forces a segment switch on a timer so the archive never trails reality by more than that interval, and the switched-out segment is written and archived at its full 16 MiB no matter how few bytes of real WAL it holds. CNPG defaults this to 5min, which permits up to 288 segments/day = 4.6 GB/day per cluster. Measured on gf-pg: the database is 21 MB and generates 4.5 MB/day of actual WAL, while its RBD volume takes 211 MB/hour -- against 192 MB/hour predicted from 12 switches of 16 MiB. Essentially every byte written to this volume is padding, roughly 1100x amplification. This resolves an anomaly that has been open since the start of the endurance work: gf-pg appeared to write gigabytes per day for a 21 MB database, and neither WAL volume nor data changes could account for it. Segment padding does, exactly. Applied to gf-pg alone as a staged rollout. It is the lowest-risk cluster and gives the cleanest signal -- its real WAL is small enough that any change in volume is unambiguously attributable to this parameter. The same default affects all seven clusters and is worth an estimated ~24 GB/day of logical writes fleet-wide, but that only goes ahead once this one is verified. The cost is RPO. On total loss of the cluster, up to 30min of Grafana state becomes unrecoverable instead of 5min. That state is dashboards, annotations and preferences, changing at ~4.5 MB/day, so the window is nearly always empty. Clusters holding credentials or job state get evaluated separately rather than inheriting this value by default. archive_timeout is in CNPG's GlobalDefaultSettings, not its FixedConfigurationParameters, so it is user-overridable -- verified against pkg/postgres/configuration.go rather than assumed. Context: drive-endurance work on three QLC NVMe OSDs at 81-87% wear.
2026-08-20 14:21:03 +02:00
354 changed files with 12454 additions and 10306 deletions
@@ -32,13 +32,13 @@
 #              mechanism as update-audit.yml):
 #                woodpecker repo cron add --repo k.renner/homelab \
 #                  --name alert-monitor --branch master --schedule '*/15 * * * *'
-#   - deploy : woodpecker-cli pipeline deploy k.renner/homelab <n> alert-monitor
-#              where <n> is an existing pipeline to promote.
+#   - manual : woodpecker pipeline create --branch master \
+#                --var PIPELINE=alert-monitor k.renner/homelab
 when:
  - event: cron
    cron: alert-monitor
-  - event: deployment
-    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "alert-monitor"'
+  - event: manual
+    evaluate: 'PIPELINE == "alert-monitor"'

 steps:
  - name: fetch-alerts
@@ -101,7 +101,7 @@ steps:
        bash scripts/ci/make-diag-kubeconfig.sh

  - name: alert-triage-agent
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.250
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.218
    environment:
      CLAUDE_CODE_OAUTH_TOKEN:
        from_secret: claude_code_oauth_token
@@ -1,17 +1,6 @@
 # Builds the CI tooling images (and the secret-rotator runtime image) with
 # kaniko and pushes them to the Gitea container registry.
 #
-# The kaniko image is NOT Google's. Google archived
-# GoogleContainerTools/kaniko on 2025-06-03, but the original authors forked
-# it to chainguard-dev/kaniko and kept releasing — the pin here is well past
-# the v1.24.0 the Google repo ended on. The fork ships no binaries or images
-# of its own, so these come from gitlab-ci-utils, which builds that fork for
-# amd64 and arm64 under versioned tags. A generic `:debug` tag exists there
-# and is deliberately unused; everything else in this repo is pinned too.
-#
-# The executor path, flags and busybox layout are unchanged from Google's
-# image, which is why the swap touched nothing but the `image:` lines (#552).
-#
 # Credential: ci_images_token — a PAT scoped to `package: read+write` and
 # NOTHING else (verified: 403 on /user/repos, /admin/users and the issues
 # API; 200 on the /v2/ registry endpoints kaniko uses). It replaces the
@@ -31,12 +20,12 @@ when:
      - "ci/images/**"
      - "secret-rotator/**"
      - ".woodpecker/build-ci-images.yml"
-  - event: deployment
-    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "build-ci-images"'
+  - event: manual
+    evaluate: 'PIPELINE == "build-ci-images"'

 steps:
  - name: build-ansible
-    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
+    image: gcr.io/kaniko-project/executor:v1.23.2-debug
    directory: ci/images/ansible
    depends_on: []
    # Only rebuild when THIS image's source (or the pipeline file) changed, or
@@ -44,7 +33,7 @@ steps:
    # secret-rotator commit needlessly rebuilt all four ~1.3G ci-* images and
    # orphaned ~5G of registry layers per run — a main driver of #336.
    when:
-      - event: deployment
+      - event: manual
      - event: push
        path: ["ci/images/ansible/**", ".woodpecker/build-ci-images.yml"]
    environment:
@@ -64,11 +53,11 @@ steps:
          --destination gitea.renner.uno/gitea_admin/ci-ansible:latest

  - name: build-k8s
-    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
+    image: gcr.io/kaniko-project/executor:v1.23.2-debug
    directory: ci/images/k8s
    depends_on: []
    when:
-      - event: deployment
+      - event: manual
      - event: push
        path: ["ci/images/k8s/**", ".woodpecker/build-ci-images.yml"]
    environment:
@@ -88,11 +77,11 @@ steps:
          --destination gitea.renner.uno/gitea_admin/ci-k8s:latest

  - name: build-tofu
-    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
+    image: gcr.io/kaniko-project/executor:v1.23.2-debug
    directory: ci/images/tofu
    depends_on: []
    when:
-      - event: deployment
+      - event: manual
      - event: push
        path: ["ci/images/tofu/**", ".woodpecker/build-ci-images.yml"]
    environment:
@@ -119,11 +108,11 @@ steps:
  # Runtime image, not CI tooling: a static musl binary in a FROM-scratch image,
  # used as the base for secret-rotation CronJobs. Replaces per-job inline Python.
  - name: build-secret-rotator
-    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
+    image: gcr.io/kaniko-project/executor:v1.23.2-debug
    directory: secret-rotator
    depends_on: []
    when:
-      - event: deployment
+      - event: manual
      - event: push
        path: ["secret-rotator/**", ".woodpecker/build-ci-images.yml"]
    environment:
@@ -148,11 +137,11 @@ steps:
  # container, but nothing about it is zigbee-specific. Exists so tools come from
  # the image rather than an `apk add` on every container start.
  - name: build-alpine-jq
-    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
+    image: gcr.io/kaniko-project/executor:v1.23.2-debug
    directory: ci/images/alpine-jq
    depends_on: []
    when:
-      - event: deployment
+      - event: manual
      - event: push
        path: ["ci/images/alpine-jq/**", ".woodpecker/build-ci-images.yml"]
    environment:
@@ -177,11 +166,11 @@ steps:
  # critical path of every node rebuild. The download now happens here, once,
  # where the sha256 is checked — see the Containerfile for the full rationale.
  - name: build-cni-plugins
-    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
+    image: gcr.io/kaniko-project/executor:v1.23.2-debug
    directory: ci/images/cni-plugins
    depends_on: []
    when:
-      - event: deployment
+      - event: manual
      - event: push
        path: ["ci/images/cni-plugins/**", ".woodpecker/build-ci-images.yml"]
    environment:
@@ -206,11 +195,11 @@ steps:
  # every paperless pod start. The openssl CLI is genuinely absent from the alpine
  # base, so this one needs an image rather than a busybox substitute.
  - name: build-alpine-openssl
-    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
+    image: gcr.io/kaniko-project/executor:v1.23.2-debug
    directory: ci/images/alpine-openssl
    depends_on: []
    when:
-      - event: deployment
+      - event: manual
      - event: push
        path: ["ci/images/alpine-openssl/**", ".woodpecker/build-ci-images.yml"]
    environment:
@@ -235,11 +224,11 @@ steps:
  # Ansible converge. Not a ci-* image -- it is a runtime artifact -- but it
  # builds the same way and lands in the same registry so the audit sees it.
  - name: build-heating-can
-    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
+    image: gcr.io/kaniko-project/executor:v1.23.2-debug
    directory: ci/images/heating-can
    depends_on: []
    when:
-      - event: deployment
+      - event: manual
      - event: push
        path: ["ci/images/heating-can/**", ".woodpecker/build-ci-images.yml"]
    environment:
@@ -18,7 +18,7 @@
 #                 consumer pins to  ci-claude:<version>  and open a PR. A human
 #                 reviews and merges it; nothing auto-merges.
 #
-# TAG SCHEME — deliberately version tags (ci-claude:<claude-version>), NOT the dated
+# TAG SCHEME — deliberately version tags (ci-claude:2.1.218), NOT the dated
 # tags the other ci-* images use. The tag encodes the thing that changes here
 # (the Claude version), which makes "did it change?" a trivial, race-free
 # string compare and keeps the weekly cron from opening no-op PRs. The other
@@ -54,8 +54,8 @@
 #
 # Triggering:
 #   - cron   : the cron NAMED exactly "ci-claude-refresh" (see above).
-#   - deploy : woodpecker-cli pipeline deploy k.renner/homelab <n> ci-claude-refresh
-#              where <n> is an existing pipeline to promote.
+#   - manual : woodpecker pipeline create --branch master \
+#                --var PIPELINE=ci-claude-refresh k.renner/homelab
 #   - push   : a change to the Containerfile (or this file) ON master forces a
 #              rebuild of the current latest version so Containerfile fixes ship
 #              promptly. Restricted to master ON PURPOSE — a feature branch must
@@ -65,8 +65,8 @@
 when:
  - event: cron
    cron: ci-claude-refresh
-  - event: deployment
-    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "ci-claude-refresh"'
+  - event: manual
+    evaluate: 'PIPELINE == "ci-claude-refresh"'
  - event: push
    branch: master
    path:
@@ -84,24 +84,14 @@ steps:
      - |
        set -eu
        mkdir -p "$$CI_WORKSPACE/.ci-refresh"
-        # Files that pin the image, DISCOVERED rather than listed. This used to
-        # be a hardcoded list carrying the instruction "keep this in sync with
-        # the consumers" — and it was not: e2e.yml and ci-scripts-test.yml were
-        # missing, so they sat on a version no bump could ever reach. A list
-        # that must be maintained by hand is the bug; deriving it removes the
-        # failure mode instead of correcting one instance of it.
-        #
-        # Matches a version-pinned reference only, which is what excludes this
-        # file: its own steps run ci-claude:latest, and :latest never needs
-        # rewriting. That exclusion is load-bearing — do not write a literal
-        # ci-claude version anywhere in THIS file, not even in prose, or it
-        # adds itself to the consumer set and the sed below rewrites it. The
-        # tag-scheme note at the top says <claude-version> for that reason.
-        CONSUMERS=$$(grep -rlE 'ci-claude:[0-9]' .woodpecker)
-        # An empty list would make the grep below read stdin and hang forever
-        # rather than fail, so it is a hard stop.
-        [ -n "$$CONSUMERS" ] || { echo "No ci-claude consumers found — refusing to continue."; exit 1; }
-        echo "consumers: $$CONSUMERS"
+        # Files that pin the image. Keep this list in sync with the consumers.
+        CONSUMERS="
+          .woodpecker/security-scan.yml
+          .woodpecker/alert-monitor.yml
+          .woodpecker/update-audit.yml
+          .woodpecker/pr-review.yml
+          .woodpecker/issue-fix.yml
+        "
        NEW=$$(npm view @anthropic-ai/claude-code version)
        case "$$NEW" in
          [0-9]*.[0-9]*.[0-9]*) : ;;
@@ -140,7 +130,7 @@ steps:
  #    --build-arg to the resolved version (not "latest") guarantees the tag and
  #    the image content agree even if a new release lands mid-run.
  - name: build
-    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
+    image: gcr.io/kaniko-project/executor:v1.23.2-debug
    directory: ci/images/claude
    depends_on: [decide]
    environment:
@@ -186,10 +176,13 @@ steps:
          echo "Branch $$BRANCH already exists — a PR for $$NEW is already open. Nothing to do."
          exit 0
        fi
-        # Same discovery as the decide step — see the comment there for why this
-        # is not a maintained list.
-        CONSUMERS=$$(grep -rlE 'ci-claude:[0-9]' .woodpecker)
-        [ -n "$$CONSUMERS" ] || { echo "No ci-claude consumers found — refusing to continue."; exit 1; }
+        CONSUMERS="
+          .woodpecker/security-scan.yml
+          .woodpecker/alert-monitor.yml
+          .woodpecker/update-audit.yml
+          .woodpecker/pr-review.yml
+          .woodpecker/issue-fix.yml
+        "
        # Exact-string swap of the current tag for the new one, consumers only.
        for f in $$CONSUMERS; do
          sed -i "s#ci-claude:$$CUR#ci-claude:$$NEW#g" "$$f"
@@ -205,7 +198,7 @@ steps:
        {
          printf 'Automated bump of the ci-claude agent image from `%s` to `%s`.\n\n' "$$CUR" "$$NEW"
          printf 'The refresh pipeline built and pushed `ci-claude:%s` (and `:latest`) from the\n' "$$NEW"
-          printf 'latest published `@anthropic-ai/claude-code`. This rewires the %s pipelines that\n' "$$(printf '%s\n' $$CONSUMERS | grep -c .)"
+          printf 'latest published `@anthropic-ai/claude-code`. This rewires the 5 pipelines that\n'
          printf 'run the image so CI stops running a stale Claude Code.\n\n'
          printf 'Merging is a human decision — review, then merge to roll it out.\n'
        } > /tmp/pr-body.md
@@ -17,6 +17,6 @@ when:

 steps:
  - name: model-selection-tests
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.250
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.218
    commands:
      - sh scripts/ci/test-model-selection.sh
@@ -6,8 +6,8 @@ when:
      - "ansible/roles/claude-workstation/**"
      - "ansible/plays/install-claude-workstation.yml"
      - ".woodpecker/deploy-claude-workstation.yml"
-  - event: deployment
-    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "deploy-claude-workstation"'
+  - event: manual
+    evaluate: 'PIPELINE == "deploy-claude-workstation"'

 timeout: 30

@@ -8,12 +8,12 @@ when:
    path:
      - "opentofu/**"
      - ".woodpecker/deploy-infrastructure.yml"
-  - event: deployment
-    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "deploy-infrastructure"'
+  - event: manual
+    evaluate: 'PIPELINE == "deploy-infrastructure"'

 steps:
  - name: validate-secrets
-    image: alpine:3.24.1
+    image: alpine:3.21
    environment:
      TF_VAR_proxmox_api_url:
        from_secret: proxmox_api_url
@@ -84,7 +84,7 @@ steps:
    image: gitea.renner.uno/gitea_admin/ci-tofu:2026-04-29
    directory: opentofu
    when:
-      - event: deployment
+      - event: manual
        evaluate: 'TOFU_APPLY == "true"'
    environment:
      TF_IN_AUTOMATION: "true"
@@ -128,6 +128,35 @@ steps:
      - printf "Host *\n\tStrictHostKeyChecking no\n" > ~/.ssh/config
      - echo "$VAULT_PASS" > /tmp/.vault_pass && chmod 600 /tmp/.vault_pass
      - export ANSIBLE_VAULT_PASSWORD_FILE=/tmp/.vault_pass
+      - |
+        # Ensure Talos image exists on all Proxmox nodes via API
+        TALOS_VERSION="v1.12.6"
+        TALOS_SCHEMATIC="88e3752db032a08082972bd0adf16a80849e3870c3f95cf5b669936c5699e5ad"
+        TALOS_URL="https://factory.talos.dev/image/$TALOS_SCHEMATIC/$TALOS_VERSION/nocloud-amd64.raw"
+        TALOS_FILENAME="talos-$TALOS_VERSION-$(echo "$TALOS_SCHEMATIC" | cut -c1-12)-nocloud-amd64.raw"
+        API_TOKEN="$TF_VAR_proxmox_user!$TF_VAR_proxmox_token_id=$TF_VAR_proxmox_token_secret"
+        for node in jupiter saturn neptune; do
+          EXISTS=$(curl -sk "$TF_VAR_proxmox_api_url/api2/json/nodes/$node/storage/local/content" \
+            -H "Authorization: PVEAPIToken=$API_TOKEN" 2>/dev/null | grep -c "$TALOS_FILENAME" || true)
+          if [ "$EXISTS" -gt 0 ]; then
+            echo "Talos image already exists on $node"
+            continue
+          fi
+          echo "Downloading Talos image to $node..."
+          curl -sk "$TF_VAR_proxmox_api_url/api2/json/nodes/$node/storage/local/download-url" \
+            -H "Authorization: PVEAPIToken=$API_TOKEN" \
+            --data-urlencode "url=$TALOS_URL" \
+            --data-urlencode "content=import" \
+            --data-urlencode "filename=$TALOS_FILENAME" \
+            --max-time 600
+          echo "Waiting for download on $node..."
+          for i in $(seq 1 60); do
+            sleep 10
+            EXISTS=$(curl -sk "$TF_VAR_proxmox_api_url/api2/json/nodes/$node/storage/local/content" \
+              -H "Authorization: PVEAPIToken=$API_TOKEN" 2>/dev/null | grep -c "$TALOS_FILENAME" || true)
+            if [ "$EXISTS" -gt 0 ]; then echo "Download complete on $node"; break; fi
+          done
+        done
      - tofu init -upgrade -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY"
      - |
        if [ -n "$$TOFU_STATE_RM" ]; then
@@ -7,8 +7,8 @@ when:
      - "ansible/plays/install-netbird-client.yml"
      - "ansible/inventory/group_vars/claude_workstation/**"
      - ".woodpecker/deploy-netbird-client.yml"
-  - event: deployment
-    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "deploy-netbird-client"'
+  - event: manual
+    evaluate: 'PIPELINE == "deploy-netbird-client"'

 timeout: 30

@@ -7,24 +7,10 @@ when:
      # install-netbird-proxy.yml also applies the netbird-proxy-watchdog role,
      # so changes to it must redeploy this host too.
      - "ansible/roles/netbird-proxy-watchdog/**"
-      # install-netbird-proxy.yml also applies the edge bouncer.
-      - "ansible/roles/crowdsec-edge-bouncer/**"
      - "ansible/plays/install-netbird-proxy.yml"
-      # The variables that steer those roles, not just the roles themselves.
-      # Without these two, a change that only touches inventory never deploys
-      # and nothing says so: arming the bouncer lives in host_vars, and its
-      # LAPI key lives in group_vars, so both would sit in git looking applied
-      # while the hosts kept running the previous configuration.
-      #
-      # pve_hosts.yml is deliberately NOT listed. It is the single inventory
-      # file for the whole fleet, so listing it would redeploy the proxies for
-      # any unrelated host edit. The cost is that a change to [netbird_proxy]
-      # group membership needs a manual deployment run.
-      - "ansible/inventory/host_vars/proxy-*.yml"
-      - "ansible/inventory/group_vars/netbird_proxy/**"
      - ".woodpecker/deploy-netbird-proxy.yml"
-  - event: deployment
-    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "deploy-netbird-proxy"'
+  - event: manual
+    evaluate: 'PIPELINE == "deploy-netbird-proxy"'

 timeout: 30

@@ -9,8 +9,8 @@ when:
      - "ansible/roles/netbird-proxy-watchdog/**"
      - "ansible/plays/install-netbird-server.yml"
      - ".woodpecker/deploy-netbird-server.yml"
-  - event: deployment
-    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "deploy-netbird-server"'
+  - event: manual
+    evaluate: 'PIPELINE == "deploy-netbird-server"'

 timeout: 30

@@ -1,6 +1,6 @@
 when:
-  - event: deployment
-    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "e2e-test"'
+  - event: manual
+    evaluate: 'PIPELINE == "e2e-test"'

 steps:
  - name: e2e
@@ -7,8 +7,7 @@
 #                was harmless at daily/weekly cadence — but alert-monitor.yml
 #                fires every 15 minutes, so piggybacking must be opt-in now.
 #                A new cron that SHOULD also run e2e gets its name added here.
-#   - deploy   : `woodpecker-cli pipeline deploy k.renner/homelab <n> e2e`, where
-#                <n> is an existing pipeline to promote.
+#   - manual   : `woodpecker pipeline create --branch master --var PIPELINE=e2e k.renner/homelab`

 when:
  - event: cron
@@ -17,8 +16,8 @@ when:
    cron: update-audit
  - event: cron
    cron: weekly-update
-  - event: deployment
-    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "e2e"'
+  - event: manual
+    evaluate: 'PIPELINE == "e2e"'

 steps:
  - name: model-selection-tests
@@ -26,7 +25,7 @@ steps:
    # fetch-models.sh response mapping. No cluster, no secrets, no network — the
    # ci-claude image just provides sh + jq. Keeps the flag-injection regression
    # test from rotting (it exercises the exact injection shapes).
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.250
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.218
    commands:
      - sh scripts/ci/test-model-selection.sh

@@ -0,0 +1,27 @@
+when:
+  - event: manual
+    evaluate: 'PIPELINE == "install-garage-synology"'
+
+steps:
+  - name: run-playbook
+    image: gitea.renner.uno/gitea_admin/ci-ansible:2026-04-29
+    environment:
+      ANSIBLE_FORCE_COLOR: "true"
+      ANSIBLE_HOST_KEY_CHECKING: "false"
+      SSH_PRIVATE_KEY:
+        from_secret: ssh_private_key
+      VAULT_PASS:
+        from_secret: vault_pass
+      PROXMOX_TOKEN_ID:
+        from_secret: proxmox_token_id
+      PROXMOX_TOKEN_SECRET:
+        from_secret: proxmox_token_secret
+    commands:
+      - mkdir -p ~/.ssh
+      - echo "$SSH_PRIVATE_KEY" | base64 -d > ~/.ssh/id_ed25519
+      - chmod 600 ~/.ssh/id_ed25519
+      - 'printf "Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile /dev/null\n" > ~/.ssh/config'
+      - cd ansible
+      - echo "$$VAULT_PASS" > .vault_pass && chmod 600 .vault_pass
+      - export ANSIBLE_VAULT_PASSWORD_FILE=.vault_pass
+      - ansible-playbook -i inventory/ plays/install-garage-synology.yml -v
@@ -34,13 +34,13 @@
 #   - cron   : a cron NAMED exactly "issue-fix":
 #                woodpecker repo cron add --repo k.renner/homelab \
 #                  --name issue-fix --branch master --schedule '41 * * * *'
-#   - deploy : woodpecker-cli pipeline deploy k.renner/homelab <n> issue-fix
-#              where <n> is an existing pipeline to promote.
+#   - manual : woodpecker pipeline create --branch master \
+#                --var PIPELINE=issue-fix k.renner/homelab
 when:
  - event: cron
    cron: issue-fix
-  - event: deployment
-    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "issue-fix"'
+  - event: manual
+    evaluate: 'PIPELINE == "issue-fix"'

 steps:
  - name: fetch-diag
@@ -62,7 +62,7 @@ steps:
      - bash scripts/ci/make-diag-kubeconfig.sh

  - name: issue-fix-agent
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.250
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.218
    environment:
      CLAUDE_CODE_OAUTH_TOKEN:
        from_secret: claude_code_oauth_token
@@ -53,9 +53,9 @@
 #
 # Triggering: pull_request only — automatic on PR open + push-to-open-PR. To
 # re-review a PR, push a commit (the new head sha clears the dedup marker below
-# and earns a fresh review). There is deliberately no on-demand trigger for a
-# specific PR: Woodpecker exposes injected variables to YAML substitution and
-# `when` filters, NOT to a step's shell environment, so a PR=<n> re-review would
+# and earns a fresh review). There is deliberately no manual PIPELINE trigger:
+# Woodpecker exposes a --var only to YAML variable substitution and `when`
+# filters, NOT to a step's shell environment, so a --var PR=<n> re-review would
 # silently no-op (the step would find no PR number and exit green).
 # (Note: keep literal dollar-brace out of THIS file entirely, even in comments —
 # Woodpecker runs its variable substitution over the whole YAML text, comments
@@ -65,7 +65,7 @@ when:

 steps:
  - name: pr-review-agent
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.250
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.218
    environment:
      CLAUDE_CODE_OAUTH_TOKEN:
        from_secret: claude_code_oauth_token
@@ -1,20 +1,8 @@
-# Trivy scan of the IaC tree and every image referenced from ansible/, with an
-# agent pass that opens a PR for CRITICAL/HIGH findings.
-#
-# Triggers:
-#   - cron       : the named `daily-security-scan` cron, matched by NAME.
-#                  Crons cannot carry variables — `woodpecker-cli repo cron add`
-#                  has no --var — so a variable gate can never match a cron.
-#   - deployment : woodpecker-cli pipeline deploy k.renner/homelab <n> security-scan
-#                  The target rides in pipeline metadata as
-#                  CI_PIPELINE_DEPLOY_TARGET, which the workflow-root filter
-#                  does see, unlike manual-run variables on Woodpecker 3.18
-#                  (#582).
 when:
  - event: cron
-    cron: daily-security-scan
-  - event: deployment
-    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "security-scan"'
+    evaluate: 'PIPELINE == "security-scan"'
+  - event: manual
+    evaluate: 'PIPELINE == "security-scan"'

 steps:
  - name: iac-scan
@@ -40,28 +28,12 @@ steps:
        cat /tmp/images.txt
        echo "---"
        i=0
-        failed=0
-        failed_names=""
        while IFS= read -r img; do
          i=$$((i+1))
          out="trivy-image-$$i.json"
          echo "=== [$$i] $$img -> $$out ==="
-          # Keep going on a bad image so one unpullable reference cannot cost
-          # the scan of every image after it, but record that it happened.
-          if ! trivy image --skip-db-update --scanners vuln --timeout 15m --severity CRITICAL,HIGH --ignore-unfixed --format json -o "$$out" "$$img"; then
-            failed=$$((failed+1))
-            failed_names="$$failed_names $$img"
-          fi
+          trivy image --skip-db-update --scanners vuln --timeout 15m --severity CRITICAL,HIGH --ignore-unfixed --format json -o "$$out" "$$img" || true
        done < /tmp/images.txt
-        echo "--- scanned $$((i - failed)) of $$i images ---"
-        if [ "$$failed" -gt 0 ]; then
-          echo "NOT SCANNED:$$failed_names"
-          exit 1
-        fi
-    # Non-zero above turns this step red without failing the workflow, so the
-    # fix agent still runs on what was scanned. Coverage is reported, not
-    # enforced: an incomplete scan is visible in the step status instead of
-    # being indistinguishable from a clean one.
    failure: ignore

  - name: security-fix-agent
@@ -70,7 +42,7 @@ steps:
    # this step is skipped — otherwise we'd fork-bomb PRs.
    when:
      - branch: master
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.250
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.218
    environment:
      CLAUDE_CODE_OAUTH_TOKEN:
        from_secret: claude_code_oauth_token
@@ -0,0 +1,32 @@
+when:
+  - event: cron
+    evaluate: 'PIPELINE == "update-all"'
+  - event: manual
+    evaluate: 'PIPELINE == "update-all"'
+
+timeout: 120
+
+steps:
+  - name: update-all
+    image: gitea.renner.uno/gitea_admin/ci-ansible:2026-04-29
+    environment:
+      ANSIBLE_FORCE_COLOR: "true"
+      ANSIBLE_HOST_KEY_CHECKING: "false"
+      ANSIBLE_ROLES_PATH: "../roles:./roles"
+      SSH_PRIVATE_KEY:
+        from_secret: ssh_private_key
+      VAULT_PASS:
+        from_secret: vault_pass
+      PROXMOX_TOKEN_ID:
+        from_secret: proxmox_token_id
+      PROXMOX_TOKEN_SECRET:
+        from_secret: proxmox_token_secret
+    commands:
+      - mkdir -p ~/.ssh
+      - echo "$SSH_PRIVATE_KEY" | base64 -d > ~/.ssh/id_ed25519
+      - chmod 600 ~/.ssh/id_ed25519
+      - printf "Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile /dev/null\n" > ~/.ssh/config
+      - cd ansible
+      - echo "$VAULT_PASS" > .vault_pass && chmod 600 .vault_pass
+      - export ANSIBLE_VAULT_PASSWORD_FILE=.vault_pass
+      - ansible-playbook -i inventory/ plays/update.yml -v
@@ -10,14 +10,14 @@
 # report; the pipeline posts it to a single persistent Gitea issue.
 #
 # Triggering:
-#   - cron   : a cron NAMED exactly "update-audit". Crons carry no variables —
-#              `repo cron add` takes name, branch, schedule and enabled — so the
-#              name filter is the only thing that can select a workflow for a
-#              scheduled run:
+#   - cron   : a cron NAMED exactly "update-audit". Unlike the other pipelines
+#              here (which gate on an injected `PIPELINE` var set at cron-create
+#              time in the UI), this uses Woodpecker's documented `cron:` name
+#              filter so the schedule can be created entirely from the CLI:
 #                woodpecker repo cron add --repo k.renner/homelab \
 #                  --name update-audit --branch master --schedule '@weekly'
-#   - deploy : woodpecker-cli pipeline deploy k.renner/homelab <n> update-audit
-#              where <n> is an existing pipeline to promote.
+#   - manual : woodpecker pipeline create --branch master \
+#                --var PIPELINE=update-audit k.renner/homelab
 #
 # NOTE: e2e.yml runs on an explicit list of cron names that includes this one,
 # so a weekly e2e run piggybacks on the audit (it was a bare `event: cron`
@@ -25,12 +25,12 @@
 when:
  - event: cron
    cron: update-audit
-  - event: deployment
-    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "update-audit"'
+  - event: manual
+    evaluate: 'PIPELINE == "update-audit"'

 steps:
  - name: update-audit-agent
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.250
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.218
    environment:
      CLAUDE_CODE_OAUTH_TOKEN:
        from_secret: claude_code_oauth_token
@@ -170,8 +170,8 @@ role keeps logs off local flash and monitors wear:
  WAL/positions sit on `/run/alloy` (tmpfs), so the agent writes nothing to
  the eMMC either. Net effect: no log writes hit local flash — the trade-off
  is that logs are RAM-only and lost on reboot unless already shipped.
- **Loki endpoint:** the old srv-network Loki
-  (`loki.srv.renner.uno:3100`) no longer exists — Loki moved into the cluster, so Alloy pushes
+- **Loki endpoint:** the unauthenticated srv-network Loki
+  (`loki.srv.renner.uno:3100`) is firewalled off the IoT VLAN, so Alloy pushes
  to the authenticated cluster ingress `https://loki.renner.uno/loki/api/v1/push`
  with basic-auth (`edge_loki_username` / `vault_edge_loki_password`).
  **Delivery only works once the `edge-1` htpasswd line has reconciled into
@@ -0,0 +1,22 @@
+plugin: community.proxmox.proxmox
+user: root@pam
+# token_id: ansible
+# token_secret: provided via PROXMOX_TOKEN_SECRET env var
+url: 'https://jupiter.renner.uno:8006'
+validate_certs: true
+want_facts: true
+want_nodes: false
+keyed_groups:
+    # proxmox_tags_parsed is an example of a fact only returned when 'want_facts=true'
+  - key: proxmox_tags_parsed
+    separator: "_"
+    prefix: proxmox
+  - key: proxmox_status
+    prefix: status
+compose:
+  ansible_host: >-
+    (proxmox_lxc_interfaces | default([]) | selectattr('name', 'equalto', 'eth0') | map(attribute='inet') | list | first | default('') | split('/') | first)
+    if proxmox_vmtype == 'lxc' else
+    (proxmox_agent_interfaces | default([]) | selectattr('name', 'equalto', 'eth0') | map(attribute='ip-addresses') | list | first | list | first | default('') | split('/') | first)
+  ansible_user: "'root' if proxmox_vmtype == 'lxc' else 'ladmin'"
+
@@ -1,13 +0,0 @@
-$ANSIBLE_VAULT;1.1;AES256
-32373134633930343261336362373335623735656634313034393731393661643361346634623439
-3561363963333533313264643036663731656465343162330a333964383462373934303766323533
-63613065313063626261383236396363656565623538666565646661353766643133306334396561
-3961376234623236620a616339326238366538333666316538396465666365373834313064653963
-38623233616633383435373836613438333763343932636239386263306238376137643531373161
-35386437313465643437343832623935353238316565366537646530323233363837656631383962
-32653332333162623966366163633732356637323461613966313263313066643966373335313162
-30306362346561633566316166313164323635356363643034663933363464663562343136646137
-66643536363734316533663032643637356136363237376365613330633961616665323430633135
-36323733633464306330666232663433616533373466633939353061663136666665323432373666
-36393930333435333238306238373931393232393861333334643139306138653336633534626362
-61323038303838396438
@@ -1,29 +1,201 @@
 $ANSIBLE_VAULT;1.1;AES256
-30353730633631376539373334376531353263396339313464373538396231616166653738383161
-6438656132616365646463653464396236313437323537620a363632643061333036353537626461
-34393065613836343536383565303066326133363937386263653961663666643430663139653961
-6462363235343063350a636165376232366636363233376637636231363839343738346134623562
-63663839326435303964313730336436396365343163343366613966323938656263303166326435
-65303931393233623064343763633161353161643635396365316362333137353732346366643162
-36386132386638623333623839373161386330353635346331393735366662393935353264646439
-36386539393432373765396562613033656339646532323864366230393638313161383363636636
-32353266376563393239393734316461313566386233333164623135303132346164353165323338
-62306333363838393236346435303339393639623231353232643966613163313134336265633636
-64366539326639303362393933303465376532626663346431613563353032383037363532646637
-38376266323537343437643635343634636164393230313632663533666434306134366261383230
-66383135303565383763646331383338646339613539623835366438393635363832366134316531
-63636466653239383231616139663231623636376564376234383762383066613030313937616533
-33396135616236643034303236353230376434313365326633303961303232363766633161353434
-64333832623535343733353462313862343934633363616237353331386264386632643538333939
-34636437393934326131363236346636663665613537343134313636623439356665636663613938
-37346634356164343537396363663730306232623964636635333333653738383538343236383535
-66326462373964333234343435336630643930333433393034303439366631386666393330343238
-61336237373236313533333732343064336637326636663433396362346631343165656163313561
-30383331326231383539343639343130323731333432306630353835343338613861376631646264
-66383539616663376237663234313431313064663566313566366664386439316161353135306338
-35373962636537653535333935306564303235356232313463386237353737313736386463396164
-62666165336539316565623634326562356434313833643833353635303361353937666663356263
-35653665323736386263333437666432623332333936373938376637633463653930326466383139
-38336239646135316565356564373731386662383234386433633632393165343331353835623437
-30663138633430356334353832633434633939373337663437326263336537356633313763353337
-64373764666430643433
+32666334326535346466653862343263373365346261313136653366366664633436356536383437
+6666326436366433373163353262383031333264633530390a346539316165633939346663623264
+30653839376635626261323234326539313235623962306639396633343533363561376363363864
+3165323130633061350a346233396530303737316361653337643264383230613635383932636133
+61336561663264393831313364366665636166306437383835303332643934393062623163343861
+31643063373432643531663538666363613331626564376539336435343161333237333036343664
+31636635316631376563326637303337386337313330323838666562303463363361346531613363
+64383062396132373536663832356266383430316465643934636537396466366335636131623339
+34316166323730316463666465613734383339636264306431623535656362633739386362393134
+30343837363361333666616332646131663738663737663434303437336130313133306236313739
+36363731386163353136376230393262653334366239653032613034323333386433666262353364
+30326364343735643739626563393661616238313265633566356639623639383138303934653737
+64356139656163666263346539663733356239386333313763653434343163353436323937663034
+35653236616135383465656536323738346662396366653435383337383233643232656435356335
+63616138666662386337323865356439303961396138613965303465373766343939333933316664
+35356561353366663361306336303339633362623332386330383533336461306139613232306363
+62613731613835336338346236373636343463383433613938393865666165653935353830633966
+61393266623961343665343136323235393764313461306138343334366631313531396663666136
+34636436666238376138633461636438356461643461356139313437393736666338623038353933
+34633032653930623065623639646231346133343231383032663438326164313136613361356232
+38306565353737623532653231303963616336323563663137316139303136623234663432373661
+37333431663233656635373433323461663164303137633831306232626465303761633934336461
+65613534633665663565366634623362303964313862333936383032363137306336636563383435
+35366666313065393463656430303436346334313930616166383861613830373962353536303866
+39636165393861363161633734343838346338303036306639316231313738383831626639303833
+62316336623135336462633439323533383931666563643839383236636134363466383561633363
+64366466336362376234336463383639323165306631633437333639333263373837666565306163
+34666162336166623831353436363831666561376562613732333938353638333134323133346435
+61663261373433333635643761373465653031303236613465643931356433656161373463316331
+34326163633037633737393338313931353130306134663839623332666365373663643330353864
+36366535313032333262373038333161336230303165646133303261333839376432393435613434
+39633830316236653962663464363637333433393665353066653230383339666130653135343332
+32623165386535323832626536316665396564623064633564663437613930313665616163326637
+61363538333761633264396430326539643833313830346565643937656137376464646664313636
+38626165306661353631393031383063646463363031633435373735653838636634316634323738
+38313832376265616631623761393666346133636135666664643661646130306237383134383934
+39643561363962323261343236393630656435653261633933656262623737613130353531386438
+39643335306436636238393734646238653465626232643161316539346435353130633138393863
+63343965346361353036646561636339303063393033633162666639303166303265333233386137
+38643835616131303263366539646130643630366139623032366532653031303336323463326263
+37313566633032313931313565616131376635643030373833323838393633653531653664333032
+33623566333731636462366133653837636238383832353362653230393164343135663062373438
+61613832336633343038373565316431643432626532636435653438383536336166383436393035
+66613037663739616337383136663034643433333933643862396330356533636439666536353338
+30313761633961383337353030333862396164376337393338303565663963633662633134653362
+30346563643537323431313137636364623231376364386364633739313438376663343339333330
+64623663303535393262376437626637316532643139353262393231376135663161663862646666
+65303463393966393633613964353132303863373565353131396435623634336364313864373465
+61376335313830383534643431383966616565346466633633613933643231303732373938313434
+38383731653938656331646239353330666266336566633232353835653239393232366634663165
+31313766643539373133396563633637336133633034633236393336663064626630376632336264
+63346136613165613063316264646561633430613630396263333562366430326432323264383233
+30616631616535666631303838326330363339343463303831376531396164333539386263343334
+64313363626363316266343735313263613038383464396564313336373831393936336237303361
+65346161323961336631366263353161616562633533323036383366613637333832323466326535
+36376661346133383461626461646237373531333532656239333566323433333139633966343465
+36653130646636336534353331623062313863666637306462343139333031313234633636613232
+64383663626566616465623938346531623362383839316361366566336166386461336634303339
+63363931303866376462303634633033386332373564393932616638323764383630643839393630
+38373533636561613131343138383765376332336434303930326630303735313462316666333861
+37346336613339643466636665326466386664376438646666363537306533303036613135303230
+65303733366163363964613135306538393234636262326639613439363063326237306337666564
+32303161373661633231333662303565383134333737343530616631363337396333346261646636
+38663439653435623236366437643334393432383636316664306530303133396561396435313936
+38316637393365343936636438666461326436353636323537343363623337656536613662393333
+32346638636532333266323036623932316265303134633265366634623336386138653165643662
+35396637636136386366663762346236343565623133623336363530653039363166383534313632
+39333162396534376335646664383362366663373531646132643462396131333932653464326532
+39326633616335633030373463303665383261323763303863383566386536646136613362323034
+35313064313533616262666439663266353136616466643562666131366636623166333030346333
+62353539396131666434356238646439326337343239366234383866613065636232613863323931
+38363339623634643433393135396264363835386135303061623964343832323936313536623634
+34396531643839623932663565656535396337353934373462336231356133303335643931323131
+33383030653362323830376435393134666339333036336333376539363033666666396534636131
+64353663663633313966336439643334616332616536336238613162363435323938313636303930
+66306135613735653437366164336237656331346365646265656362316639656162386239326632
+34613431653433306633666632363639306266303361383230316539306161396566346137363133
+30333131666530636264656263303639636161366134633964393834663436326539626264383034
+34636134663833333764613237663233346363643566353166316230396235373261393739316237
+38373466623635333432326531613637303839633033636166633364383964626134386261636536
+64616432343936363664383532323936386136393332653131343533653633323639356234353363
+62613631333864663435616633396537373035646238653565646535306163623262396136356663
+63383537653231633766613231316666303830336334323738626138366434323164643465663034
+64343965363738626638363733646136316338633161383330306363643461643664303734663863
+38656135616165366438623237333161363234356538356466646266316136313832666161626566
+65306234663338323263376437363961353033396532643132336261336662656136656530343532
+61353736626665386437376230633336656663356361626437393932633765336439613665666632
+34363763656164326565633433643836383234363962613535353230356461613133666232396336
+32316264353535353162373833666365633266376634366362646264616161363634363138303832
+30656465326562653430373962346438646138336364363833646165353636346161336336613565
+37303931393865356136333431656264666331313165373037656535333832643662343434396337
+37656263363161396438626437643839633834643265313963653337363463373530333930303163
+34353163333461353763643631303763623034333665373235656531393561363432633433393137
+63333438646565633533353466373930313831316132306662363831623732336435346334323331
+32373736396239613839396535316166386637613435643637323932303239643339613737653761
+63323062333430636161316437383035326531346466336263353761636661326363656631613063
+62623863616533333036643563333463366432386232386436356632323863663862613466613931
+35636538663263336333613664353934623338356262643832356163326161373166633833333838
+36373537363934393137663462633833373364356230356136646639633737636130396231633730
+61356536373064356665646335633435646164623164333964323135343536663162656564653961
+63626432653439373938333039386661363330373136633831656230336237376436313664653633
+61623830326330663235376534313135303566636131373835383534363730616433363536346666
+38326532376338393462363430643334343137613363626132356133363534643336663063623136
+66623261343932373662663134323538666334623764376131623530373065316338653535313165
+35313337303563373737313963303134303435643563616561323963613565646466353162353637
+62623961626630323432323133666636356531316362633331313630393761303030363736643163
+63333565616232366665356535373632626366366532633036323839393266613730373331343765
+39343132643734623162386162353438353165653537303033313439326132343036333739623832
+30316132396532626236633665333162653536633134313339306661613066343937323032376235
+64386161623639636361313865613536373338333931623438366338616338326536383232363564
+30356139373865313432366337646366613361393262333336643535656438383562343763636236
+38616263613834613030363033613839613334323964343436326136626565643838323064313037
+34663964646636363038393134343634663637663030636339613864316165666265383632373637
+32373063616130343762393832663030636334646263626236663730336135376461643134646337
+63346534353264663031336164623330633737333766633662613863343431333138363337326536
+38363937313632633537363261633133636266376534346430643230623663333464393262396564
+37323632653036653336623636646537326636383865383338366138653463336638363965356536
+30373664313863396237613363373236353966303862376561653562666231353930653233383733
+38303261343436393261653436626136353237666633613166623035636466346539363436616364
+36386438666165316137303932346435313736616434643337303038383166386135326132656265
+64633866363532333465623135323761393837623934316561633637613433316334376138343764
+64646262356631373865393363353465643733623535626434666639613965383239343736356231
+31306135666539646234333934326664353137643261613131646364363739333035396437316362
+62343531656234376438376266373932376233643630383836323032633537383961383537376631
+38303062623032636264666534313266363835636132646664366162303430393161313964616633
+32613063393965653438643766373434613237393736643635653465366533363535623932386433
+38653530316364323033383330373433663865393837653466376666633434343062376432633138
+64653365373062643431353533363764666339663334393432353466623062646632383163656365
+63633365326565353730333164303237383766386135376630613234333834626335643738376339
+31626161346562613439396263646361343464636136636663363766356263636630323366326131
+37356166343031376635346433363335373137303933343539303832613363656334663964333462
+31613531383166666261336439356434323263306364616536636333316131373930646262386366
+33333032643464313165336265326130333637323461643131393437326535316164356665313636
+61613237356331393463336431316539633133326334396134663035396531633538666331393132
+64393134643832376230656332633633396333313464333066656461366335343131613038656535
+32353037316132393936373834633535646631653464616331633830326131333361316464393762
+34306136306633323766353235353663666539663036383636393335336263353134303634626131
+31316334663733666530653330333263653135643539353439306165353339623032643633393632
+37653636663164646438626638643630393135623934646166666635316532366264623165616131
+64653736306431623639303564306431616263613966656563373861616339316466383761643439
+64323836303162613931313534393436353464333330353635663034396631636563616464653530
+34616164323362653231343433323338633930323035396630326461666461346338363039383737
+38666431373633376536323739316132316231353738306632336537633938653535316530346634
+65323937336635373736346438323133353333343235623463663264623838316239366165306535
+30323136626561636537393066383539303233366262396538383563353730356537623266623038
+33316135303833323663363733353634653763383166633663313465303838646266303264363339
+30363637396161656332323333653931336661393566663232326466646563356436306332333532
+35656331383661376538633664346537613161303135643937643161383034643532653961346533
+31396237653234363832383437613433326165636262333236393334613437386638636535616634
+34313330643139386232396332376135376332613333633938366533616435303232376366623237
+37323535383932623631363431383165323338306338633066373937323361383336616265303234
+32343132303366343866303837326566383865393764343130616366373536623766316238363430
+31633734633438313065336431316161636465333166653765653962326636663065336532393731
+33386139383466626565663666373664313030663132313233356136613435663265396332656633
+33636436646565643838643930343938326231623738393661393738343062303661353362646465
+31653335343638613238333636326337396438373263663666373263616662653634613036663364
+66323639336233623538663061366462663766383539333432336363613938346263636436336430
+36646639353163316661356636393034653934666563356335396631323166313532353361653630
+61353261656431613135313965373061313038653237626231366332306432633538656462323739
+38313831663861663738336564333731363937653663663031343261633538643535336264306230
+31656365326261633730613765393861376662656137623832363832633266346135643133633331
+39623836633337323934653464333731626361316638333565393435656132626631613739366535
+37653266333432386334333461653231346630646435343230636531326338396566396466376532
+32666632383430393064353861373033393538366266333339373436353932656333366264393936
+39373938346633616534643734313934643438333965323133313037373737303964643337653135
+31623038356263643438343635643530663362656630393164663634366461623464353532353239
+38623662666235383139663162343836316331396266313062316639373034376337393563643739
+65313564303962353061616265393135326266386266316538616430383966306163386533376137
+36393565336435636535376639623931616435353532356438666663383961356638353333366139
+63313962643638323862353738316233613862346532306631386536323261636235343565353563
+64613465343662663034643138623865363862316131363461343636333864656132626166393836
+32393737386166343630323861376562353337663030613263613664663938376337333533613432
+33343064636365643831373135623466613738303164326562386430386339653831313536366532
+32613430313461663662313966373238643934633938633764363365386139303337643533313961
+61646131653638616365346236616330363566636261636362363635323566353562326239393530
+33323934623530306436623963663735353337306162646638383934326530333163396165366139
+61653339363166376266643733653536636639653939346666663834376361373533363231636433
+35363665373031393231363161333036303239313032326134363432363034623762366637643830
+32353132353961343830303562613964376234303461623263343735336238353565353238363438
+61326164323265386431353130346364393466353334386336376139303036646634376331313561
+39386331366337646634366635623066313432623537323164646263323532303465343735326364
+33383162626139383232636362613831303832633562636136393138316334303832383666313963
+37643835663065613236303338353435663830343637326563343665616532633830623035336534
+39376536663937653939326261346263613265636363333531323931646633313837386238333435
+33646339626462386635316230636135626330643630633664636364313539626332643331356334
+32313736343031633631393662363638626137303331383263656335373966383335653133333932
+34393861303665613633366461666664376466383437303065653938323166633562643834356139
+32616564373731313134316661343839333830663338636132323265376133656337393632353931
+35633531383438656461613135323134623134393062356661326435323065616331656638306432
+62313130643763626336383762306535656238393961653537336465373064653432653665373362
+63666366343866613235376535313835623261313531653265373636383030613930353530346265
+64303531316239386630626565323966616535373936393766663063623464663864363338343466
+33666237613532326661626461343166323162366532386638333835663838343865346266646533
+66343533326562396331363137353631633965623561333963613565636264653031616339613232
+39666362643739663837326432623462646233306239326463616232383236346236383132396637
+34623332346533366362613465363263373339643033663264363332336463343337333466393866
+33333363313331376634363431366636376466336131363330323530306162666434393764393563
+35633530353839386362326262343635383434333338626563383463653263333430383465626535
+63363933343365326665343166633939313739316231383938376166646331623663343161623865
+36646630336337373865
@@ -0,0 +1,39 @@
+$ANSIBLE_VAULT;1.1;AES256
+32656264356562366161643464343065313163663461663038646531306532383532393238396566
+3831646133656131336266346531323035313234663336380a356536313965383066363135343739
+32663032373165643931663762333036393430333437366630326562653034636464336166313730
+6666366634346664330a643063343061623465393436656461303862336431393931643638333735
+36663561626134333639326635373735623739366366396564653763656661663866623639323233
+63326130303065663265356136343266613966333230653935343836343461623535363161346439
+64653130383539363933636437643035306536376439326539666662383064636337363265356135
+39623537303165356635373666643634383731333236643061343033376466643762373636333834
+65626435353563623938363566626330356662366666666263363038646334323731393631356531
+37346230303134326135633033393162333839346664363062633736363861343435623331343762
+36643266333965323861633836336234333839306330343961653636623439656432613662303865
+33313438316236653461346264373338393539326438666337346162326163313837626364623461
+37353135323866326536303138363165396339306237393064366335386339303238373563373064
+62646430313131373964346466373164306432363361626661653531356261313233616563353465
+33353661663930333037636637353234663231633339333436393362393363653066376337376364
+61663035396563643665353162356466623032643938353933393930656162386631376132383461
+65383265623136646261393863336635323634336136376534323231383930383632306161393136
+66366563633737613931633233666336363166313930613963396636633761633638323465306332
+65333764343762643133613036353937386338303336656530366433333031323336366165346435
+62626533363961636361306664616638396538646530633730373636643131303636333036326534
+65616361353435346161383366626334386536376630326334353837396161356637356431316364
+31643636326262383362336535346237666465363165643830613932336362623635366138303336
+66376362386531313063336266623033653736616430386635656134616461643238373831333062
+62343964343032366434316365626232656337613832313263353237353336643735353464656262
+66313765393332393663303562393438376139373565623734653432626134376630343863353061
+31383530643264643330313136356638646265663831363637363464636634356639313936336437
+33393961363930616435623864303732633564303937343638373761653761666339373433373432
+39653333396138383332353465396639346535666536373232653536393163353835613239386530
+62666536633636363035613063643965623735366333386335363066326238323835663535333162
+36613366326137303564653463393665666337363664363563643361373766663331333363326533
+36353265616261613232633833616362363433313562303536326537346662383239343965313964
+36353534666535313239316166343163343132323134313266653537646136383262626233376332
+31616166366234653632373732633061373362336137303831353763373166643031313466656434
+64623362663762316465633564633462363866373762383937386362623462333461666462613336
+64393633336137663631386562366339646464363433396366303466336561653938386639393563
+33613936356336346134323830366236326231653465373438623433613634656666386162336331
+32316637383637336266656637343639613233623739303438323033333935663261373230626661
+6635353534313662646665613438336264303835633564303166
@@ -0,0 +1,6 @@
+---
+# n8n group variables
+# Maps vault secrets to role defaults
+
+n8n_encryption_key: "{{ vault_n8n_encryption_key }}"
+n8n_postgres_password: "{{ vault_n8n_postgres_password }}"
@@ -1,23 +0,0 @@
---
-# Bouncer key for crowdsec-edge-bouncer, scoped to the proxy hosts.
-#
-# A single encrypted VALUE rather than a whole-file vault like its siblings in
-# group_vars/all and group_vars/edge_host. Deliberate: adding this variable to
-# an existing whole-file vault means decrypting that file first, which exposes
-# every other secret in it for the sake of appending one line. encrypt_string
-# keeps the blast radius at this one value.
-#
-# Same value as OpenBao homelab/infrastructure/crowdsec-edge-bouncer-key, which
-# the crowdsec-edge-bouncer-key ExternalSecret feeds to BOUNCER_KEY_edge on the
-# LAPI. Both sides must move together — and unlike crowdsec-bouncer-key, this
-# one is NOT in the quarterly rotator, because nothing on these hosts re-reads
-# OpenBao on a schedule. Rotating it means updating OpenBao and re-running
-# plays/install-netbird-proxy.yml.
-vault_crowdsec_edge_bouncer_key: !vault |
-          $ANSIBLE_VAULT;1.1;AES256
-          32653764333536316163303838333839616562663862666637626165303837633866643033393239
-          3363376538656236643161386432326437633232643564630a633635306365323537346139626364
-          36613033343938363366306239363638383466396337623333623136313234656366333366623433
-          6437656632373364340a346566343866366433306533353431383064356362346536383136323663
-          39333864643534326433653233303137373162663066633132623931316135323733333863306330
-          6366623463666435636362376238313639666263323163303733
@@ -0,0 +1,2 @@
+---
+ansible_user: root
@@ -0,0 +1,3 @@
+---
+# Docker hosts - tagged with 'docker' in Proxmox
+docker_monitoring_enabled: true
@@ -0,0 +1,8 @@
+---
+# --- NTP (see plays/install-ntp-ha.yml) ---
+# The PVE hosts hold the 10.2.40.4 VIP via keepalived on the VLAN 40 bridge
+# sub-interface. This role disappears when these nodes are rebuilt as bare-metal
+# Talos, which cannot run chrony or keepalived — edge-1 is being brought into
+# ntp_cluster to take over.
+ntp_keepalived_enabled: true
+ntp_keepalived_interface: "vmbr1.40"
@@ -0,0 +1,20 @@
+$ANSIBLE_VAULT;1.1;AES256
+66356134353363643130323632626530653863306635396432353539623731346138323635336632
+3536633563333231373333323564373763363936356535630a333135623336613832313833626531
+34646137633633353266653536363934303939626564353036343033643462373439643139353065
+6539383436343233300a396339356562363832373962306337663464363961346335656130653235
+38646563656538373530643531643964633934623465643065626266396130643966343166313937
+65663965613937323634323436386130653633656261653065306365316135343531653737663138
+65303339323137653362383766343361643837356531643339613431333238613564343563333761
+30616530613133633762306533363134383265313031343230313764633238626565623935313134
+38323435653264643534366366313862646161373434623135366164323236363733653533656233
+34336233396465626264666637363532646562306237363533616461613764306433366639623562
+30393234396238353038356636373632326130306431663439316363646530636563666364316139
+34326333363536343963393833616465303636323166643638623061353964303265393733366234
+33373435643237636536363364373830353833346235333534626561343465373236643165646664
+39396233363531393835656463666662616633643566383061323161663061323831356337333337
+65653336663735633536353263316634373639313238343365353833646666353131656530373361
+32643133353737366465633635663339666138323035633537653462303532366632626262373532
+32643132363062623033356134356135346361343234656636663936336338303230616665326630
+65353231356538633034626332633831396362373733646539313834386535666631613463656639
+373864663230353339666137653835656630
@@ -0,0 +1,11 @@
+---
+# Loki server - central log aggregation host
+
+# Enable syslog receiver for remote devices (Synology NAS, etc.)
+# Synology DSM's "IETF" format actually sends BSD/RFC 3164 format: <pri>MMM DD HH:MM:SS host app: msg
+# Using raw format to accept messages and parse them with regex
+alloy_syslog_receiver_enabled: true
+alloy_syslog_receiver_port: 1514
+alloy_syslog_receiver_protocol: "udp"
+alloy_syslog_receiver_format: "raw"
+alloy_stability_level: "experimental"
@@ -0,0 +1,37 @@
+---
+# PBS Client configuration for nas-1
+pbs_backup_paths:
+  - /volume1/Documents
+  - /volume1/homes
+  - /volume1/Homeassistant Backup
+  - /volume1/Games
+  - /volume1/Media
+  - /volume1/Music
+  - /volume1/video
+  - /volume2/Backup
+  - /volume1/Movies
+
+  # Add more paths as needed:
+  # - /volume1/photos
+  # - /volume1/music
+
+# PBS Server connection
+pbs_server_host: "pbs2.renner.uno"
+pbs_server_port: 8007
+pbs_datastore: "pool"
+pbs_backup_id: "nas-1"
+
+# PBS Authentication - credentials in vault.yml
+pbs_username: "root@pam!nas-1"
+
+# Garage S3 (K8s backup target)
+garage_image: "dxflrs/garage:v2.3.0"
+garage_zone: "dsm"
+garage_capacity: "2T"
+
+# SSH connection - use password auth (key not supported by DSM)
+ansible_ssh_pass: "{{ nas_ssh_password }}"
+ansible_become_pass: "{{ nas_ssh_password }}"
+
+# Python 3.9+ from Synology package
+ansible_python_interpreter: /usr/local/bin/python3.9
@@ -0,0 +1,27 @@
+$ANSIBLE_VAULT;1.1;AES256
+34663462636437633437613863366266306334666635643839313737343739643864613136613938
+3138393966306561656366396465383539303137333036300a376237376636356230343834386362
+36636535396330333732383233323263613132333166343739396137636538653435336662353138
+6565643866623235610a316266383933303830323936303735623338623165386339383733373435
+62366461363833616438373237383132366438326364363462333265366533323861376630643365
+39343036343838333639363432643738666630363063323636646661313539643465366532363438
+33303837623234613434633563666562346536346435353461373536323737376233386163326537
+61383963343737656438356136613537623538333163343466343739346135333439616432663763
+36663962633037376561326434303035333938646436663832303432346365353439666435613337
+34363636326530333233313231343566306432376532396666643761633136316333376337666135
+65363338623035663438323338663863366362653131336434323865326530326534623631336263
+36633063383031386161363131333636393364623436613331353733313235383062656131313164
+63393665633361313961363637663237616364643236383835393736643937393564656238376362
+31393465383361326264376133393233616264663636383337363962356532613438663637616536
+33653137656335323763313461323633616635346232303062323062333766383931313965346361
+66656461313239633366636463633465636463663937393932636337643861383937663662306466
+32353166643832303234643264633062616330393963333034623432613730366339306637613239
+34626161663666316339323762373639353335653663353635366664643063393432623039306237
+38363161343930363763393364663364353735343236313665313461626165636135303839363164
+37336638396564326166376233353730346563656266323535666265323062646232323938353732
+32636235333839623166633761376334316434613235333730353439343262393537363461333238
+32653435623362666662336532643832356163326332636436346132643237643030343761663332
+36663530356539303636663438646430363335353135323932613863666264363533326137353164
+31663032346236616263343637396439663334396166656364623637666535386530383535663435
+66303037303265616634373763616437333835333136336563323339666563663931396133633131
+3963653266633535303630653335666166313336653831623362
@@ -0,0 +1,8 @@
+---
+# Paperless server specific configuration
+
+# Extra scrape targets for Alloy
+alloy_extra_scrape_targets:
+  - name: "paperless"
+    address: "127.0.0.1:9199"
+    job: "paperless"
@@ -0,0 +1,7 @@
+---
+# Wiki server configuration
+
+# Home Assistant documentation generation (disabled - see HA Wiki Documentation project)
+homeassistant_docs_enabled: false
+homeassistant_url: "http://10.2.30.11:8123"
+homeassistant_token: "{{ vault_homeassistant_token }}"
@@ -1,16 +1,31 @@
-# Hosts are reached by name. An address in an inventory is allocator state
-# standing in for identity, and it pins access to the legacy family -- the names
-# resolve IPv6-first, which is the primary path on this network. DNS is the
-# authoritative source for where a host is; this file says which host, not where
-# it currently sits.
+# Reached by name, like every other host in this file. These three were the last
+# entries still addressed by IPv4 literal; authentik, netbird, proxy, nas-1,
+# grafana and edge-1 have always used their FQDN.
 #
-# Gone from this file, and why:
-#   pve_cluster (jupiter/neptune/saturn)  run Talos on bare metal; no SSH at all
-#   authentik_db / authentik_app          authentik runs in-cluster (CNPG)
-#   grafana                               runs in-cluster
-#   synology_nas (nas-1)                  powered off; Garage and the PBS client
-#                                         it hosted are gone with it
-# Their hosts no longer resolve, except the three that resolve to Talos nodes.
+# An address in an inventory is allocator state standing in for identity, and it
+# pins access to the legacy family -- the names resolve IPv6-first, which is the
+# primary path on this network. DNS is the authoritative source for where a host
+# is; this file should say which host, not where it currently sits.
+#
+# The short inventory_hostname is kept deliberately: plays key on it
+# (ntp_servers[inventory_hostname] in install-ntp-ha.yml, node names in opentofu,
+# the prometheus targets), so only ansible_host changes.
+[pve_cluster]
+jupiter ansible_host=jupiter.renner.uno ansible_user=root
+neptune ansible_host=neptune.renner.uno ansible_user=root
+saturn ansible_host=saturn.renner.uno ansible_user=root
+
+[authentik_db]
+authentik-db-1.srv.renner.uno ansible_user=root
+authentik-db-2.srv.renner.uno ansible_user=root
+
+[authentik:children]
+authentik_db
+authentik_app
+
+[authentik_app]
+authentik-app-1.srv.renner.uno ansible_user=root
+authentik-app-2.srv.renner.uno ansible_user=root

 [netbird_server]
 netbird.renner.uno ansible_user=root
@@ -19,6 +34,13 @@ netbird.renner.uno ansible_user=root
 proxy-1.renner.uno ansible_user=root
 proxy-2.renner.uno ansible_user=root

+
+[synology_nas]
+nas-1.renner.uno ansible_user=ladmin ansible_python_interpreter=/usr/bin/python3
+
+[grafana]
+grafana.srv.renner.uno ansible_user=ladmin
+
 # Every bare-metal edge thin client. Gets the host-generic roles only:
 # flash-wear handling, the log shipper, the MQTT bridge, the VLAN
 # sub-interface. Nothing here assumes any peripheral is attached.
@@ -49,21 +71,12 @@ zwave_gateway
 heating_can
 power_meter

-# NTP servers. edge-1 was added here so the role would survive the PVE hosts
-# being retired for bare-metal Talos, which cannot run chrony or keepalived
-# itself. That retirement has now happened, so edge-1 is the ONLY member.
-#
-# THIS IS NO LONGER HIGHLY AVAILABLE. The group was jupiter (MASTER, prio 100),
-# saturn (90), neptune (80) and edge-1 (BACKUP, 70); the first three are gone.
-# A single VRRP member takes the address regardless of its configured state, so
-# fd40::123 is still served — verified answering — but by one thin client with
-# no failover behind it. install-ntp-ha.yml still carries the four-host
-# ntp_servers priority map, which now describes hosts that do not exist.
-#
-# Restoring redundancy needs a second chrony host that is not a Talos node.
-# See plays/install-ntp-ha.yml for why keepalived is scoped more narrowly than
-# chrony within this group.
+# NTP servers. Includes edge-1 so the role survives the PVE hosts being
+# retired for bare-metal Talos, which cannot run chrony or keepalived itself.
+# See plays/install-ntp-ha.yml for why keepalived is scoped more narrowly
+# than chrony within this group.
 [ntp_cluster:children]
+pve_cluster
 edge_host

 # Off-site Debian VM (Synology VMM on nas-2, 10.3.40.0/24) running Claude Code.
@@ -0,0 +1,48 @@
+---
+# Backup Authentik PostgreSQL database for K8s migration
+# Creates a pg_dump on the primary DB host
+#
+# Usage (via CI):
+#   ansible-playbook -i inventory/ plays/backup-authentik-db.yml
+#
+# Output: /tmp/authentik-backup.sql on the primary DB host
+
+- name: Backup Authentik PostgreSQL database
+  hosts: authentik-db-1.srv.renner.uno
+  become: true
+  tasks:
+    - name: Create database backup
+      become_user: postgres
+      command: >
+        pg_dump
+        --clean --if-exists
+        --no-owner --no-privileges
+        --format=custom
+        --file=/tmp/authentik-backup.dump
+        {{ authentik_db_name | default('authentik') }}
+      changed_when: true
+
+    - name: Make backup readable
+      file:
+        path: /tmp/authentik-backup.dump
+        mode: "0644"
+
+    - name: Get backup size
+      stat:
+        path: /tmp/authentik-backup.dump
+      register: backup_stat
+
+    - name: Report backup size
+      debug:
+        msg: "Backup created: {{ (backup_stat.stat.size / 1024 / 1024) | round(1) }} MB"
+
+    - name: Fetch backup to controller
+      fetch:
+        src: /tmp/authentik-backup.dump
+        dest: /tmp/authentik-backup.dump
+        flat: true
+
+    - name: Clean up remote backup
+      file:
+        path: /tmp/authentik-backup.dump
+        state: absent
@@ -0,0 +1,59 @@
+---
+# Dump Paperless-ngx DB from the legacy VM to a local file on the
+# controller (CI runner), where a downstream pipeline step consumes it
+# to produce a Barman seed for the CNPG `bootstrap.recovery` path.
+# Data + media tarballs are handled separately by `backup-paperless.yml`.
+
+- name: Dump Paperless DB from legacy VM to CI runner
+  hosts: paperless.srv.renner.uno
+  become: true
+  vars:
+    paperless_db_name: paperless
+    paperless_db_user: paperless
+  tasks:
+    - name: Find running paperless DB container
+      ansible.builtin.shell: >
+        docker ps --filter "label=com.docker.compose.service=db"
+        --format '{{ "{{" }}.Names{{ "}}" }}' | head -1
+      register: paperless_db_container
+      changed_when: false
+
+    - name: Fail if DB container not found
+      ansible.builtin.fail:
+        msg: "Could not find paperless DB container (compose service=db)"
+      when: paperless_db_container.stdout | trim == ""
+
+    - name: Dump DB to /tmp on legacy host
+      ansible.builtin.shell: >
+        docker exec -i {{ paperless_db_container.stdout | trim }}
+        pg_dump --clean --if-exists --no-owner --no-privileges --format=custom
+        -U {{ paperless_db_user }} {{ paperless_db_name }}
+        > /tmp/paperless-backup.dump
+      args:
+        executable: /bin/bash
+      changed_when: true
+
+    - name: Make dump readable
+      ansible.builtin.file:
+        path: /tmp/paperless-backup.dump
+        mode: "0644"
+
+    - name: Report dump size
+      ansible.builtin.stat:
+        path: /tmp/paperless-backup.dump
+      register: dump_stat
+
+    - name: Show dump size
+      ansible.builtin.debug:
+        msg: "Dump created: {{ (dump_stat.stat.size / 1024 / 1024) | round(1) }} MB"
+
+    - name: Fetch dump to controller
+      ansible.builtin.fetch:
+        src: /tmp/paperless-backup.dump
+        dest: /tmp/paperless-backup.dump
+        flat: true
+
+    - name: Remove remote dump
+      ansible.builtin.file:
+        path: /tmp/paperless-backup.dump
+        state: absent
@@ -0,0 +1,95 @@
+---
+- name: Check for Distribution Upgrades
+  hosts: proxmox_all_qemu:&status_running:!proxmox_noansible
+  become: true
+  gather_facts: true
+  ignore_unreachable: true
+  
+  tasks:
+    - name: Check Ubuntu Release Availability
+      ansible.builtin.command: do-release-upgrade -c
+      register: ubuntu_check
+      changed_when: false
+      failed_when: false
+      when: 
+        - ansible_distribution is defined
+        - ansible_distribution == 'Ubuntu'
+
+    - name: Determine Upgrade Status
+      ansible.builtin.set_fact:
+        upgrade_needed: >-
+          {%- if ansible_distribution is not defined -%}
+            unknown
+          {%- elif ansible_distribution == 'Ubuntu' -%}
+            {%- if ubuntu_check.stdout is defined and 'New release' in ubuntu_check.stdout -%}
+              true
+            {%- else -%}
+              false
+            {%- endif -%}
+          {%- elif ansible_distribution == 'Debian' -%}
+            {%- if ansible_distribution_major_version | int < 12 -%}
+              true
+            {%- else -%}
+              false
+            {%- endif -%}
+          {%- else -%}
+            false
+          {%- endif -%}
+        current_version_str: "{{ ansible_distribution }} {{ ansible_distribution_version }}"
+        target_version_str: >-
+          {%- if ansible_distribution is not defined -%}
+            Unknown
+          {%- elif ansible_distribution == 'Ubuntu' -%}
+            {%- if ubuntu_check.stdout is defined and 'New release' in ubuntu_check.stdout -%}
+              {{ (ubuntu_check.stdout | regex_findall("New release '([^']+)'") | first) | default('Unknown') }}
+            {%- else -%}
+              -
+            {%- endif -%}
+          {%- elif ansible_distribution == 'Debian' -%}
+            {%- if ansible_distribution_major_version | int < 12 -%}
+              Debian 12 (Bookworm)
+            {%- else -%}
+              -
+            {%- endif -%}
+          {%- else -%}
+            -
+          {%- endif -%}
+      when: ansible_distribution is defined
+
+    - name: Group hosts by status
+      ansible.builtin.group_by:
+        key: "upgrade_{{ upgrade_needed | trim }}"
+      when: upgrade_needed is defined
+
+- name: Generate Report
+  hosts: localhost
+  gather_facts: false
+  tasks:
+    - name: Display Upgrade Candidates
+      ansible.builtin.debug:
+        msg: "{{ report.splitlines() }}"
+      vars:
+        report: |
+          
+          DISTRIBUTION UPGRADE REPORT
+          ===========================
+
+          REQUIRES UPGRADE
+          ================
+          {{ '%-30s' | format('HOST') }}   {{ '%-20s' | format('CURRENT') }}   {{ 'TARGET' }}
+          {{ '------------------------------' }}   {{ '--------------------' }}   {{ '--------------------' }}
+          {% for host in groups['upgrade_true'] | default([]) %}
+          {{ '%-30s' | format(host) }}   {{ '%-20s' | format(hostvars[host]['current_version_str']) }}   {{ hostvars[host]['target_version_str'] }}
+          {% endfor %}
+          {% if groups['upgrade_true'] is not defined or groups['upgrade_true'] | length == 0 %}
+          (No hosts require upgrade)
+          {% endif %}
+
+          UP TO DATE
+          ==========
+          {{ '%-30s' | format('HOST') }}   {{ 'CURRENT' }}
+          {{ '------------------------------' }}   {{ '--------------------' }}
+          {% for host in groups['upgrade_false'] | default([]) %}
+          {{ '%-30s' | format(host) }}   {{ hostvars[host]['current_version_str'] }}
+          {% endfor %}
+
@@ -0,0 +1,8 @@
+- name: Ensure debian-12 template exists on saturn
+  hosts: saturn
+  become: true
+  tasks:
+    - name: Download debian-12 template
+      command: pveam download local debian-12-standard_12.12-1_amd64.tar.zst
+      args:
+        creates: /var/lib/vz/template/cache/debian-12-standard_12.12-1_amd64.tar.zst
@@ -1,16 +1,22 @@
 ---
 - name: Install Custom CA Certificates
-  hosts: all
+  hosts: all:!proxmox_noansible
  become: true
  roles:
    - certificates

 - name: Base System Hardening & Common Configuration
-  hosts: all
+  hosts: all:!pve:!pbs:!proxmox_noansible:!proxmox_nodes:!pve_cluster
  become: true
  ignore_unreachable: true
  gather_facts: true

+  vars:
+    # Monitoring agent endpoints
+    loki_endpoint: "http://loki.srv.renner.uno:3100"
+    prometheus_remote_write_endpoint: "http://prometheus.srv.renner.uno:9090/api/v1/write"
+
  roles:
    - common
    - qemu-guest-agent
+    - monitoring-agent
@@ -0,0 +1,44 @@
+---
+# Configure Audiobookshelf permissions
+# Fixes upload issues by ensuring audiobookshelf user can write to storage directories
+
+- name: Configure Audiobookshelf
+  hosts: audiobookshelf.srv.renner.uno
+  become: true
+
+  tasks:
+    - name: Ensure audiobookshelf user exists
+      ansible.builtin.user:
+        name: audiobookshelf
+        system: true
+        create_home: false
+        state: present
+
+    - name: Set ownership on audiobooks directory
+      ansible.builtin.file:
+        path: /audiobooks
+        owner: audiobookshelf
+        group: audiobookshelf
+        mode: "0755"
+        state: directory
+        recurse: false
+
+    - name: Set ownership on audiobooks content recursively
+      ansible.builtin.command:
+        cmd: chown -R audiobookshelf:audiobookshelf /audiobooks
+      changed_when: true
+
+    - name: Set ownership on metadata directory
+      ansible.builtin.file:
+        path: /metadata
+        owner: audiobookshelf
+        group: audiobookshelf
+        mode: "0755"
+        state: directory
+        recurse: true
+
+    - name: Restart audiobookshelf service
+      ansible.builtin.systemd:
+        name: audiobookshelf
+        state: restarted
+        enabled: true
@@ -0,0 +1,40 @@
+---
+# Configure Grafana to use Authentik for authentication
+#
+# Prerequisites:
+# 1. Create OAuth2/OpenID application in Authentik:
+#    - Name: Grafana
+#    - Slug: grafana
+#    - Client type: Confidential
+#    - Redirect URIs: https://dash.renner.uno/login/generic_oauth
+#    - Signing Key: authentik Self-signed Certificate
+#
+# 2. Add to vault (ansible/inventory/group_vars/all/vault.yml):
+#    vault_grafana_authentik_client_id: "<client-id-from-authentik>"
+#    vault_grafana_authentik_client_secret: "<client-secret-from-authentik>"
+#
+# 3. (Optional) Create groups in Authentik:
+#    - "Grafana Admins" - gets Admin role
+#    - "Grafana Editors" - gets Editor role
+#    - Others get Viewer role
+
+- name: Configure Grafana Authentik Authentication
+  hosts: grafana.srv.renner.uno
+  become: yes
+  roles:
+    - grafana-authentik
+
+  post_tasks:
+    - name: Display configuration status
+      debug:
+        msg: |
+          Grafana Authentik OAuth configured!
+
+          Users can now login at: {{ grafana_root_url }}
+
+          Role mapping:
+          - Members of "Grafana Admins" -> Admin
+          - Members of "Grafana Editors" -> Editor
+          - All others -> Viewer
+
+          If auto_login is enabled, users will be redirected to Authentik automatically.
@@ -0,0 +1,68 @@
+---
+# Set sync=disabled on the zvols backing the K8s control-plane VM root disks
+# (k8s-1/2/3, VMIDs 150/151/152) to eliminate ZFS fsync amplification on the
+# shared local-zfs pool. Trades a small durability window for major fsync
+# latency reduction on etcd's WAL writes:
+#
+#   - etcd's WAL fsync returns immediately; ZFS still flushes via txgs
+#     (default ~5s)
+#   - Worst case on hard power loss: up to ~5s of writes lost per CP member
+#   - With 3-member quorum and Velero backups, this is acceptable for homelab
+#   - The whole zvol is sync=disabled (Talos OS, containerd, kubelet
+#     ephemeral, etcd WAL all share it), but etcd is the dominant fsync user;
+#     other writers tolerate small loss windows
+#
+# Each PVE host owns exactly one CP zvol (jupiter→150, saturn→151, neptune→152).
+# Auto-discovery handles whatever the host has — no need to update if a CP
+# VM ever moves.
+#
+# Idempotent: skips zvols that already have sync=disabled.
+#
+# Run manually against freshly-created CP zvols so they get the property
+# before Talos starts writing (previously wired into the now-removed
+# rebuild-k8s pipeline between `recreate-vms` and `talos-bootstrap`).
+
+- name: Set sync=disabled on K8s CP VM zvols
+  hosts: pve_cluster
+  become: yes
+  gather_facts: no
+
+  tasks:
+    - name: Discover CP-VM zvols on this host
+      shell: |
+        set -o pipefail
+        zfs list -H -t volume -o name | grep -E "/vm-(150|151|152)-disk-[0-9]+$" || true
+      args:
+        executable: /bin/bash
+      register: cp_zvols
+      changed_when: false
+
+    - name: Read current sync property
+      shell: "zfs get -H -o value sync {{ item }}"
+      register: current_sync
+      changed_when: false
+      loop: "{{ cp_zvols.stdout_lines }}"
+      when: cp_zvols.stdout_lines | length > 0
+
+    - name: Set sync=disabled where not already
+      shell: "zfs set sync=disabled {{ item.item }}"
+      when:
+        - cp_zvols.stdout_lines | length > 0
+        - item.stdout | trim != "disabled"
+      loop: "{{ current_sync.results | default([]) }}"
+      loop_control:
+        label: "{{ item.item }}"
+
+    - name: Verify sync=disabled
+      shell: "zfs get -H -o value sync {{ item }}"
+      register: verify_sync
+      changed_when: false
+      failed_when: verify_sync.stdout | trim != "disabled"
+      loop: "{{ cp_zvols.stdout_lines }}"
+      when: cp_zvols.stdout_lines | length > 0
+
+    - name: Summary
+      debug:
+        msg: >-
+          {{ inventory_hostname }}: tuned {{ cp_zvols.stdout_lines | length }}
+          zvol(s) [{{ cp_zvols.stdout_lines | join(', ') }}]
@@ -0,0 +1,53 @@
+---
+# Configure Proxmox VE Node Host Networking
+#
+# Fixes two defects found by live inspection on 2026-07-19:
+#
+#   1. All three nodes had no IPv6 address on any external interface. FRR
+#      enables ipv6 forwarding for the OSPFv3 corosync mesh, and with the
+#      kernel default accept_ra=1 the nodes silently discarded the Router
+#      Advertisements the gateway sends on VLAN 40. Fixed with accept_ra=2.
+#
+#   2. saturn's vmbr0 sits at MTU 1500 while its own uplink and both peer
+#      nodes are at 9000, so large frames between saturn's guests and the
+#      other nodes' guests blackhole instead of fragmenting.
+#
+# The sysctl fix is non-disruptive and applies on every run. The MTU fix is
+# staged as a pending PVE network change and only applied when explicitly
+# requested, because applying it reloads networking on the node and briefly
+# interrupts the Talos k8s VM and its Ceph OSD.
+#
+# Safe (converges IPv6, stages the MTU change without applying it):
+#   ansible-playbook -i inventory/ plays/configure-pve-node-network.yml
+#
+# Applying the staged MTU change, one node at a time:
+#   ansible-playbook -i inventory/ plays/configure-pve-node-network.yml \
+#     --limit saturn -e pve_node_network_apply_pending=true
+#
+# See wiki: pve-ipv6-accept-ra-forwarding-defect,
+#           pve-saturn-vmbr0-mtu-asymmetry
+
+- name: Configure PVE node host networking
+  hosts: pve_cluster
+  become: yes
+  serial: 1
+
+  roles:
+    - role: pve-node-network
+
+  post_tasks:
+    - name: Summarise node IPv6 state
+      ansible.builtin.command: ip -6 addr show dev vmbr1.40 scope global
+      register: pve_node_network_v6_summary
+      changed_when: false
+      failed_when: false
+
+    - name: Display node IPv6 state
+      ansible.builtin.debug:
+        msg: >-
+          {{ inventory_hostname }}:
+          {{ pve_node_network_v6_summary.stdout_lines
+             | select('search', 'inet6')
+             | map('trim')
+             | list
+             | default(['no global IPv6'], true) }}
@@ -0,0 +1,20 @@
+---
+- name: Setup PostgreSQL HA Cluster
+  hosts: authentik_db
+  become: true
+  vars:
+    # Set this in inventory or pass as extra var.
+    # For initial setup, we define authentik-db-1 as primary.
+    postgres_primary_host: "authentik-db-1.srv.renner.uno"
+    
+  roles:
+    - common
+    - postgres-simple-ha
+
+- name: Setup Authentik Stateless App
+  hosts: authentik_app
+  become: true
+  roles:
+    - common
+    - docker-install
+    - authentik-stateless
@@ -10,7 +10,7 @@
 - name: Deploy the Claude Code workstation
  hosts: claude_workstation
  become: true
-  # qemu-guest-agent is applied
+  # qemu-guest-agent + monitoring-agent (Alloy -> Loki/Prometheus) are applied
  # to every inventory host by the umbrella plays/common.yml, same as for the
  # netbird-proxy host — run that against this host too (see RUNBOOK Step 5).
  roles:
@@ -18,10 +18,9 @@
 #
 # Self-contained: `certificates` and `common` are the parts of plays/common.yml
 # that apply to an edge host, carried here so bootstrapping one is a single
-# unrestricted run over the group. Not carried over: qemu-guest-agent, since
-# these are bare metal rather than VMs. Log shipping is edge-log-shipper's job
-# and lives below — it pushes to the authenticated cluster ingress with a
-# per-host credential, which is the only Loki these hosts can reach.
+# unrestricted run over the group. Not carried over: qemu-guest-agent (bare
+# metal, not VMs) and monitoring-agent (Debian-oriented; these hosts ship logs
+# via edge-log-shipper).
 #
 # Order is dependency order: edge-flash-wear before docker, so the journal is
 # volatile before containers start logging into it.
@@ -0,0 +1,313 @@
+---
+# Synology-specific playbook using raw module (Python 3.8 workaround)
+# Deploys Garage (https://garagehq.deuxfleurs.fr) as a single-node S3 target
+# for K8s backups (Velero + CNPG Barman Cloud Plugin).
+#
+# Required secrets in host_vars/nas-1.renner.uno/vault.yml:
+#   garage_rpc_secret:     "<openssl rand -hex 32>"
+#   garage_admin_token:    "<openssl rand -base64 32>"
+#   garage_metrics_token:  "<openssl rand -base64 32>"
+#
+# Post-install bootstrap (run once, record the returned secret keys):
+#   docker exec garaged /garage status
+#   docker exec garaged /garage bucket create k8s-velero
+#   docker exec garaged /garage bucket create k8s-cnpg
+#   docker exec garaged /garage bucket create k8s-secrets
+#   docker exec garaged /garage bucket create k8s-migration
+#   docker exec garaged /garage key create velero-key
+#   docker exec garaged /garage key create cnpg-key
+#   docker exec garaged /garage key create secrets-key
+#   docker exec garaged /garage key create migration-key
+#   docker exec garaged /garage bucket allow --read --write --owner k8s-velero    --key velero-key
+#   docker exec garaged /garage bucket allow --read --write --owner k8s-cnpg      --key cnpg-key
+#   docker exec garaged /garage bucket allow --read --write --owner k8s-secrets   --key secrets-key
+#   docker exec garaged /garage bucket allow --read --write --owner k8s-migration --key migration-key
+- name: Install Garage S3 on Synology NAS
+  hosts: synology_nas
+  gather_facts: false
+
+  vars:
+    garage_data_dir: "/volume1/docker/garage"
+
+  tasks:
+    - name: Create Garage directory tree
+      ansible.builtin.raw: |
+        mkdir -p {{ garage_data_dir }}/config \
+                 {{ garage_data_dir }}/meta \
+                 {{ garage_data_dir }}/data
+      changed_when: true
+
+    # Secret writes use the same compare-then-write pattern as garage.toml so
+    # rotation is idempotent AND a real change can drive force-recreate below.
+    # Garage reads these files once at startup; without container restart, a
+    # rotated token leaves the running process holding the previous value.
+    - name: Write rpc_secret
+      ansible.builtin.raw: |
+        cat > {{ garage_data_dir }}/config/rpc_secret.new << 'SECRET_EOF'
+        {{ garage_rpc_secret }}
+        SECRET_EOF
+        chmod 600 {{ garage_data_dir }}/config/rpc_secret.new
+        if [ ! -f {{ garage_data_dir }}/config/rpc_secret ] \
+        || ! cmp -s {{ garage_data_dir }}/config/rpc_secret.new {{ garage_data_dir }}/config/rpc_secret; then
+          mv {{ garage_data_dir }}/config/rpc_secret.new {{ garage_data_dir }}/config/rpc_secret
+          echo RPC_SECRET_CHANGED
+        else
+          rm {{ garage_data_dir }}/config/rpc_secret.new
+        fi
+      register: rpc_secret_write
+      changed_when: "'RPC_SECRET_CHANGED' in rpc_secret_write.stdout"
+      no_log: true
+
+    - name: Write admin_token
+      ansible.builtin.raw: |
+        cat > {{ garage_data_dir }}/config/admin_token.new << 'SECRET_EOF'
+        {{ garage_admin_token }}
+        SECRET_EOF
+        chmod 600 {{ garage_data_dir }}/config/admin_token.new
+        if [ ! -f {{ garage_data_dir }}/config/admin_token ] \
+        || ! cmp -s {{ garage_data_dir }}/config/admin_token.new {{ garage_data_dir }}/config/admin_token; then
+          mv {{ garage_data_dir }}/config/admin_token.new {{ garage_data_dir }}/config/admin_token
+          echo ADMIN_TOKEN_CHANGED
+        else
+          rm {{ garage_data_dir }}/config/admin_token.new
+        fi
+      register: admin_token_write
+      changed_when: "'ADMIN_TOKEN_CHANGED' in admin_token_write.stdout"
+      no_log: true
+
+    - name: Write metrics_token
+      ansible.builtin.raw: |
+        cat > {{ garage_data_dir }}/config/metrics_token.new << 'SECRET_EOF'
+        {{ garage_metrics_token }}
+        SECRET_EOF
+        chmod 600 {{ garage_data_dir }}/config/metrics_token.new
+        if [ ! -f {{ garage_data_dir }}/config/metrics_token ] \
+        || ! cmp -s {{ garage_data_dir }}/config/metrics_token.new {{ garage_data_dir }}/config/metrics_token; then
+          mv {{ garage_data_dir }}/config/metrics_token.new {{ garage_data_dir }}/config/metrics_token
+          echo METRICS_TOKEN_CHANGED
+        else
+          rm {{ garage_data_dir }}/config/metrics_token.new
+        fi
+      register: metrics_token_write
+      changed_when: "'METRICS_TOKEN_CHANGED' in metrics_token_write.stdout"
+      no_log: true
+
+    - name: Write garage.toml
+      ansible.builtin.raw: |
+        cat > {{ garage_data_dir }}/config/garage.toml.new << 'TOML_EOF'
+        metadata_dir = "/var/lib/garage/meta"
+        data_dir     = "/var/lib/garage/data"
+
+        # SQLite chosen over LMDB: single-node deployment has no peers to
+        # rebuild metadata from on unclean-shutdown corruption. SQLite is
+        # slower but portable and has no LMDB corruption window.
+        db_engine = "sqlite"
+
+        # 6-hour auto-snapshot of the metadata DB as a backstop against
+        # corruption; last two snapshots retained.
+        metadata_auto_snapshot_interval = "6h"
+
+        # Single-zone, single-node: only coherent replication factor.
+        replication_factor = 1
+
+        rpc_bind_addr    = "[::]:3901"
+        rpc_secret_file  = "/etc/garage/rpc_secret"
+
+        compression_level = 2
+
+        # 64 MB blocks (default 1 MB; previously 20 MB). On the 5 × 7 TB
+        # Btrfs RAID 5 volume the per-block fixed cost (SQLite lookup,
+        # dir traversal, hash verify, zstd decompress, nginx buffering)
+        # is the dominant overhead per object; larger blocks amortize
+        # it. block_size is a maximum, so 16 MiB CNPG WAL segments
+        # still upload as one block each. Only applies to NEW uploads;
+        # existing objects keep their original chunking until re-uploaded.
+        block_size = "64M"
+
+        [s3_api]
+        api_bind_addr = "[::]:3900"
+        s3_region     = "garage"
+        root_domain   = ".s3.{{ inventory_hostname }}"
+
+        [s3_web]
+        bind_addr   = "[::]:3902"
+        root_domain = ".web.{{ inventory_hostname }}"
+
+        [admin]
+        api_bind_addr      = "[::]:3903"
+        admin_token_file   = "/etc/garage/admin_token"
+        metrics_token_file = "/etc/garage/metrics_token"
+        TOML_EOF
+        if [ ! -f {{ garage_data_dir }}/config/garage.toml ] \
+        || ! cmp -s {{ garage_data_dir }}/config/garage.toml.new {{ garage_data_dir }}/config/garage.toml; then
+          mv {{ garage_data_dir }}/config/garage.toml.new {{ garage_data_dir }}/config/garage.toml
+          echo GARAGE_TOML_CHANGED
+        else
+          rm {{ garage_data_dir }}/config/garage.toml.new
+        fi
+      register: garage_toml_write
+      changed_when: "'GARAGE_TOML_CHANGED' in garage_toml_write.stdout"
+
+    - name: Write docker-compose.yml
+      ansible.builtin.raw: |
+        cat > {{ garage_data_dir }}/docker-compose.yml << 'COMPOSE_EOF'
+        services:
+          garage:
+            image: {{ garage_image }}
+            container_name: garaged
+            restart: unless-stopped
+            # HTTP services (3900/3902/3903) bind to loopback; DSM
+            # reverse proxy fronts and TLS-terminates them. RPC 3901
+            # is exposed directly — the Noise handshake is raw TCP,
+            # so DSM RP cannot front it. rpc_secret gates auth.
+            ports:
+              - "3901:3901"
+              - "127.0.0.1:3900:3900"
+              - "127.0.0.1:3902:3902"
+              - "127.0.0.1:3903:3903"
+            volumes:
+              - {{ garage_data_dir }}/config/garage.toml:/etc/garage.toml:ro
+              - {{ garage_data_dir }}/config/rpc_secret:/etc/garage/rpc_secret:ro
+              - {{ garage_data_dir }}/config/admin_token:/etc/garage/admin_token:ro
+              - {{ garage_data_dir }}/config/metrics_token:/etc/garage/metrics_token:ro
+              - {{ garage_data_dir }}/meta:/var/lib/garage/meta
+              - {{ garage_data_dir }}/data:/var/lib/garage/data
+        COMPOSE_EOF
+      changed_when: true
+
+    - name: Pull image and start Garage container
+      ansible.builtin.shell: |
+        export PATH=$PATH:/usr/local/bin
+        cd {{ garage_data_dir }} && docker compose pull && docker compose up -d
+      become: true
+      changed_when: true
+
+    # Garage does not reload config on SIGHUP and compose does not notice
+    # when a bind-mounted host file changes. Force-recreate the container
+    # when garage.toml OR any of the three secret files changed so the new
+    # config / rotated tokens are picked up.
+    - name: Force-recreate garaged when garage.toml or secrets changed
+      ansible.builtin.shell: |
+        export PATH=$PATH:/usr/local/bin
+        cd {{ garage_data_dir }} && docker compose up -d --force-recreate garage
+      become: true
+      when: garage_toml_write.changed
+            or rpc_secret_write.changed
+            or admin_token_write.changed
+            or metrics_token_write.changed
+      changed_when: true
+
+    - name: Wait for Garage to accept CLI calls
+      ansible.builtin.shell: |
+        export PATH=$PATH:/usr/local/bin
+        for i in $(seq 1 30); do
+          if docker exec garaged /garage status >/dev/null 2>&1; then
+            exit 0
+          fi
+          sleep 2
+        done
+        exit 1
+      become: true
+      changed_when: false
+
+    - name: Check whether cluster layout is already applied
+      ansible.builtin.shell: |
+        export PATH=$PATH:/usr/local/bin
+        docker exec garaged /garage status
+      become: true
+      register: garage_status
+      changed_when: false
+
+    - name: Apply single-node layout (idempotent — only on first run)
+      when: "'NO ROLE ASSIGNED' in garage_status.stdout"
+      ansible.builtin.shell: |
+        export PATH=$PATH:/usr/local/bin
+        NODE_ID=$(docker exec garaged /garage node id | head -n 1 | cut -d@ -f1)
+        docker exec garaged /garage layout assign "$NODE_ID" \
+          -z {{ garage_zone }} \
+          -c {{ garage_capacity }} \
+          -t {{ inventory_hostname_short }}
+        docker exec garaged /garage layout apply --version 1
+      become: true
+      changed_when: true
+
+    - name: Report Garage status
+      ansible.builtin.shell: |
+        export PATH=$PATH:/usr/local/bin
+        docker exec garaged /garage status
+      become: true
+      register: garage_status_final
+      changed_when: false
+
+    - name: Show Garage status
+      ansible.builtin.debug:
+        var: garage_status_final.stdout_lines
+
+    # Bootstrap buckets + keys. Idempotent: bucket_info / key_info return
+    # non-zero when missing, so we only create on first run. Access-key
+    # secrets are captured at create time and persisted to chmod-600 files
+    # under {{ garage_data_dir }}/config/keys/<key>.creds on nas-1. These
+    # files are the only post-install record of the secrets — Garage does
+    # not expose them again.
+    - name: Bootstrap Garage buckets, keys, and grants
+      ansible.builtin.shell: |
+        set -e
+        export PATH=$PATH:/usr/local/bin
+        KEYS_DIR={{ garage_data_dir }}/config/keys
+        mkdir -p "$KEYS_DIR"
+        chmod 700 "$KEYS_DIR"
+
+        bootstrap_pair() {
+          local BUCKET="$1"
+          local KEY="$2"
+          local CREDS_FILE="$KEYS_DIR/$KEY.creds"
+
+          if ! docker exec garaged /garage bucket info "$BUCKET" >/dev/null 2>&1; then
+            docker exec garaged /garage bucket create "$BUCKET"
+          fi
+
+          if ! docker exec garaged /garage key info "$KEY" >/dev/null 2>&1; then
+            CREATE_OUT=$(docker exec garaged /garage key create "$KEY")
+            KEY_ID=$(echo "$CREATE_OUT"   | awk '/^Key ID:/     {print $3}')
+            SECRET=$(echo "$CREATE_OUT"   | awk '/^Secret key:/ {print $3}')
+            umask 177
+            printf 'KEY_ID=%s\nSECRET_KEY=%s\n' "$KEY_ID" "$SECRET" > "$CREDS_FILE"
+          fi
+
+          docker exec garaged /garage bucket allow \
+            --read --write --owner \
+            "$BUCKET" --key "$KEY"
+        }
+
+        bootstrap_pair k8s-velero    velero-key
+        bootstrap_pair k8s-cnpg      cnpg-key
+        bootstrap_pair k8s-secrets   secrets-key
+        bootstrap_pair k8s-migration migration-key
+        bootstrap_pair tfstate       tfstate-key
+      become: true
+      changed_when: true
+      no_log: true
+
+    - name: Report bucket + key state (no secrets)
+      ansible.builtin.shell: |
+        export PATH=$PATH:/usr/local/bin
+        docker exec garaged /garage bucket list
+        echo "---"
+        docker exec garaged /garage key list
+        echo "---"
+        ls -l {{ garage_data_dir }}/config/keys/ 2>&1
+      become: true
+      register: garage_bootstrap_state
+      changed_when: false
+
+    - name: Show bucket + key state
+      ansible.builtin.debug:
+        var: garage_bootstrap_state.stdout_lines
+
+    - name: Post-install note
+      ansible.builtin.debug:
+        msg:
+          - "Buckets: k8s-velero, k8s-cnpg, k8s-secrets, k8s-migration, tfstate"
+          - "Keys:    velero-key, cnpg-key, secrets-key, migration-key, tfstate-key"
+          - "Secrets are written to {{ garage_data_dir }}/config/keys/<key>.creds on nas-1 (chmod 600, root-owned)."
+          - "SSH to nas-1 and `sudo cat` the file to retrieve KEY_ID + SECRET_KEY for Velero / CNPG / migration / tfstate config."
@@ -0,0 +1,8 @@
+---
+- name: Install and Configure Gatus
+  hosts: proxmox_gatus
+  become: true
+  roles:
+    - role: common
+    - role: docker-install
+    - role: gatus
@@ -0,0 +1,8 @@
+---
+- name: Install and Configure Grist
+  hosts: grist
+  become: true
+  roles:
+    - role: common
+    - role: docker-install
+    - role: grist
@@ -0,0 +1,8 @@
+---
+- name: Install and Configure ical-proxy
+  hosts: proxmox_ical_proxy
+  become: true
+  roles:
+    - role: common
+    - role: docker-install
+    - role: ical-proxy
@@ -0,0 +1,19 @@
+---
+# Deploy MkDocs wiki from Obsidian vault
+#
+# Usage:
+#   ansible-playbook -i inventory plays/install-mkdocs-wiki.yml
+#
+- name: Deploy MkDocs Wiki
+  hosts: proxmox_wiki
+  become: true
+
+  pre_tasks:
+    - name: Ensure wiki group exists
+      ansible.builtin.group:
+        name: wiki
+        system: true
+        state: present
+
+  roles:
+    - role: mkdocs-wiki
@@ -0,0 +1,8 @@
+---
+- name: Install and Configure Music Assistant
+  hosts: proxmox_music_assistant
+  become: true
+  roles:
+    - role: common
+    - role: docker-install
+    - role: music-assistant
@@ -0,0 +1,8 @@
+---
+- name: Install and Configure n8n
+  hosts: n8n
+  become: true
+  roles:
+    - role: common
+    - role: docker-install
+    - role: n8n
@@ -0,0 +1,38 @@
+---
+# Deploy monitoring agent to NetBird server (external datacenter)
+# Uses NetBird mesh IPs to reach Loki and Prometheus
+# Usage: ansible-playbook -i inventory/ plays/setup-netbird-monitoring.yml
+
+- name: Deploy Monitoring Agent to NetBird Server
+  hosts: netbird_server
+  become: yes
+  gather_facts: yes
+
+  vars:
+    # Use ARM64 binary for Hetzner ARM server
+    alloy_arch: "linux-arm64"
+
+    # Monitoring endpoints via NetBird mesh
+    loki_endpoint: "http://loki.nb.renner.uno:3100"
+    prometheus_remote_write_endpoint: "http://prometheus.nb.renner.uno:9090/api/v1/write"
+
+    # NetBird server labels
+    alloy_extra_labels:
+      environment: "production"
+      datacenter: "hetzner"
+      type: "netbird-server"
+      service: "netbird"
+
+  roles:
+    - monitoring-agent
+
+  post_tasks:
+    - name: Verify Alloy agent is running
+      systemd:
+        name: alloy
+        state: started
+      register: alloy_status
+
+    - name: Display agent status
+      debug:
+        msg: "Monitoring agent deployed to {{ inventory_hostname }} via NetBird mesh"
@@ -26,7 +26,3 @@
    - docker-install
    - netbird-proxy
    - netbird-proxy-watchdog
-    # Enforces the cluster's CrowdSec decisions here, where public traffic
-    # enters. Deployed in dry-run: it logs what it would block and creates no
-    # nftables rules until crowdsec_bouncer_mode is flipped per host.
-    - crowdsec-edge-bouncer
@@ -0,0 +1,41 @@
+---
+# Install Node Exporter and Chrony Exporter on Proxmox VE hosts
+# Usage: ansible-playbook -i inventory/ plays/install-node-exporter-pve.yml
+
+- name: Install Exporters on PVE Cluster
+  hosts: pve_cluster
+  become: yes
+  gather_facts: yes
+
+  roles:
+    - node-exporter
+    - chrony-exporter
+
+  post_tasks:
+    - name: Verify node-exporter is running
+      uri:
+        url: "http://127.0.0.1:9100/metrics"
+        status_code: 200
+      register: node_exporter_health
+      retries: 3
+      delay: 5
+      until: node_exporter_health.status == 200
+
+    - name: Verify chrony-exporter is running
+      uri:
+        url: "http://127.0.0.1:9123/metrics"
+        status_code: 200
+      register: chrony_exporter_health
+      retries: 3
+      delay: 5
+      until: chrony_exporter_health.status == 200
+
+    - name: Verify chrony metrics include stratum
+      uri:
+        url: "http://127.0.0.1:9123/metrics"
+        return_content: yes
+      register: chrony_metrics
+
+    - name: Check chrony exporter output
+      debug:
+        msg: "Chrony stratum metrics available: {{ 'chrony_tracking_stratum' in chrony_metrics.content }}"
@@ -3,7 +3,7 @@
 # Usage: ansible-playbook -i inventory/ plays/install-node-exporter.yml

 - name: Install Node Exporter
-  hosts: all
+  hosts: all:!pve_cluster
  become: yes
  gather_facts: yes

@@ -0,0 +1,8 @@
+---
+- name: Install and Configure ownCloud Infinite Scale
+  hosts: proxmox_owncloud
+  become: true
+  roles:
+    - role: common
+    - role: docker-install
+    - role: owncloud
@@ -0,0 +1,104 @@
+---
+# Synology-specific playbook using raw module (Python 3.8 workaround)
+- name: Install PBS Client on Synology NAS
+  hosts: synology_nas
+  gather_facts: false
+
+  vars:
+    pbs_client_data_dir: "/volume1/docker/pbs-client"
+
+  tasks:
+    - name: Create PBS client directory
+      ansible.builtin.raw: |
+        mkdir -p {{ pbs_client_data_dir }}/state
+      changed_when: true
+
+    - name: Create docker-compose.yml
+      ansible.builtin.raw: |
+        cat > {{ pbs_client_data_dir }}/docker-compose.yml << 'COMPOSE_EOF'
+        services:
+          pbs-client:
+            image: proxmox/proxmox-backup-client:latest
+            container_name: pbs-client
+            restart: unless-stopped
+            hostname: {{ pbs_backup_id }}
+            env_file:
+              - .env
+            volumes:
+              - {{ pbs_client_data_dir }}/state:/root/.proxmox-backup
+        {% for path in pbs_backup_paths %}
+              - "{{ path }}:{{ path }}:ro"
+        {% endfor %}
+            mem_limit: 2g
+            command: ["sleep", "infinity"]
+        COMPOSE_EOF
+      changed_when: true
+
+    - name: Create .env file
+      ansible.builtin.raw: |
+        cat > {{ pbs_client_data_dir }}/.env << 'ENV_EOF'
+        PBS_REPOSITORY={{ pbs_username }}@{{ pbs_server_host }}:{{ pbs_server_port }}:{{ pbs_datastore }}
+        PBS_PASSWORD={{ pbs_password }}
+        ENV_EOF
+        chmod 600 {{ pbs_client_data_dir }}/.env
+      changed_when: true
+
+    - name: Create backup script
+      ansible.builtin.raw: |
+        cat > {{ pbs_client_data_dir }}/backup.sh << 'SCRIPT_EOF'
+        #!/bin/bash
+        set -e
+        source {{ pbs_client_data_dir }}/.env
+
+        BACKUP_SOURCES=""
+        {% for path in pbs_backup_paths %}
+        BACKUP_SOURCES="$BACKUP_SOURCES {{ path | basename }}.pxar:{{ path }}"
+        {% endfor %}
+
+        echo "[$(date '+%Y-%m-%d %H:%M:%S')] Starting backup for {{ pbs_backup_id }}"
+        docker exec pbs-client proxmox-backup-client backup \
+            $BACKUP_SOURCES \
+            --backup-id {{ pbs_backup_id }} \
+            --repository "${PBS_REPOSITORY}"
+
+        echo "[$(date '+%Y-%m-%d %H:%M:%S')] Backup completed, pruning..."
+        docker exec pbs-client proxmox-backup-client prune \
+            --repository "${PBS_REPOSITORY}" \
+            --backup-id {{ pbs_backup_id }} \
+            --keep-last {{ pbs_keep_last | default(3) }} \
+            --keep-daily {{ pbs_keep_daily | default(7) }} \
+            --keep-weekly {{ pbs_keep_weekly | default(4) }} \
+            --keep-monthly {{ pbs_keep_monthly | default(6) }}
+
+        echo "[$(date '+%Y-%m-%d %H:%M:%S')] Done"
+        SCRIPT_EOF
+        chmod 755 {{ pbs_client_data_dir }}/backup.sh
+      changed_when: true
+
+    - name: Start PBS client container
+      ansible.builtin.shell: |
+        export PATH=$PATH:/usr/local/bin
+        cd {{ pbs_client_data_dir }} && docker compose pull && docker compose up -d
+      become: true
+      changed_when: true
+
+    - name: Setup cron job for daily backup at 2 AM (root crontab)
+      ansible.builtin.shell: |
+        # Remove existing entry if present
+        crontab -l 2>/dev/null | grep -v "pbs-client/backup.sh" | crontab - 2>/dev/null || true
+        # Add new entry
+        (crontab -l 2>/dev/null; echo "0 2 * * * {{ pbs_client_data_dir }}/backup.sh >> /var/log/pbs-backup.log 2>&1") | crontab -
+      become: true
+      changed_when: true
+
+    - name: Verify container is running
+      ansible.builtin.shell: |
+        export PATH=$PATH:/usr/local/bin
+        docker ps --filter name=pbs-client --format "{{ '{{' }}.Status{{ '}}' }}"
+      become: true
+      register: container_status
+      changed_when: false
+
+    - name: Show container status
+      ansible.builtin.debug:
+        msg: "PBS client container status: {{ container_status.stdout }}"
@@ -0,0 +1,11 @@
+---
+- name: Install PBS Client on Synology NAS
+  hosts: synology_nas
+  become: true
+  # Variables are set in:
+  # - Role defaults: ansible/roles/pbs-client/defaults/main.yml
+  # - Host overrides: ansible/inventory/host_vars/<hostname>/vars.yml
+  # - Secrets: ansible/inventory/host_vars/<hostname>/vault.yml
+
+  roles:
+    - pbs-client
@@ -0,0 +1,62 @@
+---
+# Setup PostgreSQL monitoring user and exporter
+# Usage: ansible-playbook -i inventory/ plays/setup-postgres-monitoring.yml
+
+- name: Create PostgreSQL monitoring user on primary
+  hosts: authentik-db-1.srv.renner.uno
+  become: yes
+  vars_files:
+    - ../inventory/group_vars/all/vault.yml
+  tasks:
+    - name: Install psycopg2 for Ansible PostgreSQL modules
+      apt:
+        name: python3-psycopg2
+        state: present
+
+    - name: Create postgres_exporter user
+      become_user: postgres
+      community.postgresql.postgresql_user:
+        name: postgres_exporter
+        password: "{{ vault_postgres_exporter_password }}"
+        role_attr_flags: "NOSUPERUSER,NOCREATEDB,NOCREATEROLE"
+        state: present
+
+    - name: Grant pg_monitor role to exporter user
+      become_user: postgres
+      community.postgresql.postgresql_privs:
+        database: postgres
+        type: group
+        objs: pg_monitor
+        roles: postgres_exporter
+        grant_option: no
+
+    - name: Grant connect on authentik database
+      become_user: postgres
+      community.postgresql.postgresql_privs:
+        database: authentik
+        type: database
+        privs: CONNECT
+        roles: postgres_exporter
+
+- name: Deploy postgres-exporter to Prometheus server
+  hosts: prometheus.srv.renner.uno
+  become: yes
+  vars_files:
+    - ../inventory/group_vars/all/vault.yml
+  vars:
+    postgres_exporter_db_host: "10.2.40.135"  # PostgreSQL HA VIP
+    postgres_exporter_db_user: "postgres_exporter"
+    postgres_exporter_db_password: "{{ vault_postgres_exporter_password }}"
+    postgres_exporter_db_name: "postgres"
+  roles:
+    - postgres-exporter
+
+  post_tasks:
+    - name: Verify postgres-exporter is running
+      uri:
+        url: "http://127.0.0.1:9187/metrics"
+        status_code: 200
+      register: exporter_health
+      retries: 3
+      delay: 5
+      until: exporter_health.status == 200
@@ -0,0 +1,33 @@
+---
+# Deploy monitoring agents to Proxmox VE nodes
+# Usage: ansible-playbook -i inventory/ plays/setup-pve-monitoring.yml
+
+- name: Deploy Monitoring Agents to Proxmox Nodes
+  hosts: pve_cluster
+  become: yes
+  gather_facts: yes
+
+  vars:
+    # Monitoring endpoints
+    loki_endpoint: "http://loki.srv.renner.uno:3100"
+    prometheus_remote_write_endpoint: "http://prometheus.srv.renner.uno:9090/api/v1/write"
+
+    # PVE-specific labels
+    alloy_extra_labels:
+      environment: "production"
+      datacenter: "homelab"
+      type: "hypervisor"
+
+  roles:
+    - monitoring-agent
+
+  post_tasks:
+    - name: Verify Alloy agent is running
+      systemd:
+        name: alloy
+        state: started
+      register: alloy_status
+
+    - name: Display agent status
+      debug:
+        msg: "Monitoring agent deployed to {{ inventory_hostname }}"
@@ -0,0 +1,7 @@
+---
+- name: Install HashiCorp Vault
+  hosts: vault.srv.renner.uno
+  become: true
+  roles:
+    - common
+    - vault
@@ -1,250 +0,0 @@
---
-# Rotate the NetBird shared secrets that Ansible owns BOTH ends of.
-#
-# Deliberately opt-in. Rotation is not idempotent and it interrupts the mesh
-# briefly, so this play refuses to run unless invoked explicitly:
-#
-#   ansible-playbook plays/rotate-netbird-secrets.yml -e rotate_confirm=yes
-#
-# ONE-TIME MIGRATION REQUIRED BEFORE FIRST USE. The three keys below currently
-# live in group_vars/all/vault.yml. They must be deleted from there, because
-# group_vars files merge alphabetically and vault.yml sorts after
-# netbird-rotatable.yml — leaving them would let a later ordinary deploy
-# overwrite a freshly rotated secret with the old one. The play checks for this
-# and refuses to run until it is done:
-#
-#   ansible-vault edit inventory/group_vars/all/vault.yml
-#   # delete vault_netbird_turn_secret, vault_netbird_relay_secret,
-#   #        and netbird_turn_secret
-#
-# That last one is easy to miss and breaks the rotation silently. vault.yml
-# carried BOTH vault_netbird_turn_secret and a bare netbird_turn_secret. The
-# role default is `netbird_turn_secret: "{{ vault_netbird_turn_secret }}"`, but
-# a group var outranks a role default — so the bare key wins, and rotating only
-# the vault_-prefixed one changes nothing that reaches the template. The check
-# below matches both spellings for that reason.
-#
-# The first rotation then populates netbird-rotatable.yml with fresh values, so
-# nothing needs copying across.
-#
-# WHAT IS ROTATED, and why each is safe to rotate from here:
-#
-#   vault_netbird_turn_secret   coturn's `user=netbird:<secret>` in
-#                               turnserver.conf and the management server's
-#                               Turn.Secret/Password in management.json. Both
-#                               templates, one role, one host — the two ends
-#                               cannot disagree across a run.
-#   vault_netbird_relay_secret  management.json on the server and relay.env on
-#                               the proxies. Two hosts, both managed here.
-#
-# The test for "safe to rotate here" is whether the value appears in
-# management.json.j2. If the management server does not read it from a file
-# Ansible writes, Ansible does not own both ends, whatever else it templates.
-#
-# WHAT IS NOT, and must not be added:
-#
-#   vault_netbird_proxy_token    LOOKS symmetric — it is in proxy.env on both
-#                                the server and the proxies, all Ansible
-#                                templates — but the management server does not
-#                                validate it from any of them. It is registered
-#                                in the management DATABASE. Rotating it changes
-#                                only what the proxies present, so every proxy
-#                                is locked out: "proxy auth failed: too many
-#                                failed authentication attempts", and the
-#                                proxies stop receiving their domain
-#                                configuration. Learned by doing it.
-#
-#   vault_netbird_datastore_key  encrypts NetBird's datastore at rest. Changing
-#                                it without re-encrypting the store makes the
-#                                existing data unreadable. Rotating this needs a
-#                                dump/restore, not a template rewrite.
-#   vault_netbird_auth_client_secret  the other half lives in Authentik. Ansible
-#                                only writes the consumer side, so rotating here
-#                                breaks login until Authentik is changed too.
-#
-# These are Ansible's OWN credentials, not a copy of the cluster's. OpenBao
-# carries credentials for some of the same systems (see
-# kubernetes/infrastructure/openbao/rotation/), and those are deliberately DIFFERENT
-# values: one consumer, one credential, independently revocable. Rotating here
-# must not touch OpenBao and does not invalidate anything the cluster holds —
-# that is the point of them being distinct, not drift to be reconciled.
-#
-# Corollary: if a credential ever has to be shared between the two, that is the
-# bug. Mint a second one instead.
-#
-# There IS a brief interruption. The server is reconfigured before the proxies,
-# so relay/proxy auth fails for the seconds between the two plays. Clients
-# reconnect; existing peer-to-peer tunnels are not torn down.
-
- name: Generate and store new NetBird shared secrets
-  hosts: localhost
-  connection: local
-  gather_facts: false
-
-  vars:
-    rotatable_vault: "{{ playbook_dir }}/../inventory/group_vars/all/netbird-rotatable.yml"
-
-  pre_tasks:
-    - name: Require explicit confirmation
-      ansible.builtin.assert:
-        that:
-          - rotate_confirm | default('') == 'yes'
-        fail_msg: >-
-          Refusing to rotate without -e rotate_confirm=yes. This changes live
-          credentials and briefly interrupts the NetBird mesh.
-        success_msg: Confirmed; rotating NetBird shared secrets.
-
-    - name: Verify the vault password is available before touching anything
-      ansible.builtin.command:
-        cmd: "{{ playbook_dir }}/../vault-password.sh"
-      register: _vault_pw_check
-      changed_when: false
-      no_log: true
-
-    # Files inside group_vars/<group>/ are merged ALPHABETICALLY and the last
-    # one wins. netbird-rotatable.yml sorts before vault.yml, so if vault.yml
-    # still defines these keys it silently overrides everything this play
-    # writes. The rotation itself would still work — the plays below pass the
-    # new values explicitly, and play vars outrank group vars — but the NEXT
-    # ordinary run of install-netbird-server.yml or install-netbird-proxy.yml
-    # would read the stale value from vault.yml and push it back out, breaking
-    # server<->proxy auth. The deploy pipelines run unattended, so that would
-    # land on an unrelated change to those roles.
-    #
-    # Counting matches rather than printing them: the value never reaches
-    # stdout, only the key name does.
-    # Proving the vault is readable FIRST. Without this, a failed decrypt (wrong
-    # password, moved file) sends empty input to grep, which then reports zero
-    # matches — indistinguishable from a clean vault, so the guard below would
-    # wave the rotation through precisely when it knows least.
-    - name: Prove vault.yml is readable before trusting the key check
-      ansible.builtin.shell:
-        cmd: >-
-          set -o pipefail;
-          ansible-vault view
-          --vault-password-file {{ playbook_dir }}/../vault-password.sh
-          {{ playbook_dir }}/../inventory/group_vars/all/vault.yml
-          | wc -l
-        executable: /bin/bash
-      register: _vault_lines
-      changed_when: false
-
-    - name: Check vault.yml does not still define the rotated keys
-      ansible.builtin.shell:
-        cmd: >-
-          ansible-vault view
-          --vault-password-file {{ playbook_dir }}/../vault-password.sh
-          {{ playbook_dir }}/../inventory/group_vars/all/vault.yml
-          | grep -cE '^(vault_)?netbird_(turn_secret|relay_secret):'
-      register: _stale_keys
-      changed_when: false
-      # grep exits 1 on "no matches", which is the good case here.
-      failed_when: false
-
-    - name: Refuse to rotate while vault.yml still owns these keys
-      ansible.builtin.assert:
-        that:
-          - _vault_lines.stdout | int > 0
-          - _stale_keys.stdout | int == 0
-        fail_msg: >-
-          vault.yml still defines {{ _stale_keys.stdout }} of the keys this play
-          rotates. group_vars files merge alphabetically and vault.yml sorts
-          last, so it would override netbird-rotatable.yml on the next deploy
-          and revert the secrets. Remove vault_netbird_turn_secret,
-          netbird_turn_secret and vault_netbird_relay_secret from vault.yml
-          first (one-time migration), then re-run.
-        success_msg: vault.yml does not shadow the rotated keys.
-
-  tasks:
-    # 48 alphanumerics. coturn's shared secret and the relay secret are both
-    # opaque strings; length is the only property that matters.
-    #
-    # ansible.builtin.password, NOT community.general.random_string: the CI
-    # image (ci/images/ansible/Containerfile) installs only community.proxmox
-    # and community.docker, so anything from community.general fails at this
-    # task when the play runs through the project's own tooling — even though
-    # it resolves fine on a workstation that happens to have the collection.
-    #
-    # `/dev/null` as the path makes the lookup generate without persisting;
-    # the value exists only in this fact.
-    - name: Generate replacement secrets
-      ansible.builtin.set_fact:
-        new_turn_secret: "{{ lookup('ansible.builtin.password', '/dev/null length=48 chars=ascii_letters,digits') }}"
-        new_relay_secret: "{{ lookup('ansible.builtin.password', '/dev/null length=48 chars=ascii_letters,digits') }}"
-      no_log: true
-
-    - name: Render the new values to a temporary file
-      ansible.builtin.copy:
-        dest: "{{ rotatable_vault }}.new"
-        mode: "0600"
-        content: |
-          ---
-          # Generated by plays/rotate-netbird-secrets.yml. Do not hand-edit —
-          # the next rotation overwrites this file wholesale.
-          vault_netbird_turn_secret: "{{ new_turn_secret }}"
-          vault_netbird_relay_secret: "{{ new_relay_secret }}"
-      no_log: true
-
-    # Encrypting the whole file rather than inline !vault strings: these three
-    # values are always rewritten together, so per-value encryption buys nothing
-    # and makes the write non-atomic.
-    - name: Encrypt the new file
-      ansible.builtin.command:
-        cmd: >-
-          ansible-vault encrypt
-          --vault-password-file {{ playbook_dir }}/../vault-password.sh
-          {{ rotatable_vault }}.new
-      changed_when: true
-
-    # Move into place only after encryption succeeded, so a failure above never
-    # leaves plaintext where the inventory would load it.
-    - name: Install the encrypted file
-      ansible.builtin.command:
-        cmd: "mv {{ rotatable_vault }}.new {{ rotatable_vault }}"
-      changed_when: true
-
-    - name: Confirm the installed file is encrypted
-      ansible.builtin.command:
-        cmd: "head -1 {{ rotatable_vault }}"
-      register: _vault_header
-      changed_when: false
-      failed_when: "'$ANSIBLE_VAULT' not in _vault_header.stdout"
-
- name: Apply the new secrets to the NetBird server
-  hosts: netbird_server
-  become: true
-
-  vars:
-    netbird_turn_secret: "{{ hostvars['localhost']['new_turn_secret'] }}"
-    netbird_relay_secret: "{{ hostvars['localhost']['new_relay_secret'] }}"
-
-  roles:
-    - netbird-server
-
- name: Apply the new secrets to the NetBird proxies
-  hosts: netbird_proxy
-  become: true
-  # One at a time so at least one relay stays reachable while the other
-  # restarts. The server is already on the new secret at this point, so a
-  # proxy is broken only between its own template write and its restart.
-  serial: 1
-
-  vars:
-    netbird_relay_secret: "{{ hostvars['localhost']['new_relay_secret'] }}"
-
-  roles:
-    - netbird-proxy
-
- name: Report what still needs doing by hand
-  hosts: localhost
-  connection: local
-  gather_facts: false
-
-  tasks:
-    - name: Remaining manual steps
-      ansible.builtin.debug:
-        msg:
-          - "Rotated: turn_secret and relay_secret."
-          - "Commit inventory/group_vars/all/netbird-rotatable.yml — it is encrypted."
-          - "NOT rotated: datastore_key (needs dump/restore), auth_client_secret (Authentik side)."
-          - "OpenBao is unaffected by design — its NetBird credentials are separate ones."
@@ -0,0 +1,10 @@
+---
+- name: Update Authentik HA
+  hosts: authentik_app
+  become: true
+  serial: 1
+  vars:
+    authentik_path: /home/ladmin
+
+  roles:
+    - authentik-update
@@ -0,0 +1,13 @@
+---
+- name: Update Docker Containers
+  hosts: proxmox_docker
+  become: true
+  gather_facts: false
+
+  vars:
+    # Define the root directory where your docker-compose files live.
+    # You can override this per-host in host_vars if they differ.
+    docker_compose_dir: "/opt" 
+
+  roles:
+    - docker-update
@@ -0,0 +1,32 @@
+---
+- name: Update LXC Containers
+  hosts: proxmox_all_lxc:!proxmox_noansible
+  become: true
+  gather_facts: true
+  
+  tasks:
+    - name: Update APT Cache
+      ansible.builtin.apt:
+        update_cache: true
+        cache_valid_time: 3600
+      when: ansible_pkg_mgr == 'apt'
+
+    - name: Upgrade all packages
+      ansible.builtin.apt:
+        upgrade: dist
+      when: ansible_pkg_mgr == 'apt'
+      register: apt_upgrade
+
+    - name: Check if reboot required
+      ansible.builtin.stat:
+        path: /var/run/reboot-required
+      register: reboot_required_file
+      when: ansible_pkg_mgr == 'apt'
+
+    - name: Reboot if required
+      ansible.builtin.reboot:
+        msg: "Rebooting due to system updates"
+      when: 
+        - ansible_pkg_mgr == 'apt'
+        - reboot_required_file.stat.exists
+        - apt_upgrade.changed
@@ -0,0 +1,9 @@
+---
+- name: Proxmox VE Rolling Update
+  hosts: proxmox_nodes  # Replace with your inventory group name
+  become: true
+  serial: 1           # CRITICAL: Only update one node at a time
+  any_errors_fatal: true
+  
+  roles:
+    - pve-update
@@ -0,0 +1,9 @@
+---
+- name: Update Proxmox VMs (Linux)
+  hosts: proxmox_all_qemu:&status_running:!proxmox_noansible:!proxmox_eol
+  become: true
+  serial: 1
+  gather_facts: false # We gather manually after checking connection
+
+  roles:
+    - vm-update
@@ -0,0 +1,27 @@
+---
+- name: Full Infrastructure Update
+  hosts: localhost
+  gather_facts: false
+  tasks:
+    - name: Banner
+      debug:
+        msg: "Starting Full Infrastructure Update Sequence"
+
+- import_playbook: update-vm.yml
+- import_playbook: update-lxc.yml
+- import_playbook: update-authentik.yml
+# Re-render docker-compose templates before pulling updates
+# (immich decommissioned — migrated to in-cluster deployment, see kubernetes/apps/immich/)
+- import_playbook: install-n8n.yml
+- import_playbook: install-grist.yml
+- import_playbook: install-music-assistant.yml
+- import_playbook: update-docker.yml
+- import_playbook: update-pve-rolling.yml
+
+- name: Update Summary
+  hosts: localhost
+  gather_facts: false
+  tasks:
+    - name: Done
+      debug:
+        msg: "Full Infrastructure Update Completed Successfully"
@@ -0,0 +1,18 @@
+---
+authentik_path: /home/ladmin
+authentik_db_host: "authentik-db.renner.uno"
+authentik_db_user: "authentik"
+authentik_db_name: "authentik"
+authentik_db_password: "changeMeInVault"
+authentik_secret_key: "changeMeInVault"
+authentik_tag: "latest" # or "2025.10.x"
+
+authentik_port_http: 9000
+authentik_port_https: 9443
+
+authentik_email_host: "smtp.example.com"
+authentik_email_port: 587
+authentik_email_user: "user"
+authentik_email_password: "password"
+authentik_email_use_tls: true
+authentik_email_from: "authentik@example.com"
@@ -0,0 +1,5 @@
+---
+- name: Restart Authentik
+  community.docker.docker_compose_v2: 
+    project_src: "{{ authentik_path }}"
+    state: restarted
@@ -0,0 +1,42 @@
+---
+- name: Ensure Authentik directory exists
+  ansible.builtin.file:
+    path: "{{ authentik_path }}"
+    state: directory
+    owner: "{{ ansible_user | default('root') }}"
+    group: "{{ ansible_user | default('root') }}"
+
+- name: Get latest Authentik release if no tag specified
+  ansible.builtin.uri:
+    url: https://api.github.com/repos/goauthentik/authentik/releases/latest
+    return_content: true
+  register: github_response
+  when: authentik_tag is not defined
+
+- name: Set target version fact
+  set_fact:
+    authentik_version: "{{ github_response.json.tag_name | regex_replace('^version/', '') if authentik_tag is not defined else authentik_tag }}"
+
+- name: Get Docker group ID
+  ansible.builtin.shell: "getent group docker | cut -d: -f3"
+  register: docker_gid_result
+  changed_when: false
+
+- name: Deploy .env configuration
+  ansible.builtin.template:
+    src: .env.j2
+    dest: "{{ authentik_path }}/.env"
+    mode: '0600'
+  notify: Restart Authentik
+
+- name: Deploy Docker Compose file
+  ansible.builtin.template:
+    src: docker-compose.yml.j2
+    dest: "{{ authentik_path }}/docker-compose.yml"
+  notify: Restart Authentik
+
+- name: Ensure Authentik is started
+  community.docker.docker_compose_v2:
+    project_src: "{{ authentik_path }}"
+    state: present
+    pull: always
@@ -0,0 +1,38 @@
+PG_PASS={{ authentik_db_password }}
+PG_HOST={{ authentik_db_host }}
+PG_USER={{ authentik_db_user }}
+PG_DB={{ authentik_db_name }}
+AUTHENTIK_SECRET_KEY={{ authentik_secret_key }}
+AUTHENTIK_ERROR_REPORTING__ENABLED={{ authentik_error_reporting_enabled | default('true') }}
+AUTHENTIK_EMAIL__HOST={{ authentik_email_host }}
+AUTHENTIK_EMAIL__PORT={{ authentik_email_port }}
+AUTHENTIK_EMAIL__USERNAME={{ authentik_email_username }}
+AUTHENTIK_EMAIL__PASSWORD={{ authentik_email_password }}
+AUTHENTIK_EMAIL__USE_TLS={{ authentik_email_use_tls | lower }}
+AUTHENTIK_EMAIL__USE_SSL={{ authentik_email_use_ssl | lower }}
+AUTHENTIK_EMAIL__TIMEOUT={{ authentik_email_timeout }}
+AUTHENTIK_EMAIL__FROM={{ authentik_email_from }}
+AUTHENTIK_LOG_LEVEL={{ authentik_log_level | default('info') }}
+AUTHENTIK_LISTEN__TRUSTED_PROXY_CIDRS={{ authentik_listen_trusted_proxy_cidrs | default('10.0.0.0/8,172.16.0.0/12') }}
+AUTHENTIK_TAG={{ authentik_tag | default(authentik_version) }}
+AUTHENTIK_ERROR_REPORTING__ENABLED=true
+AUTHENTIK_EMAIL__HOST={{ authentik_email_host }}
+AUTHENTIK_EMAIL__PORT={{ authentik_email_port }}
+AUTHENTIK_EMAIL__USERNAME={{ authentik_email_user }}
+AUTHENTIK_EMAIL__PASSWORD={{ authentik_email_password }}
+AUTHENTIK_EMAIL__USE_TLS={{ authentik_email_use_tls }}
+AUTHENTIK_EMAIL__FROM={{ authentik_email_from }}
+AUTHENTIK_PORT_HTTP={{ authentik_port_http }}
+AUTHENTIK_PORT_HTTPS={{ authentik_port_https }}
+AUTHENTIK_CSRF_TRUSTED_ORIGINS=https://authentik.renner.uno,https://auth.renner.uno,https://authentik-app-1.srv.renner.uno,https://authentik-app-2.srv.renner.uno
+
+# HA specific config:
+# Disable Redis (Use Postgres)
+AUTHENTIK_REDIS__HOST=""
+# Worker needs to use shared storage for media
+# Assuming nfs/glusterfs or just volume syncing?
+# Or maybe just use local volume if media uploads ar rare?
+# For true stateless, external S3 storage is recommended:
+# AUTHENTIK_STORAGE__MEDIA__BACKEND="aws"
+# AUTHENTIK_STORAGE__MEDIA__S3__BUCKET="..."
+# For now, local volume (will not sync across nodes without NFS).
@@ -0,0 +1,41 @@
+services:
+  server:
+    image: ghcr.io/goauthentik/server:${AUTHENTIK_TAG:-latest}
+    restart: unless-stopped
+    command: server
+    env_file:
+      - .env
+    environment:
+      AUTHENTIK_REDIS__HOST: ""
+      AUTHENTIK_POSTGRESQL__HOST: ${PG_HOST}
+      AUTHENTIK_POSTGRESQL__NAME: ${PG_DB}
+      AUTHENTIK_POSTGRESQL__USER: ${PG_USER}
+      AUTHENTIK_POSTGRESQL__PASSWORD: ${PG_PASS}
+      AUTHENTIK_LISTEN__METRICS: "0.0.0.0:9300"
+    volumes:
+      - ./media:/media
+      - ./custom-templates:/templates
+      - ./certs:/certs
+    ports:
+      - "${AUTHENTIK_PORT_HTTP:-9000}:9000"
+      - "${AUTHENTIK_PORT_HTTPS:-9443}:9443"
+      - "9300:9300"
+
+  worker:
+    image: ghcr.io/goauthentik/server:${AUTHENTIK_TAG:-latest}
+    restart: unless-stopped
+    command: worker
+    user: "1000:{{ docker_gid_result.stdout }}"
+    env_file:
+      - .env
+    environment:
+      AUTHENTIK_REDIS__HOST: ""
+      AUTHENTIK_POSTGRESQL__HOST: ${PG_HOST}
+      AUTHENTIK_POSTGRESQL__NAME: ${PG_DB}
+      AUTHENTIK_POSTGRESQL__USER: ${PG_USER}
+      AUTHENTIK_POSTGRESQL__PASSWORD: ${PG_PASS}
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+      - ./media:/media
+      - ./certs:/certs
+      - ./custom-templates:/templates
@@ -0,0 +1,47 @@
+---
+- name: Get latest Authentik release
+  ansible.builtin.uri:
+    url: https://api.github.com/repos/goauthentik/authentik/releases/latest
+    return_content: true
+  register: github_response
+
+- name: Set target version
+  ansible.builtin.set_fact:
+    target_version: "{{ github_response.json.tag_name | regex_replace('^version/', '') }}"
+
+- name: Display target version
+  ansible.builtin.debug:
+    msg: "Updating Authentik to version {{ target_version }}"
+
+- name: Update AUTHENTIK_TAG in .env
+  ansible.builtin.lineinfile:
+    path: "{{ authentik_path | default('/home/ladmin') }}/.env"
+    regexp: '^AUTHENTIK_TAG='
+    line: "AUTHENTIK_TAG={{ target_version }}"
+    create: true
+
+- name: Pull new images
+  ansible.builtin.command:
+    cmd: docker compose pull
+    chdir: "{{ authentik_path | default('/home/ladmin') }}"
+
+- name: Pull Outpost images
+  ansible.builtin.shell: |
+    docker pull ghcr.io/goauthentik/proxy:{{ target_version }}
+    docker pull ghcr.io/goauthentik/ldap:{{ target_version }}
+    docker pull ghcr.io/goauthentik/radius:{{ target_version }}
+  changed_when: true
+
+- name: Apply updates
+  ansible.builtin.command:
+    cmd: docker compose up -d --force-recreate
+    chdir: "{{ authentik_path | default('/home/ladmin') }}"
+
+- name: Restart worker to trigger outpost update
+  ansible.builtin.command: 
+    cmd: docker compose restart worker
+    chdir: "{{ authentik_path | default('/home/ladmin') }}"
+
+- name: Prune unused images
+  ansible.builtin.command:
+    cmd: docker image prune -f
@@ -169,7 +169,8 @@ Merge the PR, then trigger the pipeline (Woodpecker → Run → `PIPELINE ==
 "deploy-claude-workstation"`), or push a change under the role's paths. CI SSHes
 to `10.3.40.60` over the tunnel and applies `common`, `netbird-client`, and this
 role. Also run the umbrella `plays/common.yml` against the new host once, so it
-picks up `qemu-guest-agent` like every other inventory host.
+picks up `qemu-guest-agent` and `monitoring-agent` (Alloy → Loki/Prometheus)
+like every other inventory host.

 ---

@@ -3,8 +3,8 @@
 # Debian 12 VM on the off-site nas-2 (Synology VMM). See README for the manual
 # bootstrap (VM creation, Netbird route, secret material) that precedes this.
 #
-# common.yml already applies `common` and `qemu-guest-agent`
-# to every inventory host, so this role owns only the
+# common.yml already applies `common`, `qemu-guest-agent` and `monitoring-agent`
+# (Alloy -> Loki/Prometheus) to every inventory host, so this role owns only the
 # Claude-specific surface: toolchain, the two credential-isolated users, the
 # read-only creds, notes sync, the unattended timers, and egress lockdown.

@@ -1,56 +0,0 @@
---
-# CrowdSec firewall bouncer on the NetBird proxy hosts.
-#
-# Enforces the decisions the in-cluster CrowdSec already detects, at the point
-# where public traffic actually enters: these hosts terminate TLS for every
-# *.renner.uno name. A drop here never reaches the mesh, the relay or the
-# cluster.
-#
-# Detection stays in the cluster. The reverse-proxy produces no parseable
-# access log and there is no CrowdSec collection for it, while Traefik has a
-# working one — and every scenario in use needs HTTP detail (paths, status
-# codes, user agents) that does not exist at this layer.
-
-# Pinned. A release tarball URL is immutable, and this checksum is of the
-# amd64 asset for exactly this tag.
-crowdsec_bouncer_version: v0.0.36
-crowdsec_bouncer_sha256: f86e4b72693549d99f40a9402abefb894108f047a3fbe6e72fada25ee17ce88b
-
-# The LAPI over the public path these hosts already serve, see
-# kubernetes/infrastructure/crowdsec/lapi-ingress.yaml. The proxies cannot
-# reach it over the mesh: the reverse-proxy container runs non-root, so its
-# embedded NetBird client terminates the tunnel in userspace and the tunnel
-# belongs to that process rather than to the host netns.
-#
-# No ca_cert_path: Traefik serves a Let's Encrypt certificate for this name,
-# and the bouncer adds ca_cert_path to the system pool rather than replacing
-# it — the system pool already trusts LE.
-crowdsec_bouncer_api_url: https://crowdsec.renner.uno/
-crowdsec_bouncer_api_key: "{{ vault_crowdsec_edge_bouncer_key }}"
-
-# Enforcing. The bouncer creates two nftables tables (ip crowdsec, ip6
-# crowdsec6) holding the decisions it pulled, touches no existing chain and
-# sets no chain policy — SSH and :443 stay open unless the source is banned.
-#
-# `dry-run` is the other value: it pulls and logs what it would block without
-# creating a single rule. That is what a new host should be given before it
-# enforces, and what an existing one can be dropped back to, per host through
-# inventory/host_vars, without disarming the rest.
-#
-# Worth knowing before arming anything new: the first bouncer on a path
-# enforces the whole CAPI set at once — around 15k addresses today — so the
-# false-positive window belongs before the switch, not after it. And because
-# this chain sits on the INPUT hook while these hosts run no other firewall,
-# there is no conntrack rule accepting established flows ahead of it: a banned
-# address is dropped even when it is answering our own outbound connection.
-# That is why capi_whitelists.yaml carries the proxies and netbird.renner.uno.
-crowdsec_bouncer_mode: nftables
-
-crowdsec_bouncer_update_frequency: 10s
-crowdsec_bouncer_log_level: info
-
-# Bound to loopback deliberately. These hosts are public and have no metrics
-# collection path at all today — opening :60601 to the internet to gain a
-# scrape target is the wrong trade. Revisit when host metrics exist.
-crowdsec_bouncer_metrics_listen: 127.0.0.1
-crowdsec_bouncer_metrics_port: 60601
@@ -1,10 +0,0 @@
---
-# A restart drops both nftables tables and rebuilds them from the next
-# successful LAPI pull, so there is a short window with no blocklist. That is
-# the documented behaviour of a clean shutdown, not a defect — and it is why
-# retry_initial_connect matters.
- name: Restart crowdsec firewall bouncer
-  ansible.builtin.systemd:
-    name: crowdsec-firewall-bouncer.service
-    state: restarted
-    daemon_reload: true
@@ -1,111 +0,0 @@
---
-# See defaults/main.yml for why this exists and why it starts inert.
-#
-# The static release tarball, not a distribution package: the AUR package
-# `crowdsec-firewall-bouncer-nftables` depends on `crowdsec` and would pull the
-# whole Security Engine onto hosts that only need the bouncer, the group is
-# mixed (proxy-1 Arch, VPS-1 Debian), and the AUR would be a new pattern this
-# repo does not otherwise have. The tarball is the same artefact the AUR
-# package downloads.
-
- name: Ensure nftables is present (the enforcing backend)
-  ansible.builtin.package:
-    name: nftables
-    state: present
-
- name: Ensure the bouncer config directory exists
-  ansible.builtin.file:
-    path: /etc/crowdsec/bouncers
-    state: directory
-    mode: "0755"
-
-# Fetched on the CONTROLLER and pushed, not pulled by the host.
-#
-# proxy-1 is IPv6-only, and neither github.com nor objects.githubusercontent.com
-# publishes an AAAA record — a get_url on the host fails with "Network is
-# unreachable" and no amount of retrying fixes it. Mirroring into
-# gitea.renner.uno is no help either: its public records are the internal LB
-# VIP (10.2.99.86 / fd99::86), so it is unreachable from outside the LAN. And
-# upstream ships no container image for this bouncer, which is expected —
-# it manipulates host netfilter.
-#
-# The CI runner has v4, so it does the fetching. The checksum is verified
-# there, before anything is copied to a host.
- name: Fetch the pinned bouncer release on the controller
-  ansible.builtin.get_url:
-    url: >-
-      https://github.com/crowdsecurity/cs-firewall-bouncer/releases/download/{{
-      crowdsec_bouncer_version }}/crowdsec-firewall-bouncer-linux-amd64.tgz
-    dest: /tmp/crowdsec-firewall-bouncer-{{ crowdsec_bouncer_version }}.tgz
-    checksum: sha256:{{ crowdsec_bouncer_sha256 }}
-    mode: "0644"
-  delegate_to: localhost
-  become: false
-  run_once: true
-
-# Only the download had to move to the controller — the extraction stays on
-# the host. The ci-ansible image ships a tar that ansible rejects ("detected as
-# tar type None. GNU tar required"), while both host distributions have GNU
-# tar. Shipping the archive rather than the unpacked binary also keeps the
-# artefact that the checksum was verified against intact all the way to the
-# host.
- name: Copy the bouncer archive to the host
-  ansible.builtin.copy:
-    src: /tmp/crowdsec-firewall-bouncer-{{ crowdsec_bouncer_version }}.tgz
-    dest: /opt/crowdsec-firewall-bouncer-{{ crowdsec_bouncer_version }}.tgz
-    mode: "0644"
-
-# Only the binary. The tarball also carries install.sh / upgrade.sh /
-# uninstall.sh, which manage systemd and config themselves — that is this
-# role's job, and letting them run would leave two owners for the same files.
- name: Unpack the bouncer binary
-  ansible.builtin.unarchive:
-    src: /opt/crowdsec-firewall-bouncer-{{ crowdsec_bouncer_version }}.tgz
-    dest: /opt
-    remote_src: true
-    include:
-      - crowdsec-firewall-bouncer-{{ crowdsec_bouncer_version }}/crowdsec-firewall-bouncer
-    creates: /opt/crowdsec-firewall-bouncer-{{ crowdsec_bouncer_version }}/crowdsec-firewall-bouncer
-
- name: Install the bouncer binary
-  ansible.builtin.copy:
-    src: /opt/crowdsec-firewall-bouncer-{{ crowdsec_bouncer_version }}/crowdsec-firewall-bouncer
-    dest: /usr/local/bin/crowdsec-firewall-bouncer
-    remote_src: true
-    mode: "0755"
-  notify: Restart crowdsec firewall bouncer
-
-# 0600: this file carries the LAPI key.
- name: Deploy the bouncer configuration
-  ansible.builtin.template:
-    src: crowdsec-firewall-bouncer.yaml.j2
-    dest: /etc/crowdsec/bouncers/crowdsec-firewall-bouncer.yaml
-    mode: "0600"
-  notify: Restart crowdsec firewall bouncer
-
- name: Deploy the bouncer systemd unit
-  ansible.builtin.template:
-    src: crowdsec-firewall-bouncer.service.j2
-    dest: /etc/systemd/system/crowdsec-firewall-bouncer.service
-    mode: "0644"
-  notify: Restart crowdsec firewall bouncer
-
- name: Enable and start the bouncer
-  ansible.builtin.systemd:
-    name: crowdsec-firewall-bouncer.service
-    enabled: true
-    state: started
-    daemon_reload: true
-
- name: Report how to read it
-  ansible.builtin.debug:
-    msg: |
-      crowdsec-firewall-bouncer deployed in mode "{{ crowdsec_bouncer_mode }}".
-
-      What it would block:  journalctl -u crowdsec-firewall-bouncer -f
-      What it does block:   nft list table ip crowdsec ; nft list table ip6 crowdsec6
-      Metrics:              curl -s {{ crowdsec_bouncer_metrics_listen }}:{{ crowdsec_bouncer_metrics_port }}/metrics
-
-      Rollback: systemctl stop crowdsec-firewall-bouncer. A clean stop deletes
-      both nftables tables by itself. Only after a kill -9 do they need
-      removing by hand: nft delete table ip crowdsec ; nft delete table ip6 crowdsec6
@@ -1,24 +0,0 @@
-# Managed by Ansible (crowdsec-edge-bouncer role) — do not edit.
-#
-# Upstream's shipped unit orders itself After=crowdsec.service. There is no
-# local Security Engine on these hosts — the LAPI is in the cluster — so that
-# dependency is dropped; nss-lookup instead, because api_url is a public name
-# that has to resolve before the first connect.
-[Unit]
-Description=CrowdSec firewall bouncer (edge enforcement for the NetBird proxy)
-After=network-online.target nss-lookup.target
-Wants=network-online.target
-
-[Service]
-Type=notify
-# -t validates the config and exits non-zero on a bad one, so a broken config
-# fails the start instead of taking the running rules down with it.
-ExecStartPre=/usr/local/bin/crowdsec-firewall-bouncer -c /etc/crowdsec/bouncers/crowdsec-firewall-bouncer.yaml -t
-ExecStart=/usr/local/bin/crowdsec-firewall-bouncer -c /etc/crowdsec/bouncers/crowdsec-firewall-bouncer.yaml
-Restart=always
-RestartSec=10
-LimitNOFILE=65536
-KillMode=mixed
-
-[Install]
-WantedBy=multi-user.target
@@ -1,55 +0,0 @@
-# Managed by Ansible (crowdsec-edge-bouncer role) — do not edit.
-mode: {{ crowdsec_bouncer_mode }}
-update_frequency: {{ crowdsec_bouncer_update_frequency }}
-
-# stdout, so journald owns the log and there is no second rotation policy to
-# keep in sync. The shipped example writes to /var/log; that only matters for
-# hosts without journald.
-log_mode: stdout
-log_level: {{ crowdsec_bouncer_log_level }}
-
-api_url: {{ crowdsec_bouncer_api_url }}
-api_key: {{ crowdsec_bouncer_api_key }}
-
-# Without this the process EXITS when the LAPI is unreachable at startup —
-# there is no retry on the first connect. And because a clean shutdown deletes
-# the nftables tables, the systemd restart loop would leave the host with no
-# blocklist at all until the first successful connect. Present in the code,
-# absent from the shipped example config.
-retry_initial_connect: true
-
-disable_ipv4: false
-disable_ipv6: false
-deny_action: DROP
-
-supported_decisions_types:
-  - ban
-
-# Separate tables per family; the bouncer never touches an existing table or
-# chain, and never sets a chain policy. Priority -10 puts it ahead of anything
-# at the default filter priority.
-nftables:
-  ipv4:
-    enabled: true
-    set-only: false
-    table: crowdsec
-    chain: crowdsec-chain
-    priority: -10
-  ipv6:
-    enabled: true
-    set-only: false
-    table: crowdsec6
-    chain: crowdsec6-chain
-    priority: -10
-
-# input is the traffic this exists for: the reverse-proxy is host-networked and
-# owns :443, so its packets traverse INPUT. forward covers the bridge-networked
-# http-redirect sidecar, whose packets never do.
-nftables_hooks:
-  - input
-  - forward
-
-prometheus:
-  enabled: true
-  listen_addr: {{ crowdsec_bouncer_metrics_listen }}
-  listen_port: {{ crowdsec_bouncer_metrics_port }}
@@ -1,8 +1,8 @@
 ---
 # Journal -> Loki shipping.
 #
-# The srv-network Loki (loki.srv.renner.uno:3100) no longer exists — it moved
-# into the cluster. The only endpoint these hosts can reach is the authenticated
+# The srv-network Loki (loki.srv.renner.uno:3100, no auth) is firewalled off
+# the IoT VLAN; the only endpoint these hosts can reach is the authenticated
 # cluster ingress. edge_loki_password comes from the vault.

 edge_alloy_config_file: /etc/alloy/edge.alloy
@@ -7,8 +7,8 @@
 // Loki. Its own WAL/positions live on /run (tmpfs, see edge-alloy.service),
 // so this agent writes nothing to the eMMC either.
 //
-// The srv-network Loki (loki.srv.renner.uno:3100) no longer exists — it moved
-// into the cluster. The only reachable endpoint is the cluster
+// The srv-network Loki (loki.srv.renner.uno:3100, unauthenticated) is not
+// reachable from the IoT VLAN; the only reachable endpoint is the cluster
 // ingress, which enforces Traefik basic-auth — hence the credential below.

 loki.relabel "journal" {
@@ -0,0 +1,15 @@
+---
+gatus_version: "v5.35.0"
+gatus_port: 8080
+gatus_data_dir: "/opt/gatus"
+
+# SMTP configuration (override in vault)
+gatus_smtp_host: ""
+gatus_smtp_port: 587
+gatus_smtp_from: ""
+gatus_smtp_username: ""
+gatus_smtp_password: ""
+gatus_alert_email: ""
+
+# Webhook URL for outage notifications (override in vault or group_vars)
+gatus_webhook_url: ""
@@ -0,0 +1,5 @@
+---
+- name: Restart Gatus
+  community.docker.docker_compose_v2:
+    project_src: "{{ gatus_data_dir }}"
+    state: restarted
@@ -0,0 +1,29 @@
+---
+- name: Install required system packages
+  ansible.builtin.apt:
+    name:
+      - python3-docker
+    state: present
+    update_cache: true
+
+- name: Create Gatus data directory
+  ansible.builtin.file:
+    path: "{{ gatus_data_dir }}"
+    state: directory
+    mode: '0755'
+
+- name: Copy Gatus configuration files
+  ansible.builtin.template:
+    src: "{{ item.src }}"
+    dest: "{{ gatus_data_dir }}/{{ item.dest }}"
+    mode: '0644'
+  loop:
+    - { src: 'config.yaml.j2', dest: 'config.yaml' }
+    - { src: 'docker-compose.yml.j2', dest: 'docker-compose.yml' }
+  notify: Restart Gatus
+
+- name: Start Gatus via Docker Compose
+  community.docker.docker_compose_v2:
+    project_src: "{{ gatus_data_dir }}"
+    state: present
+    pull: always
@@ -0,0 +1,282 @@
+storage:
+  type: sqlite
+  path: /data/gatus.db
+
+ui:
+  title: Homelab Status
+  description: Service health monitoring for renner.uno
+
+{% if gatus_smtp_host %}
+alerting:
+  email:
+    host: {{ gatus_smtp_host }}
+    port: {{ gatus_smtp_port }}
+    from: {{ gatus_smtp_from }}
+    username: {{ gatus_smtp_username }}
+    password: {{ gatus_smtp_password }}
+    to: {{ gatus_alert_email }}
+    default-alert:
+      enabled: true
+      failure-threshold: 3
+      success-threshold: 2
+      send-on-resolved: true
+{% if gatus_webhook_url %}
+  custom:
+    url: {{ gatus_webhook_url }}
+    method: POST
+    body: |
+      {
+        "service": "[ALERT_TRIGGERED_OR_RESOLVED]",
+        "description": "[ENDPOINT_NAME] - [RESULT_ERRORS]",
+        "status": "[ALERT_TRIGGERED_OR_RESOLVED]"
+      }
+    default-alert:
+      enabled: true
+      failure-threshold: 3
+      success-threshold: 2
+      send-on-resolved: true
+{% endif %}
+{% endif %}
+
+endpoints:
+  # Web Services
+  - name: ownCloud
+    group: services
+    url: https://owncloud.renner.uno/ocs/v2.php/cloud/capabilities?format=json
+    interval: 60s
+    conditions:
+      - "[STATUS] == 200"
+      - "[RESPONSE_TIME] < 2000"
+      - "[BODY].ocs.meta.status == ok"
+    alerts:
+      - type: email
+{% if gatus_webhook_url %}
+      - type: custom
+{% endif %}
+
+  - name: ONLYOFFICE
+    group: services
+    url: https://office.renner.uno/healthcheck
+    interval: 60s
+    conditions:
+      - "[STATUS] == 200"
+      - "[BODY] == true"
+    alerts:
+      - type: email
+{% if gatus_webhook_url %}
+      - type: custom
+{% endif %}
+
+  - name: Authentik
+    group: services
+    url: https://auth.renner.uno/-/health/ready/
+    interval: 60s
+    conditions:
+      - "[STATUS] == 200"
+      - "[RESPONSE_TIME] < 2000"
+    alerts:
+      - type: email
+{% if gatus_webhook_url %}
+      - type: custom
+{% endif %}
+
+  - name: Gitea
+    group: services
+    url: https://gitea.renner.uno/api/v1/settings/api
+    interval: 60s
+    conditions:
+      - "[STATUS] == 200"
+      - "[RESPONSE_TIME] < 2000"
+    alerts:
+      - type: email
+{% if gatus_webhook_url %}
+      - type: custom
+{% endif %}
+
+  - name: Immich
+    group: services
+    url: https://immich.renner.uno/api/server/ping
+    interval: 60s
+    conditions:
+      - "[STATUS] == 200"
+      - "[BODY].res == pong"
+    alerts:
+      - type: email
+{% if gatus_webhook_url %}
+      - type: custom
+{% endif %}
+
+  - name: Woodpecker CI
+    group: services
+    url: https://ci.renner.uno/healthz
+    interval: 60s
+    conditions:
+      - "[STATUS] == any(200, 204)"
+    alerts:
+      - type: email
+{% if gatus_webhook_url %}
+      - type: custom
+{% endif %}
+
+  - name: Grafana
+    group: services
+    url: https://dash.renner.uno/api/health
+    interval: 60s
+    conditions:
+      - "[STATUS] == 200"
+      - "[BODY].database == ok"
+    alerts:
+      - type: email
+{% if gatus_webhook_url %}
+      - type: custom
+{% endif %}
+
+  - name: Home Assistant
+    group: services
+    url: https://home.renner.uno
+    interval: 60s
+    conditions:
+      - "[STATUS] == any(200, 301, 302)"
+    alerts:
+      - type: email
+{% if gatus_webhook_url %}
+      - type: custom
+{% endif %}
+
+  - name: Paperless
+    group: services
+    url: https://file.renner.uno
+    interval: 60s
+    conditions:
+      - "[STATUS] == any(200, 301, 302)"
+    alerts:
+      - type: email
+{% if gatus_webhook_url %}
+      - type: custom
+{% endif %}
+
+  - name: Audiobookshelf
+    group: services
+    url: https://audiobooks.renner.uno/healthcheck
+    interval: 60s
+    conditions:
+      - "[STATUS] == 200"
+    alerts:
+      - type: email
+{% if gatus_webhook_url %}
+      - type: custom
+{% endif %}
+
+  - name: Jellyfin
+    group: services
+    url: https://media.renner.uno/health
+    interval: 60s
+    conditions:
+      - "[STATUS] == 200"
+    alerts:
+      - type: email
+{% if gatus_webhook_url %}
+      - type: custom
+{% endif %}
+
+  - name: Wiki
+    group: services
+    url: https://wiki.renner.uno
+    interval: 60s
+    conditions:
+      - "[STATUS] == 200"
+    alerts:
+      - type: email
+{% if gatus_webhook_url %}
+      - type: custom
+{% endif %}
+
+  - name: Music Assistant
+    group: services
+    url: https://music.renner.uno
+    interval: 60s
+    conditions:
+      - "[STATUS] == any(200, 301, 302)"
+    alerts:
+      - type: email
+{% if gatus_webhook_url %}
+      - type: custom
+{% endif %}
+
+  - name: InfluxDB
+    group: services
+    url: https://influx.renner.uno/health
+    interval: 60s
+    conditions:
+      - "[STATUS] == 200"
+    alerts:
+      - type: email
+{% if gatus_webhook_url %}
+      - type: custom
+{% endif %}
+
+  - name: PBS (pbs.obg.renner.uno)
+    group: infrastructure
+    url: https://pbs.obg.renner.uno:8007
+    interval: 60s
+    client:
+      insecure-skip-verify: true
+    conditions:
+      - "[STATUS] == any(200, 301, 302)"
+
+  - name: PBS 2 (pbs2.renner.uno)
+    group: infrastructure
+    url: https://pbs2.renner.uno:8007
+    interval: 60s
+    client:
+      insecure-skip-verify: true
+    conditions:
+      - "[STATUS] == any(200, 301, 302)"
+    alerts:
+      - type: email
+{% if gatus_webhook_url %}
+      - type: custom
+{% endif %}
+
+  - name: Proxmox VE
+    group: infrastructure
+    url: https://pve.renner.uno
+    interval: 60s
+    client:
+      insecure-skip-verify: true
+    conditions:
+      - "[STATUS] == any(200, 301, 302)"
+    alerts:
+      - type: email
+{% if gatus_webhook_url %}
+      - type: custom
+{% endif %}
+
+  # Infrastructure
+  - name: Proxmox DNS (VIP)
+    group: infrastructure
+    url: 10.2.40.53
+    interval: 30s
+    dns:
+      query-name: k8s-1.srv.renner.uno
+      query-type: A
+    conditions:
+      - "[DNS_RCODE] == NOERROR"
+      - "[RESPONSE_TIME] < 500"
+    alerts:
+      - type: email
+{% if gatus_webhook_url %}
+      - type: custom
+{% endif %}
+
+  - name: PostgreSQL HA (VIP)
+    group: infrastructure
+    url: tcp://10.2.40.135:5432
+    interval: 30s
+    conditions:
+      - "[CONNECTED] == true"
+    alerts:
+      - type: email
+{% if gatus_webhook_url %}
+      - type: custom
+{% endif %}
@@ -0,0 +1,17 @@
+services:
+  gatus:
+    container_name: gatus
+    image: twinproduction/gatus:{{ gatus_version }}
+    ports:
+      - "{{ gatus_port }}:8080"
+    volumes:
+      - {{ gatus_data_dir }}/config.yaml:/config/config.yaml:ro
+      - {{ gatus_data_dir }}/data:/data
+      - /etc/localtime:/etc/localtime:ro
+    restart: always
+    healthcheck:
+      test: ["CMD", "wget", "-q", "--spider", "http://localhost:8080/health"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 10s
@@ -0,0 +1,28 @@
+---
+# Grafana Authentik OAuth Configuration
+
+# Authentik OAuth settings (must be set in vault)
+# Add to ansible/inventory/group_vars/all/vault.yml:
+#   vault_grafana_authentik_client_id: "<client-id>"
+#   vault_grafana_authentik_client_secret: "<client-secret>"
+grafana_authentik_client_id: "{{ vault_grafana_authentik_client_id | default('') }}"
+grafana_authentik_client_secret: "{{ vault_grafana_authentik_client_secret | default('') }}"
+
+# Authentik URLs
+authentik_base_url: "https://auth.renner.uno"
+grafana_authentik_auth_url: "{{ authentik_base_url }}/application/o/authorize/"
+grafana_authentik_token_url: "{{ authentik_base_url }}/application/o/token/"
+grafana_authentik_api_url: "{{ authentik_base_url }}/application/o/userinfo/"
+
+# Grafana settings
+grafana_root_url: "https://dash.renner.uno"
+grafana_config_dir: "/etc/grafana"
+
+# Role mapping from Authentik groups
+grafana_role_attribute_path: "contains(groups, 'Grafana Admins') && 'Admin' || contains(groups, 'Grafana Editors') && 'Editor' || 'Viewer'"
+
+# Allow users to sign up automatically
+grafana_oauth_allow_sign_up: true
+
+# Auto login (skip Grafana login page, go directly to Authentik)
+grafana_oauth_auto_login: false
@@ -0,0 +1,5 @@
+---
+- name: Restart grafana
+  systemd:
+    name: grafana-server
+    state: restarted
@@ -0,0 +1,48 @@
+---
+# Configure Grafana to use Authentik for OAuth authentication
+
+- name: Verify Authentik credentials are configured
+  assert:
+    that:
+      - grafana_authentik_client_id | length > 0
+      - grafana_authentik_client_secret | length > 0
+    fail_msg: |
+      Authentik OAuth credentials not configured!
+
+      Please add to ansible/inventory/group_vars/all/vault.yml:
+        vault_grafana_authentik_client_id: "<client-id-from-authentik>"
+        vault_grafana_authentik_client_secret: "<client-secret-from-authentik>"
+
+      To create the OAuth application in Authentik:
+      1. Go to https://auth.renner.uno/if/admin/
+      2. Applications > Providers > Create
+      3. Select "OAuth2/OpenID Provider"
+      4. Name: Grafana, Slug: grafana
+      5. Client type: Confidential
+      6. Redirect URIs: https://grafana.renner.uno/login/generic_oauth
+      6. Redirect URIs: https://dash.renner.uno/login/generic_oauth
+      7. Copy Client ID and Secret to vault
+
+- name: Ensure Grafana provisioning directory exists
+  file:
+    path: "{{ grafana_config_dir }}/provisioning"
+    state: directory
+    owner: grafana
+    group: grafana
+    mode: '0755'
+
+- name: Deploy Authentik OAuth configuration
+  template:
+    src: grafana-auth.ini.j2
+    dest: "{{ grafana_config_dir }}/grafana.ini"
+    owner: root
+    group: grafana
+    mode: '0640'
+    backup: yes
+  notify: Restart grafana
+
+- name: Ensure Grafana service is running
+  systemd:
+    name: grafana-server
+    state: started
+    enabled: yes
@@ -0,0 +1,26 @@
+# Grafana Authentik OAuth Configuration
+# Managed by Ansible - Do not edit manually
+
+[server]
+root_url = {{ grafana_root_url }}
+
+[auth]
+# Disable login form if using auto_login
+disable_login_form = {{ grafana_oauth_auto_login | lower }}
+
+[auth.generic_oauth]
+enabled = true
+name = Authentik
+icon = signin
+allow_sign_up = {{ grafana_oauth_allow_sign_up | lower }}
+auto_login = {{ grafana_oauth_auto_login | lower }}
+client_id = {{ grafana_authentik_client_id }}
+client_secret = {{ grafana_authentik_client_secret }}
+scopes = openid profile email
+auth_url = {{ grafana_authentik_auth_url }}
+token_url = {{ grafana_authentik_token_url }}
+api_url = {{ grafana_authentik_api_url }}
+role_attribute_path = {{ grafana_role_attribute_path }}
+role_attribute_strict = false
+allow_assign_grafana_admin = true
+use_pkce = true
@@ -0,0 +1,27 @@
+---
+# Grist configuration
+grist_version: "latest"
+grist_port: 8484
+grist_data_dir: "/opt/grist"
+
+# Public URL (required for proper link generation)
+grist_public_url: "https://grist.renner.uno"
+
+# Session secret (override in vault!)
+grist_session_secret: "CHANGE_ME_IN_VAULT"
+
+# Default admin email
+grist_default_email: "admin@renner.uno"
+
+# Sandbox flavor: gvisor for security, unsandboxed for compatibility
+grist_sandbox_flavor: "gvisor"
+grist_default_locale: "de"
+
+# OIDC Authentication (Authentik)
+grist_oidc_enabled: true
+grist_oidc_issuer: "https://auth.renner.uno/application/o/grist/"
+grist_oidc_client_id: ""
+grist_oidc_client_secret: ""
+grist_oidc_sp_profile_name_attr: "name"
+grist_oidc_sp_profile_email_attr: "email"
+grist_oidc_ignore_email_verified: true
@@ -0,0 +1,5 @@
+---
+- name: Restart Grist
+  community.docker.docker_compose_v2:
+    project_src: "{{ grist_data_dir }}"
+    state: restarted
@@ -0,0 +1,35 @@
+---
+- name: Install required system packages
+  ansible.builtin.apt:
+    name:
+      - python3-docker
+    state: present
+    update_cache: true
+
+- name: Create Grist data directory
+  ansible.builtin.file:
+    path: "{{ grist_data_dir }}"
+    state: directory
+    mode: '0755'
+
+- name: Create Grist persist directory
+  ansible.builtin.file:
+    path: "{{ grist_data_dir }}/persist"
+    state: directory
+    mode: '0755'
+
+- name: Copy Grist configuration files
+  ansible.builtin.template:
+    src: "{{ item.src }}"
+    dest: "{{ grist_data_dir }}/{{ item.dest }}"
+    mode: '0644'
+  loop:
+    - { src: '.env.j2', dest: '.env' }
+    - { src: 'docker-compose.yml.j2', dest: 'docker-compose.yml' }
+  notify: Restart Grist
+
+- name: Start Grist via Docker Compose
+  community.docker.docker_compose_v2:
+    project_src: "{{ grist_data_dir }}"
+    state: present
+    pull: always
@@ -0,0 +1,29 @@
+# Grist Environment Configuration
+# Managed by Ansible - Do not edit manually
+
+# Application URL
+APP_HOME_URL={{ grist_public_url }}
+
+# Security
+GRIST_SESSION_SECRET={{ grist_session_secret }}
+GRIST_SANDBOX_FLAVOR={{ grist_sandbox_flavor }}
+
+# Default admin
+GRIST_DEFAULT_EMAIL={{ grist_default_email }}
+
+{% if grist_oidc_enabled and grist_oidc_client_id %}
+# OIDC Authentication (Authentik)
+GRIST_OIDC_IDP_ENABLED=true
+GRIST_OIDC_IDP_ISSUER={{ grist_oidc_issuer }}
+GRIST_OIDC_IDP_CLIENT_ID={{ grist_oidc_client_id }}
+GRIST_OIDC_IDP_CLIENT_SECRET={{ grist_oidc_client_secret }}
+GRIST_OIDC_SP_PROFILE_NAME_ATTR={{ grist_oidc_sp_profile_name_attr }}
+GRIST_OIDC_SP_PROFILE_EMAIL_ATTR={{ grist_oidc_sp_profile_email_attr }}
+GRIST_OIDC_SP_IGNORE_EMAIL_VERIFIED={{ grist_oidc_ignore_email_verified | lower }}
+{% endif %}
+
+# Locale
+GRIST_DEFAULT_LOCALE={{ grist_default_locale }}
+
+# Telemetry (disabled)
+GRIST_TELEMETRY_LEVEL=off
@@ -0,0 +1,20 @@
+version: "3.8"
+
+services:
+  grist:
+    container_name: grist
+    image: gristlabs/grist:{{ grist_version }}
+    env_file:
+      - .env
+    ports:
+      - "{{ grist_port }}:8484"
+    volumes:
+      - {{ grist_data_dir }}/persist:/persist
+      - /etc/localtime:/etc/localtime:ro
+    restart: always
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8484/status"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 30s
@@ -0,0 +1,20 @@
+---
+- name: Ensure install directory exists
+  file:
+    path: /opt/ical-proxy
+    state: directory
+    mode: '0755'
+
+- name: Checkout ical-proxy repository
+  git:
+    repo: 'https://github.com/konairius/ical-proxy.git'
+    dest: /opt/ical-proxy
+    version: main
+    force: yes
+  register: git_checkout
+
+- name: Deploy ical-proxy with docker compose
+  community.docker.docker_compose_v2:
+    project_src: /opt/ical-proxy
+    state: present
+    build: always
--- a/Show More
+++ b/Show More