fix: [alert] KubePodNotReady (rook-ceph) (#591 )

2026-08-25 22:45:27 +00:00
319 changed files with 12311 additions and 7734 deletions
@@ -32,13 +32,13 @@
 #              mechanism as update-audit.yml):
 #                woodpecker repo cron add --repo k.renner/homelab \
 #                  --name alert-monitor --branch master --schedule '*/15 * * * *'
-#   - deploy : woodpecker-cli pipeline deploy k.renner/homelab <n> alert-monitor
-#              where <n> is an existing pipeline to promote.
+#   - manual : woodpecker pipeline create --branch master \
+#                --var PIPELINE=alert-monitor k.renner/homelab
 when:
  - event: cron
    cron: alert-monitor
-  - event: deployment
-    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "alert-monitor"'
+  - event: manual
+    evaluate: 'PIPELINE == "alert-monitor"'

 steps:
  - name: fetch-alerts
@@ -101,7 +101,7 @@ steps:
        bash scripts/ci/make-diag-kubeconfig.sh

  - name: alert-triage-agent
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.250
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.218
    environment:
      CLAUDE_CODE_OAUTH_TOKEN:
        from_secret: claude_code_oauth_token
@@ -1,17 +1,6 @@
 # Builds the CI tooling images (and the secret-rotator runtime image) with
 # kaniko and pushes them to the Gitea container registry.
 #
-# The kaniko image is NOT Google's. Google archived
-# GoogleContainerTools/kaniko on 2025-06-03, but the original authors forked
-# it to chainguard-dev/kaniko and kept releasing — the pin here is well past
-# the v1.24.0 the Google repo ended on. The fork ships no binaries or images
-# of its own, so these come from gitlab-ci-utils, which builds that fork for
-# amd64 and arm64 under versioned tags. A generic `:debug` tag exists there
-# and is deliberately unused; everything else in this repo is pinned too.
-#
-# The executor path, flags and busybox layout are unchanged from Google's
-# image, which is why the swap touched nothing but the `image:` lines (#552).
-#
 # Credential: ci_images_token — a PAT scoped to `package: read+write` and
 # NOTHING else (verified: 403 on /user/repos, /admin/users and the issues
 # API; 200 on the /v2/ registry endpoints kaniko uses). It replaces the
@@ -31,12 +20,12 @@ when:
      - "ci/images/**"
      - "secret-rotator/**"
      - ".woodpecker/build-ci-images.yml"
-  - event: deployment
-    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "build-ci-images"'
+  - event: manual
+    evaluate: 'PIPELINE == "build-ci-images"'

 steps:
  - name: build-ansible
-    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
+    image: gcr.io/kaniko-project/executor:v1.23.2-debug
    directory: ci/images/ansible
    depends_on: []
    # Only rebuild when THIS image's source (or the pipeline file) changed, or
@@ -44,7 +33,7 @@ steps:
    # secret-rotator commit needlessly rebuilt all four ~1.3G ci-* images and
    # orphaned ~5G of registry layers per run — a main driver of #336.
    when:
-      - event: deployment
+      - event: manual
      - event: push
        path: ["ci/images/ansible/**", ".woodpecker/build-ci-images.yml"]
    environment:
@@ -64,11 +53,11 @@ steps:
          --destination gitea.renner.uno/gitea_admin/ci-ansible:latest

  - name: build-k8s
-    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
+    image: gcr.io/kaniko-project/executor:v1.23.2-debug
    directory: ci/images/k8s
    depends_on: []
    when:
-      - event: deployment
+      - event: manual
      - event: push
        path: ["ci/images/k8s/**", ".woodpecker/build-ci-images.yml"]
    environment:
@@ -88,11 +77,11 @@ steps:
          --destination gitea.renner.uno/gitea_admin/ci-k8s:latest

  - name: build-tofu
-    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
+    image: gcr.io/kaniko-project/executor:v1.23.2-debug
    directory: ci/images/tofu
    depends_on: []
    when:
-      - event: deployment
+      - event: manual
      - event: push
        path: ["ci/images/tofu/**", ".woodpecker/build-ci-images.yml"]
    environment:
@@ -119,11 +108,11 @@ steps:
  # Runtime image, not CI tooling: a static musl binary in a FROM-scratch image,
  # used as the base for secret-rotation CronJobs. Replaces per-job inline Python.
  - name: build-secret-rotator
-    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
+    image: gcr.io/kaniko-project/executor:v1.23.2-debug
    directory: secret-rotator
    depends_on: []
    when:
-      - event: deployment
+      - event: manual
      - event: push
        path: ["secret-rotator/**", ".woodpecker/build-ci-images.yml"]
    environment:
@@ -148,11 +137,11 @@ steps:
  # container, but nothing about it is zigbee-specific. Exists so tools come from
  # the image rather than an `apk add` on every container start.
  - name: build-alpine-jq
-    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
+    image: gcr.io/kaniko-project/executor:v1.23.2-debug
    directory: ci/images/alpine-jq
    depends_on: []
    when:
-      - event: deployment
+      - event: manual
      - event: push
        path: ["ci/images/alpine-jq/**", ".woodpecker/build-ci-images.yml"]
    environment:
@@ -177,11 +166,11 @@ steps:
  # critical path of every node rebuild. The download now happens here, once,
  # where the sha256 is checked — see the Containerfile for the full rationale.
  - name: build-cni-plugins
-    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
+    image: gcr.io/kaniko-project/executor:v1.23.2-debug
    directory: ci/images/cni-plugins
    depends_on: []
    when:
-      - event: deployment
+      - event: manual
      - event: push
        path: ["ci/images/cni-plugins/**", ".woodpecker/build-ci-images.yml"]
    environment:
@@ -206,11 +195,11 @@ steps:
  # every paperless pod start. The openssl CLI is genuinely absent from the alpine
  # base, so this one needs an image rather than a busybox substitute.
  - name: build-alpine-openssl
-    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
+    image: gcr.io/kaniko-project/executor:v1.23.2-debug
    directory: ci/images/alpine-openssl
    depends_on: []
    when:
-      - event: deployment
+      - event: manual
      - event: push
        path: ["ci/images/alpine-openssl/**", ".woodpecker/build-ci-images.yml"]
    environment:
@@ -235,11 +224,11 @@ steps:
  # Ansible converge. Not a ci-* image -- it is a runtime artifact -- but it
  # builds the same way and lands in the same registry so the audit sees it.
  - name: build-heating-can
-    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
+    image: gcr.io/kaniko-project/executor:v1.23.2-debug
    directory: ci/images/heating-can
    depends_on: []
    when:
-      - event: deployment
+      - event: manual
      - event: push
        path: ["ci/images/heating-can/**", ".woodpecker/build-ci-images.yml"]
    environment:
@@ -18,7 +18,7 @@
 #                 consumer pins to  ci-claude:<version>  and open a PR. A human
 #                 reviews and merges it; nothing auto-merges.
 #
-# TAG SCHEME — deliberately version tags (ci-claude:<claude-version>), NOT the dated
+# TAG SCHEME — deliberately version tags (ci-claude:2.1.218), NOT the dated
 # tags the other ci-* images use. The tag encodes the thing that changes here
 # (the Claude version), which makes "did it change?" a trivial, race-free
 # string compare and keeps the weekly cron from opening no-op PRs. The other
@@ -54,8 +54,8 @@
 #
 # Triggering:
 #   - cron   : the cron NAMED exactly "ci-claude-refresh" (see above).
-#   - deploy : woodpecker-cli pipeline deploy k.renner/homelab <n> ci-claude-refresh
-#              where <n> is an existing pipeline to promote.
+#   - manual : woodpecker pipeline create --branch master \
+#                --var PIPELINE=ci-claude-refresh k.renner/homelab
 #   - push   : a change to the Containerfile (or this file) ON master forces a
 #              rebuild of the current latest version so Containerfile fixes ship
 #              promptly. Restricted to master ON PURPOSE — a feature branch must
@@ -65,8 +65,8 @@
 when:
  - event: cron
    cron: ci-claude-refresh
-  - event: deployment
-    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "ci-claude-refresh"'
+  - event: manual
+    evaluate: 'PIPELINE == "ci-claude-refresh"'
  - event: push
    branch: master
    path:
@@ -84,24 +84,14 @@ steps:
      - |
        set -eu
        mkdir -p "$$CI_WORKSPACE/.ci-refresh"
-        # Files that pin the image, DISCOVERED rather than listed. This used to
-        # be a hardcoded list carrying the instruction "keep this in sync with
-        # the consumers" — and it was not: e2e.yml and ci-scripts-test.yml were
-        # missing, so they sat on a version no bump could ever reach. A list
-        # that must be maintained by hand is the bug; deriving it removes the
-        # failure mode instead of correcting one instance of it.
-        #
-        # Matches a version-pinned reference only, which is what excludes this
-        # file: its own steps run ci-claude:latest, and :latest never needs
-        # rewriting. That exclusion is load-bearing — do not write a literal
-        # ci-claude version anywhere in THIS file, not even in prose, or it
-        # adds itself to the consumer set and the sed below rewrites it. The
-        # tag-scheme note at the top says <claude-version> for that reason.
-        CONSUMERS=$$(grep -rlE 'ci-claude:[0-9]' .woodpecker)
-        # An empty list would make the grep below read stdin and hang forever
-        # rather than fail, so it is a hard stop.
-        [ -n "$$CONSUMERS" ] || { echo "No ci-claude consumers found — refusing to continue."; exit 1; }
-        echo "consumers: $$CONSUMERS"
+        # Files that pin the image. Keep this list in sync with the consumers.
+        CONSUMERS="
+          .woodpecker/security-scan.yml
+          .woodpecker/alert-monitor.yml
+          .woodpecker/update-audit.yml
+          .woodpecker/pr-review.yml
+          .woodpecker/issue-fix.yml
+        "
        NEW=$$(npm view @anthropic-ai/claude-code version)
        case "$$NEW" in
          [0-9]*.[0-9]*.[0-9]*) : ;;
@@ -140,7 +130,7 @@ steps:
  #    --build-arg to the resolved version (not "latest") guarantees the tag and
  #    the image content agree even if a new release lands mid-run.
  - name: build
-    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
+    image: gcr.io/kaniko-project/executor:v1.23.2-debug
    directory: ci/images/claude
    depends_on: [decide]
    environment:
@@ -186,10 +176,13 @@ steps:
          echo "Branch $$BRANCH already exists — a PR for $$NEW is already open. Nothing to do."
          exit 0
        fi
-        # Same discovery as the decide step — see the comment there for why this
-        # is not a maintained list.
-        CONSUMERS=$$(grep -rlE 'ci-claude:[0-9]' .woodpecker)
-        [ -n "$$CONSUMERS" ] || { echo "No ci-claude consumers found — refusing to continue."; exit 1; }
+        CONSUMERS="
+          .woodpecker/security-scan.yml
+          .woodpecker/alert-monitor.yml
+          .woodpecker/update-audit.yml
+          .woodpecker/pr-review.yml
+          .woodpecker/issue-fix.yml
+        "
        # Exact-string swap of the current tag for the new one, consumers only.
        for f in $$CONSUMERS; do
          sed -i "s#ci-claude:$$CUR#ci-claude:$$NEW#g" "$$f"
@@ -205,7 +198,7 @@ steps:
        {
          printf 'Automated bump of the ci-claude agent image from `%s` to `%s`.\n\n' "$$CUR" "$$NEW"
          printf 'The refresh pipeline built and pushed `ci-claude:%s` (and `:latest`) from the\n' "$$NEW"
-          printf 'latest published `@anthropic-ai/claude-code`. This rewires the %s pipelines that\n' "$$(printf '%s\n' $$CONSUMERS | grep -c .)"
+          printf 'latest published `@anthropic-ai/claude-code`. This rewires the 5 pipelines that\n'
          printf 'run the image so CI stops running a stale Claude Code.\n\n'
          printf 'Merging is a human decision — review, then merge to roll it out.\n'
        } > /tmp/pr-body.md
@@ -17,6 +17,6 @@ when:

 steps:
  - name: model-selection-tests
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.250
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.218
    commands:
      - sh scripts/ci/test-model-selection.sh
@@ -6,8 +6,8 @@ when:
      - "ansible/roles/claude-workstation/**"
      - "ansible/plays/install-claude-workstation.yml"
      - ".woodpecker/deploy-claude-workstation.yml"
-  - event: deployment
-    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "deploy-claude-workstation"'
+  - event: manual
+    evaluate: 'PIPELINE == "deploy-claude-workstation"'

 timeout: 30

@@ -8,8 +8,8 @@ when:
    path:
      - "opentofu/**"
      - ".woodpecker/deploy-infrastructure.yml"
-  - event: deployment
-    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "deploy-infrastructure"'
+  - event: manual
+    evaluate: 'PIPELINE == "deploy-infrastructure"'

 steps:
  - name: validate-secrets
@@ -84,7 +84,7 @@ steps:
    image: gitea.renner.uno/gitea_admin/ci-tofu:2026-04-29
    directory: opentofu
    when:
-      - event: deployment
+      - event: manual
        evaluate: 'TOFU_APPLY == "true"'
    environment:
      TF_IN_AUTOMATION: "true"
@@ -128,6 +128,42 @@ steps:
      - printf "Host *\n\tStrictHostKeyChecking no\n" > ~/.ssh/config
      - echo "$VAULT_PASS" > /tmp/.vault_pass && chmod 600 /tmp/.vault_pass
      - export ANSIBLE_VAULT_PASSWORD_FILE=/tmp/.vault_pass
+      - |
+        # Ensure Talos image exists on all Proxmox nodes via API.
+        #
+        # Only nodes still running Proxmox belong in this list. jupiter and
+        # saturn run Talos on bare metal now, so the storage API returns
+        # nothing for them and the download-wait loop below would burn its full
+        # 60x10s per node before giving up. Bare-metal nodes boot the
+        # metal-amd64 image from their own installed disk instead, and the
+        # whole step goes away once neptune is converted too.
+        TALOS_VERSION="v1.12.6"
+        TALOS_SCHEMATIC="88e3752db032a08082972bd0adf16a80849e3870c3f95cf5b669936c5699e5ad"
+        TALOS_URL="https://factory.talos.dev/image/$TALOS_SCHEMATIC/$TALOS_VERSION/nocloud-amd64.raw"
+        TALOS_FILENAME="talos-$TALOS_VERSION-$(echo "$TALOS_SCHEMATIC" | cut -c1-12)-nocloud-amd64.raw"
+        API_TOKEN="$TF_VAR_proxmox_user!$TF_VAR_proxmox_token_id=$TF_VAR_proxmox_token_secret"
+        for node in neptune; do
+          EXISTS=$(curl -sk "$TF_VAR_proxmox_api_url/api2/json/nodes/$node/storage/local/content" \
+            -H "Authorization: PVEAPIToken=$API_TOKEN" 2>/dev/null | grep -c "$TALOS_FILENAME" || true)
+          if [ "$EXISTS" -gt 0 ]; then
+            echo "Talos image already exists on $node"
+            continue
+          fi
+          echo "Downloading Talos image to $node..."
+          curl -sk "$TF_VAR_proxmox_api_url/api2/json/nodes/$node/storage/local/download-url" \
+            -H "Authorization: PVEAPIToken=$API_TOKEN" \
+            --data-urlencode "url=$TALOS_URL" \
+            --data-urlencode "content=import" \
+            --data-urlencode "filename=$TALOS_FILENAME" \
+            --max-time 600
+          echo "Waiting for download on $node..."
+          for i in $(seq 1 60); do
+            sleep 10
+            EXISTS=$(curl -sk "$TF_VAR_proxmox_api_url/api2/json/nodes/$node/storage/local/content" \
+              -H "Authorization: PVEAPIToken=$API_TOKEN" 2>/dev/null | grep -c "$TALOS_FILENAME" || true)
+            if [ "$EXISTS" -gt 0 ]; then echo "Download complete on $node"; break; fi
+          done
+        done
      - tofu init -upgrade -backend-config="access_key=$AWS_ACCESS_KEY_ID" -backend-config="secret_key=$AWS_SECRET_ACCESS_KEY"
      - |
        if [ -n "$$TOFU_STATE_RM" ]; then
@@ -7,8 +7,8 @@ when:
      - "ansible/plays/install-netbird-client.yml"
      - "ansible/inventory/group_vars/claude_workstation/**"
      - ".woodpecker/deploy-netbird-client.yml"
-  - event: deployment
-    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "deploy-netbird-client"'
+  - event: manual
+    evaluate: 'PIPELINE == "deploy-netbird-client"'

 timeout: 30

@@ -7,24 +7,10 @@ when:
      # install-netbird-proxy.yml also applies the netbird-proxy-watchdog role,
      # so changes to it must redeploy this host too.
      - "ansible/roles/netbird-proxy-watchdog/**"
-      # install-netbird-proxy.yml also applies the edge bouncer.
-      - "ansible/roles/crowdsec-edge-bouncer/**"
      - "ansible/plays/install-netbird-proxy.yml"
-      # The variables that steer those roles, not just the roles themselves.
-      # Without these two, a change that only touches inventory never deploys
-      # and nothing says so: arming the bouncer lives in host_vars, and its
-      # LAPI key lives in group_vars, so both would sit in git looking applied
-      # while the hosts kept running the previous configuration.
-      #
-      # pve_hosts.yml is deliberately NOT listed. It is the single inventory
-      # file for the whole fleet, so listing it would redeploy the proxies for
-      # any unrelated host edit. The cost is that a change to [netbird_proxy]
-      # group membership needs a manual deployment run.
-      - "ansible/inventory/host_vars/proxy-*.yml"
-      - "ansible/inventory/group_vars/netbird_proxy/**"
      - ".woodpecker/deploy-netbird-proxy.yml"
-  - event: deployment
-    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "deploy-netbird-proxy"'
+  - event: manual
+    evaluate: 'PIPELINE == "deploy-netbird-proxy"'

 timeout: 30

@@ -9,8 +9,8 @@ when:
      - "ansible/roles/netbird-proxy-watchdog/**"
      - "ansible/plays/install-netbird-server.yml"
      - ".woodpecker/deploy-netbird-server.yml"
-  - event: deployment
-    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "deploy-netbird-server"'
+  - event: manual
+    evaluate: 'PIPELINE == "deploy-netbird-server"'

 timeout: 30

@@ -1,6 +1,6 @@
 when:
-  - event: deployment
-    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "e2e-test"'
+  - event: manual
+    evaluate: 'PIPELINE == "e2e-test"'

 steps:
  - name: e2e
@@ -7,8 +7,7 @@
 #                was harmless at daily/weekly cadence — but alert-monitor.yml
 #                fires every 15 minutes, so piggybacking must be opt-in now.
 #                A new cron that SHOULD also run e2e gets its name added here.
-#   - deploy   : `woodpecker-cli pipeline deploy k.renner/homelab <n> e2e`, where
-#                <n> is an existing pipeline to promote.
+#   - manual   : `woodpecker pipeline create --branch master --var PIPELINE=e2e k.renner/homelab`

 when:
  - event: cron
@@ -17,8 +16,8 @@ when:
    cron: update-audit
  - event: cron
    cron: weekly-update
-  - event: deployment
-    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "e2e"'
+  - event: manual
+    evaluate: 'PIPELINE == "e2e"'

 steps:
  - name: model-selection-tests
@@ -26,7 +25,7 @@ steps:
    # fetch-models.sh response mapping. No cluster, no secrets, no network — the
    # ci-claude image just provides sh + jq. Keeps the flag-injection regression
    # test from rotting (it exercises the exact injection shapes).
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.250
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.218
    commands:
      - sh scripts/ci/test-model-selection.sh

@@ -0,0 +1,27 @@
+when:
+  - event: manual
+    evaluate: 'PIPELINE == "install-garage-synology"'
+
+steps:
+  - name: run-playbook
+    image: gitea.renner.uno/gitea_admin/ci-ansible:2026-04-29
+    environment:
+      ANSIBLE_FORCE_COLOR: "true"
+      ANSIBLE_HOST_KEY_CHECKING: "false"
+      SSH_PRIVATE_KEY:
+        from_secret: ssh_private_key
+      VAULT_PASS:
+        from_secret: vault_pass
+      PROXMOX_TOKEN_ID:
+        from_secret: proxmox_token_id
+      PROXMOX_TOKEN_SECRET:
+        from_secret: proxmox_token_secret
+    commands:
+      - mkdir -p ~/.ssh
+      - echo "$SSH_PRIVATE_KEY" | base64 -d > ~/.ssh/id_ed25519
+      - chmod 600 ~/.ssh/id_ed25519
+      - 'printf "Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile /dev/null\n" > ~/.ssh/config'
+      - cd ansible
+      - echo "$$VAULT_PASS" > .vault_pass && chmod 600 .vault_pass
+      - export ANSIBLE_VAULT_PASSWORD_FILE=.vault_pass
+      - ansible-playbook -i inventory/ plays/install-garage-synology.yml -v
@@ -34,13 +34,13 @@
 #   - cron   : a cron NAMED exactly "issue-fix":
 #                woodpecker repo cron add --repo k.renner/homelab \
 #                  --name issue-fix --branch master --schedule '41 * * * *'
-#   - deploy : woodpecker-cli pipeline deploy k.renner/homelab <n> issue-fix
-#              where <n> is an existing pipeline to promote.
+#   - manual : woodpecker pipeline create --branch master \
+#                --var PIPELINE=issue-fix k.renner/homelab
 when:
  - event: cron
    cron: issue-fix
-  - event: deployment
-    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "issue-fix"'
+  - event: manual
+    evaluate: 'PIPELINE == "issue-fix"'

 steps:
  - name: fetch-diag
@@ -62,7 +62,7 @@ steps:
      - bash scripts/ci/make-diag-kubeconfig.sh

  - name: issue-fix-agent
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.250
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.218
    environment:
      CLAUDE_CODE_OAUTH_TOKEN:
        from_secret: claude_code_oauth_token
@@ -53,9 +53,9 @@
 #
 # Triggering: pull_request only — automatic on PR open + push-to-open-PR. To
 # re-review a PR, push a commit (the new head sha clears the dedup marker below
-# and earns a fresh review). There is deliberately no on-demand trigger for a
-# specific PR: Woodpecker exposes injected variables to YAML substitution and
-# `when` filters, NOT to a step's shell environment, so a PR=<n> re-review would
+# and earns a fresh review). There is deliberately no manual PIPELINE trigger:
+# Woodpecker exposes a --var only to YAML variable substitution and `when`
+# filters, NOT to a step's shell environment, so a --var PR=<n> re-review would
 # silently no-op (the step would find no PR number and exit green).
 # (Note: keep literal dollar-brace out of THIS file entirely, even in comments —
 # Woodpecker runs its variable substitution over the whole YAML text, comments
@@ -65,7 +65,7 @@ when:

 steps:
  - name: pr-review-agent
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.250
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.218
    environment:
      CLAUDE_CODE_OAUTH_TOKEN:
        from_secret: claude_code_oauth_token
@@ -1,20 +1,8 @@
-# Trivy scan of the IaC tree and every image referenced from ansible/, with an
-# agent pass that opens a PR for CRITICAL/HIGH findings.
-#
-# Triggers:
-#   - cron       : the named `daily-security-scan` cron, matched by NAME.
-#                  Crons cannot carry variables — `woodpecker-cli repo cron add`
-#                  has no --var — so a variable gate can never match a cron.
-#   - deployment : woodpecker-cli pipeline deploy k.renner/homelab <n> security-scan
-#                  The target rides in pipeline metadata as
-#                  CI_PIPELINE_DEPLOY_TARGET, which the workflow-root filter
-#                  does see, unlike manual-run variables on Woodpecker 3.18
-#                  (#582).
 when:
  - event: cron
-    cron: daily-security-scan
-  - event: deployment
-    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "security-scan"'
+    evaluate: 'PIPELINE == "security-scan"'
+  - event: manual
+    evaluate: 'PIPELINE == "security-scan"'

 steps:
  - name: iac-scan
@@ -40,28 +28,12 @@ steps:
        cat /tmp/images.txt
        echo "---"
        i=0
-        failed=0
-        failed_names=""
        while IFS= read -r img; do
          i=$$((i+1))
          out="trivy-image-$$i.json"
          echo "=== [$$i] $$img -> $$out ==="
-          # Keep going on a bad image so one unpullable reference cannot cost
-          # the scan of every image after it, but record that it happened.
-          if ! trivy image --skip-db-update --scanners vuln --timeout 15m --severity CRITICAL,HIGH --ignore-unfixed --format json -o "$$out" "$$img"; then
-            failed=$$((failed+1))
-            failed_names="$$failed_names $$img"
-          fi
+          trivy image --skip-db-update --scanners vuln --timeout 15m --severity CRITICAL,HIGH --ignore-unfixed --format json -o "$$out" "$$img" || true
        done < /tmp/images.txt
-        echo "--- scanned $$((i - failed)) of $$i images ---"
-        if [ "$$failed" -gt 0 ]; then
-          echo "NOT SCANNED:$$failed_names"
-          exit 1
-        fi
-    # Non-zero above turns this step red without failing the workflow, so the
-    # fix agent still runs on what was scanned. Coverage is reported, not
-    # enforced: an incomplete scan is visible in the step status instead of
-    # being indistinguishable from a clean one.
    failure: ignore

  - name: security-fix-agent
@@ -70,7 +42,7 @@ steps:
    # this step is skipped — otherwise we'd fork-bomb PRs.
    when:
      - branch: master
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.250
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.218
    environment:
      CLAUDE_CODE_OAUTH_TOKEN:
        from_secret: claude_code_oauth_token
@@ -0,0 +1,32 @@
+when:
+  - event: cron
+    evaluate: 'PIPELINE == "update-all"'
+  - event: manual
+    evaluate: 'PIPELINE == "update-all"'
+
+timeout: 120
+
+steps:
+  - name: update-all
+    image: gitea.renner.uno/gitea_admin/ci-ansible:2026-04-29
+    environment:
+      ANSIBLE_FORCE_COLOR: "true"
+      ANSIBLE_HOST_KEY_CHECKING: "false"
+      ANSIBLE_ROLES_PATH: "../roles:./roles"
+      SSH_PRIVATE_KEY:
+        from_secret: ssh_private_key
+      VAULT_PASS:
+        from_secret: vault_pass
+      PROXMOX_TOKEN_ID:
+        from_secret: proxmox_token_id
+      PROXMOX_TOKEN_SECRET:
+        from_secret: proxmox_token_secret
+    commands:
+      - mkdir -p ~/.ssh
+      - echo "$SSH_PRIVATE_KEY" | base64 -d > ~/.ssh/id_ed25519
+      - chmod 600 ~/.ssh/id_ed25519
+      - printf "Host *\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile /dev/null\n" > ~/.ssh/config
+      - cd ansible
+      - echo "$VAULT_PASS" > .vault_pass && chmod 600 .vault_pass
+      - export ANSIBLE_VAULT_PASSWORD_FILE=.vault_pass
+      - ansible-playbook -i inventory/ plays/update.yml -v
@@ -10,14 +10,14 @@
 # report; the pipeline posts it to a single persistent Gitea issue.
 #
 # Triggering:
-#   - cron   : a cron NAMED exactly "update-audit". Crons carry no variables —
-#              `repo cron add` takes name, branch, schedule and enabled — so the
-#              name filter is the only thing that can select a workflow for a
-#              scheduled run:
+#   - cron   : a cron NAMED exactly "update-audit". Unlike the other pipelines
+#              here (which gate on an injected `PIPELINE` var set at cron-create
+#              time in the UI), this uses Woodpecker's documented `cron:` name
+#              filter so the schedule can be created entirely from the CLI:
 #                woodpecker repo cron add --repo k.renner/homelab \
 #                  --name update-audit --branch master --schedule '@weekly'
-#   - deploy : woodpecker-cli pipeline deploy k.renner/homelab <n> update-audit
-#              where <n> is an existing pipeline to promote.
+#   - manual : woodpecker pipeline create --branch master \
+#                --var PIPELINE=update-audit k.renner/homelab
 #
 # NOTE: e2e.yml runs on an explicit list of cron names that includes this one,
 # so a weekly e2e run piggybacks on the audit (it was a bare `event: cron`
@@ -25,12 +25,12 @@
 when:
  - event: cron
    cron: update-audit
-  - event: deployment
-    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "update-audit"'
+  - event: manual
+    evaluate: 'PIPELINE == "update-audit"'

 steps:
  - name: update-audit-agent
-    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.250
+    image: gitea.renner.uno/gitea_admin/ci-claude:2.1.218
    environment:
      CLAUDE_CODE_OAUTH_TOKEN:
        from_secret: claude_code_oauth_token
@@ -170,8 +170,8 @@ role keeps logs off local flash and monitors wear:
  WAL/positions sit on `/run/alloy` (tmpfs), so the agent writes nothing to
  the eMMC either. Net effect: no log writes hit local flash — the trade-off
  is that logs are RAM-only and lost on reboot unless already shipped.
- **Loki endpoint:** the old srv-network Loki
-  (`loki.srv.renner.uno:3100`) no longer exists — Loki moved into the cluster, so Alloy pushes
+- **Loki endpoint:** the unauthenticated srv-network Loki
+  (`loki.srv.renner.uno:3100`) is firewalled off the IoT VLAN, so Alloy pushes
  to the authenticated cluster ingress `https://loki.renner.uno/loki/api/v1/push`
  with basic-auth (`edge_loki_username` / `vault_edge_loki_password`).
  **Delivery only works once the `edge-1` htpasswd line has reconciled into
@@ -0,0 +1,22 @@
+plugin: community.proxmox.proxmox
+user: root@pam
+# token_id: ansible
+# token_secret: provided via PROXMOX_TOKEN_SECRET env var
+url: 'https://jupiter.renner.uno:8006'
+validate_certs: true
+want_facts: true
+want_nodes: false
+keyed_groups:
+    # proxmox_tags_parsed is an example of a fact only returned when 'want_facts=true'
+  - key: proxmox_tags_parsed
+    separator: "_"
+    prefix: proxmox
+  - key: proxmox_status
+    prefix: status
+compose:
+  ansible_host: >-
+    (proxmox_lxc_interfaces | default([]) | selectattr('name', 'equalto', 'eth0') | map(attribute='inet') | list | first | default('') | split('/') | first)
+    if proxmox_vmtype == 'lxc' else
+    (proxmox_agent_interfaces | default([]) | selectattr('name', 'equalto', 'eth0') | map(attribute='ip-addresses') | list | first | list | first | default('') | split('/') | first)
+  ansible_user: "'root' if proxmox_vmtype == 'lxc' else 'ladmin'"
+
@@ -1,13 +0,0 @@
-$ANSIBLE_VAULT;1.1;AES256
-32373134633930343261336362373335623735656634313034393731393661643361346634623439
-3561363963333533313264643036663731656465343162330a333964383462373934303766323533
-63613065313063626261383236396363656565623538666565646661353766643133306334396561
-3961376234623236620a616339326238366538333666316538396465666365373834313064653963
-38623233616633383435373836613438333763343932636239386263306238376137643531373161
-35386437313465643437343832623935353238316565366537646530323233363837656631383962
-32653332333162623966366163633732356637323461613966313263313066643966373335313162
-30306362346561633566316166313164323635356363643034663933363464663562343136646137
-66643536363734316533663032643637356136363237376365613330633961616665323430633135
-36323733633464306330666232663433616533373466633939353061663136666665323432373666
-36393930333435333238306238373931393232393861333334643139306138653336633534626362
-61323038303838396438
@@ -1,29 +1,201 @@
 $ANSIBLE_VAULT;1.1;AES256
-30353730633631376539373334376531353263396339313464373538396231616166653738383161
-6438656132616365646463653464396236313437323537620a363632643061333036353537626461
-34393065613836343536383565303066326133363937386263653961663666643430663139653961
-6462363235343063350a636165376232366636363233376637636231363839343738346134623562
-63663839326435303964313730336436396365343163343366613966323938656263303166326435
-65303931393233623064343763633161353161643635396365316362333137353732346366643162
-36386132386638623333623839373161386330353635346331393735366662393935353264646439
-36386539393432373765396562613033656339646532323864366230393638313161383363636636
-32353266376563393239393734316461313566386233333164623135303132346164353165323338
-62306333363838393236346435303339393639623231353232643966613163313134336265633636
-64366539326639303362393933303465376532626663346431613563353032383037363532646637
-38376266323537343437643635343634636164393230313632663533666434306134366261383230
-66383135303565383763646331383338646339613539623835366438393635363832366134316531
-63636466653239383231616139663231623636376564376234383762383066613030313937616533
-33396135616236643034303236353230376434313365326633303961303232363766633161353434
-64333832623535343733353462313862343934633363616237353331386264386632643538333939
-34636437393934326131363236346636663665613537343134313636623439356665636663613938
-37346634356164343537396363663730306232623964636635333333653738383538343236383535
-66326462373964333234343435336630643930333433393034303439366631386666393330343238
-61336237373236313533333732343064336637326636663433396362346631343165656163313561
-30383331326231383539343639343130323731333432306630353835343338613861376631646264
-66383539616663376237663234313431313064663566313566366664386439316161353135306338
-35373962636537653535333935306564303235356232313463386237353737313736386463396164
-62666165336539316565623634326562356434313833643833353635303361353937666663356263
-35653665323736386263333437666432623332333936373938376637633463653930326466383139
-38336239646135316565356564373731386662383234386433633632393165343331353835623437
-30663138633430356334353832633434633939373337663437326263336537356633313763353337
-64373764666430643433
+32666334326535346466653862343263373365346261313136653366366664633436356536383437
+6666326436366433373163353262383031333264633530390a346539316165633939346663623264
+30653839376635626261323234326539313235623962306639396633343533363561376363363864
+3165323130633061350a346233396530303737316361653337643264383230613635383932636133
+61336561663264393831313364366665636166306437383835303332643934393062623163343861
+31643063373432643531663538666363613331626564376539336435343161333237333036343664
+31636635316631376563326637303337386337313330323838666562303463363361346531613363
+64383062396132373536663832356266383430316465643934636537396466366335636131623339
+34316166323730316463666465613734383339636264306431623535656362633739386362393134
+30343837363361333666616332646131663738663737663434303437336130313133306236313739
+36363731386163353136376230393262653334366239653032613034323333386433666262353364
+30326364343735643739626563393661616238313265633566356639623639383138303934653737
+64356139656163666263346539663733356239386333313763653434343163353436323937663034
+35653236616135383465656536323738346662396366653435383337383233643232656435356335
+63616138666662386337323865356439303961396138613965303465373766343939333933316664
+35356561353366663361306336303339633362623332386330383533336461306139613232306363
+62613731613835336338346236373636343463383433613938393865666165653935353830633966
+61393266623961343665343136323235393764313461306138343334366631313531396663666136
+34636436666238376138633461636438356461643461356139313437393736666338623038353933
+34633032653930623065623639646231346133343231383032663438326164313136613361356232
+38306565353737623532653231303963616336323563663137316139303136623234663432373661
+37333431663233656635373433323461663164303137633831306232626465303761633934336461
+65613534633665663565366634623362303964313862333936383032363137306336636563383435
+35366666313065393463656430303436346334313930616166383861613830373962353536303866
+39636165393861363161633734343838346338303036306639316231313738383831626639303833
+62316336623135336462633439323533383931666563643839383236636134363466383561633363
+64366466336362376234336463383639323165306631633437333639333263373837666565306163
+34666162336166623831353436363831666561376562613732333938353638333134323133346435
+61663261373433333635643761373465653031303236613465643931356433656161373463316331
+34326163633037633737393338313931353130306134663839623332666365373663643330353864
+36366535313032333262373038333161336230303165646133303261333839376432393435613434
+39633830316236653962663464363637333433393665353066653230383339666130653135343332
+32623165386535323832626536316665396564623064633564663437613930313665616163326637
+61363538333761633264396430326539643833313830346565643937656137376464646664313636
+38626165306661353631393031383063646463363031633435373735653838636634316634323738
+38313832376265616631623761393666346133636135666664643661646130306237383134383934
+39643561363962323261343236393630656435653261633933656262623737613130353531386438
+39643335306436636238393734646238653465626232643161316539346435353130633138393863
+63343965346361353036646561636339303063393033633162666639303166303265333233386137
+38643835616131303263366539646130643630366139623032366532653031303336323463326263
+37313566633032313931313565616131376635643030373833323838393633653531653664333032
+33623566333731636462366133653837636238383832353362653230393164343135663062373438
+61613832336633343038373565316431643432626532636435653438383536336166383436393035
+66613037663739616337383136663034643433333933643862396330356533636439666536353338
+30313761633961383337353030333862396164376337393338303565663963633662633134653362
+30346563643537323431313137636364623231376364386364633739313438376663343339333330
+64623663303535393262376437626637316532643139353262393231376135663161663862646666
+65303463393966393633613964353132303863373565353131396435623634336364313864373465
+61376335313830383534643431383966616565346466633633613933643231303732373938313434
+38383731653938656331646239353330666266336566633232353835653239393232366634663165
+31313766643539373133396563633637336133633034633236393336663064626630376632336264
+63346136613165613063316264646561633430613630396263333562366430326432323264383233
+30616631616535666631303838326330363339343463303831376531396164333539386263343334
+64313363626363316266343735313263613038383464396564313336373831393936336237303361
+65346161323961336631366263353161616562633533323036383366613637333832323466326535
+36376661346133383461626461646237373531333532656239333566323433333139633966343465
+36653130646636336534353331623062313863666637306462343139333031313234633636613232
+64383663626566616465623938346531623362383839316361366566336166386461336634303339
+63363931303866376462303634633033386332373564393932616638323764383630643839393630
+38373533636561613131343138383765376332336434303930326630303735313462316666333861
+37346336613339643466636665326466386664376438646666363537306533303036613135303230
+65303733366163363964613135306538393234636262326639613439363063326237306337666564
+32303161373661633231333662303565383134333737343530616631363337396333346261646636
+38663439653435623236366437643334393432383636316664306530303133396561396435313936
+38316637393365343936636438666461326436353636323537343363623337656536613662393333
+32346638636532333266323036623932316265303134633265366634623336386138653165643662
+35396637636136386366663762346236343565623133623336363530653039363166383534313632
+39333162396534376335646664383362366663373531646132643462396131333932653464326532
+39326633616335633030373463303665383261323763303863383566386536646136613362323034
+35313064313533616262666439663266353136616466643562666131366636623166333030346333
+62353539396131666434356238646439326337343239366234383866613065636232613863323931
+38363339623634643433393135396264363835386135303061623964343832323936313536623634
+34396531643839623932663565656535396337353934373462336231356133303335643931323131
+33383030653362323830376435393134666339333036336333376539363033666666396534636131
+64353663663633313966336439643334616332616536336238613162363435323938313636303930
+66306135613735653437366164336237656331346365646265656362316639656162386239326632
+34613431653433306633666632363639306266303361383230316539306161396566346137363133
+30333131666530636264656263303639636161366134633964393834663436326539626264383034
+34636134663833333764613237663233346363643566353166316230396235373261393739316237
+38373466623635333432326531613637303839633033636166633364383964626134386261636536
+64616432343936363664383532323936386136393332653131343533653633323639356234353363
+62613631333864663435616633396537373035646238653565646535306163623262396136356663
+63383537653231633766613231316666303830336334323738626138366434323164643465663034
+64343965363738626638363733646136316338633161383330306363643461643664303734663863
+38656135616165366438623237333161363234356538356466646266316136313832666161626566
+65306234663338323263376437363961353033396532643132336261336662656136656530343532
+61353736626665386437376230633336656663356361626437393932633765336439613665666632
+34363763656164326565633433643836383234363962613535353230356461613133666232396336
+32316264353535353162373833666365633266376634366362646264616161363634363138303832
+30656465326562653430373962346438646138336364363833646165353636346161336336613565
+37303931393865356136333431656264666331313165373037656535333832643662343434396337
+37656263363161396438626437643839633834643265313963653337363463373530333930303163
+34353163333461353763643631303763623034333665373235656531393561363432633433393137
+63333438646565633533353466373930313831316132306662363831623732336435346334323331
+32373736396239613839396535316166386637613435643637323932303239643339613737653761
+63323062333430636161316437383035326531346466336263353761636661326363656631613063
+62623863616533333036643563333463366432386232386436356632323863663862613466613931
+35636538663263336333613664353934623338356262643832356163326161373166633833333838
+36373537363934393137663462633833373364356230356136646639633737636130396231633730
+61356536373064356665646335633435646164623164333964323135343536663162656564653961
+63626432653439373938333039386661363330373136633831656230336237376436313664653633
+61623830326330663235376534313135303566636131373835383534363730616433363536346666
+38326532376338393462363430643334343137613363626132356133363534643336663063623136
+66623261343932373662663134323538666334623764376131623530373065316338653535313165
+35313337303563373737313963303134303435643563616561323963613565646466353162353637
+62623961626630323432323133666636356531316362633331313630393761303030363736643163
+63333565616232366665356535373632626366366532633036323839393266613730373331343765
+39343132643734623162386162353438353165653537303033313439326132343036333739623832
+30316132396532626236633665333162653536633134313339306661613066343937323032376235
+64386161623639636361313865613536373338333931623438366338616338326536383232363564
+30356139373865313432366337646366613361393262333336643535656438383562343763636236
+38616263613834613030363033613839613334323964343436326136626565643838323064313037
+34663964646636363038393134343634663637663030636339613864316165666265383632373637
+32373063616130343762393832663030636334646263626236663730336135376461643134646337
+63346534353264663031336164623330633737333766633662613863343431333138363337326536
+38363937313632633537363261633133636266376534346430643230623663333464393262396564
+37323632653036653336623636646537326636383865383338366138653463336638363965356536
+30373664313863396237613363373236353966303862376561653562666231353930653233383733
+38303261343436393261653436626136353237666633613166623035636466346539363436616364
+36386438666165316137303932346435313736616434643337303038383166386135326132656265
+64633866363532333465623135323761393837623934316561633637613433316334376138343764
+64646262356631373865393363353465643733623535626434666639613965383239343736356231
+31306135666539646234333934326664353137643261613131646364363739333035396437316362
+62343531656234376438376266373932376233643630383836323032633537383961383537376631
+38303062623032636264666534313266363835636132646664366162303430393161313964616633
+32613063393965653438643766373434613237393736643635653465366533363535623932386433
+38653530316364323033383330373433663865393837653466376666633434343062376432633138
+64653365373062643431353533363764666339663334393432353466623062646632383163656365
+63633365326565353730333164303237383766386135376630613234333834626335643738376339
+31626161346562613439396263646361343464636136636663363766356263636630323366326131
+37356166343031376635346433363335373137303933343539303832613363656334663964333462
+31613531383166666261336439356434323263306364616536636333316131373930646262386366
+33333032643464313165336265326130333637323461643131393437326535316164356665313636
+61613237356331393463336431316539633133326334396134663035396531633538666331393132
+64393134643832376230656332633633396333313464333066656461366335343131613038656535
+32353037316132393936373834633535646631653464616331633830326131333361316464393762
+34306136306633323766353235353663666539663036383636393335336263353134303634626131
+31316334663733666530653330333263653135643539353439306165353339623032643633393632
+37653636663164646438626638643630393135623934646166666635316532366264623165616131
+64653736306431623639303564306431616263613966656563373861616339316466383761643439
+64323836303162613931313534393436353464333330353635663034396631636563616464653530
+34616164323362653231343433323338633930323035396630326461666461346338363039383737
+38666431373633376536323739316132316231353738306632336537633938653535316530346634
+65323937336635373736346438323133353333343235623463663264623838316239366165306535
+30323136626561636537393066383539303233366262396538383563353730356537623266623038
+33316135303833323663363733353634653763383166633663313465303838646266303264363339
+30363637396161656332323333653931336661393566663232326466646563356436306332333532
+35656331383661376538633664346537613161303135643937643161383034643532653961346533
+31396237653234363832383437613433326165636262333236393334613437386638636535616634
+34313330643139386232396332376135376332613333633938366533616435303232376366623237
+37323535383932623631363431383165323338306338633066373937323361383336616265303234
+32343132303366343866303837326566383865393764343130616366373536623766316238363430
+31633734633438313065336431316161636465333166653765653962326636663065336532393731
+33386139383466626565663666373664313030663132313233356136613435663265396332656633
+33636436646565643838643930343938326231623738393661393738343062303661353362646465
+31653335343638613238333636326337396438373263663666373263616662653634613036663364
+66323639336233623538663061366462663766383539333432336363613938346263636436336430
+36646639353163316661356636393034653934666563356335396631323166313532353361653630
+61353261656431613135313965373061313038653237626231366332306432633538656462323739
+38313831663861663738336564333731363937653663663031343261633538643535336264306230
+31656365326261633730613765393861376662656137623832363832633266346135643133633331
+39623836633337323934653464333731626361316638333565393435656132626631613739366535
+37653266333432386334333461653231346630646435343230636531326338396566396466376532
+32666632383430393064353861373033393538366266333339373436353932656333366264393936
+39373938346633616534643734313934643438333965323133313037373737303964643337653135
+31623038356263643438343635643530663362656630393164663634366461623464353532353239
+38623662666235383139663162343836316331396266313062316639373034376337393563643739
+65313564303962353061616265393135326266386266316538616430383966306163386533376137
+36393565336435636535376639623931616435353532356438666663383961356638353333366139
+63313962643638323862353738316233613862346532306631386536323261636235343565353563
+64613465343662663034643138623865363862316131363461343636333864656132626166393836
+32393737386166343630323861376562353337663030613263613664663938376337333533613432
+33343064636365643831373135623466613738303164326562386430386339653831313536366532
+32613430313461663662313966373238643934633938633764363365386139303337643533313961
+61646131653638616365346236616330363566636261636362363635323566353562326239393530
+33323934623530306436623963663735353337306162646638383934326530333163396165366139
+61653339363166376266643733653536636639653939346666663834376361373533363231636433
+35363665373031393231363161333036303239313032326134363432363034623762366637643830
+32353132353961343830303562613964376234303461623263343735336238353565353238363438
+61326164323265386431353130346364393466353334386336376139303036646634376331313561
+39386331366337646634366635623066313432623537323164646263323532303465343735326364
+33383162626139383232636362613831303832633562636136393138316334303832383666313963
+37643835663065613236303338353435663830343637326563343665616532633830623035336534
+39376536663937653939326261346263613265636363333531323931646633313837386238333435
+33646339626462386635316230636135626330643630633664636364313539626332643331356334
+32313736343031633631393662363638626137303331383263656335373966383335653133333932
+34393861303665613633366461666664376466383437303065653938323166633562643834356139
+32616564373731313134316661343839333830663338636132323265376133656337393632353931
+35633531383438656461613135323134623134393062356661326435323065616331656638306432
+62313130643763626336383762306535656238393961653537336465373064653432653665373362
+63666366343866613235376535313835623261313531653265373636383030613930353530346265
+64303531316239386630626565323966616535373936393766663063623464663864363338343466
+33666237613532326661626461343166323162366532386638333835663838343865346266646533
+66343533326562396331363137353631633965623561333963613565636264653031616339613232
+39666362643739663837326432623462646233306239326463616232383236346236383132396637
+34623332346533366362613465363263373339643033663264363332336463343337333466393866
+33333363313331376634363431366636376466336131363330323530306162666434393764393563
+35633530353839386362326262343635383434333338626563383463653263333430383465626535
+63363933343365326665343166633939313739316231383938376166646331623663343161623865
+36646630336337373865
@@ -0,0 +1,39 @@
+$ANSIBLE_VAULT;1.1;AES256
+32656264356562366161643464343065313163663461663038646531306532383532393238396566
+3831646133656131336266346531323035313234663336380a356536313965383066363135343739
+32663032373165643931663762333036393430333437366630326562653034636464336166313730
+6666366634346664330a643063343061623465393436656461303862336431393931643638333735
+36663561626134333639326635373735623739366366396564653763656661663866623639323233
+63326130303065663265356136343266613966333230653935343836343461623535363161346439
+64653130383539363933636437643035306536376439326539666662383064636337363265356135
+39623537303165356635373666643634383731333236643061343033376466643762373636333834
+65626435353563623938363566626330356662366666666263363038646334323731393631356531
+37346230303134326135633033393162333839346664363062633736363861343435623331343762
+36643266333965323861633836336234333839306330343961653636623439656432613662303865
+33313438316236653461346264373338393539326438666337346162326163313837626364623461
+37353135323866326536303138363165396339306237393064366335386339303238373563373064
+62646430313131373964346466373164306432363361626661653531356261313233616563353465
+33353661663930333037636637353234663231633339333436393362393363653066376337376364
+61663035396563643665353162356466623032643938353933393930656162386631376132383461
+65383265623136646261393863336635323634336136376534323231383930383632306161393136
+66366563633737613931633233666336363166313930613963396636633761633638323465306332
+65333764343762643133613036353937386338303336656530366433333031323336366165346435
+62626533363961636361306664616638396538646530633730373636643131303636333036326534
+65616361353435346161383366626334386536376630326334353837396161356637356431316364
+31643636326262383362336535346237666465363165643830613932336362623635366138303336
+66376362386531313063336266623033653736616430386635656134616461643238373831333062
+62343964343032366434316365626232656337613832313263353237353336643735353464656262
+66313765393332393663303562393438376139373565623734653432626134376630343863353061
+31383530643264643330313136356638646265663831363637363464636634356639313936336437
+33393961363930616435623864303732633564303937343638373761653761666339373433373432
+39653333396138383332353465396639346535666536373232653536393163353835613239386530
+62666536633636363035613063643965623735366333386335363066326238323835663535333162
+36613366326137303564653463393665666337363664363563643361373766663331333363326533
+36353265616261613232633833616362363433313562303536326537346662383239343965313964
+36353534666535313239316166343163343132323134313266653537646136383262626233376332
+31616166366234653632373732633061373362336137303831353763373166643031313466656434
+64623362663762316465633564633462363866373762383937386362623462333461666462613336
+64393633336137663631386562366339646464363433396366303466336561653938386639393563
+33613936356336346134323830366236326231653465373438623433613634656666386162336331
+32316637383637336266656637343639613233623739303438323033333935663261373230626661
+6635353534313662646665613438336264303835633564303166
@@ -0,0 +1,6 @@
+---
+# n8n group variables
+# Maps vault secrets to role defaults
+
+n8n_encryption_key: "{{ vault_n8n_encryption_key }}"
+n8n_postgres_password: "{{ vault_n8n_postgres_password }}"
@@ -1,23 +0,0 @@
---
-# Bouncer key for crowdsec-edge-bouncer, scoped to the proxy hosts.
-#
-# A single encrypted VALUE rather than a whole-file vault like its siblings in
-# group_vars/all and group_vars/edge_host. Deliberate: adding this variable to
-# an existing whole-file vault means decrypting that file first, which exposes
-# every other secret in it for the sake of appending one line. encrypt_string
-# keeps the blast radius at this one value.
-#
-# Same value as OpenBao homelab/infrastructure/crowdsec-edge-bouncer-key, which
-# the crowdsec-edge-bouncer-key ExternalSecret feeds to BOUNCER_KEY_edge on the
-# LAPI. Both sides must move together — and unlike crowdsec-bouncer-key, this
-# one is NOT in the quarterly rotator, because nothing on these hosts re-reads
-# OpenBao on a schedule. Rotating it means updating OpenBao and re-running
-# plays/install-netbird-proxy.yml.
-vault_crowdsec_edge_bouncer_key: !vault |
-          $ANSIBLE_VAULT;1.1;AES256
-          32653764333536316163303838333839616562663862666637626165303837633866643033393239
-          3363376538656236643161386432326437633232643564630a633635306365323537346139626364
-          36613033343938363366306239363638383466396337623333623136313234656366333366623433
-          6437656632373364340a346566343866366433306533353431383064356362346536383136323663
-          39333864643534326433653233303137373162663066633132623931316135323733333863306330
-          6366623463666435636362376238313639666263323163303733
@@ -0,0 +1,2 @@
+---
+ansible_user: root
@@ -0,0 +1,3 @@
+---
+# Docker hosts - tagged with 'docker' in Proxmox
+docker_monitoring_enabled: true
@@ -0,0 +1,8 @@
+---
+# --- NTP (see plays/install-ntp-ha.yml) ---
+# The PVE hosts hold the 10.2.40.4 VIP via keepalived on the VLAN 40 bridge
+# sub-interface. This role disappears when these nodes are rebuilt as bare-metal
+# Talos, which cannot run chrony or keepalived — edge-1 is being brought into
+# ntp_cluster to take over.
+ntp_keepalived_enabled: true
+ntp_keepalived_interface: "vmbr1.40"
@@ -0,0 +1,20 @@
+$ANSIBLE_VAULT;1.1;AES256
+66356134353363643130323632626530653863306635396432353539623731346138323635336632
+3536633563333231373333323564373763363936356535630a333135623336613832313833626531
+34646137633633353266653536363934303939626564353036343033643462373439643139353065
+6539383436343233300a396339356562363832373962306337663464363961346335656130653235
+38646563656538373530643531643964633934623465643065626266396130643966343166313937
+65663965613937323634323436386130653633656261653065306365316135343531653737663138
+65303339323137653362383766343361643837356531643339613431333238613564343563333761
+30616530613133633762306533363134383265313031343230313764633238626565623935313134
+38323435653264643534366366313862646161373434623135366164323236363733653533656233
+34336233396465626264666637363532646562306237363533616461613764306433366639623562
+30393234396238353038356636373632326130306431663439316363646530636563666364316139
+34326333363536343963393833616465303636323166643638623061353964303265393733366234
+33373435643237636536363364373830353833346235333534626561343465373236643165646664
+39396233363531393835656463666662616633643566383061323161663061323831356337333337
+65653336663735633536353263316634373639313238343365353833646666353131656530373361
+32643133353737366465633635663339666138323035633537653462303532366632626262373532
+32643132363062623033356134356135346361343234656636663936336338303230616665326630
+65353231356538633034626332633831396362373733646539313834386535666631613463656639
+373864663230353339666137653835656630
@@ -0,0 +1,11 @@
+---
+# Loki server - central log aggregation host
+
+# Enable syslog receiver for remote devices (Synology NAS, etc.)
+# Synology DSM's "IETF" format actually sends BSD/RFC 3164 format: <pri>MMM DD HH:MM:SS host app: msg
+# Using raw format to accept messages and parse them with regex
+alloy_syslog_receiver_enabled: true
+alloy_syslog_receiver_port: 1514
+alloy_syslog_receiver_protocol: "udp"
+alloy_syslog_receiver_format: "raw"
+alloy_stability_level: "experimental"
@@ -0,0 +1,37 @@
+---
+# PBS Client configuration for nas-1
+pbs_backup_paths:
+  - /volume1/Documents
+  - /volume1/homes
+  - /volume1/Homeassistant Backup
+  - /volume1/Games
+  - /volume1/Media
+  - /volume1/Music
+  - /volume1/video
+  - /volume2/Backup
+  - /volume1/Movies
+
+  # Add more paths as needed:
+  # - /volume1/photos
+  # - /volume1/music
+
+# PBS Server connection
+pbs_server_host: "pbs2.renner.uno"
+pbs_server_port: 8007
+pbs_datastore: "pool"
+pbs_backup_id: "nas-1"
+
+# PBS Authentication - credentials in vault.yml
+pbs_username: "root@pam!nas-1"
+
+# Garage S3 (K8s backup target)
+garage_image: "dxflrs/garage:v2.3.0"
+garage_zone: "dsm"
+garage_capacity: "2T"
+
+# SSH connection - use password auth (key not supported by DSM)
+ansible_ssh_pass: "{{ nas_ssh_password }}"
+ansible_become_pass: "{{ nas_ssh_password }}"
+
+# Python 3.9+ from Synology package
+ansible_python_interpreter: /usr/local/bin/python3.9
@@ -0,0 +1,27 @@
+$ANSIBLE_VAULT;1.1;AES256
+34663462636437633437613863366266306334666635643839313737343739643864613136613938
+3138393966306561656366396465383539303137333036300a376237376636356230343834386362
+36636535396330333732383233323263613132333166343739396137636538653435336662353138
+6565643866623235610a316266383933303830323936303735623338623165386339383733373435
+62366461363833616438373237383132366438326364363462333265366533323861376630643365
+39343036343838333639363432643738666630363063323636646661313539643465366532363438
+33303837623234613434633563666562346536346435353461373536323737376233386163326537
+61383963343737656438356136613537623538333163343466343739346135333439616432663763
+36663962633037376561326434303035333938646436663832303432346365353439666435613337
+34363636326530333233313231343566306432376532396666643761633136316333376337666135
+65363338623035663438323338663863366362653131336434323865326530326534623631336263
+36633063383031386161363131333636393364623436613331353733313235383062656131313164
+63393665633361313961363637663237616364643236383835393736643937393564656238376362
+31393465383361326264376133393233616264663636383337363962356532613438663637616536
+33653137656335323763313461323633616635346232303062323062333766383931313965346361
+66656461313239633366636463633465636463663937393932636337643861383937663662306466
+32353166643832303234643264633062616330393963333034623432613730366339306637613239
+34626161663666316339323762373639353335653663353635366664643063393432623039306237
+38363161343930363763393364663364353735343236313665313461626165636135303839363164
+37336638396564326166376233353730346563656266323535666265323062646232323938353732
+32636235333839623166633761376334316434613235333730353439343262393537363461333238
+32653435623362666662336532643832356163326332636436346132643237643030343761663332
+36663530356539303636663438646430363335353135323932613863666264363533326137353164
+31663032346236616263343637396439663334396166656364623637666535386530383535663435
+66303037303265616634373763616437333835333136336563323339666563663931396133633131
+3963653266633535303630653335666166313336653831623362
@@ -0,0 +1,8 @@
+---
+# Paperless server specific configuration
+
+# Extra scrape targets for Alloy
+alloy_extra_scrape_targets:
+  - name: "paperless"
+    address: "127.0.0.1:9199"
+    job: "paperless"
@@ -0,0 +1,7 @@
+---
+# Wiki server configuration
+
+# Home Assistant documentation generation (disabled - see HA Wiki Documentation project)
+homeassistant_docs_enabled: false
+homeassistant_url: "http://10.2.30.11:8123"
+homeassistant_token: "{{ vault_homeassistant_token }}"
@@ -1,16 +1,31 @@
-# Hosts are reached by name. An address in an inventory is allocator state
-# standing in for identity, and it pins access to the legacy family -- the names
-# resolve IPv6-first, which is the primary path on this network. DNS is the
-# authoritative source for where a host is; this file says which host, not where
-# it currently sits.
+# Reached by name, like every other host in this file. These three were the last
+# entries still addressed by IPv4 literal; authentik, netbird, proxy, nas-1,
+# grafana and edge-1 have always used their FQDN.
 #
-# Gone from this file, and why:
-#   pve_cluster (jupiter/neptune/saturn)  run Talos on bare metal; no SSH at all
-#   authentik_db / authentik_app          authentik runs in-cluster (CNPG)
-#   grafana                               runs in-cluster
-#   synology_nas (nas-1)                  powered off; Garage and the PBS client
-#                                         it hosted are gone with it
-# Their hosts no longer resolve, except the three that resolve to Talos nodes.
+# An address in an inventory is allocator state standing in for identity, and it
+# pins access to the legacy family -- the names resolve IPv6-first, which is the
+# primary path on this network. DNS is the authoritative source for where a host
+# is; this file should say which host, not where it currently sits.
+#
+# The short inventory_hostname is kept deliberately: plays key on it
+# (ntp_servers[inventory_hostname] in install-ntp-ha.yml, node names in opentofu,
+# the prometheus targets), so only ansible_host changes.
+[pve_cluster]
+jupiter ansible_host=jupiter.renner.uno ansible_user=root
+neptune ansible_host=neptune.renner.uno ansible_user=root
+saturn ansible_host=saturn.renner.uno ansible_user=root
+
+[authentik_db]
+authentik-db-1.srv.renner.uno ansible_user=root
+authentik-db-2.srv.renner.uno ansible_user=root
+
+[authentik:children]
+authentik_db
+authentik_app
+
+[authentik_app]
+authentik-app-1.srv.renner.uno ansible_user=root
+authentik-app-2.srv.renner.uno ansible_user=root

 [netbird_server]
 netbird.renner.uno ansible_user=root
@@ -19,6 +34,13 @@ netbird.renner.uno ansible_user=root
 proxy-1.renner.uno ansible_user=root
 proxy-2.renner.uno ansible_user=root

+
+[synology_nas]
+nas-1.renner.uno ansible_user=ladmin ansible_python_interpreter=/usr/bin/python3
+
+[grafana]
+grafana.srv.renner.uno ansible_user=ladmin
+
 # Every bare-metal edge thin client. Gets the host-generic roles only:
 # flash-wear handling, the log shipper, the MQTT bridge, the VLAN
 # sub-interface. Nothing here assumes any peripheral is attached.
@@ -49,21 +71,12 @@ zwave_gateway
 heating_can
 power_meter

-# NTP servers. edge-1 was added here so the role would survive the PVE hosts
-# being retired for bare-metal Talos, which cannot run chrony or keepalived
-# itself. That retirement has now happened, so edge-1 is the ONLY member.
-#
-# THIS IS NO LONGER HIGHLY AVAILABLE. The group was jupiter (MASTER, prio 100),
-# saturn (90), neptune (80) and edge-1 (BACKUP, 70); the first three are gone.
-# A single VRRP member takes the address regardless of its configured state, so
-# fd40::123 is still served — verified answering — but by one thin client with
-# no failover behind it. install-ntp-ha.yml still carries the four-host
-# ntp_servers priority map, which now describes hosts that do not exist.
-#
-# Restoring redundancy needs a second chrony host that is not a Talos node.
-# See plays/install-ntp-ha.yml for why keepalived is scoped more narrowly than
-# chrony within this group.
+# NTP servers. Includes edge-1 so the role survives the PVE hosts being
+# retired for bare-metal Talos, which cannot run chrony or keepalived itself.
+# See plays/install-ntp-ha.yml for why keepalived is scoped more narrowly
+# than chrony within this group.
 [ntp_cluster:children]
+pve_cluster
 edge_host

 # Off-site Debian VM (Synology VMM on nas-2, 10.3.40.0/24) running Claude Code.
@@ -0,0 +1,48 @@
+---
+# Backup Authentik PostgreSQL database for K8s migration
+# Creates a pg_dump on the primary DB host
+#
+# Usage (via CI):
+#   ansible-playbook -i inventory/ plays/backup-authentik-db.yml
+#
+# Output: /tmp/authentik-backup.sql on the primary DB host
+
+- name: Backup Authentik PostgreSQL database
+  hosts: authentik-db-1.srv.renner.uno
+  become: true
+  tasks:
+    - name: Create database backup
+      become_user: postgres
+      command: >
+        pg_dump
+        --clean --if-exists
+        --no-owner --no-privileges
+        --format=custom
+        --file=/tmp/authentik-backup.dump
+        {{ authentik_db_name | default('authentik') }}
+      changed_when: true
+
+    - name: Make backup readable
+      file:
+        path: /tmp/authentik-backup.dump
+        mode: "0644"
+
+    - name: Get backup size
+      stat:
+        path: /tmp/authentik-backup.dump
+      register: backup_stat
+
+    - name: Report backup size
+      debug:
+        msg: "Backup created: {{ (backup_stat.stat.size / 1024 / 1024) | round(1) }} MB"
+
+    - name: Fetch backup to controller
+      fetch:
+        src: /tmp/authentik-backup.dump
+        dest: /tmp/authentik-backup.dump
+        flat: true
+
+    - name: Clean up remote backup
+      file:
+        path: /tmp/authentik-backup.dump
+        state: absent
@@ -0,0 +1,59 @@
+---
+# Dump Paperless-ngx DB from the legacy VM to a local file on the
+# controller (CI runner), where a downstream pipeline step consumes it
+# to produce a Barman seed for the CNPG `bootstrap.recovery` path.
+# Data + media tarballs are handled separately by `backup-paperless.yml`.
+
+- name: Dump Paperless DB from legacy VM to CI runner
+  hosts: paperless.srv.renner.uno
+  become: true
+  vars:
+    paperless_db_name: paperless
+    paperless_db_user: paperless
+  tasks:
+    - name: Find running paperless DB container
+      ansible.builtin.shell: >
+        docker ps --filter "label=com.docker.compose.service=db"
+        --format '{{ "{{" }}.Names{{ "}}" }}' | head -1
+      register: paperless_db_container
+      changed_when: false
+
+    - name: Fail if DB container not found
+      ansible.builtin.fail:
+        msg: "Could not find paperless DB container (compose service=db)"
+      when: paperless_db_container.stdout | trim == ""
+
+    - name: Dump DB to /tmp on legacy host
+      ansible.builtin.shell: >
+        docker exec -i {{ paperless_db_container.stdout | trim }}
+        pg_dump --clean --if-exists --no-owner --no-privileges --format=custom
+        -U {{ paperless_db_user }} {{ paperless_db_name }}
+        > /tmp/paperless-backup.dump
+      args:
+        executable: /bin/bash
+      changed_when: true
+
+    - name: Make dump readable
+      ansible.builtin.file:
+        path: /tmp/paperless-backup.dump
+        mode: "0644"
+
+    - name: Report dump size
+      ansible.builtin.stat:
+        path: /tmp/paperless-backup.dump
+      register: dump_stat
+
+    - name: Show dump size
+      ansible.builtin.debug:
+        msg: "Dump created: {{ (dump_stat.stat.size / 1024 / 1024) | round(1) }} MB"
+
+    - name: Fetch dump to controller
+      ansible.builtin.fetch:
+        src: /tmp/paperless-backup.dump
+        dest: /tmp/paperless-backup.dump
+        flat: true
+
+    - name: Remove remote dump
+      ansible.builtin.file:
+        path: /tmp/paperless-backup.dump
+        state: absent
@@ -0,0 +1,95 @@
+---
+- name: Check for Distribution Upgrades
+  hosts: proxmox_all_qemu:&status_running:!proxmox_noansible
+  become: true
+  gather_facts: true
+  ignore_unreachable: true
+  
+  tasks:
+    - name: Check Ubuntu Release Availability
+      ansible.builtin.command: do-release-upgrade -c
+      register: ubuntu_check
+      changed_when: false
+      failed_when: false
+      when: 
+        - ansible_distribution is defined
+        - ansible_distribution == 'Ubuntu'
+
+    - name: Determine Upgrade Status
+      ansible.builtin.set_fact:
+        upgrade_needed: >-
+          {%- if ansible_distribution is not defined -%}
+            unknown
+          {%- elif ansible_distribution == 'Ubuntu' -%}
+            {%- if ubuntu_check.stdout is defined and 'New release' in ubuntu_check.stdout -%}
+              true
+            {%- else -%}
+              false
+            {%- endif -%}
+          {%- elif ansible_distribution == 'Debian' -%}
+            {%- if ansible_distribution_major_version | int < 12 -%}
+              true
+            {%- else -%}
+              false
+            {%- endif -%}
+          {%- else -%}
+            false
+          {%- endif -%}
+        current_version_str: "{{ ansible_distribution }} {{ ansible_distribution_version }}"
+        target_version_str: >-
+          {%- if ansible_distribution is not defined -%}
+            Unknown
+          {%- elif ansible_distribution == 'Ubuntu' -%}
+            {%- if ubuntu_check.stdout is defined and 'New release' in ubuntu_check.stdout -%}
+              {{ (ubuntu_check.stdout | regex_findall("New release '([^']+)'") | first) | default('Unknown') }}
+            {%- else -%}
+              -
+            {%- endif -%}
+          {%- elif ansible_distribution == 'Debian' -%}
+            {%- if ansible_distribution_major_version | int < 12 -%}
+              Debian 12 (Bookworm)
+            {%- else -%}
+              -
+            {%- endif -%}
+          {%- else -%}
+            -
+          {%- endif -%}
+      when: ansible_distribution is defined
+
+    - name: Group hosts by status
+      ansible.builtin.group_by:
+        key: "upgrade_{{ upgrade_needed | trim }}"
+      when: upgrade_needed is defined
+
+- name: Generate Report
+  hosts: localhost
+  gather_facts: false
+  tasks:
+    - name: Display Upgrade Candidates
+      ansible.builtin.debug:
+        msg: "{{ report.splitlines() }}"
+      vars:
+        report: |
+          
+          DISTRIBUTION UPGRADE REPORT
+          ===========================
+
+          REQUIRES UPGRADE
+          ================
+          {{ '%-30s' | format('HOST') }}   {{ '%-20s' | format('CURRENT') }}   {{ 'TARGET' }}
+          {{ '------------------------------' }}   {{ '--------------------' }}   {{ '--------------------' }}
+          {% for host in groups['upgrade_true'] | default([]) %}
+          {{ '%-30s' | format(host) }}   {{ '%-20s' | format(hostvars[host]['current_version_str']) }}   {{ hostvars[host]['target_version_str'] }}
+          {% endfor %}
+          {% if groups['upgrade_true'] is not defined or groups['upgrade_true'] | length == 0 %}
+          (No hosts require upgrade)
+          {% endif %}
+
+          UP TO DATE
+          ==========
+          {{ '%-30s' | format('HOST') }}   {{ 'CURRENT' }}
+          {{ '------------------------------' }}   {{ '--------------------' }}
+          {% for host in groups['upgrade_false'] | default([]) %}
+          {{ '%-30s' | format(host) }}   {{ hostvars[host]['current_version_str'] }}
+          {% endfor %}
+
@@ -0,0 +1,8 @@
+- name: Ensure debian-12 template exists on saturn
+  hosts: saturn
+  become: true
+  tasks:
+    - name: Download debian-12 template
+      command: pveam download local debian-12-standard_12.12-1_amd64.tar.zst
+      args:
+        creates: /var/lib/vz/template/cache/debian-12-standard_12.12-1_amd64.tar.zst
@@ -1,16 +1,22 @@
 ---
 - name: Install Custom CA Certificates
-  hosts: all
+  hosts: all:!proxmox_noansible
  become: true
  roles:
    - certificates

 - name: Base System Hardening & Common Configuration
-  hosts: all
+  hosts: all:!pve:!pbs:!proxmox_noansible:!proxmox_nodes:!pve_cluster
  become: true
  ignore_unreachable: true
  gather_facts: true

+  vars:
+    # Monitoring agent endpoints
+    loki_endpoint: "http://loki.srv.renner.uno:3100"
+    prometheus_remote_write_endpoint: "http://prometheus.srv.renner.uno:9090/api/v1/write"
+
  roles:
    - common
    - qemu-guest-agent
+    - monitoring-agent
@@ -0,0 +1,44 @@
+---
+# Configure Audiobookshelf permissions
+# Fixes upload issues by ensuring audiobookshelf user can write to storage directories
+
+- name: Configure Audiobookshelf
+  hosts: audiobookshelf.srv.renner.uno
+  become: true
+
+  tasks:
+    - name: Ensure audiobookshelf user exists
+      ansible.builtin.user:
+        name: audiobookshelf
+        system: true
+        create_home: false
+        state: present
+
+    - name: Set ownership on audiobooks directory
+      ansible.builtin.file:
+        path: /audiobooks
+        owner: audiobookshelf
+        group: audiobookshelf
+        mode: "0755"
+        state: directory
+        recurse: false
+
+    - name: Set ownership on audiobooks content recursively
+      ansible.builtin.command:
+        cmd: chown -R audiobookshelf:audiobookshelf /audiobooks
+      changed_when: true
+
+    - name: Set ownership on metadata directory
+      ansible.builtin.file:
+        path: /metadata
+        owner: audiobookshelf
+        group: audiobookshelf
+        mode: "0755"
+        state: directory
+        recurse: true
+
+    - name: Restart audiobookshelf service
+      ansible.builtin.systemd:
+        name: audiobookshelf
+        state: restarted
+        enabled: true
@@ -0,0 +1,40 @@
+---
+# Configure Grafana to use Authentik for authentication
+#
+# Prerequisites:
+# 1. Create OAuth2/OpenID application in Authentik:
+#    - Name: Grafana
+#    - Slug: grafana
+#    - Client type: Confidential
+#    - Redirect URIs: https://dash.renner.uno/login/generic_oauth
+#    - Signing Key: authentik Self-signed Certificate
+#
+# 2. Add to vault (ansible/inventory/group_vars/all/vault.yml):
+#    vault_grafana_authentik_client_id: "<client-id-from-authentik>"
+#    vault_grafana_authentik_client_secret: "<client-secret-from-authentik>"
+#
+# 3. (Optional) Create groups in Authentik:
+#    - "Grafana Admins" - gets Admin role
+#    - "Grafana Editors" - gets Editor role
+#    - Others get Viewer role
+
+- name: Configure Grafana Authentik Authentication
+  hosts: grafana.srv.renner.uno
+  become: yes
+  roles:
+    - grafana-authentik
+
+  post_tasks:
+    - name: Display configuration status
+      debug:
+        msg: |
+          Grafana Authentik OAuth configured!
+
+          Users can now login at: {{ grafana_root_url }}
+
+          Role mapping:
+          - Members of "Grafana Admins" -> Admin
+          - Members of "Grafana Editors" -> Editor
+          - All others -> Viewer
+
+          If auto_login is enabled, users will be redirected to Authentik automatically.
@@ -0,0 +1,68 @@
+---
+# Set sync=disabled on the zvols backing the K8s control-plane VM root disks
+# (k8s-1/2/3, VMIDs 150/151/152) to eliminate ZFS fsync amplification on the
+# shared local-zfs pool. Trades a small durability window for major fsync
+# latency reduction on etcd's WAL writes:
+#
+#   - etcd's WAL fsync returns immediately; ZFS still flushes via txgs
+#     (default ~5s)
+#   - Worst case on hard power loss: up to ~5s of writes lost per CP member
+#   - With 3-member quorum and Velero backups, this is acceptable for homelab
+#   - The whole zvol is sync=disabled (Talos OS, containerd, kubelet
+#     ephemeral, etcd WAL all share it), but etcd is the dominant fsync user;
+#     other writers tolerate small loss windows
+#
+# Each PVE host owns exactly one CP zvol (jupiter→150, saturn→151, neptune→152).
+# Auto-discovery handles whatever the host has — no need to update if a CP
+# VM ever moves.
+#
+# Idempotent: skips zvols that already have sync=disabled.
+#
+# Run manually against freshly-created CP zvols so they get the property
+# before Talos starts writing (previously wired into the now-removed
+# rebuild-k8s pipeline between `recreate-vms` and `talos-bootstrap`).
+
+- name: Set sync=disabled on K8s CP VM zvols
+  hosts: pve_cluster
+  become: yes
+  gather_facts: no
+
+  tasks:
+    - name: Discover CP-VM zvols on this host
+      shell: |
+        set -o pipefail
+        zfs list -H -t volume -o name | grep -E "/vm-(150|151|152)-disk-[0-9]+$" || true
+      args:
+        executable: /bin/bash
+      register: cp_zvols
+      changed_when: false
+
+    - name: Read current sync property
+      shell: "zfs get -H -o value sync {{ item }}"
+      register: current_sync
+      changed_when: false
+      loop: "{{ cp_zvols.stdout_lines }}"
+      when: cp_zvols.stdout_lines | length > 0
+
+    - name: Set sync=disabled where not already
+      shell: "zfs set sync=disabled {{ item.item }}"
+      when:
+        - cp_zvols.stdout_lines | length > 0
+        - item.stdout | trim != "disabled"
+      loop: "{{ current_sync.results | default([]) }}"
+      loop_control:
+        label: "{{ item.item }}"
+
+    - name: Verify sync=disabled
+      shell: "zfs get -H -o value sync {{ item }}"
+      register: verify_sync
+      changed_when: false
+      failed_when: verify_sync.stdout | trim != "disabled"
+      loop: "{{ cp_zvols.stdout_lines }}"
+      when: cp_zvols.stdout_lines | length > 0
+
+    - name: Summary
+      debug:
+        msg: >-
+          {{ inventory_hostname }}: tuned {{ cp_zvols.stdout_lines | length }}
+          zvol(s) [{{ cp_zvols.stdout_lines | join(', ') }}]
@@ -0,0 +1,53 @@
+---
+# Configure Proxmox VE Node Host Networking
+#
+# Fixes two defects found by live inspection on 2026-07-19:
+#
+#   1. All three nodes had no IPv6 address on any external interface. FRR
+#      enables ipv6 forwarding for the OSPFv3 corosync mesh, and with the
+#      kernel default accept_ra=1 the nodes silently discarded the Router
+#      Advertisements the gateway sends on VLAN 40. Fixed with accept_ra=2.
+#
+#   2. saturn's vmbr0 sits at MTU 1500 while its own uplink and both peer
+#      nodes are at 9000, so large frames between saturn's guests and the
+#      other nodes' guests blackhole instead of fragmenting.
+#
+# The sysctl fix is non-disruptive and applies on every run. The MTU fix is
+# staged as a pending PVE network change and only applied when explicitly
+# requested, because applying it reloads networking on the node and briefly
+# interrupts the Talos k8s VM and its Ceph OSD.
+#
+# Safe (converges IPv6, stages the MTU change without applying it):
+#   ansible-playbook -i inventory/ plays/configure-pve-node-network.yml
+#
+# Applying the staged MTU change, one node at a time:
+#   ansible-playbook -i inventory/ plays/configure-pve-node-network.yml \
+#     --limit saturn -e pve_node_network_apply_pending=true
+#
+# See wiki: pve-ipv6-accept-ra-forwarding-defect,
+#           pve-saturn-vmbr0-mtu-asymmetry
+
+- name: Configure PVE node host networking
+  hosts: pve_cluster
+  become: yes
+  serial: 1
+
+  roles:
+    - role: pve-node-network
+
+  post_tasks:
+    - name: Summarise node IPv6 state
+      ansible.builtin.command: ip -6 addr show dev vmbr1.40 scope global
+      register: pve_node_network_v6_summary
+      changed_when: false
+      failed_when: false
+
+    - name: Display node IPv6 state
+      ansible.builtin.debug:
+        msg: >-
+          {{ inventory_hostname }}:
+          {{ pve_node_network_v6_summary.stdout_lines
+             | select('search', 'inet6')
+             | map('trim')
+             | list
+             | default(['no global IPv6'], true) }}
@@ -0,0 +1,20 @@
+---
+- name: Setup PostgreSQL HA Cluster
+  hosts: authentik_db
+  become: true
+  vars:
+    # Set this in inventory or pass as extra var.
+    # For initial setup, we define authentik-db-1 as primary.
+    postgres_primary_host: "authentik-db-1.srv.renner.uno"
+    
+  roles:
+    - common
+    - postgres-simple-ha
+
+- name: Setup Authentik Stateless App
+  hosts: authentik_app
+  become: true
+  roles:
+    - common
+    - docker-install
+    - authentik-stateless
@@ -10,7 +10,7 @@
 - name: Deploy the Claude Code workstation
  hosts: claude_workstation
  become: true
-  # qemu-guest-agent is applied
+  # qemu-guest-agent + monitoring-agent (Alloy -> Loki/Prometheus) are applied
  # to every inventory host by the umbrella plays/common.yml, same as for the
  # netbird-proxy host — run that against this host too (see RUNBOOK Step 5).
  roles:
@@ -18,10 +18,9 @@
 #
 # Self-contained: `certificates` and `common` are the parts of plays/common.yml
 # that apply to an edge host, carried here so bootstrapping one is a single
-# unrestricted run over the group. Not carried over: qemu-guest-agent, since
-# these are bare metal rather than VMs. Log shipping is edge-log-shipper's job
-# and lives below — it pushes to the authenticated cluster ingress with a
-# per-host credential, which is the only Loki these hosts can reach.
+# unrestricted run over the group. Not carried over: qemu-guest-agent (bare
+# metal, not VMs) and monitoring-agent (Debian-oriented; these hosts ship logs
+# via edge-log-shipper).
 #
 # Order is dependency order: edge-flash-wear before docker, so the journal is
 # volatile before containers start logging into it.
@@ -0,0 +1,313 @@
+---
+# Synology-specific playbook using raw module (Python 3.8 workaround)
+# Deploys Garage (https://garagehq.deuxfleurs.fr) as a single-node S3 target
+# for K8s backups (Velero + CNPG Barman Cloud Plugin).
+#
+# Required secrets in host_vars/nas-1.renner.uno/vault.yml:
+#   garage_rpc_secret:     "<openssl rand -hex 32>"
+#   garage_admin_token:    "<openssl rand -base64 32>"
+#   garage_metrics_token:  "<openssl rand -base64 32>"
+#
+# Post-install bootstrap (run once, record the returned secret keys):
+#   docker exec garaged /garage status
+#   docker exec garaged /garage bucket create k8s-velero
+#   docker exec garaged /garage bucket create k8s-cnpg
+#   docker exec garaged /garage bucket create k8s-secrets
+#   docker exec garaged /garage bucket create k8s-migration
+#   docker exec garaged /garage key create velero-key
+#   docker exec garaged /garage key create cnpg-key
+#   docker exec garaged /garage key create secrets-key
+#   docker exec garaged /garage key create migration-key
+#   docker exec garaged /garage bucket allow --read --write --owner k8s-velero    --key velero-key
+#   docker exec garaged /garage bucket allow --read --write --owner k8s-cnpg      --key cnpg-key
+#   docker exec garaged /garage bucket allow --read --write --owner k8s-secrets   --key secrets-key
+#   docker exec garaged /garage bucket allow --read --write --owner k8s-migration --key migration-key
+- name: Install Garage S3 on Synology NAS
+  hosts: synology_nas
+  gather_facts: false
+
+  vars:
+    garage_data_dir: "/volume1/docker/garage"
+
+  tasks:
+    - name: Create Garage directory tree
+      ansible.builtin.raw: |
+        mkdir -p {{ garage_data_dir }}/config \
+                 {{ garage_data_dir }}/meta \
+                 {{ garage_data_dir }}/data
+      changed_when: true
+
+    # Secret writes use the same compare-then-write pattern as garage.toml so
+    # rotation is idempotent AND a real change can drive force-recreate below.
+    # Garage reads these files once at startup; without container restart, a
+    # rotated token leaves the running process holding the previous value.
+    - name: Write rpc_secret
+      ansible.builtin.raw: |
+        cat > {{ garage_data_dir }}/config/rpc_secret.new << 'SECRET_EOF'
+        {{ garage_rpc_secret }}
+        SECRET_EOF
+        chmod 600 {{ garage_data_dir }}/config/rpc_secret.new
+        if [ ! -f {{ garage_data_dir }}/config/rpc_secret ] \
+        || ! cmp -s {{ garage_data_dir }}/config/rpc_secret.new {{ garage_data_dir }}/config/rpc_secret; then
+          mv {{ garage_data_dir }}/config/rpc_secret.new {{ garage_data_dir }}/config/rpc_secret
+          echo RPC_SECRET_CHANGED
+        else
+          rm {{ garage_data_dir }}/config/rpc_secret.new
+        fi
+      register: rpc_secret_write
+      changed_when: "'RPC_SECRET_CHANGED' in rpc_secret_write.stdout"
+      no_log: true
+
+    - name: Write admin_token
+      ansible.builtin.raw: |
+        cat > {{ garage_data_dir }}/config/admin_token.new << 'SECRET_EOF'
+        {{ garage_admin_token }}
+        SECRET_EOF
+        chmod 600 {{ garage_data_dir }}/config/admin_token.new
+        if [ ! -f {{ garage_data_dir }}/config/admin_token ] \
+        || ! cmp -s {{ garage_data_dir }}/config/admin_token.new {{ garage_data_dir }}/config/admin_token; then
+          mv {{ garage_data_dir }}/config/admin_token.new {{ garage_data_dir }}/config/admin_token
+          echo ADMIN_TOKEN_CHANGED
+        else
+          rm {{ garage_data_dir }}/config/admin_token.new
+        fi
+      register: admin_token_write
+      changed_when: "'ADMIN_TOKEN_CHANGED' in admin_token_write.stdout"
+      no_log: true
+
+    - name: Write metrics_token
+      ansible.builtin.raw: |
+        cat > {{ garage_data_dir }}/config/metrics_token.new << 'SECRET_EOF'
+        {{ garage_metrics_token }}
+        SECRET_EOF
+        chmod 600 {{ garage_data_dir }}/config/metrics_token.new
+        if [ ! -f {{ garage_data_dir }}/config/metrics_token ] \
+        || ! cmp -s {{ garage_data_dir }}/config/metrics_token.new {{ garage_data_dir }}/config/metrics_token; then
+          mv {{ garage_data_dir }}/config/metrics_token.new {{ garage_data_dir }}/config/metrics_token
+          echo METRICS_TOKEN_CHANGED
+        else
+          rm {{ garage_data_dir }}/config/metrics_token.new
+        fi
+      register: metrics_token_write
+      changed_when: "'METRICS_TOKEN_CHANGED' in metrics_token_write.stdout"
+      no_log: true
+
+    - name: Write garage.toml
+      ansible.builtin.raw: |
+        cat > {{ garage_data_dir }}/config/garage.toml.new << 'TOML_EOF'
+        metadata_dir = "/var/lib/garage/meta"
+        data_dir     = "/var/lib/garage/data"
+
+        # SQLite chosen over LMDB: single-node deployment has no peers to
+        # rebuild metadata from on unclean-shutdown corruption. SQLite is
+        # slower but portable and has no LMDB corruption window.
+        db_engine = "sqlite"
+
+        # 6-hour auto-snapshot of the metadata DB as a backstop against
+        # corruption; last two snapshots retained.
+        metadata_auto_snapshot_interval = "6h"
+
+        # Single-zone, single-node: only coherent replication factor.
+        replication_factor = 1
+
+        rpc_bind_addr    = "[::]:3901"
+        rpc_secret_file  = "/etc/garage/rpc_secret"
+
+        compression_level = 2
+
+        # 64 MB blocks (default 1 MB; previously 20 MB). On the 5 × 7 TB
+        # Btrfs RAID 5 volume the per-block fixed cost (SQLite lookup,
+        # dir traversal, hash verify, zstd decompress, nginx buffering)
+        # is the dominant overhead per object; larger blocks amortize
+        # it. block_size is a maximum, so 16 MiB CNPG WAL segments
+        # still upload as one block each. Only applies to NEW uploads;
+        # existing objects keep their original chunking until re-uploaded.
+        block_size = "64M"
+
+        [s3_api]
+        api_bind_addr = "[::]:3900"
+        s3_region     = "garage"
+        root_domain   = ".s3.{{ inventory_hostname }}"
+
+        [s3_web]
+        bind_addr   = "[::]:3902"
+        root_domain = ".web.{{ inventory_hostname }}"
+
+        [admin]
+        api_bind_addr      = "[::]:3903"
+        admin_token_file   = "/etc/garage/admin_token"
+        metrics_token_file = "/etc/garage/metrics_token"
+        TOML_EOF
+        if [ ! -f {{ garage_data_dir }}/config/garage.toml ] \
+        || ! cmp -s {{ garage_data_dir }}/config/garage.toml.new {{ garage_data_dir }}/config/garage.toml; then
+          mv {{ garage_data_dir }}/config/garage.toml.new {{ garage_data_dir }}/config/garage.toml
+          echo GARAGE_TOML_CHANGED
+        else
+          rm {{ garage_data_dir }}/config/garage.toml.new
+        fi
+      register: garage_toml_write
+      changed_when: "'GARAGE_TOML_CHANGED' in garage_toml_write.stdout"
+
+    - name: Write docker-compose.yml
+      ansible.builtin.raw: |
+        cat > {{ garage_data_dir }}/docker-compose.yml << 'COMPOSE_EOF'
+        services:
+          garage:
+            image: {{ garage_image }}
+            container_name: garaged
+            restart: unless-stopped
+            # HTTP services (3900/3902/3903) bind to loopback; DSM
+            # reverse proxy fronts and TLS-terminates them. RPC 3901
+            # is exposed directly — the Noise handshake is raw TCP,
+            # so DSM RP cannot front it. rpc_secret gates auth.
+            ports:
+              - "3901:3901"
+              - "127.0.0.1:3900:3900"
+              - "127.0.0.1:3902:3902"
+              - "127.0.0.1:3903:3903"
+            volumes:
+              - {{ garage_data_dir }}/config/garage.toml:/etc/garage.toml:ro
+              - {{ garage_data_dir }}/config/rpc_secret:/etc/garage/rpc_secret:ro
+              - {{ garage_data_dir }}/config/admin_token:/etc/garage/admin_token:ro
+              - {{ garage_data_dir }}/config/metrics_token:/etc/garage/metrics_token:ro
+              - {{ garage_data_dir }}/meta:/var/lib/garage/meta
+              - {{ garage_data_dir }}/data:/var/lib/garage/data
+        COMPOSE_EOF
+      changed_when: true
+
+    - name: Pull image and start Garage container
+      ansible.builtin.shell: |
+        export PATH=$PATH:/usr/local/bin
+        cd {{ garage_data_dir }} && docker compose pull && docker compose up -d
+      become: true
+      changed_when: true
+
+    # Garage does not reload config on SIGHUP and compose does not notice
+    # when a bind-mounted host file changes. Force-recreate the container
+    # when garage.toml OR any of the three secret files changed so the new
+    # config / rotated tokens are picked up.
+    - name: Force-recreate garaged when garage.toml or secrets changed
+      ansible.builtin.shell: |
+        export PATH=$PATH:/usr/local/bin
+        cd {{ garage_data_dir }} && docker compose up -d --force-recreate garage
+      become: true
+      when: garage_toml_write.changed
+            or rpc_secret_write.changed
+            or admin_token_write.changed
+            or metrics_token_write.changed
+      changed_when: true
+
+    - name: Wait for Garage to accept CLI calls
+      ansible.builtin.shell: |
+        export PATH=$PATH:/usr/local/bin
+        for i in $(seq 1 30); do
+          if docker exec garaged /garage status >/dev/null 2>&1; then
+            exit 0
+          fi
+          sleep 2
+        done
+        exit 1
+      become: true
+      changed_when: false
+
+    - name: Check whether cluster layout is already applied
+      ansible.builtin.shell: |
+        export PATH=$PATH:/usr/local/bin
+        docker exec garaged /garage status
+      become: true
+      register: garage_status
+      changed_when: false
+
+    - name: Apply single-node layout (idempotent — only on first run)
+      when: "'NO ROLE ASSIGNED' in garage_status.stdout"
+      ansible.builtin.shell: |
+        export PATH=$PATH:/usr/local/bin
+        NODE_ID=$(docker exec garaged /garage node id | head -n 1 | cut -d@ -f1)
+        docker exec garaged /garage layout assign "$NODE_ID" \
+          -z {{ garage_zone }} \
+          -c {{ garage_capacity }} \
+          -t {{ inventory_hostname_short }}
+        docker exec garaged /garage layout apply --version 1
+      become: true
+      changed_when: true
+
+    - name: Report Garage status
+      ansible.builtin.shell: |
+        export PATH=$PATH:/usr/local/bin
+        docker exec garaged /garage status
+      become: true
+      register: garage_status_final
+      changed_when: false
+
+    - name: Show Garage status
+      ansible.builtin.debug:
+        var: garage_status_final.stdout_lines
+
+    # Bootstrap buckets + keys. Idempotent: bucket_info / key_info return
+    # non-zero when missing, so we only create on first run. Access-key
+    # secrets are captured at create time and persisted to chmod-600 files
+    # under {{ garage_data_dir }}/config/keys/<key>.creds on nas-1. These
+    # files are the only post-install record of the secrets — Garage does
+    # not expose them again.
+    - name: Bootstrap Garage buckets, keys, and grants
+      ansible.builtin.shell: |
+        set -e
+        export PATH=$PATH:/usr/local/bin
+        KEYS_DIR={{ garage_data_dir }}/config/keys
+        mkdir -p "$KEYS_DIR"
+        chmod 700 "$KEYS_DIR"
+
+        bootstrap_pair() {
+          local BUCKET="$1"
+          local KEY="$2"
+          local CREDS_FILE="$KEYS_DIR/$KEY.creds"
+
+          if ! docker exec garaged /garage bucket info "$BUCKET" >/dev/null 2>&1; then
+            docker exec garaged /garage bucket create "$BUCKET"
+          fi
+
+          if ! docker exec garaged /garage key info "$KEY" >/dev/null 2>&1; then
+            CREATE_OUT=$(docker exec garaged /garage key create "$KEY")
+            KEY_ID=$(echo "$CREATE_OUT"   | awk '/^Key ID:/     {print $3}')
+            SECRET=$(echo "$CREATE_OUT"   | awk '/^Secret key:/ {print $3}')
+            umask 177
+            printf 'KEY_ID=%s\nSECRET_KEY=%s\n' "$KEY_ID" "$SECRET" > "$CREDS_FILE"
+          fi
+
+          docker exec garaged /garage bucket allow \
+            --read --write --owner \
+            "$BUCKET" --key "$KEY"
+        }
+
+        bootstrap_pair k8s-velero    velero-key
+        bootstrap_pair k8s-cnpg      cnpg-key
+        bootstrap_pair k8s-secrets   secrets-key
+        bootstrap_pair k8s-migration migration-key
+        bootstrap_pair tfstate       tfstate-key
+      become: true
+      changed_when: true
+      no_log: true
+
+    - name: Report bucket + key state (no secrets)
+      ansible.builtin.shell: |
+        export PATH=$PATH:/usr/local/bin
+        docker exec garaged /garage bucket list
+        echo "---"
+        docker exec garaged /garage key list
+        echo "---"
+        ls -l {{ garage_data_dir }}/config/keys/ 2>&1
+      become: true
+      register: garage_bootstrap_state
+      changed_when: false
+
+    - name: Show bucket + key state
+      ansible.builtin.debug:
+        var: garage_bootstrap_state.stdout_lines
+
+    - name: Post-install note
+      ansible.builtin.debug:
+        msg:
+          - "Buckets: k8s-velero, k8s-cnpg, k8s-secrets, k8s-migration, tfstate"
+          - "Keys:    velero-key, cnpg-key, secrets-key, migration-key, tfstate-key"
+          - "Secrets are written to {{ garage_data_dir }}/config/keys/<key>.creds on nas-1 (chmod 600, root-owned)."
+          - "SSH to nas-1 and `sudo cat` the file to retrieve KEY_ID + SECRET_KEY for Velero / CNPG / migration / tfstate config."
@@ -0,0 +1,8 @@
+---
+- name: Install and Configure Gatus
+  hosts: proxmox_gatus
+  become: true
+  roles:
+    - role: common
+    - role: docker-install
+    - role: gatus
@@ -0,0 +1,8 @@
+---
+- name: Install and Configure Grist
+  hosts: grist
+  become: true
+  roles:
+    - role: common
+    - role: docker-install
+    - role: grist
@@ -0,0 +1,8 @@
+---
+- name: Install and Configure ical-proxy
+  hosts: proxmox_ical_proxy
+  become: true
+  roles:
+    - role: common
+    - role: docker-install
+    - role: ical-proxy
@@ -0,0 +1,19 @@
+---
+# Deploy MkDocs wiki from Obsidian vault
+#
+# Usage:
+#   ansible-playbook -i inventory plays/install-mkdocs-wiki.yml
+#
+- name: Deploy MkDocs Wiki
+  hosts: proxmox_wiki
+  become: true
+
+  pre_tasks:
+    - name: Ensure wiki group exists
+      ansible.builtin.group:
+        name: wiki
+        system: true
+        state: present
+
+  roles:
+    - role: mkdocs-wiki
@@ -0,0 +1,8 @@
+---
+- name: Install and Configure Music Assistant
+  hosts: proxmox_music_assistant
+  become: true
+  roles:
+    - role: common
+    - role: docker-install
+    - role: music-assistant
@@ -0,0 +1,8 @@
+---
+- name: Install and Configure n8n
+  hosts: n8n
+  become: true
+  roles:
+    - role: common
+    - role: docker-install
+    - role: n8n
@@ -0,0 +1,38 @@
+---
+# Deploy monitoring agent to NetBird server (external datacenter)
+# Uses NetBird mesh IPs to reach Loki and Prometheus
+# Usage: ansible-playbook -i inventory/ plays/setup-netbird-monitoring.yml
+
+- name: Deploy Monitoring Agent to NetBird Server
+  hosts: netbird_server
+  become: yes
+  gather_facts: yes
+
+  vars:
+    # Use ARM64 binary for Hetzner ARM server
+    alloy_arch: "linux-arm64"
+
+    # Monitoring endpoints via NetBird mesh
+    loki_endpoint: "http://loki.nb.renner.uno:3100"
+    prometheus_remote_write_endpoint: "http://prometheus.nb.renner.uno:9090/api/v1/write"
+
+    # NetBird server labels
+    alloy_extra_labels:
+      environment: "production"
+      datacenter: "hetzner"
+      type: "netbird-server"
+      service: "netbird"
+
+  roles:
+    - monitoring-agent
+
+  post_tasks:
+    - name: Verify Alloy agent is running
+      systemd:
+        name: alloy
+        state: started
+      register: alloy_status
+
+    - name: Display agent status
+      debug:
+        msg: "Monitoring agent deployed to {{ inventory_hostname }} via NetBird mesh"
@@ -26,7 +26,3 @@
    - docker-install
    - netbird-proxy
    - netbird-proxy-watchdog
-    # Enforces the cluster's CrowdSec decisions here, where public traffic
-    # enters. Deployed in dry-run: it logs what it would block and creates no
-    # nftables rules until crowdsec_bouncer_mode is flipped per host.
-    - crowdsec-edge-bouncer
@@ -0,0 +1,41 @@
+---
+# Install Node Exporter and Chrony Exporter on Proxmox VE hosts
+# Usage: ansible-playbook -i inventory/ plays/install-node-exporter-pve.yml
+
+- name: Install Exporters on PVE Cluster
+  hosts: pve_cluster
+  become: yes
+  gather_facts: yes
+
+  roles:
+    - node-exporter
+    - chrony-exporter
+
+  post_tasks:
+    - name: Verify node-exporter is running
+      uri:
+        url: "http://127.0.0.1:9100/metrics"
+        status_code: 200
+      register: node_exporter_health
+      retries: 3
+      delay: 5
+      until: node_exporter_health.status == 200
+
+    - name: Verify chrony-exporter is running
+      uri:
+        url: "http://127.0.0.1:9123/metrics"
+        status_code: 200
+      register: chrony_exporter_health
+      retries: 3
+      delay: 5
+      until: chrony_exporter_health.status == 200
+
+    - name: Verify chrony metrics include stratum
+      uri:
+        url: "http://127.0.0.1:9123/metrics"
+        return_content: yes
+      register: chrony_metrics
+
+    - name: Check chrony exporter output
+      debug:
+        msg: "Chrony stratum metrics available: {{ 'chrony_tracking_stratum' in chrony_metrics.content }}"
@@ -3,7 +3,7 @@
 # Usage: ansible-playbook -i inventory/ plays/install-node-exporter.yml

 - name: Install Node Exporter
-  hosts: all
+  hosts: all:!pve_cluster
  become: yes
  gather_facts: yes

@@ -0,0 +1,8 @@
+---
+- name: Install and Configure ownCloud Infinite Scale
+  hosts: proxmox_owncloud
+  become: true
+  roles:
+    - role: common
+    - role: docker-install
+    - role: owncloud
@@ -0,0 +1,104 @@
+---
+# Synology-specific playbook using raw module (Python 3.8 workaround)
+- name: Install PBS Client on Synology NAS
+  hosts: synology_nas
+  gather_facts: false
+
+  vars:
+    pbs_client_data_dir: "/volume1/docker/pbs-client"
+
+  tasks:
+    - name: Create PBS client directory
+      ansible.builtin.raw: |
+        mkdir -p {{ pbs_client_data_dir }}/state
+      changed_when: true
+
+    - name: Create docker-compose.yml
+      ansible.builtin.raw: |
+        cat > {{ pbs_client_data_dir }}/docker-compose.yml << 'COMPOSE_EOF'
+        services:
+          pbs-client:
+            image: proxmox/proxmox-backup-client:latest
+            container_name: pbs-client
+            restart: unless-stopped
+            hostname: {{ pbs_backup_id }}
+            env_file:
+              - .env
+            volumes:
+              - {{ pbs_client_data_dir }}/state:/root/.proxmox-backup
+        {% for path in pbs_backup_paths %}
+              - "{{ path }}:{{ path }}:ro"
+        {% endfor %}
+            mem_limit: 2g
+            command: ["sleep", "infinity"]
+        COMPOSE_EOF
+      changed_when: true
+
+    - name: Create .env file
+      ansible.builtin.raw: |
+        cat > {{ pbs_client_data_dir }}/.env << 'ENV_EOF'
+        PBS_REPOSITORY={{ pbs_username }}@{{ pbs_server_host }}:{{ pbs_server_port }}:{{ pbs_datastore }}
+        PBS_PASSWORD={{ pbs_password }}
+        ENV_EOF
+        chmod 600 {{ pbs_client_data_dir }}/.env
+      changed_when: true
+
+    - name: Create backup script
+      ansible.builtin.raw: |
+        cat > {{ pbs_client_data_dir }}/backup.sh << 'SCRIPT_EOF'
+        #!/bin/bash
+        set -e
+        source {{ pbs_client_data_dir }}/.env
+
+        BACKUP_SOURCES=""
+        {% for path in pbs_backup_paths %}
+        BACKUP_SOURCES="$BACKUP_SOURCES {{ path | basename }}.pxar:{{ path }}"
+        {% endfor %}
+
+        echo "[$(date '+%Y-%m-%d %H:%M:%S')] Starting backup for {{ pbs_backup_id }}"
+        docker exec pbs-client proxmox-backup-client backup \
+            $BACKUP_SOURCES \
+            --backup-id {{ pbs_backup_id }} \
+            --repository "${PBS_REPOSITORY}"
+
+        echo "[$(date '+%Y-%m-%d %H:%M:%S')] Backup completed, pruning..."
+        docker exec pbs-client proxmox-backup-client prune \
+            --repository "${PBS_REPOSITORY}" \
+            --backup-id {{ pbs_backup_id }} \
+            --keep-last {{ pbs_keep_last | default(3) }} \
+            --keep-daily {{ pbs_keep_daily | default(7) }} \
+            --keep-weekly {{ pbs_keep_weekly | default(4) }} \
+            --keep-monthly {{ pbs_keep_monthly | default(6) }}
+
+        echo "[$(date '+%Y-%m-%d %H:%M:%S')] Done"
+        SCRIPT_EOF
+        chmod 755 {{ pbs_client_data_dir }}/backup.sh
+      changed_when: true
+
+    - name: Start PBS client container
+      ansible.builtin.shell: |
+        export PATH=$PATH:/usr/local/bin
+        cd {{ pbs_client_data_dir }} && docker compose pull && docker compose up -d
+      become: true
+      changed_when: true
+
+    - name: Setup cron job for daily backup at 2 AM (root crontab)
+      ansible.builtin.shell: |
+        # Remove existing entry if present
+        crontab -l 2>/dev/null | grep -v "pbs-client/backup.sh" | crontab - 2>/dev/null || true
+        # Add new entry
+        (crontab -l 2>/dev/null; echo "0 2 * * * {{ pbs_client_data_dir }}/backup.sh >> /var/log/pbs-backup.log 2>&1") | crontab -
+      become: true
+      changed_when: true
+
+    - name: Verify container is running
+      ansible.builtin.shell: |
+        export PATH=$PATH:/usr/local/bin
+        docker ps --filter name=pbs-client --format "{{ '{{' }}.Status{{ '}}' }}"
+      become: true
+      register: container_status
+      changed_when: false
+
+    - name: Show container status
+      ansible.builtin.debug:
+        msg: "PBS client container status: {{ container_status.stdout }}"
@@ -0,0 +1,11 @@
+---
+- name: Install PBS Client on Synology NAS
+  hosts: synology_nas
+  become: true
+  # Variables are set in:
+  # - Role defaults: ansible/roles/pbs-client/defaults/main.yml
+  # - Host overrides: ansible/inventory/host_vars/<hostname>/vars.yml
+  # - Secrets: ansible/inventory/host_vars/<hostname>/vault.yml
+
+  roles:
+    - pbs-client
@@ -0,0 +1,62 @@
+---
+# Setup PostgreSQL monitoring user and exporter
+# Usage: ansible-playbook -i inventory/ plays/setup-postgres-monitoring.yml
+
+- name: Create PostgreSQL monitoring user on primary
+  hosts: authentik-db-1.srv.renner.uno
+  become: yes
+  vars_files:
+    - ../inventory/group_vars/all/vault.yml
+  tasks:
+    - name: Install psycopg2 for Ansible PostgreSQL modules
+      apt:
+        name: python3-psycopg2
+        state: present
+
+    - name: Create postgres_exporter user
+      become_user: postgres
+      community.postgresql.postgresql_user:
+        name: postgres_exporter
+        password: "{{ vault_postgres_exporter_password }}"
+        role_attr_flags: "NOSUPERUSER,NOCREATEDB,NOCREATEROLE"
+        state: present
+
+    - name: Grant pg_monitor role to exporter user
+      become_user: postgres
+      community.postgresql.postgresql_privs:
+        database: postgres
+        type: group
+        objs: pg_monitor
+        roles: postgres_exporter
+        grant_option: no
+
+    - name: Grant connect on authentik database
+      become_user: postgres
+      community.postgresql.postgresql_privs:
+        database: authentik
+        type: database
+        privs: CONNECT
+        roles: postgres_exporter
+
+- name: Deploy postgres-exporter to Prometheus server
+  hosts: prometheus.srv.renner.uno
+  become: yes
+  vars_files:
+    - ../inventory/group_vars/all/vault.yml
+  vars:
+    postgres_exporter_db_host: "10.2.40.135"  # PostgreSQL HA VIP
+    postgres_exporter_db_user: "postgres_exporter"
+    postgres_exporter_db_password: "{{ vault_postgres_exporter_password }}"
+    postgres_exporter_db_name: "postgres"
+  roles:
+    - postgres-exporter
+
+  post_tasks:
+    - name: Verify postgres-exporter is running
+      uri:
+        url: "http://127.0.0.1:9187/metrics"
+        status_code: 200
+      register: exporter_health
+      retries: 3
+      delay: 5
+      until: exporter_health.status == 200
@@ -0,0 +1,33 @@
+---
+# Deploy monitoring agents to Proxmox VE nodes
+# Usage: ansible-playbook -i inventory/ plays/setup-pve-monitoring.yml
+
+- name: Deploy Monitoring Agents to Proxmox Nodes
+  hosts: pve_cluster
+  become: yes
+  gather_facts: yes
+
+  vars:
+    # Monitoring endpoints
+    loki_endpoint: "http://loki.srv.renner.uno:3100"
+    prometheus_remote_write_endpoint: "http://prometheus.srv.renner.uno:9090/api/v1/write"
+
+    # PVE-specific labels
+    alloy_extra_labels:
+      environment: "production"
+      datacenter: "homelab"
+      type: "hypervisor"
+
+  roles:
+    - monitoring-agent
+
+  post_tasks:
+    - name: Verify Alloy agent is running
+      systemd:
+        name: alloy
+        state: started
+      register: alloy_status
+
+    - name: Display agent status
+      debug:
+        msg: "Monitoring agent deployed to {{ inventory_hostname }}"
@@ -0,0 +1,7 @@
+---
+- name: Install HashiCorp Vault
+  hosts: vault.srv.renner.uno
+  become: true
+  roles:
+    - common
+    - vault
@@ -1,250 +0,0 @@
---
-# Rotate the NetBird shared secrets that Ansible owns BOTH ends of.
-#
-# Deliberately opt-in. Rotation is not idempotent and it interrupts the mesh
-# briefly, so this play refuses to run unless invoked explicitly:
-#
-#   ansible-playbook plays/rotate-netbird-secrets.yml -e rotate_confirm=yes
-#
-# ONE-TIME MIGRATION REQUIRED BEFORE FIRST USE. The three keys below currently
-# live in group_vars/all/vault.yml. They must be deleted from there, because
-# group_vars files merge alphabetically and vault.yml sorts after
-# netbird-rotatable.yml — leaving them would let a later ordinary deploy
-# overwrite a freshly rotated secret with the old one. The play checks for this
-# and refuses to run until it is done:
-#
-#   ansible-vault edit inventory/group_vars/all/vault.yml
-#   # delete vault_netbird_turn_secret, vault_netbird_relay_secret,
-#   #        and netbird_turn_secret
-#
-# That last one is easy to miss and breaks the rotation silently. vault.yml
-# carried BOTH vault_netbird_turn_secret and a bare netbird_turn_secret. The
-# role default is `netbird_turn_secret: "{{ vault_netbird_turn_secret }}"`, but
-# a group var outranks a role default — so the bare key wins, and rotating only
-# the vault_-prefixed one changes nothing that reaches the template. The check
-# below matches both spellings for that reason.
-#
-# The first rotation then populates netbird-rotatable.yml with fresh values, so
-# nothing needs copying across.
-#
-# WHAT IS ROTATED, and why each is safe to rotate from here:
-#
-#   vault_netbird_turn_secret   coturn's `user=netbird:<secret>` in
-#                               turnserver.conf and the management server's
-#                               Turn.Secret/Password in management.json. Both
-#                               templates, one role, one host — the two ends
-#                               cannot disagree across a run.
-#   vault_netbird_relay_secret  management.json on the server and relay.env on
-#                               the proxies. Two hosts, both managed here.
-#
-# The test for "safe to rotate here" is whether the value appears in
-# management.json.j2. If the management server does not read it from a file
-# Ansible writes, Ansible does not own both ends, whatever else it templates.
-#
-# WHAT IS NOT, and must not be added:
-#
-#   vault_netbird_proxy_token    LOOKS symmetric — it is in proxy.env on both
-#                                the server and the proxies, all Ansible
-#                                templates — but the management server does not
-#                                validate it from any of them. It is registered
-#                                in the management DATABASE. Rotating it changes
-#                                only what the proxies present, so every proxy
-#                                is locked out: "proxy auth failed: too many
-#                                failed authentication attempts", and the
-#                                proxies stop receiving their domain
-#                                configuration. Learned by doing it.
-#
-#   vault_netbird_datastore_key  encrypts NetBird's datastore at rest. Changing
-#                                it without re-encrypting the store makes the
-#                                existing data unreadable. Rotating this needs a
-#                                dump/restore, not a template rewrite.
-#   vault_netbird_auth_client_secret  the other half lives in Authentik. Ansible
-#                                only writes the consumer side, so rotating here
-#                                breaks login until Authentik is changed too.
-#
-# These are Ansible's OWN credentials, not a copy of the cluster's. OpenBao
-# carries credentials for some of the same systems (see
-# kubernetes/infrastructure/openbao/rotation/), and those are deliberately DIFFERENT
-# values: one consumer, one credential, independently revocable. Rotating here
-# must not touch OpenBao and does not invalidate anything the cluster holds —
-# that is the point of them being distinct, not drift to be reconciled.
-#
-# Corollary: if a credential ever has to be shared between the two, that is the
-# bug. Mint a second one instead.
-#
-# There IS a brief interruption. The server is reconfigured before the proxies,
-# so relay/proxy auth fails for the seconds between the two plays. Clients
-# reconnect; existing peer-to-peer tunnels are not torn down.
-
- name: Generate and store new NetBird shared secrets
-  hosts: localhost
-  connection: local
-  gather_facts: false
-
-  vars:
-    rotatable_vault: "{{ playbook_dir }}/../inventory/group_vars/all/netbird-rotatable.yml"
-
-  pre_tasks:
-    - name: Require explicit confirmation
-      ansible.builtin.assert:
-        that:
-          - rotate_confirm | default('') == 'yes'
-        fail_msg: >-
-          Refusing to rotate without -e rotate_confirm=yes. This changes live
-          credentials and briefly interrupts the NetBird mesh.
-        success_msg: Confirmed; rotating NetBird shared secrets.
-
-    - name: Verify the vault password is available before touching anything
-      ansible.builtin.command:
-        cmd: "{{ playbook_dir }}/../vault-password.sh"
-      register: _vault_pw_check
-      changed_when: false
-      no_log: true
-
-    # Files inside group_vars/<group>/ are merged ALPHABETICALLY and the last
-    # one wins. netbird-rotatable.yml sorts before vault.yml, so if vault.yml
-    # still defines these keys it silently overrides everything this play
-    # writes. The rotation itself would still work — the plays below pass the
-    # new values explicitly, and play vars outrank group vars — but the NEXT
-    # ordinary run of install-netbird-server.yml or install-netbird-proxy.yml
-    # would read the stale value from vault.yml and push it back out, breaking
-    # server<->proxy auth. The deploy pipelines run unattended, so that would
-    # land on an unrelated change to those roles.
-    #
-    # Counting matches rather than printing them: the value never reaches
-    # stdout, only the key name does.
-    # Proving the vault is readable FIRST. Without this, a failed decrypt (wrong
-    # password, moved file) sends empty input to grep, which then reports zero
-    # matches — indistinguishable from a clean vault, so the guard below would
-    # wave the rotation through precisely when it knows least.
-    - name: Prove vault.yml is readable before trusting the key check
-      ansible.builtin.shell:
-        cmd: >-
-          set -o pipefail;
-          ansible-vault view
-          --vault-password-file {{ playbook_dir }}/../vault-password.sh
-          {{ playbook_dir }}/../inventory/group_vars/all/vault.yml
-          | wc -l
-        executable: /bin/bash
-      register: _vault_lines
-      changed_when: false
-
-    - name: Check vault.yml does not still define the rotated keys
-      ansible.builtin.shell:
-        cmd: >-
-          ansible-vault view
-          --vault-password-file {{ playbook_dir }}/../vault-password.sh
-          {{ playbook_dir }}/../inventory/group_vars/all/vault.yml
-          | grep -cE '^(vault_)?netbird_(turn_secret|relay_secret):'
-      register: _stale_keys
-      changed_when: false
-      # grep exits 1 on "no matches", which is the good case here.
-      failed_when: false
-
-    - name: Refuse to rotate while vault.yml still owns these keys
-      ansible.builtin.assert:
-        that:
-          - _vault_lines.stdout | int > 0
-          - _stale_keys.stdout | int == 0
-        fail_msg: >-
-          vault.yml still defines {{ _stale_keys.stdout }} of the keys this play
-          rotates. group_vars files merge alphabetically and vault.yml sorts
-          last, so it would override netbird-rotatable.yml on the next deploy
-          and revert the secrets. Remove vault_netbird_turn_secret,
-          netbird_turn_secret and vault_netbird_relay_secret from vault.yml
-          first (one-time migration), then re-run.
-        success_msg: vault.yml does not shadow the rotated keys.
-
-  tasks:
-    # 48 alphanumerics. coturn's shared secret and the relay secret are both
-    # opaque strings; length is the only property that matters.
-    #
-    # ansible.builtin.password, NOT community.general.random_string: the CI
-    # image (ci/images/ansible/Containerfile) installs only community.proxmox
-    # and community.docker, so anything from community.general fails at this
-    # task when the play runs through the project's own tooling — even though
-    # it resolves fine on a workstation that happens to have the collection.
-    #
-    # `/dev/null` as the path makes the lookup generate without persisting;
-    # the value exists only in this fact.
-    - name: Generate replacement secrets
-      ansible.builtin.set_fact:
-        new_turn_secret: "{{ lookup('ansible.builtin.password', '/dev/null length=48 chars=ascii_letters,digits') }}"
-        new_relay_secret: "{{ lookup('ansible.builtin.password', '/dev/null length=48 chars=ascii_letters,digits') }}"
-      no_log: true
-
-    - name: Render the new values to a temporary file
-      ansible.builtin.copy:
-        dest: "{{ rotatable_vault }}.new"
-        mode: "0600"
-        content: |
-          ---
-          # Generated by plays/rotate-netbird-secrets.yml. Do not hand-edit —
-          # the next rotation overwrites this file wholesale.
-          vault_netbird_turn_secret: "{{ new_turn_secret }}"
-          vault_netbird_relay_secret: "{{ new_relay_secret }}"
-      no_log: true
-
-    # Encrypting the whole file rather than inline !vault strings: these three
-    # values are always rewritten together, so per-value encryption buys nothing
-    # and makes the write non-atomic.
-    - name: Encrypt the new file
-      ansible.builtin.command:
-        cmd: >-
-          ansible-vault encrypt
-          --vault-password-file {{ playbook_dir }}/../vault-password.sh
-          {{ rotatable_vault }}.new
-      changed_when: true
-
-    # Move into place only after encryption succeeded, so a failure above never
-    # leaves plaintext where the inventory would load it.
-    - name: Install the encrypted file
-      ansible.builtin.command:
-        cmd: "mv {{ rotatable_vault }}.new {{ rotatable_vault }}"
-      changed_when: true
-
-    - name: Confirm the installed file is encrypted
-      ansible.builtin.command:
-        cmd: "head -1 {{ rotatable_vault }}"
-      register: _vault_header
-      changed_when: false
-      failed_when: "'$ANSIBLE_VAULT' not in _vault_header.stdout"
-
- name: Apply the new secrets to the NetBird server
-  hosts: netbird_server
-  become: true
-
-  vars:
-    netbird_turn_secret: "{{ hostvars['localhost']['new_turn_secret'] }}"
-    netbird_relay_secret: "{{ hostvars['localhost']['new_relay_secret'] }}"
-
-  roles:
-    - netbird-server
-
- name: Apply the new secrets to the NetBird proxies
-  hosts: netbird_proxy
-  become: true
-  # One at a time so at least one relay stays reachable while the other
-  # restarts. The server is already on the new secret at this point, so a
-  # proxy is broken only between its own template write and its restart.
-  serial: 1
-
-  vars:
-    netbird_relay_secret: "{{ hostvars['localhost']['new_relay_secret'] }}"
-
-  roles:
-    - netbird-proxy
-
- name: Report what still needs doing by hand
-  hosts: localhost
-  connection: local
-  gather_facts: false
-
-  tasks:
-    - name: Remaining manual steps
-      ansible.builtin.debug:
-        msg:
-          - "Rotated: turn_secret and relay_secret."
-          - "Commit inventory/group_vars/all/netbird-rotatable.yml — it is encrypted."
-          - "NOT rotated: datastore_key (needs dump/restore), auth_client_secret (Authentik side)."
-          - "OpenBao is unaffected by design — its NetBird credentials are separate ones."
@@ -0,0 +1,10 @@
+---
+- name: Update Authentik HA
+  hosts: authentik_app
+  become: true
+  serial: 1
+  vars:
+    authentik_path: /home/ladmin
+
+  roles:
+    - authentik-update
@@ -0,0 +1,13 @@
+---
+- name: Update Docker Containers
+  hosts: proxmox_docker
+  become: true
+  gather_facts: false
+
+  vars:
+    # Define the root directory where your docker-compose files live.
+    # You can override this per-host in host_vars if they differ.
+    docker_compose_dir: "/opt" 
+
+  roles:
+    - docker-update
@@ -0,0 +1,32 @@
+---
+- name: Update LXC Containers
+  hosts: proxmox_all_lxc:!proxmox_noansible
+  become: true
+  gather_facts: true
+  
+  tasks:
+    - name: Update APT Cache
+      ansible.builtin.apt:
+        update_cache: true
+        cache_valid_time: 3600
+      when: ansible_pkg_mgr == 'apt'
+
+    - name: Upgrade all packages
+      ansible.builtin.apt:
+        upgrade: dist
+      when: ansible_pkg_mgr == 'apt'
+      register: apt_upgrade
+
+    - name: Check if reboot required
+      ansible.builtin.stat:
+        path: /var/run/reboot-required
+      register: reboot_required_file
+      when: ansible_pkg_mgr == 'apt'
+
+    - name: Reboot if required
+      ansible.builtin.reboot:
+        msg: "Rebooting due to system updates"
+      when: 
+        - ansible_pkg_mgr == 'apt'
+        - reboot_required_file.stat.exists
+        - apt_upgrade.changed
@@ -0,0 +1,9 @@
+---
+- name: Proxmox VE Rolling Update
+  hosts: proxmox_nodes  # Replace with your inventory group name
+  become: true
+  serial: 1           # CRITICAL: Only update one node at a time
+  any_errors_fatal: true
+  
+  roles:
+    - pve-update
@@ -0,0 +1,9 @@
+---
+- name: Update Proxmox VMs (Linux)
+  hosts: proxmox_all_qemu:&status_running:!proxmox_noansible:!proxmox_eol
+  become: true
+  serial: 1
+  gather_facts: false # We gather manually after checking connection
+
+  roles:
+    - vm-update
@@ -0,0 +1,27 @@
+---
+- name: Full Infrastructure Update
+  hosts: localhost
+  gather_facts: false
+  tasks:
+    - name: Banner
+      debug:
+        msg: "Starting Full Infrastructure Update Sequence"
+
+- import_playbook: update-vm.yml
+- import_playbook: update-lxc.yml
+- import_playbook: update-authentik.yml
+# Re-render docker-compose templates before pulling updates
+# (immich decommissioned — migrated to in-cluster deployment, see kubernetes/apps/immich/)
+- import_playbook: install-n8n.yml
+- import_playbook: install-grist.yml
+- import_playbook: install-music-assistant.yml
+- import_playbook: update-docker.yml
+- import_playbook: update-pve-rolling.yml
+
+- name: Update Summary
+  hosts: localhost
+  gather_facts: false
+  tasks:
+    - name: Done
+      debug:
+        msg: "Full Infrastructure Update Completed Successfully"
@@ -0,0 +1,18 @@
+---
+authentik_path: /home/ladmin
+authentik_db_host: "authentik-db.renner.uno"
+authentik_db_user: "authentik"
+authentik_db_name: "authentik"
+authentik_db_password: "changeMeInVault"
+authentik_secret_key: "changeMeInVault"
+authentik_tag: "latest" # or "2025.10.x"
+
+authentik_port_http: 9000
+authentik_port_https: 9443
+
+authentik_email_host: "smtp.example.com"
+authentik_email_port: 587
+authentik_email_user: "user"
+authentik_email_password: "password"
+authentik_email_use_tls: true
+authentik_email_from: "authentik@example.com"
@@ -0,0 +1,5 @@
+---
+- name: Restart Authentik
+  community.docker.docker_compose_v2: 
+    project_src: "{{ authentik_path }}"
+    state: restarted
@@ -0,0 +1,42 @@
+---
+- name: Ensure Authentik directory exists
+  ansible.builtin.file:
+    path: "{{ authentik_path }}"
+    state: directory
+    owner: "{{ ansible_user | default('root') }}"
+    group: "{{ ansible_user | default('root') }}"
+
+- name: Get latest Authentik release if no tag specified
+  ansible.builtin.uri:
+    url: https://api.github.com/repos/goauthentik/authentik/releases/latest
+    return_content: true
+  register: github_response
+  when: authentik_tag is not defined
+
+- name: Set target version fact
+  set_fact:
+    authentik_version: "{{ github_response.json.tag_name | regex_replace('^version/', '') if authentik_tag is not defined else authentik_tag }}"
+
+- name: Get Docker group ID
+  ansible.builtin.shell: "getent group docker | cut -d: -f3"
+  register: docker_gid_result
+  changed_when: false
+
+- name: Deploy .env configuration
+  ansible.builtin.template:
+    src: .env.j2
+    dest: "{{ authentik_path }}/.env"
+    mode: '0600'
+  notify: Restart Authentik
+
+- name: Deploy Docker Compose file
+  ansible.builtin.template:
+    src: docker-compose.yml.j2
+    dest: "{{ authentik_path }}/docker-compose.yml"
+  notify: Restart Authentik
+
+- name: Ensure Authentik is started
+  community.docker.docker_compose_v2:
+    project_src: "{{ authentik_path }}"
+    state: present
+    pull: always
@@ -0,0 +1,38 @@
+PG_PASS={{ authentik_db_password }}
+PG_HOST={{ authentik_db_host }}
+PG_USER={{ authentik_db_user }}
+PG_DB={{ authentik_db_name }}
+AUTHENTIK_SECRET_KEY={{ authentik_secret_key }}
+AUTHENTIK_ERROR_REPORTING__ENABLED={{ authentik_error_reporting_enabled | default('true') }}
+AUTHENTIK_EMAIL__HOST={{ authentik_email_host }}
+AUTHENTIK_EMAIL__PORT={{ authentik_email_port }}
+AUTHENTIK_EMAIL__USERNAME={{ authentik_email_username }}
+AUTHENTIK_EMAIL__PASSWORD={{ authentik_email_password }}
+AUTHENTIK_EMAIL__USE_TLS={{ authentik_email_use_tls | lower }}
+AUTHENTIK_EMAIL__USE_SSL={{ authentik_email_use_ssl | lower }}
+AUTHENTIK_EMAIL__TIMEOUT={{ authentik_email_timeout }}
+AUTHENTIK_EMAIL__FROM={{ authentik_email_from }}
+AUTHENTIK_LOG_LEVEL={{ authentik_log_level | default('info') }}
+AUTHENTIK_LISTEN__TRUSTED_PROXY_CIDRS={{ authentik_listen_trusted_proxy_cidrs | default('10.0.0.0/8,172.16.0.0/12') }}
+AUTHENTIK_TAG={{ authentik_tag | default(authentik_version) }}
+AUTHENTIK_ERROR_REPORTING__ENABLED=true
+AUTHENTIK_EMAIL__HOST={{ authentik_email_host }}
+AUTHENTIK_EMAIL__PORT={{ authentik_email_port }}
+AUTHENTIK_EMAIL__USERNAME={{ authentik_email_user }}
+AUTHENTIK_EMAIL__PASSWORD={{ authentik_email_password }}
+AUTHENTIK_EMAIL__USE_TLS={{ authentik_email_use_tls }}
+AUTHENTIK_EMAIL__FROM={{ authentik_email_from }}
+AUTHENTIK_PORT_HTTP={{ authentik_port_http }}
+AUTHENTIK_PORT_HTTPS={{ authentik_port_https }}
+AUTHENTIK_CSRF_TRUSTED_ORIGINS=https://authentik.renner.uno,https://auth.renner.uno,https://authentik-app-1.srv.renner.uno,https://authentik-app-2.srv.renner.uno
+
+# HA specific config:
+# Disable Redis (Use Postgres)
+AUTHENTIK_REDIS__HOST=""
+# Worker needs to use shared storage for media
+# Assuming nfs/glusterfs or just volume syncing?
+# Or maybe just use local volume if media uploads ar rare?
+# For true stateless, external S3 storage is recommended:
+# AUTHENTIK_STORAGE__MEDIA__BACKEND="aws"
+# AUTHENTIK_STORAGE__MEDIA__S3__BUCKET="..."
+# For now, local volume (will not sync across nodes without NFS).
@@ -0,0 +1,41 @@
+services:
+  server:
+    image: ghcr.io/goauthentik/server:${AUTHENTIK_TAG:-latest}
+    restart: unless-stopped
+    command: server
+    env_file:
+      - .env
+    environment:
+      AUTHENTIK_REDIS__HOST: ""
+      AUTHENTIK_POSTGRESQL__HOST: ${PG_HOST}
+      AUTHENTIK_POSTGRESQL__NAME: ${PG_DB}
+      AUTHENTIK_POSTGRESQL__USER: ${PG_USER}
+      AUTHENTIK_POSTGRESQL__PASSWORD: ${PG_PASS}
+      AUTHENTIK_LISTEN__METRICS: "0.0.0.0:9300"
+    volumes:
+      - ./media:/media
+      - ./custom-templates:/templates
+      - ./certs:/certs
+    ports:
+      - "${AUTHENTIK_PORT_HTTP:-9000}:9000"
+      - "${AUTHENTIK_PORT_HTTPS:-9443}:9443"
+      - "9300:9300"
+
+  worker:
+    image: ghcr.io/goauthentik/server:${AUTHENTIK_TAG:-latest}
+    restart: unless-stopped
+    command: worker
+    user: "1000:{{ docker_gid_result.stdout }}"
+    env_file:
+      - .env
+    environment:
+      AUTHENTIK_REDIS__HOST: ""
+      AUTHENTIK_POSTGRESQL__HOST: ${PG_HOST}
+      AUTHENTIK_POSTGRESQL__NAME: ${PG_DB}
+      AUTHENTIK_POSTGRESQL__USER: ${PG_USER}
+      AUTHENTIK_POSTGRESQL__PASSWORD: ${PG_PASS}
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+      - ./media:/media
+      - ./certs:/certs
+      - ./custom-templates:/templates
@@ -0,0 +1,47 @@
+---
+- name: Get latest Authentik release
+  ansible.builtin.uri:
+    url: https://api.github.com/repos/goauthentik/authentik/releases/latest
+    return_content: true
+  register: github_response
+
+- name: Set target version
+  ansible.builtin.set_fact:
+    target_version: "{{ github_response.json.tag_name | regex_replace('^version/', '') }}"
+
+- name: Display target version
+  ansible.builtin.debug:
+    msg: "Updating Authentik to version {{ target_version }}"
+
+- name: Update AUTHENTIK_TAG in .env
+  ansible.builtin.lineinfile:
+    path: "{{ authentik_path | default('/home/ladmin') }}/.env"
+    regexp: '^AUTHENTIK_TAG='
+    line: "AUTHENTIK_TAG={{ target_version }}"
+    create: true
+
+- name: Pull new images
+  ansible.builtin.command:
+    cmd: docker compose pull
+    chdir: "{{ authentik_path | default('/home/ladmin') }}"
+
+- name: Pull Outpost images
+  ansible.builtin.shell: |
+    docker pull ghcr.io/goauthentik/proxy:{{ target_version }}
+    docker pull ghcr.io/goauthentik/ldap:{{ target_version }}
+    docker pull ghcr.io/goauthentik/radius:{{ target_version }}
+  changed_when: true
+
+- name: Apply updates
+  ansible.builtin.command:
+    cmd: docker compose up -d --force-recreate
+    chdir: "{{ authentik_path | default('/home/ladmin') }}"
+
+- name: Restart worker to trigger outpost update
+  ansible.builtin.command: 
+    cmd: docker compose restart worker
+    chdir: "{{ authentik_path | default('/home/ladmin') }}"
+
+- name: Prune unused images
+  ansible.builtin.command:
+    cmd: docker image prune -f
@@ -169,7 +169,8 @@ Merge the PR, then trigger the pipeline (Woodpecker → Run → `PIPELINE ==
 "deploy-claude-workstation"`), or push a change under the role's paths. CI SSHes
 to `10.3.40.60` over the tunnel and applies `common`, `netbird-client`, and this
 role. Also run the umbrella `plays/common.yml` against the new host once, so it
-picks up `qemu-guest-agent` like every other inventory host.
+picks up `qemu-guest-agent` and `monitoring-agent` (Alloy → Loki/Prometheus)
+like every other inventory host.

 ---

@@ -3,8 +3,8 @@
 # Debian 12 VM on the off-site nas-2 (Synology VMM). See README for the manual
 # bootstrap (VM creation, Netbird route, secret material) that precedes this.
 #
-# common.yml already applies `common` and `qemu-guest-agent`
-# to every inventory host, so this role owns only the
+# common.yml already applies `common`, `qemu-guest-agent` and `monitoring-agent`
+# (Alloy -> Loki/Prometheus) to every inventory host, so this role owns only the
 # Claude-specific surface: toolchain, the two credential-isolated users, the
 # read-only creds, notes sync, the unattended timers, and egress lockdown.

@@ -1,56 +0,0 @@
---
-# CrowdSec firewall bouncer on the NetBird proxy hosts.
-#
-# Enforces the decisions the in-cluster CrowdSec already detects, at the point
-# where public traffic actually enters: these hosts terminate TLS for every
-# *.renner.uno name. A drop here never reaches the mesh, the relay or the
-# cluster.
-#
-# Detection stays in the cluster. The reverse-proxy produces no parseable
-# access log and there is no CrowdSec collection for it, while Traefik has a
-# working one — and every scenario in use needs HTTP detail (paths, status
-# codes, user agents) that does not exist at this layer.
-
-# Pinned. A release tarball URL is immutable, and this checksum is of the
-# amd64 asset for exactly this tag.
-crowdsec_bouncer_version: v0.0.36
-crowdsec_bouncer_sha256: f86e4b72693549d99f40a9402abefb894108f047a3fbe6e72fada25ee17ce88b
-
-# The LAPI over the public path these hosts already serve, see
-# kubernetes/infrastructure/crowdsec/lapi-ingress.yaml. The proxies cannot
-# reach it over the mesh: the reverse-proxy container runs non-root, so its
-# embedded NetBird client terminates the tunnel in userspace and the tunnel
-# belongs to that process rather than to the host netns.
-#
-# No ca_cert_path: Traefik serves a Let's Encrypt certificate for this name,
-# and the bouncer adds ca_cert_path to the system pool rather than replacing
-# it — the system pool already trusts LE.
-crowdsec_bouncer_api_url: https://crowdsec.renner.uno/
-crowdsec_bouncer_api_key: "{{ vault_crowdsec_edge_bouncer_key }}"
-
-# Enforcing. The bouncer creates two nftables tables (ip crowdsec, ip6
-# crowdsec6) holding the decisions it pulled, touches no existing chain and
-# sets no chain policy — SSH and :443 stay open unless the source is banned.
-#
-# `dry-run` is the other value: it pulls and logs what it would block without
-# creating a single rule. That is what a new host should be given before it
-# enforces, and what an existing one can be dropped back to, per host through
-# inventory/host_vars, without disarming the rest.
-#
-# Worth knowing before arming anything new: the first bouncer on a path
-# enforces the whole CAPI set at once — around 15k addresses today — so the
-# false-positive window belongs before the switch, not after it. And because
-# this chain sits on the INPUT hook while these hosts run no other firewall,
-# there is no conntrack rule accepting established flows ahead of it: a banned
-# address is dropped even when it is answering our own outbound connection.
-# That is why capi_whitelists.yaml carries the proxies and netbird.renner.uno.
-crowdsec_bouncer_mode: nftables
-
-crowdsec_bouncer_update_frequency: 10s
-crowdsec_bouncer_log_level: info
-
-# Bound to loopback deliberately. These hosts are public and have no metrics
-# collection path at all today — opening :60601 to the internet to gain a
-# scrape target is the wrong trade. Revisit when host metrics exist.
-crowdsec_bouncer_metrics_listen: 127.0.0.1
-crowdsec_bouncer_metrics_port: 60601
@@ -1,10 +0,0 @@
---
-# A restart drops both nftables tables and rebuilds them from the next
-# successful LAPI pull, so there is a short window with no blocklist. That is
-# the documented behaviour of a clean shutdown, not a defect — and it is why
-# retry_initial_connect matters.
- name: Restart crowdsec firewall bouncer
-  ansible.builtin.systemd:
-    name: crowdsec-firewall-bouncer.service
-    state: restarted
-    daemon_reload: true
@@ -1,111 +0,0 @@
---
-# See defaults/main.yml for why this exists and why it starts inert.
-#
-# The static release tarball, not a distribution package: the AUR package
-# `crowdsec-firewall-bouncer-nftables` depends on `crowdsec` and would pull the
-# whole Security Engine onto hosts that only need the bouncer, the group is
-# mixed (proxy-1 Arch, VPS-1 Debian), and the AUR would be a new pattern this
-# repo does not otherwise have. The tarball is the same artefact the AUR
-# package downloads.
-
- name: Ensure nftables is present (the enforcing backend)
-  ansible.builtin.package:
-    name: nftables
-    state: present
-
- name: Ensure the bouncer config directory exists
-  ansible.builtin.file:
-    path: /etc/crowdsec/bouncers
-    state: directory
-    mode: "0755"
-
-# Fetched on the CONTROLLER and pushed, not pulled by the host.
-#
-# proxy-1 is IPv6-only, and neither github.com nor objects.githubusercontent.com
-# publishes an AAAA record — a get_url on the host fails with "Network is
-# unreachable" and no amount of retrying fixes it. Mirroring into
-# gitea.renner.uno is no help either: its public records are the internal LB
-# VIP (10.2.99.86 / fd99::86), so it is unreachable from outside the LAN. And
-# upstream ships no container image for this bouncer, which is expected —
-# it manipulates host netfilter.
-#
-# The CI runner has v4, so it does the fetching. The checksum is verified
-# there, before anything is copied to a host.
- name: Fetch the pinned bouncer release on the controller
-  ansible.builtin.get_url:
-    url: >-
-      https://github.com/crowdsecurity/cs-firewall-bouncer/releases/download/{{
-      crowdsec_bouncer_version }}/crowdsec-firewall-bouncer-linux-amd64.tgz
-    dest: /tmp/crowdsec-firewall-bouncer-{{ crowdsec_bouncer_version }}.tgz
-    checksum: sha256:{{ crowdsec_bouncer_sha256 }}
-    mode: "0644"
-  delegate_to: localhost
-  become: false
-  run_once: true
-
-# Only the download had to move to the controller — the extraction stays on
-# the host. The ci-ansible image ships a tar that ansible rejects ("detected as
-# tar type None. GNU tar required"), while both host distributions have GNU
-# tar. Shipping the archive rather than the unpacked binary also keeps the
-# artefact that the checksum was verified against intact all the way to the
-# host.
- name: Copy the bouncer archive to the host
-  ansible.builtin.copy:
-    src: /tmp/crowdsec-firewall-bouncer-{{ crowdsec_bouncer_version }}.tgz
-    dest: /opt/crowdsec-firewall-bouncer-{{ crowdsec_bouncer_version }}.tgz
-    mode: "0644"
-
-# Only the binary. The tarball also carries install.sh / upgrade.sh /
-# uninstall.sh, which manage systemd and config themselves — that is this
-# role's job, and letting them run would leave two owners for the same files.
- name: Unpack the bouncer binary
-  ansible.builtin.unarchive:
-    src: /opt/crowdsec-firewall-bouncer-{{ crowdsec_bouncer_version }}.tgz
-    dest: /opt
-    remote_src: true
-    include:
-      - crowdsec-firewall-bouncer-{{ crowdsec_bouncer_version }}/crowdsec-firewall-bouncer
-    creates: /opt/crowdsec-firewall-bouncer-{{ crowdsec_bouncer_version }}/crowdsec-firewall-bouncer
-
- name: Install the bouncer binary
-  ansible.builtin.copy:
-    src: /opt/crowdsec-firewall-bouncer-{{ crowdsec_bouncer_version }}/crowdsec-firewall-bouncer
-    dest: /usr/local/bin/crowdsec-firewall-bouncer
-    remote_src: true
-    mode: "0755"
-  notify: Restart crowdsec firewall bouncer
-
-# 0600: this file carries the LAPI key.
- name: Deploy the bouncer configuration
-  ansible.builtin.template:
-    src: crowdsec-firewall-bouncer.yaml.j2
-    dest: /etc/crowdsec/bouncers/crowdsec-firewall-bouncer.yaml
-    mode: "0600"
-  notify: Restart crowdsec firewall bouncer
-
- name: Deploy the bouncer systemd unit
-  ansible.builtin.template:
-    src: crowdsec-firewall-bouncer.service.j2
-    dest: /etc/systemd/system/crowdsec-firewall-bouncer.service
-    mode: "0644"
-  notify: Restart crowdsec firewall bouncer
-
- name: Enable and start the bouncer
-  ansible.builtin.systemd:
-    name: crowdsec-firewall-bouncer.service
-    enabled: true
-    state: started
-    daemon_reload: true
-
- name: Report how to read it
-  ansible.builtin.debug:
-    msg: |
-      crowdsec-firewall-bouncer deployed in mode "{{ crowdsec_bouncer_mode }}".
-
-      What it would block:  journalctl -u crowdsec-firewall-bouncer -f
-      What it does block:   nft list table ip crowdsec ; nft list table ip6 crowdsec6
-      Metrics:              curl -s {{ crowdsec_bouncer_metrics_listen }}:{{ crowdsec_bouncer_metrics_port }}/metrics
-
-      Rollback: systemctl stop crowdsec-firewall-bouncer. A clean stop deletes
-      both nftables tables by itself. Only after a kill -9 do they need
-      removing by hand: nft delete table ip crowdsec ; nft delete table ip6 crowdsec6
@@ -1,24 +0,0 @@
-# Managed by Ansible (crowdsec-edge-bouncer role) — do not edit.
-#
-# Upstream's shipped unit orders itself After=crowdsec.service. There is no
-# local Security Engine on these hosts — the LAPI is in the cluster — so that
-# dependency is dropped; nss-lookup instead, because api_url is a public name
-# that has to resolve before the first connect.
-[Unit]
-Description=CrowdSec firewall bouncer (edge enforcement for the NetBird proxy)
-After=network-online.target nss-lookup.target
-Wants=network-online.target
-
-[Service]
-Type=notify
-# -t validates the config and exits non-zero on a bad one, so a broken config
-# fails the start instead of taking the running rules down with it.
-ExecStartPre=/usr/local/bin/crowdsec-firewall-bouncer -c /etc/crowdsec/bouncers/crowdsec-firewall-bouncer.yaml -t
-ExecStart=/usr/local/bin/crowdsec-firewall-bouncer -c /etc/crowdsec/bouncers/crowdsec-firewall-bouncer.yaml
-Restart=always
-RestartSec=10
-LimitNOFILE=65536
-KillMode=mixed
-
-[Install]
-WantedBy=multi-user.target
@@ -1,55 +0,0 @@
-# Managed by Ansible (crowdsec-edge-bouncer role) — do not edit.
-mode: {{ crowdsec_bouncer_mode }}
-update_frequency: {{ crowdsec_bouncer_update_frequency }}
-
-# stdout, so journald owns the log and there is no second rotation policy to
-# keep in sync. The shipped example writes to /var/log; that only matters for
-# hosts without journald.
-log_mode: stdout
-log_level: {{ crowdsec_bouncer_log_level }}
-
-api_url: {{ crowdsec_bouncer_api_url }}
-api_key: {{ crowdsec_bouncer_api_key }}
-
-# Without this the process EXITS when the LAPI is unreachable at startup —
-# there is no retry on the first connect. And because a clean shutdown deletes
-# the nftables tables, the systemd restart loop would leave the host with no
-# blocklist at all until the first successful connect. Present in the code,
-# absent from the shipped example config.
-retry_initial_connect: true
-
-disable_ipv4: false
-disable_ipv6: false
-deny_action: DROP
-
-supported_decisions_types:
-  - ban
-
-# Separate tables per family; the bouncer never touches an existing table or
-# chain, and never sets a chain policy. Priority -10 puts it ahead of anything
-# at the default filter priority.
-nftables:
-  ipv4:
-    enabled: true
-    set-only: false
-    table: crowdsec
-    chain: crowdsec-chain
-    priority: -10
-  ipv6:
-    enabled: true
-    set-only: false
-    table: crowdsec6
-    chain: crowdsec6-chain
-    priority: -10
-
-# input is the traffic this exists for: the reverse-proxy is host-networked and
-# owns :443, so its packets traverse INPUT. forward covers the bridge-networked
-# http-redirect sidecar, whose packets never do.
-nftables_hooks:
-  - input
-  - forward
-
-prometheus:
-  enabled: true
-  listen_addr: {{ crowdsec_bouncer_metrics_listen }}
-  listen_port: {{ crowdsec_bouncer_metrics_port }}
@@ -1,8 +1,8 @@
 ---
 # Journal -> Loki shipping.
 #
-# The srv-network Loki (loki.srv.renner.uno:3100) no longer exists — it moved
-# into the cluster. The only endpoint these hosts can reach is the authenticated
+# The srv-network Loki (loki.srv.renner.uno:3100, no auth) is firewalled off
+# the IoT VLAN; the only endpoint these hosts can reach is the authenticated
 # cluster ingress. edge_loki_password comes from the vault.

 edge_alloy_config_file: /etc/alloy/edge.alloy
@@ -7,8 +7,8 @@
 // Loki. Its own WAL/positions live on /run (tmpfs, see edge-alloy.service),
 // so this agent writes nothing to the eMMC either.
 //
-// The srv-network Loki (loki.srv.renner.uno:3100) no longer exists — it moved
-// into the cluster. The only reachable endpoint is the cluster
+// The srv-network Loki (loki.srv.renner.uno:3100, unauthenticated) is not
+// reachable from the IoT VLAN; the only reachable endpoint is the cluster
 // ingress, which enforces Traefik basic-auth — hence the credential below.

 loki.relabel "journal" {
@@ -0,0 +1,15 @@
+---
+gatus_version: "v5.35.0"
+gatus_port: 8080
+gatus_data_dir: "/opt/gatus"
+
+# SMTP configuration (override in vault)
+gatus_smtp_host: ""
+gatus_smtp_port: 587
+gatus_smtp_from: ""
+gatus_smtp_username: ""
+gatus_smtp_password: ""
+gatus_alert_email: ""
+
+# Webhook URL for outage notifications (override in vault or group_vars)
+gatus_webhook_url: ""
@@ -0,0 +1,5 @@
+---
+- name: Restart Gatus
+  community.docker.docker_compose_v2:
+    project_src: "{{ gatus_data_dir }}"
+    state: restarted
@@ -0,0 +1,29 @@
+---
+- name: Install required system packages
+  ansible.builtin.apt:
+    name:
+      - python3-docker
+    state: present
+    update_cache: true
+
+- name: Create Gatus data directory
+  ansible.builtin.file:
+    path: "{{ gatus_data_dir }}"
+    state: directory
+    mode: '0755'
+
+- name: Copy Gatus configuration files
+  ansible.builtin.template:
+    src: "{{ item.src }}"
+    dest: "{{ gatus_data_dir }}/{{ item.dest }}"
+    mode: '0644'
+  loop:
+    - { src: 'config.yaml.j2', dest: 'config.yaml' }
+    - { src: 'docker-compose.yml.j2', dest: 'docker-compose.yml' }
+  notify: Restart Gatus
+
+- name: Start Gatus via Docker Compose
+  community.docker.docker_compose_v2:
+    project_src: "{{ gatus_data_dir }}"
+    state: present
+    pull: always
@@ -0,0 +1,282 @@
+storage:
+  type: sqlite
+  path: /data/gatus.db
+
+ui:
+  title: Homelab Status
+  description: Service health monitoring for renner.uno
+
+{% if gatus_smtp_host %}
+alerting:
+  email:
+    host: {{ gatus_smtp_host }}
+    port: {{ gatus_smtp_port }}
+    from: {{ gatus_smtp_from }}
+    username: {{ gatus_smtp_username }}
+    password: {{ gatus_smtp_password }}
+    to: {{ gatus_alert_email }}
+    default-alert:
+      enabled: true
+      failure-threshold: 3
+      success-threshold: 2
+      send-on-resolved: true
+{% if gatus_webhook_url %}
+  custom:
+    url: {{ gatus_webhook_url }}
+    method: POST
+    body: |
+      {
+        "service": "[ALERT_TRIGGERED_OR_RESOLVED]",
+        "description": "[ENDPOINT_NAME] - [RESULT_ERRORS]",
+        "status": "[ALERT_TRIGGERED_OR_RESOLVED]"
+      }
+    default-alert:
+      enabled: true
+      failure-threshold: 3
+      success-threshold: 2
+      send-on-resolved: true
+{% endif %}
+{% endif %}
+
+endpoints:
+  # Web Services
+  - name: ownCloud
+    group: services
+    url: https://owncloud.renner.uno/ocs/v2.php/cloud/capabilities?format=json
+    interval: 60s
+    conditions:
+      - "[STATUS] == 200"
+      - "[RESPONSE_TIME] < 2000"
+      - "[BODY].ocs.meta.status == ok"
+    alerts:
+      - type: email
+{% if gatus_webhook_url %}
+      - type: custom
+{% endif %}
+
+  - name: ONLYOFFICE
+    group: services
+    url: https://office.renner.uno/healthcheck
+    interval: 60s
+    conditions:
+      - "[STATUS] == 200"
+      - "[BODY] == true"
+    alerts:
+      - type: email
+{% if gatus_webhook_url %}
+      - type: custom
+{% endif %}
+
+  - name: Authentik
+    group: services
+    url: https://auth.renner.uno/-/health/ready/
+    interval: 60s
+    conditions:
+      - "[STATUS] == 200"
+      - "[RESPONSE_TIME] < 2000"
+    alerts:
+      - type: email
+{% if gatus_webhook_url %}
+      - type: custom
+{% endif %}
+
+  - name: Gitea
+    group: services
+    url: https://gitea.renner.uno/api/v1/settings/api
+    interval: 60s
+    conditions:
+      - "[STATUS] == 200"
+      - "[RESPONSE_TIME] < 2000"
+    alerts:
+      - type: email
+{% if gatus_webhook_url %}
+      - type: custom
+{% endif %}
+
+  - name: Immich
+    group: services
+    url: https://immich.renner.uno/api/server/ping
+    interval: 60s
+    conditions:
+      - "[STATUS] == 200"
+      - "[BODY].res == pong"
+    alerts:
+      - type: email
+{% if gatus_webhook_url %}
+      - type: custom
+{% endif %}
+
+  - name: Woodpecker CI
+    group: services
+    url: https://ci.renner.uno/healthz
+    interval: 60s
+    conditions:
+      - "[STATUS] == any(200, 204)"
+    alerts:
+      - type: email
+{% if gatus_webhook_url %}
+      - type: custom
+{% endif %}
+
+  - name: Grafana
+    group: services
+    url: https://dash.renner.uno/api/health
+    interval: 60s
+    conditions:
+      - "[STATUS] == 200"
+      - "[BODY].database == ok"
+    alerts:
+      - type: email
+{% if gatus_webhook_url %}
+      - type: custom
+{% endif %}
+
+  - name: Home Assistant
+    group: services
+    url: https://home.renner.uno
+    interval: 60s
+    conditions:
+      - "[STATUS] == any(200, 301, 302)"
+    alerts:
+      - type: email
+{% if gatus_webhook_url %}
+      - type: custom
+{% endif %}
+
+  - name: Paperless
+    group: services
+    url: https://file.renner.uno
+    interval: 60s
+    conditions:
+      - "[STATUS] == any(200, 301, 302)"
+    alerts:
+      - type: email
+{% if gatus_webhook_url %}
+      - type: custom
+{% endif %}
+
+  - name: Audiobookshelf
+    group: services
+    url: https://audiobooks.renner.uno/healthcheck
+    interval: 60s
+    conditions:
+      - "[STATUS] == 200"
+    alerts:
+      - type: email
+{% if gatus_webhook_url %}
+      - type: custom
+{% endif %}
+
+  - name: Jellyfin
+    group: services
+    url: https://media.renner.uno/health
+    interval: 60s
+    conditions:
+      - "[STATUS] == 200"
+    alerts:
+      - type: email
+{% if gatus_webhook_url %}
+      - type: custom
+{% endif %}
+
+  - name: Wiki
+    group: services
+    url: https://wiki.renner.uno
+    interval: 60s
+    conditions:
+      - "[STATUS] == 200"
+    alerts:
+      - type: email
+{% if gatus_webhook_url %}
+      - type: custom
+{% endif %}
+
+  - name: Music Assistant
+    group: services
+    url: https://music.renner.uno
+    interval: 60s
+    conditions:
+      - "[STATUS] == any(200, 301, 302)"
+    alerts:
+      - type: email
+{% if gatus_webhook_url %}
+      - type: custom
+{% endif %}
+
+  - name: InfluxDB
+    group: services
+    url: https://influx.renner.uno/health
+    interval: 60s
+    conditions:
+      - "[STATUS] == 200"
+    alerts:
+      - type: email
+{% if gatus_webhook_url %}
+      - type: custom
+{% endif %}
+
+  - name: PBS (pbs.obg.renner.uno)
+    group: infrastructure
+    url: https://pbs.obg.renner.uno:8007
+    interval: 60s
+    client:
+      insecure-skip-verify: true
+    conditions:
+      - "[STATUS] == any(200, 301, 302)"
+
+  - name: PBS 2 (pbs2.renner.uno)
+    group: infrastructure
+    url: https://pbs2.renner.uno:8007
+    interval: 60s
+    client:
+      insecure-skip-verify: true
+    conditions:
+      - "[STATUS] == any(200, 301, 302)"
+    alerts:
+      - type: email
+{% if gatus_webhook_url %}
+      - type: custom
+{% endif %}
+
+  - name: Proxmox VE
+    group: infrastructure
+    url: https://pve.renner.uno
+    interval: 60s
+    client:
+      insecure-skip-verify: true
+    conditions:
+      - "[STATUS] == any(200, 301, 302)"
+    alerts:
+      - type: email
+{% if gatus_webhook_url %}
+      - type: custom
+{% endif %}
+
+  # Infrastructure
+  - name: Proxmox DNS (VIP)
+    group: infrastructure
+    url: 10.2.40.53
+    interval: 30s
+    dns:
+      query-name: k8s-1.srv.renner.uno
+      query-type: A
+    conditions:
+      - "[DNS_RCODE] == NOERROR"
+      - "[RESPONSE_TIME] < 500"
+    alerts:
+      - type: email
+{% if gatus_webhook_url %}
+      - type: custom
+{% endif %}
+
+  - name: PostgreSQL HA (VIP)
+    group: infrastructure
+    url: tcp://10.2.40.135:5432
+    interval: 30s
+    conditions:
+      - "[CONNECTED] == true"
+    alerts:
+      - type: email
+{% if gatus_webhook_url %}
+      - type: custom
+{% endif %}
@@ -0,0 +1,17 @@
+services:
+  gatus:
+    container_name: gatus
+    image: twinproduction/gatus:{{ gatus_version }}
+    ports:
+      - "{{ gatus_port }}:8080"
+    volumes:
+      - {{ gatus_data_dir }}/config.yaml:/config/config.yaml:ro
+      - {{ gatus_data_dir }}/data:/data
+      - /etc/localtime:/etc/localtime:ro
+    restart: always
+    healthcheck:
+      test: ["CMD", "wget", "-q", "--spider", "http://localhost:8080/health"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 10s
@@ -0,0 +1,28 @@
+---
+# Grafana Authentik OAuth Configuration
+
+# Authentik OAuth settings (must be set in vault)
+# Add to ansible/inventory/group_vars/all/vault.yml:
+#   vault_grafana_authentik_client_id: "<client-id>"
+#   vault_grafana_authentik_client_secret: "<client-secret>"
+grafana_authentik_client_id: "{{ vault_grafana_authentik_client_id | default('') }}"
+grafana_authentik_client_secret: "{{ vault_grafana_authentik_client_secret | default('') }}"
+
+# Authentik URLs
+authentik_base_url: "https://auth.renner.uno"
+grafana_authentik_auth_url: "{{ authentik_base_url }}/application/o/authorize/"
+grafana_authentik_token_url: "{{ authentik_base_url }}/application/o/token/"
+grafana_authentik_api_url: "{{ authentik_base_url }}/application/o/userinfo/"
+
+# Grafana settings
+grafana_root_url: "https://dash.renner.uno"
+grafana_config_dir: "/etc/grafana"
+
+# Role mapping from Authentik groups
+grafana_role_attribute_path: "contains(groups, 'Grafana Admins') && 'Admin' || contains(groups, 'Grafana Editors') && 'Editor' || 'Viewer'"
+
+# Allow users to sign up automatically
+grafana_oauth_allow_sign_up: true
+
+# Auto login (skip Grafana login page, go directly to Authentik)
+grafana_oauth_auto_login: false
@@ -0,0 +1,5 @@
+---
+- name: Restart grafana
+  systemd:
+    name: grafana-server
+    state: restarted
@@ -0,0 +1,48 @@
+---
+# Configure Grafana to use Authentik for OAuth authentication
+
+- name: Verify Authentik credentials are configured
+  assert:
+    that:
+      - grafana_authentik_client_id | length > 0
+      - grafana_authentik_client_secret | length > 0
+    fail_msg: |
+      Authentik OAuth credentials not configured!
+
+      Please add to ansible/inventory/group_vars/all/vault.yml:
+        vault_grafana_authentik_client_id: "<client-id-from-authentik>"
+        vault_grafana_authentik_client_secret: "<client-secret-from-authentik>"
+
+      To create the OAuth application in Authentik:
+      1. Go to https://auth.renner.uno/if/admin/
+      2. Applications > Providers > Create
+      3. Select "OAuth2/OpenID Provider"
+      4. Name: Grafana, Slug: grafana
+      5. Client type: Confidential
+      6. Redirect URIs: https://grafana.renner.uno/login/generic_oauth
+      6. Redirect URIs: https://dash.renner.uno/login/generic_oauth
+      7. Copy Client ID and Secret to vault
+
+- name: Ensure Grafana provisioning directory exists
+  file:
+    path: "{{ grafana_config_dir }}/provisioning"
+    state: directory
+    owner: grafana
+    group: grafana
+    mode: '0755'
+
+- name: Deploy Authentik OAuth configuration
+  template:
+    src: grafana-auth.ini.j2
+    dest: "{{ grafana_config_dir }}/grafana.ini"
+    owner: root
+    group: grafana
+    mode: '0640'
+    backup: yes
+  notify: Restart grafana
+
+- name: Ensure Grafana service is running
+  systemd:
+    name: grafana-server
+    state: started
+    enabled: yes
@@ -0,0 +1,26 @@
+# Grafana Authentik OAuth Configuration
+# Managed by Ansible - Do not edit manually
+
+[server]
+root_url = {{ grafana_root_url }}
+
+[auth]
+# Disable login form if using auto_login
+disable_login_form = {{ grafana_oauth_auto_login | lower }}
+
+[auth.generic_oauth]
+enabled = true
+name = Authentik
+icon = signin
+allow_sign_up = {{ grafana_oauth_allow_sign_up | lower }}
+auto_login = {{ grafana_oauth_auto_login | lower }}
+client_id = {{ grafana_authentik_client_id }}
+client_secret = {{ grafana_authentik_client_secret }}
+scopes = openid profile email
+auth_url = {{ grafana_authentik_auth_url }}
+token_url = {{ grafana_authentik_token_url }}
+api_url = {{ grafana_authentik_api_url }}
+role_attribute_path = {{ grafana_role_attribute_path }}
+role_attribute_strict = false
+allow_assign_grafana_admin = true
+use_pkce = true
@@ -0,0 +1,27 @@
+---
+# Grist configuration
+grist_version: "latest"
+grist_port: 8484
+grist_data_dir: "/opt/grist"
+
+# Public URL (required for proper link generation)
+grist_public_url: "https://grist.renner.uno"
+
+# Session secret (override in vault!)
+grist_session_secret: "CHANGE_ME_IN_VAULT"
+
+# Default admin email
+grist_default_email: "admin@renner.uno"
+
+# Sandbox flavor: gvisor for security, unsandboxed for compatibility
+grist_sandbox_flavor: "gvisor"
+grist_default_locale: "de"
+
+# OIDC Authentication (Authentik)
+grist_oidc_enabled: true
+grist_oidc_issuer: "https://auth.renner.uno/application/o/grist/"
+grist_oidc_client_id: ""
+grist_oidc_client_secret: ""
+grist_oidc_sp_profile_name_attr: "name"
+grist_oidc_sp_profile_email_attr: "email"
+grist_oidc_ignore_email_verified: true
@@ -0,0 +1,5 @@
+---
+- name: Restart Grist
+  community.docker.docker_compose_v2:
+    project_src: "{{ grist_data_dir }}"
+    state: restarted
@@ -0,0 +1,35 @@
+---
+- name: Install required system packages
+  ansible.builtin.apt:
+    name:
+      - python3-docker
+    state: present
+    update_cache: true
+
+- name: Create Grist data directory
+  ansible.builtin.file:
+    path: "{{ grist_data_dir }}"
+    state: directory
+    mode: '0755'
+
+- name: Create Grist persist directory
+  ansible.builtin.file:
+    path: "{{ grist_data_dir }}/persist"
+    state: directory
+    mode: '0755'
+
+- name: Copy Grist configuration files
+  ansible.builtin.template:
+    src: "{{ item.src }}"
+    dest: "{{ grist_data_dir }}/{{ item.dest }}"
+    mode: '0644'
+  loop:
+    - { src: '.env.j2', dest: '.env' }
+    - { src: 'docker-compose.yml.j2', dest: 'docker-compose.yml' }
+  notify: Restart Grist
+
+- name: Start Grist via Docker Compose
+  community.docker.docker_compose_v2:
+    project_src: "{{ grist_data_dir }}"
+    state: present
+    pull: always
@@ -0,0 +1,29 @@
+# Grist Environment Configuration
+# Managed by Ansible - Do not edit manually
+
+# Application URL
+APP_HOME_URL={{ grist_public_url }}
+
+# Security
+GRIST_SESSION_SECRET={{ grist_session_secret }}
+GRIST_SANDBOX_FLAVOR={{ grist_sandbox_flavor }}
+
+# Default admin
+GRIST_DEFAULT_EMAIL={{ grist_default_email }}
+
+{% if grist_oidc_enabled and grist_oidc_client_id %}
+# OIDC Authentication (Authentik)
+GRIST_OIDC_IDP_ENABLED=true
+GRIST_OIDC_IDP_ISSUER={{ grist_oidc_issuer }}
+GRIST_OIDC_IDP_CLIENT_ID={{ grist_oidc_client_id }}
+GRIST_OIDC_IDP_CLIENT_SECRET={{ grist_oidc_client_secret }}
+GRIST_OIDC_SP_PROFILE_NAME_ATTR={{ grist_oidc_sp_profile_name_attr }}
+GRIST_OIDC_SP_PROFILE_EMAIL_ATTR={{ grist_oidc_sp_profile_email_attr }}
+GRIST_OIDC_SP_IGNORE_EMAIL_VERIFIED={{ grist_oidc_ignore_email_verified | lower }}
+{% endif %}
+
+# Locale
+GRIST_DEFAULT_LOCALE={{ grist_default_locale }}
+
+# Telemetry (disabled)
+GRIST_TELEMETRY_LEVEL=off
@@ -0,0 +1,20 @@
+version: "3.8"
+
+services:
+  grist:
+    container_name: grist
+    image: gristlabs/grist:{{ grist_version }}
+    env_file:
+      - .env
+    ports:
+      - "{{ grist_port }}:8484"
+    volumes:
+      - {{ grist_data_dir }}/persist:/persist
+      - /etc/localtime:/etc/localtime:ro
+    restart: always
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8484/status"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 30s
@@ -0,0 +1,20 @@
+---
+- name: Ensure install directory exists
+  file:
+    path: /opt/ical-proxy
+    state: directory
+    mode: '0755'
+
+- name: Checkout ical-proxy repository
+  git:
+    repo: 'https://github.com/konairius/ical-proxy.git'
+    dest: /opt/ical-proxy
+    version: main
+    force: yes
+  register: git_checkout
+
+- name: Deploy ical-proxy with docker compose
+  community.docker.docker_compose_v2:
+    project_src: /opt/ical-proxy
+    state: present
+    build: always
--- a/Show More
+++ b/Show More