fix(llama-gate): name the authentication flow, as Audiobookshelf does

The gate set only authorization_flow, so authentication fell back to
Authentik's default flow and the step-up ran on top of a password prompt
instead of replacing it — a clumsy sign-in for a UI meant to be opened often.

Audiobookshelf and immich set both halves, and that is the pattern to copy for
a user-facing app:

  authentication_flow: passkey-passwordless-authentication
  authorization_flow:  passkey-stepup-authorization

Both are referenced with !Find rather than !KeyOf because they are defined in
the passkey-stepup document of this ConfigMap, and !KeyOf resolves only within
a single document.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

kubernetes/apps/authentik/blueprints-configmap.yaml

@@ -877,6 +877,105 @@ data:
             kubernetes_ingress_secret_name: "authentik-outpost-tls"
             kubernetes_json_patches: null
 
+      # llama-server WebUI gate
+      #
+      # llama-server has no authentication of its own — it serves its chat UI
+      # and a full OpenAI-compatible API to anyone who can reach the port. That
+      # is why it was LAN-only until now. This gate is what makes a public
+      # hostname defensible: the outpost terminates every request, enforces the
+      # session, and only then forwards to llama-server.
+      #
+      # Same reasoning as vscode-gate, and the same passkey step-up: what sits
+      # behind the gate is entirely unprotected, so the gate is the whole of the
+      # access control rather than a second factor on top of an app login.
+      #
+      # In-cluster consumers (Home Assistant, Paperless, CI) are unaffected:
+      # they address llama-server.ai.svc.cluster.local directly and never pass
+      # through Traefik or this outpost.
+      #
+      # No group and no policy binding: any user who can authenticate gets in,
+      # as with Immich, Audiobookshelf, Home Assistant and Music Assistant. The
+      # gates that pin membership to a named user in git — vscode, esphome,
+      # zigbee2mqtt, shelly-manager, prometheus — are admin interfaces, where
+      # the allowlist is the point. A chat UI is not one of those, and deciding
+      # its membership here would put an access-control choice in a manifest
+      # instead of leaving it where the user directory belongs.
+      # ------------------------------------------------------------------
+      - id: llama-gate-provider
+        model: authentik_providers_proxy.proxyprovider
+        identifiers:
+          name: llama-gate
+        state: present
+        attrs:
+          # Both flows, mirroring the Audiobookshelf provider. Setting only
+          # authorization_flow leaves authentication on Authentik's default
+          # flow, which is what made the first login clumsy — the step-up ran
+          # on top of a password prompt instead of replacing it. With the
+          # passwordless flow named explicitly, signing in is the passkey
+          # itself.
+          #
+          # Referenced by !Find rather than !KeyOf: both flows are defined in
+          # the passkey-stepup document of this ConfigMap, and !KeyOf only
+          # resolves within one document.
+          authentication_flow: !Find [authentik_flows.flow, [slug, passkey-passwordless-authentication]]
+          authorization_flow: !Find [authentik_flows.flow, [slug, passkey-stepup-authorization]]
+          invalidation_flow: !Find [authentik_flows.flow, [slug, default-provider-invalidation-flow]]
+          mode: proxy
+          internal_host: http://llama-server.ai.svc.cluster.local:8080
+          internal_host_ssl_validation: false
+          external_host: https://llm.renner.uno
+          cookie_domain: renner.uno
+          access_token_validity: hours=24
+          refresh_token_validity: days=30
+          intercept_header_auth: true
+          # The WebUI ships `<link rel="manifest">` and registers a service
+          # worker. Per the W3C spec a manifest is fetched with
+          # `credentials: 'omit'`, so it carries no session cookie and the gate
+          # answers it with a 302 to auth.renner.uno — which turns every page
+          # load into a redirect round-trip. This is the failure that forced
+          # paperless-gate's own carve-out and that retired music-assistant-gate
+          # outright. Both paths serve no authenticated content, so letting them
+          # through anonymously costs nothing.
+          skip_path_regex: ^/(manifest\.webmanifest|sw\.js)$
+
+      - model: authentik_core.application
+        identifiers:
+          slug: llama-gate
+        state: present
+        attrs:
+          name: Local LLM (Gate)
+          slug: llama-gate
+          provider: !KeyOf llama-gate-provider
+          meta_launch_url: https://llm.renner.uno
+          meta_description: "Forward-auth gate in front of the local llama-server WebUI"
+
+      - model: authentik_outposts.outpost
+        identifiers:
+          name: llama-gate
+        state: present
+        attrs:
+          type: proxy
+          service_connection: !KeyOf k8s-conn
+          providers:
+            - !KeyOf llama-gate-provider
+          config:
+            log_level: info
+            authentik_host: "http://authentik-server.authentik.svc.cluster.local"
+            authentik_host_insecure: false
+            authentik_host_browser: "https://auth.renner.uno/"
+            refresh_interval: "minutes=5"
+            container_image: null
+            object_naming_template: "ak-outpost-%(name)s"
+            kubernetes_namespace: authentik
+            kubernetes_replicas: 1
+            kubernetes_service_type: ClusterIP
+            kubernetes_disabled_components: ["ingress"]
+            kubernetes_image_pull_secrets: []
+            kubernetes_ingress_annotations: {}
+            kubernetes_ingress_class_name: null
+            kubernetes_ingress_secret_name: "authentik-outpost-tls"
+            kubernetes_json_patches: null
+
       - model: authentik_outposts.outpost
         identifiers:
           name: authentik Embedded Outpost

kubernetes/apps/authentik/kustomization.yaml

@@ -15,6 +15,7 @@ resources:
   - alertmanager-gate-egress.yaml
   - prometheus-gate-egress.yaml
   - wordpress-gate-egress.yaml
+  - llama-gate-egress.yaml
   - outpost-servicemonitors.yaml
   - ldap-outpost-cert.yaml
   - ldap-outpost-lb.yaml

kubernetes/apps/authentik/llama-gate-egress.yaml

@@ -0,0 +1,30 @@
+---
+# Lets the llama-gate outpost reach llama-server. The cluster baseline CNP
+# (kubernetes/infrastructure/network-policies/authentik-ns.yaml) otherwise stops
+# any authentik-namespace pod from entering the ai namespace.
+#
+# The outpost is the trust boundary: it forwards to llama-server only after
+# validating the Authentik session, and the matching ingress rule in
+# ai-ns.yaml admits nothing else — so nothing in the cluster can route around
+# the gate to reach the unauthenticated API through the public hostname.
+#
+# Scoped to this outpost's pod label rather than the whole namespace, so the
+# other gates cannot reach llama-server either.
+apiVersion: cilium.io/v2
+kind: CiliumNetworkPolicy
+metadata:
+  name: llama-gate-egress
+  namespace: authentik
+spec:
+  endpointSelector:
+    matchLabels:
+      goauthentik.io/outpost-name: llama-gate
+  egress:
+    - toEndpoints:
+        - matchLabels:
+            "k8s:io.kubernetes.pod.namespace": ai
+            app.kubernetes.io/name: llama-server
+      toPorts:
+        - ports:
+            - port: "8080"
+              protocol: TCP

kubernetes/apps/llama-server/gate-service.yaml

@@ -0,0 +1,23 @@
+---
+# ExternalName mirror so the ai-namespace Ingress can backend to the llama-gate
+# outpost that Authentik deploys into the authentik namespace (cross-namespace
+# Service references are not supported in a standard Ingress).
+#
+# Traffic path:
+#   browser → traefik (llm.renner.uno)
+#           → ak-outpost-llama-gate (ExternalName → authentik ns)
+#           → outpost validates the session + passkey step-up
+#           → http://llama-server.ai.svc.cluster.local:8080
+#
+# In-cluster consumers skip all of this and address llama-server directly.
+apiVersion: v1
+kind: Service
+metadata:
+  name: ak-outpost-llama-gate
+  namespace: ai
+spec:
+  type: ExternalName
+  externalName: ak-outpost-llama-gate.authentik.svc.cluster.local
+  ports:
+    - port: 9000
+      targetPort: 9000

kubernetes/apps/llama-server/ingress.yaml

@@ -1,12 +1,31 @@
 ---
-# LAN-only: no `dns.renner.uno/public` label, so the Cloudflare external-dns
-# instance skips it and only the UDM instance publishes the record. The API is
-# unauthenticated, so it must not be reachable from the internet.
+# llm.renner.uno — the chat WebUI, public and behind Authentik.
+#
+# Traefik no longer reaches llama-server directly. This Ingress backends the
+# llama-gate proxy outpost (ExternalName → authentik ns), which enforces a
+# passkey step-up and forwards only after the session validates. The ai
+# CiliumNetworkPolicy is tightened so ONLY that outpost pod may reach
+# llama-server, so nothing in the cluster can route around the gate. Pattern
+# mirrors kubernetes/apps/home-assistant/vscode-ingress.yaml.
+#
+# The gate is the entire access control here: llama-server has no
+# authentication of its own and serves a full OpenAI-compatible API on the same
+# port as the UI. That is what kept this hostname LAN-only until now.
+#
+# In-cluster consumers — Home Assistant, Paperless, CI — address
+# llama-server.ai.svc.cluster.local:8080 directly and never pass through
+# Traefik or the outpost, so they need no credentials and are unaffected by the
+# gate.
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
   name: llama-server
   namespace: ai
+  labels:
+    # Public: Cloudflare wildcard → tunnel → NetBird proxy → Traefik. The UDM
+    # external-dns instance skips anything carrying this, and LAN clients reach
+    # the same hostname through the hairpin NAT.
+    dns.renner.uno/public: "true"
   annotations:
     cert-manager.io/cluster-issuer: letsencrypt-cluster
 spec:
@@ -23,6 +42,6 @@ spec:
             pathType: Prefix
             backend:
               service:
-                name: llama-server
+                name: ak-outpost-llama-gate
                 port:
-                  number: 8080
+                  number: 9000

kubernetes/apps/llama-server/kustomization.yaml

@@ -4,4 +4,5 @@ resources:
   - pvc.yaml
   - deployment.yaml
   - embed.yaml
+  - gate-service.yaml
   - ingress.yaml

kubernetes/e2e-test.sh

@@ -530,24 +530,6 @@ check "alertmanager: no severity=critical alerts active" bash -c '
 # manifests were valid in each case and the endpoint still misbehaved.
 # ---------------------------------------------------------------------------
 
-# llm_chat: POST a chat completion through the ingress. Body lands in
-# /tmp/llm_body for the caller to assert against. Generous --max-time: the
-# model runs on an iGPU at ~12 tok/s, so a short answer still takes seconds.
-llm_chat() {
-    local model="$1" prompt="$2" max="${3:-64}" out body code payload
-    payload=$(jq -nc --arg m "$model" --arg p "$prompt" --argjson n "$max" \
-        '{model:$m,messages:[{role:"user",content:$p}],max_tokens:$n,temperature:0}')
-    out=$(curl -sk --resolve "llm.renner.uno:443:$LB_IP" --max-time 300 \
-          -H 'Content-Type: application/json' -d "$payload" \
-          -w $'\n__CODE__:%{http_code}' \
-          "https://llm.renner.uno/v1/chat/completions" 2>&1)
-    code=${out##*__CODE__:}
-    body=${out%$'\n'__CODE__:*}
-    printf 'code=%s body=%s\n' "$code" "${body:0:300}"
-    [[ "$code" == "200" ]] || return 1
-    printf '%s' "$body" > /tmp/llm_body
-}
-export -f llm_chat
 export LB_IP
 
 # helper_post: POST JSON from inside e2e-helper, for Services with no ingress.
@@ -571,54 +553,66 @@ helper_post() {
 }
 export -f helper_post
 
+# llm_chat: POST a chat completion to llama-server from inside the cluster.
+# It goes to the Service, not the public hostname: llm.renner.uno now answers
+# with an Authentik redirect, and the API is deliberately in-cluster only —
+# which is also how every real consumer reaches it. Result lands in
+# /tmp/helper_body.
+llm_chat() {
+    local prompt="$1" max="${2:-64}"
+    helper_post "http://llama-server.ai.svc.cluster.local:8080/v1/chat/completions" \
+        "$(jq -nc --arg p "$prompt" --argjson n "$max" \
+            '{model:"qwen3.6-35b-a3b",messages:[{role:"user",content:$p}],max_tokens:$n,temperature:0}')"
+}
+export -f llm_chat
+
 echo "=== Local AI inference ==="
 
-check "llama-server /health returns 200" \
-    http_code "https://llm.renner.uno/health" 200
+# The gate is the whole of the access control on the public hostname:
+# llama-server has no authentication and serves the API on the same port as the
+# UI. An anonymous request must be bounced to Authentik, never answered. A 200
+# here would mean the ingress is reaching llama-server directly again.
+check "public hostname is gated by authentik" \
+    bash -c 'code=$(curl -sk --resolve "llm.renner.uno:443:$LB_IP" --max-time 15 \
+                 -o /dev/null -w "%{http_code}" "https://llm.renner.uno/v1/models")
+             echo "got=$code"; [[ "$code" =~ ^(302|307|401|403)$ ]]'
 
-# The chat model is named for what it is, so a picker and a log line say which
-# model answered. `chat` stays valid alongside it as a stable handle, so a
-# consumer's config survives the next model swap.
 # Asserts the id specifically, not merely that the string appears somewhere:
 # every configured name shows up in the `aliases` array regardless of order, so
 # a substring match passes even when the picker is showing the wrong one.
 check "descriptive name is the advertised id" \
-    http_body "https://llm.renner.uno/v1/models" '"id":"qwen3.6-35b-a3b"'
+    bash -c 'helper_curl "http://llama-server.ai.svc.cluster.local:8080/v1/models" &&
+             jq -e ".data[0].id == \"qwen3.6-35b-a3b\"" /tmp/helper_body >/dev/null'
+
+# Only one name is configured. A second would be advertised alongside it and,
+# if it sorted first, would take the id — which is what a picker displays.
+check "no second name competes for the id" \
+    bash -c 'jq -e ".data[0].aliases | length == 1" /tmp/helper_body >/dev/null'
 
 # Regression: the embedding model used to share this port. It appeared in the
 # chat picker, and selecting it returned fluent nonsense ("capital capital
 # capital ...") instead of an error, because --embeddings does not refuse chat
-# completions. It now lives on a ClusterIP-only Service; if it ever reappears
-# here, it is selectable again and this must fail.
+# completions. It now lives on a separate Service; if it ever reappears here,
+# it is selectable again and this must fail.
 check "embedding model absent from chat endpoint" \
-    bash -c '! curl -sk --resolve "llm.renner.uno:443:$LB_IP" --max-time 15 \
-        "https://llm.renner.uno/v1/models" | grep -qi "embed"'
-
-check "embeddings not exposed through the ingress" \
-    http_code "https://llm.renner.uno/v1/embeddings" 501
+    bash -c '! grep -qi embed /tmp/helper_body'
 
 # Regression: with thinking enabled by default the model spent its whole token
 # budget on reasoning_content and returned an EMPTY content with
 # finish_reason "length" — a 200 response that answers nothing. Asserting on
 # the status code alone would have passed.
 check "chat completion returns non-empty content" \
-    bash -c 'llm_chat chat "Nenne die Hauptstadt von Frankreich." 64 &&
-             jq -e ".choices[0].message.content | length > 0" /tmp/llm_body >/dev/null'
+    bash -c 'llm_chat "Nenne die Hauptstadt von Frankreich." 64 &&
+             jq -e ".choices[0].message.content | length > 0" /tmp/helper_body >/dev/null'
 
 check "chat completion stops rather than truncating" \
-    bash -c 'jq -e ".choices[0].finish_reason == \"stop\"" /tmp/llm_body >/dev/null'
+    bash -c 'jq -e ".choices[0].finish_reason == \"stop\"" /tmp/helper_body >/dev/null'
 
 # Coherence, not just liveness. This is what catches a model answering from the
 # wrong weights or a numerically broken backend — llama-bench measures
 # throughput and would report excellent numbers while emitting garbage.
 check "chat answer is coherent (knows Paris)" \
-    bash -c 'jq -re ".choices[0].message.content" /tmp/llm_body | grep -qi paris'
-
-# Only one name is configured. A second would be advertised alongside it and,
-# if it sorted first, would take the id — which is what a picker displays.
-check "no second name competes for the id" \
-    bash -c 'test "$(curl -sk --resolve "llm.renner.uno:443:$LB_IP" --max-time 15 \
-        "https://llm.renner.uno/v1/models" | jq -r ".data[0].aliases | length")" = "1"'
+    bash -c 'jq -re ".choices[0].message.content" /tmp/helper_body | grep -qi paris'
 
 # Reached from the monitoring namespace, which ai-ns.yaml admits; there is no
 # ingress for this Service by design.

kubernetes/infrastructure/network-policies/ai-ns.yaml

@@ -9,10 +9,21 @@ spec:
     - fromEndpoints:
         - matchLabels:
             "k8s:io.kubernetes.pod.namespace": ai
+    # Traefik is deliberately NOT admitted. llm.renner.uno is public and
+    # llama-server has no authentication of its own, so the Authentik llama-gate
+    # outpost is the only thing standing in front of it. Traefik routes the
+    # hostname to that outpost and never to llama-server, and this policy is
+    # what makes that structural rather than a matter of configuration: with no
+    # ingress rule for traefik, an Ingress that pointed straight at the Service
+    # — by accident or otherwise — simply would not connect.
     - fromEndpoints:
         - matchLabels:
-            "k8s:io.kubernetes.pod.namespace": ingress
-            app.kubernetes.io/name: traefik
+            "k8s:io.kubernetes.pod.namespace": authentik
+            goauthentik.io/outpost-name: llama-gate
+      toPorts:
+        - ports:
+            - port: "8080"
+              protocol: TCP
     - fromEndpoints:
         - matchLabels:
             "k8s:io.kubernetes.pod.namespace": monitoring

kubernetes/infrastructure/velero/restore-test.sh

@@ -5,8 +5,9 @@
 #   1. Velero Restore reaches phase=Completed against yesterday's
 #      scheduled Backup (real artifact, not a synthetic one).
 #   2. Each restored PVC is mountable and its file count is within ±10%
-#      of the live source volume, ignoring the transient subtrees listed
-#      in volatile_paths_for().
+#      (or ±ABS_PARITY_SLACK files, whichever is wider) of the live
+#      source volume, ignoring the transient subtrees listed in
+#      volatile_paths_for().
 #
 # What this does NOT exercise:
 #   - CNPG database restore (barman-cloud, separate code path).
@@ -317,11 +318,37 @@ EOF
     count_files_in_pvc "$src_ns" "$tmp_pvc" "src-${pvc:0:40}-${RUN_ID}" "$excludes"
 }
 
-# Restored count must be within ±10% of the source's current file count.
-# That tolerance absorbs day-of drift since the backup ran without
-# letting "lost half the data" pass silently. Volatile subtrees are pruned
-# from BOTH sides — keyed off the source identity, since the PVC name is
-# preserved across the restore but the destination namespace is not.
+# Restored count must be within max(±10%, ±ABS_PARITY_SLACK files) of the
+# source's current file count. That tolerance absorbs day-of drift since
+# the backup ran without letting "lost half the data" pass silently.
+# Volatile subtrees are pruned from BOTH sides — keyed off the source
+# identity, since the PVC name is preserved across the restore but the
+# destination namespace is not.
+#
+# Why the absolute floor exists. A purely relative window is structurally
+# unable to pass a small volume: ±10% of 9 files is ±0.9, i.e. a budget of
+# ZERO files, and ±10% of 2 files is ±0.2. The 2026-09-01 run failed on
+# exactly that — audiobookshelf/libation-config source=9 restored=8 and
+# trmnl/trmnl-renderer-data source=2 restored=3, both off by one file and
+# in opposite directions, with all 12 restores at phase=Completed and the
+# other 46 checks green. Parity compares a backup taken up to ~24 h ago
+# against the live volume *now*, so a single config write on a tiny volume
+# between those two moments is expected, not a defect in the backup.
+#
+# The floor only governs volumes below ~10*ABS_PARITY_SLACK files; above
+# that the 10% window still binds, so real data loss on the volumes that
+# carry data keeps failing loudly. Two files of slack rather than one
+# because a single logical write on a config volume routinely moves more
+# than one file (an app writing a settings file plus a transient sidecar),
+# and a floor that buys exactly one month is not worth the change.
+#
+# What the floor must NOT swallow, and does not: a non-empty source that
+# restored to an *empty* volume fails unconditionally below, before any
+# tolerance is applied. The old relative-only check caught that case as a
+# side effect (delta = 1.0 > 0.10); it is now explicit so the floor cannot
+# erode it on a volume of ABS_PARITY_SLACK files or fewer.
+ABS_PARITY_SLACK=${ABS_PARITY_SLACK:-2}
+
 verify_pvc_file_count_parity() {
     local src_ns="$1" dst_ns="$2" pvc="$3"
     local restored source excludes
@@ -339,11 +366,15 @@ verify_pvc_file_count_parity() {
     if [[ "$source" -eq 0 && "$restored" -eq 0 ]]; then
         return 0
     fi
-    # Use awk to compute a 10% window without bc.
-    awk -v s="$source" -v r="$restored" 'BEGIN {
-        if (s == 0) { exit !(r == 0) }
-        delta = (r - s) / s; if (delta < 0) delta = -delta
-        exit !(delta <= 0.10)
+    # Empty on either side while the other has data: never tolerable.
+    if [[ "$source" -eq 0 || "$restored" -eq 0 ]]; then
+        return 1
+    fi
+    # Use awk to compute the window without bc.
+    awk -v s="$source" -v r="$restored" -v slack="$ABS_PARITY_SLACK" 'BEGIN {
+        delta = r - s; if (delta < 0) delta = -delta
+        tol = 0.10 * s; if (tol < slack) tol = slack
+        exit !(delta <= tol)
     }'
 }
 
@@ -492,7 +523,7 @@ EOF
         check "${src_ns}: at least one PVC was restored" false
     else
         for pvc in "${PVCS[@]}"; do
-            check "${src_ns}: PVC ${pvc} file count parity (±10%)" \
+            check "${src_ns}: PVC ${pvc} file count parity (±10% / ±${ABS_PARITY_SLACK})" \
                 verify_pvc_file_count_parity "$src_ns" "$dst_ns" "$pvc"
         done
     fi