fix: [alert] KubeHpaMaxedOut (authentik) (#761)

.woodpecker/deploy-netbird-proxy.yml

@@ -10,18 +10,6 @@ when:
       # install-netbird-proxy.yml also applies the edge bouncer.
       - "ansible/roles/crowdsec-edge-bouncer/**"
       - "ansible/plays/install-netbird-proxy.yml"
-      # The variables that steer those roles, not just the roles themselves.
-      # Without these two, a change that only touches inventory never deploys
-      # and nothing says so: arming the bouncer lives in host_vars, and its
-      # LAPI key lives in group_vars, so both would sit in git looking applied
-      # while the hosts kept running the previous configuration.
-      #
-      # pve_hosts.yml is deliberately NOT listed. It is the single inventory
-      # file for the whole fleet, so listing it would redeploy the proxies for
-      # any unrelated host edit. The cost is that a change to [netbird_proxy]
-      # group membership needs a manual deployment run.
-      - "ansible/inventory/host_vars/proxy-*.yml"
-      - "ansible/inventory/group_vars/netbird_proxy/**"
       - ".woodpecker/deploy-netbird-proxy.yml"
   - event: deployment
     evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "deploy-netbird-proxy"'

ansible/roles/crowdsec-edge-bouncer/defaults/main.yml

@@ -28,23 +28,14 @@ crowdsec_bouncer_sha256: f86e4b72693549d99f40a9402abefb894108f047a3fbe6e72fada25
 crowdsec_bouncer_api_url: https://crowdsec.renner.uno/
 crowdsec_bouncer_api_key: "{{ vault_crowdsec_edge_bouncer_key }}"
 
-# Enforcing. The bouncer creates two nftables tables (ip crowdsec, ip6
-# crowdsec6) holding the decisions it pulled, touches no existing chain and
-# sets no chain policy — SSH and :443 stay open unless the source is banned.
+# STARTS INERT, and that is the point.
 #
-# `dry-run` is the other value: it pulls and logs what it would block without
-# creating a single rule. That is what a new host should be given before it
-# enforces, and what an existing one can be dropped back to, per host through
-# inventory/host_vars, without disarming the rest.
-#
-# Worth knowing before arming anything new: the first bouncer on a path
-# enforces the whole CAPI set at once — around 15k addresses today — so the
-# false-positive window belongs before the switch, not after it. And because
-# this chain sits on the INPUT hook while these hosts run no other firewall,
-# there is no conntrack rule accepting established flows ahead of it: a banned
-# address is dropped even when it is answering our own outbound connection.
-# That is why capi_whitelists.yaml carries the proxies and netbird.renner.uno.
-crowdsec_bouncer_mode: nftables
+# dry-run pulls decisions and logs what it would block without creating a
+# single nftables rule. The first bouncer to arm anywhere enforces ~24k CAPI
+# decisions at once on the only external entry path, so the false-positive
+# window has to come before the enforcement, not after it. Flip to `nftables`
+# one host at a time once the log looks right.
+crowdsec_bouncer_mode: dry-run
 
 crowdsec_bouncer_update_frequency: 10s
 crowdsec_bouncer_log_level: info

kubernetes/apps/authentik/helmrelease.yaml

@@ -120,7 +120,14 @@ spec:
     server:
       resources:
         requests:
-          cpu: 200m
+          # 500m, not 200m: the HPA below targets 70% of *this* request, so at
+          # 200m the scale-up threshold sat at 140m — below the pod's own
+          # ordinary peak. Measured over 7d, a warm server pod p99s at ~108m
+          # and peaks at ~190m, and a freshly-started one burns ~300m booting
+          # Django. That made the HPA oscillate 2<->3<->4 all day and pin at
+          # maxReplicas (KubeHpaMaxedOut, issue #761). At 500m the threshold
+          # is 350m/pod, above the boot burst, so only real load scales out.
+          cpu: 500m
           memory: 1Gi
         limits:
           memory: 2Gi
@@ -154,7 +161,11 @@ spec:
     worker:
       resources:
         requests:
-          cpu: 200m
+          # Same reasoning as server above, and the worker is burstier still:
+          # over 7d its per-pod p99 reaches ~336m and peaks hit ~630m while
+          # running blueprint applies and scheduled tasks. 200m understated
+          # that badly enough that routine task batches read as saturation.
+          cpu: 500m
           memory: 1Gi
         limits:
           memory: 2Gi

kubernetes/apps/llama-server/deployment.yaml

@@ -75,31 +75,11 @@ spec:
             # many small per-expert matmuls that sets the pace, not bandwidth.
             - --threads
             - "4"
-            # 32k because Home Assistant's Assist agent measured 18910 tokens
-            # for a single request: ~12k of exposed-entity overview plus ~7k of
-            # base instructions and tool schemas, whose parameters enumerate
-            # every exposed name, area and floor. At 16384 the request was
-            # refused outright.
-            #
-            # --ctx-size is the total and llama.cpp divides it across the
-            # slots, so 65536 over two slots is 32768 each.
-            #
-            # It does not read that way without --parallel, and the reason is
-            # in `llama-server --help`: --kv-unified defaults to enabled only
-            # "if number of slots is auto". Leaving --parallel unset therefore
-            # bought a single KV buffer shared across sequences, where every
-            # slot could draw the full --ctx-size — which is why this server
-            # once reported n_ctx_slot 16384 across four slots. Naming a slot
-            # count turns that off and splits the context instead.
-            #
-            # Two slots is what is actually in use: one holds Home Assistant's
-            # prompt, one holds Paperless's. A third caller would queue rather
-            # than evict either, which is the better failure mode given a cold
-            # prefill of this prompt costs around two minutes.
+            # 16k rather than 8k because the measured prefill falloff is mild
+            # and this model's KV cache is unusually cheap — only 10 of its 40
+            # layers use full attention.
             - --ctx-size
-            - "65536"
-            - --parallel
-            - "2"
+            - "16384"
             # A micro-batch of 2048 or more makes this Vulkan/Mesa stack fail
             # with ErrorDeviceLost.
             - --ubatch-size

kubernetes/apps/llama-server/embed.yaml

@@ -64,27 +64,16 @@ spec:
             - "99"
             - --threads
             - "2"
-            # Paperless chunks at 1024 tokens, but counts them with its own
-            # tokenizer while this server measures the batch in bge-m3's.
-            # Those disagree: XLM-RoBERTa splits German into more pieces than
-            # llama_index's counter does, so a chunk Paperless considers full
-            # at 1024 arrived here as 1069 and the request failed outright —
-            # "input (1069 tokens) is too large to process".
-            #
-            # The headroom therefore has to absorb a tokenizer ratio, not just
-            # a rounding error, and 1.5x covers far more than the 1.044 that
-            # broke it. The defaults are batch 2048 / ubatch 512, and with
-            # --embeddings llama-server clamps both to the smaller rather than
-            # run them mismatched, so both are set.
-            #
-            # 2048 stays avoided: that is where the chat model met
-            # ErrorDeviceLost on this Vulkan stack. That failure came under a
-            # 20 GB model's memory pressure and may not bind a 600 MB one, but
-            # this does not need to find out.
+            # Paperless chunks documents at 1024 tokens. The defaults are
+            # batch 2048 / ubatch 512, and with --embeddings llama-server
+            # clamps both down to 512 rather than run them mismatched — below
+            # the chunk size. Matching both to 1024 fits a whole chunk in one
+            # pass; 2048 is deliberately avoided, since that is where this
+            # Vulkan stack fails with ErrorDeviceLost.
             - --batch-size
-            - "1536"
+            - "1024"
             - --ubatch-size
-            - "1536"
+            - "1024"
             # Room for a full chunk plus overhead rather than exactly one.
             - --ctx-size
             - "4096"

kubernetes/apps/monitoring-rules/crowdsec-servicemonitor.yaml

@@ -1,36 +0,0 @@
-# Scrapes CrowdSec. Until this landed, nothing did: `up{namespace="crowdsec"}`
-# returned zero series and Prometheus held no cs_* metric at all, so the
-# detection side of the ingress path was invisible — including whether the
-# agents were reading logs and whether any bouncer was pulling decisions.
-#
-# One monitor for both services. They carry the same shape (a `metrics` port on
-# 6060) and differ only in `app`, so a matchExpressions selector keeps this to
-# a single object rather than two near-identical ones:
-#
-#   crowdsec-service        the LAPI      — decisions, bouncer + machine requests
-#   crowdsec-agent-service  the agents    — acquisition, parsers, scenarios
-#
-# The crowdsec-ns CiliumNetworkPolicy already admits the monitoring namespace
-# without a port restriction, so no policy change is needed.
-#
-# `release: kube-prometheus-stack` is what the chart's Prometheus selects on.
-apiVersion: monitoring.coreos.com/v1
-kind: ServiceMonitor
-metadata:
-  name: crowdsec
-  namespace: crowdsec
-  labels:
-    app.kubernetes.io/name: crowdsec
-    release: kube-prometheus-stack
-spec:
-  selector:
-    matchExpressions:
-      - key: app
-        operator: In
-        values:
-          - crowdsec-service
-          - crowdsec-agent-service
-  endpoints:
-    - port: metrics
-      path: /metrics
-      interval: 30s

kubernetes/apps/monitoring-rules/kustomization.yaml

@@ -23,7 +23,6 @@ resources:
   - prometheusrule-cnpg-backup.yaml
   - alloy-servicemonitor.yaml
   - cnpg-podmonitors.yaml
-  - crowdsec-servicemonitor.yaml
   - home-assistant-servicemonitor.yaml
   - immich-servicemonitor.yaml
   - loki-servicemonitor.yaml

kubernetes/apps/monitoring/dashboards/crowdsec-overview.json

@@ -1,163 +0,0 @@
-{
-  "annotations": {"list": []},
-  "description": "CrowdSec: Erkennung im Cluster, Durchsetzung am NetBird-Edge",
-  "editable": true,
-  "fiscalYearStartMonth": 0,
-  "graphTooltip": 1,
-  "id": null,
-  "links": [],
-  "panels": [
-    {
-      "datasource": {"type": "prometheus", "uid": "prometheus"},
-      "description": "Aktive Ban-Entscheidungen in der LAPI, ganz überwiegend aus den CAPI-Community-Blocklisten. Das ist die Menge, die ein scharfgeschalteter Bouncer durchsetzt.",
-      "fieldConfig": {
-        "defaults": {"color": {"mode": "thresholds"}, "mappings": [], "thresholds": {"mode": "absolute", "steps": [{"color": "text", "value": null}]}, "unit": "short"},
-        "overrides": []
-      },
-      "gridPos": {"h": 4, "w": 6, "x": 0, "y": 0},
-      "id": 1,
-      "options": {"colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto"},
-      "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "sum(cs_active_decisions)", "refId": "A"}],
-      "title": "Aktive Decisions",
-      "type": "stat"
-    },
-    {
-      "datasource": {"type": "prometheus", "uid": "prometheus"},
-      "description": "Bouncer, die in den letzten 10 Minuten Decisions abgeholt haben. Fällt das unter die erwartete Zahl, holt ein Bouncer nichts mehr — und setzt damit einen veralteten Stand durch, ohne dass irgendetwas ausfällt.",
-      "fieldConfig": {
-        "defaults": {"color": {"mode": "thresholds"}, "mappings": [], "thresholds": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "green", "value": 1}]}, "unit": "short"},
-        "overrides": []
-      },
-      "gridPos": {"h": 4, "w": 6, "x": 6, "y": 0},
-      "id": 2,
-      "options": {"colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto"},
-      "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "count(sum by (bouncer) (rate(cs_lapi_bouncer_requests_total[10m])) > 0)", "refId": "A"}],
-      "title": "Bouncer, die abholen",
-      "type": "stat"
-    },
-    {
-      "datasource": {"type": "prometheus", "uid": "prometheus"},
-      "description": "Vom Agent gelesene Logzeilen pro Sekunde. Steht das auf 0, detektiert CrowdSec nichts mehr — der häufigste stille Ausfall, weil alles andere weiterläuft.",
-      "fieldConfig": {
-        "defaults": {"color": {"mode": "thresholds"}, "mappings": [], "thresholds": {"mode": "absolute", "steps": [{"color": "red", "value": null}, {"color": "green", "value": 0.01}]}, "unit": "reqps"},
-        "overrides": []
-      },
-      "gridPos": {"h": 4, "w": 6, "x": 12, "y": 0},
-      "id": 3,
-      "options": {"colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto"},
-      "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "sum(rate(cs_filesource_hits_total[5m]))", "refId": "A"}],
-      "title": "Gelesene Logzeilen/s",
-      "type": "stat"
-    },
-    {
-      "datasource": {"type": "prometheus", "uid": "prometheus"},
-      "description": "Szenario-Overflows der letzten 24 h — also lokal erkannte Angriffe, im Gegensatz zu den von CAPI übernommenen.",
-      "fieldConfig": {
-        "defaults": {"color": {"mode": "thresholds"}, "mappings": [], "thresholds": {"mode": "absolute", "steps": [{"color": "text", "value": null}]}, "unit": "short"},
-        "overrides": []
-      },
-      "gridPos": {"h": 4, "w": 6, "x": 18, "y": 0},
-      "id": 4,
-      "options": {"colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}, "textMode": "auto"},
-      "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "sum(increase(cs_bucket_overflowed_total[24h]))", "refId": "A"}],
-      "title": "Lokale Overflows (24 h)",
-      "type": "stat"
-    },
-    {
-      "datasource": {"type": "prometheus", "uid": "prometheus"},
-      "description": "Abholrate je Bouncer. Der Firewall-Bouncer auf den Proxies fragt alle 10 s. Die LAPI legt bei jeder neuen beobachteten Client-Adresse einen Eintrag `<name>@<ip>` an, und beobachtet wird die Pod-IP des netbird-ingress-Peers — deshalb erscheint ein logischer Bouncer hier mehrfach und nach Pod-Rollen unter neuen Namen.",
-      "fieldConfig": {
-        "defaults": {"color": {"mode": "palette-classic"}, "custom": {"axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": {"legend": false, "tooltip": false, "viz": false}, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": {"type": "linear"}, "showPoints": "never", "spanNulls": false, "stacking": {"group": "A", "mode": "none"}, "thresholdsStyle": {"mode": "off"}}, "mappings": [], "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}]}, "unit": "reqps"},
-        "overrides": []
-      },
-      "gridPos": {"h": 8, "w": 12, "x": 0, "y": 4},
-      "id": 5,
-      "options": {"legend": {"calcs": ["lastNotNull"], "displayMode": "table", "placement": "right", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
-      "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "sum by (bouncer) (rate(cs_lapi_bouncer_requests_total[5m]))", "legendFormat": "{{bouncer}}", "refId": "A"}],
-      "title": "Bouncer-Abholrate",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {"type": "prometheus", "uid": "prometheus"},
-      "description": "Aktive Decisions nach Grund. `http:scan` dominiert und stammt aus den CAPI-Blocklisten.",
-      "fieldConfig": {
-        "defaults": {"color": {"mode": "palette-classic"}, "custom": {"axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", "hideFrom": {"legend": false, "tooltip": false, "viz": false}, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": {"type": "linear"}, "showPoints": "never", "spanNulls": false, "stacking": {"group": "A", "mode": "normal"}, "thresholdsStyle": {"mode": "off"}}, "mappings": [], "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}]}, "unit": "short"},
-        "overrides": []
-      },
-      "gridPos": {"h": 8, "w": 12, "x": 12, "y": 4},
-      "id": 6,
-      "options": {"legend": {"calcs": ["lastNotNull"], "displayMode": "table", "placement": "right", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
-      "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "sum by (reason) (cs_active_decisions)", "legendFormat": "{{reason}}", "refId": "A"}],
-      "title": "Aktive Decisions nach Grund",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {"type": "prometheus", "uid": "prometheus"},
-      "description": "Welche Szenarien lokal auslösen. Das ist das, was CrowdSec selbst aus den Traefik-Logs erkennt.",
-      "fieldConfig": {
-        "defaults": {"color": {"mode": "palette-classic"}, "custom": {"axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "bars", "fillOpacity": 70, "gradientMode": "none", "hideFrom": {"legend": false, "tooltip": false, "viz": false}, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": {"type": "linear"}, "showPoints": "never", "spanNulls": false, "stacking": {"group": "A", "mode": "normal"}, "thresholdsStyle": {"mode": "off"}}, "mappings": [], "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}]}, "unit": "short"},
-        "overrides": []
-      },
-      "gridPos": {"h": 8, "w": 12, "x": 0, "y": 12},
-      "id": 7,
-      "options": {"legend": {"calcs": ["sum"], "displayMode": "table", "placement": "right", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
-      "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "sum by (name) (increase(cs_bucket_overflowed_total[1h]))", "legendFormat": "{{name}}", "refId": "A"}],
-      "title": "Szenario-Overflows (je Stunde)",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {"type": "prometheus", "uid": "prometheus"},
-      "description": "Geparste gegen nicht geparste Zeilen. Ein Anstieg bei `ko` heißt meist, dass sich ein Logformat geändert hat und der Parser nicht mehr greift — CrowdSec liest dann weiter, erkennt aber nichts mehr.",
-      "fieldConfig": {
-        "defaults": {"color": {"mode": "palette-classic"}, "custom": {"axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": {"legend": false, "tooltip": false, "viz": false}, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": {"type": "linear"}, "showPoints": "never", "spanNulls": false, "stacking": {"group": "A", "mode": "none"}, "thresholdsStyle": {"mode": "off"}}, "mappings": [], "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}]}, "unit": "reqps"},
-        "overrides": []
-      },
-      "gridPos": {"h": 8, "w": 12, "x": 12, "y": 12},
-      "id": 8,
-      "options": {"legend": {"calcs": ["lastNotNull"], "displayMode": "list", "placement": "bottom", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
-      "targets": [
-        {"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "sum(rate(cs_parser_hits_ok_total[5m]))", "legendFormat": "geparst", "refId": "A"},
-        {"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "sum(rate(cs_parser_hits_ko_total[5m]))", "legendFormat": "nicht geparst", "refId": "B"}
-      ],
-      "title": "Parser",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {"type": "prometheus", "uid": "prometheus"},
-      "description": "Whitelist-Treffer nach Grund. Nützlich beim Scharfschalten: was hier zählt, wäre sonst ein Kandidat für einen Ban gewesen.",
-      "fieldConfig": {
-        "defaults": {"color": {"mode": "palette-classic"}, "custom": {"axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": {"legend": false, "tooltip": false, "viz": false}, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": {"type": "linear"}, "showPoints": "never", "spanNulls": false, "stacking": {"group": "A", "mode": "none"}, "thresholdsStyle": {"mode": "off"}}, "mappings": [], "thresholds": {"mode": "absolute", "steps": [{"color": "green", "value": null}]}, "unit": "reqps"},
-        "overrides": []
-      },
-      "gridPos": {"h": 8, "w": 12, "x": 0, "y": 20},
-      "id": 9,
-      "options": {"legend": {"calcs": ["lastNotNull"], "displayMode": "table", "placement": "right", "showLegend": true}, "tooltip": {"mode": "multi", "sort": "desc"}},
-      "targets": [{"datasource": {"type": "prometheus", "uid": "prometheus"}, "expr": "sum by (reason) (rate(cs_node_wl_hits_ok_total[5m]))", "legendFormat": "{{reason}}", "refId": "A"}],
-      "title": "Whitelist-Treffer",
-      "type": "timeseries"
-    },
-    {
-      "datasource": {"type": "prometheus", "uid": "prometheus"},
-      "gridPos": {"h": 8, "w": 12, "x": 12, "y": 20},
-      "id": 10,
-      "options": {
-        "code": {"language": "plaintext", "showLineNumbers": false, "showMiniMap": false},
-        "content": "## Was dieses Dashboard NICHT zeigt\n\nAlle Panels stammen aus der LAPI und den Agents im Cluster. Die **Bouncer auf proxy-1 und proxy-2 exportieren eigene Metriken**, die hier fehlen:\n\n- `fw_bouncer_active_decisions` — was tatsächlich in den nftables-Sets steht\n- `fw_bouncer_dropped_packets` / `_bytes` — was wirklich verworfen wurde\n- `lapi_requests_total` / `_failures_total` — aus Sicht des Bouncers\n\nSie liegen auf `127.0.0.1:60601` der Proxy-Hosts. Die Hosts sind öffentlich und haben bisher keinerlei Metrik-Pfad; den Port ins Internet zu öffnen, nur um ein Scrape-Ziel zu bekommen, wäre der falsche Tausch.\n\n**Konsequenz:** „Bouncer-Abholrate\" oben zeigt, dass ein Bouncer *fragt* — nicht, dass er die Antwort auch angewandt hat. Solange der Modus `dry-run` ist, wendet er ohnehin nichts an.\n\nEin Scrape-Pfad für die Hosts ist eigene Arbeit und Voraussetzung dafür, dass „was wird wirklich geblockt\" beantwortbar wird."
-      },
-      "title": "Blinder Fleck",
-      "type": "text"
-    }
-  ],
-  "refresh": "30s",
-  "schemaVersion": 38,
-  "style": "dark",
-  "tags": ["homelab", "crowdsec", "security"],
-  "templating": {"list": []},
-  "time": {"from": "now-6h", "to": "now"},
-  "timepicker": {},
-  "timezone": "",
-  "title": "CrowdSec Overview",
-  "uid": "crowdsec-overview",
-  "version": 1,
-  "weekStart": ""
-}

kubernetes/apps/monitoring/dashboards/kustomization.yaml

@@ -38,9 +38,6 @@ configMapGenerator:
   - name: dashboard-blackbox-overview
     files:
       - blackbox-overview.json
-  - name: dashboard-crowdsec-overview
-    files:
-      - crowdsec-overview.json
   - name: dashboard-disk-health
     files:
       - disk-health.json

kubernetes/infrastructure/crowdsec/helmrelease.yaml

@@ -65,21 +65,6 @@ spec:
           - 45.157.177.26/32
           - 2a03:4000:4b:c1e:4474:43ff:fedf:8f85/128
           - 2a03:4000:28:f6e:a4c7:59ff:fe1e:9a29/128
-          # netbird.renner.uno — management, signal, relay and TURN.
-          #
-          # Not optional once a bouncer is armed. Its chain sits on the INPUT
-          # hook, the proxy hosts run no other firewall, and there is therefore
-          # no conntrack rule accepting established flows ahead of it. A ban on
-          # this address would drop the RETURN traffic of the proxies' own
-          # outbound control connection and take them off the mesh — with no
-          # inbound attack involved at all.
-          #
-          # Checked when this was added: none of these addresses appear in the
-          # current decision set. That is the state today, not a guarantee —
-          # these are rented addresses on a hoster whose ranges do end up on
-          # community blocklists.
-          - 162.55.50.97/32
-          - 2a01:4f8:c2c:4579::1/128
 
     # mTLS between agents and the LAPI. Do not switch back to password auth:
     # the LAPI runs without persistence, so its machine-registration DB is
@@ -123,24 +108,6 @@ spec:
 
     lapi:
       replicas: 1
-      service:
-        # Backend wiring for the public LAPI Ingress (lapi-ingress.yaml).
-        #
-        # These belong on the SERVICE, not on the Ingress: under the Kubernetes
-        # Ingress provider `traefik.ingress.kubernetes.io/service.*` are service
-        # annotations, and on an Ingress they are silently ignored. The symptom
-        # is Traefik speaking plaintext to a TLS-only backend, which the LAPI
-        # answers with "Client sent an HTTP request to an HTTPS server." and a
-        # 400 — an error that looks like a malformed request rather than a
-        # scheme mismatch.
-        #
-        # With tls.enabled the LAPI serves HTTPS on 8080 and offers no
-        # plaintext port, so https here is mandatory rather than a hardening
-        # choice. The ServersTransport carries the chart's own CA, since that
-        # certificate is self-signed.
-        annotations:
-          traefik.ingress.kubernetes.io/service.serversscheme: https
-          traefik.ingress.kubernetes.io/service.serverstransport: crowdsec-crowdsec-lapi@kubernetescrd
       # Far below the chart default, which reserves half a core per LAPI for a
       # workload that draws single-digit millicores — enough to leave the
       # cluster unschedulable with one node out. Only the reservation shrinks;

kubernetes/infrastructure/crowdsec/lapi-ingress.yaml

@@ -44,8 +44,8 @@ metadata:
   namespace: crowdsec
 spec:
   serverName: crowdsec-service.crowdsec.svc.cluster.local
-  rootCAs:
-    - secret: crowdsec-root-secret
+  rootCAsSecrets:
+    - crowdsec-root-secret
 ---
 apiVersion: networking.k8s.io/v1
 kind: Ingress
@@ -60,12 +60,8 @@ metadata:
     cert-manager.io/cluster-issuer: letsencrypt-cluster
     # NOT behind the Authentik gate: the client is a daemon with an API key,
     # not a browser that can complete an OIDC flow.
-    #
-    # The backend scheme and ServersTransport are NOT set here. Under the
-    # Kubernetes Ingress provider, `traefik.ingress.kubernetes.io/service.*`
-    # are SERVICE annotations — on an Ingress they are silently ignored, and
-    # Traefik then talks plaintext to a TLS-only backend. They live on the
-    # chart's Service instead, via lapi.service.annotations in helmrelease.yaml.
+    traefik.ingress.kubernetes.io/service.serversscheme: https
+    traefik.ingress.kubernetes.io/service.serverstransport: crowdsec-crowdsec-lapi@kubernetescrd
 spec:
   ingressClassName: traefik
   rules:

kubernetes/infrastructure/network-policies/ai-ns.yaml

@@ -34,17 +34,6 @@ spec:
         - ports:
             - port: "8080"
               protocol: TCP
-    # Home Assistant's llama.cpp conversation agent calls the chat model only —
-    # it has no use for the embedding endpoint, so unlike paperless above this
-    # grant does not extend to llama-embed.
-    - fromEndpoints:
-        - matchLabels:
-            "k8s:io.kubernetes.pod.namespace": home-assistant
-            app.kubernetes.io/name: home-assistant
-      toPorts:
-        - ports:
-            - port: "8080"
-              protocol: TCP
     - fromEndpoints:
         - matchLabels:
             "k8s:io.kubernetes.pod.namespace": monitoring

kubernetes/infrastructure/network-policies/home-assistant-ns.yaml

@@ -167,18 +167,6 @@ spec:
     - toEndpoints:
         - matchLabels:
             "k8s:io.kubernetes.pod.namespace": influxdb
-    # The cluster's own chat model, for the llama.cpp conversation agent that
-    # backs Assist. Scoped to the one pod and port rather than the ai namespace
-    # as a whole, so this grant cannot widen if something else is placed there —
-    # the embedding endpoint next to it is deliberately not included.
-    - toEndpoints:
-        - matchLabels:
-            "k8s:io.kubernetes.pod.namespace": ai
-            app.kubernetes.io/name: llama-server
-      toPorts:
-        - ports:
-            - port: "8080"
-              protocol: TCP
     # traefik for east-west OIDC roundtrips that go through the
     # public hostname via k8s_gateway.
     - toEndpoints: