homelab/kubernetes/apps/monitoring-rules/probes-blackbox.yaml

# HTTP probes — fan out to blackbox-exporter. Probe targets mirror the
# legacy Ansible role's blackbox_http_targets list.
#
# The `release` label on every Probe here IS load-bearing: probeSelector still
# matches on it, unlike serviceMonitorSelector and podMonitorSelector, which
# are empty. Removing it silently stops the probe from being discovered.
apiVersion: monitoring.coreos.com/v1
kind: Probe
metadata:
  name: blackbox-http
  namespace: monitoring
  labels:
    release: kube-prometheus-stack
spec:
  jobName: blackbox-http
  prober:
    url: blackbox-exporter-prometheus-blackbox-exporter.monitoring.svc.cluster.local:9115
  module: http_2xx
  interval: 30s
  scrapeTimeout: 10s
  targets:
    staticConfig:
      static:
        - https://gitea.renner.uno
        - https://immich.renner.uno
        - https://auth.renner.uno
        - https://home.renner.uno
        # prometheus.renner.uno and alerts.renner.uno are Authentik-gated
        # (PR #147 / #146). These EXTERNAL probes now measure the full
        # gated path — DNS, TLS, Traefik, the gate outpost — following the
        # 302 to the Authentik login (200). They are REACHABILITY checks,
        # not app health: with the gate up, they stay green even if the app
        # behind it is dead (the probe never reaches the app). Real app
        # health comes from the two INTERNAL /-/healthy probes below.
        - https://prometheus.renner.uno
        - https://alerts.renner.uno
        - https://dash.renner.uno
        # Offsite backup target (Hetzner Object Storage). Bare endpoint
        # returns 200. CNPG WAL archiving + Velero both depend on it;
        # this probe replaces the old GarageDown signal on nas-1.
        - https://fsn1.your-objectstorage.com
        # NetBird controller VPS (Hetzner) — management/signal/relay/
        # dashboard. Not hairpinned and not split-horizon, so this
        # traverses the WAN and detects a controller/DC outage (the
        # 2026-07-24 one ran ~20 h with no signal). Dashboard root, 200.
        - https://netbird.renner.uno
        # Dead-man probe for the off-site Uptime Kuma.
        - https://kuma.renner.uno
        # NetBird relay health endpoints on the netcup members (relay
        # binary serves :9000/health, verified on 0.73.2). Plain http,
        # ports outside the hairpin's 80/443 — real WAN path.
        - http://proxy-1.renner.uno:9000/health
        - http://proxy-2.renner.uno:9000/health
        # Woodpecker. Ungated, root returns 200 with no redirect. Most
        # scheduled jobs in this repo run here — the nightly
        # scans, the deploy targets — so an outage here is silent by
        # construction: the thing that would have told you is the thing that
        # is down.
        - https://ci.renner.uno
        # Hermes. /api/status is public and answers 200 while the gateway
        # runs. It runs the alert-monitor job that files the [alert] issues,
        # so its outage is silent the same way Woodpecker's is.
        - https://hermes.renner.uno/api/status
        # OpenBao. Probes the health API rather than the UI on purpose: a
        # SEALED instance still serves the UI (307 -> /ui/ -> 200) while every
        # ExternalSecret in the cluster fails, which is exactly the
        # green-while-broken shape this file warns about elsewhere.
        #
        # standbyok=true because the endpoint answers 200 only on the ACTIVE
        # node and 429 on a standby — the external route currently lands on a
        # standby, so the bare path would be permanently red. With the
        # parameter, 200 means initialized, unsealed and serving, on whichever
        # node answers; 503 sealed and 501 uninitialized both stay red.
        - https://openbao.renner.uno/v1/sys/health?standbyok=true
        # ical-proxy serves an unauthenticated /health that answers 200 with
        # {"status":"healthy"}. Unlike the gated hostnames this is real app
        # health, not reachability — the root path returns 404 by design, since
        # the service only answers on calendar paths.
        - https://ical-proxy.renner.uno/health
        # APP-HEALTH probes for the gated monitoring UIs, from the correct
        # in-cluster vantage (monitoring-vantage-point-rule): the gate is
        # only on the ingress, so the ClusterIP /-/healthy endpoints are
        # reachable and unauthenticated in-cluster. These go red when the
        # app is actually down, regardless of the gate — closing the blind
        # spot where a dead app behind a live gate showed green. The
        # blackbox-exporter runs in this namespace, so intra-namespace
        # reach to :9093/:9090 is allowed. (The legacy srv-host Prometheus
        # on VLAN 40 cannot reach cluster ClusterIPs, so it keeps only the
        # external reachability probe above; it is not the health vantage.)
        - http://kube-prometheus-stack-alertmanager.monitoring.svc.cluster.local:9093/-/healthy
        - http://kube-prometheus-stack-prometheus.monitoring.svc.cluster.local:9090/-/healthy
        # Loki has no external vantage at all: its ingress requires auth on
        # every path, so an outside probe can only ever report 401. In-cluster
        # /ready answers 200 unauthenticated and is in this namespace, so the
        # same intra-namespace allowance the two above rely on covers it.
        - http://loki.monitoring.svc.cluster.local:3100/ready
        # APP-HEALTH probes for the two gated navidrome-namespace apps. Same
        # rule as the block above (monitoring-vantage-point-rule), but these
        # are the first targets here that leave the monitoring namespace.
        #
        # Both sit entirely behind an Authentik gate outpost (ingress class
        # authentik-gate; ../navidrome/ingress.yaml, ../navidrome/lidarr.yaml) — so an
        # EXTERNAL probe of either name measures the gate, not the app, and
        # stays green while the app behind
        # a live gate is dead. That external signal is still wanted, and REN-2
        # adds it from UptimeRobot; these are the other half of the pair.
        #
        # NO CiliumNetworkPolicy change is needed — but NOT because
        # cnp-cross-namespace-allow-apps-gatus-to-monitored-services fails to
        # apply here. It applies: its Generalization section names
        # blackbox-exporter by hand. The reason we are safe is that the allow
        # that note demands is PRESENT AND DEDICATED — the navidrome baseline
        # CNP (infrastructure/network-policies/navidrome-ns.yaml) admits
        # blackbox-exporter by pod label on exactly the two listeners these
        # two probes reach, :4533 and :8686. It used to admit the whole
        # monitoring namespace on every port; REN-28 narrowed it, and the
        # ports in that rule are the ones these probes depend on, so a probe
        # added here against a THIRD listener in that namespace would need
        # that rule widened first. Run that check for the next namespace you
        # add; do not skip it on the grounds that the note "is about gatus".
        #
        # EGRESS — the half the wiki note is silent on, and the half that is
        # actually load-bearing here. Cilium enforces at BOTH ends, so the
        # destination ingress allow above is worthless if the prober itself
        # is in egress enforcement without a matching rule. It is not: an
        # endpoint only enters egress enforcement once some policy selecting
        # it declares an `egress:` block, and none does for monitoring. The
        # namespace has exactly two Cilium policies and both are ingress-only:
        # infrastructure/network-policies/monitoring-ns.yaml
        # (default-deny-ingress) and monitoring-alloy-syslog.yaml
        # (allow-syslog-world-ingress). blackbox-exporter runs in that same
        # namespace (../monitoring/helmrelease-blackbox-exporter.yaml#namespace).
        # Re-verified at 2b71dda1 across all 635 yaml files in the repo — 73
        # Cilium objects, 38 carrying an egress or egressDeny block, none of
        # them in the monitoring namespace, and the single
        # CiliumClusterwideNetworkPolicy is scoped to `reserved.health` so it
        # cannot select a pod. 17 other namespaces here already have an egress
        # block. If a routine hardening pass ever adds one to monitoring, it
        # MUST list the navidrome and paperless namespaces or these three
        # cross-namespace targets go red while the *.monitoring.svc ones above
        # keep working — a failure mode nobody will connect back to the
        # hardening commit. The cnp-cross-namespace-* notes document only the
        # ingress half; that silence is what makes this the trap.
        #
        # BLAST RADIUS if that egress dependency is ever broken. Nothing
        # user-facing breaks: these are Prometheus-side Probe targets, not
        # gatus checks, so status.renner.uno is unaffected and no workload is
        # restarted. What breaks is the signal, and it breaks misleadingly.
        # All three targets start failing BlackboxProbeFailing
        # (prometheusrule-blackbox.yaml#blackbox-probe-failing — severity
        # warning, for: 10m); none of the three is in that rule's exclusion
        # list. BlackboxProbesAbsent does NOT fire, because the other 17
        # targets keep producing
        # probe_success — so it reads as three apps going down, not as a
        # policy change. WHO NOTICES: warning routes to the default `email`
        # receiver, admin@renner.uno (helmrelease-prometheus.yaml alertmanager
        # route, group_wait 30s, grouped by alertname), so the owner gets one
        # email roughly 10-11 minutes after the policy lands. HOW FAST UNDONE:
        # the REVERT recipe below is a one-line deletion; from merge it is
        # live within ~11 minutes (GitRepository interval 1m0s,
        # monitoring-rules Kustomization interval 10m), or immediately with
        # `flux reconcile kustomization monitoring-rules --with-source`.
        #
        # /ping on both: unauthenticated, and already the pods' own kubelet
        # probe path (../navidrome/deployment.yaml#kubelet-probes,
        # ../navidrome/lidarr.yaml#kubelet-probes). For lidarr that it is
        # ungated is proven independently of the gate — its reconcile script
        # (../navidrome/lidarr.yaml#reconcile-script) curls /ping in a
        # loop BEFORE it has read the API key out of config.xml.
        #
        # STATUS CODE, unproven for all three of the new targets: module
        # http_2xx follows redirects and passes only on a FINAL 2xx, while
        # every piece of repo evidence behind these paths establishes only
        # `< 400` — a kubelet httpGet accepts 200-399, and that reconcile
        # script's `curl -f` without -L exits 0 on a 302. That evidence proves the
        # paths are UNGATED, not that they answer 200. Check
        # probe_http_status_code on the first scrapes; a target that settles
        # on a 3xx gets REVERTED, not widened to accept 3xx.
        #
        # REVERT — an executed recipe, not a claim. Every command below was
        # actually run against master 2b71dda1 on 2026-09-25; the results are
        # what is recorded here.
        #
        # NO `git revert` works. All four candidates conflict in THIS file at
        # 2b71dda1, because the comment blocks around these targets were
        # rewritten after the probes landed (#979):
        #
        #   git revert -m 1 0a82d61f   merge of #977, paperless     CONFLICT
        #   git revert -m 1 16238d39   merge of #976, nav + lidarr  CONFLICT
        #   git revert cb9e73bd        the paperless commit         CONFLICT
        #   git revert 2219c928        the nav + lidarr commit      CONFLICT
        #
        # Hand-resolving any of them is the trap: the natural --theirs
        # resolution of the 16238d39 conflict also drops the paperless target
        # below, leaving NO app-health probes and no signal that it happened.
        #
        # Delete the target lines forward instead. Only the target lines are
        # part of the object; the comments are inert. Both of these were run
        # and verified:
        #
        #   # all three app-health targets — blackbox-http 20 -> 17
        #   sed -i '/svc\.cluster\.local:80\/ping$/d;/paperless\.paperless\.svc\.cluster\.local:80\/accounts\/login\/$/d' \
        #     kubernetes/apps/monitoring-rules/probes-blackbox.yaml
        #
        #   # the paperless target alone — blackbox-http 20 -> 19
        #   sed -i '/paperless\.paperless\.svc\.cluster\.local:80\/accounts\/login\/$/d' \
        #     kubernetes/apps/monitoring-rules/probes-blackbox.yaml
        #
        # Verified after each run: the file still parses, blackbox-http drops
        # to 17 and to 19 respectively, blackbox-edge-wan (3) and
        # blackbox-tcp-relays (2) are untouched, and `git diff` contains
        # nothing but the deleted target lines. Then drop whichever comment
        # block is left orphaned, open a PR, and merge — timing as in the
        # BLAST RADIUS paragraph above.
        - http://navidrome.navidrome.svc.cluster.local:80/ping
        - http://lidarr.navidrome.svc.cluster.local:80/ping
        # anchor: paperless-app-health
        # APP-HEALTH probe for paperless. Same rule as the block above
        # (monitoring-vantage-point-rule); the blind spot it closes is the one
        # gatus already documents in its Web Service 11 comment
        # (../gatus/configmap.yaml#gatus-endpoints) — through the
        # gate outpost EVERY path, /api/ included,
        # answers 302 to an unauthenticated caller, so no external probe of
        # paperless.renner.uno can tell a live app from a live gate. That
        # external signal is still wanted, and REN-2 adds it from UptimeRobot;
        # this is the other half of the pair.
        #
        # That gatus comment says the in-cluster probe "needs a
        # CiliumNetworkPolicy first". That sentence was WRONG when written,
        # and still is, on different grounds. The probe is admitted by the
        # dedicated policy
        # ../../infrastructure/network-policies/paperless-ns.yaml#paperless-allow-blackbox,
        # scoped to blackbox-exporter on the paperless pod's :8000 (REN-28);
        # the namespace-wide `monitoring` admit this probe used to rely on is
        # gone. Cilium unions the allow rules of every CNP selecting an
        # endpoint (the mechanism ../paperless/network-policy.yaml#cnp-union
        # relies on), so that policy is all the probe needs. The EGRESS
        # paragraph in the block above applies here identically.
        #
        # The stale sentence is deliberately left in place in
        # apps/gatus/configmap.yaml: it sits INSIDE the `config.yaml: |`
        # block scalar, so editing it changes ConfigMap data and
        # `configmap.reloader.stakater.com/reload`
        # (../gatus/deployment.yaml#reloader) rolls the gatus Deployment. The
        # correction is written at the top of
        # that file instead, outside `data:`, where it costs no restart.
        #
        # /accounts/login/ because paperless-ngx has no unauthenticated health
        # endpoint at all. It is the app's own established "the webserver
        # answers" signal — the same path its startup, liveness and readiness
        # probes use (../paperless/paperless.yaml#http-probes) — and it
        # renders a Django view
        # rather than a static file. It does NOT cover the consumer/celery
        # workers: see process-liveness-vs-protocol-liveness.
        #
        # The cluster-local Host header is accepted only because
        # PAPERLESS_ALLOWED_HOSTS is "*"
        # (../paperless/paperless.yaml#allowed-hosts). Narrow that and
        # Django answers 400 here and this probe goes permanently red.
        #
        # This is the target most likely to fail the STATUS CODE check in the
        # block above — a Django login view can answer 302 — so check it
        # first, and revert rather than widen if it does. Use the REVERT
        # recipe in that block: `git revert -m 1 0a82d61f` conflicts here too
        # at 2b71dda1, and the tested one-line deletion for this target alone
        # is the second command listed there (20 -> 19 targets).
        - http://paperless.paperless.svc.cluster.local:80/accounts/login/
        # anchor: shelly-manager-app-health
        # APP-HEALTH probe for shelly-manager (REN-531/REN-533). Same rule as
        # the blocks above (monitoring-vantage-point-rule): shelly.renner.uno
        # sits behind a gate outpost, so only the in-cluster
        # Service can tell a dead app from a live gate.
        #
        # /health is unauthenticated and answers a plain 200 — not a 3xx —
        # per the app's own test (shelly-manager repo,
        # AuthenticationTests.Health_IsUnauthenticated); it is also the pod's
        # kubelet liveness/readiness path
        # (../home-assistant-iot/shelly-manager-deployment.yaml). AllowedHosts is
        # "*", so the cluster-local Host header is accepted. Service :80 ->
        # container :8080 (../home-assistant-iot/shelly-manager-service.yaml).
        #
        # NO CiliumNetworkPolicy change: the home-assistant default-deny
        # admits the whole monitoring namespace on every port
        # (../../infrastructure/network-policies/home-assistant-ns.yaml,
        # "Prometheus scrape" rule). If that rule is ever narrowed to
        # Prometheus alone, add a blackbox-exporter grant on :8080 in a
        # DEDICATED policy selecting shelly-manager — not on the {} selector,
        # where :8080 is shared by three apps. The EGRESS paragraph above
        # applies too: a future monitoring egress block must list
        # home-assistant.
        #
        # REVERT: delete the target line below.
        - http://shelly-manager.home-assistant.svc.cluster.local:80/health
# WAN-path probes of the public edge BY PROXY IP. The name-based
# home.renner.uno probe above resolves in-cluster (CoreDNS/k8s_gateway)
# and never leaves the LAN — the vantage-point blind spot that let the
# 2026-07-24 outage run silent. IP-literal targets dodge both the
# split-horizon DNS and the UniFi hairpin (absent on the Server/Ingress
# VLANs the nodes use), so these genuinely go WAN → netcup proxy →
# mesh → cluster. One target per edge member/family; keep the
# addresses in sync with proxy DNS + opentofu/unifi_nat_hairpin.tf.
---
apiVersion: monitoring.coreos.com/v1
kind: Probe
metadata:
  name: blackbox-edge-wan
  namespace: monitoring
  labels:
    release: kube-prometheus-stack
spec:
  jobName: blackbox-edge-wan
  prober:
    url: blackbox-exporter-prometheus-blackbox-exporter.monitoring.svc.cluster.local:9115
  module: http_2xx_wan_edge
  interval: 60s
  scrapeTimeout: 15s
  targets:
    staticConfig:
      static:
        # Member attribution follows DNS, which is the operational truth
        # (verified 2026-07-26: dig + live ssh — the v6-only host with
        # the dhcpcd/no-IPv4 lease is proxy-1.renner.uno = 2a03:…:28:f6e…;
        # proxy-2.renner.uno is the dual-stack member; re-verified
        # 2026-07-27). opentofu/unifi_nat_hairpin.tf now agrees; the
        # mismatch noted here previously has been corrected.
        # proxy-2 v4 (dual-stack member)
        - https://45.157.177.26
        # proxy-2 v6
        - https://[2a03:4000:4b:c1e:4474:43ff:fedf:8f85]
        # proxy-1 v6 (v6-only member)
        - https://[2a03:4000:28:f6e:a4c7:59ff:fe1e:9a29]
---
# TCP reachability of the NetBird relay listeners on the netcup members
# (rel:// WebSocket on :33080). Complements the :9000/health probes in
# blackbox-http: health says the process is alive, this says the actual
# relay port is reachable from outside.
apiVersion: monitoring.coreos.com/v1
kind: Probe
metadata:
  name: blackbox-tcp-relays
  namespace: monitoring
  labels:
    release: kube-prometheus-stack
spec:
  jobName: blackbox-tcp-relays
  prober:
    url: blackbox-exporter-prometheus-blackbox-exporter.monitoring.svc.cluster.local:9115
  module: tcp_connect
  interval: 60s
  scrapeTimeout: 10s
  targets:
    staticConfig:
      static:
        - proxy-1.renner.uno:33080
        - proxy-2.renner.uno:33080
---
# anchor: negative-control-paperless
# NEGATIVE CONTROLS for the paperless ingress scoping (REN-28 PR 2a).
#
# These four targets are here TO BE REFUSED. They are not availability
# monitoring: every one of them is a workload in the `paperless` namespace that
# blackbox-exporter has no business reaching, and each must read
# `probe_success == 1` before REN-28's PR 2b and `0` after it. That flip is the
# test. Without them, "the whole-namespace admit is gone" is an assertion about
# a CNP diff rather than a measured property of the cluster, and a later
# re-widening of the grant would be silent.
#
# Why this can only be the ingress policy under test: blackbox-exporter runs in
# `monitoring`, whose baseline CNP (infrastructure/network-policies/
# monitoring-ns.yaml) is ingress-only — it declares no `egress:` at all — and
# the cluster's default-deny.yaml is scoped `namespace: apps`, which covers
# neither `monitoring` nor `paperless`. So the only thing that can turn these
# red is paperless's own ingress policy, which is exactly the variable. Nothing
# else can make the test pass or fail.
#
# WHY A SEPARATE jobName, and why these must NEVER be folded into the
# blackbox-http Probe above: after PR 2b these targets fail permanently and BY
# DESIGN. BlackboxProbeFailing and BlackboxProbesAbsent
# (prometheusrule-blackbox.yaml#blackbox-probe-failing and
# prometheusrule-blackbox.yaml#blackbox-probes-absent) are both scoped to
# job="blackbox-http", so a distinct job is outside both and stays silent — and
# the hand-maintained instance!~ exclusion regex in the first of them, whose
# fragility that file warns about, does not have to grow a fifth entry. Adding
# these to blackbox-http instead would
# page the on-call forever.
#
# The inverted alert that gives these targets teeth — CNPNegativeControlReachable
# (prometheusrule-blackbox.yaml#cnp-negative-control-reachable),
# `probe_success{job="blackbox-negative-control"} == 1` — landed with PR 2b,
# together with the policy change, not before: until then these controls were
# SUPPOSED to read 1, so the alert would have fired on the correct state.
#
# TWO WAYS A BROKEN HARNESS READS AS A PASSING TEST. The inverted alert fires
# only on `1`, so anything that forces `0` for a reason other than the policy
# disarms it silently. Both of these are load-bearing:
#
#   * The IPv4 fallback. The shared tcp_connect module
#     (monitoring/helmrelease-blackbox-exporter.yaml) prefers ip6 with
#     ip_protocol_fallback: true. All four Services are SingleStack IPv4, so
#     ip6 never resolves and every `1` these controls ever read came through
#     the fallback (probe_ip_protocol == 4). Set the fallback to false and all
#     four drop to `0` with the policy wide open. Do not change the module for
#     these targets — blackbox-tcp-relays pins ip6-first on purpose; give this
#     job its own module instead.
#   * scrapeTimeout: 10s below. Cilium drops the SYN, so a refused connect runs
#     to the module's own 5s timeout. 5s < 10s is what keeps the SCRAPE
#     succeeding, so `up` stays 1 and probe_success reads 0 instead of going
#     absent. Do not trim scrapeTimeout toward 5s: a scrape that times out
#     leaves no probe_success series for the inverted alert to read, and sets
#     `up` to 0, which the chart's job-agnostic TargetDown (defaultRules are on
#     in monitoring/helmrelease-prometheus.yaml) can page on.
#
# A bare TCP connect is the right probe: it completes or it times out, with no
# application-layer ambiguity about whether a 403 meant the policy or the app.
# Known, accepted side effect: the pprl-pg-rw connect opens and closes without a
# startup packet, so Postgres logs "incomplete startup packet" at LOG level once
# a minute until PR 2b lands and the connect stops arriving.
apiVersion: monitoring.coreos.com/v1
kind: Probe
metadata:
  name: blackbox-negative-control-paperless
  namespace: monitoring
  labels:
    # Load-bearing, as noted at the top of this file: probeSelector matches on
    # it. Without it this Probe is never discovered and the control reads
    # `absent`, not `1` — which would look like a passing negative test.
    release: kube-prometheus-stack
spec:
  jobName: blackbox-negative-control
  prober:
    url: blackbox-exporter-prometheus-blackbox-exporter.monitoring.svc.cluster.local:9115
  module: tcp_connect
  interval: 60s
  scrapeTimeout: 10s
  targets:
    staticConfig:
      static:
        # Proves BOTH halves of the pprl-pg scoping PR 2b will introduce: the
        # source (blackbox is not Prometheus, and the pprl-pg-allow-prometheus
        # rule 2b adds admits only Prometheus) and the port (5432 is not the
        # exporter's 9187). Both that rule and paperless-allow-blackbox
        # (../../infrastructure/network-policies/paperless-ns.yaml#pprl-pg-allow-prometheus,
        # ...#paperless-allow-blackbox) now exist, which is why this target
        # must read 0. Service is CNPG-generated from Cluster `pprl-pg`
        # (../paperless/cluster.yaml#pprl-pg) and is the name paperless itself
        # dials (../paperless/paperless.yaml#db-host).
        - pprl-pg-rw.paperless.svc.cluster.local:5432
        # Three witnesses that the whole-namespace `monitoring` admit is gone
        # for workloads no new rule selects. Ports from the Services themselves:
        # ../paperless/redis.yaml#service-port,
        # ../paperless/tika.yaml#service-port,
        # ../paperless/gotenberg.yaml#service-port.
        - redis.paperless.svc.cluster.local:6379
        - tika.paperless.svc.cluster.local:9998
        - gotenberg.paperless.svc.cluster.local:3000