claude-botandClaude Opus 5.5
ac44925baf
crowdsec, external-dns-unifi, netbird-ingress and velero were in the infrastructure Kustomization, while the ExternalSecrets that create their Secrets were in infrastructure-crds, which depends on infrastructure. On a cold start each was therefore installed before its Secret could exist: the pods sat in CreateContainerConfigError until OpenBao was unsealed and ESO synced. For the HelmReleases (crowdsec, external-dns-unifi, velero) Helm's wait timed out meanwhile; with remediation.retries: 3 they give up after four attempts and stay failed even once the Secret appears. Velero failing that way stops every restore. | workload | Secrets | |---|---| | crowdsec | crowdsec-bouncer-key (crowdsec, ingress), crowdsec-edge-bouncer-key | | external-dns-unifi | external-dns-unifi-secret | | netbird-ingress | netbird-ingress-setup-key | | velero (+ backup-verify CronJob) | cloud-credentials, cloud-credentials-hetzner, hcloud-backup-verify | Each now has a Kustomization of its own (clusters/homelab/ infrastructure-services.yaml), depending on infrastructure and infrastructure-crds, with its ExternalSecrets beside it. The openbao ClusterSecretStore in infrastructure-crds turns Ready only once OpenBao is unsealed, so the Secret exists before the consumer installs. One Kustomization each so that a broken service holds back only itself. Velero's Schedules need the velero.io CRDs from its HelmRelease, so they move from post-crds to infrastructure/velero-schedules with a Kustomization after velero. post-crds keeps cloudflare-api-token (consumed by its own ClusterIssuer) and flux-git. The directories themselves stay where they are, so paths such as the CI test infrastructure/velero/backup-verify/tests/run.sh are unchanged. netbird-ingress gets 10m to roll its DaemonSet over three nodes (maxUnavailable 1, minReadySeconds 45). The crowdsec comments name the Kustomization it now runs in, and flux-graph.sh colours the new nodes. Production rollout: infrastructure and infrastructure-crds must run with prune: false (the stack's first PR) when this merges. Pruned, the velero HelmRelease would be uninstalled, and the moved ExternalSecrets would take their Secrets with them. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
85 lines
3.0 KiB
Bash
Executable File
85 lines
3.0 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
# Draws the Flux Kustomization dependency graph from the dependsOn fields in
|
|
# kubernetes/clusters/homelab. Edges point from a dependency to its dependent.
|
|
#
|
|
# flux-graph.sh DOT on stdout, every edge
|
|
# flux-graph.sh out.dot the same, written to out.dot
|
|
# flux-graph.sh out.svg rendered; out.png likewise
|
|
#
|
|
# Rendered output is the transitive reduction (tred): an edge implied by a
|
|
# longer path is left out, the order is the same.
|
|
#
|
|
# Green: infrastructure. Blue: the Velero restore chain. Dashed: nothing is
|
|
# health-checked. Grey fill: suspended.
|
|
#
|
|
# Needs jq, either yq (mikefarah's or the Python wrapper) and, to render,
|
|
# graphviz.
|
|
|
|
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
CLUSTER_DIR="${REPO_ROOT}/kubernetes/clusters/homelab"
|
|
OUT="${1:-}"
|
|
|
|
command -v jq >/dev/null || { echo "jq not found" >&2; exit 1; }
|
|
command -v yq >/dev/null || { echo "yq not found" >&2; exit 1; }
|
|
|
|
# One JSON document per YAML document.
|
|
yaml_to_json() {
|
|
if [[ "$(yq --version)" == *mikefarah* ]]; then
|
|
yq eval-all -o=json -I=0 '.' "$@"
|
|
else
|
|
yq -c '.' "$@"
|
|
fi
|
|
}
|
|
|
|
to_dot() {
|
|
yaml_to_json "${CLUSTER_DIR}"/*.yaml | jq -r -s '
|
|
[ .[]
|
|
| select(.kind == "Kustomization"
|
|
and .apiVersion == "kustomize.toolkit.fluxcd.io/v1") ] as $ks
|
|
| "digraph flux {",
|
|
" rankdir=LR;",
|
|
" node [shape=box, style=\"rounded\", fontname=\"Helvetica\", fontsize=10];",
|
|
" edge [color=\"#888888\"];",
|
|
( $ks[]
|
|
| .metadata.name as $n
|
|
# healthChecks are ignored under wait: true; without either, nothing
|
|
# is checked, healthCheckExprs included.
|
|
| (if .spec.wait == true then "wait"
|
|
elif ((.spec.healthChecks // []) | length) > 0 then "health checks"
|
|
else "wait: false" end) as $wait
|
|
| (if .spec.suspend == true then ", style=\"rounded,filled\", fillcolor=\"#dddddd\""
|
|
elif $wait == "wait: false" then ", style=\"rounded,dashed\""
|
|
else "" end) as $dash
|
|
| (if .spec.suspend == true then "\\nsuspended" else "" end) as $suspended
|
|
| (if ($n | startswith("restore-")) or ($n | IN("velero", "velero-backups-synced"))
|
|
then ", color=\"#1f77b4\", fontcolor=\"#1f77b4\""
|
|
elif ($n | startswith("infrastructure")) or ($n | IN("crowdsec", "external-dns-unifi", "netbird-ingress", "velero-schedules"))
|
|
then ", color=\"#2ca02c\", fontcolor=\"#2ca02c\""
|
|
else "" end) as $colour
|
|
| " \"\($n)\" [label=\"\($n)\\n\($wait) · \(.spec.timeout // "-")\($suspended)\"\($dash)\($colour)];" ),
|
|
( $ks[]
|
|
| .metadata.name as $n
|
|
| (.spec.dependsOn // [])[]
|
|
| " \"\(.name)\" -> \"\($n)\";" ),
|
|
"}"'
|
|
}
|
|
|
|
case "${OUT}" in
|
|
"")
|
|
to_dot
|
|
;;
|
|
*.dot)
|
|
to_dot > "${OUT}"
|
|
;;
|
|
*.svg|*.png)
|
|
command -v dot >/dev/null || { echo "graphviz (dot) not found" >&2; exit 1; }
|
|
to_dot | tred | dot -T"${OUT##*.}" -Gdpi=110 -o "${OUT}"
|
|
;;
|
|
*)
|
|
echo "usage: $0 [out.dot|out.svg|out.png]" >&2
|
|
exit 1
|
|
;;
|
|
esac
|