diff --git a/kubernetes/e2e-test.sh b/kubernetes/e2e-test.sh index 97240042..dfa8aac4 100755 --- a/kubernetes/e2e-test.sh +++ b/kubernetes/e2e-test.sh @@ -77,12 +77,11 @@ http_body() { purge_released_pvs() { local pattern="$1" # regex applied to claimRef.namespace + "/" + claimRef.name local pv pvs - pvs=$(kubectl get pv -o json 2>/dev/null \ - | jq -r --arg p "$pattern" ' - .items[] - | select(.status.phase == "Released") - | select(((.spec.claimRef.namespace // "") + "/" + (.spec.claimRef.name // "")) | test($p)) - | .metadata.name') + # No jq here: the ci-k8s image doesn't ship it, and the original jq + # version silently produced an empty list for months, leaking Released + # PVs (and their CephFS/RBD data) on every run — issue #262. + pvs=$(kubectl get pv -o jsonpath='{range .items[?(@.status.phase=="Released")]}{.metadata.name}{" "}{.spec.claimRef.namespace}{"/"}{.spec.claimRef.name}{"\n"}{end}' 2>/dev/null \ + | awk -v p="$pattern" '$2 ~ p {print $1}') for pv in $pvs; do kubectl patch pv "$pv" --type=merge \ -p '{"spec":{"persistentVolumeReclaimPolicy":"Delete"}}' >/dev/null 2>&1 || true @@ -103,8 +102,8 @@ tls_valid() { } echo "=== Cluster ===" -check "4 nodes Ready" \ - bash -c 'test "$(kubectl get nodes --no-headers | grep -c " Ready ")" -eq 4' +check "3 nodes Ready" \ + bash -c 'test "$(kubectl get nodes --no-headers | grep -c " Ready ")" -eq 3' echo "=== Flux Kustomizations ===" for ks in flux-system infrastructure infrastructure-crds infrastructure-ceph-cluster \ @@ -124,9 +123,12 @@ check "all HelmReleases Ready" bash -c ' echo "=== Pods ===" check "no failing pods" bash -c ' + # woodpecker-ci is excluded: the runner pod executing THIS e2e run (and + # any parallel build) is legitimately Pending/ContainerCreating for a + # few seconds and made the check flap (issue #262). bad=$(kubectl get pods -A \ --field-selector=status.phase!=Running,status.phase!=Succeeded \ - --no-headers 2>/dev/null || true) + --no-headers 2>/dev/null | grep -v "^woodpecker-ci " || true) [[ -z "$bad" ]] || { echo "$bad"; exit 1; } ' @@ -367,16 +369,27 @@ check "auth.renner.uno resolves via 1.1.1.1" \ echo "=== Data presence (post-restore) ===" # Floors (>=) chosen to be well below current values so adding/removing real # data doesn't break the test, while a fully empty restore would. +# CNPG instance pods are NOT stable names — the operator replaces instances +# on failover/maintenance (pprl-pg-1 no longer exists; issue #262). Resolve +# the current primary by role label instead of hardcoding a pod. +cnpg_primary() { # + kubectl -n "$1" get pod \ + -l "cnpg.io/cluster=$2,cnpg.io/instanceRole=primary" \ + -o jsonpath='{.items[0].metadata.name}' 2>/dev/null +} +# Exported so the `bash -c '...'` subshells the `check` helper uses can call +# it (same mechanism as helper_curl below). +export -f cnpg_primary check "paperless: documents_document rows >= 100" \ - bash -c 'n=$(kubectl -n paperless exec pprl-pg-1 -c postgres -- \ + bash -c 'n=$(kubectl -n paperless exec "$(cnpg_primary paperless pprl-pg)" -c postgres -- \ psql -U postgres -d paperless -tAc "select count(*) from documents_document" 2>/dev/null \ | tr -d "[:space:]"); echo "n=$n"; test "$n" -ge 100' check "authentik: users >= 1" \ - bash -c 'n=$(kubectl -n authentik exec ak-pg-1 -c postgres -- \ + bash -c 'n=$(kubectl -n authentik exec "$(cnpg_primary authentik ak-pg)" -c postgres -- \ psql -U postgres -d authentik -tAc "select count(*) from authentik_core_user" 2>/dev/null \ | tr -d "[:space:]"); echo "n=$n"; test "$n" -ge 1' check "authentik: applications >= 1" \ - bash -c 'n=$(kubectl -n authentik exec ak-pg-1 -c postgres -- \ + bash -c 'n=$(kubectl -n authentik exec "$(cnpg_primary authentik ak-pg)" -c postgres -- \ psql -U postgres -d authentik -tAc "select count(*) from authentik_core_application" 2>/dev/null \ | tr -d "[:space:]"); echo "n=$n"; test "$n" -ge 1' check "audiobookshelf: /audiobooks has >= 50 entries" \ @@ -388,11 +401,15 @@ check "audiobookshelf: /audiobooks has >= 50 entries" \ # of #730's empty observability bodies (kubectl exec against a missing/dying # pod returns no stdout, so the per-check $out was empty and the failure tail # was uninformative). -kubectl -n default delete pod e2e-helper --ignore-not-found --wait=true --timeout=30s >/dev/null 2>&1 || true +# The helper lives in the MONITORING namespace: its whole purpose is to curl +# Loki/Prometheus/Alertmanager, and monitoring's default-deny-ingress CNP +# only admits intra-namespace traffic (plus a short allow-list that `default` +# is not on — running it there timed out every obs check, issue #262). +kubectl -n monitoring delete pod e2e-helper --ignore-not-found --wait=true --timeout=30s >/dev/null 2>&1 || true cat <<'EOF' | kubectl apply -f - >/dev/null apiVersion: v1 kind: Pod -metadata: { name: e2e-helper, namespace: default } +metadata: { name: e2e-helper, namespace: monitoring } spec: restartPolicy: Never securityContext: @@ -409,15 +426,15 @@ spec: allowPrivilegeEscalation: false capabilities: { drop: ["ALL"] } EOF -if ! kubectl -n default wait --for=condition=ready pod/e2e-helper --timeout=180s; then +if ! kubectl -n monitoring wait --for=condition=ready pod/e2e-helper --timeout=180s; then echo "ERROR: e2e-helper pod did not become ready in 180s. Diagnostic:" - kubectl -n default describe pod e2e-helper || true + kubectl -n monitoring describe pod e2e-helper || true exit 1 fi # Sanity check: confirm exec actually works before the obs section depends on it. -if ! kubectl -n default exec e2e-helper -- echo "helper-alive" | grep -q helper-alive; then +if ! kubectl -n monitoring exec e2e-helper -- echo "helper-alive" | grep -q helper-alive; then echo "ERROR: kubectl exec into e2e-helper failed despite Ready status." - kubectl -n default describe pod e2e-helper || true + kubectl -n monitoring describe pod e2e-helper || true exit 1 fi @@ -432,7 +449,7 @@ helper_curl() { local url="$1" local out body code attempt for attempt in 1 2 3 4 5; do - out=$(kubectl -n default exec e2e-helper -- \ + out=$(kubectl -n monitoring exec e2e-helper -- \ curl -sk --max-time 10 -w $'\n__CODE__:%{http_code}' "$url" 2>&1) body=${out%$'\n'__CODE__:*} code=${out##*__CODE__:} @@ -492,7 +509,7 @@ for slug in audiobookshelf grafana; do done # Tear down the helper pod regardless of pass/fail so reruns are clean. -kubectl -n default delete pod e2e-helper --ignore-not-found --wait=false >/dev/null 2>&1 || true +kubectl -n monitoring delete pod e2e-helper --ignore-not-found --wait=false >/dev/null 2>&1 || true echo "" echo "========================================"