kubernetes/e2e-test.sh

@@ -77,12 +77,11 @@ http_body() {
 purge_released_pvs() {
     local pattern="$1"   # regex applied to claimRef.namespace + "/" + claimRef.name
     local pv pvs
-    pvs=$(kubectl get pv -o json 2>/dev/null \
-        | jq -r --arg p "$pattern" '
-            .items[]
-            | select(.status.phase == "Released")
-            | select(((.spec.claimRef.namespace // "") + "/" + (.spec.claimRef.name // "")) | test($p))
-            | .metadata.name')
+    # No jq here: the ci-k8s image doesn't ship it, and the original jq
+    # version silently produced an empty list for months, leaking Released
+    # PVs (and their CephFS/RBD data) on every run — issue #262.
+    pvs=$(kubectl get pv -o jsonpath='{range .items[?(@.status.phase=="Released")]}{.metadata.name}{" "}{.spec.claimRef.namespace}{"/"}{.spec.claimRef.name}{"\n"}{end}' 2>/dev/null \
+        | awk -v p="$pattern" '$2 ~ p {print $1}')
     for pv in $pvs; do
         kubectl patch pv "$pv" --type=merge \
             -p '{"spec":{"persistentVolumeReclaimPolicy":"Delete"}}' >/dev/null 2>&1 || true
@@ -103,8 +102,8 @@ tls_valid() {
 }
 
 echo "=== Cluster ==="
-check "4 nodes Ready" \
-    bash -c 'test "$(kubectl get nodes --no-headers | grep -c " Ready ")" -eq 4'
+check "3 nodes Ready" \
+    bash -c 'test "$(kubectl get nodes --no-headers | grep -c " Ready ")" -eq 3'
 
 echo "=== Flux Kustomizations ==="
 for ks in flux-system infrastructure infrastructure-crds infrastructure-ceph-cluster \
@@ -124,9 +123,12 @@ check "all HelmReleases Ready" bash -c '
 
 echo "=== Pods ==="
 check "no failing pods" bash -c '
+    # woodpecker-ci is excluded: the runner pod executing THIS e2e run (and
+    # any parallel build) is legitimately Pending/ContainerCreating for a
+    # few seconds and made the check flap (issue #262).
     bad=$(kubectl get pods -A \
         --field-selector=status.phase!=Running,status.phase!=Succeeded \
-        --no-headers 2>/dev/null || true)
+        --no-headers 2>/dev/null | grep -v "^woodpecker-ci " || true)
     [[ -z "$bad" ]] || { echo "$bad"; exit 1; }
 '
 
@@ -367,16 +369,27 @@ check "auth.renner.uno resolves via 1.1.1.1" \
 echo "=== Data presence (post-restore) ==="
 # Floors (>=) chosen to be well below current values so adding/removing real
 # data doesn't break the test, while a fully empty restore would.
+# CNPG instance pods are NOT stable names — the operator replaces instances
+# on failover/maintenance (pprl-pg-1 no longer exists; issue #262). Resolve
+# the current primary by role label instead of hardcoding a pod.
+cnpg_primary() { # <namespace> <cluster>
+    kubectl -n "$1" get pod \
+      -l "cnpg.io/cluster=$2,cnpg.io/instanceRole=primary" \
+      -o jsonpath='{.items[0].metadata.name}' 2>/dev/null
+}
+# Exported so the `bash -c '...'` subshells the `check` helper uses can call
+# it (same mechanism as helper_curl below).
+export -f cnpg_primary
 check "paperless: documents_document rows >= 100" \
-    bash -c 'n=$(kubectl -n paperless exec pprl-pg-1 -c postgres -- \
+    bash -c 'n=$(kubectl -n paperless exec "$(cnpg_primary paperless pprl-pg)" -c postgres -- \
         psql -U postgres -d paperless -tAc "select count(*) from documents_document" 2>/dev/null \
         | tr -d "[:space:]"); echo "n=$n"; test "$n" -ge 100'
 check "authentik: users >= 1" \
-    bash -c 'n=$(kubectl -n authentik exec ak-pg-1 -c postgres -- \
+    bash -c 'n=$(kubectl -n authentik exec "$(cnpg_primary authentik ak-pg)" -c postgres -- \
         psql -U postgres -d authentik -tAc "select count(*) from authentik_core_user" 2>/dev/null \
         | tr -d "[:space:]"); echo "n=$n"; test "$n" -ge 1'
 check "authentik: applications >= 1" \
-    bash -c 'n=$(kubectl -n authentik exec ak-pg-1 -c postgres -- \
+    bash -c 'n=$(kubectl -n authentik exec "$(cnpg_primary authentik ak-pg)" -c postgres -- \
         psql -U postgres -d authentik -tAc "select count(*) from authentik_core_application" 2>/dev/null \
         | tr -d "[:space:]"); echo "n=$n"; test "$n" -ge 1'
 check "audiobookshelf: /audiobooks has >= 50 entries" \
@@ -388,11 +401,15 @@ check "audiobookshelf: /audiobooks has >= 50 entries" \
 # of #730's empty observability bodies (kubectl exec against a missing/dying
 # pod returns no stdout, so the per-check $out was empty and the failure tail
 # was uninformative).
-kubectl -n default delete pod e2e-helper --ignore-not-found --wait=true --timeout=30s >/dev/null 2>&1 || true
+# The helper lives in the MONITORING namespace: its whole purpose is to curl
+# Loki/Prometheus/Alertmanager, and monitoring's default-deny-ingress CNP
+# only admits intra-namespace traffic (plus a short allow-list that `default`
+# is not on — running it there timed out every obs check, issue #262).
+kubectl -n monitoring delete pod e2e-helper --ignore-not-found --wait=true --timeout=30s >/dev/null 2>&1 || true
 cat <<'EOF' | kubectl apply -f - >/dev/null
 apiVersion: v1
 kind: Pod
-metadata: { name: e2e-helper, namespace: default }
+metadata: { name: e2e-helper, namespace: monitoring }
 spec:
   restartPolicy: Never
   securityContext:
@@ -409,15 +426,15 @@ spec:
         allowPrivilegeEscalation: false
         capabilities: { drop: ["ALL"] }
 EOF
-if ! kubectl -n default wait --for=condition=ready pod/e2e-helper --timeout=180s; then
+if ! kubectl -n monitoring wait --for=condition=ready pod/e2e-helper --timeout=180s; then
     echo "ERROR: e2e-helper pod did not become ready in 180s. Diagnostic:"
-    kubectl -n default describe pod e2e-helper || true
+    kubectl -n monitoring describe pod e2e-helper || true
     exit 1
 fi
 # Sanity check: confirm exec actually works before the obs section depends on it.
-if ! kubectl -n default exec e2e-helper -- echo "helper-alive" | grep -q helper-alive; then
+if ! kubectl -n monitoring exec e2e-helper -- echo "helper-alive" | grep -q helper-alive; then
     echo "ERROR: kubectl exec into e2e-helper failed despite Ready status."
-    kubectl -n default describe pod e2e-helper || true
+    kubectl -n monitoring describe pod e2e-helper || true
     exit 1
 fi
 
@@ -432,7 +449,7 @@ helper_curl() {
     local url="$1"
     local out body code attempt
     for attempt in 1 2 3 4 5; do
-        out=$(kubectl -n default exec e2e-helper -- \
+        out=$(kubectl -n monitoring exec e2e-helper -- \
               curl -sk --max-time 10 -w $'\n__CODE__:%{http_code}' "$url" 2>&1)
         body=${out%$'\n'__CODE__:*}
         code=${out##*__CODE__:}
@@ -492,7 +509,7 @@ for slug in audiobookshelf grafana; do
 done
 
 # Tear down the helper pod regardless of pass/fail so reruns are clean.
-kubectl -n default delete pod e2e-helper --ignore-not-found --wait=false >/dev/null 2>&1 || true
+kubectl -n monitoring delete pod e2e-helper --ignore-not-found --wait=false >/dev/null 2>&1 || true
 
 echo ""
 echo "========================================"