fix: [alert] KubeJobFailed (gitea) (#1214 )

2026-10-09 01:42:49 +00:00
1 changed files with 20 additions and 0 deletions
@@ -8,6 +8,8 @@ import os
 import re
 import ssl
 import sys
+import time
+import urllib.error
 import urllib.parse
 import urllib.request
 from datetime import datetime
@@ -21,6 +23,7 @@ GROUP_SECONDS = int(os.environ["GROUP_SECONDS"])
 MAX_DELETES = int(os.environ["MAX_DELETES"])
 EXCLUDE = re.compile(os.environ["EXCLUDE_PACKAGES"])
 DRY_RUN = os.environ.get("DRY_RUN", "false") == "true"
+WAIT_SECONDS = int(os.environ.get("WAIT_SECONDS", "1200"))

 # A package is a CI build package if at least one of its tags looks like this.
 BUILD_TAG = re.compile(r"^(build\.[0-9]+|main-[0-9a-f]{7,40})$")
@@ -42,6 +45,22 @@ def gitea(method, path):
        return json.loads(body) if body else None


+def wait_for_gitea():
+    """Gitea may be restarting at 22:30 (upgrade, crashloop; REN-1367). With
+    backoffLimit 0 a single refused connection fails the run, so poll
+    /api/healthz for up to WAIT_SECONDS (inside activeDeadlineSeconds) first."""
+    deadline = time.monotonic() + WAIT_SECONDS
+    while True:
+        try:
+            with urllib.request.urlopen(GITEA + "/api/healthz", timeout=10):
+                return
+        except (urllib.error.URLError, OSError) as e:
+            if time.monotonic() >= deadline:
+                raise
+            log(f"gitea not ready ({e}); retrying in 30s")
+            time.sleep(30)
+
+
 def ts(s):
    # Gitea returns RFC 3339, e.g. 2026-10-07T18:07:32Z or +02:00.
    return datetime.fromisoformat(s.replace("Z", "+00:00")).timestamp()
@@ -116,6 +135,7 @@ def plan(versions, protected):


 def main():
+    wait_for_gitea()
    in_use, unsafe = running_images()
    log(f"running tags from {REGISTRY_HOST}: {sum(len(t) for t in in_use.values())}")
    work = []