2026-10-06 14:28:17 +00:00
2 changed files with 77 additions and 0 deletions
@@ -16,3 +16,4 @@ resources:
  - ingress.yaml
  - api-door.yaml
  - network-policy.yaml
+  - lease-receipt-repair-job.yaml
@@ -0,0 +1,76 @@
+# One-shot repair (REN-841). The kubernetes sandbox plugin returns no
+# termination receipt from release/destroy, so leases of the cancelled runs
+# below were released without metadata.remoteExecutionTermination, and the
+# queued owner comments wait on `remote_cleanup` forever
+# (server/src/services/explicit-native-continuation.ts:228). The
+# agent-sandbox controller log shows both Sandbox CRs deleted (13:13:26Z and
+# 11:16:35Z on 2026-10-06), and paperclip-renner has no pods. This writes the
+# receipt the plugin should have returned; the server's periodic
+# resumeExecutionWaitComments then starts the saved comment.
+# Remove this file in a follow-up PR after the Job has run.
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: pclp-lease-receipt-repair-ren841
+  namespace: paperclip
+spec:
+  backoffLimit: 0
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: pclp-lease-receipt-repair
+    spec:
+      restartPolicy: Never
+      automountServiceAccountToken: false
+      securityContext:
+        runAsUser: 26
+        runAsGroup: 26
+        runAsNonRoot: true
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: repair
+          # Same image the pclp-pg instances run (psql 17).
+          image: ghcr.io/cloudnative-pg/postgresql:17@sha256:70664ebcfa1100361b5bdc28bbf06fdbe08db2dc4ad7bd14de33c5e05fe8ea8e
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: ["ALL"]
+          env:
+            - { name: PGHOST, value: pclp-pg-rw.paperclip.svc.cluster.local }
+            - { name: PGDATABASE, value: paperclip }
+            - { name: PGSSLMODE, value: require }
+            - name: PGUSER
+              valueFrom:
+                secretKeyRef: { name: pclp-pg-app, key: username }
+            - name: PGPASSWORD
+              valueFrom:
+                secretKeyRef: { name: pclp-pg-app, key: password }
+          command:
+            - psql
+            - -X
+            - -v
+            - ON_ERROR_STOP=1
+            - -c
+            - >-
+              SELECT id, heartbeat_run_id, provider, provider_lease_id, lease_policy, status,
+              cleanup_status, released_at, metadata ? 'remoteExecutionTermination' AS has_receipt
+              FROM environment_leases
+              WHERE heartbeat_run_id IN ('e08976cd-b520-4c42-8248-35b4b841a98d', '144ae3bf-f67b-4421-8a4c-e16b2bc19973')
+            - -c
+            - >-
+              UPDATE environment_leases SET updated_at = now(),
+              metadata = coalesce(metadata, '{}'::jsonb) || jsonb_build_object('remoteExecutionTermination', jsonb_build_object(
+              'schema', 'paperclip.remote-termination.v1', 'companyId', company_id, 'runId', heartbeat_run_id,
+              'leaseId', id, 'provider', provider, 'providerLeaseId', provider_lease_id, 'state', 'destroyed',
+              'confirmedAt', to_char(now() AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"'),
+              'repair', 'REN-841: agent-sandbox controller log shows the Sandbox CR deleted'))
+              WHERE heartbeat_run_id IN ('e08976cd-b520-4c42-8248-35b4b841a98d', '144ae3bf-f67b-4421-8a4c-e16b2bc19973')
+              AND provider IS NOT NULL AND provider <> 'local' AND provider_lease_id IS NOT NULL
+              AND lease_policy = 'ephemeral' AND released_at IS NOT NULL AND cleanup_status = 'success'
+              AND status IN ('released', 'expired', 'failed')
+              AND NOT (coalesce(metadata, '{}'::jsonb) ? 'remoteExecutionTermination')
+              RETURNING id, heartbeat_run_id, provider_lease_id
+          resources:
+            requests: { cpu: 10m, memory: 32Mi }
+            limits: { memory: 64Mi }