paperclip: build upstream a027f76 + PR #13733 and track it (REN-401)

k8s sandbox runs fail because the run path never sets the lease remoteCwd
(paperclipai/paperclip#13864). The fix, paperclipai/paperclip#13733, is not
merged upstream yet. Build our own image from the master commit the nightly
runs now plus that patch, and point Flux image automation at it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

.woodpecker/build-paperclip.yml

@@ -0,0 +1,65 @@
+# Paperclip server image (REN-401): upstream paperclipai/paperclip at a pinned
+# master commit plus the patches in ci/images/paperclip/. Needed while the k8s
+# sandbox fix (upstream PR #13733) is not merged; see the README there for how
+# to go back to upstream nightly.
+#
+# Own workflow for the reason given in build-agent-runtime.yml: a step in
+# build-ci-images.yml runs on every change to that file and competes for the
+# woodpecker-ci-quota. Re-run alone with a deployment to target
+# `build-paperclip`.
+#
+# Tag `build.<unix-ts>` is what the Flux ImagePolicy `paperclip` sorts on
+# (image-automation/paperclip.yaml); image automation then rolls it out.
+# konairius/ and ci_images_token as for the agent-runtime images.
+when:
+  - event: push
+    branch: master
+    path:
+      - "ci/images/paperclip/**"
+      - ".woodpecker/build-paperclip.yml"
+  - event: deployment
+    evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "build-paperclip"'
+
+steps:
+  - name: prepare-source
+    image: gitea.renner.uno/gitea_admin/ci-k8s:2026-09-08
+    commands:
+      - sh ci/images/paperclip/prepare.sh build/paperclip-src
+
+  - name: build-paperclip
+    image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug
+    directory: build/paperclip-src
+    depends_on: [prepare-source]
+    # Rust runner + UI + server build in one pod. The server build alone sets
+    # a 4 GiB Node heap, so the memory limit is the LimitRange max (8Gi).
+    backend_options:
+      kubernetes:
+        resources:
+          requests:
+            cpu: "1"
+            memory: 2Gi
+          limits:
+            cpu: "4"
+            memory: 8Gi
+    environment:
+      GITEA_USER: gitea_admin
+      GITEA_TOKEN:
+        from_secret: ci_images_token
+    commands:
+      - . ../../ci/images/paperclip/upstream.env
+      - mkdir -p /kaniko/.docker
+      - AUTH=$(printf '%s:%s' "$GITEA_USER" "$GITEA_TOKEN" | base64 | tr -d '\n')
+      - printf '{"auths":{"gitea.renner.uno":{"auth":"%s"}}}' "$AUTH" > /kaniko/.docker/config.json
+      - TAG=build.$(date +%s)
+      # Same target and build args as upstream's docker.yml for nightly.
+      - |
+        /kaniko/executor \
+          --context . \
+          --dockerfile Dockerfile.kaniko \
+          --target production \
+          --skip-unused-stages \
+          --snapshot-mode=redo \
+          --build-arg PAPERCLIP_BUILD_COMMIT=$PAPERCLIP_COMMIT \
+          --build-arg PAPERCLIP_BUILD_VERSION=$PAPERCLIP_VERSION \
+          --build-arg CLI_TOOLS_CACHE_EPOCH=$(date -u +%G-W%V) \
+          --destination gitea.renner.uno/konairius/paperclip:$TAG

ci/images/paperclip/README.md

@@ -0,0 +1,43 @@
+# Paperclip server image (REN-401)
+
+Upstream `paperclipai/paperclip` at the commit in `upstream.env`, plus every
+`*.patch` in this directory. Built by `.woodpecker/build-paperclip.yml`,
+pushed as `gitea.renner.uno/konairius/paperclip:build.<unix-ts>`, rolled out
+by Flux image automation (`kubernetes/infrastructure/image-automation/paperclip.yaml`).
+
+## Why
+
+k8s sandbox runs fail: the run path never sets the lease `remoteCwd`, so file
+sync into the pod fails (paperclipai/paperclip#13864). The fix is
+paperclipai/paperclip#13733, open and not merged.
+`pr-13733-lease-remote-cwd.patch` is that PR's diff (head `5f320bf6`).
+
+## Files
+
+- `upstream.env` — pinned commit and the version string stamped into the
+  image. Bump both together.
+- `prepare.sh` — fetches the commit, applies the patches, writes
+  `Dockerfile.kaniko` (upstream's `COPY --parents` line split into plain
+  `COPY` lines; kaniko does not support `--parents`).
+- `*.patch` — applied in name order with `git apply`.
+
+## Moving to a newer upstream commit
+
+Set `PAPERCLIP_COMMIT` to the new master commit and `PAPERCLIP_VERSION` to
+`git describe --tags --match 'v*' --long <commit>`, then run
+`sh prepare.sh /tmp/src`: it fails if a patch no longer applies.
+
+## Exit: when #13733 is merged upstream
+
+Renovate cannot watch an upstream PR, so this is a manual step. Once
+upstream nightly contains the fix:
+
+1. `kubernetes/infrastructure/image-automation/paperclip.yaml`: back to
+   `ghcr.io/paperclipai/paperclip` with `filterTags.pattern: '^nightly$'`
+   (the version before REN-401 in git history).
+2. `renovate.json`: swap `gitea.renner.uno/konairius/paperclip` back to
+   `ghcr.io/paperclipai/paperclip` in the image-automation exclusion list.
+3. Delete this directory and `.woodpecker/build-paperclip.yml`.
+
+Image automation then rewrites `kubernetes/apps/paperclip/paperclip.yaml`
+to the nightly digest by itself.

ci/images/paperclip/pr-13733-lease-remote-cwd.patch

@@ -0,0 +1,93 @@
+diff --git a/server/src/__tests__/environment-run-orchestrator.test.ts b/server/src/__tests__/environment-run-orchestrator.test.ts
+index f3d1fc659..812cae357 100644
+--- a/server/src/__tests__/environment-run-orchestrator.test.ts
++++ b/server/src/__tests__/environment-run-orchestrator.test.ts
+@@ -430,6 +430,51 @@ describe("environmentRunOrchestrator — realizeForRun", () => {
+     expect(result.persistedExecutionWorkspace).toEqual(updatedEw);
+   });
+ 
++  it("persists the realized cwd as remoteCwd when realization returns no workspaceRealization payload", async () => {
++    const runtime = makeMockRuntime({
++      realizeWorkspace: vi.fn().mockResolvedValue({
++        cwd: "/workspace",
++        metadata: { provider: "kubernetes", remoteCwd: "/workspace" },
++      }),
++    } as Partial<EnvironmentRuntimeService>);
++    mockResolveEnvironmentExecutionTarget.mockResolvedValue({ kind: "local", environmentId: "env-1", leaseId: "lease-1" });
++
++    const orchestrator = environmentRunOrchestrator(mockDb, { environmentRuntime: runtime });
++
++    await orchestrator.realizeForRun(
++      makeRealizeInput({ environment: makeEnvironment("sandbox") }),
++    );
++
++    expect(mockUpdateLeaseMetadata).toHaveBeenCalledWith(
++      "lease-1",
++      expect.objectContaining({ remoteCwd: "/workspace" }),
++    );
++    // Nothing to record about the realization shape, so the execution workspace
++    // row is left untouched.
++    expect(mockUpdateExecutionWorkspace).not.toHaveBeenCalled();
++  });
++
++  it("does not rewrite lease metadata when the realized cwd already matches the lease", async () => {
++    const runtime = makeMockRuntime({
++      realizeWorkspace: vi.fn().mockResolvedValue({
++        cwd: "/workspace",
++        metadata: { provider: "kubernetes", remoteCwd: "/workspace" },
++      }),
++    } as Partial<EnvironmentRuntimeService>);
++    mockResolveEnvironmentExecutionTarget.mockResolvedValue({ kind: "local", environmentId: "env-1", leaseId: "lease-1" });
++
++    const orchestrator = environmentRunOrchestrator(mockDb, { environmentRuntime: runtime });
++
++    await orchestrator.realizeForRun(
++      makeRealizeInput({
++        environment: makeEnvironment("sandbox"),
++        lease: makeLease({ metadata: { remoteCwd: "/workspace" } }),
++      }),
++    );
++
++    expect(mockUpdateLeaseMetadata).not.toHaveBeenCalled();
++  });
++
+   it("runs a remote provision command after workspace realization when configured", async () => {
+     mockBuildWorkspaceRealizationRequest.mockReturnValue({
+       version: 1,
+diff --git a/server/src/services/environment-run-orchestrator.ts b/server/src/services/environment-run-orchestrator.ts
+index 97cca667e..e76f64913 100644
+--- a/server/src/services/environment-run-orchestrator.ts
++++ b/server/src/services/environment-run-orchestrator.ts
+@@ -485,16 +485,29 @@ export function environmentRunOrchestrator(
+     }
+ 
+     // Step 3: Persist realization metadata on lease and execution workspace
+-    if (Object.keys(workspaceRealization).length > 0) {
++    const hasWorkspaceRealization = Object.keys(workspaceRealization).length > 0;
++    // A driver that realizes the workspace reports the directory it prepared
++    // through the realization `cwd`. Persist it on the lease as `remoteCwd`, so
++    // every later step that receives only the lease — native file sync, a
++    // resumed lease, a second run against the same lease — confines itself to
++    // the same root. Without it a driver whose realization returns no
++    // `workspaceRealization` payload leaves the lease without a root at all,
++    // and the sync step fails even though realization succeeded.
++    const currentRemoteCwd =
++      typeof lease.metadata?.remoteCwd === "string" ? lease.metadata.remoteCwd.trim() : "";
++    const realizedRemoteCwd =
++      realizedWorkspaceCwd && realizedWorkspaceCwd !== currentRemoteCwd ? realizedWorkspaceCwd : null;
++    if (hasWorkspaceRealization || realizedRemoteCwd) {
+       const nextLeaseMetadata = {
+         ...(lease.metadata ?? {}),
+-        workspaceRealization,
++        ...(hasWorkspaceRealization ? { workspaceRealization } : {}),
++        ...(realizedRemoteCwd ? { remoteCwd: realizedRemoteCwd } : {}),
+       };
+       const updatedLease = await environmentsSvc.updateLeaseMetadata(lease.id, nextLeaseMetadata);
+       if (updatedLease) {
+         lease = updatedLease;
+       }
+-      if (persistedExecutionWorkspace) {
++      if (hasWorkspaceRealization && persistedExecutionWorkspace) {
+         const updatedEw = await executionWorkspacesSvc.update(persistedExecutionWorkspace.id, {
+           metadata: {
+             ...(persistedExecutionWorkspace.metadata ?? {}),

ci/images/paperclip/prepare.sh

@@ -0,0 +1,39 @@
+#!/bin/sh
+# Fetch upstream Paperclip at the pinned commit, apply our patches and make
+# the Dockerfile buildable with kaniko. Usage: prepare.sh <target-dir>
+set -eu
+HERE=$(cd "$(dirname "$0")" && pwd)
+. "$HERE/upstream.env"
+OUT=$1
+
+rm -rf "$OUT"
+git init -q "$OUT"
+cd "$OUT"
+git fetch -q --depth 1 https://github.com/paperclipai/paperclip.git "$PAPERCLIP_COMMIT"
+git -c advice.detachedHead=false checkout -q FETCH_HEAD
+test "$(git rev-parse HEAD)" = "$PAPERCLIP_COMMIT"
+
+for p in "$HERE"/*.patch; do
+  echo "applying $(basename "$p")"
+  git apply "$p"
+done
+
+# kaniko does not know `COPY --parents` (BuildKit only). Upstream uses it for
+# one line: every sandbox provider's package.json. Replace it with one plain
+# COPY per provider directory, which copies the same files to the same place.
+grep -c -- '^COPY --parents packages/plugins/sandbox-providers/\./\*/package.json packages/plugins/sandbox-providers/$' Dockerfile | grep -qx 1
+awk '
+  /^COPY --parents packages\/plugins\/sandbox-providers\// {
+    while (("ls -d packages/plugins/sandbox-providers/*/package.json" | getline f) > 0) {
+      d = f; sub(/package.json$/, "", d)
+      print "COPY " f " " d
+    }
+    next
+  }
+  { print }
+' Dockerfile > Dockerfile.kaniko
+if grep -q -- '--parents' Dockerfile.kaniko; then
+  echo "Dockerfile.kaniko still uses COPY --parents" >&2
+  exit 1
+fi
+rm -rf .git

ci/images/paperclip/upstream.env

@@ -0,0 +1,7 @@
+# Upstream paperclipai/paperclip commit this image is built from (REN-401).
+# a027f76 is the master commit the nightly image ran when this was pinned, so
+# the only change against that image is the patch next to this file.
+# PAPERCLIP_VERSION is `git describe --tags --match 'v*' --long` of that
+# commit, the value upstream stamps into its own images. Bump both together.
+PAPERCLIP_COMMIT=a027f76a726e8a556674eded807c2de0e55f5cc0
+PAPERCLIP_VERSION=v2026.916.0-219-ga027f76a7

kubernetes/infrastructure/image-automation/paperclip.yaml

@@ -1,13 +1,18 @@
+# Our own build (REN-401): upstream master at a pinned commit plus the k8s
+# sandbox fix, built by .woodpecker/build-paperclip.yml. Back to
+# ghcr.io/paperclipai/paperclip:nightly once upstream PR #13733 is merged;
+# see ci/images/paperclip/README.md.
 apiVersion: image.toolkit.fluxcd.io/v1
 kind: ImageRepository
 metadata:
   name: paperclip
   namespace: flux-system
 spec:
-  image: ghcr.io/paperclipai/paperclip
-  interval: 1h
+  image: gitea.renner.uno/konairius/paperclip
+  interval: 10m
 ---
-# Upstream publishes no semver tags; track the nightly tag by digest.
+# The pipeline tags every build `build.<unix-ts>`; latest build wins. Pinned by
+# digest in the manifest like before.
 apiVersion: image.toolkit.fluxcd.io/v1
 kind: ImagePolicy
 metadata:
@@ -17,9 +22,10 @@ spec:
   imageRepositoryRef:
     name: paperclip
   digestReflectionPolicy: Always
-  interval: 1h
+  interval: 10m
   filterTags:
-    pattern: '^nightly$'
+    pattern: '^build\.(?P<ts>\d+)$'
+    extract: '$ts'
   policy:
-    alphabetical:
+    numerical:
       order: asc

renovate.json

@@ -41,12 +41,12 @@
         "ghcr.io/immich-app/immich-server",
         "ghcr.io/matter-js/python-matter-server",
         "ghcr.io/music-assistant/server",
-        "ghcr.io/paperclipai/paperclip",
         "ghcr.io/paperless-ngx/paperless-ngx",
         "ghcr.io/usetrmnl/terminus",
         "ghcr.io/usetrmnl/trmnl-ha-amd64",
         "gitea.renner.uno/k.renner/shelly-manager",
         "gitea.renner.uno/k.renner/vsftpd",
+        "gitea.renner.uno/konairius/paperclip",
         "gitea.renner.uno/konairius/wiki",
         "gotenberg/gotenberg",
         "lscr.io/linuxserver/lidarr",