diff --git a/.woodpecker/build-paperclip.yml b/.woodpecker/build-paperclip.yml new file mode 100644 index 00000000..ca25fdf6 --- /dev/null +++ b/.woodpecker/build-paperclip.yml @@ -0,0 +1,65 @@ +# Paperclip server image (REN-401): upstream paperclipai/paperclip at a pinned +# master commit plus the patches in ci/images/paperclip/. Needed while the k8s +# sandbox fix (upstream PR #13733) is not merged; see the README there for how +# to go back to upstream nightly. +# +# Own workflow for the reason given in build-agent-runtime.yml: a step in +# build-ci-images.yml runs on every change to that file and competes for the +# woodpecker-ci-quota. Re-run alone with a deployment to target +# `build-paperclip`. +# +# Tag `build.` is what the Flux ImagePolicy `paperclip` sorts on +# (image-automation/paperclip.yaml); image automation then rolls it out. +# konairius/ and ci_images_token as for the agent-runtime images. +when: + - event: push + branch: master + path: + - "ci/images/paperclip/**" + - ".woodpecker/build-paperclip.yml" + - event: deployment + evaluate: 'CI_PIPELINE_DEPLOY_TARGET == "build-paperclip"' + +steps: + - name: prepare-source + image: gitea.renner.uno/gitea_admin/ci-k8s:2026-09-08 + commands: + - sh ci/images/paperclip/prepare.sh build/paperclip-src + + - name: build-paperclip + image: registry.gitlab.com/gitlab-ci-utils/container-images/kaniko:v1.25.18-debug + directory: build/paperclip-src + depends_on: [prepare-source] + # Rust runner + UI + server build in one pod. The server build alone sets + # a 4 GiB Node heap, so the memory limit is the LimitRange max (8Gi). + backend_options: + kubernetes: + resources: + requests: + cpu: "1" + memory: 2Gi + limits: + cpu: "4" + memory: 8Gi + environment: + GITEA_USER: gitea_admin + GITEA_TOKEN: + from_secret: ci_images_token + commands: + - . ../../ci/images/paperclip/upstream.env + - mkdir -p /kaniko/.docker + - AUTH=$(printf '%s:%s' "$GITEA_USER" "$GITEA_TOKEN" | base64 | tr -d '\n') + - printf '{"auths":{"gitea.renner.uno":{"auth":"%s"}}}' "$AUTH" > /kaniko/.docker/config.json + - TAG=build.$(date +%s) + # Same target and build args as upstream's docker.yml for nightly. + - | + /kaniko/executor \ + --context . \ + --dockerfile Dockerfile.kaniko \ + --target production \ + --skip-unused-stages \ + --snapshot-mode=redo \ + --build-arg PAPERCLIP_BUILD_COMMIT=$PAPERCLIP_COMMIT \ + --build-arg PAPERCLIP_BUILD_VERSION=$PAPERCLIP_VERSION \ + --build-arg CLI_TOOLS_CACHE_EPOCH=$(date -u +%G-W%V) \ + --destination gitea.renner.uno/konairius/paperclip:$TAG diff --git a/ci/images/paperclip/README.md b/ci/images/paperclip/README.md new file mode 100644 index 00000000..57db480f --- /dev/null +++ b/ci/images/paperclip/README.md @@ -0,0 +1,43 @@ +# Paperclip server image (REN-401) + +Upstream `paperclipai/paperclip` at the commit in `upstream.env`, plus every +`*.patch` in this directory. Built by `.woodpecker/build-paperclip.yml`, +pushed as `gitea.renner.uno/konairius/paperclip:build.`, rolled out +by Flux image automation (`kubernetes/infrastructure/image-automation/paperclip.yaml`). + +## Why + +k8s sandbox runs fail: the run path never sets the lease `remoteCwd`, so file +sync into the pod fails (paperclipai/paperclip#13864). The fix is +paperclipai/paperclip#13733, open and not merged. +`pr-13733-lease-remote-cwd.patch` is that PR's diff (head `5f320bf6`). + +## Files + +- `upstream.env` — pinned commit and the version string stamped into the + image. Bump both together. +- `prepare.sh` — fetches the commit, applies the patches, writes + `Dockerfile.kaniko` (upstream's `COPY --parents` line split into plain + `COPY` lines; kaniko does not support `--parents`). +- `*.patch` — applied in name order with `git apply`. + +## Moving to a newer upstream commit + +Set `PAPERCLIP_COMMIT` to the new master commit and `PAPERCLIP_VERSION` to +`git describe --tags --match 'v*' --long `, then run +`sh prepare.sh /tmp/src`: it fails if a patch no longer applies. + +## Exit: when #13733 is merged upstream + +Renovate cannot watch an upstream PR, so this is a manual step. Once +upstream nightly contains the fix: + +1. `kubernetes/infrastructure/image-automation/paperclip.yaml`: back to + `ghcr.io/paperclipai/paperclip` with `filterTags.pattern: '^nightly$'` + (the version before REN-401 in git history). +2. `renovate.json`: swap `gitea.renner.uno/konairius/paperclip` back to + `ghcr.io/paperclipai/paperclip` in the image-automation exclusion list. +3. Delete this directory and `.woodpecker/build-paperclip.yml`. + +Image automation then rewrites `kubernetes/apps/paperclip/paperclip.yaml` +to the nightly digest by itself. diff --git a/ci/images/paperclip/pr-13733-lease-remote-cwd.patch b/ci/images/paperclip/pr-13733-lease-remote-cwd.patch new file mode 100644 index 00000000..5adffe99 --- /dev/null +++ b/ci/images/paperclip/pr-13733-lease-remote-cwd.patch @@ -0,0 +1,93 @@ +diff --git a/server/src/__tests__/environment-run-orchestrator.test.ts b/server/src/__tests__/environment-run-orchestrator.test.ts +index f3d1fc659..812cae357 100644 +--- a/server/src/__tests__/environment-run-orchestrator.test.ts ++++ b/server/src/__tests__/environment-run-orchestrator.test.ts +@@ -430,6 +430,51 @@ describe("environmentRunOrchestrator — realizeForRun", () => { + expect(result.persistedExecutionWorkspace).toEqual(updatedEw); + }); + ++ it("persists the realized cwd as remoteCwd when realization returns no workspaceRealization payload", async () => { ++ const runtime = makeMockRuntime({ ++ realizeWorkspace: vi.fn().mockResolvedValue({ ++ cwd: "/workspace", ++ metadata: { provider: "kubernetes", remoteCwd: "/workspace" }, ++ }), ++ } as Partial); ++ mockResolveEnvironmentExecutionTarget.mockResolvedValue({ kind: "local", environmentId: "env-1", leaseId: "lease-1" }); ++ ++ const orchestrator = environmentRunOrchestrator(mockDb, { environmentRuntime: runtime }); ++ ++ await orchestrator.realizeForRun( ++ makeRealizeInput({ environment: makeEnvironment("sandbox") }), ++ ); ++ ++ expect(mockUpdateLeaseMetadata).toHaveBeenCalledWith( ++ "lease-1", ++ expect.objectContaining({ remoteCwd: "/workspace" }), ++ ); ++ // Nothing to record about the realization shape, so the execution workspace ++ // row is left untouched. ++ expect(mockUpdateExecutionWorkspace).not.toHaveBeenCalled(); ++ }); ++ ++ it("does not rewrite lease metadata when the realized cwd already matches the lease", async () => { ++ const runtime = makeMockRuntime({ ++ realizeWorkspace: vi.fn().mockResolvedValue({ ++ cwd: "/workspace", ++ metadata: { provider: "kubernetes", remoteCwd: "/workspace" }, ++ }), ++ } as Partial); ++ mockResolveEnvironmentExecutionTarget.mockResolvedValue({ kind: "local", environmentId: "env-1", leaseId: "lease-1" }); ++ ++ const orchestrator = environmentRunOrchestrator(mockDb, { environmentRuntime: runtime }); ++ ++ await orchestrator.realizeForRun( ++ makeRealizeInput({ ++ environment: makeEnvironment("sandbox"), ++ lease: makeLease({ metadata: { remoteCwd: "/workspace" } }), ++ }), ++ ); ++ ++ expect(mockUpdateLeaseMetadata).not.toHaveBeenCalled(); ++ }); ++ + it("runs a remote provision command after workspace realization when configured", async () => { + mockBuildWorkspaceRealizationRequest.mockReturnValue({ + version: 1, +diff --git a/server/src/services/environment-run-orchestrator.ts b/server/src/services/environment-run-orchestrator.ts +index 97cca667e..e76f64913 100644 +--- a/server/src/services/environment-run-orchestrator.ts ++++ b/server/src/services/environment-run-orchestrator.ts +@@ -485,16 +485,29 @@ export function environmentRunOrchestrator( + } + + // Step 3: Persist realization metadata on lease and execution workspace +- if (Object.keys(workspaceRealization).length > 0) { ++ const hasWorkspaceRealization = Object.keys(workspaceRealization).length > 0; ++ // A driver that realizes the workspace reports the directory it prepared ++ // through the realization `cwd`. Persist it on the lease as `remoteCwd`, so ++ // every later step that receives only the lease — native file sync, a ++ // resumed lease, a second run against the same lease — confines itself to ++ // the same root. Without it a driver whose realization returns no ++ // `workspaceRealization` payload leaves the lease without a root at all, ++ // and the sync step fails even though realization succeeded. ++ const currentRemoteCwd = ++ typeof lease.metadata?.remoteCwd === "string" ? lease.metadata.remoteCwd.trim() : ""; ++ const realizedRemoteCwd = ++ realizedWorkspaceCwd && realizedWorkspaceCwd !== currentRemoteCwd ? realizedWorkspaceCwd : null; ++ if (hasWorkspaceRealization || realizedRemoteCwd) { + const nextLeaseMetadata = { + ...(lease.metadata ?? {}), +- workspaceRealization, ++ ...(hasWorkspaceRealization ? { workspaceRealization } : {}), ++ ...(realizedRemoteCwd ? { remoteCwd: realizedRemoteCwd } : {}), + }; + const updatedLease = await environmentsSvc.updateLeaseMetadata(lease.id, nextLeaseMetadata); + if (updatedLease) { + lease = updatedLease; + } +- if (persistedExecutionWorkspace) { ++ if (hasWorkspaceRealization && persistedExecutionWorkspace) { + const updatedEw = await executionWorkspacesSvc.update(persistedExecutionWorkspace.id, { + metadata: { + ...(persistedExecutionWorkspace.metadata ?? {}), diff --git a/ci/images/paperclip/prepare.sh b/ci/images/paperclip/prepare.sh new file mode 100755 index 00000000..0439289a --- /dev/null +++ b/ci/images/paperclip/prepare.sh @@ -0,0 +1,39 @@ +#!/bin/sh +# Fetch upstream Paperclip at the pinned commit, apply our patches and make +# the Dockerfile buildable with kaniko. Usage: prepare.sh +set -eu +HERE=$(cd "$(dirname "$0")" && pwd) +. "$HERE/upstream.env" +OUT=$1 + +rm -rf "$OUT" +git init -q "$OUT" +cd "$OUT" +git fetch -q --depth 1 https://github.com/paperclipai/paperclip.git "$PAPERCLIP_COMMIT" +git -c advice.detachedHead=false checkout -q FETCH_HEAD +test "$(git rev-parse HEAD)" = "$PAPERCLIP_COMMIT" + +for p in "$HERE"/*.patch; do + echo "applying $(basename "$p")" + git apply "$p" +done + +# kaniko does not know `COPY --parents` (BuildKit only). Upstream uses it for +# one line: every sandbox provider's package.json. Replace it with one plain +# COPY per provider directory, which copies the same files to the same place. +grep -c -- '^COPY --parents packages/plugins/sandbox-providers/\./\*/package.json packages/plugins/sandbox-providers/$' Dockerfile | grep -qx 1 +awk ' + /^COPY --parents packages\/plugins\/sandbox-providers\// { + while (("ls -d packages/plugins/sandbox-providers/*/package.json" | getline f) > 0) { + d = f; sub(/package.json$/, "", d) + print "COPY " f " " d + } + next + } + { print } +' Dockerfile > Dockerfile.kaniko +if grep -q -- '--parents' Dockerfile.kaniko; then + echo "Dockerfile.kaniko still uses COPY --parents" >&2 + exit 1 +fi +rm -rf .git diff --git a/ci/images/paperclip/upstream.env b/ci/images/paperclip/upstream.env new file mode 100644 index 00000000..a4c3b249 --- /dev/null +++ b/ci/images/paperclip/upstream.env @@ -0,0 +1,7 @@ +# Upstream paperclipai/paperclip commit this image is built from (REN-401). +# a027f76 is the master commit the nightly image ran when this was pinned, so +# the only change against that image is the patch next to this file. +# PAPERCLIP_VERSION is `git describe --tags --match 'v*' --long` of that +# commit, the value upstream stamps into its own images. Bump both together. +PAPERCLIP_COMMIT=a027f76a726e8a556674eded807c2de0e55f5cc0 +PAPERCLIP_VERSION=v2026.916.0-219-ga027f76a7 diff --git a/kubernetes/infrastructure/image-automation/paperclip.yaml b/kubernetes/infrastructure/image-automation/paperclip.yaml index 9f776b25..f389866e 100644 --- a/kubernetes/infrastructure/image-automation/paperclip.yaml +++ b/kubernetes/infrastructure/image-automation/paperclip.yaml @@ -1,13 +1,18 @@ +# Our own build (REN-401): upstream master at a pinned commit plus the k8s +# sandbox fix, built by .woodpecker/build-paperclip.yml. Back to +# ghcr.io/paperclipai/paperclip:nightly once upstream PR #13733 is merged; +# see ci/images/paperclip/README.md. apiVersion: image.toolkit.fluxcd.io/v1 kind: ImageRepository metadata: name: paperclip namespace: flux-system spec: - image: ghcr.io/paperclipai/paperclip - interval: 1h + image: gitea.renner.uno/konairius/paperclip + interval: 10m --- -# Upstream publishes no semver tags; track the nightly tag by digest. +# The pipeline tags every build `build.`; latest build wins. Pinned by +# digest in the manifest like before. apiVersion: image.toolkit.fluxcd.io/v1 kind: ImagePolicy metadata: @@ -17,9 +22,10 @@ spec: imageRepositoryRef: name: paperclip digestReflectionPolicy: Always - interval: 1h + interval: 10m filterTags: - pattern: '^nightly$' + pattern: '^build\.(?P\d+)$' + extract: '$ts' policy: - alphabetical: + numerical: order: asc diff --git a/renovate.json b/renovate.json index ac792da8..167949f9 100644 --- a/renovate.json +++ b/renovate.json @@ -41,12 +41,12 @@ "ghcr.io/immich-app/immich-server", "ghcr.io/matter-js/python-matter-server", "ghcr.io/music-assistant/server", - "ghcr.io/paperclipai/paperclip", "ghcr.io/paperless-ngx/paperless-ngx", "ghcr.io/usetrmnl/terminus", "ghcr.io/usetrmnl/trmnl-ha-amd64", "gitea.renner.uno/k.renner/shelly-manager", "gitea.renner.uno/k.renner/vsftpd", + "gitea.renner.uno/konairius/paperclip", "gitea.renner.uno/konairius/wiki", "gotenberg/gotenberg", "lscr.io/linuxserver/lidarr",