paperclipai
diff --git a/‎doc/DEVELOPING.md‎
Lines changed: 14 additions & 0 deletions b/‎doc/DEVELOPING.md‎
Lines changed: 14 additions & 0 deletions
diff --git a/‎docker/daytona-runner/Dockerfile‎
Lines changed: 1 addition & 1 deletion b/‎docker/daytona-runner/Dockerfile‎
Lines changed: 1 addition & 1 deletion
diff --git a/‎server/src/__tests__/chat-channels.integration.test.ts‎
Lines changed: 12 additions & 1 deletion b/‎server/src/__tests__/chat-channels.integration.test.ts‎
Lines changed: 12 additions & 1 deletion
diff --git a/‎server/src/__tests__/heartbeat-native-cleanup-admission.test.ts‎
Lines changed: 70 additions & 0 deletions b/‎server/src/__tests__/heartbeat-native-cleanup-admission.test.ts‎
Lines changed: 70 additions & 0 deletions
diff --git a/‎server/src/__tests__/native-workspace-sync.test.ts‎
Lines changed: 15 additions & 0 deletions b/‎server/src/__tests__/native-workspace-sync.test.ts‎
Lines changed: 15 additions & 0 deletions
diff --git a/‎server/src/services/heartbeat.ts‎
Lines changed: 23 additions & 2 deletions b/‎server/src/services/heartbeat.ts‎
Lines changed: 23 additions & 2 deletions
diff --git a/‎server/src/services/native-runtime/native-restart-recovery.test.ts‎
Lines changed: 59 additions & 3 deletions b/‎server/src/services/native-runtime/native-restart-recovery.test.ts‎
Lines changed: 59 additions & 3 deletions
@@ -1087,6 +1087,20 @@ that classification finishes.
 - A verified live runner re-registers its existing PRP authority and reconnects
   with the same operating-system PID. Paperclip does not spawn a competing
   runner.
+- For a running sandbox session, recovery checks the original provider lease,
+  remote workspace, durable runner identity, and process marker inside that
+  sandbox. The process marker must include the Linux boot ID and start ticks;
+  recovery compares them with the live process before adoption and signaling.
+  Collection uses the required Node runtime and is optional for fresh launches. Older
+  markers or images without that proof remain blocked for recovery. Remote
+  PIDs are never interpreted as controller-local PIDs. The runner must
+  authenticate to its existing PRP authority; reconnection neither
+  launches another provider nor consumes a provider retry. A replacement
+  sandbox or mismatched identity blocks adoption without overwriting evidence.
+- Shutdown waits up to 30 seconds for an in-progress native startup to reach
+  its detach acknowledgement. It reports a startup deadline failure instead
+  of claiming that an unfinished bootstrap detached safely. A queued turn
+  waits for the previous executor to finish releasing its task resources.
 - A verified dead runner starts a replacement from the same durable root and
   resumes the same provider checkpoint. Only the operating-system PID changes.
 - A runner that died before its first authenticated connection can restart on
 
@@ -28,7 +28,7 @@ COPY cli/package.json ./cli/package.json
 # The complete resolved lock (including transitive integrity hashes) is reviewed.
 # Reject registry-time drift BEFORE installing packages or running lifecycle code.
 # Refresh this digest together with source/provider dependency changes.
-ARG PAPERCLIP_RUNNER_LOCK_SHA256=133cc415964e4ee3f348251c315d560cc8f8f51a8dddde2da84e065ba4ae3fa6
+ARG PAPERCLIP_RUNNER_LOCK_SHA256=d7d96cf0d98cf0946f6195e29ba173b03711a947a1c38f312b67cda56c254c22
 RUN pnpm install --resolution-only --ignore-scripts --no-frozen-lockfile \
     && printf '%s  pnpm-lock.yaml\n' "${PAPERCLIP_RUNNER_LOCK_SHA256}" > /tmp/provider-lock.sha256 \
     && sha256sum -c /tmp/provider-lock.sha256 \
 
@@ -14335,7 +14335,18 @@ describeEmbeddedPostgres("chat channel control-plane integration", () => {
       .set({ nextAttemptAt: null })
       .where(eq(chatDeliveries.endpointId, endpoint.id));
 
-    const processing = service.processPendingDeliveries();
+    // This suite shares its database. Drain only this fixture's conversation;
+    // a global recovery pass can renew unrelated retained deliveries with this
+    // test's failure hook and make its ownership assertion depend on test order.
+    const [firstDelivery] = await db
+      .select({ id: chatDeliveries.id })
+      .from(chatDeliveries)
+      .where(and(
+        eq(chatDeliveries.endpointId, endpoint.id),
+        eq(chatDeliveries.providerEventId, `${thread.thread.id}:${first.id}`),
+      ));
+    if (!firstDelivery) throw new Error("Expected first delivery");
+    const processing = service.processPendingDeliveries(25, firstDelivery.id);
     await firstDeliveryEntered;
     await renewalAttempted;
     releaseFirstDelivery();
 
@@ -0,0 +1,70 @@
+import { randomUUID } from "node:crypto";
+import { eq } from "drizzle-orm";
+import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
+import { agents, companies, createDb, heartbeatRuns, issues } from "@paperclipai/db";
+import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
+import { heartbeatService } from "../services/heartbeat.js";
+
+const execute = vi.hoisted(() => vi.fn());
+vi.mock("../adapters/index.js", async () => {
+  const actual = await vi.importActual<typeof import("../adapters/index.js")>("../adapters/index.js");
+  return { ...actual, getServerAdapter: vi.fn(() => ({ supportsLocalAgentJwt: false, execute })) };
+});
+const support = await getEmbeddedPostgresTestSupport();
+const suite = support.supported ? describe : describe.skip;
+
+suite("native terminal-run cleanup admission", () => {
+  let db: ReturnType<typeof createDb>;
+  let temporary: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
+  beforeAll(async () => {
+    temporary = await startEmbeddedPostgresTestDatabase("native-cleanup-admission-");
+    db = createDb(temporary.connectionString);
+  }, 60_000);
+  afterAll(async () => {
+    await db?.$client.end({ timeout: 0 });
+    await temporary?.cleanup();
+  });
+
+  it("holds the same task until its native executor settles while allowing unrelated tasks", async () => {
+    const companyId = randomUUID(), agentId = randomUUID(), issueId = randomUUID(), otherIssueId = randomUUID();
+    await db.insert(companies).values({ id: companyId, name: "Cleanup admission", issuePrefix: "CLN", defaultResponsibleUserId: "owner" });
+    await db.insert(agents).values({ id: agentId, companyId, name: "Worker", role: "engineer", status: "idle", adapterType: "process",
+      adapterConfig: {}, permissions: {}, runtimeConfig: { heartbeat: { enabled: false, wakeOnDemand: true, maxConcurrentRuns: 20 } } });
+    await db.insert(issues).values([issueId, otherIssueId].map(id => ({ id, companyId, title: "Recovery task", status: "todo", assigneeAgentId: agentId, responsibleUserId: "owner" })));
+    let release!: () => void, entered!: () => void;
+    const cleanupGate = new Promise<void>(resolve => { release = resolve; });
+    const terminal = new Promise<void>(resolve => { entered = resolve; });
+    let first = true;
+    execute.mockImplementation(async (context: { runId: string; context: { issueId: string } }) => {
+      if (first) {
+        first = false;
+        // Native finalization commits success before workspace/lease cleanup
+        // finishes. Keep the real heartbeat executor alive across that boundary.
+        await db.update(heartbeatRuns).set({ status: "succeeded", runtimeMode: "native", finishedAt: new Date() }).where(eq(heartbeatRuns.id, context.runId));
+        entered();
+        await cleanupGate;
+      } else {
+        await db.update(issues).set({ status: "done" }).where(eq(issues.id, context.context.issueId));
+      }
+      return { exitCode: 0, signal: null, timedOut: false, summary: "Completed" };
+    });
+    const heartbeat = heartbeatService(db);
+    const nextId = randomUUID(), otherId = randomUUID();
+    try {
+      await heartbeat.invoke(agentId, "on_demand", { issueId }, "manual");
+      await terminal;
+      await db.insert(heartbeatRuns).values([
+        { id: nextId, companyId, agentId, status: "queued", invocationSource: "on_demand", responsibleUserId: "owner", contextSnapshot: { issueId } },
+        { id: otherId, companyId, agentId, status: "queued", invocationSource: "on_demand", responsibleUserId: "owner", contextSnapshot: { issueId: otherIssueId } },
+      ]);
+      await heartbeat.resumeQueuedRuns();
+      expect((await heartbeat.getRun(nextId))?.status).toBe("queued");
+      expect((await heartbeat.getRun(otherId))?.status).not.toBe("queued");
+    } finally {
+      release();
+      await heartbeat.drainActiveRunExecutions();
+    }
+    expect((await heartbeat.getRun(nextId))?.status).toBe("succeeded");
+    expect(execute).toHaveBeenCalledTimes(3);
+  }, 30_000);
+});
@@ -12,6 +12,7 @@ import {
   nativeWorkspaceSyncInternals,
   readNativeWorkspaceSyncReference,
   resumeNativeWorkspaceSync,
+  prepareNativeWorkspaceSync,
 } from "../services/native-runtime/native-workspace-sync.js";
 
 const digest = "a".repeat(64);
@@ -94,6 +95,20 @@ describe("native workspace sync durable metadata", () => {
     ).toThrow("native_workspace_sync_unexpected_existing_descriptor");
   });
 
+  it("refuses a different sandbox before preparing restart workspace sync", async () => {
+    const select = vi.fn();
+    await expect(prepareNativeWorkspaceSync({
+      db: { select } as never, runId: "run", companyId: "company", workspaceId: "workspace",
+      workspaceLocalDir: "/tmp/local-workspace", lease: { id: "lease", providerLeaseId: "replacement" } as never,
+      target: { kind: "remote", transport: "sandbox", remoteCwd: "/workspace", providerKey: "daytona",
+        sandboxLeaseAcquisition: { outcome: "replacement", providerLeaseId: "replacement" } } as never,
+      restartRecovery: { kind: "reattach_remote_runner", runId: "run", leaseOwner: "controller", controllerGeneration: 2,
+        providerAttempt: 1, restartKind: "graceful", recoveryRequestId: null,
+        remote: { providerLeaseId: "original", remoteCwd: "/workspace" } },
+    })).rejects.toThrow("native_remote_recovery_lease_mismatch");
+    expect(select).not.toHaveBeenCalled();
+  });
+
   it("reads backward-compatible references and the resource disposition", () => {
     const base = {
       schema: "paperclip.native-workspace-sync/v1",
 
@@ -14839,6 +14839,7 @@ export function heartbeatService(
     const claims = dispositions.filter(
       (disposition): disposition is NativeRestartRecoveryClaim =>
         disposition.kind === "reattach_existing_runner" ||
+        disposition.kind === "reattach_remote_runner" ||
         disposition.kind === "resume_dead_runner" ||
         disposition.kind === "bootstrap_incomplete",
     );
@@ -14847,14 +14848,16 @@ export function heartbeatService(
       if (run) {
         const isClaim =
           disposition.kind === "reattach_existing_runner" ||
+          disposition.kind === "reattach_remote_runner" ||
           disposition.kind === "resume_dead_runner" ||
           disposition.kind === "bootstrap_incomplete";
         await appendRunEvent(run, {
           eventType: "native.recovery.transition",
           stream: "system",
           level: disposition.kind === "blocked" ? "warn" : "info",
           message:
-            disposition.kind === "reattach_existing_runner"
+            disposition.kind === "reattach_existing_runner" ||
+            disposition.kind === "reattach_remote_runner"
               ? "Recovering the existing native runner process after server restart"
               : disposition.kind === "resume_dead_runner"
                 ? "Resuming the durable native provider session after runner process loss"
@@ -17129,6 +17132,23 @@ export function heartbeatService(
     }
 
     const issueId = readNonEmptyString(context.issueId);
+    if (issueId && activeRunExecutions.size > 0) {
+      // Native finalization publishes success before workspace synchronization,
+      // provider suspension, and lease release finish. A queued comment must
+      // not acquire a fresh sandbox while its predecessor still owns that work.
+      // The executor's finally block retries this agent after removing its owner.
+      const [settlingOwner] = await db.select({ id: heartbeatRuns.id })
+        .from(heartbeatRuns)
+        .where(and(
+          eq(heartbeatRuns.companyId, run.companyId),
+          eq(heartbeatRuns.agentId, run.agentId),
+          eq(heartbeatRuns.runtimeMode, "native"),
+          inArray(heartbeatRuns.id, [...activeRunExecutions]),
+          sql`${heartbeatRuns.contextSnapshot}->>'issueId' = ${issueId}`,
+        ))
+        .limit(1);
+      if (settlingOwner) return null;
+    }
     if (issueId) {
       const activePauseHold = await treeControlSvc.getActivePauseHoldGate(
         run.companyId,
@@ -19903,7 +19923,8 @@ export function heartbeatService(
     if (
       runOptions.nativeLeaseOwner &&
       run.runtimeMode === "native" &&
-      runOptions.nativeRestartRecovery?.kind !== "reattach_existing_runner"
+      runOptions.nativeRestartRecovery?.kind !== "reattach_existing_runner" &&
+      runOptions.nativeRestartRecovery?.kind !== "reattach_remote_runner"
     ) {
       // A numeric PID or process-group ID is a liveness signal, never an
       // ownership capability: the OS may have recycled it after the service
 
@@ -9,16 +9,72 @@ import {
 
 describe("native restart recovery classification", () => {
   it("does not reopen a failed checkpoint or reset an exhausted provider budget", () => {
-    const evidence = { runnerPidAlive: false, runnerGroupAlive: false, processStartMatches: false, hasCheckpoint: true, hasProviderEvidence: true };
-    expect(classifyNativeRunnerRecoveryEvidence({ ...evidence, checkpointFailed: true })).toMatchObject({ claimKind: null, reason: "provider_checkpoint_permanently_failed" });
-    expect(classifyNativeRunnerRecoveryEvidence({ ...evidence, providerAttempt: 3 })).toMatchObject({ claimKind: null, reason: "execution_recovery_budget_exhausted" });
+    const evidence = {
+      runnerPidAlive: false,
+      runnerGroupAlive: false,
+      processStartMatches: false,
+      hasCheckpoint: true,
+      hasProviderEvidence: true,
+    };
+    expect(
+      classifyNativeRunnerRecoveryEvidence({
+        ...evidence,
+        checkpointFailed: true,
+      }),
+    ).toMatchObject({
+      claimKind: null,
+      reason: "provider_checkpoint_permanently_failed",
+    });
+    expect(
+      classifyNativeRunnerRecoveryEvidence({ ...evidence, providerAttempt: 3 }),
+    ).toMatchObject({
+      claimKind: null,
+      reason: "execution_recovery_budget_exhausted",
+    });
   });
   it("keeps controller-only recovery out of the provider retry budget", () => {
     expect(nextNativeProviderAttempt(2, "reattach_existing_runner")).toBe(2);
+    expect(nextNativeProviderAttempt(2, "reattach_remote_runner")).toBe(2);
     expect(nextNativeProviderAttempt(2, "bootstrap_incomplete")).toBe(3);
     expect(nextNativeProviderAttempt(2, "resume_dead_runner")).toBe(3);
   });
 
+  it("verifies remote ownership in the sandbox instead of interpreting its PIDs locally", () => {
+    const evidence = {
+      remoteSandbox: true,
+      runnerPidAlive: true,
+      runnerGroupAlive: true,
+      processStartMatches: false,
+      knownProviderProcessAlive: true,
+      knownProviderProcessIdentityAmbiguous: true,
+      hasCheckpoint: true,
+      checkpointIdentityMatches: true,
+      hasProviderEvidence: true,
+      providerAttempt: 3,
+    };
+    expect(classifyNativeRunnerRecoveryEvidence(evidence).claimKind).toBe(
+      "reattach_remote_runner",
+    );
+    expect(
+      classifyNativeRunnerRecoveryEvidence({
+        ...evidence,
+        checkpointIdentityMatches: false,
+      }).claimKind,
+    ).toBeNull();
+    expect(
+      classifyNativeRunnerRecoveryEvidence({
+        ...evidence,
+        checkpointFailed: true,
+      }).claimKind,
+    ).toBeNull();
+    expect(
+      classifyNativeRunnerRecoveryEvidence({
+        ...evidence,
+        hasCheckpoint: false,
+      }).claimKind,
+    ).toBeNull();
+  });
+
   it.each([
     {
       name: "reattaches an exact live runner",