fix(desktop): imports arrive whole, survive the rate limit, and record their folders

- The proxy no longer runs for the desktop's raw upload routes (import, and
  the browser file transfer that has the same shape). Running it made Next
  buffer the body and cut it off at its 10 MB proxy limit. The import route
  also requires a declared length and refuses a body that does not match
  it, so a cut-off file is never stored as complete.
- A file entry with no body is refused instead of stored as an empty file.
- A name Sim cannot store (one with a backslash) is refused for the whole
  import before anything lands, and at the contract.
- A chunk that runs past the size the manifest listed is refused, like one
  that ends early.
- A rate-limited entry is sent again after the limit clears. A 429 means
  nothing was stored, so a large tree no longer fails part way.
- Folders an import creates are audited as folder creations and announced to
  the workspace's file views, as folders made by hand are.
- Tests record what the fake Sim stored instead of asserting mock calls. The
  integration suite removes its audit rows.

Author: waleedlatif1

apps/desktop/src/main/desktop-executor/runner.test.ts

@@ -3,6 +3,7 @@ import { tmpdir } from 'node:os'
 import { join } from 'node:path'
 import type { TerminalToolResponse } from '@sim/terminal-protocol'
 import { afterEach, describe, expect, it, vi } from 'vitest'
+import { DeviceRequestError } from '@/main/desktop-executor/client'
 import type {
   ClaimedDesktopCall,
   DesktopImportEntryRequest,
@@ -140,19 +141,27 @@ describe('background imports', () => {
     }
   }
 
-  /** Records what reached Sim, with each file's bytes as text. */
-  function recordingSim(fail?: (request: DesktopImportEntryRequest) => boolean) {
+  /**
+   * A fake Sim import route: records every entry it stores, with each file's bytes as text, and
+   * the token each request presented. `answer` can refuse a request instead.
+   */
+  function recordingSim(answer?: (request: DesktopImportEntryRequest) => Error | null) {
     const stored: Array<{ kind: string; relativePath: string; text?: string }> = []
-    const importEntry = vi.fn(async (request: DesktopImportEntryRequest) => {
-      if (fail?.(request)) throw new Error('Sim refused the entry')
+    const tokens: string[] = []
+    let requests = 0
+    const importEntry = async (request: DesktopImportEntryRequest) => {
+      requests += 1
+      tokens.push(request.call.executionToken)
+      const refusal = answer?.(request)
+      if (refusal) throw refusal
       stored.push({
         kind: request.kind,
         relativePath: request.relativePath,
         ...(request.content ? { text: await request.content.text() } : {}),
       })
       return { id: `id-${stored.length}`, name: request.relativePath || request.sourceName }
-    })
-    return { stored, importEntry }
+    }
+    return { stored, tokens, importEntry, requests: () => requests }
   }
 
   it("stores a folder's tree in Sim, each file with the bytes on disk", async () => {
@@ -169,10 +178,7 @@ describe('background imports', () => {
       { kind: 'directory', relativePath: 'q3' },
       { kind: 'file', relativePath: 'q3/summary.txt', text: 'quarterly numbers' },
     ])
-    expect(sim.importEntry.mock.calls[0]?.[0]).toMatchObject({
-      sourceName: 'Reports',
-      call: { executionToken: 'token-import-1' },
-    })
+    expect(new Set(sim.tokens)).toEqual(new Set(['token-import-1']))
     expect(completion.data).toMatchObject({
       success: true,
       workspaceId: 'ws-1',
@@ -188,7 +194,9 @@ describe('background imports', () => {
   })
 
   it('reports what landed when an import stops part way, and not to retry it', async () => {
-    const sim = recordingSim((request) => request.relativePath === 'q3/summary.txt')
+    const sim = recordingSim((request) =>
+      request.relativePath === 'q3/summary.txt' ? new Error('Sim refused the entry') : null
+    )
     const completion = await runner({ imports: { importEntry: sim.importEntry } }).run(
       importCall(await reportsFolder()),
       new AbortController().signal
@@ -207,18 +215,38 @@ describe('background imports', () => {
 
   it('stores nothing more once the call is stopped', async () => {
     const controller = new AbortController()
-    const sim = recordingSim()
-    sim.importEntry.mockImplementationOnce(async (request) => {
+    const sim = recordingSim(() => {
       controller.abort()
-      return { id: 'id-root', name: request.sourceName }
+      return null
     })
     const completion = await runner({ imports: { importEntry: sim.importEntry } }).run(
       importCall(await reportsFolder()),
       controller.signal
     )
 
     expect(completion.status).toBe('error')
-    expect(sim.importEntry).toHaveBeenCalledTimes(1)
+    expect(sim.stored).toEqual([{ kind: 'directory', relativePath: '' }])
+  })
+
+  it("waits out Sim's rate limit instead of failing the import part way", async () => {
+    let limited = false
+    const sim = recordingSim((request) => {
+      if (request.relativePath !== 'notes.txt' || limited) return null
+      limited = true
+      return new DeviceRequestError(429, 'Too many requests', 10)
+    })
+    const completion = await runner({ imports: { importEntry: sim.importEntry } }).run(
+      importCall(await reportsFolder()),
+      new AbortController().signal
+    )
+
+    expect(completion.status).toBe('success')
+    expect(sim.stored.map((entry) => entry.relativePath)).toEqual([
+      '',
+      'notes.txt',
+      'q3',
+      'q3/summary.txt',
+    ])
   })
 
   it('fails without storing anything when the source cannot be read', async () => {
@@ -229,7 +257,7 @@ describe('background imports', () => {
     )
 
     expect(completion.status).toBe('error')
-    expect(sim.importEntry).not.toHaveBeenCalled()
+    expect(sim.requests()).toBe(0)
     expect(completion.data).toMatchObject({ workspaceId: 'ws-1', partial: false })
   })
 })

apps/desktop/src/main/desktop-executor/runner.ts

@@ -44,6 +44,8 @@ import {
 import { getErrorMessage } from '@sim/utils/errors'
 import { interruptibleSleep } from '@sim/utils/helpers'
 import { isRecordLike } from '@sim/utils/object'
+import { backoffWithJitter } from '@sim/utils/retry'
+import { DeviceRequestError } from '@/main/desktop-executor/client'
 import type { DesktopToolRunner } from '@/main/desktop-executor/executor'
 import type {
   ClaimedDesktopCall,
@@ -54,6 +56,9 @@ import type {
 const logger = createLogger('DesktopExecutorRunner')
 
 const USER_LOCAL_TOOLS: ReadonlySet<string> = new Set(['read', 'grep', 'glob'])
+/** A rate-limited import entry is retried this many times; Sim never stored a refused one. */
+const IMPORT_RATE_LIMIT_ATTEMPTS = 8
+const IMPORT_RATE_LIMIT_MAX_WAIT_MS = 30_000
 
 /** The model learns a call never ran because a surface is switched off on this machine. */
 function surfaceOff(surface: string): DesktopToolCompletion {
@@ -231,6 +236,34 @@ export function createDesktopToolRunner(deps: DesktopToolRunnerDeps): DesktopToo
    * Imports the call's source into its workspace one entry at a time: each directory as a folder,
    * each file read in chunks and checked against the manifest that listed it.
    */
+  /**
+   * Sends one entry, waiting out Sim's rate limit: a large tree can outrun it, and a 429 means
+   * nothing was stored, so sending the entry again cannot duplicate it.
+   */
+  async function importEntry(
+    request: DesktopImportEntryRequest,
+    signal: AbortSignal
+  ): Promise<DesktopImportedEntry> {
+    for (let attempt = 1; ; attempt++) {
+      try {
+        return await deps.imports.importEntry(request, signal)
+      } catch (error) {
+        if (
+          !(error instanceof DeviceRequestError) ||
+          error.status !== 429 ||
+          attempt >= IMPORT_RATE_LIMIT_ATTEMPTS
+        ) {
+          throw error
+        }
+        await interruptibleSleep(
+          backoffWithJitter(attempt, error.retryAfterMs, { maxMs: IMPORT_RATE_LIMIT_MAX_WAIT_MS }),
+          signal
+        )
+        signal.throwIfAborted()
+      }
+    }
+  }
+
   async function runImport(
     call: ClaimedDesktopCall,
     signal: AbortSignal
@@ -250,12 +283,12 @@ export function createDesktopToolRunner(deps: DesktopToolRunnerDeps): DesktopToo
         signal.throwIfAborted()
         const target = { call, sourceName: manifest.name, relativePath: entry.relativePath }
         if (entry.kind === 'directory') {
-          const folder = await deps.imports.importEntry({ ...target, kind: 'directory' }, signal)
+          const folder = await importEntry({ ...target, kind: 'directory' }, signal)
           folders.push({ id: folder.id, relativePath: entry.relativePath })
           continue
         }
         const parts = await readImportEntry(call.toolCallId, entry, read, signal)
-        const file = await deps.imports.importEntry(
+        const file = await importEntry(
           { ...target, kind: 'file', content: new Blob(parts) },
           signal
         )

apps/sim/app/api/desktop/tool/import/route.ts

@@ -40,10 +40,16 @@ export const PUT = withRouteHandler(async (request: NextRequest) => {
   const parsed = await parseRequest(importDesktopEntryContract, request, {})
   if (!parsed.success) return parsed.response
   const query = parsed.data.query
-  const declaredLength = Number(request.headers.get('content-length') ?? 0)
+  const lengthHeader = request.headers.get('content-length')
+  const declaredLength = Number(lengthHeader ?? 0)
   if (!Number.isFinite(declaredLength) || declaredLength > MAX_DESKTOP_IMPORT_FILE_BYTES) {
     return NextResponse.json(withRequestId({ error: TOO_LARGE }), { status: 413 })
   }
+  if (query.kind === 'file' && lengthHeader === null) {
+    return NextResponse.json(withRequestId({ error: 'A file import must declare its length' }), {
+      status: 411,
+    })
+  }
 
   try {
     await admitDesktopImportEntry(principal, query)
@@ -65,6 +71,13 @@ export const PUT = withRouteHandler(async (request: NextRequest) => {
       }
       throw error
     }
+    // Anything between the device and here that cut the body short must not become a stored file.
+    if (content.length !== declaredLength) {
+      return NextResponse.json(
+        withRequestId({ error: 'The file did not arrive whole; nothing was stored' }),
+        { status: 400 }
+      )
+    }
   }
 
   try {

apps/sim/lib/api/contracts/desktop-executor.ts

@@ -183,6 +183,9 @@ export const completeDesktopToolContract = defineRouteContract({
   error: z.object({ error: z.string() }),
 })
 
+/** Workspace file names cannot hold a backslash, which a macOS or Linux file name can. */
+const BACKSLASH_NAME = 'Sim cannot store a file or folder whose name contains a backslash'
+
 /** One relative path inside an import source, as the device's manifest lists it. */
 const desktopImportRelativePathSchema = z
   .string()
@@ -193,14 +196,21 @@ const desktopImportRelativePathSchema = z
       path.split('/').every((segment) => segment !== '' && segment !== '.' && segment !== '..'),
     'Relative path must stay inside the import source'
   )
+  .refine((path) => !path.includes('\\'), BACKSLASH_NAME)
 
 const importDesktopEntryQuerySchema = z.object({
   deviceId: desktopDeviceIdSchema,
   toolCallId: desktopToolCallIdSchema,
   executionToken: z.string().min(1).max(128),
   kind: z.enum(['file', 'directory']),
   /** The import source's own name: the folder a directory import lands in, or the file. */
-  sourceName: z.string().trim().min(1, 'Source name is required').max(255),
+  sourceName: z
+    .string()
+    .trim()
+    .min(1, 'Source name is required')
+    .max(255)
+    .refine((name) => !name.includes('/'), 'Source name must be a single name')
+    .refine((name) => !name.includes('\\'), BACKSLASH_NAME),
   relativePath: desktopImportRelativePathSchema,
 })
 

apps/sim/lib/desktop/application/import.integration.ts

@@ -20,6 +20,7 @@ vi.mock('@/lib/uploads/core/setup.server', () => ({
 import type { SessionPrincipal } from '@sim/auth/principal'
 import { db } from '@sim/db'
 import {
+  auditLog,
   copilotAsyncToolCalls,
   copilotChats,
   copilotRuns,
@@ -49,6 +50,8 @@ describe('desktop imports', () => {
 
   afterAll(async () => {
     if (userIds.length) {
+      // Audit rows outlive their actor (the foreign key sets null), so they go first.
+      await db.delete(auditLog).where(inArray(auditLog.actorId, userIds))
       await db.delete(workspace).where(inArray(workspace.ownerId, userIds))
       await db.delete(user).where(inArray(user.id, userIds))
     }
@@ -185,6 +188,30 @@ describe('desktop imports', () => {
     expect(stored).toBeDefined()
   })
 
+  it('records the folders an import creates, and only those', async () => {
+    const claimed = await claimedImport()
+
+    const root = await entry(claimed, 'directory', '')
+    await entry(claimed, 'directory', '')
+
+    const auditedRoot = async () =>
+      (
+        await db
+          .select({ resourceId: auditLog.resourceId, action: auditLog.action })
+          .from(auditLog)
+          .where(eq(auditLog.workspaceId, claimed.workspaceId))
+      ).filter((row) => row.resourceId === root.id)
+    await expect.poll(auditedRoot).toEqual([{ resourceId: root.id, action: 'folder.created' }])
+  })
+
+  it('refuses a file entry with no bytes instead of storing an empty file', async () => {
+    const claimed = await claimedImport()
+
+    await expect(entry(claimed, 'file', 'empty.txt')).rejects.toMatchObject({
+      code: 'validation',
+    })
+  })
+
   it('merges into folders that already exist and never overwrites a file', async () => {
     const claimed = await claimedImport()
     const first = await entry(claimed, 'directory', '')

apps/sim/lib/desktop/application/import.ts

@@ -1,10 +1,12 @@
+import { AuditAction, AuditResourceType } from '@sim/audit'
 import type { Principal } from '@sim/auth/principal'
 import { toRecord } from '@sim/utils/object'
 import { OrchestrationError } from '@/lib/core/orchestration/types'
 import { DesktopDeviceUnrecognizedError } from '@/lib/desktop/executor/errors'
 import { getBoundDesktopCall, getBoundDesktopDevice } from '@/lib/desktop/executor/repository'
 import { ASYNC_TOOL_STATUS } from '@/lib/mothership/async-runs/lifecycle'
 import { getAsyncToolCall } from '@/lib/mothership/async-runs/repository'
+import { notifyWorkspaceFilesChanged } from '@/lib/realtime/notify'
 import {
   ensureWorkspaceFileChildFolder,
   loadActiveWorkspaceContext,
@@ -105,21 +107,25 @@ export const importDesktopEntry = defineAuthorizedWorkspaceFileUseCase({
     const segments = input.relativePath
       ? [input.sourceName, ...input.relativePath.split('/')]
       : [input.sourceName]
-    const folders = input.kind === 'directory' ? segments : segments.slice(0, -1)
+    const folderSegments = input.kind === 'directory' ? segments : segments.slice(0, -1)
     let folderId = context.rootFolderId
-    for (const name of folders) {
-      folderId = await ensureWorkspaceFileChildFolder({
+    const createdFolders: Array<{ id: string; name: string }> = []
+    for (const name of folderSegments) {
+      const folder = await ensureWorkspaceFileChildFolder({
         workspaceId: context.workspaceId,
         userId: context.importingUserId,
         parentId: folderId,
         name,
       })
+      if (folder.created) createdFolders.push({ id: folder.id, name: folder.name })
+      folderId = folder.id
     }
     const name = segments[segments.length - 1] ?? input.sourceName
     if (input.kind === 'directory') {
       if (!folderId) throw new OrchestrationError('validation', 'A directory needs a name')
-      return { kind: 'directory' as const, id: folderId, name }
+      return { kind: 'directory' as const, id: folderId, name, createdFolders }
     }
+    if (!input.content) throw new OrchestrationError('validation', 'A file import needs its bytes')
     const created = await createAuthorizedWorkspaceFile({
       principal,
       input: {
@@ -129,11 +135,30 @@ export const importDesktopEntry = defineAuthorizedWorkspaceFileUseCase({
         folderId,
         exactName: false,
       },
-      content: input.content ?? Buffer.alloc(0),
+      content: input.content,
       workspace: context,
     })
-    return { kind: 'file' as const, id: created.file.id, name: created.file.name, created }
+    return {
+      kind: 'file' as const,
+      id: created.file.id,
+      name: created.file.name,
+      createdFolders,
+      created,
+    }
+  },
+  // Folders an import creates are recorded like folders made by hand; the file records itself.
+  projectAudit: ({ result }) => [
+    ...result.createdFolders.map((folder) => ({
+      action: AuditAction.FOLDER_CREATED,
+      resourceType: AuditResourceType.FOLDER,
+      resourceId: folder.id,
+      resourceName: folder.name,
+      description: `Created file folder "${folder.name}"`,
+    })),
+    ...(result.kind === 'file' ? [projectCreateWorkspaceFileAudit(result.created)] : []),
+  ],
+  async afterSuccess({ context, result }) {
+    // A new file announces itself; a new folder is announced here, as folder creation does.
+    if (result.createdFolders.length > 0) await notifyWorkspaceFilesChanged(context.workspaceId)
   },
-  projectAudit: ({ result }) =>
-    result.kind === 'file' ? projectCreateWorkspaceFileAudit(result.created) : [],
 })