Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view

Large diffs are not rendered by default.

Large diffs are not rendered by default.

75 changes: 75 additions & 0 deletions apps/sim/lib/execution/remote-sandbox/e2b-session.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -606,3 +606,78 @@ describe('E2B session recovery', () => {
expect(killSandbox).not.toHaveBeenCalled()
})
})

describe('E2B session lease', () => {
const IDLE_MS = 20 * 60_000
const LEASE_MS = IDLE_MS + 60_000

/** Counts control-plane requests; connect and setTimeout both set the deadline, so a caller must preserve it. */
function controlPlane(remainingMs: number) {
const plane = { endAtMs: Date.now() + remainingMs, requests: 0 }
list.mockReturnValue({ nextItems, hasNext: false })
nextItems.mockResolvedValue([{ ...candidate('retained', 10), endAt: new Date(plane.endAtMs) }])
const sandbox = {
sandboxId: 'retained',
getInfo: async () => {
plane.requests++
return { endAt: new Date(plane.endAtMs) }
},
setTimeout: async (timeoutMs: number) => {
plane.requests++
plane.endAtMs = Date.now() + timeoutMs
},
}
connect.mockImplementation(async (_id: string, options: { timeoutMs: number }) => {
plane.requests++
plane.endAtMs = Date.now() + options.timeoutMs
return sandbox
})
create.mockImplementation(async (_template: string, options: { timeoutMs: number }) => {
plane.requests++
plane.endAtMs = Date.now() + options.timeoutMs
return sandbox
})
return plane
}

it('grants a reused lease in the reconnect and keeps the idle window without more requests', async () => {
const plane = controlPlane(5 * 60_000)
const sandbox = await e2bProvider.findSessionSandbox?.('chat', { lifetimeMs: LEASE_MS })
expect(plane.endAtMs).toBeGreaterThanOrEqual(Date.now() + LEASE_MS - 1000)
expect(sandbox?.outlives?.(IDLE_MS)).toBe(true)
await sandbox?.extendLifetime?.(IDLE_MS)
expect(plane.endAtMs).toBeGreaterThanOrEqual(Date.now() + IDLE_MS)
expect(plane.requests).toBe(1)
})

it('grants a created lease at creation without reading it back', async () => {
const plane = controlPlane(0)
const sandbox = await e2bProvider.create('mothership', {
sessionKey: 'chat',
lifetimeMs: LEASE_MS,
})
expect(sandbox.outlives?.(IDLE_MS)).toBe(true)
await sandbox.extendLifetime?.(IDLE_MS)
expect(plane.endAtMs).toBeGreaterThanOrEqual(Date.now() + LEASE_MS - 1000)
expect(plane.requests).toBe(1)
})

it('keeps a later deadline another job already granted', async () => {
const plane = controlPlane(2 * 3_600_000)
const sandbox = await e2bProvider.findSessionSandbox?.('chat', { lifetimeMs: LEASE_MS })
await sandbox?.extendLifetime?.(IDLE_MS)
expect(plane.endAtMs).toBeGreaterThanOrEqual(Date.now() + 2 * 3_600_000 - 1000)
expect(plane.requests).toBe(1)
})

it('reads back and extends a lease it did not grant itself', async () => {
const plane = controlPlane(5 * 60_000)
const sandbox = await e2bProvider.findSessionSandbox?.('chat', {})
expect(sandbox?.outlives?.(IDLE_MS)).toBe(false)
await sandbox?.extendLifetime?.(LEASE_MS)
expect(plane.endAtMs).toBeGreaterThanOrEqual(Date.now() + LEASE_MS - 1000)
expect(plane.requests).toBe(3)
await sandbox?.extendLifetime?.(IDLE_MS)
expect(plane.requests).toBe(3)
})
})
43 changes: 37 additions & 6 deletions apps/sim/lib/execution/remote-sandbox/e2b.ts
Original file line number Diff line number Diff line change
Expand Up @@ -359,25 +359,41 @@ class E2BSandboxHandle implements SandboxHandle {
private killed = false
private killPromise: Promise<void> | null = null

/**
* @param sessionDeadlineAtMs Earliest time the provider can reap this session sandbox, as set
* by this handle's own create, connect, or timeout request. Session deadlines only ever move
* later — every update path extends and none shortens — so it stays a valid lower bound.
*/
constructor(
private readonly sandbox: E2BSandbox,
private readonly language: CodeLanguage,
private readonly providerLimitAtMs?: number,
private readonly sessionKey?: string
private readonly sessionKey?: string,
private sessionDeadlineAtMs?: number
) {}

get sandboxId(): string {
return this.sandbox.sandboxId
}

outlives(lifetimeMs: number): boolean {
return (
this.sessionDeadlineAtMs !== undefined &&
this.sessionDeadlineAtMs >= Date.now() + e2bTimeoutMs(lifetimeMs)
)
}

async extendLifetime(lifetimeMs: number): Promise<void> {
const timeoutMs = e2bTimeoutMs(lifetimeMs)
if (this.sessionKey !== undefined) {
if (this.outlives(lifetimeMs)) return
/** Session callers serialize updates so a short job cannot shorten another job's lease. */
const info = await this.sandbox.getInfo()
if (info.endAt.getTime() >= Date.now() + timeoutMs) return
}
const requestedAtMs = Date.now()
await this.sandbox.setTimeout(timeoutMs)
if (this.sessionKey !== undefined) this.sessionDeadlineAtMs = requestedAtMs + timeoutMs
}

async runCode(
Expand Down Expand Up @@ -1072,13 +1088,16 @@ export const e2bProvider: SandboxProvider = {
effectiveLifetimeMs === E2B_MAX_SANDBOX_LIFETIME_MS
? lifetimeStartedAtMs + E2B_MAX_SANDBOX_LIFETIME_MS
: undefined,
options?.sessionKey
options?.sessionKey,
options?.sessionKey && effectiveLifetimeMs !== undefined
? lifetimeStartedAtMs + effectiveLifetimeMs
: undefined
)
},

async findSessionSandbox(
key: string,
options: { language?: CodeLanguage }
options: { language?: CodeLanguage; lifetimeMs?: number }
): Promise<SandboxHandle | null> {
const apiKey = env.E2B_API_KEY
if (!apiKey) throw new Error('E2B_API_KEY is required when E2B is enabled')
Expand All @@ -1097,9 +1116,21 @@ export const e2bProvider: SandboxProvider = {
'This workbench predates durable execution ownership and requires recovery before reuse'
)
}
// Connect also sets a timeout, including for running sandboxes. Preserve the active deadline.
const timeoutMs = Math.max(5 * 60_000, candidate.endAt.getTime() - Date.now())
// Connect also sets a timeout, including for running sandboxes. Preserve the active deadline,
// and grant the requested lease in the same request instead of a later getInfo + setTimeout.
const requestedAtMs = Date.now()
const timeoutMs = Math.max(
5 * 60_000,
candidate.endAt.getTime() - requestedAtMs,
options.lifetimeMs === undefined ? 0 : e2bTimeoutMs(options.lifetimeMs)
)
const sandbox = await Sandbox.connect(candidate.sandboxId, { apiKey, timeoutMs })
return new E2BSandboxHandle(sandbox, options.language ?? CodeLanguage.Python, undefined, key)
return new E2BSandboxHandle(
sandbox,
options.language ?? CodeLanguage.Python,
undefined,
key,
options.lifetimeMs === undefined ? undefined : requestedAtMs + timeoutMs
)
},
}
2 changes: 2 additions & 0 deletions apps/sim/lib/execution/remote-sandbox/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -135,6 +135,8 @@ async function leaseSandbox(
created,
session: status,
release: async () => {
// A deadline that already covers the idle window needs no serialized update.
if (created.sandbox.outlives?.(SESSION_SANDBOX_IDLE_MS)) return
// Cleanup failure cannot relabel a completed mutation as a failed execution.
try {
await withSandboxSessionLock(session.key, AbortSignal.timeout(30_000), async () => {
Expand Down
23 changes: 23 additions & 0 deletions apps/sim/lib/execution/remote-sandbox/session-sandbox.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -771,6 +771,29 @@ describe('session sandbox lease', () => {
expect(calls.extendLifetime.length).toBeGreaterThanOrEqual(2)
})

it('trusts a lease granted by a slow reconnect for the whole call, then refreshes nothing', async () => {
const { handle, calls } = fakeSandbox('sb-covered')
let grantedUntilMs = 0
handle.outlives = (lifetimeMs) => grantedUntilMs >= Date.now() + lifetimeMs
mockFindSessionSandbox.mockImplementation(
async (_key: string, options: { lifetimeMs?: number }) => {
grantedUntilMs = Date.now() + (options.lifetimeMs ?? 0)
await sleep(20)
return handle
}
)

const result = await executeInSandbox({
...CODE_REQUEST,
sandboxKind: 'mothership',
session: { key: 'mothership-chat:c3' },
})

expect(result.sandboxSession).toBe('reused')
expect(grantedUntilMs).toBeGreaterThanOrEqual(Date.now() + 20 * 60_000)
expect(calls.extendLifetime).toHaveLength(0)
})

it('does not rewrite an unchanged executable while earlier code can still use it', async () => {
const { handle } = fakeSandbox('unchanged-tooling')
mockFindSessionSandbox.mockResolvedValue(handle)
Expand Down
6 changes: 5 additions & 1 deletion apps/sim/lib/execution/remote-sandbox/session.ts
Original file line number Diff line number Diff line change
Expand Up @@ -35,8 +35,10 @@ export async function ensureSessionSandbox(args: {
signal.throwIfAborted()
if (!provider.findSessionSandbox) throw new Error('This deployment has no persistent workbench')
const lifetimeMs = SESSION_SANDBOX_IDLE_MS + (options.lifetimeMs ?? 0)
const requestedAtMs = Date.now()
const existing = await provider.findSessionSandbox(session.key, {
...(options.language ? { language: options.language } : {}),
lifetimeMs,
})
signal.throwIfAborted()
const created: CreatedSandbox = existing
Expand All @@ -61,7 +63,9 @@ export async function ensureSessionSandbox(args: {
providerId: provider.id,
sandboxId: created.sandbox.sandboxId,
})
await created.sandbox.extendLifetime?.(lifetimeMs)
// The budget is anchored before acquisition, so a lease granted by the lookup or create covers it.
if (!created.sandbox.outlives?.(Math.max(0, lifetimeMs - (Date.now() - requestedAtMs))))
await created.sandbox.extendLifetime?.(lifetimeMs)
signal.throwIfAborted()
if (session.cli) {
await ensureSessionCli(created.sandbox, session.cli, signal, args.bootstrapTimeoutMs)
Expand Down
10 changes: 8 additions & 2 deletions apps/sim/lib/execution/remote-sandbox/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -278,6 +278,11 @@ export interface SandboxHandle {
* served an execution. Absent on providers without session support.
*/
extendLifetime?(lifetimeMs: number): Promise<void>
/**
* True when a deadline this handle already established keeps the sandbox alive for
* `lifetimeMs` from now, so {@link extendLifetime} would make no provider request.
*/
outlives?(lifetimeMs: number): boolean
/** Reads provider metadata without materializing the file contents. */
getFileSize(path: string): Promise<number>
readFile(path: string): Promise<string>
Expand Down Expand Up @@ -475,10 +480,11 @@ export interface SandboxProvider {
* {@link CreateSandboxOptions.sessionKey}, or resolves null when none is
* available. Lookup failures must throw rather than masquerade as absence.
* Providers without session support omit this method; callers then
* run every execution in a fresh sandbox.
* run every execution in a fresh sandbox. `lifetimeMs` asks the reconnect to keep the
* sandbox alive at least that long, never shortening a later deadline.
*/
findSessionSandbox?(
key: string,
options: { language?: CodeLanguage }
options: { language?: CodeLanguage; lifetimeMs?: number }
): Promise<SandboxHandle | null>
}
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
28 changes: 24 additions & 4 deletions apps/sim/tools/cloudflare/get_tunnel.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ export const getTunnelTool: ToolConfig<CloudflareGetTunnelParams, CloudflareTunn
id: 'cloudflare_get_tunnel',
name: 'Cloudflare Get Tunnel',
description:
'Reads a single Cloudflare Tunnel (cloudflared), including its health status and active connector connections. Requires an API token with Account Cloudflare Tunnel Read.',
'Reads a single Cloudflare Tunnel (cloudflared), including its health status and active connector connections (from the dedicated connections endpoint). Requires an API token with Account Cloudflare Tunnel Read.',
version: '1.0.0',

params: {
Expand Down Expand Up @@ -37,7 +37,7 @@ export const getTunnelTool: ToolConfig<CloudflareGetTunnelParams, CloudflareTunn
headers: (params) => cloudflareHeaders(params.apiKey),
},

transformResponse: async (response: Response) => {
transformResponse: async (response: Response, params?: CloudflareGetTunnelParams) => {
const data = await response.json()

if (!data.success) {
Expand All @@ -63,6 +63,25 @@ export const getTunnelTool: ToolConfig<CloudflareGetTunnelParams, CloudflareTunn
}

const tunnel = data.result
let connections: unknown[] | null = null
if (params?.accountId && params?.tunnelId && params?.apiKey) {
try {
const connectionsUrl = `https://api.cloudflare.com/client/v4/accounts/${params.accountId.trim()}/cfd_tunnel/${params.tunnelId.trim()}/connections`
const connectionsRes = await fetch(connectionsUrl, {
method: 'GET',
headers: cloudflareHeaders(params.apiKey),
})
Comment on lines +70 to +73

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Connections request ignores cancellation

If a workflow is cancelled or its deadline expires while Cloudflare's connections request is pending, this new fetch has neither the execution signal nor a timeout. Unlike the initial tunnel request, it can keep cloudflare_get_tunnel pending after cancellation. Pass the signal supplied to transformResponse and bound the request.

const connectionsData = await connectionsRes.json()
if (connectionsData?.success && Array.isArray(connectionsData.result)) {
connections = connectionsData.result.flatMap(
(connector: { conns?: unknown[] }) => connector.conns ?? []
)
Comment on lines +75 to +78

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Connection parsing lacks tests

The new flattening of connector responses has no regression test, and the existing Cloudflare tests cover block parameter defaults rather than this response. A response-shape mistake or silently empty connections result could go unnoticed. Add focused tests for successful connector data and a failed secondary request.

Note: If this suggestion doesn't match your team's coding style, reply to this and let me know. I'll remember it for next time!

}
} catch {
connections = null
}
Comment on lines +74 to +82

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Connection failures are hidden

If Cloudflare rejects the connections request or it fails, the tool still reports success with connections: null and provides no reason. That makes a permissions or API failure hard to distinguish from unavailable connection data. Preserve a warning or error for the secondary lookup so users can diagnose it.

}

return {
success: true,
output: {
Expand All @@ -78,7 +97,7 @@ export const getTunnelTool: ToolConfig<CloudflareGetTunnelParams, CloudflareTunn
deleted_at: tunnel?.deleted_at ?? null,
conns_active_at: tunnel?.conns_active_at ?? null,
conns_inactive_at: tunnel?.conns_inactive_at ?? null,
connections: tunnel?.connections ?? null,
connections,
},
}
},
Expand Down Expand Up @@ -126,7 +145,8 @@ export const getTunnelTool: ToolConfig<CloudflareGetTunnelParams, CloudflareTunn
},
connections: {
type: 'json',
description: 'Active connector connections for the tunnel',
description:
'Active connector connections for the tunnel (from GET .../cfd_tunnel/{id}/connections)',
optional: true,
},
},
Expand Down
7 changes: 4 additions & 3 deletions apps/sim/tools/cloudflare/list_tunnels.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ export const listTunnelsTool: ToolConfig<
id: 'cloudflare_list_tunnels',
name: 'Cloudflare List Tunnels',
description:
'Lists the Cloudflare Tunnels (cloudflared) in an account, with their health status and active connections. Requires an API token with Account Cloudflare Tunnel Read.',
'Lists the Cloudflare Tunnels (cloudflared) in an account, with their health status. Connections are no longer included on list responses (Cloudflare API change 2026-10-05); use cloudflare_get_tunnel for a tunnel’s connections. Requires an API token with Account Cloudflare Tunnel Read.',
version: '1.0.0',

params: {
Expand Down Expand Up @@ -155,7 +155,7 @@ export const listTunnelsTool: ToolConfig<
deleted_at: tunnel.deleted_at ?? null,
conns_active_at: tunnel.conns_active_at ?? null,
conns_inactive_at: tunnel.conns_inactive_at ?? null,
connections: tunnel.connections ?? null,
connections: null,
})),
total_count: data.result_info?.total_count ?? tunnels.length,
},
Expand Down Expand Up @@ -215,7 +215,8 @@ export const listTunnelsTool: ToolConfig<
},
connections: {
type: 'json',
description: 'Active connector connections for the tunnel',
description:
'Always null on list. Use cloudflare_get_tunnel to fetch active connections for a specific tunnel.',
optional: true,
},
},
Expand Down