Skip to content

Commit 48b9b0b

Browse files
TheodoreSpeaksTheodoreSpeaks
authored
Merge pull request #6244 from simstudioai/feat/v2-knowledge-multipart-upload
feat(knowledge): multipart knowledge document uploads
2 parents 832d4ee + 09c27a4 commit 48b9b0b

40 files changed

Lines changed: 2827 additions & 352 deletions

File tree

‎apps/docs/openapi-v2-knowledge.json‎

Lines changed: 400 additions & 0 deletions
Large diffs are not rendered by default.

‎apps/sim/app/api/files/uploads/[uploadId]/complete/route.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ export const POST = withRouteHandler(async (request: NextRequest, context: Uploa
3232
uploadId: parsed.data.params.uploadId,
3333
workspaceId,
3434
userId: user,
35+
purpose: 'workspace_file',
3536
uploadToken: parsed.data.headers['upload-token'],
3637
})
3738
const metadata = upload.metadata as { folderId?: string | null }

‎apps/sim/app/api/files/uploads/[uploadId]/parts/route.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ export const POST = withRouteHandler(async (request: NextRequest, context: Uploa
2929
uploadId: parsed.data.params.uploadId,
3030
workspaceId,
3131
userId: user,
32+
purpose: 'workspace_file',
3233
uploadToken: parsed.data.headers['upload-token'],
3334
})
3435
const parts = await createUploadPartUrls({

‎apps/sim/app/api/files/uploads/[uploadId]/route.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ export const DELETE = withRouteHandler(async (request: NextRequest, context: Upl
2727
uploadId: parsed.data.params.uploadId,
2828
workspaceId,
2929
userId: user,
30+
purpose: 'workspace_file',
3031
uploadToken: parsed.data.headers['upload-token'],
3132
})
3233
return NextResponse.json({ data: toV2FileUpload(await abortUploadSession(upload), null) })

‎apps/sim/app/api/knowledge/[id]/documents/uploads/[uploadId]/complete/route.ts‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
import { type NextRequest, NextResponse } from 'next/server'
2+
import { completeKnowledgeDocumentUploadContract } from '@/lib/api/contracts/knowledge/upload-sessions'
3+
import { parseRequest } from '@/lib/api/server'
4+
import { generateRequestId } from '@/lib/core/utils/request'
5+
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
6+
import { completeUploadSession } from '@/lib/uploads/multipart-session/service'
7+
import { uploadSessionErrorResponse } from '@/app/api/files/uploads/utils'
8+
import {
9+
requireKnowledgeDocumentUploadAccess,
10+
requireKnowledgeDocumentUploadActor,
11+
resolveKnowledgeDocumentUploadAttribution,
12+
} from '@/app/api/knowledge/[id]/documents/uploads/utils'
13+
import {
14+
finalizeKnowledgeDocumentUpload,
15+
getOwnedKnowledgeDocumentUpload,
16+
toV2KnowledgeDocumentUpload,
17+
} from '@/app/api/v2/knowledge/[id]/documents/uploads/utils'
18+
19+
interface KnowledgeDocumentUploadRouteParams {
20+
params: Promise<{ id: string; uploadId: string }>
21+
}
22+
23+
export const POST = withRouteHandler(
24+
async (request: NextRequest, context: KnowledgeDocumentUploadRouteParams) => {
25+
const actor = await requireKnowledgeDocumentUploadActor()
26+
if (actor instanceof NextResponse) return actor
27+
const parsed = await parseRequest(completeKnowledgeDocumentUploadContract, request, context)
28+
if (!parsed.success) return parsed.response
29+
const { id: knowledgeBaseId, uploadId } = parsed.data.params
30+
const { workspaceId } = parsed.data.query
31+
const access = await requireKnowledgeDocumentUploadAccess({
32+
knowledgeBaseId,
33+
workspaceId,
34+
userId: actor.id,
35+
})
36+
if (access instanceof NextResponse) return access
37+
const requestId = generateRequestId()
38+
try {
39+
const upload = getOwnedKnowledgeDocumentUpload({
40+
knowledgeBaseId,
41+
uploadId,
42+
workspaceId,
43+
userId: actor.id,
44+
uploadToken: parsed.data.headers['upload-token'],
45+
})
46+
const completed = await completeUploadSession({
47+
session: upload,
48+
parts: parsed.data.body.parts,
49+
finalize: (claimed) =>
50+
finalizeKnowledgeDocumentUpload({
51+
claimed,
52+
knowledgeBaseId,
53+
knowledgeBaseName: access.knowledgeBase.name,
54+
workspaceId,
55+
userId: actor.id,
56+
resolveAttribution: () =>
57+
resolveKnowledgeDocumentUploadAttribution({ workspaceId, userId: actor.id }),
58+
source: 'ui',
59+
requestId,
60+
request,
61+
actorName: actor.name,
62+
actorEmail: actor.email,
63+
}),
64+
})
65+
return NextResponse.json({
66+
data: toV2KnowledgeDocumentUpload(completed.session, completed.value),
67+
})
68+
} catch (error) {
69+
const classified = uploadSessionErrorResponse(error)
70+
if (classified) return classified
71+
throw error
72+
}
73+
}
74+
)

‎apps/sim/app/api/knowledge/[id]/documents/uploads/[uploadId]/parts/route.ts‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
import { type NextRequest, NextResponse } from 'next/server'
2+
import { createKnowledgeDocumentUploadPartUrlsContract } from '@/lib/api/contracts/knowledge/upload-sessions'
3+
import { parseRequest } from '@/lib/api/server'
4+
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
5+
import { createUploadPartUrls } from '@/lib/uploads/multipart-session/service'
6+
import { uploadSessionErrorResponse } from '@/app/api/files/uploads/utils'
7+
import {
8+
requireKnowledgeDocumentUploadAccess,
9+
requireKnowledgeDocumentUploadActor,
10+
} from '@/app/api/knowledge/[id]/documents/uploads/utils'
11+
import { getOwnedKnowledgeDocumentUpload } from '@/app/api/v2/knowledge/[id]/documents/uploads/utils'
12+
13+
interface KnowledgeDocumentUploadRouteParams {
14+
params: Promise<{ id: string; uploadId: string }>
15+
}
16+
17+
export const POST = withRouteHandler(
18+
async (request: NextRequest, context: KnowledgeDocumentUploadRouteParams) => {
19+
const actor = await requireKnowledgeDocumentUploadActor()
20+
if (actor instanceof NextResponse) return actor
21+
const parsed = await parseRequest(
22+
createKnowledgeDocumentUploadPartUrlsContract,
23+
request,
24+
context
25+
)
26+
if (!parsed.success) return parsed.response
27+
const { id: knowledgeBaseId, uploadId } = parsed.data.params
28+
const { workspaceId } = parsed.data.query
29+
const access = await requireKnowledgeDocumentUploadAccess({
30+
knowledgeBaseId,
31+
workspaceId,
32+
userId: actor.id,
33+
})
34+
if (access instanceof NextResponse) return access
35+
try {
36+
const upload = getOwnedKnowledgeDocumentUpload({
37+
knowledgeBaseId,
38+
uploadId,
39+
workspaceId,
40+
userId: actor.id,
41+
uploadToken: parsed.data.headers['upload-token'],
42+
})
43+
const parts = await createUploadPartUrls({
44+
session: upload,
45+
partNumbers: parsed.data.body.partNumbers,
46+
localOrigin: request.nextUrl.origin,
47+
})
48+
return NextResponse.json({ data: { parts } })
49+
} catch (error) {
50+
const classified = uploadSessionErrorResponse(error)
51+
if (classified) return classified
52+
throw error
53+
}
54+
}
55+
)

‎apps/sim/app/api/knowledge/[id]/documents/uploads/[uploadId]/route.ts‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
import { type NextRequest, NextResponse } from 'next/server'
2+
import { abortKnowledgeDocumentUploadContract } from '@/lib/api/contracts/knowledge/upload-sessions'
3+
import { parseRequest } from '@/lib/api/server'
4+
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
5+
import { uploadSessionErrorResponse } from '@/app/api/files/uploads/utils'
6+
import {
7+
requireKnowledgeDocumentUploadAccess,
8+
requireKnowledgeDocumentUploadActor,
9+
} from '@/app/api/knowledge/[id]/documents/uploads/utils'
10+
import {
11+
abortKnowledgeDocumentUpload,
12+
getOwnedKnowledgeDocumentUpload,
13+
toV2KnowledgeDocumentUpload,
14+
} from '@/app/api/v2/knowledge/[id]/documents/uploads/utils'
15+
16+
interface KnowledgeDocumentUploadRouteParams {
17+
params: Promise<{ id: string; uploadId: string }>
18+
}
19+
20+
export const DELETE = withRouteHandler(
21+
async (request: NextRequest, context: KnowledgeDocumentUploadRouteParams) => {
22+
const actor = await requireKnowledgeDocumentUploadActor()
23+
if (actor instanceof NextResponse) return actor
24+
const parsed = await parseRequest(abortKnowledgeDocumentUploadContract, request, context)
25+
if (!parsed.success) return parsed.response
26+
const { id: knowledgeBaseId, uploadId } = parsed.data.params
27+
const { workspaceId } = parsed.data.query
28+
const access = await requireKnowledgeDocumentUploadAccess({
29+
knowledgeBaseId,
30+
workspaceId,
31+
userId: actor.id,
32+
})
33+
if (access instanceof NextResponse) return access
34+
try {
35+
const upload = getOwnedKnowledgeDocumentUpload({
36+
knowledgeBaseId,
37+
uploadId,
38+
workspaceId,
39+
userId: actor.id,
40+
uploadToken: parsed.data.headers['upload-token'],
41+
})
42+
const aborted = await abortKnowledgeDocumentUpload(upload, knowledgeBaseId)
43+
return NextResponse.json({ data: toV2KnowledgeDocumentUpload(aborted, null) })
44+
} catch (error) {
45+
const classified = uploadSessionErrorResponse(error)
46+
if (classified) return classified
47+
throw error
48+
}
49+
}
50+
)

‎apps/sim/app/api/knowledge/[id]/documents/uploads/route.test.ts‎

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
/**
2+
* @vitest-environment node
3+
*/
4+
import { NextRequest, NextResponse } from 'next/server'
5+
import { beforeEach, describe, expect, it, vi } from 'vitest'
6+
7+
const {
8+
mockCreateKnowledgeDocumentUploadSession,
9+
mockRequireKnowledgeDocumentUploadAccess,
10+
mockRequireKnowledgeDocumentUploadActor,
11+
mockRequireKnowledgeDocumentUploadBilling,
12+
} = vi.hoisted(() => ({
13+
mockCreateKnowledgeDocumentUploadSession: vi.fn(),
14+
mockRequireKnowledgeDocumentUploadAccess: vi.fn(),
15+
mockRequireKnowledgeDocumentUploadActor: vi.fn(),
16+
mockRequireKnowledgeDocumentUploadBilling: vi.fn(),
17+
}))
18+
19+
vi.mock('@/app/api/knowledge/[id]/documents/uploads/utils', () => ({
20+
requireKnowledgeDocumentUploadAccess: mockRequireKnowledgeDocumentUploadAccess,
21+
requireKnowledgeDocumentUploadActor: mockRequireKnowledgeDocumentUploadActor,
22+
requireKnowledgeDocumentUploadBilling: mockRequireKnowledgeDocumentUploadBilling,
23+
}))
24+
vi.mock('@/app/api/files/uploads/utils', () => ({ uploadSessionErrorResponse: vi.fn() }))
25+
vi.mock('@/app/api/v2/knowledge/[id]/documents/uploads/utils', () => ({
26+
createKnowledgeDocumentUploadSession: mockCreateKnowledgeDocumentUploadSession,
27+
toV2KnowledgeDocumentUpload: (session: Record<string, unknown>) => ({
28+
...session,
29+
name: session.fileName,
30+
contentType: session.contentType,
31+
size: session.fileSize,
32+
expiresAt: '2026-08-05T00:00:00.000Z',
33+
document: null,
34+
}),
35+
}))
36+
37+
import { POST } from '@/app/api/knowledge/[id]/documents/uploads/route'
38+
39+
const WORKSPACE_ID = '6fc7631d-88cd-46f8-9f0a-d4764daef7f8'
40+
41+
function request() {
42+
return POST(
43+
new NextRequest('http://localhost:3000/api/knowledge/kb-1/documents/uploads', {
44+
method: 'POST',
45+
headers: { 'Content-Type': 'application/json' },
46+
body: JSON.stringify({
47+
workspaceId: WORKSPACE_ID,
48+
name: 'guide.pdf',
49+
contentType: 'application/pdf',
50+
size: 1024,
51+
tag1: 'product',
52+
processingOptions: { recipe: 'default', lang: 'en' },
53+
}),
54+
}),
55+
{ params: Promise.resolve({ id: 'kb-1' }) }
56+
)
57+
}
58+
59+
describe('POST /api/knowledge/[id]/documents/uploads', () => {
60+
beforeEach(() => {
61+
vi.clearAllMocks()
62+
mockRequireKnowledgeDocumentUploadActor.mockResolvedValue({ id: 'user-1' })
63+
mockRequireKnowledgeDocumentUploadAccess.mockResolvedValue({
64+
knowledgeBase: { id: 'kb-1', name: 'Docs', workspaceId: WORKSPACE_ID },
65+
})
66+
mockRequireKnowledgeDocumentUploadBilling.mockResolvedValue({ actorUserId: 'user-1' })
67+
mockCreateKnowledgeDocumentUploadSession.mockResolvedValue({
68+
id: 'upload-1',
69+
knowledgeBaseId: 'kb-1',
70+
status: 'uploading',
71+
fileName: 'guide.pdf',
72+
contentType: 'application/pdf',
73+
fileSize: 1024,
74+
partSize: 8 * 1024 * 1024,
75+
partCount: 1,
76+
uploadToken: 'token',
77+
error: null,
78+
})
79+
})
80+
81+
it('authorizes and bills before allocating a first-party upload session', async () => {
82+
const response = await request()
83+
84+
expect(response.status).toBe(201)
85+
expect(mockRequireKnowledgeDocumentUploadAccess).toHaveBeenCalledWith({
86+
knowledgeBaseId: 'kb-1',
87+
workspaceId: WORKSPACE_ID,
88+
userId: 'user-1',
89+
})
90+
expect(mockCreateKnowledgeDocumentUploadSession).toHaveBeenCalledWith({
91+
workspaceId: WORKSPACE_ID,
92+
userId: 'user-1',
93+
knowledgeBaseId: 'kb-1',
94+
fileName: 'guide.pdf',
95+
contentType: 'application/pdf',
96+
fileSize: 1024,
97+
metadata: {
98+
tag1: 'product',
99+
processingOptions: { recipe: 'default', lang: 'en' },
100+
},
101+
})
102+
expect(mockRequireKnowledgeDocumentUploadBilling.mock.invocationCallOrder[0]).toBeLessThan(
103+
mockCreateKnowledgeDocumentUploadSession.mock.invocationCallOrder[0]
104+
)
105+
})
106+
107+
it('does not bill or allocate storage when write access is denied', async () => {
108+
mockRequireKnowledgeDocumentUploadAccess.mockResolvedValue(
109+
NextResponse.json({ error: 'Forbidden' }, { status: 403 })
110+
)
111+
112+
const response = await request()
113+
114+
expect(response.status).toBe(403)
115+
expect(mockRequireKnowledgeDocumentUploadBilling).not.toHaveBeenCalled()
116+
expect(mockCreateKnowledgeDocumentUploadSession).not.toHaveBeenCalled()
117+
})
118+
})

‎apps/sim/app/api/knowledge/[id]/documents/uploads/route.ts‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
import { type NextRequest, NextResponse } from 'next/server'
2+
import { createKnowledgeDocumentUploadContract } from '@/lib/api/contracts/knowledge/upload-sessions'
3+
import { parseRequest } from '@/lib/api/server'
4+
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
5+
import { validateFileType } from '@/lib/uploads/utils/validation'
6+
import { uploadSessionErrorResponse } from '@/app/api/files/uploads/utils'
7+
import {
8+
requireKnowledgeDocumentUploadAccess,
9+
requireKnowledgeDocumentUploadActor,
10+
requireKnowledgeDocumentUploadBilling,
11+
} from '@/app/api/knowledge/[id]/documents/uploads/utils'
12+
import {
13+
createKnowledgeDocumentUploadSession,
14+
toV2KnowledgeDocumentUpload,
15+
} from '@/app/api/v2/knowledge/[id]/documents/uploads/utils'
16+
17+
interface KnowledgeDocumentUploadsRouteParams {
18+
params: Promise<{ id: string }>
19+
}
20+
21+
export const POST = withRouteHandler(
22+
async (request: NextRequest, context: KnowledgeDocumentUploadsRouteParams) => {
23+
const actor = await requireKnowledgeDocumentUploadActor()
24+
if (actor instanceof NextResponse) return actor
25+
const parsed = await parseRequest(createKnowledgeDocumentUploadContract, request, context)
26+
if (!parsed.success) return parsed.response
27+
const { id: knowledgeBaseId } = parsed.data.params
28+
const { workspaceId, name, contentType, size, ...metadata } = parsed.data.body
29+
const access = await requireKnowledgeDocumentUploadAccess({
30+
knowledgeBaseId,
31+
workspaceId,
32+
userId: actor.id,
33+
})
34+
if (access instanceof NextResponse) return access
35+
const billing = await requireKnowledgeDocumentUploadBilling({
36+
workspaceId,
37+
userId: actor.id,
38+
})
39+
if (billing instanceof NextResponse) return billing
40+
const fileTypeError = validateFileType(name, contentType)
41+
if (fileTypeError) {
42+
return NextResponse.json({ error: fileTypeError.message }, { status: 415 })
43+
}
44+
try {
45+
const upload = await createKnowledgeDocumentUploadSession({
46+
workspaceId,
47+
userId: actor.id,
48+
knowledgeBaseId,
49+
fileName: name,
50+
contentType,
51+
fileSize: size,
52+
metadata,
53+
})
54+
return NextResponse.json({ data: toV2KnowledgeDocumentUpload(upload, null) }, { status: 201 })
55+
} catch (error) {
56+
const classified = uploadSessionErrorResponse(error)
57+
if (classified) return classified
58+
throw error
59+
}
60+
}
61+
)

0 commit comments

Comments
 (0)