Skip to content

Commit 64992ca

Browse files
authored
fix(file-parsers): share the event-loop slice across PDF parses and yield inside whole-document passes (#9323)
- PDF parsing yields through the shared process-wide turnstile instead of a per-parse slicer - suppressFurniture and the line passes yield between pages instead of only between passes - Text-flow reachability is precomputed linearly, replacing a per-band-line walk that was quadratic on tall pages - The e2e build artifact includes dist/xlsx-worker.mjs so acceptance runs use the spreadsheet worker
1 parent 0751de2 commit 64992ca

7 files changed

Lines changed: 310 additions & 159 deletions

File tree

‎.github/workflows/checks.yml‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -949,13 +949,15 @@ jobs:
949949
NEXT_BUILD_SKIP_TYPE_CHECK: 'true'
950950
run: bunx turbo run build --filter=@sim/app
951951

952-
# Everything `next start` reads; node_modules and the source tree come from e2e-production's own
953-
# checkout and setup, at the same workspace path and lockfile. Source maps are over half the
954-
# build and only serve stack traces and devtools, which the suite never reads.
952+
# Everything `next start` reads, including the spreadsheet parse worker it spawns (named, so a
953+
# missing bundle fails packaging as it fails docker/app.Dockerfile); node_modules and the source
954+
# tree come from e2e-production's own checkout and setup, at the same workspace path and
955+
# lockfile. Source maps are over half the build and only serve stack traces and devtools, which
956+
# the suite never reads.
955957
- name: Package the build for e2e-production
956958
run: >-
957959
tar -I 'zstd -T0' -cf "$RUNNER_TEMP/next-build.tar.zst" -C apps/sim
958-
--exclude .next/cache --exclude .next/trace --exclude '*.map' .next
960+
--exclude .next/cache --exclude .next/trace --exclude '*.map' .next dist/xlsx-worker.mjs
959961
960962
# Already zstd-compressed, so stored without zip compression. `overwrite` lets a re-run of
961963
# this job replace the earlier attempt's build; three days covers re-running a failed shard

‎apps/sim/lib/file-parsers/pdf-furniture.test.ts‎

Lines changed: 74 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import { describe, expect, it } from 'vitest'
2+
import { yieldWhenSliceExpires } from '@/lib/core/utils/event-loop'
23
import {
34
isPageNumber,
45
normalizeFurnitureText,
@@ -33,10 +34,54 @@ function texts(pages: PdfLine[][]): string[][] {
3334
}
3435

3536
describe('suppressFurniture', () => {
36-
it('drops a header repeated on enough pages but keeps its first occurrence', () => {
37+
it('lets timers run between pages instead of holding the event loop for the whole document', async () => {
38+
const pages = Array.from({ length: 3000 }, (_, i) =>
39+
page(i + 1, { header: 'Annual Report', footer: 'Confidential' })
40+
)
41+
let timerFired = false
42+
let timerFiredBeforeLastPage = false
43+
const lastPage = pages[pages.length - 1]
44+
const [first, ...rest] = lastPage.lines
45+
lastPage.lines = [
46+
{
47+
...first,
48+
get text() {
49+
timerFiredBeforeLastPage ||= timerFired
50+
return first.text
51+
},
52+
},
53+
...rest,
54+
]
55+
await yieldWhenSliceExpires()
56+
setTimeout(() => {
57+
timerFired = true
58+
}, 0)
59+
60+
await suppressFurniture(pages)
61+
62+
expect(timerFiredBeforeLastPage).toBe(true)
63+
})
64+
65+
it('resolves a tall page packed with band baselines in linear time', async () => {
66+
const lineCount = 40_000
67+
const pageHeight = 100 * lineCount
68+
const lines = Array.from({ length: lineCount }, (_, i) =>
69+
line(`w${i.toString(36)}`, pageHeight - 3 * i, 50)
70+
)
71+
const startedAt = performance.now()
72+
73+
await suppressFurniture([
74+
{ lines, pageHeight },
75+
{ lines: [], pageHeight },
76+
])
77+
78+
expect(performance.now() - startedAt).toBeLessThan(1000)
79+
})
80+
81+
it('drops a header repeated on enough pages but keeps its first occurrence', async () => {
3782
const pages = [1, 2, 3, 4].map((i) => page(i, { header: 'ACME Corp — Internal Use Only' }))
3883

39-
const result = texts(suppressFurniture(pages))
84+
const result = texts(await suppressFurniture(pages))
4085

4186
expect(result[0]).toContain('ACME Corp — Internal Use Only')
4287
for (const remaining of result.slice(1)) {
@@ -45,34 +90,34 @@ describe('suppressFurniture', () => {
4590
}
4691
})
4792

48-
it('never applies the frequency rule to a single page', () => {
49-
const result = texts(suppressFurniture([page(1, { header: 'Draft' })]))
93+
it('never applies the frequency rule to a single page', async () => {
94+
const result = texts(await suppressFurniture([page(1, { header: 'Draft' })]))
5095

5196
expect(result[0]).toContain('Draft')
5297
})
5398

54-
it('treats two matching pages as furniture when the document has exactly two pages', () => {
99+
it('treats two matching pages as furniture when the document has exactly two pages', async () => {
55100
const pages = [1, 2].map((i) =>
56101
page(i, { footer: `Confidential draft, do not distribute — Page ${i} of 2` })
57102
)
58103

59-
const result = texts(suppressFurniture(pages))
104+
const result = texts(await suppressFurniture(pages))
60105

61106
expect(result[0]).toContain('Confidential draft, do not distribute — Page 1 of 2')
62107
expect(result[1]).not.toContain('Confidential draft, do not distribute — Page 2 of 2')
63108
})
64109

65-
it('requires the fraction threshold on longer documents without a streak', () => {
110+
it('requires the fraction threshold on longer documents without a streak', async () => {
66111
const pages = Array.from({ length: 10 }, (_, i) =>
67112
page(i, { footer: i % 4 === 0 ? 'Sporadic note' : undefined })
68113
)
69114

70-
const result = texts(suppressFurniture(pages))
115+
const result = texts(await suppressFurniture(pages))
71116

72117
expect(result.flat().filter((text) => text === 'Sporadic note')).toHaveLength(3)
73118
})
74119

75-
it('detects a footer by a three-page streak even when its title changes per chapter', () => {
120+
it('detects a footer by a three-page streak even when its title changes per chapter', async () => {
76121
const pages = Array.from({ length: 12 }, (_, i) => {
77122
const chapter = i < 6 ? 'Chapter 1 Filing Information' : 'Chapter 2 Filing Status'
78123
const pageNumber = i + 6
@@ -83,7 +128,7 @@ describe('suppressFurniture', () => {
83128
return page(i, { footer })
84129
})
85130

86-
const result = texts(suppressFurniture(pages))
131+
const result = texts(await suppressFurniture(pages))
87132
const footers = result.flat().filter((text) => text.includes('Publication 17'))
88133

89134
expect(footers).toEqual([
@@ -92,21 +137,21 @@ describe('suppressFurniture', () => {
92137
])
93138
})
94139

95-
it('drops page numbers in the bands regardless of repetition', () => {
140+
it('drops page numbers in the bands regardless of repetition', async () => {
96141
const pages: PdfPageLines[] = [
97142
{ lines: [line('Page 1 of 3', 55), ...body(3, 600)], pageHeight: PAGE_HEIGHT },
98143
{ lines: [line('2', 55), ...body(3, 600), line('ii', 729)], pageHeight: PAGE_HEIGHT },
99144
{ lines: [line('- 3 -', 55), ...body(3, 600), line('925', 60)], pageHeight: PAGE_HEIGHT },
100145
]
101146

102-
const result = texts(suppressFurniture(pages))
147+
const result = texts(await suppressFurniture(pages))
103148

104149
expect(result[0]).toEqual(['Body line 1', 'Body line 2', 'Body line 3'])
105150
expect(result[1]).toEqual(['Body line 1', 'Body line 2', 'Body line 3'])
106151
expect(result[2]).toEqual(['Body line 1', 'Body line 2', 'Body line 3', '925'])
107152
})
108153

109-
it('drops a folio outside the band only when a margin gap separates it from the text', () => {
154+
it('drops a folio outside the band only when a margin gap separates it from the text', async () => {
110155
const pages = [1, 2].map((i) => ({
111156
lines: [...body(20, 700), line(`${i}`, 100)],
112157
pageHeight: PAGE_HEIGHT,
@@ -116,21 +161,21 @@ describe('suppressFurniture', () => {
116161
pageHeight: PAGE_HEIGHT,
117162
}))
118163

119-
for (const kept of texts(suppressFurniture(pages))) expect(kept).toHaveLength(20)
120-
for (const kept of texts(suppressFurniture(tableCells))) expect(kept).toContain('1')
164+
for (const kept of texts(await suppressFurniture(pages))) expect(kept).toHaveLength(20)
165+
for (const kept of texts(await suppressFurniture(tableCells))) expect(kept).toContain('1')
121166
})
122167

123-
it('ignores band text longer than the furniture cap, such as a repeated table header', () => {
168+
it('ignores band text longer than the furniture cap, such as a repeated table header', async () => {
124169
const header = `SKU Product Unit price Lead time Notes ${'Column '.repeat(14)}`.trim()
125170
expect(header.length).toBeGreaterThan(120)
126171
const pages = [1, 2, 3, 4].map((i) => page(i, { header }))
127172

128-
const result = texts(suppressFurniture(pages))
173+
const result = texts(await suppressFurniture(pages))
129174

130175
for (const kept of result) expect(kept).toContain(header)
131176
})
132177

133-
it('merges same-baseline fragments into one key before matching', () => {
178+
it('merges same-baseline fragments into one key before matching', async () => {
134179
const pages = [1, 2, 3, 4].map((i) => ({
135180
lines: [
136181
...body(3, 600),
@@ -141,13 +186,13 @@ describe('suppressFurniture', () => {
141186
pageHeight: PAGE_HEIGHT,
142187
}))
143188

144-
const result = texts(suppressFurniture(pages))
189+
const result = texts(await suppressFurniture(pages))
145190

146191
expect(result[0].slice(3)).toEqual(['1', 'Chapter 1', 'Publication 17 (2025)'])
147192
expect(result[3]).toHaveLength(3)
148193
})
149194

150-
it('keeps a table header that repeats at the top of every page while dropping the running header above it', () => {
195+
it('keeps a table header that repeats at the top of every page while dropping the running header above it', async () => {
151196
const pages = Array.from({ length: 5 }, () => {
152197
const rows = Array.from({ length: 50 }, (_, i) =>
153198
line(`${1000 + i} ${2000 + i} ${3000 + i}`, 740 - (i + 1) * 8, 7.5)
@@ -163,7 +208,7 @@ describe('suppressFurniture', () => {
163208
}
164209
})
165210

166-
const result = texts(suppressFurniture(pages))
211+
const result = texts(await suppressFurniture(pages))
167212

168213
for (const kept of result) {
169214
expect(kept).toContain('Single Married filing jointly Head of household')
@@ -174,7 +219,7 @@ describe('suppressFurniture', () => {
174219
).toHaveLength(1)
175220
})
176221

177-
it('keeps a footnote that sits directly under the body while dropping the footer below it', () => {
222+
it('keeps a footnote that sits directly under the body while dropping the footer below it', async () => {
178223
const pages = Array.from({ length: 4 }, () => ({
179224
lines: [
180225
...body(52, 700),
@@ -188,15 +233,15 @@ describe('suppressFurniture', () => {
188233
pageHeight: PAGE_HEIGHT,
189234
}))
190235

191-
const result = texts(suppressFurniture(pages))
236+
const result = texts(await suppressFurniture(pages))
192237

193238
for (const kept of result) {
194239
expect(kept).toContain('* This column must also be used by a qualifying surviving spouse.')
195240
}
196241
expect(result.flat().filter((text) => text === 'Visit IRS.gov.')).toHaveLength(1)
197242
})
198243

199-
it('keeps a band line whose text also occurs in body positions', () => {
244+
it('keeps a band line whose text also occurs in body positions', async () => {
200245
const pages = [1, 2, 3, 4].map((i) => ({
201246
lines: [
202247
line('Quarter Revenue Margin', 760),
@@ -206,29 +251,29 @@ describe('suppressFurniture', () => {
206251
pageHeight: PAGE_HEIGHT,
207252
}))
208253

209-
for (const kept of texts(suppressFurniture(pages))) {
254+
for (const kept of texts(await suppressFurniture(pages))) {
210255
expect(kept).toContain('Quarter Revenue Margin')
211256
}
212257
})
213258

214-
it('never drops a band line that would leave a hyphenated word orphaned', () => {
259+
it('never drops a band line that would leave a hyphenated word orphaned', async () => {
215260
const pages = [1, 2, 3, 4].map(() => ({
216261
lines: [line('Married filing sepa-', 760), ...body(40, 660)],
217262
pageHeight: PAGE_HEIGHT,
218263
}))
219264

220-
for (const kept of texts(suppressFurniture(pages))) {
265+
for (const kept of texts(await suppressFurniture(pages))) {
221266
expect(kept).toContain('Married filing sepa-')
222267
}
223268
})
224269

225-
it('leaves body text alone even when it repeats', () => {
270+
it('leaves body text alone even when it repeats', async () => {
226271
const pages = [1, 2, 3, 4].map(() => ({
227272
lines: [line('Repeated body sentence.', 400)],
228273
pageHeight: PAGE_HEIGHT,
229274
}))
230275

231-
for (const kept of texts(suppressFurniture(pages))) {
276+
for (const kept of texts(await suppressFurniture(pages))) {
232277
expect(kept).toEqual(['Repeated body sentence.'])
233278
}
234279
})

0 commit comments

Comments
 (0)