From a37a0423a12267589ad400fab14aeaad0200b28c Mon Sep 17 00:00:00 2001 From: baseballyama Date: Sun, 11 Oct 2026 15:43:17 +0900 Subject: [PATCH 1/2] perf(streaming): speed up iterRows on wide sheets of styled empty cells (#302) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A sheet whose rows carry hundreds of `` cells is nearly all SAX events and almost no values, so the per-event cost of the row walk dominated: each event was allocated as an object with Clark-notation names and attributes and handed through an async generator step. The walk now registers its handlers on saxes directly and only crosses an async boundary once per 64K feed, which brings it under loadWorkbook's time for the same bytes instead of 1.5–3x over it. The saxes driver moves to an internal module so no public declaration imports saxes' types, which do not type-check under skipLibCheck: false. coordinateToTuple also reads the plain "AB12" shape off char codes ahead of the regex, which both readers call once per cell. Closes #302 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0142ArZjzXh8UP4dVS623qG3 --- .changeset/fast-styled-empty-cells.md | 5 + src/streaming/read-only.ts | 233 +++++++++++++----------- src/utils/coordinate.ts | 19 ++ src/xml/iterparse.ts | 191 +------------------ src/xml/sax-feed.ts | 187 +++++++++++++++++++ tests/perf/iter-rows-throughput.test.ts | 96 ++++++++++ 6 files changed, 444 insertions(+), 287 deletions(-) create mode 100644 .changeset/fast-styled-empty-cells.md create mode 100644 src/xml/sax-feed.ts create mode 100644 tests/perf/iter-rows-throughput.test.ts diff --git a/.changeset/fast-styled-empty-cells.md b/.changeset/fast-styled-empty-cells.md new file mode 100644 index 00000000..bc8f8471 --- /dev/null +++ b/.changeset/fast-styled-empty-cells.md @@ -0,0 +1,5 @@ +--- +'@office-kit/xlsx': patch +--- + +perf: `loadWorkbookStream`'s `iterRows` / `iterValues` no longer slow down on wide sheets full of styled empty cells (``), the shape many business exports have. The row walk now drives the XML parser directly instead of handing each SAX event through an async iterator, which makes it several times faster on such sheets; cell references are also parsed without a regex, which speeds up `loadWorkbook` too (#302) diff --git a/src/streaming/read-only.ts b/src/streaming/read-only.ts index 611a81c9..4a9b3d89 100644 --- a/src/streaming/read-only.ts +++ b/src/streaming/read-only.ts @@ -3,7 +3,7 @@ // `loadWorkbookStream` opens the zip + parses workbook.xml / sharedStrings.xml // / styles.xml metadata up front (small even on million-row archives), then // exposes a lazy `openWorksheet(name)` that SAX-iterates the sheet body via -// `iterParse`. The iterator streams rows without materialising the full sheet +// `feedSax`. The iterator streams rows without materialising the full sheet // in memory. import { normalizeStrictArchive } from '../io/strict.js'; @@ -14,7 +14,7 @@ import { type SharedStringEntry, type SharedStringsTable, } from '../workbook/shared-strings.js'; -import { ARC_CONTENT_TYPES, ARC_ROOT_RELS, localNameOf } from '../xml/namespaces.js'; +import { ARC_CONTENT_TYPES, ARC_ROOT_RELS, qname } from '../xml/namespaces.js'; import { findById, findByType, makeRelationships, relsFromBytes } from '../packaging/relationships.js'; import { manifestFromBytes } from '../packaging/manifest.js'; import { parseCellNumber } from '../utils/cell-number.js'; @@ -35,7 +35,9 @@ import { parseCellDate } from '../utils/cell-date.js'; import { parseCellErrorCode } from '../utils/cell-error.js'; import { unknownCellType } from '../utils/cell-text.js'; import { parseXsdBoolean } from '../utils/xsd-boolean.js'; -import { iterParse, type SaxEvent, type SaxInput } from '../xml/iterparse.js'; +import { SaxesParser } from 'saxes'; +import type { SaxInput } from '../xml/iterparse.js'; +import { buildAttrsClark, feedSax } from '../xml/sax-feed.js'; import { el, type XmlNode } from '../xml/tree.js'; import { resolveRelTarget } from '../packaging/part-name.js'; import { parseXml } from '../xml/parser.js'; @@ -182,7 +184,6 @@ async function* iterSheetRows( const minCol = opts.minCol ?? 1; const maxCol = opts.maxCol ?? Number.POSITIVE_INFINITY; - let inSheetData = false; // -1: outside a . 0: inside one that carried no `@r`, whose number the // first located cell settles. `nextRow` is the high-water mark a row with // neither an `@r` nor a located cell falls back to. @@ -223,112 +224,126 @@ async function* iterSheetRows( const isStack: XmlNode[] = []; let checkedRoot = false; - for await (const ev of iterParse(sheetInput)) { - const e = ev as SaxEvent; - if (e.kind === 'start') { - if (!checkedRoot) { - assertNotStrictRoot(e.name); - checkedRoot = true; - } - // Inside ``, every descendant belongs to the subtree, so this runs - // ahead of the local-name switch rather than through it. - if (isStack.length > 0) { - const child = el(e.name, { ...e.attrs }); - isStack[isStack.length - 1]?.children.push(child); - isStack.push(child); - continue; - } - const local = localNameOf(e.name); - if (!inSheetData) { - if (local === 'sheetData') inSheetData = true; - continue; - } - if (local === 'is' && cellOpen) { - isNode = el(e.name); - isStack.push(isNode); - continue; + let inSheetData = false; + // Set once `` closes or the band is passed. The rest of the + // current feed still runs through saxes, but contributes nothing. + let finished = false; + // Rows completed by the current feed. The handlers below run synchronously + // inside saxes, one feed at a time, so a whole-sheet pass costs one async + // step per feed rather than one per SAX event. On a wide sheet of styled + // empty cells the per-event path is what dominated (#302). + const ready: ReadOnlyCell[][] = []; + + const parser = new SaxesParser({ xmlns: true }); + parser.on('opentag', (node) => { + if (finished) return; + if (!checkedRoot) { + assertNotStrictRoot(qname(node.uri, node.local)); + checkedRoot = true; + } + // Inside ``, every descendant belongs to the subtree, so this runs + // ahead of the local-name switch rather than through it. + if (isStack.length > 0) { + const child = el(qname(node.uri, node.local), buildAttrsClark(node.attributes)); + isStack[isStack.length - 1]?.children.push(child); + isStack.push(child); + return; + } + const local = node.local; + if (!inSheetData) { + if (local === 'sheetData') inSheetData = true; + return; + } + if (local === 'is' && cellOpen) { + isNode = el(qname(node.uri, node.local)); + isStack.push(isNode); + return; + } + switch (local) { + case 'row': { + // `@r` is optional on CT_Row (ECMA-376 §18.3.1.73); such a row waits + // for its first cell to name it. + const rRaw = node.attributes['r']?.value; + if (rRaw === undefined) { + chargeRow(budget, title, undefined); + currentRow = 0; + } else { + currentRow = rowNumberFromAttr(rRaw, 'loadWorkbookStream'); + nextRow = Math.max(nextRow, currentRow + 1); + // Charged here rather than at `` so a row past the cap is + // refused before its cells are decoded. Unnumbered rows are + // charged at their opening tag too, before buffering any cells. + chargeRow(budget, title, currentRow); + } + currentCells = []; + pendingCells = []; + nextCol = 1; + break; } - switch (local) { - case 'row': { - // `@r` is optional on CT_Row (ECMA-376 §18.3.1.73); such a row waits - // for its first cell to name it. - const rRaw = e.attrs['r']; - if (rRaw === undefined) { - chargeRow(budget, title, undefined); - currentRow = 0; - } else { - currentRow = rowNumberFromAttr(rRaw, 'loadWorkbookStream'); - nextRow = Math.max(nextRow, currentRow + 1); - // Charged here rather than at `` so a row past the cap is - // refused before its cells are decoded. Unnumbered rows are - // charged at their opening tag too, before buffering any cells. - chargeRow(budget, title, currentRow); - } - currentCells = []; - pendingCells = []; - nextCol = 1; - break; + case 'c': { + if (currentRow < 0) break; + const ref = node.attributes['r']?.value; + if (currentRow === 0 && ref) settleRow(coordinateToTuple(ref).row); + // Skip cell-attr parsing entirely when the row is outside the + // requested band — saves the parseInt + coordinateToTuple hit on + // every cell of every excluded row. + if (currentRow !== 0 && (currentRow < minRow || currentRow > maxRow)) break; + cellType = node.attributes['t']?.value ?? 'n'; + const sRaw = node.attributes['s']?.value; + cellStyleId = sRaw ? Number.parseInt(sRaw, 10) || 0 : 0; + if (ref) { + const tup = coordinateToTuple(ref); + cellRow = tup.row; + cellCol = tup.col; + } else { + cellRow = currentRow; + cellCol = nextCol; } - case 'c': { - if (currentRow < 0) break; - const ref = e.attrs['r']; - if (currentRow === 0 && ref) settleRow(coordinateToTuple(ref).row); - // Skip cell-attr parsing entirely when the row is outside the - // requested band — saves the parseInt + coordinateToTuple hit on - // every cell of every excluded row. - if (currentRow !== 0 && (currentRow < minRow || currentRow > maxRow)) break; - cellType = e.attrs['t'] ?? 'n'; - const sRaw = e.attrs['s']; - cellStyleId = sRaw ? Number.parseInt(sRaw, 10) || 0 : 0; - if (ref) { - const tup = coordinateToTuple(ref); - cellRow = tup.row; - cellCol = tup.col; - } else { - cellRow = currentRow; - cellCol = nextCol; - } - nextCol = cellCol + 1; - cellOpen = cellCol >= minCol && cellCol <= maxCol; - if (cellOpen) { - // An unresolved row might be selected later, so its buffered cells - // consume budget even if its eventual number is outside the band. - chargeCell(budget, title, cellCol, cellRow === 0 ? undefined : cellRow); - } - vText = ''; - isNode = undefined; - isStack.length = 0; - break; + nextCol = cellCol + 1; + cellOpen = cellCol >= minCol && cellCol <= maxCol; + if (cellOpen) { + // An unresolved row might be selected later, so its buffered cells + // consume budget even if its eventual number is outside the band. + chargeCell(budget, title, cellCol, cellRow === 0 ? undefined : cellRow); } - case 'v': - if (cellOpen) inV = true; - break; - default: - break; + vText = ''; + isNode = undefined; + isStack.length = 0; + break; } - continue; + case 'v': + if (cellOpen) inV = true; + break; + default: + break; } - if (e.kind === 'text') { - if (inV) vText += e.text; - else if (isStack.length > 0) { - // Text events for one element accumulate before `parseRichString` sees - // them, so a chunk boundary inside a `` cannot split an escape - // sequence, and adjacent runs cannot join into one. - const top = isStack[isStack.length - 1]; - if (top !== undefined) top.text = (top.text ?? '') + e.text; - } - continue; + }); + + const onText = (text: string): void => { + if (finished) return; + if (inV) vText += text; + else if (isStack.length > 0) { + // Text events for one element accumulate before `parseRichString` sees + // them, so a chunk boundary inside a `` cannot split an escape + // sequence, and adjacent runs cannot join into one. + const top = isStack[isStack.length - 1]; + if (top !== undefined) top.text = (top.text ?? '') + text; } - // end + }; + parser.on('text', onText); + // A CDATA section is character data like any other. + parser.on('cdata', onText); + + parser.on('closetag', (node) => { + if (finished) return; if (isStack.length > 0) { isStack.pop(); - continue; + return; } - const local = localNameOf(e.name); - if (!inSheetData) continue; - switch (local) { + if (!inSheetData) return; + switch (node.local) { case 'sheetData': - inSheetData = false; + finished = true; return; case 'row': { // A row that held no located cell still consumes a slot, so settle it @@ -337,13 +352,13 @@ async function* iterSheetRows( settleRow(derivedRowNumber(nextRow, 'loadWorkbookStream')); } if (currentRow >= minRow && currentRow <= maxRow && currentCells.length > 0) { - yield currentCells; + ready.push(currentCells); } // Once we've crossed maxRow there are no more rows to yield — every // subsequent would just be parsed and dropped. Stop iterating // early. ECMA-376 emits rows in ascending order. if (currentRow > maxRow) { - inSheetData = false; + finished = true; return; } currentRow = -1; @@ -367,6 +382,20 @@ async function* iterSheetRows( default: break; } + }); + + try { + for await (const _ of feedSax(sheetInput, parser)) { + yield* ready; + ready.length = 0; + if (finished) return; + } + } catch (err) { + // A feed that failed part-way still completed the rows ahead of the + // failure; a caller reading up to a content limit or a corrupt cell gets + // them before the error, as it would from a walk one event at a time. + yield* ready; + throw err; } } diff --git a/src/utils/coordinate.ts b/src/utils/coordinate.ts index 240da5d9..25a72329 100644 --- a/src/utils/coordinate.ts +++ b/src/utils/coordinate.ts @@ -131,6 +131,25 @@ export function coordinateFromString(coord: string): CellCoordinate { * numeric index. Thin convenience for the worksheet read path. */ export function coordinateToTuple(coord: string): CellCoordinateNumeric { + // Every `` a reader walks comes through here, almost always as plain + // upper-case "AB12", so that shape is read off the char codes without the + // regex, the upper-casing and the cache lookup. Anything else, including an + // out-of-range reference, takes the general path and its errors. + let i = 0; + let col = 0; + for (let c = coord.charCodeAt(0); c >= 65 /* A */ && c <= 90 /* Z */ && i < 3; c = coord.charCodeAt(++i)) { + col = col * 26 + (c - 64); + } + const lead = coord.charCodeAt(i); + if (i > 0 && col <= MAX_COL && lead >= 49 /* 1 */ && lead <= 57 /* 9 */ && coord.length - i <= 7) { + let row = 0; + for (; i < coord.length; i++) { + const d = coord.charCodeAt(i); + if (d < 48 /* 0 */ || d > 57 /* 9 */) break; + row = row * 10 + (d - 48); + } + if (i === coord.length && row <= MAX_ROW) return { col, row }; + } const c = coordinateFromString(coord); return { col: columnIndexFromLetter(c.column), row: c.row }; } diff --git a/src/xml/iterparse.ts b/src/xml/iterparse.ts index 83eb44b4..f6eadb13 100644 --- a/src/xml/iterparse.ts +++ b/src/xml/iterparse.ts @@ -14,8 +14,8 @@ // the prescan reads. import { SaxesParser } from 'saxes'; -import { OpenXmlSchemaError } from '../utils/exceptions.js'; import { qname } from './namespaces.js'; +import { buildAttrsClark, feedSax } from './sax-feed.js'; export type SaxEvent = | { kind: 'start'; name: string; attrs: Record } @@ -29,169 +29,13 @@ export type SaxEvent = */ export type SaxInput = Uint8Array | string | ReadableStream; -/** - * Recognize declarations only in the prologue, skipping comments and processing - * instructions. State spans chunks, but retained text is at most one keyword or - * terminator. Once a root tag begins, saxes handles all remaining syntax checks. - */ -const makeDtdScanner = (): ((chunk: string) => void) => { - let state: 'prologue' | 'markup' | 'comment' | 'pi' | 'done' = 'prologue'; - let token = ''; - return (chunk) => { - for (const char of chunk) { - if (state === 'done') return; - if (state === 'comment' || state === 'pi') { - const terminator = state === 'comment' ? '-->' : '?>'; - token = (token + char).slice(-terminator.length); - if (token === terminator) { - state = 'prologue'; - token = ''; - } - } else if (state === 'prologue') { - if (char === '<') { - state = 'markup'; - token = char; - } - } else { - token += char; - if (token === '' : '?>'; + token = (token + char).slice(-terminator.length); + if (token === terminator) { + state = 'prologue'; + token = ''; + } + } else if (state === 'prologue') { + if (char === '<') { + state = 'markup'; + token = char; + } + } else { + token += char; + if (token === '