From ba2fe7519e32a053a0c3a5c47dbbb085c7e476da Mon Sep 17 00:00:00 2001 From: Caio Pizzol Date: Wed, 12 Aug 2026 13:46:49 -0300 Subject: [PATCH 1/5] fix(spec): open results on the correct PDF page Co-authored-by: Ben Lewis --- apps/web/src/components/PdfViewer.tsx | 27 ++------------ apps/web/src/components/pdfNavigation.ts | 45 ++++++++++++++++++++++++ tests/web/pdf-navigation.test.ts | 32 +++++++++++++++++ 3 files changed, 79 insertions(+), 25 deletions(-) create mode 100644 apps/web/src/components/pdfNavigation.ts create mode 100644 tests/web/pdf-navigation.test.ts diff --git a/apps/web/src/components/PdfViewer.tsx b/apps/web/src/components/PdfViewer.tsx index 2eaa463..933a7d8 100644 --- a/apps/web/src/components/PdfViewer.tsx +++ b/apps/web/src/components/PdfViewer.tsx @@ -1,28 +1,5 @@ import { useCallback, useEffect, useRef, useState } from "react"; - -// PDF URLs and page counts for each part -const PDF_CONFIG: Record = { - 1: { - url: "https://cdn.ooxml.dev/ecma-376/part1.pdf", - totalPages: 5560, - name: "Fundamentals", - }, - 2: { - url: "https://cdn.ooxml.dev/ecma-376/part2.pdf", - totalPages: 129, - name: "OPC", - }, - 3: { - url: "https://cdn.ooxml.dev/ecma-376/part3.pdf", - totalPages: 65, - name: "Compatibility", - }, - 4: { - url: "https://cdn.ooxml.dev/ecma-376/part4.pdf", - totalPages: 4031, - name: "Transitional", - }, -}; +import { PDF_CONFIG, pdfUrlForPrintedPage } from "./pdfNavigation"; interface PdfViewerProps { partNumber: number; @@ -119,7 +96,7 @@ export function PdfViewer({ partNumber, pageNumber, onPageChange }: PdfViewerPro }, [config.totalPages, onPageChange]); const progressPercent = (currentPage / config.totalPages) * 100; - const pdfUrl = `${config.url}#page=${currentPage}&toolbar=0&navpanes=0`; + const pdfUrl = pdfUrlForPrintedPage(currentPage, config); return (
diff --git a/apps/web/src/components/pdfNavigation.ts b/apps/web/src/components/pdfNavigation.ts new file mode 100644 index 0000000..354c4c7 --- /dev/null +++ b/apps/web/src/components/pdfNavigation.ts @@ -0,0 +1,45 @@ +export interface PdfConfig { + url: string; + totalPages: number; + pageOffset: number; + name: string; +} + +// Search results use the page number printed in the spec, while browser PDF +// fragments address physical sheets, including unnumbered front matter. +export const PDF_CONFIG: Record = { + 1: { + url: "https://cdn.ooxml.dev/ecma-376/part1.pdf", + totalPages: 5016, + pageOffset: 10, + name: "Fundamentals", + }, + 2: { + url: "https://cdn.ooxml.dev/ecma-376/part2.pdf", + totalPages: 129, + pageOffset: 8, + name: "OPC", + }, + 3: { + url: "https://cdn.ooxml.dev/ecma-376/part3.pdf", + totalPages: 38, + pageOffset: 6, + name: "Compatibility", + }, + 4: { + url: "https://cdn.ooxml.dev/ecma-376/part4.pdf", + totalPages: 1534, + pageOffset: 14, + name: "Transitional", + }, +}; + +export function printedPageToSheet(printedPage: number, config: PdfConfig): number { + const clampedPage = Math.max(1, Math.min(printedPage, config.totalPages)); + return clampedPage + config.pageOffset; +} + +export function pdfUrlForPrintedPage(printedPage: number, config: PdfConfig): string { + const sheet = printedPageToSheet(printedPage, config); + return `${config.url}#page=${sheet}&toolbar=0&navpanes=0`; +} diff --git a/tests/web/pdf-navigation.test.ts b/tests/web/pdf-navigation.test.ts new file mode 100644 index 0000000..ce234dd --- /dev/null +++ b/tests/web/pdf-navigation.test.ts @@ -0,0 +1,32 @@ +import { describe, expect, test } from "bun:test"; +import { + PDF_CONFIG, + pdfUrlForPrintedPage, + printedPageToSheet, +} from "../../apps/web/src/components/pdfNavigation"; + +describe("printedPageToSheet", () => { + test("uses the printed page counts of the served PDFs", () => { + expect(PDF_CONFIG[1].totalPages).toBe(5016); + expect(PDF_CONFIG[2].totalPages).toBe(129); + expect(PDF_CONFIG[3].totalPages).toBe(38); + expect(PDF_CONFIG[4].totalPages).toBe(1534); + }); + + test("adds each PDF's front-matter offset", () => { + expect(printedPageToSheet(1, PDF_CONFIG[1])).toBe(11); + expect(printedPageToSheet(1, PDF_CONFIG[2])).toBe(9); + expect(printedPageToSheet(1, PDF_CONFIG[3])).toBe(7); + expect(printedPageToSheet(1, PDF_CONFIG[4])).toBe(15); + }); + + test("does not navigate past the last sheet", () => { + expect(printedPageToSheet(10_000, PDF_CONFIG[3])).toBe(44); + }); + + test("builds the browser URL with the physical sheet", () => { + expect(pdfUrlForPrintedPage(238, PDF_CONFIG[1])).toBe( + "https://cdn.ooxml.dev/ecma-376/part1.pdf#page=248&toolbar=0&navpanes=0", + ); + }); +}); From 674906786f4a5f83d5d6db58262db8d19a237c0a Mon Sep 17 00:00:00 2001 From: Caio Pizzol Date: Wed, 12 Aug 2026 15:03:37 -0300 Subject: [PATCH 2/5] fix(spec): rebuild page-aware PDF corpus Co-authored-by: Ben Lewis --- .github/workflows/ci.yml | 3 + .gitignore | 2 + data/sources.json | 32 +- package.json | 2 + packages/shared/src/db/index.ts | 60 +++- packages/shared/src/types/index.ts | 1 + scripts/ingest-pdf/README.md | 33 ++- scripts/ingest-pdf/audit.ts | 60 ++++ scripts/ingest-pdf/chunk.ts | 141 ++++++--- scripts/ingest-pdf/embed.ts | 42 +-- scripts/ingest-pdf/extract.py | 386 +++++++++++++++++++------ scripts/ingest-pdf/fix-page-numbers.py | 158 ---------- scripts/ingest-pdf/pipeline.ts | 37 +-- scripts/ingest-pdf/upload.ts | 50 ++-- scripts/requirements.txt | 4 +- tests/db/spec-content-replace.test.ts | 68 +++++ tests/ingest-pdf/chunk-pages.test.ts | 22 ++ tests/ingest-pdf/corpus-audit.test.ts | 19 ++ tests/ingest-pdf/test_headings.py | 205 +++++++++++++ tests/web/pdf-navigation.test.ts | 16 +- 20 files changed, 932 insertions(+), 409 deletions(-) create mode 100644 scripts/ingest-pdf/audit.ts delete mode 100644 scripts/ingest-pdf/fix-page-numbers.py create mode 100644 tests/db/spec-content-replace.test.ts create mode 100644 tests/ingest-pdf/chunk-pages.test.ts create mode 100644 tests/ingest-pdf/corpus-audit.test.ts create mode 100644 tests/ingest-pdf/test_headings.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 44b48ac..1ab5afb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,5 +26,8 @@ jobs: - name: Migration runner tests run: bun test tests/scripts/db-migrate.test.ts + - name: PDF ingest tests + run: bun run pdf:test + - name: Build run: bun run build diff --git a/.gitignore b/.gitignore index ec0c48e..b8bc912 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,8 @@ node_modules/ dist/ .DS_Store +__pycache__/ +*.py[cod] dev/ .wrangler/ .env* diff --git a/data/sources.json b/data/sources.json index 4503849..4710df8 100644 --- a/data/sources.json +++ b/data/sources.json @@ -4,38 +4,38 @@ { "name": "ecma-376-part1", "kind": "spec_pdf", - "edition": "5th", - "version": "2016-12", - "url": "https://ecma-international.org/wp-content/uploads/ECMA-376-1_5th_edition_december_2016.zip", + "edition": "4th", + "version": "2012-12", + "url": "https://cdn.ooxml.dev/ecma-376/part1.pdf", "license_note": "Published by Ecma International. See the ECMA-376 publications page for the current download and licensing terms before redistribution.", - "sha256": "9d0bcad9cf06054785b03762fcfadbf6bab7e54a5f9d69434e34b7fd464d4129" + "sha256": "4a9d481c74daeab4068408dae354e8b6f808771a1b9810ea69482344ce6ccd65" }, { "name": "ecma-376-part2", "kind": "spec_pdf", - "edition": "5th", - "version": "2021-12", - "url": "https://ecma-international.org/wp-content/uploads/ECMA-376-2_5th_edition_december_2021.zip", + "edition": "4th", + "version": "2012-12", + "url": "https://cdn.ooxml.dev/ecma-376/part2.pdf", "license_note": "Published by Ecma International. See the ECMA-376 publications page for the current download and licensing terms before redistribution.", - "sha256": "1d489dc491168ea1f9e9a59063acc8dd5f02b4ad1d21aa7ec19ba9a58d020c70" + "sha256": "1256d9d704af65b8dabfcf9e67770c0294d256387e1945ddba69fb453d174f55" }, { "name": "ecma-376-part3", "kind": "spec_pdf", - "edition": "5th", - "version": "2015-12", - "url": "https://ecma-international.org/wp-content/uploads/ECMA-376-3_5th_edition_december_2015.zip", + "edition": "4th", + "version": "2012-12", + "url": "https://cdn.ooxml.dev/ecma-376/part3.pdf", "license_note": "Published by Ecma International. See the ECMA-376 publications page for the current download and licensing terms before redistribution.", - "sha256": "42294159fbbbe9393ccadac95b859d7729cc68d908898bcbe31034dda059daa8" + "sha256": "e4e58be15925162baeb8fd3aae51381c8c6995e421c4e460b950c057861a11c6" }, { "name": "ecma-376-part4", "kind": "spec_pdf", - "edition": "5th", - "version": "2016-12", - "url": "https://ecma-international.org/wp-content/uploads/ECMA-376-4_5th_edition_december_2016.zip", + "edition": "4th", + "version": "2012-12", + "url": "https://cdn.ooxml.dev/ecma-376/part4.pdf", "license_note": "Published by Ecma International. See the ECMA-376 publications page for the current download and licensing terms before redistribution.", - "sha256": "bd25da1109f73762356596918bf5ff8b74a1331642dba5f1c1d1dfc6bed34ecd" + "sha256": "46a34fd930801e69ad5150996ca7671c8137efa4fd4bca3dcd30829594f65e02" }, { "name": "ecma-376-transitional", diff --git a/package.json b/package.json index 7f6f970..fdcfb59 100644 --- a/package.json +++ b/package.json @@ -28,9 +28,11 @@ "sources:sync": "bun scripts/sources-sync.ts", "pdf:ingest": "bun scripts/ingest-pdf/pipeline.ts", "pdf:chunk": "bun scripts/ingest-pdf/chunk.ts", + "pdf:audit": "bun scripts/ingest-pdf/audit.ts", "pdf:embed": "bun scripts/ingest-pdf/embed.ts", "pdf:upload": "bun scripts/ingest-pdf/upload.ts", "pdf:setup": "pip install -r scripts/requirements.txt", + "pdf:test": "python3 -m unittest discover -s tests/ingest-pdf && bun test tests/ingest-pdf/*.test.ts", "xsd:fetch": "bun scripts/ingest-xsd/fetch.ts", "xsd:ingest": "bun scripts/ingest-xsd/ingest.ts", "test": "export TEST_DATABASE_URL=${TEST_DATABASE_URL:-postgresql://postgres:postgres@localhost:5432/ecma_spec} && bun test tests/web/ && bun test tests/db/ && bun test tests/ingest-xsd/ && bun test tests/mcp-server/" diff --git a/packages/shared/src/db/index.ts b/packages/shared/src/db/index.ts index f00aae0..3a99bae 100644 --- a/packages/shared/src/db/index.ts +++ b/packages/shared/src/db/index.ts @@ -3,6 +3,19 @@ import type { SearchResult, SpecContent } from "../types"; export type DbClient = ReturnType; +function specContentRow(item: Omit) { + return { + part_number: item.partNumber, + section_id: item.sectionId, + title: item.title, + content: item.content, + content_type: item.contentType, + page_number: item.pageNumber, + embedding: item.embedding ? `[${item.embedding.join(",")}]` : null, + source_id: item.sourceId ?? null, + }; +} + export function createDbClient(connectionString: string) { const sql = postgres(connectionString); @@ -16,7 +29,7 @@ export function createDbClient(connectionString: string) { // Insert content async insert(content: Omit) { const [result] = await sql<[{ id: number }]>` - INSERT INTO spec_content (part_number, section_id, title, content, content_type, page_number, embedding) + INSERT INTO spec_content (part_number, section_id, title, content, content_type, page_number, embedding, source_id) VALUES ( ${content.partNumber}, ${content.sectionId}, @@ -24,7 +37,8 @@ export function createDbClient(connectionString: string) { ${content.content}, ${content.contentType}, ${content.pageNumber}, - ${content.embedding ? `[${content.embedding.join(",")}]` : null} + ${content.embedding ? `[${content.embedding.join(",")}]` : null}, + ${content.sourceId ?? null} ) RETURNING id `; @@ -33,15 +47,7 @@ export function createDbClient(connectionString: string) { // Insert multiple (batch) async insertBatch(items: Omit[]) { - const values = items.map((item) => ({ - part_number: item.partNumber, - section_id: item.sectionId, - title: item.title, - content: item.content, - content_type: item.contentType, - page_number: item.pageNumber, - embedding: item.embedding ? `[${item.embedding.join(",")}]` : null, - })); + const values = items.map(specContentRow); const result = await sql` INSERT INTO spec_content ${sql(values)} @@ -50,6 +56,38 @@ export function createDbClient(connectionString: string) { return result.map((r) => r.id as number); }, + async replacePart( + partNumber: number, + items: Omit[], + options: { batchSize?: number; onProgress?: (inserted: number) => void } = {}, + ): Promise<{ deleted: number; inserted: number }> { + if (items.length === 0) { + throw new Error(`Refusing to replace Part ${partNumber} with no content`); + } + if (items.some((item) => item.partNumber !== partNumber)) { + throw new Error(`Replacement content must all belong to Part ${partNumber}`); + } + + const { batchSize = 50, onProgress } = options; + return sql.begin(async (tx) => { + // postgres.js transaction handles are callable at runtime, but its + // TransactionSql type drops the call signature through Omit. + const transaction = tx as unknown as typeof sql; + const deleted = + await transaction`DELETE FROM spec_content WHERE part_number = ${partNumber}`; + let inserted = 0; + + for (let index = 0; index < items.length; index += batchSize) { + const batch = items.slice(index, index + batchSize).map(specContentRow); + await transaction`INSERT INTO spec_content ${transaction(batch)}`; + inserted += batch.length; + onProgress?.(inserted); + } + + return { deleted: deleted.count, inserted }; + }); + }, + // Update embedding async updateEmbedding(id: number, embedding: number[]) { await sql` diff --git a/packages/shared/src/types/index.ts b/packages/shared/src/types/index.ts index 2753776..993ee0e 100644 --- a/packages/shared/src/types/index.ts +++ b/packages/shared/src/types/index.ts @@ -8,6 +8,7 @@ export interface SpecContent { content: string; contentType: string; pageNumber: number | null; + sourceId?: number | null; embedding?: number[]; } diff --git a/scripts/ingest-pdf/README.md b/scripts/ingest-pdf/README.md index a14cb5e..608eb86 100644 --- a/scripts/ingest-pdf/README.md +++ b/scripts/ingest-pdf/README.md @@ -2,8 +2,8 @@ Builds the prose-search corpus that powers `ooxml_search` / `ooxml_section` / `ooxml_parts`. Each ECMA-376 part PDF is extracted into -section-aware markdown, chunked at ~6 KB boundaries, embedded with the -configured provider, and uploaded into `spec_content`. +section-aware markdown, chunked at ~6 KB boundaries, embedded with Voyage, +and uploaded into `spec_content`. ``` PDF -> extract (Python) -> chunk (6KB, section-aware) -> embed -> upload @@ -13,11 +13,8 @@ PDF -> extract (Python) -> chunk (6KB, section-aware) -> embed -> upload - Python with `pymupdf4llm`: `bun run pdf:setup` - `DATABASE_URL` pointed at a Postgres with `db/schema.sql` applied -- An embedding provider key (one of): - - `OPENAI_API_KEY` (default) - - `VOYAGE_API_KEY` - - `GOOGLE_API_KEY` - - `COHERE_API_KEY` +- `VOYAGE_API_KEY`. Search queries use `voyage-3`, so the corpus must use the + same model and 1,024 dimensions. ## Run the full pipeline @@ -28,14 +25,20 @@ bun run pdf:ingest 3 ./pdfs/ECMA-376-Part3.pdf bun run pdf:ingest 4 ./pdfs/ECMA-376-Part4.pdf ``` -Each run extracts to `dev/data/extracted/partN/`, chunks to -`dev/data/chunks/partN-chunks.json`, embeds to -`dev/data/embedded/partN-embedded.json`, then uploads. +Each run extracts to `data/extracted/partN/`, chunks to +`data/chunks/partN-chunks.json`, embeds to +`data/embedded/partN-embedded.json`, then replaces that part in one database +transaction. A failed upload rolls back, and an empty corpus is rejected. + +Extraction records the physical PDF sheet for each page and derives the +printed page from the document's running headers. This keeps long sections on +their actual pages and excludes contents-list entries from the corpus. ## Run individual stages ```bash bun run pdf:chunk ./extracted/part1 ./chunks/part1.json +bun run pdf:audit ./extracted/part1 bun run pdf:embed ./chunks/part1.json ./embedded/part1.json bun run pdf:upload 1 ./embedded/part1.json ``` @@ -46,8 +49,8 @@ without re-extracting. ## Files - `pipeline.ts` - orchestrator (extract -> chunk -> embed -> upload) -- `extract.py` - PDF -> section-aware markdown via pymupdf4llm -- `fix-page-numbers.py` - PDF prelude-aware page-number alignment -- `chunk.ts` - markdown -> 6 KB chunks with section IDs -- `embed.ts` - chunks -> chunks + 1024-dim embeddings -- `upload.ts` - bulk insert into `spec_content` +- `extract.py` - PDF -> page-aware, section-aware markdown via pymupdf4llm +- `chunk.ts` - markdown -> page-aware 6 KB chunks with section IDs +- `audit.ts` - rejects duplicate sections, contents entries, and invalid pages +- `embed.ts` - chunks -> chunks + Voyage embeddings +- `upload.ts` - atomic part replacement in `spec_content` diff --git a/scripts/ingest-pdf/audit.ts b/scripts/ingest-pdf/audit.ts new file mode 100644 index 0000000..1cb6c45 --- /dev/null +++ b/scripts/ingest-pdf/audit.ts @@ -0,0 +1,60 @@ +/** + * Audit an extracted corpus before embedding or upload. + * + * Usage: + * bun scripts/ingest-pdf/audit.ts + */ + +import { type Chunk, chunkSections } from "./chunk"; + +const TOC_TITLE_PATTERN = /\.{2,}|\s\d{1,4}$/; + +export function auditCorpus(chunks: Chunk[], maxPrintedPage: number): void { + if (chunks.length === 0) throw new Error("Corpus contains no chunks"); + + for (const chunk of chunks) { + if (chunk.pageNumber < 1 || chunk.pageNumber > maxPrintedPage) { + throw new Error(`${chunk.sectionId} has invalid printed page ${chunk.pageNumber}`); + } + if (TOC_TITLE_PATTERN.test(chunk.sectionTitle)) { + throw new Error(`${chunk.sectionId} has a contents-shaped title: ${chunk.sectionTitle}`); + } + if (chunk.content.includes("` markers in each section's + * body, where N is the physical page the following text came from. We track + * those markers while walking the content so a chunk records the page it + * actually falls on, not the page its section started on. Markers are stripped + * from the stored content and from the embedding text. + * + * `pageNumber` is the *printed* page (physical minus the part's front-matter + * offset, read from the extraction metadata) - the number printed on the page + * and stored in `spec_content.page_number`. + * * Usage: * bun scripts/ingest-pdf/chunk.ts * @@ -11,7 +21,7 @@ * bun scripts/ingest-pdf/chunk.ts ./extracted/part1 ./chunks/part1-chunks.json */ -interface ExtractedSection { +export interface ExtractedSection { sectionId: string; title: string; pageStart: number; @@ -21,7 +31,7 @@ interface ExtractedSection { parentId: string | null; } -interface Chunk { +export interface Chunk { sectionId: string; sectionTitle: string; content: string; @@ -35,6 +45,9 @@ interface Chunk { const CHUNK_SIZE = 6000; // ~2000-3000 tokens const CHUNK_OVERLAP = 200; +// Page markers written by extract.py (physical page number). +const PAGE_MARKER_PATTERN = //g; + // Markdown code fence pattern (pymupdf4llm outputs code in fences) const CODE_FENCE_PATTERN = /```[\s\S]*?```/g; @@ -64,11 +77,29 @@ function stripForEmbedding(content: string): string { return text.replace(/\n{3,}/g, "\n\n").trim(); } -function splitIntoChunks( +/** + * Pull page markers out of a paragraph. + * + * Returns the paragraph without markers, plus the last physical page seen in + * it (null when the paragraph carried no marker). + */ +function takePageMarkers(paragraph: string): { text: string; physicalPage: number | null } { + let physicalPage: number | null = null; + + const text = paragraph.replace(PAGE_MARKER_PATTERN, (_match, page: string) => { + physicalPage = Number.parseInt(page, 10); + return ""; + }); + + return { text: text.trim(), physicalPage }; +} + +export function splitIntoChunks( text: string, sectionId: string, sectionTitle: string, pageStart: number, + pageOffset: number, ): Chunk[] { const chunks: Chunk[] = []; @@ -76,61 +107,87 @@ function splitIntoChunks( return chunks; } - // Split full content (with code blocks and tables inline) by paragraphs + // Physical page -> printed page. Sections start on a printed page already, + // so fall back to that until the first marker is seen. + const toPrinted = (physical: number) => Math.max(1, physical - pageOffset); + const paragraphs = text.split(/\n\n+/); let currentChunk = ""; - const currentPage = pageStart; + let currentPage = pageStart; + let chunkPage = pageStart; + + const pushChunk = () => { + const content = currentChunk.trim(); + if (!content) return; + + chunks.push({ + sectionId, + sectionTitle, + content, + embeddingText: stripForEmbedding(content), + contentType: "text", + pageNumber: chunkPage, + chunkIndex: chunks.length, + }); + }; for (const para of paragraphs) { - const trimmedPara = para.trim(); + const { text: trimmedPara, physicalPage } = takePageMarkers(para); + + if (physicalPage !== null) { + currentPage = toPrinted(physicalPage); + // A marker that arrives while the chunk is still empty belongs to + // the chunk about to be started. + if (!currentChunk.trim()) { + chunkPage = currentPage; + } + } + if (!trimmedPara) continue; // Check if adding this paragraph exceeds chunk size if (currentChunk.length + trimmedPara.length > CHUNK_SIZE) { - // Save current chunk if it has content - if (currentChunk.trim()) { - const content = currentChunk.trim(); - chunks.push({ - sectionId, - sectionTitle, - content, - embeddingText: stripForEmbedding(content), - contentType: "text", - pageNumber: currentPage, - chunkIndex: chunks.length, - }); - } + pushChunk(); - // Start new chunk with overlap + // Start new chunk with overlap, on whichever page we have reached const overlap = currentChunk.slice(-CHUNK_OVERLAP); currentChunk = `${overlap}\n\n${trimmedPara}`; + chunkPage = currentPage; } else { + if (!currentChunk) { + chunkPage = currentPage; + } currentChunk += (currentChunk ? "\n\n" : "") + trimmedPara; } } - // Don't forget the last chunk - if (currentChunk.trim()) { - const content = currentChunk.trim(); - chunks.push({ - sectionId, - sectionTitle, - content, - embeddingText: stripForEmbedding(content), - contentType: "text", - pageNumber: currentPage, - chunkIndex: chunks.length, - }); - } + pushChunk(); return chunks; } -async function chunkSections(sectionsPath: string): Promise { - const sectionsJson = await Bun.file(sectionsPath).text(); +async function readPageOffset(extractedDir: string): Promise { + try { + const metadata = await Bun.file(`${extractedDir}/metadata.json`).json(); + const offset = metadata?.pageOffset; + if (typeof offset === "number") return offset; + } catch { + // Extraction predating page offsets - fall through. + } + + console.warn( + " No pageOffset in metadata.json; treating extracted pages as printed pages.\n" + + " Re-run extract.py so page numbers line up with the PDF.", + ); + return 0; +} + +export async function chunkSections(extractedDir: string): Promise { + const sectionsJson = await Bun.file(`${extractedDir}/sections.json`).text(); const sections: ExtractedSection[] = JSON.parse(sectionsJson); + const pageOffset = await readPageOffset(extractedDir); - console.log(`Processing ${sections.length} sections...`); + console.log(`Processing ${sections.length} sections (page offset ${pageOffset})...`); const allChunks: Chunk[] = []; @@ -140,6 +197,7 @@ async function chunkSections(sectionsPath: string): Promise { section.sectionId, section.title, section.pageStart, + pageOffset, ); allChunks.push(...chunks); } @@ -159,10 +217,9 @@ async function main() { } const [extractedDir, outputFile] = args; - const sectionsPath = `${extractedDir}/sections.json`; try { - const chunks = await chunkSections(sectionsPath); + const chunks = await chunkSections(extractedDir); // Save chunks await Bun.write(outputFile, JSON.stringify(chunks, null, 2)); @@ -175,15 +232,21 @@ async function main() { const avgEmbedding = Math.round( chunks.reduce((sum, c) => sum + c.embeddingText.length, 0) / chunks.length, ); + const multiPageSections = new Set( + chunks.filter((c) => c.chunkIndex > 0).map((c) => c.sectionId), + ).size; console.log("\nChunk statistics:"); console.log(` Total chunks: ${chunks.length}`); console.log(` Average content size: ${avgContent} chars`); console.log(` Average embedding text size: ${avgEmbedding} chars`); + console.log(` Sections spanning multiple chunks: ${multiPageSections}`); } catch (error) { console.error("Chunking failed:", error); process.exit(1); } } -main(); +if (import.meta.main) { + main(); +} diff --git a/scripts/ingest-pdf/embed.ts b/scripts/ingest-pdf/embed.ts index 8ab7e53..4682151 100644 --- a/scripts/ingest-pdf/embed.ts +++ b/scripts/ingest-pdf/embed.ts @@ -7,17 +7,13 @@ * bun scripts/ingest-pdf/embed.ts * * Environment variables: - * EMBEDDING_PROVIDER - openai, google, voyage, or cohere (default: openai) - * OPENAI_API_KEY / GOOGLE_API_KEY / etc. + * VOYAGE_API_KEY * * Example: - * EMBEDDING_PROVIDER=openai bun scripts/ingest-pdf/embed.ts ./chunks/part1-chunks.json ./embedded/part1-embedded.json + * bun scripts/ingest-pdf/embed.ts ./chunks/part1-chunks.json ./embedded/part1-embedded.json */ -import { - createEmbeddingClient, - type EmbeddingProvider, -} from "../../packages/shared/src/embeddings/index.ts"; +import { createEmbeddingClient } from "../../packages/shared/src/embeddings/index.ts"; interface Chunk { sectionId: string; @@ -33,26 +29,19 @@ interface EmbeddedChunk extends Chunk { embedding: number[]; } -function getApiKey(provider: EmbeddingProvider): string { - const keyMap: Record = { - openai: "OPENAI_API_KEY", - google: "GOOGLE_API_KEY", - voyage: "VOYAGE_API_KEY", - cohere: "COHERE_API_KEY", - }; - - const key = process.env[keyMap[provider]]; +function getApiKey(): string { + const key = process.env.VOYAGE_API_KEY; if (!key) { - throw new Error(`Missing ${keyMap[provider]} environment variable`); + throw new Error("Missing VOYAGE_API_KEY environment variable"); } return key; } -async function embedChunks(chunks: Chunk[], provider: EmbeddingProvider): Promise { - const apiKey = getApiKey(provider); - const client = createEmbeddingClient(provider, { apiKey }); +async function embedChunks(chunks: Chunk[]): Promise { + const apiKey = getApiKey(); + const client = createEmbeddingClient("voyage", { apiKey }); - console.log(`Using ${provider} (${client.model}, ${client.dimensions}d)`); + console.log(`Using Voyage (${client.model}, ${client.dimensions}d)`); console.log(`Embedding ${chunks.length} chunks...`); const embeddedChunks: EmbeddedChunk[] = []; @@ -96,19 +85,14 @@ async function main() { console.log("Usage: bun scripts/ingest-pdf/embed.ts "); console.log(""); console.log("Environment variables:"); - console.log(" EMBEDDING_PROVIDER - openai, google, voyage, or cohere (default: openai)"); - console.log(" OPENAI_API_KEY / GOOGLE_API_KEY / etc."); + console.log(" VOYAGE_API_KEY"); console.log(""); console.log("Example:"); - console.log( - " EMBEDDING_PROVIDER=openai bun scripts/ingest-pdf/embed.ts ./chunks/part1.json ./embedded/part1.json", - ); + console.log(" bun scripts/ingest-pdf/embed.ts ./chunks/part1.json ./embedded/part1.json"); process.exit(1); } const [chunksFile, outputFile] = args; - const provider = (process.env.EMBEDDING_PROVIDER || "openai") as EmbeddingProvider; - try { // Load chunks const chunksJson = await Bun.file(chunksFile).text(); @@ -117,7 +101,7 @@ async function main() { // Generate embeddings const startTime = Date.now(); - const embeddedChunks = await embedChunks(chunks, provider); + const embeddedChunks = await embedChunks(chunks); const duration = (Date.now() - startTime) / 1000; // Save embedded chunks diff --git a/scripts/ingest-pdf/extract.py b/scripts/ingest-pdf/extract.py index 7c895f7..232a715 100644 --- a/scripts/ingest-pdf/extract.py +++ b/scripts/ingest-pdf/extract.py @@ -5,63 +5,135 @@ Extracts text from ECMA-376 PDF files with proper markdown formatting. Produces cleaner output than pdf.js with code fences and table formatting. +Page numbers +------------ +Two numbering systems exist in these PDFs and conflating them is what made +every search result open on the wrong page: + + * physical page - the sheet index a PDF viewer addresses via `#page=N`. + * printed page - the number printed in the running header, which starts + over at 1 after the roman-numeral front matter. + +We extract with `page_chunks=True`, so every line's physical page is known +exactly rather than inferred from stray digits in the text. The printed page +is then derived as `physical - page_offset`, where the offset is measured +from the running headers (see `detect_page_offset`). + +`spec_content.page_number` stores the *printed* page, because that is the +number a reader sees on the page and cites. The web viewer adds the part's +offset back when it builds the `#page=` fragment. + +Section content keeps inline `` markers so the chunker can +attribute each chunk to the page it actually falls on. + Usage: - python scripts/ingest/extract-pdf.py [--pages START-END] + python scripts/ingest-pdf/extract.py [--pages START-END] Example: - python scripts/ingest/extract-pdf.py ./pdfs/ECMA-376-Part1.pdf ./extracted/part1 - python scripts/ingest/extract-pdf.py ./pdfs/ECMA-376-Part1.pdf ./extracted/part1 --pages 100-200 + python scripts/ingest-pdf/extract.py ./pdfs/ECMA-376-Part1.pdf ./extracted/part1 + python scripts/ingest-pdf/extract.py ./pdfs/ECMA-376-Part1.pdf ./extracted/part1 --pages 100-200 """ import sys import json import re import os +from collections import Counter from pathlib import Path +# Inline marker recording the physical page a run of lines came from. +PAGE_MARKER = "" +PAGE_MARKER_RE = re.compile(r"^$") + + +def detect_page_offset(doc) -> int: + """ + Measure how many sheets precede printed page 1. + + Every body page prints its number in the running header, within the first + few lines. `physical - printed` is constant across the document, so we take + the modal vote and ignore pages where no number is found (front matter, + full-bleed figures). + """ + votes: Counter[int] = Counter() + + for index in range(doc.page_count): + printed = printed_page_in_header(doc[index].get_text()) + if printed is not None: + votes[(index + 1) - printed] += 1 + + if not votes: + return 0 + + offset, agreed = votes.most_common(1)[0] + total = sum(votes.values()) + print(f"Page offset: {offset} (agreement {agreed}/{total} pages with a header number)") + + if agreed / total < 0.9: + print(f" WARNING: header page numbers disagree; runners-up {votes.most_common(4)[1:]}") + + return offset + + +def printed_page_in_header(page_text: str) -> int | None: + """Return the page number printed in the running header, if present.""" + lines = [line.strip() for line in page_text.split("\n")[:6] if line.strip()] + + for line in lines[:3]: + if re.fullmatch(r"\d{1,4}", line): + return int(line) + + return None + + def extract_pdf(pdf_path: str, output_dir: str, page_range: tuple[int, int] | None = None): """Extract PDF to markdown using pymupdf4llm.""" import pymupdf4llm - import fitz # pymupdf + import pymupdf print(f"Loading PDF: {pdf_path}") - # Get total page count - doc = fitz.open(pdf_path) - total_pages = len(doc) - doc.close() + doc = pymupdf.open(pdf_path) + total_pages = doc.page_count print(f"PDF loaded: {total_pages} pages") - # Determine pages to process + page_offset = detect_page_offset(doc) + + # Determine pages to process (0-based indices for pymupdf) if page_range: start_page, end_page = page_range pages = list(range(start_page - 1, min(end_page, total_pages))) print(f"Processing pages {start_page} to {min(end_page, total_pages)}") else: - pages = None # Process all pages + pages = list(range(total_pages)) print(f"Processing all {total_pages} pages") - # Extract to markdown + # Extract per page so each line's physical page is known exactly. print("Extracting text...") - md_text = pymupdf4llm.to_markdown( + page_chunks = pymupdf4llm.to_markdown( pdf_path, pages=pages, - show_progress=True + page_chunks=True, + show_progress=True, ) + doc.close() + + md_text = assemble_markdown(page_chunks, pages) + # Create output directory Path(output_dir).mkdir(parents=True, exist_ok=True) - # Save raw markdown + # Save raw markdown (page markers included - they are HTML comments) md_path = Path(output_dir) / "content.md" with open(md_path, "w", encoding="utf-8") as f: f.write(md_text) print(f"Saved markdown to {md_path}") # Parse sections from markdown - sections = parse_sections(md_text, page_range[0] if page_range else 1) + sections = parse_sections(md_text, page_offset) # Save sections sections_path = Path(output_dir) / "sections.json" @@ -87,8 +159,10 @@ def extract_pdf(pdf_path: str, output_dir: str, page_range: tuple[int, int] | No # Save metadata metadata = { "totalPages": total_pages, - "processedPages": len(pages) if pages else total_pages, + "printedPages": max(1, total_pages - page_offset), + "processedPages": len(pages), "pageRange": list(page_range) if page_range else None, + "pageOffset": page_offset, "sectionsFound": len(sections), "contentLength": len(md_text), } @@ -97,107 +171,247 @@ def extract_pdf(pdf_path: str, output_dir: str, page_range: tuple[int, int] | No with open(metadata_path, "w", encoding="utf-8") as f: json.dump(metadata, f, indent=2) - print(f"\nExtraction complete!") + print("\nExtraction complete!") print(f" Total pages: {total_pages}") - print(f" Processed pages: {metadata['processedPages']}") + print(f" Processed pages: {len(pages)}") + print(f" Page offset: {page_offset} (printed page = physical page - {page_offset})") print(f" Sections found: {len(sections)}") print(f" Content size: {len(md_text):,} chars") + print("\n Set pageOffset in apps/web/src/components/pdfNavigation.ts PDF_CONFIG to " + f"{page_offset} and totalPages to {max(1, total_pages - page_offset)} for this part.") return md_text, sections -def parse_sections(md_text: str, start_page: int) -> list[dict]: - """Parse section structure from markdown text.""" +def assemble_markdown(page_chunks, pages: list[int]) -> str: + """Join per-page markdown, prefixing each page with its physical page marker.""" + parts = [] + + for position, chunk in enumerate(page_chunks): + metadata = chunk.get("metadata") or {} + physical = metadata.get("page_number", metadata.get("page")) + + # Fall back to the requested page list if the key is absent or renamed. + if not isinstance(physical, int): + physical = pages[position] + 1 if position < len(pages) else position + 1 + + parts.append(f"\n\n{PAGE_MARKER.format(physical)}\n\n{chunk.get('text', '')}") + + return "".join(parts) + + +# Section heading patterns. Both forms have been emitted by pymupdf4llm across +# versions, so match either rather than depending on the bold styling alone. +HEADING_PATTERNS = [ + # Bold: **12.3.2** **Title** + # + # The title must be bold too. The running header on every page reads + # `**17** . WordprocessingML Reference Material` - an unbolded title - and + # accepting it would turn every page into a bogus section 17. + re.compile(r"^\*\*(\d+(?:\.\d+)*)\*\*\s*\*\*([^*]+)\*\*$"), + # ATX: #### 12.3.2 Title / ### 12.3.2. Title (optionally bolded) + re.compile(r"^#+\s*\*{0,2}(\d+(?:\.\d+)*)\.?\s+(.+?)\*{0,2}$"), +] + +# A real heading's title starts with a word. Running headers and stray +# numbering fragments start with punctuation. +TITLE_STARTS_WITH_WORD_RE = re.compile(r"^[A-Za-z0-9(]") + +# An annex number, its qualifier and its name arrive as one bold run on some +# pymupdf4llm releases and as three on others: +# +# **Annex A** **(normative)** **Namespaces** +# #### **Annex A (normative) Namespaces** +# +# Matching the runs individually meant a new marker arrangement leaked `**` +# into the stored title, so emphasis is stripped before the match instead. The +# title keeps the qualifier: ("Annex A", "(normative) Namespaces"). +ANNEX_PATTERN = re.compile(r"^Annex\s+([A-Z])\b[.:]?\s*(.*)$", re.IGNORECASE) + +# An annex whose name spilled onto the next line carries only its qualifier. +ANNEX_QUALIFIER_ONLY_RE = re.compile(r"\([^)]*\)") + +HEADING_PREFIX_RE = re.compile(r"^#{1,6}\s*") +EMPHASIS_RE = re.compile(r"\*+|__") + +# A table-of-contents entry: leader dots and/or a trailing page number. +# These render bold in some pymupdf4llm versions and so are indistinguishable +# from real headings by styling alone - they must be rejected by shape. +TOC_LEADER_RE = re.compile(r"\.{2,}") +TOC_TRAILING_PAGE_RE = re.compile(r"[.\s]\d{1,4}\s*$") + + +def strip_emphasis(text: str) -> str: + """Drop markdown emphasis markers and collapse the whitespace they leave.""" + return re.sub(r"\s+", " ", EMPHASIS_RE.sub("", text)).strip() + + +def heading_body(stripped: str) -> str: + """A heading line as plain text: no ATX prefix, no emphasis markers.""" + return strip_emphasis(HEADING_PREFIX_RE.sub("", stripped)) + + +def has_heading_markup(stripped: str) -> bool: + """True when a line is styled as a heading (ATX prefix or a bold run).""" + return stripped.startswith("#") or stripped.startswith("**") + + +def looks_like_toc(title: str, raw_line: str) -> bool: + """True when a heading candidate is really a contents-listing entry.""" + if TOC_LEADER_RE.search(raw_line): + return True + + # No leader dots, but still "Title 1047" - a contents entry whose title + # ran long enough to swallow the dots. + return bool(TOC_TRAILING_PAGE_RE.search(title)) + + +def match_heading(stripped: str) -> tuple[str, str] | None: + """Return (section_id, title) for a real section heading, else None.""" + # Markdown table rows are contents listings in these PDFs, never headings. + if stripped.startswith("|"): + return None + + # A heading carries markdown emphasis. Annex pages repeat a bare "Annex A" + # as their running header, which is not a heading. + if not has_heading_markup(stripped): + return None + + annex = ANNEX_PATTERN.match(heading_body(stripped)) + if annex: + title = annex.group(2).strip() + if not title or looks_like_toc(title, stripped): + return None + return f"Annex {annex.group(1).upper()}", title + + for pattern in HEADING_PATTERNS: + match = pattern.match(stripped) + if not match: + continue + + section_id, title = match.group(1), strip_emphasis(match.group(2) or "") + # Real top-level ATX headings use `# 4. Terms...`. Lines such as + # `#4 shrinks...` and numbered RELAX NG examples can otherwise look + # like sections after markdown extraction. + if stripped.startswith("#") and "." not in section_id: + body = heading_body(stripped) + if not re.match(rf"^{re.escape(section_id)}\.\s", body): + return None + if not title or not TITLE_STARTS_WITH_WORD_RE.match(title): + return None + if looks_like_toc(title, stripped): + return None + return section_id, title + + return None + + +def needs_title_continuation(section_id: str, title: str) -> bool: + """ + True when an annex heading carries no name beyond its qualifier. + + `**Annex A** **(normative)**` with `**Namespaces**` on the following line is + one heading split in two, so the name is picked up by `title_continuation`. + """ + if not section_id.lower().startswith("annex"): + return False + + return ANNEX_QUALIFIER_ONLY_RE.fullmatch(title) is not None + + +def title_continuation(stripped: str) -> str | None: + """ + Return the trailing run of a heading whose title spilled onto the next line. + + Only a styled run that doesn't open with a number qualifies, which keeps the + annex body - and the next heading - out of the title. + """ + if not has_heading_markup(stripped) or match_heading(stripped) is not None: + return None + + text = heading_body(stripped) + if not text or not TITLE_STARTS_WITH_WORD_RE.match(text) or text[0].isdigit(): + return None + + return text + + +def parse_sections(md_text: str, page_offset: int = 0) -> list[dict]: + """ + Parse section structure from markdown text. + + `page_offset` converts the physical pages recorded in the page markers into + the printed page numbers stored in the database. + """ sections = [] - # Section patterns for ECMA-376 - # Only match BOLD section headers (actual sections, not TOC entries) - patterns = [ - # Main section with bold: **12.3.2** **Title** - r'^\*\*(\d+(?:\.\d+)*)\*\*\s*\*\*([^*]+)\*\*$', - # Annex with bold: **Annex A** **(normative)** or **Annex A (informative)** - r'^\*\*(Annex\s+[A-Z])\*\*\s*(?:\*\*)?(?:\(([^)]+)\))?(?:\*\*)?\s*(.*)$', - ] - - # TOC pattern to skip (has page number at end: "17.3.2 Title ... 264") - toc_pattern = r'^\d+(?:\.\d+)*\s+.+\.{2,}\s*\d+$' - - # Page number patterns from pymupdf4llm output - # Arabic numerals (main content): standalone line with just digits - arabic_page_pattern = r'^(\d+)$' - # Header/footer line to skip - header_pattern = r'^ECMA-376 Part \d' - - lines = md_text.split('\n') + lines = md_text.split("\n") current_section = None - current_content = [] - current_page = start_page + current_content: list[str] = [] + current_physical = 1 + awaiting_title = False + + def printed_page() -> int: + return max(1, current_physical - page_offset) for line in lines: stripped = line.strip() - # Skip header/footer lines - if re.match(header_pattern, stripped): + marker = PAGE_MARKER_RE.match(stripped) + if marker: + current_physical = int(marker.group(1)) + # Keep the marker in the section body so the chunker can advance + # its page as it walks the content. Pad with blank lines so the + # marker survives as its own paragraph when the body is re-split. + if current_section: + current_content.extend(["", stripped, ""]) continue - # Track page numbers - standalone arabic numerals - if re.match(arabic_page_pattern, stripped): - page_num = int(stripped) - # Sanity check: page should increase or be close to current - if page_num >= current_page and page_num < current_page + 50: - current_page = page_num - continue # Don't include page number in content - - # Skip TOC entries (have page numbers at the end with dots) - if re.match(toc_pattern, stripped): - continue + heading = match_heading(stripped) - # Check for section headers (bold only - actual sections) - section_match = None - for pattern in patterns: - match = re.match(pattern, stripped, re.IGNORECASE) - if match: - section_match = match - break + # An annex name can land on the line after its number and qualifier. + # Only the first non-blank line after such a heading is considered, so + # the annex body can never be mistaken for the rest of the title. + if heading is None and awaiting_title and stripped: + awaiting_title = False + continuation = title_continuation(stripped) + if continuation: + current_section["title"] = f"{current_section['title']} {continuation}".strip() + current_content.append(line) + continue - if section_match: + if heading: # Save previous section if current_section: - current_section["content"] = '\n'.join(current_content).strip() - current_section["pageEnd"] = current_page + 1 # +1 to match TOC + current_section["content"] = "\n".join(current_content).strip() + current_section["pageEnd"] = printed_page() sections.append(current_section) - # Start new section - groups = section_match.groups() - section_id = groups[0] - title = groups[1] if len(groups) > 1 else "" + section_id, title = heading - # Calculate depth - if section_id.startswith("Annex"): + if section_id.lower().startswith("annex"): depth = 1 else: - depth = section_id.count('.') + 1 - - # Get parent ID - parent_id = get_parent_section_id(section_id) + depth = section_id.count(".") + 1 current_section = { "sectionId": section_id, - "title": (title or "").strip(), - "pageStart": current_page + 1, # +1 to match TOC page numbers - "pageEnd": current_page + 1, + "title": title, + "pageStart": printed_page(), + "pageEnd": printed_page(), "content": "", "depth": depth, - "parentId": parent_id, + "parentId": get_parent_section_id(section_id), } current_content = [line] + awaiting_title = needs_title_continuation(section_id, title) elif current_section: current_content.append(line) # Don't forget the last section if current_section: - current_section["content"] = '\n'.join(current_content).strip() - current_section["pageEnd"] = current_page + 1 # +1 to match TOC + current_section["content"] = "\n".join(current_content).strip() + current_section["pageEnd"] = printed_page() sections.append(current_section) return sections @@ -205,25 +419,25 @@ def parse_sections(md_text: str, start_page: int) -> list[dict]: def get_parent_section_id(section_id: str) -> str | None: """Get parent section ID from a section ID.""" - if section_id.startswith("Annex"): + if section_id.lower().startswith("annex"): return None - parts = section_id.split('.') + parts = section_id.split(".") if len(parts) <= 1: return None - return '.'.join(parts[:-1]) + return ".".join(parts[:-1]) def main(): args = sys.argv[1:] if len(args) < 2: - print("Usage: python scripts/ingest/extract-pdf.py [--pages START-END]") + print("Usage: python scripts/ingest-pdf/extract.py [--pages START-END]") print("") print("Example:") - print(" python scripts/ingest/extract-pdf.py ./pdfs/ECMA-376-Part1.pdf ./extracted/part1") - print(" python scripts/ingest/extract-pdf.py ./pdfs/ECMA-376-Part1.pdf ./extracted/part1 --pages 100-200") + print(" python scripts/ingest-pdf/extract.py ./pdfs/ECMA-376-Part1.pdf ./extracted/part1") + print(" python scripts/ingest-pdf/extract.py ./pdfs/ECMA-376-Part1.pdf ./extracted/part1 --pages 100-200") sys.exit(1) pdf_path = args[0] diff --git a/scripts/ingest-pdf/fix-page-numbers.py b/scripts/ingest-pdf/fix-page-numbers.py deleted file mode 100644 index e68a1f3..0000000 --- a/scripts/ingest-pdf/fix-page-numbers.py +++ /dev/null @@ -1,158 +0,0 @@ -#!/usr/bin/env python3 -""" -Fix page numbers in embedded JSON files by re-parsing content.md. - -This avoids re-running the full pipeline (re-chunk, re-embed) which costs API credits. - -Usage: - python scripts/ingest/fix-page-numbers.py - -Example: - python scripts/ingest/fix-page-numbers.py 1 -""" - -import sys -import json -import re -from pathlib import Path - - -def parse_sections_for_pages(md_text: str, start_page: int = 1) -> dict[str, int]: - """Parse section IDs and their page numbers from markdown.""" - section_pages = {} - - # Bold section header patterns - patterns = [ - # Part 1 style: **12.3.2** **Title** - r'^\*\*(\d+(?:\.\d+)*)\*\*\s*\*\*([^*]+)\*\*$', - # Part 2/3/4 style: # **12.3.2. Title** or # **12. Title** - r'^#+\s*\*\*(\d+(?:\.\d+)*)\.?\s+([^*]+)\*\*$', - # Annex - r'^\*\*(Annex\s+[A-Z])\*\*\s*(?:\*\*)?(?:\(([^)]+)\))?(?:\*\*)?\s*(.*)$', - ] - - toc_pattern = r'^\d+(?:\.\d+)*\s+.+\.{2,}\s*\d+$' - arabic_page_pattern = r'^(\d+)$' - header_pattern = r'^ECMA-376 Part \d' - - lines = md_text.split('\n') - current_page = start_page - - for line in lines: - stripped = line.strip() - - # Skip headers - if re.match(header_pattern, stripped): - continue - - # Track page numbers - if re.match(arabic_page_pattern, stripped): - page_num = int(stripped) - if page_num >= current_page and page_num < current_page + 50: - current_page = page_num - continue - - # Skip TOC entries - if re.match(toc_pattern, stripped): - continue - - # Check for section headers - for pattern in patterns: - match = re.match(pattern, stripped, re.IGNORECASE) - if match: - section_id = match.group(1) - # +1 to match TOC page numbers - section_pages[section_id] = current_page + 1 - break - - return section_pages - - -def fix_embedded_file(part_number: int): - """Fix page numbers in embedded JSON file.""" - base_dir = Path("dev/data") - content_path = base_dir / f"extracted/part{part_number}/content.md" - embedded_path = base_dir / f"embedded/part{part_number}-embedded.json" - - if not content_path.exists(): - print(f"ERROR: Content file not found: {content_path}") - return False - - if not embedded_path.exists(): - print(f"ERROR: Embedded file not found: {embedded_path}") - return False - - print(f"Processing part {part_number}...") - - # Parse content.md for section page numbers - print(f" Parsing {content_path}...") - with open(content_path) as f: - content = f.read() - - section_pages = parse_sections_for_pages(content) - print(f" Found {len(section_pages)} sections with page numbers") - - # Load embedded chunks - print(f" Loading {embedded_path}...") - with open(embedded_path) as f: - chunks = json.load(f) - - print(f" Loaded {len(chunks)} chunks") - - # Update page numbers - updated = 0 - missing = set() - for chunk in chunks: - section_id = chunk.get("sectionId") - if section_id and section_id in section_pages: - old_page = chunk.get("pageNumber") - new_page = section_pages[section_id] - if old_page != new_page: - chunk["pageNumber"] = new_page - updated += 1 - elif section_id: - missing.add(section_id) - - print(f" Updated {updated} chunks") - if missing: - print(f" Warning: {len(missing)} sections not found in parsed content") - - # Save updated file - print(f" Saving {embedded_path}...") - with open(embedded_path, "w") as f: - json.dump(chunks, f, indent=2) - - print(f" Done!") - return True - - -def main(): - if len(sys.argv) < 2: - print("Usage: python scripts/ingest/fix-page-numbers.py ") - print("") - print("Examples:") - print(" python scripts/ingest/fix-page-numbers.py 1") - print(" python scripts/ingest/fix-page-numbers.py all") - sys.exit(1) - - arg = sys.argv[1] - - if arg == "all": - parts = [1, 2, 3, 4] - else: - try: - parts = [int(arg)] - except ValueError: - print(f"Invalid part number: {arg}") - sys.exit(1) - - for part in parts: - if not fix_embedded_file(part): - sys.exit(1) - print() - - print("All done! Now run upload.ts to update the database.") - - -if __name__ == "__main__": - main() diff --git a/scripts/ingest-pdf/pipeline.ts b/scripts/ingest-pdf/pipeline.ts index dacc32a..1d12262 100644 --- a/scripts/ingest-pdf/pipeline.ts +++ b/scripts/ingest-pdf/pipeline.ts @@ -8,8 +8,7 @@ * * Environment variables: * DATABASE_URL - PostgreSQL connection string - * EMBEDDING_PROVIDER - openai, google, voyage, or cohere (default: openai) - * OPENAI_API_KEY / GOOGLE_API_KEY / etc. + * VOYAGE_API_KEY * * Example: * bun scripts/ingest-pdf/pipeline.ts 1 ./pdfs/ECMA-376-Part1.pdf @@ -25,8 +24,7 @@ async function main() { console.log(""); console.log("Environment variables:"); console.log(" DATABASE_URL - PostgreSQL connection string"); - console.log(" EMBEDDING_PROVIDER - openai, google, voyage, or cohere (default: openai)"); - console.log(" OPENAI_API_KEY / GOOGLE_API_KEY / etc."); + console.log(" VOYAGE_API_KEY"); console.log(""); console.log("Example:"); console.log(" bun scripts/ingest-pdf/pipeline.ts 1 ./pdfs/ECMA-376-Part1.pdf"); @@ -47,16 +45,8 @@ async function main() { process.exit(1); } - const provider = process.env.EMBEDDING_PROVIDER || "openai"; - const apiKeyVar = { - openai: "OPENAI_API_KEY", - google: "GOOGLE_API_KEY", - voyage: "VOYAGE_API_KEY", - cohere: "COHERE_API_KEY", - }[provider]; - - if (apiKeyVar && !process.env[apiKeyVar]) { - console.error(`Missing ${apiKeyVar} environment variable`); + if (!process.env.VOYAGE_API_KEY) { + console.error("Missing VOYAGE_API_KEY environment variable"); process.exit(1); } @@ -71,11 +61,11 @@ async function main() { console.log(`ECMA-376 Part ${partNumber} Ingestion Pipeline`); console.log("=".repeat(60)); console.log(`PDF: ${pdfPath}`); - console.log(`Embedding provider: ${provider}`); + console.log("Embedding provider: Voyage"); console.log(""); // Step 1: Extract (using Python + pymupdf4llm for better markdown output) - console.log("\n[1/4] Extracting PDF..."); + console.log("\n[1/5] Extracting PDF..."); console.log("-".repeat(40)); // Try different Python paths (pymupdf4llm may be installed in a specific version) @@ -108,17 +98,22 @@ async function main() { } // Step 2: Chunk - console.log("\n[2/4] Chunking content..."); + console.log("\n[2/5] Chunking content..."); console.log("-".repeat(40)); await $`bun scripts/ingest-pdf/chunk.ts ${extractedDir} ${chunksFile}`; - // Step 3: Embed - console.log("\n[3/4] Generating embeddings..."); + // Step 3: Audit generated content before spending embedding credits. + console.log("\n[3/5] Auditing corpus..."); + console.log("-".repeat(40)); + await $`bun scripts/ingest-pdf/audit.ts ${extractedDir}`; + + // Step 4: Embed + console.log("\n[4/5] Generating embeddings..."); console.log("-".repeat(40)); await $`bun scripts/ingest-pdf/embed.ts ${chunksFile} ${embeddedFile}`; - // Step 4: Upload - console.log("\n[4/4] Uploading to database..."); + // Step 5: Upload + console.log("\n[5/5] Uploading to database..."); console.log("-".repeat(40)); await $`bun scripts/ingest-pdf/upload.ts ${partNumber} ${embeddedFile}`; diff --git a/scripts/ingest-pdf/upload.ts b/scripts/ingest-pdf/upload.ts index c17cffc..1570c65 100644 --- a/scripts/ingest-pdf/upload.ts +++ b/scripts/ingest-pdf/upload.ts @@ -64,31 +64,31 @@ async function main() { console.log("Connecting to database..."); const db = createDbClient(databaseUrl); - // Upload in batches - console.log("Uploading..."); - const batchSize = 50; - let uploaded = 0; - - for (let i = 0; i < chunks.length; i += batchSize) { - const batch = chunks.slice(i, i + batchSize); - - const items: Omit[] = batch.map((chunk) => ({ - partNumber, - sectionId: chunk.sectionId, - title: chunk.sectionTitle, - content: chunk.content, - contentType: chunk.contentType, - pageNumber: chunk.pageNumber, - embedding: chunk.embedding, - })); - - await db.insertBatch(items); - uploaded += batch.length; - - if (uploaded % 200 === 0 || uploaded === chunks.length) { - console.log(` ${uploaded}/${chunks.length}`); - } - } + const [source] = await db.sql<{ id: number }[]>` + SELECT id FROM reference_sources WHERE name = ${`ecma-376-part${partNumber}`} + `; + if (!source) throw new Error(`Missing reference source for Part ${partNumber}`); + + const items: Omit[] = chunks.map((chunk) => ({ + partNumber, + sectionId: chunk.sectionId, + title: chunk.sectionTitle, + content: chunk.content, + contentType: chunk.contentType, + pageNumber: chunk.pageNumber, + embedding: chunk.embedding, + sourceId: source.id, + })); + + console.log("Replacing existing part..."); + const result = await db.replacePart(partNumber, items, { + onProgress: (inserted) => { + if (inserted % 200 === 0 || inserted === items.length) { + console.log(` ${inserted}/${items.length}`); + } + }, + }); + console.log(`Replaced ${result.deleted} rows with ${result.inserted}`); // Get stats const stats = await db.getStats(); diff --git a/scripts/requirements.txt b/scripts/requirements.txt index 56ae9c8..5e75cc4 100644 --- a/scripts/requirements.txt +++ b/scripts/requirements.txt @@ -1,4 +1,4 @@ # Python dependencies for PDF extraction # Install with: pip install -r scripts/requirements.txt -pymupdf4llm>=0.0.17 -pymupdf>=1.24.0 +pymupdf4llm==1.28.2 +pymupdf==1.28.2 diff --git a/tests/db/spec-content-replace.test.ts b/tests/db/spec-content-replace.test.ts new file mode 100644 index 0000000..8360492 --- /dev/null +++ b/tests/db/spec-content-replace.test.ts @@ -0,0 +1,68 @@ +import { afterAll, beforeAll, beforeEach, describe, expect, test } from "bun:test"; +import { createDbClient, type DbClient } from "../../packages/shared/src/db"; +import type { SpecContent } from "../../packages/shared/src/types"; + +const databaseUrl = process.env.TEST_DATABASE_URL; +const describeWithDatabase = databaseUrl ? describe : describe.skip; +const part = 901; + +function chunk(content: string): Omit { + return { + partNumber: part, + sectionId: "17.3.1.12", + title: "ind (Paragraph Indentation)", + content, + contentType: "text", + pageNumber: 219, + }; +} + +describeWithDatabase("replacePart", () => { + let db: DbClient; + + beforeAll(() => { + db = createDbClient(databaseUrl as string); + }); + + beforeEach(async () => { + await db.sql`DELETE FROM spec_content WHERE part_number = ${part}`; + }); + + afterAll(async () => { + await db.sql`DELETE FROM spec_content WHERE part_number = ${part}`; + await db.close(); + }); + + test("re-ingesting replaces the part instead of duplicating it", async () => { + await db.replacePart(part, [chunk("old")]); + const result = await db.replacePart(part, [chunk("new")]); + + const rows = await db.sql<{ content: string }[]>` + SELECT content FROM spec_content WHERE part_number = ${part} + `; + + expect(result).toEqual({ deleted: 1, inserted: 1 }); + expect(rows).toEqual([{ content: "new" }]); + }); + + test("refuses an empty replacement", async () => { + await db.replacePart(part, [chunk("keep")]); + await expect(db.replacePart(part, [])).rejects.toThrow("with no content"); + + const rows = await db.sql<{ content: string }[]>` + SELECT content FROM spec_content WHERE part_number = ${part} + `; + expect(rows).toEqual([{ content: "keep" }]); + }); + + test("rolls back when an insert fails", async () => { + await db.replacePart(part, [chunk("keep")]); + const invalid = { ...chunk("invalid"), content: null as unknown as string }; + await expect(db.replacePart(part, [invalid])).rejects.toThrow(); + + const rows = await db.sql<{ content: string }[]>` + SELECT content FROM spec_content WHERE part_number = ${part} + `; + expect(rows).toEqual([{ content: "keep" }]); + }); +}); diff --git a/tests/ingest-pdf/chunk-pages.test.ts b/tests/ingest-pdf/chunk-pages.test.ts new file mode 100644 index 0000000..ab62e12 --- /dev/null +++ b/tests/ingest-pdf/chunk-pages.test.ts @@ -0,0 +1,22 @@ +import { describe, expect, test } from "bun:test"; +import { splitIntoChunks } from "../../scripts/ingest-pdf/chunk"; + +describe("PDF chunk page numbers", () => { + test("uses the page marker where each chunk begins", () => { + const firstPage = "A".repeat(5_900); + const secondPage = "B".repeat(500); + const thirdPage = "C".repeat(5_900); + + const chunks = splitIntoChunks( + `\n\n${firstPage}\n\n\n\n${secondPage}\n\n\n\n${thirdPage}`, + "17.3.1.12", + "ind (Paragraph Indentation)", + 219, + 10, + ); + + expect(chunks.map((chunk) => chunk.pageNumber)).toEqual([219, 220, 222]); + expect(chunks.every((chunk) => !chunk.content.includes("contents row", + embeddingText: "contents row", + contentType: "text", + pageNumber: 229, + chunkIndex: 0, + }; + + expect(() => auditCorpus([badChunk], 219)).toThrow(); + }); +}); diff --git a/tests/ingest-pdf/test_headings.py b/tests/ingest-pdf/test_headings.py new file mode 100644 index 0000000..3d422d5 --- /dev/null +++ b/tests/ingest-pdf/test_headings.py @@ -0,0 +1,205 @@ +#!/usr/bin/env python3 +""" +Tests for the section heading parsing in scripts/ingest-pdf/extract.py. + +pymupdf4llm changes the markdown it emits between releases - a heading is bold +runs on older ones and an ATX heading on newer ones - so both shapes are +covered here, along with the lines that only look like headings: running +headers, contents listings and table rows. + +Run with: bun run pdf:test +""" + +import importlib.util +import unittest +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] + +# scripts/ingest-pdf isn't a package, so load extract.py by path. +spec = importlib.util.spec_from_file_location( + "extract", REPO_ROOT / "scripts" / "ingest-pdf" / "extract.py" +) +extract = importlib.util.module_from_spec(spec) +spec.loader.exec_module(extract) + +match_heading = extract.match_heading + + +class TestNumberedHeadings(unittest.TestCase): + def test_bold_runs(self): + self.assertEqual( + match_heading("**17.3.1.24** **pStyle (Paragraph Style Reference)**"), + ("17.3.1.24", "pStyle (Paragraph Style Reference)"), + ) + + def test_atx_heading(self): + self.assertEqual(match_heading("# **12. Package Structure**"), ("12", "Package Structure")) + self.assertEqual( + match_heading("### **12.3.2. Main Document Part**"), + ("12.3.2", "Main Document Part"), + ) + self.assertEqual(match_heading("#### 17.3.1.24 pStyle"), ("17.3.1.24", "pStyle")) + + def test_title_never_keeps_emphasis_markers(self): + for line in ( + "**17.3.1.24** **pStyle**", + "## **17.3.1.24 pStyle**", + "## 17.3.1.24 **pStyle**", + ): + _section_id, title = match_heading(line) + self.assertNotIn("*", title, line) + + def test_ignores_contents_listing(self): + self.assertIsNone(match_heading("17.3.2 Paragraphs .......... 264")) + self.assertIsNone(match_heading("**17.3.2** **Paragraphs .......... 264**")) + # A contents entry whose title swallowed the leader dots + self.assertIsNone(match_heading("**17.3.2** **Paragraphs 264**")) + self.assertIsNone(match_heading("| **17.3.2** | Paragraphs | 264 |")) + + def test_ignores_running_header(self): + # The header on every body page: the title isn't part of the bold run + self.assertIsNone(match_heading("**17** . WordprocessingML Reference Material")) + + def test_ignores_body_text_and_bare_numbers(self): + self.assertIsNone(match_heading("Some plain body paragraph.")) + self.assertIsNone(match_heading("**17.3.1.24**")) + + def test_ignores_numbered_example_lines(self): + self.assertIsNone(match_heading("# **4 shrinks when it is clicked on.**")) + self.assertIsNone(match_heading("###### 13 B.2.1.23 Worksheet Part")) + + def test_top_level_atx_heading_requires_its_period(self): + self.assertEqual(match_heading("# **4. Terms and Definitions**"), ("4", "Terms and Definitions")) + + +class TestAnnexHeadings(unittest.TestCase): + def test_qualifier_and_name_in_separate_runs(self): + self.assertEqual( + match_heading("**Annex A** **(normative)** **Namespaces**"), + ("Annex A", "(normative) Namespaces"), + ) + + def test_collapsed_into_one_run(self): + self.assertEqual( + match_heading("#### **Annex B (informative) Bibliography**"), + ("Annex B", "(informative) Bibliography"), + ) + + def test_qualifier_is_not_mistaken_for_the_name(self): + # The old parser captured "(normative)" as the whole title, markers and all + self.assertEqual(match_heading("**Annex A** **(normative)**"), ("Annex A", "(normative)")) + + def test_title_never_keeps_emphasis_markers(self): + for line in ( + "**Annex A** **(normative)** **Namespaces**", + "#### **Annex A (normative) Namespaces**", + "**Annex A** **(normative)**", + ): + _section_id, title = match_heading(line) + self.assertNotIn("*", title, line) + + def test_letter_is_normalised(self): + self.assertEqual( + match_heading("**annex c** **(informative) Notes**"), + ("Annex C", "(informative) Notes"), + ) + + def test_ignores_running_header(self): + # Annex pages repeat their number as the running header + self.assertIsNone(match_heading("Annex A")) + self.assertIsNone(match_heading("**Annex A**")) + + +class TestParseSections(unittest.TestCase): + # Ben's result appeared once in the TOC and once as real prose. + def test_ben_fixture_keeps_only_the_real_section(self): + md = "\n".join( + [ + extract.PAGE_MARKER.format(176), + "**17.3.1.12** **ind (Paragraph Indentation) .......... 219**", + extract.PAGE_MARKER.format(229), + "#### 17.3.1.12 ind (Paragraph Indentation)", + "This element specifies the set of indentation properties.", + ] + ) + + sections = extract.parse_sections(md, page_offset=10) + + self.assertEqual(len(sections), 1) + self.assertEqual(sections[0]["sectionId"], "17.3.1.12") + self.assertEqual(sections[0]["pageStart"], 219) + + def test_annex_name_on_the_line_after_its_qualifier(self): + md = "\n".join( + [ + "**Annex A** **(normative)**", + "", + "**Namespaces**", + "", + "This annex lists the namespaces.", + ] + ) + + sections = extract.parse_sections(md) + + self.assertEqual(len(sections), 1) + self.assertEqual(sections[0]["sectionId"], "Annex A") + self.assertEqual(sections[0]["title"], "(normative) Namespaces") + self.assertEqual(sections[0]["depth"], 1) + self.assertIsNone(sections[0]["parentId"]) + + def test_annex_body_is_not_swallowed_as_a_title(self): + md = "\n".join( + [ + "**Annex A** **(normative)**", + "", + "This annex lists the namespaces.", + "", + "**More prose here.**", + ] + ) + + sections = extract.parse_sections(md) + + self.assertEqual(sections[0]["title"], "(normative)") + self.assertIn("This annex lists the namespaces.", sections[0]["content"]) + self.assertIn("More prose here.", sections[0]["content"]) + + def test_sections_and_annexes_together(self): + md = "\n".join( + [ + "**17.3.1** **Paragraphs**", + "Paragraph prose.", + "**Annex A** **(normative)** **Namespaces**", + "Annex prose.", + ] + ) + + sections = extract.parse_sections(md) + + self.assertEqual( + [(s["sectionId"], s["title"]) for s in sections], + [("17.3.1", "Paragraphs"), ("Annex A", "(normative) Namespaces")], + ) + self.assertEqual(sections[0]["parentId"], "17.3") + + def test_pages_come_from_the_page_markers(self): + md = "\n".join( + [ + extract.PAGE_MARKER.format(30), + "**Annex A** **(normative)** **Namespaces**", + "Annex prose.", + extract.PAGE_MARKER.format(31), + "More annex prose.", + ] + ) + + sections = extract.parse_sections(md, page_offset=10) + + self.assertEqual(sections[0]["pageStart"], 20) + self.assertEqual(sections[0]["pageEnd"], 21) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/web/pdf-navigation.test.ts b/tests/web/pdf-navigation.test.ts index ce234dd..73884a1 100644 --- a/tests/web/pdf-navigation.test.ts +++ b/tests/web/pdf-navigation.test.ts @@ -6,11 +6,13 @@ import { } from "../../apps/web/src/components/pdfNavigation"; describe("printedPageToSheet", () => { - test("uses the printed page counts of the served PDFs", () => { - expect(PDF_CONFIG[1].totalPages).toBe(5016); - expect(PDF_CONFIG[2].totalPages).toBe(129); - expect(PDF_CONFIG[3].totalPages).toBe(38); - expect(PDF_CONFIG[4].totalPages).toBe(1534); + test("printed pages plus front matter equal the served PDF sheet counts", () => { + const servedPdfSheets = { 1: 5026, 2: 137, 3: 44, 4: 1548 }; + + for (const [part, sheets] of Object.entries(servedPdfSheets)) { + const config = PDF_CONFIG[Number(part)]; + expect(config.totalPages + config.pageOffset).toBe(sheets); + } }); test("adds each PDF's front-matter offset", () => { @@ -25,8 +27,8 @@ describe("printedPageToSheet", () => { }); test("builds the browser URL with the physical sheet", () => { - expect(pdfUrlForPrintedPage(238, PDF_CONFIG[1])).toBe( - "https://cdn.ooxml.dev/ecma-376/part1.pdf#page=248&toolbar=0&navpanes=0", + expect(pdfUrlForPrintedPage(219, PDF_CONFIG[1])).toBe( + "https://cdn.ooxml.dev/ecma-376/part1.pdf#page=229&toolbar=0&navpanes=0", ); }); }); From efa7196925c720e1fc08180c0da2107ebe05700e Mon Sep 17 00:00:00 2001 From: Caio Pizzol Date: Wed, 12 Aug 2026 15:25:20 -0300 Subject: [PATCH 3/5] refactor(spec): keep PDF fix focused Co-authored-by: Ben Lewis --- .gitignore | 2 - package.json | 3 +- packages/shared/src/db/index.ts | 33 ++++++++------- packages/shared/src/types/index.ts | 1 - scripts/ingest-pdf/README.md | 3 +- scripts/ingest-pdf/audit.ts | 60 --------------------------- scripts/ingest-pdf/chunk.ts | 33 ++++----------- scripts/ingest-pdf/extract.py | 12 ++---- scripts/ingest-pdf/pipeline.ts | 18 ++++---- scripts/ingest-pdf/upload.ts | 11 +---- tests/db/spec-content-replace.test.ts | 23 +++++++--- tests/ingest-pdf/corpus-audit.test.ts | 19 --------- tests/ingest-pdf/test_headings.py | 5 ++- 13 files changed, 62 insertions(+), 161 deletions(-) delete mode 100644 scripts/ingest-pdf/audit.ts delete mode 100644 tests/ingest-pdf/corpus-audit.test.ts diff --git a/.gitignore b/.gitignore index b8bc912..ec0c48e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,6 @@ node_modules/ dist/ .DS_Store -__pycache__/ -*.py[cod] dev/ .wrangler/ .env* diff --git a/package.json b/package.json index fdcfb59..f5187fe 100644 --- a/package.json +++ b/package.json @@ -28,11 +28,10 @@ "sources:sync": "bun scripts/sources-sync.ts", "pdf:ingest": "bun scripts/ingest-pdf/pipeline.ts", "pdf:chunk": "bun scripts/ingest-pdf/chunk.ts", - "pdf:audit": "bun scripts/ingest-pdf/audit.ts", "pdf:embed": "bun scripts/ingest-pdf/embed.ts", "pdf:upload": "bun scripts/ingest-pdf/upload.ts", "pdf:setup": "pip install -r scripts/requirements.txt", - "pdf:test": "python3 -m unittest discover -s tests/ingest-pdf && bun test tests/ingest-pdf/*.test.ts", + "pdf:test": "python3 -m unittest discover -s tests/ingest-pdf && bun test tests/ingest-pdf/chunk-pages.test.ts", "xsd:fetch": "bun scripts/ingest-xsd/fetch.ts", "xsd:ingest": "bun scripts/ingest-xsd/ingest.ts", "test": "export TEST_DATABASE_URL=${TEST_DATABASE_URL:-postgresql://postgres:postgres@localhost:5432/ecma_spec} && bun test tests/web/ && bun test tests/db/ && bun test tests/ingest-xsd/ && bun test tests/mcp-server/" diff --git a/packages/shared/src/db/index.ts b/packages/shared/src/db/index.ts index 3a99bae..8ee51d2 100644 --- a/packages/shared/src/db/index.ts +++ b/packages/shared/src/db/index.ts @@ -3,7 +3,9 @@ import type { SearchResult, SpecContent } from "../types"; export type DbClient = ReturnType; -function specContentRow(item: Omit) { +type ReplacementContent = Omit & { sourceId: number }; + +function specContentRow(item: ReplacementContent) { return { part_number: item.partNumber, section_id: item.sectionId, @@ -12,7 +14,7 @@ function specContentRow(item: Omit) { content_type: item.contentType, page_number: item.pageNumber, embedding: item.embedding ? `[${item.embedding.join(",")}]` : null, - source_id: item.sourceId ?? null, + source_id: item.sourceId, }; } @@ -29,7 +31,7 @@ export function createDbClient(connectionString: string) { // Insert content async insert(content: Omit) { const [result] = await sql<[{ id: number }]>` - INSERT INTO spec_content (part_number, section_id, title, content, content_type, page_number, embedding, source_id) + INSERT INTO spec_content (part_number, section_id, title, content, content_type, page_number, embedding) VALUES ( ${content.partNumber}, ${content.sectionId}, @@ -37,8 +39,7 @@ export function createDbClient(connectionString: string) { ${content.content}, ${content.contentType}, ${content.pageNumber}, - ${content.embedding ? `[${content.embedding.join(",")}]` : null}, - ${content.sourceId ?? null} + ${content.embedding ? `[${content.embedding.join(",")}]` : null} ) RETURNING id `; @@ -47,7 +48,15 @@ export function createDbClient(connectionString: string) { // Insert multiple (batch) async insertBatch(items: Omit[]) { - const values = items.map(specContentRow); + const values = items.map((item) => ({ + part_number: item.partNumber, + section_id: item.sectionId, + title: item.title, + content: item.content, + content_type: item.contentType, + page_number: item.pageNumber, + embedding: item.embedding ? `[${item.embedding.join(",")}]` : null, + })); const result = await sql` INSERT INTO spec_content ${sql(values)} @@ -58,8 +67,7 @@ export function createDbClient(connectionString: string) { async replacePart( partNumber: number, - items: Omit[], - options: { batchSize?: number; onProgress?: (inserted: number) => void } = {}, + items: ReplacementContent[], ): Promise<{ deleted: number; inserted: number }> { if (items.length === 0) { throw new Error(`Refusing to replace Part ${partNumber} with no content`); @@ -68,23 +76,20 @@ export function createDbClient(connectionString: string) { throw new Error(`Replacement content must all belong to Part ${partNumber}`); } - const { batchSize = 50, onProgress } = options; return sql.begin(async (tx) => { // postgres.js transaction handles are callable at runtime, but its // TransactionSql type drops the call signature through Omit. const transaction = tx as unknown as typeof sql; + await transaction`SELECT pg_advisory_xact_lock(376, ${partNumber})`; const deleted = await transaction`DELETE FROM spec_content WHERE part_number = ${partNumber}`; - let inserted = 0; - + const batchSize = 50; for (let index = 0; index < items.length; index += batchSize) { const batch = items.slice(index, index + batchSize).map(specContentRow); await transaction`INSERT INTO spec_content ${transaction(batch)}`; - inserted += batch.length; - onProgress?.(inserted); } - return { deleted: deleted.count, inserted }; + return { deleted: deleted.count, inserted: items.length }; }); }, diff --git a/packages/shared/src/types/index.ts b/packages/shared/src/types/index.ts index 993ee0e..2753776 100644 --- a/packages/shared/src/types/index.ts +++ b/packages/shared/src/types/index.ts @@ -8,7 +8,6 @@ export interface SpecContent { content: string; contentType: string; pageNumber: number | null; - sourceId?: number | null; embedding?: number[]; } diff --git a/scripts/ingest-pdf/README.md b/scripts/ingest-pdf/README.md index 608eb86..5aad8f3 100644 --- a/scripts/ingest-pdf/README.md +++ b/scripts/ingest-pdf/README.md @@ -38,8 +38,8 @@ their actual pages and excludes contents-list entries from the corpus. ```bash bun run pdf:chunk ./extracted/part1 ./chunks/part1.json -bun run pdf:audit ./extracted/part1 bun run pdf:embed ./chunks/part1.json ./embedded/part1.json +bun run sources:sync bun run pdf:upload 1 ./embedded/part1.json ``` @@ -51,6 +51,5 @@ without re-extracting. - `pipeline.ts` - orchestrator (extract -> chunk -> embed -> upload) - `extract.py` - PDF -> page-aware, section-aware markdown via pymupdf4llm - `chunk.ts` - markdown -> page-aware 6 KB chunks with section IDs -- `audit.ts` - rejects duplicate sections, contents entries, and invalid pages - `embed.ts` - chunks -> chunks + Voyage embeddings - `upload.ts` - atomic part replacement in `spec_content` diff --git a/scripts/ingest-pdf/audit.ts b/scripts/ingest-pdf/audit.ts deleted file mode 100644 index 1cb6c45..0000000 --- a/scripts/ingest-pdf/audit.ts +++ /dev/null @@ -1,60 +0,0 @@ -/** - * Audit an extracted corpus before embedding or upload. - * - * Usage: - * bun scripts/ingest-pdf/audit.ts - */ - -import { type Chunk, chunkSections } from "./chunk"; - -const TOC_TITLE_PATTERN = /\.{2,}|\s\d{1,4}$/; - -export function auditCorpus(chunks: Chunk[], maxPrintedPage: number): void { - if (chunks.length === 0) throw new Error("Corpus contains no chunks"); - - for (const chunk of chunks) { - if (chunk.pageNumber < 1 || chunk.pageNumber > maxPrintedPage) { - throw new Error(`${chunk.sectionId} has invalid printed page ${chunk.pageNumber}`); - } - if (TOC_TITLE_PATTERN.test(chunk.sectionTitle)) { - throw new Error(`${chunk.sectionId} has a contents-shaped title: ${chunk.sectionTitle}`); - } - if (chunk.content.includes("/g; @@ -149,9 +148,7 @@ export function splitIntoChunks( if (currentChunk.length + trimmedPara.length > CHUNK_SIZE) { pushChunk(); - // Start new chunk with overlap, on whichever page we have reached - const overlap = currentChunk.slice(-CHUNK_OVERLAP); - currentChunk = `${overlap}\n\n${trimmedPara}`; + currentChunk = trimmedPara; chunkPage = currentPage; } else { if (!currentChunk) { @@ -167,22 +164,15 @@ export function splitIntoChunks( } async function readPageOffset(extractedDir: string): Promise { - try { - const metadata = await Bun.file(`${extractedDir}/metadata.json`).json(); - const offset = metadata?.pageOffset; - if (typeof offset === "number") return offset; - } catch { - // Extraction predating page offsets - fall through. + const metadata = await Bun.file(`${extractedDir}/metadata.json`).json(); + const offset = metadata?.pageOffset; + if (!Number.isInteger(offset) || offset < 1) { + throw new Error("metadata.json has no valid pageOffset; run extract.py again"); } - - console.warn( - " No pageOffset in metadata.json; treating extracted pages as printed pages.\n" + - " Re-run extract.py so page numbers line up with the PDF.", - ); - return 0; + return offset; } -export async function chunkSections(extractedDir: string): Promise { +async function chunkSections(extractedDir: string): Promise { const sectionsJson = await Bun.file(`${extractedDir}/sections.json`).text(); const sections: ExtractedSection[] = JSON.parse(sectionsJson); const pageOffset = await readPageOffset(extractedDir); @@ -232,15 +222,10 @@ async function main() { const avgEmbedding = Math.round( chunks.reduce((sum, c) => sum + c.embeddingText.length, 0) / chunks.length, ); - const multiPageSections = new Set( - chunks.filter((c) => c.chunkIndex > 0).map((c) => c.sectionId), - ).size; - console.log("\nChunk statistics:"); console.log(` Total chunks: ${chunks.length}`); console.log(` Average content size: ${avgContent} chars`); console.log(` Average embedding text size: ${avgEmbedding} chars`); - console.log(` Sections spanning multiple chunks: ${multiPageSections}`); } catch (error) { console.error("Chunking failed:", error); process.exit(1); diff --git a/scripts/ingest-pdf/extract.py b/scripts/ingest-pdf/extract.py index 232a715..7bdef79 100644 --- a/scripts/ingest-pdf/extract.py +++ b/scripts/ingest-pdf/extract.py @@ -64,7 +64,7 @@ def detect_page_offset(doc) -> int: votes[(index + 1) - printed] += 1 if not votes: - return 0 + raise ValueError("Could not detect the PDF page offset") offset, agreed = votes.most_common(1)[0] total = sum(votes.values()) @@ -234,11 +234,10 @@ def assemble_markdown(page_chunks, pages: list[int]) -> str: HEADING_PREFIX_RE = re.compile(r"^#{1,6}\s*") EMPHASIS_RE = re.compile(r"\*+|__") -# A table-of-contents entry: leader dots and/or a trailing page number. +# A table-of-contents entry contains leader dots before its page number. # These render bold in some pymupdf4llm versions and so are indistinguishable # from real headings by styling alone - they must be rejected by shape. TOC_LEADER_RE = re.compile(r"\.{2,}") -TOC_TRAILING_PAGE_RE = re.compile(r"[.\s]\d{1,4}\s*$") def strip_emphasis(text: str) -> str: @@ -258,12 +257,7 @@ def has_heading_markup(stripped: str) -> bool: def looks_like_toc(title: str, raw_line: str) -> bool: """True when a heading candidate is really a contents-listing entry.""" - if TOC_LEADER_RE.search(raw_line): - return True - - # No leader dots, but still "Title 1047" - a contents entry whose title - # ran long enough to swallow the dots. - return bool(TOC_TRAILING_PAGE_RE.search(title)) + return bool(TOC_LEADER_RE.search(raw_line)) def match_heading(stripped: str) -> tuple[str, str] | None: diff --git a/scripts/ingest-pdf/pipeline.ts b/scripts/ingest-pdf/pipeline.ts index 1d12262..95becf5 100644 --- a/scripts/ingest-pdf/pipeline.ts +++ b/scripts/ingest-pdf/pipeline.ts @@ -65,7 +65,7 @@ async function main() { console.log(""); // Step 1: Extract (using Python + pymupdf4llm for better markdown output) - console.log("\n[1/5] Extracting PDF..."); + console.log("\n[1/4] Extracting PDF..."); console.log("-".repeat(40)); // Try different Python paths (pymupdf4llm may be installed in a specific version) @@ -98,23 +98,19 @@ async function main() { } // Step 2: Chunk - console.log("\n[2/5] Chunking content..."); + console.log("\n[2/4] Chunking content..."); console.log("-".repeat(40)); await $`bun scripts/ingest-pdf/chunk.ts ${extractedDir} ${chunksFile}`; - // Step 3: Audit generated content before spending embedding credits. - console.log("\n[3/5] Auditing corpus..."); - console.log("-".repeat(40)); - await $`bun scripts/ingest-pdf/audit.ts ${extractedDir}`; - - // Step 4: Embed - console.log("\n[4/5] Generating embeddings..."); + // Step 3: Embed + console.log("\n[3/4] Generating embeddings..."); console.log("-".repeat(40)); await $`bun scripts/ingest-pdf/embed.ts ${chunksFile} ${embeddedFile}`; - // Step 5: Upload - console.log("\n[5/5] Uploading to database..."); + // Step 4: Upload + console.log("\n[4/4] Uploading to database..."); console.log("-".repeat(40)); + await $`bun scripts/sources-sync.ts`; await $`bun scripts/ingest-pdf/upload.ts ${partNumber} ${embeddedFile}`; console.log(`\n${"=".repeat(60)}`); diff --git a/scripts/ingest-pdf/upload.ts b/scripts/ingest-pdf/upload.ts index 1570c65..d8c6175 100644 --- a/scripts/ingest-pdf/upload.ts +++ b/scripts/ingest-pdf/upload.ts @@ -14,7 +14,6 @@ */ import { createDbClient } from "../../packages/shared/src/db/index.ts"; -import type { SpecContent } from "../../packages/shared/src/types/index.ts"; interface EmbeddedChunk { sectionId: string; @@ -69,7 +68,7 @@ async function main() { `; if (!source) throw new Error(`Missing reference source for Part ${partNumber}`); - const items: Omit[] = chunks.map((chunk) => ({ + const items = chunks.map((chunk) => ({ partNumber, sectionId: chunk.sectionId, title: chunk.sectionTitle, @@ -81,13 +80,7 @@ async function main() { })); console.log("Replacing existing part..."); - const result = await db.replacePart(partNumber, items, { - onProgress: (inserted) => { - if (inserted % 200 === 0 || inserted === items.length) { - console.log(` ${inserted}/${items.length}`); - } - }, - }); + const result = await db.replacePart(partNumber, items); console.log(`Replaced ${result.deleted} rows with ${result.inserted}`); // Get stats diff --git a/tests/db/spec-content-replace.test.ts b/tests/db/spec-content-replace.test.ts index 8360492..44bf23c 100644 --- a/tests/db/spec-content-replace.test.ts +++ b/tests/db/spec-content-replace.test.ts @@ -1,12 +1,14 @@ import { afterAll, beforeAll, beforeEach, describe, expect, test } from "bun:test"; import { createDbClient, type DbClient } from "../../packages/shared/src/db"; import type { SpecContent } from "../../packages/shared/src/types"; +import { getTestDatabaseUrl } from "../test-db"; -const databaseUrl = process.env.TEST_DATABASE_URL; -const describeWithDatabase = databaseUrl ? describe : describe.skip; +const databaseUrl = getTestDatabaseUrl(); const part = 901; +const sourceName = "test-spec-content-replace"; +let sourceId: number; -function chunk(content: string): Omit { +function chunk(content: string): Omit & { sourceId: number } { return { partNumber: part, sectionId: "17.3.1.12", @@ -14,14 +16,22 @@ function chunk(content: string): Omit { content, contentType: "text", pageNumber: 219, + sourceId, }; } -describeWithDatabase("replacePart", () => { +describe("replacePart", () => { let db: DbClient; - beforeAll(() => { - db = createDbClient(databaseUrl as string); + beforeAll(async () => { + db = createDbClient(databaseUrl); + const [source] = await db.sql<{ id: number }[]>` + INSERT INTO reference_sources (name, kind) + VALUES (${sourceName}, 'test') + ON CONFLICT (name) DO UPDATE SET kind = EXCLUDED.kind + RETURNING id + `; + sourceId = source.id; }); beforeEach(async () => { @@ -30,6 +40,7 @@ describeWithDatabase("replacePart", () => { afterAll(async () => { await db.sql`DELETE FROM spec_content WHERE part_number = ${part}`; + await db.sql`DELETE FROM reference_sources WHERE id = ${sourceId}`; await db.close(); }); diff --git a/tests/ingest-pdf/corpus-audit.test.ts b/tests/ingest-pdf/corpus-audit.test.ts deleted file mode 100644 index 83d91d1..0000000 --- a/tests/ingest-pdf/corpus-audit.test.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { describe, expect, test } from "bun:test"; -import { auditCorpus } from "../../scripts/ingest-pdf/audit"; -import type { Chunk } from "../../scripts/ingest-pdf/chunk"; - -describe("PDF corpus audit", () => { - test("rejects contents entries and invalid page data", () => { - const badChunk: Chunk = { - sectionId: "17.3.1.12", - sectionTitle: "ind (Paragraph Indentation) .......... 219", - content: "contents row", - embeddingText: "contents row", - contentType: "text", - pageNumber: 229, - chunkIndex: 0, - }; - - expect(() => auditCorpus([badChunk], 219)).toThrow(); - }); -}); diff --git a/tests/ingest-pdf/test_headings.py b/tests/ingest-pdf/test_headings.py index 3d422d5..2178b53 100644 --- a/tests/ingest-pdf/test_headings.py +++ b/tests/ingest-pdf/test_headings.py @@ -53,10 +53,11 @@ def test_title_never_keeps_emphasis_markers(self): def test_ignores_contents_listing(self): self.assertIsNone(match_heading("17.3.2 Paragraphs .......... 264")) self.assertIsNone(match_heading("**17.3.2** **Paragraphs .......... 264**")) - # A contents entry whose title swallowed the leader dots - self.assertIsNone(match_heading("**17.3.2** **Paragraphs 264**")) self.assertIsNone(match_heading("| **17.3.2** | Paragraphs | 264 |")) + def test_keeps_real_heading_ending_in_a_number(self): + self.assertEqual(match_heading("**12.3** **Changes in Part 1**"), ("12.3", "Changes in Part 1")) + def test_ignores_running_header(self): # The header on every body page: the title isn't part of the bold run self.assertIsNone(match_heading("**17** . WordprocessingML Reference Material")) From c1e58c11c266033cc177e898b3a6c88ee480d7f7 Mon Sep 17 00:00:00 2001 From: Caio Pizzol Date: Wed, 12 Aug 2026 15:35:03 -0300 Subject: [PATCH 4/5] fix(spec): keep table chunks embeddable Co-authored-by: Ben Lewis --- scripts/ingest-pdf/chunk.ts | 2 +- tests/ingest-pdf/chunk-pages.test.ts | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/scripts/ingest-pdf/chunk.ts b/scripts/ingest-pdf/chunk.ts index 5045dfd..6b2bf95 100644 --- a/scripts/ingest-pdf/chunk.ts +++ b/scripts/ingest-pdf/chunk.ts @@ -123,7 +123,7 @@ export function splitIntoChunks( sectionId, sectionTitle, content, - embeddingText: stripForEmbedding(content), + embeddingText: stripForEmbedding(content) || content, contentType: "text", pageNumber: chunkPage, chunkIndex: chunks.length, diff --git a/tests/ingest-pdf/chunk-pages.test.ts b/tests/ingest-pdf/chunk-pages.test.ts index ab62e12..26b6a34 100644 --- a/tests/ingest-pdf/chunk-pages.test.ts +++ b/tests/ingest-pdf/chunk-pages.test.ts @@ -19,4 +19,16 @@ describe("PDF chunk page numbers", () => { expect(chunks.every((chunk) => !chunk.content.includes("\n\n| Value |\n|---|\n| `` |", + "17.3.1.12", + "ind (Paragraph Indentation)", + 219, + 10, + ); + + expect(chunk.embeddingText).not.toBe(""); + }); }); From 17481c08b78365b97a0c26c58689bbf126b960c4 Mon Sep 17 00:00:00 2001 From: Caio Pizzol Date: Wed, 12 Aug 2026 15:43:53 -0300 Subject: [PATCH 5/5] fix(spec): verify PDF before ingestion Co-authored-by: Ben Lewis --- scripts/ingest-pdf/pipeline.ts | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/scripts/ingest-pdf/pipeline.ts b/scripts/ingest-pdf/pipeline.ts index 95becf5..c38d1b4 100644 --- a/scripts/ingest-pdf/pipeline.ts +++ b/scripts/ingest-pdf/pipeline.ts @@ -16,6 +16,20 @@ import { $ } from "bun"; +async function verifyPdf(partNumber: number, pdfPath: string) { + const manifest = await Bun.file("./data/sources.json").json(); + const source = manifest.sources?.find( + (entry: { name?: string }) => entry.name === `ecma-376-part${partNumber}`, + ); + if (!source?.sha256) throw new Error(`Missing source hash for Part ${partNumber}`); + + const bytes = await Bun.file(pdfPath).arrayBuffer(); + const actual = new Bun.CryptoHasher("sha256").update(bytes).digest("hex"); + if (actual !== source.sha256) { + throw new Error(`Part ${partNumber} PDF does not match data/sources.json`); + } +} + async function main() { const args = process.argv.slice(2); @@ -50,6 +64,8 @@ async function main() { process.exit(1); } + await verifyPdf(partNumber, pdfPath); + // Create directories const extractedDir = `./data/extracted/part${partNumber}`; const chunksFile = `./data/chunks/part${partNumber}-chunks.json`;