fix(spec): correct PDF search result pages

.github/workflows/ci.yml

@@ -26,5 +26,8 @@ jobs:
       - name: Migration runner tests
         run: bun test tests/scripts/db-migrate.test.ts
 
+      - name: PDF ingest tests
+        run: bun run pdf:test
+
       - name: Build
         run: bun run build

apps/web/src/components/PdfViewer.tsx

@@ -1,28 +1,5 @@
 import { useCallback, useEffect, useRef, useState } from "react";
-
-// PDF URLs and page counts for each part
-const PDF_CONFIG: Record<number, { url: string; totalPages: number; name: string }> = {
-	1: {
-		url: "https://cdn.ooxml.dev/ecma-376/part1.pdf",
-		totalPages: 5560,
-		name: "Fundamentals",
-	},
-	2: {
-		url: "https://cdn.ooxml.dev/ecma-376/part2.pdf",
-		totalPages: 129,
-		name: "OPC",
-	},
-	3: {
-		url: "https://cdn.ooxml.dev/ecma-376/part3.pdf",
-		totalPages: 65,
-		name: "Compatibility",
-	},
-	4: {
-		url: "https://cdn.ooxml.dev/ecma-376/part4.pdf",
-		totalPages: 4031,
-		name: "Transitional",
-	},
-};
+import { PDF_CONFIG, pdfUrlForPrintedPage } from "./pdfNavigation";
 
 interface PdfViewerProps {
 	partNumber: number;
@@ -119,7 +96,7 @@ export function PdfViewer({ partNumber, pageNumber, onPageChange }: PdfViewerPro
 	}, [config.totalPages, onPageChange]);
 
 	const progressPercent = (currentPage / config.totalPages) * 100;
-	const pdfUrl = `${config.url}#page=${currentPage}&toolbar=0&navpanes=0`;
+	const pdfUrl = pdfUrlForPrintedPage(currentPage, config);
 
 	return (
 		<div className="flex h-full flex-col bg-[var(--color-bg-secondary)]">

apps/web/src/components/pdfNavigation.ts

@@ -0,0 +1,45 @@
+export interface PdfConfig {
+	url: string;
+	totalPages: number;
+	pageOffset: number;
+	name: string;
+}
+
+// Search results use the page number printed in the spec, while browser PDF
+// fragments address physical sheets, including unnumbered front matter.
+export const PDF_CONFIG: Record<number, PdfConfig> = {
+	1: {
+		url: "https://cdn.ooxml.dev/ecma-376/part1.pdf",
+		totalPages: 5016,
+		pageOffset: 10,
+		name: "Fundamentals",
+	},
+	2: {
+		url: "https://cdn.ooxml.dev/ecma-376/part2.pdf",
+		totalPages: 129,
+		pageOffset: 8,
+		name: "OPC",
+	},
+	3: {
+		url: "https://cdn.ooxml.dev/ecma-376/part3.pdf",
+		totalPages: 38,
+		pageOffset: 6,
+		name: "Compatibility",
+	},
+	4: {
+		url: "https://cdn.ooxml.dev/ecma-376/part4.pdf",
+		totalPages: 1534,
+		pageOffset: 14,
+		name: "Transitional",
+	},
+};
+
+export function printedPageToSheet(printedPage: number, config: PdfConfig): number {
+	const clampedPage = Math.max(1, Math.min(printedPage, config.totalPages));
+	return clampedPage + config.pageOffset;
+}
+
+export function pdfUrlForPrintedPage(printedPage: number, config: PdfConfig): string {
+	const sheet = printedPageToSheet(printedPage, config);
+	return `${config.url}#page=${sheet}&toolbar=0&navpanes=0`;
+}

data/sources.json

@@ -4,38 +4,38 @@
     {
       "name": "ecma-376-part1",
       "kind": "spec_pdf",
-      "edition": "5th",
-      "version": "2016-12",
-      "url": "https://ecma-international.org/wp-content/uploads/ECMA-376-1_5th_edition_december_2016.zip",
+      "edition": "4th",
+      "version": "2012-12",
+      "url": "https://cdn.ooxml.dev/ecma-376/part1.pdf",
       "license_note": "Published by Ecma International. See the ECMA-376 publications page for the current download and licensing terms before redistribution.",
-      "sha256": "9d0bcad9cf06054785b03762fcfadbf6bab7e54a5f9d69434e34b7fd464d4129"
+      "sha256": "4a9d481c74daeab4068408dae354e8b6f808771a1b9810ea69482344ce6ccd65"
     },
     {
       "name": "ecma-376-part2",
       "kind": "spec_pdf",
-      "edition": "5th",
-      "version": "2021-12",
-      "url": "https://ecma-international.org/wp-content/uploads/ECMA-376-2_5th_edition_december_2021.zip",
+      "edition": "4th",
+      "version": "2012-12",
+      "url": "https://cdn.ooxml.dev/ecma-376/part2.pdf",
       "license_note": "Published by Ecma International. See the ECMA-376 publications page for the current download and licensing terms before redistribution.",
-      "sha256": "1d489dc491168ea1f9e9a59063acc8dd5f02b4ad1d21aa7ec19ba9a58d020c70"
+      "sha256": "1256d9d704af65b8dabfcf9e67770c0294d256387e1945ddba69fb453d174f55"
     },
     {
       "name": "ecma-376-part3",
       "kind": "spec_pdf",
-      "edition": "5th",
-      "version": "2015-12",
-      "url": "https://ecma-international.org/wp-content/uploads/ECMA-376-3_5th_edition_december_2015.zip",
+      "edition": "4th",
+      "version": "2012-12",
+      "url": "https://cdn.ooxml.dev/ecma-376/part3.pdf",
       "license_note": "Published by Ecma International. See the ECMA-376 publications page for the current download and licensing terms before redistribution.",
-      "sha256": "42294159fbbbe9393ccadac95b859d7729cc68d908898bcbe31034dda059daa8"
+      "sha256": "e4e58be15925162baeb8fd3aae51381c8c6995e421c4e460b950c057861a11c6"
     },
     {
       "name": "ecma-376-part4",
       "kind": "spec_pdf",
-      "edition": "5th",
-      "version": "2016-12",
-      "url": "https://ecma-international.org/wp-content/uploads/ECMA-376-4_5th_edition_december_2016.zip",
+      "edition": "4th",
+      "version": "2012-12",
+      "url": "https://cdn.ooxml.dev/ecma-376/part4.pdf",
       "license_note": "Published by Ecma International. See the ECMA-376 publications page for the current download and licensing terms before redistribution.",
-      "sha256": "bd25da1109f73762356596918bf5ff8b74a1331642dba5f1c1d1dfc6bed34ecd"
+      "sha256": "46a34fd930801e69ad5150996ca7671c8137efa4fd4bca3dcd30829594f65e02"
     },
     {
       "name": "ecma-376-transitional",

package.json

@@ -31,6 +31,7 @@
     "pdf:embed": "bun scripts/ingest-pdf/embed.ts",
     "pdf:upload": "bun scripts/ingest-pdf/upload.ts",
     "pdf:setup": "pip install -r scripts/requirements.txt",
+    "pdf:test": "python3 -m unittest discover -s tests/ingest-pdf && bun test tests/ingest-pdf/chunk-pages.test.ts",
     "xsd:fetch": "bun scripts/ingest-xsd/fetch.ts",
     "xsd:ingest": "bun scripts/ingest-xsd/ingest.ts",
     "test": "export TEST_DATABASE_URL=${TEST_DATABASE_URL:-postgresql://postgres:postgres@localhost:5432/ecma_spec} && bun test tests/web/ && bun test tests/db/ && bun test tests/ingest-xsd/ && bun test tests/mcp-server/"

packages/shared/src/db/index.ts

@@ -3,6 +3,21 @@ import type { SearchResult, SpecContent } from "../types";
 
 export type DbClient = ReturnType<typeof createDbClient>;
 
+type ReplacementContent = Omit<SpecContent, "id"> & { sourceId: number };
+
+function specContentRow(item: ReplacementContent) {
+	return {
+		part_number: item.partNumber,
+		section_id: item.sectionId,
+		title: item.title,
+		content: item.content,
+		content_type: item.contentType,
+		page_number: item.pageNumber,
+		embedding: item.embedding ? `[${item.embedding.join(",")}]` : null,
+		source_id: item.sourceId,
+	};
+}
+
 export function createDbClient(connectionString: string) {
 	const sql = postgres(connectionString);
 
@@ -50,6 +65,34 @@ export function createDbClient(connectionString: string) {
 			return result.map((r) => r.id as number);
 		},
 
+		async replacePart(
+			partNumber: number,
+			items: ReplacementContent[],
+		): Promise<{ deleted: number; inserted: number }> {
+			if (items.length === 0) {
+				throw new Error(`Refusing to replace Part ${partNumber} with no content`);
+			}
+			if (items.some((item) => item.partNumber !== partNumber)) {
+				throw new Error(`Replacement content must all belong to Part ${partNumber}`);
+			}
+
+			return sql.begin(async (tx) => {
+				// postgres.js transaction handles are callable at runtime, but its
+				// TransactionSql type drops the call signature through Omit.
+				const transaction = tx as unknown as typeof sql;
+				await transaction`SELECT pg_advisory_xact_lock(376, ${partNumber})`;
+				const deleted =
+					await transaction`DELETE FROM spec_content WHERE part_number = ${partNumber}`;
+				const batchSize = 50;
+				for (let index = 0; index < items.length; index += batchSize) {
+					const batch = items.slice(index, index + batchSize).map(specContentRow);
+					await transaction`INSERT INTO spec_content ${transaction(batch)}`;
+				}
+
+				return { deleted: deleted.count, inserted: items.length };
+			});
+		},
+
 		// Update embedding
 		async updateEmbedding(id: number, embedding: number[]) {
 			await sql`

scripts/ingest-pdf/README.md

@@ -2,8 +2,8 @@
 
 Builds the prose-search corpus that powers `ooxml_search` /
 `ooxml_section` / `ooxml_parts`. Each ECMA-376 part PDF is extracted into
-section-aware markdown, chunked at ~6 KB boundaries, embedded with the
-configured provider, and uploaded into `spec_content`.
+section-aware markdown, chunked at ~6 KB boundaries, embedded with Voyage,
+and uploaded into `spec_content`.
 
 ```
 PDF -> extract (Python) -> chunk (6KB, section-aware) -> embed -> upload
@@ -13,11 +13,8 @@ PDF -> extract (Python) -> chunk (6KB, section-aware) -> embed -> upload
 
 - Python with `pymupdf4llm`: `bun run pdf:setup`
 - `DATABASE_URL` pointed at a Postgres with `db/schema.sql` applied
-- An embedding provider key (one of):
-  - `OPENAI_API_KEY` (default)
-  - `VOYAGE_API_KEY`
-  - `GOOGLE_API_KEY`
-  - `COHERE_API_KEY`
+- `VOYAGE_API_KEY`. Search queries use `voyage-3`, so the corpus must use the
+  same model and 1,024 dimensions.
 
 ## Run the full pipeline
 
@@ -28,15 +25,21 @@ bun run pdf:ingest 3 ./pdfs/ECMA-376-Part3.pdf
 bun run pdf:ingest 4 ./pdfs/ECMA-376-Part4.pdf
 ```
 
-Each run extracts to `dev/data/extracted/partN/`, chunks to
-`dev/data/chunks/partN-chunks.json`, embeds to
-`dev/data/embedded/partN-embedded.json`, then uploads.
+Each run extracts to `data/extracted/partN/`, chunks to
+`data/chunks/partN-chunks.json`, embeds to
+`data/embedded/partN-embedded.json`, then replaces that part in one database
+transaction. A failed upload rolls back, and an empty corpus is rejected.
+
+Extraction records the physical PDF sheet for each page and derives the
+printed page from the document's running headers. This keeps long sections on
+their actual pages and excludes contents-list entries from the corpus.
 
 ## Run individual stages
 
 ```bash
 bun run pdf:chunk    ./extracted/part1            ./chunks/part1.json
 bun run pdf:embed    ./chunks/part1.json          ./embedded/part1.json
+bun run sources:sync
 bun run pdf:upload   1                            ./embedded/part1.json
 ```
 
@@ -46,8 +49,7 @@ without re-extracting.
 ## Files
 
 - `pipeline.ts` - orchestrator (extract -> chunk -> embed -> upload)
-- `extract.py` - PDF -> section-aware markdown via pymupdf4llm
-- `fix-page-numbers.py` - PDF prelude-aware page-number alignment
-- `chunk.ts` - markdown -> 6 KB chunks with section IDs
-- `embed.ts` - chunks -> chunks + 1024-dim embeddings
-- `upload.ts` - bulk insert into `spec_content`
+- `extract.py` - PDF -> page-aware, section-aware markdown via pymupdf4llm
+- `chunk.ts` - markdown -> page-aware 6 KB chunks with section IDs
+- `embed.ts` - chunks -> chunks + Voyage embeddings
+- `upload.ts` - atomic part replacement in `spec_content`