fix(spec): open search results on the correct PDF page

.gitignore

@@ -11,4 +11,7 @@ dev/
 PLAN.md
 
 # XSD/spec artifacts: pulled by scripts/fetch-xsd.ts; never committed.
-data/xsd-cache/
+data/xsd-cache/
+# Python bytecode
+__pycache__/
+*.pyc

apps/web/src/components/PdfViewer.tsx

@@ -1,71 +1,102 @@
 import { useCallback, useEffect, useRef, useState } from "react";
 
-// PDF URLs and page counts for each part
-const PDF_CONFIG: Record<number, { url: string; totalPages: number; name: string }> = {
+/**
+ * PDF URLs, sheet counts, and front-matter offsets for each part.
+ *
+ * `totalPages` is the number of sheets in the file - what `#page=` addresses.
+ * `pageOffset` is how many sheets precede printed page 1 (cover, contents,
+ * foreword). Both are measured from the published PDFs; `extract.py` prints
+ * the pair for a part at the end of every extraction run.
+ *
+ * Search results carry the *printed* page (`spec_content.page_number`), so
+ * navigating to one means adding `pageOffset` back.
+ */
+const PDF_CONFIG: Record<
+	number,
+	{ url: string; totalPages: number; pageOffset: number; name: string }
+> = {
 	1: {
 		url: "https://cdn.ooxml.dev/ecma-376/part1.pdf",
-		totalPages: 5560,
+		totalPages: 5026,
+		pageOffset: 10,
 		name: "Fundamentals",
 	},
 	2: {
 		url: "https://cdn.ooxml.dev/ecma-376/part2.pdf",
-		totalPages: 129,
+		totalPages: 137,
+		pageOffset: 8,
 		name: "OPC",
 	},
 	3: {
 		url: "https://cdn.ooxml.dev/ecma-376/part3.pdf",
-		totalPages: 65,
+		totalPages: 44,
+		pageOffset: 6,
 		name: "Compatibility",
 	},
 	4: {
 		url: "https://cdn.ooxml.dev/ecma-376/part4.pdf",
-		totalPages: 4031,
+		totalPages: 1548,
+		pageOffset: 14,
 		name: "Transitional",
 	},
 };
 
 interface PdfViewerProps {
 	partNumber: number;
-	pageNumber: number;
-	onPageChange?: (page: number) => void;
+	/** Printed page from the spec, as stored in `spec_content.page_number`. Null opens the cover. */
+	pageNumber: number | null;
+	/** Fires with the printed page now shown, or null while inside the front matter. */
+	onPageChange?: (printedPage: number | null) => void;
 }
 
 export function PdfViewer({ partNumber, pageNumber, onPageChange }: PdfViewerProps) {
 	const config = PDF_CONFIG[partNumber] || PDF_CONFIG[1];
-	const [currentPage, setCurrentPage] = useState(pageNumber);
+
+	// State is the physical sheet - that is what `#page=` and the scrubber address.
+	const toSheet = useCallback(
+		(printed: number | null) =>
+			printed === null ? 1 : Math.max(1, Math.min(printed + config.pageOffset, config.totalPages)),
+		[config.pageOffset, config.totalPages],
+	);
+	const toPrinted = useCallback(
+		(sheet: number) => (sheet > config.pageOffset ? sheet - config.pageOffset : null),
+		[config.pageOffset],
+	);
+
+	const [currentSheet, setCurrentSheet] = useState(() => toSheet(pageNumber));
 	const [isDragging, setIsDragging] = useState(false);
 	const progressRef = useRef<HTMLDivElement>(null);
 
 	// Sync with prop changes
 	useEffect(() => {
-		setCurrentPage(pageNumber);
-	}, [pageNumber]);
-
-	const updatePage = useCallback(
-		(newPage: number) => {
-			const clamped = Math.max(1, Math.min(newPage, config.totalPages));
-			setCurrentPage(clamped);
-			onPageChange?.(clamped);
+		setCurrentSheet(toSheet(pageNumber));
+	}, [pageNumber, toSheet]);
+
+	const updateSheet = useCallback(
+		(newSheet: number) => {
+			const clamped = Math.max(1, Math.min(newSheet, config.totalPages));
+			setCurrentSheet(clamped);
+			onPageChange?.(toPrinted(clamped));
 		},
-		[config.totalPages, onPageChange],
+		[config.totalPages, onPageChange, toPrinted],
 	);
 
-	const handlePrev = () => updatePage(currentPage - 1);
-	const handleNext = () => updatePage(currentPage + 1);
+	const handlePrev = () => updateSheet(currentSheet - 1);
+	const handleNext = () => updateSheet(currentSheet + 1);
 
 	// Progress bar interaction
-	const getPageFromPosition = useCallback(
+	const getSheetFromPosition = useCallback(
 		(clientX: number) => {
-			if (!progressRef.current) return currentPage;
+			if (!progressRef.current) return currentSheet;
 			const rect = progressRef.current.getBoundingClientRect();
 			const ratio = Math.max(0, Math.min(1, (clientX - rect.left) / rect.width));
 			return Math.max(1, Math.round(ratio * config.totalPages));
 		},
-		[config.totalPages, currentPage],
+		[config.totalPages, currentSheet],
 	);
 
 	const handleProgressClick = (e: React.MouseEvent) => {
-		updatePage(getPageFromPosition(e.clientX));
+		updateSheet(getSheetFromPosition(e.clientX));
 	};
 
 	const handleDragStart = (e: React.MouseEvent) => {
@@ -77,7 +108,7 @@ export function PdfViewer({ partNumber, pageNumber, onPageChange }: PdfViewerPro
 		if (!isDragging) return;
 
 		const handleMove = (e: MouseEvent) => {
-			updatePage(getPageFromPosition(e.clientX));
+			updateSheet(getSheetFromPosition(e.clientX));
 		};
 
 		const handleUp = () => {
@@ -91,35 +122,37 @@ export function PdfViewer({ partNumber, pageNumber, onPageChange }: PdfViewerPro
 			document.removeEventListener("mousemove", handleMove);
 			document.removeEventListener("mouseup", handleUp);
 		};
-	}, [isDragging, getPageFromPosition, updatePage]);
+	}, [isDragging, getSheetFromPosition, updateSheet]);
 
 	// Keyboard navigation
 	useEffect(() => {
 		const handleKeyDown = (e: KeyboardEvent) => {
 			if (e.key === "ArrowLeft") {
 				e.preventDefault();
-				setCurrentPage((p) => {
-					const newPage = Math.max(1, p - 1);
-					onPageChange?.(newPage);
-					return newPage;
+				setCurrentSheet((p) => {
+					const newSheet = Math.max(1, p - 1);
+					onPageChange?.(toPrinted(newSheet));
+					return newSheet;
 				});
 			}
 			if (e.key === "ArrowRight") {
 				e.preventDefault();
-				setCurrentPage((p) => {
-					const newPage = Math.min(config.totalPages, p + 1);
-					onPageChange?.(newPage);
-					return newPage;
+				setCurrentSheet((p) => {
+					const newSheet = Math.min(config.totalPages, p + 1);
+					onPageChange?.(toPrinted(newSheet));
+					return newSheet;
 				});
 			}
 		};
 
 		document.addEventListener("keydown", handleKeyDown);
 		return () => document.removeEventListener("keydown", handleKeyDown);
-	}, [config.totalPages, onPageChange]);
+	}, [config.totalPages, onPageChange, toPrinted]);
 
-	const progressPercent = (currentPage / config.totalPages) * 100;
-	const pdfUrl = `${config.url}#page=${currentPage}&toolbar=0&navpanes=0`;
+	const progressPercent = (currentSheet / config.totalPages) * 100;
+	const printedPage = toPrinted(currentSheet);
+	const printedTotal = config.totalPages - config.pageOffset;
+	const pdfUrl = `${config.url}#page=${currentSheet}&toolbar=0&navpanes=0`;
 
 	return (
 		<div className="flex h-full flex-col bg-[var(--color-bg-secondary)]">
@@ -139,7 +172,7 @@ export function PdfViewer({ partNumber, pageNumber, onPageChange }: PdfViewerPro
 							<button
 								type="button"
 								onClick={handlePrev}
-								disabled={currentPage <= 1}
+								disabled={currentSheet <= 1}
 								className="flex h-8 w-8 items-center justify-center rounded-lg bg-[var(--color-bg-tertiary)] text-[var(--color-text-secondary)] transition hover:bg-[var(--color-border)] hover:text-[var(--color-text-primary)] disabled:opacity-40 disabled:cursor-not-allowed"
 								aria-label="Previous page"
 							>
@@ -148,16 +181,24 @@ export function PdfViewer({ partNumber, pageNumber, onPageChange }: PdfViewerPro
 							<button
 								type="button"
 								onClick={handleNext}
-								disabled={currentPage >= config.totalPages}
+								disabled={currentSheet >= config.totalPages}
 								className="flex h-8 w-8 items-center justify-center rounded-lg bg-[var(--color-bg-tertiary)] text-[var(--color-text-secondary)] transition hover:bg-[var(--color-border)] hover:text-[var(--color-text-primary)] disabled:opacity-40 disabled:cursor-not-allowed"
 								aria-label="Next page"
 							>
 								→
 							</button>
 						</div>
 						<div className="flex items-baseline gap-1 text-sm">
-							<span className="font-semibold text-[var(--color-text-primary)]">{currentPage}</span>
-							<span className="text-[var(--color-text-muted)]">of {config.totalPages}</span>
+							{printedPage === null ? (
+								<span className="text-[var(--color-text-muted)]">Front matter</span>
+							) : (
+								<>
+									<span className="font-semibold text-[var(--color-text-primary)]">
+										{printedPage}
+									</span>
+									<span className="text-[var(--color-text-muted)]">of {printedTotal}</span>
+								</>
+							)}
 						</div>
 					</div>
 				</div>
@@ -186,7 +227,11 @@ export function PdfViewer({ partNumber, pageNumber, onPageChange }: PdfViewerPro
 					key={pdfUrl}
 					src={pdfUrl}
 					className="h-full w-full border-0"
-					title={`ECMA-376 Part ${partNumber} - Page ${currentPage}`}
+					title={
+						printedPage === null
+							? `ECMA-376 Part ${partNumber} - front matter`
+							: `ECMA-376 Part ${partNumber} - page ${printedPage}`
+					}
 				/>
 			</div>
 		</div>

apps/web/src/pages/SpecExplorer.tsx

@@ -201,10 +201,9 @@ export function SpecExplorer() {
 		return () => document.removeEventListener("keydown", handleKeyDown);
 	}, [results]);
 
-	// Handle page change from PDF viewer
-	const handlePageChange = useCallback((page: number) => {
+	// Handle page change from PDF viewer (printed spec page, null in front matter)
+	const handlePageChange = useCallback((_printedPage: number | null) => {
 		// Could sync page changes back to state if needed
-		console.log("Page changed to:", page);
 	}, []);
 
 	return (
@@ -309,7 +308,7 @@ export function SpecExplorer() {
 				<div className="overflow-hidden">
 					<PdfViewer
 						partNumber={selectedResult?.partNumber ?? 1}
-						pageNumber={selectedResult?.pageNumber ?? 1}
+						pageNumber={selectedResult?.pageNumber ?? null}
 						onPageChange={handlePageChange}
 					/>
 				</div>

db/migrations/0006_remove_toc_chunks.sql

@@ -0,0 +1,46 @@
+-- Remove table-of-contents entries from the prose corpus.
+--
+-- extract.py matched section headings by their bold styling. In the ingested
+-- markdown the contents listing is bold too, so every TOC line became its own
+-- "section": a title, a section ID, leader dots, and the page number it points
+-- at. Those rows were embedded and uploaded alongside real spec prose.
+--
+-- They are actively harmful to search. A TOC row is pure title text, so it
+-- matches a title-shaped query better than the actual prose does and outranks
+-- it - while carrying the page number of the contents listing itself, typically
+-- 50-100 pages from the section it names. Measured across 12 representative
+-- queries, 55% of returned results were TOC rows.
+--
+-- The matching fix is in scripts/ingest-pdf/extract.py (see `looks_like_toc`),
+-- which keeps them out of future ingests. This migration clears the rows that
+-- are already in the database.
+--
+-- Predicate: a short chunk that opens with a section ID and either carries
+-- leader dots, or is a single unbroken line ending in a page number. Verified
+-- against 459 production rows sampled across Part 1: 217 deleted, zero real
+-- prose chunks caught, zero TOC rows left behind.
+--
+-- Idempotent: re-running deletes nothing further.
+
+-- Dry run - inspect before applying:
+--
+--   SELECT part_number, count(*)
+--   FROM spec_content
+--   WHERE <the WHERE clause below>
+--   GROUP BY part_number ORDER BY part_number;
+
+DELETE FROM spec_content
+WHERE length(btrim(content)) < 400
+  -- opens with a section ID: "17.3.1.12 ..." or "**17.3.1.12** ..."
+  AND btrim(content) ~ '^\*{0,2}[[:space:]]*[0-9]+(\.[0-9]+)*\.?\*{0,2}[[:space:]]'
+  AND (
+    -- leader dots are unambiguous
+    btrim(content) ~ '\.{4,}'
+    -- or a single unbroken line ending in a page number, for entries whose
+    -- title ran long enough to swallow the dots. The blank-line test keeps
+    -- real prose (which has paragraph breaks) out of the match.
+    OR (
+      btrim(content) ~ '[.[:space:]][0-9]{1,4}[[:space:]]*\*{0,2}$'
+      AND strpos(content, E'\n\n') = 0
+    )
+  );

scripts/ingest-pdf/README.md

@@ -43,11 +43,36 @@ bun run pdf:upload   1                            ./embedded/part1.json
 Useful when iterating on chunking or trying a different embedding provider
 without re-extracting.
 
+## Page numbers
+
+These PDFs carry two numbering systems, and mixing them up sends every search
+result to the wrong page:
+
+- **physical page** - the sheet index, what a viewer's `#page=N` addresses.
+- **printed page** - the number in the running header, which restarts at 1
+  after the roman-numeral front matter.
+
+`extract.py` extracts with `page_chunks=True`, so each line's physical page is
+known exactly rather than guessed from stray digits in the text. It measures the
+front-matter offset from the running headers and writes it to
+`metadata.json` as `pageOffset`.
+
+`spec_content.page_number` stores the **printed** page - the number a reader
+sees on the page and cites. `PdfViewer` adds the part's `pageOffset` back when
+building the `#page=` fragment.
+
+Every extraction run prints the `totalPages` / `pageOffset` pair for that part.
+When a PDF is replaced (a new edition, a re-paginated release), copy those two
+numbers into `PDF_CONFIG` in `apps/web/src/components/PdfViewer.tsx`.
+
+Section bodies keep inline `<!--page:N-->` markers so `chunk.ts` can attribute
+each chunk to the page it actually falls on rather than to its section's first
+page. The markers are stripped from stored content and embedding text.
+
 ## Files
 
 - `pipeline.ts` - orchestrator (extract -> chunk -> embed -> upload)
 - `extract.py` - PDF -> section-aware markdown via pymupdf4llm
-- `fix-page-numbers.py` - PDF prelude-aware page-number alignment
 - `chunk.ts` - markdown -> 6 KB chunks with section IDs
 - `embed.ts` - chunks -> chunks + 1024-dim embeddings
 - `upload.ts` - bulk insert into `spec_content`