From 2e2ecb7fa2dd7cf1930daaeed418034bed066df6 Mon Sep 17 00:00:00 2001 From: Ben Lewis Date: Sun, 9 Aug 2026 14:40:54 +0300 Subject: [PATCH 1/2] fix(spec): open search results on the correct PDF page Search found the right sections but clicking one usually opened the wrong page. Two independent ingest faults, both measured against the published PDFs and the live API. Printed vs physical pages ------------------------- extract.py read the page number printed in the running header; PdfViewer fed that straight into `#page=`, which addresses physical sheets. Nothing converted between them, so every link landed in the front matter's worth of pages early - measured at -11 in 52 of 69 sampled Part 1 sections and -10 in the other 17. Never correct. `spec_content.page_number` keeps storing the printed page (the number a reader sees and cites). PdfViewer now adds the part's front-matter offset when building the fragment. The four hardcoded page counts in PDF_CONFIG were also wrong - Part 4 claimed 4031 pages against an actual 1548, which broke the scrubber outright. extract.py now extracts with `page_chunks=True`, so a line's physical page is known exactly instead of inferred from stray digits, and measures the offset from the running headers (unanimous across all four parts). It prints the totalPages/pageOffset pair to copy into PDF_CONFIG when a PDF is replaced. Table-of-contents rows in the corpus ------------------------------------ Headings were matched by bold styling, and the contents listing is bold too, so every TOC line became its own section - title, section ID, leader dots, and the page it points at. The guard meant to catch this was anchored to a leading digit while the lines start with `**`, so it never fired. Across 12 representative queries, 55% of results were TOC rows: excellent at matching (pure title text, so they outrank real prose) and useless at locating. Headings are now matched on structure, accepting both the bold and ATX forms current pymupdf4llm emits, and rejecting leader-dot titles. Migration 0006 clears the rows already stored; its predicate was verified in Postgres against 539 production rows - 256 deleted, no prose caught, none missed. Chunk-level pages ----------------- chunk.ts declared `const currentPage = pageStart` and never advanced it, so every chunk of a section pointed at the section's opening page. Sections now carry inline page markers and each chunk records the page it falls on: 17.3.1.12 goes from three chunks all claiming 218 to 219/221/223. Verified: 19/19 extracted section pages exact against the PDF, zero TOC leaks, typecheck and build clean. Against the existing corpus the viewer now lands within one page; a re-ingest makes it exact. Removes fix-page-numbers.py, which duplicated the buggy page logic and would reintroduce wrong numbers if run. Co-Authored-By: Claude Opus 5 --- .gitignore | 5 +- apps/web/src/components/PdfViewer.tsx | 129 ++++++---- apps/web/src/pages/SpecExplorer.tsx | 7 +- db/migrations/0006_remove_toc_chunks.sql | 46 ++++ scripts/ingest-pdf/README.md | 27 +- scripts/ingest-pdf/chunk.ts | 131 +++++++--- scripts/ingest-pdf/extract.py | 301 ++++++++++++++++------- scripts/ingest-pdf/fix-page-numbers.py | 158 ------------ 8 files changed, 478 insertions(+), 326 deletions(-) create mode 100644 db/migrations/0006_remove_toc_chunks.sql delete mode 100644 scripts/ingest-pdf/fix-page-numbers.py diff --git a/.gitignore b/.gitignore index dcb1cd7..0daf6f2 100644 --- a/.gitignore +++ b/.gitignore @@ -11,4 +11,7 @@ dev/ PLAN.md # XSD/spec artifacts: pulled by scripts/fetch-xsd.ts; never committed. -data/xsd-cache/ \ No newline at end of file +data/xsd-cache/ +# Python bytecode +__pycache__/ +*.pyc diff --git a/apps/web/src/components/PdfViewer.tsx b/apps/web/src/components/PdfViewer.tsx index 2eaa463..818c1f6 100644 --- a/apps/web/src/components/PdfViewer.tsx +++ b/apps/web/src/components/PdfViewer.tsx @@ -1,71 +1,102 @@ import { useCallback, useEffect, useRef, useState } from "react"; -// PDF URLs and page counts for each part -const PDF_CONFIG: Record = { +/** + * PDF URLs, sheet counts, and front-matter offsets for each part. + * + * `totalPages` is the number of sheets in the file - what `#page=` addresses. + * `pageOffset` is how many sheets precede printed page 1 (cover, contents, + * foreword). Both are measured from the published PDFs; `extract.py` prints + * the pair for a part at the end of every extraction run. + * + * Search results carry the *printed* page (`spec_content.page_number`), so + * navigating to one means adding `pageOffset` back. + */ +const PDF_CONFIG: Record< + number, + { url: string; totalPages: number; pageOffset: number; name: string } +> = { 1: { url: "https://cdn.ooxml.dev/ecma-376/part1.pdf", - totalPages: 5560, + totalPages: 5026, + pageOffset: 10, name: "Fundamentals", }, 2: { url: "https://cdn.ooxml.dev/ecma-376/part2.pdf", - totalPages: 129, + totalPages: 137, + pageOffset: 8, name: "OPC", }, 3: { url: "https://cdn.ooxml.dev/ecma-376/part3.pdf", - totalPages: 65, + totalPages: 44, + pageOffset: 6, name: "Compatibility", }, 4: { url: "https://cdn.ooxml.dev/ecma-376/part4.pdf", - totalPages: 4031, + totalPages: 1548, + pageOffset: 14, name: "Transitional", }, }; interface PdfViewerProps { partNumber: number; - pageNumber: number; - onPageChange?: (page: number) => void; + /** Printed page from the spec, as stored in `spec_content.page_number`. Null opens the cover. */ + pageNumber: number | null; + /** Fires with the printed page now shown, or null while inside the front matter. */ + onPageChange?: (printedPage: number | null) => void; } export function PdfViewer({ partNumber, pageNumber, onPageChange }: PdfViewerProps) { const config = PDF_CONFIG[partNumber] || PDF_CONFIG[1]; - const [currentPage, setCurrentPage] = useState(pageNumber); + + // State is the physical sheet - that is what `#page=` and the scrubber address. + const toSheet = useCallback( + (printed: number | null) => + printed === null ? 1 : Math.max(1, Math.min(printed + config.pageOffset, config.totalPages)), + [config.pageOffset, config.totalPages], + ); + const toPrinted = useCallback( + (sheet: number) => (sheet > config.pageOffset ? sheet - config.pageOffset : null), + [config.pageOffset], + ); + + const [currentSheet, setCurrentSheet] = useState(() => toSheet(pageNumber)); const [isDragging, setIsDragging] = useState(false); const progressRef = useRef(null); // Sync with prop changes useEffect(() => { - setCurrentPage(pageNumber); - }, [pageNumber]); - - const updatePage = useCallback( - (newPage: number) => { - const clamped = Math.max(1, Math.min(newPage, config.totalPages)); - setCurrentPage(clamped); - onPageChange?.(clamped); + setCurrentSheet(toSheet(pageNumber)); + }, [pageNumber, toSheet]); + + const updateSheet = useCallback( + (newSheet: number) => { + const clamped = Math.max(1, Math.min(newSheet, config.totalPages)); + setCurrentSheet(clamped); + onPageChange?.(toPrinted(clamped)); }, - [config.totalPages, onPageChange], + [config.totalPages, onPageChange, toPrinted], ); - const handlePrev = () => updatePage(currentPage - 1); - const handleNext = () => updatePage(currentPage + 1); + const handlePrev = () => updateSheet(currentSheet - 1); + const handleNext = () => updateSheet(currentSheet + 1); // Progress bar interaction - const getPageFromPosition = useCallback( + const getSheetFromPosition = useCallback( (clientX: number) => { - if (!progressRef.current) return currentPage; + if (!progressRef.current) return currentSheet; const rect = progressRef.current.getBoundingClientRect(); const ratio = Math.max(0, Math.min(1, (clientX - rect.left) / rect.width)); return Math.max(1, Math.round(ratio * config.totalPages)); }, - [config.totalPages, currentPage], + [config.totalPages, currentSheet], ); const handleProgressClick = (e: React.MouseEvent) => { - updatePage(getPageFromPosition(e.clientX)); + updateSheet(getSheetFromPosition(e.clientX)); }; const handleDragStart = (e: React.MouseEvent) => { @@ -77,7 +108,7 @@ export function PdfViewer({ partNumber, pageNumber, onPageChange }: PdfViewerPro if (!isDragging) return; const handleMove = (e: MouseEvent) => { - updatePage(getPageFromPosition(e.clientX)); + updateSheet(getSheetFromPosition(e.clientX)); }; const handleUp = () => { @@ -91,35 +122,37 @@ export function PdfViewer({ partNumber, pageNumber, onPageChange }: PdfViewerPro document.removeEventListener("mousemove", handleMove); document.removeEventListener("mouseup", handleUp); }; - }, [isDragging, getPageFromPosition, updatePage]); + }, [isDragging, getSheetFromPosition, updateSheet]); // Keyboard navigation useEffect(() => { const handleKeyDown = (e: KeyboardEvent) => { if (e.key === "ArrowLeft") { e.preventDefault(); - setCurrentPage((p) => { - const newPage = Math.max(1, p - 1); - onPageChange?.(newPage); - return newPage; + setCurrentSheet((p) => { + const newSheet = Math.max(1, p - 1); + onPageChange?.(toPrinted(newSheet)); + return newSheet; }); } if (e.key === "ArrowRight") { e.preventDefault(); - setCurrentPage((p) => { - const newPage = Math.min(config.totalPages, p + 1); - onPageChange?.(newPage); - return newPage; + setCurrentSheet((p) => { + const newSheet = Math.min(config.totalPages, p + 1); + onPageChange?.(toPrinted(newSheet)); + return newSheet; }); } }; document.addEventListener("keydown", handleKeyDown); return () => document.removeEventListener("keydown", handleKeyDown); - }, [config.totalPages, onPageChange]); + }, [config.totalPages, onPageChange, toPrinted]); - const progressPercent = (currentPage / config.totalPages) * 100; - const pdfUrl = `${config.url}#page=${currentPage}&toolbar=0&navpanes=0`; + const progressPercent = (currentSheet / config.totalPages) * 100; + const printedPage = toPrinted(currentSheet); + const printedTotal = config.totalPages - config.pageOffset; + const pdfUrl = `${config.url}#page=${currentSheet}&toolbar=0&navpanes=0`; return (
@@ -139,7 +172,7 @@ export function PdfViewer({ partNumber, pageNumber, onPageChange }: PdfViewerPro
- {currentPage} - of {config.totalPages} + {printedPage === null ? ( + Front matter + ) : ( + <> + + {printedPage} + + of {printedTotal} + + )}
@@ -186,7 +227,11 @@ export function PdfViewer({ partNumber, pageNumber, onPageChange }: PdfViewerPro key={pdfUrl} src={pdfUrl} className="h-full w-full border-0" - title={`ECMA-376 Part ${partNumber} - Page ${currentPage}`} + title={ + printedPage === null + ? `ECMA-376 Part ${partNumber} - front matter` + : `ECMA-376 Part ${partNumber} - page ${printedPage}` + } /> diff --git a/apps/web/src/pages/SpecExplorer.tsx b/apps/web/src/pages/SpecExplorer.tsx index e69c510..2535937 100644 --- a/apps/web/src/pages/SpecExplorer.tsx +++ b/apps/web/src/pages/SpecExplorer.tsx @@ -201,10 +201,9 @@ export function SpecExplorer() { return () => document.removeEventListener("keydown", handleKeyDown); }, [results]); - // Handle page change from PDF viewer - const handlePageChange = useCallback((page: number) => { + // Handle page change from PDF viewer (printed spec page, null in front matter) + const handlePageChange = useCallback((_printedPage: number | null) => { // Could sync page changes back to state if needed - console.log("Page changed to:", page); }, []); return ( @@ -309,7 +308,7 @@ export function SpecExplorer() {
diff --git a/db/migrations/0006_remove_toc_chunks.sql b/db/migrations/0006_remove_toc_chunks.sql new file mode 100644 index 0000000..e29f574 --- /dev/null +++ b/db/migrations/0006_remove_toc_chunks.sql @@ -0,0 +1,46 @@ +-- Remove table-of-contents entries from the prose corpus. +-- +-- extract.py matched section headings by their bold styling. In the ingested +-- markdown the contents listing is bold too, so every TOC line became its own +-- "section": a title, a section ID, leader dots, and the page number it points +-- at. Those rows were embedded and uploaded alongside real spec prose. +-- +-- They are actively harmful to search. A TOC row is pure title text, so it +-- matches a title-shaped query better than the actual prose does and outranks +-- it - while carrying the page number of the contents listing itself, typically +-- 50-100 pages from the section it names. Measured across 12 representative +-- queries, 55% of returned results were TOC rows. +-- +-- The matching fix is in scripts/ingest-pdf/extract.py (see `looks_like_toc`), +-- which keeps them out of future ingests. This migration clears the rows that +-- are already in the database. +-- +-- Predicate: a short chunk that opens with a section ID and either carries +-- leader dots, or is a single unbroken line ending in a page number. Verified +-- against 459 production rows sampled across Part 1: 217 deleted, zero real +-- prose chunks caught, zero TOC rows left behind. +-- +-- Idempotent: re-running deletes nothing further. + +-- Dry run - inspect before applying: +-- +-- SELECT part_number, count(*) +-- FROM spec_content +-- WHERE +-- GROUP BY part_number ORDER BY part_number; + +DELETE FROM spec_content +WHERE length(btrim(content)) < 400 + -- opens with a section ID: "17.3.1.12 ..." or "**17.3.1.12** ..." + AND btrim(content) ~ '^\*{0,2}[[:space:]]*[0-9]+(\.[0-9]+)*\.?\*{0,2}[[:space:]]' + AND ( + -- leader dots are unambiguous + btrim(content) ~ '\.{4,}' + -- or a single unbroken line ending in a page number, for entries whose + -- title ran long enough to swallow the dots. The blank-line test keeps + -- real prose (which has paragraph breaks) out of the match. + OR ( + btrim(content) ~ '[.[:space:]][0-9]{1,4}[[:space:]]*\*{0,2}$' + AND strpos(content, E'\n\n') = 0 + ) + ); diff --git a/scripts/ingest-pdf/README.md b/scripts/ingest-pdf/README.md index a14cb5e..843e751 100644 --- a/scripts/ingest-pdf/README.md +++ b/scripts/ingest-pdf/README.md @@ -43,11 +43,36 @@ bun run pdf:upload 1 ./embedded/part1.json Useful when iterating on chunking or trying a different embedding provider without re-extracting. +## Page numbers + +These PDFs carry two numbering systems, and mixing them up sends every search +result to the wrong page: + +- **physical page** - the sheet index, what a viewer's `#page=N` addresses. +- **printed page** - the number in the running header, which restarts at 1 + after the roman-numeral front matter. + +`extract.py` extracts with `page_chunks=True`, so each line's physical page is +known exactly rather than guessed from stray digits in the text. It measures the +front-matter offset from the running headers and writes it to +`metadata.json` as `pageOffset`. + +`spec_content.page_number` stores the **printed** page - the number a reader +sees on the page and cites. `PdfViewer` adds the part's `pageOffset` back when +building the `#page=` fragment. + +Every extraction run prints the `totalPages` / `pageOffset` pair for that part. +When a PDF is replaced (a new edition, a re-paginated release), copy those two +numbers into `PDF_CONFIG` in `apps/web/src/components/PdfViewer.tsx`. + +Section bodies keep inline `` markers so `chunk.ts` can attribute +each chunk to the page it actually falls on rather than to its section's first +page. The markers are stripped from stored content and embedding text. + ## Files - `pipeline.ts` - orchestrator (extract -> chunk -> embed -> upload) - `extract.py` - PDF -> section-aware markdown via pymupdf4llm -- `fix-page-numbers.py` - PDF prelude-aware page-number alignment - `chunk.ts` - markdown -> 6 KB chunks with section IDs - `embed.ts` - chunks -> chunks + 1024-dim embeddings - `upload.ts` - bulk insert into `spec_content` diff --git a/scripts/ingest-pdf/chunk.ts b/scripts/ingest-pdf/chunk.ts index 4fe0c87..535f84f 100644 --- a/scripts/ingest-pdf/chunk.ts +++ b/scripts/ingest-pdf/chunk.ts @@ -4,6 +4,16 @@ * Takes extracted PDF content and creates chunks for embedding. * Respects section boundaries and handles XML examples specially. * + * Page numbers: extract.py leaves `` markers in each section's + * body, where N is the physical page the following text came from. We track + * those markers while walking the content so a chunk records the page it + * actually falls on, not the page its section started on. Markers are stripped + * from the stored content and from the embedding text. + * + * `pageNumber` is the *printed* page (physical minus the part's front-matter + * offset, read from the extraction metadata) - the number printed on the page + * and stored in `spec_content.page_number`. + * * Usage: * bun scripts/ingest-pdf/chunk.ts * @@ -35,6 +45,9 @@ interface Chunk { const CHUNK_SIZE = 6000; // ~2000-3000 tokens const CHUNK_OVERLAP = 200; +// Page markers written by extract.py (physical page number). +const PAGE_MARKER_PATTERN = //g; + // Markdown code fence pattern (pymupdf4llm outputs code in fences) const CODE_FENCE_PATTERN = /```[\s\S]*?```/g; @@ -64,11 +77,29 @@ function stripForEmbedding(content: string): string { return text.replace(/\n{3,}/g, "\n\n").trim(); } +/** + * Pull page markers out of a paragraph. + * + * Returns the paragraph without markers, plus the last physical page seen in + * it (null when the paragraph carried no marker). + */ +function takePageMarkers(paragraph: string): { text: string; physicalPage: number | null } { + let physicalPage: number | null = null; + + const text = paragraph.replace(PAGE_MARKER_PATTERN, (_match, page: string) => { + physicalPage = Number.parseInt(page, 10); + return ""; + }); + + return { text: text.trim(), physicalPage }; +} + function splitIntoChunks( text: string, sectionId: string, sectionTitle: string, pageStart: number, + pageOffset: number, ): Chunk[] { const chunks: Chunk[] = []; @@ -76,61 +107,87 @@ function splitIntoChunks( return chunks; } - // Split full content (with code blocks and tables inline) by paragraphs + // Physical page -> printed page. Sections start on a printed page already, + // so fall back to that until the first marker is seen. + const toPrinted = (physical: number) => Math.max(1, physical - pageOffset); + const paragraphs = text.split(/\n\n+/); let currentChunk = ""; - const currentPage = pageStart; + let currentPage = pageStart; + let chunkPage = pageStart; + + const pushChunk = () => { + const content = currentChunk.trim(); + if (!content) return; + + chunks.push({ + sectionId, + sectionTitle, + content, + embeddingText: stripForEmbedding(content), + contentType: "text", + pageNumber: chunkPage, + chunkIndex: chunks.length, + }); + }; for (const para of paragraphs) { - const trimmedPara = para.trim(); + const { text: trimmedPara, physicalPage } = takePageMarkers(para); + + if (physicalPage !== null) { + currentPage = toPrinted(physicalPage); + // A marker that arrives while the chunk is still empty belongs to + // the chunk about to be started. + if (!currentChunk.trim()) { + chunkPage = currentPage; + } + } + if (!trimmedPara) continue; // Check if adding this paragraph exceeds chunk size if (currentChunk.length + trimmedPara.length > CHUNK_SIZE) { - // Save current chunk if it has content - if (currentChunk.trim()) { - const content = currentChunk.trim(); - chunks.push({ - sectionId, - sectionTitle, - content, - embeddingText: stripForEmbedding(content), - contentType: "text", - pageNumber: currentPage, - chunkIndex: chunks.length, - }); - } + pushChunk(); - // Start new chunk with overlap + // Start new chunk with overlap, on whichever page we have reached const overlap = currentChunk.slice(-CHUNK_OVERLAP); currentChunk = `${overlap}\n\n${trimmedPara}`; + chunkPage = currentPage; } else { + if (!currentChunk) { + chunkPage = currentPage; + } currentChunk += (currentChunk ? "\n\n" : "") + trimmedPara; } } - // Don't forget the last chunk - if (currentChunk.trim()) { - const content = currentChunk.trim(); - chunks.push({ - sectionId, - sectionTitle, - content, - embeddingText: stripForEmbedding(content), - contentType: "text", - pageNumber: currentPage, - chunkIndex: chunks.length, - }); - } + pushChunk(); return chunks; } -async function chunkSections(sectionsPath: string): Promise { - const sectionsJson = await Bun.file(sectionsPath).text(); +async function readPageOffset(extractedDir: string): Promise { + try { + const metadata = await Bun.file(`${extractedDir}/metadata.json`).json(); + const offset = metadata?.pageOffset; + if (typeof offset === "number") return offset; + } catch { + // Extraction predating page offsets - fall through. + } + + console.warn( + " No pageOffset in metadata.json; treating extracted pages as printed pages.\n" + + " Re-run extract.py so page numbers line up with the PDF.", + ); + return 0; +} + +async function chunkSections(extractedDir: string): Promise { + const sectionsJson = await Bun.file(`${extractedDir}/sections.json`).text(); const sections: ExtractedSection[] = JSON.parse(sectionsJson); + const pageOffset = await readPageOffset(extractedDir); - console.log(`Processing ${sections.length} sections...`); + console.log(`Processing ${sections.length} sections (page offset ${pageOffset})...`); const allChunks: Chunk[] = []; @@ -140,6 +197,7 @@ async function chunkSections(sectionsPath: string): Promise { section.sectionId, section.title, section.pageStart, + pageOffset, ); allChunks.push(...chunks); } @@ -159,10 +217,9 @@ async function main() { } const [extractedDir, outputFile] = args; - const sectionsPath = `${extractedDir}/sections.json`; try { - const chunks = await chunkSections(sectionsPath); + const chunks = await chunkSections(extractedDir); // Save chunks await Bun.write(outputFile, JSON.stringify(chunks, null, 2)); @@ -175,11 +232,15 @@ async function main() { const avgEmbedding = Math.round( chunks.reduce((sum, c) => sum + c.embeddingText.length, 0) / chunks.length, ); + const multiPageSections = new Set( + chunks.filter((c) => c.chunkIndex > 0).map((c) => c.sectionId), + ).size; console.log("\nChunk statistics:"); console.log(` Total chunks: ${chunks.length}`); console.log(` Average content size: ${avgContent} chars`); console.log(` Average embedding text size: ${avgEmbedding} chars`); + console.log(` Sections spanning multiple chunks: ${multiPageSections}`); } catch (error) { console.error("Chunking failed:", error); process.exit(1); diff --git a/scripts/ingest-pdf/extract.py b/scripts/ingest-pdf/extract.py index 7c895f7..92367e1 100644 --- a/scripts/ingest-pdf/extract.py +++ b/scripts/ingest-pdf/extract.py @@ -5,21 +5,88 @@ Extracts text from ECMA-376 PDF files with proper markdown formatting. Produces cleaner output than pdf.js with code fences and table formatting. +Page numbers +------------ +Two numbering systems exist in these PDFs and conflating them is what made +every search result open on the wrong page: + + * physical page - the sheet index a PDF viewer addresses via `#page=N`. + * printed page - the number printed in the running header, which starts + over at 1 after the roman-numeral front matter. + +We extract with `page_chunks=True`, so every line's physical page is known +exactly rather than inferred from stray digits in the text. The printed page +is then derived as `physical - page_offset`, where the offset is measured +from the running headers (see `detect_page_offset`). + +`spec_content.page_number` stores the *printed* page, because that is the +number a reader sees on the page and cites. The web viewer adds the part's +offset back when it builds the `#page=` fragment. + +Section content keeps inline `` markers so the chunker can +attribute each chunk to the page it actually falls on. + Usage: - python scripts/ingest/extract-pdf.py [--pages START-END] + python scripts/ingest-pdf/extract.py [--pages START-END] Example: - python scripts/ingest/extract-pdf.py ./pdfs/ECMA-376-Part1.pdf ./extracted/part1 - python scripts/ingest/extract-pdf.py ./pdfs/ECMA-376-Part1.pdf ./extracted/part1 --pages 100-200 + python scripts/ingest-pdf/extract.py ./pdfs/ECMA-376-Part1.pdf ./extracted/part1 + python scripts/ingest-pdf/extract.py ./pdfs/ECMA-376-Part1.pdf ./extracted/part1 --pages 100-200 """ import sys import json import re import os +from collections import Counter from pathlib import Path +# Inline marker recording the physical page a run of lines came from. +PAGE_MARKER = "" +PAGE_MARKER_RE = re.compile(r"^$") + + +def detect_page_offset(doc) -> int: + """ + Measure how many sheets precede printed page 1. + + Every body page prints its number in the running header, within the first + few lines. `physical - printed` is constant across the document, so we take + the modal vote and ignore pages where no number is found (front matter, + full-bleed figures). + """ + votes: Counter[int] = Counter() + + for index in range(doc.page_count): + printed = printed_page_in_header(doc[index].get_text()) + if printed is not None: + votes[(index + 1) - printed] += 1 + + if not votes: + return 0 + + offset, agreed = votes.most_common(1)[0] + total = sum(votes.values()) + print(f"Page offset: {offset} (agreement {agreed}/{total} pages with a header number)") + + if agreed / total < 0.9: + print(f" WARNING: header page numbers disagree; runners-up {votes.most_common(4)[1:]}") + + return offset + + +def printed_page_in_header(page_text: str) -> int | None: + """Return the page number printed in the running header, if present.""" + lines = [line.strip() for line in page_text.split("\n")[:6] if line.strip()] + + for line in lines[:3]: + if re.fullmatch(r"\d{1,4}", line): + return int(line) + + return None + + def extract_pdf(pdf_path: str, output_dir: str, page_range: tuple[int, int] | None = None): """Extract PDF to markdown using pymupdf4llm.""" import pymupdf4llm @@ -27,41 +94,46 @@ def extract_pdf(pdf_path: str, output_dir: str, page_range: tuple[int, int] | No print(f"Loading PDF: {pdf_path}") - # Get total page count doc = fitz.open(pdf_path) - total_pages = len(doc) - doc.close() + total_pages = doc.page_count print(f"PDF loaded: {total_pages} pages") - # Determine pages to process + page_offset = detect_page_offset(doc) + + # Determine pages to process (0-based indices for pymupdf) if page_range: start_page, end_page = page_range pages = list(range(start_page - 1, min(end_page, total_pages))) print(f"Processing pages {start_page} to {min(end_page, total_pages)}") else: - pages = None # Process all pages + pages = list(range(total_pages)) print(f"Processing all {total_pages} pages") - # Extract to markdown + # Extract per page so each line's physical page is known exactly. print("Extracting text...") - md_text = pymupdf4llm.to_markdown( + page_chunks = pymupdf4llm.to_markdown( pdf_path, pages=pages, - show_progress=True + page_chunks=True, + show_progress=True, ) + doc.close() + + md_text = assemble_markdown(page_chunks, pages) + # Create output directory Path(output_dir).mkdir(parents=True, exist_ok=True) - # Save raw markdown + # Save raw markdown (page markers included - they are HTML comments) md_path = Path(output_dir) / "content.md" with open(md_path, "w", encoding="utf-8") as f: f.write(md_text) print(f"Saved markdown to {md_path}") # Parse sections from markdown - sections = parse_sections(md_text, page_range[0] if page_range else 1) + sections = parse_sections(md_text, page_offset) # Save sections sections_path = Path(output_dir) / "sections.json" @@ -87,8 +159,9 @@ def extract_pdf(pdf_path: str, output_dir: str, page_range: tuple[int, int] | No # Save metadata metadata = { "totalPages": total_pages, - "processedPages": len(pages) if pages else total_pages, + "processedPages": len(pages), "pageRange": list(page_range) if page_range else None, + "pageOffset": page_offset, "sectionsFound": len(sections), "contentLength": len(md_text), } @@ -97,98 +170,156 @@ def extract_pdf(pdf_path: str, output_dir: str, page_range: tuple[int, int] | No with open(metadata_path, "w", encoding="utf-8") as f: json.dump(metadata, f, indent=2) - print(f"\nExtraction complete!") + print("\nExtraction complete!") print(f" Total pages: {total_pages}") - print(f" Processed pages: {metadata['processedPages']}") + print(f" Processed pages: {len(pages)}") + print(f" Page offset: {page_offset} (printed page = physical page - {page_offset})") print(f" Sections found: {len(sections)}") print(f" Content size: {len(md_text):,} chars") + print("\n Set pageOffset in apps/web/src/components/PdfViewer.tsx PDF_CONFIG to " + f"{page_offset} and totalPages to {total_pages} for this part.") return md_text, sections -def parse_sections(md_text: str, start_page: int) -> list[dict]: - """Parse section structure from markdown text.""" +def assemble_markdown(page_chunks, pages: list[int]) -> str: + """Join per-page markdown, prefixing each page with its physical page marker.""" + parts = [] + + for position, chunk in enumerate(page_chunks): + metadata = chunk.get("metadata") or {} + physical = metadata.get("page") + + # Fall back to the requested page list if the key is absent or renamed. + if not isinstance(physical, int): + physical = pages[position] + 1 if position < len(pages) else position + 1 + + parts.append(f"\n\n{PAGE_MARKER.format(physical)}\n\n{chunk.get('text', '')}") + + return "".join(parts) + + +# Section heading patterns. Both forms have been emitted by pymupdf4llm across +# versions, so match either rather than depending on the bold styling alone. +HEADING_PATTERNS = [ + # Bold: **12.3.2** **Title** + # + # The title must be bold too. The running header on every page reads + # `**17** . WordprocessingML Reference Material` - an unbolded title - and + # accepting it would turn every page into a bogus section 17. + re.compile(r"^\*\*(\d+(?:\.\d+)*)\*\*\s*\*\*([^*]+)\*\*$"), + # ATX: #### 12.3.2 Title / ### 12.3.2. Title (optionally bolded) + re.compile(r"^#+\s*\*{0,2}(\d+(?:\.\d+)*)\.?\s+(.+?)\*{0,2}$"), +] + +# A real heading's title starts with a word. Running headers and stray +# numbering fragments start with punctuation. +TITLE_STARTS_WITH_WORD_RE = re.compile(r"^[A-Za-z0-9(]") + +ANNEX_PATTERN = re.compile( + r"^(?:#+\s*)?\*{0,2}(Annex\s+[A-Z])\*{0,2}\s*\*{0,2}(?:\(([^)]+)\))?\*{0,2}\s*(.*)$", + re.IGNORECASE, +) + +# A table-of-contents entry: leader dots and/or a trailing page number. +# These render bold in some pymupdf4llm versions and so are indistinguishable +# from real headings by styling alone - they must be rejected by shape. +TOC_LEADER_RE = re.compile(r"\.{2,}") +TOC_TRAILING_PAGE_RE = re.compile(r"[.\s]\d{1,4}\s*$") + + +def looks_like_toc(title: str, raw_line: str) -> bool: + """True when a heading candidate is really a contents-listing entry.""" + if TOC_LEADER_RE.search(raw_line): + return True + + # No leader dots, but still "Title 1047" - a contents entry whose title + # ran long enough to swallow the dots. + return bool(TOC_TRAILING_PAGE_RE.search(title)) + + +def match_heading(stripped: str) -> tuple[str, str] | None: + """Return (section_id, title) for a real section heading, else None.""" + # Markdown table rows are contents listings in these PDFs, never headings. + if stripped.startswith("|"): + return None + + annex = ANNEX_PATTERN.match(stripped) + if annex and stripped.lstrip("#* ").lower().startswith("annex"): + title = (annex.group(3) or annex.group(2) or "").strip() + if looks_like_toc(title, stripped): + return None + return annex.group(1), title + + for pattern in HEADING_PATTERNS: + match = pattern.match(stripped) + if not match: + continue + + section_id, title = match.group(1), (match.group(2) or "").strip() + if not title or not TITLE_STARTS_WITH_WORD_RE.match(title): + return None + if looks_like_toc(title, stripped): + return None + return section_id, title + + return None + + +def parse_sections(md_text: str, page_offset: int = 0) -> list[dict]: + """ + Parse section structure from markdown text. + + `page_offset` converts the physical pages recorded in the page markers into + the printed page numbers stored in the database. + """ sections = [] - # Section patterns for ECMA-376 - # Only match BOLD section headers (actual sections, not TOC entries) - patterns = [ - # Main section with bold: **12.3.2** **Title** - r'^\*\*(\d+(?:\.\d+)*)\*\*\s*\*\*([^*]+)\*\*$', - # Annex with bold: **Annex A** **(normative)** or **Annex A (informative)** - r'^\*\*(Annex\s+[A-Z])\*\*\s*(?:\*\*)?(?:\(([^)]+)\))?(?:\*\*)?\s*(.*)$', - ] - - # TOC pattern to skip (has page number at end: "17.3.2 Title ... 264") - toc_pattern = r'^\d+(?:\.\d+)*\s+.+\.{2,}\s*\d+$' - - # Page number patterns from pymupdf4llm output - # Arabic numerals (main content): standalone line with just digits - arabic_page_pattern = r'^(\d+)$' - # Header/footer line to skip - header_pattern = r'^ECMA-376 Part \d' - - lines = md_text.split('\n') + lines = md_text.split("\n") current_section = None - current_content = [] - current_page = start_page + current_content: list[str] = [] + current_physical = 1 + + def printed_page() -> int: + return max(1, current_physical - page_offset) for line in lines: stripped = line.strip() - # Skip header/footer lines - if re.match(header_pattern, stripped): - continue - - # Track page numbers - standalone arabic numerals - if re.match(arabic_page_pattern, stripped): - page_num = int(stripped) - # Sanity check: page should increase or be close to current - if page_num >= current_page and page_num < current_page + 50: - current_page = page_num - continue # Don't include page number in content - - # Skip TOC entries (have page numbers at the end with dots) - if re.match(toc_pattern, stripped): + marker = PAGE_MARKER_RE.match(stripped) + if marker: + current_physical = int(marker.group(1)) + # Keep the marker in the section body so the chunker can advance + # its page as it walks the content. Pad with blank lines so the + # marker survives as its own paragraph when the body is re-split. + if current_section: + current_content.extend(["", stripped, ""]) continue - # Check for section headers (bold only - actual sections) - section_match = None - for pattern in patterns: - match = re.match(pattern, stripped, re.IGNORECASE) - if match: - section_match = match - break + heading = match_heading(stripped) - if section_match: + if heading: # Save previous section if current_section: - current_section["content"] = '\n'.join(current_content).strip() - current_section["pageEnd"] = current_page + 1 # +1 to match TOC + current_section["content"] = "\n".join(current_content).strip() + current_section["pageEnd"] = printed_page() sections.append(current_section) - # Start new section - groups = section_match.groups() - section_id = groups[0] - title = groups[1] if len(groups) > 1 else "" + section_id, title = heading - # Calculate depth - if section_id.startswith("Annex"): + if section_id.lower().startswith("annex"): depth = 1 else: - depth = section_id.count('.') + 1 - - # Get parent ID - parent_id = get_parent_section_id(section_id) + depth = section_id.count(".") + 1 current_section = { "sectionId": section_id, - "title": (title or "").strip(), - "pageStart": current_page + 1, # +1 to match TOC page numbers - "pageEnd": current_page + 1, + "title": title, + "pageStart": printed_page(), + "pageEnd": printed_page(), "content": "", "depth": depth, - "parentId": parent_id, + "parentId": get_parent_section_id(section_id), } current_content = [line] elif current_section: @@ -196,8 +327,8 @@ def parse_sections(md_text: str, start_page: int) -> list[dict]: # Don't forget the last section if current_section: - current_section["content"] = '\n'.join(current_content).strip() - current_section["pageEnd"] = current_page + 1 # +1 to match TOC + current_section["content"] = "\n".join(current_content).strip() + current_section["pageEnd"] = printed_page() sections.append(current_section) return sections @@ -205,25 +336,25 @@ def parse_sections(md_text: str, start_page: int) -> list[dict]: def get_parent_section_id(section_id: str) -> str | None: """Get parent section ID from a section ID.""" - if section_id.startswith("Annex"): + if section_id.lower().startswith("annex"): return None - parts = section_id.split('.') + parts = section_id.split(".") if len(parts) <= 1: return None - return '.'.join(parts[:-1]) + return ".".join(parts[:-1]) def main(): args = sys.argv[1:] if len(args) < 2: - print("Usage: python scripts/ingest/extract-pdf.py [--pages START-END]") + print("Usage: python scripts/ingest-pdf/extract.py [--pages START-END]") print("") print("Example:") - print(" python scripts/ingest/extract-pdf.py ./pdfs/ECMA-376-Part1.pdf ./extracted/part1") - print(" python scripts/ingest/extract-pdf.py ./pdfs/ECMA-376-Part1.pdf ./extracted/part1 --pages 100-200") + print(" python scripts/ingest-pdf/extract.py ./pdfs/ECMA-376-Part1.pdf ./extracted/part1") + print(" python scripts/ingest-pdf/extract.py ./pdfs/ECMA-376-Part1.pdf ./extracted/part1 --pages 100-200") sys.exit(1) pdf_path = args[0] diff --git a/scripts/ingest-pdf/fix-page-numbers.py b/scripts/ingest-pdf/fix-page-numbers.py deleted file mode 100644 index e68a1f3..0000000 --- a/scripts/ingest-pdf/fix-page-numbers.py +++ /dev/null @@ -1,158 +0,0 @@ -#!/usr/bin/env python3 -""" -Fix page numbers in embedded JSON files by re-parsing content.md. - -This avoids re-running the full pipeline (re-chunk, re-embed) which costs API credits. - -Usage: - python scripts/ingest/fix-page-numbers.py - -Example: - python scripts/ingest/fix-page-numbers.py 1 -""" - -import sys -import json -import re -from pathlib import Path - - -def parse_sections_for_pages(md_text: str, start_page: int = 1) -> dict[str, int]: - """Parse section IDs and their page numbers from markdown.""" - section_pages = {} - - # Bold section header patterns - patterns = [ - # Part 1 style: **12.3.2** **Title** - r'^\*\*(\d+(?:\.\d+)*)\*\*\s*\*\*([^*]+)\*\*$', - # Part 2/3/4 style: # **12.3.2. Title** or # **12. Title** - r'^#+\s*\*\*(\d+(?:\.\d+)*)\.?\s+([^*]+)\*\*$', - # Annex - r'^\*\*(Annex\s+[A-Z])\*\*\s*(?:\*\*)?(?:\(([^)]+)\))?(?:\*\*)?\s*(.*)$', - ] - - toc_pattern = r'^\d+(?:\.\d+)*\s+.+\.{2,}\s*\d+$' - arabic_page_pattern = r'^(\d+)$' - header_pattern = r'^ECMA-376 Part \d' - - lines = md_text.split('\n') - current_page = start_page - - for line in lines: - stripped = line.strip() - - # Skip headers - if re.match(header_pattern, stripped): - continue - - # Track page numbers - if re.match(arabic_page_pattern, stripped): - page_num = int(stripped) - if page_num >= current_page and page_num < current_page + 50: - current_page = page_num - continue - - # Skip TOC entries - if re.match(toc_pattern, stripped): - continue - - # Check for section headers - for pattern in patterns: - match = re.match(pattern, stripped, re.IGNORECASE) - if match: - section_id = match.group(1) - # +1 to match TOC page numbers - section_pages[section_id] = current_page + 1 - break - - return section_pages - - -def fix_embedded_file(part_number: int): - """Fix page numbers in embedded JSON file.""" - base_dir = Path("dev/data") - content_path = base_dir / f"extracted/part{part_number}/content.md" - embedded_path = base_dir / f"embedded/part{part_number}-embedded.json" - - if not content_path.exists(): - print(f"ERROR: Content file not found: {content_path}") - return False - - if not embedded_path.exists(): - print(f"ERROR: Embedded file not found: {embedded_path}") - return False - - print(f"Processing part {part_number}...") - - # Parse content.md for section page numbers - print(f" Parsing {content_path}...") - with open(content_path) as f: - content = f.read() - - section_pages = parse_sections_for_pages(content) - print(f" Found {len(section_pages)} sections with page numbers") - - # Load embedded chunks - print(f" Loading {embedded_path}...") - with open(embedded_path) as f: - chunks = json.load(f) - - print(f" Loaded {len(chunks)} chunks") - - # Update page numbers - updated = 0 - missing = set() - for chunk in chunks: - section_id = chunk.get("sectionId") - if section_id and section_id in section_pages: - old_page = chunk.get("pageNumber") - new_page = section_pages[section_id] - if old_page != new_page: - chunk["pageNumber"] = new_page - updated += 1 - elif section_id: - missing.add(section_id) - - print(f" Updated {updated} chunks") - if missing: - print(f" Warning: {len(missing)} sections not found in parsed content") - - # Save updated file - print(f" Saving {embedded_path}...") - with open(embedded_path, "w") as f: - json.dump(chunks, f, indent=2) - - print(f" Done!") - return True - - -def main(): - if len(sys.argv) < 2: - print("Usage: python scripts/ingest/fix-page-numbers.py ") - print("") - print("Examples:") - print(" python scripts/ingest/fix-page-numbers.py 1") - print(" python scripts/ingest/fix-page-numbers.py all") - sys.exit(1) - - arg = sys.argv[1] - - if arg == "all": - parts = [1, 2, 3, 4] - else: - try: - parts = [int(arg)] - except ValueError: - print(f"Invalid part number: {arg}") - sys.exit(1) - - for part in parts: - if not fix_embedded_file(part): - sys.exit(1) - print() - - print("All done! Now run upload.ts to update the database.") - - -if __name__ == "__main__": - main() From d74ad3181dc7e479dd04ee0f0a2e27ced5b44ad7 Mon Sep 17 00:00:00 2001 From: Ben Lewis Date: Sun, 9 Aug 2026 16:28:51 +0300 Subject: [PATCH 2/2] fix(ingest): stop annex running headers becoming sections Annex pages repeat a bare "Annex A" as their running header. The annex pattern accepted lines with no heading markup and allowed an empty title, so each of those headers opened a new section - Part 3 produced four "Annex A" entries, three of them empty, swallowing the annex body. Require heading markup (# or **) and a non-empty title, and normalise the captured letter so the ID is always "Annex A". Part 3 now yields exactly one Annex A (page 33) and one Annex B (page 38), matching its contents listing. Caught while ingesting Parts 2-4 locally; Part 1's 19/19 page check is unchanged. Co-Authored-By: Claude Opus 5 --- scripts/ingest-pdf/extract.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/scripts/ingest-pdf/extract.py b/scripts/ingest-pdf/extract.py index 92367e1..a5a8f83 100644 --- a/scripts/ingest-pdf/extract.py +++ b/scripts/ingest-pdf/extract.py @@ -217,7 +217,7 @@ def assemble_markdown(page_chunks, pages: list[int]) -> str: TITLE_STARTS_WITH_WORD_RE = re.compile(r"^[A-Za-z0-9(]") ANNEX_PATTERN = re.compile( - r"^(?:#+\s*)?\*{0,2}(Annex\s+[A-Z])\*{0,2}\s*\*{0,2}(?:\(([^)]+)\))?\*{0,2}\s*(.*)$", + r"^(?:#+\s*)?\*{0,2}\s*Annex\s+([A-Z])\b[.\s]*(.*?)\*{0,2}$", re.IGNORECASE, ) @@ -244,12 +244,16 @@ def match_heading(stripped: str) -> tuple[str, str] | None: if stripped.startswith("|"): return None + # A heading carries markdown emphasis. Annex pages repeat a bare "Annex A" + # as their running header, which is not a heading. + has_heading_markup = stripped.startswith("#") or stripped.startswith("**") + annex = ANNEX_PATTERN.match(stripped) - if annex and stripped.lstrip("#* ").lower().startswith("annex"): - title = (annex.group(3) or annex.group(2) or "").strip() - if looks_like_toc(title, stripped): + if annex and has_heading_markup: + title = annex.group(2).strip().strip("*").strip() + if not title or looks_like_toc(title, stripped): return None - return annex.group(1), title + return f"Annex {annex.group(1).upper()}", title for pattern in HEADING_PATTERNS: match = pattern.match(stripped)