diff --git a/.gitignore b/.gitignore index dcb1cd7..0daf6f2 100644 --- a/.gitignore +++ b/.gitignore @@ -11,4 +11,7 @@ dev/ PLAN.md # XSD/spec artifacts: pulled by scripts/fetch-xsd.ts; never committed. -data/xsd-cache/ \ No newline at end of file +data/xsd-cache/ +# Python bytecode +__pycache__/ +*.pyc diff --git a/apps/web/src/components/PdfViewer.tsx b/apps/web/src/components/PdfViewer.tsx index 2eaa463..818c1f6 100644 --- a/apps/web/src/components/PdfViewer.tsx +++ b/apps/web/src/components/PdfViewer.tsx @@ -1,71 +1,102 @@ import { useCallback, useEffect, useRef, useState } from "react"; -// PDF URLs and page counts for each part -const PDF_CONFIG: Record = { +/** + * PDF URLs, sheet counts, and front-matter offsets for each part. + * + * `totalPages` is the number of sheets in the file - what `#page=` addresses. + * `pageOffset` is how many sheets precede printed page 1 (cover, contents, + * foreword). Both are measured from the published PDFs; `extract.py` prints + * the pair for a part at the end of every extraction run. + * + * Search results carry the *printed* page (`spec_content.page_number`), so + * navigating to one means adding `pageOffset` back. + */ +const PDF_CONFIG: Record< + number, + { url: string; totalPages: number; pageOffset: number; name: string } +> = { 1: { url: "https://cdn.ooxml.dev/ecma-376/part1.pdf", - totalPages: 5560, + totalPages: 5026, + pageOffset: 10, name: "Fundamentals", }, 2: { url: "https://cdn.ooxml.dev/ecma-376/part2.pdf", - totalPages: 129, + totalPages: 137, + pageOffset: 8, name: "OPC", }, 3: { url: "https://cdn.ooxml.dev/ecma-376/part3.pdf", - totalPages: 65, + totalPages: 44, + pageOffset: 6, name: "Compatibility", }, 4: { url: "https://cdn.ooxml.dev/ecma-376/part4.pdf", - totalPages: 4031, + totalPages: 1548, + pageOffset: 14, name: "Transitional", }, }; interface PdfViewerProps { partNumber: number; - pageNumber: number; - onPageChange?: (page: number) => void; + /** Printed page from the spec, as stored in `spec_content.page_number`. Null opens the cover. */ + pageNumber: number | null; + /** Fires with the printed page now shown, or null while inside the front matter. */ + onPageChange?: (printedPage: number | null) => void; } export function PdfViewer({ partNumber, pageNumber, onPageChange }: PdfViewerProps) { const config = PDF_CONFIG[partNumber] || PDF_CONFIG[1]; - const [currentPage, setCurrentPage] = useState(pageNumber); + + // State is the physical sheet - that is what `#page=` and the scrubber address. + const toSheet = useCallback( + (printed: number | null) => + printed === null ? 1 : Math.max(1, Math.min(printed + config.pageOffset, config.totalPages)), + [config.pageOffset, config.totalPages], + ); + const toPrinted = useCallback( + (sheet: number) => (sheet > config.pageOffset ? sheet - config.pageOffset : null), + [config.pageOffset], + ); + + const [currentSheet, setCurrentSheet] = useState(() => toSheet(pageNumber)); const [isDragging, setIsDragging] = useState(false); const progressRef = useRef(null); // Sync with prop changes useEffect(() => { - setCurrentPage(pageNumber); - }, [pageNumber]); - - const updatePage = useCallback( - (newPage: number) => { - const clamped = Math.max(1, Math.min(newPage, config.totalPages)); - setCurrentPage(clamped); - onPageChange?.(clamped); + setCurrentSheet(toSheet(pageNumber)); + }, [pageNumber, toSheet]); + + const updateSheet = useCallback( + (newSheet: number) => { + const clamped = Math.max(1, Math.min(newSheet, config.totalPages)); + setCurrentSheet(clamped); + onPageChange?.(toPrinted(clamped)); }, - [config.totalPages, onPageChange], + [config.totalPages, onPageChange, toPrinted], ); - const handlePrev = () => updatePage(currentPage - 1); - const handleNext = () => updatePage(currentPage + 1); + const handlePrev = () => updateSheet(currentSheet - 1); + const handleNext = () => updateSheet(currentSheet + 1); // Progress bar interaction - const getPageFromPosition = useCallback( + const getSheetFromPosition = useCallback( (clientX: number) => { - if (!progressRef.current) return currentPage; + if (!progressRef.current) return currentSheet; const rect = progressRef.current.getBoundingClientRect(); const ratio = Math.max(0, Math.min(1, (clientX - rect.left) / rect.width)); return Math.max(1, Math.round(ratio * config.totalPages)); }, - [config.totalPages, currentPage], + [config.totalPages, currentSheet], ); const handleProgressClick = (e: React.MouseEvent) => { - updatePage(getPageFromPosition(e.clientX)); + updateSheet(getSheetFromPosition(e.clientX)); }; const handleDragStart = (e: React.MouseEvent) => { @@ -77,7 +108,7 @@ export function PdfViewer({ partNumber, pageNumber, onPageChange }: PdfViewerPro if (!isDragging) return; const handleMove = (e: MouseEvent) => { - updatePage(getPageFromPosition(e.clientX)); + updateSheet(getSheetFromPosition(e.clientX)); }; const handleUp = () => { @@ -91,35 +122,37 @@ export function PdfViewer({ partNumber, pageNumber, onPageChange }: PdfViewerPro document.removeEventListener("mousemove", handleMove); document.removeEventListener("mouseup", handleUp); }; - }, [isDragging, getPageFromPosition, updatePage]); + }, [isDragging, getSheetFromPosition, updateSheet]); // Keyboard navigation useEffect(() => { const handleKeyDown = (e: KeyboardEvent) => { if (e.key === "ArrowLeft") { e.preventDefault(); - setCurrentPage((p) => { - const newPage = Math.max(1, p - 1); - onPageChange?.(newPage); - return newPage; + setCurrentSheet((p) => { + const newSheet = Math.max(1, p - 1); + onPageChange?.(toPrinted(newSheet)); + return newSheet; }); } if (e.key === "ArrowRight") { e.preventDefault(); - setCurrentPage((p) => { - const newPage = Math.min(config.totalPages, p + 1); - onPageChange?.(newPage); - return newPage; + setCurrentSheet((p) => { + const newSheet = Math.min(config.totalPages, p + 1); + onPageChange?.(toPrinted(newSheet)); + return newSheet; }); } }; document.addEventListener("keydown", handleKeyDown); return () => document.removeEventListener("keydown", handleKeyDown); - }, [config.totalPages, onPageChange]); + }, [config.totalPages, onPageChange, toPrinted]); - const progressPercent = (currentPage / config.totalPages) * 100; - const pdfUrl = `${config.url}#page=${currentPage}&toolbar=0&navpanes=0`; + const progressPercent = (currentSheet / config.totalPages) * 100; + const printedPage = toPrinted(currentSheet); + const printedTotal = config.totalPages - config.pageOffset; + const pdfUrl = `${config.url}#page=${currentSheet}&toolbar=0&navpanes=0`; return (
@@ -139,7 +172,7 @@ export function PdfViewer({ partNumber, pageNumber, onPageChange }: PdfViewerPro
- {currentPage} - of {config.totalPages} + {printedPage === null ? ( + Front matter + ) : ( + <> + + {printedPage} + + of {printedTotal} + + )}
@@ -186,7 +227,11 @@ export function PdfViewer({ partNumber, pageNumber, onPageChange }: PdfViewerPro key={pdfUrl} src={pdfUrl} className="h-full w-full border-0" - title={`ECMA-376 Part ${partNumber} - Page ${currentPage}`} + title={ + printedPage === null + ? `ECMA-376 Part ${partNumber} - front matter` + : `ECMA-376 Part ${partNumber} - page ${printedPage}` + } /> diff --git a/apps/web/src/pages/SpecExplorer.tsx b/apps/web/src/pages/SpecExplorer.tsx index e69c510..2535937 100644 --- a/apps/web/src/pages/SpecExplorer.tsx +++ b/apps/web/src/pages/SpecExplorer.tsx @@ -201,10 +201,9 @@ export function SpecExplorer() { return () => document.removeEventListener("keydown", handleKeyDown); }, [results]); - // Handle page change from PDF viewer - const handlePageChange = useCallback((page: number) => { + // Handle page change from PDF viewer (printed spec page, null in front matter) + const handlePageChange = useCallback((_printedPage: number | null) => { // Could sync page changes back to state if needed - console.log("Page changed to:", page); }, []); return ( @@ -309,7 +308,7 @@ export function SpecExplorer() {
diff --git a/db/migrations/0006_remove_toc_chunks.sql b/db/migrations/0006_remove_toc_chunks.sql new file mode 100644 index 0000000..e29f574 --- /dev/null +++ b/db/migrations/0006_remove_toc_chunks.sql @@ -0,0 +1,46 @@ +-- Remove table-of-contents entries from the prose corpus. +-- +-- extract.py matched section headings by their bold styling. In the ingested +-- markdown the contents listing is bold too, so every TOC line became its own +-- "section": a title, a section ID, leader dots, and the page number it points +-- at. Those rows were embedded and uploaded alongside real spec prose. +-- +-- They are actively harmful to search. A TOC row is pure title text, so it +-- matches a title-shaped query better than the actual prose does and outranks +-- it - while carrying the page number of the contents listing itself, typically +-- 50-100 pages from the section it names. Measured across 12 representative +-- queries, 55% of returned results were TOC rows. +-- +-- The matching fix is in scripts/ingest-pdf/extract.py (see `looks_like_toc`), +-- which keeps them out of future ingests. This migration clears the rows that +-- are already in the database. +-- +-- Predicate: a short chunk that opens with a section ID and either carries +-- leader dots, or is a single unbroken line ending in a page number. Verified +-- against 459 production rows sampled across Part 1: 217 deleted, zero real +-- prose chunks caught, zero TOC rows left behind. +-- +-- Idempotent: re-running deletes nothing further. + +-- Dry run - inspect before applying: +-- +-- SELECT part_number, count(*) +-- FROM spec_content +-- WHERE +-- GROUP BY part_number ORDER BY part_number; + +DELETE FROM spec_content +WHERE length(btrim(content)) < 400 + -- opens with a section ID: "17.3.1.12 ..." or "**17.3.1.12** ..." + AND btrim(content) ~ '^\*{0,2}[[:space:]]*[0-9]+(\.[0-9]+)*\.?\*{0,2}[[:space:]]' + AND ( + -- leader dots are unambiguous + btrim(content) ~ '\.{4,}' + -- or a single unbroken line ending in a page number, for entries whose + -- title ran long enough to swallow the dots. The blank-line test keeps + -- real prose (which has paragraph breaks) out of the match. + OR ( + btrim(content) ~ '[.[:space:]][0-9]{1,4}[[:space:]]*\*{0,2}$' + AND strpos(content, E'\n\n') = 0 + ) + ); diff --git a/scripts/ingest-pdf/README.md b/scripts/ingest-pdf/README.md index a14cb5e..843e751 100644 --- a/scripts/ingest-pdf/README.md +++ b/scripts/ingest-pdf/README.md @@ -43,11 +43,36 @@ bun run pdf:upload 1 ./embedded/part1.json Useful when iterating on chunking or trying a different embedding provider without re-extracting. +## Page numbers + +These PDFs carry two numbering systems, and mixing them up sends every search +result to the wrong page: + +- **physical page** - the sheet index, what a viewer's `#page=N` addresses. +- **printed page** - the number in the running header, which restarts at 1 + after the roman-numeral front matter. + +`extract.py` extracts with `page_chunks=True`, so each line's physical page is +known exactly rather than guessed from stray digits in the text. It measures the +front-matter offset from the running headers and writes it to +`metadata.json` as `pageOffset`. + +`spec_content.page_number` stores the **printed** page - the number a reader +sees on the page and cites. `PdfViewer` adds the part's `pageOffset` back when +building the `#page=` fragment. + +Every extraction run prints the `totalPages` / `pageOffset` pair for that part. +When a PDF is replaced (a new edition, a re-paginated release), copy those two +numbers into `PDF_CONFIG` in `apps/web/src/components/PdfViewer.tsx`. + +Section bodies keep inline `` markers so `chunk.ts` can attribute +each chunk to the page it actually falls on rather than to its section's first +page. The markers are stripped from stored content and embedding text. + ## Files - `pipeline.ts` - orchestrator (extract -> chunk -> embed -> upload) - `extract.py` - PDF -> section-aware markdown via pymupdf4llm -- `fix-page-numbers.py` - PDF prelude-aware page-number alignment - `chunk.ts` - markdown -> 6 KB chunks with section IDs - `embed.ts` - chunks -> chunks + 1024-dim embeddings - `upload.ts` - bulk insert into `spec_content` diff --git a/scripts/ingest-pdf/chunk.ts b/scripts/ingest-pdf/chunk.ts index 4fe0c87..535f84f 100644 --- a/scripts/ingest-pdf/chunk.ts +++ b/scripts/ingest-pdf/chunk.ts @@ -4,6 +4,16 @@ * Takes extracted PDF content and creates chunks for embedding. * Respects section boundaries and handles XML examples specially. * + * Page numbers: extract.py leaves `` markers in each section's + * body, where N is the physical page the following text came from. We track + * those markers while walking the content so a chunk records the page it + * actually falls on, not the page its section started on. Markers are stripped + * from the stored content and from the embedding text. + * + * `pageNumber` is the *printed* page (physical minus the part's front-matter + * offset, read from the extraction metadata) - the number printed on the page + * and stored in `spec_content.page_number`. + * * Usage: * bun scripts/ingest-pdf/chunk.ts * @@ -35,6 +45,9 @@ interface Chunk { const CHUNK_SIZE = 6000; // ~2000-3000 tokens const CHUNK_OVERLAP = 200; +// Page markers written by extract.py (physical page number). +const PAGE_MARKER_PATTERN = //g; + // Markdown code fence pattern (pymupdf4llm outputs code in fences) const CODE_FENCE_PATTERN = /```[\s\S]*?```/g; @@ -64,11 +77,29 @@ function stripForEmbedding(content: string): string { return text.replace(/\n{3,}/g, "\n\n").trim(); } +/** + * Pull page markers out of a paragraph. + * + * Returns the paragraph without markers, plus the last physical page seen in + * it (null when the paragraph carried no marker). + */ +function takePageMarkers(paragraph: string): { text: string; physicalPage: number | null } { + let physicalPage: number | null = null; + + const text = paragraph.replace(PAGE_MARKER_PATTERN, (_match, page: string) => { + physicalPage = Number.parseInt(page, 10); + return ""; + }); + + return { text: text.trim(), physicalPage }; +} + function splitIntoChunks( text: string, sectionId: string, sectionTitle: string, pageStart: number, + pageOffset: number, ): Chunk[] { const chunks: Chunk[] = []; @@ -76,61 +107,87 @@ function splitIntoChunks( return chunks; } - // Split full content (with code blocks and tables inline) by paragraphs + // Physical page -> printed page. Sections start on a printed page already, + // so fall back to that until the first marker is seen. + const toPrinted = (physical: number) => Math.max(1, physical - pageOffset); + const paragraphs = text.split(/\n\n+/); let currentChunk = ""; - const currentPage = pageStart; + let currentPage = pageStart; + let chunkPage = pageStart; + + const pushChunk = () => { + const content = currentChunk.trim(); + if (!content) return; + + chunks.push({ + sectionId, + sectionTitle, + content, + embeddingText: stripForEmbedding(content), + contentType: "text", + pageNumber: chunkPage, + chunkIndex: chunks.length, + }); + }; for (const para of paragraphs) { - const trimmedPara = para.trim(); + const { text: trimmedPara, physicalPage } = takePageMarkers(para); + + if (physicalPage !== null) { + currentPage = toPrinted(physicalPage); + // A marker that arrives while the chunk is still empty belongs to + // the chunk about to be started. + if (!currentChunk.trim()) { + chunkPage = currentPage; + } + } + if (!trimmedPara) continue; // Check if adding this paragraph exceeds chunk size if (currentChunk.length + trimmedPara.length > CHUNK_SIZE) { - // Save current chunk if it has content - if (currentChunk.trim()) { - const content = currentChunk.trim(); - chunks.push({ - sectionId, - sectionTitle, - content, - embeddingText: stripForEmbedding(content), - contentType: "text", - pageNumber: currentPage, - chunkIndex: chunks.length, - }); - } + pushChunk(); - // Start new chunk with overlap + // Start new chunk with overlap, on whichever page we have reached const overlap = currentChunk.slice(-CHUNK_OVERLAP); currentChunk = `${overlap}\n\n${trimmedPara}`; + chunkPage = currentPage; } else { + if (!currentChunk) { + chunkPage = currentPage; + } currentChunk += (currentChunk ? "\n\n" : "") + trimmedPara; } } - // Don't forget the last chunk - if (currentChunk.trim()) { - const content = currentChunk.trim(); - chunks.push({ - sectionId, - sectionTitle, - content, - embeddingText: stripForEmbedding(content), - contentType: "text", - pageNumber: currentPage, - chunkIndex: chunks.length, - }); - } + pushChunk(); return chunks; } -async function chunkSections(sectionsPath: string): Promise { - const sectionsJson = await Bun.file(sectionsPath).text(); +async function readPageOffset(extractedDir: string): Promise { + try { + const metadata = await Bun.file(`${extractedDir}/metadata.json`).json(); + const offset = metadata?.pageOffset; + if (typeof offset === "number") return offset; + } catch { + // Extraction predating page offsets - fall through. + } + + console.warn( + " No pageOffset in metadata.json; treating extracted pages as printed pages.\n" + + " Re-run extract.py so page numbers line up with the PDF.", + ); + return 0; +} + +async function chunkSections(extractedDir: string): Promise { + const sectionsJson = await Bun.file(`${extractedDir}/sections.json`).text(); const sections: ExtractedSection[] = JSON.parse(sectionsJson); + const pageOffset = await readPageOffset(extractedDir); - console.log(`Processing ${sections.length} sections...`); + console.log(`Processing ${sections.length} sections (page offset ${pageOffset})...`); const allChunks: Chunk[] = []; @@ -140,6 +197,7 @@ async function chunkSections(sectionsPath: string): Promise { section.sectionId, section.title, section.pageStart, + pageOffset, ); allChunks.push(...chunks); } @@ -159,10 +217,9 @@ async function main() { } const [extractedDir, outputFile] = args; - const sectionsPath = `${extractedDir}/sections.json`; try { - const chunks = await chunkSections(sectionsPath); + const chunks = await chunkSections(extractedDir); // Save chunks await Bun.write(outputFile, JSON.stringify(chunks, null, 2)); @@ -175,11 +232,15 @@ async function main() { const avgEmbedding = Math.round( chunks.reduce((sum, c) => sum + c.embeddingText.length, 0) / chunks.length, ); + const multiPageSections = new Set( + chunks.filter((c) => c.chunkIndex > 0).map((c) => c.sectionId), + ).size; console.log("\nChunk statistics:"); console.log(` Total chunks: ${chunks.length}`); console.log(` Average content size: ${avgContent} chars`); console.log(` Average embedding text size: ${avgEmbedding} chars`); + console.log(` Sections spanning multiple chunks: ${multiPageSections}`); } catch (error) { console.error("Chunking failed:", error); process.exit(1); diff --git a/scripts/ingest-pdf/extract.py b/scripts/ingest-pdf/extract.py index 7c895f7..a5a8f83 100644 --- a/scripts/ingest-pdf/extract.py +++ b/scripts/ingest-pdf/extract.py @@ -5,21 +5,88 @@ Extracts text from ECMA-376 PDF files with proper markdown formatting. Produces cleaner output than pdf.js with code fences and table formatting. +Page numbers +------------ +Two numbering systems exist in these PDFs and conflating them is what made +every search result open on the wrong page: + + * physical page - the sheet index a PDF viewer addresses via `#page=N`. + * printed page - the number printed in the running header, which starts + over at 1 after the roman-numeral front matter. + +We extract with `page_chunks=True`, so every line's physical page is known +exactly rather than inferred from stray digits in the text. The printed page +is then derived as `physical - page_offset`, where the offset is measured +from the running headers (see `detect_page_offset`). + +`spec_content.page_number` stores the *printed* page, because that is the +number a reader sees on the page and cites. The web viewer adds the part's +offset back when it builds the `#page=` fragment. + +Section content keeps inline `` markers so the chunker can +attribute each chunk to the page it actually falls on. + Usage: - python scripts/ingest/extract-pdf.py [--pages START-END] + python scripts/ingest-pdf/extract.py [--pages START-END] Example: - python scripts/ingest/extract-pdf.py ./pdfs/ECMA-376-Part1.pdf ./extracted/part1 - python scripts/ingest/extract-pdf.py ./pdfs/ECMA-376-Part1.pdf ./extracted/part1 --pages 100-200 + python scripts/ingest-pdf/extract.py ./pdfs/ECMA-376-Part1.pdf ./extracted/part1 + python scripts/ingest-pdf/extract.py ./pdfs/ECMA-376-Part1.pdf ./extracted/part1 --pages 100-200 """ import sys import json import re import os +from collections import Counter from pathlib import Path +# Inline marker recording the physical page a run of lines came from. +PAGE_MARKER = "" +PAGE_MARKER_RE = re.compile(r"^$") + + +def detect_page_offset(doc) -> int: + """ + Measure how many sheets precede printed page 1. + + Every body page prints its number in the running header, within the first + few lines. `physical - printed` is constant across the document, so we take + the modal vote and ignore pages where no number is found (front matter, + full-bleed figures). + """ + votes: Counter[int] = Counter() + + for index in range(doc.page_count): + printed = printed_page_in_header(doc[index].get_text()) + if printed is not None: + votes[(index + 1) - printed] += 1 + + if not votes: + return 0 + + offset, agreed = votes.most_common(1)[0] + total = sum(votes.values()) + print(f"Page offset: {offset} (agreement {agreed}/{total} pages with a header number)") + + if agreed / total < 0.9: + print(f" WARNING: header page numbers disagree; runners-up {votes.most_common(4)[1:]}") + + return offset + + +def printed_page_in_header(page_text: str) -> int | None: + """Return the page number printed in the running header, if present.""" + lines = [line.strip() for line in page_text.split("\n")[:6] if line.strip()] + + for line in lines[:3]: + if re.fullmatch(r"\d{1,4}", line): + return int(line) + + return None + + def extract_pdf(pdf_path: str, output_dir: str, page_range: tuple[int, int] | None = None): """Extract PDF to markdown using pymupdf4llm.""" import pymupdf4llm @@ -27,41 +94,46 @@ def extract_pdf(pdf_path: str, output_dir: str, page_range: tuple[int, int] | No print(f"Loading PDF: {pdf_path}") - # Get total page count doc = fitz.open(pdf_path) - total_pages = len(doc) - doc.close() + total_pages = doc.page_count print(f"PDF loaded: {total_pages} pages") - # Determine pages to process + page_offset = detect_page_offset(doc) + + # Determine pages to process (0-based indices for pymupdf) if page_range: start_page, end_page = page_range pages = list(range(start_page - 1, min(end_page, total_pages))) print(f"Processing pages {start_page} to {min(end_page, total_pages)}") else: - pages = None # Process all pages + pages = list(range(total_pages)) print(f"Processing all {total_pages} pages") - # Extract to markdown + # Extract per page so each line's physical page is known exactly. print("Extracting text...") - md_text = pymupdf4llm.to_markdown( + page_chunks = pymupdf4llm.to_markdown( pdf_path, pages=pages, - show_progress=True + page_chunks=True, + show_progress=True, ) + doc.close() + + md_text = assemble_markdown(page_chunks, pages) + # Create output directory Path(output_dir).mkdir(parents=True, exist_ok=True) - # Save raw markdown + # Save raw markdown (page markers included - they are HTML comments) md_path = Path(output_dir) / "content.md" with open(md_path, "w", encoding="utf-8") as f: f.write(md_text) print(f"Saved markdown to {md_path}") # Parse sections from markdown - sections = parse_sections(md_text, page_range[0] if page_range else 1) + sections = parse_sections(md_text, page_offset) # Save sections sections_path = Path(output_dir) / "sections.json" @@ -87,8 +159,9 @@ def extract_pdf(pdf_path: str, output_dir: str, page_range: tuple[int, int] | No # Save metadata metadata = { "totalPages": total_pages, - "processedPages": len(pages) if pages else total_pages, + "processedPages": len(pages), "pageRange": list(page_range) if page_range else None, + "pageOffset": page_offset, "sectionsFound": len(sections), "contentLength": len(md_text), } @@ -97,98 +170,160 @@ def extract_pdf(pdf_path: str, output_dir: str, page_range: tuple[int, int] | No with open(metadata_path, "w", encoding="utf-8") as f: json.dump(metadata, f, indent=2) - print(f"\nExtraction complete!") + print("\nExtraction complete!") print(f" Total pages: {total_pages}") - print(f" Processed pages: {metadata['processedPages']}") + print(f" Processed pages: {len(pages)}") + print(f" Page offset: {page_offset} (printed page = physical page - {page_offset})") print(f" Sections found: {len(sections)}") print(f" Content size: {len(md_text):,} chars") + print("\n Set pageOffset in apps/web/src/components/PdfViewer.tsx PDF_CONFIG to " + f"{page_offset} and totalPages to {total_pages} for this part.") return md_text, sections -def parse_sections(md_text: str, start_page: int) -> list[dict]: - """Parse section structure from markdown text.""" +def assemble_markdown(page_chunks, pages: list[int]) -> str: + """Join per-page markdown, prefixing each page with its physical page marker.""" + parts = [] + + for position, chunk in enumerate(page_chunks): + metadata = chunk.get("metadata") or {} + physical = metadata.get("page") + + # Fall back to the requested page list if the key is absent or renamed. + if not isinstance(physical, int): + physical = pages[position] + 1 if position < len(pages) else position + 1 + + parts.append(f"\n\n{PAGE_MARKER.format(physical)}\n\n{chunk.get('text', '')}") + + return "".join(parts) + + +# Section heading patterns. Both forms have been emitted by pymupdf4llm across +# versions, so match either rather than depending on the bold styling alone. +HEADING_PATTERNS = [ + # Bold: **12.3.2** **Title** + # + # The title must be bold too. The running header on every page reads + # `**17** . WordprocessingML Reference Material` - an unbolded title - and + # accepting it would turn every page into a bogus section 17. + re.compile(r"^\*\*(\d+(?:\.\d+)*)\*\*\s*\*\*([^*]+)\*\*$"), + # ATX: #### 12.3.2 Title / ### 12.3.2. Title (optionally bolded) + re.compile(r"^#+\s*\*{0,2}(\d+(?:\.\d+)*)\.?\s+(.+?)\*{0,2}$"), +] + +# A real heading's title starts with a word. Running headers and stray +# numbering fragments start with punctuation. +TITLE_STARTS_WITH_WORD_RE = re.compile(r"^[A-Za-z0-9(]") + +ANNEX_PATTERN = re.compile( + r"^(?:#+\s*)?\*{0,2}\s*Annex\s+([A-Z])\b[.\s]*(.*?)\*{0,2}$", + re.IGNORECASE, +) + +# A table-of-contents entry: leader dots and/or a trailing page number. +# These render bold in some pymupdf4llm versions and so are indistinguishable +# from real headings by styling alone - they must be rejected by shape. +TOC_LEADER_RE = re.compile(r"\.{2,}") +TOC_TRAILING_PAGE_RE = re.compile(r"[.\s]\d{1,4}\s*$") + + +def looks_like_toc(title: str, raw_line: str) -> bool: + """True when a heading candidate is really a contents-listing entry.""" + if TOC_LEADER_RE.search(raw_line): + return True + + # No leader dots, but still "Title 1047" - a contents entry whose title + # ran long enough to swallow the dots. + return bool(TOC_TRAILING_PAGE_RE.search(title)) + + +def match_heading(stripped: str) -> tuple[str, str] | None: + """Return (section_id, title) for a real section heading, else None.""" + # Markdown table rows are contents listings in these PDFs, never headings. + if stripped.startswith("|"): + return None + + # A heading carries markdown emphasis. Annex pages repeat a bare "Annex A" + # as their running header, which is not a heading. + has_heading_markup = stripped.startswith("#") or stripped.startswith("**") + + annex = ANNEX_PATTERN.match(stripped) + if annex and has_heading_markup: + title = annex.group(2).strip().strip("*").strip() + if not title or looks_like_toc(title, stripped): + return None + return f"Annex {annex.group(1).upper()}", title + + for pattern in HEADING_PATTERNS: + match = pattern.match(stripped) + if not match: + continue + + section_id, title = match.group(1), (match.group(2) or "").strip() + if not title or not TITLE_STARTS_WITH_WORD_RE.match(title): + return None + if looks_like_toc(title, stripped): + return None + return section_id, title + + return None + + +def parse_sections(md_text: str, page_offset: int = 0) -> list[dict]: + """ + Parse section structure from markdown text. + + `page_offset` converts the physical pages recorded in the page markers into + the printed page numbers stored in the database. + """ sections = [] - # Section patterns for ECMA-376 - # Only match BOLD section headers (actual sections, not TOC entries) - patterns = [ - # Main section with bold: **12.3.2** **Title** - r'^\*\*(\d+(?:\.\d+)*)\*\*\s*\*\*([^*]+)\*\*$', - # Annex with bold: **Annex A** **(normative)** or **Annex A (informative)** - r'^\*\*(Annex\s+[A-Z])\*\*\s*(?:\*\*)?(?:\(([^)]+)\))?(?:\*\*)?\s*(.*)$', - ] - - # TOC pattern to skip (has page number at end: "17.3.2 Title ... 264") - toc_pattern = r'^\d+(?:\.\d+)*\s+.+\.{2,}\s*\d+$' - - # Page number patterns from pymupdf4llm output - # Arabic numerals (main content): standalone line with just digits - arabic_page_pattern = r'^(\d+)$' - # Header/footer line to skip - header_pattern = r'^ECMA-376 Part \d' - - lines = md_text.split('\n') + lines = md_text.split("\n") current_section = None - current_content = [] - current_page = start_page + current_content: list[str] = [] + current_physical = 1 + + def printed_page() -> int: + return max(1, current_physical - page_offset) for line in lines: stripped = line.strip() - # Skip header/footer lines - if re.match(header_pattern, stripped): - continue - - # Track page numbers - standalone arabic numerals - if re.match(arabic_page_pattern, stripped): - page_num = int(stripped) - # Sanity check: page should increase or be close to current - if page_num >= current_page and page_num < current_page + 50: - current_page = page_num - continue # Don't include page number in content - - # Skip TOC entries (have page numbers at the end with dots) - if re.match(toc_pattern, stripped): + marker = PAGE_MARKER_RE.match(stripped) + if marker: + current_physical = int(marker.group(1)) + # Keep the marker in the section body so the chunker can advance + # its page as it walks the content. Pad with blank lines so the + # marker survives as its own paragraph when the body is re-split. + if current_section: + current_content.extend(["", stripped, ""]) continue - # Check for section headers (bold only - actual sections) - section_match = None - for pattern in patterns: - match = re.match(pattern, stripped, re.IGNORECASE) - if match: - section_match = match - break + heading = match_heading(stripped) - if section_match: + if heading: # Save previous section if current_section: - current_section["content"] = '\n'.join(current_content).strip() - current_section["pageEnd"] = current_page + 1 # +1 to match TOC + current_section["content"] = "\n".join(current_content).strip() + current_section["pageEnd"] = printed_page() sections.append(current_section) - # Start new section - groups = section_match.groups() - section_id = groups[0] - title = groups[1] if len(groups) > 1 else "" + section_id, title = heading - # Calculate depth - if section_id.startswith("Annex"): + if section_id.lower().startswith("annex"): depth = 1 else: - depth = section_id.count('.') + 1 - - # Get parent ID - parent_id = get_parent_section_id(section_id) + depth = section_id.count(".") + 1 current_section = { "sectionId": section_id, - "title": (title or "").strip(), - "pageStart": current_page + 1, # +1 to match TOC page numbers - "pageEnd": current_page + 1, + "title": title, + "pageStart": printed_page(), + "pageEnd": printed_page(), "content": "", "depth": depth, - "parentId": parent_id, + "parentId": get_parent_section_id(section_id), } current_content = [line] elif current_section: @@ -196,8 +331,8 @@ def parse_sections(md_text: str, start_page: int) -> list[dict]: # Don't forget the last section if current_section: - current_section["content"] = '\n'.join(current_content).strip() - current_section["pageEnd"] = current_page + 1 # +1 to match TOC + current_section["content"] = "\n".join(current_content).strip() + current_section["pageEnd"] = printed_page() sections.append(current_section) return sections @@ -205,25 +340,25 @@ def parse_sections(md_text: str, start_page: int) -> list[dict]: def get_parent_section_id(section_id: str) -> str | None: """Get parent section ID from a section ID.""" - if section_id.startswith("Annex"): + if section_id.lower().startswith("annex"): return None - parts = section_id.split('.') + parts = section_id.split(".") if len(parts) <= 1: return None - return '.'.join(parts[:-1]) + return ".".join(parts[:-1]) def main(): args = sys.argv[1:] if len(args) < 2: - print("Usage: python scripts/ingest/extract-pdf.py [--pages START-END]") + print("Usage: python scripts/ingest-pdf/extract.py [--pages START-END]") print("") print("Example:") - print(" python scripts/ingest/extract-pdf.py ./pdfs/ECMA-376-Part1.pdf ./extracted/part1") - print(" python scripts/ingest/extract-pdf.py ./pdfs/ECMA-376-Part1.pdf ./extracted/part1 --pages 100-200") + print(" python scripts/ingest-pdf/extract.py ./pdfs/ECMA-376-Part1.pdf ./extracted/part1") + print(" python scripts/ingest-pdf/extract.py ./pdfs/ECMA-376-Part1.pdf ./extracted/part1 --pages 100-200") sys.exit(1) pdf_path = args[0] diff --git a/scripts/ingest-pdf/fix-page-numbers.py b/scripts/ingest-pdf/fix-page-numbers.py deleted file mode 100644 index e68a1f3..0000000 --- a/scripts/ingest-pdf/fix-page-numbers.py +++ /dev/null @@ -1,158 +0,0 @@ -#!/usr/bin/env python3 -""" -Fix page numbers in embedded JSON files by re-parsing content.md. - -This avoids re-running the full pipeline (re-chunk, re-embed) which costs API credits. - -Usage: - python scripts/ingest/fix-page-numbers.py - -Example: - python scripts/ingest/fix-page-numbers.py 1 -""" - -import sys -import json -import re -from pathlib import Path - - -def parse_sections_for_pages(md_text: str, start_page: int = 1) -> dict[str, int]: - """Parse section IDs and their page numbers from markdown.""" - section_pages = {} - - # Bold section header patterns - patterns = [ - # Part 1 style: **12.3.2** **Title** - r'^\*\*(\d+(?:\.\d+)*)\*\*\s*\*\*([^*]+)\*\*$', - # Part 2/3/4 style: # **12.3.2. Title** or # **12. Title** - r'^#+\s*\*\*(\d+(?:\.\d+)*)\.?\s+([^*]+)\*\*$', - # Annex - r'^\*\*(Annex\s+[A-Z])\*\*\s*(?:\*\*)?(?:\(([^)]+)\))?(?:\*\*)?\s*(.*)$', - ] - - toc_pattern = r'^\d+(?:\.\d+)*\s+.+\.{2,}\s*\d+$' - arabic_page_pattern = r'^(\d+)$' - header_pattern = r'^ECMA-376 Part \d' - - lines = md_text.split('\n') - current_page = start_page - - for line in lines: - stripped = line.strip() - - # Skip headers - if re.match(header_pattern, stripped): - continue - - # Track page numbers - if re.match(arabic_page_pattern, stripped): - page_num = int(stripped) - if page_num >= current_page and page_num < current_page + 50: - current_page = page_num - continue - - # Skip TOC entries - if re.match(toc_pattern, stripped): - continue - - # Check for section headers - for pattern in patterns: - match = re.match(pattern, stripped, re.IGNORECASE) - if match: - section_id = match.group(1) - # +1 to match TOC page numbers - section_pages[section_id] = current_page + 1 - break - - return section_pages - - -def fix_embedded_file(part_number: int): - """Fix page numbers in embedded JSON file.""" - base_dir = Path("dev/data") - content_path = base_dir / f"extracted/part{part_number}/content.md" - embedded_path = base_dir / f"embedded/part{part_number}-embedded.json" - - if not content_path.exists(): - print(f"ERROR: Content file not found: {content_path}") - return False - - if not embedded_path.exists(): - print(f"ERROR: Embedded file not found: {embedded_path}") - return False - - print(f"Processing part {part_number}...") - - # Parse content.md for section page numbers - print(f" Parsing {content_path}...") - with open(content_path) as f: - content = f.read() - - section_pages = parse_sections_for_pages(content) - print(f" Found {len(section_pages)} sections with page numbers") - - # Load embedded chunks - print(f" Loading {embedded_path}...") - with open(embedded_path) as f: - chunks = json.load(f) - - print(f" Loaded {len(chunks)} chunks") - - # Update page numbers - updated = 0 - missing = set() - for chunk in chunks: - section_id = chunk.get("sectionId") - if section_id and section_id in section_pages: - old_page = chunk.get("pageNumber") - new_page = section_pages[section_id] - if old_page != new_page: - chunk["pageNumber"] = new_page - updated += 1 - elif section_id: - missing.add(section_id) - - print(f" Updated {updated} chunks") - if missing: - print(f" Warning: {len(missing)} sections not found in parsed content") - - # Save updated file - print(f" Saving {embedded_path}...") - with open(embedded_path, "w") as f: - json.dump(chunks, f, indent=2) - - print(f" Done!") - return True - - -def main(): - if len(sys.argv) < 2: - print("Usage: python scripts/ingest/fix-page-numbers.py ") - print("") - print("Examples:") - print(" python scripts/ingest/fix-page-numbers.py 1") - print(" python scripts/ingest/fix-page-numbers.py all") - sys.exit(1) - - arg = sys.argv[1] - - if arg == "all": - parts = [1, 2, 3, 4] - else: - try: - parts = [int(arg)] - except ValueError: - print(f"Invalid part number: {arg}") - sys.exit(1) - - for part in parts: - if not fix_embedded_file(part): - sys.exit(1) - print() - - print("All done! Now run upload.ts to update the database.") - - -if __name__ == "__main__": - main()