From cf833c054c67a08e949f515ca422c0c7be4103dc Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Sat, 17 Oct 2026 06:17:48 +0100 Subject: [PATCH 01/11] feat(pdf-codec): leave document-level surfaces out of a read ReadPdfOptions.surfaces takes one boolean per document-level surface (outline, destinations, form, layers, structure, residue), each read unless set to false, on a full read or a page-selected one. A skipped surface's reader is never entered, its field is absent from the result, and the result names it in skippedSurfaces, so "not read" can be told apart from "none". Page items lose only what depended on a skipped surface: internal links without destinations, layer names without layers, structure owners without the structure tree. writePdf refuses a layout carrying skippedSurfaces with PdfSkippedSurfacesError. Refs #1692 --- packages/pdf-codec/README.md | 20 +- packages/pdf-codec/src/index.ts | 3 +- packages/pdf-codec/src/layout.ts | 17 +- packages/pdf-codec/src/read-surfaces.test.ts | 327 ++++++++++++++++++ packages/pdf-codec/src/read.ts | 127 +++++-- .../src/test-support/pdf-surfaces.ts | 115 ++++++ .../pdf-codec/src/write-skipped-surfaces.ts | 17 + packages/pdf-codec/src/write.ts | 4 + 8 files changed, 594 insertions(+), 36 deletions(-) create mode 100644 packages/pdf-codec/src/read-surfaces.test.ts create mode 100644 packages/pdf-codec/src/test-support/pdf-surfaces.ts create mode 100644 packages/pdf-codec/src/write-skipped-surfaces.ts diff --git a/packages/pdf-codec/README.md b/packages/pdf-codec/README.md index 1a8f58834..a9c0e987f 100644 --- a/packages/pdf-codec/README.md +++ b/packages/pdf-codec/README.md @@ -93,7 +93,25 @@ An encrypted PDF that opens without a password decrypts transparently — no ext Both accept an optional `signal` (`AbortSignal`); `readPdf` additionally takes a `sink` (`PdfDiagnosticSink`, called once per recoverable parse diagnostic — see the three-tier failure policy under [Conventions](#conventions)), and `writePdf` an `onSubstitution` callback (called once per character not representable in a standard-14 font — see [Fidelity](#fidelity)). -**Reading a subset of pages.** `readPdf(bytes, { pages: [0, 1, 2, -1] })` reads only the listed pages: zero-based indices into the page tree, a negative index counted from the end (`-1` is the last page). What a selected read skips: an unselected page's content stream is never fetched, decoded or interpreted, so the fonts and images only unselected pages use are never resolved, and no embedded file is decoded, so `attachments` is absent from a page-selected result (an attachment is document-level payload whose size is independent of the pages asked for); an absent `attachments` therefore means "not read" whenever `pageCount` is set, which only a page-selected read does, so check `pageCount` before concluding a document has no attachments. The case this exists for is identifying a document from its first pages, or reading a code from its last page, inside a memory-capped runtime such as a Cloudflare Worker isolate, without paying for every page's content. What it still reads whole, because page references resolve against it or because it is document-level and holds no page content stream: the page tree, the outline, named destinations, optional-content layers, the AcroForm, the structure element tree and the `/ParentTree` number tree (both of which scale with the document), metadata and package residue. Two surfaces are built from page content and so cover selected pages only: the structure-ownership association an item's `structure` field names (with its parent-tree diagnostics) and the `page-boxes` residue row. A destination a link states as a direct array is minted into `destinations` (as `dest1`, `dest2`, ...) only when its page is interpreted, so an unselected page's direct link destinations are absent and a minted name depends on which pages were read; named destinations are unaffected. Duplicates collapse and the result's `pages` holds the selected pages in ascending document order, each carrying its source position as `index` (read it through `sourcePageIndex(page, position)`, which falls back to the position on a full read), with the page tree's length as the document's `pageCount`; every document-level page reference keeps naming source indices, so a reference to a selected page resolves through `index`, and one to an unselected page names a page the result does not hold. A full read sets neither field, because there `pages.length` and each page's position already state them. An empty selection is valid and reads no page, which makes `{ pages: [] }` the cheapest way to learn a file's page count and metadata. An entry naming no page (outside `-pageCount` to `pageCount - 1`, or not an integer, `NaN` and the infinities included; `-0` is page 0) throws `PdfPageSelectionError` (a `PdfParseError` with code `pdf/page-out-of-range` or `pdf/page-index-not-integer`, carrying `pageIndex` and `pageCount`) before any page is read. A page-selected `LayoutDocument` is a reading of part of a file, not a document to re-emit: `writePdf` refuses one (any document carrying `pageCount`) with `PdfPartialDocumentError`, since it places pages by position and the references would point at the wrong pages. +**Reading a subset of pages.** `readPdf(bytes, { pages: [0, 1, 2, -1] })` reads only the listed pages: zero-based indices into the page tree, a negative index counted from the end (`-1` is the last page). What a selected read skips: an unselected page's content stream is never fetched, decoded or interpreted, so the fonts and images only unselected pages use are never resolved, and no embedded file is decoded, so `attachments` is absent from a page-selected result (an attachment is document-level payload whose size is independent of the pages asked for); an absent `attachments` therefore means "not read" whenever `pageCount` is set, which only a page-selected read does, so check `pageCount` before concluding a document has no attachments. The case this exists for is identifying a document from its first pages, or reading a code from its last page, inside a memory-capped runtime such as a Cloudflare Worker isolate, without paying for every page's content. What it still reads whole, because page references resolve against it or because it is document-level and holds no page content stream: the page tree, the outline, named destinations, optional-content layers, the AcroForm, the structure element tree and the `/ParentTree` number tree (both of which scale with the document), metadata and package residue; every one of those but the page tree and metadata can be left out too, with `surfaces` (below). Two surfaces are built from page content and so cover selected pages only: the structure-ownership association an item's `structure` field names (with its parent-tree diagnostics) and the `page-boxes` residue row. A destination a link states as a direct array is minted into `destinations` (as `dest1`, `dest2`, ...) only when its page is interpreted, so an unselected page's direct link destinations are absent and a minted name depends on which pages were read; named destinations are unaffected. Duplicates collapse and the result's `pages` holds the selected pages in ascending document order, each carrying its source position as `index` (read it through `sourcePageIndex(page, position)`, which falls back to the position on a full read), with the page tree's length as the document's `pageCount`; every document-level page reference keeps naming source indices, so a reference to a selected page resolves through `index`, and one to an unselected page names a page the result does not hold. A full read sets neither field, because there `pages.length` and each page's position already state them. An empty selection is valid and reads no page, which makes `{ pages: [] }` the cheapest way to learn a file's page count and metadata. An entry naming no page (outside `-pageCount` to `pageCount - 1`, or not an integer, `NaN` and the infinities included; `-0` is page 0) throws `PdfPageSelectionError` (a `PdfParseError` with code `pdf/page-out-of-range` or `pdf/page-index-not-integer`, carrying `pageIndex` and `pageCount`) before any page is read. A page-selected `LayoutDocument` is a reading of part of a file, not a document to re-emit: `writePdf` refuses one (any document carrying `pageCount`) with `PdfPartialDocumentError`, since it places pages by position and the references would point at the wrong pages. + +**Leaving document-level surfaces out.** `readPdf(bytes, { surfaces: { structure: false, outline: false } })` reads the document without the surfaces turned off. `ReadPdfOptions.surfaces` is one object of independent booleans (`ReadPdfSurfaces`), one per surface, each read unless set to `false`, so omitting the setting reads everything exactly as before. It is independent of `pages`, applying to a full read and a page-selected read alike, and combined with a small selection it is what keeps identifying a document cheap: on a long tagged PDF the structure element tree and its `/ParentTree` can be most of what a page-selected read still parses, and `{ pages: [0, 1, 2, -1], surfaces: { outline: false, destinations: false, form: false, layers: false, structure: false, residue: false } }` reads nothing beyond the page tree, metadata and those four pages. A surface turned off is never parsed (its part of the catalog is not walked, so a malformed outline, form, layer list or structure tree reports no diagnostic), its `LayoutDocument` field is absent whatever the file holds, and the result names it in `skippedSurfaces` (in the order `LayoutDocumentSurfaceSchema` lists them, absent when every surface was read). An absent `outline`, `destinations`, `form`, `layers`, `structure` or `source` therefore means "not read" whenever `skippedSurfaces` names it, so check `skippedSurfaces` before concluding a document has none, as with `pageCount` and attachments. + +Each surface, and what page items lose without it (everything else about every page item is exactly what a read with the surface on produces): + +- `outline`: the `/Outlines` bookmark tree (`outline`). Page items do not depend on it. + +- `destinations`: the named destinations of the catalog `/Dests` dictionary and the `/Names /Dests` name tree, and the registry links and outline entries name their targets through (`destinations`). Without it nothing can name an internal target, so a link whose target is internal (a `/Dest`, or a `/GoTo` action, whether it names a destination or states one directly) is left out of its page's items, and an outline entry, if the outline is read, carries no `destination`. A link to a URI is kept. + +- `form`: the AcroForm field tree (`form`). Page items do not depend on it, since a widget annotation is never a page item or annotation either way. + +- `layers`: optional content, the `/OCProperties` groups and their default visibility (`layers`). Without it no item carries a `layer`. The items themselves are all still read, so content the default configuration hides cannot be told from visible content, and a consumer that drops hidden-layer content (as documents.js's reconstruction does) keeps it. + +- `structure`: the `/StructTreeRoot` element tree and its `/ParentTree` ownership map (`structure`). Without it no item carries a `structure` owner, and the parent-tree diagnostics are not reported. + +- `residue`: the package-level residue rows (`source`), the per-page `page-boxes` row included. Page items do not depend on it, and `metadata` still reads the XMP packet it falls back to. + +A `LayoutDocument` read with any surface skipped is a reading with parts of the file deliberately unread, so `writePdf` refuses one (any document carrying `skippedSurfaces`) with `PdfSkippedSurfacesError`, which names the skipped surfaces; a document that is also page-selected reports `PdfPartialDocumentError` first. **Page boundaries: the crop box is the visible region** (ISO 32000-1 14.11.2). A page's reported `widthPt`/`heightPt` and coordinate frame come from its effective `/CropBox` — the rectangle a viewer displays and prints, inherited through the page tree like `/MediaBox` and defaulting to it — not from the media box: content an author placed outside the crop box (printer's marks, bleed, off-page slugs) is not visible and does not extract. Content wholly outside the crop box is dropped; content straddling the boundary keeps its original unclipped geometry (the item layer records what the file states — a viewer's clipping is a rendering fact, not source data); link annotations are anchored constructs rather than painted content and are never filtered, the same line the optional-content filter draws. The declared boundary rectangles a distinct crop box hides — plus `/BleedBox`/`/TrimBox`/`/ArtBox`, print-production facts with no field in the layout model — are quarantined verbatim as the `page-boxes` package-level residue row rather than silently dropped. A degenerate `/CropBox` (zero width or height) falls back to the media box with a `pdf/invalid-crop-box` warning. diff --git a/packages/pdf-codec/src/index.ts b/packages/pdf-codec/src/index.ts index 05953787d..3dd2da227 100644 --- a/packages/pdf-codec/src/index.ts +++ b/packages/pdf-codec/src/index.ts @@ -1,10 +1,11 @@ // pdf-codec's public surface: a curated barrel export with exactly one deliberate companion entry — `pdf-codec/read` (package.json's explicit ./read export onto src/read.ts, the read pipeline's own module), the read-only entry whose module graph excludes this barrel's write half and the vendored font assets it eagerly imports. Everything else matches document-schema.js/odf.js/ooxml.js's own barrel-only precedent. What's exported here is every symbol a real external consumer needs — headline read/write/codec entry points, the Layout item family this package owns outright as its native document model (src/layout.ts, exported wholesale below), the formula/math port documents.js's own MathML layout engine passes real values through, the text-layout and font-resolution helpers every layout engine built on this codec needs, and the full bytes/image surface (re-exported from byte-codec, the neutral shared package these pure utilities now live in — see the export site below for exactly why the public surface sources them from there rather than this package's own internal src/bytes/ and src/image/ copies). Internal plumbing (objects.ts, serialize.ts, lexer.ts, parse.ts, xref.ts, document.ts, interpret.ts, content-read.ts, content-write.ts, filters.ts, predictors.ts, images-read.ts, cmap.ts, font-read.ts, font-style.ts, and the cmap-table/hmtx-table/font-tables/glyf/sfnt/sfnt-subset/cff/cff-probe/cff-bounds/tounicode/ot-layout-common/gpos-table/math-font-write/math-content-write/embedded-font-write font-parsing, font-subsetting, and font-embedding internals) stays unexported — math-table.ts is a partial exception, exporting its MathVariants types alone (see below), and glyph-bounds.ts another, exporting the GlyphInkBounds shape those outline readers report through — nothing outside this package's own src/ consumes it today. embedded-font.ts is the one partial exception: its EmbeddedFace is the type ResolvedFace's own 'embedded' variant carries, and its EmbeddedFaceSubstitution is what WritePdfOptions.onMissingGlyph reports, so both must be nameable by an external caller even though nothing else in that module is exported. // Headline: read/write/diagnostics/codec. readPdf/writePdf speak LayoutDocument and there is deliberately no DocumentTree-native pair beside them. On read, PDF states positions, not structure — readPdf yields layout cheaply, and semantic content (paragraphs, headings, tables) only through a separate, lossy reconstruction pass that infers structure from geometry, genuine semantic policy that lives in documents.js (src/layout/reconstruct.ts). On write, a DocumentTree reaches PDF bytes only once a prior layout pass has already stamped frames onto it (a package with no pages, e.g. a bridge conversion's own odt-to-docx dump, cannot reach PDF at all): documents.js's layoutDocumentFromPackage walks those already-stamped frames back into a LayoutDocument, a mechanical inverse rather than a font-measuring, line-breaking pass of its own — the actual layout engine runs earlier, wherever the package first passed through an X-to-PDF or PDF-to-X conversion. ExaDev/pdf-codec#65 scoped the frames-mapping half of this boundary to this package ("readPdf/writePdf ... gain mappings to and from package frames at the edge"); that half was never implemented here and instead lives in documents.js as layoutDocumentFromPackage — a deliberate reassignment, not an oversight, but one #65 itself doesn't record. Adding a DocumentTree-facing wrapper here would mean this package owning either a reconstruction heuristic or a frame-walking pass that depends on a layout stage documents.js runs, and would make pdf-codec depend on the package that depends on it. -export type { ReadPdfOptions } from "./read"; +export type { ReadPdfOptions, ReadPdfSurfaces } from "./read"; export { readPdf } from "./read"; export type { WritePdfOptions } from "./write"; export { PdfPartialDocumentError, writePdf } from "./write"; +export { PdfSkippedSurfacesError } from "./write-skipped-surfaces"; // Write-side standard security handler: encrypts a document written by writePdf under WritePdfOptions.encryption. See encrypt-write.ts's own module comment for the four schemes and their scope. export type { PdfEncryptionOptions, diff --git a/packages/pdf-codec/src/layout.ts b/packages/pdf-codec/src/layout.ts index 72f4be218..c21c7e498 100644 --- a/packages/pdf-codec/src/layout.ts +++ b/packages/pdf-codec/src/layout.ts @@ -9,7 +9,7 @@ import { // pdf-codec's own native document model: LayoutDocument, the positioned item layer readPdf assembles from a PDF's bytes and writePdf draws into new ones. Ported verbatim from document-schema.js's own src/layout.ts (its home from the content pivot until that package's 4.0.0 promoted DocumentTree and dropped it) because a codec's native model belongs in the codec — the same family pattern as ooxml.js's Package/XmlElement and markdown-codec's AST; only PDF's native model was ever a public shared-schema export, an accident of this package predating the content pivot. The item layer remains the honest boundary between what the format says (positions) and what we think it means (structure): when documents.js's reconstruction misjudges a wrapped paragraph, these items stay inspectable as the PDF's actual testimony. Reconstruction heuristics are semantic policy and stay in documents.js, not here. The shared leaf shapes the family composes from (Color, ContentStrokeStyleSchema, LayoutFont, LayoutMetadata) stay in document-schema.js and are imported above, so content and layout keep one definition of each. -// Bumped whenever LayoutDocumentSchema's shape changes incompatibly, so a value serialized by one version of a consumer can be recognised (and rejected, rather than silently misread) by another. Still 1: moving the schemas between packages changed where the family lives, not what it accepts. Additive optional fields do not bump it either, following every field added since (structure, layers, attachments, writingMode among them); that holds for LayoutPage.index and LayoutDocument.pageCount too, which only a page-selected read writes, because the package has no serialized form of a page-selected layout and writePdf refuses one, so an older consumer never receives one from this package with those fields stripped. +// Bumped whenever LayoutDocumentSchema's shape changes incompatibly, so a value serialized by one version of a consumer can be recognised (and rejected, rather than silently misread) by another. Still 1: moving the schemas between packages changed where the family lives, not what it accepts. Additive optional fields do not bump it either, following every field added since (structure, layers, attachments, writingMode among them); that holds for LayoutPage.index and LayoutDocument.pageCount too, which only a page-selected read writes, and for LayoutDocument.skippedSurfaces, which only a read with ReadPdfOptions.surfaces writes, because the package has no serialized form of either kind of partial layout and writePdf refuses both, so an older consumer never receives one from this package with those fields stripped. export const LAYOUT_FORMAT_VERSION = 1; // sourcePath is assigned by each format's reader at read time and copied onto emitted LayoutItems by the layout engine; this module only defines the field, it doesn't generate values. Known limitation: sourcePath values are stable within one read+layout pass over a single document, not across edits — inserting content earlier in a document shifts every later path. This is not a stable identity scheme for incremental re-layout; it exists for tagged/accessible-PDF-style traceability and debugging, not edit-tracking. @@ -381,6 +381,19 @@ export interface LayoutStructureElement { readonly children: readonly LayoutStructureElement[]; } +/** + * The document-level surfaces a read can leave out (`ReadPdfOptions.surfaces`), each named by the `LayoutDocument` field it fills: `outline`, `destinations`, `form`, `layers` and `structure` are those fields, and `residue` is the package-level `source` channel. A `LayoutDocument` lists the ones its read skipped in `skippedSurfaces`. + */ +export const LayoutDocumentSurfaceSchema = z.enum([ + "outline", + "destinations", + "form", + "layers", + "structure", + "residue", +]); +export type LayoutDocumentSurface = z.infer; + export const LayoutDocumentSchema = z.object({ formatVersion: z.literal(LAYOUT_FORMAT_VERSION), metadata: LayoutMetadataSchema, @@ -396,5 +409,7 @@ export const LayoutDocumentSchema = z.object({ form: z.array(LayoutFormFieldSchema).optional(), // The package-level half of the quarantined residue channel (document-schema.js's SourceResidue): whole-document PDF facts no content node owns — the raw XMP packet, output intents, viewer/session behaviour, private application data, the trailer /ID. Keyed by the fact's own name, opaque to everything downstream exactly as the channel's contract states. source: z.record(z.string(), SourceResidueSchema).optional(), + // The document-level surfaces readPdf left unread because ReadPdfOptions.surfaces turned them off, in LayoutDocumentSurfaceSchema's order. Each one's field is absent from the document whatever the file holds, so an absent field here means "not read" rather than "none"; this list is how a consumer tells the two apart, as pageCount does for attachments on a page-selected read. Absent when every surface was read, and never empty. + skippedSurfaces: z.array(LayoutDocumentSurfaceSchema).min(1).optional(), }); export type LayoutDocument = z.infer; diff --git a/packages/pdf-codec/src/read-surfaces.test.ts b/packages/pdf-codec/src/read-surfaces.test.ts new file mode 100644 index 000000000..75447a5f2 --- /dev/null +++ b/packages/pdf-codec/src/read-surfaces.test.ts @@ -0,0 +1,327 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import type * as Form from "./form"; +import type * as Navigation from "./navigation"; +import type * as OptionalContent from "./optional-content"; +import type * as Structure from "./structure"; +import type { PdfDiagnostic } from "./diagnostics"; +import { readAcroForm } from "./form"; +import type { + LayoutDocument, + LayoutDocumentSurface, + LayoutItem, + LayoutOutlineItem, + LayoutPage, +} from "./layout"; +import { LayoutDocumentSchema, LayoutDocumentSurfaceSchema } from "./layout"; +import { createDestinationRegistry, readOutline } from "./navigation"; +import { readOptionalContent } from "./optional-content"; +import type { ReadPdfSurfaces } from "./read"; +import { readPdf } from "./read"; +import { readStructure } from "./structure"; +import { PdfPartialDocumentError, writePdf } from "./write"; +import { PdfSkippedSurfacesError } from "./write-skipped-surfaces"; +import { + DOCUMENT_SURFACE_TRAP_OBJECTS, + documentSurfacesPdf, +} from "./test-support/pdf-surfaces"; + +// Each document-level surface reader, wrapped so a test can see whether a read entered it. The real implementations still run: these observe, they do not stub. +vi.mock("./navigation", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + createDestinationRegistry: vi.fn(actual.createDestinationRegistry), + readOutline: vi.fn(actual.readOutline), + }; +}); +vi.mock("./form", async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, readAcroForm: vi.fn(actual.readAcroForm) }; +}); +vi.mock("./optional-content", async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, readOptionalContent: vi.fn(actual.readOptionalContent) }; +}); +vi.mock("./structure", async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, readStructure: vi.fn(actual.readStructure) }; +}); + +const SURFACES = LayoutDocumentSurfaceSchema.options; + +type ReaderSurface = Exclude; + +// The reader each surface's parse starts in; residue is read by read.ts itself and is covered by its trap alone. +const SURFACE_READERS: Readonly unknown>> = { + outline: () => readOutline, + destinations: () => createDestinationRegistry, + form: () => readAcroForm, + layers: () => readOptionalContent, + structure: () => readStructure, +}; + +// The LayoutDocument field each surface fills. +const SURFACE_FIELD = { + outline: "outline", + destinations: "destinations", + form: "form", + layers: "layers", + structure: "structure", + residue: "source", +} as const satisfies Record; + +function only(surface: LayoutDocumentSurface): ReadPdfSurfaces { + return { [surface]: false }; +} + +const NONE: ReadPdfSurfaces = Object.fromEntries( + SURFACES.map((surface) => [surface, false]), +); + +function readWith( + surfaces: ReadPdfSurfaces | undefined, + pages?: readonly number[], +): { readonly doc: LayoutDocument; readonly diagnostics: PdfDiagnostic[] } { + const diagnostics: PdfDiagnostic[] = []; + const doc = readPdf(documentSurfacesPdf(), { + surfaces, + pages, + sink: (diagnostic) => { + diagnostics.push(diagnostic); + }, + }); + return { doc, diagnostics }; +} + +// The trap object numbers a read's diagnostics name, sorted. +function trapsReported(diagnostics: readonly PdfDiagnostic[]): number[] { + return Object.values(DOCUMENT_SURFACE_TRAP_OBJECTS) + .filter((num) => + diagnostics.some((diagnostic) => + diagnostic.message.includes(`object ${String(num)} `), + ), + ) + .sort((a, b) => a - b); +} + +// An item, page or document as a plain record minus one field: what a skipped surface's result is compared against, field by field. +function withoutField(value: object, field: string): Record { + return Object.fromEntries( + Object.entries(value).filter(([key]) => key !== field), + ); +} + +function mapItems( + pages: readonly LayoutPage[], + map: (items: readonly LayoutItem[]) => readonly object[], +): Record[] { + return pages.map((page) => ({ ...page, items: map(page.items) })); +} + +function withoutDestinations( + outline: readonly LayoutOutlineItem[], +): Record[] { + return outline.map((item) => ({ + ...withoutField(item, "destination"), + children: withoutDestinations(item.children), + })); +} + +// What a full read becomes with one surface skipped, by the rules ReadPdfOptions.surfaces states: the surface's field is gone and named in skippedSurfaces, and page items (and, for destinations, outline entries) lose exactly what depended on it. Everything else is the full read unchanged. +function expectedWithout( + full: LayoutDocument, + surface: LayoutDocumentSurface, +): Record { + const pages = + surface === "structure" + ? mapItems(full.pages, (items) => + items.map((item) => withoutField(item, "structure")), + ) + : surface === "layers" + ? mapItems(full.pages, (items) => + items.map((item) => withoutField(item, "layer")), + ) + : surface === "destinations" + ? mapItems(full.pages, (items) => + items.filter((item) => item.kind !== "internalLink"), + ) + : full.pages; + return { + ...withoutField(full, SURFACE_FIELD[surface]), + pages, + ...(surface === "destinations" && full.outline !== undefined + ? { outline: withoutDestinations(full.outline) } + : {}), + skippedSurfaces: [surface], + }; +} + +beforeEach(() => { + vi.mocked(readOutline).mockClear(); + vi.mocked(createDestinationRegistry).mockClear(); + vi.mocked(readAcroForm).mockClear(); + vi.mocked(readOptionalContent).mockClear(); + vi.mocked(readStructure).mockClear(); +}); + +describe("readPdf surfaces: the fixture carries every surface and every dependency", () => { + it("reads all six surfaces and every page-item dependency on a full read, and springs every trap", () => { + const { doc, diagnostics } = readWith(undefined); + for (const surface of SURFACES) { + expect(doc[SURFACE_FIELD[surface]]).toBeDefined(); + } + const items = doc.pages[0]?.items ?? []; + expect(items.some((item) => "structure" in item)).toBe(true); + expect(items.some((item) => "layer" in item)).toBe(true); + expect(items.filter((item) => item.kind === "internalLink")).toHaveLength( + 2, + ); + expect(items.filter((item) => item.kind === "link")).toHaveLength(1); + expect(doc.outline?.[0]?.destination).toBe("second"); + expect(doc.source?.["page-boxes"]).toBeDefined(); + expect(doc.skippedSurfaces).toBeUndefined(); + expect(trapsReported(diagnostics)).toEqual( + Object.values(DOCUMENT_SURFACE_TRAP_OBJECTS).sort((a, b) => a - b), + ); + }); +}); + +describe("readPdf surfaces: a skipped surface is never parsed", () => { + it.each( + SURFACES.filter( + (surface): surface is ReaderSurface => surface !== "residue", + ), + )( + "never enters the %s reader when it is skipped, and does enter it otherwise", + (surface) => { + const reader = SURFACE_READERS[surface](); + readWith(only(surface)); + expect(reader).not.toHaveBeenCalled(); + for (const other of SURFACES) { + if (other !== surface && other !== "residue") { + expect(SURFACE_READERS[other]()).toHaveBeenCalledTimes(1); + } + } + }, + ); + + it.each(SURFACES)( + "reports no diagnostic from the %s trap when that surface is skipped, while every other trap still springs", + (surface) => { + const { diagnostics } = readWith(only(surface)); + const trap = DOCUMENT_SURFACE_TRAP_OBJECTS[surface]; + expect(trapsReported(diagnostics)).toEqual( + Object.values(DOCUMENT_SURFACE_TRAP_OBJECTS) + .filter((num) => num !== trap) + .sort((a, b) => a - b), + ); + }, + ); + + it("reports no diagnostic at all when every surface is skipped, and enters no surface reader", () => { + const { diagnostics } = readWith(NONE); + expect(diagnostics).toEqual([]); + for (const reader of Object.values(SURFACE_READERS)) { + expect(reader()).not.toHaveBeenCalled(); + } + }); +}); + +describe("readPdf surfaces: what a skipped surface's result holds", () => { + it.each(SURFACES)( + "with %s skipped, equals the full read minus that field and what depended on it, naming it in skippedSurfaces", + (surface) => { + const full = readWith(undefined).doc; + expect(readWith(only(surface)).doc).toEqual( + expectedWithout(full, surface), + ); + }, + ); + + it("omits a skipped surface's field rather than leaving it present and empty", () => { + const { doc } = readWith(NONE); + for (const surface of SURFACES) { + expect(Object.hasOwn(doc, SURFACE_FIELD[surface])).toBe(false); + } + }); + + it("lists every skipped surface in schema order, however the setting was spelled", () => { + expect( + readWith({ residue: false, outline: false, layers: false }).doc + .skippedSurfaces, + ).toEqual(["outline", "layers", "residue"]); + expect(readWith(NONE).doc.skippedSurfaces).toEqual([...SURFACES]); + }); + + it("reads a surface whose entry is true or absent, exactly as a read without the setting", () => { + const full = readWith(undefined).doc; + expect(readWith({}).doc).toEqual(full); + expect( + readWith(Object.fromEntries(SURFACES.map((s) => [s, true]))).doc, + ).toEqual(full); + }); + + it("validates against LayoutDocumentSchema, which refuses an empty skippedSurfaces", () => { + const { doc } = readWith(NONE); + expect(LayoutDocumentSchema.parse(doc)).toEqual(doc); + expect( + LayoutDocumentSchema.safeParse({ ...doc, skippedSurfaces: [] }).success, + ).toBe(false); + }); +}); + +describe("readPdf surfaces: independent of pages", () => { + it("skips surfaces on a page-selected read too, its selected pages losing what the same rules say", () => { + const fullSelected = readWith(undefined, [0]).doc; + const selected = readWith( + { structure: false, destinations: false }, + [0], + ).doc; + expect(selected.pages).toEqual( + mapItems(fullSelected.pages, (items) => + items + .filter((item) => item.kind !== "internalLink") + .map((item) => withoutField(item, "structure")), + ), + ); + expect(selected.pageCount).toBe(fullSelected.pageCount); + expect(selected.skippedSurfaces).toEqual(["destinations", "structure"]); + }); + + it("reads nothing beyond the page tree and metadata for an empty selection with every surface skipped", () => { + const { doc, diagnostics } = readWith(NONE, []); + expect(diagnostics).toEqual([]); + expect(doc).toEqual({ + formatVersion: readWith(undefined).doc.formatVersion, + metadata: readWith(undefined).doc.metadata, + pages: [], + pageCount: 2, + images: {}, + skippedSurfaces: [...SURFACES], + }); + }); +}); + +describe("writePdf refuses a read that skipped a surface", () => { + it.each(SURFACES)( + "throws PdfSkippedSurfacesError naming %s, and still writes the full read", + (surface) => { + const skipped = readWith(only(surface)).doc; + expect(() => writePdf(skipped)).toThrow(PdfSkippedSurfacesError); + expect(() => writePdf(skipped)).toThrow( + expect.objectContaining({ skippedSurfaces: [surface] }), + ); + expect(() => writePdf(skipped)).toThrow(surface); + }, + ); + + it("writes a full read", () => { + expect(writePdf(readWith(undefined).doc).length).toBeGreaterThan(0); + }); + + it("reports the page selection first when a read both selected pages and skipped surfaces", () => { + expect(() => writePdf(readWith(NONE, [0]).doc)).toThrow( + PdfPartialDocumentError, + ); + }); +}); diff --git a/packages/pdf-codec/src/read.ts b/packages/pdf-codec/src/read.ts index d1ab1bfca..519a0e6db 100644 --- a/packages/pdf-codec/src/read.ts +++ b/packages/pdf-codec/src/read.ts @@ -16,8 +16,9 @@ import type { LayoutImageAsset, LayoutItem, LayoutPage, + LayoutDocumentSurface, } from "./layout"; -import { LAYOUT_FORMAT_VERSION } from "./layout"; +import { LAYOUT_FORMAT_VERSION, LayoutDocumentSurfaceSchema } from "./layout"; import { openPdfDocument } from "./document"; import type { PdfDocument } from "./document"; import type { PdfDiagnosticSink } from "./diagnostics"; @@ -66,13 +67,50 @@ export interface ReadPdfOptions { * * What a selected read skips: an unselected page's content stream is never fetched, decoded or interpreted, so the fonts and images only unselected pages use are never resolved; embedded-file payloads are never decoded, so `attachments` is absent from a page-selected result (an attachment is document-level payload whose size is independent of the pages asked for, and decoding it would cost the same whichever pages were selected). * - * What it still reads whole, because a negative index and every document-level page reference resolve against it or because it is document-level and holds no page content stream: the page tree, the outline, named destinations, optional-content layers, the AcroForm, the structure element tree and the /ParentTree number tree (both of which scale with the document), metadata and package residue. Two surfaces are built from page content and so cover selected pages only: the structure-ownership association an item's `structure` field names (and its parent-tree diagnostics), and the `page-boxes` residue row. Destinations a link states as a direct array are minted into `destinations` (as `dest1`, `dest2`, ...) only when their page is interpreted, so an unselected page's direct link destinations are absent and a minted name depends on which pages were read; named destinations are unaffected. + * What it still reads whole, because a negative index and every document-level page reference resolve against it or because it is document-level and holds no page content stream: the page tree, the outline, named destinations, optional-content layers, the AcroForm, the structure element tree and the /ParentTree number tree (both of which scale with the document), metadata and package residue; every one of those but the page tree and metadata can be left out as well with `surfaces`. Two surfaces are built from page content and so cover selected pages only: the structure-ownership association an item's `structure` field names (and its parent-tree diagnostics), and the `page-boxes` residue row. Destinations a link states as a direct array are minted into `destinations` (as `dest1`, `dest2`, ...) only when their page is interpreted, so an unselected page's direct link destinations are absent and a minted name depends on which pages were read; named destinations are unaffected. * * Duplicates collapse, and the result's `pages` holds the selected pages in ascending document order whatever order they were asked for in, each carrying its source position as `index`, with the document's total as `pageCount`; every document-level reference keeps naming source indices. An empty array is a valid selection: it reads no page and returns the document-level surfaces with `pageCount`, the cheapest way to learn a file's length and metadata. An entry naming no page throws `PdfPageSelectionError` rather than being skipped. */ readonly pages?: readonly number[]; + /** + * Which document-level surfaces to read. Each is read unless its entry is `false`, so omitting the setting, or an entry, reads it as before. Independent of `pages`: it applies to a full read and a page-selected read alike. + * + * A surface turned off is never parsed (its part of the catalog is not walked, so a malformed outline, form, layer list or structure tree reports no diagnostic) and its field is absent from the result whatever the file holds; the result lists it in `skippedSurfaces`, which is how a caller tells "not read" from "none". What each one covers, and what page items lose without it: + * + * - `outline`: the `/Outlines` bookmark tree (`outline`). Page items do not depend on it. + * - `destinations`: the named destinations of the catalog `/Dests` dictionary and the `/Names /Dests` name tree, and the registry links and outline entries name their targets through (`destinations`). Without it nothing can name an internal target, so a link whose target is internal (a `/Dest`, or a `/GoTo` action, named or direct) is left out of its page's items, and a read outline entry carries no `destination`; a link to a URI is kept as usual. + * - `form`: the AcroForm field tree (`form`). Page items do not depend on it: a widget annotation is never a page item or annotation either way. + * - `layers`: optional content, the `/OCProperties` groups and their default visibility (`layers`). Without it no item carries a `layer`, since an `/OC` membership resolves to no name; the items themselves are all still read, so content the default view hides is indistinguishable from visible content. + * - `structure`: the `/StructTreeRoot` element tree and its `/ParentTree` ownership map (`structure`). Without it no item carries a `structure` owner, and the parent-tree diagnostics are not reported. + * - `residue`: the package-level residue rows (`source`), including the per-page `page-boxes` row. Page items do not depend on it; `metadata` still reads the XMP packet it falls back to. + * + * Apart from those stated losses, every page item is exactly what a read with the surface on produces. + */ + readonly surfaces?: ReadPdfSurfaces; } +/** + * One entry per document-level surface `readPdf` can leave out, each read unless set to `false` (see `ReadPdfOptions.surfaces` for what each covers and what page items lose without it). + */ +export type ReadPdfSurfaces = Readonly< + Partial> +>; + +// The surfaces a setting turns off, in LayoutDocumentSurfaceSchema's order, so the result's skippedSurfaces lists them the same way however the caller spelled the object. +function skippedSurfacesOf( + surfaces: ReadPdfSurfaces | undefined, +): LayoutDocumentSurface[] { + return LayoutDocumentSurfaceSchema.options.filter( + (surface) => surfaces?.[surface] === false, + ); +} + +// A registry for a read that skips destinations: it holds no entry and names nothing, so a link or outline entry asking it for a target gets none, without a diagnostic (nothing was looked up, so nothing dangles). +const UNREAD_DESTINATION_REGISTRY: DestinationRegistry = { + entries: [], + intern: () => undefined, +}; + // Resolves a page selection against the page tree's length: negative indices counted from the end, duplicates collapsed, and the result as a set, since readPdf walks the tree in document order and asks it per page. Throws on the first entry that names no page, before any page is read. function resolvePageSelection( selection: readonly number[], @@ -137,33 +175,42 @@ export function readPdf( options?.pages === undefined ? undefined : resolvePageSelection(options.pages, pageDicts.length); - const destinationRegistry = createDestinationRegistry( - doc.catalog, - doc, - (obj) => doc.pageIndex(obj), - sink, - ); - const outline = readOutline(doc.catalog, destinationRegistry, doc, sink); + // Each surface ReadPdfOptions.surfaces turns off is never parsed: its reader is not called, and what page interpretation would have asked of it answers nothing (see ReadPdfOptions.surfaces for what page items lose). + const skippedSurfaces = skippedSurfacesOf(options?.surfaces); + const reads = (surface: LayoutDocumentSurface): boolean => + !skippedSurfaces.includes(surface); + const destinationRegistry = reads("destinations") + ? createDestinationRegistry( + doc.catalog, + doc, + (obj) => doc.pageIndex(obj), + sink, + ) + : UNREAD_DESTINATION_REGISTRY; + const outline = reads("outline") + ? readOutline(doc.catalog, destinationRegistry, doc, sink) + : undefined; // A page-selected read decodes no embedded file: see ReadPdfOptions.pages. const attachments = selection === undefined ? readAttachments(doc.catalog, pageDicts, doc, sink) : []; - const optionalContent = readOptionalContent(doc.catalog, doc, sink); - const form = readAcroForm( - doc.catalog, - doc, - (obj) => doc.pageIndex(obj), - sink, - ); - const structure = readStructure( - doc.catalog, - pageDicts, - (index) => selection === undefined || selection.has(index), - doc, - sink, - ); - const source = readDocumentResidue(doc, sink); + const optionalContent = reads("layers") + ? readOptionalContent(doc.catalog, doc, sink) + : undefined; + const form = reads("form") + ? readAcroForm(doc.catalog, doc, (obj) => doc.pageIndex(obj), sink) + : undefined; + const structure = reads("structure") + ? readStructure( + doc.catalog, + pageDicts, + (index) => selection === undefined || selection.has(index), + doc, + sink, + ) + : undefined; + const source = reads("residue") ? readDocumentResidue(doc, sink) : undefined; const pageBoxRows: PdfObject[] = []; // An unselected page is skipped before readPage, the only path that fetches a page's /Contents, interprets it, and resolves the fonts and images it names: those costs are paid for selected pages alone. Each page keeps its source position as the page index readPage stamps (diagnostics, the structure-ownership lookup, the page-boxes residue row), and a selected page also carries it as `index`, since its position in `pages` no longer says which page it is. @@ -180,8 +227,8 @@ export function readPdf( images, imageIdCache, destinationRegistry, - optionalContent.layerNameOf, - structure.ownerOf, + optionalContent?.layerNameOf ?? noLayerName, + structure?.ownerOf ?? noStructureOwner, sink, ); if (read.boxRow !== undefined) { @@ -190,8 +237,8 @@ export function readPdf( return [selection === undefined ? read.page : { ...read.page, index }]; }); - // The per-page boundary declarations a distinct crop box or a print-production box left behind, quarantined with the other package-level residue rows. - if (pageBoxRows.length > 0) { + // The per-page boundary declarations a distinct crop box or a print-production box left behind, quarantined with the other package-level residue rows, and so left out with them when residue is skipped. + if (source !== undefined && pageBoxRows.length > 0) { source["page-boxes"] = { format: "pdf", xml: serializeObjectToText(pdfArray(pageBoxRows)), @@ -207,17 +254,31 @@ export function readPdf( ...(destinationRegistry.entries.length > 0 ? { destinations: [...destinationRegistry.entries] } : {}), - ...(outline.length > 0 ? { outline } : {}), + ...(outline !== undefined && outline.length > 0 ? { outline } : {}), ...(attachments.length > 0 ? { attachments } : {}), - ...(optionalContent.layers.length > 0 + ...(optionalContent !== undefined && optionalContent.layers.length > 0 ? { layers: [...optionalContent.layers] } : {}), - ...(form.length > 0 ? { form } : {}), - ...(structure.tree.length > 0 ? { structure: [...structure.tree] } : {}), - ...(Object.keys(source).length > 0 ? { source } : {}), + ...(form !== undefined && form.length > 0 ? { form } : {}), + ...(structure !== undefined && structure.tree.length > 0 + ? { structure: [...structure.tree] } + : {}), + ...(source !== undefined && Object.keys(source).length > 0 + ? { source } + : {}), + ...(skippedSurfaces.length > 0 ? { skippedSurfaces } : {}), }; } +// What page interpretation asks of a skipped layers or structure surface: an /OC membership names no layer, and marked content has no owner. +function noLayerName(): undefined { + return undefined; +} + +function noStructureOwner(): undefined { + return undefined; +} + // --- Page geometry: MediaBox origin shift + /Rotate, composed into one matrix applied to every extracted item on the page. --- interface PageBoxRect { diff --git a/packages/pdf-codec/src/test-support/pdf-surfaces.ts b/packages/pdf-codec/src/test-support/pdf-surfaces.ts new file mode 100644 index 000000000..77a39356e --- /dev/null +++ b/packages/pdf-codec/src/test-support/pdf-surfaces.ts @@ -0,0 +1,115 @@ +import { + EMC, + EMPTY_DICT, + enc, + FixtureBuilder, + HELVETICA_FONT_DICT, + PAGE_OBJ, +} from "./pdf"; + +// A two-page document carrying every document-level surface ReadPdfOptions.surfaces can leave out, each with a trap: one reference to an object number the cross-reference table does not hold (DOCUMENT_SURFACE_TRAP_OBJECTS), reached only by that surface's own reader, so resolving it reports pdf/object-missing-value naming that number exactly when the surface was parsed. The traps sit in the outline's /Next chain, the old-style /Dests dictionary, the AcroForm /Fields array, the /OCGs array, the structure root's /K array and the catalog /OutputIntents (residue). Page 0 also carries what page items take from those surfaces: a structure-owned marked-content span (MCID 0), a span in the "Hidden" layer (off in the default configuration), an internal link naming a name-tree destination, an internal link stating a direct destination array, a URI link, and a /TrimBox for the page-boxes residue row. Page 1 exists so destinations resolve to a real second page. +export const DOCUMENT_SURFACE_TRAP_OBJECTS = { + outline: 901, + form: 902, + layers: 903, + structure: 904, + destinations: 905, + residue: 906, +} as const; +export function documentSurfacesPdf(): Uint8Array { + const b = new FixtureBuilder().header(); + const PAGE_1_OBJ = 4; + const SHARED_FONT_OBJ = 5; + const CONTENT_PAGE_0_OBJ = 6; + const LINK_NAMED_OBJ = 7; + const LINK_DIRECT_OBJ = 8; + const LINK_URI_OBJ = 9; + const OUTLINES_ROOT_OBJ = 10; + const OLD_STYLE_DESTS_OBJ = 11; + const NAME_TREE_OBJ = 12; + const FIELD_OBJ = 13; + const OCG_HIDDEN_OBJ = 14; + const STRUCT_TREE_ROOT_OBJ = 15; + const STRUCT_PARA_OBJ = 16; + const CONTENT_PAGE_1_OBJ = 17; + const OUTLINE_ITEM_OBJ = 18; + const PARENT_TREE_OBJ = 19; + b.object( + 1, + `<< /Type /Catalog /Pages 2 0 R /Outlines ${OUTLINES_ROOT_OBJ} 0 R /Dests ${OLD_STYLE_DESTS_OBJ} 0 R /Names << /Dests ${NAME_TREE_OBJ} 0 R >> /AcroForm << /Fields [${FIELD_OBJ} 0 R ${DOCUMENT_SURFACE_TRAP_OBJECTS.form} 0 R] >> /OCProperties << /OCGs [${OCG_HIDDEN_OBJ} 0 R ${DOCUMENT_SURFACE_TRAP_OBJECTS.layers} 0 R] /D << /OFF [${OCG_HIDDEN_OBJ} 0 R] >> >> /StructTreeRoot ${STRUCT_TREE_ROOT_OBJ} 0 R /PageMode /UseOutlines /OutputIntents [${DOCUMENT_SURFACE_TRAP_OBJECTS.residue} 0 R] >>`, + ); + b.object(2, `<< /Type /Pages /Kids [3 0 R ${PAGE_1_OBJ} 0 R] /Count 2 >>`); + b.object( + PAGE_OBJ, + `<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] /TrimBox [10 10 190 190] /Resources << /Font << /F1 ${SHARED_FONT_OBJ} 0 R >> /Properties << /L1 << /OC ${OCG_HIDDEN_OBJ} 0 R >> >> >> /Contents ${CONTENT_PAGE_0_OBJ} 0 R /StructParents 0 /Annots [${LINK_NAMED_OBJ} 0 R ${LINK_DIRECT_OBJ} 0 R ${LINK_URI_OBJ} 0 R] >>`, + ); + b.object( + PAGE_1_OBJ, + `<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] /Resources << /Font << /F1 ${SHARED_FONT_OBJ} 0 R >> >> /Contents ${CONTENT_PAGE_1_OBJ} 0 R >>`, + ); + b.object(SHARED_FONT_OBJ, HELVETICA_FONT_DICT); + b.stream( + CONTENT_PAGE_0_OBJ, + EMPTY_DICT, + enc( + [ + "/P << /MCID 0 >> BDC", + "BT /F1 12 Tf 10 180 Td (Owned paragraph) Tj ET", + EMC, + "/OC /L1 BDC", + "BT /F1 12 Tf 10 150 Td (Hidden layer text) Tj ET", + EMC, + "BT /F1 12 Tf 10 120 Td (Plain text) Tj ET", + ].join("\n"), + ), + ); + b.object( + LINK_NAMED_OBJ, + "<< /Type /Annot /Subtype /Link /Rect [10 10 60 24] /Dest (second) >>", + ); + b.object( + LINK_DIRECT_OBJ, + `<< /Type /Annot /Subtype /Link /Rect [70 10 120 24] /Dest [${PAGE_1_OBJ} 0 R /Fit] >>`, + ); + b.object( + LINK_URI_OBJ, + "<< /Type /Annot /Subtype /Link /Rect [130 10 180 24] /A << /S /URI /URI (https://example.com/) >> >>", + ); + b.object( + OUTLINES_ROOT_OBJ, + `<< /Type /Outlines /First ${OUTLINE_ITEM_OBJ} 0 R /Last ${OUTLINE_ITEM_OBJ} 0 R /Count 1 >>`, + ); + b.object( + OLD_STYLE_DESTS_OBJ, + `<< /first [3 0 R /Fit] /broken ${DOCUMENT_SURFACE_TRAP_OBJECTS.destinations} 0 R >>`, + ); + b.object( + NAME_TREE_OBJ, + `<< /Names [(second) [${PAGE_1_OBJ} 0 R /XYZ 0 100 null]] >>`, + ); + b.object( + FIELD_OBJ, + "<< /Type /Annot /Subtype /Widget /FT /Tx /T (fullname) /V (Jane Doe) /Rect [10 40 110 56] /P 3 0 R >>", + ); + b.object(OCG_HIDDEN_OBJ, "<< /Type /OCG /Name (Hidden) >>"); + b.object( + STRUCT_TREE_ROOT_OBJ, + `<< /Type /StructTreeRoot /K [${STRUCT_PARA_OBJ} 0 R ${DOCUMENT_SURFACE_TRAP_OBJECTS.structure} 0 R] /ParentTree ${PARENT_TREE_OBJ} 0 R >>`, + ); + b.object( + STRUCT_PARA_OBJ, + `<< /Type /StructElem /S /P /P ${STRUCT_TREE_ROOT_OBJ} 0 R /K [0] >>`, + ); + b.stream( + CONTENT_PAGE_1_OBJ, + EMPTY_DICT, + enc("BT /F1 12 Tf 10 180 Td (Second page) Tj ET"), + ); + b.object( + OUTLINE_ITEM_OBJ, + `<< /Title (Introduction) /Parent ${OUTLINES_ROOT_OBJ} 0 R /Dest (second) /Next ${DOCUMENT_SURFACE_TRAP_OBJECTS.outline} 0 R >>`, + ); + b.object(PARENT_TREE_OBJ, `<< /Nums [0 [${STRUCT_PARA_OBJ} 0 R]] >>`); + b.classicXrefAndTrailer(PARENT_TREE_OBJ, "/Root 1 0 R"); + return b.bytes(); +} diff --git a/packages/pdf-codec/src/write-skipped-surfaces.ts b/packages/pdf-codec/src/write-skipped-surfaces.ts new file mode 100644 index 000000000..2f21eccc4 --- /dev/null +++ b/packages/pdf-codec/src/write-skipped-surfaces.ts @@ -0,0 +1,17 @@ +import type { LayoutDocumentSurface } from "./layout"; + +/** + * writePdf was handed a LayoutDocument from a read that left document-level surfaces out (one carrying `skippedSurfaces`, which only `ReadPdfOptions.surfaces` sets). Such a document is a reading with parts of the file deliberately unread, not a document to re-emit: each skipped surface's field is absent whatever the source holds, and its page items lack what depended on it (internal links without destinations, structure owners without the structure tree, layer names without layers), so writing it would silently drop the source's outline, form, layers, structure or residue. Thrown rather than writing a file that looks complete. Checked after `PdfPartialDocumentError`, so a page-selected read that also skipped surfaces reports the page selection. + */ +export class PdfSkippedSurfacesError extends Error { + /** The surfaces the read skipped, as the document's `skippedSurfaces` lists them. */ + readonly skippedSurfaces: readonly LayoutDocumentSurface[]; + + constructor(skippedSurfaces: readonly LayoutDocumentSurface[]) { + super( + `this LayoutDocument was read with ${skippedSurfaces.join(", ")} left out (it carries skippedSurfaces, which only a read with ReadPdfOptions.surfaces sets); writing it would drop whatever the source holds there, so it cannot be written as a document of its own`, + ); + this.name = "PdfSkippedSurfacesError"; + this.skippedSurfaces = skippedSurfaces; + } +} diff --git a/packages/pdf-codec/src/write.ts b/packages/pdf-codec/src/write.ts index 1314bcaad..1190b1065 100644 --- a/packages/pdf-codec/src/write.ts +++ b/packages/pdf-codec/src/write.ts @@ -18,6 +18,7 @@ import { emitFontObjects } from "./write-font-objects"; import { emitPageObjects } from "./write-page-emit"; import { emitOutlineObjects } from "./write-outline"; import { buildInfoDict } from "./write-strings"; +import { PdfSkippedSurfacesError } from "./write-skipped-surfaces"; import type { PositionedFormula } from "document-schema.js"; import type { LayoutDocument, @@ -189,6 +190,9 @@ export function writePdf( if (doc.pageCount !== undefined) { throw new PdfPartialDocumentError(doc.pageCount, doc.pages.length); } + if (doc.skippedSurfaces !== undefined) { + throw new PdfSkippedSurfacesError(doc.skippedSurfaces); + } const compress = options.compress ?? true; const registry = options.fonts; const measurer = createFontMeasurer(registry); From 3f6ffb0a3c75fd9f00f2f3c172726b821c1583e7 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Sat, 17 Oct 2026 06:20:36 +0100 Subject: [PATCH 02/11] feat(documents.js): thread ReadPdfOptions.surfaces through the pdf conversions The pdfTo* conversions, convertDocument with a pdf source and readNativeDocumentTree/readContentDocument take a surfaces option, handed to the same first hop pages goes to, so an intermediate pdf a route renders is always read whole. readDocumentMetadata now reads a pdf with every document-level surface skipped as well as no page, and returns the same metadata a full read does. openPdf takes neither option. ReadPdfSurfaces, LayoutDocumentSurface and PdfSkippedSurfacesError are exported from the root barrel. Refs #1692 --- packages/documents.js/README.md | 28 ++- .../documents.js/src/convert/composition.ts | 9 +- .../src/convert/from-pdf-surfaces.test.ts | 189 ++++++++++++++++++ packages/documents.js/src/convert/from-pdf.ts | 28 ++- packages/documents.js/src/edit/pdf/editor.ts | 4 +- packages/documents.js/src/index.ts | 9 +- 6 files changed, 258 insertions(+), 9 deletions(-) create mode 100644 packages/documents.js/src/convert/from-pdf-surfaces.test.ts diff --git a/packages/documents.js/README.md b/packages/documents.js/README.md index 374a4bb36..1e2db2c41 100644 --- a/packages/documents.js/README.md +++ b/packages/documents.js/README.md @@ -387,7 +387,7 @@ It carries the ten `pdfTo*` conversions, `PdfToDocumentOptions`, `readDocumentMe ### Converting a selection of a PDF's pages -The PDF-sourced entry points that produce content take a `pages` option: the `pdfTo*` conversions (`PdfToDocumentOptions.pages`), `convertDocument` with a pdf source (`UnifiedConversionOptions.pages`), and `readNativeDocumentTree`/`readContentDocument` (`ReadNativeDocumentTreeOptions.pages`). It is pdf-codec's `ReadPdfOptions.pages` threaded through: zero-based page-tree indices, a negative index counted from the end (`-1` is the last page), duplicates collapsed, the result in document order, and `PdfPageSelectionError` for an index naming no page. Two entry points never take it: `readDocumentMetadata` always reads with an empty selection, since metadata never depends on page content, so identifying a document reads no page at all; and `openPdf` always reads every page, since an editor holds a document to write back. `convertDocument` applies `pages` only when the source itself is pdf: a conversion from another format ignores it, including one whose route renders an intermediate pdf internally (docx to ods), because that pdf is not the caller's document. +The PDF-sourced entry points that produce content take a `pages` option: the `pdfTo*` conversions (`PdfToDocumentOptions.pages`), `convertDocument` with a pdf source (`UnifiedConversionOptions.pages`), and `readNativeDocumentTree`/`readContentDocument` (`ReadNativeDocumentTreeOptions.pages`). It is pdf-codec's `ReadPdfOptions.pages` threaded through: zero-based page-tree indices, a negative index counted from the end (`-1` is the last page), duplicates collapsed, the result in document order, and `PdfPageSelectionError` for an index naming no page. Two entry points never take it: `readDocumentMetadata` always reads with an empty selection (and every document-level surface skipped, below), since metadata never depends on page content, so identifying a document reads no page at all; and `openPdf` always reads every page, since an editor holds a document to write back. `convertDocument` applies `pages` only when the source itself is pdf: a conversion from another format ignores it, including one whose route renders an intermediate pdf internally (docx to ods), because that pdf is not the caller's document. ```ts import { pdfToMarkdown } from "documents.js/read"; @@ -403,6 +403,30 @@ Markdown output labels each selected page with its original one-based page numbe Every reconstructed node's frames name the page's index in the source PDF (the same index every destination and form widget names), which is why a page-selected conversion's reported `DocumentTree` carries no `pages` envelope: that array is indexed by frame page index, and a selection cannot fill it. Such a tree cannot be rebuilt to pdf either: `buildDocumentBytes(tree, "pdf")` needs that envelope and throws without it. A later stage's own page-addressing option, such as `page` for an svg target, counts within the selection. A page-selected `LayoutDocument` cannot be written back: `writePdf` refuses one with `PdfPartialDocumentError`. `PdfPageSelectionError`, `PdfPartialDocumentError` and `sourcePageIndex` are exported from the root barrel; the `documents.js/read` entry re-exports nothing (this package allows re-exports only in its root barrel), so a caller of that entry recognises a bad selection by the thrown error's `code` (`pdf/page-out-of-range` or `pdf/page-index-not-integer`), and resolves a page's source index as `page.index ?? position`. +### Leaving a PDF's document-level surfaces out + +The same entry points take a `surfaces` option (`PdfToDocumentOptions.surfaces`, `UnifiedConversionOptions.surfaces`, `ReadNativeDocumentTreeOptions.surfaces`): pdf-codec's `ReadPdfOptions.surfaces` threaded through, one boolean per document-level surface (`outline`, `destinations`, `form`, `layers`, `structure`, `residue`), each read unless set to `false`, so omitting it converts exactly as before. It is independent of `pages`, and `convertDocument` hands it to the same hop `pages` goes to, so an intermediate pdf a route renders internally is always read whole. Combined with a small page selection it is what keeps identifying a document from a few pages cheap on a long tagged PDF, whose structure tree and `/ParentTree` a page selection alone still parses whole. + +```ts +import { pdfToMarkdown } from "documents.js/read"; + +const firstPages = pdfToMarkdown(pdfBytes, { + pages: [0, 1, 2, -1], + surfaces: { + outline: false, + destinations: false, + form: false, + layers: false, + structure: false, + residue: false, + }, +}); +``` + +A skipped surface is never parsed, and the conversion carries nothing built from it: the reported `DocumentTree` has no outline, destinations, layers or residue tables for a skipped surface, internal links are left out without `destinations` (links to a URI are kept), and form fields are absent without `form`. Two skips change reconstruction itself. Without `structure`, a tagged PDF's headings, table cells and divisions are inferred from geometry exactly as for an untagged file, so they can come out differently than in a full conversion. Without `layers`, nothing marks content as belonging to a hidden layer, so content the default view hides is kept as if it were visible. A `DocumentTree` from a full-page conversion that skipped surfaces still carries its `pages` envelope, so `buildDocumentBytes(tree, "pdf")` rebuilds it, without the skipped surfaces. The `LayoutDocument` such a read produces lists them in `skippedSurfaces`, so an absent field there means "not read" rather than "none", and `writePdf` refuses it with `PdfSkippedSurfacesError`. + +`readDocumentMetadata` always reads a pdf with an empty page selection and every surface skipped, since metadata comes from the trailer's `/Info`, the catalog's `/Lang` and the XMP packet alone, so identifying a document parses neither its pages nor its outline, destinations, form, layers, structure tree or residue, and returns the same metadata a full read does. `openPdf` takes neither option, since an editor holds a whole document to write back. `ReadPdfSurfaces`, `LayoutDocumentSurface` and `PdfSkippedSurfacesError` are exported from the root barrel. + ### Reading and building xlsx content directly Every other content format has its own standalone `readXContent`-shaped entry point (`readDocxContent`, `readPptxContent`, `readOdtContent`, `readOdpContent`, `readOdsContent`, `readOdgContent`) — xlsx is no longer the exception. `readXlsxContent`/`buildXlsxPackage` are this package's names for `ooxml.js`'s own spreadsheet `ContentDocument` read/build pair — the same one the `ods⇄xlsx` bridge and every xlsx metadata-rebuild path already use internally — re-exported here directly rather than wrapped, since `readXlsxContent` already produces the right shape on its own. (Since `ooxml.js` 4.0.0 the upstream flat builder is named `buildXlsxPackageFromContent` — the bare `buildXlsxPackage` name moved to that package's tree-form `DocumentTree` builder — so this package re-exports the flat builder under its own long-standing `buildXlsxPackage` name and the `ContentDocument`-in/`Package`-out contract is unchanged.) csv's `readCsvContent`/`buildCsvText` are the same kind of directly-exported stage pair, one level further in: they operate on RFC 4180 text rather than a decoded package (see `src/csv/` under Architecture). svg's `readSvgContent`/`buildSvgText` are the drawing-variant counterpart of csv's pair, operating on SVG text rather than a decoded package (see `src/svg/` under Architecture). @@ -840,7 +864,7 @@ The package is layered from generic primitives outward to the two conversion dir - **`src/odb/`** — decoder-selection and pivot-mapping: `read.ts` routes to the right tier, `spreadsheet.ts`/`csv.ts` map to output formats. `odb/sql/` is the bounded SQL engine, `odb/formula/` is the rpt formula engine, `odb/report/` is the renderer, `odb/values.ts` is shared comparison/aggregation semantics. - **`src/convert/`** — the composition layer: `convert.ts` (the named conversion functions and their option types), `composition.ts` (the pathfinder, the primitive registry, the bridge/fromPdf executors, and `convertDocumentFromPdf` — the read half of the engine), `composition-to-pdf.ts` (`executeToPdf`, the layout-engine registry, and the full `convertDocument` binding — the write half, split out so a read-only consumer never statically imports a renderer), `from-pdf.ts` (the `pdfTo*` family, `readDocumentMetadata`, `readNativeDocumentTree`, `readContentDocument`, and the `documents.js/read` entry), `codec.ts` (`z.codec()` pairs), `port.ts`/`local.ts` (the `DocumentConverter` port), `variant-bridges.ts` (cross-variant semantic transforms), and `from-package.ts` (`buildDocumentBytes`, which flattens once at the boundary). The tree ⇄ flat transform those construction sites call — `assembleTree`, `decompose`, `flattenTree`, `factorStyles` — is not implemented here: it lives in `document-schema.js`, which owns both encodings. `bijection.test.ts` is this package's own gate on it, re-running the three laws over the real corpus every reader, editor, and conversion here produces. - **`src/codecs/`** — `DOCUMENT_FORMAT_CODECS`: every format's read/build capability as data, so `setDocumentMetadata`/`buildDocumentBytes` dispatch through one registry; its read halves live in `read.ts`, whose `CONTENT_READERS` `readDocumentMetadata` uses for content formats (it reads pdf through pdf-codec's `readPdf` directly, with no page selected). -- **`src/metadata/`**: cross-format metadata write via `DOCUMENT_FORMAT_CODECS`; the read side is `readDocumentMetadata` in `src/convert/from-pdf.ts`, which reads a content format through `CONTENT_READERS` and a pdf through `readPdf` with an empty page selection. +- **`src/metadata/`**: cross-format metadata write via `DOCUMENT_FORMAT_CODECS`; the read side is `readDocumentMetadata` in `src/convert/from-pdf.ts`, which reads a content format through `CONTENT_READERS` and a pdf through `readPdf` with an empty page selection and every document-level surface skipped. - **`src/package-codec.ts`** — `decodeDocumentPackage`/`encodeDocumentPackage`/`decodeOdbPackage`. Dependency direction is downward and checkable. Twelve external dependencies each own a distinct concern: `ooxml.js` (docx/pptx/xlsx), `odf.js` (odt/ods/odp/odg), `document-schema.js` (shared schemas + port contracts), `pdf-codec` (PDF codec + text-layout/font primitives), `byte-codec` (byte/image utilities), `markdown-codec` (markdown), `rtf-codec` (rtf), `wpd-codec` (wpd, read-only), `doc-codec` (doc), `xls-codec` (xls), `ppt-codec` (ppt, through this package's own `src/ppt/` envelope adapter), `archive-codec` (the `[MS-CFB]` compound-file detection all three legacy binary codecs' own bytes schemas build on). No `PdfObject`/`PdfDict`/`PdfStream` type appears anywhere in this package. diff --git a/packages/documents.js/src/convert/composition.ts b/packages/documents.js/src/convert/composition.ts index 937b62bf4..fa493e812 100644 --- a/packages/documents.js/src/convert/composition.ts +++ b/packages/documents.js/src/convert/composition.ts @@ -24,6 +24,7 @@ import { } from "odf.js"; import { sourcePageIndex } from "pdf-codec/layout"; import { readPdf } from "pdf-codec/read"; +import type { ReadPdfSurfaces } from "pdf-codec/read"; import type { LayoutDocument, PdfDiagnosticSink, @@ -324,6 +325,10 @@ export interface UnifiedConversionOptions { * The output holds only those pages. A markdown target labels each with its original one-based page number (`` at the start of the page, beside the usual `` between pages), so the text says which pages it holds. A page-addressing option of a later stage, such as `page` for an svg target, counts within the selection. Reconstruction infers from the selected pages alone, so a page can reconstruct differently than it would in a full conversion: the body text size heading detection compares against is the most common size across the selection, which on a cover page dominated by large title text is the title's own size, leaving the title a paragraph rather than a heading. The DocumentTree `onDocument` reports carries no `pages` envelope for a page-selected conversion: its frames name source page indices, which an envelope holding only the selected pages could not be indexed by. */ readonly pages?: readonly number[]; + /** + * The document-level surfaces of a pdf source to read, as pdf-codec's `ReadPdfOptions.surfaces` takes them: one boolean per surface (`outline`, `destinations`, `form`, `layers`, `structure`, `residue`), each read unless set to `false`, so omitting it reads everything as before. Independent of `pages`, and handed to the same hop: only the plan's first hop, and only when the conversion's source is pdf, so an intermediate pdf a route renders internally is always read whole. A skipped surface is never parsed and is absent from the read, so what the conversion builds from it is absent too: no outline or bookmark tables, no destinations and no internal links (a link to a URI is kept), no form fields, no layer tables, no structure, no package residue. Two skips also change reconstruction: without `structure` a tagged PDF's headings, table cells and divisions are inferred from geometry as for an untagged file, and without `layers` content the default view hides cannot be told apart and is kept as if visible. See `ReadPdfOptions.surfaces` for exactly what each covers. + */ + readonly surfaces?: ReadPdfSurfaces; } // What the fromPdf executor knows about a conversion's pages that a text build needs and the caller's options do not carry, passed to a text node's build alongside those options: the original one-based page number of each page the read returned, set only for a page-selected read (see UnifiedConversionOptions.pages), where a page's position in the output no longer says which page it is. Internal to the executors, never caller-supplied. @@ -651,6 +656,7 @@ export function executeFromPdf( signal: options?.signal, sink: options?.sink, pages: options?.pages, + surfaces: options?.surfaces, }); const content = RECONSTRUCTORS[node.variant](layout, { signal: options?.signal, @@ -720,7 +726,7 @@ export function runCompositionPlan( } // onDocument fires on the last hop only: null out onDocument for every hop before the last, then pass the caller's options through unchanged on the final hop so the callback receives the package that actually produced the output bytes. This mirrors the convention the original hand-written composed functions (xlsxToPdf/pdfToXlsx/xlsxToMarkdown/markdownToXlsx) already followed — each forwarded onDocument to its own last hop, never the first — so a caller of xlsxToPdf still receives the odsToPdf hop's content+layout package rather than the intermediate xlsx->ods bridge's content-only one. const isLastHop = i === plan.hops.length - 1; - // pages selects pages of the caller's own pdf source, so only the hop that reads that source receives it: the first hop of a plan whose source is pdf. Every other hop's pdf (an intermediate rendering, as on a docx -> ods route through toPdf then fromPdf) is not the caller's document, and selecting its pages would silently drop content or throw for an index the caller never meant against it. + // pages and surfaces select what to read of the caller's own pdf source, so only the hop that reads that source receives them: the first hop of a plan whose source is pdf. Every other hop's pdf (an intermediate rendering, as on a docx -> ods route through toPdf then fromPdf) is not the caller's document, and selecting its pages would silently drop content or throw for an index the caller never meant against it, and skipping its surfaces would drop what the caller's own source carried through it. const readsCallerPdf = i === 0 && hop.from === "pdf"; const hopOptions: UnifiedConversionOptions | undefined = options === undefined @@ -729,6 +735,7 @@ export function runCompositionPlan( ...options, onDocument: isLastHop ? options.onDocument : undefined, pages: readsCallerPdf ? options.pages : undefined, + surfaces: readsCallerPdf ? options.surfaces : undefined, }; if (hop.executor === "toPdf") { if (!isSourceContentFormat(hop.from)) { diff --git a/packages/documents.js/src/convert/from-pdf-surfaces.test.ts b/packages/documents.js/src/convert/from-pdf-surfaces.test.ts new file mode 100644 index 000000000..22a503494 --- /dev/null +++ b/packages/documents.js/src/convert/from-pdf-surfaces.test.ts @@ -0,0 +1,189 @@ +import { DEFAULT_LAYOUT_FONT } from "document-schema.js"; +import type { LayoutDocument, ReadPdfSurfaces } from "pdf-codec"; +import { LAYOUT_FORMAT_VERSION, writePdf } from "pdf-codec"; +import type * as PdfCodecRead from "pdf-codec/read"; +import { readPdf } from "pdf-codec/read"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { convertDocument } from "./composition-to-pdf"; +import { markdownToPdf } from "./convert"; +import { + pdfToDocx, + pdfToMarkdown, + readContentDocument, + readDocumentMetadata, + readNativeDocumentTree, +} from "./from-pdf"; +import { decodeMarkdownText, encodeMarkdownText } from "../markdown/text"; + +// readPdf wrapped so a test can see which surfaces each read asked for. The real reader still runs. +vi.mock("pdf-codec/read", async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, readPdf: vi.fn(actual.readPdf) }; +}); + +const BLACK = { r: 0, g: 0, b: 0 }; +const PAGE = { widthPt: 300, heightPt: 300 }; +const NO_SURFACES: Required = { + outline: false, + destinations: false, + form: false, + layers: false, + structure: false, + residue: false, +}; +const XMP_PACKET = + ''; + +// A two-page PDF carrying every document-level surface readPdf can leave out (an outline, named destinations, a form field, a layer, a structure tree, XMP residue) plus /Info metadata, written by pdf-codec's own writer so the reader sees each of them. +function everySurfacePdf(): Uint8Array { + const text = (value: string, yPt: number) => ({ + kind: "text" as const, + text: value, + xPt: 20, + yPt, + font: DEFAULT_LAYOUT_FONT, + sizePt: 12, + color: BLACK, + }); + const doc: LayoutDocument = { + formatVersion: LAYOUT_FORMAT_VERSION, + metadata: { + title: "Surface fixture", + author: "A. Author", + subject: "Every surface", + keywords: ["outline", "structure"], + }, + images: {}, + pages: [ + { + ...PAGE, + items: [ + { ...text("Tagged heading", 250), structure: "s1" }, + { ...text("Layered note", 200), layer: "Notes" }, + { + kind: "internalLink", + destination: "second", + xPt: 20, + yPt: 100, + widthPt: 60, + heightPt: 14, + }, + ], + }, + { ...PAGE, items: [text("Second page", 250)] }, + ], + destinations: [{ name: "second", pageIndex: 1, target: { kind: "fit" } }], + outline: [{ title: "Second page", destination: "second", children: [] }], + layers: [{ name: "Notes", visible: false }], + form: [ + { + name: "fullname", + fieldType: "text", + value: "Jane Doe", + widgets: [ + { pageIndex: 0, xPt: 20, yPt: 40, widthPt: 100, heightPt: 16 }, + ], + children: [], + }, + ], + structure: [{ id: "s1", type: "H1", children: [] }], + source: { xmp: { format: "pdf", xml: XMP_PACKET } }, + }; + return writePdf(doc); +} + +beforeEach(() => { + vi.mocked(readPdf).mockClear(); +}); + +function surfacesRead(): (ReadPdfSurfaces | undefined)[] { + return vi.mocked(readPdf).mock.calls.map(([, options]) => options?.surfaces); +} + +describe("readDocumentMetadata for a pdf", () => { + it("reads no page and no document-level surface, and reports exactly the metadata a full read does", () => { + const pdf = everySurfacePdf(); + const full = readPdf(pdf); + for (const field of [ + "outline", + "destinations", + "form", + "layers", + "structure", + "source", + ] as const) { + expect(full[field]).toBeDefined(); + } + vi.mocked(readPdf).mockClear(); + const metadata = readDocumentMetadata("pdf", pdf); + expect( + vi.mocked(readPdf).mock.calls.map(([, options]) => ({ + pages: options?.pages, + surfaces: options?.surfaces, + })), + ).toEqual([{ pages: [], surfaces: NO_SURFACES }]); + expect(metadata).toEqual(full.metadata); + expect(metadata.title).toBe("Surface fixture"); + expect(metadata.author).toBe("A. Author"); + }); +}); + +describe("the pdf conversions with surfaces", () => { + it("hand surfaces to readPdf from every pdfTo* conversion and from readNativeDocumentTree/readContentDocument", () => { + const pdf = everySurfacePdf(); + const surfaces: ReadPdfSurfaces = { structure: false, outline: false }; + pdfToMarkdown(pdf, { surfaces }); + pdfToDocx(pdf, { surfaces }); + readNativeDocumentTree("pdf", pdf, { surfaces }); + readContentDocument("pdf", pdf, { surfaces }); + expect(surfacesRead()).toEqual([surfaces, surfaces, surfaces, surfaces]); + }); + + it("leaves a skipped surface's tables off the reported tree, and keeps the rest", () => { + const pdf = everySurfacePdf(); + const full = readNativeDocumentTree("pdf", pdf); + const skipped = readNativeDocumentTree("pdf", pdf, { + surfaces: { layers: false }, + }); + expect(full.layers).toBeDefined(); + expect(skipped.layers).toBeUndefined(); + expect(skipped.source).toEqual(full.source); + }); + + it("keeps hidden-layer content when layers are skipped, since nothing then marks it hidden", () => { + const pdf = everySurfacePdf(); + const markdown = (surfaces?: ReadPdfSurfaces): string => + decodeMarkdownText(pdfToMarkdown(pdf, { surfaces })); + expect(markdown()).not.toContain("Layered note"); + expect(markdown({ layers: false })).toContain("Layered note"); + }); + + it("converts identically with surfaces omitted, empty, or every surface on", () => { + const pdf = everySurfacePdf(); + const full = pdfToMarkdown(pdf); + expect(pdfToMarkdown(pdf, { surfaces: {} })).toEqual(full); + expect( + pdfToMarkdown(pdf, { + surfaces: Object.fromEntries( + Object.keys(NO_SURFACES).map((surface) => [surface, true]), + ), + }), + ).toEqual(full); + }); + + it("applies surfaces only to the caller's own pdf: an intermediate pdf a route renders is read whole", () => { + const markdown = encodeMarkdownText("# Title\n\nBody.\n"); + convertDocument("markdown", "ods", markdown, { surfaces: NO_SURFACES }); + const intermediateReads = surfacesRead(); + expect(intermediateReads.length).toBeGreaterThan(0); + expect(intermediateReads.every((surfaces) => surfaces === undefined)).toBe( + true, + ); + + vi.mocked(readPdf).mockClear(); + convertDocument("pdf", "markdown", markdownToPdf(markdown), { + surfaces: NO_SURFACES, + }); + expect(surfacesRead()).toEqual([NO_SURFACES]); + }); +}); diff --git a/packages/documents.js/src/convert/from-pdf.ts b/packages/documents.js/src/convert/from-pdf.ts index a58837e69..9b1c8f63c 100644 --- a/packages/documents.js/src/convert/from-pdf.ts +++ b/packages/documents.js/src/convert/from-pdf.ts @@ -13,6 +13,7 @@ import { import type { MarkdownImageResolver } from "markdown-codec"; import type { LayoutDocument, PdfDiagnosticSink } from "pdf-codec"; import { readPdf } from "pdf-codec/read"; +import type { ReadPdfSurfaces } from "pdf-codec/read"; import { convertDocumentFromPdf } from "./composition"; import type { CsvWriteOptions, SvgWriteOptions } from "./convert"; import { stampPdfPackageTables } from "./pdf-package-tables"; @@ -20,6 +21,16 @@ import type { DocumentFormat } from "./port"; import { CONTENT_READERS } from "../codecs/read"; import { reconstructWordprocessing } from "../layout/reconstruct"; +// Every document-level surface off, for readDocumentMetadata's pdf read. Required rather than a partial object so a surface pdf-codec adds later fails to typecheck here until it is decided, instead of being read by default. +const METADATA_SURFACES: Required = { + outline: false, + destinations: false, + form: false, + layers: false, + structure: false, + residue: false, +}; + export interface ReadDocumentMetadataOptions { readonly signal?: AbortSignal; } @@ -31,8 +42,12 @@ export function readDocumentMetadata( options?: ReadDocumentMetadataOptions, ): LayoutMetadata { if (format === "pdf") { - // Metadata is document-level, so no page is read: an empty selection skips every page's content streams and the fonts and images they use, and the source's embedded files. It does not reduce the read to the catalog and /Info: the outline, destinations, AcroForm, structure tree and /ParentTree, and package residue are still read whole (see ReadPdfOptions.pages). - return readPdf(bytes, { signal: options?.signal, pages: [] }).metadata; + // Metadata is read from the trailer's /Info, the catalog's /Lang and the XMP packet alone, so the read takes nothing else: an empty selection skips every page's content streams, the fonts and images they use and the source's embedded files, and METADATA_SURFACES skips every document-level surface (outline, destinations, AcroForm, layers, structure tree and /ParentTree, residue), none of which metadata is built from. + return readPdf(bytes, { + signal: options?.signal, + pages: [], + surfaces: METADATA_SURFACES, + }).metadata; } return CONTENT_READERS[format](bytes, { signal: options?.signal }).metadata; } @@ -47,6 +62,10 @@ export interface ReadNativeDocumentTreeOptions { * The pages of a pdf source to read, exactly as UnifiedConversionOptions.pages describes them (zero-based, negative from the end, document order, `PdfPageSelectionError` for an index naming no page). Consulted only for a pdf source, since no other format's reader has pages to select. The unselected pages' content streams, the fonts and images only they use, and the source's embedded files are never read, and the selected pages are reconstructed from themselves alone (see PdfToDocumentOptions.pages for how that can change a cover page's headings). Every frame in the result names the page's index in the source PDF, so a DocumentTree read this way carries no `pages` envelope: that array is indexed by those source indices and a selection cannot fill it. */ readonly pages?: readonly number[]; + /** + * The document-level surfaces of a pdf source to read, exactly as UnifiedConversionOptions.surfaces describes them: one boolean per surface, each read unless set to `false`, independent of `pages`. Consulted only for a pdf source. A skipped surface is absent from the read, so the tree carries none of its tables, and reconstruction loses what it took from it: without `structure` a tagged PDF's headings, table cells and divisions are inferred from geometry, and without `layers` hidden-layer content is kept as if visible. + */ + readonly surfaces?: ReadPdfSurfaces; } // readContentDocument's own options are the identical shape — one options type for both, so a caller forwarding {signal, images, sink} through either entry point never needs to know which fields the OTHER one ignores. @@ -63,6 +82,7 @@ function readNativeContent( signal: options?.signal, sink: options?.sink, pages: options?.pages, + surfaces: options?.surfaces, }); const content = reconstructWordprocessing(layout, { signal: options?.signal, @@ -117,6 +137,10 @@ export interface PdfToDocumentOptions { * Reconstruction sees only the selected pages, so a page can come out differently than it would in a full conversion. Heading detection measures text against the body size, taken as the most common size across the pages read: a cover page read on its own, dominated by large title text, takes the title's size as its body, and the title is then a paragraph rather than a heading. Reading the cover together with a few ordinary pages restores the document's real body size. */ readonly pages?: readonly number[]; + /** + * The document-level surfaces to read, exactly as UnifiedConversionOptions.surfaces describes them: one boolean per surface (`outline`, `destinations`, `form`, `layers`, `structure`, `residue`), each read unless set to `false`, independent of `pages`. A skipped surface is never parsed, and the conversion carries nothing built from it. Without `structure` a tagged PDF's headings, table cells and divisions are inferred from geometry as for an untagged file, and without `layers` content the default view hides is kept as if visible. + */ + readonly surfaces?: ReadPdfSurfaces; // Called exactly once, synchronously, with the DocumentTree (the readPdf-produced layout, and the reconstructed content built from it) this conversion built internally, before the function returns its bytes — see convert.ts's DocumentToPdfOptions.onDocument for the mirrored X-to-PDF side of this same side channel. readonly onDocument?: (pkg: DocumentTree) => void; } diff --git a/packages/documents.js/src/edit/pdf/editor.ts b/packages/documents.js/src/edit/pdf/editor.ts index 215f66547..becda78df 100644 --- a/packages/documents.js/src/edit/pdf/editor.ts +++ b/packages/documents.js/src/edit/pdf/editor.ts @@ -69,10 +69,10 @@ export class PdfEditor { // // SECURITY NOTE: if `bytes` is encrypted with the PDF standard security handler and an empty user password (the common "restrict editing, not reading" case), readPdf decrypts it transparently and this call succeeds with a perfectly ordinary-looking PdfEditor — there is no field on the resulting document recording that this happened. toBytes() on the returned editor writes a plain, UNENCRYPTED PDF regardless, even though the source was encrypted. A caller that needs to preserve or re-apply the source's encryption must do so itself; this editor has no re-encryption path at all. A source requiring a real (non-empty) password throws PdfPasswordRequiredError, and one using an unsupported security handler throws PdfEncryptedError, both from readPdf itself before this function returns. // -// Always reads every page, which is why `pages` is not among the options: an editor holds a document to change and write back, and a page-selected read is only part of one (its references still name source pages, and writePdf refuses it with PdfPartialDocumentError). The two options it does take are forwarded field by field, so a wider-typed options value carrying `pages` cannot slip a selection through either. +// Always reads every page and every document-level surface, which is why neither `pages` nor `surfaces` is among the options: an editor holds a document to change and write back, and a page-selected read or one that skipped a surface is only part of one (writePdf refuses the first with PdfPartialDocumentError and the second with PdfSkippedSurfacesError). The two options it does take are forwarded field by field, so a wider-typed options value carrying `pages` or `surfaces` cannot slip either through. export function openPdf( bytes: Uint8Array, - options?: Omit, + options?: Omit, ): PdfEditor { return new PdfEditor( readPdf(bytes, { sink: options?.sink, signal: options?.signal }), diff --git a/packages/documents.js/src/index.ts b/packages/documents.js/src/index.ts index d86ba66cc..226639e40 100644 --- a/packages/documents.js/src/index.ts +++ b/packages/documents.js/src/index.ts @@ -149,6 +149,7 @@ export { // --- The PDF-side pivot model. The Layout* item family moved to pdf-codec at document-schema.js 4.0.0 (the demotion: a LayoutDocument is the pdf codec's own read/write artefact, not a shared pivot), so the item types and LAYOUT_FORMAT_VERSION re-export from pdf-codec here, while LayoutMetadata/LayoutFont/LayoutFrame — the members the content model itself references — stay re-exported from the schema with the shared primitives below. --- export type { LayoutDocument, + LayoutDocumentSurface, LayoutEllipse, LayoutImage, LayoutImageAsset, @@ -390,10 +391,14 @@ export { } from "./edit/pdf/item"; // --- The hand-written PDF codec, now an external dependency — see pdf-codec's own README (https://github.com/ExaDev/pdf-codec) for its internals. --- -export type { ReadPdfOptions } from "pdf-codec"; +export type { ReadPdfOptions, ReadPdfSurfaces } from "pdf-codec"; export { readPdf } from "pdf-codec"; export type { WritePdfOptions } from "pdf-codec"; -export { PdfPartialDocumentError, writePdf } from "pdf-codec"; +export { + PdfPartialDocumentError, + PdfSkippedSurfacesError, + writePdf, +} from "pdf-codec"; export type { PdfDiagnostic, PdfDiagnosticSeverity, From 4ac3d3f06b22702d82f1f57709cfdcbc8da6e4bf Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Sat, 17 Oct 2026 06:48:39 +0100 Subject: [PATCH 03/11] docs(pdf-codec): state that a skipped outline renumbers minted destinations The reader names each direct destination array with the first unused destN in the order it meets them, and the outline is read before any page, so leaving the outline out renumbers the names minted for page links and drops the entries only the outline minted. The surfaces doc comment and README said the outline was page-item neutral; they now state the renumbering, and the surfaces fixture carries a direct-array outline destination so the per-surface comparison and an explicit renumbering test cover it. Opening the document is named among what an every-surface-skipped read still does. Refs #1692 --- packages/pdf-codec/README.md | 8 +- packages/pdf-codec/src/read-surfaces.test.ts | 76 ++++++++++++++++++- packages/pdf-codec/src/read.ts | 6 +- .../src/test-support/pdf-surfaces.ts | 13 +++- 4 files changed, 89 insertions(+), 14 deletions(-) diff --git a/packages/pdf-codec/README.md b/packages/pdf-codec/README.md index a9c0e987f..5255a6f2c 100644 --- a/packages/pdf-codec/README.md +++ b/packages/pdf-codec/README.md @@ -93,13 +93,13 @@ An encrypted PDF that opens without a password decrypts transparently — no ext Both accept an optional `signal` (`AbortSignal`); `readPdf` additionally takes a `sink` (`PdfDiagnosticSink`, called once per recoverable parse diagnostic — see the three-tier failure policy under [Conventions](#conventions)), and `writePdf` an `onSubstitution` callback (called once per character not representable in a standard-14 font — see [Fidelity](#fidelity)). -**Reading a subset of pages.** `readPdf(bytes, { pages: [0, 1, 2, -1] })` reads only the listed pages: zero-based indices into the page tree, a negative index counted from the end (`-1` is the last page). What a selected read skips: an unselected page's content stream is never fetched, decoded or interpreted, so the fonts and images only unselected pages use are never resolved, and no embedded file is decoded, so `attachments` is absent from a page-selected result (an attachment is document-level payload whose size is independent of the pages asked for); an absent `attachments` therefore means "not read" whenever `pageCount` is set, which only a page-selected read does, so check `pageCount` before concluding a document has no attachments. The case this exists for is identifying a document from its first pages, or reading a code from its last page, inside a memory-capped runtime such as a Cloudflare Worker isolate, without paying for every page's content. What it still reads whole, because page references resolve against it or because it is document-level and holds no page content stream: the page tree, the outline, named destinations, optional-content layers, the AcroForm, the structure element tree and the `/ParentTree` number tree (both of which scale with the document), metadata and package residue; every one of those but the page tree and metadata can be left out too, with `surfaces` (below). Two surfaces are built from page content and so cover selected pages only: the structure-ownership association an item's `structure` field names (with its parent-tree diagnostics) and the `page-boxes` residue row. A destination a link states as a direct array is minted into `destinations` (as `dest1`, `dest2`, ...) only when its page is interpreted, so an unselected page's direct link destinations are absent and a minted name depends on which pages were read; named destinations are unaffected. Duplicates collapse and the result's `pages` holds the selected pages in ascending document order, each carrying its source position as `index` (read it through `sourcePageIndex(page, position)`, which falls back to the position on a full read), with the page tree's length as the document's `pageCount`; every document-level page reference keeps naming source indices, so a reference to a selected page resolves through `index`, and one to an unselected page names a page the result does not hold. A full read sets neither field, because there `pages.length` and each page's position already state them. An empty selection is valid and reads no page, which makes `{ pages: [] }` the cheapest way to learn a file's page count and metadata. An entry naming no page (outside `-pageCount` to `pageCount - 1`, or not an integer, `NaN` and the infinities included; `-0` is page 0) throws `PdfPageSelectionError` (a `PdfParseError` with code `pdf/page-out-of-range` or `pdf/page-index-not-integer`, carrying `pageIndex` and `pageCount`) before any page is read. A page-selected `LayoutDocument` is a reading of part of a file, not a document to re-emit: `writePdf` refuses one (any document carrying `pageCount`) with `PdfPartialDocumentError`, since it places pages by position and the references would point at the wrong pages. +**Reading a subset of pages.** `readPdf(bytes, { pages: [0, 1, 2, -1] })` reads only the listed pages: zero-based indices into the page tree, a negative index counted from the end (`-1` is the last page). What a selected read skips: an unselected page's content stream is never fetched, decoded or interpreted, so the fonts and images only unselected pages use are never resolved, and no embedded file is decoded, so `attachments` is absent from a page-selected result (an attachment is document-level payload whose size is independent of the pages asked for); an absent `attachments` therefore means "not read" whenever `pageCount` is set, which only a page-selected read does, so check `pageCount` before concluding a document has no attachments. The case this exists for is identifying a document from its first pages, or reading a code from its last page, inside a memory-capped runtime such as a Cloudflare Worker isolate, without paying for every page's content. What it still reads whole, because page references resolve against it or because it is document-level and holds no page content stream: the page tree, the outline, named destinations, optional-content layers, the AcroForm, the structure element tree and the `/ParentTree` number tree (both of which scale with the document), metadata and package residue; every one of those but the page tree and metadata can be left out too, with `surfaces` (below). Two surfaces are built from page content and so cover selected pages only: the structure-ownership association an item's `structure` field names (with its parent-tree diagnostics) and the `page-boxes` residue row. A destination a link states as a direct array is minted into `destinations` (as `dest1`, `dest2`, ...) only when its page is interpreted, so an unselected page's direct link destinations are absent and a minted name depends on which pages were read (and on whether the outline was read, below); named destinations are unaffected. Duplicates collapse and the result's `pages` holds the selected pages in ascending document order, each carrying its source position as `index` (read it through `sourcePageIndex(page, position)`, which falls back to the position on a full read), with the page tree's length as the document's `pageCount`; every document-level page reference keeps naming source indices, so a reference to a selected page resolves through `index`, and one to an unselected page names a page the result does not hold. A full read sets neither field, because there `pages.length` and each page's position already state them. An empty selection is valid and reads no page, which makes `{ pages: [] }` the cheapest way to learn a file's page count and metadata. An entry naming no page (outside `-pageCount` to `pageCount - 1`, or not an integer, `NaN` and the infinities included; `-0` is page 0) throws `PdfPageSelectionError` (a `PdfParseError` with code `pdf/page-out-of-range` or `pdf/page-index-not-integer`, carrying `pageIndex` and `pageCount`) before any page is read. A page-selected `LayoutDocument` is a reading of part of a file, not a document to re-emit: `writePdf` refuses one (any document carrying `pageCount`) with `PdfPartialDocumentError`, since it places pages by position and the references would point at the wrong pages. -**Leaving document-level surfaces out.** `readPdf(bytes, { surfaces: { structure: false, outline: false } })` reads the document without the surfaces turned off. `ReadPdfOptions.surfaces` is one object of independent booleans (`ReadPdfSurfaces`), one per surface, each read unless set to `false`, so omitting the setting reads everything exactly as before. It is independent of `pages`, applying to a full read and a page-selected read alike, and combined with a small selection it is what keeps identifying a document cheap: on a long tagged PDF the structure element tree and its `/ParentTree` can be most of what a page-selected read still parses, and `{ pages: [0, 1, 2, -1], surfaces: { outline: false, destinations: false, form: false, layers: false, structure: false, residue: false } }` reads nothing beyond the page tree, metadata and those four pages. A surface turned off is never parsed (its part of the catalog is not walked, so a malformed outline, form, layer list or structure tree reports no diagnostic), its `LayoutDocument` field is absent whatever the file holds, and the result names it in `skippedSurfaces` (in the order `LayoutDocumentSurfaceSchema` lists them, absent when every surface was read). An absent `outline`, `destinations`, `form`, `layers`, `structure` or `source` therefore means "not read" whenever `skippedSurfaces` names it, so check `skippedSurfaces` before concluding a document has none, as with `pageCount` and attachments. +**Leaving document-level surfaces out.** `readPdf(bytes, { surfaces: { structure: false, outline: false } })` reads the document without the surfaces turned off. `ReadPdfOptions.surfaces` is one object of independent booleans (`ReadPdfSurfaces`), one per surface, each read unless set to `false`, so omitting the setting reads everything exactly as before. It is independent of `pages`, applying to a full read and a page-selected read alike, and combined with a small selection it is what keeps identifying a document cheap: on a long tagged PDF the structure element tree and its `/ParentTree` can be most of what a page-selected read still parses, and `{ pages: [0, 1, 2, -1], surfaces: { outline: false, destinations: false, form: false, layers: false, structure: false, residue: false } }` reads nothing beyond opening the document (cross-reference table, trailer, catalog and any encryption dictionary), the page tree, metadata and those four pages. A surface turned off is never parsed (its part of the catalog is not walked, so a malformed outline, form, layer list or structure tree reports no diagnostic), its `LayoutDocument` field is absent whatever the file holds, and the result names it in `skippedSurfaces` (in the order `LayoutDocumentSurfaceSchema` lists them, absent when every surface was read). An absent `outline`, `destinations`, `form`, `layers`, `structure` or `source` therefore means "not read" whenever `skippedSurfaces` names it, so check `skippedSurfaces` before concluding a document has none, as with `pageCount` and attachments. -Each surface, and what page items lose without it (everything else about every page item is exactly what a read with the surface on produces): +Each surface, and what page items lose without it (everything else about every page item is exactly what a read with the surface on produces, apart from the renumbering a skipped outline causes): -- `outline`: the `/Outlines` bookmark tree (`outline`). Page items do not depend on it. +- `outline`: the `/Outlines` bookmark tree (`outline`). Page items do not depend on it, except through minted names. A destination stated as a direct array is named with the first unused `destN` in the order the reader meets it, and the outline is read before any page, so an outline entry's direct destination takes a lower number than any link's. Without the outline, the names minted for page links' direct destinations are renumbered and `destinations` loses the entries only the outline minted, while each link still resolves to the same target. - `destinations`: the named destinations of the catalog `/Dests` dictionary and the `/Names /Dests` name tree, and the registry links and outline entries name their targets through (`destinations`). Without it nothing can name an internal target, so a link whose target is internal (a `/Dest`, or a `/GoTo` action, whether it names a destination or states one directly) is left out of its page's items, and an outline entry, if the outline is read, carries no `destination`. A link to a URI is kept. diff --git a/packages/pdf-codec/src/read-surfaces.test.ts b/packages/pdf-codec/src/read-surfaces.test.ts index 75447a5f2..4e9a74ac7 100644 --- a/packages/pdf-codec/src/read-surfaces.test.ts +++ b/packages/pdf-codec/src/read-surfaces.test.ts @@ -156,6 +156,49 @@ function expectedWithout( }; } +// The destinations table entry a link or outline entry names, or undefined when the document holds none by that name. +function destinationNamed( + doc: LayoutDocument, + name: string, +): Record | undefined { + const entry = doc.destinations?.find( + (destination) => destination.name === name, + ); + return entry === undefined + ? undefined + : { pageIndex: entry.pageIndex, target: entry.target }; +} + +// The rule a skipped outline adds to the per-surface comparison: the reader mints a `destN` name for each direct destination array in the order it meets them, and the outline is read before any page, so leaving it out renumbers the names minted for page links and drops the entries only the outline minted. A document compared under this rule has each internal link's destination name replaced by the entry it resolves to, and its destinations table reduced to the entries some page item names, keyed by what they resolve to rather than by name. +function withLinksByTarget( + doc: LayoutDocument | Record, + source: LayoutDocument, +): Record { + const linkedNames = new Set( + source.pages.flatMap((page) => + page.items.flatMap((item) => + item.kind === "internalLink" ? [item.destination] : [], + ), + ), + ); + return { + ...withoutField(doc, "destinations"), + pages: mapItems(source.pages, (items) => + items.map((item) => + item.kind === "internalLink" + ? { + ...item, + destination: destinationNamed(source, item.destination), + } + : item, + ), + ), + linkedDestinations: (source.destinations ?? []) + .filter((destination) => linkedNames.has(destination.name)) + .map((destination) => destinationNamed(source, destination.name)), + }; +} + beforeEach(() => { vi.mocked(readOutline).mockClear(); vi.mocked(createDestinationRegistry).mockClear(); @@ -232,12 +275,39 @@ describe("readPdf surfaces: what a skipped surface's result holds", () => { "with %s skipped, equals the full read minus that field and what depended on it, naming it in skippedSurfaces", (surface) => { const full = readWith(undefined).doc; - expect(readWith(only(surface)).doc).toEqual( - expectedWithout(full, surface), - ); + const skipped = readWith(only(surface)).doc; + if (surface === "outline") { + // A skipped outline also renumbers minted destination names (see withLinksByTarget), so links are compared by the entry they resolve to. + expect(withLinksByTarget(skipped, skipped)).toEqual( + withLinksByTarget(expectedWithout(full, surface), full), + ); + return; + } + expect(skipped).toEqual(expectedWithout(full, surface)); }, ); + it("renumbers the destN names minted for page links when the outline is skipped, and drops the entries only the outline minted", () => { + const full = readWith(undefined).doc; + const skipped = readWith(only("outline")).doc; + const directLinkOf = (doc: LayoutDocument): string | undefined => + doc.pages[0]?.items.flatMap((item) => + item.kind === "internalLink" && item.destination !== "second" + ? [item.destination] + : [], + )[0]; + expect(full.outline?.[1]?.destination).toBe("dest1"); + expect(directLinkOf(full)).toBe("dest2"); + expect(directLinkOf(skipped)).toBe("dest1"); + expect(destinationNamed(skipped, "dest1")).toEqual( + destinationNamed(full, "dest2"), + ); + expect(destinationNamed(skipped, "dest2")).toBeUndefined(); + expect( + skipped.destinations?.filter((d) => !/^dest\d+$/u.test(d.name)), + ).toEqual(full.destinations?.filter((d) => !/^dest\d+$/u.test(d.name))); + }); + it("omits a skipped surface's field rather than leaving it present and empty", () => { const { doc } = readWith(NONE); for (const surface of SURFACES) { diff --git a/packages/pdf-codec/src/read.ts b/packages/pdf-codec/src/read.ts index 519a0e6db..70b00d747 100644 --- a/packages/pdf-codec/src/read.ts +++ b/packages/pdf-codec/src/read.ts @@ -67,7 +67,7 @@ export interface ReadPdfOptions { * * What a selected read skips: an unselected page's content stream is never fetched, decoded or interpreted, so the fonts and images only unselected pages use are never resolved; embedded-file payloads are never decoded, so `attachments` is absent from a page-selected result (an attachment is document-level payload whose size is independent of the pages asked for, and decoding it would cost the same whichever pages were selected). * - * What it still reads whole, because a negative index and every document-level page reference resolve against it or because it is document-level and holds no page content stream: the page tree, the outline, named destinations, optional-content layers, the AcroForm, the structure element tree and the /ParentTree number tree (both of which scale with the document), metadata and package residue; every one of those but the page tree and metadata can be left out as well with `surfaces`. Two surfaces are built from page content and so cover selected pages only: the structure-ownership association an item's `structure` field names (and its parent-tree diagnostics), and the `page-boxes` residue row. Destinations a link states as a direct array are minted into `destinations` (as `dest1`, `dest2`, ...) only when their page is interpreted, so an unselected page's direct link destinations are absent and a minted name depends on which pages were read; named destinations are unaffected. + * What it still reads whole, because a negative index and every document-level page reference resolve against it or because it is document-level and holds no page content stream: the page tree, the outline, named destinations, optional-content layers, the AcroForm, the structure element tree and the /ParentTree number tree (both of which scale with the document), metadata and package residue; every one of those but the page tree and metadata can be left out as well with `surfaces`. Two surfaces are built from page content and so cover selected pages only: the structure-ownership association an item's `structure` field names (and its parent-tree diagnostics), and the `page-boxes` residue row. Destinations a link states as a direct array are minted into `destinations` (as `dest1`, `dest2`, ...) only when their page is interpreted, so an unselected page's direct link destinations are absent and a minted name depends on which pages were read (and on whether the outline was read: see `surfaces`); named destinations are unaffected. * * Duplicates collapse, and the result's `pages` holds the selected pages in ascending document order whatever order they were asked for in, each carrying its source position as `index`, with the document's total as `pageCount`; every document-level reference keeps naming source indices. An empty array is a valid selection: it reads no page and returns the document-level surfaces with `pageCount`, the cheapest way to learn a file's length and metadata. An entry naming no page throws `PdfPageSelectionError` rather than being skipped. */ @@ -77,14 +77,14 @@ export interface ReadPdfOptions { * * A surface turned off is never parsed (its part of the catalog is not walked, so a malformed outline, form, layer list or structure tree reports no diagnostic) and its field is absent from the result whatever the file holds; the result lists it in `skippedSurfaces`, which is how a caller tells "not read" from "none". What each one covers, and what page items lose without it: * - * - `outline`: the `/Outlines` bookmark tree (`outline`). Page items do not depend on it. + * - `outline`: the `/Outlines` bookmark tree (`outline`). Page items do not depend on it, except through minted names: a destination stated as a direct array is named with the first unused `destN` in the order the reader meets it, and the outline is read before any page, so an outline entry's direct destination takes a lower number than any link's. Without the outline, the names minted for page links' direct destinations are renumbered, and `destinations` loses the entries only the outline minted; each link still resolves to the same target. * - `destinations`: the named destinations of the catalog `/Dests` dictionary and the `/Names /Dests` name tree, and the registry links and outline entries name their targets through (`destinations`). Without it nothing can name an internal target, so a link whose target is internal (a `/Dest`, or a `/GoTo` action, named or direct) is left out of its page's items, and a read outline entry carries no `destination`; a link to a URI is kept as usual. * - `form`: the AcroForm field tree (`form`). Page items do not depend on it: a widget annotation is never a page item or annotation either way. * - `layers`: optional content, the `/OCProperties` groups and their default visibility (`layers`). Without it no item carries a `layer`, since an `/OC` membership resolves to no name; the items themselves are all still read, so content the default view hides is indistinguishable from visible content. * - `structure`: the `/StructTreeRoot` element tree and its `/ParentTree` ownership map (`structure`). Without it no item carries a `structure` owner, and the parent-tree diagnostics are not reported. * - `residue`: the package-level residue rows (`source`), including the per-page `page-boxes` row. Page items do not depend on it; `metadata` still reads the XMP packet it falls back to. * - * Apart from those stated losses, every page item is exactly what a read with the surface on produces. + * Apart from those stated losses and the renumbering a skipped outline causes, every page item is exactly what a read with the surface on produces. */ readonly surfaces?: ReadPdfSurfaces; } diff --git a/packages/pdf-codec/src/test-support/pdf-surfaces.ts b/packages/pdf-codec/src/test-support/pdf-surfaces.ts index 77a39356e..de4b40a1f 100644 --- a/packages/pdf-codec/src/test-support/pdf-surfaces.ts +++ b/packages/pdf-codec/src/test-support/pdf-surfaces.ts @@ -7,7 +7,7 @@ import { PAGE_OBJ, } from "./pdf"; -// A two-page document carrying every document-level surface ReadPdfOptions.surfaces can leave out, each with a trap: one reference to an object number the cross-reference table does not hold (DOCUMENT_SURFACE_TRAP_OBJECTS), reached only by that surface's own reader, so resolving it reports pdf/object-missing-value naming that number exactly when the surface was parsed. The traps sit in the outline's /Next chain, the old-style /Dests dictionary, the AcroForm /Fields array, the /OCGs array, the structure root's /K array and the catalog /OutputIntents (residue). Page 0 also carries what page items take from those surfaces: a structure-owned marked-content span (MCID 0), a span in the "Hidden" layer (off in the default configuration), an internal link naming a name-tree destination, an internal link stating a direct destination array, a URI link, and a /TrimBox for the page-boxes residue row. Page 1 exists so destinations resolve to a real second page. +// A two-page document carrying every document-level surface ReadPdfOptions.surfaces can leave out, each with a trap: one reference to an object number the cross-reference table does not hold (DOCUMENT_SURFACE_TRAP_OBJECTS), reached only by that surface's own reader, so resolving it reports pdf/object-missing-value naming that number exactly when the surface was parsed. The outline's second entry states a direct destination array (to page 0), so the reader mints a `destN` name for it before any page's link is read. The traps sit in the outline's /Next chain, the old-style /Dests dictionary, the AcroForm /Fields array, the /OCGs array, the structure root's /K array and the catalog /OutputIntents (residue). Page 0 also carries what page items take from those surfaces: a structure-owned marked-content span (MCID 0), a span in the "Hidden" layer (off in the default configuration), an internal link naming a name-tree destination, an internal link stating a direct destination array, a URI link, and a /TrimBox for the page-boxes residue row. Page 1 exists so destinations resolve to a real second page. export const DOCUMENT_SURFACE_TRAP_OBJECTS = { outline: 901, form: 902, @@ -34,6 +34,7 @@ export function documentSurfacesPdf(): Uint8Array { const CONTENT_PAGE_1_OBJ = 17; const OUTLINE_ITEM_OBJ = 18; const PARENT_TREE_OBJ = 19; + const OUTLINE_DIRECT_ITEM_OBJ = 20; b.object( 1, `<< /Type /Catalog /Pages 2 0 R /Outlines ${OUTLINES_ROOT_OBJ} 0 R /Dests ${OLD_STYLE_DESTS_OBJ} 0 R /Names << /Dests ${NAME_TREE_OBJ} 0 R >> /AcroForm << /Fields [${FIELD_OBJ} 0 R ${DOCUMENT_SURFACE_TRAP_OBJECTS.form} 0 R] >> /OCProperties << /OCGs [${OCG_HIDDEN_OBJ} 0 R ${DOCUMENT_SURFACE_TRAP_OBJECTS.layers} 0 R] /D << /OFF [${OCG_HIDDEN_OBJ} 0 R] >> >> /StructTreeRoot ${STRUCT_TREE_ROOT_OBJ} 0 R /PageMode /UseOutlines /OutputIntents [${DOCUMENT_SURFACE_TRAP_OBJECTS.residue} 0 R] >>`, @@ -77,7 +78,7 @@ export function documentSurfacesPdf(): Uint8Array { ); b.object( OUTLINES_ROOT_OBJ, - `<< /Type /Outlines /First ${OUTLINE_ITEM_OBJ} 0 R /Last ${OUTLINE_ITEM_OBJ} 0 R /Count 1 >>`, + `<< /Type /Outlines /First ${OUTLINE_ITEM_OBJ} 0 R /Last ${OUTLINE_DIRECT_ITEM_OBJ} 0 R /Count 2 >>`, ); b.object( OLD_STYLE_DESTS_OBJ, @@ -107,9 +108,13 @@ export function documentSurfacesPdf(): Uint8Array { ); b.object( OUTLINE_ITEM_OBJ, - `<< /Title (Introduction) /Parent ${OUTLINES_ROOT_OBJ} 0 R /Dest (second) /Next ${DOCUMENT_SURFACE_TRAP_OBJECTS.outline} 0 R >>`, + `<< /Title (Introduction) /Parent ${OUTLINES_ROOT_OBJ} 0 R /Dest (second) /Next ${OUTLINE_DIRECT_ITEM_OBJ} 0 R >>`, + ); + b.object( + OUTLINE_DIRECT_ITEM_OBJ, + `<< /Title (Opening) /Parent ${OUTLINES_ROOT_OBJ} 0 R /Dest [3 0 R /XYZ 0 50 null] /Next ${DOCUMENT_SURFACE_TRAP_OBJECTS.outline} 0 R >>`, ); b.object(PARENT_TREE_OBJ, `<< /Nums [0 [${STRUCT_PARA_OBJ} 0 R]] >>`); - b.classicXrefAndTrailer(PARENT_TREE_OBJ, "/Root 1 0 R"); + b.classicXrefAndTrailer(OUTLINE_DIRECT_ITEM_OBJ, "/Root 1 0 R"); return b.bytes(); } From 1b1da93ac20cf2f617c6a336e4df48cd0b3adc68 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Sat, 17 Oct 2026 06:49:40 +0100 Subject: [PATCH 04/11] docs(documents.js): describe what a pdf rebuild and a skipped outline change The surfaces section said a tree read with surfaces skipped rebuilds to pdf without them, but the rebuild never re-emits the outline, destinations, form, layers, structure or residue in any case. It now names the two differences that do reach a rebuild (hidden-layer text drawn visibly, geometry-inferred blocks), lists the structure definitions and form contentControl constructs among what a skip leaves out, and states that skipping the outline renumbers the minted destination names internal link anchors carry. Refs #1692 --- packages/documents.js/README.md | 2 +- packages/documents.js/src/convert/composition.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/documents.js/README.md b/packages/documents.js/README.md index 1e2db2c41..8b4f52ce9 100644 --- a/packages/documents.js/README.md +++ b/packages/documents.js/README.md @@ -423,7 +423,7 @@ const firstPages = pdfToMarkdown(pdfBytes, { }); ``` -A skipped surface is never parsed, and the conversion carries nothing built from it: the reported `DocumentTree` has no outline, destinations, layers or residue tables for a skipped surface, internal links are left out without `destinations` (links to a URI are kept), and form fields are absent without `form`. Two skips change reconstruction itself. Without `structure`, a tagged PDF's headings, table cells and divisions are inferred from geometry exactly as for an untagged file, so they can come out differently than in a full conversion. Without `layers`, nothing marks content as belonging to a hidden layer, so content the default view hides is kept as if it were visible. A `DocumentTree` from a full-page conversion that skipped surfaces still carries its `pages` envelope, so `buildDocumentBytes(tree, "pdf")` rebuilds it, without the skipped surfaces. The `LayoutDocument` such a read produces lists them in `skippedSurfaces`, so an absent field there means "not read" rather than "none", and `writePdf` refuses it with `PdfSkippedSurfacesError`. +A skipped surface is never parsed, and the conversion carries nothing built from it. The reported `DocumentTree` has no outline, destinations, layers or residue tables for a skipped surface, and no structure entries in `definitions` without `structure`; internal link constructs are left out without `destinations` (links to a URI are kept), and the form's `contentControl` constructs are absent without `form`. Skipping the outline also renames internal link anchors: a destination stated as a direct array gets a reader-minted `destN` name, numbered in the order the reader meets them, and the outline is read before any page, so without it the names minted for page links (and the anchors their link constructs carry) are renumbered, while each still names the same target. Two skips change reconstruction itself. Without `structure`, a tagged PDF's headings, table cells and divisions are inferred from geometry exactly as for an untagged file, so they can come out differently than in a full conversion. Without `layers`, nothing marks content as belonging to a hidden layer, so content the default view hides is kept as if it were visible. Those are also the two differences that reach a rebuild: a `DocumentTree` from a conversion without `pages` still carries its `pages` envelope, so `buildDocumentBytes(tree, "pdf")` rebuilds it, and the rebuilt PDF draws hidden-layer text visibly when `layers` was skipped and holds the geometry-inferred blocks when `structure` was skipped. The rebuild never re-emits the outline, destinations, form, layers, structure or residue whether or not a surface was skipped, since it builds the PDF from the tree's metadata, pages and images alone. The `LayoutDocument` such a read produces lists them in `skippedSurfaces`, so an absent field there means "not read" rather than "none", and `writePdf` refuses it with `PdfSkippedSurfacesError`. `readDocumentMetadata` always reads a pdf with an empty page selection and every surface skipped, since metadata comes from the trailer's `/Info`, the catalog's `/Lang` and the XMP packet alone, so identifying a document parses neither its pages nor its outline, destinations, form, layers, structure tree or residue, and returns the same metadata a full read does. `openPdf` takes neither option, since an editor holds a whole document to write back. `ReadPdfSurfaces`, `LayoutDocumentSurface` and `PdfSkippedSurfacesError` are exported from the root barrel. diff --git a/packages/documents.js/src/convert/composition.ts b/packages/documents.js/src/convert/composition.ts index fa493e812..c037f0133 100644 --- a/packages/documents.js/src/convert/composition.ts +++ b/packages/documents.js/src/convert/composition.ts @@ -326,7 +326,7 @@ export interface UnifiedConversionOptions { */ readonly pages?: readonly number[]; /** - * The document-level surfaces of a pdf source to read, as pdf-codec's `ReadPdfOptions.surfaces` takes them: one boolean per surface (`outline`, `destinations`, `form`, `layers`, `structure`, `residue`), each read unless set to `false`, so omitting it reads everything as before. Independent of `pages`, and handed to the same hop: only the plan's first hop, and only when the conversion's source is pdf, so an intermediate pdf a route renders internally is always read whole. A skipped surface is never parsed and is absent from the read, so what the conversion builds from it is absent too: no outline or bookmark tables, no destinations and no internal links (a link to a URI is kept), no form fields, no layer tables, no structure, no package residue. Two skips also change reconstruction: without `structure` a tagged PDF's headings, table cells and divisions are inferred from geometry as for an untagged file, and without `layers` content the default view hides cannot be told apart and is kept as if visible. See `ReadPdfOptions.surfaces` for exactly what each covers. + * The document-level surfaces of a pdf source to read, as pdf-codec's `ReadPdfOptions.surfaces` takes them: one boolean per surface (`outline`, `destinations`, `form`, `layers`, `structure`, `residue`), each read unless set to `false`, so omitting it reads everything as before. Independent of `pages`, and handed to the same hop: only the plan's first hop, and only when the conversion's source is pdf, so an intermediate pdf a route renders internally is always read whole. A skipped surface is never parsed and is absent from the read, so what the conversion builds from it is absent too: no outline or bookmark tables, no destinations and no internal links (a link to a URI is kept), no form fields, no layer tables, no structure, no package residue; skipping the outline also renumbers the reader-minted `destN` names internal link anchors carry, since the outline mints its direct destinations first. Two skips also change reconstruction: without `structure` a tagged PDF's headings, table cells and divisions are inferred from geometry as for an untagged file, and without `layers` content the default view hides cannot be told apart and is kept as if visible. See `ReadPdfOptions.surfaces` for exactly what each covers. */ readonly surfaces?: ReadPdfSurfaces; } From a723268d98acfacbf8536cfebc5a1c288a8c9974 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Sat, 17 Oct 2026 06:51:09 +0100 Subject: [PATCH 05/11] test(documents.js): cover readDocumentMetadata's XMP fallback with residue skipped The surfaces fixture's XMP packet was empty, so nothing showed that a metadata read with every surface skipped still fills fields /Info does not carry from the XMP packet. A pdf whose title and author live only in XMP now reads the same metadata through readDocumentMetadata as through a full read. Refs #1692 --- .../src/convert/from-pdf-surfaces.test.ts | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/packages/documents.js/src/convert/from-pdf-surfaces.test.ts b/packages/documents.js/src/convert/from-pdf-surfaces.test.ts index 22a503494..b350a2c48 100644 --- a/packages/documents.js/src/convert/from-pdf-surfaces.test.ts +++ b/packages/documents.js/src/convert/from-pdf-surfaces.test.ts @@ -128,6 +128,37 @@ describe("readDocumentMetadata for a pdf", () => { }); }); +// An XMP packet carrying a title and creator the /Info dictionary below does not, so a reader can only report them through the XMP fallback, which reads the catalog's /Metadata stream (the same stream the residue surface quarantines). +const XMP_ONLY_FIELDS_PACKET = [ + '', + '', + '', + '', + 'Title from XMP', + "XMP Author", + "", + "", + "", + '', +].join("\n"); + +describe("readDocumentMetadata for a pdf whose metadata comes partly from XMP", () => { + it("still falls back to the XMP packet with residue skipped, reporting exactly what a full read does", () => { + const pdf = writePdf({ + formatVersion: LAYOUT_FORMAT_VERSION, + metadata: { subject: "Only in /Info" }, + images: {}, + pages: [{ ...PAGE, items: [] }], + source: { xmp: { format: "pdf", xml: XMP_ONLY_FIELDS_PACKET } }, + }); + const full = readPdf(pdf); + expect(full.metadata.title).toBe("Title from XMP"); + expect(full.metadata.author).toBe("XMP Author"); + expect(full.metadata.subject).toBe("Only in /Info"); + expect(readDocumentMetadata("pdf", pdf)).toEqual(full.metadata); + }); +}); + describe("the pdf conversions with surfaces", () => { it("hand surfaces to readPdf from every pdfTo* conversion and from readNativeDocumentTree/readContentDocument", () => { const pdf = everySurfacePdf(); From ede1c7b609dcc913bfb8dd9b97967a4b593c0979 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Sat, 17 Oct 2026 07:19:03 +0100 Subject: [PATCH 06/11] docs(pdf-codec): say a shared direct destination array mints twice The destination registry comment said a direct array shared by a link and a bookmark mints one destination. intern mints a fresh entry for every direct array with no identity or value check, so one indirect array both reference mints two entries naming the same target. The comment now says what the code does; behaviour is unchanged. Refs #1692 --- packages/pdf-codec/src/navigation.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pdf-codec/src/navigation.ts b/packages/pdf-codec/src/navigation.ts index ac32c9815..f177d9c45 100644 --- a/packages/pdf-codec/src/navigation.ts +++ b/packages/pdf-codec/src/navigation.ts @@ -150,7 +150,7 @@ function optionalCoords( return out; } -// The one table every navigation consumer resolves against: the document's named destinations plus reader-minted entries for direct destination arrays. `intern` is the single entry point both link annotations and outline items go through, so a direct array shared by a link and a bookmark mints ONE destination, not two. +// The one table every navigation consumer resolves against: the document's named destinations plus reader-minted entries for direct destination arrays. `intern` is the single entry point both link annotations and outline items go through. Every direct array it is given mints its own entry, with no identity or value check, so a direct array shared by a link and a bookmark (even one indirect object both reference) mints two destinations, each naming the same target. export interface DestinationRegistry { readonly entries: readonly LayoutDestination[]; // A named destination (PDF string) resolves to its own name when the table carries it; a direct destination array mints a fresh collision-free `destN` entry. Returns undefined (with a diagnostic) for a dangling name or an unparseable array. From af3c80f3a227eb6a04775e8c4d148469f08deebc Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Sat, 17 Oct 2026 07:19:06 +0100 Subject: [PATCH 07/11] test(pdf-codec): compare a skipped-outline read by link target from one document The outline case's link-target normalisation took a document and a separate source for its pages and destinations, so the expected side's own pages were silently replaced by the full read's. It now normalises one document, and the expected side is the full read normalised with its outline removed. Refs #1692 --- packages/pdf-codec/src/read-surfaces.test.ts | 22 +++++++++----------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/packages/pdf-codec/src/read-surfaces.test.ts b/packages/pdf-codec/src/read-surfaces.test.ts index 4e9a74ac7..66f2f7706 100644 --- a/packages/pdf-codec/src/read-surfaces.test.ts +++ b/packages/pdf-codec/src/read-surfaces.test.ts @@ -170,12 +170,9 @@ function destinationNamed( } // The rule a skipped outline adds to the per-surface comparison: the reader mints a `destN` name for each direct destination array in the order it meets them, and the outline is read before any page, so leaving it out renumbers the names minted for page links and drops the entries only the outline minted. A document compared under this rule has each internal link's destination name replaced by the entry it resolves to, and its destinations table reduced to the entries some page item names, keyed by what they resolve to rather than by name. -function withLinksByTarget( - doc: LayoutDocument | Record, - source: LayoutDocument, -): Record { +function withLinksByTarget(doc: LayoutDocument): Record { const linkedNames = new Set( - source.pages.flatMap((page) => + doc.pages.flatMap((page) => page.items.flatMap((item) => item.kind === "internalLink" ? [item.destination] : [], ), @@ -183,19 +180,19 @@ function withLinksByTarget( ); return { ...withoutField(doc, "destinations"), - pages: mapItems(source.pages, (items) => + pages: mapItems(doc.pages, (items) => items.map((item) => item.kind === "internalLink" ? { ...item, - destination: destinationNamed(source, item.destination), + destination: destinationNamed(doc, item.destination), } : item, ), ), - linkedDestinations: (source.destinations ?? []) + linkedDestinations: (doc.destinations ?? []) .filter((destination) => linkedNames.has(destination.name)) - .map((destination) => destinationNamed(source, destination.name)), + .map((destination) => destinationNamed(doc, destination.name)), }; } @@ -278,9 +275,10 @@ describe("readPdf surfaces: what a skipped surface's result holds", () => { const skipped = readWith(only(surface)).doc; if (surface === "outline") { // A skipped outline also renumbers minted destination names (see withLinksByTarget), so links are compared by the entry they resolve to. - expect(withLinksByTarget(skipped, skipped)).toEqual( - withLinksByTarget(expectedWithout(full, surface), full), - ); + expect(withLinksByTarget(skipped)).toEqual({ + ...withoutField(withLinksByTarget(full), SURFACE_FIELD.outline), + skippedSurfaces: [surface], + }); return; } expect(skipped).toEqual(expectedWithout(full, surface)); From 5f846ae1e98d145bf35561ddc0b7dcd92e8605d1 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Sat, 17 Oct 2026 07:19:31 +0100 Subject: [PATCH 08/11] docs(documents.js): name what a pdf rebuild draws from a tree The surfaces section said a rebuild draws the PDF from the tree's metadata, pages and images alone, but it also emits the positioned text, vectors, formulas and URI links and uses the tree's embedded fonts. It now says so, states the skippedSurfaces refusal before the rebuild sentences and that a rebuild is never refused, and notes that a skipped outline drops the destN entries only it minted from the tree's destinations table. Refs #1692 --- packages/documents.js/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/documents.js/README.md b/packages/documents.js/README.md index 8b4f52ce9..f04f595e6 100644 --- a/packages/documents.js/README.md +++ b/packages/documents.js/README.md @@ -423,7 +423,7 @@ const firstPages = pdfToMarkdown(pdfBytes, { }); ``` -A skipped surface is never parsed, and the conversion carries nothing built from it. The reported `DocumentTree` has no outline, destinations, layers or residue tables for a skipped surface, and no structure entries in `definitions` without `structure`; internal link constructs are left out without `destinations` (links to a URI are kept), and the form's `contentControl` constructs are absent without `form`. Skipping the outline also renames internal link anchors: a destination stated as a direct array gets a reader-minted `destN` name, numbered in the order the reader meets them, and the outline is read before any page, so without it the names minted for page links (and the anchors their link constructs carry) are renumbered, while each still names the same target. Two skips change reconstruction itself. Without `structure`, a tagged PDF's headings, table cells and divisions are inferred from geometry exactly as for an untagged file, so they can come out differently than in a full conversion. Without `layers`, nothing marks content as belonging to a hidden layer, so content the default view hides is kept as if it were visible. Those are also the two differences that reach a rebuild: a `DocumentTree` from a conversion without `pages` still carries its `pages` envelope, so `buildDocumentBytes(tree, "pdf")` rebuilds it, and the rebuilt PDF draws hidden-layer text visibly when `layers` was skipped and holds the geometry-inferred blocks when `structure` was skipped. The rebuild never re-emits the outline, destinations, form, layers, structure or residue whether or not a surface was skipped, since it builds the PDF from the tree's metadata, pages and images alone. The `LayoutDocument` such a read produces lists them in `skippedSurfaces`, so an absent field there means "not read" rather than "none", and `writePdf` refuses it with `PdfSkippedSurfacesError`. +A skipped surface is never parsed, and the conversion carries nothing built from it. The `LayoutDocument` such a read produces lists the skipped surfaces in `skippedSurfaces`, so an absent field there means "not read" rather than "none", and `writePdf` refuses that `LayoutDocument` with `PdfSkippedSurfacesError`. The reported `DocumentTree` has no outline, destinations, layers or residue tables for a skipped surface, and no structure entries in `definitions` without `structure`; internal link constructs are left out without `destinations` (links to a URI are kept), and the form's `contentControl` constructs are absent without `form`. Skipping the outline also renames internal link anchors: a destination stated as a direct array gets a reader-minted `destN` name, numbered in the order the reader meets them, and the outline is read before any page, so without it the names minted for page links (and the anchors their link constructs carry) are renumbered, while each still names the same target, and the tree's `destinations` table loses the `destN` entries only the outline minted. Two skips change reconstruction itself. Without `structure`, a tagged PDF's headings, table cells and divisions are inferred from geometry exactly as for an untagged file, so they can come out differently than in a full conversion. Without `layers`, nothing marks content as belonging to a hidden layer, so content the default view hides is kept as if it were visible. Those are also the two differences that reach a rebuild: a `DocumentTree` from a conversion without `pages` still carries its `pages` envelope, so `buildDocumentBytes(tree, "pdf")` rebuilds it, and the rebuilt PDF draws hidden-layer text visibly when `layers` was skipped and holds the geometry-inferred blocks when `structure` was skipped. The rebuild never re-emits the outline, destinations, form, layers, structure or residue whether or not a surface was skipped, since it builds the PDF from the tree's metadata, page sizes, embedded fonts and the positioned content (text, images, vectors, formulas, URI links) alone. Its `LayoutDocument` never carries `skippedSurfaces`, so `writePdf` never refuses a rebuild. `readDocumentMetadata` always reads a pdf with an empty page selection and every surface skipped, since metadata comes from the trailer's `/Info`, the catalog's `/Lang` and the XMP packet alone, so identifying a document parses neither its pages nor its outline, destinations, form, layers, structure tree or residue, and returns the same metadata a full read does. `openPdf` takes neither option, since an editor holds a whole document to write back. `ReadPdfSurfaces`, `LayoutDocumentSurface` and `PdfSkippedSurfacesError` are exported from the root barrel. From 265886beaba9836d97e8c1964264b64cc6c9571f Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Sat, 17 Oct 2026 07:31:49 +0100 Subject: [PATCH 09/11] test(pdf-codec): describe the outline comparison's destination list exactly The helper comment said linked destinations are keyed by what they resolve to; they are an ordered list of resolved targets compared in order. The comment now says so and why the order matches in both reads. Refs #1692 --- packages/pdf-codec/src/read-surfaces.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/pdf-codec/src/read-surfaces.test.ts b/packages/pdf-codec/src/read-surfaces.test.ts index 66f2f7706..9467b7b4e 100644 --- a/packages/pdf-codec/src/read-surfaces.test.ts +++ b/packages/pdf-codec/src/read-surfaces.test.ts @@ -169,7 +169,7 @@ function destinationNamed( : { pageIndex: entry.pageIndex, target: entry.target }; } -// The rule a skipped outline adds to the per-surface comparison: the reader mints a `destN` name for each direct destination array in the order it meets them, and the outline is read before any page, so leaving it out renumbers the names minted for page links and drops the entries only the outline minted. A document compared under this rule has each internal link's destination name replaced by the entry it resolves to, and its destinations table reduced to the entries some page item names, keyed by what they resolve to rather than by name. +// The rule a skipped outline adds to the per-surface comparison: the reader mints a `destN` name for each direct destination array in the order it meets them, and the outline is read before any page, so leaving it out renumbers the names minted for page links and drops the entries only the outline minted. A document compared under this rule has each internal link's destination name replaced by the entry it resolves to, and its destinations table reduced to the entries some page item names, as an ordered list of what each resolves to (`{ pageIndex, target }`) with the names dropped. Comparing those lists in order is sound because both reads list the named destinations first and then the entries minted for page links in page order, and the outline's own mints, which a full read lists between them, are named by no page item and so are not in the list. function withLinksByTarget(doc: LayoutDocument): Record { const linkedNames = new Set( doc.pages.flatMap((page) => From e0233bded4156c1c4d4c59787a62805ffc71e4c5 Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Sat, 17 Oct 2026 07:32:23 +0100 Subject: [PATCH 10/11] docs(documents.js): say where the outline lives in the tree and what a rebuild draws The outline is not a table of its own: its items are outline-N entries in the destinations table, so skipping the outline removes those entries rather than a table. The rebuild's content list now includes table-cell rectangles and border lines and sheet-cell rectangles, and says a URI link is re-emitted only when it is attached to a recovered run. Refs #1692 --- packages/documents.js/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/documents.js/README.md b/packages/documents.js/README.md index f04f595e6..c02b38a5b 100644 --- a/packages/documents.js/README.md +++ b/packages/documents.js/README.md @@ -423,7 +423,7 @@ const firstPages = pdfToMarkdown(pdfBytes, { }); ``` -A skipped surface is never parsed, and the conversion carries nothing built from it. The `LayoutDocument` such a read produces lists the skipped surfaces in `skippedSurfaces`, so an absent field there means "not read" rather than "none", and `writePdf` refuses that `LayoutDocument` with `PdfSkippedSurfacesError`. The reported `DocumentTree` has no outline, destinations, layers or residue tables for a skipped surface, and no structure entries in `definitions` without `structure`; internal link constructs are left out without `destinations` (links to a URI are kept), and the form's `contentControl` constructs are absent without `form`. Skipping the outline also renames internal link anchors: a destination stated as a direct array gets a reader-minted `destN` name, numbered in the order the reader meets them, and the outline is read before any page, so without it the names minted for page links (and the anchors their link constructs carry) are renumbered, while each still names the same target, and the tree's `destinations` table loses the `destN` entries only the outline minted. Two skips change reconstruction itself. Without `structure`, a tagged PDF's headings, table cells and divisions are inferred from geometry exactly as for an untagged file, so they can come out differently than in a full conversion. Without `layers`, nothing marks content as belonging to a hidden layer, so content the default view hides is kept as if it were visible. Those are also the two differences that reach a rebuild: a `DocumentTree` from a conversion without `pages` still carries its `pages` envelope, so `buildDocumentBytes(tree, "pdf")` rebuilds it, and the rebuilt PDF draws hidden-layer text visibly when `layers` was skipped and holds the geometry-inferred blocks when `structure` was skipped. The rebuild never re-emits the outline, destinations, form, layers, structure or residue whether or not a surface was skipped, since it builds the PDF from the tree's metadata, page sizes, embedded fonts and the positioned content (text, images, vectors, formulas, URI links) alone. Its `LayoutDocument` never carries `skippedSurfaces`, so `writePdf` never refuses a rebuild. +A skipped surface is never parsed, and the conversion carries nothing built from it. The `LayoutDocument` such a read produces lists the skipped surfaces in `skippedSurfaces`, so an absent field there means "not read" rather than "none", and `writePdf` refuses that `LayoutDocument` with `PdfSkippedSurfacesError`. The reported `DocumentTree` keeps its outline as `outline-N` entries inside the `destinations` table: without `outline` those entries are absent (and so are the `destN` entries only the outline minted, below), and without `destinations` the table holds none of the document's destinations. There are no layers or residue tables for a skipped surface, and no structure entries in `definitions` without `structure`; internal link constructs are left out without `destinations` (links to a URI are kept), and the form's `contentControl` constructs are absent without `form`. Skipping the outline also renames internal link anchors: a destination stated as a direct array gets a reader-minted `destN` name, numbered in the order the reader meets them, and the outline is read before any page, so without it the names minted for page links (and the anchors their link constructs carry) are renumbered, while each still names the same target, and the tree's `destinations` table loses the `destN` entries only the outline minted. Two skips change reconstruction itself. Without `structure`, a tagged PDF's headings, table cells and divisions are inferred from geometry exactly as for an untagged file, so they can come out differently than in a full conversion. Without `layers`, nothing marks content as belonging to a hidden layer, so content the default view hides is kept as if it were visible. Those are also the two differences that reach a rebuild: a `DocumentTree` from a conversion without `pages` still carries its `pages` envelope, so `buildDocumentBytes(tree, "pdf")` rebuilds it, and the rebuilt PDF draws hidden-layer text visibly when `layers` was skipped and holds the geometry-inferred blocks when `structure` was skipped. The rebuild never re-emits the outline, destinations, form, layers, structure or residue whether or not a surface was skipped, since it builds the PDF from the tree's metadata, page sizes, embedded fonts and the positioned content alone: text runs, images, vectors, formulas, table-cell background rectangles and border lines, sheet-cell rectangles, and URI links attached to a recovered run. An external link whose rectangle covers no recovered run (a link over an image, say) stays a link construct in the tree and is not re-emitted. Its `LayoutDocument` never carries `skippedSurfaces`, so `writePdf` never refuses a rebuild. `readDocumentMetadata` always reads a pdf with an empty page selection and every surface skipped, since metadata comes from the trailer's `/Info`, the catalog's `/Lang` and the XMP packet alone, so identifying a document parses neither its pages nor its outline, destinations, form, layers, structure tree or residue, and returns the same metadata a full read does. `openPdf` takes neither option, since an editor holds a whole document to write back. `ReadPdfSurfaces`, `LayoutDocumentSurface` and `PdfSkippedSurfacesError` are exported from the root barrel. From 886481b4a0c22ef0b5ae6f9f3635998f20924c9e Mon Sep 17 00:00:00 2001 From: Joseph Mearman Date: Sat, 17 Oct 2026 07:42:24 +0100 Subject: [PATCH 11/11] docs(documents.js): name what a skipped surface removes from the tree The surfaces doc comments spoke of outline or bookmark tables, but the outline lives as outline-N entries in the tree's destinations table. They now list what each skip removes, matching the README. The README adds the border lines a sheet cell rebuilds with and says a read outline's outline-N entries stay, without a destination, when only destinations is skipped. Refs #1692 --- packages/documents.js/README.md | 2 +- packages/documents.js/src/convert/composition.ts | 2 +- packages/documents.js/src/convert/from-pdf.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/documents.js/README.md b/packages/documents.js/README.md index c02b38a5b..bf00740a1 100644 --- a/packages/documents.js/README.md +++ b/packages/documents.js/README.md @@ -423,7 +423,7 @@ const firstPages = pdfToMarkdown(pdfBytes, { }); ``` -A skipped surface is never parsed, and the conversion carries nothing built from it. The `LayoutDocument` such a read produces lists the skipped surfaces in `skippedSurfaces`, so an absent field there means "not read" rather than "none", and `writePdf` refuses that `LayoutDocument` with `PdfSkippedSurfacesError`. The reported `DocumentTree` keeps its outline as `outline-N` entries inside the `destinations` table: without `outline` those entries are absent (and so are the `destN` entries only the outline minted, below), and without `destinations` the table holds none of the document's destinations. There are no layers or residue tables for a skipped surface, and no structure entries in `definitions` without `structure`; internal link constructs are left out without `destinations` (links to a URI are kept), and the form's `contentControl` constructs are absent without `form`. Skipping the outline also renames internal link anchors: a destination stated as a direct array gets a reader-minted `destN` name, numbered in the order the reader meets them, and the outline is read before any page, so without it the names minted for page links (and the anchors their link constructs carry) are renumbered, while each still names the same target, and the tree's `destinations` table loses the `destN` entries only the outline minted. Two skips change reconstruction itself. Without `structure`, a tagged PDF's headings, table cells and divisions are inferred from geometry exactly as for an untagged file, so they can come out differently than in a full conversion. Without `layers`, nothing marks content as belonging to a hidden layer, so content the default view hides is kept as if it were visible. Those are also the two differences that reach a rebuild: a `DocumentTree` from a conversion without `pages` still carries its `pages` envelope, so `buildDocumentBytes(tree, "pdf")` rebuilds it, and the rebuilt PDF draws hidden-layer text visibly when `layers` was skipped and holds the geometry-inferred blocks when `structure` was skipped. The rebuild never re-emits the outline, destinations, form, layers, structure or residue whether or not a surface was skipped, since it builds the PDF from the tree's metadata, page sizes, embedded fonts and the positioned content alone: text runs, images, vectors, formulas, table-cell background rectangles and border lines, sheet-cell rectangles, and URI links attached to a recovered run. An external link whose rectangle covers no recovered run (a link over an image, say) stays a link construct in the tree and is not re-emitted. Its `LayoutDocument` never carries `skippedSurfaces`, so `writePdf` never refuses a rebuild. +A skipped surface is never parsed, and the conversion carries nothing built from it. The `LayoutDocument` such a read produces lists the skipped surfaces in `skippedSurfaces`, so an absent field there means "not read" rather than "none", and `writePdf` refuses that `LayoutDocument` with `PdfSkippedSurfacesError`. The reported `DocumentTree` keeps its outline as `outline-N` entries inside the `destinations` table: without `outline` those entries are absent (and so are the `destN` entries only the outline minted, below), and without `destinations` the table holds none of the document's destinations, while a read outline's `outline-N` entries stay in it, each without a `destination`. There are no layers or residue tables for a skipped surface, and no structure entries in `definitions` without `structure`; internal link constructs are left out without `destinations` (links to a URI are kept), and the form's `contentControl` constructs are absent without `form`. Skipping the outline also renames internal link anchors: a destination stated as a direct array gets a reader-minted `destN` name, numbered in the order the reader meets them, and the outline is read before any page, so without it the names minted for page links (and the anchors their link constructs carry) are renumbered, while each still names the same target, and the tree's `destinations` table loses the `destN` entries only the outline minted. Two skips change reconstruction itself. Without `structure`, a tagged PDF's headings, table cells and divisions are inferred from geometry exactly as for an untagged file, so they can come out differently than in a full conversion. Without `layers`, nothing marks content as belonging to a hidden layer, so content the default view hides is kept as if it were visible. Those are also the two differences that reach a rebuild: a `DocumentTree` from a conversion without `pages` still carries its `pages` envelope, so `buildDocumentBytes(tree, "pdf")` rebuilds it, and the rebuilt PDF draws hidden-layer text visibly when `layers` was skipped and holds the geometry-inferred blocks when `structure` was skipped. The rebuild never re-emits the outline, destinations, form, layers, structure or residue whether or not a surface was skipped, since it builds the PDF from the tree's metadata, page sizes, embedded fonts and the positioned content alone: text runs, images, vectors, formulas, table-cell background rectangles and border lines, sheet-cell rectangles and border lines, and URI links attached to a recovered run. An external link whose rectangle covers no recovered run (a link over an image, say) stays a link construct in the tree and is not re-emitted. Its `LayoutDocument` never carries `skippedSurfaces`, so `writePdf` never refuses a rebuild. `readDocumentMetadata` always reads a pdf with an empty page selection and every surface skipped, since metadata comes from the trailer's `/Info`, the catalog's `/Lang` and the XMP packet alone, so identifying a document parses neither its pages nor its outline, destinations, form, layers, structure tree or residue, and returns the same metadata a full read does. `openPdf` takes neither option, since an editor holds a whole document to write back. `ReadPdfSurfaces`, `LayoutDocumentSurface` and `PdfSkippedSurfacesError` are exported from the root barrel. diff --git a/packages/documents.js/src/convert/composition.ts b/packages/documents.js/src/convert/composition.ts index c037f0133..9ee1e6681 100644 --- a/packages/documents.js/src/convert/composition.ts +++ b/packages/documents.js/src/convert/composition.ts @@ -326,7 +326,7 @@ export interface UnifiedConversionOptions { */ readonly pages?: readonly number[]; /** - * The document-level surfaces of a pdf source to read, as pdf-codec's `ReadPdfOptions.surfaces` takes them: one boolean per surface (`outline`, `destinations`, `form`, `layers`, `structure`, `residue`), each read unless set to `false`, so omitting it reads everything as before. Independent of `pages`, and handed to the same hop: only the plan's first hop, and only when the conversion's source is pdf, so an intermediate pdf a route renders internally is always read whole. A skipped surface is never parsed and is absent from the read, so what the conversion builds from it is absent too: no outline or bookmark tables, no destinations and no internal links (a link to a URI is kept), no form fields, no layer tables, no structure, no package residue; skipping the outline also renumbers the reader-minted `destN` names internal link anchors carry, since the outline mints its direct destinations first. Two skips also change reconstruction: without `structure` a tagged PDF's headings, table cells and divisions are inferred from geometry as for an untagged file, and without `layers` content the default view hides cannot be told apart and is kept as if visible. See `ReadPdfOptions.surfaces` for exactly what each covers. + * The document-level surfaces of a pdf source to read, as pdf-codec's `ReadPdfOptions.surfaces` takes them: one boolean per surface (`outline`, `destinations`, `form`, `layers`, `structure`, `residue`), each read unless set to `false`, so omitting it reads everything as before. Independent of `pages`, and handed to the same hop: only the plan's first hop, and only when the conversion's source is pdf, so an intermediate pdf a route renders internally is always read whole. A skipped surface is never parsed and is absent from the read, so what the conversion builds from it is absent too: no `outline-N` entries in the tree's `destinations` table without `outline`, no destinations and no internal link constructs without `destinations` (a link to a URI is kept, and a read outline's `outline-N` entries stay, each without a `destination`), no form `contentControl` constructs without `form`, no layers table without `layers`, no structure entries in `definitions` without `structure`, no residue without `residue`; skipping the outline also renumbers the reader-minted `destN` names internal link anchors carry, since the outline mints its direct destinations first. Two skips also change reconstruction: without `structure` a tagged PDF's headings, table cells and divisions are inferred from geometry as for an untagged file, and without `layers` content the default view hides cannot be told apart and is kept as if visible. See `ReadPdfOptions.surfaces` for exactly what each covers. */ readonly surfaces?: ReadPdfSurfaces; } diff --git a/packages/documents.js/src/convert/from-pdf.ts b/packages/documents.js/src/convert/from-pdf.ts index 9b1c8f63c..1db3eed15 100644 --- a/packages/documents.js/src/convert/from-pdf.ts +++ b/packages/documents.js/src/convert/from-pdf.ts @@ -63,7 +63,7 @@ export interface ReadNativeDocumentTreeOptions { */ readonly pages?: readonly number[]; /** - * The document-level surfaces of a pdf source to read, exactly as UnifiedConversionOptions.surfaces describes them: one boolean per surface, each read unless set to `false`, independent of `pages`. Consulted only for a pdf source. A skipped surface is absent from the read, so the tree carries none of its tables, and reconstruction loses what it took from it: without `structure` a tagged PDF's headings, table cells and divisions are inferred from geometry, and without `layers` hidden-layer content is kept as if visible. + * The document-level surfaces of a pdf source to read, exactly as UnifiedConversionOptions.surfaces describes them: one boolean per surface, each read unless set to `false`, independent of `pages`. Consulted only for a pdf source. A skipped surface is absent from the read, so the tree carries nothing built from it (see UnifiedConversionOptions.surfaces for exactly what: `outline-N` entries in `destinations`, destinations and internal link constructs, form `contentControl` constructs, the layers table, structure entries in `definitions`, residue), and reconstruction loses what it took from it: without `structure` a tagged PDF's headings, table cells and divisions are inferred from geometry, and without `layers` hidden-layer content is kept as if visible. */ readonly surfaces?: ReadPdfSurfaces; }