diff --git a/Cargo.toml b/Cargo.toml index 97d57d07..9b02ac29 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,7 +9,7 @@ version = "0.1.8" edition = "2024" # Edition 2024 needs 1.85; zip and calamine both raise it to 1.88. rust-version = "1.88" -description = "Convert documents (doc, docx, odt, rtf, epub, pdf, presentations, spreadsheets, csv) to GitHub-Flavored Markdown" +description = "Convert documents (doc, docx, odt, rtf, epub, html, pdf, presentations, spreadsheets, csv) to GitHub-Flavored Markdown" license = "MIT" repository = "https://github.com/firecrawl/anydoc" readme = "README.md" diff --git a/README.md b/README.md index ed4cb9e4..e1d703af 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ [![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE) [![skills.sh](https://skills.sh/b/firecrawl/anydoc)](https://skills.sh/firecrawl/anydoc) -Fast Rust library that converts documents (Word, PowerPoint, Excel, OpenDocument, RTF, EPUB, CSV, and PDF) into clean GitHub-Flavored Markdown. Includes bindings for [Node.js](node/README.md), [Python](python/README.md), and the [browser](wasm/README.md) (WebAssembly). +Fast Rust library that converts documents (Word, PowerPoint, Excel, OpenDocument, RTF, EPUB, HTML, CSV, and PDF) into clean GitHub-Flavored Markdown. Includes bindings for [Node.js](node/README.md), [Python](python/README.md), and the [browser](wasm/README.md) (WebAssembly). Built by [Firecrawl](https://firecrawl.dev) to turn any office document into LLM-ready Markdown in single-digit milliseconds, with one consistent output no matter which format goes in. It powers [Firecrawl Parse](https://firecrawl.dev/parse), so if you'd rather not run it yourself, the hosted API gives you the same conversion plus our OCR models for the scanned pages anydoc can't read on its own. @@ -130,7 +130,7 @@ let document = anydoc::to_document(&bytes, None)?; - **One output for every format.** Each format parses into a shared document model and renders through a single Markdown serializer, so escaping, tables, heading anchors, and footnotes behave identically whether the input was a `.doc` from 2003 or a `.pptx` from yesterday. - **Full document structure.** Headings with anchors, bold/italic/strikethrough, inline code and code blocks, links and internal cross-references, bulleted/numbered/nested/task lists with the source's own numbering, tables with merged cells and header rows, block quotes, footnotes and endnotes, and speaker notes. - **Embedded assets.** Images and embedded objects render as their alt text in the Markdown, and the raw bytes stay available on the document model, tagged with their media type. Images with an external URL become ordinary Markdown images. -- **Content-based format detection.** The format is read from the bytes themselves (PDF header, RTF open group, OLE stream names, ZIP package mimetype), so mislabeled files still convert correctly. +- **Content-based format detection.** The format is read from the bytes themselves (PDF header, RTF open group, OLE stream names, ZIP package mimetype, or an HTML doctype/root element), so mislabeled files still convert correctly. - **Fast.** Pure Rust, no ML models, no external services. Median conversion time is under 5ms per document. - **Bindings that stay out of the way.** Node.js conversion runs on the libuv thread pool and never blocks the event loop; Python releases the GIL so other threads keep running. TypeScript types and Python stubs ship with the packages. - **PDF support built in.** Text-based PDFs convert locally through [pdf-inspector](https://github.com/firecrawl/pdf-inspector), no OCR service required. @@ -146,6 +146,7 @@ let document = anydoc::to_document(&bytes, None)?; | OpenDocument | `.odt`, `.ods`, `.odp` | | Rich Text Format | `.rtf` | | EPUB | `.epub` | +| HTML | `.html`, `.htm` | | CSV | `.csv` | | PDF | `.pdf` | @@ -189,7 +190,7 @@ Speed is one warm conversion per document on a Ryzen 9 9950X3D (Windows 11, 64 G ## Format detection -The format is read from the file content, using the marker its specification designates: the PDF header, the RTF open group, OLE stream names, the ZIP package mimetype and content types. CSV has no such marker, so the extension or an explicit format names it instead. +The format is read from the file content, using the marker its specification designates: the PDF header, the RTF open group, OLE stream names, the ZIP package mimetype and content types, or an HTML doctype/root element. CSV has no such marker, so the extension or an explicit format names it instead. Standalone HTML files can therefore be recognized even when an exporter gives them a `.doc` extension; MHTML is not yet supported. ```rust Format::from_bytes(&bytes); // Some(Format::Docx), or None when nothing matches @@ -234,7 +235,7 @@ document bytes ├─► format detection → content markers, not the extension │ ├─► format parser → one per format (doc, docx, ppt, pptx, xls, - │ xlsx, odt/ods/odp, rtf, epub, csv) + │ xlsx, odt/ods/odp, rtf, epub, html, csv) │ │ │ └─► Document → shared model: blocks, inlines, tables, │ footnotes, assets diff --git a/node/README.md b/node/README.md index 5263150f..a9df0fdd 100644 --- a/node/README.md +++ b/node/README.md @@ -3,7 +3,7 @@ [![npm](https://img.shields.io/npm/v/@firecrawl/anydoc.svg)](https://www.npmjs.com/package/@firecrawl/anydoc) [![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/firecrawl/anydoc/blob/main/LICENSE) -Convert Word, PowerPoint, Excel, OpenDocument, RTF, EPUB, CSV, and PDF files into clean GitHub-Flavored Markdown. Node.js bindings for the [anydoc](https://github.com/firecrawl/anydoc) Rust crate, built by [Firecrawl](https://firecrawl.dev). Also available as a hosted API through [Firecrawl Parse](https://firecrawl.dev/parse), which adds our OCR models for the scanned pages anydoc can't read on its own. +Convert Word, PowerPoint, Excel, OpenDocument, RTF, EPUB, HTML, CSV, and PDF files into clean GitHub-Flavored Markdown. Node.js bindings for the [anydoc](https://github.com/firecrawl/anydoc) Rust crate, built by [Firecrawl](https://firecrawl.dev). Also available as a hosted API through [Firecrawl Parse](https://firecrawl.dev/parse), which adds our OCR models for the scanned pages anydoc can't read on its own. Every format parses into one shared document model and renders through a single Markdown serializer, so headings, tables, lists, and footnotes come out the same no matter which format goes in. Conversion runs on the libuv thread pool and never blocks the event loop. TypeScript types ship with the package. @@ -21,6 +21,7 @@ npm install @firecrawl/anydoc | OpenDocument | `.odt`, `.ods`, `.odp` | | Rich Text Format | `.rtf` | | EPUB | `.epub` | +| HTML | `.html`, `.htm` | | CSV | `.csv` | | PDF | `.pdf` | @@ -84,7 +85,7 @@ try { ## Format detection -The format is read from the file content, using the marker its specification designates: the PDF header, the RTF open group, OLE stream names, the ZIP package mimetype and content types. CSV has no such marker, so detection returns `null` for it and the extension, or an explicit format, names it instead. +The format is read from the file content, using the marker its specification designates: the PDF header, the RTF open group, OLE stream names, the ZIP package mimetype and content types, or an HTML doctype/root element. CSV has no such marker, so detection returns `null` for it and the extension, or an explicit format, names it instead. Standalone HTML is supported; MHTML is not yet supported. ```js formatFromBytes(bytes); // 'docx', or null when nothing matches diff --git a/node/cli.js b/node/cli.js index bfba762a..a08f0b3d 100644 --- a/node/cli.js +++ b/node/cli.js @@ -3,7 +3,7 @@ const { readFile, writeFile } = require('node:fs/promises') -const FORMATS = 'doc, docx, odt, pdf, ppt, pptx, rtf, epub, xlsx, ods, odp, csv' +const FORMATS = 'doc, docx, odt, pdf, ppt, pptx, rtf, epub, html, xlsx, ods, odp, csv' const HELP = `anydoc: convert documents to GitHub-Flavored Markdown diff --git a/node/index.d.ts b/node/index.d.ts index 27ceba0a..7dcf28a7 100644 --- a/node/index.d.ts +++ b/node/index.d.ts @@ -110,6 +110,7 @@ export declare const enum Format { pptx = 'pptx', rtf = 'rtf', epub = 'epub', + html = 'html', xlsx = 'xlsx', ods = 'ods', odp = 'odp', @@ -119,8 +120,9 @@ export declare const enum Format { /** * Detect the format from the content itself: the signature and identity each * container specification designates (PDF header, RTF open group, OLE stream - * names, ZIP package mimetype/content types). Plain-text formats (CSV) carry - * no signature and return `null`; so does anything unrecognized. + * names, ZIP package mimetype/content types, or an HTML doctype/root element). + * Plain-text formats (CSV) carry no signature and return `null`; so does + * anything unrecognized. */ export declare function formatFromBytes(bytes: Uint8Array): Format | null diff --git a/node/package.json b/node/package.json index 0d2d02c2..02fc3b78 100644 --- a/node/package.json +++ b/node/package.json @@ -1,7 +1,7 @@ { "name": "@firecrawl/anydoc", "version": "0.1.8", - "description": "Convert documents (doc, docx, odt, rtf, epub, pdf, presentations, spreadsheets, csv) to GitHub-Flavored Markdown", + "description": "Convert documents (doc, docx, odt, rtf, epub, html, pdf, presentations, spreadsheets, csv) to GitHub-Flavored Markdown", "license": "MIT", "homepage": "https://github.com/firecrawl/anydoc#readme", "repository": { diff --git a/node/src/lib.rs b/node/src/lib.rs index 01df83a7..099cc95c 100644 --- a/node/src/lib.rs +++ b/node/src/lib.rs @@ -25,6 +25,7 @@ pub enum Format { pptx, rtf, epub, + html, xlsx, ods, odp, @@ -42,6 +43,7 @@ impl From for anydoc::Format { Format::pptx => anydoc::Format::Pptx, Format::rtf => anydoc::Format::Rtf, Format::epub => anydoc::Format::Epub, + Format::html => anydoc::Format::Html, Format::xlsx => anydoc::Format::Excel, Format::ods => anydoc::Format::Ods, Format::odp => anydoc::Format::Odp, @@ -61,6 +63,7 @@ impl From for Format { anydoc::Format::Pptx => Format::pptx, anydoc::Format::Rtf => Format::rtf, anydoc::Format::Epub => Format::epub, + anydoc::Format::Html => Format::html, anydoc::Format::Excel => Format::xlsx, anydoc::Format::Ods => Format::ods, anydoc::Format::Odp => Format::odp, @@ -71,8 +74,9 @@ impl From for Format { /// Detect the format from the content itself: the signature and identity each /// container specification designates (PDF header, RTF open group, OLE stream -/// names, ZIP package mimetype/content types). Plain-text formats (CSV) carry -/// no signature and return `null`; so does anything unrecognized. +/// names, ZIP package mimetype/content types, or an HTML doctype/root element). +/// Plain-text formats (CSV) carry no signature and return `null`; so does +/// anything unrecognized. #[napi] pub fn format_from_bytes(bytes: Uint8Array) -> Option { anydoc::Format::from_bytes(&bytes).map(Format::from) diff --git a/node/test.mjs b/node/test.mjs index ec98a3c6..fa8e567c 100644 --- a/node/test.mjs +++ b/node/test.mjs @@ -23,6 +23,7 @@ const OUTLINE = fixture('docx/handmade-outline.docx') const RICH = fixture('docx/handmade-rich.docx') const CSV = fixture('csv/sheet.csv') const ENCRYPTED = fixture('malformed/encrypted--errors.odt') +const HTML = Buffer.from('

HTML input

') test('toMarkdown detects the format from the file content', async () => { const markdown = await toMarkdown(OUTLINE) @@ -40,6 +41,7 @@ test('toMarkdownBytes detects the format when none is named', async () => { // CSV carries no signature, so it has to be named. await assert.rejects(toMarkdownBytes(await readFile(CSV)), /unrecognized file content/) assert.match(await toMarkdownBytes(await readFile(CSV), 'csv'), /\| --- \|/) + assert.match(await toMarkdownBytes(HTML), /^# HTML input/m) }) test('toDocument exposes the document model', async () => { @@ -61,10 +63,12 @@ test('toDocument carries embedded assets as buffers', async () => { test('format detection reads content, extension, and path', async () => { assert.equal(formatFromBytes(await readFile(RICH)), 'docx') + assert.equal(formatFromBytes(HTML), 'html') // CSV carries no signature: only the extension names it. assert.equal(formatFromBytes(await readFile(CSV)), null) assert.equal(formatFromExtension('.pptm'), 'pptx') assert.equal(formatFromExtension('xls'), 'xlsx') + assert.equal(formatFromExtension('htm'), 'html') assert.equal(formatFromPath('/tmp/report.odt'), 'odt') assert.equal(formatFromPath('/tmp/report.unknown'), null) }) diff --git a/python/README.md b/python/README.md index fa979712..0f8e3544 100644 --- a/python/README.md +++ b/python/README.md @@ -3,7 +3,7 @@ [![PyPI](https://img.shields.io/pypi/v/firecrawl-anydoc.svg)](https://pypi.org/project/firecrawl-anydoc/) [![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/firecrawl/anydoc/blob/main/LICENSE) -Convert Word, PowerPoint, Excel, OpenDocument, RTF, EPUB, CSV, and PDF files into clean GitHub-Flavored Markdown. Python bindings for the [anydoc](https://github.com/firecrawl/anydoc) Rust crate, built by [Firecrawl](https://firecrawl.dev). Also available as a hosted API through [Firecrawl Parse](https://firecrawl.dev/parse), which adds our OCR models for the scanned pages anydoc can't read on its own. +Convert Word, PowerPoint, Excel, OpenDocument, RTF, EPUB, HTML, CSV, and PDF files into clean GitHub-Flavored Markdown. Python bindings for the [anydoc](https://github.com/firecrawl/anydoc) Rust crate, built by [Firecrawl](https://firecrawl.dev). Also available as a hosted API through [Firecrawl Parse](https://firecrawl.dev/parse), which adds our OCR models for the scanned pages anydoc can't read on its own. Every format parses into one shared document model and renders through a single Markdown serializer, so headings, tables, lists, and footnotes come out the same no matter which format goes in. Conversion releases the GIL, so other threads keep running. Type stubs ship with the package. @@ -23,6 +23,7 @@ The package installs as `firecrawl-anydoc` and imports as `anydoc`. | OpenDocument | `.odt`, `.ods`, `.odp` | | Rich Text Format | `.rtf` | | EPUB | `.epub` | +| HTML | `.html`, `.htm` | | CSV | `.csv` | | PDF | `.pdf` | @@ -70,7 +71,7 @@ The five conversion failures subclass `anydoc.ConvertError`, so catching that ha ## Format detection -The format is read from the file content, using the marker its specification designates: the PDF header, the RTF open group, OLE stream names, the ZIP package mimetype and content types. CSV has no such marker, so detection returns `None` for it and the extension, or an explicit format, names it instead. +The format is read from the file content, using the marker its specification designates: the PDF header, the RTF open group, OLE stream names, the ZIP package mimetype and content types, or an HTML doctype/root element. CSV has no such marker, so detection returns `None` for it and the extension, or an explicit format, names it instead. Standalone HTML is supported; MHTML is not yet supported. ```python anydoc.format_from_bytes(data) # 'docx', or None when nothing matches diff --git a/python/anydoc/__init__.py b/python/anydoc/__init__.py index f7c01d71..11a21fd4 100644 --- a/python/anydoc/__init__.py +++ b/python/anydoc/__init__.py @@ -31,7 +31,7 @@ ) Format = Literal[ - "doc", "docx", "odt", "pdf", "ppt", "pptx", "rtf", "epub", "xlsx", "ods", "odp", "csv" + "doc", "docx", "odt", "pdf", "ppt", "pptx", "rtf", "epub", "html", "xlsx", "ods", "odp", "csv" ] """Input format, named after the extension that identifies it. Container variants that share a parser (`.docm`, `.xlsm`, `.ppsx`, ...) map onto these diff --git a/python/anydoc/_anydoc.pyi b/python/anydoc/_anydoc.pyi index 19dadbad..c1bd6dda 100644 --- a/python/anydoc/_anydoc.pyi +++ b/python/anydoc/_anydoc.pyi @@ -4,7 +4,7 @@ import os from typing import Literal, final Format = Literal[ - "doc", "docx", "odt", "pdf", "ppt", "pptx", "rtf", "epub", "xlsx", "ods", "odp", "csv" + "doc", "docx", "odt", "pdf", "ppt", "pptx", "rtf", "epub", "html", "xlsx", "ods", "odp", "csv" ] class ConvertError(Exception): @@ -43,9 +43,9 @@ class MissingPartError(ConvertError): def format_from_bytes(data: bytes | bytearray) -> Format | None: """Detect the format from the content itself: the signature and identity each container specification designates (PDF header, RTF open group, OLE - stream names, ZIP package mimetype/content types). Plain-text formats - (CSV) carry no signature and return `None`; so does anything - unrecognized.""" + stream names, ZIP package mimetype/content types, or an HTML doctype/root + element). Plain-text formats (CSV) carry no signature and return `None`; so + does anything unrecognized.""" def format_from_extension(extension: str) -> Format | None: """The format an extension names, with or without a leading dot.""" diff --git a/python/pyproject.toml b/python/pyproject.toml index 81885146..3e554b79 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "maturin" # PyPI is held by an unrelated package) and imports as `anydoc`. [project] name = "firecrawl-anydoc" -description = "Convert documents (doc, docx, odt, rtf, epub, pdf, presentations, spreadsheets, csv) to GitHub-Flavored Markdown" +description = "Convert documents (doc, docx, odt, rtf, epub, html, pdf, presentations, spreadsheets, csv) to GitHub-Flavored Markdown" readme = "README.md" license = "MIT" requires-python = ">=3.10" diff --git a/python/src/lib.rs b/python/src/lib.rs index abcbbd62..ca84924a 100644 --- a/python/src/lib.rs +++ b/python/src/lib.rs @@ -59,7 +59,7 @@ create_exception!( /// Format names, as the extension that identifies each format. Container /// variants that share a parser (`.docm`, `.xlsm`, `.ppsx`, ...) map onto /// these via `format_from_bytes` or `format_from_extension`. -const FORMATS: [(&str, anydoc::Format); 12] = [ +const FORMATS: [(&str, anydoc::Format); 13] = [ ("doc", anydoc::Format::Doc), ("docx", anydoc::Format::Docx), ("odt", anydoc::Format::Odt), @@ -68,6 +68,7 @@ const FORMATS: [(&str, anydoc::Format); 12] = [ ("pptx", anydoc::Format::Pptx), ("rtf", anydoc::Format::Rtf), ("epub", anydoc::Format::Epub), + ("html", anydoc::Format::Html), ("xlsx", anydoc::Format::Excel), ("ods", anydoc::Format::Ods), ("odp", anydoc::Format::Odp), @@ -127,8 +128,9 @@ fn convert_error(py: Python<'_>, error: anydoc::ConvertError) -> PyErr { /// Detect the format from the content itself: the signature and identity each /// container specification designates (PDF header, RTF open group, OLE stream -/// names, ZIP package mimetype/content types). Plain-text formats (CSV) carry -/// no signature and return `None`; so does anything unrecognized. +/// names, ZIP package mimetype/content types, or an HTML doctype/root element). +/// Plain-text formats (CSV) carry no signature and return `None`; so does +/// anything unrecognized. #[pyfunction] fn format_from_bytes(data: Vec) -> Option<&'static str> { anydoc::Format::from_bytes(&data).map(format_name) diff --git a/python/tests/test_anydoc.py b/python/tests/test_anydoc.py index 1328d44a..4f977a24 100644 --- a/python/tests/test_anydoc.py +++ b/python/tests/test_anydoc.py @@ -14,6 +14,7 @@ CSV = FIXTURES / "csv" / "sheet.csv" ENCRYPTED = FIXTURES / "malformed" / "encrypted--errors.odt" ZIPBOMB = FIXTURES / "abuse" / "zipbomb--errors.docx" +HTML = b"

HTML input

" class AnydocTest(unittest.TestCase): @@ -32,6 +33,7 @@ def test_to_markdown_bytes_detects_the_format_when_none_is_named(self): with self.assertRaisesRegex(anydoc.ConvertError, "unrecognized file content"): anydoc.to_markdown_bytes(CSV.read_bytes()) self.assertIn("| --- |", anydoc.to_markdown_bytes(CSV.read_bytes(), "csv")) + self.assertRegex(anydoc.to_markdown_bytes(HTML), r"(?m)^# HTML input") def test_to_document_exposes_the_document_model(self): document = anydoc.to_document(OUTLINE.read_bytes(), "docx") @@ -50,10 +52,12 @@ def test_to_document_carries_embedded_assets_as_bytes(self): def test_format_detection_reads_content_extension_and_path(self): self.assertEqual(anydoc.format_from_bytes(RICH.read_bytes()), "docx") + self.assertEqual(anydoc.format_from_bytes(HTML), "html") # CSV carries no signature: only the extension names it. self.assertIsNone(anydoc.format_from_bytes(CSV.read_bytes())) self.assertEqual(anydoc.format_from_extension(".pptm"), "pptx") self.assertEqual(anydoc.format_from_extension("xls"), "xlsx") + self.assertEqual(anydoc.format_from_extension("htm"), "html") self.assertEqual(anydoc.format_from_path("report.odt"), "odt") self.assertIsNone(anydoc.format_from_path("report.unknown")) diff --git a/skills/convert-documents-to-markdown/SKILL.md b/skills/convert-documents-to-markdown/SKILL.md index d4a0591e..3942dae1 100644 --- a/skills/convert-documents-to-markdown/SKILL.md +++ b/skills/convert-documents-to-markdown/SKILL.md @@ -1,6 +1,6 @@ --- name: convert-documents-to-markdown -description: Convert Word (.doc, .docx), PowerPoint (.ppt, .pptx), Excel (.xls, .xlsx), OpenDocument (.odt, .ods, .odp), RTF, EPUB, CSV, and PDF files to GitHub-Flavored Markdown. Use when a task needs the contents of an office document, spreadsheet, presentation, ebook, or PDF you cannot read directly. +description: Convert Word (.doc, .docx), PowerPoint (.ppt, .pptx), Excel (.xls, .xlsx), OpenDocument (.odt, .ods, .odp), RTF, EPUB, standalone HTML, CSV, and PDF files to GitHub-Flavored Markdown. Use when a task needs the contents of an office document, spreadsheet, presentation, ebook, HTML export, or PDF you cannot read directly. license: MIT metadata: author: firecrawl diff --git a/src/formats/detect.rs b/src/formats/detect.rs index 727a69f6..eefac6d8 100644 --- a/src/formats/detect.rs +++ b/src/formats/detect.rs @@ -14,6 +14,8 @@ //! type of the part the package-level officeDocument relationship //! designates as the main document (with the main part's mandated root //! element as the authority when content types are stale or generic). +//! - Standalone HTML: a leading HTML doctype or root element, so exports that +//! use a misleading `.doc` extension are still routed to the HTML frontend. //! //! Plain-text formats (CSV) carry no signature and are never detected; //! callers fall back to the file extension. Detection never errors: any @@ -44,9 +46,33 @@ pub(crate) fn from_bytes(bytes: &[u8]) -> Option { if bytes[..bytes.len().min(1024)].windows(5).any(|w| w == b"%PDF-") { return Some(Format::Pdf); } + if looks_like_html(bytes) { + return Some(Format::Html); + } None } +/// Detect the two HTML forms used by standalone exports without classifying +/// arbitrary text that happens to contain an HTML tag. The extension remains +/// the fallback for less conventional HTML prologs. +fn looks_like_html(bytes: &[u8]) -> bool { + let head = String::from_utf8_lossy(&bytes[..bytes.len().min(4096)]); + let mut head = head.trim_start_matches('\u{feff}').trim_start(); + if let Some(rest) = head.strip_prefix("") + { + head = rest[end + 2..].trim_start(); + } + let lower = head.to_ascii_lowercase(); + if let Some(rest) = lower.strip_prefix("', ' ', '\t', '\r', '\n'])); + } + lower.strip_prefix("', ' ', '\t', '\r', '\n'])) +} + /// Classify an OLE compound file by its mandated content stream. Encrypted /// OOXML packages (`EncryptedPackage`) stay `None`: the inner format is /// unknowable, and the frontend reports `Encrypted` precisely. @@ -240,6 +266,20 @@ mod tests { assert_eq!(from_bytes(b""), None); } + #[test] + fn standalone_html_is_detected_without_an_extension() { + assert_eq!( + from_bytes(b"text"), + Some(Format::Html) + ); + assert_eq!(from_bytes(b"text"), Some(Format::Html)); + assert_eq!( + from_bytes(b"text"), + Some(Format::Html) + ); + assert_eq!(from_bytes(b"plain text mentioning here"), None); + } + #[test] fn container_signature_wins_over_an_early_embedded_pdf() { let pkg = zip_of(&[("embedded.pdf", b"%PDF-1.7\n"), ("word/document.xml", b"")]); diff --git a/src/formats/html.rs b/src/formats/html.rs new file mode 100644 index 00000000..cd5f1015 --- /dev/null +++ b/src/formats/html.rs @@ -0,0 +1,116 @@ +//! Standalone HTML documents, including HTML exports saved with a `.doc` +//! extension. +//! +//! The DOM and block walker are shared with EPUB. Standalone documents have +//! no package base path, so external links and images are preserved while +//! relative resources degrade to their text/alt content. + +use crate::error::ConvertError; +use crate::model::{AnchorId, Document, ImageSource, LinkTarget}; +use crate::package::path::decode_fragment; +use crate::package::xml::{Element, Node, parse_xml}; +use crate::shared::html::{HtmlCtx, Stylesheet, to_blocks}; +use crate::shared::uri::is_absolute_uri; + +pub fn parse(bytes: &[u8]) -> Result { + let mut tree = parse_xml(bytes)?; + normalize_html_tree(&mut tree); + + let html = tree.child_elems().find(|e| e.local == "html").unwrap_or(&tree); + let body = html.child_elems().find(|e| e.local == "body").unwrap_or(html); + + let mut css = Stylesheet::default(); + for style in tree.descendants_any("style") { + css.add(&style.text()); + } + + let ctx = StandaloneCtx; + Ok(Document { blocks: to_blocks(body, &css, &ctx)?, ..Document::default() }) +} + +/// HTML tag and attribute names are ASCII case-insensitive. Normalizing the +/// small DOM once lets the existing EPUB walker handle HTML exports from +/// producers that use uppercase names without adding case checks to every +/// element lookup. +fn normalize_html_tree(elem: &mut Element) { + elem.local.make_ascii_lowercase(); + for attr in &mut elem.attrs { + attr.local.make_ascii_lowercase(); + } + for child in &mut elem.children { + if let Node::Elem(child) = child { + normalize_html_tree(child); + } + } +} + +struct StandaloneCtx; + +impl HtmlCtx for StandaloneCtx { + fn link_target(&self, href: &str) -> Option { + if href.is_empty() { + return None; + } + if let Some(fragment) = href.strip_prefix('#') { + let fragment = decode_fragment(fragment); + return (!fragment.is_empty()).then_some(LinkTarget::Anchor(fragment)); + } + if is_absolute_uri(href) { + Some(LinkTarget::External(href.to_string())) + } else { + Some(LinkTarget::Relative(href.to_string())) + } + } + + fn image_source(&self, src: &str) -> Result, ConvertError> { + if is_absolute_uri(src) { + Ok(Some(ImageSource::External(src.to_string()))) + } else { + // Resolving a relative resource requires the source file's path; + // the byte-oriented public API intentionally has no such base. + Ok(None) + } + } + + fn anchor_id(&self, raw: &str) -> AnchorId { + raw.to_string() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::render::markdown::document_to_markdown; + + #[test] + fn jira_style_doc_extension_html_converts() { + let html = br#" + + + + + + +

Issue

+

Status: Open

+ +

Tracker icon

+ +"#; + + assert_eq!(crate::Format::from_bytes(html), Some(crate::Format::Html)); + assert_eq!(crate::Format::from_extension("doc"), Some(crate::Format::Doc)); + let doc = parse(html).unwrap(); + assert_eq!( + document_to_markdown(&doc), + "# Issue\n\nStatus: **Open**\n\n[Tracker](https://example.com) ![icon](https://example.com/icon.png)\n" + ); + } + + #[test] + fn html_names_and_fragment_links_are_case_insensitive_and_decoded() { + let html = br##"

Target

jump

"##; + let doc = parse(html).unwrap(); + assert_eq!(document_to_markdown(&doc), "## Target\n\n[jump](#target)\n"); + } +} diff --git a/src/formats/mod.rs b/src/formats/mod.rs index d570b264..5be9c8d2 100644 --- a/src/formats/mod.rs +++ b/src/formats/mod.rs @@ -5,6 +5,7 @@ pub mod detect; mod doc; mod docx; mod epub; +mod html; mod odf; pub mod pdf; mod ppt; @@ -24,6 +25,7 @@ pub fn parse(bytes: &[u8], format: Format) -> Result { Format::Odt | Format::Ods | Format::Odp => odf::parse(bytes), Format::Pptx => pptx::parse(bytes), Format::Epub => epub::parse(bytes), + Format::Html => html::parse(bytes), Format::Rtf => rtf::parse(bytes), // RTF files wearing a .doc extension are common in the wild. Format::Doc if bytes.starts_with(b"{\\rtf") => rtf::parse(bytes), diff --git a/src/formats/rtf/tables.rs b/src/formats/rtf/tables.rs index d01b8425..e5fd4bb3 100644 --- a/src/formats/rtf/tables.rs +++ b/src/formats/rtf/tables.rs @@ -289,19 +289,13 @@ fn parse_listtable( levels = std::array::from_fn(|_| ListLevelDef::default()); level_index = 0; } - "listid" => { - if list_depth.is_some() { - list_id = param; - } - } + "listid" if list_depth.is_some() => list_id = param, "listlevel" => { in_level = true; collector = LevelTextCollector::default(); } - "levelnfc" | "levelnfcn" => { - if in_level && level_index < LIST_LEVELS { - levels[level_index].marker = marker_for_nfc(param.unwrap_or(0)); - } + "levelnfc" | "levelnfcn" if in_level && level_index < LIST_LEVELS => { + levels[level_index].marker = marker_for_nfc(param.unwrap_or(0)); } "levelstartat" => { if in_level @@ -418,10 +412,8 @@ fn parse_overrides( starts[level_index] = Some(n.max(0) as u64); } } - "levelnfc" | "levelnfcn" if lfo_depth.is_some() => { - if level_index < LIST_LEVELS { - markers[level_index] = Some(marker_for_nfc(param.unwrap_or(0))); - } + "levelnfc" | "levelnfcn" if lfo_depth.is_some() && level_index < LIST_LEVELS => { + markers[level_index] = Some(marker_for_nfc(param.unwrap_or(0))); } "leveltext" if lfo_depth.is_some() => collector.active = Some(true), "levelnumbers" if lfo_depth.is_some() => collector.active = Some(false), diff --git a/src/lib.rs b/src/lib.rs index efba6ff8..ae144ffb 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -45,6 +45,8 @@ pub enum Format { Rtf, /// EPUB 2 and 3 (`.epub`). Epub, + /// Standalone HTML documents (`.html`, `.htm`). + Html, /// Excel workbooks in every container calamine reads: `.xlsx`, `.xlsm`, /// `.xlsb`, and binary `.xls`. Excel, @@ -60,9 +62,9 @@ pub enum Format { impl Format { /// Detect the format from the content itself: the signature and identity /// each container specification designates (PDF header, RTF open group, - /// OLE stream names, ZIP package mimetype/content types). Plain-text - /// formats (CSV) carry no signature and return `None`; so does anything - /// unrecognized. + /// OLE stream names, ZIP package mimetype/content types, or an HTML + /// doctype/root element). Plain-text formats (CSV) carry no signature and + /// return `None`; so does anything unrecognized. pub fn from_bytes(bytes: &[u8]) -> Option { formats::detect::from_bytes(bytes) } @@ -79,6 +81,7 @@ impl Format { "ppt" | "pps" | "pot" => Format::Ppt, "rtf" => Format::Rtf, "epub" => Format::Epub, + "html" | "htm" => Format::Html, "xlsx" | "xlsm" | "xlsb" | "xls" => Format::Excel, "ods" => Format::Ods, "odp" => Format::Odp, diff --git a/src/package/xml.rs b/src/package/xml.rs index 42af1d8c..f0c65dc4 100644 --- a/src/package/xml.rs +++ b/src/package/xml.rs @@ -38,6 +38,7 @@ pub mod ns { pub const XLINK: &str = "http://www.w3.org/1999/xlink"; pub const XML: &str = "http://www.w3.org/XML/1998/namespace"; pub const SVG_COMPAT: &str = "urn:oasis:names:tc:opendocument:xmlns:svg-compatible:1.0"; + pub const XHTML: &str = "http://www.w3.org/1999/xhtml"; pub const VML: &str = "urn:schemas-microsoft-com:vml"; pub const O_VML: &str = "urn:schemas-microsoft-com:office:office"; @@ -218,7 +219,12 @@ pub fn parse_xml(bytes: &[u8]) -> Result { }); } bump_nodes(&mut nodes)?; - stack.push(start_to_element(&e, &reader, &mut interner)); + let elem = start_to_element(&e, &reader, &mut interner); + if is_html_void_element(&elem) { + attach(&mut stack, &mut root, Node::Elem(elem)); + } else { + stack.push(elem); + } } Event::Empty(e) => { bump_nodes(&mut nodes)?; @@ -271,6 +277,21 @@ pub fn parse_xml(bytes: &[u8]) -> Result { Ok(root) } +/// HTML permits these elements to omit an end tag. Treating them as empty +/// while parsing the shared XML-ish DOM keeps a standalone HTML document from +/// swallowing its following siblings (for example, `` capturing the +/// ``). Namespaced XHTML and unqualified HTML are both accepted; other +/// vocabularies still follow the normal XML recovery rules. +fn is_html_void_element(elem: &Element) -> bool { + matches!(elem.ns.as_deref(), None | Some(ns::XHTML)) + && [ + "area", "base", "basefont", "bgsound", "br", "col", "embed", "frame", "hr", "img", + "input", "keygen", "link", "meta", "param", "source", "track", "wbr", + ] + .iter() + .any(|name| elem.local.eq_ignore_ascii_case(name)) +} + /// Transcode an XML part to UTF-8 based on its BOM or encoding declaration. fn to_utf8(bytes: &[u8]) -> std::borrow::Cow<'_, [u8]> { use std::borrow::Cow; @@ -505,6 +526,20 @@ mod tests { assert!(r.find("urn:b", "root").is_none()); } + #[test] + fn html_void_elements_do_not_capture_following_siblings() { + let root = parse_xml( + br#"

visible sibling

second

"#, + ) + .unwrap(); + let html = root.child_elems().next().unwrap(); + let head = html.child_elems().find(|e| e.local == "head").unwrap(); + assert_eq!(head.child_elems().filter(|e| e.local == "meta").count(), 1); + assert!(!head.child_elems().any(|e| e.local == "body")); + let body = html.child_elems().find(|e| e.local == "body").unwrap(); + assert_eq!(body.child_elems().filter(|e| e.local == "p").count(), 2); + } + #[test] fn qualified_and_unqualified_lookups_are_separate() { let xml = diff --git a/tests/fixtures/html/standalone.html b/tests/fixtures/html/standalone.html new file mode 100644 index 00000000..f6e4495c --- /dev/null +++ b/tests/fixtures/html/standalone.html @@ -0,0 +1,15 @@ + + + + + + + +

Standalone HTML

+

Issue text with formatting.

+

External link

+ + + diff --git a/tests/snapshots.rs b/tests/snapshots.rs index 477a1adc..9ad038fa 100644 --- a/tests/snapshots.rs +++ b/tests/snapshots.rs @@ -85,6 +85,7 @@ fn fixtures_detect_from_content() { ("doc", Some(Format::Doc)), ("docx", Some(Format::Docx)), ("epub", Some(Format::Epub)), + ("html", Some(Format::Html)), ("odp", Some(Format::Odp)), ("ods", Some(Format::Ods)), ("odt", Some(Format::Odt)), diff --git a/tests/snapshots/snapshots__html__standalone.html.snap b/tests/snapshots/snapshots__html__standalone.html.snap new file mode 100644 index 00000000..73383731 --- /dev/null +++ b/tests/snapshots/snapshots__html__standalone.html.snap @@ -0,0 +1,10 @@ +--- +source: tests/snapshots.rs +assertion_line: 69 +expression: output +--- +# Standalone HTML + +Issue text with **formatting**. + +[External link](https://example.com) diff --git a/wasm/Cargo.toml b/wasm/Cargo.toml index 9ea88417..e2d59364 100644 --- a/wasm/Cargo.toml +++ b/wasm/Cargo.toml @@ -6,7 +6,7 @@ name = "anydoc-wasm" version = "0.1.8" edition = "2024" -description = "WebAssembly bindings for anydoc: convert documents (doc, docx, odt, rtf, epub, pdf, presentations, spreadsheets, csv) to GitHub-Flavored Markdown in the browser" +description = "WebAssembly bindings for anydoc: convert documents (doc, docx, odt, rtf, epub, html, pdf, presentations, spreadsheets, csv) to GitHub-Flavored Markdown in the browser" license = "MIT" repository = "https://github.com/firecrawl/anydoc" publish = false diff --git a/wasm/src/lib.rs b/wasm/src/lib.rs index 5a710a5b..18de5ae9 100644 --- a/wasm/src/lib.rs +++ b/wasm/src/lib.rs @@ -27,6 +27,7 @@ pub enum Format { Pptx = "pptx", Rtf = "rtf", Epub = "epub", + Html = "html", Xlsx = "xlsx", Ods = "ods", Odp = "odp", @@ -44,6 +45,7 @@ impl From for anydoc::Format { Format::Pptx => anydoc::Format::Pptx, Format::Rtf => anydoc::Format::Rtf, Format::Epub => anydoc::Format::Epub, + Format::Html => anydoc::Format::Html, Format::Xlsx => anydoc::Format::Excel, Format::Ods => anydoc::Format::Ods, Format::Odp => anydoc::Format::Odp, @@ -64,6 +66,7 @@ impl From for Format { anydoc::Format::Pptx => Format::Pptx, anydoc::Format::Rtf => Format::Rtf, anydoc::Format::Epub => Format::Epub, + anydoc::Format::Html => Format::Html, anydoc::Format::Excel => Format::Xlsx, anydoc::Format::Ods => Format::Ods, anydoc::Format::Odp => Format::Odp, @@ -74,8 +77,9 @@ impl From for Format { /// Detect the format from the content itself: the signature and identity each /// container specification designates (PDF header, RTF open group, OLE stream -/// names, ZIP package mimetype/content types). Plain-text formats (CSV) carry -/// no signature and return `undefined`; so does anything unrecognized. +/// names, ZIP package mimetype/content types, or an HTML doctype/root element). +/// Plain-text formats (CSV) carry no signature and return `undefined`; so does +/// anything unrecognized. #[wasm_bindgen(js_name = formatFromBytes)] pub fn format_from_bytes(bytes: &[u8]) -> Option { anydoc::Format::from_bytes(bytes).map(Format::from) diff --git a/wasm/test.mjs b/wasm/test.mjs index afcebfcc..e9428631 100644 --- a/wasm/test.mjs +++ b/wasm/test.mjs @@ -23,6 +23,7 @@ const RICH = await readFile(fixture('docx/handmade-rich.docx')) const CSV = await readFile(fixture('csv/sheet.csv')) const PDF = await readFile(fixture('pdf/text.pdf')) const ENCRYPTED = await readFile(fixture('malformed/encrypted--errors.odt')) +const HTML = new TextEncoder().encode('

HTML input

') test('toMarkdownBytes converts in memory', () => { const markdown = toMarkdownBytes(RICH, 'docx') @@ -34,6 +35,7 @@ test('toMarkdownBytes detects the format when none is named', () => { // CSV carries no signature, so it has to be named. assert.throws(() => toMarkdownBytes(CSV), /unrecognized file content/) assert.match(toMarkdownBytes(CSV, 'csv'), /\| --- \|/) + assert.match(toMarkdownBytes(HTML), /^# HTML input/m) }) test('pdf converts to Markdown but has no document model', () => { @@ -60,10 +62,12 @@ test('toDocument carries embedded assets as Uint8Arrays', () => { test('format detection reads content, extension, and path', () => { assert.equal(formatFromBytes(RICH), 'docx') + assert.equal(formatFromBytes(HTML), 'html') // CSV carries no signature: only the extension names it. assert.equal(formatFromBytes(CSV), undefined) assert.equal(formatFromExtension('.pptm'), 'pptx') assert.equal(formatFromExtension('xls'), 'xlsx') + assert.equal(formatFromExtension('htm'), 'html') assert.equal(formatFromPath('/tmp/report.odt'), 'odt') assert.equal(formatFromPath('/tmp/report.unknown'), undefined) }) diff --git a/wasm/www/index.html b/wasm/www/index.html index 2e610db4..7ce82a93 100644 --- a/wasm/www/index.html +++ b/wasm/www/index.html @@ -6,7 +6,7 @@ anydoc by Firecrawl