diff --git a/Cargo.lock b/Cargo.lock index 4f390e21..0ad2afc3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -104,6 +104,16 @@ dependencies = [ "zip", ] +[[package]] +name = "anydoc-dotnet" +version = "0.1.7" +dependencies = [ + "anydoc", + "base64", + "csbindgen", + "serde_json", +] + [[package]] name = "anydoc-node" version = "0.1.0" @@ -150,6 +160,12 @@ version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + [[package]] name = "bitflags" version = "1.3.2" @@ -406,6 +422,16 @@ dependencies = [ "hybrid-array", ] +[[package]] +name = "csbindgen" +version = "1.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "950f59b281d7e20f050b4efd56d7c36c0deb853bf9ea1f20b985a75ae5b03b34" +dependencies = [ + "regex", + "syn 2.0.119", +] + [[package]] name = "csv" version = "1.4.0" @@ -1365,6 +1391,19 @@ dependencies = [ "syn 3.0.3", ] +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + [[package]] name = "sha2" version = "0.10.9" @@ -1750,6 +1789,12 @@ version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b142a20ec14a91d5bc708c1dc21b080c550113d8aa77afa29635673a65dd02c5" +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" + [[package]] name = "zopfli" version = "0.8.3" diff --git a/Cargo.toml b/Cargo.toml index 871b9bdf..8a90eb54 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace] -members = ["node", "python", "wasm"] +members = ["node", "python", "wasm", "dotnet/native"] # cargo-fuzz keeps its own lockfile and nightly-only dependencies. exclude = ["fuzz"] diff --git a/README.md b/README.md index ed4cb9e4..9d590530 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ [![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE) [![skills.sh](https://skills.sh/b/firecrawl/anydoc)](https://skills.sh/firecrawl/anydoc) -Fast Rust library that converts documents (Word, PowerPoint, Excel, OpenDocument, RTF, EPUB, CSV, and PDF) into clean GitHub-Flavored Markdown. Includes bindings for [Node.js](node/README.md), [Python](python/README.md), and the [browser](wasm/README.md) (WebAssembly). +Fast Rust library that converts documents (Word, PowerPoint, Excel, OpenDocument, RTF, EPUB, CSV, and PDF) into clean GitHub-Flavored Markdown. Includes bindings for [Node.js](node/README.md), [Python](python/README.md), the [browser](wasm/README.md) (WebAssembly), and [.NET](dotnet/README.md) (NuGet `Firecrawl.Anydoc`). Built by [Firecrawl](https://firecrawl.dev) to turn any office document into LLM-ready Markdown in single-digit milliseconds, with one consistent output no matter which format goes in. It powers [Firecrawl Parse](https://firecrawl.dev/parse), so if you'd rather not run it yourself, the hosted API gives you the same conversion plus our OCR models for the scanned pages anydoc can't read on its own. diff --git a/dotnet/.gitignore b/dotnet/.gitignore new file mode 100644 index 00000000..61203f29 --- /dev/null +++ b/dotnet/.gitignore @@ -0,0 +1,490 @@ +## Ignore Visual Studio temporary files, build results, and +## files generated by popular Visual Studio add-ons. +## +## Get latest from `dotnet new gitignore` + +# dotenv files +.env + +# User-specific files +*.rsuser +*.suo +*.user +*.userosscache +*.sln.docstates + +# User-specific files (MonoDevelop/Xamarin Studio) +*.userprefs + +# Mono auto generated files +mono_crash.* + +# Build results +[Dd]ebug/ +[Dd]ebugPublic/ +[Rr]elease/ +[Rr]eleases/ +x64/ +x86/ +[Ww][Ii][Nn]32/ +[Aa][Rr][Mm]/ +[Aa][Rr][Mm]64/ +bld/ +[Bb]in/ +[Oo]bj/ +[Ll]og/ +[Ll]ogs/ + +# Visual Studio 2015/2017 cache/options directory +.vs/ +# Uncomment if you have tasks that create the project's static files in wwwroot +#wwwroot/ + +# Visual Studio 2017 auto generated files +Generated\ Files/ + +# MSTest test Results +[Tt]est[Rr]esult*/ +[Bb]uild[Ll]og.* + +# NUnit +*.VisualState.xml +TestResult.xml +nunit-*.xml + +# Build Results of an ATL Project +[Dd]ebugPS/ +[Rr]eleasePS/ +dlldata.c + +# Benchmark Results +BenchmarkDotNet.Artifacts/ + +# .NET +project.lock.json +project.fragment.lock.json +artifacts/ + +# Tye +.tye/ + +# ASP.NET Scaffolding +ScaffoldingReadMe.txt + +# StyleCop +StyleCopReport.xml + +# Files built by Visual Studio +*_i.c +*_p.c +*_h.h +*.ilk +*.meta +*.obj +*.iobj +*.pch +*.pdb +*.ipdb +*.pgc +*.pgd +*.rsp +# but not Directory.Build.rsp, as it configures directory-level build defaults +!Directory.Build.rsp +*.sbr +*.tlb +*.tli +*.tlh +*.tmp +*.tmp_proj +*_wpftmp.csproj +*.log +*.tlog +*.vspscc +*.vssscc +.builds +*.pidb +*.svclog +*.scc + +# Chutzpah Test files +_Chutzpah* + +# Visual C++ cache files +ipch/ +*.aps +*.ncb +*.opendb +*.opensdf +*.sdf +*.cachefile +*.VC.db +*.VC.VC.opendb + +# Visual Studio profiler +*.psess +*.vsp +*.vspx +*.sap + +# Visual Studio Trace Files +*.e2e + +# TFS 2012 Local Workspace +$tf/ + +# Guidance Automation Toolkit +*.gpState + +# ReSharper is a .NET coding add-in +_ReSharper*/ +*.[Rr]e[Ss]harper +*.DotSettings.user + +# TeamCity is a build add-in +_TeamCity* + +# DotCover is a Code Coverage Tool +*.dotCover + +# AxoCover is a Code Coverage Tool +.axoCover/* +!.axoCover/settings.json + +# Coverlet is a free, cross platform Code Coverage Tool +coverage*.json +coverage*.xml +coverage*.info + +# Visual Studio code coverage results +*.coverage +*.coveragexml + +# NCrunch +_NCrunch_* +.*crunch*.local.xml +nCrunchTemp_* + +# MightyMoose +*.mm.* +AutoTest.Net/ + +# Web workbench (sass) +.sass-cache/ + +# Installshield output folder +[Ee]xpress/ + +# DocProject is a documentation generator add-in +DocProject/buildhelp/ +DocProject/Help/*.HxT +DocProject/Help/*.HxC +DocProject/Help/*.hhc +DocProject/Help/*.hhk +DocProject/Help/*.hhp +DocProject/Help/Html2 +DocProject/Help/html + +# Click-Once directory +publish/ + +# Publish Web Output +*.[Pp]ublish.xml +*.azurePubxml +# Note: Comment the next line if you want to checkin your web deploy settings, +# but database connection strings (with potential passwords) will be unencrypted +*.pubxml +*.publishproj + +# Microsoft Azure Web App publish settings. Comment the next line if you want to +# checkin your Azure Web App publish settings, but sensitive information contained +# in these scripts will be unencrypted +PublishScripts/ + +# NuGet Packages +*.nupkg +# NuGet Symbol Packages +*.snupkg +# The packages folder can be ignored because of Package Restore +**/[Pp]ackages/* +# except build/, which is used as an MSBuild target. +!**/[Pp]ackages/build/ +# Uncomment if necessary however generally it will be regenerated when needed +#!**/[Pp]ackages/repositories.config +# NuGet v3's project.json files produces more ignorable files +*.nuget.props +*.nuget.targets + +# Microsoft Azure Build Output +csx/ +*.build.csdef + +# Microsoft Azure Emulator +ecf/ +rcf/ + +# Windows Store app package directories and files +AppPackages/ +BundleArtifacts/ +Package.StoreAssociation.xml +_pkginfo.txt +*.appx +*.appxbundle +*.appxupload + +# Visual Studio cache files +# files ending in .cache can be ignored +*.[Cc]ache +# but keep track of directories ending in .cache +!?*.[Cc]ache/ + +# Others +ClientBin/ +~$* +*~ +*.dbmdl +*.dbproj.schemaview +*.jfm +*.pfx +*.publishsettings +orleans.codegen.cs + +# Including strong name files can present a security risk +# (https://github.com/github/gitignore/pull/2483#issue-259490424) +#*.snk + +# Since there are multiple workflows, uncomment next line to ignore bower_components +# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) +#bower_components/ + +# RIA/Silverlight projects +Generated_Code/ + +# Backup & report files from converting an old project file +# to a newer Visual Studio version. Backup files are not needed, +# because we have git ;-) +_UpgradeReport_Files/ +Backup*/ +UpgradeLog*.XML +UpgradeLog*.htm +ServiceFabricBackup/ +*.rptproj.bak + +# SQL Server files +*.mdf +*.ldf +*.ndf + +# Business Intelligence projects +*.rdl.data +*.bim.layout +*.bim_*.settings +*.rptproj.rsuser +*- [Bb]ackup.rdl +*- [Bb]ackup ([0-9]).rdl +*- [Bb]ackup ([0-9][0-9]).rdl + +# Microsoft Fakes +FakesAssemblies/ + +# GhostDoc plugin setting file +*.GhostDoc.xml + +# Node.js Tools for Visual Studio +.ntvs_analysis.dat +node_modules/ + +# Visual Studio 6 build log +*.plg + +# Visual Studio 6 workspace options file +*.opt + +# Visual Studio 6 auto-generated workspace file (contains which files were open etc.) +*.vbw + +# Visual Studio 6 auto-generated project file (contains which files were open etc.) +*.vbp + +# Visual Studio 6 workspace and project file (working project files containing files to include in project) +*.dsw +*.dsp + +# Visual Studio 6 technical files +*.ncb +*.aps + +# Visual Studio LightSwitch build output +**/*.HTMLClient/GeneratedArtifacts +**/*.DesktopClient/GeneratedArtifacts +**/*.DesktopClient/ModelManifest.xml +**/*.Server/GeneratedArtifacts +**/*.Server/ModelManifest.xml +_Pvt_Extensions + +# Paket dependency manager +.paket/paket.exe +paket-files/ + +# FAKE - F# Make +.fake/ + +# CodeRush personal settings +.cr/personal + +# Python Tools for Visual Studio (PTVS) +__pycache__/ +*.pyc + +# Cake - Uncomment if you are using it +# tools/** +# !tools/packages.config + +# Tabs Studio +*.tss + +# Telerik's JustMock configuration file +*.jmconfig + +# BizTalk build output +*.btp.cs +*.btm.cs +*.odx.cs +*.xsd.cs + +# OpenCover UI analysis results +OpenCover/ + +# Azure Stream Analytics local run output +ASALocalRun/ + +# MSBuild Binary and Structured Log +*.binlog + +# NVidia Nsight GPU debugger configuration file +*.nvuser + +# MFractors (Xamarin productivity tool) working folder +.mfractor/ + +# Local History for Visual Studio +.localhistory/ + +# Visual Studio History (VSHistory) files +.vshistory/ + +# BeatPulse healthcheck temp database +healthchecksdb + +# Backup folder for Package Reference Convert tool in Visual Studio 2017 +MigrationBackup/ + +# Ionide (cross platform F# VS Code tools) working folder +.ionide/ + +# Fody - auto-generated XML schema +FodyWeavers.xsd + +# VS Code files for those working on multiple tools +.vscode/* +!.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json +*.code-workspace + +# Local History for Visual Studio Code +.history/ + +# Windows Installer files from build outputs +*.cab +*.msi +*.msix +*.msm +*.msp + +# JetBrains Rider +*.sln.iml +.idea/ + +## +## Visual studio for Mac +## + + +# globs +Makefile.in +*.userprefs +*.usertasks +config.make +config.status +aclocal.m4 +install-sh +autom4te.cache/ +*.tar.gz +tarballs/ +test-results/ + +# content below from: https://github.com/github/gitignore/blob/main/Global/macOS.gitignore +# General +.DS_Store +.AppleDouble +.LSOverride + +# Icon must end with two \r +Icon + + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns +.com.apple.timemachine.donotpresent + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + +# content below from: https://github.com/github/gitignore/blob/main/Global/Windows.gitignore +# Windows thumbnail cache files +Thumbs.db +ehthumbs.db +ehthumbs_vista.db + +# Dump file +*.stackdump + +# Folder config file +[Dd]esktop.ini + +# Recycle Bin used on file shares +$RECYCLE.BIN/ + +# Windows Installer files +*.cab +*.msi +*.msix +*.msm +*.msp + +# Windows shortcuts +*.lnk + +# Vim temporary swap files +*.swp + +runtimes/ + +# compiled WebAssembly engine (produced by build-wasm.sh) +Firecrawl.Anydoc.Wasm/wwwroot/anydoc_wasm.js +Firecrawl.Anydoc.Wasm/wwwroot/anydoc_wasm_bg.wasm +Firecrawl.Anydoc.Wasm/wwwroot/anydoc_wasm.d.ts +Firecrawl.Anydoc.Wasm/wwwroot/anydoc_wasm_bg.wasm.d.ts diff --git a/dotnet/Firecrawl.Anydoc.Shared/Model.cs b/dotnet/Firecrawl.Anydoc.Shared/Model.cs new file mode 100644 index 00000000..2d96b4e2 --- /dev/null +++ b/dotnet/Firecrawl.Anydoc.Shared/Model.cs @@ -0,0 +1,197 @@ +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Firecrawl.Anydoc.Model; + +/// A parsed document: its body, its notes, and the bytes of everything +/// it embedded. +public sealed class Document +{ + public List Blocks { get; set; } = new(); + /// Footnote and endnote bodies, referenced from text by a + /// inline. + public List Notes { get; set; } = new(); + public List Assets { get; set; } = new(); + + internal static Document FromJson(string json) => + JsonSerializer.Deserialize(json, Json.Options)!; +} + +/// One block-level piece of a document body. +public sealed class Block +{ + /// heading, paragraph, list, table, + /// block_quote, code_block, or rule. + public string Kind { get; set; } = ""; + /// heading: 1-6. + public int? Level { get; set; } + /// heading: stable anchor id when the source document targets this + /// heading (bookmark, chapter fragment, ...). + public string? Anchor { get; set; } + /// heading, paragraph: + public List? Content { get; set; } + public Firecrawl.Anydoc.Model.List? List { get; set; } + public Table? Table { get; set; } + /// block_quote: nested blocks. + public List? Blocks { get; set; } + /// code_block: language hint when the source names one. + public string? Lang { get; set; } + /// code_block: the literal text, newlines intact. + public string? Text { get; set; } +} + +/// One span of inline content. +public sealed class Inline +{ + /// text, link, image, anchor (a + /// zero-width marker for an internal link target), note_ref, or + /// line_break. + public string Kind { get; set; } = ""; + /// text. + public string? Text { get; set; } + /// text: fully resolved character style. + public Style? Style { get; set; } + /// link: nested inline content. + public List? Content { get; set; } + /// link. + public LinkTarget? Target { get; set; } + /// image: alt text, empty when the source gives none. + public string? Alt { get; set; } + /// image. + public ImageSource? Source { get; set; } + /// anchor: the anchor id. + public string? Anchor { get; set; } + /// note_ref: the id of the note in . + public string? NoteId { get; set; } +} + +/// Fully resolved character style. +public sealed class Style +{ + public bool Bold { get; set; } + public bool Italic { get; set; } + public bool Strike { get; set; } + public bool Code { get; set; } +} + +/// Where a link points. +public sealed class LinkTarget +{ + /// external (absolute URL with a scheme), relative + /// (scheme-less relative reference), or anchor (internal target: a + /// heading anchor or an anchor inline). + public string Kind { get; set; } = ""; + /// The URL, relative reference, or anchor id. + public string Value { get; set; } = ""; +} + +/// Where an image's bytes live. +public sealed class ImageSource +{ + /// external (absolute URL), asset (embedded image, + /// carried in ), or unavailable (no + /// usable source: only the alt text remains). + public string Kind { get; set; } = ""; + /// external. + public string? Url { get; set; } + /// asset: index into . + public int? AssetId { get; set; } +} + +/// The marker family a list uses in the source document. +public sealed class List +{ + /// bullet, decimal, lower_alpha, + /// upper_alpha, lower_roman, or upper_roman. + public string Marker { get; set; } = ""; + /// Ordinal the first item counts from. + public ulong Start { get; set; } + public List Items { get; set; } = new(); +} + +/// One item of a , which may hold nested blocks +/// including further lists. +public sealed class ListItem +{ + public List Blocks { get; set; } = new(); + /// Task-list state, when the item carries a checkbox. + public bool? Checked { get; set; } + /// Literal marker text that overrides the list marker when the + /// source number text cannot be reproduced (composite number text such as + /// 1-a)). + public string? MarkerLabel { get; set; } +} + +/// Canonical table grid: every logical grid position appears exactly +/// once. Content and spans live on the origin slot, and each position a span +/// covers holds a covered slot pointing back at that origin. +public sealed class Table +{ + public List> Grid { get; set; } = new(); + /// Number of leading rows that are header rows (0 = no header). + public int HeaderRows { get; set; } + /// data (a real data table) or layout (layout + /// scaffolding: text boxes, positioning tables). + public string Kind { get; set; } = ""; +} + +/// One position in a : either a cell or the +/// shadow of one. +public sealed class CellSlot +{ + /// origin or covered. + public string Kind { get; set; } = ""; + /// origin. + public Cell? Cell { get; set; } + /// covered: row of the origin this position belongs to. + public int? OriginRow { get; set; } + /// covered: column of the origin this position belongs to. + public int? OriginCol { get; set; } +} + +/// A table cell and the extent it spans. +public sealed class Cell +{ + public List Blocks { get; set; } = new(); + /// Columns covered, at least 1. + public uint ColSpan { get; set; } + /// Rows covered, at least 1. + public uint RowSpan { get; set; } +} + +/// Footnote or endnote body, referenced from text by an +/// . id is document-scoped. +public sealed class Note +{ + public string Id { get; set; } = ""; + /// footnote or endnote. + public string Kind { get; set; } = ""; + public List Blocks { get; set; } = new(); +} + +/// An embedded binary asset (image, object payload). Bytes are always +/// retained, so a document stays self-contained. +public sealed class Asset +{ + /// Index into , as referenced by an + /// image source. + public int Id { get; set; } + /// MIME type, e.g. image/png. + public string MediaType { get; set; } = ""; + /// Package part or stream the asset came from, for provenance. + public string OriginPart { get; set; } = ""; + /// The payload, exactly as stored in the source. + public byte[] Data { get; set; } = Array.Empty(); +} + +internal static class Json +{ + public static readonly JsonSerializerOptions Options = new() + { + // The Rust serializers emit snake_case fields (col_span, media_type, ...). + PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower, + PropertyNameCaseInsensitive = false, + // Optional fields are omited when unset. + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + }; +} \ No newline at end of file diff --git a/dotnet/Firecrawl.Anydoc.Shared/Types.cs b/dotnet/Firecrawl.Anydoc.Shared/Types.cs new file mode 100644 index 00000000..0b5be0bf --- /dev/null +++ b/dotnet/Firecrawl.Anydoc.Shared/Types.cs @@ -0,0 +1,91 @@ +namespace Firecrawl.Anydoc; + +/// Input format, named after the extension that identifies it. +/// Container variants that share a parser (`.docm`, `.xlsm`, `.ppsx`, ...) map +/// onto these via or +/// . +public enum Format +{ + Doc = 0, + Docx = 1, + Odt = 2, + /// Converted with pdf-inspector, which emits Markdown directly; + /// is unsupported for PDFs. Scanned or + /// image-only PDFs (needing OCR) error as unsupported. + Pdf = 3, + Ppt = 4, + Pptx = 5, + Rtf = 6, + Epub = 7, + Excel = 8, + Ods = 9, + Odp = 10, + Csv = 11, +} + +/// The kind of a failed conversion, matching the stable +/// error.code() strings the Rust engine and the other bindings +/// publish. +public enum ConvertErrorKind +{ + /// The format is unknown, or cannot be converted at all: a scanned + /// or image-only PDF needs OCR, which anydoc does not do. + Unsupported, + /// The document is structurally unusable; no meaningful content + /// could be extracted. + Malformed, + /// The document is encrypted or password-protected. + Encrypted, + /// A fixed safety limit was crossed (decompression, nesting depth, + /// node count, repeat expansion, or retained asset bytes). + ResourceLimit, + /// A part required for any meaningful output is absent. + MissingPart, + /// The input could not be read. + Io, + /// An error kind this binding version does not know yet. + Unknown, +} + +/// Thrown when meaningful conversion is impossible. +/// names the failure the same way callers of the Node and Python bindings +/// branch on. +public sealed class AnydocException : Exception +{ + public AnydocException(ConvertErrorKind kind, string? message) : base(message) + { + Kind = kind; + Code = kind switch + { + ConvertErrorKind.Unsupported => "unsupported", + ConvertErrorKind.Malformed => "malformed", + ConvertErrorKind.Encrypted => "encrypted", + ConvertErrorKind.ResourceLimit => "resourceLimit", + ConvertErrorKind.MissingPart => "missingPart", + ConvertErrorKind.Io => "io", + _ => "unknown", + }; + } + + /// The kind of failure. + public ConvertErrorKind Kind { get; } + + /// Stable, machine-readable name for the kind: what callers branch + /// on, identical to the `code` the Node bindings put on their errors. + public string Code { get; } + + internal static AnydocException From(string? code, string? message) + { + ConvertErrorKind kind = code switch + { + "unsupported" => ConvertErrorKind.Unsupported, + "malformed" => ConvertErrorKind.Malformed, + "encrypted" => ConvertErrorKind.Encrypted, + "resourceLimit" => ConvertErrorKind.ResourceLimit, + "missingPart" => ConvertErrorKind.MissingPart, + "io" => ConvertErrorKind.Io, + _ => ConvertErrorKind.Unknown, + }; + return new AnydocException(kind, message); + } +} \ No newline at end of file diff --git a/dotnet/Firecrawl.Anydoc.Tests/AnydocTests.cs b/dotnet/Firecrawl.Anydoc.Tests/AnydocTests.cs new file mode 100644 index 00000000..ce00edcd --- /dev/null +++ b/dotnet/Firecrawl.Anydoc.Tests/AnydocTests.cs @@ -0,0 +1,149 @@ +using System.Text; +using Firecrawl.Anydoc.Model; +using Xunit; + +namespace Firecrawl.Anydoc.Tests; + +/// Smoke test: the bindings load and every entry point round-trips a +/// fixture, mirroring the Node and Python test suites. +public class AnydocTests +{ + private readonly AnydocConverter converter = new(); + + private static string Fixture(params string[] path) => + Path.Combine(new[] { AppContext.BaseDirectory, "fixtures" }.Concat(path).ToArray()); + + private static readonly string Outline = Fixture("docx", "handmade-outline.docx"); + private static readonly string Rich = Fixture("docx", "handmade-rich.docx"); + private static readonly string Csv = Fixture("csv", "sheet.csv"); + private static readonly string Encrypted = Fixture("malformed", "encrypted--errors.odt"); + private static readonly string Zipbomb = Fixture("abuse", "zipbomb--errors.docx"); + + [Fact] + public void ToMarkdown_detects_the_format_from_the_file_content() + { + string markdown = converter.ToMarkdown(Outline); + Assert.Matches("(?m)^# ", markdown); + } + + [Fact] + public void ToMarkdownBytes_converts_in_memory_and_round_trips_unicode() + { + string markdown = converter.ToMarkdownBytes(File.ReadAllBytes(Rich), Format.Docx); + Assert.Contains("| Quarter | Widgets |", markdown); + } + + [Fact] + public void ToMarkdownBytes_detects_the_format_when_none_is_named() + { + string markdown = converter.ToMarkdownBytes(File.ReadAllBytes(Rich)); + Assert.Contains("| Quarter | Widgets |", markdown); + // CSV carries no signature, so it has to be named. + var csvBytes = File.ReadAllBytes(Csv); + var exception = Assert.Throws(() => converter.ToMarkdownBytes(csvBytes)); + Assert.Equal(ConvertErrorKind.Unsupported, exception.Kind); + Assert.Matches("unrecognized file content", exception.Message); + Assert.Contains("| --- |", converter.ToMarkdownBytes(csvBytes, Format.Csv)); + } + + [Fact] + public void ToDocument_exposes_the_document_model() + { + Document document = converter.ToDocument(File.ReadAllBytes(Outline)); + Block heading = document.Blocks.First(b => b.Kind == "heading"); + Assert.InRange(heading.Level!.Value, 1, 6); + Assert.IsType(heading.Content![0].Text); + Assert.Equal("text", heading.Content[0].Kind); + Assert.IsType(heading.Content[0].Style!.Bold); + } + + [Fact] + public void ToDocument_carries_embedded_assets_as_bytes() + { + Document document = converter.ToDocument(File.ReadAllBytes(Rich)); + Asset image = document.Assets.Single(a => a.MediaType == "image/png"); + Assert.NotEmpty(image.Data); + Assert.Equal(image.Id, document.Assets.IndexOf(image)); + } + + [Theory] + [InlineData(".pptm", Format.Pptx)] + [InlineData("xls", Format.Excel)] + public void DetectFormatByExtension_maps_container_variants(string extension, Format expected) + { + Assert.Equal(expected, converter.DetectFormatByExtension(extension)); + } + + [Fact] + public void DetectFormat_reads_content_extension_and_path() + { + Assert.Equal(Format.Docx, converter.DetectFormat(File.ReadAllBytes(Rich))); + // CSV carries no signature: only the extension names it. + Assert.Null(converter.DetectFormat(File.ReadAllBytes(Csv))); + Assert.Equal(Format.Odt, converter.DetectFormatByPath("report.odt")); + Assert.Null(converter.DetectFormatByPath("report.unknown")); + } + + [Fact] + public void Conversion_errors_throw_the_kind_that_names_the_failure() + { + // Nothing about these bytes is a package part (Malformed). + var malformed = Assert.Throws(() => + converter.ToMarkdownBytes(Encoding.UTF8.GetBytes("not a document"), Format.Docx)); + Assert.Equal(ConvertErrorKind.Malformed, malformed.Kind); + + var unsupported = Assert.Throws(() => + converter.ToMarkdownBytes(File.ReadAllBytes(Csv))); + Assert.Equal(ConvertErrorKind.Unsupported, unsupported.Kind); + + var encrypted = Assert.Throws(() => + converter.ToMarkdownBytes(File.ReadAllBytes(Encrypted), Format.Odt)); + Assert.Equal(ConvertErrorKind.Encrypted, encrypted.Kind); + + var limit = Assert.Throws(() => + converter.ToMarkdownBytes(File.ReadAllBytes(Zipbomb), Format.Docx)); + Assert.Equal(ConvertErrorKind.ResourceLimit, limit.Kind); + // The message carries the limit name, matching the Rust detail. + Assert.Contains("max_entry_bytes", limit.Message); + + // A readable package carrying none of the parts a docx is made of. + Assert.Equal(ConvertErrorKind.MissingPart, + Assert.Throws(() => ToMarkdownOfEmptyDocx()).Kind); + } + + [Fact] + public async Task Async_variants_match_the_sync_results() + { + byte[] outline = File.ReadAllBytes(Outline); + byte[] rich = File.ReadAllBytes(Rich); + + Assert.Equal(converter.ToMarkdown(Outline), await converter.ToMarkdownAsync(Outline)); + Assert.Equal(converter.ToMarkdownBytes(outline), await converter.ToMarkdownBytesAsync(outline)); + Assert.Equal( + converter.ToMarkdownBytes(rich, Format.Docx), + await converter.ToMarkdownBytesAsync(rich, Format.Docx)); + + Document document = await converter.ToDocumentAsync(outline); + Assert.Contains(document.Blocks, b => b.Kind == "heading"); + } + + [Fact] + public void An_unreadable_file_raises_the_io_kind() + { + var exception = Assert.Throws(() => converter.ToMarkdown("no-such-file.docx")); + Assert.Equal(ConvertErrorKind.Io, exception.Kind); + } + + /// A ZIP package without the parts a docx needs -> missingPart. + private static string ToMarkdownOfEmptyDocx() + { + using var package = new MemoryStream(); + using (var archive = new System.IO.Compression.ZipArchive(package, System.IO.Compression.ZipArchiveMode.Create, true)) + { + var entry = archive.CreateEntry("[Content_Types].xml"); + using var writer = new StreamWriter(entry.Open()); + writer.Write(""); + } + return new AnydocConverter().ToMarkdownBytes(package.ToArray(), Format.Docx); + } +} \ No newline at end of file diff --git a/dotnet/Firecrawl.Anydoc.Tests/Firecrawl.Anydoc.Tests.csproj b/dotnet/Firecrawl.Anydoc.Tests/Firecrawl.Anydoc.Tests.csproj new file mode 100644 index 00000000..737de469 --- /dev/null +++ b/dotnet/Firecrawl.Anydoc.Tests/Firecrawl.Anydoc.Tests.csproj @@ -0,0 +1,31 @@ + + + + net8.0 + enable + enable + latest + false + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/dotnet/Firecrawl.Anydoc.Wasm.Tests/Firecrawl.Anydoc.Wasm.Tests.csproj b/dotnet/Firecrawl.Anydoc.Wasm.Tests/Firecrawl.Anydoc.Wasm.Tests.csproj new file mode 100644 index 00000000..faa1e771 --- /dev/null +++ b/dotnet/Firecrawl.Anydoc.Wasm.Tests/Firecrawl.Anydoc.Wasm.Tests.csproj @@ -0,0 +1,22 @@ + + + + net8.0 + enable + enable + latest + false + + + + + + + + + + + + + + \ No newline at end of file diff --git a/dotnet/Firecrawl.Anydoc.Wasm.Tests/WasmAnydocTests.cs b/dotnet/Firecrawl.Anydoc.Wasm.Tests/WasmAnydocTests.cs new file mode 100644 index 00000000..4cf8eeb8 --- /dev/null +++ b/dotnet/Firecrawl.Anydoc.Wasm.Tests/WasmAnydocTests.cs @@ -0,0 +1,250 @@ +using System.Collections.Concurrent; +using Firecrawl.Anydoc; +using Firecrawl.Anydoc.Model; +using Microsoft.JSInterop; +using Xunit; + +namespace Firecrawl.Anydoc.Wasm.Tests; + +/// Exercises the managed Wasm binding against a fake JS runtime: a +/// browser cannot run under `dotnet test`, so the interop layer is stubbed with +/// a module returning canned results. This validates the C# glue (arg +/// marshalling, enum-name mapping, error mapping) independent of the Engine. +public class AnydocWasmTests +{ + private static readonly byte[] Docx = new byte[] { 0x50, 0x4B, 0x03 }; + + [Fact] + public async Task ToMarkdownBytesAsync_pass_bytes_and_returns_the_markdown() + { + var js = FakeJs.Empty(); + string? seenFormat = null; + js.Module.On("toMarkdownBytes", args => + { + seenFormat = args[1] as string; + return ""; + }); + await using var converter = new AnydocConverter(js); + + string markdown = await converter.ToMarkdownBytesAsync(Docx, Format.Docx); + + Assert.Equal("", markdown); + Assert.Equal("docx", seenFormat); // the enum is marshalled by its wasm name + } + + [Fact] + public async Task ToMarkdownBytesAsync_leaves_the_format_undefined_when_detecting() + { + var js = FakeJs.Empty(); + js.Module.On("toMarkdownBytes", args => ""); + await using var converter = new AnydocConverter(js); + + string markdown = await converter.ToMarkdownBytesAsync(Docx); + + Assert.Equal("", markdown); + } + + [Fact] + public async Task Detecting_the_format_maps_the_wasm_name_back_to_the_enum() + { + foreach ((string name, Format expected) in new[] + { + ("docx", Format.Docx), + ("odt", Format.Odt), + ("pdf", Format.Pdf), + ("xlsx", Format.Excel), + ("csv", Format.Csv), + }) + { + var js = FakeJs.Returning("formatFromBytes", name); + await using var converter = new AnydocConverter(js); + Assert.Equal(expected, await converter.DetectFormatAsync(Docx)); + } + } + + [Fact] + public async Task Detect_the_format_returns_null_when_nothing_matches() + { + var js = FakeJs.Returning("formatFromBytes", null); + await using var converter = new AnydocConverter(js); + Assert.Null(await converter.DetectFormatAsync(Docx)); + } + + [Fact] + public async Task Detecting_by_extension_and_path_use_their_own_entry_points() + { + var ext = FakeJs.Returning("formatFromExtension", "xlsx"); + await using var cExt = new AnydocConverter(ext); + Assert.Equal(Format.Excel, await cExt.DetectFormatByExtensionAsync(".xlsx")); + + var path = FakeJs.Returning("formatFromPath", "pdf"); + await using var cPath = new AnydocConverter(path); + Assert.Equal(Format.Pdf, await cPath.DetectFormatByPathAsync("report.pdf")); + } + + [Fact] + public async Task ToDocumentAsync_returns_the_deserialized_model() + { + var js = FakeJs.Returning("toDocument", new Document + { + Blocks = + { + new Block + { + Kind = "heading", + Level = 1, + Content = new List { new Inline { Kind = "text" } }, + }, + }, + }); + await using var converter = new AnydocConverter(js); + + Document document = await converter.ToDocumentAsync(Docx); + + Assert.Equal("heading", Assert.Single(document.Blocks).Kind); + Assert.Equal("text", Assert.Single(Assert.Single(document.Blocks).Content!).Kind); + } + + [Fact] + public async Task A_js_engine_error_becomes_an_AnydocException_with_the_right_kind() + { + var js = FakeJs.Throwing("toMarkdownBytes", new JSException("anydoc:encrypted:the file is password-protected")); + await using var converter = new AnydocConverter(js); + + var exception = await Assert.ThrowsAsync(() => converter.ToMarkdownBytesAsync(Docx)); + + Assert.Equal(ConvertErrorKind.Encrypted, exception.Kind); + Assert.Equal("encrypted", exception.Code); + } + + [Fact] + public async Task A_js_error_with_a_future_code_maps_to_unknown() + { + var js = FakeJs.Throwing("toMarkdownBytes", new JSException("anydoc:negotiator hatchback:surprise!")); + await using var converter = new AnydocConverter(js); + + var exception = await Assert.ThrowsAsync(() => converter.ToMarkdownBytesAsync(Docx)); + + Assert.Equal(ConvertErrorKind.Unknown, exception.Kind); + } + + [Fact] + public void The_sync_members_require_the_async_variant() + { + var js = FakeJs.Empty(); + var converter = new AnydocConverter(js); + + foreach (Action act in new Action[] + { + () => converter.DetectFormat(Docx), + () => converter.DetectFormatByExtension(".docx"), + () => converter.DetectFormatByPath("a.docx"), + () => converter.ToMarkdownBytes(Docx), + () => converter.ToMarkdownBytes(Docx, Format.Docx), + () => converter.ToDocument(Docx), + }) + { + Assert.Throws(act); + } + } + + [Fact] + public async Task Path_conversion_is_unsupported_and_its_message_points_at_the_byte_api() + { + await using var converter = new AnydocConverter(FakeJs.Empty()); + + var exception = await Assert.ThrowsAsync( + () => converter.ToMarkdownAsync("a.docx")); + + Assert.Contains("ToMarkdownBytesAsync", exception.Message); + } + + [Fact] + public async Task Reusing_one_converter_reuses_a_single_imported_module() + { + var js = FakeJs.Empty(); + js.Module.On("toMarkdownBytes", args => ""); + await using var converter = new AnydocConverter(js); + + await converter.ToMarkdownBytesAsync(Docx); + await converter.ToMarkdownBytesAsync(Docx); + + Assert.Equal(1, js.ImportCount); + } +} + +/// A minimal IJSRuntime that returns a hand-driven module for the +/// `import` call and routes every module invocation to a registered handler. +internal sealed class FakeJs : IJSRuntime +{ + public FakeModule Module { get; } = new(); + public int ImportCount { get; private set; } + + public ValueTask InvokeAsync(string identifier, object?[]? args) => + InvokeAsync(identifier, CancellationToken.None, args); + + public ValueTask InvokeAsync( + string identifier, CancellationToken cancellationToken, object?[]? args) + { + if (identifier == "import") + { + ImportCount++; + var task = Task.FromResult((TValue)(object)Module); + return new ValueTask(task); + } + throw new InvalidOperationException($"unexpected js identifier {identifier}"); + } + + public static FakeJs Empty() => new(); + + public static FakeJs Returning(string identifier, object? result) + { + var js = new FakeJs(); + js.Module.On(identifier, _ => result); + return js; + } + + public static FakeJs Throwing(string identifier, JSException exception) + { + var js = new FakeJs(); + js.Module.ThrowOn(identifier, exception); + return js; + } +} + +internal sealed class FakeModule : IJSObjectReference +{ + private readonly ConcurrentDictionary> _handlers = new(); + private readonly ConcurrentDictionary _throwing = new(); + + public void On(string identifier, Func handler) => _handlers[identifier] = handler; + public void ThrowOn(string identifier, JSException exception) => _throwing[identifier] = exception; + + public ValueTask InvokeAsync(string identifier, object?[]? args) => + InvokeAsync(identifier, CancellationToken.None, args); + + public ValueTask InvokeAsync( + string identifier, CancellationToken cancellationToken, object?[]? args) + { + try + { + if (_throwing.TryGetValue(identifier, out JSException? thrown)) + { + throw thrown; + } + if (_handlers.TryGetValue(identifier, out var handler)) + { + object? result = handler(args ?? Array.Empty()); + var task = Task.FromResult((TValue)result!); + return new ValueTask(task); + } + } + catch (JSException) + { + throw; + } + throw new InvalidOperationException($"no handler registered for {identifier}"); + } + + public ValueTask DisposeAsync() => ValueTask.CompletedTask; +} \ No newline at end of file diff --git a/dotnet/Firecrawl.Anydoc.Wasm/AnydocConverter.cs b/dotnet/Firecrawl.Anydoc.Wasm/AnydocConverter.cs new file mode 100644 index 00000000..a987b7e8 --- /dev/null +++ b/dotnet/Firecrawl.Anydoc.Wasm/AnydocConverter.cs @@ -0,0 +1,210 @@ +using Firecrawl.Anydoc.Model; +using Microsoft.JSInterop; + +namespace Firecrawl.Anydoc; + +/// Converts documents to GitHub-Flavored Markdown inside a Blazor +/// WebAssembly app. Backed by the anydoc Rust engine compiled to WebAssembly +/// and reached through JavaScript interop, so no native platform library is +/// loaded. +/// +/// Blazor WASM has no synchronous JS interop, so every entry point is +/// async (`DetectFormatAsync`, `ToMarkdownBytesAsync`, ...). The sync-named +/// members exist for API parity with the native Firecrawl.Anydoc binding +/// but throw ; a browser has no +/// filesystem either, so path-based conversion is unsupported. +public sealed class AnydocConverter : IAsyncDisposable +{ + private const string ModulePath = "./_content/Firecrawl.Anydoc.Wasm/anydoc-wasm.js"; + + private readonly IJSRuntime _js; + private IJSObjectReference? _module; + + /// Create a converter for the app's JS runtime. Safe to reuse + /// across conversions; the WASM module is instantiated once, lazily. + public AnydocConverter(IJSRuntime js) => _js = js; + + /// Detect the format from the content itself: the signature and + /// identity each container specification designates (PDF header, RTF open + /// group, OLE stream names, ZIP package mimetype/content types). Plain-text + /// formats (CSV) carry no signature and return ; so + /// does anything unrecognized. + public async Task DetectFormatAsync(byte[] bytes) + { + IJSObjectReference module = await ModuleAsync(); + string? name = await module.InvokeAsync("formatFromBytes", bytes); + return FormatFromWasmName(name); + } + + /// The format an extension names, with or without a leading + /// dot. + public async Task DetectFormatByExtensionAsync(string extension) + { + IJSObjectReference module = await ModuleAsync(); + string? name = await module.InvokeAsync("formatFromExtension", extension); + return FormatFromWasmName(name); + } + + /// The format a path's extension names. + public async Task DetectFormatByPathAsync(string path) + { + IJSObjectReference module = await ModuleAsync(); + string? name = await module.InvokeAsync("formatFromPath", path); + return FormatFromWasmName(name); + } + + /// Convert an in-memory document to Markdown. Without a format, it + /// is detected from the content, which signature-less formats (CSV) have to + /// name explicitly. + public async Task ToMarkdownBytesAsync(byte[] data) => + await ToMarkdownBytesCoreAsync(data, format: null); + + /// Convert an in-memory document to Markdown, naming the format + /// explicitly. + public async Task ToMarkdownBytesAsync(byte[] data, Format format) => + await ToMarkdownBytesCoreAsync(data, format); + + /// Parse an in-memory document into the document model, which also + /// carries the embedded assets. The format is detected from the content. + /// Unsupported for : PDF conversion produces + /// Markdown directly and has no document-model form; use + /// . + public async Task ToDocumentAsync(byte[] bytes) + { + IJSObjectReference module = await ModuleAsync(); + try + { + return await module.InvokeAsync("toDocument", bytes, (string?)null); + } + catch (JSException e) + { + throw FromJsException(e); + } + } + + /// Not supported in the browser: a WebAssembly host has no + /// filesystem to read the file from. Fetch the bytes first and use + /// . + public Task ToMarkdownAsync(string path) => + throw new PlatformNotSupportedException( + "The WebAssembly binding has no filesystem; fetch the file's bytes and call ToMarkdownBytesAsync instead."); + + /// + /// always: Blazor WASM has + /// no synchronous JS interop; use . + public Format? DetectFormat(ReadOnlySpan bytes) => + throw SyncUnsupported(); + + /// + public Format? DetectFormatByExtension(string extension) => + throw SyncUnsupported(); + + /// + public Format? DetectFormatByPath(string path) => + throw SyncUnsupported(); + + /// + public string ToMarkdownBytes(ReadOnlySpan data) => + throw SyncUnsupported(); + + /// + public string ToMarkdownBytes(ReadOnlySpan data, Format format) => + throw SyncUnsupported(); + + /// + public Document ToDocument(ReadOnlySpan bytes) => + throw SyncUnsupported(); + + private async Task ToMarkdownBytesCoreAsync(byte[] data, Format? format) + { + IJSObjectReference module = await ModuleAsync(); + string? name = FormatToWasmName(format); + try + { + return await module.InvokeAsync("toMarkdownBytes", data, name); + } + catch (JSException e) + { + throw FromJsException(e); + } + } + + private async Task ModuleAsync() + { + if (_module is null) + { + _module = await _js.InvokeAsync("import", ModulePath); + } + return _module; + } + + private static PlatformNotSupportedException SyncUnsupported() => + new( + "Blazor WebAssembly has no synchronous JS interop; use the Async variant " + + "(for example ToMarkdownBytesAsync instead of ToMarkdownBytes)."); + + private static AnydocException FromJsException(JSException e) + { + // The shim re-throws engine errors as `anydoc::`. + const string Prefix = "anydoc:"; + string text = e.Message; + if (text.StartsWith(Prefix, StringComparison.Ordinal)) + { + string rest = text[Prefix.Length..]; + int colon = rest.IndexOf(':'); + if (colon > 0) + { + return AnydocException.From(rest[..colon], rest[(colon + 1)..]); + } + } + return AnydocException.From("unknown", text); + } + + /// The name the wasm-bindgen enum uses for a format (the JS + /// string enum is what crosses the boundary, not the ordinal). + private static string? FormatToWasmName(Format? format) => format switch + { + null => null, + Format.Doc => "doc", + Format.Docx => "docx", + Format.Odt => "odt", + Format.Pdf => "pdf", + Format.Ppt => "ppt", + Format.Pptx => "pptx", + Format.Rtf => "rtf", + Format.Epub => "epub", + Format.Excel => "xlsx", + Format.Ods => "ods", + Format.Odp => "odp", + Format.Csv => "csv", + _ => null, + }; + + private static Format? FormatFromWasmName(string? name) => name switch + { + "doc" => Format.Doc, + "docx" => Format.Docx, + "odt" => Format.Odt, + "pdf" => Format.Pdf, + "ppt" => Format.Ppt, + "pptx" => Format.Pptx, + "rtf" => Format.Rtf, + "epub" => Format.Epub, + "xlsx" => Format.Excel, + "ods" => Format.Ods, + "odp" => Format.Odp, + "csv" => Format.Csv, + _ => null, + }; + + /// Release the JavaScript module reference. Idempotent; the + /// converter is unusable afterwards. + public async ValueTask DisposeAsync() + { + if (_module is not null) + { + await _module.DisposeAsync(); + _module = null; + } + } +} \ No newline at end of file diff --git a/dotnet/Firecrawl.Anydoc.Wasm/Firecrawl.Anydoc.Wasm.csproj b/dotnet/Firecrawl.Anydoc.Wasm/Firecrawl.Anydoc.Wasm.csproj new file mode 100644 index 00000000..e6447b6b --- /dev/null +++ b/dotnet/Firecrawl.Anydoc.Wasm/Firecrawl.Anydoc.Wasm.csproj @@ -0,0 +1,49 @@ + + + + net8.0 + enable + enable + latest + + Firecrawl.Anydoc.Wasm + Firecrawl.Anydoc + + 0.1.7 + WebAssembly binding for anydoc: convert documents (doc, docx, odt, rtf, epub, pdf, presentations, spreadsheets, csv) to GitHub-Flavored Markdown inside a Blazor WebAssembly app, without any native platform library. + Firecrawl.Anydoc.Wasm + Firecrawl;Junian Triajianto + Firecrawl;junian + MIT + https://github.com/firecrawl/anydoc + https://github.com/firecrawl/anydoc + markdown; docx; pptx; xlsx; epub; pdf; converter; blazor; wasm; document + README.md + false + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/dotnet/Firecrawl.Anydoc.Wasm/wwwroot/anydoc-wasm.js b/dotnet/Firecrawl.Anydoc.Wasm/wwwroot/anydoc-wasm.js new file mode 100644 index 00000000..de014028 --- /dev/null +++ b/dotnet/Firecrawl.Anydoc.Wasm/wwwroot/anydoc-wasm.js @@ -0,0 +1,65 @@ +// Firecrawl.Anydoc.Wasm interop shim. +// +// Loaded once from managed code via Blazor's `import` helper. It lazy-loads +// the wasm-bindgen glue produced from the anydoc `wasm/` crate, instantiates +// the engine, and re-exposes the conversion entry points as plain functions +// that the C# AnydocConverter calls through IJSRuntime. +// +// Conversion errors thrown by the engine (a JS Error with a `code` property) +// are re-thrown with a parseable `anydoc::` message so the +// managed side can reconstruct an AnydocException with the right Kind. + +let gluePromise = null; + +function glue() { + if (!gluePromise) { + gluePromise = (async () => { + const m = await import("./anydoc_wasm.js"); + // The wasm binary sits next to this shim; resolve it against + // import.meta.url, not the app's base path. + const wasmUrl = new URL("./anydoc_wasm_bg.wasm", import.meta.url); + await m.default(wasmUrl); + return m; + })(); + } + return gluePromise; +} + +function rethrowConversion(error) { + const code = (error && error.code) || "unknown"; + const message = (error && error.message) || String(error); + throw new Error(`anydoc:${code}:${message}`); +} + +export async function formatFromBytes(bytes) { + const m = await glue(); + return m.formatFromBytes(bytes); +} + +export async function formatFromExtension(extension) { + const m = await glue(); + return m.formatFromExtension(extension); +} + +export async function formatFromPath(path) { + const m = await glue(); + return m.formatFromPath(path); +} + +export async function toMarkdownBytes(bytes, format) { + const m = await glue(); + try { + return m.toMarkdownBytes(bytes, format || undefined); + } catch (error) { + throw rethrowConversion(error); + } +} + +export async function toDocument(bytes, format) { + const m = await glue(); + try { + return m.toDocument(bytes, format || undefined); + } catch (error) { + throw rethrowConversion(error); + } +} \ No newline at end of file diff --git a/dotnet/Firecrawl.Anydoc.slnx b/dotnet/Firecrawl.Anydoc.slnx new file mode 100644 index 00000000..9741466a --- /dev/null +++ b/dotnet/Firecrawl.Anydoc.slnx @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + \ No newline at end of file diff --git a/dotnet/Firecrawl.Anydoc/AnydocConverter.cs b/dotnet/Firecrawl.Anydoc/AnydocConverter.cs new file mode 100644 index 00000000..741fc29a --- /dev/null +++ b/dotnet/Firecrawl.Anydoc/AnydocConverter.cs @@ -0,0 +1,177 @@ +using System.Runtime.InteropServices; +using System.Text; +using Firecrawl.Anydoc.Native; +using Firecrawl.Anydoc.Model; + +namespace Firecrawl.Anydoc; + +/// Converts documents to GitHub-Flavored Markdown, backed by the anydoc +/// Rust engine loaded as a native library. Safe to share across threads and +/// reuse: the underlying native calls are stateless, so instances hold no +/// process-lifetime resources and do not need to be disposed. +public sealed class AnydocConverter +{ + private const int NoFormat = -1; + + /// Detect the format from the content itself: the signature and + /// identity each container specification designates (PDF header, RTF open + /// group, OLE stream names, ZIP package mimetype/content types). Plain-text + /// formats (CSV) carry no signature and return ; so + /// does anything unrecognized. + public unsafe Format? DetectFormat(ReadOnlySpan bytes) + { + unsafe + { + fixed (byte* p = bytes) + { + return FormatFromCode(AnydocNative.anydoc_format_from_bytes(p, (nuint)bytes.Length)); + } + } + } + + /// The format an extension names, with or without a leading + /// dot. + public unsafe Format? DetectFormatByExtension(string extension) + { + byte[] text = Encoding.UTF8.GetBytes(extension); + unsafe + { + fixed (byte* p = text) + { + return FormatFromCode(AnydocNative.anydoc_format_from_extension(p, (nuint)text.Length)); + } + } + } + + /// The format a path's extension names. + public unsafe Format? DetectFormatByPath(string path) + { + byte[] text = Encoding.UTF8.GetBytes(path); + unsafe + { + fixed (byte* p = text) + { + return FormatFromCode(AnydocNative.anydoc_format_from_path(p, (nuint)text.Length)); + } + } + } + + /// Convert a document file to Markdown. The format is detected from + /// the file content; the extension is the fallback for signature-less + /// formats (CSV) and unrecognizable containers. + /// when conversion is impossible; a file + /// that cannot be read has . + public unsafe string ToMarkdown(string path) + { + byte[] text = Encoding.UTF8.GetBytes(path); + AnyResult result = default; + unsafe + { + fixed (byte* p = text) + { + AnydocNative.anydoc_to_markdown_path(p, (nuint)text.Length, &result); + } + } + byte[] data = TakeResult(&result); + return Encoding.UTF8.GetString(data); + } + + /// Asynchronous , offloaded to the + /// thread pool so the calling thread is not blocked by the native + /// conversion. + public Task ToMarkdownAsync(string path) => Task.Run(() => ToMarkdown(path)); + + /// Convert an in-memory document to Markdown. Without a format, it is + /// detected from the content, which signature-less formats (CSV) have to + /// name explicitly. + public string ToMarkdownBytes(ReadOnlySpan data) => ToMarkdownBytesCore(data, null); + + /// Convert an in-memory document to Markdown, naming the format + /// explicitly. + public string ToMarkdownBytes(ReadOnlySpan data, Format format) => + ToMarkdownBytesCore(data, format); + + /// Asynchronous . + public Task ToMarkdownBytesAsync(ReadOnlySpan data) + { + byte[] copy = data.ToArray(); + return Task.Run(() => ToMarkdownBytes(copy)); + } + + /// Asynchronous . + public Task ToMarkdownBytesAsync(ReadOnlySpan data, Format format) + { + byte[] copy = data.ToArray(); + return Task.Run(() => ToMarkdownBytes(copy, format)); + } + + /// Parse an in-memory document into the document model, which also + /// carries the embedded assets. The format is detected from the content. + /// Unsupported for : PDF conversion produces + /// Markdown directly and has no document-model form; use + /// . + public unsafe Document ToDocument(ReadOnlySpan bytes) + { + AnyResult result = default; + unsafe + { + fixed (byte* p = bytes) + { + AnydocNative.anydoc_to_document(p, (nuint)bytes.Length, NoFormat, &result); + } + } + byte[] data = TakeResult(&result); + return Document.FromJson(Encoding.UTF8.GetString(data)); + } + + /// Asynchronous . + public Task ToDocumentAsync(ReadOnlySpan bytes) + { + byte[] copy = bytes.ToArray(); + return Task.Run(() => ToDocument(copy)); + } + + private static unsafe string ToMarkdownBytesCore(ReadOnlySpan data, Format? format) + { + AnyResult result = default; + unsafe + { + fixed (byte* p = data) + { + AnydocNative.anydoc_to_markdown_bytes(p, (nuint)data.Length, CodeFor(format), &result); + } + } + byte[] bytes = TakeResult(&result); + return Encoding.UTF8.GetString(bytes); + } + + private static int CodeFor(Format? format) => format is null ? NoFormat : (int)format; + + private static Format? FormatFromCode(int code) => code < 0 ? null : (Format)code; + + /// Turn a filled into managed + /// bytes, throwing the matching when the + /// conversion failed, and always releasing the Rust-side allocation. + private static unsafe byte[] TakeResult(AnyResult* result) + { + try + { + if (!result->ok) + { + string? code = Marshal.PtrToStringUTF8((nint)result->error_code); + string? message = Marshal.PtrToStringUTF8((nint)result->error_message); + throw AnydocException.From(code, message); + } + byte[] data = new byte[(int)result->data.len]; + if (data.Length > 0) + { + Marshal.Copy((nint)result->data.ptr, data, 0, data.Length); + } + return data; + } + finally + { + AnydocNative.anydoc_free_result(result); + } + } +} diff --git a/dotnet/Firecrawl.Anydoc/Firecrawl.Anydoc.csproj b/dotnet/Firecrawl.Anydoc/Firecrawl.Anydoc.csproj new file mode 100644 index 00000000..abbd7820 --- /dev/null +++ b/dotnet/Firecrawl.Anydoc/Firecrawl.Anydoc.csproj @@ -0,0 +1,53 @@ + + + + net8.0 + enable + enable + true + latest + + Firecrawl.Anydoc + Firecrawl.Anydoc + + 0.1.7 + Convert documents (doc, docx, odt, rtf, epub, pdf, presentations, spreadsheets, csv) to GitHub-Flavored Markdown, backed by the anydoc Rust engine. + Firecrawl.Anydoc + Firecrawl;Junian Triajianto + Firecrawl;junian + MIT + https://github.com/firecrawl/anydoc + https://github.com/firecrawl/anydoc + markdown; docx; pptx; xlsx; epub; pdf; converter; document + README.md + false + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/dotnet/Firecrawl.Anydoc/NativeMethods.cs b/dotnet/Firecrawl.Anydoc/NativeMethods.cs new file mode 100644 index 00000000..199fc4f2 --- /dev/null +++ b/dotnet/Firecrawl.Anydoc/NativeMethods.cs @@ -0,0 +1,60 @@ +using System.Runtime.InteropServices; + +namespace Firecrawl.Anydoc.Native +{ + /// + /// DllImport surface, extended with the platform-specific loader that maps + /// the logical library name to runtimes/{platform}-{arch}/native/, + /// the layout both NuGet and this project's build output use. + /// + internal static unsafe partial class AnydocNative + { + static AnydocNative() + { + NativeLibrary.SetDllImportResolver(typeof(AnydocNative).Assembly, ResolveLibrary); + } + + private static nint ResolveLibrary(string libraryName, System.Reflection.Assembly assembly, DllImportSearchPath? searchPath) + { + if (libraryName != __DllName) + { + return nint.Zero; + } + + string platform = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "win" + : RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? "osx" + : RuntimeInformation.IsOSPlatform(OSPlatform.Linux) ? "linux" + : throw new PlatformNotSupportedException($"anydoc has no native library for {RuntimeInformation.OSDescription}"); + + string arch = RuntimeInformation.OSArchitecture switch + { + Architecture.X64 => "x64", + Architecture.Arm64 => "arm64", + _ => throw new PlatformNotSupportedException($"anydoc has no native library for {RuntimeInformation.OSArchitecture}"), + }; + + string name = libraryName; + string prefix = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "" : "lib"; + string ext = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? ".dll" + : RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? ".dylib" + : ".so"; + + string relative = $"runtimes/{platform}-{arch}/native/{prefix}{name}{ext}"; + foreach (string baseDir in new[] { AppContext.BaseDirectory, Path.GetDirectoryName(typeof(AnydocNative).Assembly.Location) ?? "" }) + { + if (string.IsNullOrEmpty(baseDir)) + { + continue; + } + string candidate = Path.Combine(baseDir, relative); + if (File.Exists(candidate)) + { + return NativeLibrary.Load(candidate, assembly, searchPath); + } + } + // Last chance: let the default resolution attempt it (NuGet hosts + // native libs on the default search path too). + return NativeLibrary.Load(relative, assembly, searchPath); + } + } +} diff --git a/dotnet/Firecrawl.Anydoc/NativeMethods.g.cs b/dotnet/Firecrawl.Anydoc/NativeMethods.g.cs new file mode 100644 index 00000000..2737ebee --- /dev/null +++ b/dotnet/Firecrawl.Anydoc/NativeMethods.g.cs @@ -0,0 +1,102 @@ +// +// This code is generated by csbindgen. +// DON'T CHANGE THIS DIRECTLY. +// +#pragma warning disable CS8500 +#pragma warning disable CS8981 +using System; +using System.Runtime.InteropServices; + + +namespace Firecrawl.Anydoc.Native +{ + internal static unsafe partial class AnydocNative + { + const string __DllName = "anydoc_dotnet"; + + + + + + /// + /// `format_from_bytes`: detect the format from content. + /// + [DllImport(__DllName, EntryPoint = "anydoc_format_from_bytes", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)] + internal static extern int anydoc_format_from_bytes(byte* bytes, nuint len); + + /// + /// `format_from_extension`: the format a (possibly dot-prefixed) extension names. + /// + [DllImport(__DllName, EntryPoint = "anydoc_format_from_extension", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)] + internal static extern int anydoc_format_from_extension(byte* bytes, nuint len); + + /// + /// `format_from_path`: the format a path's extension names. + /// + [DllImport(__DllName, EntryPoint = "anydoc_format_from_path", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)] + internal static extern int anydoc_format_from_path(byte* bytes, nuint len); + + /// + /// `to_markdown(path)`: convert a file, detecting the format from its content. + /// + [DllImport(__DllName, EntryPoint = "anydoc_to_markdown_path", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)] + internal static extern void anydoc_to_markdown_path(byte* path, nuint len, AnyResult* result); + + /// + /// `to_markdown_bytes`: convert in-memory bytes; `format` of `-1` detects it. + /// + [DllImport(__DllName, EntryPoint = "anydoc_to_markdown_bytes", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)] + internal static extern void anydoc_to_markdown_bytes(byte* bytes, nuint len, int format, AnyResult* result); + + /// + /// `to_document`: parse in-memory bytes into the document model, returned as + /// JSON. Unsupported for PDF, like the other bindings. + /// + [DllImport(__DllName, EntryPoint = "anydoc_to_document", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)] + internal static extern void anydoc_to_document(byte* bytes, nuint len, int format, AnyResult* result); + + /// + /// Release everything `*result` holds: the error strings and the data buffer. + /// + [DllImport(__DllName, EntryPoint = "anydoc_free_result", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)] + internal static extern void anydoc_free_result(AnyResult* result); + + + } + + /// + /// An owned byte buffer; `ptr` is null and `len` zero when empty. + /// + [StructLayout(LayoutKind.Sequential)] + internal unsafe partial struct ByteBuffer + { + public byte* ptr; + public nuint len; + public nuint capacity; + } + + /// + /// The outcome of a conversion. `ok` selects between the payload and the error + /// fields; the caller hands the whole struct back to [`anydoc_free_result`]. + /// + [StructLayout(LayoutKind.Sequential)] + internal unsafe partial struct AnyResult + { + [MarshalAs(UnmanagedType.U1)] public bool ok; + /// + /// Machine-readable kind (`error.code()`), null when `ok`. + /// + public byte* error_code; + /// + /// Human-readable message, null when `ok`. + /// + public byte* error_message; + /// + /// Markdown, or the document model as JSON, when `ok`. + /// + public ByteBuffer data; + } + + + +} diff --git a/dotnet/README.md b/dotnet/README.md new file mode 100644 index 00000000..4a3b7525 --- /dev/null +++ b/dotnet/README.md @@ -0,0 +1,140 @@ +# Firecrawl.Anydoc + +.NET bindings for **anydoc**, the fast Rust library that converts documents +(Word, PowerPoint, Excel, OpenDocument, RTF, EPUB, CSV, and PDF) into clean +GitHub-Flavored Markdown. This package bundles the native engine for every +supported platform, so it installs and runs with no extra setup. + +``` +dotnet add package Firecrawl.Anydoc +``` + +Supported platforms: **macOS, Windows, and Linux** on both **x64** and **arm64**. + +## Usage + +```csharp +using Firecrawl.Anydoc; + +// Construct once and reuse for every conversion. +// Instances are thread-safe and hold no disposable state. +var anydoc = new AnydocConverter(); + +// From a file path (format detected from the content): +string markdown = await anydoc.ToMarkdownAsync("report.docx"); + +// From bytes, with the format detected from the content: +string fromBytes = await anydoc.ToMarkdownBytesAsync(bytes); + +// Or name it, which signature-less formats (CSV) need: +string fromCsv = await anydoc.ToMarkdownBytesAsync(bytes, Format.Csv); + +// Or stop at the document model, which also carries embedded assets: +Document document = await anydoc.ToDocumentAsync(bytes); +``` + +Synchronous variants (`ToMarkdown`, `ToMarkdownBytes`, `ToDocument`) behave the +same way when you are not in an `async` context. + +### Format detection + +```csharp +Format? byBytes = anydoc.DetectFormat(bytes); // content signature +Format? byExt = anydoc.DetectFormatByExtension(".pptm"); // => Format.Pptx +Format? byPath = anydoc.DetectFormatByPath("sheet.csv"); // => Format.Csv +``` + +### Errors + +Failures throw `AnydocException`, whose `Kind` (a `ConvertErrorKind`) names the +reason the same way the Node and Python bindings do: + +```csharp +try +{ + string md = anydoc.ToMarkdown(bytes, Format.Docx); +} +catch (AnydocException e) +{ + switch (e.Kind) + { + case ConvertErrorKind.Unsupported: // scanned PDF, unknown format + case ConvertErrorKind.Malformed: // structurally unusable document + case ConvertErrorKind.Encrypted: // password-protected + case ConvertErrorKind.ResourceLimit:// a fixed safety limit was crossed + case ConvertErrorKind.MissingPart: // a required part is absent + case ConvertErrorKind.Io: // the file could not be read + } +} +``` + +The document model (`Firecrawl.Anydoc.Model`) encodes every block, inline, +list, table grid, note, and embedded asset, mirroring the Node and Python +bindings exactly (`Kind` string discriminants, `NoteId`, `MediaType`, etc.). + +## Blazor WebAssembly: `Firecrawl.Anydoc.Wasm` + +For client-side Blazor there is a sibling package, `Firecrawl.Anydoc.Wasm`, +that runs the same engine compiled to WebAssembly. No native library is +loaded, so a `browser-wasm` publish needs no RID-specific runtime: the engine +ships as static web assets next to the managed wrapper. + +```csharp +using Firecrawl.Anydoc; +using Microsoft.JSInterop; + +// Injected into a Blazor component or service: +var anydoc = new AnydocConverter(jsRuntime); + +// A browser has no filesystem, so conversion always starts from bytes and +// every call is async (Blazor WASM has no synchronous JS interop): +byte[] bytes = await httpClient.GetByteArrayAsync("report.docx"); +string markdown = await anydoc.ToMarkdownBytesAsync(bytes); +Document document = await anydoc.ToDocumentAsync(bytes); + +Format? format = await anydoc.DetectFormatAsync(bytes); +``` + +One converter instance is enough for the whole app: it instantiates the WASM +module lazily and caches it. The API matches the native package — `Format`, +`ConvertErrorKind`, `AnydocException`, and the `Firecrawl.Anydoc.Model` types +are the same shared source — but the sync-named members (`ToMarkdownBytes`, +`ToDocument`, `DetectFormat`, ...) throw `PlatformNotSupportedException` +because Blazor WASM cannot block on JavaScript interop. + +### Building the WebAssembly engine + +The compiled assets are gitignored and produced from the `wasm/` crate: + +```sh +sh dotnet/build-wasm.sh # needs the wasm32-unknown-unknown target and wasm-pack +``` + +This writes `anydoc_wasm.js` (wasm-bindgen glue) and `anydoc_wasm_bg.wasm` +into `Firecrawl.Anydoc.Wasm/wwwroot/`, which the package ships as static web +assets. `wwwroot/anydoc-wasm.js` — the small interop shim the managed wrapper +loads via Blazor's `import` — is checked in and never regenerated. + +## Building + +The C# wrapper is generated from the Rust FFI with +[csbindgen](https://github.com/Cysharp/csbindgen): `cargo build -p anydoc-dotnet` +regenerates `src/NativeMethods.g.cs`. + +To rebuild the native binaries and produce the NuGet package: + +```sh +sh dotnet/build.sh # build the host's native library and pack +sh dotnet/build.sh --all # build every supported platform's library +``` + +Each architecture's library is laid out under +`src/runtimes/{rid}/native/` — NuGet's native-library convention — so a single +package carries all six targets. `dotnet pack` bundles them with the managed +wrapper. + +Run the tests with: + +```sh +dotnet test dotnet/Firecrawl.Anydoc.slnx +``` \ No newline at end of file diff --git a/dotnet/build-wasm.sh b/dotnet/build-wasm.sh new file mode 100755 index 00000000..9ef3a3d3 --- /dev/null +++ b/dotnet/build-wasm.sh @@ -0,0 +1,47 @@ +#!/bin/sh +# Build the anydoc WASM engine from the repo's wasm/ crate and lay the +# wasm-bindgen output into Firecrawl.Anydoc.Wasm/wwwroot so the NuGet package +# ships them as static web assets. +# +# Produces: +# Firecrawl.Anydoc.Wasm/wwwroot/anydoc_wasm.js wasm-bindgen glue +# Firecrawl.Anydoc.Wasm/wwwroot/anydoc_wasm_bg.wasm engine binary +# +# The static wwwroot/anydoc-wasm.js interop shim is checked in (not generated). +# +# Prerequisites: +# rustup target add wasm32-unknown-unknown +# cargo install wasm-pack (or use the wasm-bindgen CLI directly) +set -eu + +cd "$(dirname "$0")" +root=$(pwd) +wasm_dir="$root/../wasm" +out="$root/Firecrawl.Anydoc.Wasm/wwwroot" + +build_with_wasm_pack() { + (cd "$wasm_dir" && wasm-pack build --release --target web --out-dir pkg) + cp "$wasm_dir/pkg/anydoc_wasm.js" "$out/anydoc_wasm.js" + cp "$wasm_dir/pkg/anydoc_wasm_bg.wasm" "$out/anydoc_wasm_bg.wasm" +} + +build_with_wasm_bindgen() { + rustup target add wasm32-unknown-unknown >/dev/null + (cd "$wasm_dir" && cargo build --release --target wasm32-unknown-unknown --package anydoc-wasm) + wasm-bindgen "$root/../target/wasm32-unknown-unknown/release/anydoc_wasm.wasm" \ + --target web --out-dir "$out" --out-name anydoc_wasm +} + +rm -f "$out/anydoc_wasm.js" "$out/anydoc_wasm_bg.wasm" +if command -v wasm-pack >/dev/null 2>&1; then + build_with_wasm_pack +elif command -v wasm-bindgen >/dev/null 2>&1; then + build_with_wasm_bindgen +else + echo "no wasm-pack or wasm-bindgen toolchain found." >&2 + echo "install one: cargo install wasm-pack (or cargo install wasm-bindgen-cli)" >&2 + exit 1 +fi + +echo "-> $out/anydoc_wasm.js" +echo "-> $out/anydoc_wasm_bg.wasm" \ No newline at end of file diff --git a/dotnet/build.sh b/dotnet/build.sh new file mode 100755 index 00000000..a316fa74 --- /dev/null +++ b/dotnet/build.sh @@ -0,0 +1,79 @@ +#!/bin/sh +# Build the native cdylib for the listed targets and lay each one out under +# dotnet/Firecrawl.Anydoc/runtimes/{rid}/native so `dotnet pack` bundles a NuGet package +# (Firecrawl.Anydoc) that carries macOS, Windows, and Linux binaries for both +# x64 and arm64. +# +# Cross-compiling to a target you do not natively develop on needs that target +# installed (`rustup target add `). Windows/MSVC targets cannot be +# produced on macOS or Linux hosts; build those on Windows runners. +# +# Usage (from the repo root, or dotnet/): +# sh dotnet/build.sh # host target only +# sh dotnet/build.sh --all # every supported native target +# sh dotnet/build.sh osx-arm64 ... # a specific RID subset +set -eu + +cd "$(dirname "$0")" +root=$(pwd) +repo=$(cd .. && pwd) + +# RID -> rust target triple -> artifact filename +# (cargo names cdylibs lib.dylib/.so and .dll on their platforms) +targets() { + echo osx-x64:x86_64-apple-darwin:libanydoc_dotnet.dylib + echo osx-arm64:aarch64-apple-darwin:libanydoc_dotnet.dylib + echo win-x64:x86_64-pc-windows-msvc:anydoc_dotnet.dll + echo win-arm64:aarch64-pc-windows-msvc:anydoc_dotnet.dll + echo linux-x64:x86_64-unknown-linux-gnu:libanydoc_dotnet.so + echo linux-arm64:aarch64-unknown-linux-gnu:libanydoc_dotnet.so +} + +host_rid() { + case "$(uname -s)-$(uname -m)" in + Darwin-arm64) echo osx-arm64 ;; + Darwin-x86_64|Darwin-i386) echo osx-x64 ;; + Linux-x86_64|Linux-amd64) echo linux-x64 ;; + Linux-aarch64|Linux-arm64) echo linux-arm64 ;; + *) echo "unsupported host $(uname -s) $(uname -m)" >&2; exit 2 ;; + esac +} + +build_rid() { + rid=$1; shift + line=$(targets | grep "^$rid:" || true) + [ -n "$line" ] || { echo "unknown RID: $rid" >&2; exit 2; } + rest=${line#*:} # strip rid + triple=${rest%%:*} # rust target triple + artifact=${rest#*:} # cdylib filename + + echo "-- building $rid ($triple)" + if [ "$triple" = unknown ] || ! rustup target list --installed 2>/dev/null | grep -qx "$triple"; then + echo " target $triple not installed; skipping (add it with: rustup target add $triple)" >&2 + return 1 + fi + cargo build --manifest-path "$root/native/Cargo.toml" --release --target "$triple" -p anydoc-dotnet + + out=$(find "$root/../target" "$root/native/target" -type f -name "$artifact" 2>/dev/null | head -n 1) + [ -n "$out" ] || { echo " artifact $artifact not found" >&2; return 1; } + mkdir -p "$root/Firecrawl.Anydoc/runtimes/$rid/native" + cp "$out" "$root/Firecrawl.Anydoc/runtimes/$rid/native/$artifact" + echo " -> dotnet/Firecrawl.Anydoc/runtimes/$rid/native/$artifact" +} + +if [ "$#" -eq 0 ]; then + build_rid "$(host_rid)" +elif [ "$1" = --all ]; then + for rid in $(targets | cut -d: -f1); do + build_rid "$rid" || true + done +else + for rid in "$@"; do + build_rid "$rid" + done +fi + +echo "--packing dotnet" +dotnet pack "$root/Firecrawl.Anydoc/Firecrawl.Anydoc.csproj" -c Release -o "$root/artifacts" +echo "package:" +ls -1 "$root/artifacts"/Firecrawl.Anydoc.*.nupkg 2>/dev/null || true \ No newline at end of file diff --git a/dotnet/examples/AnydocAvaloniaDemo/AnydocAvaloniaDemo.csproj b/dotnet/examples/AnydocAvaloniaDemo/AnydocAvaloniaDemo.csproj new file mode 100644 index 00000000..0f1550cb --- /dev/null +++ b/dotnet/examples/AnydocAvaloniaDemo/AnydocAvaloniaDemo.csproj @@ -0,0 +1,29 @@ + + + WinExe + net10.0 + enable + app.manifest + + + + + + + + + + + + + + + + + + None + All + + + + diff --git a/dotnet/examples/AnydocAvaloniaDemo/App.axaml b/dotnet/examples/AnydocAvaloniaDemo/App.axaml new file mode 100644 index 00000000..c8f723c1 --- /dev/null +++ b/dotnet/examples/AnydocAvaloniaDemo/App.axaml @@ -0,0 +1,15 @@ + + + + + + + + + + + \ No newline at end of file diff --git a/dotnet/examples/AnydocAvaloniaDemo/App.axaml.cs b/dotnet/examples/AnydocAvaloniaDemo/App.axaml.cs new file mode 100644 index 00000000..625edfd7 --- /dev/null +++ b/dotnet/examples/AnydocAvaloniaDemo/App.axaml.cs @@ -0,0 +1,28 @@ +using Avalonia; +using Avalonia.Controls.ApplicationLifetimes; +using Avalonia.Markup.Xaml; +using AnydocAvaloniaDemo.ViewModels; +using AnydocAvaloniaDemo.Views; + +namespace AnydocAvaloniaDemo; + +public partial class App : Application +{ + public override void Initialize() + { + AvaloniaXamlLoader.Load(this); + } + + public override void OnFrameworkInitializationCompleted() + { + if (ApplicationLifetime is IClassicDesktopStyleApplicationLifetime desktop) + { + desktop.MainWindow = new MainWindow + { + DataContext = new MainViewModel(), + }; + } + + base.OnFrameworkInitializationCompleted(); + } +} \ No newline at end of file diff --git a/dotnet/examples/AnydocAvaloniaDemo/Assets/avalonia-logo.ico b/dotnet/examples/AnydocAvaloniaDemo/Assets/avalonia-logo.ico new file mode 100644 index 00000000..f7da8bb5 Binary files /dev/null and b/dotnet/examples/AnydocAvaloniaDemo/Assets/avalonia-logo.ico differ diff --git a/dotnet/examples/AnydocAvaloniaDemo/Program.cs b/dotnet/examples/AnydocAvaloniaDemo/Program.cs new file mode 100644 index 00000000..b0d03561 --- /dev/null +++ b/dotnet/examples/AnydocAvaloniaDemo/Program.cs @@ -0,0 +1,24 @@ +using Avalonia; +using System; + +namespace AnydocAvaloniaDemo; + +sealed class Program +{ + // Initialization code. Don't use any Avalonia, third-party APIs or any + // SynchronizationContext-reliant code before AppMain is called: things aren't initialized + // yet and stuff might break. + [STAThread] + public static void Main(string[] args) => BuildAvaloniaApp() + .StartWithClassicDesktopLifetime(args); + + // Avalonia configuration, don't remove; also used by visual designer. + public static AppBuilder BuildAvaloniaApp() + => AppBuilder.Configure() + .UsePlatformDetect() +#if DEBUG + .WithDeveloperTools() +#endif + .WithInterFont() + .LogToTrace(); +} diff --git a/dotnet/examples/AnydocAvaloniaDemo/ViewLocator.cs b/dotnet/examples/AnydocAvaloniaDemo/ViewLocator.cs new file mode 100644 index 00000000..104e53d3 --- /dev/null +++ b/dotnet/examples/AnydocAvaloniaDemo/ViewLocator.cs @@ -0,0 +1,37 @@ +using System; +using System.Diagnostics.CodeAnalysis; +using Avalonia.Controls; +using Avalonia.Controls.Templates; +using AnydocAvaloniaDemo.ViewModels; + +namespace AnydocAvaloniaDemo; + +/// +/// Given a view model, returns the corresponding view if possible. +/// +[RequiresUnreferencedCode( + "Default implementation of ViewLocator involves reflection which may be trimmed away.", + Url = "https://docs.avaloniaui.net/docs/concepts/view-locator")] +public class ViewLocator : IDataTemplate +{ + public Control? Build(object? param) + { + if (param is null) + return null; + + var name = param.GetType().FullName!.Replace("ViewModel", "View", StringComparison.Ordinal); + var type = Type.GetType(name); + + if (type != null) + { + return (Control)Activator.CreateInstance(type)!; + } + + return new TextBlock { Text = "Not Found: " + name }; + } + + public bool Match(object? data) + { + return data is ViewModelBase; + } +} diff --git a/dotnet/examples/AnydocAvaloniaDemo/ViewModels/MainViewModel.cs b/dotnet/examples/AnydocAvaloniaDemo/ViewModels/MainViewModel.cs new file mode 100644 index 00000000..824dacae --- /dev/null +++ b/dotnet/examples/AnydocAvaloniaDemo/ViewModels/MainViewModel.cs @@ -0,0 +1,91 @@ +using System; +using System.IO; +using System.Threading.Tasks; +using CommunityToolkit.Mvvm.ComponentModel; +using CommunityToolkit.Mvvm.Input; +using Firecrawl.Anydoc; + +namespace AnydocAvaloniaDemo.ViewModels; + +public partial class MainViewModel : ViewModelBase +{ + private readonly AnydocConverter _converter = new(); + + [ObservableProperty] + public partial string SelectedFile { get; set; } = ""; + + [ObservableProperty] + public partial string Status { get; set; } = "Select a document to convert."; + + [ObservableProperty] + public partial string Markdown { get; set; } = ""; + + [ObservableProperty] + public partial bool IsBusy { get; set; } + + [ObservableProperty] + public partial bool HasError { get; set; } + + [ObservableProperty] + public partial string? Format { get; set; } + + [ObservableProperty] + public partial bool HasResult { get; set; } + + [RelayCommand] + public void SetFile(string path) + { + SelectedFile = path; + Status = Path.GetFileName(path); + Markdown = ""; + HasError = false; + HasResult = false; + Format = null; + } + + [RelayCommand] + public async Task ConvertAsync() + { + if (string.IsNullOrWhiteSpace(SelectedFile)) + { + return; + } + + IsBusy = true; + HasError = false; + HasResult = false; + Markdown = ""; + Status = "Converting…"; + + try + { + byte[] bytes = await File.ReadAllBytesAsync(SelectedFile); + + Format? format = _converter.DetectFormatByExtension(Path.GetExtension(SelectedFile)) + ?? _converter.DetectFormat(bytes); + + string markdown = format is null + ? await _converter.ToMarkdownBytesAsync(bytes) + : await _converter.ToMarkdownBytesAsync(bytes, format.Value); + + Format = format?.ToString(); + Markdown = markdown; + HasResult = true; + Status = $"Converted {bytes.Length:N0} bytes to Markdown."; + } + catch (AnydocException ex) + { + HasError = true; + Status = $"{ex.Code}: {ex.Message}"; + } + catch (Exception ex) + { + HasError = true; + Status = ex.Message; + } + finally + { + IsBusy = false; + } + } +} diff --git a/dotnet/examples/AnydocAvaloniaDemo/ViewModels/ViewModelBase.cs b/dotnet/examples/AnydocAvaloniaDemo/ViewModels/ViewModelBase.cs new file mode 100644 index 00000000..326f2107 --- /dev/null +++ b/dotnet/examples/AnydocAvaloniaDemo/ViewModels/ViewModelBase.cs @@ -0,0 +1,7 @@ +using CommunityToolkit.Mvvm.ComponentModel; + +namespace AnydocAvaloniaDemo.ViewModels; + +public abstract class ViewModelBase : ObservableObject +{ +} diff --git a/dotnet/examples/AnydocAvaloniaDemo/Views/MainWindow.axaml b/dotnet/examples/AnydocAvaloniaDemo/Views/MainWindow.axaml new file mode 100644 index 00000000..fcb5c836 --- /dev/null +++ b/dotnet/examples/AnydocAvaloniaDemo/Views/MainWindow.axaml @@ -0,0 +1,59 @@ + + + + + + + + + + + + +