diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..d6035a4 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,66 @@ +name: CI + +on: + push: + branches: ["main"] + pull_request: + workflow_dispatch: + +env: + CARGO_TERM_COLOR: always + RUSTFLAGS: "-D warnings" + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + test: + name: test (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + steps: + - uses: actions/checkout@v4 + + - uses: dtolnay/rust-toolchain@stable + + - uses: Swatinem/rust-cache@v2 + with: + key: ${{ matrix.os }} + + - name: Build + run: cargo build --workspace --all-targets + + - name: Test + run: cargo test --workspace + + # The corpus is generated, not hand-committed: if the generator and the + # committed binaries ever disagree, the goldens are testing fiction. + - name: Corpus is up to date + run: python3 corpus/generate.py --check + + lint: + name: fmt + clippy + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: dtolnay/rust-toolchain@stable + with: + components: rustfmt, clippy + + - uses: Swatinem/rust-cache@v2 + + - name: Format + run: cargo fmt --all -- --check + + - name: Clippy + run: cargo clippy --workspace --all-targets -- -D warnings + + - name: Docs + run: cargo doc --workspace --no-deps + env: + RUSTDOCFLAGS: "-D warnings" diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..081f38a --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +/target +**/*.rs.bk +*.pdb +__pycache__/ diff --git a/AGENTS.md b/AGENTS.md index ce05b00..0b46dda 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -10,8 +10,12 @@ Office (`.doc`, `.docx`, `.xls`, `.xlsx`) y LibreOffice (`.odt`, `.ods`) a un Ma extendido llamado **DocMark**, y de vuelta, con pérdida mínima de formato. Se invoca como CLI o como servidor **MCP por stdio**. -**Estado actual**: solo existe documentación de diseño. La implementación comienza en la -Fase 0 del plan. No hay todavía `Cargo.toml` en la raíz. +**Estado actual**: **Fases 0 y 1 cerradas**. Existe el workspace con los siete crates, el IR +(`docsai-model`), el lector `.docx` (`docsai-office`), el serializador DocMark +(`docsai-docmark`), la orquestación (`docsai-convert`) y la CLI con `convert` y `formats`. +`docsai-odf` y `docsai-mcp` son esqueletos que sólo fijan las reglas de dependencia. + +Lo siguiente es la **Fase 2**: parser DocMark, writer `.docx` y el comando `roundtrip`. ## 2. Documentos que debes leer antes de implementar @@ -26,6 +30,10 @@ Orden de lectura obligatorio: 4. `docs/analisis-tecnico.md` — por qué se eligió cada librería. No sustituyas una dependencia clave (calamine, docx-rs, rmcp, comrak…) sin dejar constancia escrita del motivo en ese documento. +5. `kb/` — base de conocimiento de lo **ya construido**: resumen de las fases cerradas, + estructura real del código, decisiones técnicas con su motivo, y las consideraciones para + la fase que vayas a abordar. Empieza por `kb/README.md`. Si tu cambio altera la estructura + o toma una decisión técnica no obvia, actualízala en el mismo PR. ## 3. Estructura del repositorio (objetivo) @@ -63,6 +71,13 @@ cargo test --workspace # unit + integración + round-trip cargo clippy --workspace --all-targets -- -D warnings cargo fmt --all -- --check cargo run -p docsai-cli -- convert corpus/docx/basic-styles.docx -o /tmp/out.dmk.md +python3 corpus/generate.py --check # el corpus es generado; CI comprueba que está al día +``` + +Actualizar un golden es un acto deliberado, y su diff se revisa a mano: + +```bash +DOCSAI_UPDATE_GOLDENS=1 cargo test -p docsai-convert --test goldens ``` CI (GitHub Actions) ejecuta la matriz `{ubuntu-latest, windows-latest, macos-latest}` × diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..0a257ea --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,1053 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" +dependencies = [ + "derive_arbitrary", +] + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "bit-set" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" + +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "caseless" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b6fd507454086c8edfd769ca6ada439193cdb209c7681712ef6275cccbfe5d8" +dependencies = [ + "unicode-normalization", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "clap" +version = "4.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "301b56658598e48f3648647ac6fc887be7e7108eddfa4e9b63fcf3ec58c0cadf" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94a65403d1a1bd28f7dc68eb8506e8874808ee5eecb59298de588e2e1407a078" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "comrak" +version = "0.39.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fefab951771fc3beeed0773ce66a4f7b706273fc6c4c95b08dd1615744abcf5" +dependencies = [ + "caseless", + "entities", + "memchr", + "slug", + "typed-arena", + "unicode_categories", +] + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" + +[[package]] +name = "derive_arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "deunicode" +version = "1.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abd57806937c9cc163efc8ea3910e00a62e2aeb0b8119f1793a978088f8f6b04" + +[[package]] +name = "displaydoc" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "docsai-cli" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap", + "docsai-convert", + "docsai-model", + "serde_json", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "docsai-convert" +version = "0.1.0" +dependencies = [ + "comrak", + "docsai-docmark", + "docsai-model", + "docsai-odf", + "docsai-office", + "thiserror", + "tracing", + "zip", +] + +[[package]] +name = "docsai-docmark" +version = "0.1.0" +dependencies = [ + "docsai-model", + "thiserror", +] + +[[package]] +name = "docsai-mcp" +version = "0.1.0" +dependencies = [ + "docsai-convert", +] + +[[package]] +name = "docsai-model" +version = "0.1.0" +dependencies = [ + "proptest", + "serde", + "serde_json", + "thiserror", +] + +[[package]] +name = "docsai-odf" +version = "0.1.0" +dependencies = [ + "docsai-model", +] + +[[package]] +name = "docsai-office" +version = "0.1.0" +dependencies = [ + "docsai-model", + "quick-xml", + "thiserror", + "tracing", + "zip", +] + +[[package]] +name = "entities" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5320ae4c3782150d900b79807611a59a99fc9a1d61d686faafc24b93fc8d7ca" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi 5.3.0", + "wasip2", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "log" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "proptest" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b45fcc2344c680f5025fe57779faef368840d0bd1f42f216291f0dc4ace4744" +dependencies = [ + "bit-set", + "bit-vec", + "bitflags", + "num-traits", + "rand", + "rand_chacha", + "rand_xorshift", + "regex-syntax", + "rusty-fork", + "tempfile", + "unarray", +] + +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + +[[package]] +name = "quick-xml" +version = "0.37.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "331e97a1af0bf59823e6eadffe373d7b27f485be8748f71471c662c1f269b7fb" +dependencies = [ + "memchr", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" +dependencies = [ + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "rand_xorshift" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "513962919efc330f829edb2535844d1b912b0fbe2ca165d613e4e8788bb05a5a" +dependencies = [ + "rand_core", +] + +[[package]] +name = "regex-automata" +version = "0.4.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "rusty-fork" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc6bf79ff24e648f6da1f8d1f011e9cac26491b619e6b9280f2b47f1774e6ee2" +dependencies = [ + "fnv", + "quick-error", + "tempfile", + "wait-timeout", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + +[[package]] +name = "slug" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "882a80f72ee45de3cc9a5afeb2da0331d58df69e4e7d8eeb5d3c7784ae67e724" +dependencies = [ + "deunicode", + "wasm-bindgen", +] + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.3", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "thiserror" +version = "2.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09a43598840e33d5b0331f38c5e30d13bb11c11210a4b58f0d9b18a5a5eefcd9" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43cbfe0cf76104d42a574802844187e84a305e531ed54455f11fbde0f10541cd" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "thread_local" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "tinyvec" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + +[[package]] +name = "typed-arena" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6af6ae20167a9ece4bcb41af5b80f8a1f1df981f6391189ce00fd257af04126a" + +[[package]] +name = "unarray" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "unicode_categories" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "wait-timeout" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ac3b126d3914f9849036f826e054cbabdc8519970b8998ddaf3b5bd3c65f11" +dependencies = [ + "libc", +] + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 2.0.119", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "zerocopy" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zip" +version = "2.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fabe6324e908f85a1c52063ce7aa26b68dcb7eb6dbc83a2d148403c9bc3eba50" +dependencies = [ + "arbitrary", + "crc32fast", + "crossbeam-utils", + "displaydoc", + "flate2", + "indexmap", + "memchr", + "thiserror", + "zopfli", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" + +[[package]] +name = "zopfli" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249" +dependencies = [ + "bumpalo", + "crc32fast", + "log", + "simd-adler32", +] diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..5ad3084 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,45 @@ +[workspace] +resolver = "2" +members = [ + "crates/docsai-model", + "crates/docsai-docmark", + "crates/docsai-office", + "crates/docsai-odf", + "crates/docsai-convert", + "crates/docsai-cli", + "crates/docsai-mcp", +] + +[workspace.package] +version = "0.1.0" +edition = "2021" +rust-version = "1.82" +license = "MIT" +repository = "https://github.com/digiogithub/docsai" +authors = ["docsai contributors"] + +[workspace.dependencies] +docsai-model = { path = "crates/docsai-model", version = "0.1.0" } +docsai-docmark = { path = "crates/docsai-docmark", version = "0.1.0" } +docsai-office = { path = "crates/docsai-office", version = "0.1.0" } +docsai-odf = { path = "crates/docsai-odf", version = "0.1.0" } +docsai-convert = { path = "crates/docsai-convert", version = "0.1.0" } + +anyhow = "1" +clap = { version = "4", features = ["derive"] } +comrak = { version = "0.39", default-features = false } +proptest = "1" +quick-xml = "0.37" +serde = { version = "1", features = ["derive"] } +# `float_roundtrip`: without it serde_json's fast float parser can shift a +# value by 1 ULP, which would silently corrupt spreadsheet numbers. +serde_json = { version = "1", features = ["float_roundtrip"] } +thiserror = "2" +tracing = "0.1" +tracing-subscriber = { version = "0.3", features = ["env-filter"] } +zip = { version = "2", default-features = false, features = ["deflate"] } + +[profile.release] +lto = true +strip = true +codegen-units = 1 diff --git a/README.md b/README.md index 1cfa686..72ab3cc 100644 --- a/README.md +++ b/README.md @@ -4,25 +4,34 @@ `docsai` es un binario único y multiplataforma (Windows, Linux, macOS) que convierte documentos ofimáticos a un perfil de Markdown extendido — **DocMark** — diseñado para conservar la máxima información posible (estilos, imágenes, propiedades, fórmulas) y permitir la **conversión inversa con pérdida mínima de formato**. Puede usarse como herramienta CLI o como **servidor MCP (Model Context Protocol) por stdio**, para integrarse con asistentes de IA como Claude. -> **Estado del proyecto: fase de diseño / documentación.** -> Este repositorio contiene actualmente la documentación inicial completa del proyecto -> (análisis técnico, especificación de formato, arquitectura y plan de desarrollo) -> para que el equipo de implementación pueda comenzar el desarrollo. -> Todavía no hay código funcional. +> **Estado del proyecto: Fases 0 y 1 completadas.** +> Ya se convierte `.docx` → DocMark con estilos, listas, tablas, imágenes (con toda su +> geometría), cabeceras, pies, notas al pie, campos y propiedades. La conversión inversa +> (DocMark → `.docx`) es la Fase 2 y todavía no existe; las hojas de cálculo llegan en la +> Fase 3. Ver [`docs/plan-desarrollo.md`](docs/plan-desarrollo.md). + +```bash +cargo run -p docsai-cli -- convert informe.docx -o informe.dmk.md +cargo run -p docsai-cli -- formats +``` --- ## Formatos soportados (objetivo) +✅ = funciona hoy · 🕓 = planificado, con la fase que lo trae · ➖ = fuera de alcance + | Formato | Extensión | Lectura | Escritura | Notas | |---|---|---|---|---| -| Word OOXML | `.docx` | ✅ Fase 1 | ✅ Fase 2 | Estilos, imágenes, tablas, listas, cabeceras/pies, propiedades | -| Word binario | `.doc` | ✅ Fase 5 | ➖ | Solo lectura (vía parser nativo o fallback LibreOffice headless) | -| Excel OOXML | `.xlsx` | ✅ Fase 3 | ✅ Fase 3 | Valores **y fórmulas**, formatos de número, celdas combinadas, imágenes ancladas | -| Excel binario | `.xls` | ✅ Fase 3 | ➖ | Solo lectura (calamine) | -| OpenDocument Text | `.odt` | ✅ Fase 4 | ✅ Fase 4 | Equivalente libre de `.docx` | -| OpenDocument Spreadsheet | `.ods` | ✅ Fase 4 | ✅ Fase 4 | Equivalente libre de `.xlsx` | -| Markdown extendido | `.dmk.md` | ✅ | ✅ | Formato pivote **DocMark** (superconjunto de CommonMark + GFM) | +| Word OOXML | `.docx` | ✅ | 🕓 Fase 2 | Estilos, imágenes, tablas, listas, cabeceras/pies, notas al pie, campos, propiedades | +| Word binario | `.doc` | 🕓 Fase 5 | ➖ | Solo lectura (vía parser nativo o fallback LibreOffice headless) | +| Excel OOXML | `.xlsx` | 🕓 Fase 3 | 🕓 Fase 3 | Valores **y fórmulas**, formatos de número, celdas combinadas, imágenes ancladas | +| Excel binario | `.xls` | 🕓 Fase 3 | ➖ | Solo lectura (calamine) | +| OpenDocument Text | `.odt` | 🕓 Fase 4 | 🕓 Fase 4 | Equivalente libre de `.docx` | +| OpenDocument Spreadsheet | `.ods` | 🕓 Fase 4 | 🕓 Fase 4 | Equivalente libre de `.xlsx` | +| Markdown extendido | `.dmk.md` | 🕓 Fase 2 | ✅ | Formato pivote **DocMark** (superconjunto de CommonMark + GFM) | + +`docsai formats` imprime esta misma matriz según lo que el binario realmente sabe hacer. La escritura de `.doc` y `.xls` (formatos binarios legados) queda fuera de alcance de forma deliberada: la ruta de salida recomendada hacia el ecosistema Microsoft es siempre OOXML (`.docx` / `.xlsx`). @@ -55,24 +64,34 @@ Texto con **negrita** y [color]{color="#FF0000"} personalizado. ![Diagrama de ventas](assets/img-001.png){width=450px height=300px anchor=inline} ``` -## Uso previsto (CLI) +## Uso (CLI) + +Lo que funciona hoy: ```bash -# Office → DocMark -docsai convert informe.docx -o informe.dmk.md # extrae assets/ junto al .md -docsai convert ventas.xlsx -o ventas.dmk.md # valores + fórmulas - -# DocMark → Office -docsai convert informe.dmk.md -o informe.docx -docsai convert ventas.dmk.md -o ventas.xlsx - -# Inspección y verificación -docsai inspect informe.docx # estructura, estilos, medios, propiedades (JSON) -docsai roundtrip informe.docx # convierte ida y vuelta y reporta métricas de fidelidad -docsai --help +docsai convert informe.docx -o informe.dmk.md # extrae assets/ junto al .md +docsai convert informe.docx # a stdout, sin escribir medios al lado +docsai convert informe.docx --fidelity plain # CommonMark limpio, para LLM/RAG +docsai convert informe.docx -o out.md --json # informe de conversión en JSON +docsai formats # matriz de soporte de este binario ``` -## Uso previsto (servidor MCP) +Grados de fidelidad (`--fidelity`, spec §6): `full` (defecto, apto para ida y vuelta), +`standard` (Markdown rico sin catálogos ni raw-blocks) y `plain` (CommonMark+GFM puro). + +Códigos de salida: `0` correcto, `1` conversión con pérdidas (`--strict` incluye las advertencias +menores), `2` error de entrada, `3` formato no soportado. + +Planificado: + +```bash +docsai convert informe.dmk.md -o informe.docx # Fase 2 +docsai convert ventas.xlsx -o ventas.dmk.md # Fase 3 +docsai inspect informe.docx # Fase 6 +docsai roundtrip informe.docx # Fase 2 +``` + +## Uso previsto (servidor MCP, Fase 7) ```bash docsai mcp # arranca el servidor MCP por stdio @@ -99,6 +118,21 @@ Tools MCP previstas: `convert_to_markdown`, `convert_from_markdown`, `inspect_do | [`docs/arquitectura.md`](docs/arquitectura.md) | Arquitectura del software: workspace de crates, modelo de documento intermedio (IR), CLI, servidor MCP | | [`docs/plan-desarrollo.md`](docs/plan-desarrollo.md) | Plan de desarrollo detallado en 9 fases, con entregables, criterios de aceptación, estimaciones y estrategia de pruebas | | [`AGENTS.md`](AGENTS.md) | Guía operativa para desarrolladores y agentes de IA que trabajen en este repositorio | +| [`corpus/README.md`](corpus/README.md) | El corpus de pruebas: qué aísla cada documento y cómo se regenera | +| [`docs/spikes/`](docs/spikes/) | Informes de los spikes de riesgo, con la decisión que cerró cada uno | +| [`kb/`](kb/) | Base de conocimiento: qué hay construido, cómo está estructurado, las decisiones técnicas y lo que espera a las fases siguientes | + +## Desarrollo + +```bash +cargo test --workspace +cargo clippy --workspace --all-targets -- -D warnings +cargo fmt --all -- --check +python3 corpus/generate.py --check # el corpus es generado, no dibujado a mano +``` + +Los golden files viven junto al corpus (`corpus/docx/*.expected.dmk.md`). Para actualizarlos: +`DOCSAI_UPDATE_GOLDENS=1 cargo test -p docsai-convert --test goldens`, y **se revisa el diff**. ## Principios de diseño diff --git a/corpus/README.md b/corpus/README.md new file mode 100644 index 0000000..7bd3bdb --- /dev/null +++ b/corpus/README.md @@ -0,0 +1,83 @@ +# Corpus de pruebas + +Documentos de prueba versionados. **Un rasgo por fichero**: cuando un golden falla, el fichero +que falla señala el área del lector que se ha roto. + +Ningún documento contiene datos reales ni privados (`AGENTS.md` §6). + +## Cómo se generan + +Todos los ficheros los produce `corpus/generate.py`, sin dependencias más allá de la biblioteca +estándar de Python 3: + +```bash +python3 corpus/generate.py # regenera todo +python3 corpus/generate.py --check # falla si el árbol está desfasado (lo ejecuta CI) +``` + +Que el corpus sea **generado y no dibujado a mano** es deliberado: + +- El XML de cada documento vive en el generador, donde se revisa en un `git diff` normal; un + `.docx` hecho con Word es una caja opaca en la revisión. +- Los paquetes se escriben con marca de tiempo y orden de miembros fijos, así que regenerar + produce archivos byte a byte idénticos y el repositorio no acumula ruido binario. +- Los medios (PNG, GIF, EMF) se sintetizan en Python puro, sin Pillow, para que el generador + funcione igual en las tres plataformas de CI. + +La contrapartida: son documentos *mínimos*, no documentos de Word reales. Los documentos reales +anonimizados que pide la Fase 1 (tarea 10) y los corpus de rendimiento y adversarios de la Fase 8 +se añadirán aparte; el test de rendimiento de 50 páginas sintetiza su propio documento en tiempo +de ejecución (`crates/docsai-convert/tests/goldens.rs`). + +## Golden files + +Cada `docx/.docx` tiene al lado su DocMark esperado, `docx/.expected.dmk.md`. +Los comparan los tests de `crates/docsai-convert/tests/goldens.rs`. Para actualizarlos: + +```bash +DOCSAI_UPDATE_GOLDENS=1 cargo test -p docsai-convert --test goldens +``` + +El diff resultante **se revisa a mano** antes de confirmarlo: un golden actualizado sin mirar es +un test que ha dejado de comprobar nada. + +## Documentos de texto (`docx/`) + +| Fichero | Rasgo que aísla | +|---|---| +| `basic-text.docx` | Párrafos, salto de línea manual, párrafo vacío y los caracteres que Markdown escapa | +| `basic-styles.docx` | Negrita, cursiva, tachado, subrayado, color, resaltado, fuente y tamaño, sub/superíndice, hipervínculo, alineación y sangrías | +| `nested-lists.docx` | `numbering.xml` con tres niveles numerados y dos de viñetas; reconstrucción del árbol desde pares `(numId, ilvl)` | +| `table-simple.docx` | Tabla regular con estilo de tabla y rejilla | +| `table-merged.docx` | `gridSpan` y `vMerge` → `colspan`/`rowspan` y celdas absorbidas | +| `images-inline.docx` | Imágenes `wp:inline`: PNG con alt/título/nombre, GIF entre texto, EMF vectorial | +| `images-floating.docx` | `wp:anchor`: offsets relativos al margen con `wrapSquare`, alineación simbólica relativa a la página con `wrapTopAndBottom`, y marca de agua con `behindDoc` | +| `images-transformed.docx` | Rotación 45°, recorte `a:srcRect` con borde `a:ln`, volteo H+V y escala ≠ 100 % | +| `images-duplicated.docx` | El mismo mapa de bits en tres partes distintas del paquete con geometrías distintas: prueba la deduplicación del `AssetStore` | +| `images-vml.docx` | `w:pict` con VML heredado (documentos convertidos desde `.doc`) | +| `headers-footers.docx` | `sectPr` con cabecera por defecto y de primera página, pie con campos `PAGE`/`NUMPAGES`, dos columnas y `titlePg` | +| `footnotes.docx` | `footnotes.xml` con dos notas, una con formato dentro | +| `custom-styles.docx` | Estilo personalizado, estilo heredado con delta directo, estilo de carácter y propiedades personalizadas del documento | +| `fields-raw.docx` | Control de contenido `w:sdt`, campo complejo `TOC` y campo simple `DATE` | + +## Hojas de cálculo (`xlsx/`) + +Generadas en la Fase 0 para que el corpus esté completo; las consume la **Fase 3**, que es cuando +existirá el lector de `xlsx`. + +| Fichero | Rasgo que aísla | +|---|---| +| `values-types.xlsx` | Los seis tipos de celda: entero, decimal, booleano, error, fecha (serial + `numFmt`) y cadena en línea | +| `formulas-basic.xlsx` | Fórmulas con valor cacheado, referencia entre celdas y un nombre definido | +| `formulas-shared.xlsx` | Fórmulas compartidas (`t="shared"`) y de matriz (`t="array"`) | +| `number-formats.xlsx` | Moneda, fecha, porcentaje y millares por `numFmtId` | +| `merged-cells.xlsx` | `mergeCells` horizontal y vertical, y ancho de columna personalizado | +| `images-anchored.xlsx` | Los tres anclajes de hoja: `twoCellAnchor`, `oneCellAnchor` y `absoluteAnchor` | + +## Añadir un documento + +1. Escribe una función `docx_()` en `generate.py` y añádela a `GENERATORS`. +2. Regenera (`python3 corpus/generate.py`) y añade la fila a la tabla de arriba. +3. Genera su golden y **revisa el diff**. +4. Si el rasgo aún no está implementado, el golden documentará la degradación actual (un + raw-block, por ejemplo). Eso es correcto: hace visible lo que falta. diff --git a/corpus/docx/basic-styles.docx b/corpus/docx/basic-styles.docx new file mode 100644 index 0000000..10f8a56 Binary files /dev/null and b/corpus/docx/basic-styles.docx differ diff --git a/corpus/docx/basic-styles.expected.dmk.md b/corpus/docx/basic-styles.expected.dmk.md new file mode 100644 index 0000000..742808b --- /dev/null +++ b/corpus/docx/basic-styles.expected.dmk.md @@ -0,0 +1,52 @@ +--- +docmark: "1.0" +source-format: docx +title: "Estilos basicos" +author: "docsai corpus" +last-modified-by: "docsai corpus" +created: 2026-01-01T00:00:00Z +modified: 2026-01-02T00:00:00Z +language: "es-ES" +application: "docsai-corpus" +page: + size: A4 + margins: { top: 70.85pt, bottom: 70.85pt, left: 85.05pt, right: 85.05pt, header: 35.4pt, footer: 35.4pt } + orientation: portrait +style-defaults: + font: { name: "Calibri", size: 11pt } + paragraph: { space-after: 8pt, line-height: 1.079 } +styles: + Heading1: + type: paragraph + name: "heading 1" + based-on: Normal + font: { name: "Calibri Light", size: 16pt, color: "#2E74B5" } + paragraph: { space-before: 12pt, space-after: 4pt, keep-with-next: true, outline-level: 0 } + Heading2: + type: paragraph + name: "heading 2" + based-on: Normal + font: { size: 13pt, color: "#2E74B5" } + paragraph: { outline-level: 1 } + Hyperlink: + type: character + font: { color: "#0563C1", underline: single } + Normal: + type: paragraph + default: true + TableGrid: + type: table + name: "Table Grid" +--- + +# Titulo de nivel 1 {.Heading1} + +**negrita** *cursiva* ~~tachado~~ [subrayado]{.underline} + +## Titulo de nivel 2 {.Heading2} + +[rojo]{color=#FF0000} [resaltado]{highlight=yellow} [Arial 14]{font=Arial size=14pt} x[2]{.sup} H[2]{.sub}O + +Visita [el sitio](https://example.com/docsai){.Hyperlink} para mas informacion. + +Parrafo centrado con sangria. {align=center indent-first-line=24px indent-left=48px space-after=6pt space-before=12pt} diff --git a/corpus/docx/basic-text.docx b/corpus/docx/basic-text.docx new file mode 100644 index 0000000..025fd40 Binary files /dev/null and b/corpus/docx/basic-text.docx differ diff --git a/corpus/docx/basic-text.expected.dmk.md b/corpus/docx/basic-text.expected.dmk.md new file mode 100644 index 0000000..5c3d5d7 --- /dev/null +++ b/corpus/docx/basic-text.expected.dmk.md @@ -0,0 +1,51 @@ +--- +docmark: "1.0" +source-format: docx +title: "Texto basico" +author: "docsai corpus" +last-modified-by: "docsai corpus" +created: 2026-01-01T00:00:00Z +modified: 2026-01-02T00:00:00Z +language: "es-ES" +application: "docsai-corpus" +page: + size: A4 + margins: { top: 70.85pt, bottom: 70.85pt, left: 85.05pt, right: 85.05pt, header: 35.4pt, footer: 35.4pt } + orientation: portrait +style-defaults: + font: { name: "Calibri", size: 11pt } + paragraph: { space-after: 8pt, line-height: 1.079 } +styles: + Heading1: + type: paragraph + name: "heading 1" + based-on: Normal + font: { name: "Calibri Light", size: 16pt, color: "#2E74B5" } + paragraph: { space-before: 12pt, space-after: 4pt, keep-with-next: true, outline-level: 0 } + Heading2: + type: paragraph + name: "heading 2" + based-on: Normal + font: { size: 13pt, color: "#2E74B5" } + paragraph: { outline-level: 1 } + Hyperlink: + type: character + font: { color: "#0563C1", underline: single } + Normal: + type: paragraph + default: true + TableGrid: + type: table + name: "Table Grid" +--- + +Primer parrafo del documento. + +Segundo parrafo con un salto +de linea manual. + +Caracteres que Markdown escapa: \*asterisco\* \_guion\_ #almohadilla |tuberia| \[corchete\] \ \`backtick\` \\barra. + +[]{.empty} + +Parrafo final tras uno vacio. diff --git a/corpus/docx/custom-styles.docx b/corpus/docx/custom-styles.docx new file mode 100644 index 0000000..ba78299 Binary files /dev/null and b/corpus/docx/custom-styles.docx differ diff --git a/corpus/docx/custom-styles.expected.dmk.md b/corpus/docx/custom-styles.expected.dmk.md new file mode 100644 index 0000000..4a5bf27 --- /dev/null +++ b/corpus/docx/custom-styles.expected.dmk.md @@ -0,0 +1,64 @@ +--- +docmark: "1.0" +source-format: docx +title: "Estilos personalizados" +author: "docsai corpus" +last-modified-by: "docsai corpus" +created: 2026-01-01T00:00:00Z +modified: 2026-01-02T00:00:00Z +language: "es-ES" +application: "docsai-corpus" +custom-properties: + Departamento: "Ventas" + Revision: "3" +page: + size: A4 + margins: { top: 70.85pt, bottom: 70.85pt, left: 85.05pt, right: 85.05pt, header: 35.4pt, footer: 35.4pt } + orientation: portrait +style-defaults: + font: { name: "Calibri", size: 11pt } + paragraph: { space-after: 8pt, line-height: 1.079 } +styles: + Destacado: + type: paragraph + based-on: Normal + font: { size: 12pt, color: "#C00000", italic: true } + paragraph: { align: center, space-before: 6pt, space-after: 6pt, background: "#F2F2F2" } + DestacadoFuerte: + type: paragraph + name: "Destacado fuerte" + based-on: Destacado + font: { bold: true } + Enfatico: + type: character + font: { color: "#C00000", italic: true } + Heading1: + type: paragraph + name: "heading 1" + based-on: Normal + font: { name: "Calibri Light", size: 16pt, color: "#2E74B5" } + paragraph: { space-before: 12pt, space-after: 4pt, keep-with-next: true, outline-level: 0 } + Heading2: + type: paragraph + name: "heading 2" + based-on: Normal + font: { size: 13pt, color: "#2E74B5" } + paragraph: { outline-level: 1 } + Hyperlink: + type: character + font: { color: "#0563C1", underline: single } + Normal: + type: paragraph + default: true + TableGrid: + type: table + name: "Table Grid" +--- + +Parrafo normal sin atributos. + +Parrafo con estilo Destacado. {.Destacado} + +Estilo heredado con delta directo. {.DestacadoFuerte align=right} + +Texto con [estilo de caracter]{.Enfatico} y delta [**adicional**]{.Enfatico}. diff --git a/corpus/docx/fields-raw.docx b/corpus/docx/fields-raw.docx new file mode 100644 index 0000000..f727897 Binary files /dev/null and b/corpus/docx/fields-raw.docx differ diff --git a/corpus/docx/fields-raw.expected.dmk.md b/corpus/docx/fields-raw.expected.dmk.md new file mode 100644 index 0000000..8ac0077 --- /dev/null +++ b/corpus/docx/fields-raw.expected.dmk.md @@ -0,0 +1,50 @@ +--- +docmark: "1.0" +source-format: docx +title: "Campos y bloques opacos" +author: "docsai corpus" +last-modified-by: "docsai corpus" +created: 2026-01-01T00:00:00Z +modified: 2026-01-02T00:00:00Z +language: "es-ES" +application: "docsai-corpus" +page: + size: A4 + margins: { top: 70.85pt, bottom: 70.85pt, left: 85.05pt, right: 85.05pt, header: 35.4pt, footer: 35.4pt } + orientation: portrait +style-defaults: + font: { name: "Calibri", size: 11pt } + paragraph: { space-after: 8pt, line-height: 1.079 } +styles: + Heading1: + type: paragraph + name: "heading 1" + based-on: Normal + font: { name: "Calibri Light", size: 16pt, color: "#2E74B5" } + paragraph: { space-before: 12pt, space-after: 4pt, keep-with-next: true, outline-level: 0 } + Heading2: + type: paragraph + name: "heading 2" + based-on: Normal + font: { size: 13pt, color: "#2E74B5" } + paragraph: { outline-level: 1 } + Hyperlink: + type: character + font: { color: "#0563C1", underline: single } + Normal: + type: paragraph + default: true + TableGrid: + type: table + name: "Table Grid" +--- + +Antes del control de contenido. + +Contenido dentro de un control de contenido. + +[Tabla de contenido generada]{.field field=TOC instr="TOC \\o \"1-3\" \\h"} + +Fecha: [01/01/2026]{.field field=DATE instr="DATE \\@ \"dd/MM/yyyy\""} + +Despues. diff --git a/corpus/docx/footnotes.docx b/corpus/docx/footnotes.docx new file mode 100644 index 0000000..0327b4b Binary files /dev/null and b/corpus/docx/footnotes.docx differ diff --git a/corpus/docx/footnotes.expected.dmk.md b/corpus/docx/footnotes.expected.dmk.md new file mode 100644 index 0000000..0290040 --- /dev/null +++ b/corpus/docx/footnotes.expected.dmk.md @@ -0,0 +1,52 @@ +--- +docmark: "1.0" +source-format: docx +title: "Notas al pie" +author: "docsai corpus" +last-modified-by: "docsai corpus" +created: 2026-01-01T00:00:00Z +modified: 2026-01-02T00:00:00Z +language: "es-ES" +application: "docsai-corpus" +page: + size: A4 + margins: { top: 70.85pt, bottom: 70.85pt, left: 85.05pt, right: 85.05pt, header: 35.4pt, footer: 35.4pt } + orientation: portrait +style-defaults: + font: { name: "Calibri", size: 11pt } + paragraph: { space-after: 8pt, line-height: 1.079 } +styles: + FootnoteReference: + type: character + name: "footnote reference" + font: { vertical-align: superscript } + Heading1: + type: paragraph + name: "heading 1" + based-on: Normal + font: { name: "Calibri Light", size: 16pt, color: "#2E74B5" } + paragraph: { space-before: 12pt, space-after: 4pt, keep-with-next: true, outline-level: 0 } + Heading2: + type: paragraph + name: "heading 2" + based-on: Normal + font: { size: 13pt, color: "#2E74B5" } + paragraph: { outline-level: 1 } + Hyperlink: + type: character + font: { color: "#0563C1", underline: single } + Normal: + type: paragraph + default: true + TableGrid: + type: table + name: "Table Grid" +--- + +Texto con nota[^1] y sigue. + +Otro parrafo con segunda nota[^2]. + +[^1]: Primera nota al pie. + +[^2]: Segunda nota, con **negrita**. diff --git a/corpus/docx/headers-footers.docx b/corpus/docx/headers-footers.docx new file mode 100644 index 0000000..ef58a83 Binary files /dev/null and b/corpus/docx/headers-footers.docx differ diff --git a/corpus/docx/headers-footers.expected.dmk.md b/corpus/docx/headers-footers.expected.dmk.md new file mode 100644 index 0000000..1ceeb3d --- /dev/null +++ b/corpus/docx/headers-footers.expected.dmk.md @@ -0,0 +1,56 @@ +--- +docmark: "1.0" +source-format: docx +title: "Cabeceras y pies" +author: "docsai corpus" +last-modified-by: "docsai corpus" +created: 2026-01-01T00:00:00Z +modified: 2026-01-02T00:00:00Z +language: "es-ES" +application: "docsai-corpus" +page: + size: A4 + margins: { top: 70.85pt, bottom: 70.85pt, left: 85.05pt, right: 85.05pt, header: 35.4pt, footer: 35.4pt } + orientation: portrait + columns: 2 + title-page: true +style-defaults: + font: { name: "Calibri", size: 11pt } + paragraph: { space-after: 8pt, line-height: 1.079 } +styles: + Heading1: + type: paragraph + name: "heading 1" + based-on: Normal + font: { name: "Calibri Light", size: 16pt, color: "#2E74B5" } + paragraph: { space-before: 12pt, space-after: 4pt, keep-with-next: true, outline-level: 0 } + Heading2: + type: paragraph + name: "heading 2" + based-on: Normal + font: { size: 13pt, color: "#2E74B5" } + paragraph: { outline-level: 1 } + Hyperlink: + type: character + font: { color: "#0563C1", underline: single } + Normal: + type: paragraph + default: true + TableGrid: + type: table + name: "Table Grid" +--- + +::: {.header scope=default} +Cabecera por defecto +::: + +::: {.header scope=first} +Cabecera de primera pagina +::: + +::: {.footer scope=default} +Pagina [1]{.field field=PAGE} de [3]{.field field=NUMPAGES} +::: + +Cuerpo del documento con cabeceras y pies. diff --git a/corpus/docx/images-duplicated.docx b/corpus/docx/images-duplicated.docx new file mode 100644 index 0000000..145ff7f Binary files /dev/null and b/corpus/docx/images-duplicated.docx differ diff --git a/corpus/docx/images-duplicated.expected.dmk.md b/corpus/docx/images-duplicated.expected.dmk.md new file mode 100644 index 0000000..c3f4a14 --- /dev/null +++ b/corpus/docx/images-duplicated.expected.dmk.md @@ -0,0 +1,48 @@ +--- +docmark: "1.0" +source-format: docx +title: "Imagenes duplicadas" +author: "docsai corpus" +last-modified-by: "docsai corpus" +created: 2026-01-01T00:00:00Z +modified: 2026-01-02T00:00:00Z +language: "es-ES" +application: "docsai-corpus" +page: + size: A4 + margins: { top: 70.85pt, bottom: 70.85pt, left: 85.05pt, right: 85.05pt, header: 35.4pt, footer: 35.4pt } + orientation: portrait +style-defaults: + font: { name: "Calibri", size: 11pt } + paragraph: { space-after: 8pt, line-height: 1.079 } +styles: + Heading1: + type: paragraph + name: "heading 1" + based-on: Normal + font: { name: "Calibri Light", size: 16pt, color: "#2E74B5" } + paragraph: { space-before: 12pt, space-after: 4pt, keep-with-next: true, outline-level: 0 } + Heading2: + type: paragraph + name: "heading 2" + based-on: Normal + font: { size: 13pt, color: "#2E74B5" } + paragraph: { outline-level: 1 } + Hyperlink: + type: character + font: { color: "#0563C1", underline: single } + Normal: + type: paragraph + default: true + TableGrid: + type: table + name: "Table Grid" +--- + +El mismo mapa de bits tres veces con geometrias distintas. + +![Original](assets/img-40e10599.png){height=90px name="Copia A" native-size=120x90 width=120px} + +![Mitad](assets/img-40e10599.png){height=45px name="Copia B" native-size=120x90 width=60px} + +![Flotante](assets/img-40e10599.png){anchor=floating height=3.75cm name="Copia C" native-size=120x90 relative-to=margin relative-to-v=paragraph width=5cm wrap=square wrap-side=largest x=2cm y=0px z-index=251658240}Tercera aparicion, flotante. diff --git a/corpus/docx/images-floating.docx b/corpus/docx/images-floating.docx new file mode 100644 index 0000000..c567fab Binary files /dev/null and b/corpus/docx/images-floating.docx differ diff --git a/corpus/docx/images-floating.expected.dmk.md b/corpus/docx/images-floating.expected.dmk.md new file mode 100644 index 0000000..2bbd75a --- /dev/null +++ b/corpus/docx/images-floating.expected.dmk.md @@ -0,0 +1,46 @@ +--- +docmark: "1.0" +source-format: docx +title: "Imagenes flotantes" +author: "docsai corpus" +last-modified-by: "docsai corpus" +created: 2026-01-01T00:00:00Z +modified: 2026-01-02T00:00:00Z +language: "es-ES" +application: "docsai-corpus" +page: + size: A4 + margins: { top: 70.85pt, bottom: 70.85pt, left: 85.05pt, right: 85.05pt, header: 35.4pt, footer: 35.4pt } + orientation: portrait +style-defaults: + font: { name: "Calibri", size: 11pt } + paragraph: { space-after: 8pt, line-height: 1.079 } +styles: + Heading1: + type: paragraph + name: "heading 1" + based-on: Normal + font: { name: "Calibri Light", size: 16pt, color: "#2E74B5" } + paragraph: { space-before: 12pt, space-after: 4pt, keep-with-next: true, outline-level: 0 } + Heading2: + type: paragraph + name: "heading 2" + based-on: Normal + font: { size: 13pt, color: "#2E74B5" } + paragraph: { outline-level: 1 } + Hyperlink: + type: character + font: { color: "#0563C1", underline: single } + Normal: + type: paragraph + default: true + TableGrid: + type: table + name: "Table Grid" +--- + +![Logo flotante cuadrado](assets/img-40e10599.png){anchor=floating height=2.6cm name=Logo native-size=120x90 relative-to=margin relative-to-v=paragraph width=3.5cm wrap=square wrap-side=right x=1.2cm y=0.5cm z-index=2}Texto que rodea a la imagen flotante anclada al margen. + +![Banner centrado](assets/img-8cbae2d9.png){align-h=center align-v=top anchor=floating height=4cm name=Banner native-size=64x64 relative-to=page width=6cm wrap=top-bottom z-index=3}Parrafo con imagen anclada a la pagina y alineacion simbolica. + +![Marca de agua](assets/img-40e10599.png){anchor=behind height=7.5cm name="Marca de agua" native-size=120x90 relative-to=page width=10cm wrap=none x=2cm y=8cm z-index=1}Parrafo sobre la marca de agua detras del texto. diff --git a/corpus/docx/images-inline.docx b/corpus/docx/images-inline.docx new file mode 100644 index 0000000..cb41d0c Binary files /dev/null and b/corpus/docx/images-inline.docx differ diff --git a/corpus/docx/images-inline.expected.dmk.md b/corpus/docx/images-inline.expected.dmk.md new file mode 100644 index 0000000..b651460 --- /dev/null +++ b/corpus/docx/images-inline.expected.dmk.md @@ -0,0 +1,50 @@ +--- +docmark: "1.0" +source-format: docx +title: "Imagenes en linea" +author: "docsai corpus" +last-modified-by: "docsai corpus" +created: 2026-01-01T00:00:00Z +modified: 2026-01-02T00:00:00Z +language: "es-ES" +application: "docsai-corpus" +page: + size: A4 + margins: { top: 70.85pt, bottom: 70.85pt, left: 85.05pt, right: 85.05pt, header: 35.4pt, footer: 35.4pt } + orientation: portrait +style-defaults: + font: { name: "Calibri", size: 11pt } + paragraph: { space-after: 8pt, line-height: 1.079 } +styles: + Heading1: + type: paragraph + name: "heading 1" + based-on: Normal + font: { name: "Calibri Light", size: 16pt, color: "#2E74B5" } + paragraph: { space-before: 12pt, space-after: 4pt, keep-with-next: true, outline-level: 0 } + Heading2: + type: paragraph + name: "heading 2" + based-on: Normal + font: { size: 13pt, color: "#2E74B5" } + paragraph: { outline-level: 1 } + Hyperlink: + type: character + font: { color: "#0563C1", underline: single } + Normal: + type: paragraph + default: true + TableGrid: + type: table + name: "Table Grid" +--- + +Imagen en linea a tamano nativo: + +![Diagrama de ventas](assets/img-40e10599.png){height=90px name=Diagrama native-size=120x90 title="Figura 1" width=120px} + +Imagen GIF en medio ![Icono verde](assets/img-7fcbd432.gif){height=32px name=Icono native-size=48x32 width=48px} del parrafo. + +Imagen EMF vectorial (no renderizable): + +![Logo vectorial](assets/img-a03c5ab4.emf){height=2cm name=LogoEMF render=unsupported width=4cm} diff --git a/corpus/docx/images-transformed.docx b/corpus/docx/images-transformed.docx new file mode 100644 index 0000000..4714576 Binary files /dev/null and b/corpus/docx/images-transformed.docx differ diff --git a/corpus/docx/images-transformed.expected.dmk.md b/corpus/docx/images-transformed.expected.dmk.md new file mode 100644 index 0000000..c5ab59a --- /dev/null +++ b/corpus/docx/images-transformed.expected.dmk.md @@ -0,0 +1,52 @@ +--- +docmark: "1.0" +source-format: docx +title: "Imagenes transformadas" +author: "docsai corpus" +last-modified-by: "docsai corpus" +created: 2026-01-01T00:00:00Z +modified: 2026-01-02T00:00:00Z +language: "es-ES" +application: "docsai-corpus" +page: + size: A4 + margins: { top: 70.85pt, bottom: 70.85pt, left: 85.05pt, right: 85.05pt, header: 35.4pt, footer: 35.4pt } + orientation: portrait +style-defaults: + font: { name: "Calibri", size: 11pt } + paragraph: { space-after: 8pt, line-height: 1.079 } +styles: + Heading1: + type: paragraph + name: "heading 1" + based-on: Normal + font: { name: "Calibri Light", size: 16pt, color: "#2E74B5" } + paragraph: { space-before: 12pt, space-after: 4pt, keep-with-next: true, outline-level: 0 } + Heading2: + type: paragraph + name: "heading 2" + based-on: Normal + font: { size: 13pt, color: "#2E74B5" } + paragraph: { outline-level: 1 } + Hyperlink: + type: character + font: { color: "#0563C1", underline: single } + Normal: + type: paragraph + default: true + TableGrid: + type: table + name: "Table Grid" +--- + +Imagen rotada 45 grados: + +![Rotada](assets/img-40e10599.png){anchor=floating height=3cm name=Rotada native-size=120x90 relative-to=margin relative-to-v=paragraph rotation=45 width=4cm wrap=square x=0px y=0px z-index=251658240} + +Imagen recortada con borde: + +![Recortada](assets/img-40e10599.png){border="1pt solid #000000" crop="10%,5%,20%,0%" height=2.25cm name=Recortada native-size=120x90 width=3cm} + +Imagen volteada y escalada al 50 %: + +![Volteada](assets/img-8cbae2d9.png){anchor=floating flip=hv height=1.5cm name=Volteada native-size=64x64 relative-to=margin relative-to-v=paragraph width=2cm wrap=square x=0px y=0px z-index=251658240} diff --git a/corpus/docx/images-vml.docx b/corpus/docx/images-vml.docx new file mode 100644 index 0000000..931a8d9 Binary files /dev/null and b/corpus/docx/images-vml.docx differ diff --git a/corpus/docx/images-vml.expected.dmk.md b/corpus/docx/images-vml.expected.dmk.md new file mode 100644 index 0000000..d8cdec0 --- /dev/null +++ b/corpus/docx/images-vml.expected.dmk.md @@ -0,0 +1,42 @@ +--- +docmark: "1.0" +source-format: docx +title: "Imagen VML" +author: "docsai corpus" +last-modified-by: "docsai corpus" +created: 2026-01-01T00:00:00Z +modified: 2026-01-02T00:00:00Z +language: "es-ES" +application: "docsai-corpus" +page: + size: A4 + margins: { top: 70.85pt, bottom: 70.85pt, left: 85.05pt, right: 85.05pt, header: 35.4pt, footer: 35.4pt } + orientation: portrait +style-defaults: + font: { name: "Calibri", size: 11pt } + paragraph: { space-after: 8pt, line-height: 1.079 } +styles: + Heading1: + type: paragraph + name: "heading 1" + based-on: Normal + font: { name: "Calibri Light", size: 16pt, color: "#2E74B5" } + paragraph: { space-before: 12pt, space-after: 4pt, keep-with-next: true, outline-level: 0 } + Heading2: + type: paragraph + name: "heading 2" + based-on: Normal + font: { size: 13pt, color: "#2E74B5" } + paragraph: { outline-level: 1 } + Hyperlink: + type: character + font: { color: "#0563C1", underline: single } + Normal: + type: paragraph + default: true + TableGrid: + type: table + name: "Table Grid" +--- + +![Imagen VML heredada](assets/img-8cbae2d9.png){anchor=floating height=90px name=_x0000_s1026 native-size=64x64 relative-to=margin relative-to-v=paragraph title=Heredada width=120px wrap=square wrap-side=right x=10pt y=5pt z-index=1}Parrafo con imagen VML heredada. diff --git a/corpus/docx/nested-lists.docx b/corpus/docx/nested-lists.docx new file mode 100644 index 0000000..54d0ed9 Binary files /dev/null and b/corpus/docx/nested-lists.docx differ diff --git a/corpus/docx/nested-lists.expected.dmk.md b/corpus/docx/nested-lists.expected.dmk.md new file mode 100644 index 0000000..027459a --- /dev/null +++ b/corpus/docx/nested-lists.expected.dmk.md @@ -0,0 +1,70 @@ +--- +docmark: "1.0" +source-format: docx +title: "Listas anidadas" +author: "docsai corpus" +last-modified-by: "docsai corpus" +created: 2026-01-01T00:00:00Z +modified: 2026-01-02T00:00:00Z +language: "es-ES" +application: "docsai-corpus" +page: + size: A4 + margins: { top: 70.85pt, bottom: 70.85pt, left: 85.05pt, right: 85.05pt, header: 35.4pt, footer: 35.4pt } + orientation: portrait +style-defaults: + font: { name: "Calibri", size: 11pt } + paragraph: { space-after: 8pt, line-height: 1.079 } +styles: + Heading1: + type: paragraph + name: "heading 1" + based-on: Normal + font: { name: "Calibri Light", size: 16pt, color: "#2E74B5" } + paragraph: { space-before: 12pt, space-after: 4pt, keep-with-next: true, outline-level: 0 } + Heading2: + type: paragraph + name: "heading 2" + based-on: Normal + font: { size: 13pt, color: "#2E74B5" } + paragraph: { outline-level: 1 } + Hyperlink: + type: character + font: { color: "#0563C1", underline: single } + ListParagraph: + type: paragraph + name: "List Paragraph" + based-on: Normal + Normal: + type: paragraph + default: true + TableGrid: + type: table + name: "Table Grid" +list-definitions: + L1: + levels: + - { format: decimal, text: "%1.", start: 1, indent: 48px, hanging: 24px } + - { format: lowerLetter, text: "%2)", start: 1, indent: 96px, hanging: 24px } + - { format: lowerRoman, text: "%3.", start: 1, indent: 144px, hanging: 12px } + L2: + levels: + - { format: bullet, text: "•", indent: 48px, hanging: 24px } + - { format: bullet, text: "o", indent: 96px, hanging: 24px } +--- + +Lista numerada anidada: + +1. Primer punto {.ListParagraph list=L1} + 1. Sub-punto a {.ListParagraph list=L1} + 1. Sub-sub-punto i {.ListParagraph list=L1} + 2. Sub-punto b {.ListParagraph} +2. Segundo punto {.ListParagraph} + +Lista de vinetas: + +- Vineta uno {.ListParagraph list=L2} + - Vineta anidada {.ListParagraph list=L2} +- Vineta dos {.ListParagraph} + +Parrafo posterior fuera de la lista. diff --git a/corpus/docx/table-merged.docx b/corpus/docx/table-merged.docx new file mode 100644 index 0000000..a2c5477 Binary files /dev/null and b/corpus/docx/table-merged.docx differ diff --git a/corpus/docx/table-merged.expected.dmk.md b/corpus/docx/table-merged.expected.dmk.md new file mode 100644 index 0000000..34400fa --- /dev/null +++ b/corpus/docx/table-merged.expected.dmk.md @@ -0,0 +1,50 @@ +--- +docmark: "1.0" +source-format: docx +title: "Tabla combinada" +author: "docsai corpus" +last-modified-by: "docsai corpus" +created: 2026-01-01T00:00:00Z +modified: 2026-01-02T00:00:00Z +language: "es-ES" +application: "docsai-corpus" +page: + size: A4 + margins: { top: 70.85pt, bottom: 70.85pt, left: 85.05pt, right: 85.05pt, header: 35.4pt, footer: 35.4pt } + orientation: portrait +style-defaults: + font: { name: "Calibri", size: 11pt } + paragraph: { space-after: 8pt, line-height: 1.079 } +styles: + Heading1: + type: paragraph + name: "heading 1" + based-on: Normal + font: { name: "Calibri Light", size: 16pt, color: "#2E74B5" } + paragraph: { space-before: 12pt, space-after: 4pt, keep-with-next: true, outline-level: 0 } + Heading2: + type: paragraph + name: "heading 2" + based-on: Normal + font: { size: 13pt, color: "#2E74B5" } + paragraph: { outline-level: 1 } + Hyperlink: + type: character + font: { color: "#0563C1", underline: single } + Normal: + type: paragraph + default: true + TableGrid: + type: table + name: "Table Grid" +--- + +Tabla con combinaciones: + +::: {.table col-widths="125pt,125pt,100pt,100pt" header-row=false style=TableGrid} +| | | | | +| ----------------- | ---------------------- | --- | ----- | +| Region | Trimestres {colspan=2} | | Total | +| Norte {rowspan=2} | 100 | 200 | 300 | +| | 150 | 250 | 400 | +::: diff --git a/corpus/docx/table-simple.docx b/corpus/docx/table-simple.docx new file mode 100644 index 0000000..2662745 Binary files /dev/null and b/corpus/docx/table-simple.docx differ diff --git a/corpus/docx/table-simple.expected.dmk.md b/corpus/docx/table-simple.expected.dmk.md new file mode 100644 index 0000000..3f08289 --- /dev/null +++ b/corpus/docx/table-simple.expected.dmk.md @@ -0,0 +1,52 @@ +--- +docmark: "1.0" +source-format: docx +title: "Tabla simple" +author: "docsai corpus" +last-modified-by: "docsai corpus" +created: 2026-01-01T00:00:00Z +modified: 2026-01-02T00:00:00Z +language: "es-ES" +application: "docsai-corpus" +page: + size: A4 + margins: { top: 70.85pt, bottom: 70.85pt, left: 85.05pt, right: 85.05pt, header: 35.4pt, footer: 35.4pt } + orientation: portrait +style-defaults: + font: { name: "Calibri", size: 11pt } + paragraph: { space-after: 8pt, line-height: 1.079 } +styles: + Heading1: + type: paragraph + name: "heading 1" + based-on: Normal + font: { name: "Calibri Light", size: 16pt, color: "#2E74B5" } + paragraph: { space-before: 12pt, space-after: 4pt, keep-with-next: true, outline-level: 0 } + Heading2: + type: paragraph + name: "heading 2" + based-on: Normal + font: { size: 13pt, color: "#2E74B5" } + paragraph: { outline-level: 1 } + Hyperlink: + type: character + font: { color: "#0563C1", underline: single } + Normal: + type: paragraph + default: true + TableGrid: + type: table + name: "Table Grid" +--- + +Tabla regular: + +::: {.table col-widths="200px,200px,200px" header-row=false style=TableGrid} +| | | | +| -------- | --- | --- | +| Concepto | T1 | T2 | +| Ventas | 100 | 200 | +| Costes | 40 | 60 | +::: + +Fin. diff --git a/corpus/generate.py b/corpus/generate.py new file mode 100644 index 0000000..95920ab --- /dev/null +++ b/corpus/generate.py @@ -0,0 +1,1296 @@ +#!/usr/bin/env python3 +"""Deterministic generator for the docsai test corpus. + +Every file under ``corpus/docx`` and ``corpus/xlsx`` is produced by this script +so that the corpus is reproducible, reviewable as source (the XML lives here, +not inside an opaque binary) and free of real user data. + +Usage:: + + python3 corpus/generate.py # regenerate everything + python3 corpus/generate.py --check # fail if the tree is out of date + +Design notes: + +* Packages are written with a fixed ZIP timestamp and fixed member order, so + regenerating produces byte-identical archives. +* Media are synthesised in pure Python (no Pillow): PNG via zlib, GIF and EMF + by hand. That keeps the generator dependency-free on the three CI platforms. +* Each document isolates *one* feature so a failing golden test points at a + single area of the reader. +""" + +from __future__ import annotations + +import argparse +import hashlib +import os +import struct +import sys +import zlib +import zipfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parent +FIXED_DATE = (2026, 1, 1, 0, 0, 0) + +# -------------------------------------------------------------------------- +# Media synthesis +# -------------------------------------------------------------------------- + + +def png(width: int, height: int, rgb: tuple[int, int, int]) -> bytes: + """Minimal, valid, solid-colour RGB PNG.""" + + def chunk(kind: bytes, data: bytes) -> bytes: + return ( + struct.pack(">I", len(data)) + + kind + + data + + struct.pack(">I", zlib.crc32(kind + data) & 0xFFFFFFFF) + ) + + raw = b"".join(b"\x00" + bytes(rgb) * width for _ in range(height)) + return ( + b"\x89PNG\r\n\x1a\n" + + chunk(b"IHDR", struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)) + + chunk(b"IDAT", zlib.compress(raw, 9)) + + chunk(b"IEND", b"") + ) + + +def gif(width: int, height: int, rgb: tuple[int, int, int]) -> bytes: + """Minimal single-colour GIF87a. Used to exercise content sniffing.""" + header = b"GIF87a" + struct.pack(" bytes: + """Minimal EMR_HEADER + EMR_EOF record pair. + + Not renderable, but structurally an EMF: enough for format sniffing and for + the "extract as-is, warn, never rasterise" path. + """ + bounds = struct.pack(" int: + return int(round(n * EMU_PER_PX)) + + +def cm(n: float) -> int: + return int(round(n * EMU_PER_CM)) + + +# -------------------------------------------------------------------------- +# Package plumbing +# -------------------------------------------------------------------------- + +XML_DECL = '\n' + +W_NS = ( + 'xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main" ' + 'xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships" ' + 'xmlns:wp="http://schemas.openxmlformats.org/drawingml/2006/wordprocessingDrawing" ' + 'xmlns:a="http://schemas.openxmlformats.org/drawingml/2006/main" ' + 'xmlns:pic="http://schemas.openxmlformats.org/drawingml/2006/picture" ' + 'xmlns:v="urn:schemas-microsoft-com:vml" ' + 'xmlns:o="urn:schemas-microsoft-com:office:office" ' + 'xmlns:w10="urn:schemas-microsoft-com:office:word"' +) + +RELS_NS = 'xmlns="http://schemas.openxmlformats.org/package/2006/relationships"' +REL_BASE = "http://schemas.openxmlformats.org/officeDocument/2006/relationships" + +CONTENT_TYPE = { + ".png": "image/png", + ".gif": "image/gif", + ".emf": "image/x-emf", +} + + +def write_package(path: Path, parts: dict[str, bytes]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with zipfile.ZipFile(path, "w", zipfile.ZIP_DEFLATED) as zf: + for name in sorted(parts): + info = zipfile.ZipInfo(name, date_time=FIXED_DATE) + info.compress_type = zipfile.ZIP_DEFLATED + info.external_attr = 0o600 << 16 + zf.writestr(info, parts[name]) + + +def core_props(title: str, author: str = "docsai corpus", lang: str = "es-ES") -> bytes: + return ( + XML_DECL + + '' + f"{title}{author}" + f"{author}" + f"{lang}" + '2026-01-01T00:00:00Z' + '2026-01-02T00:00:00Z' + "" + ).encode() + + +APP_PROPS = ( + XML_DECL + + '' + "docsai-corpusdocsai" + "" +).encode() + + +def custom_props(props: dict[str, str]) -> bytes: + items = [] + for i, (k, v) in enumerate(props.items(), start=2): + items.append( + f'' + f'{v}' + "" + ) + return ( + XML_DECL + + '' + + "".join(items) + + "" + ).encode() + + +DEFAULT_STYLES = """ + + + + + + + + + + + + + + + + + + + + + + + + + +""".format(ns=W_NS) + + +def build_docx( + name: str, + body: str, + *, + title: str, + styles: str = DEFAULT_STYLES, + numbering: str | None = None, + media: dict[str, bytes] | None = None, + rels: list[tuple[str, str, str]] | None = None, + headers: dict[str, str] | None = None, + footnotes: str | None = None, + custom: dict[str, str] | None = None, +) -> None: + """Assemble a .docx package. + + ``rels`` entries are ``(rId, type_suffix, target)`` and ``headers`` maps a + part name (``header1.xml``) to its XML body. + """ + media = media or {} + rels = list(rels or []) + headers = headers or {} + + parts: dict[str, bytes] = {} + overrides = [ + ('/word/document.xml', "application/vnd.openxmlformats-officedocument.wordprocessingml.document.main+xml"), + ("/word/styles.xml", "application/vnd.openxmlformats-officedocument.wordprocessingml.styles+xml"), + ("/docProps/core.xml", "application/vnd.openxmlformats-package.core-properties+xml"), + ("/docProps/app.xml", "application/vnd.openxmlformats-officedocument.extended-properties+xml"), + ] + + parts["word/document.xml"] = (XML_DECL + body).encode() + parts["word/styles.xml"] = (XML_DECL + styles).encode() + parts["docProps/core.xml"] = core_props(title) + parts["docProps/app.xml"] = APP_PROPS + + rels.append(("rIdStyles", "styles", "styles.xml")) + + if numbering is not None: + parts["word/numbering.xml"] = (XML_DECL + numbering).encode() + rels.append(("rIdNum", "numbering", "numbering.xml")) + overrides.append( + ("/word/numbering.xml", "application/vnd.openxmlformats-officedocument.wordprocessingml.numbering+xml") + ) + + if footnotes is not None: + parts["word/footnotes.xml"] = (XML_DECL + footnotes).encode() + rels.append(("rIdFn", "footnotes", "footnotes.xml")) + overrides.append( + ("/word/footnotes.xml", "application/vnd.openxmlformats-officedocument.wordprocessingml.footnotes+xml") + ) + + for part, xml in sorted(headers.items()): + parts[f"word/{part}"] = (XML_DECL + xml).encode() + kind = "header" if part.startswith("header") else "footer" + overrides.append( + ( + f"/word/{part}", + f"application/vnd.openxmlformats-officedocument.wordprocessingml.{kind}+xml", + ) + ) + + if custom is not None: + parts["docProps/custom.xml"] = custom_props(custom) + overrides.append( + ("/docProps/custom.xml", "application/vnd.openxmlformats-officedocument.custom-properties+xml") + ) + + for fname, blob in sorted(media.items()): + parts[f"word/media/{fname}"] = blob + + exts = sorted({Path(f).suffix for f in media}) + defaults = ['', + ''] + defaults += [f'' for e in exts] + + parts["[Content_Types].xml"] = ( + XML_DECL + + '' + + "".join(defaults) + + "".join(f'' for p, c in overrides) + + "" + ).encode() + + pkg_rels = [ + ("rId1", "officeDocument", "word/document.xml"), + ("rId2", "metadata/core-properties", "docProps/core.xml"), + ("rId3", "extended-properties", "docProps/app.xml"), + ] + if custom is not None: + pkg_rels.append(("rId4", "custom-properties", "docProps/custom.xml")) + pkg_rel_xml = "".join( + f'' for i, t, tg in pkg_rels + ) + # core-properties lives in a different namespace family + pkg_rel_xml = pkg_rel_xml.replace( + f"{REL_BASE}/metadata/core-properties", + "http://schemas.openxmlformats.org/package/2006/relationships/metadata/core-properties", + ) + parts["_rels/.rels"] = ( + XML_DECL + f"" + pkg_rel_xml + "" + ).encode() + + doc_rel_xml = [] + for rid, kind, target in rels: + mode = ' TargetMode="External"' if target.startswith("http") else "" + doc_rel_xml.append( + f'' + ) + parts["word/_rels/document.xml.rels"] = ( + XML_DECL + f"" + "".join(doc_rel_xml) + "" + ).encode() + + write_package(ROOT / "docx" / name, parts) + + +# -------------------------------------------------------------------------- +# Drawing helpers +# -------------------------------------------------------------------------- + + +def blip_fill(rid: str, crop: str = "") -> str: + return ( + f'{crop}' + "" + ) + + +def graphic(rid: str, w: int, h: int, *, xfrm: str = "", crop: str = "", ln: str = "", + pic_name: str = "Picture") -> str: + if not xfrm: + xfrm = f'' + return ( + '' + f'' + + blip_fill(rid, crop) + + f'{xfrm}{ln}' + "" + ) + + +def inline_image(rid: str, w: int, h: int, *, alt: str = "", title: str = "", + name: str = "Picture 1", doc_id: int = 1, xfrm: str = "", + crop: str = "", ln: str = "", hlink: str = "") -> str: + title_attr = f' title="{title}"' if title else "" + doc_pr = f'{hlink}' + return ( + "" + f'' + f'' + + doc_pr + + '' + + graphic(rid, w, h, xfrm=xfrm, crop=crop, ln=ln) + + "" + ) + + +def anchor_image( + rid: str, + w: int, + h: int, + *, + rel_h: str = "margin", + rel_v: str = "paragraph", + pos_h: str = "", + pos_v: str = "", + wrap: str = '', + behind: bool = False, + z: int = 251658240, + alt: str = "", + name: str = "Picture", + doc_id: int = 10, + xfrm: str = "", + crop: str = "", + ln: str = "", +) -> str: + pos_h = pos_h or "0" + pos_v = pos_v or "0" + return ( + "" + f'' + '' + f'{pos_h}' + f'{pos_v}' + f'' + f"{wrap}" + f'' + '' + + graphic(rid, w, h, xfrm=xfrm, crop=crop, ln=ln) + + "" + ) + + +SECT_PR = ( + '' + '' +) + + +def document(body: str, sect: str = SECT_PR) -> str: + return f"{body}{sect}" + + +def p(runs: str, ppr: str = "") -> str: + return f"{ppr}{runs}" + + +def r(text: str, rpr: str = "") -> str: + return f'{rpr}{text}' + + +# -------------------------------------------------------------------------- +# DOCX corpus +# -------------------------------------------------------------------------- + + +def docx_basic_text() -> None: + body = ( + p(r("Primer parrafo del documento.")) + + p(r("Segundo parrafo con un salto") + "" + r("de linea manual.")) + + p(r("Caracteres que Markdown escapa: *asterisco* _guion_ #almohadilla " + "|tuberia| [corchete] <angulo> `backtick` \\barra.")) + + p(r("")) + + p(r("Parrafo final tras uno vacio.")) + ) + build_docx("basic-text.docx", document(body), title="Texto basico") + + +def docx_basic_styles() -> None: + body = ( + p(r("Titulo de nivel 1"), '') + + p( + r("negrita", "") + + r(" ") + + r("cursiva", "") + + r(" ") + + r("tachado", "") + + r(" ") + + r("subrayado", '') + ) + + p(r("Titulo de nivel 2"), '') + + p( + r("rojo", '') + + r(" ") + + r("resaltado", '') + + r(" ") + + r("Arial 14", '') + + r(" x") + + r("2", '') + + r(" H") + + r("2", '') + + r("O") + ) + + p( + r("Visita ") + + '' + + r("el sitio", '') + + "" + + r(" para mas informacion.") + ) + + p(r("Parrafo centrado con sangria."), + '' + '') + ) + build_docx( + "basic-styles.docx", + document(body), + title="Estilos basicos", + rels=[("rIdLink", "hyperlink", "https://example.com/docsai")], + ) + + +NUMBERING = """ + + + + + + + + + + + + + + + + + + +""".format(ns=W_NS) + + +def li(text: str, num_id: int, ilvl: int) -> str: + return p( + r(text), + f'' + f'', + ) + + +def docx_nested_lists() -> None: + body = ( + p(r("Lista numerada anidada:")) + + li("Primer punto", 1, 0) + + li("Sub-punto a", 1, 1) + + li("Sub-sub-punto i", 1, 2) + + li("Sub-punto b", 1, 1) + + li("Segundo punto", 1, 0) + + p(r("Lista de vinetas:")) + + li("Vineta uno", 2, 0) + + li("Vineta anidada", 2, 1) + + li("Vineta dos", 2, 0) + + p(r("Parrafo posterior fuera de la lista.")) + ) + styles = DEFAULT_STYLES.replace( + "", + '' + '' + "", + ) + build_docx( + "nested-lists.docx", + document(body), + title="Listas anidadas", + styles=styles, + numbering=NUMBERING, + ) + + +def tc(text: str, tcpr: str = "", width: int = 3000) -> str: + return ( + f'{tcpr}' + + p(r(text)) + + "" + ) + + +def docx_table_simple() -> None: + grid = '' + rows = ( + "" + tc("Concepto") + tc("T1") + tc("T2") + "" + "" + tc("Ventas") + tc("100") + tc("200") + "" + "" + tc("Costes") + tc("40") + tc("60") + "" + ) + tbl = ( + '' + '' + grid + rows + "" + ) + build_docx( + "table-simple.docx", + document(p(r("Tabla regular:")) + tbl + p(r("Fin."))), + title="Tabla simple", + ) + + +def docx_table_merged() -> None: + grid = ('' + '') + rows = ( + "" + + tc("Region", "", 2500) + + tc("Trimestres", '', 4500) + + tc("Total", "", 2000) + + "" + "" + + tc("Norte", '', 2500) + + tc("100", "", 2500) + + tc("200", "", 2000) + + tc("300", "", 2000) + + "" + "" + + tc("", '', 2500) + + tc("150", "", 2500) + + tc("250", "", 2000) + + tc("400", "", 2000) + + "" + ) + tbl = ( + '' + '' + grid + rows + "" + ) + build_docx( + "table-merged.docx", + document(p(r("Tabla con combinaciones:")) + tbl), + title="Tabla combinada", + ) + + +def docx_images_inline() -> None: + body = ( + p(r("Imagen en linea a tamano nativo:")) + + p(inline_image("rIdImg1", px(120), px(90), alt="Diagrama de ventas", + title="Figura 1", name="Diagrama")) + + p(r("Imagen GIF en medio ") + inline_image("rIdImg2", px(48), px(32), + alt="Icono verde", doc_id=2, + name="Icono") + r(" del parrafo.")) + + p(r("Imagen EMF vectorial (no renderizable):")) + + p(inline_image("rIdImg3", cm(4), cm(2), alt="Logo vectorial", doc_id=3, name="LogoEMF")) + ) + build_docx( + "images-inline.docx", + document(body), + title="Imagenes en linea", + media={"image1.png": BLUE_PNG, "image2.gif": GREEN_GIF, "image3.emf": LOGO_EMF}, + rels=[ + ("rIdImg1", "image", "media/image1.png"), + ("rIdImg2", "image", "media/image2.gif"), + ("rIdImg3", "image", "media/image3.emf"), + ], + ) + + +def docx_images_floating() -> None: + body = ( + p( + anchor_image( + "rIdImg1", cm(3.5), cm(2.6), + rel_h="margin", rel_v="paragraph", + pos_h=f"{cm(1.2)}", + pos_v=f"{cm(0.5)}", + wrap='', + z=2, alt="Logo flotante cuadrado", name="Logo", + ) + + r("Texto que rodea a la imagen flotante anclada al margen.") + ) + + p( + anchor_image( + "rIdImg2", cm(6), cm(4), + rel_h="page", rel_v="page", + pos_h="center", + pos_v="top", + wrap="", + z=3, alt="Banner centrado", name="Banner", doc_id=11, + ) + + r("Parrafo con imagen anclada a la pagina y alineacion simbolica.") + ) + + p( + anchor_image( + "rIdImg1", cm(10), cm(7.5), + rel_h="page", rel_v="page", + pos_h=f"{cm(2)}", + pos_v=f"{cm(8)}", + wrap="", behind=True, z=1, + alt="Marca de agua", name="Marca de agua", doc_id=12, + ) + + r("Parrafo sobre la marca de agua detras del texto.") + ) + ) + build_docx( + "images-floating.docx", + document(body), + title="Imagenes flotantes", + media={"image1.png": BLUE_PNG, "image2.png": RED_PNG}, + rels=[ + ("rIdImg1", "image", "media/image1.png"), + ("rIdImg2", "image", "media/image2.png"), + ], + ) + + +def docx_images_transformed() -> None: + rot45 = 45 * 60000 + xfrm_rot = f'' + xfrm_flip = (f'' + f'') + crop = '' + border = ('' + '') + body = ( + p(r("Imagen rotada 45 grados:")) + + p(anchor_image("rIdImg1", cm(4), cm(3), xfrm=xfrm_rot, + alt="Rotada", name="Rotada")) + + p(r("Imagen recortada con borde:")) + + p(inline_image("rIdImg1", cm(3), cm(2.25), alt="Recortada", doc_id=2, + name="Recortada", crop=crop, ln=border)) + + p(r("Imagen volteada y escalada al 50 %:")) + + p(anchor_image("rIdImg2", cm(2), cm(1.5), xfrm=xfrm_flip, + alt="Volteada", name="Volteada", doc_id=13)) + ) + build_docx( + "images-transformed.docx", + document(body), + title="Imagenes transformadas", + media={"image1.png": BLUE_PNG, "image2.png": RED_PNG}, + rels=[ + ("rIdImg1", "image", "media/image1.png"), + ("rIdImg2", "image", "media/image2.png"), + ], + ) + + +def docx_images_duplicated() -> None: + body = ( + p(r("El mismo mapa de bits tres veces con geometrias distintas.")) + + p(inline_image("rIdA", px(120), px(90), alt="Original", name="Copia A")) + + p(inline_image("rIdB", px(60), px(45), alt="Mitad", doc_id=2, name="Copia B")) + + p( + anchor_image( + "rIdC", cm(5), cm(3.75), + rel_h="margin", rel_v="paragraph", + pos_h=f"{cm(2)}", + wrap='', + alt="Flotante", name="Copia C", doc_id=3, + ) + + r("Tercera aparicion, flotante.") + ) + ) + # Three distinct relationships and three distinct parts holding identical bytes: + # the AssetStore must collapse them into one asset. + build_docx( + "images-duplicated.docx", + document(body), + title="Imagenes duplicadas", + media={"image1.png": BLUE_PNG, "image2.png": BLUE_PNG, "image3.png": BLUE_PNG}, + rels=[ + ("rIdA", "image", "media/image1.png"), + ("rIdB", "image", "media/image2.png"), + ("rIdC", "image", "media/image3.png"), + ], + ) + + +def docx_headers_footers() -> None: + header_default = ( + f"" + p(r("Cabecera por defecto")) + "" + ) + header_first = f"" + p(r("Cabecera de primera pagina")) + "" + footer_default = ( + f"" + + p( + r("Pagina ") + + '' + r("1") + "" + + r(" de ") + + '' + r("3") + "" + ) + + "" + ) + sect = ( + '' + '' + '' + '' + '' + ) + body = p(r("Cuerpo del documento con cabeceras y pies.")) + build_docx( + "headers-footers.docx", + document(body, sect), + title="Cabeceras y pies", + headers={ + "header1.xml": header_default, + "header2.xml": header_first, + "footer1.xml": footer_default, + }, + rels=[ + ("rIdH1", "header", "header1.xml"), + ("rIdH2", "header", "header2.xml"), + ("rIdF1", "footer", "footer1.xml"), + ], + ) + + +def docx_footnotes() -> None: + fn = ( + f"" + '' + p(r("")) + "" + '' + p(r("")) + "" + '' + + p('' + + r(" Primera nota al pie.")) + + "" + '' + + p('' + + r(" Segunda nota, con ") + r("negrita", "") + r(".")) + + "" + "" + ) + body = ( + p(r("Texto con nota") + + '' + '' + + r(" y sigue.")) + + p(r("Otro parrafo con segunda nota") + + '' + '' + + r(".")) + ) + styles = DEFAULT_STYLES.replace( + "", + '' + '' + "", + ) + build_docx("footnotes.docx", document(body), title="Notas al pie", + styles=styles, footnotes=fn) + + +CUSTOM_STYLES = DEFAULT_STYLES.replace( + "", + """ + + + + + + + + + + + + """, +) + + +def docx_custom_styles() -> None: + body = ( + p(r("Parrafo normal sin atributos.")) + + p(r("Parrafo con estilo Destacado."), '') + + p(r("Estilo heredado con delta directo."), + '') + + p(r("Texto con ") + r("estilo de caracter", '') + + r(" y delta ") + + r("adicional", '') + + r(".")) + ) + build_docx( + "custom-styles.docx", + document(body), + title="Estilos personalizados", + styles=CUSTOM_STYLES, + custom={"Departamento": "Ventas", "Revision": "3"}, + ) + + +def docx_images_vml() -> None: + """Legacy VML picture (`w:pict`), typical of documents converted from .doc.""" + pict = ( + "" + '' + '' + '' + "" + "" + ) + body = p(pict + r("Parrafo con imagen VML heredada.")) + build_docx( + "images-vml.docx", + document(body), + title="Imagen VML", + media={"image1.png": RED_PNG}, + rels=[("rIdVml", "image", "media/image1.png")], + ) + + +def docx_fields_raw() -> None: + """Fields and structures with no DocMark representation (raw-block path).""" + sdt = ( + "" + '' + "" + p(r("Contenido dentro de un control de contenido.")) + "" + "" + ) + toc = ( + p('' + ' TOC \\o "1-3" \\h ' + '' + + r("Tabla de contenido generada") + + '') + ) + body = ( + p(r("Antes del control de contenido.")) + + sdt + + toc + + p(r("Fecha: ") + '' + + r("01/01/2026") + "") + + p(r("Despues.")) + ) + build_docx("fields-raw.docx", document(body), title="Campos y bloques opacos") + + +# -------------------------------------------------------------------------- +# XLSX corpus (consumed from Fase 3; generated now so the corpus is complete) +# -------------------------------------------------------------------------- + +S_NS = 'xmlns="http://schemas.spreadsheetml.org/2006/main"' +X_NS = ( + 'xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main" ' + 'xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships"' +) + +BASE_XL_STYLES = f""" + + + + + + + + + + + + + + + + + + + + + + + + + + + + +""" + + +def build_xlsx( + name: str, + sheets: list[tuple[str, str]], + *, + title: str, + shared: list[str] | None = None, + defined_names: str = "", + media: dict[str, bytes] | None = None, + drawings: dict[str, str] | None = None, + sheet_rels: dict[int, list[tuple[str, str, str]]] | None = None, +) -> None: + media = media or {} + drawings = drawings or {} + sheet_rels = sheet_rels or {} + parts: dict[str, bytes] = {} + overrides = [ + ("/xl/workbook.xml", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet.main+xml"), + ("/xl/styles.xml", "application/vnd.openxmlformats-officedocument.spreadsheetml.styles+xml"), + ("/docProps/core.xml", "application/vnd.openxmlformats-package.core-properties+xml"), + ("/docProps/app.xml", "application/vnd.openxmlformats-officedocument.extended-properties+xml"), + ] + + sheet_tags = "".join( + f'' + for i, (sname, _) in enumerate(sheets, start=1) + ) + parts["xl/workbook.xml"] = ( + XML_DECL + + f"{sheet_tags}{defined_names}" + ).encode() + parts["xl/styles.xml"] = (XML_DECL + BASE_XL_STYLES).encode() + parts["docProps/core.xml"] = core_props(title) + parts["docProps/app.xml"] = APP_PROPS + + wb_rels = [ + (f"rIdSheet{i}", "worksheet", f"worksheets/sheet{i}.xml") + for i in range(1, len(sheets) + 1) + ] + wb_rels.append(("rIdStyles", "styles", "styles.xml")) + + for i, (_, xml) in enumerate(sheets, start=1): + parts[f"xl/worksheets/sheet{i}.xml"] = (XML_DECL + xml).encode() + overrides.append( + ( + f"/xl/worksheets/sheet{i}.xml", + "application/vnd.openxmlformats-officedocument.spreadsheetml.worksheet+xml", + ) + ) + if i in sheet_rels: + rel_xml = "".join( + f'' + for rid, kind, tgt in sheet_rels[i] + ) + parts[f"xl/worksheets/_rels/sheet{i}.xml.rels"] = ( + XML_DECL + f"" + rel_xml + "" + ).encode() + + if shared: + items = "".join(f"{s}" for s in shared) + parts["xl/sharedStrings.xml"] = ( + XML_DECL + + f'{items}' + ).encode() + wb_rels.append(("rIdSst", "sharedStrings", "sharedStrings.xml")) + overrides.append( + ("/xl/sharedStrings.xml", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sharedStrings+xml") + ) + + for dname, dxml in sorted(drawings.items()): + parts[f"xl/drawings/{dname}"] = (XML_DECL + dxml).encode() + overrides.append( + (f"/xl/drawings/{dname}", "application/vnd.openxmlformats-officedocument.drawing+xml") + ) + + if media: + rel_xml = "".join( + f'' + for n, f in enumerate(sorted(media), start=1) + ) + parts["xl/drawings/_rels/drawing1.xml.rels"] = ( + XML_DECL + f"" + rel_xml + "" + ).encode() + for fname, blob in sorted(media.items()): + parts[f"xl/media/{fname}"] = blob + + exts = sorted({Path(f).suffix for f in media}) + defaults = ['', + ''] + defaults += [f'' for e in exts] + + parts["[Content_Types].xml"] = ( + XML_DECL + + '' + + "".join(defaults) + + "".join(f'' for p_, c in overrides) + + "" + ).encode() + + parts["xl/_rels/workbook.xml.rels"] = ( + XML_DECL + + f"" + + "".join( + f'' + for rid, kind, tgt in wb_rels + ) + + "" + ).encode() + + parts["_rels/.rels"] = ( + XML_DECL + + f"" + + f'' + + '' + + f'' + + "" + ).encode() + + write_package(ROOT / "xlsx" / name, parts) + + +def sheet(rows: str, extra: str = "", dim: str = "A1:D10") -> str: + return ( + f'' + '' + f"{rows}{extra}" + ) + + +def row(idx: int, cells: str) -> str: + return f'{cells}' + + +def c(ref: str, value: str = "", *, t: str = "", s: int = 0, f: str = "") -> str: + attrs = f' r="{ref}"' + if t: + attrs += f' t="{t}"' + if s: + attrs += f' s="{s}"' + inner = (f"{f}" if f else "") + (f"{value}" if value != "" else "") + return f"{inner}" + + +def xlsx_values_types() -> None: + rows = ( + row(1, c("A1", "0", t="s", s=4) + c("B1", "1", t="s", s=4)) + + row(2, c("A2", "2", t="s") + c("B2", "42")) + + row(3, c("A3", "3", t="s") + c("B3", "3.14159")) + + row(4, c("A4", "4", t="s") + c("B4", "1", t="b")) + + row(5, c("A5", "5", t="s") + c("B5", "#DIV/0!", t="e")) + + row(6, c("A6", "6", t="s") + c("B6", "45658", s=2)) + + row(7, c("A7", "7", t="s") + 'en linea') + ) + build_xlsx( + "values-types.xlsx", + [("Tipos", sheet(rows, dim="A1:B7"))], + title="Tipos de valor", + shared=["Concepto", "Valor", "Entero", "Decimal", "Booleano", "Error", "Fecha", "Cadena"], + ) + + +def xlsx_formulas_basic() -> None: + rows = ( + row(1, c("A1", "0", t="s", s=4) + c("B1", "1", t="s", s=4) + c("C1", "2", t="s", s=4)) + + row(2, c("A2", "3", t="s") + c("B2", "100") + c("C2", "200")) + + row(3, c("A3", "4", t="s") + c("B3", "150") + c("C3", "250")) + + row(4, c("A4", "5", t="s") + c("B4", "250", f="SUM(B2:B3)") + + c("C4", "450", f="SUM(C2:C3)")) + + row(5, c("A5", "6", t="s") + c("B5", "700", f="B4+C4")) + ) + build_xlsx( + "formulas-basic.xlsx", + [("Datos", sheet(rows, dim="A1:C5"))], + title="Formulas basicas", + shared=["Producto", "T1", "T2", "Widgets", "Gadgets", "Total", "Suma general"], + defined_names='Datos!$B$5', + ) + + +def xlsx_formulas_shared() -> None: + rows = ( + row(1, c("A1", "10") + c("B1", "20") + + 'A1+B130') + + row(2, c("A2", "11") + c("B2", "21") + + '32') + + row(3, c("A3", "12") + c("B3", "22") + + '34') + + row(4, 'SUM(A1:A3*B1:B3)1274') + ) + build_xlsx( + "formulas-shared.xlsx", + [("Compartidas", sheet(rows, dim="A1:C4"))], + title="Formulas compartidas y de matriz", + ) + + +def xlsx_number_formats() -> None: + rows = ( + row(1, c("A1", "0", t="s", s=4) + c("B1", "1", t="s", s=4)) + + row(2, c("A2", "2", t="s") + c("B2", "1234.5", s=1)) + + row(3, c("A3", "3", t="s") + c("B3", "45658", s=2)) + + row(4, c("A4", "4", t="s") + c("B4", "0.175", s=3)) + + row(5, c("A5", "5", t="s") + c("B5", "1234567", s=5)) + ) + build_xlsx( + "number-formats.xlsx", + [("Formatos", sheet(rows, dim="A1:B5"))], + title="Formatos de numero", + shared=["Concepto", "Valor", "Moneda", "Fecha", "Porcentaje", "Millares"], + ) + + +def xlsx_merged_cells() -> None: + rows = ( + row(1, c("A1", "0", t="s", s=4) + c("B1", "") + c("C1", "1", t="s", s=4)) + + row(2, c("A2", "2", t="s") + c("B2", "100") + c("C2", "200")) + + row(3, c("A3", "3", t="s") + c("B3", "150") + c("C3", "250")) + ) + extra = ( + '' + '' + "" + ) + cols = '' + body = sheet(rows, extra, dim="A1:C3").replace("", cols + "") + build_xlsx( + "merged-cells.xlsx", + [("Combinadas", body)], + title="Celdas combinadas", + shared=["Cabecera combinada", "Total", "Norte", "Sur"], + ) + + +def xlsx_images_anchored() -> None: + xdr = ( + 'xmlns:xdr="http://schemas.openxmlformats.org/drawingml/2006/spreadsheetDrawing" ' + 'xmlns:a="http://schemas.openxmlformats.org/drawingml/2006/main" ' + 'xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships"' + ) + + def pic(rid: str, ident: int, name: str, alt: str, w: int = 0, h: int = 0) -> str: + ext = f'' if w else "" + return ( + f'' + "" + f'' + f'{ext}' + '' + ) + + drawing = ( + f"" + "" + f'1{px(12)}' + f'1{px(3)}' + "30" + "70" + + pic("rIdMedia1", 1, "Logo", "Logo de la empresa") + + "" + "" + "50" + "190" + f'' + + pic("rIdMedia2", 2, "Firma", "Firma", px(180), px(60)) + + "" + "" + f'' + + pic("rIdMedia1", 3, "Marca de agua", "Marca de agua", cm(10), cm(7.5)) + + "" + "" + ) + rows = row(1, c("A1", "0", t="s", s=4)) + row(2, c("A2", "1", t="s")) + body = sheet(rows, '', dim="A1:A2") + build_xlsx( + "images-anchored.xlsx", + [("Hoja", body)], + title="Imagenes ancladas", + shared=["Hoja con imagenes", "Contenido"], + media={"image1.png": BLUE_PNG, "image2.gif": GREEN_GIF}, + drawings={"drawing1.xml": drawing}, + sheet_rels={1: [("rIdDraw", "drawing", "../drawings/drawing1.xml")]}, + ) + + +# -------------------------------------------------------------------------- + + +GENERATORS = [ + docx_basic_text, + docx_basic_styles, + docx_nested_lists, + docx_table_simple, + docx_table_merged, + docx_images_inline, + docx_images_floating, + docx_images_transformed, + docx_images_duplicated, + docx_headers_footers, + docx_footnotes, + docx_custom_styles, + docx_images_vml, + docx_fields_raw, + xlsx_values_types, + xlsx_formulas_basic, + xlsx_formulas_shared, + xlsx_number_formats, + xlsx_merged_cells, + xlsx_images_anchored, +] + + +def digest_tree() -> dict[str, str]: + out = {} + for sub in ("docx", "xlsx"): + d = ROOT / sub + if not d.exists(): + continue + for f in sorted(d.iterdir()): + if f.is_file(): + out[f"{sub}/{f.name}"] = hashlib.sha256(f.read_bytes()).hexdigest() + return out + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--check", action="store_true", + help="regenerate into memory and fail if the tree differs") + args = ap.parse_args() + + before = digest_tree() + for gen in GENERATORS: + gen() + after = digest_tree() + + if args.check and before != after: + changed = sorted(set(before) ^ set(after)) or [ + k for k in after if before.get(k) != after[k] + ] + print("corpus out of date, regenerate with corpus/generate.py:", file=sys.stderr) + for k in changed: + print(f" {k}", file=sys.stderr) + return 1 + + print(f"generated {len(after)} corpus files under {ROOT}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/corpus/xlsx/formulas-basic.xlsx b/corpus/xlsx/formulas-basic.xlsx new file mode 100644 index 0000000..0ccdc42 Binary files /dev/null and b/corpus/xlsx/formulas-basic.xlsx differ diff --git a/corpus/xlsx/formulas-shared.xlsx b/corpus/xlsx/formulas-shared.xlsx new file mode 100644 index 0000000..29ae313 Binary files /dev/null and b/corpus/xlsx/formulas-shared.xlsx differ diff --git a/corpus/xlsx/images-anchored.xlsx b/corpus/xlsx/images-anchored.xlsx new file mode 100644 index 0000000..7b98574 Binary files /dev/null and b/corpus/xlsx/images-anchored.xlsx differ diff --git a/corpus/xlsx/merged-cells.xlsx b/corpus/xlsx/merged-cells.xlsx new file mode 100644 index 0000000..8db41fc Binary files /dev/null and b/corpus/xlsx/merged-cells.xlsx differ diff --git a/corpus/xlsx/number-formats.xlsx b/corpus/xlsx/number-formats.xlsx new file mode 100644 index 0000000..727f815 Binary files /dev/null and b/corpus/xlsx/number-formats.xlsx differ diff --git a/corpus/xlsx/values-types.xlsx b/corpus/xlsx/values-types.xlsx new file mode 100644 index 0000000..c2943ca Binary files /dev/null and b/corpus/xlsx/values-types.xlsx differ diff --git a/crates/docsai-cli/Cargo.toml b/crates/docsai-cli/Cargo.toml new file mode 100644 index 0000000..e02a302 --- /dev/null +++ b/crates/docsai-cli/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "docsai-cli" +description = "The docsai command-line interface" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +authors.workspace = true + +[[bin]] +name = "docsai" +path = "src/main.rs" + +[dependencies] +anyhow = { workspace = true } +clap = { workspace = true } +docsai-convert = { workspace = true } +docsai-model = { workspace = true } +serde_json = { workspace = true } +tracing = { workspace = true } +tracing-subscriber = { workspace = true } diff --git a/crates/docsai-cli/src/main.rs b/crates/docsai-cli/src/main.rs new file mode 100644 index 0000000..9fb8677 --- /dev/null +++ b/crates/docsai-cli/src/main.rs @@ -0,0 +1,205 @@ +//! The `docsai` command-line interface. + +use std::io::Write; +use std::path::PathBuf; +use std::process::ExitCode; + +use clap::{Parser, Subcommand}; +use docsai_convert::{ConvertOptions, Fidelity, SUPPORT}; +use docsai_model::Format; + +/// Exit codes (arquitectura §5). +const EXIT_OK: u8 = 0; +const EXIT_WARNINGS: u8 = 1; +const EXIT_INPUT: u8 = 2; +const EXIT_UNSUPPORTED: u8 = 3; + +#[derive(Parser)] +#[command( + name = "docsai", + version, + about = "Bidirectional converter between Office/LibreOffice documents and DocMark", + long_about = None +)] +struct Cli { + #[command(subcommand)] + command: Command, + + /// Print warnings in detail. + #[arg(long, global = true)] + verbose: bool, +} + +#[derive(Subcommand)] +enum Command { + /// Convert a document. + Convert { + /// Input document. + input: PathBuf, + /// Output file; without it the result goes to stdout. + #[arg(short, long)] + output: Option, + /// Force the target format instead of inferring it from `--output`. + #[arg(long, value_name = "FMT")] + to: Option, + /// How much of the source survives: full, standard or plain. + #[arg(long, default_value = "full")] + fidelity: String, + /// Where to extract media. Defaults to `assets/` next to the output. + #[arg(long, value_name = "DIR")] + assets_dir: Option, + /// Print the conversion report as JSON on stdout. + #[arg(long)] + json: bool, + /// Treat severe warnings as failures. + #[arg(long)] + strict: bool, + }, + /// Show which formats this build can read and write. + Formats { + /// Print the matrix as JSON. + #[arg(long)] + json: bool, + }, +} + +fn main() -> ExitCode { + // Logs always go to stderr: stdout may carry the converted document. + tracing_subscriber::fmt() + .with_writer(std::io::stderr) + .with_env_filter( + tracing_subscriber::EnvFilter::try_from_default_env() + .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("warn")), + ) + .init(); + + let cli = Cli::parse(); + match run(&cli) { + Ok(code) => ExitCode::from(code), + Err(error) => { + eprintln!("docsai: {error}"); + ExitCode::from(exit_code_for(&error)) + } + } +} + +fn run(cli: &Cli) -> anyhow::Result { + match &cli.command { + Command::Formats { json } => { + print_formats(*json); + Ok(EXIT_OK) + } + Command::Convert { + input, + output, + to, + fidelity, + assets_dir, + json, + strict, + } => { + let fidelity = Fidelity::parse(fidelity).ok_or_else(|| { + anyhow::anyhow!("unknown --fidelity `{fidelity}`; use full, standard or plain") + })?; + let target = match to { + Some(name) => Some( + Format::parse(name) + .ok_or_else(|| anyhow::anyhow!("unknown --to format `{name}`"))?, + ), + None => None, + }; + let options = ConvertOptions { + fidelity, + assets_dir: assets_dir.clone(), + target, + }; + let outcome = docsai_convert::convert_file(input, output.as_deref(), &options)?; + + if output.is_none() && !*json { + let mut stdout = std::io::stdout().lock(); + stdout.write_all(outcome.markdown.as_bytes())?; + } + if *json { + let report = serde_json::to_string_pretty(&outcome.report)?; + println!("{report}"); + } else { + report_to_stderr(&outcome, cli.verbose); + } + + // Exit code 1 marks a conversion that lost something; `--strict` + // additionally makes minor warnings count as severe. + let lost = + outcome.report.has_severe() || (*strict && !outcome.report.warnings.is_empty()); + Ok(if lost { EXIT_WARNINGS } else { EXIT_OK }) + } + } +} + +fn report_to_stderr(outcome: &docsai_convert::Outcome, verbose: bool) { + let report = &outcome.report; + if report.warnings.is_empty() { + return; + } + let severe = report + .warnings + .iter() + .filter(|w| w.severity() == docsai_model::Severity::Severe) + .count(); + eprintln!( + "docsai: {} warning(s){}, {} raw-block(s)", + report.warnings.len(), + if severe > 0 { + format!(", {severe} severe") + } else { + String::new() + }, + report.raw_blocks_emitted + ); + if verbose { + for warning in &report.warnings { + eprintln!(" [{:?}] {}", warning.severity(), warning.message()); + } + } else { + eprintln!(" run with --verbose (or --json) for the detail"); + } +} + +fn print_formats(json: bool) { + if json { + let rows: Vec<_> = SUPPORT + .iter() + .map(|s| { + serde_json::json!({ + "format": s.format.as_str(), + "read": s.read, + "write": s.write, + "note": s.note, + }) + }) + .collect(); + println!( + "{}", + serde_json::to_string_pretty(&rows).unwrap_or_default() + ); + return; + } + println!("{:<10} {:<6} {:<6} NOTE", "FORMAT", "READ", "WRITE"); + for support in SUPPORT { + println!( + "{:<10} {:<6} {:<6} {}", + support.format.as_str(), + if support.read { "yes" } else { "no" }, + if support.write { "yes" } else { "no" }, + support.note + ); + } +} + +fn exit_code_for(error: &anyhow::Error) -> u8 { + match error.downcast_ref::() { + Some(docsai_convert::ConvertError::Unsupported { .. }) => EXIT_UNSUPPORTED, + Some(docsai_convert::ConvertError::UnknownFormat(_)) => EXIT_UNSUPPORTED, + Some(_) => EXIT_INPUT, + None => EXIT_INPUT, + } +} diff --git a/crates/docsai-convert/Cargo.toml b/crates/docsai-convert/Cargo.toml new file mode 100644 index 0000000..127ca0b --- /dev/null +++ b/crates/docsai-convert/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "docsai-convert" +description = "Conversion pipelines, format detection and asset management for docsai" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +authors.workspace = true + +[dependencies] +docsai-docmark = { workspace = true } +docsai-model = { workspace = true } +docsai-odf = { workspace = true } +docsai-office = { workspace = true } +thiserror = { workspace = true } +tracing = { workspace = true } + +[dev-dependencies] +comrak.workspace = true +zip = { workspace = true } diff --git a/crates/docsai-convert/src/assets.rs b/crates/docsai-convert/src/assets.rs new file mode 100644 index 0000000..f4361b2 --- /dev/null +++ b/crates/docsai-convert/src/assets.rs @@ -0,0 +1,108 @@ +//! [`AssetStore`] implementations (arquitectura §3.2). + +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; + +use docsai_model::assets::{describe, AssetError, AssetId, AssetInfo, AssetStore}; + +/// An [`AssetStore`] that materialises every asset into a directory. +/// +/// Files are written as they are stored, under the deterministic +/// `img-.` name, so two conversions of the same document produce +/// the same directory and `git diff` on `assets/` stays empty. +#[derive(Debug)] +pub struct DirAssetStore { + dir: PathBuf, + entries: BTreeMap)>, + written: Vec, +} + +impl DirAssetStore { + /// Creates the directory lazily: it is only made when the first asset + /// arrives, so documents without media leave no empty folder behind. + pub fn new(dir: impl Into) -> Self { + DirAssetStore { + dir: dir.into(), + entries: BTreeMap::new(), + written: Vec::new(), + } + } + + pub fn dir(&self) -> &Path { + &self.dir + } + + /// Paths written so far, in storage order. + pub fn written(&self) -> &[PathBuf] { + &self.written + } +} + +impl AssetStore for DirAssetStore { + fn put(&mut self, bytes: &[u8]) -> Result { + if bytes.is_empty() { + return Err(AssetError::Empty); + } + let id = AssetId::new(docsai_model::assets::content_hash(bytes)); + if self.entries.contains_key(&id) { + return Ok(id); // deduplicated: nothing more to write + } + let info = describe(&id, bytes); + std::fs::create_dir_all(&self.dir).map_err(|e| AssetError::Io(e.to_string()))?; + let path = self.dir.join(&info.file_name); + std::fs::write(&path, bytes).map_err(|e| AssetError::Io(e.to_string()))?; + self.written.push(path); + self.entries.insert(id.clone(), (info, bytes.to_vec())); + Ok(id) + } + + fn get(&self, id: &AssetId) -> Option<&[u8]> { + self.entries.get(id).map(|(_, bytes)| bytes.as_slice()) + } + + fn info(&self, id: &AssetId) -> Option<&AssetInfo> { + self.entries.get(id).map(|(info, _)| info) + } + + fn ids(&self) -> Vec { + self.entries.keys().cloned().collect() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn temp_dir(name: &str) -> PathBuf { + let dir = std::env::temp_dir().join(format!("docsai-test-{name}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + dir + } + + #[test] + fn writes_files_with_deterministic_names_and_deduplicates() { + let dir = temp_dir("assets"); + let mut store = DirAssetStore::new(&dir); + let png = b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x78\x00\x00\x00\x5a\x08\x02\x00\x00\x00"; + + let first = store.put(png).unwrap(); + let second = store.put(png).unwrap(); + assert_eq!(first, second); + assert_eq!(store.written().len(), 1, "written once"); + + let info = store.info(&first).unwrap(); + assert!(info.file_name.starts_with("img-")); + assert!(info.file_name.ends_with(".png")); + assert_eq!(std::fs::read(dir.join(&info.file_name)).unwrap(), png); + + let _ = std::fs::remove_dir_all(&dir); + } + + #[test] + fn no_media_leaves_no_directory_behind() { + let dir = temp_dir("empty"); + let store = DirAssetStore::new(&dir); + assert!(store.is_empty()); + assert!(!dir.exists(), "the directory is only created on demand"); + } +} diff --git a/crates/docsai-convert/src/lib.rs b/crates/docsai-convert/src/lib.rs new file mode 100644 index 0000000..d61fc38 --- /dev/null +++ b/crates/docsai-convert/src/lib.rs @@ -0,0 +1,136 @@ +//! Conversion orchestration: format detection, pipelines and asset management. +//! +//! This is the only crate that knows about more than one format; the readers +//! and writers never talk to each other (arquitectura §1). + +#![forbid(unsafe_code)] + +pub mod assets; +mod pipeline; + +pub use assets::DirAssetStore; +pub use docsai_docmark::{Fidelity, Options as DocMarkOptions}; +pub use pipeline::{convert_file, read_document, ConvertOptions, Outcome}; + +use docsai_model::Format; + +/// Errors the orchestration layer can return. +#[derive(Debug, thiserror::Error)] +pub enum ConvertError { + #[error("{0}")] + Read(#[from] docsai_office::ReadError), + + #[error("i/o error on `{path}`: {source}")] + Io { + path: String, + #[source] + source: std::io::Error, + }, + + #[error("unsupported conversion: {from} -> {to}")] + Unsupported { from: Format, to: Format }, + + #[error("could not tell what kind of document `{0}` is")] + UnknownFormat(String), + + #[error("the produced document breaks an IR invariant: {0}")] + Invalid(String), +} + +/// One row of the support matrix shown by `docsai formats`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct FormatSupport { + pub format: Format, + pub read: bool, + pub write: bool, + /// Phase of `docs/plan-desarrollo.md` that lands the missing direction. + pub note: &'static str, +} + +/// The support matrix as of this build. +pub const SUPPORT: &[FormatSupport] = &[ + FormatSupport { + format: Format::Docx, + read: true, + write: false, + note: "writing arrives in Fase 2", + }, + FormatSupport { + format: Format::Doc, + read: false, + write: false, + note: "reading arrives in Fase 5; writing is out of scope", + }, + FormatSupport { + format: Format::Xlsx, + read: false, + write: false, + note: "arrives in Fase 3", + }, + FormatSupport { + format: Format::Xls, + read: false, + write: false, + note: "reading arrives in Fase 3; writing is out of scope", + }, + FormatSupport { + format: Format::Odt, + read: false, + write: false, + note: "arrives in Fase 4", + }, + FormatSupport { + format: Format::Ods, + read: false, + write: false, + note: "arrives in Fase 4", + }, + FormatSupport { + format: Format::DocMark, + read: false, + write: true, + note: "reading arrives in Fase 2", + }, +]; + +/// Whether this build can read a format. +pub fn can_read(format: Format) -> bool { + SUPPORT.iter().any(|s| s.format == format && s.read) +} + +/// Whether this build can write a format. +pub fn can_write(format: Format) -> bool { + SUPPORT.iter().any(|s| s.format == format && s.write) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_matrix_matches_what_the_crates_actually_do() { + // The matrix is what `docsai formats` prints, so it must not drift + // ahead of the readers. + for support in SUPPORT.iter().filter(|s| s.format != Format::DocMark) { + assert_eq!( + support.read, + docsai_office::READABLE.contains(&support.format), + "read support for {} is out of step with the office crate", + support.format + ); + } + assert!(can_read(Format::Docx)); + assert!(!can_write(Format::Docx), "docx writing is Fase 2"); + assert!(can_write(Format::DocMark)); + } + + #[test] + fn every_format_appears_exactly_once() { + let mut seen: Vec = SUPPORT.iter().map(|s| s.format).collect(); + let total = seen.len(); + seen.sort(); + seen.dedup(); + assert_eq!(seen.len(), total); + assert_eq!(total, 7); + } +} diff --git a/crates/docsai-convert/src/pipeline.rs b/crates/docsai-convert/src/pipeline.rs new file mode 100644 index 0000000..b3f4ee2 --- /dev/null +++ b/crates/docsai-convert/src/pipeline.rs @@ -0,0 +1,259 @@ +//! The conversion pipelines. + +use std::io::{Read, Seek}; +use std::path::{Path, PathBuf}; + +use docsai_docmark::{Fidelity, Options as DocMarkOptions}; +use docsai_model::assets::AssetStore; +use docsai_model::{ConversionReport, Document, Format}; + +use crate::assets::DirAssetStore; +use crate::ConvertError; + +/// Settings for one conversion. +#[derive(Debug, Clone)] +pub struct ConvertOptions { + pub fidelity: Fidelity, + /// Where media go. Defaults to `assets/` next to the output file. + pub assets_dir: Option, + /// Target format. Inferred from the output extension when absent. + pub target: Option, +} + +impl Default for ConvertOptions { + fn default() -> Self { + ConvertOptions { + fidelity: Fidelity::Full, + assets_dir: None, + target: None, + } + } +} + +/// What a conversion produced. +#[derive(Debug)] +pub struct Outcome { + pub source_format: Format, + pub target_format: Format, + /// The DocMark text, also written to `output_path` when one was given. + pub markdown: String, + pub output_path: Option, + pub assets_written: Vec, + pub report: ConversionReport, +} + +/// Reads any supported document into the IR, detecting its format by content. +pub fn read_document( + reader: R, + hint: Option<&str>, + assets: &mut dyn AssetStore, +) -> Result<(Document, Format, ConversionReport), ConvertError> { + let mut reader = reader; + let (format, score) = docsai_office::detect(&mut reader, hint); + if score == docsai_office::DetectScore::No || !crate::can_read(format) { + return Err(match crate::SUPPORT.iter().find(|s| s.format == format) { + Some(support) if score != docsai_office::DetectScore::No => ConvertError::Unsupported { + from: support.format, + to: Format::DocMark, + }, + _ => ConvertError::UnknownFormat(hint.unwrap_or("").to_string()), + }); + } + let (document, report) = docsai_office::read(format, reader, assets)?; + docsai_model::validate::validate(&document).map_err(|errors| { + ConvertError::Invalid( + errors + .iter() + .map(|e| e.to_string()) + .collect::>() + .join("; "), + ) + })?; + Ok((document, format, report)) +} + +/// Converts a file on disk. +/// +/// With `output` set, the DocMark is written there and media go to the assets +/// directory; without it the text is returned for the caller to print. +pub fn convert_file( + input: &Path, + output: Option<&Path>, + options: &ConvertOptions, +) -> Result { + let target = resolve_target(output, options)?; + if !crate::can_write(target) { + return Err(ConvertError::Unsupported { + from: Format::DocMark, + to: target, + }); + } + + let file = std::fs::File::open(input).map_err(|source| ConvertError::Io { + path: input.display().to_string(), + source, + })?; + let hint = input.file_name().and_then(|n| n.to_str()); + + let assets_dir = options + .assets_dir + .clone() + .unwrap_or_else(|| default_assets_dir(input, output)); + let mut store = DirAssetStore::new(assets_dir.clone()); + + let (document, source_format, mut report) = read_document(file, hint, &mut store)?; + + let docmark_options = DocMarkOptions { + fidelity: options.fidelity, + assets_dir: relative_assets_dir(&assets_dir, output), + source_format, + }; + let (markdown, write_report) = docsai_docmark::serialize(&document, &store, &docmark_options); + report.merge(write_report); + + if let Some(output) = output { + if let Some(parent) = output.parent().filter(|p| !p.as_os_str().is_empty()) { + std::fs::create_dir_all(parent).map_err(|source| ConvertError::Io { + path: parent.display().to_string(), + source, + })?; + } + std::fs::write(output, markdown.as_bytes()).map_err(|source| ConvertError::Io { + path: output.display().to_string(), + source, + })?; + } + + Ok(Outcome { + source_format, + target_format: target, + markdown, + output_path: output.map(Path::to_path_buf), + assets_written: store.written().to_vec(), + report, + }) +} + +fn resolve_target(output: Option<&Path>, options: &ConvertOptions) -> Result { + if let Some(target) = options.target { + return Ok(target); + } + let Some(output) = output else { + // Writing to stdout: DocMark is the only text format there is. + return Ok(Format::DocMark); + }; + let name = output.file_name().and_then(|n| n.to_str()).unwrap_or(""); + // `.dmk.md` and `.md` both mean DocMark. + match name.rsplit('.').next().and_then(Format::parse) { + Some(format) => Ok(format), + None => Err(ConvertError::UnknownFormat(output.display().to_string())), + } +} + +fn default_assets_dir(input: &Path, output: Option<&Path>) -> PathBuf { + let base = output + .and_then(|o| o.parent()) + .filter(|p| !p.as_os_str().is_empty()) + .or_else(|| input.parent()) + .unwrap_or_else(|| Path::new(".")); + base.join("assets") +} + +/// The path DocMark links should use: relative to the output file when +/// possible, so the `.dmk.md` stays portable. +fn relative_assets_dir(assets_dir: &Path, output: Option<&Path>) -> String { + let parent = output.and_then(|o| o.parent()).unwrap_or(Path::new("")); + let relative = assets_dir.strip_prefix(parent).unwrap_or(assets_dir); + // DocMark links always use `/`, on every platform. + relative.to_string_lossy().replace('\\', "/") +} + +#[cfg(test)] +mod tests { + use super::*; + + fn corpus(name: &str) -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../corpus/docx") + .join(name) + } + + fn temp_dir(name: &str) -> PathBuf { + let dir = + std::env::temp_dir().join(format!("docsai-pipeline-{name}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + dir + } + + #[test] + fn converts_a_docx_to_a_file_with_its_assets() { + let dir = temp_dir("images"); + let output = dir.join("out.dmk.md"); + let outcome = convert_file( + &corpus("images-inline.docx"), + Some(&output), + &ConvertOptions::default(), + ) + .expect("converts"); + + assert_eq!(outcome.source_format, Format::Docx); + assert_eq!(outcome.target_format, Format::DocMark); + assert_eq!(outcome.assets_written.len(), 3); + assert!(output.exists()); + + let written = std::fs::read_to_string(&output).unwrap(); + assert_eq!(written, outcome.markdown); + assert!(written.contains("](assets/img-"), "links point at assets/"); + for path in &outcome.assets_written { + assert!(path.starts_with(dir.join("assets"))); + } + let _ = std::fs::remove_dir_all(&dir); + } + + #[test] + fn without_an_output_path_nothing_is_written() { + let outcome = convert_file(&corpus("basic-text.docx"), None, &ConvertOptions::default()) + .expect("converts"); + assert!(outcome.output_path.is_none()); + assert!(outcome.markdown.contains("Primer parrafo")); + } + + #[test] + fn an_unsupported_target_is_rejected_before_reading() { + let options = ConvertOptions { + target: Some(Format::Docx), + ..Default::default() + }; + let error = convert_file(&corpus("basic-text.docx"), None, &options).unwrap_err(); + assert!(matches!(error, ConvertError::Unsupported { .. })); + } + + #[test] + fn a_missing_file_is_an_io_error() { + let error = convert_file( + Path::new("no-such-file.docx"), + None, + &ConvertOptions::default(), + ) + .unwrap_err(); + assert!(matches!(error, ConvertError::Io { .. })); + } + + #[test] + fn asset_links_stay_relative_to_the_output() { + assert_eq!( + relative_assets_dir(Path::new("out/assets"), Some(Path::new("out/doc.md"))), + "assets" + ); + assert_eq!( + relative_assets_dir(Path::new("media"), Some(Path::new("doc.md"))), + "media" + ); + assert_eq!( + relative_assets_dir(Path::new("/tmp/a1"), None), + "/tmp/a1", + "an absolute directory keeps exactly one leading slash" + ); + } +} diff --git a/crates/docsai-convert/tests/goldens.rs b/crates/docsai-convert/tests/goldens.rs new file mode 100644 index 0000000..2f100af --- /dev/null +++ b/crates/docsai-convert/tests/goldens.rs @@ -0,0 +1,291 @@ +//! Golden tests over the whole docx corpus (`AGENTS.md` §6). +//! +//! Each `corpus/docx/.docx` has its expected DocMark beside it as +//! `.expected.dmk.md`. Updating a golden is a deliberate act: +//! +//! ```text +//! DOCSAI_UPDATE_GOLDENS=1 cargo test -p docsai-convert --test goldens +//! ``` +//! +//! …and the resulting diff has to be reviewed by hand. + +use std::path::{Path, PathBuf}; + +use docsai_docmark::{Fidelity, Options}; +use docsai_model::{ConversionReport, Document, MemoryAssetStore}; + +fn corpus_dir() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("../../corpus/docx") +} + +fn documents() -> Vec { + let mut paths: Vec = std::fs::read_dir(corpus_dir()) + .expect("the corpus is present") + .filter_map(|entry| entry.ok().map(|e| e.path())) + .filter(|p| p.extension().and_then(|e| e.to_str()) == Some("docx")) + .collect(); + paths.sort(); + assert!(paths.len() >= 14, "only {} documents found", paths.len()); + paths +} + +/// Converts a corpus document, with a stable `assets/` prefix so the golden +/// does not depend on where the test ran. +fn convert(path: &Path, fidelity: Fidelity) -> (String, ConversionReport, MemoryAssetStore) { + let file = std::fs::File::open(path).unwrap_or_else(|e| panic!("{}: {e}", path.display())); + let mut assets = MemoryAssetStore::new(); + let (document, mut report) = docsai_office::read_docx(file, &mut assets) + .unwrap_or_else(|e| panic!("{}: {e}", path.display())); + + if let Err(errors) = docsai_model::validate::validate(&document) { + panic!("{}: invalid IR: {errors:?}", path.display()); + } + + let options = Options { + fidelity, + assets_dir: "assets".into(), + source_format: docsai_model::Format::Docx, + }; + let (markdown, write_report) = docsai_docmark::serialize(&document, &assets, &options); + report.merge(write_report); + (markdown, report, assets) +} + +fn golden_path(document: &Path) -> PathBuf { + let stem = document.file_stem().unwrap().to_string_lossy().into_owned(); + document.with_file_name(format!("{stem}.expected.dmk.md")) +} + +#[test] +fn the_corpus_matches_its_goldens() { + let updating = std::env::var_os("DOCSAI_UPDATE_GOLDENS").is_some(); + let mut mismatches = Vec::new(); + + for document in documents() { + let (markdown, _, _) = convert(&document, Fidelity::Full); + let golden = golden_path(&document); + if updating { + std::fs::write(&golden, &markdown).expect("writes the golden"); + continue; + } + match std::fs::read_to_string(&golden) { + Ok(expected) if expected == markdown => {} + Ok(expected) => mismatches.push(format!( + "{}:\n{}", + golden.display(), + first_difference(&expected, &markdown) + )), + Err(_) => mismatches.push(format!("{}: golden missing", golden.display())), + } + } + + assert!( + mismatches.is_empty(), + "{} golden(s) differ; review the diff, then regenerate with \ + DOCSAI_UPDATE_GOLDENS=1\n\n{}", + mismatches.len(), + mismatches.join("\n\n") + ); +} + +fn first_difference(expected: &str, actual: &str) -> String { + for (index, (want, got)) in expected.lines().zip(actual.lines()).enumerate() { + if want != got { + return format!(" line {}\n - {want}\n + {got}", index + 1); + } + } + format!( + " line count differs: expected {}, got {}", + expected.lines().count(), + actual.lines().count() + ) +} + +#[test] +fn serialisation_is_deterministic() { + for document in documents() { + let (first, _, _) = convert(&document, Fidelity::Full); + let (second, _, _) = convert(&document, Fidelity::Full); + assert_eq!( + first, + second, + "{} serialises differently on a second run", + document.display() + ); + } +} + +#[test] +fn every_document_uses_unix_line_endings_and_no_bom() { + for document in documents() { + let (markdown, _, _) = convert(&document, Fidelity::Full); + assert!( + !markdown.contains('\r'), + "{} contains a CR", + document.display() + ); + assert!( + !markdown.starts_with('\u{feff}'), + "{} has a BOM", + document.display() + ); + assert!( + markdown.ends_with('\n') && !markdown.ends_with("\n\n"), + "{} does not end in exactly one newline", + document.display() + ); + } +} + +/// Fase 1 acceptance criterion: *«La salida en `--fidelity plain` es CommonMark +/// limpio verificado con comrak»*. +#[test] +fn plain_fidelity_is_clean_commonmark() { + let mut options = comrak::Options::default(); + options.extension.table = true; + options.extension.strikethrough = true; + options.extension.autolink = true; + + for document in documents() { + let (markdown, _, _) = convert(&document, Fidelity::Plain); + let name = document.display(); + + assert!( + !markdown.starts_with("---"), + "{name}: plain output must not carry front matter" + ); + assert!( + !markdown.contains(":::"), + "{name}: plain output must not carry fenced divs" + ); + + // comrak must reach the same text, and no attribute block may survive + // into the rendered HTML. + let html = comrak::markdown_to_html(&markdown, &options); + assert!( + !html.contains("{."), + "{name}: an attribute block reached the HTML:\n{html}" + ); + assert!( + !html.contains("</") && !html.contains(" 50_000, + "the document should be substantial" + ); + assert!( + report.raw_blocks_emitted < 10, + "{} raw-blocks in an ordinary document", + report.raw_blocks_emitted + ); + + // The budget of arquitectura §8 is for an optimised build; a debug build + // runs roughly an order of magnitude slower, so it gets a looser bound + // rather than no bound at all. + let budget = if cfg!(debug_assertions) { + std::time::Duration::from_secs(10) + } else { + std::time::Duration::from_secs(1) + }; + assert!( + elapsed < budget, + "conversion took {elapsed:?}, over the {budget:?} budget" + ); +} + +/// Writes a synthetic document of roughly 50 pages: paragraphs, headings and +/// tables, all of them things the reader must handle at speed. +fn build_large_docx(path: &Path, paragraphs: usize) { + use std::io::Write; + + let ns = concat!( + r#"xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main" "#, + r#"xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships""# + ); + let mut body = String::with_capacity(paragraphs * 200); + for index in 0..paragraphs { + if index % 40 == 0 { + body.push_str(&format!( + r#"Seccion {index}"# + )); + } + body.push_str(&format!( + r#"Parrafo {index} con texto suficiente para ocupar una linea completa de la pagina. Negrita."# + )); + if index % 100 == 0 { + body.push_str( + r#" + a + b"#, + ); + } + } + let document = format!("{body}"); + let styles = format!( + r#" + "# + ); + + let file = std::fs::File::create(path).expect("creates the fixture"); + let mut zip = zip::ZipWriter::new(file); + let options: zip::write::FileOptions<'_, ()> = Default::default(); + for (name, content) in [ + ("word/document.xml", document.as_str()), + ("word/styles.xml", styles.as_str()), + ] { + zip.start_file(name, options).unwrap(); + zip.write_all(content.as_bytes()).unwrap(); + } + zip.finish().unwrap(); +} diff --git a/crates/docsai-docmark/Cargo.toml b/crates/docsai-docmark/Cargo.toml new file mode 100644 index 0000000..470e57f --- /dev/null +++ b/crates/docsai-docmark/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "docsai-docmark" +description = "DocMark serializer and parser (IR <-> extended Markdown)" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +authors.workspace = true + +[dependencies] +docsai-model = { workspace = true } +thiserror = { workspace = true } diff --git a/crates/docsai-docmark/src/attrs.rs b/crates/docsai-docmark/src/attrs.rs new file mode 100644 index 0000000..8823361 --- /dev/null +++ b/crates/docsai-docmark/src/attrs.rs @@ -0,0 +1,155 @@ +//! Pandoc-style attribute blocks `{#id .class key="value"}`. +//! +//! The canonical order of spec §8 is enforced here, in one place: id first, +//! then classes alphabetically, then keys alphabetically. That ordering is +//! what makes the serialiser deterministic and the golden diffs stable. + +use std::collections::BTreeMap; + +use crate::escape::{escape_attr_value, is_bare_value}; + +/// A set of attributes being built for one node. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct Attrs { + id: Option, + classes: Vec, + pairs: BTreeMap, +} + +impl Attrs { + pub fn new() -> Self { + Self::default() + } + + pub fn id(&mut self, id: impl Into) -> &mut Self { + self.id = Some(id.into()); + self + } + + /// Adds a class. Empty names are ignored so callers can pass through + /// optional style ids without branching. + pub fn class(&mut self, class: impl Into) -> &mut Self { + let class = class.into(); + if !class.is_empty() && !self.classes.contains(&class) { + self.classes.push(class); + } + self + } + + /// Adds a key/value pair. + pub fn set(&mut self, key: impl Into, value: impl Into) -> &mut Self { + self.pairs.insert(key.into(), value.into()); + self + } + + /// Adds a pair only when the value is `Some`. + pub fn set_opt(&mut self, key: &str, value: Option>) -> &mut Self { + if let Some(value) = value { + self.set(key, value); + } + self + } + + /// Adds a boolean pair only when it is `true`. + pub fn set_flag(&mut self, key: &str, value: bool) -> &mut Self { + if value { + self.set(key, "true"); + } + self + } + + pub fn is_empty(&self) -> bool { + self.id.is_none() && self.classes.is_empty() && self.pairs.is_empty() + } + + /// Renders the block, or the empty string when there is nothing to render. + /// + /// ``` + /// use docsai_docmark::attrs::Attrs; + /// let mut a = Attrs::new(); + /// a.set("width", "450px").class("Quote").id("img-1").set("title", "Figura 1"); + /// assert_eq!(a.render(), r#"{#img-1 .Quote title="Figura 1" width=450px}"#); + /// ``` + pub fn render(&self) -> String { + if self.is_empty() { + return String::new(); + } + let mut parts: Vec = Vec::new(); + if let Some(id) = &self.id { + parts.push(format!("#{id}")); + } + let mut classes = self.classes.clone(); + classes.sort(); + parts.extend(classes.into_iter().map(|c| format!(".{c}"))); + for (key, value) in &self.pairs { + if is_bare_value(value) { + parts.push(format!("{key}={value}")); + } else { + parts.push(format!("{key}=\"{}\"", escape_attr_value(value))); + } + } + format!("{{{}}}", parts.join(" ")) + } + + /// Renders the block preceded by a space, for appending to a line. + pub fn suffix(&self) -> String { + if self.is_empty() { + String::new() + } else { + format!(" {}", self.render()) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn renders_in_the_canonical_order() { + let mut a = Attrs::new(); + a.set("z", "1") + .class("Beta") + .set("a", "2") + .class("Alpha") + .id("x"); + assert_eq!(a.render(), "{#x .Alpha .Beta a=2 z=1}"); + } + + #[test] + fn quotes_only_what_needs_quoting() { + let mut a = Attrs::new(); + a.set("width", "450px") + .set("color", "#FF0000") + .set("name", "Logo corporativo") + .set("link", "https://example.com/a b"); + assert_eq!( + a.render(), + r#"{color=#FF0000 link="https://example.com/a b" name="Logo corporativo" width=450px}"# + ); + } + + #[test] + fn is_empty_when_nothing_was_added() { + assert!(Attrs::new().is_empty()); + assert_eq!(Attrs::new().render(), ""); + assert_eq!(Attrs::new().suffix(), ""); + } + + #[test] + fn empty_classes_and_absent_options_are_skipped() { + let mut a = Attrs::new(); + a.class("") + .set_opt("missing", None::) + .set_flag("off", false) + .set_flag("on", true); + assert_eq!(a.render(), "{on=true}"); + } + + #[test] + fn later_values_replace_earlier_ones() { + let mut a = Attrs::new(); + a.set("k", "1").set("k", "2"); + assert_eq!(a.render(), "{k=2}"); + } +} diff --git a/crates/docsai-docmark/src/escape.rs b/crates/docsai-docmark/src/escape.rs new file mode 100644 index 0000000..b0a6dde --- /dev/null +++ b/crates/docsai-docmark/src/escape.rs @@ -0,0 +1,142 @@ +//! Escaping rules (spec §8). +//! +//! Normative and deliberately small: only characters that would change the +//! meaning of the surrounding CommonMark are escaped, with a fixed decision +//! table so that two serialisations of the same IR are byte-identical. + +/// Where a run of text is being written, which decides what must be escaped. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TextContext { + /// Ordinary block content. + Block, + /// Inside a GFM table cell, where `|` also ends the cell. + TableCell, + /// Inside a link label, where `[` and `]` also close it early. + LinkLabel, +} + +/// Characters escaped everywhere: they open inline constructs. +const ALWAYS: &[char] = &['\\', '`', '*', '_', '[', ']', '<']; + +/// Escapes `text` for the given context. +/// +/// ``` +/// use docsai_docmark::escape::{escape, TextContext}; +/// assert_eq!(escape("a*b", TextContext::Block), r"a\*b"); +/// assert_eq!(escape("a|b", TextContext::TableCell), r"a\|b"); +/// assert_eq!(escape("a|b", TextContext::Block), "a|b"); +/// ``` +pub fn escape(text: &str, context: TextContext) -> String { + let mut out = String::with_capacity(text.len()); + let mut at_line_start = true; + let mut chars = text.chars().peekable(); + + while let Some(c) = chars.next() { + let needs_escape = ALWAYS.contains(&c) + || (context == TextContext::TableCell && c == '|') + || (at_line_start && matches!(c, '#' | '>' | '-' | '+' | '=' | '~')) + // A digit that starts a line and is followed by `.` or `)` would + // become an ordered list marker. + || (at_line_start + && c.is_ascii_digit() + && chars.peek().is_some_and(|n| matches!(n, '.' | ')'))) + // `&` only matters when it could open an entity reference. + || (c == '&' && chars.peek().is_some_and(|n| n.is_ascii_alphanumeric() || *n == '#')); + + if needs_escape { + out.push('\\'); + } + out.push(c); + + at_line_start = c == '\n'; + // A leading run of spaces still counts as the line start. + if c == ' ' && at_line_start { + at_line_start = true; + } + } + out +} + +/// Escapes a value for use inside a double-quoted attribute. +pub fn escape_attr_value(value: &str) -> String { + let mut out = String::with_capacity(value.len()); + for c in value.chars() { + match c { + '"' => out.push_str("\\\""), + '\\' => out.push_str("\\\\"), + '\n' => out.push_str("\\n"), + _ => out.push(c), + } + } + out +} + +/// True when a value can be written without quotes: a number or a simple +/// identifier (spec §8). +pub fn is_bare_value(value: &str) -> bool { + !value.is_empty() + && value + .chars() + .all(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.' | '%' | '#' | '+')) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn escapes_inline_markers_but_not_ordinary_punctuation() { + assert_eq!( + escape("*asterisco* _guion_ `code` [x] \\z", TextContext::Block), + // `>` only opens a block quote, and only at the start of a line. + r"\*asterisco\* \_guion\_ \`code\` \[x\] \ \\z" + ); + assert_eq!( + escape("coma, punto. dos: puntos; ¿eñe?", TextContext::Block), + "coma, punto. dos: puntos; ¿eñe?" + ); + } + + #[test] + fn escapes_block_markers_only_at_the_start_of_a_line() { + assert_eq!(escape("# titulo", TextContext::Block), r"\# titulo"); + assert_eq!(escape("a # b", TextContext::Block), "a # b"); + assert_eq!(escape("- item", TextContext::Block), r"\- item"); + assert_eq!(escape("1. uno", TextContext::Block), r"\1. uno"); + assert_eq!(escape("1 uno", TextContext::Block), "1 uno", "not a marker"); + assert_eq!(escape("x\n> cita", TextContext::Block), "x\n\\> cita"); + } + + #[test] + fn pipes_are_escaped_only_inside_table_cells() { + assert_eq!(escape("a|b", TextContext::TableCell), r"a\|b"); + assert_eq!(escape("a|b", TextContext::Block), "a|b"); + } + + #[test] + fn ampersands_are_escaped_only_when_they_could_be_entities() { + assert_eq!(escape("Tom & Jerry", TextContext::Block), "Tom & Jerry"); + assert_eq!(escape("a&b", TextContext::Block), r"a\&b"); + assert_eq!(escape("a&b", TextContext::Block), r"a\&b"); + } + + #[test] + fn attribute_values_choose_bare_or_quoted() { + assert!(is_bare_value("450px")); + assert!(is_bare_value("#2E74B5")); + assert!(is_bare_value("top-bottom")); + assert!(!is_bare_value("Logo corporativo")); + assert!(!is_bare_value("")); + assert_eq!(escape_attr_value(r#"a "b" \c"#), r#"a \"b\" \\c"#); + } + + #[test] + fn escaping_is_idempotent_in_shape() { + // Escaping twice must not double the backslashes of the *first* pass + // in a way the parser cannot undo: `\*` becomes `\\\*`, which unescapes + // back to `\*`. This is the property the round-trip relies on. + let once = escape("a*b", TextContext::Block); + let twice = escape(&once, TextContext::Block); + assert_eq!(twice, r"a\\\*b"); + } +} diff --git a/crates/docsai-docmark/src/frontmatter.rs b/crates/docsai-docmark/src/frontmatter.rs new file mode 100644 index 0000000..8063548 --- /dev/null +++ b/crates/docsai-docmark/src/frontmatter.rs @@ -0,0 +1,477 @@ +//! The YAML front matter (spec §2). +//! +//! Written by hand rather than through a YAML library: the spec demands byte +//! determinism, and the schema is small and fully known. Doing it here also +//! keeps a YAML serialiser out of the dependency tree. + +use docsai_model::list::{ListCatalog, NumFormat}; +use docsai_model::sheet::Workbook; +use docsai_model::style::{DocDefaults, FontProps, ParaProps, Style, StyleCatalog, Underline}; +use docsai_model::text::{DocumentMeta, PageGeometry}; +use docsai_model::units::Length; +use docsai_model::{Document, Format, DOCMARK_VERSION}; + +use crate::units::{len, pt}; +use crate::Fidelity; + +/// Writes the front matter block, delimiters included, ending in a blank line. +pub fn write(out: &mut String, doc: &Document, source: Format, fidelity: Fidelity) { + if fidelity == Fidelity::Plain { + return; // plain output is CommonMark only (spec §6) + } + + out.push_str("---\n"); + scalar(out, "docmark", "ed(DOCMARK_VERSION)); + scalar(out, "source-format", source.as_str()); + write_meta(out, doc.meta()); + + match doc { + Document::Text(text) => { + if let Some(section) = text.sections.first() { + write_page(out, §ion.page); + } + if fidelity == Fidelity::Full { + write_styles(out, &text.styles); + write_lists(out, &text.list_defs); + } + } + Document::Workbook(book) => { + write_workbook(out, book); + if fidelity == Fidelity::Full { + write_styles(out, &book.styles); + } + } + } + + out.push_str("---\n\n"); +} + +fn write_meta(out: &mut String, meta: &DocumentMeta) { + let text = |out: &mut String, key: &str, value: &Option| { + if let Some(value) = value { + scalar(out, key, "ed(value)); + } + }; + text(out, "title", &meta.title); + text(out, "author", &meta.author); + text(out, "last-modified-by", &meta.last_modified_by); + text(out, "subject", &meta.subject); + text(out, "keywords", &meta.keywords); + text(out, "description", &meta.description); + // Timestamps are ISO-8601 and safe unquoted. + if let Some(created) = &meta.created { + scalar(out, "created", created); + } + if let Some(modified) = &meta.modified { + scalar(out, "modified", modified); + } + text(out, "language", &meta.language); + text(out, "application", &meta.application); + + if !meta.custom.is_empty() { + out.push_str("custom-properties:\n"); + for (key, value) in &meta.custom { + out.push_str(&format!(" {}: {}\n", yaml_key(key), quoted(value))); + } + } +} + +fn write_page(out: &mut String, page: &PageGeometry) { + if page.size.width.is_zero() && page.size.height.is_zero() { + return; + } + out.push_str("page:\n"); + match page.paper_name() { + Some(name) => out.push_str(&format!(" size: {name}\n")), + None => out.push_str(&format!( + " size: {{ width: {}, height: {} }}\n", + len(page.size.width), + len(page.size.height) + )), + } + let m = &page.margins; + out.push_str(&format!( + " margins: {{ top: {}, bottom: {}, left: {}, right: {}, header: {}, footer: {} }}\n", + len(m.top), + len(m.bottom), + len(m.left), + len(m.right), + len(m.header), + len(m.footer) + )); + out.push_str(&format!(" orientation: {}\n", page.orientation.as_str())); + if page.columns > 1 { + out.push_str(&format!(" columns: {}\n", page.columns)); + } + if page.title_page { + out.push_str(" title-page: true\n"); + } +} + +fn write_workbook(out: &mut String, book: &Workbook) { + if book.active_sheet.is_none() && book.defined_names.is_empty() { + return; + } + out.push_str("workbook:\n"); + if let Some(active) = &book.active_sheet { + out.push_str(&format!(" active-sheet: {}\n", quoted(active))); + } + if !book.defined_names.is_empty() { + out.push_str(" defined-names:\n"); + for name in &book.defined_names { + out.push_str(&format!( + " {}: {}\n", + yaml_key(&name.name), + quoted(&name.refers_to) + )); + } + } +} + +fn write_styles(out: &mut String, catalog: &StyleCatalog) { + if !catalog.defaults.is_empty() { + write_defaults(out, &catalog.defaults); + } + if catalog.styles.is_empty() { + return; + } + out.push_str("styles:\n"); + for style in catalog.styles.values() { + write_style(out, style); + } +} + +fn write_defaults(out: &mut String, defaults: &DocDefaults) { + out.push_str("style-defaults:\n"); + if let Some(font) = font_flow(&defaults.font) { + out.push_str(&format!(" font: {font}\n")); + } + if let Some(para) = para_flow(&defaults.paragraph) { + out.push_str(&format!(" paragraph: {para}\n")); + } +} + +fn write_style(out: &mut String, style: &Style) { + out.push_str(&format!(" {}:\n", yaml_key(style.id.as_str()))); + out.push_str(&format!(" type: {}\n", style.style_type.as_str())); + if style.name != style.id.as_str() { + out.push_str(&format!(" name: {}\n", quoted(&style.name))); + } + if let Some(based_on) = &style.based_on { + out.push_str(&format!(" based-on: {}\n", yaml_key(based_on.as_str()))); + } + if let Some(next) = &style.next { + out.push_str(&format!(" next: {}\n", yaml_key(next.as_str()))); + } + if style.is_default { + out.push_str(" default: true\n"); + } + if let Some(font) = font_flow(&style.font) { + out.push_str(&format!(" font: {font}\n")); + } + if let Some(para) = para_flow(&style.paragraph) { + out.push_str(&format!(" paragraph: {para}\n")); + } +} + +/// A `FontProps` as a YAML flow mapping, or `None` when it carries nothing. +/// +/// Field order is fixed by this function, not alphabetical: it mirrors the +/// spec's example and stays stable across releases. +pub fn font_flow(font: &FontProps) -> Option { + let mut parts: Vec = Vec::new(); + if let Some(name) = &font.name { + parts.push(format!("name: {}", quoted(name))); + } + if let Some(size) = font.size { + parts.push(format!("size: {}", pt(size))); + } + if let Some(color) = &font.color { + parts.push(format!("color: {}", quoted(color))); + } + push_flag(&mut parts, "bold", font.bold); + push_flag(&mut parts, "italic", font.italic); + push_flag(&mut parts, "strike", font.strike); + if let Some(underline) = font.underline { + parts.push(format!("underline: {}", underline_name(underline))); + } + if let Some(highlight) = &font.highlight { + parts.push(format!("highlight: {}", quoted(highlight))); + } + if let Some(vert) = font.vert_align { + parts.push(format!( + "vertical-align: {}", + match vert { + docsai_model::style::VertAlign::Superscript => "superscript", + docsai_model::style::VertAlign::Subscript => "subscript", + docsai_model::style::VertAlign::Baseline => "baseline", + } + )); + } + push_flag(&mut parts, "small-caps", font.small_caps); + push_flag(&mut parts, "caps", font.caps); + flow(parts) +} + +/// A `ParaProps` as a YAML flow mapping, or `None` when it carries nothing. +pub fn para_flow(para: &ParaProps) -> Option { + let mut parts: Vec = Vec::new(); + if let Some(align) = para.align { + parts.push(format!("align: {}", align.as_str())); + } + push_len(&mut parts, "indent-left", para.indent_left); + push_len(&mut parts, "indent-right", para.indent_right); + push_len(&mut parts, "indent-first-line", para.indent_first_line); + push_len(&mut parts, "indent-hanging", para.indent_hanging); + if let Some(value) = para.space_before { + parts.push(format!("space-before: {}", pt(value))); + } + if let Some(value) = para.space_after { + parts.push(format!("space-after: {}", pt(value))); + } + if let Some(line) = para.line_height { + parts.push(format!("line-height: {}", line_height(line))); + } + push_flag(&mut parts, "keep-with-next", para.keep_with_next); + push_flag(&mut parts, "page-break-before", para.page_break_before); + if let Some(background) = ¶.background { + parts.push(format!("background: {}", quoted(background))); + } + if let Some(level) = para.outline_level { + parts.push(format!("outline-level: {level}")); + } + flow(parts) +} + +fn line_height(value: docsai_model::style::LineHeight) -> String { + use docsai_model::style::LineHeight::*; + match value { + Multiple(thousandths) => docsai_model::units::trim_float(thousandths as f64 / 1000.0, 3), + Exact(length) => format!("exact {}", pt(length)), + AtLeast(length) => format!("at-least {}", pt(length)), + } +} + +fn write_lists(out: &mut String, catalog: &ListCatalog) { + if catalog.is_empty() { + return; + } + out.push_str("list-definitions:\n"); + for (id, def) in &catalog.defs { + out.push_str(&format!(" {}:\n levels:\n", yaml_key(id.as_str()))); + for level in &def.levels { + let mut parts = vec![format!("format: {}", num_format(&level.format))]; + if !level.text.is_empty() { + parts.push(format!("text: {}", quoted(&level.text))); + } + if let Some(start) = level.start { + parts.push(format!("start: {start}")); + } + push_len(&mut parts, "indent", level.indent); + push_len(&mut parts, "hanging", level.hanging); + out.push_str(&format!(" - {{ {} }}\n", parts.join(", "))); + } + } +} + +fn num_format(format: &NumFormat) -> String { + let name = format.as_str(); + if name.is_empty() { + "none".into() + } else { + name.to_string() + } +} + +fn underline_name(underline: Underline) -> &'static str { + match underline { + Underline::None => "none", + Underline::Single => "single", + Underline::Double => "double", + Underline::Thick => "thick", + Underline::Dotted => "dotted", + Underline::Dashed => "dashed", + Underline::Wave => "wave", + } +} + +fn push_flag(parts: &mut Vec, key: &str, value: Option) { + if let Some(value) = value { + parts.push(format!("{key}: {value}")); + } +} + +fn push_len(parts: &mut Vec, key: &str, value: Option) { + if let Some(value) = value { + parts.push(format!("{key}: {}", len(value))); + } +} + +fn flow(parts: Vec) -> Option { + if parts.is_empty() { + None + } else { + Some(format!("{{ {} }}", parts.join(", "))) + } +} + +fn scalar(out: &mut String, key: &str, value: &str) { + out.push_str(&format!("{key}: {value}\n")); +} + +/// A YAML double-quoted scalar. +fn quoted(value: &str) -> String { + let mut out = String::with_capacity(value.len() + 2); + out.push('"'); + for c in value.chars() { + match c { + '"' => out.push_str("\\\""), + '\\' => out.push_str("\\\\"), + '\n' => out.push_str("\\n"), + '\r' => out.push_str("\\r"), + '\t' => out.push_str("\\t"), + _ => out.push(c), + } + } + out.push('"'); + out +} + +/// A mapping key: bare when it is a plain identifier, quoted otherwise. +fn yaml_key(key: &str) -> String { + let plain = !key.is_empty() + && key + .chars() + .all(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '_')) + && !key.chars().next().is_some_and(|c| c.is_ascii_digit()); + if plain { + key.to_string() + } else { + quoted(key) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use docsai_model::style::{Align, StyleId, StyleType}; + use docsai_model::text::{Margins, Orientation, Section, TextDocument}; + use docsai_model::units::Size; + + fn render(doc: &Document) -> String { + let mut out = String::new(); + write(&mut out, doc, Format::Docx, Fidelity::Full); + out + } + + #[test] + fn writes_the_mandatory_header_and_the_metadata() { + let doc = Document::Text(TextDocument { + meta: DocumentMeta { + title: Some("Informe \"Anual\"".into()), + author: Some("Ana Pérez".into()), + created: Some("2026-03-01T10:00:00Z".into()), + language: Some("es-ES".into()), + custom: [("Departamento".to_string(), "Ventas".to_string())] + .into_iter() + .collect(), + ..Default::default() + }, + ..Default::default() + }); + let out = render(&doc); + assert!(out.starts_with("---\ndocmark: \"1.0\"\nsource-format: docx\n")); + assert!(out.contains(r#"title: "Informe \"Anual\"""#)); + assert!(out.contains("created: 2026-03-01T10:00:00Z\n")); + assert!(out.contains("custom-properties:\n Departamento: \"Ventas\"\n")); + assert!(out.ends_with("---\n\n")); + } + + #[test] + fn recognises_the_paper_size_and_writes_the_margins() { + let doc = Document::Text(TextDocument { + sections: vec![Section { + page: PageGeometry { + size: Size::new(Length::from_twips(11906), Length::from_twips(16838)), + margins: Margins { + top: Length::from_cm(2.5), + bottom: Length::from_cm(2.5), + left: Length::from_cm(3.0), + right: Length::from_cm(3.0), + header: Length::from_cm(1.25), + footer: Length::from_cm(1.25), + }, + orientation: Orientation::Landscape, + columns: 2, + title_page: true, + }, + ..Default::default() + }], + ..Default::default() + }); + let out = render(&doc); + assert!(out.contains("page:\n size: A4\n")); + assert!(out.contains("margins: { top: 2.5cm, bottom: 2.5cm, left: 3cm, right: 3cm")); + assert!(out.contains("orientation: landscape\n")); + assert!(out.contains("columns: 2\n")); + assert!(out.contains("title-page: true\n")); + } + + #[test] + fn writes_the_style_catalogue_with_its_inheritance() { + let mut styles = StyleCatalog::default(); + styles.defaults.font.size = Some(Length::from_pt(11.0)); + let mut heading = Style::new("Heading1", StyleType::Paragraph); + heading.name = "heading 1".into(); + heading.based_on = Some(StyleId::new("Normal")); + heading.font.name = Some("Calibri Light".into()); + heading.font.size = Some(Length::from_pt(16.0)); + heading.font.color = Some("#2E74B5".into()); + heading.paragraph.align = Some(Align::Center); + heading.paragraph.outline_level = Some(0); + styles.insert(heading); + + let doc = Document::Text(TextDocument { + styles, + ..Default::default() + }); + let out = render(&doc); + assert!(out.contains("style-defaults:\n font: { size: 11pt }\n")); + assert!(out.contains("styles:\n Heading1:\n type: paragraph\n")); + assert!(out.contains(" name: \"heading 1\"\n")); + assert!(out.contains(" based-on: Normal\n")); + assert!( + out.contains(" font: { name: \"Calibri Light\", size: 16pt, color: \"#2E74B5\" }") + ); + assert!(out.contains(" paragraph: { align: center, outline-level: 0 }\n")); + } + + #[test] + fn plain_fidelity_has_no_front_matter() { + let doc = Document::Text(TextDocument::default()); + let mut out = String::new(); + write(&mut out, &doc, Format::Docx, Fidelity::Plain); + assert!(out.is_empty()); + } + + #[test] + fn standard_fidelity_drops_the_catalogues() { + let mut styles = StyleCatalog::default(); + styles.insert(Style::new("Heading1", StyleType::Paragraph)); + let doc = Document::Text(TextDocument { + styles, + ..Default::default() + }); + let mut out = String::new(); + write(&mut out, &doc, Format::Docx, Fidelity::Standard); + assert!(!out.contains("styles:")); + assert!(out.contains("docmark:")); + } + + #[test] + fn keys_that_are_not_identifiers_get_quoted() { + assert_eq!(yaml_key("Heading1"), "Heading1"); + assert_eq!(yaml_key("mi estilo"), "\"mi estilo\""); + assert_eq!(yaml_key("2col"), "\"2col\""); + } +} diff --git a/crates/docsai-docmark/src/lib.rs b/crates/docsai-docmark/src/lib.rs new file mode 100644 index 0000000..24c780d --- /dev/null +++ b/crates/docsai-docmark/src/lib.rs @@ -0,0 +1,116 @@ +//! DocMark: the extended-Markdown pivot format. +//! +//! Fase 1 lands the **serialiser** (IR → DocMark). The parser (DocMark → IR) +//! is Fase 2 and deliberately absent here. +//! +//! The output is normative and deterministic (spec §8): two serialisations of +//! the same IR are byte-identical, line endings are always `\n`, and the +//! encoding is UTF-8 without BOM. +//! +//! ``` +//! use docsai_docmark::{serialize, Options}; +//! use docsai_model::{Document, MemoryAssetStore, text::TextDocument}; +//! +//! let doc = Document::Text(TextDocument::default()); +//! let (markdown, _report) = serialize(&doc, &MemoryAssetStore::new(), &Options::default()); +//! assert!(markdown.starts_with("---\ndocmark: \"1.0\"\n")); +//! ``` + +#![forbid(unsafe_code)] + +pub mod attrs; +pub mod escape; +mod frontmatter; +pub mod units; +mod writer; + +use docsai_model::assets::AssetStore; +use docsai_model::{ConversionReport, Document, Format}; + +/// How much of the IR reaches the output (spec §6). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum Fidelity { + /// Everything: attributes, catalogues and raw-blocks. Round-trip grade. + #[default] + Full, + /// The main attributes, without catalogues or raw-blocks. + Standard, + /// Plain CommonMark + GFM, for LLM and RAG consumption. + Plain, +} + +impl Fidelity { + pub fn as_str(self) -> &'static str { + match self { + Fidelity::Full => "full", + Fidelity::Standard => "standard", + Fidelity::Plain => "plain", + } + } + + /// Parses a `--fidelity` value. + pub fn parse(value: &str) -> Option { + match value.trim().to_ascii_lowercase().as_str() { + "full" => Some(Fidelity::Full), + "standard" => Some(Fidelity::Standard), + "plain" => Some(Fidelity::Plain), + _ => None, + } + } +} + +impl std::fmt::Display for Fidelity { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +/// Serialisation settings. +#[derive(Debug, Clone)] +pub struct Options { + pub fidelity: Fidelity, + /// Directory the image links point at, relative to the `.dmk.md` file. + pub assets_dir: String, + /// The format the document came from, recorded in the front matter. + pub source_format: Format, +} + +impl Default for Options { + fn default() -> Self { + Options { + fidelity: Fidelity::Full, + assets_dir: "assets".into(), + source_format: Format::Docx, + } + } +} + +/// Serialises a document to DocMark. +pub fn serialize( + doc: &Document, + assets: &dyn AssetStore, + options: &Options, +) -> (String, ConversionReport) { + let mut out = String::new(); + frontmatter::write(&mut out, doc, options.source_format, options.fidelity); + + match doc { + Document::Text(text) => { + let writer = writer::Writer::new(options, &text.styles, assets); + let (body, mut report) = writer.finish(text); + out.push_str(&body); + report.stats.styles = text.styles.styles.len() as u32; + (out, report) + } + Document::Workbook(_) => { + // Spreadsheet serialisation is Fase 3; emitting a half-formed sheet + // would be worse than saying so. + let mut report = ConversionReport::new(); + report.warn(docsai_model::Warning::Degraded { + what: "workbook".into(), + why: "spreadsheet serialisation arrives in Fase 3".into(), + }); + (out, report) + } + } +} diff --git a/crates/docsai-docmark/src/units.rs b/crates/docsai-docmark/src/units.rs new file mode 100644 index 0000000..bb26789 --- /dev/null +++ b/crates/docsai-docmark/src/units.rs @@ -0,0 +1,60 @@ +//! How lengths and numbers appear in DocMark. +//! +//! Centralised so that every attribute renders a length the same way, which +//! the idempotence test of Fase 2 will depend on. + +use docsai_model::units::{trim_float, Length}; + +/// A length as DocMark writes it: whole pixels when it lands exactly on one at +/// 96 dpi, centimetres otherwise. +/// +/// ``` +/// use docsai_docmark::units::len; +/// use docsai_model::units::Length; +/// assert_eq!(len(Length::from_px(450.0)), "450px"); +/// assert_eq!(len(Length::from_cm(3.5)), "3.5cm"); +/// ``` +pub fn len(value: Length) -> String { + value.to_string() +} + +/// A length as points, for typographic measures (font sizes, spacing). +pub fn pt(value: Length) -> String { + format!("{}pt", trim_float(value.pt(), 2)) +} + +/// A percentage with at most two decimals. +pub fn percent(value: f32) -> String { + format!("{}%", trim_float(value as f64, 2)) +} + +/// A plain number with at most two decimals. +pub fn number(value: f32) -> String { + trim_float(value as f64, 2) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn lengths_prefer_exact_pixels() { + assert_eq!(len(Length::from_px(120.0)), "120px"); + assert_eq!(len(Length::from_cm(3.5)), "3.5cm"); + assert_eq!(len(Length::ZERO), "0px"); + } + + #[test] + fn typographic_measures_are_points() { + assert_eq!(pt(Length::from_half_points(22)), "11pt"); + assert_eq!(pt(Length::from_twips(240)), "12pt"); + } + + #[test] + fn percentages_and_numbers_drop_trailing_zeros() { + assert_eq!(percent(10.0), "10%"); + assert_eq!(percent(12.5), "12.5%"); + assert_eq!(number(45.0), "45"); + assert_eq!(number(-0.0), "0"); + } +} diff --git a/crates/docsai-docmark/src/writer.rs b/crates/docsai-docmark/src/writer.rs new file mode 100644 index 0000000..9ad1238 --- /dev/null +++ b/crates/docsai-docmark/src/writer.rs @@ -0,0 +1,899 @@ +//! IR → DocMark (spec §3, §7, §8). +//! +//! The serialiser is written by hand rather than delegated to a Markdown +//! library so that every byte of the output is under our control: the +//! idempotence test of Fase 2 (`serialize(parse(md)) == md`) has no room for a +//! library's own formatting opinions. + +use docsai_model::assets::AssetStore; +use docsai_model::image::{Anchor, AxisPos, Flip, ImageRef}; +use docsai_model::report::{ConversionReport, Warning}; +use docsai_model::style::{FontProps, ResolvedStyle, StyleCatalog, Underline, VertAlign}; +use docsai_model::text::{ + Block, BreakKind, HeaderFooter, Inline, List, Paragraph, RawFragment, RunProps, Section, Table, + TableCell, TextDocument, +}; +use docsai_model::units::Length; + +use crate::attrs::Attrs; +use crate::escape::{escape, TextContext}; +use crate::units::{len, number, percent, pt}; +use crate::{Fidelity, Options}; + +/// Serialises a text document body (front matter excluded). +pub struct Writer<'a> { + out: String, + options: &'a Options, + styles: &'a StyleCatalog, + assets: &'a dyn AssetStore, + report: ConversionReport, + /// Rendered footnote definitions, emitted at the very end. + footnotes: Vec, +} + +impl<'a> Writer<'a> { + pub fn new(options: &'a Options, styles: &'a StyleCatalog, assets: &'a dyn AssetStore) -> Self { + Writer { + out: String::new(), + options, + styles, + assets, + report: ConversionReport::new(), + footnotes: Vec::new(), + } + } + + fn full(&self) -> bool { + self.options.fidelity == Fidelity::Full + } + + fn plain(&self) -> bool { + self.options.fidelity == Fidelity::Plain + } + + /// Writes the whole document body and returns the text and the report. + pub fn finish(mut self, doc: &TextDocument) -> (String, ConversionReport) { + let multi_section = doc.sections.len() > 1; + for section in &doc.sections { + self.write_section(section, multi_section); + } + if !self.footnotes.is_empty() { + let definitions = std::mem::take(&mut self.footnotes); + for definition in definitions { + self.block(&definition); + } + } + // A document always ends with exactly one newline. + while self.out.ends_with("\n\n") { + self.out.pop(); + } + if !self.out.is_empty() && !self.out.ends_with('\n') { + self.out.push('\n'); + } + (self.out, self.report) + } + + /// Appends a block, separated from the previous one by a blank line. + fn block(&mut self, text: &str) { + if text.is_empty() { + return; + } + if !self.out.is_empty() && !self.out.ends_with("\n\n") { + if !self.out.ends_with('\n') { + self.out.push('\n'); + } + self.out.push('\n'); + } + self.out.push_str(text.trim_end_matches('\n')); + self.out.push('\n'); + } + + // ---------------------------------------------------------------------- + // Sections, headers and footers + // ---------------------------------------------------------------------- + + fn write_section(&mut self, section: &Section, wrap: bool) { + if !self.plain() { + for header in §ion.headers { + self.write_header_footer(header, "header"); + } + for footer in §ion.footers { + self.write_header_footer(footer, "footer"); + } + } + + let body = self.render_blocks(§ion.blocks, 0); + if wrap && !self.plain() { + let mut attrs = Attrs::new(); + attrs.class("section"); + if section.page.columns > 1 { + attrs.set("columns", section.page.columns.to_string()); + } + if let Some(paper) = section.page.paper_name() { + attrs.set("page-size", paper); + } + attrs.set("orientation", section.page.orientation.as_str()); + let rendered = format!("::: {}\n{}\n:::", attrs.render(), body.trim_end()); + self.block(&rendered); + } else { + self.block(&body); + } + } + + fn write_header_footer(&mut self, part: &HeaderFooter, kind: &str) { + let body = self.render_blocks(&part.blocks, 0); + if body.trim().is_empty() { + return; + } + let mut attrs = Attrs::new(); + attrs.class(kind).set("scope", part.scope.as_str()); + let rendered = format!("::: {}\n{}\n:::", attrs.render(), body.trim_end()); + self.block(&rendered); + } + + // ---------------------------------------------------------------------- + // Blocks + // ---------------------------------------------------------------------- + + /// Renders a run of blocks into one string, blank-line separated. + /// + /// One exception: a list directly after a paragraph is joined with a + /// single newline, which is what keeps a nested list *tight* in + /// CommonMark instead of wrapping every item in a `

`. + fn render_blocks(&mut self, blocks: &[Block], depth: usize) -> String { + let mut out = String::new(); + let mut previous: Option<&Block> = None; + for block in blocks { + let text = self.render_block(block, depth); + if text.trim().is_empty() { + continue; + } + if !out.is_empty() { + // Only *inside* a list item: at the top level a blank line is + // what separates a paragraph from the list that follows it. + let tight = depth > 0 + && matches!(previous, Some(Block::Paragraph(_))) + && matches!(block, Block::List(_)); + out.push_str(if tight { "\n" } else { "\n\n" }); + } + out.push_str(&text); + previous = Some(block); + } + out + } + + fn render_block(&mut self, block: &Block, depth: usize) -> String { + match block { + Block::Paragraph(p) => self.render_paragraph(p, None, None), + Block::Heading(h) => { + let hashes = "#".repeat(h.level.clamp(1, 6) as usize); + self.render_paragraph(&h.paragraph, Some(&hashes), None) + } + Block::List(list) => self.render_list(list, depth), + Block::Table(table) => self.render_table(table), + Block::Image(image) => self.render_image(image), + Block::TextBox(text_box) => { + let body = self.render_blocks(&text_box.blocks, depth); + if self.plain() { + return body; + } + let mut attrs = Attrs::new(); + attrs.class("textbox"); + if let Some(size) = text_box.size { + attrs + .set("width", len(size.width)) + .set("height", len(size.height)); + } + if let Some(x) = text_box.x { + attrs.set("x", len(x)); + } + if let Some(y) = text_box.y { + attrs.set("y", len(y)); + } + format!("::: {}\n{}\n:::", attrs.render(), body.trim_end()) + } + Block::Raw(raw) => self.render_raw(raw), + } + } + + /// A raw-block (spec §7). Dropped in `standard` and `plain`, which is a + /// documented loss and therefore a warning. + fn render_raw(&mut self, raw: &RawFragment) -> String { + if !self.full() { + self.report.warn(Warning::RawBlockDropped { + id: raw.id.as_str().to_string(), + format: raw.format.clone(), + }); + return String::new(); + } + self.report.raw_blocks_emitted += 1; + let mut attrs = Attrs::new(); + attrs + .class("raw") + .set("format", &raw.format) + .set("part", &raw.part) + .id(raw.id.as_str()); + // The fence must be longer than the longest backtick run inside. + let longest = raw + .content + .split(|c| c != '`') + .map(str::len) + .max() + .unwrap_or(0); + let fence = "`".repeat(longest.max(2) + 1); + format!( + "::: {}\n{fence}xml\n{}\n{fence}\n:::", + attrs.render(), + raw.content.trim_end() + ) + } + + /// Renders a paragraph. + /// + /// `prefix` turns it into a heading; `extra` merges one more pair into its + /// attribute block, which is how a list item names its definition without + /// growing a second `{...}` (spec §3.3). + fn render_paragraph( + &mut self, + paragraph: &Paragraph, + prefix: Option<&str>, + extra: Option<(&str, &str)>, + ) -> String { + self.report.stats.paragraphs += 1; + let content = self.render_inlines(¶graph.content, TextContext::Block); + let mut attrs = if self.plain() { + Attrs::new() + } else { + self.paragraph_attrs(paragraph, prefix.is_some()) + }; + if let Some((key, value)) = extra.filter(|_| !self.plain()) { + attrs.set(key, value); + } + + let body = content.trim_end_matches(' '); + match prefix { + Some(hashes) => format!("{hashes} {body}{}", attrs.suffix()), + // An empty paragraph is real content in a word processor (it is + // how people space a document out), and a blank line cannot carry + // it: Markdown would merge it away. Full fidelity marks it with an + // empty span so the reverse conversion can put it back; the lossy + // modes drop it, which is exactly what they promise (spec §6). + None if body.is_empty() && attrs.is_empty() => { + if self.full() { + "[]{.empty}".to_string() + } else { + String::new() + } + } + None if body.is_empty() => format!("[]{{.empty}}{}", attrs.suffix()), + None => format!("{body}{}", attrs.suffix()), + } + } + + /// The attribute block of a paragraph: its style plus whatever direct + /// formatting the style does not already imply (the economy rule, §3.1). + fn paragraph_attrs(&self, paragraph: &Paragraph, is_heading: bool) -> Attrs { + let mut attrs = Attrs::new(); + let style = paragraph.format.style.as_ref(); + if let Some(style) = style { + attrs.class(style.as_str()); + } + let resolved = self.styles.resolve(style); + let direct = paragraph.format.direct.minus(&resolved.paragraph); + + if let Some(align) = direct.align { + attrs.set("align", align.as_str()); + } + push_len(&mut attrs, "indent-left", direct.indent_left); + push_len(&mut attrs, "indent-right", direct.indent_right); + push_len(&mut attrs, "indent-first-line", direct.indent_first_line); + push_len(&mut attrs, "indent-hanging", direct.indent_hanging); + if let Some(value) = direct.space_before { + attrs.set("space-before", pt(value)); + } + if let Some(value) = direct.space_after { + attrs.set("space-after", pt(value)); + } + if let Some(value) = direct.background { + attrs.set("background", value); + } + if direct.keep_with_next == Some(true) { + attrs.set("keep-with-next", "true"); + } + if direct.page_break_before == Some(true) { + attrs.set("page-break-before", "true"); + } + // The `#` count already carries the heading depth. + if !is_heading { + if let Some(level) = direct.outline_level { + attrs.set("outline-level", level.to_string()); + } + } + attrs + } + + fn render_list(&mut self, list: &List, depth: usize) -> String { + self.report.stats.lists += 1; + let mut out = String::new(); + for (index, item) in list.items.iter().enumerate() { + let marker = if list.ordered { + format!("{}. ", index + 1) + } else { + "- ".to_string() + }; + let indent = " ".repeat(marker.len()); + // The list definition is named once, on the first item, inside + // that item's own attribute block. + let names_definition = index == 0 && !self.plain(); + let definition = list.def.as_ref().map(|d| d.as_str()); + let body = match (names_definition, definition, item.blocks.split_first()) { + (true, Some(def), Some((Block::Paragraph(first), rest))) => { + let head = self.render_paragraph(first, None, Some(("list", def))); + let tail = self.render_blocks(rest, depth + 1); + match rest.first() { + None => head, + Some(Block::List(_)) => format!("{head}\n{tail}"), + Some(_) => format!("{head}\n\n{tail}"), + } + } + _ => self.render_blocks(&item.blocks, depth + 1), + }; + + let mut lines = body.lines(); + let first = lines.next().unwrap_or(""); + out.push_str(&format!("{marker}{first}\n")); + for line in lines { + if line.is_empty() { + out.push('\n'); + } else { + out.push_str(&format!("{indent}{line}\n")); + } + } + } + out.trim_end().to_string() + } + + // ---------------------------------------------------------------------- + // Tables + // ---------------------------------------------------------------------- + + fn render_table(&mut self, table: &Table) -> String { + self.report.stats.tables += 1; + if table.rows.is_empty() { + return String::new(); + } + if table.is_complex() && !self.plain() { + return self.render_complex_table(table); + } + + let width = table.width().max(1); + let mut grid: Vec> = Vec::new(); + for row in &table.rows { + let mut cells: Vec = Vec::new(); + for cell in &row.cells { + cells.push(self.render_table_cell(cell)); + // A horizontal span leaves the absorbed columns empty so that + // the GFM grid stays rectangular. + for _ in 1..cell.colspan.max(1) { + cells.push(String::new()); + } + } + cells.resize(width, String::new()); + grid.push(cells); + } + + // A GFM table always has a header row; when the source had none we + // keep an empty one and record the fact in the container. + let has_header = table.header_row; + if !has_header { + grid.insert(0, vec![String::new(); width]); + } + + let widths = column_widths(&grid, width); + let mut out = String::new(); + for (index, row) in grid.iter().enumerate() { + out.push_str(&render_row(row, &widths)); + if index == 0 { + out.push_str(&render_delimiter(&widths)); + } + } + + if self.plain() { + return out.trim_end().to_string(); + } + let mut attrs = Attrs::new(); + if let Some(style) = &table.style { + attrs.set("style", style.as_str()); + } + if !table.col_widths.is_empty() { + let widths: Vec = table.col_widths.iter().map(|w| len(*w)).collect(); + attrs.set("col-widths", widths.join(",")); + } + if !has_header { + attrs.set("header-row", "false"); + } + if attrs.is_empty() { + return out.trim_end().to_string(); + } + attrs.class("table"); + format!("::: {}\n{}\n:::", attrs.render(), out.trim_end()) + } + + /// Cell content plus its span attributes, on one line. + fn render_table_cell(&mut self, cell: &TableCell) -> String { + let mut text = match cell.blocks.as_slice() { + [] => String::new(), + [Block::Paragraph(p)] => self.render_inlines(&p.content, TextContext::TableCell), + blocks => { + let rendered = self.render_blocks(blocks, 0); + rendered.replace('\n', " ") + } + }; + text = text.trim().to_string(); + + if self.plain() { + return text; + } + let mut attrs = Attrs::new(); + if cell.colspan > 1 { + attrs.set("colspan", cell.colspan.to_string()); + } + if cell.rowspan > 1 { + attrs.set("rowspan", cell.rowspan.to_string()); + } + if let Some(background) = &cell.background { + attrs.set("background", background); + } + format!("{text}{}", attrs.suffix()).trim().to_string() + } + + /// Tables GFM cannot hold (multi-paragraph or nested content, spec §3.4). + fn render_complex_table(&mut self, table: &Table) -> String { + let mut attrs = Attrs::new(); + attrs.class("table").set("complex", "true"); + if let Some(style) = &table.style { + attrs.set("style", style.as_str()); + } + let mut out = format!("::: {}\n", attrs.render()); + for row in &table.rows { + out.push_str("::: {.row}\n"); + for cell in &row.cells { + let mut cell_attrs = Attrs::new(); + cell_attrs.class("cell"); + if cell.colspan > 1 { + cell_attrs.set("colspan", cell.colspan.to_string()); + } + if cell.rowspan > 1 { + cell_attrs.set("rowspan", cell.rowspan.to_string()); + } + let body = self.render_blocks(&cell.blocks, 0); + out.push_str(&format!( + "::: {}\n{}\n:::\n", + cell_attrs.render(), + body.trim_end() + )); + } + out.push_str(":::\n"); + } + out.push_str(":::"); + out + } + + // ---------------------------------------------------------------------- + // Inlines + // ---------------------------------------------------------------------- + + fn render_inlines(&mut self, inlines: &[Inline], context: TextContext) -> String { + let mut out = String::new(); + for inline in inlines { + out.push_str(&self.render_inline(inline, context)); + } + out + } + + fn render_inline(&mut self, inline: &Inline, context: TextContext) -> String { + match inline { + Inline::Text(text) => escape(text, context), + Inline::Styled { content, props } => { + let inner = self.render_inlines(content, context); + self.wrap_styled(inner, props) + } + Inline::Link { + target, + content, + props, + } => { + let content = strip_redundant_style(content, props); + let label = self.render_inlines(&content, TextContext::LinkLabel); + let link = format!("[{label}]({})", escape_link_target(target)); + if self.plain() || props.is_empty() { + link + } else { + let mut attrs = Attrs::new(); + self.style_attrs(&mut attrs, props); + format!("{link}{}", attrs.render()) + } + } + Inline::Footnote(blocks) => { + self.report.stats.footnotes += 1; + let index = self.footnotes.len() + 1; + let body = self.render_blocks(blocks, 0); + let mut lines = body.lines(); + // The source separates its own marker from the text with a + // space; DocMark draws the marker itself. + let first = lines.next().unwrap_or("").trim_start(); + let mut definition = format!("[^{index}]: {first}"); + for line in lines { + definition.push_str(&format!("\n {line}")); + } + self.footnotes.push(definition); + format!("[^{index}]") + } + Inline::Field { + kind, + cached, + instruction, + } => { + let text = escape(cached, context); + if self.plain() { + return text; + } + let mut attrs = Attrs::new(); + attrs.class("field").set("field", kind.as_str()); + if self.full() && !instruction.is_empty() && instruction != kind.as_str() { + attrs.set("instr", instruction); + } + format!("[{text}]{}", attrs.render()) + } + Inline::Break(BreakKind::Line) => " \n".to_string(), + Inline::Break(kind) => { + if self.plain() { + "\n".to_string() + } else { + let mut attrs = Attrs::new(); + attrs.class("break").set("kind", kind.as_str()); + format!("[]{}", attrs.render()) + } + } + Inline::Image(image) => self.render_image(image), + Inline::Raw(raw) => { + if self.full() { + // An inline raw fragment still travels as a block-level + // raw-block; the spec has no inline form (§7). + let rendered = self.render_raw(raw); + format!("\n\n{rendered}\n\n") + } else { + self.report.warn(Warning::RawBlockDropped { + id: raw.id.as_str().to_string(), + format: raw.format.clone(), + }); + String::new() + } + } + } + } + + /// Wraps rendered text in the emphasis markers and the span its formatting + /// needs. Order is fixed: strike inside italic inside bold inside the span. + fn wrap_styled(&mut self, inner: String, props: &RunProps) -> String { + if inner.is_empty() { + return inner; + } + let resolved = self.styles.resolve(props.style.as_ref()); + let font = props.direct.minus(&resolved.font); + + let mut text = inner; + if font.strike == Some(true) { + text = format!("~~{text}~~"); + } + if font.italic == Some(true) { + text = format!("*{text}*"); + } + if font.bold == Some(true) { + text = format!("**{text}**"); + } + if self.plain() { + return text; + } + + let mut attrs = Attrs::new(); + self.style_attrs(&mut attrs, props); + if attrs.is_empty() { + text + } else { + format!("[{text}]{}", attrs.render()) + } + } + + /// Everything about a run that the emphasis markers cannot express. + fn style_attrs(&self, attrs: &mut Attrs, props: &RunProps) { + if let Some(style) = &props.style { + attrs.class(style.as_str()); + } + let resolved: ResolvedStyle = self.styles.resolve(props.style.as_ref()); + let font: FontProps = props.direct.minus(&resolved.font); + + match font.underline { + Some(Underline::None) | None => {} + Some(Underline::Single) => { + attrs.class("underline"); + } + Some(other) => { + attrs.class("underline"); + attrs.set("underline", underline_name(other)); + } + } + match font.vert_align { + Some(VertAlign::Superscript) => { + attrs.class("sup"); + } + Some(VertAlign::Subscript) => { + attrs.class("sub"); + } + _ => {} + } + if font.bold == Some(false) { + attrs.set("bold", "false"); + } + if font.italic == Some(false) { + attrs.set("italic", "false"); + } + attrs.set_opt("color", font.color); + attrs.set_opt("highlight", font.highlight); + attrs.set_opt("font", font.name); + if let Some(size) = font.size { + attrs.set("size", pt(size)); + } + if font.small_caps == Some(true) { + attrs.class("small-caps"); + } + if font.caps == Some(true) { + attrs.class("caps"); + } + } + + // ---------------------------------------------------------------------- + // Images + // ---------------------------------------------------------------------- + + fn render_image(&mut self, image: &ImageRef) -> String { + self.report.stats.images += 1; + let alt = escape(&image.alt, TextContext::LinkLabel); + let path = match self.assets.info(&image.asset) { + Some(info) => format!("{}/{}", self.options.assets_dir, info.file_name), + None => image + .external_src + .clone() + .unwrap_or_else(|| format!("{}/missing", self.options.assets_dir)), + }; + + if self.plain() { + return format!("![{alt}]({path})"); + } + + let mut attrs = Attrs::new(); + let geometry = &image.geometry; + + // `width`/`height` are mandatory except on two-cell sheet anchors, + // where the grid defines the size (spec §4.1). + let two_cell = matches!(geometry.anchor, Anchor::SheetTwoCell { .. }); + if !two_cell { + attrs + .set("width", len(geometry.display_size.width)) + .set("height", len(geometry.display_size.height)); + } + if let Some((w, h)) = geometry.native_size_px { + attrs.set("native-size", format!("{w}x{h}")); + } + if let Some(dpi) = geometry.dpi.filter(|d| *d != 96) { + attrs.set("dpi", dpi.to_string()); + } + self.anchor_attrs(&mut attrs, &geometry.anchor); + + if geometry.rotation_deg != 0.0 { + attrs.set("rotation", number(geometry.rotation_deg)); + } + if geometry.flip != Flip::None { + attrs.set("flip", geometry.flip.as_str()); + } + if let Some(crop) = geometry.crop.filter(|c| !c.is_empty()) { + attrs.set( + "crop", + format!( + "{},{},{},{}", + percent(crop.left), + percent(crop.top), + percent(crop.right), + percent(crop.bottom) + ), + ); + } + if let Some(border) = &geometry.border { + attrs.set( + "border", + format!("{} {} {}", pt(border.width), border.style, border.color), + ); + } + if let Some(z) = geometry.z_index { + attrs.set("z-index", z.to_string()); + } + + attrs.set_opt("name", image.name.clone()); + attrs.set_opt("title", image.title.clone()); + attrs.set_opt("link", image.link.clone()); + attrs.set_opt("external-src", image.external_src.clone()); + if let Some(raw) = &image.effects_raw { + attrs.set("effects-raw", raw.as_str()); + } + // A hint for viewers: these are preserved but not displayable. + if self + .assets + .info(&image.asset) + .is_some_and(|i| matches!(i.content_type.as_str(), "image/x-emf" | "image/x-wmf")) + { + attrs.set("render", "unsupported"); + } + + format!("![{alt}]({path}){}", attrs.render()) + } + + fn anchor_attrs(&self, attrs: &mut Attrs, anchor: &Anchor) { + match anchor { + Anchor::Inline => {} + Anchor::Floating { + relative_to_h, + relative_to_v, + position, + wrap, + wrap_side, + behind_text, + } => { + attrs.set("anchor", if *behind_text { "behind" } else { "floating" }); + attrs.set("relative-to", relative_to_h.as_str()); + if relative_to_v != relative_to_h { + attrs.set("relative-to-v", relative_to_v.as_str()); + } + match position.h { + AxisPos::Offset(x) => attrs.set("x", len(x)), + AxisPos::Align(keyword) => attrs.set("align-h", keyword.as_str()), + }; + match position.v { + AxisPos::Offset(y) => attrs.set("y", len(y)), + AxisPos::Align(keyword) => attrs.set("align-v", keyword.as_str()), + }; + attrs.set("wrap", wrap.as_str()); + if *wrap_side != docsai_model::image::WrapSide::Both { + attrs.set("wrap-side", wrap_side.as_str()); + } + } + Anchor::SheetTwoCell { + from, + to, + move_with_cells, + size_with_cells, + } => { + attrs + .set("anchor", "two-cell") + .set("from", from.cell.a1()) + .set("from-offset", offset(from.offset_x, from.offset_y)) + .set("to", to.cell.a1()) + .set("to-offset", offset(to.offset_x, to.offset_y)) + .set("move-with-cells", move_with_cells.to_string()) + .set("size-with-cells", size_with_cells.to_string()); + } + Anchor::SheetOneCell { from } => { + attrs + .set("anchor", "one-cell") + .set("from", from.cell.a1()) + .set("from-offset", offset(from.offset_x, from.offset_y)); + } + Anchor::SheetAbsolute { pos } => { + attrs + .set("anchor", "absolute") + .set("x", len(pos.x)) + .set("y", len(pos.y)); + } + } + } +} + +/// Removes the link's own character style from its child runs, so that +/// `[texto](url){.Hyperlink}` does not also carry `{.Hyperlink}` inside the +/// label. +fn strip_redundant_style(content: &[Inline], props: &RunProps) -> Vec { + let Some(style) = &props.style else { + return content.to_vec(); + }; + content + .iter() + .map(|inline| match inline { + Inline::Styled { + content: inner, + props: inner_props, + } if inner_props.style.as_ref() == Some(style) => { + let stripped = RunProps { + style: None, + direct: inner_props.direct.clone(), + }; + if stripped.is_empty() { + // Nothing else to say about the run: unwrap it entirely. + Inline::Styled { + content: inner.clone(), + props: RunProps::default(), + } + } else { + Inline::Styled { + content: inner.clone(), + props: stripped, + } + } + } + other => other.clone(), + }) + .collect() +} + +fn offset(x: Length, y: Length) -> String { + format!("{},{}", len(x), len(y)) +} + +fn push_len(attrs: &mut Attrs, key: &str, value: Option) { + if let Some(value) = value { + attrs.set(key, len(value)); + } +} + +fn underline_name(underline: Underline) -> &'static str { + match underline { + Underline::None => "none", + Underline::Single => "single", + Underline::Double => "double", + Underline::Thick => "thick", + Underline::Dotted => "dotted", + Underline::Dashed => "dashed", + Underline::Wave => "wave", + } +} + +/// A link destination needs `<>` when it holds spaces or parentheses. +fn escape_link_target(target: &str) -> String { + if target.contains(|c: char| c.is_whitespace() || c == '(' || c == ')') { + format!("<{}>", target.replace('>', "%3E")) + } else { + target.to_string() + } +} + +/// Column widths for a padded GFM table (spec §8: padded below 120 columns). +fn column_widths(grid: &[Vec], columns: usize) -> Vec { + const MAX_PADDED_COLUMNS: usize = 120; + if columns > MAX_PADDED_COLUMNS { + return vec![0; columns]; + } + let mut widths = vec![3usize; columns]; + for row in grid { + for (index, cell) in row.iter().enumerate() { + if index < columns { + widths[index] = widths[index].max(cell.chars().count()); + } + } + } + widths +} + +fn render_row(row: &[String], widths: &[usize]) -> String { + let cells: Vec = row + .iter() + .zip(widths) + .map(|(cell, width)| { + let pad = width.saturating_sub(cell.chars().count()); + format!("{cell}{}", " ".repeat(pad)) + }) + .collect(); + format!("| {} |\n", cells.join(" | ")) +} + +fn render_delimiter(widths: &[usize]) -> String { + let cells: Vec = widths.iter().map(|w| "-".repeat((*w).max(3))).collect(); + format!("| {} |\n", cells.join(" | ")) +} diff --git a/crates/docsai-mcp/Cargo.toml b/crates/docsai-mcp/Cargo.toml new file mode 100644 index 0000000..6783c3c --- /dev/null +++ b/crates/docsai-mcp/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "docsai-mcp" +description = "MCP (stdio) server for docsai" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +authors.workspace = true + +[dependencies] +docsai-convert = { workspace = true } diff --git a/crates/docsai-mcp/src/lib.rs b/crates/docsai-mcp/src/lib.rs new file mode 100644 index 0000000..ef35a84 --- /dev/null +++ b/crates/docsai-mcp/src/lib.rs @@ -0,0 +1,26 @@ +//! MCP server over stdio. +//! +//! Scheduled for **Fase 7** of `docs/plan-desarrollo.md`. It is a library, not +//! a binary, so that `docsai mcp` runs inside the single distributed +//! executable (arquitectura §2). +//! +//! One rule already applies to anything added here: **stdout carries the +//! JSON-RPC protocol and nothing else**; every log goes to stderr. + +#![forbid(unsafe_code)] + +/// Names of the tools the server will expose (arquitectura §6). +pub const TOOLS: &[&str] = &[ + "convert_to_markdown", + "convert_from_markdown", + "inspect_document", + "list_supported_formats", +]; + +#[cfg(test)] +mod tests { + #[test] + fn the_v1_tool_set_is_four_tools() { + assert_eq!(super::TOOLS.len(), 4); + } +} diff --git a/crates/docsai-model/Cargo.toml b/crates/docsai-model/Cargo.toml new file mode 100644 index 0000000..1d9d459 --- /dev/null +++ b/crates/docsai-model/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "docsai-model" +description = "Intermediate document model (IR) shared by every docsai converter" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +authors.workspace = true + +[dependencies] +serde = { workspace = true } +thiserror = { workspace = true } + +[dev-dependencies] +proptest = { workspace = true } +serde_json = { workspace = true } diff --git a/crates/docsai-model/src/assets.rs b/crates/docsai-model/src/assets.rs new file mode 100644 index 0000000..d8c2251 --- /dev/null +++ b/crates/docsai-model/src/assets.rs @@ -0,0 +1,308 @@ +//! Media storage abstraction (arquitectura §3.2). +//! +//! The IR never holds image bytes: an [`ImageRef`](crate::image::ImageRef) +//! points at an [`AssetId`] resolved by an [`AssetStore`]. `docsai-convert` +//! supplies the real implementations (a directory for the CLI, memory/base64 +//! for MCP); [`MemoryAssetStore`] here is the reference implementation the +//! readers use in tests — it performs no I/O, so `docsai-model` stays +//! I/O-free. + +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; + +/// Key of a stored asset. Derived from the content hash, so identical bytes +/// always yield the same id and the store deduplicates for free. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[serde(transparent)] +pub struct AssetId(pub String); + +impl AssetId { + pub fn new(id: impl Into) -> Self { + AssetId(id.into()) + } + + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl std::fmt::Display for AssetId { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.0) + } +} + +/// What the store knows about one asset. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub struct AssetInfo { + pub id: AssetId, + /// Deterministic name `img-.`; the extension comes from + /// sniffing the content, never from the source document's naming. + pub file_name: String, + pub content_type: String, + pub byte_len: usize, + /// Native pixel dimensions, when the format header exposes them. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub native_size_px: Option<(u32, u32)>, +} + +/// Errors an asset store can return. +#[derive(Debug, thiserror::Error)] +pub enum AssetError { + #[error("asset {0} not found")] + NotFound(AssetId), + #[error("asset is empty")] + Empty, + #[error("asset storage failed: {0}")] + Io(String), +} + +/// Content-addressed media storage. +pub trait AssetStore { + /// Stores `bytes` and returns their id. Storing the same bytes twice must + /// return the same id and must not duplicate storage. + fn put(&mut self, bytes: &[u8]) -> Result; + + /// The stored bytes, if present. + fn get(&self, id: &AssetId) -> Option<&[u8]>; + + /// The manifest entry, if present. + fn info(&self, id: &AssetId) -> Option<&AssetInfo>; + + /// Every id in the store, in deterministic order. + fn ids(&self) -> Vec; + + /// Number of stored assets. + fn len(&self) -> usize { + self.ids().len() + } + + fn is_empty(&self) -> bool { + self.len() == 0 + } +} + +/// An in-memory [`AssetStore`]; the reference implementation. +#[derive(Debug, Clone, Default)] +pub struct MemoryAssetStore { + entries: BTreeMap)>, +} + +impl MemoryAssetStore { + pub fn new() -> Self { + Self::default() + } + + /// The manifest, ordered by id. + pub fn manifest(&self) -> Vec<&AssetInfo> { + self.entries.values().map(|(info, _)| info).collect() + } +} + +impl AssetStore for MemoryAssetStore { + fn put(&mut self, bytes: &[u8]) -> Result { + if bytes.is_empty() { + return Err(AssetError::Empty); + } + let id = AssetId::new(content_hash(bytes)); + if !self.entries.contains_key(&id) { + let info = describe(&id, bytes); + self.entries.insert(id.clone(), (info, bytes.to_vec())); + } + Ok(id) + } + + fn get(&self, id: &AssetId) -> Option<&[u8]> { + self.entries.get(id).map(|(_, bytes)| bytes.as_slice()) + } + + fn info(&self, id: &AssetId) -> Option<&AssetInfo> { + self.entries.get(id).map(|(info, _)| info) + } + + fn ids(&self) -> Vec { + self.entries.keys().cloned().collect() + } +} + +/// Builds the manifest entry for some bytes: id, deterministic file name, +/// sniffed content type and native dimensions. +pub fn describe(id: &AssetId, bytes: &[u8]) -> AssetInfo { + let (content_type, ext) = sniff(bytes); + AssetInfo { + id: id.clone(), + file_name: format!("img-{}.{}", &id.0[..id.0.len().min(8)], ext), + content_type: content_type.to_string(), + byte_len: bytes.len(), + native_size_px: dimensions(bytes), + } +} + +/// FNV-1a 64-bit over the content, rendered as 16 lowercase hex digits. +/// +/// A non-cryptographic hash is enough here: the id only has to be stable and +/// collision-free in practice for the media of one document, and avoiding a +/// crypto dependency keeps `docsai-model` free of heavy deps. The convert +/// layer is free to substitute a stronger hash without changing this trait. +pub fn content_hash(bytes: &[u8]) -> String { + let mut hash: u64 = 0xcbf2_9ce4_8422_2325; + for b in bytes { + hash ^= *b as u64; + hash = hash.wrapping_mul(0x0000_0100_0000_01b3); + } + // Mix in the length so that same-content-different-length is impossible. + hash ^= bytes.len() as u64; + hash = hash.wrapping_mul(0x0000_0100_0000_01b3); + format!("{hash:016x}") +} + +/// Guesses `(content-type, extension)` from the magic bytes. +/// +/// Deliberately header-only and dependency-free: docsai never re-encodes a +/// bitmap, so it only needs to name and measure it. +pub fn sniff(bytes: &[u8]) -> (&'static str, &'static str) { + match bytes { + b if b.starts_with(b"\x89PNG\r\n\x1a\n") => ("image/png", "png"), + b if b.starts_with(b"\xff\xd8\xff") => ("image/jpeg", "jpg"), + b if b.starts_with(b"GIF87a") || b.starts_with(b"GIF89a") => ("image/gif", "gif"), + b if b.starts_with(b"BM") => ("image/bmp", "bmp"), + b if b.starts_with(b"RIFF") && b.len() > 12 && &b[8..12] == b"WEBP" => { + ("image/webp", "webp") + } + b if b.starts_with(b"II*\x00") || b.starts_with(b"MM\x00*") => ("image/tiff", "tiff"), + b if is_emf(b) => ("image/x-emf", "emf"), + [0xd7, 0xcd, 0xc6, 0x9a, ..] => ("image/x-wmf", "wmf"), + b if b.starts_with(b" ("image/svg+xml", "svg"), + _ => ("application/octet-stream", "bin"), + } +} + +fn is_emf(bytes: &[u8]) -> bool { + bytes.len() >= 44 + && u32::from_le_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]) == 1 + && &bytes[40..44] == b" EMF" +} + +/// Native pixel dimensions read straight from the format header. +/// +/// Returns `None` for formats whose size is not a pixel count (EMF/WMF are +/// vector) or that this function does not parse. +pub fn dimensions(bytes: &[u8]) -> Option<(u32, u32)> { + match sniff(bytes).1 { + "png" if bytes.len() >= 24 => Some((be32(&bytes[16..20]), be32(&bytes[20..24]))), + "gif" if bytes.len() >= 10 => Some(( + u16::from_le_bytes([bytes[6], bytes[7]]) as u32, + u16::from_le_bytes([bytes[8], bytes[9]]) as u32, + )), + "bmp" if bytes.len() >= 26 => Some(( + u32::from_le_bytes([bytes[18], bytes[19], bytes[20], bytes[21]]), + u32::from_le_bytes([bytes[22], bytes[23], bytes[24], bytes[25]]), + )), + "jpg" => jpeg_dimensions(bytes), + _ => None, + } +} + +fn be32(b: &[u8]) -> u32 { + u32::from_be_bytes([b[0], b[1], b[2], b[3]]) +} + +/// Walks JPEG markers to the first frame header. +fn jpeg_dimensions(bytes: &[u8]) -> Option<(u32, u32)> { + let mut i = 2usize; + while i + 9 < bytes.len() { + if bytes[i] != 0xFF { + i += 1; + continue; + } + let marker = bytes[i + 1]; + // SOF0..SOF15 except the non-frame markers DHT/JPG/DAC. + if (0xC0..=0xCF).contains(&marker) && !matches!(marker, 0xC4 | 0xC8 | 0xCC) { + let h = u16::from_be_bytes([bytes[i + 5], bytes[i + 6]]) as u32; + let w = u16::from_be_bytes([bytes[i + 7], bytes[i + 8]]) as u32; + return Some((w, h)); + } + let len = u16::from_be_bytes([bytes[i + 2], bytes[i + 3]]) as usize; + if len < 2 { + return None; + } + i += 2 + len; + } + None +} + +#[cfg(test)] +mod tests { + use super::*; + + fn png_bytes() -> Vec { + let mut v = b"\x89PNG\r\n\x1a\n".to_vec(); + v.extend_from_slice(&[0, 0, 0, 13]); + v.extend_from_slice(b"IHDR"); + v.extend_from_slice(&120u32.to_be_bytes()); + v.extend_from_slice(&90u32.to_be_bytes()); + v.extend_from_slice(&[8, 2, 0, 0, 0]); + v + } + + #[test] + fn identical_bytes_share_one_asset() { + let mut store = MemoryAssetStore::new(); + let a = store.put(&png_bytes()).unwrap(); + let b = store.put(&png_bytes()).unwrap(); + assert_eq!(a, b); + assert_eq!(store.len(), 1, "deduplicated by content"); + } + + #[test] + fn different_bytes_get_different_ids() { + let mut store = MemoryAssetStore::new(); + let a = store.put(&png_bytes()).unwrap(); + let b = store.put(b"GIF87a\x30\x00\x20\x00rest").unwrap(); + assert_ne!(a, b); + assert_eq!(store.len(), 2); + } + + #[test] + fn empty_assets_are_rejected() { + let mut store = MemoryAssetStore::new(); + assert!(matches!(store.put(&[]), Err(AssetError::Empty))); + } + + #[test] + fn file_names_are_deterministic_and_content_typed() { + let mut store = MemoryAssetStore::new(); + let id = store.put(&png_bytes()).unwrap(); + let info = store.info(&id).unwrap(); + assert!(info.file_name.starts_with("img-")); + assert!(info.file_name.ends_with(".png")); + assert_eq!(info.content_type, "image/png"); + assert_eq!(info.native_size_px, Some((120, 90))); + } + + #[test] + fn sniffs_the_formats_the_readers_meet() { + assert_eq!(sniff(&png_bytes()).1, "png"); + assert_eq!(sniff(b"GIF89a....").1, "gif"); + assert_eq!(sniff(b"\xff\xd8\xff\xe0").1, "jpg"); + assert_eq!(sniff(b"BM....").1, "bmp"); + assert_eq!(sniff(&[0xd7, 0xcd, 0xc6, 0x9a, 0]).1, "wmf"); + assert_eq!(sniff(b"nothing recognisable").1, "bin"); + } + + #[test] + fn sniffs_emf_by_signature_not_extension() { + let mut emf = vec![0u8; 44]; + emf[0] = 1; + emf[40..44].copy_from_slice(b" EMF"); + assert_eq!(sniff(&emf), ("image/x-emf", "emf")); + assert_eq!(dimensions(&emf), None, "vector formats have no pixel size"); + } + + #[test] + fn gif_dimensions_are_little_endian() { + let gif = b"GIF87a\x30\x00\x20\x00\xf0\x00"; + assert_eq!(dimensions(gif), Some((48, 32))); + } +} diff --git a/crates/docsai-model/src/image.rs b/crates/docsai-model/src/image.rs new file mode 100644 index 0000000..dc7685c --- /dev/null +++ b/crates/docsai-model/src/image.rs @@ -0,0 +1,435 @@ +//! Normalised image model (arquitectura §3.1). +//! +//! Every source format has its own graphics model — DrawingML, VML, Escher, +//! SpreadsheetDrawingML, ODF `draw:frame`. Readers translate *into* this model +//! rather than around it: whatever cannot be mapped degrades with a typed +//! warning, and effects with no representation move to a raw-block through +//! [`ImageRef::effects_raw`]. + +use serde::{Deserialize, Serialize}; + +use crate::assets::AssetId; +use crate::sheet::CellRef; +use crate::units::{Length, Point, Size}; + +/// Identifier of a raw fragment kept aside for round-trip fidelity. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[serde(transparent)] +pub struct RawId(pub String); + +impl RawId { + pub fn new(id: impl Into) -> Self { + RawId(id.into()) + } + + pub fn as_str(&self) -> &str { + &self.0 + } +} + +/// A picture placed in a document, with everything needed to put it back. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub struct ImageRef { + /// Key in the [`AssetStore`](crate::assets::AssetStore): the content hash, + /// so N appearances of one bitmap share a single stored file. + pub asset: AssetId, + pub geometry: ImageGeometry, + /// Alternative text. Travels in the Markdown `![…]` slot, never as an + /// attribute, so plain viewers show it (spec §3.5). + #[serde(default, skip_serializing_if = "String::is_empty")] + pub alt: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub title: Option, + /// Internal object name (`wp:docPr @name`). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub name: Option, + /// Hyperlink applied to the image itself. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub link: Option, + /// Set when the image was *linked* rather than embedded. The referenced + /// content is never fetched (arquitectura §3.2, security). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub external_src: Option, + /// Effects with no DocMark representation, kept as an associated raw-block. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub effects_raw: Option, +} + +impl ImageRef { + pub fn new(asset: AssetId, geometry: ImageGeometry) -> Self { + ImageRef { + asset, + geometry, + alt: String::new(), + title: None, + name: None, + link: None, + external_src: None, + effects_raw: None, + } + } +} + +/// Placement and transform of an image. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub struct ImageGeometry { + /// Size as displayed, which need not match the bitmap's native size. + pub display_size: Size, + /// Native pixel dimensions of the bitmap, when known. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub native_size_px: Option<(u32, u32)>, + /// Declared resolution, when it differs from 96 dpi. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub dpi: Option, + pub anchor: Anchor, + /// Clockwise rotation in degrees. + #[serde(default, skip_serializing_if = "is_zero_f32")] + pub rotation_deg: f32, + #[serde(default, skip_serializing_if = "Flip::is_none")] + pub flip: Flip, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub crop: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub border: Option, + /// Stacking order among floating objects. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub z_index: Option, +} + +fn is_zero_f32(v: &f32) -> bool { + *v == 0.0 +} + +impl ImageGeometry { + /// An inline image of the given displayed size. + pub fn inline(display_size: Size) -> Self { + ImageGeometry { + display_size, + native_size_px: None, + dpi: None, + anchor: Anchor::Inline, + rotation_deg: 0.0, + flip: Flip::None, + crop: None, + border: None, + z_index: None, + } + } + + /// True when the anchor only makes sense inside a spreadsheet. + pub fn is_sheet_anchored(&self) -> bool { + self.anchor.is_sheet() + } +} + +/// How an image is attached to its surroundings. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case", tag = "kind")] +pub enum Anchor { + /// Flows with the text. + Inline, + /// Positioned relative to a page element, with text wrapping around it. + Floating { + relative_to_h: RelBase, + relative_to_v: RelBase, + position: HVPos, + wrap: WrapMode, + wrap_side: WrapSide, + behind_text: bool, + }, + /// Spreadsheet only: stretches and moves with the grid. + SheetTwoCell { + from: CellAnchor, + to: CellAnchor, + move_with_cells: bool, + size_with_cells: bool, + }, + /// Spreadsheet only: anchored to one cell, fixed size. + SheetOneCell { from: CellAnchor }, + /// Spreadsheet only: absolute position on the sheet. + SheetAbsolute { pos: Point }, +} + +impl Anchor { + /// True for the three spreadsheet-only anchors. + pub fn is_sheet(&self) -> bool { + matches!( + self, + Anchor::SheetTwoCell { .. } + | Anchor::SheetOneCell { .. } + | Anchor::SheetAbsolute { .. } + ) + } + + /// The DocMark `anchor=` value for this anchor (spec §3.5 / §4.1). + pub fn keyword(&self) -> &'static str { + match self { + Anchor::Inline => "inline", + Anchor::Floating { behind_text, .. } => { + if *behind_text { + "behind" + } else { + "floating" + } + } + Anchor::SheetTwoCell { .. } => "two-cell", + Anchor::SheetOneCell { .. } => "one-cell", + Anchor::SheetAbsolute { .. } => "absolute", + } + } +} + +/// What a floating position is measured from. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum RelBase { + Page, + Margin, + Paragraph, + Character, + Line, + Column, +} + +impl RelBase { + pub fn as_str(self) -> &'static str { + match self { + RelBase::Page => "page", + RelBase::Margin => "margin", + RelBase::Paragraph => "paragraph", + RelBase::Character => "character", + RelBase::Line => "line", + RelBase::Column => "column", + } + } +} + +/// A floating position: either an offset or a symbolic alignment, per axis. +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub struct HVPos { + pub h: AxisPos, + pub v: AxisPos, +} + +/// Position along one axis. +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum AxisPos { + /// Offset from the relative base. + Offset(Length), + /// Symbolic alignment (`left`, `center`, `top`…) as the source expressed it. + Align(AlignKeyword), +} + +/// Symbolic alignment keywords shared by both axes. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum AlignKeyword { + Left, + Center, + Right, + Inside, + Outside, + Top, + Middle, + Bottom, +} + +impl AlignKeyword { + pub fn as_str(self) -> &'static str { + match self { + AlignKeyword::Left => "left", + AlignKeyword::Center => "center", + AlignKeyword::Right => "right", + AlignKeyword::Inside => "inside", + AlignKeyword::Outside => "outside", + AlignKeyword::Top => "top", + AlignKeyword::Middle => "middle", + AlignKeyword::Bottom => "bottom", + } + } +} + +/// How text flows around a floating image. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum WrapMode { + Square, + Tight, + Through, + TopBottom, + None, +} + +impl WrapMode { + pub fn as_str(self) -> &'static str { + match self { + WrapMode::Square => "square", + WrapMode::Tight => "tight", + WrapMode::Through => "through", + WrapMode::TopBottom => "top-bottom", + WrapMode::None => "none", + } + } +} + +/// Which sides of a floating image text may flow on. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum WrapSide { + Both, + Left, + Right, + Largest, +} + +impl WrapSide { + pub fn as_str(self) -> &'static str { + match self { + WrapSide::Both => "both", + WrapSide::Left => "left", + WrapSide::Right => "right", + WrapSide::Largest => "largest", + } + } +} + +/// Mirroring. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum Flip { + #[default] + None, + H, + V, + HV, +} + +impl Flip { + pub fn is_none(&self) -> bool { + *self == Flip::None + } + + pub fn from_flags(h: bool, v: bool) -> Flip { + match (h, v) { + (false, false) => Flip::None, + (true, false) => Flip::H, + (false, true) => Flip::V, + (true, true) => Flip::HV, + } + } + + pub fn as_str(self) -> &'static str { + match self { + Flip::None => "none", + Flip::H => "h", + Flip::V => "v", + Flip::HV => "hv", + } + } +} + +/// Crop, as a percentage of the original taken off each side. +#[derive(Debug, Clone, Copy, PartialEq, Default, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub struct CropRect { + pub left: f32, + pub top: f32, + pub right: f32, + pub bottom: f32, +} + +impl CropRect { + /// True when nothing is actually cropped. + pub fn is_empty(&self) -> bool { + self.left == 0.0 && self.top == 0.0 && self.right == 0.0 && self.bottom == 0.0 + } +} + +/// A single-line border around an image. Anything richer goes to a raw-block. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub struct SimpleBorder { + pub width: Length, + /// `solid`, `dashed`, `dotted`… as named by the source. + pub style: String, + /// `#RRGGBB`. + pub color: String, +} + +/// A cell-relative anchor point: a cell plus an offset inside it. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub struct CellAnchor { + pub cell: CellRef, + pub offset_x: Length, + pub offset_y: Length, +} + +impl CellAnchor { + pub fn new(cell: CellRef, offset_x: Length, offset_y: Length) -> Self { + CellAnchor { + cell, + offset_x, + offset_y, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn anchor_keywords_match_the_spec() { + assert_eq!(Anchor::Inline.keyword(), "inline"); + let floating = Anchor::Floating { + relative_to_h: RelBase::Margin, + relative_to_v: RelBase::Paragraph, + position: HVPos { + h: AxisPos::Offset(Length::from_cm(1.2)), + v: AxisPos::Align(AlignKeyword::Top), + }, + wrap: WrapMode::Square, + wrap_side: WrapSide::Right, + behind_text: false, + }; + assert_eq!(floating.keyword(), "floating"); + assert!(!floating.is_sheet()); + } + + #[test] + fn behind_text_changes_the_keyword() { + let behind = Anchor::Floating { + relative_to_h: RelBase::Page, + relative_to_v: RelBase::Page, + position: HVPos { + h: AxisPos::Offset(Length::ZERO), + v: AxisPos::Offset(Length::ZERO), + }, + wrap: WrapMode::None, + wrap_side: WrapSide::Both, + behind_text: true, + }; + assert_eq!(behind.keyword(), "behind"); + } + + #[test] + fn sheet_anchors_are_flagged() { + let a = Anchor::SheetOneCell { + from: CellAnchor::new(CellRef::new(1, 1), Length::ZERO, Length::ZERO), + }; + assert!(a.is_sheet()); + assert_eq!(a.keyword(), "one-cell"); + } + + #[test] + fn flip_flags_combine() { + assert_eq!(Flip::from_flags(true, true), Flip::HV); + assert_eq!(Flip::from_flags(false, false), Flip::None); + assert!(Flip::None.is_none()); + } +} diff --git a/crates/docsai-model/src/lib.rs b/crates/docsai-model/src/lib.rs new file mode 100644 index 0000000..20d2213 --- /dev/null +++ b/crates/docsai-model/src/lib.rs @@ -0,0 +1,199 @@ +//! The docsai intermediate document model (IR). +//! +//! Every converter reads *into* this model and writes *from* it; no converter +//! ever talks to another converter (arquitectura §1). The crate is +//! deliberately free of I/O and of heavy dependencies: it knows nothing about +//! ZIP, XML or the filesystem, and media live behind the +//! [`AssetStore`] trait. +//! +//! Design rules the types encode: +//! +//! * **Style = reference + delta**, never flattened formatting ([`style`]). +//! * **Lengths are normalised to EMU** with a newtype ([`units::Length`]). +//! * **One image model for every format** ([`image`]), whose invariants are +//! checked by [`validate`]. +//! * **Everything is `serde`-serialisable**, which gives `inspect --json` and +//! round-trip diffing for free. +//! +//! ``` +//! use docsai_model::{Document, text::{TextDocument, Section, Block, Paragraph}}; +//! +//! let doc = Document::Text(TextDocument { +//! sections: vec![Section { +//! blocks: vec![Block::Paragraph(Paragraph::text("Hola"))], +//! ..Default::default() +//! }], +//! ..Default::default() +//! }); +//! assert!(docsai_model::validate::validate(&doc).is_ok()); +//! ``` + +pub mod assets; +pub mod image; +pub mod list; +pub mod report; +pub mod sheet; +pub mod style; +pub mod text; +pub mod units; +pub mod validate; + +use serde::{Deserialize, Serialize}; + +pub use assets::{AssetId, AssetInfo, AssetStore, MemoryAssetStore}; +pub use report::{ConversionReport, ConversionStats, Severity, Warning}; +pub use sheet::Workbook; +pub use style::{StyleCatalog, StyleId}; +pub use text::{DocumentMeta, TextDocument}; +pub use units::Length; + +/// Version of the DocMark specification this model targets. +pub const DOCMARK_VERSION: &str = "1.0"; + +/// The two shapes a document can take. +/// +/// Externally tagged on purpose: an internally tagged enum makes serde buffer +/// the payload through `Content`, which rewrites every map key as a string and +/// would break the integer-keyed maps inside [`Workbook`]. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum Document { + Text(TextDocument), + Workbook(Workbook), +} + +impl Document { + /// Document properties, whichever shape it is. + pub fn meta(&self) -> &DocumentMeta { + match self { + Document::Text(d) => &d.meta, + Document::Workbook(w) => &w.meta, + } + } + + pub fn meta_mut(&mut self) -> &mut DocumentMeta { + match self { + Document::Text(d) => &mut d.meta, + Document::Workbook(w) => &mut w.meta, + } + } + + /// The style catalogue, whichever shape it is. + pub fn styles(&self) -> &StyleCatalog { + match self { + Document::Text(d) => &d.styles, + Document::Workbook(w) => &w.styles, + } + } + + pub fn is_text(&self) -> bool { + matches!(self, Document::Text(_)) + } + + pub fn is_workbook(&self) -> bool { + matches!(self, Document::Workbook(_)) + } +} + +/// The source formats docsai knows about. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum Format { + Docx, + Doc, + Xlsx, + Xls, + Odt, + Ods, + /// The DocMark pivot itself. + DocMark, +} + +impl Format { + /// The name used in the DocMark front matter and on the CLI. + pub fn as_str(self) -> &'static str { + match self { + Format::Docx => "docx", + Format::Doc => "doc", + Format::Xlsx => "xlsx", + Format::Xls => "xls", + Format::Odt => "odt", + Format::Ods => "ods", + Format::DocMark => "docmark", + } + } + + /// Parses a format name; also accepts `md`/`markdown` for DocMark. + pub fn parse(name: &str) -> Option { + match name + .trim() + .trim_start_matches('.') + .to_ascii_lowercase() + .as_str() + { + "docx" | "docm" => Some(Format::Docx), + "doc" => Some(Format::Doc), + "xlsx" | "xlsm" => Some(Format::Xlsx), + "xls" => Some(Format::Xls), + "odt" => Some(Format::Odt), + "ods" => Some(Format::Ods), + "docmark" | "dmk" | "md" | "markdown" => Some(Format::DocMark), + _ => None, + } + } + + /// True when documents of this format become a [`Document::Workbook`]. + pub fn is_spreadsheet(self) -> bool { + matches!(self, Format::Xlsx | Format::Xls | Format::Ods) + } +} + +impl std::fmt::Display for Format { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn format_names_round_trip() { + for f in [ + Format::Docx, + Format::Doc, + Format::Xlsx, + Format::Xls, + Format::Odt, + Format::Ods, + Format::DocMark, + ] { + assert_eq!(Format::parse(f.as_str()), Some(f)); + } + } + + #[test] + fn format_parsing_is_forgiving_about_shape() { + assert_eq!(Format::parse(".DOCX"), Some(Format::Docx)); + assert_eq!(Format::parse("md"), Some(Format::DocMark)); + assert_eq!( + Format::parse("docm"), + Some(Format::Docx), + "macro-enabled files are read as their macro-free equivalent" + ); + assert_eq!(Format::parse("pdf"), None); + } + + #[test] + fn document_exposes_meta_for_both_shapes() { + let mut text = Document::Text(TextDocument::default()); + text.meta_mut().title = Some("T".into()); + assert_eq!(text.meta().title.as_deref(), Some("T")); + assert!(text.is_text()); + + let book = Document::Workbook(Workbook::default()); + assert!(book.is_workbook()); + assert_eq!(book.meta().title, None); + } +} diff --git a/crates/docsai-model/src/list.rs b/crates/docsai-model/src/list.rs new file mode 100644 index 0000000..d9b3312 --- /dev/null +++ b/crates/docsai-model/src/list.rs @@ -0,0 +1,180 @@ +//! List definitions. +//! +//! OOXML lists are not nested syntactically: paragraphs carry flat +//! `(numId, ilvl)` pairs and the tree has to be rebuilt. The IR stores the +//! *rebuilt tree* in [`crate::text::List`] and the *typographic definition* +//! here, so DocMark can keep clean Markdown lists and still regenerate +//! `numbering.xml`. + +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; + +use crate::units::Length; + +/// Identifier of a list definition (`L1`, `L2`… as written to the front matter). +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[serde(transparent)] +pub struct ListId(pub String); + +impl ListId { + pub fn new(id: impl Into) -> Self { + ListId(id.into()) + } + + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl std::fmt::Display for ListId { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.0) + } +} + +/// Numbering style of a list level. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum NumFormat { + Decimal, + LowerLetter, + UpperLetter, + LowerRoman, + UpperRoman, + Bullet, + /// The level shows no marker at all. + None, + /// Any other format, kept verbatim so the writer can restore it. + Other(String), +} + +impl NumFormat { + /// True for formats that produce an ordered Markdown list. + pub fn is_ordered(&self) -> bool { + !matches!(self, NumFormat::Bullet | NumFormat::None) + } + + pub fn as_str(&self) -> &str { + match self { + NumFormat::Decimal => "decimal", + NumFormat::LowerLetter => "lowerLetter", + NumFormat::UpperLetter => "upperLetter", + NumFormat::LowerRoman => "lowerRoman", + NumFormat::UpperRoman => "upperRoman", + NumFormat::Bullet => "bullet", + NumFormat::None => "none", + NumFormat::Other(s) => s, + } + } + + /// Maps an OOXML `w:numFmt` value. + pub fn from_ooxml(value: &str) -> NumFormat { + match value { + "decimal" => NumFormat::Decimal, + "lowerLetter" => NumFormat::LowerLetter, + "upperLetter" => NumFormat::UpperLetter, + "lowerRoman" => NumFormat::LowerRoman, + "upperRoman" => NumFormat::UpperRoman, + "bullet" => NumFormat::Bullet, + "none" => NumFormat::None, + other => NumFormat::Other(other.to_string()), + } + } +} + +/// One level of a list definition. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub struct ListLevel { + pub format: NumFormat, + /// Marker template as the source writes it (`%1.`, `%2)`, `•`). + pub text: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub start: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub indent: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub hanging: Option, +} + +impl ListLevel { + pub fn new(format: NumFormat, text: impl Into) -> Self { + ListLevel { + format, + text: text.into(), + start: None, + indent: None, + hanging: None, + } + } +} + +/// A whole list definition. +#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub struct ListDef { + pub levels: Vec, +} + +impl ListDef { + pub fn level(&self, ilvl: usize) -> Option<&ListLevel> { + self.levels.get(ilvl) + } + + /// Whether a level renders as an ordered list; unknown levels default to + /// ordered, matching Word's behaviour for out-of-range `ilvl`. + pub fn is_ordered_at(&self, ilvl: usize) -> bool { + self.level(ilvl).is_none_or(|l| l.format.is_ordered()) + } +} + +/// The document's list definitions, ordered by id for deterministic output. +#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case", transparent)] +pub struct ListCatalog { + pub defs: BTreeMap, +} + +impl ListCatalog { + pub fn insert(&mut self, id: ListId, def: ListDef) { + self.defs.insert(id, def); + } + + pub fn get(&self, id: &ListId) -> Option<&ListDef> { + self.defs.get(id) + } + + pub fn is_empty(&self) -> bool { + self.defs.is_empty() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn bullets_are_unordered_everything_else_is_ordered() { + assert!(NumFormat::Decimal.is_ordered()); + assert!(NumFormat::LowerRoman.is_ordered()); + assert!(!NumFormat::Bullet.is_ordered()); + assert!(!NumFormat::None.is_ordered()); + assert!(NumFormat::Other("chicago".into()).is_ordered()); + } + + #[test] + fn unknown_ooxml_formats_survive_verbatim() { + let f = NumFormat::from_ooxml("ideographDigital"); + assert_eq!(f, NumFormat::Other("ideographDigital".into())); + assert_eq!(f.as_str(), "ideographDigital"); + } + + #[test] + fn missing_levels_default_to_ordered() { + let def = ListDef { + levels: vec![ListLevel::new(NumFormat::Bullet, "•")], + }; + assert!(!def.is_ordered_at(0)); + assert!(def.is_ordered_at(5)); + } +} diff --git a/crates/docsai-model/src/report.rs b/crates/docsai-model/src/report.rs new file mode 100644 index 0000000..19b5d1a --- /dev/null +++ b/crates/docsai-model/src/report.rs @@ -0,0 +1,218 @@ +//! Structured conversion reports. +//! +//! Rule 3 of `AGENTS.md`: information is never lost silently. Every +//! degradation a reader or writer performs shows up here, typed, and travels +//! to the CLI (stderr / `--json`) and to the MCP response. + +use serde::{Deserialize, Serialize}; + +/// How much a warning should worry the user. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum Severity { + /// Nothing was lost; the user may still want to know. + Info, + /// Something was simplified but is still recognisable. + Minor, + /// Content was dropped or degraded in a way that shows. + Severe, +} + +/// A typed conversion warning. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case", tag = "warning")] +pub enum Warning { + /// An element with no IR representation was met. + UnsupportedElement { + kind: String, + location: String, + /// What was done about it (`raw-block`, `skipped`…). + action: String, + }, + /// Something was mapped, but imperfectly. + Degraded { what: String, why: String }, + /// Image geometry could not be mapped in full (arquitectura §3.1). + ImageGeometryDegraded { what: String, why: String }, + /// A media part could not be stored or read. + AssetIssue { asset: String, why: String }, + /// A linked (not embedded) image was found; remote content is never + /// fetched, so only the URL survives. + ExternalImageNotFetched { url: String }, + /// Macros are always ignored (`.docm`/`.xlsm` read as their macro-free + /// equivalents). + MacrosIgnored { part: String }, + /// Tracked changes were resolved to their accepted state. + RevisionsAccepted { count: u32 }, + /// A raw-block could not be re-injected because the target dialect differs. + RawBlockDropped { id: String, format: String }, +} + +impl Warning { + pub fn severity(&self) -> Severity { + match self { + Warning::UnsupportedElement { action, .. } if action == "raw-block" => Severity::Minor, + Warning::UnsupportedElement { .. } => Severity::Severe, + Warning::Degraded { .. } | Warning::ImageGeometryDegraded { .. } => Severity::Minor, + Warning::AssetIssue { .. } | Warning::RawBlockDropped { .. } => Severity::Severe, + Warning::ExternalImageNotFetched { .. } => Severity::Minor, + Warning::MacrosIgnored { .. } | Warning::RevisionsAccepted { .. } => Severity::Info, + } + } + + /// One-line human rendering, for stderr. + pub fn message(&self) -> String { + match self { + Warning::UnsupportedElement { + kind, + location, + action, + } => format!("unsupported element <{kind}> at {location}: {action}"), + Warning::Degraded { what, why } => format!("degraded {what}: {why}"), + Warning::ImageGeometryDegraded { what, why } => { + format!("image geometry degraded ({what}): {why}") + } + Warning::AssetIssue { asset, why } => format!("asset {asset}: {why}"), + Warning::ExternalImageNotFetched { url } => { + format!("linked image not fetched (by design): {url}") + } + Warning::MacrosIgnored { part } => format!("macros ignored in {part}"), + Warning::RevisionsAccepted { count } => { + format!("{count} tracked change(s) taken as accepted") + } + Warning::RawBlockDropped { id, format } => { + format!("raw-block {id} dropped: source dialect is {format}") + } + } + } +} + +/// Counters describing what a conversion processed. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case", default)] +pub struct ConversionStats { + pub paragraphs: u32, + pub headings: u32, + pub lists: u32, + pub tables: u32, + pub images: u32, + pub footnotes: u32, + pub sheets: u32, + pub cells: u32, + pub formulas: u32, + pub styles: u32, +} + +/// The outcome of a conversion, beyond the document itself. +#[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case", default)] +pub struct ConversionReport { + pub warnings: Vec, + pub raw_blocks_emitted: u32, + pub stats: ConversionStats, +} + +impl ConversionReport { + pub fn new() -> Self { + Self::default() + } + + pub fn warn(&mut self, warning: Warning) { + self.warnings.push(warning); + } + + /// Records an unsupported element that was preserved as a raw-block. + pub fn raw_block(&mut self, kind: impl Into, location: impl Into) { + self.raw_blocks_emitted += 1; + self.warn(Warning::UnsupportedElement { + kind: kind.into(), + location: location.into(), + action: "raw-block".into(), + }); + } + + /// The worst severity present, if any. + pub fn max_severity(&self) -> Option { + self.warnings.iter().map(|w| w.severity()).max() + } + + /// True when at least one warning is [`Severity::Severe`]; drives the + /// CLI's exit code 1 and `--strict`. + pub fn has_severe(&self) -> bool { + self.max_severity() == Some(Severity::Severe) + } + + /// Folds another report into this one. + pub fn merge(&mut self, other: ConversionReport) { + self.warnings.extend(other.warnings); + self.raw_blocks_emitted += other.raw_blocks_emitted; + let s = other.stats; + self.stats.paragraphs += s.paragraphs; + self.stats.headings += s.headings; + self.stats.lists += s.lists; + self.stats.tables += s.tables; + self.stats.images += s.images; + self.stats.footnotes += s.footnotes; + self.stats.sheets += s.sheets; + self.stats.cells += s.cells; + self.stats.formulas += s.formulas; + self.stats.styles += s.styles; + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn raw_blocks_are_counted_and_reported() { + let mut report = ConversionReport::new(); + report.raw_block("w:sdt", "word/document.xml:12"); + assert_eq!(report.raw_blocks_emitted, 1); + assert_eq!(report.warnings.len(), 1); + assert_eq!(report.max_severity(), Some(Severity::Minor)); + assert!(!report.has_severe()); + } + + #[test] + fn dropping_an_element_is_severe_preserving_it_is_not() { + let dropped = Warning::UnsupportedElement { + kind: "w:object".into(), + location: "x".into(), + action: "skipped".into(), + }; + let kept = Warning::UnsupportedElement { + kind: "w:object".into(), + location: "x".into(), + action: "raw-block".into(), + }; + assert_eq!(dropped.severity(), Severity::Severe); + assert_eq!(kept.severity(), Severity::Minor); + } + + #[test] + fn merge_accumulates_everything() { + let mut a = ConversionReport::new(); + a.stats.paragraphs = 2; + a.raw_block("x", "y"); + let mut b = ConversionReport::new(); + b.stats.paragraphs = 3; + b.stats.images = 1; + b.warn(Warning::MacrosIgnored { + part: "word/vbaProject.bin".into(), + }); + a.merge(b); + assert_eq!(a.stats.paragraphs, 5); + assert_eq!(a.stats.images, 1); + assert_eq!(a.warnings.len(), 2); + assert_eq!(a.raw_blocks_emitted, 1); + } + + #[test] + fn messages_are_one_line() { + let w = Warning::ImageGeometryDegraded { + what: "VML shape".into(), + why: "no z-order in source".into(), + }; + assert!(!w.message().contains('\n')); + } +} diff --git a/crates/docsai-model/src/sheet.rs b/crates/docsai-model/src/sheet.rs new file mode 100644 index 0000000..4053ee5 --- /dev/null +++ b/crates/docsai-model/src/sheet.rs @@ -0,0 +1,340 @@ +//! Spreadsheet side of the IR. +//! +//! Populated from Fase 3 onwards; defined now because [`CellRef`] is part of +//! the shared image model and because the IR validator must know the +//! `Workbook`-only invariants from day one. + +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; + +use crate::image::ImageRef; +use crate::style::{StyleCatalog, StyleId}; +use crate::text::{DocumentMeta, RawFragment}; +use crate::units::Length; + +/// A zero-based `(column, row)` cell coordinate. +/// +/// Serialises as its A1 string (`"B2"`) rather than as a pair, so that it can +/// be a JSON map key — [`Sheet::cells`] is keyed by it — and so that +/// `inspect --json` stays readable. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct CellRef { + pub col: u32, + pub row: u32, +} + +impl Serialize for CellRef { + fn serialize(&self, s: S) -> Result { + s.serialize_str(&self.a1()) + } +} + +impl<'de> Deserialize<'de> for CellRef { + fn deserialize>(d: D) -> Result { + let text = String::deserialize(d)?; + CellRef::parse_a1(&text) + .ok_or_else(|| serde::de::Error::custom(format!("invalid A1 reference `{text}`"))) + } +} + +impl std::fmt::Display for CellRef { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.a1()) + } +} + +impl CellRef { + pub const fn new(col: u32, row: u32) -> Self { + CellRef { col, row } + } + + /// The A1-style reference (`B2`), as DocMark writes it. + /// + /// ``` + /// # use docsai_model::sheet::CellRef; + /// assert_eq!(CellRef::new(0, 0).a1(), "A1"); + /// assert_eq!(CellRef::new(26, 9).a1(), "AA10"); + /// ``` + pub fn a1(&self) -> String { + let mut name = String::new(); + let mut col = self.col as u64 + 1; + while col > 0 { + let rem = ((col - 1) % 26) as u8; + name.insert(0, (b'A' + rem) as char); + col = (col - 1) / 26; + } + name.push_str(&(self.row + 1).to_string()); + name + } + + /// Parses an A1-style reference. `$` anchors are accepted and ignored. + pub fn parse_a1(text: &str) -> Option { + let text = text.replace('$', ""); + let split = text.find(|c: char| c.is_ascii_digit())?; + let (letters, digits) = text.split_at(split); + if letters.is_empty() || !letters.bytes().all(|b| b.is_ascii_uppercase()) { + return None; + } + let mut col: u64 = 0; + for b in letters.bytes() { + col = col * 26 + (b - b'A' + 1) as u64; + } + let row: u32 = digits.parse().ok()?; + if row == 0 || col == 0 { + return None; + } + Some(CellRef::new((col - 1) as u32, row - 1)) + } +} + +/// An inclusive rectangular range of cells. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub struct CellRange { + pub start: CellRef, + pub end: CellRef, +} + +impl CellRange { + pub const fn new(start: CellRef, end: CellRef) -> Self { + CellRange { start, end } + } + + /// `A1:C3`, or just `A1` when the range is a single cell. + pub fn a1(&self) -> String { + if self.start == self.end { + self.start.a1() + } else { + format!("{}:{}", self.start.a1(), self.end.a1()) + } + } +} + +/// The value stored in a cell. +#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case", tag = "type", content = "value")] +pub enum CellValue { + #[default] + Empty, + Number(f64), + Text(String), + Bool(bool), + /// ISO-8601. Serial numbers are converted on read and back on write so a + /// hand edit of the DocMark cannot corrupt them. + DateTime(String), + /// Spreadsheet error literal (`#DIV/0!`…). + Error(String), +} + +impl CellValue { + pub fn is_empty(&self) -> bool { + matches!(self, CellValue::Empty) + } + + /// The DocMark `type=` keyword. + pub fn type_keyword(&self) -> &'static str { + match self { + CellValue::Empty => "empty", + CellValue::Number(_) => "number", + CellValue::Text(_) => "text", + CellValue::Bool(_) => "bool", + CellValue::DateTime(_) => "date", + CellValue::Error(_) => "error", + } + } +} + +/// Which formula language a formula is written in. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum FormulaDialect { + Ooxml, + OpenFormula, +} + +/// A cell formula, kept in its original dialect (risk R5). +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub struct Formula { + /// The expression *without* the leading `=`. + pub text: String, + pub dialect: FormulaDialect, + /// Range this formula is shared over, when it was a shared formula. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub shared_over: Option, + /// Range this formula spills into, when it was an array formula. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub array_over: Option, +} + +/// A number format code (`#,##0.00`), plus the source's numeric id when it had one. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub struct NumFmt { + pub code: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub id: Option, +} + +/// One cell. +#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub struct Cell { + pub value: CellValue, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub formula: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub num_fmt: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub style: Option, +} + +/// Column layout. +#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub struct ColProps { + /// Width in characters, the unit spreadsheets present to the user. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub width_chars: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub hidden: Option, +} + +/// Row layout. +#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub struct RowProps { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub height: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub hidden: Option, +} + +/// A frozen/split pane. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub struct Pane { + /// Top-left cell of the unfrozen area. + pub top_left: CellRef, + pub frozen: bool, +} + +/// A named range. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub struct DefinedName { + pub name: String, + /// The formula text of the reference (`Ventas!$D$10`), source dialect. + pub refers_to: String, + /// Sheet index when the name is sheet-scoped. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub sheet: Option, +} + +/// A worksheet: a sparse grid plus layout and anchored images. +#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case", default)] +pub struct Sheet { + pub name: String, + /// Sparse cell grid; absent keys are empty cells. + pub cells: BTreeMap, + pub merges: Vec, + pub cols: BTreeMap, + pub rows: BTreeMap, + #[serde(skip_serializing_if = "Option::is_none")] + pub pane: Option, + /// Images anchored to this sheet. + pub images: Vec, + /// Sheet-level fragments with no DocMark representation (charts…). + pub raw: Vec, + pub hidden: bool, +} + +impl Sheet { + pub fn new(name: impl Into) -> Self { + Sheet { + name: name.into(), + ..Default::default() + } + } + + /// The used range, or `None` for an empty sheet. + pub fn used_range(&self) -> Option { + let mut iter = self.cells.keys(); + let first = *iter.next()?; + let (mut min, mut max) = (first, first); + for &c in iter { + min.col = min.col.min(c.col); + min.row = min.row.min(c.row); + max.col = max.col.max(c.col); + max.row = max.row.max(c.row); + } + Some(CellRange::new(min, max)) + } +} + +/// A spreadsheet document. +#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case", default)] +pub struct Workbook { + pub meta: DocumentMeta, + pub styles: StyleCatalog, + pub defined_names: Vec, + pub sheets: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub active_sheet: Option, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a1_notation_round_trips() { + for (col, row, text) in [ + (0u32, 0u32, "A1"), + (1, 1, "B2"), + (25, 0, "Z1"), + (26, 0, "AA1"), + (51, 99, "AZ100"), + (702, 0, "AAA1"), + ] { + let r = CellRef::new(col, row); + assert_eq!(r.a1(), text); + assert_eq!(CellRef::parse_a1(text), Some(r)); + } + } + + #[test] + fn parse_a1_tolerates_absolute_refs_and_rejects_junk() { + assert_eq!(CellRef::parse_a1("$D$10"), Some(CellRef::new(3, 9))); + assert_eq!(CellRef::parse_a1("A0"), None); + assert_eq!(CellRef::parse_a1("1A"), None); + assert_eq!(CellRef::parse_a1(""), None); + assert_eq!( + CellRef::parse_a1("a1"), + None, + "lowercase is not A1 notation" + ); + } + + #[test] + fn ranges_collapse_when_single_cell() { + let one = CellRange::new(CellRef::new(0, 0), CellRef::new(0, 0)); + assert_eq!(one.a1(), "A1"); + let many = CellRange::new(CellRef::new(1, 1), CellRef::new(3, 2)); + assert_eq!(many.a1(), "B2:D3"); + } + + #[test] + fn used_range_covers_every_cell() { + let mut sheet = Sheet::new("Hoja"); + sheet.cells.insert(CellRef::new(3, 9), Cell::default()); + sheet.cells.insert(CellRef::new(1, 2), Cell::default()); + assert_eq!( + sheet.used_range(), + Some(CellRange::new(CellRef::new(1, 2), CellRef::new(3, 9))) + ); + assert_eq!(Sheet::new("vacia").used_range(), None); + } +} diff --git a/crates/docsai-model/src/style.rs b/crates/docsai-model/src/style.rs new file mode 100644 index 0000000..0000e24 --- /dev/null +++ b/crates/docsai-model/src/style.rs @@ -0,0 +1,507 @@ +//! Named styles with inheritance. +//! +//! The cardinal rule (análisis §5.2): the IR stores **a reference plus a +//! delta**, never flattened formatting. Every field of [`FontProps`] and +//! [`ParaProps`] is an `Option`; `None` means "inherit", not "off". + +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; + +use crate::units::Length; + +/// Identifier of a named style, as it appears in the source document. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[serde(transparent)] +pub struct StyleId(pub String); + +impl StyleId { + pub fn new(id: impl Into) -> Self { + StyleId(id.into()) + } + + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl std::fmt::Display for StyleId { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.0) + } +} + +/// What a style can be applied to. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum StyleType { + Paragraph, + Character, + Table, + Numbering, +} + +impl StyleType { + pub fn as_str(self) -> &'static str { + match self { + StyleType::Paragraph => "paragraph", + StyleType::Character => "character", + StyleType::Table => "table", + StyleType::Numbering => "numbering", + } + } +} + +/// Underline styles that DocMark can express; anything else degrades to +/// [`Underline::Single`] with a warning from the reader. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum Underline { + None, + Single, + Double, + Thick, + Dotted, + Dashed, + Wave, +} + +/// Sub/superscript. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum VertAlign { + Baseline, + Superscript, + Subscript, +} + +/// Character-level formatting. Every field is a delta over what is inherited. +#[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case", default)] +pub struct FontProps { + #[serde(skip_serializing_if = "Option::is_none")] + pub name: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub size: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub bold: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub italic: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub strike: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub underline: Option, + /// `#RRGGBB`. + #[serde(skip_serializing_if = "Option::is_none")] + pub color: Option, + /// Highlight colour name as the source names it (`yellow`, `cyan`…). + #[serde(skip_serializing_if = "Option::is_none")] + pub highlight: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub vert_align: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub small_caps: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub caps: Option, +} + +impl FontProps { + /// True when the delta carries no information. + pub fn is_empty(&self) -> bool { + *self == FontProps::default() + } + + /// Layers `self` on top of `base`: fields set in `self` win, the rest are + /// inherited. This is the merge step of the four-level OOXML cascade. + pub fn over(&self, base: &FontProps) -> FontProps { + macro_rules! pick { + ($($f:ident),+ $(,)?) => { + FontProps { $($f: self.$f.clone().or_else(|| base.$f.clone())),+ } + }; + } + pick!( + name, size, bold, italic, strike, underline, color, highlight, vert_align, small_caps, + caps + ) + } + + /// The subset of `self` that is not already implied by `base`. + /// + /// Used by the serialiser to honour the "economy rule" of the spec (§3.1): + /// formatting identical to the applied style is not emitted. + pub fn minus(&self, base: &FontProps) -> FontProps { + macro_rules! drop_same { + ($($f:ident),+ $(,)?) => { + FontProps { + $($f: match (&self.$f, &base.$f) { + (Some(a), Some(b)) if a == b => None, + (a, _) => a.clone(), + }),+ + } + }; + } + drop_same!( + name, size, bold, italic, strike, underline, color, highlight, vert_align, small_caps, + caps + ) + } +} + +/// Horizontal alignment of a paragraph. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum Align { + Left, + Center, + Right, + Justify, +} + +impl Align { + pub fn as_str(self) -> &'static str { + match self { + Align::Left => "left", + Align::Center => "center", + Align::Right => "right", + Align::Justify => "justify", + } + } +} + +/// Line spacing, kept in the source's own terms so the round-trip is exact. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum LineHeight { + /// A multiple of the single line height, in 1/1000 (240 → 1.0 in OOXML). + Multiple(i32), + /// An exact height. + Exact(Length), + /// A minimum height. + AtLeast(Length), +} + +/// Paragraph-level formatting. Every field is a delta over what is inherited. +#[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case", default)] +pub struct ParaProps { + #[serde(skip_serializing_if = "Option::is_none")] + pub align: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub indent_left: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub indent_right: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub indent_first_line: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub indent_hanging: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub space_before: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub space_after: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub line_height: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub keep_with_next: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub page_break_before: Option, + /// Shading colour, `#RRGGBB`. + #[serde(skip_serializing_if = "Option::is_none")] + pub background: Option, + /// Outline level, 0-based: the heading depth the paragraph belongs to. + #[serde(skip_serializing_if = "Option::is_none")] + pub outline_level: Option, +} + +impl ParaProps { + pub fn is_empty(&self) -> bool { + *self == ParaProps::default() + } + + /// Layers `self` on top of `base`; see [`FontProps::over`]. + pub fn over(&self, base: &ParaProps) -> ParaProps { + macro_rules! pick { + ($($f:ident),+ $(,)?) => { + ParaProps { $($f: self.$f.clone().or_else(|| base.$f.clone())),+ } + }; + } + pick!( + align, + indent_left, + indent_right, + indent_first_line, + indent_hanging, + space_before, + space_after, + line_height, + keep_with_next, + page_break_before, + background, + outline_level, + ) + } + + /// See [`FontProps::minus`]. + pub fn minus(&self, base: &ParaProps) -> ParaProps { + macro_rules! drop_same { + ($($f:ident),+ $(,)?) => { + ParaProps { + $($f: match (&self.$f, &base.$f) { + (Some(a), Some(b)) if a == b => None, + (a, _) => a.clone(), + }),+ + } + }; + } + drop_same!( + align, + indent_left, + indent_right, + indent_first_line, + indent_hanging, + space_before, + space_after, + line_height, + keep_with_next, + page_break_before, + background, + outline_level, + ) + } +} + +/// A named style as declared by the source document. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub struct Style { + pub id: StyleId, + /// Human-readable name (`w:name`), which may differ from the id. + pub name: String, + #[serde(rename = "type")] + pub style_type: StyleType, + #[serde(skip_serializing_if = "Option::is_none")] + pub based_on: Option, + /// Style applied to the next paragraph (`w:next`). + #[serde(skip_serializing_if = "Option::is_none")] + pub next: Option, + #[serde(skip_serializing_if = "FontProps::is_empty", default)] + pub font: FontProps, + #[serde(skip_serializing_if = "ParaProps::is_empty", default)] + pub paragraph: ParaProps, + /// True for the document's default style of its kind. + #[serde(skip_serializing_if = "std::ops::Not::not", default)] + pub is_default: bool, +} + +impl Style { + pub fn new(id: impl Into, style_type: StyleType) -> Self { + let id = StyleId::new(id); + Style { + name: id.0.clone(), + id, + style_type, + based_on: None, + next: None, + font: FontProps::default(), + paragraph: ParaProps::default(), + is_default: false, + } + } +} + +/// Document-wide defaults (`w:docDefaults`), the bottom of the cascade. +#[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case", default)] +pub struct DocDefaults { + #[serde(skip_serializing_if = "FontProps::is_empty")] + pub font: FontProps, + #[serde(skip_serializing_if = "ParaProps::is_empty")] + pub paragraph: ParaProps, +} + +/// Formatting with the whole cascade already applied. +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub struct ResolvedStyle { + pub font: FontProps, + pub paragraph: ParaProps, +} + +/// The document's style catalogue. +/// +/// Kept ordered by id so that serialisation is deterministic. +#[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case", default)] +pub struct StyleCatalog { + #[serde(skip_serializing_if = "DocDefaults::is_empty")] + pub defaults: DocDefaults, + pub styles: BTreeMap, +} + +impl DocDefaults { + pub fn is_empty(&self) -> bool { + self.font.is_empty() && self.paragraph.is_empty() + } +} + +/// Depth limit for `based-on` chains. Word itself refuses to nest this deep; +/// the limit also breaks cycles in malformed documents without recursion. +const MAX_INHERITANCE_DEPTH: usize = 32; + +impl StyleCatalog { + pub fn insert(&mut self, style: Style) { + self.styles.insert(style.id.clone(), style); + } + + pub fn get(&self, id: &StyleId) -> Option<&Style> { + self.styles.get(id) + } + + pub fn contains(&self, id: &StyleId) -> bool { + self.styles.contains_key(id) + } + + /// The default paragraph style, if the document declared one. + pub fn default_paragraph_style(&self) -> Option<&Style> { + self.styles + .values() + .find(|s| s.is_default && s.style_type == StyleType::Paragraph) + } + + /// Walks the `based_on` chain from the document defaults up to `id` and + /// merges every level, producing fully resolved formatting. + /// + /// Cycles are broken by a depth limit and by a visited set, so + /// a malformed `styles.xml` cannot hang or overflow the stack. + /// + /// ``` + /// # use docsai_model::style::*; + /// let mut cat = StyleCatalog::default(); + /// let mut base = Style::new("Normal", StyleType::Paragraph); + /// base.font.size = Some(docsai_model::units::Length::from_pt(11.0)); + /// let mut child = Style::new("Quote", StyleType::Paragraph); + /// child.based_on = Some(StyleId::new("Normal")); + /// child.font.italic = Some(true); + /// cat.insert(base); + /// cat.insert(child); + /// let r = cat.resolve(Some(&StyleId::new("Quote"))); + /// assert_eq!(r.font.italic, Some(true)); + /// assert_eq!(r.font.size, Some(docsai_model::units::Length::from_pt(11.0))); + /// ``` + pub fn resolve(&self, id: Option<&StyleId>) -> ResolvedStyle { + let mut chain: Vec<&Style> = Vec::new(); + let mut visited: Vec<&StyleId> = Vec::new(); + let mut cursor = id; + while let Some(current) = cursor { + if visited.contains(¤t) || chain.len() >= MAX_INHERITANCE_DEPTH { + break; + } + visited.push(current); + match self.styles.get(current) { + Some(style) => { + chain.push(style); + cursor = style.based_on.as_ref(); + } + None => break, + } + } + + let mut resolved = ResolvedStyle { + font: self.defaults.font.clone(), + paragraph: self.defaults.paragraph.clone(), + }; + // `chain` runs child → ancestor; apply ancestors first. + for style in chain.iter().rev() { + resolved.font = style.font.over(&resolved.font); + resolved.paragraph = style.paragraph.over(&resolved.paragraph); + } + resolved + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::units::Length; + + fn catalog() -> StyleCatalog { + let mut cat = StyleCatalog::default(); + cat.defaults.font.name = Some("Calibri".into()); + cat.defaults.font.size = Some(Length::from_pt(11.0)); + + let mut normal = Style::new("Normal", StyleType::Paragraph); + normal.is_default = true; + cat.insert(normal); + + let mut destacado = Style::new("Destacado", StyleType::Paragraph); + destacado.based_on = Some(StyleId::new("Normal")); + destacado.font.italic = Some(true); + destacado.font.color = Some("#C00000".into()); + destacado.paragraph.align = Some(Align::Center); + cat.insert(destacado); + + let mut fuerte = Style::new("DestacadoFuerte", StyleType::Paragraph); + fuerte.based_on = Some(StyleId::new("Destacado")); + fuerte.font.bold = Some(true); + cat.insert(fuerte); + cat + } + + #[test] + fn resolves_through_the_whole_chain() { + let r = catalog().resolve(Some(&StyleId::new("DestacadoFuerte"))); + assert_eq!(r.font.bold, Some(true)); // own + assert_eq!(r.font.italic, Some(true)); // parent + assert_eq!(r.font.name.as_deref(), Some("Calibri")); // docDefaults + assert_eq!(r.paragraph.align, Some(Align::Center)); + } + + #[test] + fn child_wins_over_ancestor() { + let mut cat = catalog(); + let mut fuerte = cat.get(&StyleId::new("DestacadoFuerte")).unwrap().clone(); + fuerte.font.color = Some("#000000".into()); + cat.insert(fuerte); + let r = cat.resolve(Some(&StyleId::new("DestacadoFuerte"))); + assert_eq!(r.font.color.as_deref(), Some("#000000")); + } + + #[test] + fn unknown_style_falls_back_to_defaults() { + let r = catalog().resolve(Some(&StyleId::new("NoExiste"))); + assert_eq!(r.font.name.as_deref(), Some("Calibri")); + assert_eq!(r.font.italic, None); + } + + #[test] + fn inheritance_cycle_terminates() { + let mut cat = StyleCatalog::default(); + let mut a = Style::new("A", StyleType::Paragraph); + a.based_on = Some(StyleId::new("B")); + a.font.bold = Some(true); + let mut b = Style::new("B", StyleType::Paragraph); + b.based_on = Some(StyleId::new("A")); + b.font.italic = Some(true); + cat.insert(a); + cat.insert(b); + let r = cat.resolve(Some(&StyleId::new("A"))); + assert_eq!(r.font.bold, Some(true)); + assert_eq!(r.font.italic, Some(true)); + } + + #[test] + fn minus_drops_redundant_formatting() { + let base = FontProps { + italic: Some(true), + color: Some("#C00000".into()), + ..Default::default() + }; + let direct = FontProps { + italic: Some(true), + bold: Some(true), + color: Some("#000000".into()), + ..Default::default() + }; + let delta = direct.minus(&base); + assert_eq!(delta.italic, None, "identical to the style, not emitted"); + assert_eq!(delta.bold, Some(true)); + assert_eq!(delta.color.as_deref(), Some("#000000"), "differs, kept"); + } +} diff --git a/crates/docsai-model/src/text.rs b/crates/docsai-model/src/text.rs new file mode 100644 index 0000000..909a086 --- /dev/null +++ b/crates/docsai-model/src/text.rs @@ -0,0 +1,657 @@ +//! Text-document side of the IR (`docx`, `odt`, `doc`). + +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; + +use crate::image::{ImageRef, RawId}; +use crate::list::{ListCatalog, ListId}; +use crate::style::{FontProps, ParaProps, StyleCatalog, StyleId}; +use crate::units::{Length, Size}; + +/// Document properties: `docProps/*` in OOXML, `meta.xml` in ODF. +#[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case", default)] +pub struct DocumentMeta { + #[serde(skip_serializing_if = "Option::is_none")] + pub title: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub author: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub last_modified_by: Option, + /// ISO-8601, verbatim from the source (no date library in the IR). + #[serde(skip_serializing_if = "Option::is_none")] + pub created: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub modified: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub language: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub subject: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub keywords: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub description: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub application: Option, + /// Custom properties, kept as-is and ordered for determinism. + #[serde(skip_serializing_if = "BTreeMap::is_empty")] + pub custom: BTreeMap, +} + +/// Page margins. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub struct Margins { + pub top: Length, + pub right: Length, + pub bottom: Length, + pub left: Length, + pub header: Length, + pub footer: Length, +} + +/// Page orientation. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum Orientation { + #[default] + Portrait, + Landscape, +} + +impl Orientation { + pub fn as_str(self) -> &'static str { + match self { + Orientation::Portrait => "portrait", + Orientation::Landscape => "landscape", + } + } +} + +/// Page geometry of a section. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub struct PageGeometry { + pub size: Size, + pub margins: Margins, + pub orientation: Orientation, + /// Number of text columns. + pub columns: u16, + /// True when the first page uses its own header/footer. + pub title_page: bool, +} + +impl PageGeometry { + /// The standard paper name matching this size, within 1 mm. + /// + /// ``` + /// # use docsai_model::text::PageGeometry; + /// # use docsai_model::units::{Length, Size}; + /// let mut g = PageGeometry::default(); + /// g.size = Size::new(Length::from_twips(11906), Length::from_twips(16838)); + /// assert_eq!(g.paper_name(), Some("A4")); + /// ``` + pub fn paper_name(&self) -> Option<&'static str> { + const TOLERANCE_MM: f64 = 1.0; + const PAPERS: &[(&str, f64, f64)] = &[ + ("A3", 297.0, 420.0), + ("A4", 210.0, 297.0), + ("A5", 148.0, 210.0), + ("Letter", 215.9, 279.4), + ("Legal", 215.9, 355.6), + ("Tabloid", 279.4, 431.8), + ]; + let (w, h) = (self.size.width.mm(), self.size.height.mm()); + let (short, long) = if w <= h { (w, h) } else { (h, w) }; + PAPERS + .iter() + .find(|(_, pw, ph)| { + (short - pw).abs() <= TOLERANCE_MM && (long - ph).abs() <= TOLERANCE_MM + }) + .map(|(name, _, _)| *name) + } +} + +/// Which pages a header or footer applies to. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum HeaderScope { + Default, + First, + Even, +} + +impl HeaderScope { + pub fn as_str(self) -> &'static str { + match self { + HeaderScope::Default => "default", + HeaderScope::First => "first", + HeaderScope::Even => "even", + } + } +} + +/// A header or footer. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub struct HeaderFooter { + pub scope: HeaderScope, + pub blocks: Vec, +} + +/// A document section: page geometry, its headers/footers and its content. +#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case", default)] +pub struct Section { + pub page: PageGeometry, + pub headers: Vec, + pub footers: Vec, + pub blocks: Vec, +} + +/// A text document. +#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case", default)] +pub struct TextDocument { + pub meta: DocumentMeta, + pub styles: StyleCatalog, + pub list_defs: ListCatalog, + pub sections: Vec

, +} + +impl TextDocument { + /// Every block of every section, in document order. + pub fn blocks(&self) -> impl Iterator { + self.sections.iter().flat_map(|s| s.blocks.iter()) + } +} + +/// A block-level element. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +// Adjacently tagged: `Inline::Text(String)` cannot be internally tagged, and +// both enums stay consistent in the JSON that `inspect --json` emits. +#[serde(rename_all = "kebab-case", tag = "block", content = "data")] +pub enum Block { + Paragraph(Paragraph), + Heading(Heading), + List(List), + Table(Table), + /// A picture that stands on its own line. + Image(ImageRef), + TextBox(TextBox), + /// Fidelity hatch: source bytes with no DocMark representation (spec §7). + Raw(RawFragment), +} + +/// Paragraph formatting: a style reference plus direct deltas. +#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case", default)] +pub struct ParaFormat { + #[serde(skip_serializing_if = "Option::is_none")] + pub style: Option, + /// Direct paragraph formatting, over and above the style. + #[serde(skip_serializing_if = "ParaProps::is_empty")] + pub direct: ParaProps, + /// Direct run formatting applied to the whole paragraph mark. + #[serde(skip_serializing_if = "FontProps::is_empty")] + pub run_direct: FontProps, +} + +impl ParaFormat { + pub fn is_empty(&self) -> bool { + self.style.is_none() && self.direct.is_empty() && self.run_direct.is_empty() + } + + pub fn styled(style: impl Into) -> Self { + ParaFormat { + style: Some(StyleId::new(style)), + ..Default::default() + } + } +} + +/// A paragraph. +#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case", default)] +pub struct Paragraph { + pub format: ParaFormat, + pub content: Vec, +} + +impl Paragraph { + pub fn new(content: Vec) -> Self { + Paragraph { + format: ParaFormat::default(), + content, + } + } + + /// A paragraph holding a single text run. + pub fn text(text: impl Into) -> Self { + Paragraph::new(vec![Inline::Text(text.into())]) + } + + /// True when the paragraph carries no content at all. + pub fn is_empty(&self) -> bool { + self.content.iter().all(|i| i.is_empty()) + } + + /// Concatenation of every text fragment, ignoring formatting. + pub fn plain_text(&self) -> String { + let mut out = String::new(); + for inline in &self.content { + inline.push_plain_text(&mut out); + } + out + } +} + +/// A heading: an outline level plus a paragraph. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub struct Heading { + /// 1-based heading depth, as `#` count in Markdown. + pub level: u8, + pub paragraph: Paragraph, +} + +/// A rebuilt list tree. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub struct List { + /// The definition in [`crate::list::ListCatalog`], when the + /// list is not the document's default one. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub def: Option, + pub ordered: bool, + /// Nesting depth, 0 for a top-level list. + #[serde(default, skip_serializing_if = "is_zero_u8")] + pub level: u8, + pub items: Vec, +} + +fn is_zero_u8(v: &u8) -> bool { + *v == 0 +} + +/// One list item; nested lists appear as [`Block::List`] inside `blocks`. +#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub struct ListItem { + pub blocks: Vec, +} + +/// A table. +#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case", default)] +pub struct Table { + #[serde(skip_serializing_if = "Option::is_none")] + pub style: Option, + /// Width of each grid column. + pub col_widths: Vec, + pub rows: Vec, + /// True when the first row is a header row. + pub header_row: bool, +} + +impl Table { + /// Number of grid columns, taking spans into account. + pub fn width(&self) -> usize { + self.rows + .iter() + .map(|r| r.cells.iter().map(|c| c.colspan.max(1) as usize).sum()) + .max() + .unwrap_or(0) + } + + /// True when the table needs the `complex=true` container of spec §3.4: + /// a cell holding anything other than a single paragraph cannot live in a + /// GFM table row. + pub fn is_complex(&self) -> bool { + self.rows.iter().flat_map(|r| r.cells.iter()).any(|c| { + c.blocks.len() > 1 + || c.blocks + .iter() + .any(|b| !matches!(b, Block::Paragraph(_) | Block::Image(_))) + }) + } +} + +/// A table row. +#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case", default)] +pub struct TableRow { + pub cells: Vec, + /// True for rows repeated as a header on each page (`w:tblHeader`). + pub is_header: bool, +} + +/// A table cell. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case", default)] +pub struct TableCell { + pub blocks: Vec, + /// Horizontal span; 1 for a normal cell. + pub colspan: u16, + /// Vertical span; 1 for a normal cell. + pub rowspan: u16, + /// True for a cell absorbed by a span above or to its left. + pub covered: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub width: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub background: Option, +} + +impl Default for TableCell { + fn default() -> Self { + TableCell { + blocks: Vec::new(), + colspan: 1, + rowspan: 1, + covered: false, + width: None, + background: None, + } + } +} + +impl TableCell { + pub fn text(text: impl Into) -> Self { + TableCell { + blocks: vec![Block::Paragraph(Paragraph::text(text))], + ..Default::default() + } + } +} + +/// A text box or frame. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub struct TextBox { + pub blocks: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub size: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub x: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub y: Option, +} + +/// A verbatim fragment of the source, preserved for the reverse conversion. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub struct RawFragment { + pub id: RawId, + /// Source dialect (`ooxml`, `odf`). A writer for another dialect drops the + /// fragment with a warning rather than emitting nonsense. + pub format: String, + /// Package part the fragment came from (`word/document.xml`). + pub part: String, + /// The original markup. + pub content: String, +} + +/// Character formatting of a run: style reference plus direct deltas. +#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case", default)] +pub struct RunProps { + #[serde(skip_serializing_if = "Option::is_none")] + pub style: Option, + #[serde(skip_serializing_if = "FontProps::is_empty")] + pub direct: FontProps, +} + +impl RunProps { + pub fn is_empty(&self) -> bool { + self.style.is_none() && self.direct.is_empty() + } + + pub fn direct(direct: FontProps) -> Self { + RunProps { + style: None, + direct, + } + } +} + +/// A dynamic field. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum FieldKind { + Page, + NumPages, + Date, + Time, + Toc, + Ref, + /// Any other field; the name is the first token of the instruction. + Other(String), +} + +impl FieldKind { + /// Classifies a field instruction (`" PAGE "`, `" TOC \\o \"1-3\" "`). + pub fn from_instruction(instr: &str) -> FieldKind { + let name = instr.split_whitespace().next().unwrap_or(""); + match name.to_ascii_uppercase().as_str() { + "PAGE" => FieldKind::Page, + "NUMPAGES" => FieldKind::NumPages, + "DATE" => FieldKind::Date, + "TIME" => FieldKind::Time, + "TOC" => FieldKind::Toc, + "REF" => FieldKind::Ref, + "" => FieldKind::Other(String::new()), + other => FieldKind::Other(other.to_string()), + } + } + + pub fn as_str(&self) -> &str { + match self { + FieldKind::Page => "PAGE", + FieldKind::NumPages => "NUMPAGES", + FieldKind::Date => "DATE", + FieldKind::Time => "TIME", + FieldKind::Toc => "TOC", + FieldKind::Ref => "REF", + FieldKind::Other(s) => s, + } + } +} + +/// A manual break. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum BreakKind { + Line, + Page, + Column, +} + +impl BreakKind { + pub fn as_str(self) -> &'static str { + match self { + BreakKind::Line => "line", + BreakKind::Page => "page", + BreakKind::Column => "column", + } + } +} + +/// An inline element. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case", tag = "inline", content = "data")] +pub enum Inline { + Text(String), + /// Text with formatting applied. + Styled { + content: Vec, + props: RunProps, + }, + Link { + target: String, + content: Vec, + #[serde(default, skip_serializing_if = "RunProps::is_empty")] + props: RunProps, + }, + /// A footnote; its content is block-level. + Footnote(Vec), + Field { + kind: FieldKind, + /// Last known displayed value. + cached: String, + /// The full instruction, needed to regenerate the field. + #[serde(default, skip_serializing_if = "String::is_empty")] + instruction: String, + }, + Break(BreakKind), + /// An image flowing with the text. + Image(ImageRef), + /// Inline-level fidelity hatch. + Raw(RawFragment), +} + +impl Inline { + /// True when the element contributes nothing visible. + pub fn is_empty(&self) -> bool { + match self { + Inline::Text(t) => t.is_empty(), + Inline::Styled { content, .. } => content.iter().all(|i| i.is_empty()), + _ => false, + } + } + + fn push_plain_text(&self, out: &mut String) { + match self { + Inline::Text(t) => out.push_str(t), + Inline::Styled { content, .. } | Inline::Link { content, .. } => { + for i in content { + i.push_plain_text(out); + } + } + Inline::Field { cached, .. } => out.push_str(cached), + Inline::Break(BreakKind::Line) => out.push('\n'), + _ => {} + } + } + + /// Wraps `content` in a [`Inline::Styled`], or returns it unchanged when + /// there is no formatting to apply. + pub fn styled(content: Vec, props: RunProps) -> Vec { + if props.is_empty() { + content + } else { + vec![Inline::Styled { content, props }] + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::units::Size; + + #[test] + fn recognises_standard_paper_sizes() { + let mut g = PageGeometry { + size: Size::new(Length::from_twips(11906), Length::from_twips(16838)), + ..Default::default() + }; + assert_eq!(g.paper_name(), Some("A4")); + g.size = Size::new(Length::from_twips(16838), Length::from_twips(11906)); + assert_eq!(g.paper_name(), Some("A4"), "landscape is the same paper"); + g.size = Size::new(Length::from_cm(10.0), Length::from_cm(10.0)); + assert_eq!(g.paper_name(), None); + } + + #[test] + fn field_instructions_are_classified() { + assert_eq!(FieldKind::from_instruction(" PAGE "), FieldKind::Page); + assert_eq!( + FieldKind::from_instruction(r#" TOC \o "1-3" \h "#), + FieldKind::Toc + ); + assert_eq!( + FieldKind::from_instruction(" HYPERLINK x "), + FieldKind::Other("HYPERLINK".into()) + ); + } + + #[test] + fn plain_text_walks_nested_inlines() { + let p = Paragraph::new(vec![ + Inline::Text("Hola ".into()), + Inline::Styled { + content: vec![Inline::Text("mundo".into())], + props: RunProps::default(), + }, + Inline::Link { + target: "https://example.com".into(), + content: vec![Inline::Text(" enlace".into())], + props: RunProps::default(), + }, + Inline::Field { + kind: FieldKind::Page, + cached: " 1".into(), + instruction: " PAGE ".into(), + }, + ]); + assert_eq!(p.plain_text(), "Hola mundo enlace 1"); + } + + #[test] + fn table_width_accounts_for_colspans() { + let table = Table { + rows: vec![ + TableRow { + cells: vec![ + TableCell::text("a"), + TableCell { + colspan: 2, + ..TableCell::text("b") + }, + ], + is_header: true, + }, + TableRow { + cells: vec![ + TableCell::text("c"), + TableCell::text("d"), + TableCell::text("e"), + ], + is_header: false, + }, + ], + ..Default::default() + }; + assert_eq!(table.width(), 3); + assert!(!table.is_complex()); + } + + #[test] + fn multi_block_cells_make_a_table_complex() { + let table = Table { + rows: vec![TableRow { + cells: vec![TableCell { + blocks: vec![ + Block::Paragraph(Paragraph::text("uno")), + Block::Paragraph(Paragraph::text("dos")), + ], + ..Default::default() + }], + is_header: false, + }], + ..Default::default() + }; + assert!(table.is_complex()); + } + + #[test] + fn styled_is_skipped_when_there_is_nothing_to_apply() { + let content = vec![Inline::Text("x".into())]; + assert_eq!( + Inline::styled(content.clone(), RunProps::default()), + content, + "no empty Styled wrappers in the IR" + ); + } +} diff --git a/crates/docsai-model/src/units.rs b/crates/docsai-model/src/units.rs new file mode 100644 index 0000000..98e92af --- /dev/null +++ b/crates/docsai-model/src/units.rs @@ -0,0 +1,333 @@ +//! Normalised lengths. +//! +//! Every length in the IR is stored in EMU (English Metric Units), the unit +//! OOXML uses for drawings: 914 400 per inch, which divides exactly into +//! points, centimetres, millimetres and twips. Conversion to the human units +//! DocMark exposes (`pt`, `cm`, `px`) happens only at serialisation time. +//! +//! ``` +//! use docsai_model::units::Length; +//! assert_eq!(Length::from_cm(2.54).emu(), Length::from_inch(1.0).emu()); +//! assert_eq!(Length::from_pt(72.0), Length::from_inch(1.0)); +//! ``` + +use serde::{Deserialize, Serialize}; +use std::fmt; + +/// EMU in one inch. The definition all other constants derive from. +pub const EMU_PER_INCH: i64 = 914_400; +/// EMU in one typographic point (1/72 in). +pub const EMU_PER_POINT: i64 = EMU_PER_INCH / 72; +/// EMU in one centimetre. +pub const EMU_PER_CM: i64 = 360_000; +/// EMU in one millimetre. +pub const EMU_PER_MM: i64 = EMU_PER_CM / 10; +/// EMU in one twip (1/20 pt), the unit WordprocessingML uses for layout. +pub const EMU_PER_TWIP: i64 = EMU_PER_POINT / 20; +/// Resolution assumed when a bitmap declares none. +pub const DEFAULT_DPI: u32 = 96; + +/// A length, stored in EMU. +/// +/// `Length` is deliberately a newtype over `i64`: negative values are legal +/// (indents and drawing offsets can be negative) and integer EMU keep the +/// round-trip exact for every source format. +#[derive( + Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Default, Serialize, Deserialize, +)] +#[serde(transparent)] +pub struct Length(i64); + +impl Length { + /// The zero length. + pub const ZERO: Length = Length(0); + + /// Builds a length from raw EMU. + pub const fn from_emu(emu: i64) -> Self { + Length(emu) + } + + /// The length in EMU. + pub const fn emu(self) -> i64 { + self.0 + } + + /// Builds a length from twips (1/1440 in), the WordprocessingML layout unit. + pub const fn from_twips(twips: i64) -> Self { + Length(twips * EMU_PER_TWIP) + } + + /// The length in twips, rounded to nearest. + pub fn twips(self) -> i64 { + div_round(self.0, EMU_PER_TWIP) + } + + /// Builds a length from half-points, the unit of `w:sz` (font sizes). + pub const fn from_half_points(half_points: i64) -> Self { + Length(half_points * (EMU_PER_POINT / 2)) + } + + /// The length in half-points, rounded to nearest. + pub fn half_points(self) -> i64 { + div_round(self.0, EMU_PER_POINT / 2) + } + + /// Builds a length from eighth-points, the unit of border widths (`w:sz`). + /// + /// An eighth of a point is 1587.5 EMU, so this is the one Word unit that + /// does not divide EMU exactly; the result is rounded to nearest. + pub const fn from_eighth_points(eighth_points: i64) -> Self { + Length(div_round(eighth_points * EMU_PER_POINT, 8)) + } + + /// Builds a length from points. + pub fn from_pt(pt: f64) -> Self { + Length((pt * EMU_PER_POINT as f64).round() as i64) + } + + /// The length in points. + pub fn pt(self) -> f64 { + self.0 as f64 / EMU_PER_POINT as f64 + } + + /// Builds a length from centimetres. + pub fn from_cm(cm: f64) -> Self { + Length((cm * EMU_PER_CM as f64).round() as i64) + } + + /// The length in centimetres. + pub fn cm(self) -> f64 { + self.0 as f64 / EMU_PER_CM as f64 + } + + /// Builds a length from millimetres. + pub fn from_mm(mm: f64) -> Self { + Length((mm * EMU_PER_MM as f64).round() as i64) + } + + /// The length in millimetres. + pub fn mm(self) -> f64 { + self.0 as f64 / EMU_PER_MM as f64 + } + + /// Builds a length from inches. + pub fn from_inch(inch: f64) -> Self { + Length((inch * EMU_PER_INCH as f64).round() as i64) + } + + /// The length in inches. + pub fn inch(self) -> f64 { + self.0 as f64 / EMU_PER_INCH as f64 + } + + /// Builds a length from pixels at the given resolution. + pub fn from_px_at(px: f64, dpi: u32) -> Self { + Length((px * EMU_PER_INCH as f64 / dpi.max(1) as f64).round() as i64) + } + + /// Builds a length from pixels at 96 dpi, the CSS reference resolution. + pub fn from_px(px: f64) -> Self { + Self::from_px_at(px, DEFAULT_DPI) + } + + /// The length in pixels at the given resolution. + pub fn px_at(self, dpi: u32) -> f64 { + self.0 as f64 * dpi.max(1) as f64 / EMU_PER_INCH as f64 + } + + /// The length in pixels at 96 dpi. + pub fn px(self) -> f64 { + self.px_at(DEFAULT_DPI) + } + + /// True when the length is exactly zero. + pub const fn is_zero(self) -> bool { + self.0 == 0 + } +} + +const fn div_round(value: i64, divisor: i64) -> i64 { + let half = divisor / 2; + if value >= 0 { + (value + half) / divisor + } else { + (value - half) / divisor + } +} + +/// EMU in one pixel at 96 dpi. +const EMU_PER_PX: i64 = EMU_PER_INCH / DEFAULT_DPI as i64; +/// EMU in a hundredth of a centimetre. +const EMU_PER_CENTI_CM: i64 = EMU_PER_CM / 100; +/// EMU in a hundredth of a point. +const EMU_PER_CENTI_PT: i64 = EMU_PER_POINT / 100; + +impl fmt::Display for Length { + /// Renders the length the way DocMark does, always **losslessly**: the + /// first unit that represents it exactly wins, in the order `px`, `cm`, + /// `pt`, and raw `emu` as the last resort. + /// + /// Choosing readability over exactness here would silently move margins + /// and image sizes on every round-trip, so exactness comes first: a Word + /// margin of 1417 twips prints as `70.85pt`, not as an approximate + /// `2.499cm`. + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + if self.0 % EMU_PER_PX == 0 { + write!(f, "{}px", self.0 / EMU_PER_PX) + } else if self.0 % EMU_PER_CENTI_CM == 0 { + write!(f, "{}cm", trim_float(self.cm(), 2)) + } else if self.0 % EMU_PER_CENTI_PT == 0 { + write!(f, "{}pt", trim_float(self.pt(), 2)) + } else { + write!(f, "{}emu", self.0) + } + } +} + +/// Formats a float with at most `decimals` digits and no trailing zeros. +/// +/// Used by every unit rendered into DocMark, so that the serialiser stays +/// deterministic (`2.5cm`, never `2.50cm` or `2.4999999cm`). +pub fn trim_float(value: f64, decimals: usize) -> String { + let mut s = format!("{value:.decimals$}"); + if s.contains('.') { + while s.ends_with('0') { + s.pop(); + } + if s.ends_with('.') { + s.pop(); + } + } + if s == "-0" { + s = "0".into(); + } + s +} + +/// A rectangular size. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] +pub struct Size { + pub width: Length, + pub height: Length, +} + +impl Size { + pub const fn new(width: Length, height: Length) -> Self { + Size { width, height } + } +} + +/// A point in a two-dimensional coordinate space. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] +pub struct Point { + pub x: Length, + pub y: Length, +} + +impl Point { + pub const fn new(x: Length, y: Length) -> Self { + Point { x, y } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn inch_is_the_anchor_of_every_unit() { + let inch = Length::from_inch(1.0); + assert_eq!(inch.emu(), 914_400); + assert_eq!(inch, Length::from_pt(72.0)); + assert_eq!(inch, Length::from_cm(2.54)); + assert_eq!(inch, Length::from_mm(25.4)); + assert_eq!(inch, Length::from_twips(1440)); + assert_eq!(inch, Length::from_px(96.0)); + } + + #[test] + fn round_trips_through_every_unit() { + // A length representable in every unit: 1944 twips = 97.2 pt = 129.6 px. + let l = Length::from_emu(1_234_440); + assert_eq!(Length::from_pt(l.pt()), l); + assert_eq!(Length::from_cm(l.cm()), l); + assert_eq!(Length::from_twips(l.twips()), l); + assert_eq!(Length::from_px(l.px()), l); + } + + #[test] + fn coarser_units_round_rather_than_truncate() { + // 1 234 560 EMU is 1944.19 twips: the nearest twip, not the floor. + assert_eq!(Length::from_emu(1_234_560).twips(), 1944); + assert_eq!(Length::from_emu(1_234_880).twips(), 1945); + } + + #[test] + fn px_conversion_honours_dpi() { + let l = Length::from_px_at(300.0, 300); + assert_eq!(l, Length::from_inch(1.0)); + assert_eq!(l.px_at(300), 300.0); + assert_eq!(l.px(), 96.0); + } + + #[test] + fn word_units_map_exactly() { + // w:sz="22" is 11 pt; w:ind w:left="720" is half an inch. + assert_eq!(Length::from_half_points(22), Length::from_pt(11.0)); + assert_eq!(Length::from_twips(720), Length::from_inch(0.5)); + assert_eq!(Length::from_eighth_points(8), Length::from_pt(1.0)); + } + + #[test] + fn negative_lengths_round_symmetrically() { + assert_eq!(Length::from_twips(-720).twips(), -720); + assert_eq!(Length::from_pt(-11.0).pt(), -11.0); + } + + #[test] + fn display_prefers_whole_pixels() { + assert_eq!(Length::from_px(96.0).to_string(), "96px"); + assert_eq!(Length::from_cm(3.5).to_string(), "3.5cm"); + } + + #[test] + fn display_is_always_lossless() { + // Parsing back what Display wrote must give the very same EMU, or a + // round-trip would move margins and image sizes. + fn reparse(text: &str) -> Length { + let split = text.find(|c: char| c.is_alphabetic()).unwrap(); + let (number, unit) = text.split_at(split); + let value: f64 = number.parse().unwrap(); + match unit { + "px" => Length::from_px(value), + "cm" => Length::from_cm(value), + "pt" => Length::from_pt(value), + "emu" => Length::from_emu(value as i64), + other => panic!("unexpected unit {other}"), + } + } + let samples = [ + Length::from_twips(1417), // a Word margin: not a whole px or cm + Length::from_twips(2500), // a table column width + Length::from_cm(3.0), + Length::from_px(120.0), + Length::from_pt(11.0), + Length::from_emu(1), // nothing but EMU can express this + Length::from_emu(-899_795), + ]; + for length in samples { + let text = length.to_string(); + assert_eq!(reparse(&text), length, "`{text}` did not round-trip"); + } + assert_eq!(Length::from_twips(1417).to_string(), "70.85pt"); + assert_eq!(Length::from_emu(1).to_string(), "1emu"); + } + + #[test] + fn trim_float_is_deterministic() { + assert_eq!(trim_float(2.500, 3), "2.5"); + assert_eq!(trim_float(2.0, 3), "2"); + assert_eq!(trim_float(-0.0001, 3), "0"); + assert_eq!(trim_float(1.0 / 3.0, 3), "0.333"); + } +} diff --git a/crates/docsai-model/src/validate.rs b/crates/docsai-model/src/validate.rs new file mode 100644 index 0000000..3f26766 --- /dev/null +++ b/crates/docsai-model/src/validate.rs @@ -0,0 +1,280 @@ +//! IR invariant checking. +//! +//! The invariants a reader can violate but the writers rely on. Chief among +//! them (arquitectura §3.1): sheet anchors are only legal inside a +//! [`crate::sheet::Workbook`], and document anchors only inside a +//! [`crate::text::TextDocument`]. Cross-anchoring does not +//! exist, and the validator is what makes that a checked rule instead of a +//! comment. + +use serde::{Deserialize, Serialize}; + +use crate::image::{Anchor, ImageRef}; +use crate::text::{Block, Inline, TextDocument}; +use crate::Document; + +/// A violated invariant. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case", tag = "error")] +pub enum ValidationError { + /// A spreadsheet anchor was found in a text document. + SheetAnchorInTextDocument { location: String, anchor: String }, + /// A document anchor was found on a sheet image. + DocumentAnchorInSheet { sheet: String, anchor: String }, + /// A table row has more cells than the table's grid. + RowWiderThanGrid { + table: String, + row: usize, + width: usize, + grid: usize, + }, + /// A `two-cell` anchor whose end is before its start. + InvertedCellAnchor { location: String, range: String }, +} + +impl std::fmt::Display for ValidationError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + ValidationError::SheetAnchorInTextDocument { location, anchor } => write!( + f, + "sheet anchor `{anchor}` on an image of a text document, at {location}" + ), + ValidationError::DocumentAnchorInSheet { sheet, anchor } => write!( + f, + "document anchor `{anchor}` on an image of sheet `{sheet}`" + ), + ValidationError::RowWiderThanGrid { + table, + row, + width, + grid, + } => write!( + f, + "table {table} row {row} spans {width} columns but the grid has {grid}" + ), + ValidationError::InvertedCellAnchor { location, range } => { + write!( + f, + "two-cell anchor {range} ends before it starts, at {location}" + ) + } + } + } +} + +impl std::error::Error for ValidationError {} + +/// Checks every invariant, returning all violations rather than the first. +pub fn validate(doc: &Document) -> Result<(), Vec> { + let mut errors = Vec::new(); + match doc { + Document::Text(text) => validate_text(text, &mut errors), + Document::Workbook(book) => { + for sheet in &book.sheets { + for image in &sheet.images { + if !image.geometry.anchor.is_sheet() { + errors.push(ValidationError::DocumentAnchorInSheet { + sheet: sheet.name.clone(), + anchor: image.geometry.anchor.keyword().into(), + }); + } + check_two_cell(image, &format!("sheet {}", sheet.name), &mut errors); + } + } + } + } + if errors.is_empty() { + Ok(()) + } else { + Err(errors) + } +} + +fn validate_text(doc: &TextDocument, errors: &mut Vec) { + for (si, section) in doc.sections.iter().enumerate() { + let path = format!("section {si}"); + walk_blocks(§ion.blocks, &path, errors); + for h in §ion.headers { + walk_blocks( + &h.blocks, + &format!("{path} header {}", h.scope.as_str()), + errors, + ); + } + for f in §ion.footers { + walk_blocks( + &f.blocks, + &format!("{path} footer {}", f.scope.as_str()), + errors, + ); + } + } +} + +fn walk_blocks(blocks: &[Block], path: &str, errors: &mut Vec) { + for (bi, block) in blocks.iter().enumerate() { + let path = format!("{path}/block {bi}"); + match block { + Block::Paragraph(p) => walk_inlines(&p.content, &path, errors), + Block::Heading(h) => walk_inlines(&h.paragraph.content, &path, errors), + Block::Image(image) => check_image(image, &path, errors), + Block::List(list) => { + for (ii, item) in list.items.iter().enumerate() { + walk_blocks(&item.blocks, &format!("{path}/item {ii}"), errors); + } + } + Block::Table(table) => { + let grid = table.width(); + for (ri, row) in table.rows.iter().enumerate() { + let width: usize = row.cells.iter().map(|c| c.colspan.max(1) as usize).sum(); + if width > grid { + errors.push(ValidationError::RowWiderThanGrid { + table: path.clone(), + row: ri, + width, + grid, + }); + } + for (ci, cell) in row.cells.iter().enumerate() { + walk_blocks(&cell.blocks, &format!("{path}/r{ri}c{ci}"), errors); + } + } + } + Block::TextBox(tb) => walk_blocks(&tb.blocks, &path, errors), + Block::Raw(_) => {} + } + } +} + +fn walk_inlines(inlines: &[Inline], path: &str, errors: &mut Vec) { + for inline in inlines { + match inline { + Inline::Image(image) => check_image(image, path, errors), + Inline::Styled { content, .. } | Inline::Link { content, .. } => { + walk_inlines(content, path, errors) + } + Inline::Footnote(blocks) => walk_blocks(blocks, &format!("{path}/footnote"), errors), + _ => {} + } + } +} + +fn check_image(image: &ImageRef, path: &str, errors: &mut Vec) { + if image.geometry.anchor.is_sheet() { + errors.push(ValidationError::SheetAnchorInTextDocument { + location: path.to_string(), + anchor: image.geometry.anchor.keyword().into(), + }); + } +} + +fn check_two_cell(image: &ImageRef, path: &str, errors: &mut Vec) { + if let Anchor::SheetTwoCell { from, to, .. } = &image.geometry.anchor { + if to.cell.col < from.cell.col || to.cell.row < from.cell.row { + errors.push(ValidationError::InvertedCellAnchor { + location: path.to_string(), + range: format!("{}:{}", from.cell.a1(), to.cell.a1()), + }); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::assets::AssetId; + use crate::image::{CellAnchor, ImageGeometry}; + use crate::sheet::{CellRef, Sheet, Workbook}; + use crate::text::{Paragraph, Section}; + use crate::units::{Length, Size}; + + fn image_with(anchor: Anchor) -> ImageRef { + let mut geometry = ImageGeometry::inline(Size::new(Length::ZERO, Length::ZERO)); + geometry.anchor = anchor; + ImageRef::new(AssetId::new("a"), geometry) + } + + #[test] + fn a_plain_text_document_validates() { + let doc = Document::Text(TextDocument { + sections: vec![Section { + blocks: vec![Block::Paragraph(Paragraph::text("hola"))], + ..Default::default() + }], + ..Default::default() + }); + assert!(validate(&doc).is_ok()); + } + + #[test] + fn sheet_anchors_are_rejected_in_text_documents() { + let anchor = Anchor::SheetOneCell { + from: CellAnchor::new(CellRef::new(0, 0), Length::ZERO, Length::ZERO), + }; + let doc = Document::Text(TextDocument { + sections: vec![Section { + blocks: vec![Block::Image(image_with(anchor))], + ..Default::default() + }], + ..Default::default() + }); + let errors = validate(&doc).unwrap_err(); + assert!(matches!( + errors[0], + ValidationError::SheetAnchorInTextDocument { .. } + )); + } + + #[test] + fn nested_images_are_reached_too() { + let anchor = Anchor::SheetAbsolute { + pos: Default::default(), + }; + let doc = Document::Text(TextDocument { + sections: vec![Section { + blocks: vec![Block::Paragraph(Paragraph::new(vec![Inline::Footnote( + vec![Block::Image(image_with(anchor))], + )]))], + ..Default::default() + }], + ..Default::default() + }); + assert_eq!(validate(&doc).unwrap_err().len(), 1); + } + + #[test] + fn document_anchors_are_rejected_on_sheets() { + let mut sheet = Sheet::new("Hoja"); + sheet.images.push(image_with(Anchor::Inline)); + let doc = Document::Workbook(Workbook { + sheets: vec![sheet], + ..Default::default() + }); + let errors = validate(&doc).unwrap_err(); + assert!(matches!( + errors[0], + ValidationError::DocumentAnchorInSheet { .. } + )); + } + + #[test] + fn inverted_two_cell_anchors_are_caught() { + let anchor = Anchor::SheetTwoCell { + from: CellAnchor::new(CellRef::new(3, 8), Length::ZERO, Length::ZERO), + to: CellAnchor::new(CellRef::new(1, 2), Length::ZERO, Length::ZERO), + move_with_cells: true, + size_with_cells: false, + }; + let mut sheet = Sheet::new("Hoja"); + sheet.images.push(image_with(anchor)); + let doc = Document::Workbook(Workbook { + sheets: vec![sheet], + ..Default::default() + }); + let errors = validate(&doc).unwrap_err(); + assert!(matches!( + errors[0], + ValidationError::InvertedCellAnchor { .. } + )); + } +} diff --git a/crates/docsai-model/tests/json_roundtrip.proptest-regressions b/crates/docsai-model/tests/json_roundtrip.proptest-regressions new file mode 100644 index 0000000..711d8bd --- /dev/null +++ b/crates/docsai-model/tests/json_roundtrip.proptest-regressions @@ -0,0 +1,9 @@ +# Seeds for failure cases proptest has generated in the past. It is +# automatically read and these particular cases re-run before any +# novel cases are generated. +# +# It is recommended to check this file in to source control so that +# everyone who runs the test benefits from these saved cases. +cc d39b8c98c84b4ac05040305143709c8ae812fb08f36df0af61227b4147a58663 # shrinks to numbers = [0.0], texts = [] +cc b6d546bba511082d918c04fa30486795c6ee13a44655b21e24f66ee018b621da # shrinks to blocks = [Paragraph(Paragraph { format: ParaFormat { style: None, direct: ParaProps { align: None, indent_left: None, indent_right: None, indent_first_line: None, indent_hanging: None, space_before: None, space_after: None, line_height: None, keep_with_next: None, page_break_before: None, background: None, outline_level: None }, run_direct: FontProps { name: None, size: None, bold: None, italic: None, strike: None, underline: None, color: None, highlight: None, vert_align: None, small_caps: None, caps: None } }, content: [Styled { content: [Styled { content: [Text("")], props: RunProps { style: None, direct: FontProps { name: None, size: None, bold: None, italic: None, strike: None, underline: None, color: None, highlight: None, vert_align: None, small_caps: None, caps: None } } }], props: RunProps { style: None, direct: FontProps { name: None, size: None, bold: None, italic: None, strike: None, underline: None, color: None, highlight: None, vert_align: None, small_caps: None, caps: None } } }] })], title = None +cc ca330d585bd79b97599a3e9d33e1a9c3b78948b986a710931926dabcbbe4bff5 # shrinks to numbers = [-990038535528.8899], texts = [] diff --git a/crates/docsai-model/tests/json_roundtrip.rs b/crates/docsai-model/tests/json_roundtrip.rs new file mode 100644 index 0000000..5ee077a --- /dev/null +++ b/crates/docsai-model/tests/json_roundtrip.rs @@ -0,0 +1,491 @@ +//! Fase 0 acceptance criterion: the IR serialises to JSON and back +//! identically. Checked with hand-written documents covering every node kind +//! and with `proptest` over randomly generated IRs. + +use docsai_model::assets::AssetId; +use docsai_model::image::*; +use docsai_model::list::*; +use docsai_model::sheet::*; +use docsai_model::style::*; +use docsai_model::text::*; +use docsai_model::units::{Length, Point, Size}; +use docsai_model::Document; +use proptest::prelude::*; + +fn assert_json_round_trip(doc: &Document) { + let json = serde_json::to_string(doc).expect("serialises"); + let back: Document = serde_json::from_str(&json).expect("deserialises"); + assert_eq!(doc, &back, "round-trip changed the document\n{json}"); + // Serialising the reconstructed value must produce the very same bytes. + assert_eq!(json, serde_json::to_string(&back).unwrap()); +} + +#[test] +fn round_trips_a_document_using_every_block_kind() { + let asset = AssetId::new("deadbeefcafe0001"); + let mut geometry = ImageGeometry::inline(Size::new(Length::from_cm(4.0), Length::from_cm(3.0))); + geometry.anchor = Anchor::Floating { + relative_to_h: RelBase::Margin, + relative_to_v: RelBase::Paragraph, + position: HVPos { + h: AxisPos::Offset(Length::from_cm(1.2)), + v: AxisPos::Align(AlignKeyword::Top), + }, + wrap: WrapMode::Square, + wrap_side: WrapSide::Right, + behind_text: false, + }; + geometry.rotation_deg = 45.0; + geometry.flip = Flip::HV; + geometry.crop = Some(CropRect { + left: 10.0, + top: 5.0, + right: 20.0, + bottom: 0.0, + }); + geometry.border = Some(SimpleBorder { + width: Length::from_pt(1.0), + style: "solid".into(), + color: "#000000".into(), + }); + geometry.z_index = Some(2); + geometry.native_size_px = Some((800, 600)); + geometry.dpi = Some(300); + + let mut image = ImageRef::new(asset, geometry); + image.alt = "Diagrama".into(); + image.title = Some("Figura 1".into()); + image.name = Some("Logo".into()); + image.link = Some("https://example.com".into()); + image.effects_raw = Some(RawId::new("raw-0001")); + + let mut styles = StyleCatalog::default(); + styles.defaults.font.size = Some(Length::from_pt(11.0)); + let mut heading = Style::new("Heading1", StyleType::Paragraph); + heading.based_on = Some(StyleId::new("Normal")); + heading.font.color = Some("#2E74B5".into()); + heading.paragraph.outline_level = Some(0); + heading.paragraph.line_height = Some(LineHeight::Multiple(240)); + styles.insert(heading); + + let mut list_defs = ListCatalog::default(); + list_defs.insert( + ListId::new("L1"), + ListDef { + levels: vec![ + ListLevel { + format: NumFormat::Decimal, + text: "%1.".into(), + start: Some(1), + indent: Some(Length::from_twips(720)), + hanging: Some(Length::from_twips(360)), + }, + ListLevel::new(NumFormat::Other("ideographDigital".into()), "%2"), + ], + }, + ); + + let doc = Document::Text(TextDocument { + meta: DocumentMeta { + title: Some("Informe".into()), + author: Some("Ana".into()), + created: Some("2026-03-01T10:00:00Z".into()), + language: Some("es-ES".into()), + custom: [("Departamento".to_string(), "Ventas".to_string())] + .into_iter() + .collect(), + ..Default::default() + }, + styles, + list_defs, + sections: vec![Section { + page: PageGeometry { + size: Size::new(Length::from_twips(11906), Length::from_twips(16838)), + margins: Margins { + top: Length::from_twips(1417), + right: Length::from_twips(1701), + bottom: Length::from_twips(1417), + left: Length::from_twips(1701), + header: Length::from_twips(708), + footer: Length::from_twips(708), + }, + orientation: Orientation::Landscape, + columns: 2, + title_page: true, + }, + headers: vec![HeaderFooter { + scope: HeaderScope::Default, + blocks: vec![Block::Paragraph(Paragraph::text("Cabecera"))], + }], + footers: vec![HeaderFooter { + scope: HeaderScope::First, + blocks: vec![Block::Paragraph(Paragraph::new(vec![Inline::Field { + kind: FieldKind::Page, + cached: "1".into(), + instruction: " PAGE ".into(), + }]))], + }], + blocks: vec![ + Block::Heading(Heading { + level: 1, + paragraph: Paragraph { + format: ParaFormat::styled("Heading1"), + content: vec![Inline::Text("Título".into())], + }, + }), + Block::Paragraph(Paragraph::new(vec![ + Inline::Styled { + content: vec![Inline::Text("negrita".into())], + props: RunProps::direct(FontProps { + bold: Some(true), + underline: Some(Underline::Wave), + vert_align: Some(VertAlign::Superscript), + ..Default::default() + }), + }, + Inline::Link { + target: "https://example.com".into(), + content: vec![Inline::Text("enlace".into())], + props: RunProps { + style: Some(StyleId::new("Hyperlink")), + direct: FontProps::default(), + }, + }, + Inline::Break(BreakKind::Page), + Inline::Footnote(vec![Block::Paragraph(Paragraph::text("nota"))]), + Inline::Image(image.clone()), + Inline::Raw(RawFragment { + id: RawId::new("raw-0002"), + format: "ooxml".into(), + part: "word/document.xml".into(), + content: "".into(), + }), + ])), + Block::List(List { + def: Some(ListId::new("L1")), + ordered: true, + level: 0, + items: vec![ListItem { + blocks: vec![ + Block::Paragraph(Paragraph::text("uno")), + Block::List(List { + def: Some(ListId::new("L1")), + ordered: false, + level: 1, + items: vec![ListItem { + blocks: vec![Block::Paragraph(Paragraph::text("anidado"))], + }], + }), + ], + }], + }), + Block::Table(Table { + style: Some(StyleId::new("TableGrid")), + col_widths: vec![Length::from_twips(2500), Length::from_twips(2500)], + header_row: true, + rows: vec![TableRow { + cells: vec![ + TableCell { + colspan: 2, + rowspan: 1, + background: Some("#F2F2F2".into()), + ..TableCell::text("combinada") + }, + TableCell { + covered: true, + ..Default::default() + }, + ], + is_header: true, + }], + }), + Block::Image(image), + Block::TextBox(TextBox { + blocks: vec![Block::Paragraph(Paragraph::text("caja"))], + size: Some(Size::new(Length::from_cm(6.0), Length::from_cm(3.0))), + x: Some(Length::from_cm(5.0)), + y: Some(Length::from_cm(2.0)), + }), + Block::Raw(RawFragment { + id: RawId::new("raw-0003"), + format: "ooxml".into(), + part: "word/document.xml".into(), + content: "".into(), + }), + ], + }], + }); + + assert_json_round_trip(&doc); +} + +#[test] +fn round_trips_a_workbook_with_every_anchor_kind() { + let asset = AssetId::new("deadbeefcafe0002"); + let sheet_image = |anchor: Anchor| { + let mut g = ImageGeometry::inline(Size::new(Length::from_px(180.0), Length::from_px(60.0))); + g.anchor = anchor; + ImageRef::new(asset.clone(), g) + }; + + let mut sheet = Sheet::new("Ventas"); + sheet.cells.insert( + CellRef::new(3, 1), + Cell { + value: CellValue::Number(300.0), + formula: Some(Formula { + text: "SUM(B2:C2)".into(), + dialect: FormulaDialect::Ooxml, + shared_over: Some(CellRange::new(CellRef::new(3, 1), CellRef::new(3, 3))), + array_over: None, + }), + num_fmt: Some(NumFmt { + code: "#,##0".into(), + id: Some(3), + }), + style: Some(StyleId::new("HeaderRow")), + }, + ); + sheet.cells.insert( + CellRef::new(0, 0), + Cell { + value: CellValue::DateTime("2026-01-15T00:00:00".into()), + ..Default::default() + }, + ); + sheet.cells.insert( + CellRef::new(1, 0), + Cell { + value: CellValue::Error("#DIV/0!".into()), + ..Default::default() + }, + ); + sheet + .merges + .push(CellRange::new(CellRef::new(0, 4), CellRef::new(2, 4))); + sheet.cols.insert( + 0, + ColProps { + width_chars: Some(18.5), + hidden: Some(false), + }, + ); + sheet.rows.insert( + 0, + RowProps { + height: Some(Length::from_pt(15.0)), + hidden: None, + }, + ); + sheet.pane = Some(Pane { + top_left: CellRef::new(0, 1), + frozen: true, + }); + sheet.images = vec![ + sheet_image(Anchor::SheetTwoCell { + from: CellAnchor::new( + CellRef::new(1, 1), + Length::from_px(12.0), + Length::from_px(3.0), + ), + to: CellAnchor::new(CellRef::new(3, 7), Length::ZERO, Length::ZERO), + move_with_cells: true, + size_with_cells: false, + }), + sheet_image(Anchor::SheetOneCell { + from: CellAnchor::new(CellRef::new(5, 19), Length::ZERO, Length::ZERO), + }), + sheet_image(Anchor::SheetAbsolute { + pos: Point::new(Length::from_cm(5.0), Length::from_cm(8.0)), + }), + ]; + + let doc = Document::Workbook(Workbook { + meta: DocumentMeta { + title: Some("Libro".into()), + ..Default::default() + }, + styles: StyleCatalog::default(), + defined_names: vec![DefinedName { + name: "TOTAL_ANUAL".into(), + refers_to: "Ventas!$D$10".into(), + sheet: None, + }], + sheets: vec![sheet], + active_sheet: Some("Ventas".into()), + }); + + assert_json_round_trip(&doc); +} + +// -------------------------------------------------------------------------- +// Property testing +// -------------------------------------------------------------------------- + +prop_compose! { + fn arb_length()(emu in -10_000_000i64..10_000_000i64) -> Length { + Length::from_emu(emu) + } +} + +fn arb_font() -> impl Strategy { + ( + proptest::option::of("[a-zA-Z ]{1,12}"), + proptest::option::of(arb_length()), + proptest::option::of(any::()), + proptest::option::of(any::()), + proptest::option::of("#[0-9A-F]{6}"), + ) + .prop_map(|(name, size, bold, italic, color)| FontProps { + name, + size, + bold, + italic, + color, + ..Default::default() + }) +} + +fn arb_inline() -> impl Strategy { + let leaf = prop_oneof![ + ".{0,24}".prop_map(Inline::Text), + Just(Inline::Break(BreakKind::Line)), + Just(Inline::Break(BreakKind::Page)), + (".{0,8}", ".{0,8}").prop_map(|(cached, instr)| Inline::Field { + kind: FieldKind::from_instruction(&instr), + cached, + instruction: instr, + }), + ]; + leaf.prop_recursive(3, 12, 3, |inner| { + prop_oneof![ + (proptest::collection::vec(inner.clone(), 0..3), arb_font()).prop_map( + |(content, direct)| Inline::Styled { + content, + props: RunProps::direct(direct), + } + ), + ( + "https?://[a-z]{1,8}\\.test", + proptest::collection::vec(inner, 0..3) + ) + .prop_map(|(target, content)| Inline::Link { + target, + content, + props: RunProps::default(), + }), + ] + }) +} + +fn arb_block() -> impl Strategy { + let leaf = prop_oneof![ + ( + proptest::collection::vec(arb_inline(), 0..4), + proptest::option::of("[A-Za-z]{1,10}") + ) + .prop_map(|(content, style)| Block::Paragraph(Paragraph { + format: ParaFormat { + style: style.map(StyleId::new), + ..Default::default() + }, + content, + })), + (1u8..=6, ".{0,16}").prop_map(|(level, text)| Block::Heading(Heading { + level, + paragraph: Paragraph::text(text), + })), + (".{0,8}", ".{0,32}").prop_map(|(part, content)| Block::Raw(RawFragment { + id: RawId::new("raw-0001"), + format: "ooxml".into(), + part, + content, + })), + ]; + leaf.prop_recursive(3, 16, 3, |inner| { + prop_oneof![ + ( + any::(), + proptest::collection::vec(inner.clone(), 1..3) + ) + .prop_map(|(ordered, blocks)| Block::List(List { + def: None, + ordered, + level: 0, + items: vec![ListItem { blocks }], + })), + proptest::collection::vec(inner, 1..3).prop_map(|blocks| Block::Table(Table { + style: None, + col_widths: vec![], + header_row: false, + rows: vec![TableRow { + cells: vec![TableCell { + blocks, + ..Default::default() + }], + is_header: false, + }], + })), + ] + }) +} + +proptest! { + #![proptest_config(ProptestConfig::with_cases(96))] + + #[test] + fn arbitrary_text_documents_round_trip_through_json( + blocks in proptest::collection::vec(arb_block(), 0..6), + title in proptest::option::of(".{0,20}"), + ) { + let doc = Document::Text(TextDocument { + meta: DocumentMeta { title, ..Default::default() }, + sections: vec![Section { blocks, ..Default::default() }], + ..Default::default() + }); + let json = serde_json::to_string(&doc).unwrap(); + let back: Document = serde_json::from_str(&json).unwrap(); + prop_assert_eq!(&doc, &back); + prop_assert_eq!(json, serde_json::to_string(&back).unwrap()); + } + + #[test] + fn arbitrary_cells_round_trip_through_json( + numbers in proptest::collection::vec(-1e12f64..1e12, 0..8), + texts in proptest::collection::vec(".{0,16}", 0..8), + ) { + let mut sheet = Sheet::new("Hoja"); + for (i, n) in numbers.iter().enumerate() { + sheet.cells.insert( + CellRef::new(0, i as u32), + Cell { value: CellValue::Number(*n), ..Default::default() }, + ); + } + for (i, t) in texts.iter().enumerate() { + sheet.cells.insert( + CellRef::new(1, i as u32), + Cell { value: CellValue::Text(t.clone()), ..Default::default() }, + ); + } + let doc = Document::Workbook(Workbook { sheets: vec![sheet], ..Default::default() }); + let json = serde_json::to_string(&doc).unwrap(); + let back: Document = serde_json::from_str(&json).unwrap(); + prop_assert_eq!(&doc, &back); + } + + #[test] + fn style_resolution_never_panics_on_arbitrary_chains( + edges in proptest::collection::vec((0u8..8, 0u8..8), 0..16), + ) { + let mut catalog = StyleCatalog::default(); + for (child, parent) in edges { + let mut style = Style::new(format!("S{child}"), StyleType::Paragraph); + style.based_on = Some(StyleId::new(format!("S{parent}"))); + style.font.bold = Some(child % 2 == 0); + catalog.insert(style); + } + for i in 0..8u8 { + let _ = catalog.resolve(Some(&StyleId::new(format!("S{i}")))); + } + } +} diff --git a/crates/docsai-odf/Cargo.toml b/crates/docsai-odf/Cargo.toml new file mode 100644 index 0000000..c118196 --- /dev/null +++ b/crates/docsai-odf/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "docsai-odf" +description = "OpenDocument (odt/ods) readers and writers for docsai" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +authors.workspace = true + +[dependencies] +docsai-model = { workspace = true } diff --git a/crates/docsai-odf/src/lib.rs b/crates/docsai-odf/src/lib.rs new file mode 100644 index 0000000..4322616 --- /dev/null +++ b/crates/docsai-odf/src/lib.rs @@ -0,0 +1,21 @@ +//! OpenDocument readers and writers (`.odt`, `.ods`). +//! +//! Scheduled for **Fase 4** of `docs/plan-desarrollo.md`. The crate exists +//! from Fase 0 so that the workspace layout and the dependency rules of +//! `AGENTS.md` §3 are enforced by the compiler from the start: this crate may +//! depend on `docsai-model` and on nothing else in the workspace. + +#![forbid(unsafe_code)] + +use docsai_model::Format; + +/// Formats this crate will handle. +pub const FORMATS: &[Format] = &[Format::Odt, Format::Ods]; + +#[cfg(test)] +mod tests { + #[test] + fn formats_are_the_odf_pair() { + assert_eq!(super::FORMATS.len(), 2); + } +} diff --git a/crates/docsai-office/Cargo.toml b/crates/docsai-office/Cargo.toml new file mode 100644 index 0000000..122b334 --- /dev/null +++ b/crates/docsai-office/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "docsai-office" +description = "OOXML and legacy Office readers/writers for docsai" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +authors.workspace = true + +[dependencies] +docsai-model = { workspace = true } +quick-xml = { workspace = true } +thiserror = { workspace = true } +tracing = { workspace = true } +zip = { workspace = true } diff --git a/crates/docsai-office/src/detect.rs b/crates/docsai-office/src/detect.rs new file mode 100644 index 0000000..f5b252d --- /dev/null +++ b/crates/docsai-office/src/detect.rs @@ -0,0 +1,168 @@ +//! Format detection **by content**, not by extension (arquitectura §4). +//! +//! A file called `informe.txt` that is really a `.docx` must still convert, +//! and a `.docx` extension on a ZIP of holiday photos must not. + +use docsai_model::Format; +use std::io::{Read, Seek, SeekFrom}; + +/// How confident detection is about a format. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub enum DetectScore { + /// The content does not look like this format at all. + No, + /// The container matches but the payload was not inspected. + Maybe, + /// The defining part of the format was found. + Certain, +} + +/// The OLE2/CFB compound file signature, shared by `.doc` and `.xls`. +const OLE2_MAGIC: &[u8] = &[0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A, 0xE1]; +/// Local file header of a ZIP archive. +const ZIP_MAGIC: &[u8] = b"PK\x03\x04"; + +/// Detects the format of a document. +/// +/// `hint` is the file name, used only to break ties the content cannot: `.doc` +/// and `.xls` share a container, and so do `.odt` and `.ods` when the +/// `mimetype` entry is missing. +pub fn detect(mut reader: R, hint: Option<&str>) -> (Format, DetectScore) { + let mut magic = [0u8; 8]; + let read = read_head(&mut reader, &mut magic); + let _ = reader.seek(SeekFrom::Start(0)); + + if read >= 8 && magic == OLE2_MAGIC { + // Both legacy formats are the same container; only the name tells them + // apart without parsing the directory (Fase 5 does that properly). + let format = match extension(hint).as_deref() { + Some("xls") | Some("xlt") => Format::Xls, + _ => Format::Doc, + }; + return (format, DetectScore::Maybe); + } + + if read >= 4 && magic[..4] == *ZIP_MAGIC { + if let Ok(zip) = zip::ZipArchive::new(&mut reader) { + let names: Vec = zip.file_names().map(str::to_string).collect(); + let has = |name: &str| names.iter().any(|n| n == name); + let _ = reader.seek(SeekFrom::Start(0)); + + if has(crate::docx::DOCUMENT_PART) { + return (Format::Docx, DetectScore::Certain); + } + if has("xl/workbook.xml") { + return (Format::Xlsx, DetectScore::Certain); + } + if has("content.xml") && has("mimetype") { + let format = match extension(hint).as_deref() { + Some("ods") => Format::Ods, + _ => Format::Odt, + }; + return (format, DetectScore::Maybe); + } + } + let _ = reader.seek(SeekFrom::Start(0)); + return (Format::Docx, DetectScore::No); + } + + // DocMark is text; the front matter is what identifies it. + if looks_like_docmark(&magic[..read]) { + return (Format::DocMark, DetectScore::Maybe); + } + match extension(hint).and_then(|e| Format::parse(&e)) { + Some(format) => (format, DetectScore::Maybe), + None => (Format::DocMark, DetectScore::No), + } +} + +fn read_head(reader: &mut R, buffer: &mut [u8; 8]) -> usize { + let mut filled = 0; + while filled < buffer.len() { + match reader.read(&mut buffer[filled..]) { + Ok(0) => break, + Ok(n) => filled += n, + Err(_) => break, + } + } + filled +} + +fn looks_like_docmark(head: &[u8]) -> bool { + head.starts_with(b"---") +} + +fn extension(hint: Option<&str>) -> Option { + let name = hint?; + let ext = name.rsplit('.').next()?; + if ext == name { + return None; + } + Some(ext.to_ascii_lowercase()) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Cursor; + + fn corpus(name: &str) -> Cursor> { + let path = format!("{}/../../corpus/{name}", env!("CARGO_MANIFEST_DIR")); + Cursor::new(std::fs::read(&path).unwrap_or_else(|e| panic!("{path}: {e}"))) + } + + #[test] + fn recognises_docx_and_xlsx_by_their_defining_part() { + assert_eq!( + detect(corpus("docx/basic-text.docx"), Some("basic-text.docx")), + (Format::Docx, DetectScore::Certain) + ); + assert_eq!( + detect(corpus("xlsx/values-types.xlsx"), Some("values-types.xlsx")), + (Format::Xlsx, DetectScore::Certain) + ); + } + + #[test] + fn the_extension_does_not_override_the_content() { + let (format, score) = detect(corpus("docx/basic-text.docx"), Some("notas.txt")); + assert_eq!((format, score), (Format::Docx, DetectScore::Certain)); + } + + #[test] + fn a_plain_zip_is_not_an_office_document() { + let mut buffer = Vec::new(); + { + let mut zip = zip::ZipWriter::new(Cursor::new(&mut buffer)); + zip.start_file::<_, ()>("photo.jpg", Default::default()) + .unwrap(); + zip.finish().unwrap(); + } + assert_eq!( + detect(Cursor::new(buffer), Some("x.docx")).1, + DetectScore::No + ); + } + + #[test] + fn legacy_containers_are_told_apart_by_name() { + let mut ole = OLE2_MAGIC.to_vec(); + ole.extend_from_slice(&[0u8; 32]); + assert_eq!( + detect(Cursor::new(ole.clone()), Some("a.doc")).0, + Format::Doc + ); + assert_eq!(detect(Cursor::new(ole), Some("a.xls")).0, Format::Xls); + } + + #[test] + fn docmark_is_recognised_by_its_front_matter() { + let text = b"---\ndocmark: \"1.0\"\n---\n# Hola\n".to_vec(); + assert_eq!(detect(Cursor::new(text), None).0, Format::DocMark); + } + + #[test] + fn an_empty_file_is_not_a_panic() { + assert_eq!(detect(Cursor::new(Vec::new()), None).1, DetectScore::No); + } +} diff --git a/crates/docsai-office/src/docx/body.rs b/crates/docsai-office/src/docx/body.rs new file mode 100644 index 0000000..443b7e2 --- /dev/null +++ b/crates/docsai-office/src/docx/body.rs @@ -0,0 +1,969 @@ +//! `word/document.xml` (and headers, footers and footnotes) → IR blocks. + +use std::collections::BTreeMap; + +use docsai_model::assets::AssetStore; +use docsai_model::image::RawId; +use docsai_model::list::ListId; +use docsai_model::report::{ConversionReport, Warning}; +use docsai_model::style::{StyleCatalog, StyleId}; +use docsai_model::text::{ + Block, BreakKind, FieldKind, Heading, Inline, List, ListItem, ParaFormat, Paragraph, + RawFragment, RunProps, Table, TableCell, TableRow, +}; +use docsai_model::units::Length; + +use super::drawing::{read_drawing, read_vml_picture, DrawingContext}; +use super::format::{hex_color, paragraph_props, run_props}; +use super::numbering::Numbering; +use super::styles::heading_level; +use crate::package::{Package, Relationships}; +use crate::xml::Element; + +/// Everything a body walk needs to read, but never mutates. +pub struct Ctx<'a> { + pub package: &'a Package, + pub rels: &'a Relationships, + /// Part being read, used for warning locations and raw-block provenance. + pub part: &'a str, + /// The part's text, so raw-blocks can quote the original bytes. + pub source: &'a str, + pub styles: &'a StyleCatalog, + pub numbering: &'a Numbering, + /// Footnote id → its blocks. Empty while reading `footnotes.xml` itself. + pub footnotes: &'a BTreeMap>, +} + +impl Ctx<'_> { + fn drawing(&self) -> DrawingContext<'_> { + DrawingContext { + package: self.package, + rels: self.rels, + part: self.part, + } + } +} + +/// Everything a body walk mutates. +pub struct State<'a> { + pub assets: &'a mut dyn AssetStore, + pub report: &'a mut ConversionReport, + /// Sequence behind `raw-0001`, `raw-0002`… Shared across the document so + /// that ids are unique and stable. + pub raw_seq: u32, + /// Accepted tracked changes, reported once at the end. + pub revisions: u32, +} + +impl State<'_> { + /// Preserves an unrecognised element verbatim (spec §7). + fn raw(&mut self, element: &Element, ctx: &Ctx<'_>) -> RawFragment { + self.raw_seq += 1; + self.report.raw_block( + qualified(element), + format!("{}:{}", ctx.part, element.span.start), + ); + RawFragment { + id: RawId::new(format!("raw-{:04}", self.raw_seq)), + format: "ooxml".into(), + part: ctx.part.to_string(), + content: element.raw(ctx.source).to_string(), + } + } +} + +fn qualified(element: &Element) -> String { + if element.prefix.is_empty() { + element.name.clone() + } else { + format!("{}:{}", element.prefix, element.name) + } +} + +/// Elements that carry no content and need neither a block nor a warning. +const IGNORED: &[&str] = &[ + "bookmarkStart", + "bookmarkEnd", + "proofErr", + "commentRangeStart", + "commentRangeEnd", + "permStart", + "permEnd", + "lastRenderedPageBreak", + "sectPr", + "tblPr", + "tblGrid", + "trPr", + "tcPr", + "pPr", + "rPr", +]; + +// -------------------------------------------------------------------------- +// Blocks +// -------------------------------------------------------------------------- + +/// One item of the flat body stream, before lists are rebuilt into a tree. +enum Flat { + Block(Block), + /// A paragraph carrying `w:numPr`, still flat as OOXML stores it. + ListParagraph { + num_id: i64, + ilvl: u8, + paragraph: Paragraph, + }, +} + +/// Reads the block children of an element (a body, a table cell, a header…). +pub fn read_blocks(parent: &Element, ctx: &Ctx<'_>, st: &mut State<'_>) -> Vec { + read_blocks_of(parent.children(), ctx, st) +} + +/// Reads an explicit sequence of block elements. +/// +/// Sections split the body at the paragraph that carries `w:sectPr`, so the +/// document reader needs to hand over slices rather than a whole parent. +pub fn read_blocks_of<'e>( + children: impl IntoIterator, + ctx: &Ctx<'_>, + st: &mut State<'_>, +) -> Vec { + let mut flat = Vec::new(); + collect_blocks(children, ctx, st, &mut flat); + rebuild_lists(flat, ctx, st) +} + +fn collect_blocks<'e>( + children: impl IntoIterator, + ctx: &Ctx<'_>, + st: &mut State<'_>, + out: &mut Vec, +) { + for child in children { + match child.name.as_str() { + "p" => { + let (paragraph, numbering) = read_paragraph(child, ctx, st); + match numbering { + Some((num_id, ilvl)) => out.push(Flat::ListParagraph { + num_id, + ilvl, + paragraph, + }), + None => { + st.report.stats.paragraphs += 1; + out.push(Flat::Block(as_heading_or_paragraph(paragraph, ctx, st))); + } + } + } + "tbl" => { + st.report.stats.tables += 1; + out.push(Flat::Block(Block::Table(read_table(child, ctx, st)))); + } + "sdt" => { + // A content control has no DocMark representation, but its + // text does: flatten to the content and say so, rather than + // hiding readable text inside an opaque raw-block. + st.report.warn(Warning::Degraded { + what: "w:sdt".into(), + why: "structured document tag flattened to its content".into(), + }); + if let Some(content) = child.child("sdtContent") { + collect_blocks(content.children(), ctx, st, out); + } + } + "customXml" => collect_blocks(child.children(), ctx, st, out), + "ins" => { + st.revisions += 1; + collect_blocks(child.children(), ctx, st, out); + } + "del" => { + st.revisions += 1; + } + name if IGNORED.contains(&name) => {} + _ => { + let fragment = st.raw(child, ctx); + out.push(Flat::Block(Block::Raw(fragment))); + } + } + } +} + +fn as_heading_or_paragraph(paragraph: Paragraph, ctx: &Ctx<'_>, st: &mut State<'_>) -> Block { + let level = paragraph + .format + .direct + .outline_level + .map(|l| l + 1) + .or_else(|| { + paragraph + .format + .style + .as_ref() + .and_then(|id| heading_level(ctx.styles, id)) + }); + match level { + Some(level) if (1..=9).contains(&level) && !paragraph.is_empty() => { + st.report.stats.headings += 1; + Block::Heading(Heading { level, paragraph }) + } + _ => Block::Paragraph(paragraph), + } +} + +/// Rebuilds the list tree from the flat `(numId, ilvl)` pairs. +fn rebuild_lists(flat: Vec, ctx: &Ctx<'_>, st: &mut State<'_>) -> Vec { + let mut out = Vec::new(); + let mut run: Vec<(i64, u8, Paragraph)> = Vec::new(); + + for item in flat { + match item { + Flat::ListParagraph { + num_id, + ilvl, + paragraph, + } => { + // A different list definition starts a different list. + if run.first().is_some_and(|(id, _, _)| *id != num_id) { + flush_list(&mut run, ctx, st, &mut out); + } + run.push((num_id, ilvl, paragraph)); + } + Flat::Block(block) => { + flush_list(&mut run, ctx, st, &mut out); + out.push(block); + } + } + } + flush_list(&mut run, ctx, st, &mut out); + out +} + +fn flush_list( + run: &mut Vec<(i64, u8, Paragraph)>, + ctx: &Ctx<'_>, + st: &mut State<'_>, + out: &mut Vec, +) { + if run.is_empty() { + return; + } + st.report.stats.lists += 1; + st.report.stats.paragraphs += run.len() as u32; + let items = std::mem::take(run); + out.push(build_list(&items, ctx)); +} + +/// Turns a run of same-list paragraphs into a nested [`List`]. +fn build_list(items: &[(i64, u8, Paragraph)], ctx: &Ctx<'_>) -> Block { + let (num_id, base_level, _) = &items[0]; + let base = *base_level; + let def: Option = ctx.numbering.list_id(*num_id).cloned(); + let mut list = List { + def, + ordered: ctx.numbering.is_ordered(*num_id, base as usize), + level: base, + items: Vec::new(), + }; + + let mut i = 0; + while i < items.len() { + if items[i].1 <= base { + list.items.push(ListItem { + blocks: vec![Block::Paragraph(items[i].2.clone())], + }); + i += 1; + } else { + let start = i; + while i < items.len() && items[i].1 > base { + i += 1; + } + let nested = build_list(&items[start..i], ctx); + match list.items.last_mut() { + // A deeper item with no parent above it (a malformed document) + // still becomes a list, just an orphan one. + Some(parent) => parent.blocks.push(nested), + None => list.items.push(ListItem { + blocks: vec![nested], + }), + } + } + } + Block::List(list) +} + +// -------------------------------------------------------------------------- +// Paragraphs +// -------------------------------------------------------------------------- + +/// Reads a `w:p`, returning its `(numId, ilvl)` when it is a list item. +fn read_paragraph( + p: &Element, + ctx: &Ctx<'_>, + st: &mut State<'_>, +) -> (Paragraph, Option<(i64, u8)>) { + let ppr = p.child("pPr"); + let mut format = ParaFormat::default(); + let mut numbering = None; + + if let Some(ppr) = ppr { + format.style = ppr + .child("pStyle") + .and_then(|e| e.attr("val")) + .map(StyleId::new); + format.direct = paragraph_props(ppr); + if let Some(rpr) = ppr.child("rPr") { + format.run_direct = run_props(rpr); + } + if let Some(numpr) = ppr.child("numPr") { + let num_id = numpr.child("numId").and_then(|e| e.attr_i64("val")); + let ilvl = numpr + .child("ilvl") + .and_then(|e| e.attr_i64("val")) + .unwrap_or(0) + .clamp(0, 8) as u8; + // `numId=0` explicitly means "no numbering". + numbering = num_id.filter(|id| *id > 0).map(|id| (id, ilvl)); + } + } + + let content = read_inlines(p, ctx, st); + (Paragraph { format, content }, numbering) +} + +/// A run may emit content *or* field-control markers; the paragraph-level loop +/// assembles the markers into fields. +enum Piece { + // Boxed: `Inline` dwarfs the field-control markers, and a paragraph builds + // a whole vector of these. + Inline(Box), + FieldBegin, + FieldSeparate, + FieldEnd, + Instruction(String), +} + +impl Piece { + fn inline(inline: Inline) -> Piece { + Piece::Inline(Box::new(inline)) + } +} + +struct FieldBuilder { + instruction: String, + result: Vec, + in_result: bool, +} + +/// Reads the inline children of a paragraph (or of a hyperlink, or of an +/// inserted run range). +fn read_inlines(parent: &Element, ctx: &Ctx<'_>, st: &mut State<'_>) -> Vec { + let mut pieces = Vec::new(); + collect_pieces(parent, ctx, st, &mut pieces); + + let mut out: Vec = Vec::new(); + let mut fields: Vec = Vec::new(); + + for piece in pieces { + match piece { + Piece::FieldBegin => fields.push(FieldBuilder { + instruction: String::new(), + result: Vec::new(), + in_result: false, + }), + Piece::Instruction(text) => { + if let Some(field) = fields.last_mut() { + field.instruction.push_str(&text); + } + } + Piece::FieldSeparate => { + if let Some(field) = fields.last_mut() { + field.in_result = true; + } + } + Piece::FieldEnd => { + let Some(field) = fields.pop() else { continue }; + let inline = build_field(field); + match fields.last_mut() { + Some(parent) => parent.result.push(inline), + None => out.push(inline), + } + } + Piece::Inline(inline) => match fields.last_mut() { + Some(field) if field.in_result => field.result.push(*inline), + // Content between `begin` and `separate` is the instruction + // rendering, which the field itself already carries. + Some(_) => {} + None => out.push(*inline), + }, + } + } + + // An unterminated field still has to surface its result text. + for field in fields { + out.push(build_field(field)); + } + out +} + +fn build_field(field: FieldBuilder) -> Inline { + let cached = Paragraph::new(field.result).plain_text(); + Inline::Field { + kind: FieldKind::from_instruction(&field.instruction), + cached, + instruction: field.instruction.trim().to_string(), + } +} + +fn collect_pieces(parent: &Element, ctx: &Ctx<'_>, st: &mut State<'_>, out: &mut Vec) { + for child in parent.children() { + match child.name.as_str() { + "r" => read_run(child, ctx, st, out), + "hyperlink" => { + let target = hyperlink_target(child, ctx); + let content = read_inlines(child, ctx, st); + let props = RunProps { + style: child + .path(&["r", "rPr", "rStyle"]) + .and_then(|e| e.attr("val")) + .map(StyleId::new), + ..Default::default() + }; + match target { + Some(target) => out.push(Piece::inline(Inline::Link { + target, + content, + props, + })), + // A hyperlink with no resolvable target is still text. + None => out.extend(content.into_iter().map(Piece::inline)), + } + } + "fldSimple" => { + let instruction = child.attr("instr").unwrap_or_default().trim().to_string(); + let cached = Paragraph::new(read_inlines(child, ctx, st)).plain_text(); + out.push(Piece::inline(Inline::Field { + kind: FieldKind::from_instruction(&instruction), + cached, + instruction, + })); + } + "ins" | "smartTag" => { + if child.name == "ins" { + st.revisions += 1; + } + collect_pieces(child, ctx, st, out); + } + "del" => { + st.revisions += 1; + } + "sdt" => { + if let Some(content) = child.child("sdtContent") { + collect_pieces(content, ctx, st, out); + } + } + "commentReference" => st.report.warn(Warning::Degraded { + what: "w:commentReference".into(), + why: "comments are out of scope in v1".into(), + }), + name if IGNORED.contains(&name) => {} + _ => { + let fragment = st.raw(child, ctx); + out.push(Piece::inline(Inline::Raw(fragment))); + } + } + } +} + +fn hyperlink_target(link: &Element, ctx: &Ctx<'_>) -> Option { + if let Some(id) = link.attr_qualified("r:id") { + if let Some(rel) = ctx.rels.get(id) { + let anchor = link.attr("anchor").unwrap_or_default(); + return Some(if anchor.is_empty() { + rel.target.clone() + } else { + format!("{}#{anchor}", rel.target) + }); + } + } + link.attr("anchor").map(|a| format!("#{a}")) +} + +fn read_run(r: &Element, ctx: &Ctx<'_>, st: &mut State<'_>, out: &mut Vec) { + let props = r + .child("rPr") + .map(|rpr| RunProps { + style: rpr + .child("rStyle") + .and_then(|e| e.attr("val")) + .map(StyleId::new), + direct: run_props(rpr), + }) + .unwrap_or_default(); + + // Content of this run, buffered so that formatting wraps it in one span. + let mut buffer: Vec = Vec::new(); + macro_rules! flush { + () => { + if !buffer.is_empty() { + let content = std::mem::take(&mut buffer); + out.extend( + Inline::styled(content, props.clone()) + .into_iter() + .map(Piece::inline), + ); + } + }; + } + + for child in r.children() { + match child.name.as_str() { + "t" => buffer.push(Inline::Text(child.text())), + "tab" => buffer.push(Inline::Text("\t".into())), + "noBreakHyphen" => buffer.push(Inline::Text("\u{2011}".into())), + "softHyphen" => {} + "br" => buffer.push(Inline::Break(match child.attr("type") { + Some("page") => BreakKind::Page, + Some("column") => BreakKind::Column, + _ => BreakKind::Line, + })), + "cr" => buffer.push(Inline::Break(BreakKind::Line)), + "sym" => { + // A symbol font character: the code point is meaningless + // without the font, so preserve it verbatim. + let fragment = st.raw(child, ctx); + buffer.push(Inline::Raw(fragment)); + } + "drawing" => match read_drawing(child, &ctx.drawing(), st.assets, st.report) { + Ok(Some(image)) => buffer.push(Inline::Image(image)), + Ok(None) => { + let fragment = st.raw(child, ctx); + buffer.push(Inline::Raw(fragment)); + } + Err(e) => st.report.warn(Warning::AssetIssue { + asset: ctx.part.to_string(), + why: e.to_string(), + }), + }, + "pict" | "object" => { + match read_vml_picture(child, &ctx.drawing(), st.assets, st.report) { + Ok(Some(image)) => buffer.push(Inline::Image(image)), + Ok(None) => { + let fragment = st.raw(child, ctx); + buffer.push(Inline::Raw(fragment)); + } + Err(e) => st.report.warn(Warning::AssetIssue { + asset: ctx.part.to_string(), + why: e.to_string(), + }), + } + } + "footnoteReference" | "endnoteReference" => { + let id = child.attr_i64("id").unwrap_or(-1); + match ctx.footnotes.get(&id) { + Some(blocks) => { + st.report.stats.footnotes += 1; + // The reference sits *outside* the run's character + // style: `FootnoteReference` only makes the marker + // superscript, and DocMark draws its own marker. + flush!(); + out.push(Piece::inline(Inline::Footnote(blocks.clone()))); + } + None => st.report.warn(Warning::Degraded { + what: format!("footnote {id}"), + why: "referenced note not found in the package".into(), + }), + } + } + "footnoteRef" | "endnoteRef" | "separator" | "continuationSeparator" => {} + "fldChar" => { + flush!(); + match child.attr("fldCharType") { + Some("begin") => out.push(Piece::FieldBegin), + Some("separate") => out.push(Piece::FieldSeparate), + Some("end") => out.push(Piece::FieldEnd), + _ => {} + } + } + "instrText" => { + flush!(); + out.push(Piece::Instruction(child.text())); + } + name if IGNORED.contains(&name) => {} + _ => { + let fragment = st.raw(child, ctx); + buffer.push(Inline::Raw(fragment)); + } + } + } + flush!(); +} + +// -------------------------------------------------------------------------- +// Tables +// -------------------------------------------------------------------------- + +fn read_table(tbl: &Element, ctx: &Ctx<'_>, st: &mut State<'_>) -> Table { + let mut table = Table { + style: tbl + .path(&["tblPr", "tblStyle"]) + .and_then(|e| e.attr("val")) + .map(StyleId::new), + col_widths: tbl + .child("tblGrid") + .map(|grid| { + grid.children_named("gridCol") + .map(|c| Length::from_twips(c.attr_i64("w").unwrap_or(0))) + .collect() + }) + .unwrap_or_default(), + ..Default::default() + }; + + // `vMerge` continuations point back at the restart cell above them, so + // rowspans are resolved after the grid is known. + let mut open_merges: BTreeMap = BTreeMap::new(); + + for tr in tbl.children_named("tr") { + let mut row = TableRow { + is_header: tr.path(&["trPr", "tblHeader"]).is_some(), + ..Default::default() + }; + let mut grid_col = 0usize; + + for tc in tr.children_named("tc") { + let tcpr = tc.child("tcPr"); + let colspan = tcpr + .and_then(|p| p.child("gridSpan")) + .and_then(|e| e.attr_i64("val")) + .unwrap_or(1) + .clamp(1, 1024) as u16; + let vmerge = tcpr.and_then(|p| p.child("vMerge")); + let continues = vmerge.is_some_and(|e| !matches!(e.attr("val"), Some("restart"))); + + let mut cell = TableCell { + blocks: read_blocks(tc, ctx, st), + colspan, + rowspan: 1, + covered: false, + width: tcpr + .and_then(|p| p.child("tcW")) + .filter(|w| w.attr("type") != Some("auto")) + .and_then(|w| w.attr_i64("w")) + .map(Length::from_twips), + background: tcpr + .and_then(|p| p.child("shd")) + .and_then(|s| s.attr("fill")) + .and_then(hex_color), + }; + + if vmerge.is_some() { + if continues { + if let Some((row_index, cell_index)) = open_merges.get(&grid_col).copied() { + if let Some(target) = table + .rows + .get_mut(row_index) + .and_then(|r| r.cells.get_mut(cell_index)) + { + target.rowspan = target.rowspan.saturating_add(1); + } + } + cell.covered = true; + cell.blocks.clear(); + } else { + open_merges.insert(grid_col, (table.rows.len(), row.cells.len())); + } + } else { + open_merges.remove(&grid_col); + } + + grid_col += colspan as usize; + row.cells.push(cell); + } + table.rows.push(row); + } + + table.header_row = table.rows.first().is_some_and(|r| r.is_header); + table +} + +#[cfg(test)] +mod tests { + use super::*; + use docsai_model::MemoryAssetStore; + + #[derive(Default)] + struct Fixture { + package: Package, + styles: StyleCatalog, + numbering: Numbering, + footnotes: BTreeMap>, + } + + fn read(xml: &str, fixture: &Fixture) -> (Vec, ConversionReport) { + let root = Element::parse("word/document.xml", xml.as_bytes()).unwrap(); + let rels = Relationships::default(); + let ctx = Ctx { + package: &fixture.package, + rels: &rels, + part: "word/document.xml", + source: xml, + styles: &fixture.styles, + numbering: &fixture.numbering, + footnotes: &fixture.footnotes, + }; + let mut assets = MemoryAssetStore::new(); + let mut report = ConversionReport::new(); + let mut st = State { + assets: &mut assets, + report: &mut report, + raw_seq: 0, + revisions: 0, + }; + let blocks = read_blocks(&root, &ctx, &mut st); + (blocks, report) + } + + const NS: &str = r#"xmlns:w="urn:w" xmlns:r="urn:r""#; + + #[test] + fn reads_runs_and_their_formatting() { + let xml = format!( + r#" + normal + negrita + "# + ); + let (blocks, _) = read(&xml, &Fixture::default()); + let Block::Paragraph(p) = &blocks[0] else { + panic!("expected a paragraph, got {:?}", blocks[0]) + }; + assert_eq!(p.plain_text(), "normal negrita"); + assert!(matches!(p.content[1], Inline::Styled { .. })); + } + + #[test] + fn breaks_and_tabs_survive() { + let xml = format!( + r#"ab + "# + ); + let (blocks, _) = read(&xml, &Fixture::default()); + let Block::Paragraph(p) = &blocks[0] else { + unreachable!() + }; + assert!(p + .content + .iter() + .any(|i| matches!(i, Inline::Break(BreakKind::Line)))); + assert!(p + .content + .iter() + .any(|i| matches!(i, Inline::Break(BreakKind::Page)))); + assert!(p.plain_text().contains('\t')); + } + + #[test] + fn rebuilds_a_nested_list_from_flat_numbering() { + let mut fixture = Fixture::default(); + let numbering_xml = r#" + + + + + "#; + let mut discard = ConversionReport::new(); + fixture.numbering = super::super::numbering::read_numbering( + &Element::parse("n.xml", numbering_xml.as_bytes()).unwrap(), + &mut discard, + ); + + let item = |text: &str, lvl: u8| { + format!( + r#" + {text}"# + ) + }; + let xml = format!( + "{}{}{}{}", + item("uno", 0), + item("uno.a", 1), + item("uno.b", 1), + item("dos", 0) + ); + let (blocks, _) = read(&xml, &fixture); + assert_eq!(blocks.len(), 1, "one list, not four paragraphs"); + let Block::List(list) = &blocks[0] else { + panic!("expected a list") + }; + assert!(list.ordered); + assert_eq!(list.items.len(), 2); + let nested = list.items[0].blocks.iter().find_map(|b| match b { + Block::List(l) => Some(l), + _ => None, + }); + assert_eq!(nested.expect("nested list").items.len(), 2); + } + + #[test] + fn num_id_zero_is_not_a_list() { + let xml = format!( + r#" + x"# + ); + let (blocks, _) = read(&xml, &Fixture::default()); + assert!(matches!(blocks[0], Block::Paragraph(_))); + } + + #[test] + fn resolves_vertical_and_horizontal_merges() { + let xml = format!( + r#" + + + Region + Trimestres + + + Norte + 100 + 200 + + + + 150 + 250 + + "# + ); + let (blocks, _) = read(&xml, &Fixture::default()); + let Block::Table(table) = &blocks[0] else { + panic!("expected a table") + }; + assert_eq!(table.col_widths.len(), 3); + assert!(table.header_row); + assert_eq!(table.rows[0].cells[1].colspan, 2); + assert_eq!(table.rows[1].cells[0].rowspan, 2, "vMerge became a rowspan"); + assert!(table.rows[2].cells[0].covered); + assert_eq!(table.width(), 3); + } + + #[test] + fn assembles_complex_fields_from_their_run_sequence() { + let xml = format!( + r#" + + PAGE + + 7 + + "# + ); + let (blocks, _) = read(&xml, &Fixture::default()); + let Block::Paragraph(p) = &blocks[0] else { + unreachable!() + }; + match &p.content[0] { + Inline::Field { + kind, + cached, + instruction, + } => { + assert_eq!(*kind, FieldKind::Page); + assert_eq!(cached, "7"); + assert_eq!(instruction, "PAGE"); + } + other => panic!("expected a field, got {other:?}"), + } + } + + #[test] + fn simple_fields_keep_their_instruction() { + let xml = format!( + r#" + 01/01/2026"# + ); + let (blocks, _) = read(&xml, &Fixture::default()); + let Block::Paragraph(p) = &blocks[0] else { + unreachable!() + }; + let Inline::Field { + kind, + cached, + instruction, + } = &p.content[0] + else { + panic!("expected a field") + }; + assert_eq!(*kind, FieldKind::Date); + assert_eq!(cached, "01/01/2026"); + assert!(instruction.contains("dd/MM/yyyy")); + } + + #[test] + fn unknown_elements_become_raw_blocks_with_their_bytes() { + let xml = format!(r#""#); + let (blocks, report) = read(&xml, &Fixture::default()); + let Block::Raw(raw) = &blocks[0] else { + panic!("expected a raw block") + }; + assert_eq!(raw.format, "ooxml"); + assert!(raw.content.starts_with("")); + assert_eq!(report.raw_blocks_emitted, 1); + } + + #[test] + fn content_controls_keep_their_text_visible() { + let xml = format!( + r#" + dentro"# + ); + let (blocks, report) = read(&xml, &Fixture::default()); + assert_eq!(blocks.len(), 1); + let Block::Paragraph(p) = &blocks[0] else { + panic!("sdt content should be flattened to blocks") + }; + assert_eq!(p.plain_text(), "dentro"); + assert!(report + .warnings + .iter() + .any(|w| w.message().contains("w:sdt"))); + } + + #[test] + fn tracked_changes_are_taken_as_accepted() { + let xml = format!( + r#" + añadido + borrado + final + "# + ); + let (blocks, _) = read(&xml, &Fixture::default()); + let Block::Paragraph(p) = &blocks[0] else { + unreachable!() + }; + assert_eq!(p.plain_text(), "añadido final"); + } + + #[test] + fn headings_come_from_the_style_catalogue() { + let mut fixture = Fixture::default(); + let styles_xml = r#" + + "#; + fixture.styles = super::super::styles::read_styles( + &Element::parse("s.xml", styles_xml.as_bytes()).unwrap(), + ); + let xml = format!( + r#" + Titulo"# + ); + let (blocks, _) = read(&xml, &fixture); + let Block::Heading(h) = &blocks[0] else { + panic!("expected a heading") + }; + assert_eq!(h.level, 2); + assert_eq!(h.paragraph.format.style, Some(StyleId::new("Heading2"))); + } +} diff --git a/crates/docsai-office/src/docx/drawing.rs b/crates/docsai-office/src/docx/drawing.rs new file mode 100644 index 0000000..17ffbe5 --- /dev/null +++ b/crates/docsai-office/src/docx/drawing.rs @@ -0,0 +1,520 @@ +//! DrawingML (`w:drawing`) and legacy VML (`w:pict`) → the normalised image +//! model of arquitectura §3.1. +//! +//! This is where the spike (docs/spikes/R1) concluded the existing crates fall +//! short, so it is deliberately exhaustive: wrap mode and side, `behindDoc`, +//! crop, rotation, flips, alt text, title, object name, the hyperlink on the +//! image and linked-but-not-embedded sources are all mapped, and anything left +//! over becomes a typed warning rather than a silent loss. + +use docsai_model::assets::AssetStore; +use docsai_model::image::{ + AlignKeyword, Anchor, AxisPos, CropRect, Flip, HVPos, ImageGeometry, ImageRef, RelBase, + SimpleBorder, WrapMode, WrapSide, +}; +use docsai_model::report::{ConversionReport, Warning}; +use docsai_model::units::{Length, Size}; + +use super::format::hex_color; +use crate::error::ReadError; +use crate::package::{Package, Relationships}; +use crate::xml::Element; + +/// Sixty-thousandths of a degree per degree, the DrawingML rotation unit. +const ROT_UNITS_PER_DEGREE: f32 = 60_000.0; +/// Thousandths of a percent, the `a:srcRect` unit. +const CROP_UNITS_PER_PERCENT: f32 = 1_000.0; + +/// Context a drawing needs to resolve its media and report problems. +pub struct DrawingContext<'a> { + pub package: &'a Package, + pub rels: &'a Relationships, + /// Part the drawing lives in, for warning locations. + pub part: &'a str, +} + +/// Reads a `w:drawing` element. +/// +/// Returns `Ok(None)` when the drawing holds something that is not a picture +/// (a chart, SmartArt, a text box); the caller preserves those as raw-blocks. +pub fn read_drawing( + drawing: &Element, + ctx: &DrawingContext<'_>, + assets: &mut dyn AssetStore, + report: &mut ConversionReport, +) -> Result, ReadError> { + let (frame, anchored) = match (drawing.child("inline"), drawing.child("anchor")) { + (Some(inline), _) => (inline, false), + (None, Some(anchor)) => (anchor, true), + (None, None) => return Ok(None), + }; + + let Some(graphic_data) = frame.path(&["graphic", "graphicData"]) else { + return Ok(None); + }; + let Some(pic) = graphic_data.child("pic") else { + // Charts, diagrams and shapes live here too and have no image model. + report.warn(Warning::UnsupportedElement { + kind: graphic_data + .attr("uri") + .unwrap_or("a:graphicData") + .rsplit('/') + .next() + .unwrap_or("graphicData") + .to_string(), + location: ctx.part.to_string(), + action: "raw-block".into(), + }); + return Ok(None); + }; + + let display_size = frame + .child("extent") + .map(|e| { + Size::new( + Length::from_emu(e.attr_i64("cx").unwrap_or(0)), + Length::from_emu(e.attr_i64("cy").unwrap_or(0)), + ) + }) + .unwrap_or_default(); + + let mut geometry = ImageGeometry::inline(display_size); + read_shape_properties(pic.child("spPr"), &mut geometry, report, ctx.part); + read_crop(pic.path(&["blipFill", "srcRect"]), &mut geometry); + if anchored { + geometry.anchor = read_anchor(frame); + // `relativeHeight` is the z-order; it lives on the geometry, not on + // the anchor, because sheet-anchored images stack the same way. + geometry.z_index = frame.attr_i64("relativeHeight").map(|v| v as i32); + } + + let doc_pr = frame.child("docPr"); + let alt = doc_pr + .and_then(|e| e.attr("descr")) + .or_else(|| { + pic.path(&["nvPicPr", "cNvPr"]) + .and_then(|e| e.attr("descr")) + }) + .unwrap_or_default() + .to_string(); + + let blip = pic.path(&["blipFill", "blip"]); + let mut image = match resolve_media(blip, ctx, assets, report)? { + Some(mut image) => { + image.geometry = geometry; + image.alt = alt; + image + } + None => return Ok(None), + }; + + image.title = doc_pr.and_then(|e| e.attr("title")).map(str::to_string); + image.name = doc_pr + .and_then(|e| e.attr("name")) + .or_else(|| pic.path(&["nvPicPr", "cNvPr"]).and_then(|e| e.attr("name"))) + .filter(|n| !n.is_empty()) + .map(str::to_string); + image.link = doc_pr + .and_then(|e| e.child("hlinkClick")) + .and_then(|e| e.attr_qualified("r:id")) + .and_then(|id| ctx.rels.get(id)) + .map(|rel| rel.target.clone()); + + // Native dimensions come from the stored asset's own header. + if let Some(info) = assets.info(&image.asset) { + image.geometry.native_size_px = info.native_size_px; + } + + // DrawingML effects have no DocMark representation; flag them so the + // caller can attach a raw-block instead of losing them silently. + if pic + .child("spPr") + .is_some_and(|spr| spr.child("effectLst").is_some() || spr.child("scene3d").is_some()) + { + report.warn(Warning::ImageGeometryDegraded { + what: image.name.clone().unwrap_or_else(|| "picture".into()), + why: "DrawingML effects (shadow/bevel/3-D) have no DocMark model".into(), + }); + } + + Ok(Some(image)) +} + +fn read_shape_properties( + sppr: Option<&Element>, + geometry: &mut ImageGeometry, + report: &mut ConversionReport, + part: &str, +) { + let Some(sppr) = sppr else { return }; + + if let Some(xfrm) = sppr.child("xfrm") { + if let Some(rot) = xfrm.attr_i64("rot") { + geometry.rotation_deg = rot as f32 / ROT_UNITS_PER_DEGREE; + } + geometry.flip = Flip::from_flags( + xfrm.attr("flipH").is_some_and(is_true), + xfrm.attr("flipV").is_some_and(is_true), + ); + } + + if let Some(ln) = sppr.child("ln") { + let color = ln + .path(&["solidFill", "srgbClr"]) + .and_then(|c| c.attr("val")) + .and_then(hex_color) + .unwrap_or_else(|| "#000000".into()); + geometry.border = Some(SimpleBorder { + width: Length::from_emu(ln.attr_i64("w").unwrap_or(9525)), + style: ln + .child("prstDash") + .and_then(|d| d.attr("val")) + .map(dash_style) + .unwrap_or("solid") + .to_string(), + color, + }); + if ln.child("gradFill").is_some() || ln.child("pattFill").is_some() { + report.warn(Warning::ImageGeometryDegraded { + what: format!("image border in {part}"), + why: "only single-colour borders are modelled; the fill was simplified".into(), + }); + } + } + + // A non-rectangular preset geometry changes the visible shape. + if let Some(prst) = sppr.child("prstGeom").and_then(|g| g.attr("prst")) { + if prst != "rect" { + report.warn(Warning::ImageGeometryDegraded { + what: format!("image in {part}"), + why: format!("preset shape `{prst}` is rendered as a rectangle"), + }); + } + } +} + +fn read_crop(src_rect: Option<&Element>, geometry: &mut ImageGeometry) { + let Some(rect) = src_rect else { return }; + let side = |name: &str| rect.attr_i64(name).unwrap_or(0) as f32 / CROP_UNITS_PER_PERCENT; + let crop = CropRect { + left: side("l"), + top: side("t"), + right: side("r"), + bottom: side("b"), + }; + if !crop.is_empty() { + geometry.crop = Some(crop); + } +} + +fn read_anchor(anchor: &Element) -> Anchor { + let axis = |name: &str| -> (RelBase, AxisPos) { + let Some(position) = anchor.child(name) else { + return (RelBase::Paragraph, AxisPos::Offset(Length::ZERO)); + }; + let base = rel_base(position.attr("relativeFrom").unwrap_or("column")); + let value = match position.child("align").map(|a| a.text()) { + Some(keyword) => AxisPos::Align(align_keyword(keyword.trim())), + None => AxisPos::Offset(Length::from_emu( + position + .child("posOffset") + .and_then(|o| o.text().trim().parse::().ok()) + .unwrap_or(0), + )), + }; + (base, value) + }; + + let (relative_to_h, h) = axis("positionH"); + let (relative_to_v, v) = axis("positionV"); + let (wrap, wrap_side) = read_wrap(anchor); + + Anchor::Floating { + relative_to_h, + relative_to_v, + position: HVPos { h, v }, + wrap, + wrap_side, + behind_text: anchor.attr("behindDoc").is_some_and(is_true), + } +} + +fn read_wrap(anchor: &Element) -> (WrapMode, WrapSide) { + for child in anchor.children() { + let mode = match child.name.as_str() { + "wrapSquare" => WrapMode::Square, + "wrapTight" => WrapMode::Tight, + "wrapThrough" => WrapMode::Through, + "wrapTopAndBottom" => WrapMode::TopBottom, + "wrapNone" => WrapMode::None, + _ => continue, + }; + let side = match child.attr("wrapText") { + Some("left") => WrapSide::Left, + Some("right") => WrapSide::Right, + Some("largest") => WrapSide::Largest, + _ => WrapSide::Both, + }; + return (mode, side); + } + (WrapMode::None, WrapSide::Both) +} + +/// Resolves `a:blip` to a stored asset, honouring embedded and linked images. +fn resolve_media( + blip: Option<&Element>, + ctx: &DrawingContext<'_>, + assets: &mut dyn AssetStore, + report: &mut ConversionReport, +) -> Result, ReadError> { + let Some(blip) = blip else { return Ok(None) }; + + if let Some(embed) = blip.attr_qualified("r:embed") { + let Some(rel) = ctx.rels.get(embed) else { + report.warn(Warning::AssetIssue { + asset: embed.to_string(), + why: "relationship not found".into(), + }); + return Ok(None); + }; + let Some(bytes) = ctx.package.part(&rel.target) else { + report.warn(Warning::AssetIssue { + asset: rel.target.clone(), + why: "media part missing from the package".into(), + }); + return Ok(None); + }; + let id = assets.put(bytes)?; + report.stats.images += 1; + let mut image = ImageRef::new(id, ImageGeometry::inline(Size::default())); + warn_if_unrenderable(&image, assets, report); + image.alt.clear(); + return Ok(Some(image)); + } + + if let Some(link) = blip.attr_qualified("r:link") { + // Linked images are never fetched: doing so would turn a document into + // an outbound network request (arquitectura §3.2). + let url = ctx + .rels + .get(link) + .map(|r| r.target.clone()) + .unwrap_or_else(|| link.to_string()); + report.warn(Warning::ExternalImageNotFetched { url: url.clone() }); + let mut image = ImageRef::new( + docsai_model::AssetId::new(String::new()), + ImageGeometry::inline(Size::default()), + ); + image.external_src = Some(url); + return Ok(Some(image)); + } + + Ok(None) +} + +fn warn_if_unrenderable(image: &ImageRef, assets: &dyn AssetStore, report: &mut ConversionReport) { + let Some(info) = assets.info(&image.asset) else { + return; + }; + if matches!(info.content_type.as_str(), "image/x-emf" | "image/x-wmf") { + report.warn(Warning::Degraded { + what: info.file_name.clone(), + why: "WMF/EMF is preserved byte for byte but Markdown viewers cannot render it".into(), + }); + } else if info.content_type == "application/octet-stream" { + report.warn(Warning::AssetIssue { + asset: info.file_name.clone(), + why: "unrecognised media type; stored verbatim".into(), + }); + } +} + +/// Reads a legacy VML picture (`w:pict`), typical of documents that once were +/// `.doc`. Only what the normalised model covers is mapped; the rest is +/// reported as degraded (análisis §1.4). +pub fn read_vml_picture( + pict: &Element, + ctx: &DrawingContext<'_>, + assets: &mut dyn AssetStore, + report: &mut ConversionReport, +) -> Result, ReadError> { + let Some(shape) = pict.child("shape").or_else(|| pict.child("rect")) else { + return Ok(None); + }; + let Some(data) = shape.child("imagedata") else { + return Ok(None); + }; + let Some(rel_id) = data + .attr_qualified("r:id") + .or_else(|| data.attr_qualified("o:relid")) + else { + return Ok(None); + }; + let Some(rel) = ctx.rels.get(rel_id) else { + report.warn(Warning::AssetIssue { + asset: rel_id.to_string(), + why: "VML relationship not found".into(), + }); + return Ok(None); + }; + let Some(bytes) = ctx.package.part(&rel.target) else { + report.warn(Warning::AssetIssue { + asset: rel.target.clone(), + why: "media part missing from the package".into(), + }); + return Ok(None); + }; + + let style = VmlStyle::parse(shape.attr("style").unwrap_or_default()); + let mut geometry = ImageGeometry::inline(Size::new( + style.length("width").unwrap_or_default(), + style.length("height").unwrap_or_default(), + )); + geometry.rotation_deg = style.number("rotation").unwrap_or(0.0); + geometry.z_index = style.number("z-index").map(|z| z as i32); + + let floating = style.get("position").is_some_and(|p| p == "absolute"); + if floating { + let (wrap, wrap_side) = match shape.child("wrap") { + Some(w) => ( + match w.attr("type") { + Some("square") => WrapMode::Square, + Some("tight") => WrapMode::Tight, + Some("through") => WrapMode::Through, + Some("topAndBottom") => WrapMode::TopBottom, + _ => WrapMode::None, + }, + match w.attr("side") { + Some("left") => WrapSide::Left, + Some("right") => WrapSide::Right, + Some("largest") => WrapSide::Largest, + _ => WrapSide::Both, + }, + ), + None => (WrapMode::None, WrapSide::Both), + }; + geometry.anchor = Anchor::Floating { + relative_to_h: rel_base( + style + .get("mso-position-horizontal-relative") + .unwrap_or("column"), + ), + relative_to_v: rel_base( + style + .get("mso-position-vertical-relative") + .unwrap_or("paragraph"), + ), + position: HVPos { + h: AxisPos::Offset(style.length("margin-left").unwrap_or_default()), + v: AxisPos::Offset(style.length("margin-top").unwrap_or_default()), + }, + wrap, + wrap_side, + behind_text: geometry.z_index.is_some_and(|z| z < 0), + }; + } + + let id = assets.put(bytes)?; + report.stats.images += 1; + let mut image = ImageRef::new(id, geometry); + image.alt = shape.attr("alt").unwrap_or_default().to_string(); + image.title = data + .attr_qualified("o:title") + .filter(|t| !t.is_empty()) + .map(str::to_string); + image.name = shape.attr("id").map(str::to_string); + if let Some(info) = assets.info(&image.asset) { + image.geometry.native_size_px = info.native_size_px; + } + warn_if_unrenderable(&image, assets, report); + + report.warn(Warning::ImageGeometryDegraded { + what: image.name.clone().unwrap_or_else(|| "VML shape".into()), + why: "read from legacy VML; only the normalised attributes are mapped".into(), + }); + Ok(Some(image)) +} + +/// The `style` attribute of a VML shape, which is a CSS-like `k:v;` list. +struct VmlStyle { + entries: Vec<(String, String)>, +} + +impl VmlStyle { + fn parse(text: &str) -> VmlStyle { + let entries = text + .split(';') + .filter_map(|pair| pair.split_once(':')) + .map(|(k, v)| (k.trim().to_ascii_lowercase(), v.trim().to_string())) + .collect(); + VmlStyle { entries } + } + + fn get(&self, key: &str) -> Option<&str> { + self.entries + .iter() + .find(|(k, _)| k == key) + .map(|(_, v)| v.as_str()) + } + + fn number(&self, key: &str) -> Option { + self.get(key)? + .trim_end_matches(|c: char| c.is_alphabetic()) + .parse() + .ok() + } + + /// A CSS length. VML writes points by default; `in`, `cm`, `mm` and `px` + /// also appear in documents converted from `.doc`. + fn length(&self, key: &str) -> Option { + let raw = self.get(key)?.trim(); + let split = raw.find(|c: char| c.is_alphabetic()).unwrap_or(raw.len()); + let (number, unit) = raw.split_at(split); + let value: f64 = number.trim().parse().ok()?; + Some(match unit.trim() { + "in" => Length::from_inch(value), + "cm" => Length::from_cm(value), + "mm" => Length::from_mm(value), + "px" => Length::from_px(value), + "pc" => Length::from_pt(value * 12.0), + _ => Length::from_pt(value), + }) + } +} + +fn rel_base(value: &str) -> RelBase { + match value { + "page" => RelBase::Page, + "margin" | "leftMargin" | "rightMargin" | "topMargin" | "bottomMargin" | "insideMargin" + | "outsideMargin" => RelBase::Margin, + "paragraph" => RelBase::Paragraph, + "character" => RelBase::Character, + "line" => RelBase::Line, + _ => RelBase::Column, + } +} + +fn align_keyword(value: &str) -> AlignKeyword { + match value { + "center" => AlignKeyword::Center, + "right" => AlignKeyword::Right, + "inside" => AlignKeyword::Inside, + "outside" => AlignKeyword::Outside, + "top" => AlignKeyword::Top, + "bottom" => AlignKeyword::Bottom, + "middle" => AlignKeyword::Middle, + _ => AlignKeyword::Left, + } +} + +fn dash_style(value: &str) -> &'static str { + match value { + "solid" => "solid", + "dot" | "sysDot" => "dotted", + _ => "dashed", + } +} + +fn is_true(value: &str) -> bool { + matches!(value, "1" | "true" | "on") +} diff --git a/crates/docsai-office/src/docx/format.rs b/crates/docsai-office/src/docx/format.rs new file mode 100644 index 0000000..4e1dc98 --- /dev/null +++ b/crates/docsai-office/src/docx/format.rs @@ -0,0 +1,246 @@ +//! `w:rPr` and `w:pPr` → the IR's delta types. +//! +//! Nothing here flattens: a property absent from the XML stays `None` in the +//! IR so that the style it inherits from keeps supplying it. + +use docsai_model::style::{Align, FontProps, LineHeight, ParaProps, Underline, VertAlign}; +use docsai_model::units::Length; + +use crate::xml::Element; + +/// Reads a `w:rPr` into character-level deltas. +pub fn run_props(rpr: &Element) -> FontProps { + let mut font = FontProps::default(); + + if let Some(fonts) = rpr.child("rFonts") { + font.name = fonts + .attr("ascii") + .or_else(|| fonts.attr("hAnsi")) + .or_else(|| fonts.attr("cs")) + .map(str::to_string); + } + if let Some(sz) = rpr.child("sz").and_then(|e| e.attr_i64("val")) { + font.size = Some(Length::from_half_points(sz)); + } + if let Some(b) = rpr.child("b") { + font.bold = Some(b.ooxml_flag()); + } + if let Some(i) = rpr.child("i") { + font.italic = Some(i.ooxml_flag()); + } + if let Some(s) = rpr.child("strike") { + font.strike = Some(s.ooxml_flag()); + } + if let Some(u) = rpr.child("u") { + font.underline = Some(underline(u.attr("val").unwrap_or("single"))); + } + if let Some(color) = rpr.child("color").and_then(|e| e.attr("val")) { + // `auto` means "let the renderer decide": not a colour, so not a delta. + if let Some(hex) = hex_color(color) { + font.color = Some(hex); + } + } + if let Some(h) = rpr.child("highlight").and_then(|e| e.attr("val")) { + if h != "none" { + font.highlight = Some(h.to_string()); + } + } + if let Some(v) = rpr.child("vertAlign").and_then(|e| e.attr("val")) { + font.vert_align = Some(match v { + "superscript" => VertAlign::Superscript, + "subscript" => VertAlign::Subscript, + _ => VertAlign::Baseline, + }); + } + if let Some(c) = rpr.child("smallCaps") { + font.small_caps = Some(c.ooxml_flag()); + } + if let Some(c) = rpr.child("caps") { + font.caps = Some(c.ooxml_flag()); + } + font +} + +/// Reads a `w:pPr` into paragraph-level deltas. +/// +/// `w:pStyle` and `w:numPr` are *not* read here: they are references, not +/// formatting, and the body reader handles them. +pub fn paragraph_props(ppr: &Element) -> ParaProps { + let mut para = ParaProps::default(); + + if let Some(jc) = ppr.child("jc").and_then(|e| e.attr("val")) { + para.align = Some(match jc { + "center" => Align::Center, + "right" | "end" => Align::Right, + "both" | "distribute" | "justify" => Align::Justify, + _ => Align::Left, + }); + } + if let Some(ind) = ppr.child("ind") { + para.indent_left = twips_attr(ind, &["left", "start"]); + para.indent_right = twips_attr(ind, &["right", "end"]); + para.indent_first_line = twips_attr(ind, &["firstLine"]); + para.indent_hanging = twips_attr(ind, &["hanging"]); + } + if let Some(spacing) = ppr.child("spacing") { + para.space_before = twips_attr(spacing, &["before"]); + para.space_after = twips_attr(spacing, &["after"]); + if let Some(line) = spacing.attr_i64("line") { + para.line_height = Some(match spacing.attr("lineRule") { + // `auto` counts in 240ths of a line; the IR stores 1000ths. + Some("exact") => LineHeight::Exact(Length::from_twips(line)), + Some("atLeast") => LineHeight::AtLeast(Length::from_twips(line)), + _ => LineHeight::Multiple((line * 1000 / 240) as i32), + }); + } + } + if let Some(k) = ppr.child("keepNext") { + para.keep_with_next = Some(k.ooxml_flag()); + } + if let Some(b) = ppr.child("pageBreakBefore") { + para.page_break_before = Some(b.ooxml_flag()); + } + if let Some(shd) = ppr.child("shd") { + if let Some(fill) = shd.attr("fill").and_then(hex_color) { + para.background = Some(fill); + } + } + if let Some(lvl) = ppr.child("outlineLvl").and_then(|e| e.attr_i64("val")) { + if (0..=8).contains(&lvl) { + para.outline_level = Some(lvl as u8); + } + } + para +} + +fn twips_attr(element: &Element, names: &[&str]) -> Option { + names + .iter() + .find_map(|n| element.attr_i64(n)) + .map(Length::from_twips) +} + +/// Normalises an OOXML colour to `#RRGGBB`, or `None` for `auto`/garbage. +pub fn hex_color(value: &str) -> Option { + let value = value.trim().trim_start_matches('#'); + if value.eq_ignore_ascii_case("auto") { + return None; + } + // Word writes both RRGGBB and (in DrawingML themes) AARRGGBB. + let hex = match value.len() { + 6 => value, + 8 => &value[2..], + _ => return None, + }; + if !hex.bytes().all(|b| b.is_ascii_hexdigit()) { + return None; + } + Some(format!("#{}", hex.to_ascii_uppercase())) +} + +fn underline(value: &str) -> Underline { + match value { + "none" => Underline::None, + "double" => Underline::Double, + "thick" => Underline::Thick, + "dotted" | "dottedHeavy" => Underline::Dotted, + "dash" | "dashed" | "dashLong" | "dashDotHeavy" => Underline::Dashed, + "wave" | "wavyHeavy" | "wavyDouble" => Underline::Wave, + _ => Underline::Single, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn parse(xml: &str) -> Element { + Element::parse("t.xml", xml.as_bytes()).unwrap() + } + + #[test] + fn reads_the_inline_formatting_of_the_spec_table() { + let rpr = parse( + r#" + + + + + "#, + ); + let font = run_props(&rpr); + assert_eq!(font.name.as_deref(), Some("Arial")); + assert_eq!(font.size, Some(Length::from_pt(14.0))); + assert_eq!(font.bold, Some(true)); + assert_eq!(font.italic, Some(true)); + assert_eq!(font.strike, Some(true)); + assert_eq!(font.underline, Some(Underline::Double)); + assert_eq!(font.color.as_deref(), Some("#FF0000")); + assert_eq!(font.highlight.as_deref(), Some("yellow")); + assert_eq!(font.vert_align, Some(VertAlign::Superscript)); + } + + #[test] + fn explicit_off_is_a_delta_not_an_absence() { + let rpr = parse(r#""#); + assert_eq!( + run_props(&rpr).bold, + Some(false), + "turning bold off must override the style" + ); + let empty = parse(r#""#); + assert_eq!(run_props(&empty).bold, None, "absent means inherit"); + } + + #[test] + fn auto_colour_is_not_a_colour() { + let rpr = parse(r#""#); + assert_eq!(run_props(&rpr).color, None); + assert_eq!(hex_color("2e74b5").as_deref(), Some("#2E74B5")); + assert_eq!(hex_color("FF2E74B5").as_deref(), Some("#2E74B5")); + assert_eq!(hex_color("zzz"), None); + } + + #[test] + fn reads_paragraph_geometry() { + let ppr = parse( + r#" + + + + + + "#, + ); + let para = paragraph_props(&ppr); + assert_eq!(para.align, Some(Align::Center)); + assert_eq!(para.indent_left, Some(Length::from_twips(720))); + assert_eq!(para.indent_first_line, Some(Length::from_twips(360))); + assert_eq!(para.indent_right, Some(Length::from_twips(240))); + assert_eq!(para.space_before, Some(Length::from_twips(240))); + assert_eq!(para.line_height, Some(LineHeight::Multiple(1500))); + assert_eq!(para.keep_with_next, Some(true)); + assert_eq!(para.background.as_deref(), Some("#F2F2F2")); + assert_eq!(para.outline_level, Some(1)); + } + + #[test] + fn exact_line_spacing_keeps_its_unit() { + let ppr = + parse(r#""#); + assert_eq!( + paragraph_props(&ppr).line_height, + Some(LineHeight::Exact(Length::from_twips(240))) + ); + } + + #[test] + fn nonsense_values_are_ignored_rather_than_fatal() { + let ppr = parse( + r#""#, + ); + let para = paragraph_props(&ppr); + assert_eq!(para.outline_level, None); + assert_eq!(para.indent_left, None); + } +} diff --git a/crates/docsai-office/src/docx/mod.rs b/crates/docsai-office/src/docx/mod.rs new file mode 100644 index 0000000..906edc2 --- /dev/null +++ b/crates/docsai-office/src/docx/mod.rs @@ -0,0 +1,469 @@ +//! The `.docx` reader (Fase 1). +//! +//! Built directly on `zip` + `quick-xml` after the R1 spike +//! (`docs/spikes/R1-estrategia-docx.md`): one pass over `document.xml` that +//! produces the IR and preserves everything it does not recognise. + +mod body; +mod drawing; +mod format; +mod numbering; +mod styles; + +use std::collections::BTreeMap; +use std::io::{Read, Seek}; + +use docsai_model::assets::AssetStore; +use docsai_model::report::{ConversionReport, Warning}; +use docsai_model::text::{ + Block, DocumentMeta, HeaderFooter, HeaderScope, Margins, Orientation, PageGeometry, Section, + TextDocument, +}; +use docsai_model::units::{Length, Size}; +use docsai_model::Document; + +use crate::error::ReadError; +use crate::package::{Package, Relationships}; +use crate::xml::Element; + +use body::{read_blocks, read_blocks_of, Ctx, State}; + +pub(crate) const DOCUMENT_PART: &str = "word/document.xml"; + +/// Reads a `.docx` into the IR. +/// +/// Media are stored in `assets`; everything that could not be represented +/// faithfully is reported in the returned [`ConversionReport`]. +pub fn read( + reader: R, + assets: &mut dyn AssetStore, +) -> Result<(Document, ConversionReport), ReadError> { + let package = Package::open(reader)?; + read_package(&package, assets) +} + +pub(crate) fn read_package( + package: &Package, + assets: &mut dyn AssetStore, +) -> Result<(Document, ConversionReport), ReadError> { + if !package.has_part(DOCUMENT_PART) { + return Err(ReadError::MissingPart(DOCUMENT_PART.into())); + } + let mut report = ConversionReport::new(); + + if package.part_names().any(|n| n.ends_with("vbaProject.bin")) { + // Macros are ignored always and by design (AGENTS.md §7, plan Fase 8). + report.warn(Warning::MacrosIgnored { + part: "word/vbaProject.bin".into(), + }); + } + + let meta = read_meta(package); + let styles = match package.optional_xml("word/styles.xml")? { + Some(root) => styles::read_styles(&root), + None => Default::default(), + }; + report.stats.styles = styles.styles.len() as u32; + + let numbering = match package.optional_xml("word/numbering.xml")? { + Some(root) => numbering::read_numbering(&root, &mut report), + None => Default::default(), + }; + + let document_rels = package.relationships(DOCUMENT_PART); + let mut state = State { + assets, + report: &mut report, + raw_seq: 0, + revisions: 0, + }; + + // Footnotes first: the body needs their blocks to inline them, and they + // themselves cannot contain footnote references. + let footnotes = read_footnotes(package, &styles, &numbering, &document_rels, &mut state)?; + + let source = package.text(DOCUMENT_PART)?; + let root = Element::parse(DOCUMENT_PART, source.as_bytes())?; + let doc_body = root.child("body").ok_or_else(|| ReadError::WrongShape { + part: DOCUMENT_PART.into(), + expected: "WordprocessingML document".into(), + })?; + + let ctx = Ctx { + package, + rels: &document_rels, + part: DOCUMENT_PART, + source, + styles: &styles, + numbering: &numbering, + footnotes: &footnotes, + }; + + let sections = read_sections(doc_body, &ctx, &mut state, package, &styles, &numbering)?; + let revisions = state.revisions; + if revisions > 0 { + report.warn(Warning::RevisionsAccepted { count: revisions }); + } + + let document = Document::Text(TextDocument { + meta, + styles, + list_defs: numbering.catalog, + sections, + }); + Ok((document, report)) +} + +// -------------------------------------------------------------------------- +// Sections +// -------------------------------------------------------------------------- + +/// Splits the body at every `w:sectPr` and reads each section. +/// +/// A section ends at the paragraph whose `pPr` carries a `sectPr`; the body's +/// own trailing `sectPr` closes the last one. +fn read_sections( + doc_body: &Element, + ctx: &Ctx<'_>, + state: &mut State<'_>, + package: &Package, + styles: &docsai_model::StyleCatalog, + numbering: &numbering::Numbering, +) -> Result, ReadError> { + let mut sections = Vec::new(); + let mut pending: Vec<&Element> = Vec::new(); + + for child in doc_body.children() { + if child.name == "sectPr" { + continue; // the trailing one, handled below + } + pending.push(child); + if let Some(sect_pr) = child.path(&["pPr", "sectPr"]) { + let blocks = read_blocks_of(pending.drain(..), ctx, state); + sections.push(build_section( + sect_pr, blocks, ctx, state, package, styles, numbering, + )?); + } + } + + let trailing = doc_body.child("sectPr"); + let blocks = read_blocks_of(pending, ctx, state); + if !blocks.is_empty() || sections.is_empty() { + let section = match trailing { + Some(sect_pr) => { + build_section(sect_pr, blocks, ctx, state, package, styles, numbering)? + } + None => Section { + blocks, + ..Default::default() + }, + }; + sections.push(section); + } + Ok(sections) +} + +fn build_section( + sect_pr: &Element, + blocks: Vec, + ctx: &Ctx<'_>, + state: &mut State<'_>, + package: &Package, + styles: &docsai_model::StyleCatalog, + numbering: &numbering::Numbering, +) -> Result { + let mut section = Section { + page: read_page_geometry(sect_pr), + blocks, + ..Default::default() + }; + + for (element, scope) in sect_pr + .children_named("headerReference") + .map(|e| (e, "header")) + .chain( + sect_pr + .children_named("footerReference") + .map(|e| (e, "footer")), + ) + { + let Some(rel_id) = element.attr_qualified("r:id") else { + continue; + }; + let Some(rel) = ctx.rels.get(rel_id) else { + continue; + }; + let Some(source) = package.part(&rel.target) else { + continue; + }; + let Ok(source) = std::str::from_utf8(source) else { + continue; + }; + let root = Element::parse(&rel.target, source.as_bytes())?; + let part_rels = package.relationships(&rel.target); + let inner = Ctx { + package, + rels: &part_rels, + part: &rel.target, + source, + styles, + numbering, + footnotes: ctx.footnotes, + }; + let header = HeaderFooter { + scope: match element.attr("type") { + Some("first") => HeaderScope::First, + Some("even") => HeaderScope::Even, + _ => HeaderScope::Default, + }, + blocks: read_blocks(&root, &inner, state), + }; + if scope == "header" { + section.headers.push(header); + } else { + section.footers.push(header); + } + } + + section.headers.sort_by_key(|h| h.scope); + section.footers.sort_by_key(|h| h.scope); + Ok(section) +} + +fn read_page_geometry(sect_pr: &Element) -> PageGeometry { + let mut page = PageGeometry { + columns: 1, + ..Default::default() + }; + if let Some(sz) = sect_pr.child("pgSz") { + page.size = Size::new( + Length::from_twips(sz.attr_i64("w").unwrap_or(0)), + Length::from_twips(sz.attr_i64("h").unwrap_or(0)), + ); + page.orientation = match sz.attr("orient") { + Some("landscape") => Orientation::Landscape, + // Word omits `orient` and simply swaps the dimensions. + _ if page.size.width > page.size.height => Orientation::Landscape, + _ => Orientation::Portrait, + }; + } + if let Some(mar) = sect_pr.child("pgMar") { + let twips = |name: &str| Length::from_twips(mar.attr_i64(name).unwrap_or(0)); + page.margins = Margins { + top: twips("top"), + right: twips("right"), + bottom: twips("bottom"), + left: twips("left"), + header: twips("header"), + footer: twips("footer"), + }; + } + if let Some(cols) = sect_pr.child("cols").and_then(|c| c.attr_i64("num")) { + page.columns = cols.clamp(1, 64) as u16; + } + page.title_page = sect_pr.child("titlePg").is_some_and(|e| e.ooxml_flag()); + page +} + +// -------------------------------------------------------------------------- +// Footnotes and document properties +// -------------------------------------------------------------------------- + +fn read_footnotes( + package: &Package, + styles: &docsai_model::StyleCatalog, + numbering: &numbering::Numbering, + document_rels: &Relationships, + state: &mut State<'_>, +) -> Result>, ReadError> { + let part = match document_rels.first_of_kind("footnotes") { + Some(rel) => rel.target.clone(), + None => "word/footnotes.xml".to_string(), + }; + let Some(bytes) = package.part(&part) else { + return Ok(BTreeMap::new()); + }; + let Ok(source) = std::str::from_utf8(bytes) else { + return Ok(BTreeMap::new()); + }; + let root = Element::parse(&part, source.as_bytes())?; + let rels = package.relationships(&part); + let empty = BTreeMap::new(); + let ctx = Ctx { + package, + rels: &rels, + part: &part, + source, + styles, + numbering, + footnotes: &empty, + }; + + let mut out = BTreeMap::new(); + for note in root.children_named("footnote") { + // The separator pseudo-notes are layout, not content. + if matches!( + note.attr("type"), + Some("separator") | Some("continuationSeparator") + ) { + continue; + } + let Some(id) = note.attr_i64("id") else { + continue; + }; + out.insert(id, read_blocks(note, &ctx, state)); + } + Ok(out) +} + +fn read_meta(package: &Package) -> DocumentMeta { + let mut meta = DocumentMeta::default(); + + if let Ok(Some(core)) = package.optional_xml("docProps/core.xml") { + let text = |name: &str| core.child(name).map(|e| e.text()).filter(|t| !t.is_empty()); + meta.title = text("title"); + meta.author = text("creator"); + meta.last_modified_by = text("lastModifiedBy"); + meta.created = text("created"); + meta.modified = text("modified"); + meta.language = text("language"); + meta.subject = text("subject"); + meta.keywords = text("keywords"); + meta.description = text("description"); + } + + if let Ok(Some(app)) = package.optional_xml("docProps/app.xml") { + meta.application = app + .child("Application") + .map(|e| e.text()) + .filter(|t| !t.is_empty()); + } + + if let Ok(Some(custom)) = package.optional_xml("docProps/custom.xml") { + for property in custom.children_named("property") { + let Some(name) = property.attr("name") else { + continue; + }; + let value = property.children().map(|v| v.text()).collect::(); + meta.custom.insert(name.to_string(), value); + } + } + + meta +} + +#[cfg(test)] +mod tests { + use super::*; + use docsai_model::MemoryAssetStore; + + fn read_fixture(name: &str) -> (Document, ConversionReport) { + let path = format!( + "{}/../../corpus/docx/{name}.docx", + env!("CARGO_MANIFEST_DIR") + ); + let file = std::fs::File::open(&path).unwrap_or_else(|e| panic!("{path}: {e}")); + let mut assets = MemoryAssetStore::new(); + read(file, &mut assets).unwrap_or_else(|e| panic!("{name}: {e}")) + } + + fn text_doc(doc: &Document) -> &TextDocument { + match doc { + Document::Text(d) => d, + Document::Workbook(_) => panic!("expected a text document"), + } + } + + #[test] + fn reads_page_geometry_and_document_properties() { + let (doc, _) = read_fixture("basic-text"); + let doc = text_doc(&doc); + assert_eq!(doc.meta.title.as_deref(), Some("Texto basico")); + assert_eq!(doc.meta.author.as_deref(), Some("docsai corpus")); + assert_eq!(doc.meta.language.as_deref(), Some("es-ES")); + let page = doc.sections[0].page; + assert_eq!(page.paper_name(), Some("A4")); + assert_eq!(page.orientation, Orientation::Portrait); + assert_eq!(page.margins.top, Length::from_twips(1417)); + } + + #[test] + fn reads_custom_properties() { + let (doc, _) = read_fixture("custom-styles"); + let doc = text_doc(&doc); + assert_eq!( + doc.meta.custom.get("Departamento").map(String::as_str), + Some("Ventas") + ); + assert_eq!( + doc.meta.custom.get("Revision").map(String::as_str), + Some("3") + ); + } + + #[test] + fn resolves_headers_footers_and_section_settings() { + let (doc, _) = read_fixture("headers-footers"); + let doc = text_doc(&doc); + let section = &doc.sections[0]; + assert_eq!(section.page.columns, 2); + assert!(section.page.title_page); + assert_eq!(section.headers.len(), 2, "default and first-page headers"); + assert_eq!(section.headers[0].scope, HeaderScope::Default); + assert_eq!(section.headers[1].scope, HeaderScope::First); + assert_eq!(section.footers.len(), 1); + + let footer_text: String = section.footers[0] + .blocks + .iter() + .filter_map(|b| match b { + Block::Paragraph(p) => Some(p.plain_text()), + _ => None, + }) + .collect(); + assert!(footer_text.contains("Pagina 1 de 3"), "got {footer_text}"); + } + + #[test] + fn inlines_footnote_content_at_the_reference() { + use docsai_model::text::Inline; + let (doc, report) = read_fixture("footnotes"); + let doc = text_doc(&doc); + // Footnote references carry a character style, so the inline sits + // inside a `Styled` wrapper: collect recursively. + fn collect<'a>(inlines: &'a [Inline], out: &mut Vec<&'a Vec>) { + for inline in inlines { + match inline { + Inline::Footnote(blocks) => out.push(blocks), + Inline::Styled { content, .. } | Inline::Link { content, .. } => { + collect(content, out) + } + _ => {} + } + } + } + let mut notes = Vec::new(); + for block in doc.blocks() { + if let Block::Paragraph(p) = block { + collect(&p.content, &mut notes); + } + } + assert_eq!(notes.len(), 2); + assert_eq!(report.stats.footnotes, 2); + let first = match ¬es[0][0] { + Block::Paragraph(p) => p.plain_text(), + other => panic!("{other:?}"), + }; + assert!(first.contains("Primera nota al pie"), "got {first}"); + } + + #[test] + fn a_missing_document_part_is_an_error() { + let package = Package::default(); + let mut assets = MemoryAssetStore::new(); + assert!(matches!( + read_package(&package, &mut assets), + Err(ReadError::MissingPart(_)) + )); + } +} diff --git a/crates/docsai-office/src/docx/numbering.rs b/crates/docsai-office/src/docx/numbering.rs new file mode 100644 index 0000000..df0f8f3 --- /dev/null +++ b/crates/docsai-office/src/docx/numbering.rs @@ -0,0 +1,198 @@ +//! `word/numbering.xml` → [`ListCatalog`]. +//! +//! OOXML keeps two levels of indirection: a paragraph names a `numId`, that +//! `w:num` points at an `abstractNumId`, and the abstract definition holds the +//! levels. The IR flattens the indirection into one definition per `numId`, +//! named `L`, which is what the DocMark front matter shows. + +use docsai_model::list::{ListCatalog, ListDef, ListId, ListLevel, NumFormat}; +use docsai_model::report::{ConversionReport, Warning}; +use docsai_model::units::Length; +use std::collections::BTreeMap; + +use crate::xml::Element; + +/// The list definitions of a document, plus the `numId` → definition mapping. +#[derive(Debug, Default)] +pub struct Numbering { + pub catalog: ListCatalog, + /// `numId` as written in `w:numPr`, mapped to its catalogue entry. + ids: BTreeMap, +} + +impl Numbering { + /// The catalogue id for a `w:numId` value. + pub fn list_id(&self, num_id: i64) -> Option<&ListId> { + self.ids.get(&num_id) + } + + /// Whether a `(numId, ilvl)` pair renders as an ordered list. + pub fn is_ordered(&self, num_id: i64, ilvl: usize) -> bool { + self.list_id(num_id) + .and_then(|id| self.catalog.get(id)) + .is_none_or(|def| def.is_ordered_at(ilvl)) + } +} + +/// Reads `numbering.xml`. +pub fn read_numbering(root: &Element, report: &mut ConversionReport) -> Numbering { + let mut abstracts: BTreeMap = BTreeMap::new(); + for element in root.children_named("abstractNum") { + let Some(id) = element.attr_i64("abstractNumId") else { + continue; + }; + abstracts.insert(id, read_definition(element)); + } + + let mut numbering = Numbering::default(); + for element in root.children_named("num") { + let Some(num_id) = element.attr_i64("numId") else { + continue; + }; + let abstract_id = element + .child("abstractNumId") + .and_then(|e| e.attr_i64("val")); + let Some(def) = abstract_id.and_then(|id| abstracts.get(&id)).cloned() else { + continue; + }; + if element.child("lvlOverride").is_some() { + report.warn(Warning::Degraded { + what: format!("list {num_id}"), + why: "level overrides (w:lvlOverride) are not modelled in v1".into(), + }); + } + let list_id = ListId::new(format!("L{num_id}")); + numbering.catalog.insert(list_id.clone(), def); + numbering.ids.insert(num_id, list_id); + } + numbering +} + +fn read_definition(element: &Element) -> ListDef { + let mut levels: Vec<(usize, ListLevel)> = Vec::new(); + for lvl in element.children_named("lvl") { + let index = lvl.attr_i64("ilvl").unwrap_or(0).max(0) as usize; + let format = lvl + .child("numFmt") + .and_then(|e| e.attr("val")) + .map(NumFormat::from_ooxml) + .unwrap_or(NumFormat::Decimal); + let mut level = ListLevel::new( + format, + lvl.child("lvlText") + .and_then(|e| e.attr("val")) + .unwrap_or(""), + ); + level.start = lvl + .child("start") + .and_then(|e| e.attr_i64("val")) + .map(|v| v as i32); + if let Some(ind) = lvl.path(&["pPr", "ind"]) { + level.indent = ind + .attr_i64("left") + .or_else(|| ind.attr_i64("start")) + .map(Length::from_twips); + level.hanging = ind.attr_i64("hanging").map(Length::from_twips); + } + levels.push((index, level)); + } + + // Levels may be declared out of order or with gaps; fill so that indexing + // by `ilvl` is always safe. + let depth = levels.iter().map(|(i, _)| *i + 1).max().unwrap_or(0); + let mut ordered = vec![ListLevel::new(NumFormat::Decimal, ""); depth]; + for (index, level) in levels { + ordered[index] = level; + } + ListDef { levels: ordered } +} + +#[cfg(test)] +mod tests { + use super::*; + + const NUMBERING: &str = r#" + + + + + + + + + + + "#; + + fn numbering() -> (Numbering, ConversionReport) { + let mut report = ConversionReport::new(); + let root = Element::parse("numbering.xml", NUMBERING.as_bytes()).unwrap(); + (read_numbering(&root, &mut report), report) + } + + #[test] + fn maps_num_ids_through_to_their_levels() { + let (n, _) = numbering(); + let id = n.list_id(1).unwrap().clone(); + assert_eq!(id, ListId::new("L1")); + let def = n.catalog.get(&id).unwrap(); + assert_eq!(def.levels.len(), 2); + assert_eq!(def.levels[0].format, NumFormat::Decimal); + assert_eq!(def.levels[0].text, "%1."); + assert_eq!(def.levels[0].indent, Some(Length::from_twips(720))); + assert_eq!(def.levels[0].hanging, Some(Length::from_twips(360))); + assert_eq!(def.levels[1].format, NumFormat::LowerLetter); + } + + #[test] + fn bullets_and_numbers_are_distinguished_per_level() { + let (n, _) = numbering(); + assert!(n.is_ordered(1, 0)); + assert!(!n.is_ordered(2, 0)); + assert!(n.is_ordered(99, 0), "unknown lists default to ordered"); + } + + #[test] + fn gaps_in_ilvl_do_not_shift_the_levels() { + let xml = r#" + + + + + "#; + let mut report = ConversionReport::new(); + let n = read_numbering( + &Element::parse("n.xml", xml.as_bytes()).unwrap(), + &mut report, + ); + let def = n.catalog.get(n.list_id(7).unwrap()).unwrap(); + assert_eq!(def.levels.len(), 3); + assert_eq!(def.levels[2].format, NumFormat::Bullet); + } + + #[test] + fn level_overrides_are_reported_not_silently_dropped() { + let xml = r#" + + + + "#; + let mut report = ConversionReport::new(); + read_numbering( + &Element::parse("n.xml", xml.as_bytes()).unwrap(), + &mut report, + ); + assert!(matches!(report.warnings[0], Warning::Degraded { .. })); + } + + #[test] + fn a_num_pointing_nowhere_is_skipped_not_fatal() { + let xml = r#""#; + let mut report = ConversionReport::new(); + let n = read_numbering( + &Element::parse("n.xml", xml.as_bytes()).unwrap(), + &mut report, + ); + assert!(n.list_id(1).is_none()); + } +} diff --git a/crates/docsai-office/src/docx/styles.rs b/crates/docsai-office/src/docx/styles.rs new file mode 100644 index 0000000..be6c7d2 --- /dev/null +++ b/crates/docsai-office/src/docx/styles.rs @@ -0,0 +1,170 @@ +//! `word/styles.xml` → [`StyleCatalog`]. + +use docsai_model::style::{Style, StyleCatalog, StyleId, StyleType}; + +use super::format::{paragraph_props, run_props}; +use crate::xml::Element; + +/// Reads the style catalogue, including `w:docDefaults`. +pub fn read_styles(root: &Element) -> StyleCatalog { + let mut catalog = StyleCatalog::default(); + + if let Some(defaults) = root.child("docDefaults") { + if let Some(rpr) = defaults.path(&["rPrDefault", "rPr"]) { + catalog.defaults.font = run_props(rpr); + } + if let Some(ppr) = defaults.path(&["pPrDefault", "pPr"]) { + catalog.defaults.paragraph = paragraph_props(ppr); + } + } + + for element in root.children_named("style") { + let Some(id) = element.attr("styleId") else { + continue; + }; + let Some(style_type) = style_type(element.attr("type")) else { + continue; + }; + let mut style = Style::new(id, style_type); + style.name = element + .path(&["name"]) + .and_then(|e| e.attr("val")) + .unwrap_or(id) + .to_string(); + style.based_on = element + .child("basedOn") + .and_then(|e| e.attr("val")) + .map(StyleId::new); + style.next = element + .child("next") + .and_then(|e| e.attr("val")) + .map(StyleId::new); + style.is_default = element + .attr("default") + .is_some_and(|v| v != "0" && v != "false"); + if let Some(rpr) = element.child("rPr") { + style.font = run_props(rpr); + } + if let Some(ppr) = element.child("pPr") { + style.paragraph = paragraph_props(ppr); + } + catalog.insert(style); + } + + catalog +} + +fn style_type(value: Option<&str>) -> Option { + match value { + Some("paragraph") | None => Some(StyleType::Paragraph), + Some("character") => Some(StyleType::Character), + Some("table") => Some(StyleType::Table), + Some("numbering") => Some(StyleType::Numbering), + Some(_) => None, + } +} + +/// The heading depth a paragraph style implies, if any. +/// +/// Word marks headings two ways and documents in the wild use both: the +/// outline level of the style's `pPr`, and the conventional style name +/// (`heading 1`) or id (`Heading1`). +pub fn heading_level(catalog: &StyleCatalog, id: &StyleId) -> Option { + let style = catalog.get(id)?; + // The outline level is inherited, so resolve the chain rather than reading + // this style's own `pPr` only. + if let Some(level) = catalog.resolve(Some(id)).paragraph.outline_level { + if level <= 8 { + return Some(level + 1); + } + } + for candidate in [style.name.as_str(), id.as_str()] { + let lowercase = candidate.to_ascii_lowercase(); + let Some(rest) = lowercase.strip_prefix("heading") else { + continue; + }; + if let Ok(level) = rest.trim().parse::() { + if (1..=9).contains(&level) { + return Some(level); + } + } + } + None +} + +#[cfg(test)] +mod tests { + use super::*; + use docsai_model::units::Length; + + const STYLES: &str = r#" + + + + + + + + + + + + + + "#; + + fn catalog() -> StyleCatalog { + read_styles(&Element::parse("styles.xml", STYLES.as_bytes()).unwrap()) + } + + #[test] + fn reads_doc_defaults_as_the_bottom_of_the_cascade() { + let cat = catalog(); + assert_eq!(cat.defaults.font.name.as_deref(), Some("Calibri")); + assert_eq!(cat.defaults.font.size, Some(Length::from_pt(11.0))); + assert_eq!( + cat.defaults.paragraph.space_after, + Some(Length::from_twips(160)) + ); + } + + #[test] + fn reads_every_style_kind_with_its_inheritance() { + let cat = catalog(); + assert_eq!(cat.styles.len(), 4); + let h1 = cat.get(&StyleId::new("Heading1")).unwrap(); + assert_eq!(h1.name, "heading 1"); + assert_eq!(h1.based_on, Some(StyleId::new("Normal"))); + assert_eq!(h1.next, Some(StyleId::new("Normal"))); + assert_eq!(h1.style_type, StyleType::Paragraph); + assert!(cat.default_paragraph_style().is_some()); + assert_eq!( + cat.get(&StyleId::new("Enfatico")).unwrap().style_type, + StyleType::Character + ); + } + + #[test] + fn resolution_reaches_the_document_defaults() { + let resolved = catalog().resolve(Some(&StyleId::new("Heading1"))); + assert_eq!(resolved.font.name.as_deref(), Some("Calibri"), "inherited"); + assert_eq!( + resolved.font.size, + Some(Length::from_pt(16.0)), + "overridden" + ); + } + + #[test] + fn heading_level_comes_from_outline_level_or_the_name() { + let cat = catalog(); + assert_eq!(heading_level(&cat, &StyleId::new("Heading1")), Some(1)); + assert_eq!(heading_level(&cat, &StyleId::new("Normal")), None); + + let xml = r#" + + "#; + let named = read_styles(&Element::parse("s.xml", xml.as_bytes()).unwrap()); + assert_eq!(heading_level(&named, &StyleId::new("Heading3")), Some(3)); + } +} diff --git a/crates/docsai-office/src/error.rs b/crates/docsai-office/src/error.rs new file mode 100644 index 0000000..3e67b22 --- /dev/null +++ b/crates/docsai-office/src/error.rs @@ -0,0 +1,48 @@ +//! Reader errors. +//! +//! Fase 1 acceptance criterion: *no panics on corrupt input, always `Err`*. +//! Every failure path in this crate ends in one of these variants. + +/// Something went wrong reading an Office document. +#[derive(Debug, thiserror::Error)] +pub enum ReadError { + #[error("not a ZIP container: {0}")] + NotAZip(String), + + #[error("required part `{0}` is missing")] + MissingPart(String), + + #[error("malformed XML in `{part}`: {source}")] + Xml { + part: String, + #[source] + source: quick_xml::Error, + }, + + #[error("`{part}` is not valid UTF-8")] + Encoding { part: String }, + + #[error("`{part}` is not a {expected} document")] + WrongShape { part: String, expected: String }, + + #[error("document exceeds the safety limit: {0}")] + TooLarge(String), + + #[error("document is encrypted or password-protected")] + Encrypted, + + #[error("i/o error: {0}")] + Io(#[from] std::io::Error), + + #[error("asset storage: {0}")] + Asset(#[from] docsai_model::assets::AssetError), +} + +impl From for ReadError { + fn from(e: zip::result::ZipError) -> Self { + match e { + zip::result::ZipError::Io(io) => ReadError::Io(io), + other => ReadError::NotAZip(other.to_string()), + } + } +} diff --git a/crates/docsai-office/src/lib.rs b/crates/docsai-office/src/lib.rs new file mode 100644 index 0000000..b5ec7a0 --- /dev/null +++ b/crates/docsai-office/src/lib.rs @@ -0,0 +1,58 @@ +//! Office readers and writers for docsai. +//! +//! Fase 1 lands the `.docx` reader; `.xlsx`/`.xls` arrive in Fase 3 and `.doc` +//! in Fase 5. The crate depends on `docsai-model` and on nothing else in the +//! workspace (`AGENTS.md` §3). +//! +//! ```no_run +//! use docsai_model::MemoryAssetStore; +//! let file = std::fs::File::open("informe.docx")?; +//! let mut assets = MemoryAssetStore::new(); +//! let (document, report) = docsai_office::read_docx(file, &mut assets)?; +//! println!("{} warnings", report.warnings.len()); +//! # Ok::<(), Box>(()) +//! ``` + +#![forbid(unsafe_code)] + +pub mod detect; +mod docx; +mod error; +mod package; +mod xml; + +pub use detect::{detect, DetectScore}; +pub use error::ReadError; + +use docsai_model::assets::AssetStore; +use docsai_model::{ConversionReport, Document, Format}; +use std::io::{Read, Seek}; + +/// Formats this crate can read today. +pub const READABLE: &[Format] = &[Format::Docx]; + +/// Formats this crate can write today. +pub const WRITABLE: &[Format] = &[]; + +/// Reads a `.docx` document. +pub fn read_docx( + reader: R, + assets: &mut dyn AssetStore, +) -> Result<(Document, ConversionReport), ReadError> { + docx::read(reader, assets) +} + +/// Reads any Office document this crate supports. +pub fn read( + format: Format, + reader: R, + assets: &mut dyn AssetStore, +) -> Result<(Document, ConversionReport), ReadError> { + match format { + Format::Docx => read_docx(reader, assets), + other => Err(ReadError::WrongShape { + part: other.to_string(), + expected: "a format supported in this phase (docx)".into(), + }), + } +} diff --git a/crates/docsai-office/src/package.rs b/crates/docsai-office/src/package.rs new file mode 100644 index 0000000..f922d7a --- /dev/null +++ b/crates/docsai-office/src/package.rs @@ -0,0 +1,292 @@ +//! OPC package handling: the ZIP container and its relationship graph. + +use std::collections::BTreeMap; +use std::io::{Read, Seek}; + +use crate::error::ReadError; +use crate::xml::Element; + +/// Cap on the total uncompressed size of a package. +/// +/// A first line of defence against decompression bombs; Fase 8 hardens this +/// further with a dedicated adversarial suite. +const MAX_TOTAL_BYTES: u64 = 512 * 1024 * 1024; +/// Cap on a single part. +const MAX_PART_BYTES: u64 = 128 * 1024 * 1024; + +/// An OPC package held in memory, keyed by part name. +#[derive(Debug, Default)] +pub struct Package { + parts: BTreeMap>, +} + +impl Package { + /// Reads every part of the ZIP container. + pub fn open(reader: R) -> Result { + let mut zip = zip::ZipArchive::new(reader)?; + let mut parts = BTreeMap::new(); + let mut total: u64 = 0; + + for i in 0..zip.len() { + let mut entry = match zip.by_index(i) { + Ok(entry) => entry, + // A single unreadable entry must not sink the document. + Err(zip::result::ZipError::Io(e)) => return Err(ReadError::Io(e)), + Err(_) => continue, + }; + if entry.is_dir() { + continue; + } + if entry.encrypted() { + return Err(ReadError::Encrypted); + } + let size = entry.size(); + if size > MAX_PART_BYTES { + return Err(ReadError::TooLarge(format!( + "part `{}` declares {size} bytes", + entry.name() + ))); + } + total = total.saturating_add(size); + if total > MAX_TOTAL_BYTES { + return Err(ReadError::TooLarge(format!( + "package expands to more than {MAX_TOTAL_BYTES} bytes" + ))); + } + let Some(name) = normalise_part_name(entry.name()) else { + // Absolute or traversing member name: never trust it. + continue; + }; + let mut bytes = Vec::with_capacity(size.min(1 << 20) as usize); + entry.read_to_end(&mut bytes)?; + parts.insert(name, bytes); + } + + if parts.is_empty() { + return Err(ReadError::NotAZip("archive has no readable parts".into())); + } + Ok(Package { parts }) + } + + pub fn part(&self, name: &str) -> Option<&[u8]> { + self.parts.get(name).map(|v| v.as_slice()) + } + + pub fn has_part(&self, name: &str) -> bool { + self.parts.contains_key(name) + } + + /// Part names, in deterministic order. + pub fn part_names(&self) -> impl Iterator { + self.parts.keys().map(|s| s.as_str()) + } + + /// A required part, as UTF-8 text. + pub fn text(&self, name: &str) -> Result<&str, ReadError> { + let bytes = self + .part(name) + .ok_or_else(|| ReadError::MissingPart(name.to_string()))?; + std::str::from_utf8(bytes).map_err(|_| ReadError::Encoding { + part: name.to_string(), + }) + } + + /// Parses a part into an XML tree, or `None` if the part is absent. + pub fn optional_xml(&self, name: &str) -> Result, ReadError> { + match self.part(name) { + Some(bytes) => Element::parse(name, bytes).map(Some), + None => Ok(None), + } + } + + /// The relationships declared for `part` (`/_rels/.rels`). + pub fn relationships(&self, part: &str) -> Relationships { + let Some(rels_name) = rels_part_name(part) else { + return Relationships::default(); + }; + let Some(bytes) = self.part(&rels_name) else { + return Relationships::default(); + }; + let Ok(root) = Element::parse(&rels_name, bytes) else { + return Relationships::default(); + }; + + let base = parent_dir(part); + let mut map = BTreeMap::new(); + for rel in root.children_named("Relationship") { + let (Some(id), Some(kind), Some(target)) = + (rel.attr("Id"), rel.attr("Type"), rel.attr("Target")) + else { + continue; + }; + let external = rel.attr("TargetMode") == Some("External"); + let resolved = if external { + target.to_string() + } else { + match resolve_target(&base, target) { + Some(path) => path, + None => continue, + } + }; + map.insert( + id.to_string(), + Relationship { + kind: kind.rsplit('/').next().unwrap_or(kind).to_string(), + target: resolved, + external, + }, + ); + } + Relationships { map } + } +} + +/// One OPC relationship. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Relationship { + /// Last path segment of the relationship type (`image`, `hyperlink`…). + pub kind: String, + /// Package-absolute part name, or the URL for external targets. + pub target: String, + pub external: bool, +} + +/// The relationships of one part. +#[derive(Debug, Default, Clone)] +pub struct Relationships { + map: BTreeMap, +} + +impl Relationships { + pub fn get(&self, id: &str) -> Option<&Relationship> { + self.map.get(id) + } + + /// The first relationship of a given kind, if any. + pub fn first_of_kind(&self, kind: &str) -> Option<&Relationship> { + self.map.values().find(|r| r.kind == kind) + } +} + +/// Normalises a ZIP member name, rejecting anything that escapes the package. +/// +/// Media file names inside a document are attacker-controlled, so a member +/// called `../../etc/passwd` must never become a part name (arquitectura §3.2). +fn normalise_part_name(raw: &str) -> Option { + let raw = raw.replace('\\', "/"); + if raw.starts_with('/') || raw.contains(':') { + return None; + } + let mut out: Vec<&str> = Vec::new(); + for segment in raw.split('/') { + match segment { + "" | "." => continue, + ".." => return None, + s => out.push(s), + } + } + if out.is_empty() { + None + } else { + Some(out.join("/")) + } +} + +fn parent_dir(part: &str) -> String { + match part.rfind('/') { + Some(i) => part[..i].to_string(), + None => String::new(), + } +} + +fn rels_part_name(part: &str) -> Option { + let dir = parent_dir(part); + let file = part.rsplit('/').next()?; + Some(if dir.is_empty() { + format!("_rels/{file}.rels") + } else { + format!("{dir}/_rels/{file}.rels") + }) +} + +/// Resolves a relationship target against the part's directory. +fn resolve_target(base: &str, target: &str) -> Option { + let target = target.replace('\\', "/"); + let joined = if let Some(stripped) = target.strip_prefix('/') { + stripped.to_string() + } else if base.is_empty() { + target + } else { + format!("{base}/{target}") + }; + + let mut out: Vec<&str> = Vec::new(); + for segment in joined.split('/') { + match segment { + "" | "." => continue, + ".." => { + out.pop()?; + } + s => out.push(s), + } + } + if out.is_empty() { + None + } else { + Some(out.join("/")) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn targets_resolve_relative_to_their_part() { + assert_eq!( + resolve_target("word", "media/image1.png").as_deref(), + Some("word/media/image1.png") + ); + assert_eq!( + resolve_target("xl/worksheets", "../drawings/drawing1.xml").as_deref(), + Some("xl/drawings/drawing1.xml") + ); + assert_eq!( + resolve_target("word", "/word/styles.xml").as_deref(), + Some("word/styles.xml") + ); + } + + #[test] + fn targets_cannot_escape_the_package() { + assert_eq!(resolve_target("word", "../../etc/passwd"), None); + assert_eq!(resolve_target("", ".."), None); + } + + #[test] + fn member_names_are_sanitised() { + assert_eq!( + normalise_part_name("word//media/./image1.png").as_deref(), + Some("word/media/image1.png") + ); + assert_eq!( + normalise_part_name("word\\media\\image1.png").as_deref(), + Some("word/media/image1.png") + ); + assert_eq!(normalise_part_name("../secret"), None); + assert_eq!(normalise_part_name("/abs/path"), None); + assert_eq!(normalise_part_name("C:/win"), None); + } + + #[test] + fn rels_part_names_follow_the_opc_convention() { + assert_eq!( + rels_part_name("word/document.xml").as_deref(), + Some("word/_rels/document.xml.rels") + ); + assert_eq!( + rels_part_name("[Content_Types].xml").as_deref(), + Some("_rels/[Content_Types].xml.rels") + ); + } +} diff --git a/crates/docsai-office/src/xml.rs b/crates/docsai-office/src/xml.rs new file mode 100644 index 0000000..705ba00 --- /dev/null +++ b/crates/docsai-office/src/xml.rs @@ -0,0 +1,354 @@ +//! A minimal XML tree over `quick-xml`. +//! +//! OOXML parts are small enough to hold in memory, and a tree makes the +//! readers far easier to review than a hand-rolled event state machine. Each +//! node also records the **byte span** it occupies in the source, which is what +//! makes the raw-block fidelity hatch (spec §7) exact: an unrecognised element +//! is preserved as the original bytes, not as a re-serialisation. +//! +//! Elements are matched by *local* name. OOXML prefixes are conventional but +//! not guaranteed, and every lookup in this crate is contextual (children of a +//! known parent), so local names are unambiguous while being robust to a +//! document that renames its prefixes. + +use quick_xml::events::Event; +use quick_xml::Reader; +use std::ops::Range; + +use crate::error::ReadError; + +/// An XML element. +#[derive(Debug, Clone)] +pub struct Element { + /// Local name, without prefix (`p` for `w:p`). + pub name: String, + /// Namespace prefix as written (`w`), empty when there was none. + pub prefix: String, + /// Attributes, in document order, with their qualified names. + pub attrs: Vec<(String, String)>, + pub children: Vec, + /// Byte range of the whole element in the source part. + pub span: Range, +} + +/// A child of an element. +#[derive(Debug, Clone)] +pub enum Node { + Element(Element), + Text(String), +} + +/// Depth limit; a document nested deeper than this is pathological and would +/// otherwise let a crafted file exhaust memory. +const MAX_DEPTH: usize = 256; + +impl Element { + /// Parses a whole part into a tree rooted at its single top element. + pub fn parse(part: &str, source: &[u8]) -> Result { + let text = std::str::from_utf8(source).map_err(|_| ReadError::Encoding { + part: part.to_string(), + })?; + let mut reader = Reader::from_str(text); + let config = reader.config_mut(); + config.trim_text(false); + config.expand_empty_elements = false; + config.check_end_names = false; + // A stray closing tag should not sink a document that is otherwise + // readable; a *missing* one still does, because a truncated part means + // truncated content and the caller must be told. + config.allow_unmatched_ends = true; + + let mut stack: Vec = Vec::new(); + let mut root: Option = None; + let mut start = 0usize; + + loop { + let event = reader.read_event().map_err(|source| ReadError::Xml { + part: part.to_string(), + source, + })?; + let end = reader.buffer_position() as usize; + match event { + Event::Start(e) => { + if stack.len() >= MAX_DEPTH { + return Err(ReadError::TooLarge(format!( + "`{part}` nests deeper than {MAX_DEPTH} elements" + ))); + } + stack.push(new_element(e.name().as_ref(), e.attributes(), start, part)?); + } + Event::Empty(e) => { + let element = new_element(e.name().as_ref(), e.attributes(), start, part)?; + let element = Element { + span: start..end, + ..element + }; + match stack.last_mut() { + Some(parent) => parent.children.push(Node::Element(element)), + None if root.is_none() => root = Some(element), + None => {} + } + } + Event::End(_) => { + let Some(mut element) = stack.pop() else { + // Stray closing tag: ignore rather than fail; the + // surrounding content is still usable. + start = end; + continue; + }; + element.span = element.span.start..end; + match stack.last_mut() { + Some(parent) => parent.children.push(Node::Element(element)), + None if root.is_none() => root = Some(element), + None => {} + } + } + Event::Text(t) => { + if let Some(parent) = stack.last_mut() { + let text = t.unescape().map_err(|source| ReadError::Xml { + part: part.to_string(), + source, + })?; + if !text.is_empty() { + parent.children.push(Node::Text(text.into_owned())); + } + } + } + Event::CData(t) => { + if let Some(parent) = stack.last_mut() { + let text = String::from_utf8_lossy(t.as_ref()).into_owned(); + parent.children.push(Node::Text(text)); + } + } + Event::Eof => break, + _ => {} + } + start = end; + } + + root.ok_or_else(|| ReadError::WrongShape { + part: part.to_string(), + expected: "XML with a root element".into(), + }) + } + + /// The value of an attribute, matched on its local name. + /// + /// `w:val` and `val` both match a lookup for `"val"`, which keeps the + /// readers independent of prefix conventions. + pub fn attr(&self, local: &str) -> Option<&str> { + self.attrs + .iter() + .find(|(name, _)| local_of(name) == local) + .map(|(_, value)| value.as_str()) + } + + /// The value of an attribute, matched on its *qualified* name. Needed when + /// two namespaces put different meanings on the same local name, as + /// `r:embed` and `r:link` do next to `a:blip`'s own attributes. + pub fn attr_qualified(&self, qualified: &str) -> Option<&str> { + self.attrs + .iter() + .find(|(name, _)| name == qualified) + .map(|(_, value)| value.as_str()) + } + + /// An OOXML boolean attribute/element: absent value means true, and + /// `0`/`false`/`off` mean false. + pub fn ooxml_flag(&self) -> bool { + !matches!(self.attr("val"), Some("0") | Some("false") | Some("off")) + } + + /// The first child element with this local name. + pub fn child(&self, local: &str) -> Option<&Element> { + self.children().find(|e| e.name == local) + } + + /// Every child element with this local name. + pub fn children_named<'a>(&'a self, local: &'a str) -> impl Iterator + 'a { + self.children().filter(move |e| e.name == local) + } + + /// Every child element. + pub fn children(&self) -> impl Iterator { + self.children.iter().filter_map(|n| match n { + Node::Element(e) => Some(e), + Node::Text(_) => None, + }) + } + + /// A descendant reached by a path of local names. + pub fn path(&self, path: &[&str]) -> Option<&Element> { + let mut cursor = self; + for step in path { + cursor = cursor.child(step)?; + } + Some(cursor) + } + + /// Concatenation of the direct text children. + pub fn text(&self) -> String { + self.children + .iter() + .filter_map(|n| match n { + Node::Text(t) => Some(t.as_str()), + Node::Element(_) => None, + }) + .collect() + } + + /// Concatenation of every descendant text node. + #[cfg(test)] + pub fn deep_text(&self) -> String { + let mut out = String::new(); + self.push_deep_text(&mut out); + out + } + + #[cfg(test)] + fn push_deep_text(&self, out: &mut String) { + for node in &self.children { + match node { + Node::Text(t) => out.push_str(t), + Node::Element(e) => e.push_deep_text(out), + } + } + } + + /// The original bytes of this element, for raw-block preservation. + pub fn raw<'a>(&self, source: &'a str) -> &'a str { + source + .get(self.span.clone()) + .unwrap_or_default() + .trim_matches(|c: char| c == '\n' || c == '\r') + } + + /// An integer attribute, or `None` when absent or unparsable. + pub fn attr_i64(&self, local: &str) -> Option { + self.attr(local)?.trim().parse().ok() + } +} + +fn new_element( + raw_name: &[u8], + attributes: quick_xml::events::attributes::Attributes<'_>, + start: usize, + part: &str, +) -> Result { + let qualified = String::from_utf8_lossy(raw_name).into_owned(); + let (prefix, name) = match qualified.split_once(':') { + Some((p, n)) => (p.to_string(), n.to_string()), + None => (String::new(), qualified.clone()), + }; + + let mut attrs = Vec::new(); + for attr in attributes { + let attr = attr.map_err(|e| ReadError::Xml { + part: part.to_string(), + source: quick_xml::Error::InvalidAttr(e), + })?; + let key = String::from_utf8_lossy(attr.key.as_ref()).into_owned(); + let value = attr + .unescape_value() + .map(|v| v.into_owned()) + .unwrap_or_else(|_| String::from_utf8_lossy(&attr.value).into_owned()); + attrs.push((key, value)); + } + + Ok(Element { + name, + prefix, + attrs, + children: Vec::new(), + span: start..start, + }) +} + +fn local_of(qualified: &str) -> &str { + qualified.split_once(':').map_or(qualified, |(_, n)| n) +} + +#[cfg(test)] +mod tests { + use super::*; + + const DOC: &str = r#" + + Hola mundo + celda +"#; + + #[test] + fn parses_the_tree_and_matches_on_local_names() { + let root = Element::parse("test.xml", DOC.as_bytes()).unwrap(); + assert_eq!(root.name, "document"); + assert_eq!(root.prefix, "w"); + let body = root.child("body").unwrap(); + assert_eq!(body.children_named("p").count(), 1); + let p = body.child("p").unwrap(); + assert_eq!( + p.path(&["pPr", "pStyle"]).and_then(|e| e.attr("val")), + Some("Heading1") + ); + assert_eq!(p.deep_text(), "Hola mundo"); + } + + #[test] + fn spans_point_at_the_original_bytes() { + let root = Element::parse("test.xml", DOC.as_bytes()).unwrap(); + let tbl = root.path(&["body", "tbl"]).unwrap(); + let raw = tbl.raw(DOC); + assert!(raw.starts_with(""), "got {raw}"); + assert!(raw.ends_with("")); + assert!(raw.contains("celda")); + } + + #[test] + fn empty_elements_get_a_span_too() { + let root = Element::parse("test.xml", DOC.as_bytes()).unwrap(); + let style = root.path(&["body", "p", "pPr", "pStyle"]).unwrap(); + assert_eq!(style.raw(DOC), r#""#); + } + + #[test] + fn ooxml_flags_default_to_true() { + let xml = + r#""#; + let root = Element::parse("t.xml", xml.as_bytes()).unwrap(); + assert!(root.child("b").unwrap().ooxml_flag()); + assert!(!root.child("i").unwrap().ooxml_flag()); + assert!(root.child("strike").unwrap().ooxml_flag()); + } + + #[test] + fn qualified_attribute_lookup_distinguishes_namespaces() { + let xml = r#""#; + let root = Element::parse("t.xml", xml.as_bytes()).unwrap(); + assert_eq!(root.attr_qualified("r:embed"), Some("rId1")); + assert_eq!(root.attr_qualified("r:link"), Some("rId2")); + } + + #[test] + fn entities_are_unescaped_in_text_and_attributes() { + let xml = r#"x & y"#; + let root = Element::parse("t.xml", xml.as_bytes()).unwrap(); + assert_eq!(root.text(), "x & y"); + assert_eq!(root.attr("val"), Some("a < b")); + } + + #[test] + fn malformed_xml_is_an_error_not_a_panic() { + assert!( + Element::parse("t.xml", b"").is_err(), + "a truncated part is an error, not partial content" + ); + assert!(Element::parse("t.xml", b"").is_err()); + assert!(Element::parse("t.xml", &[0xff, 0xfe, 0x00]).is_err()); + } + + #[test] + fn stray_closing_tags_do_not_underflow() { + let root = Element::parse("t.xml", b"").unwrap(); + assert_eq!(root.name, "a", "the first complete root wins"); + } +} diff --git a/crates/docsai-office/tests/docx_images.rs b/crates/docsai-office/tests/docx_images.rs new file mode 100644 index 0000000..5ead7a9 --- /dev/null +++ b/crates/docsai-office/tests/docx_images.rs @@ -0,0 +1,259 @@ +//! The image half of the Fase 1 acceptance criteria: floating, transformed and +//! duplicated pictures must reach the IR with their geometry intact. + +use docsai_model::assets::AssetStore; +use docsai_model::image::{ + AlignKeyword, Anchor, AxisPos, Flip, ImageRef, RelBase, WrapMode, WrapSide, +}; +use docsai_model::text::{Block, Inline, TextDocument}; +use docsai_model::units::Length; +use docsai_model::{ConversionReport, Document, MemoryAssetStore}; + +fn read(name: &str) -> (TextDocument, ConversionReport, MemoryAssetStore) { + let path = format!( + "{}/../../corpus/docx/{name}.docx", + env!("CARGO_MANIFEST_DIR") + ); + let file = std::fs::File::open(&path).unwrap_or_else(|e| panic!("{path}: {e}")); + let mut assets = MemoryAssetStore::new(); + let (document, report) = docsai_office::read_docx(file, &mut assets).expect("reads"); + match document { + Document::Text(doc) => (doc, report, assets), + Document::Workbook(_) => panic!("expected a text document"), + } +} + +/// Every image in the document, in order. +fn images(doc: &TextDocument) -> Vec<&ImageRef> { + fn from_inlines<'a>(inlines: &'a [Inline], out: &mut Vec<&'a ImageRef>) { + for inline in inlines { + match inline { + Inline::Image(image) => out.push(image), + Inline::Styled { content, .. } | Inline::Link { content, .. } => { + from_inlines(content, out) + } + _ => {} + } + } + } + let mut out = Vec::new(); + for block in doc.blocks() { + match block { + Block::Paragraph(p) => from_inlines(&p.content, &mut out), + Block::Image(image) => out.push(image), + _ => {} + } + } + out +} + +#[test] +fn inline_images_carry_size_alt_title_and_name() { + let (doc, report, assets) = read("images-inline"); + let images = images(&doc); + assert_eq!(images.len(), 3, "png, gif and emf"); + assert_eq!(report.stats.images, 3); + + let first = images[0]; + assert_eq!(first.geometry.anchor, Anchor::Inline); + assert_eq!(first.geometry.display_size.width, Length::from_px(120.0)); + assert_eq!(first.geometry.display_size.height, Length::from_px(90.0)); + assert_eq!(first.alt, "Diagrama de ventas"); + assert_eq!(first.title.as_deref(), Some("Figura 1")); + assert_eq!(first.name.as_deref(), Some("Diagrama")); + assert_eq!( + first.geometry.native_size_px, + Some((120, 90)), + "native size is read from the bitmap header, not from the document" + ); + + let info = assets.info(&images[1].asset).expect("gif stored"); + assert_eq!(info.content_type, "image/gif"); + assert!(info.file_name.ends_with(".gif")); +} + +#[test] +fn emf_is_preserved_verbatim_with_a_warning() { + let (doc, report, assets) = read("images-inline"); + let emf = images(&doc) + .into_iter() + .find(|i| { + assets + .info(&i.asset) + .is_some_and(|info| info.content_type == "image/x-emf") + }) + .expect("the EMF was stored"); + assert_eq!(emf.geometry.display_size.width, Length::from_cm(4.0)); + assert_eq!(emf.geometry.native_size_px, None, "vector, no pixel size"); + assert!( + report + .warnings + .iter() + .any(|w| w.message().contains("cannot render")), + "the user must be told EMF will not display: {:?}", + report.warnings + ); +} + +#[test] +fn floating_images_keep_position_wrap_and_z_order() { + let (doc, _, _) = read("images-floating"); + let images = images(&doc); + assert_eq!(images.len(), 3); + + // Anchored to the margin with explicit offsets and square wrap on the right. + match &images[0].geometry.anchor { + Anchor::Floating { + relative_to_h, + relative_to_v, + position, + wrap, + wrap_side, + behind_text, + } => { + assert_eq!(*relative_to_h, RelBase::Margin); + assert_eq!(*relative_to_v, RelBase::Paragraph); + assert_eq!(position.h, AxisPos::Offset(Length::from_cm(1.2))); + assert_eq!(position.v, AxisPos::Offset(Length::from_cm(0.5))); + assert_eq!(*wrap, WrapMode::Square); + assert_eq!(*wrap_side, WrapSide::Right); + assert!(!behind_text); + } + other => panic!("expected a floating anchor, got {other:?}"), + } + assert_eq!(images[0].geometry.z_index, Some(2)); + + // Anchored to the page with symbolic alignment and top/bottom wrap. + match &images[1].geometry.anchor { + Anchor::Floating { + relative_to_h, + position, + wrap, + .. + } => { + assert_eq!(*relative_to_h, RelBase::Page); + assert_eq!(position.h, AxisPos::Align(AlignKeyword::Center)); + assert_eq!(position.v, AxisPos::Align(AlignKeyword::Top)); + assert_eq!(*wrap, WrapMode::TopBottom); + } + other => panic!("expected a floating anchor, got {other:?}"), + } + + // Behind the text: the watermark. + assert_eq!(images[2].geometry.anchor.keyword(), "behind"); + assert_eq!(images[2].alt, "Marca de agua"); +} + +#[test] +fn transforms_survive_rotation_flip_crop_and_border() { + let (doc, _, _) = read("images-transformed"); + let images = images(&doc); + assert_eq!(images.len(), 3); + + let rotated = &images[0]; + assert!( + (rotated.geometry.rotation_deg - 45.0).abs() < 0.01, + "got {}", + rotated.geometry.rotation_deg + ); + + let cropped = &images[1]; + let crop = cropped.geometry.crop.expect("srcRect became a crop"); + assert!((crop.left - 10.0).abs() < 0.01); + assert!((crop.top - 5.0).abs() < 0.01); + assert!((crop.right - 20.0).abs() < 0.01); + assert_eq!(crop.bottom, 0.0); + let border = cropped + .geometry + .border + .as_ref() + .expect("a:ln became a border"); + assert_eq!(border.width, Length::from_pt(1.0)); + assert_eq!(border.color, "#000000"); + assert_eq!(border.style, "solid"); + + let flipped = &images[2]; + assert_eq!(flipped.geometry.flip, Flip::HV); + assert_eq!(flipped.geometry.display_size.width, Length::from_cm(2.0)); + assert_eq!( + flipped.geometry.native_size_px, + Some((64, 64)), + "scaled down from its native size" + ); +} + +#[test] +fn one_bitmap_used_three_times_is_stored_once() { + let (doc, report, assets) = read("images-duplicated"); + let images = images(&doc); + assert_eq!(images.len(), 3); + assert_eq!(report.stats.images, 3); + + let ids: std::collections::BTreeSet<_> = images.iter().map(|i| &i.asset).collect(); + assert_eq!(ids.len(), 1, "three appearances, one asset"); + assert_eq!(assets.len(), 1, "and one stored file"); + + // …each keeping its own geometry. + assert_eq!( + images[0].geometry.display_size.width, + Length::from_px(120.0) + ); + assert_eq!(images[1].geometry.display_size.width, Length::from_px(60.0)); + assert!(matches!(images[2].geometry.anchor, Anchor::Floating { .. })); +} + +#[test] +fn legacy_vml_pictures_are_read_into_the_same_model() { + let (doc, report, _) = read("images-vml"); + let images = images(&doc); + assert_eq!(images.len(), 1, "the VML shape became an image"); + + let image = images[0]; + assert_eq!(image.alt, "Imagen VML heredada"); + assert_eq!(image.title.as_deref(), Some("Heredada")); + assert_eq!(image.geometry.display_size.width, Length::from_pt(90.0)); + assert_eq!(image.geometry.display_size.height, Length::from_pt(67.5)); + match &image.geometry.anchor { + Anchor::Floating { + relative_to_h, + position, + wrap, + wrap_side, + .. + } => { + assert_eq!(*relative_to_h, RelBase::Margin); + assert_eq!(position.h, AxisPos::Offset(Length::from_pt(10.0))); + assert_eq!(*wrap, WrapMode::Square); + assert_eq!(*wrap_side, WrapSide::Right); + } + other => panic!("expected a floating anchor, got {other:?}"), + } + assert!( + report + .warnings + .iter() + .any(|w| w.message().contains("legacy VML")), + "reading VML is a degradation and must say so" + ); +} + +#[test] +fn every_document_of_the_corpus_satisfies_the_ir_invariants() { + let dir = format!("{}/../../corpus/docx", env!("CARGO_MANIFEST_DIR")); + let mut checked = 0; + for entry in std::fs::read_dir(&dir).expect("corpus present") { + let path = entry.expect("entry").path(); + if path.extension().and_then(|e| e.to_str()) != Some("docx") { + continue; + } + let file = std::fs::File::open(&path).expect("open"); + let mut assets = MemoryAssetStore::new(); + let (document, _) = docsai_office::read_docx(file, &mut assets) + .unwrap_or_else(|e| panic!("{}: {e}", path.display())); + if let Err(errors) = docsai_model::validate::validate(&document) { + panic!("{}: {errors:?}", path.display()); + } + checked += 1; + } + assert!(checked >= 14, "only {checked} documents checked"); +} diff --git a/crates/docsai-office/tests/robustness.rs b/crates/docsai-office/tests/robustness.rs new file mode 100644 index 0000000..870d1e8 --- /dev/null +++ b/crates/docsai-office/tests/robustness.rs @@ -0,0 +1,123 @@ +//! Fase 1 acceptance criterion: *«Cero pánicos con corpus corrupto sintético +//! (ZIP truncado, XML malformado): siempre `Err`»*. +//! +//! The R1 spike measured 204 panics out of 903 corrupt inputs for `docx-rs`; +//! this test is the standing guarantee that the own parser does better. + +use docsai_model::assets::AssetStore; +use docsai_model::MemoryAssetStore; +use std::io::Cursor; + +fn corpus(name: &str) -> Vec { + let path = format!( + "{}/../../corpus/docx/{name}.docx", + env!("CARGO_MANIFEST_DIR") + ); + std::fs::read(&path).unwrap_or_else(|e| panic!("{path}: {e}")) +} + +/// Reads a byte slice, reporting whether it panicked. +fn read_catching(bytes: Vec) -> Result { + let previous = std::panic::take_hook(); + std::panic::set_hook(Box::new(|_| {})); + let outcome = std::panic::catch_unwind(move || { + let mut assets = MemoryAssetStore::new(); + docsai_office::read_docx(Cursor::new(bytes), &mut assets).is_ok() + }); + std::panic::set_hook(previous); + outcome.map_err(|_| ()) +} + +#[test] +fn truncation_never_panics() { + let base = corpus("images-floating"); + let mut checked = 0; + for cut in (1..base.len()).step_by(7) { + let truncated = base[..cut].to_vec(); + assert!( + read_catching(truncated).is_ok(), + "panicked on a {cut}-byte prefix" + ); + checked += 1; + } + assert!(checked > 100, "only {checked} prefixes exercised"); +} + +#[test] +fn byte_corruption_never_panics() { + let base = corpus("nested-lists"); + for index in (0..base.len()).step_by(11) { + let mut corrupted = base.clone(); + corrupted[index] ^= 0xFF; + assert!( + read_catching(corrupted).is_ok(), + "panicked with byte {index} flipped" + ); + } +} + +#[test] +fn malformed_xml_inside_a_valid_zip_is_an_error() { + // A structurally fine package whose main part is broken XML. + let mut buffer = Vec::new(); + { + let mut zip = zip::ZipWriter::new(Cursor::new(&mut buffer)); + let options: zip::write::FileOptions<'_, ()> = Default::default(); + zip.start_file("word/document.xml", options).unwrap(); + std::io::Write::write_all(&mut zip, b"").unwrap(); + zip.finish().unwrap(); + } + let mut assets = MemoryAssetStore::new(); + let result = docsai_office::read_docx(Cursor::new(buffer), &mut assets); + assert!(result.is_err(), "a truncated part must not read as success"); +} + +#[test] +fn a_package_without_a_document_part_is_an_error() { + let mut buffer = Vec::new(); + { + let mut zip = zip::ZipWriter::new(Cursor::new(&mut buffer)); + let options: zip::write::FileOptions<'_, ()> = Default::default(); + zip.start_file("hello.txt", options).unwrap(); + std::io::Write::write_all(&mut zip, b"not a document").unwrap(); + zip.finish().unwrap(); + } + let mut assets = MemoryAssetStore::new(); + assert!(docsai_office::read_docx(Cursor::new(buffer), &mut assets).is_err()); +} + +#[test] +fn random_bytes_are_an_error() { + for seed in 0u8..32 { + let noise: Vec = (0..4096u32) + .map(|i| (i as u8).wrapping_mul(31).wrapping_add(seed)) + .collect(); + let mut assets = MemoryAssetStore::new(); + assert!(docsai_office::read_docx(Cursor::new(noise), &mut assets).is_err()); + } +} + +#[test] +fn a_media_part_escaping_the_package_is_ignored() { + // `word/media/../../evil.png` must never be reachable as a part. + let mut buffer = Vec::new(); + { + let mut zip = zip::ZipWriter::new(Cursor::new(&mut buffer)); + let options: zip::write::FileOptions<'_, ()> = Default::default(); + zip.start_file("word/document.xml", options).unwrap(); + std::io::Write::write_all( + &mut zip, + br#""#, + ) + .unwrap(); + zip.start_file("word/media/../../evil.png", options) + .unwrap(); + std::io::Write::write_all(&mut zip, b"\x89PNG\r\n\x1a\n").unwrap(); + zip.finish().unwrap(); + } + let mut assets = MemoryAssetStore::new(); + let (_, report) = docsai_office::read_docx(Cursor::new(buffer), &mut assets) + .expect("the document itself is still readable"); + assert_eq!(report.stats.images, 0); + assert!(assets.is_empty(), "the traversing member was not stored"); +} diff --git a/docs/analisis-tecnico.md b/docs/analisis-tecnico.md index d4d4bef..99ea37c 100644 --- a/docs/analisis-tecnico.md +++ b/docs/analisis-tecnico.md @@ -181,7 +181,7 @@ gratuita con todo el ecosistema Pandoc (PDF vía LaTeX, HTML, EPUB…). | **`calamine`** | Lectura `.xls`, `.xlsx`, `.xlsb`, `.ods` (valores **y fórmulas**) | Maduro, mantenido, muy usado | Lectura perezosa y rápida; lee fórmulas vía `worksheet_formula()`. **No lee estilos/formatos de número con detalle suficiente** → se complementa con lectura propia de `xl/styles.xml` | | **`umya-spreadsheet`** | Lectura+escritura `.xlsx` con estilos | Mantenido | Único crate que lee Y escribe xlsx con estilos; parsea todo el workbook en memoria (coste en ficheros grandes; existe `lazy_read`). Candidato principal para la **escritura** xlsx | | **`rust_xlsxwriter`** | Escritura `.xlsx` (alternativa) | Muy mantenido (port de XlsxWriter) | Excelente API de escritura con fórmulas y formatos, pero solo escritura y no permite "editar" — válido porque docsai regenera desde IR. Decidir vs umya en spike de Fase 3 | -| **`docx-rs` (bokuweb)** | Lectura+escritura `.docx` | El más usado (1M+ descargas) | JSON-friendly, escribe bien; la lectura no expone el 100 % de styles.xml/numbering.xml → complementar con `quick-xml` propio donde falte | +| **`docx-rs` (bokuweb)** | ~~Lectura~~ + posible escritura `.docx` | El más usado (1M+ descargas) | **Descartado para la lectura** tras el spike R1 (`docs/spikes/R1-estrategia-docx.md`, agosto 2026): resuelve bien estilos y numeración, pero pierde casi todo el modelo de imagen (wrap, `behindDoc`, recorte, volteo, rotación, alt, título, hipervínculo), las notas al pie y el `w:instr` de los campos simples, no conserva elementos desconocidos para raw-blocks, y entra en *panic* en el 23 % de 903 entradas corruptas medidas. La lectura docx usa parser propio sobre `zip` + `quick-xml`. Sigue siendo candidato para el **writer** de la Fase 2, decisión independiente | | **`docx-rust`** | Alternativa lectura `.docx` | Menor actividad | Mapeo XML más directo; mantener como referencia | | **`spreadsheet-ods`** | Lectura+escritura `.ods` | Mantenido | Cubre estilos y fórmulas ODS; evita escribir un writer ODF-spreadsheet propio | | **(ninguno)** | `.odt` | — | No hay crate maduro para ODT con estilos: **parser/writer propio** sobre `zip` + `quick-xml` (ODF es regular; esfuerzo acotado) | @@ -207,11 +207,12 @@ idempotencia del round-trip. | `serde` / `serde_yaml` / `serde_json` | Front matter, `inspect --json`, config | | `clap` (derive) | CLI | | `rmcp` (SDK oficial de MCP, transporte stdio) | Servidor MCP; macros `#[tool]`; implementa spec 2026-07-28 con compatibilidad hacia atrás | -| `image` | Detección de dimensiones/re-codificación de imágenes; sin soporte WMF/EMF (limitación aceptada) | +| ~~`image`~~ | **No se usa en la Fase 1.** docsai nunca recodifica un mapa de bits, solo necesita nombrarlo y medirlo, y eso se resuelve leyendo la cabecera del formato (PNG, JPEG, GIF, BMP, TIFF, WebP, EMF, WMF) en `docsai-model::assets`, sin dependencia pesada. Se reevaluará si alguna fase necesita recodificar de verdad | +| ~~`serde_yaml`~~ | **No se usa.** El front matter tiene un esquema pequeño y conocido y la spec exige determinismo byte a byte, así que se escribe a mano; además el crate está sin mantenimiento activo | | `thiserror` / `anyhow` | Errores | | `tracing` + `tracing-subscriber` | Logs (siempre a stderr) | | `tokio` | Solo en `docsai-mcp` (rmcp lo requiere); el núcleo de conversión es síncrono | -| `insta` | Snapshot testing de golden files | +| ~~`insta`~~ | **No se usa.** Los golden files son `.expected.dmk.md` junto al corpus, como prescribe `AGENTS.md` §6: se revisan como texto normal en el diff y no dependen de un formato de snapshot | | `cargo-fuzz` | Fuzzing de parsers (Fase 8) | | `cargo-dist` | Empaquetado de releases multiplataforma | diff --git a/docs/especificacion-docmark.md b/docs/especificacion-docmark.md index 64e72af..ac087e2 100644 --- a/docs/especificacion-docmark.md +++ b/docs/especificacion-docmark.md @@ -1,12 +1,16 @@ -# Especificación DocMark v1.0-draft +# Especificación DocMark v1.0 **DocMark** es el perfil de Markdown extendido usado por `docsai` como formato pivote textual. Objetivo: representar documentos de texto (docx/odt/doc) y hojas de cálculo (xlsx/xls/ods) de forma **legible, editable a mano y con información suficiente para regenerar el documento original con pérdida mínima**. -Estado: **borrador**. El equipo de implementación debe congelar esta especificación al final de -la Fase 0 (ver plan). Todo cambio posterior incrementa la versión declarada en el front matter. +Estado: **congelada (v1.0)** al cierre de la Fase 0. Todo cambio posterior incrementa la versión +declarada en el front matter y exige documentar la migración (`AGENTS.md` §2). + +El anexo A (tablas complejas) queda resuelto en §3.4. Los TODO del borrador están cerrados; las +adiciones que la implementación de la Fase 1 hizo explícitas están marcadas en el texto y +recogidas en §10. ## 0. Principios @@ -45,6 +49,10 @@ page: # geometría de página / sección por defecto ( margins: { top: 2.5cm, bottom: 2.5cm, left: 3cm, right: 3cm } orientation: portrait +style-defaults: # `w:docDefaults` de OOXML: el fondo de la cascada + font: { name: "Calibri", size: 11pt } + paragraph: { space-after: 8pt } + styles: # catálogo de estilos del documento original (§3) Heading1: type: paragraph @@ -64,7 +72,12 @@ list-definitions: # numbering.xml / estilos de lista ODF normaliza ``` Reglas: -- Claves en kebab-case. Unidades explícitas (`pt`, `cm`, `px`, `%`); colores `#RRGGBB`. +- Claves en kebab-case. Unidades explícitas (`px`, `cm`, `pt`, `emu`, `%`); colores `#RRGGBB`. +- **Regla de unidades (normativa)**: el serializador elige la **primera unidad que representa la + longitud de forma exacta**, en el orden `px` (a 96 dpi) → `cm` → `pt` → `emu`. La legibilidad + cede ante la exactitud: un margen de Word de 1417 twips se escribe `70.85pt`, nunca un + `2.499cm` aproximado, porque redondear movería el margen en cada ida y vuelta. `emu` es la + escotilla final y siempre existe. - El bloque `styles` es un **catálogo**: el cuerpo referencia estilos por nombre mediante clases (`{.Heading1}`); el writer inverso lo usa para regenerar `styles.xml`/`styles.xml` ODF. - Los campos desconocidos se conservan (los parsers no deben rechazarlos): forward-compatible. @@ -82,8 +95,13 @@ Párrafo con estilo y formato directo. {.Quote align=center space-after=12pt} ``` - El nivel `#` refleja el nivel de esquema (outline level); la clase indica el estilo real. -- Atributos de párrafo van en `{...}` **al final del bloque**: `align`, `indent`, - `space-before/after`, `line-height`, `background`, `border`, `page-break-before`, etc. +- Atributos de párrafo van en `{...}` **al final del bloque**: `align`, + `indent-left`, `indent-right`, `indent-first-line`, `indent-hanging`, + `space-before`, `space-after`, `line-height`, `background`, `keep-with-next`, + `page-break-before`, `outline-level`. +- Un párrafo **vacío** es contenido real (así se espacia un documento) y una línea en blanco no + puede representarlo, porque Markdown la absorbe: en modo `full` se escribe `[]{.empty}`. Los + modos `standard` y `plain` lo descartan, que es justo lo que prometen (§6). - Regla de economía: si el formato coincide exactamente con lo que define el estilo, **no** se emiten atributos redundantes (mantiene el Markdown limpio y el diff estable). @@ -96,9 +114,14 @@ Párrafo con estilo y formato directo. {.Quote align=center space-after=12pt} | color, resaltado, fuente, tamaño | `[texto]{color="#FF0000" highlight="yellow" font="Arial" size=14pt}` | | estilo de carácter | `[texto]{.Emphatic}` | | sub/superíndice | `[x]{.sub}` / `[x]{.sup}` | +| versalitas / mayúsculas | `[x]{.small-caps}` / `[x]{.caps}` | +| subrayado no simple | `[x]{.underline underline=double}` (valores: `double`, `thick`, `dotted`, `dashed`, `wave`) | +| formato desactivado sobre un estilo | `[x]{bold=false}` / `[x]{italic=false}` | | salto de línea manual | doble espacio final (hard break) | +| salto de página o columna | `[]{.break kind=page}` / `[]{.break kind=column}` | | hipervínculo | `[texto](https://...)` — con atributos si tiene estilo: `[texto](url){.Hyperlink}` | | nota al pie | `[^1]` + definición al final (sintaxis Pandoc/GFM footnotes) | +| campo dinámico | `[valor]{.field field=PAGE}`, con `instr="…"` en modo `full` | ### 3.3 Listas @@ -106,11 +129,18 @@ Listas Markdown estándar; la definición tipográfica vive en `list-definitions en el primer ítem cuando no es la lista por defecto: ```markdown -1. Primer punto {list=L1} -2. Segundo punto - a) Sub-punto (el marcador real lo define L1 nivel 2) +1. Primer punto {.ListParagraph list=L1} +2. Segundo punto {.ListParagraph} + 1. Sub-punto (el marcador real lo define L1 nivel 2) {.ListParagraph list=L1} ``` +- `list=` viaja **dentro del bloque de atributos del primer ítem**, no en un bloque aparte: un + ítem nunca lleva dos `{...}`. +- Las listas anidadas se escriben *tight* (sin línea en blanco entre el ítem y su sublista) para + que un visor CommonMark no envuelva cada ítem en un `

`. +- El marcador ordinal que se escribe es `1.`, `2.`… El marcador real de presentación lo define + el nivel correspondiente de `list-definitions`. + ### 3.4 Tablas (documentos de texto) Tablas GFM cuando son regulares. Una tabla con celdas combinadas, anchos fijos o estilo de tabla @@ -126,9 +156,31 @@ se envuelve en un contenedor que aporta los metadatos: ``` - `rowspan`/`colspan` en la **primera celda** del área combinada; las celdas absorbidas quedan vacías. +- El contenedor admite `style`, `col-widths` y `header-row=false`. GFM exige una fila de + cabecera: cuando el original no la tenía se emite una fila vacía y `header-row=false` lo + registra, de modo que la vuelta no invente una cabecera. - Si la estructura excede lo representable en GFM (tablas anidadas, celdas multipárrafo), la tabla - completa se emite como contenedor `::: {.table complex=true}` con filas como sub-bloques - `::: {.row}` / `::: {.cell}` (formato detallado en el anexo A del futuro doc de implementación). + completa se emite como contenedor `::: {.table complex=true}` con filas y celdas como + sub-bloques: + +````markdown +::: {.table complex=true style=TableGrid} +::: {.row} +::: {.cell rowspan=2} +Primer párrafo de la celda. + +Segundo párrafo de la misma celda. +::: +::: {.cell} +Otra celda. +::: +::: +::: +```` + + Reglas del formato complejo: un `::: {.row}` por fila, un `::: {.cell}` por celda **incluidas + las absorbidas** (que van vacías), `colspan`/`rowspan` en la celda que abre el área, y dentro + de cada celda cualquier bloque DocMark válido. Esto cierra el anexo A del borrador. ### 3.5 Imágenes y objetos @@ -159,7 +211,8 @@ Imagen flotante con posición y ajuste de texto: | `native-size` | Dimensiones en píxeles del bitmap original (`AxB`) | cabecera del fichero | | `dpi` | Resolución declarada, si difiere de 96 | metadatos del bitmap | | `anchor` | `inline` (defecto) \| `floating` \| `behind` \| `front` | `wp:inline`/`wp:anchor`, `text:anchor-type` | -| `relative-to` | Referencia de posición: `page` \| `margin` \| `paragraph` \| `character` | `wp:positionH/V @relativeFrom` | +| `relative-to` | Referencia de posición horizontal: `page` \| `margin` \| `paragraph` \| `character` \| `line` \| `column` | `wp:positionH @relativeFrom` | +| `relative-to-v` | Referencia vertical, **solo si difiere** de `relative-to` | `wp:positionV @relativeFrom` | | `x`, `y` | Offsets desde la referencia (solo flotantes) | `wp:posOffset`, `svg:x/y` | | `align-h`, `align-v` | Alineación simbólica (`left/center/right`, `top/middle/bottom`) cuando el original usa alineación en vez de offset | `wp:align` | | `wrap` | `square` \| `tight` \| `through` \| `top-bottom` \| `none` | `wp:wrapSquare…`, `style:wrap` | @@ -167,12 +220,14 @@ Imagen flotante con posición y ajuste de texto: | `z-index` | Orden de apilamiento entre objetos flotantes | `@relativeHeight` | | `rotation` | Grados en sentido horario | `a:xfrm @rot` | | `flip` | `h` \| `v` \| `hv` | `a:xfrm @flipH/V` | -| `crop` | Recorte `"izq,arr,der,abj"` en % del original | `a:srcRect`, `fo:clip` | +| `crop` | Recorte `"izq,arr,der,abj"`, los cuatro lados con sufijo `%` | `a:srcRect`, `fo:clip` | | `border` | Borde simple `"1pt solid #000000"` (bordes complejos → raw) | `pic:spPr` | | `name` | Nombre interno del objeto en el documento | `wp:docPr @name` | | `title` | Título/leyenda | `wp:docPr @title` | | `link` | Hipervínculo sobre la imagen | `a:hlinkClick` | | `external-src` | URL original si la imagen estaba **enlazada**, no embebida | `r:link`, `xlink:href` | +| `render` | `unsupported` para formatos que un visor Markdown no dibuja (WMF/EMF) | sniffing del contenido | +| `effects-raw` | Id del raw-block con los efectos irrepresentables | `a:effectLst`, `a:scene3d` | Reglas: - El texto alternativo (accesibilidad, `wp:docPr @descr` / `svg:desc`) va en el campo alt @@ -333,3 +388,28 @@ DocMark en modo `full` es parseable por `pandoc -f markdown` con estas salvedade los bloques `cell-meta` y `raw` aparecen como divs genéricos, y los atributos no estándar se conservan como atributos de div/span. Esto es intencional: da salida gratuita a PDF/HTML/EPUB vía Pandoc sin que docsai tenga que implementarla. + +--- + +## 10. Registro de cambios de la v1.0 + +Añadidos respecto al borrador, decididos durante la Fase 1 al implementar el serializador. Todos +son adiciones: ningún documento escrito contra el borrador deja de ser válido. + +| Cambio | Motivo | +|---|---| +| Regla de unidades exactas (§2) con `emu` como escotilla | El borrador dejaba la unidad al gusto del serializador; redondear a `cm` movía márgenes y tamaños en cada ida y vuelta | +| `style-defaults` en el front matter | `w:docDefaults` es el fondo de la cascada de 4 niveles y no tenía sitio | +| `indent-left/right/first-line/hanging` en vez de un `indent` único | OOXML y ODF distinguen los cuatro; uno solo perdía información | +| `[]{.empty}` para párrafos vacíos | Una línea en blanco no sobrevive a Markdown, y los párrafos vacíos son contenido real | +| `[]{.break kind=page\|column}` | El borrador solo cubría el salto de línea | +| `.small-caps`, `.caps`, `underline=`, `bold=false`, `italic=false` | Formato directo que la tabla §3.2 no cubría | +| `{.field field=X instr="…"}` | El borrador describía los campos en prosa pero no fijaba la sintaxis | +| `list=` dentro del bloque de atributos del primer ítem | Evita que un ítem lleve dos `{...}` seguidos | +| `header-row=false` en el contenedor de tabla | GFM obliga a una fila de cabecera que el original podía no tener | +| Formato completo de `::: {.table complex=true}` | Cierra el anexo A del borrador | +| `relative-to-v`, `render=unsupported`, `effects-raw` en imágenes | Ejes con referencias distintas, medios no dibujables y efectos sin modelo | + +**Extensiones no implementadas todavía** (la spec las define, la Fase 1 no las emite): `border` de +párrafo, `::: {.textbox}` (los cuadros de texto viajan como raw-block), y todo lo de §4 (hojas de +cálculo), que llega en la Fase 3. diff --git a/docs/plan-desarrollo.md b/docs/plan-desarrollo.md index 1b807be..681c575 100644 --- a/docs/plan-desarrollo.md +++ b/docs/plan-desarrollo.md @@ -44,10 +44,19 @@ Tareas: Entregables: workspace verde en CI, `docsai-model` v1, spec congelada, corpus v1, informe del spike. Criterios de aceptación: -- [ ] `cargo test --workspace` verde en las 3 plataformas. -- [ ] IR serializa/deserializa a JSON con round-trip idéntico (proptest básico). -- [ ] Spike documentado con decisión firmada sobre la estrategia docx. -- [ ] DocMark v1.0 sin TODOs abiertos. +- [x] `cargo test --workspace` verde en las 3 plataformas. +- [x] IR serializa/deserializa a JSON con round-trip idéntico (proptest básico). +- [x] Spike documentado con decisión firmada sobre la estrategia docx. +- [x] DocMark v1.0 sin TODOs abiertos. + +**Estado**: cerrada. Notas de ejecución: +- El spike R1 (`docs/spikes/R1-estrategia-docx.md`) descartó `docx-rs`: pierde el modelo de + imagen casi entero, las notas al pie y los campos simples, y provoca *panic* en el 23 % de las + entradas corruptas. La lectura docx se hace con parser propio sobre `zip` + `quick-xml`. +- El corpus se **genera** con `corpus/generate.py` en vez de dibujarse a mano: el XML queda + revisable en el diff y los paquetes son reproducibles byte a byte (ver `corpus/README.md`). +- Los golden files son `.expected.dmk.md` junto al corpus, como prescribe `AGENTS.md` §6, en + lugar de snapshots de `insta`; `insta` sale del árbol de dependencias. --- @@ -79,11 +88,20 @@ Tareas: 10. Golden tests de todo el corpus docx; añadir 5+ documentos "del mundo real" anonimizados. Criterios de aceptación: -- [ ] Los golden docx del corpus pasan (incluidos los tres de imágenes: flotantes, +- [x] Los golden docx del corpus pasan (incluidos los tres de imágenes: flotantes, transformadas y duplicadas — con geometría completa en los atributos DocMark). -- [ ] Cero pánicos con corpus corrupto sintético (ZIP truncado, XML malformado): siempre `Err`. -- [ ] Un docx real de 50+ páginas convierte en < 1 s con < 10 raw-blocks. -- [ ] La salida en `--fidelity plain` es CommonMark limpio verificado con comrak. +- [x] Cero pánicos con corpus corrupto sintético (ZIP truncado, XML malformado): siempre `Err`. +- [x] Un docx real de 50+ páginas convierte en < 1 s con < 10 raw-blocks. +- [x] La salida en `--fidelity plain` es CommonMark limpio verificado con comrak. + +**Estado**: cerrada salvo la tarea 10 (documentos reales anonimizados), que depende de disponer +de documentos reales y no bloquea la Fase 2. Notas de ejecución: +- Los cuadros de texto DrawingML (`wps:txbx`) se conservan como raw-block, no como + `::: {.textbox}`: el tipo existe en el IR y en la spec, pero emitirlo se pospone a la Fase 2, + donde el writer tendrá que reconstruirlos. +- Tres defectos que el corpus destapó y que quedan corregidos: el formato de longitudes era + lisiado por redondeo (un margen de 1417 twips se escribía `2.499cm`), los párrafos vacíos + desaparecían en silencio, y el estilo de carácter de un hipervínculo se emitía dos veces. --- diff --git a/docs/spikes/R1-estrategia-docx.md b/docs/spikes/R1-estrategia-docx.md new file mode 100644 index 0000000..6e0eb8f --- /dev/null +++ b/docs/spikes/R1-estrategia-docx.md @@ -0,0 +1,157 @@ +# Spike R1 — Estrategia de lectura DOCX + +**Riesgo mitigado**: R1 del análisis técnico §6 — *«La cascada de estilos OOXML resulta más +costosa de lo previsto y retrasa Fase 1»*. + +**Pregunta**: ¿basta `docx-rs` (+ complemento propio con `quick-xml` donde falte) para la +Fase 1, o hace falta un parser OOXML propio completo? + +**Fecha**: agosto 2026 · **Versión evaluada**: `docx-rs` 0.4.22 · **Toolchain**: rustc 1.94.1 + +**Decisión**: **parser propio sobre `zip` + `quick-xml`.** `docx-rs` no se usa en el camino de +lectura. Se mantiene como referencia de mapeo XML y como posible base del *writer* de Fase 2 +(decisión independiente, a revisar en su momento). + +--- + +## 1. Método + +Se generó el corpus de la Fase 0 (`corpus/generate.py`) y se leyó con `docx_rs::read_docx`, +serializando el resultado a JSON para inspeccionar qué información sobrevive. Se comprobó +rasgo a rasgo contra lo que exige la especificación DocMark y el IR de `arquitectura.md` §3. + +Adicionalmente se sometió al lector a 903 entradas corruptas sintéticas (truncados cada 7 bytes +y volteos de un byte cada 13 bytes sobre `images-floating.docx`) capturando *panics*. + +Documentos usados: `basic-styles`, `custom-styles`, `nested-lists`, `footnotes`, `fields-raw`, +`headers-footers`, `images-inline`, `images-floating`, `images-transformed`, `images-vml`. + +## 2. Resultados + +### 2.1 Lo que `docx-rs` sí resuelve + +| Rasgo | Estado | Nota | +|---|---|---| +| `styles.xml` con `docDefaults` | ✅ | `runPropertyDefault` y `paragraphPropertyDefault` expuestos | +| `basedOn`, `styleType`, `name` | ✅ | Suficiente para reconstruir la herencia | +| Formato de párrafo/run como **delta** | ✅ | El modelo separa `property.style` del formato directo, que es exactamente el principio «referencia + delta» | +| `numbering.xml` | ✅ | `abstractNums` + `numberings` + niveles con `format`, `lvlText`, indentación | +| Cabeceras y pies | ✅ | Resueltos y adjuntos a `sectionProperty` (`header`, `firstHeader`, `footer`) | +| `sectPr` (tamaño, márgenes, `titlePg`) | ✅ | `columns` se expone pero no el resto de `w:cols` | +| Campos complejos (`fldChar`/`instrText`) | ✅ | `instrTextString` conservado | +| Tablas con `gridSpan`/`vMerge` | ✅ | Presentes en `tableCellProperty` | + +La cascada de 4 niveles **es resoluble** con lo que expone: el riesgo R1, tal como estaba +formulado (los *estilos*), no se materializa. Es el resto lo que falla. + +### 2.2 Lo que `docx-rs` pierde + +Medido sobre el corpus, no inferido de la documentación. + +**Imágenes** — el modelo `Pic` expone `size`, `positionType`, `positionH/V`, `relativeFromH/V`, +`distT/B/L/R`, `relativeHeight` y `rot`. No expone: + +| Atributo DocMark §3.5 | Origen OOXML | En `docx-rs` | +|---|---|---| +| `wrap`, `wrap-side` | `wp:wrapSquare/Tight/Through/TopAndBottom` | ❌ ausente | +| `anchor=behind` | `wp:anchor @behindDoc` | ❌ ausente | +| `crop` | `a:srcRect` | ❌ ausente | +| `flip` | `a:xfrm @flipH/@flipV` | ❌ ausente | +| `rotation` | `a:xfrm @rot` | ⚠️ campo `rot: u16` presente pero devuelve `0` para `rot="2700000"` (45°); además `u16` no puede representar 60000ᵃᵛᵒˢ de grado ni valores negativos | +| alt (`![…]`) | `wp:docPr @descr` | ❌ ausente | +| `title`, `name` | `wp:docPr @title/@name` | ❌ ausente | +| `link` | `a:hlinkClick` | ❌ ausente | +| `external-src` | `r:link` | ❌ ausente | +| `border` | `pic:spPr/a:ln` | ❌ ausente | +| bytes del medio | `word/media/*` | ❌ no se cargan al leer (`image` es «for writer only») | + +**VML legado** (`w:pict`, `images-vml.docx`): se colapsa a un nodo `shape` genérico de 813 bytes +de JSON total; se pierden el `r:id` de `v:imagedata`, el `style` (posición y tamaño), el `alt` +y el `w10:wrap`. Pérdida total del objeto. + +**Notas al pie** (`footnotes.docx`): `w:footnoteReference` se descarta — el run queda con +`children: []`. `word/footnotes.xml` no se expone en la API. + +**Campos simples** (`fields-raw.docx`): `w:fldSimple` pierde el atributo `w:instr`; solo queda +el texto cacheado. «Fecha: 01/01/2026» deja de ser un campo `DATE`. + +**Escotilla de fidelidad**: `w:sdt` se representa como `structuredDataTag` con `alias: null` y +sin el XML original. No hay ningún mecanismo genérico que conserve los bytes de un elemento +desconocido, que es justo lo que necesita el `raw-block` de la spec §7 y el criterio de +cobertura medible de la Fase 1 (tarea 9). + +**Ruido en el modelo**: cada estilo leído sale con un `tableProperty` con bordes inventados +(`single/2/000000`) aunque el estilo sea de párrafo — habría que filtrarlo para no contaminar +el catálogo del front matter. + +### 2.3 Robustez frente a entrada corrupta + +``` +903 entradas corruptas (truncados + volteos de byte) + → Ok: 88 Err: 611 PANIC: 204 +``` + +Un 23 % de las entradas corruptas provoca *panic*. El criterio de aceptación de la Fase 1 es +explícito: *«Cero pánicos con corpus corrupto sintético (ZIP truncado, XML malformado): siempre +`Err`»*. Envolver todo el reader en `catch_unwind` no es una mitigación aceptable (no funciona +con `panic=abort`, y `AGENTS.md` §6 exige que los parsers devuelvan `Err`, no que se recuperen). + +## 3. Análisis + +El complemento propio que haría falta para cerrar los huecos incluye: todo `w:drawing` +(DrawingML completo), todo `w:pict` (VML), `word/footnotes.xml`, `w:fldSimple`, la captura de +elementos desconocidos para raw-blocks y la carga de `word/media/*`. Es decir: **el grueso de +`document.xml`**. Lo que quedaría delegado en `docx-rs` es `styles.xml`, `numbering.xml` y el +árbol de párrafos/tablas — la parte más mecánica y mejor documentada del formato. + +Mantener las dos rutas implica además: + +- Dos parsers XML en el binario (`xml-rs` dentro de `docx-rs`, `quick-xml` en el nuestro). +- Reconciliar dos árboles distintos del mismo `document.xml` (el de `docx-rs` y el nuestro para + drawings/campos), con el riesgo de desincronización de posiciones. +- Una dependencia de la que dependemos para lo fácil y no para lo difícil, con un riesgo de + *panic* que hay que asumir o parchear aguas arriba. + +## 4. Decisión + +**Parser propio sobre `zip` + `quick-xml`** en `docsai-office`, con estas consecuencias: + +1. **Sin `docx-rs` en `docsai-office`.** Se anota en `analisis-tecnico.md` §4.1 conforme a la + regla de `AGENTS.md` §2 (no se sustituye una dependencia clave sin dejar constancia). +2. **Un solo recorrido** de `document.xml` con `quick-xml` en modo evento, que produce el IR + directamente y captura como raw-block cualquier elemento no reconocido, con su parte y su + ruta. Esto hace la cobertura medible desde el primer día. +3. **Sin `unwrap`/`expect`/índices sin comprobar** en el camino de lectura: todo error es un + `ReadError` tipado. El criterio «cero pánicos» se verifica con un test de corrupción + sintética equivalente al de este spike, ejecutado en CI desde la Fase 1. +4. **`quick-xml` con entidades externas deshabilitadas** (comportamiento por defecto: no expande + entidades externas), lo que adelanta parte de la Fase 8. +5. El coste estimado del parser propio (≈2 semanas de las 4–6 de la Fase 1) es comparable al del + complemento que habría que escribir de todos modos, y elimina la reconciliación de árboles. + +### Riesgos que introduce esta decisión + +| Riesgo | Mitigación | +|---|---| +| Más superficie propia que mantener | Corpus + golden tests desde la Fase 0; el parser cubre solo lo que el IR modela, el resto va a raw-block | +| Rasgos OOXML olvidados por desconocimiento | La captura genérica de elementos desconocidos los hace **visibles** (advertencia + raw-block) en vez de silenciosos | +| El writer de Fase 2 podría necesitar `docx-rs` | Decisión independiente y posterior; escribir es mucho más simple que leer (controlamos el XML de salida) y probablemente también se haga a mano | + +## 5. Estado del riesgo R1 + +**Cerrado.** La cascada de estilos no es el cuello de botella; el modelo de imágenes sí lo era, y +la decisión de parser propio lo neutraliza. R8 (diversidad de modelos de imagen) queda cubierto +por la misma decisión: DrawingML y VML se leen en el mismo recorrido hacia `ImageGeometry`. + +## 6. Reproducir este spike + +El programa de sondeo vivió fuera del árbol (no se versiona una dependencia que hemos +descartado). Para reproducirlo: + +```bash +python3 corpus/generate.py +cargo new /tmp/spike-docx && cd /tmp/spike-docx +cargo add docx-rs@0.4.22 serde_json +# leer los documentos de corpus/docx con read_docx() y serializar `docx.document` a JSON; +# para la prueba de robustez, truncar y voltear bytes del .docx y contar catch_unwind(Err) +``` diff --git a/kb/01-resumen-fases-0-1.md b/kb/01-resumen-fases-0-1.md new file mode 100644 index 0000000..bf2d476 --- /dev/null +++ b/kb/01-resumen-fases-0-1.md @@ -0,0 +1,112 @@ +# 01 — Resumen de las fases 0 y 1 + +## Qué se puede hacer hoy + +```bash +docsai convert informe.docx -o informe.dmk.md # extrae assets/ junto al .md +docsai convert informe.docx # a stdout +docsai convert informe.docx --fidelity plain # CommonMark limpio para LLM/RAG +docsai convert informe.docx -o out.md --json # informe de conversión en JSON +docsai formats # matriz de soporte de este binario +``` + +Un `.docx` entra y sale un fichero DocMark con: + +- **Propiedades** del documento (core, app y personalizadas) en el front matter. +- **Catálogo de estilos** completo, con `w:docDefaults` y la cadena `basedOn`, sin aplanar. +- **Párrafos y runs**: negrita, cursiva, tachado, subrayado (los siete estilos), color, + resaltado, fuente, tamaño, sub/superíndice, versalitas, mayúsculas, hipervínculos, saltos de + línea, página y columna, y tabuladores. +- **Listas** reconstruidas en árbol desde los pares planos `(numId, ilvl)`, con su definición + tipográfica en `list-definitions`. +- **Tablas** con `gridSpan` y `vMerge` resueltos a `colspan`/`rowspan` y celdas absorbidas + marcadas; contenedor `::: {.table complex=true}` para lo que GFM no admite. +- **Imágenes** con la geometría entera: tamaño mostrado y nativo, anclaje en línea o flotante, + base de posición por eje, offsets o alineación simbólica, ajuste de texto y lado, orden Z, + rotación, volteos, recorte, borde, texto alternativo, título, nombre interno e hipervínculo. + Tanto DrawingML como **VML heredado**. +- **Secciones** partidas en cada `sectPr`, con cabeceras y pies por ámbito (defecto, primera + página, pares). +- **Notas al pie** insertadas en el punto de su referencia. +- **Campos** simples y complejos, conservando la instrucción original. +- **Raw-blocks** con los bytes exactos de todo elemento OOXML no reconocido. + +Lo que **no** se puede hacer: la conversión inversa (Fase 2), hojas de cálculo (Fase 3), ODF +(Fase 4), `.doc` (Fase 5) y el servidor MCP (Fase 7). + +## Fase 0 — Fundamentos + +| Tarea del plan | Estado | +|---|---| +| 1. Workspace con los 7 crates | ✅ | +| 2. `docsai-model` v1 (IR, unidades, imágenes, `AssetStore`, validador) | ✅ | +| 3. Spike de riesgo R1 | ✅ [`docs/spikes/R1-estrategia-docx.md`](../docs/spikes/R1-estrategia-docx.md) | +| 4. Congelar la spec DocMark v1.0 | ✅ anexo A resuelto, registro de cambios en §10 | +| 5. Corpus inicial | ✅ 14 docx + 6 xlsx generados por `corpus/generate.py` | +| 6. CI GitHub Actions en 3 SO | ✅ build, test, corpus al día, fmt, clippy, rustdoc | +| 7. Esqueleto de CLI | ✅ superado: la CLI ya hace `convert` completo | + +**Criterios de aceptación**: los cuatro cumplidos y verificados con tests +(`cargo test --workspace` verde, round-trip JSON del IR con proptest, spike documentado con +decisión firmada, spec sin TODO). + +## Fase 1 — Lectura DOCX → DocMark + +| Tarea del plan | Estado | +|---|---| +| 1. Reader docx (ZIP, `document.xml`, relaciones) | ✅ | +| 2. Estilos con resolución referencia + delta | ✅ | +| 3. Párrafos y runs con todo el formato inline de la spec §3.2 | ✅ | +| 4. Listas desde `numbering.xml` | ✅ | +| 5. Tablas con spans, anchos y estilo | ✅ | +| 6. Imágenes DrawingML + VML, con deduplicación por hash | ✅ | +| 7. Secciones, cabeceras/pies, notas al pie, campos, propiedades | ✅ | +| 8. Serializador DocMark con modos de fidelidad | ✅ | +| 9. Raw-block + advertencia tipada para lo no reconocido | ✅ | +| 10. Golden tests del corpus + 5 documentos reales anonimizados | ⚠️ goldens sí; documentos reales **pendientes** | + +**Criterios de aceptación**, todos verificados por tests: + +- ✅ Los goldens del corpus pasan, incluidos los tres de imágenes con geometría completa. +- ✅ Cero pánicos con corpus corrupto sintético: 900+ entradas (truncados y volteos de byte) + devuelven `Err` (`crates/docsai-office/tests/robustness.rs`). +- ✅ Documento de 50+ páginas en menos de 1 s con menos de 10 raw-blocks. +- ✅ `--fidelity plain` es CommonMark limpio, verificado parseándolo con comrak. + +## Métricas + +| | | +|---|---| +| Tests que pasan | 164 | +| Código de producción | ~9 500 líneas de Rust | +| Código de test | ~1 200 líneas en ficheros de integración, más los módulos `#[cfg(test)]` | +| Documentos del corpus | 14 docx + 6 xlsx, con sus 14 goldens | +| Dependencias externas del núcleo | `serde`, `thiserror`, `quick-xml`, `zip`, `tracing` | +| `unsafe` | Cero (`#![forbid(unsafe_code)]` en todos los crates de biblioteca) | + +## Tres defectos que el corpus destapó + +Los tres estaban en el camino de la fidelidad, y ninguno se habría visto sin pasar documentos +reales por el pipeline: + +1. **Formato de longitudes con pérdida.** Un margen de Word de 1417 twips se escribía `2.499cm` + y al releerlo daba un valor distinto: habría movido márgenes e imágenes en cada ida y vuelta. + Ahora se elige la primera unidad que representa la longitud de forma **exacta** + (`px` → `cm` → `pt` → `emu`), y hay un test que lo verifica para cada caso. +2. **Párrafos vacíos desaparecían en silencio.** Una línea en blanco no sobrevive a Markdown. + En modo `full` se escriben ahora como `[]{.empty}`. +3. **Estilo de hipervínculo emitido dos veces**, dentro y fuera de la etiqueta. + +Además, el propio spike destapó un error en el generador del corpus (`w:drawing` colocado fuera +de un `w:r`, que es OOXML inválido) antes de que contaminara los goldens. + +## Qué quedó fuera, y por qué + +| Elemento | Motivo | +|---|---| +| **Documentos reales anonimizados** (Fase 1, tarea 10) | Requiere disponer de documentos reales. No bloquea la Fase 2; el corpus sintético ya cubre los rasgos uno a uno | +| **Cuadros de texto DrawingML** (`wps:txbx`) | Viajan como raw-block. El tipo `Block::TextBox` existe en el IR y `::: {.textbox}` está en la spec, pero emitirlo sin el writer que lo reconstruya es adelantar trabajo de la Fase 2 | +| **Validación en Word y LibreOffice** | Criterio de la Fase 2 (aún no se generan `.docx`), y no hay LibreOffice en el entorno | +| **Comentarios y control de cambios** | Fuera de alcance de la v1 según el análisis §1.1. Las revisiones se toman como aceptadas y se reporta cuántas | +| **`w:lvlOverride` en listas** | No modelado en v1; se emite advertencia `Degraded` | +| **Lector xlsx** | Fase 3. Los ficheros del corpus ya existen para entonces | diff --git a/kb/02-estructura-proyecto.md b/kb/02-estructura-proyecto.md new file mode 100644 index 0000000..1bfc3fa --- /dev/null +++ b/kb/02-estructura-proyecto.md @@ -0,0 +1,155 @@ +# 02 — Estructura del proyecto + +## Árbol + +``` +docsai/ +├── Cargo.toml # workspace: 7 miembros + dependencias compartidas +├── .github/workflows/ci.yml # build/test en 3 SO + fmt/clippy/rustdoc +├── crates/ +│ ├── docsai-model/ # el IR. Sin I/O, sin dependencias pesadas +│ ├── docsai-docmark/ # serializador DocMark (el parser es Fase 2) +│ ├── docsai-office/ # lector .docx (xlsx Fase 3, doc Fase 5) +│ ├── docsai-odf/ # esqueleto (Fase 4) +│ ├── docsai-convert/ # orquestación: detección, pipelines, assets +│ ├── docsai-cli/ # binario `docsai` +│ └── docsai-mcp/ # esqueleto (Fase 7) +├── corpus/ +│ ├── generate.py # genera TODOS los ficheros del corpus +│ ├── README.md # qué aísla cada documento +│ ├── docx/*.docx # 14 documentos + 14 *.expected.dmk.md (goldens) +│ └── xlsx/*.xlsx # 6 libros, para la Fase 3 +├── docs/ # documentación de diseño (español) +│ └── spikes/ # informes de spikes con su decisión +└── kb/ # esta base de conocimiento +``` + +## Los siete crates + +| Crate | Tipo | Líneas | Responsabilidad | Depende de | +|---|---|---|---|---| +| `docsai-model` | lib | ~3 500 | El IR y los tipos comunes | *(nada del workspace)* | +| `docsai-docmark` | lib | ~1 850 | IR ⇄ DocMark | model | +| `docsai-office` | lib | ~3 500 | Lectores/escritores OOXML y legados | model | +| `docsai-odf` | lib | 21 | Lectores/escritores ODF | model | +| `docsai-convert` | lib | ~500 | Detección, pipelines, gestión de assets | los cuatro anteriores | +| `docsai-cli` | bin | ~200 | El binario `docsai` | convert, model | +| `docsai-mcp` | lib | 26 | Servidor MCP por stdio | convert | + +**Regla de dependencia (`AGENTS.md` §3)**: ningún crate de formato importa a otro crate de +formato. `docsai-odf` y `docsai-mcp` existen ya, vacíos, precisamente para que el compilador +haga cumplir esa regla desde el primer día en lugar de descubrirla rota en la Fase 4. + +## `docsai-model` — el IR + +| Módulo | Contenido | +|---|---| +| `units` | `Length` (newtype sobre EMU), `Size`, `Point`, conversiones exactas y el formato de salida | +| `style` | `StyleCatalog`, `Style`, `FontProps`, `ParaProps`, `DocDefaults` y la resolución de la cascada | +| `list` | `ListCatalog`, `ListDef`, `ListLevel`, `NumFormat` | +| `text` | `TextDocument`, `Section`, `PageGeometry`, `Block`, `Inline`, `Paragraph`, `Table`, `RawFragment` | +| `sheet` | `Workbook`, `Sheet`, `Cell`, `CellRef` (A1), `Formula`, `NumFmt` — poblado en la Fase 3 | +| `image` | `ImageRef`, `ImageGeometry`, `Anchor` y su cortejo (wrap, crop, flip, bordes) | +| `assets` | `AssetStore` (trait), `MemoryAssetStore`, hash de contenido, sniffing y dimensiones | +| `report` | `ConversionReport`, `Warning` tipada, `Severity`, `ConversionStats` | +| `validate` | Invariantes: anclajes de hoja solo en `Workbook`, filas más anchas que la rejilla, anclajes `two-cell` invertidos | + +Tres principios que los tipos codifican, y que conviene no romper: + +1. **Estilo = referencia + delta.** Cada campo de `FontProps`/`ParaProps` es `Option`: `None` + significa *heredar*, no *desactivado*. `over()` mezcla la cascada; `minus()` calcula el delta + que hay que emitir (la «regla de economía» de la spec §3.1). +2. **Todas las longitudes en EMU.** Un solo entero, sin pérdidas, para OOXML (EMU/twips), + ODF (cm/in) y `.doc` (twips). +3. **El IR no conoce el I/O.** Las imágenes son un `AssetId` detrás de un trait. + +## `docsai-office` — el lector docx + +| Módulo | Responsabilidad | +|---|---| +| `xml` | Árbol XML sobre `quick-xml` con **spans de bytes** por nodo | +| `package` | ZIP, relaciones OPC, límites anti-bomba, saneado de nombres | +| `detect` | Detección de formato **por contenido**, no por extensión | +| `docx/format` | `w:rPr` y `w:pPr` → los tipos delta del IR | +| `docx/styles` | `styles.xml` → `StyleCatalog`; nivel de encabezado | +| `docx/numbering` | `numbering.xml` → `ListCatalog` (aplana la doble indirección) | +| `docx/drawing` | DrawingML y VML → `ImageRef`/`ImageGeometry` | +| `docx/body` | El recorrido del cuerpo: bloques, inlines, campos, tablas, listas | +| `docx/mod` | Ensamblaje: propiedades, secciones, cabeceras/pies, notas al pie | + +El detalle que lo sostiene todo: **cada nodo del árbol XML recuerda su rango de bytes en el +origen**. Un elemento no reconocido se conserva citando los bytes originales, no una +re-serialización, y por eso el raw-block es exacto. + +## `docsai-docmark` — el serializador + +| Módulo | Responsabilidad | +|---|---| +| `escape` | Tabla de escapado fija por contexto (bloque, celda de tabla, etiqueta de enlace) | +| `attrs` | Bloques `{#id .clase clave="valor"}` con el orden canónico de la spec §8 | +| `units` | Cómo se escribe una longitud, un porcentaje o un número | +| `frontmatter` | El YAML, escrito a mano para garantizar determinismo byte a byte | +| `writer` | El recorrido del IR: bloques, inlines, imágenes, tablas, contenedores | + +## Flujo de una conversión + +``` +fichero.docx + │ docsai-convert::read_document + ▼ +detect (por contenido) ──► docsai-office::read_docx + │ Package::open → partes en memoria, nombres saneados + │ Element::parse → árbol XML con spans + │ styles + numbering + footnotes + │ read_sections → bloques, con DirAssetStore recibiendo medios + ▼ + (Document, ConversionReport) + │ docsai_model::validate + ▼ + docsai-docmark::serialize + │ front matter + cuerpo, según --fidelity + ▼ + fichero.dmk.md + assets/img-. +``` + +## Dependencias externas + +Todo el núcleo se apoya en cinco crates: + +| Crate | Para qué | Dónde | +|---|---|---| +| `serde` | Serialización del IR | model y derivados | +| `thiserror` | Errores tipados de biblioteca | model, docmark, office, convert | +| `quick-xml` | Parsing XML | office | +| `zip` | Contenedores OOXML | office | +| `tracing` | Logs, siempre a stderr | office, convert, cli | + +Y en los binarios: `clap` (CLI), `anyhow` (errores de binario), `serde_json` (`--json`), +`tracing-subscriber`. En tests: `proptest` y `comrak`. + +Deliberadamente **ausentes**: `docx-rs` (descartado por el spike R1), `image` (basta leer la +cabecera del formato), `serde_yaml` (el front matter se escribe a mano y el crate no está +mantenido) e `insta` (los goldens son ficheros de texto revisables). Cada ausencia está +justificada en [`docs/analisis-tecnico.md`](../docs/analisis-tecnico.md) §4, como exige +`AGENTS.md` §2. + +## Dónde están los tests + +| Ubicación | Qué cubre | +|---|---| +| Módulos `#[cfg(test)]` de cada fichero | Unidades: conversiones, escapado, cascada de estilos, parsing de fragmentos XML | +| `crates/docsai-model/tests/json_roundtrip.rs` | Round-trip JSON del IR: documentos a mano con todos los nodos + proptest | +| `crates/docsai-office/tests/docx_images.rs` | Geometría de imágenes sobre el corpus real | +| `crates/docsai-office/tests/robustness.rs` | Los 900+ inputs corruptos, path traversal, ZIP sin documento | +| `crates/docsai-convert/tests/goldens.rs` | Goldens, determinismo, finales de línea, `plain` con comrak, rendimiento | + +## Convenciones que conviene respetar + +- Nombres de código, comentarios y mensajes de commit en **inglés**; la documentación de `docs/` + y `kb/` en **español** (`AGENTS.md` §5). +- `thiserror` en bibliotecas, `anyhow` sólo en binarios. +- Los parsers **nunca entran en pánico**: sin `unwrap`, `expect` ni indexación sin comprobar en + el camino de lectura. +- El serializador es **determinista**: mismo IR ⇒ mismos bytes. Cualquier iteración sobre un + mapa usa `BTreeMap`, nunca `HashMap`. +- Rutas con `std::path`, nunca concatenando `/`. diff --git a/kb/03-decisiones-tecnicas.md b/kb/03-decisiones-tecnicas.md new file mode 100644 index 0000000..cdb730a --- /dev/null +++ b/kb/03-decisiones-tecnicas.md @@ -0,0 +1,177 @@ +# 03 — Decisiones técnicas + +Las decisiones que no se deducen del plan y que un futuro colaborador podría deshacer sin +querer. Cada una lleva su motivo y su coste. + +## 1. Parser docx propio, sin `docx-rs` + +**Decisión**: la lectura `.docx` se hace con `zip` + `quick-xml` propios. + +**Motivo** (spike R1, medido sobre el corpus, no supuesto): + +- `docx-rs` resuelve bien estilos, numeración y cabeceras/pies. Pero pierde **casi todo el modelo + de imagen**: ajuste de texto y lado, `behindDoc`, recorte, volteos, texto alternativo, título, + nombre del objeto, hipervínculo sobre la imagen y las imágenes enlazadas. La rotación existe + como campo pero devuelve `0` para un `rot="2700000"`, y el tipo es `u16`, que no puede + representar sesentamilésimas de grado ni valores negativos. +- Descarta `w:footnoteReference` y el `w:instr` de `w:fldSimple`. +- El VML (`w:pict`) se colapsa a un nodo genérico sin `r:id`, sin estilo y sin `alt`. +- No conserva elementos desconocidos, que es justo lo que el raw-block necesita. +- **204 pánicos de 903 entradas corruptas** (23 %), contra un criterio de aceptación que exige + siempre `Err`. + +Las imágenes son requisito de primera clase del proyecto y la mitad de los criterios de la +Fase 1. Lo que quedaba delegable en `docx-rs` era la parte mecánica; el complemento propio +habría sido el grueso de `document.xml` de todos modos, con dos parsers XML en el binario y dos +árboles del mismo documento que reconciliar. + +**Coste asumido**: más superficie propia que mantener. Se mitiga con el corpus, los goldens y la +captura genérica de elementos desconocidos, que hace *visible* lo que falta en vez de silencioso. + +**Detalle**: `docx-rs` sigue siendo candidato para el **writer** de la Fase 2. Es una decisión +independiente y aún abierta: escribir es mucho más simple que leer, porque el XML de salida lo +controlamos nosotros. + +## 2. Árbol XML con spans de bytes + +**Decisión**: `quick-xml` alimenta un árbol en memoria donde **cada nodo recuerda el rango de +bytes que ocupa en el origen**. + +**Motivo**: un raw-block tiene que conservar los bytes originales. Re-serializar el subárbol +daría XML equivalente pero no idéntico, y la re-inyección de la Fase 2 dejaría de ser exacta. Con +el rango basta cortar la cadena de origen. Añadido: un árbol es mucho más revisable que una +máquina de estados sobre eventos, y las partes OOXML caben en memoria de sobra. + +**Coste**: memoria proporcional al tamaño de la parte XML. Aceptable para documentos; habrá que +revisarlo en la Fase 3 para hojas de 100 k celdas, donde `calamine` ya hace streaming por hoja. + +## 3. Coincidencia por nombre local, no por espacio de nombres + +**Decisión**: los elementos se buscan por su nombre local (`p`, no `w:p`). + +**Motivo**: los prefijos OOXML son convencionales pero no obligatorios, y toda la navegación del +lector es contextual (hijos de un padre conocido), así que no hay ambigüedad. Un documento que +renombre sus prefijos se lee igual. + +**Excepción**: los atributos con significado distinto según el espacio de nombres —`r:embed` y +`r:link` junto a un `a:blip`— se buscan por nombre **cualificado** con `attr_qualified`. + +## 4. Formato de longitudes: exactitud antes que legibilidad + +**Decisión**: al escribir una longitud se elige la **primera unidad que la representa +exactamente**, en el orden `px` (96 dpi) → `cm` → `pt` → `emu`. + +**Motivo**: un margen de Word de 1417 twips no es un número redondo de centímetros. Escribirlo +`2.499cm` lo mueve 155 EMU en cada ida y vuelta, y esos errores se acumulan. Ahora sale +`70.85pt`, que es exacto. `emu` es la escotilla final y siempre existe. + +**Coste**: alguna longitud se lee peor (`85.05pt` en vez de «unos 3 cm»). Es el precio de que el +round-trip de la Fase 2 pueda aspirar al 95 % de fidelidad. + +## 5. `Document` externamente etiquetado en serde + +**Decisión**: `#[serde(rename_all = "kebab-case")]` sin `tag`, a diferencia de `Block` e +`Inline`, que sí son adyacentemente etiquetados. + +**Motivo**: un enum *internamente* etiquetado hace que serde almacene el contenido en un búfer +`Content` intermedio, y ese búfer **reescribe toda clave de mapa como cadena**. Eso rompe los +mapas con clave entera de `Workbook` (`cols`, `rows`). Es un fallo que sólo aparece al +deserializar y con un mensaje poco claro (`invalid type: string "0", expected u32`). + +**Relacionado**: `CellRef` implementa `Serialize`/`Deserialize` a mano como su referencia A1 +(`"B2"`), para poder ser clave de mapa y de paso dejar el `inspect --json` legible. + +## 6. `serde_json` con la feature `float_roundtrip` + +**Decisión**: activada en el workspace. + +**Motivo**: sin ella, el parser rápido de coma flotante de `serde_json` puede desplazar un valor +un ULP. Sobre celdas de hoja de cálculo eso es **corromper datos del usuario en silencio**. Se +detectó comparando IR antes y después de un round-trip JSON en el test de proptest. + +## 7. Hash no criptográfico para los assets + +**Decisión**: FNV-1a de 64 bits, mezclando la longitud, renderizado en 16 dígitos hex. + +**Motivo**: el id sólo tiene que ser estable y libre de colisiones en la práctica para los medios +de un documento. Evita meter `sha2` en `docsai-model`, que es el crate que debe permanecer +ligero. El trait `AssetStore` permite que la capa de conversión sustituya el hash sin tocar el +modelo. + +**Cuándo revisarlo**: si algún día el nombre del asset se usa como frontera de confianza. Hoy no +lo es. + +## 8. Sniffing de imágenes propio, sin el crate `image` + +**Decisión**: leer la cabecera de PNG, JPEG, GIF, BMP, TIFF, WebP, EMF y WMF a mano. + +**Motivo**: docsai **nunca recodifica** un mapa de bits; sólo necesita nombrarlo (extensión y +content-type) y medirlo (dimensiones nativas). Eso son unas decenas de líneas contra un árbol de +dependencias considerable en el binario final. + +**Detalle de seguridad**: la extensión sale del **contenido**, nunca del nombre que traiga el +documento. + +## 9. Front matter YAML escrito a mano + +**Decisión**: sin librería YAML. + +**Motivo**: la spec exige determinismo byte a byte, el esquema es pequeño y conocido, y +`serde_yaml` está sin mantenimiento activo. Escribirlo a mano da control total sobre el orden de +las claves y el entrecomillado. + +**Coste**: hay que mantener el emisor a mano. Cuando la Fase 2 escriba el parser, ambos lados +tendrán que moverse juntos; el test de idempotencia lo hará evidente si se desincronizan. + +## 10. Golden files como ficheros de texto, no snapshots de `insta` + +**Decisión**: `corpus/docx/.expected.dmk.md` junto al documento. + +**Motivo**: es lo que prescribe `AGENTS.md` §6, y un golden que es DocMark legible se revisa en +el diff como cualquier otro texto. Un `.snap` de `insta` añade un formato intermedio entre el +revisor y lo que el programa produce. + +Actualizar es deliberado: `DOCSAI_UPDATE_GOLDENS=1 cargo test -p docsai-convert --test goldens`, +y **se revisa el diff**. + +## 11. Corpus generado, no dibujado a mano + +**Decisión**: `corpus/generate.py` produce los 20 ficheros; CI comprueba que el árbol está al día. + +**Motivo**: el XML de cada documento vive en el generador, donde se revisa; un `.docx` hecho con +Word es opaco en la revisión. Los paquetes salen con marca de tiempo y orden de miembros fijos, +así que regenerar da archivos idénticos byte a byte y el repositorio no acumula ruido binario. +Los medios se sintetizan en Python puro para que funcione igual en las tres plataformas de CI. + +**Coste**: son documentos *mínimos*, no documentos de Word reales. Los reales anonimizados siguen +pendientes (Fase 1, tarea 10). La decisión valió su precio de inmediato: el spike destapó que el +generador colocaba `w:drawing` fuera de un `w:r`, que es OOXML inválido, antes de que eso +contaminara ningún golden. + +## 12. `w:sdt` se aplana en lugar de conservarse opaco + +**Decisión**: un control de contenido emite sus bloques hijos y una advertencia `Degraded`, en +vez de un raw-block con todo dentro. + +**Motivo**: meterlo entero en un raw-block escondería texto perfectamente legible detrás de un +bloque opaco, y en modo `plain` desaparecería del todo. El texto es lo que el usuario quiere ver. + +**Coste**: se pierden las propiedades del control (`w:alias`, binding de datos). Está reportado, +no es silencioso, y la Fase 2 tendrá que decidir si las necesita. + +## 13. La advertencia es parte de la salida, no un adorno + +Cada degradación emite una `Warning` **tipada** con severidad. La CLI las cuenta en stderr, las +detalla con `--verbose` y las serializa con `--json`; el código de salida 1 marca una conversión +que perdió algo. `--strict` sube el listón para que cuenten también las menores. + +Es la regla 3 de `AGENTS.md` hecha código: **nada se degrada en silencio**. + +## 14. Límites de seguridad desde el principio + +Aunque el endurecimiento es la Fase 8, tres cosas eran demasiado baratas como para posponerlas: + +- **Tope de descompresión**: 512 MB por paquete, 128 MB por parte. +- **Límite de profundidad XML**: 256 niveles. +- **Saneado de nombres de miembro**: un `word/media/../../evil.png` nunca llega a ser una parte + ni un asset. Hay un test dedicado. diff --git a/kb/04-siguientes-fases.md b/kb/04-siguientes-fases.md new file mode 100644 index 0000000..f62811b --- /dev/null +++ b/kb/04-siguientes-fases.md @@ -0,0 +1,162 @@ +# 04 — Consideraciones para las siguientes fases + +Lo que las fases siguientes van a encontrarse ya resuelto, lo que les espera, y las trampas que +ya se conocen. + +--- + +## Fase 2 — Escritura DocMark → DOCX + round-trip + +Es la fase inmediata. Cierra el ciclo y monta la infraestructura de fidelidad. + +### Lo que ya está hecho para ella + +| | | +|---|---| +| **La spec está congelada** | v1.0, con el registro de cambios en §10. El parser tiene un contrato fijo contra el que trabajar, no un borrador móvil | +| **El serializador es determinista** | El test de idempotencia `serialize(parse(md)) == md` tiene la mitad de la ecuación ya garantizada | +| **Las longitudes no pierden** | Toda longitud escrita se relee exacta. Sin esto, la métrica de fidelidad tendría un suelo artificial | +| **El IR es completo** | `Block::TextBox`, `Workbook`, anclajes de hoja… ya existen, aunque el lector docx no los produzca todos | +| **El validador de invariantes** | Ya se invoca en cada conversión; el parser hereda esa red | +| **Los raw-blocks son exactos** | Guardan los bytes originales, así que la re-inyección `format=ooxml` es copiar y pegar | +| **Los goldens son la referencia** | 14 documentos con su DocMark esperado: el parser tiene 14 entradas conocidas y su IR esperado | + +### Lo que hay que construir + +1. **Parser DocMark** (`docsai-docmark`): comrak + capa propia de atributos `{...}` y fenced divs + `:::`. Comrak ya está en el árbol como dependencia de test, así que la evaluación está hecha. +2. **Writer docx** (`docsai-office`): IR → `document.xml`, `styles.xml`, `numbering.xml`, medios y + propiedades. +3. **Comando `roundtrip`** con diff estructural del IR normalizado y métrica por categoría. +4. **Test de idempotencia** del serializador sobre todos los goldens. +5. **Property testing**: IR aleatorio → md → IR debe ser la identidad. El generador de IRs + arbitrarios **ya existe** en `crates/docsai-model/tests/json_roundtrip.rs` y se puede reutilizar + casi tal cual. + +### Trampas ya identificadas + +- **El escapado tiene que ser reversible, no sólo correcto.** `escape()` escapa `\` primero, de + modo que desescapar es determinista. El test `escaping_is_idempotent_in_shape` fija esa + propiedad; el parser tiene que ser su inverso exacto. +- **Los tres modos de fidelidad no son simétricos.** Sólo `full` es reversible. `standard` y + `plain` pierden a propósito, y el `roundtrip` sólo tiene sentido sobre `full`. +- **`[]{.empty}`, `[]{.break kind=page}` y `{.field ...}` son sintaxis propia** por encima de + CommonMark: el parser los necesita explícitamente. Están en la spec §3.1, §3.2 y §10. +- **Un ítem de lista nunca lleva dos bloques `{...}`.** `list=` va dentro del bloque de atributos + del primer ítem. +- **El writer docx tiene que volver a poner las celdas absorbidas.** El IR marca `covered=true` y + `colspan`/`rowspan` en la celda que abre el área; OOXML espera `w:gridSpan` y `w:vMerge`. +- **Los cuadros de texto siguen siendo raw-block.** Si la Fase 2 quiere `::: {.textbox}` de + verdad, hay que ampliar antes el lector — está anotado en el plan. +- **Decidir si `docx-rs` sirve como writer.** Sigue abierto. El spike R1 sólo cerró la lectura. +- **Validar en Word y LibreOffice** es criterio de aceptación y no hay LibreOffice en el entorno + de CI actual: hay que preverlo (checklist manual por release, o `soffice` headless en el runner + Linux). + +--- + +## Fase 3 — Hojas de cálculo (XLSX/XLS) + +### Lo que ya está hecho + +- **El corpus xlsx existe**: `values-types`, `formulas-basic`, `formulas-shared`, + `number-formats`, `merged-cells` e `images-anchored` (los tres anclajes). Generado en la Fase 0 + precisamente para no tener que inventarlo con prisa. +- **El lado `Workbook` del IR está definido y probado**: `Sheet`, `Cell`, `CellValue`, `Formula` + con su dialecto, `NumFmt`, `CellRef` con notación A1 en ambos sentidos, `ColProps`, `RowProps`, + `Pane`, `DefinedName`. Todo con round-trip JSON verificado. +- **Los tres anclajes de hoja** están en el modelo de imágenes y el validador ya rechaza usarlos + fuera de un `Workbook`. +- **`float_roundtrip`** está activado: los valores numéricos no se corrompen al pasar por JSON. + +### Lo que hay que decidir y construir + +1. **Spike del writer xlsx**: `umya-spreadsheet` contra `rust_xlsxwriter`. Criterio: cuál + regenera con mayor fidelidad estilos + `numFmt` desde el IR. Documentar en `docs/spikes/`. +2. **Serialización de hojas a DocMark** (spec §4): la sintaxis está especificada pero **no + implementada**. `docsai_docmark::serialize` devuelve hoy front matter y una advertencia + `Degraded` explícita para `Document::Workbook`; ese es el punto de entrada. +3. **Lectura de `xl/drawings/drawing*.xml`** propia con `quick-xml`: ni `calamine` ni `umya` + exponen la geometría completa. El módulo `docx/drawing.rs` es el patrón a imitar; el modelo de + destino es el mismo. + +### Trampas ya identificadas + +- **Streaming.** El árbol XML en memoria es cómodo para documentos pero el presupuesto de la hoja + de 100 k celdas es < 3 s y < 500 MB. Habrá que usar `calamine` (que es perezoso por hoja) para + valores y reservar el árbol propio para partes pequeñas (`styles.xml`, `drawing*.xml`). +- **Las fechas son números de serie más formato.** El IR guarda `CellValue::DateTime` en ISO-8601 + para que una edición a mano del DocMark no pueda corromperlas; la conversión de ida y vuelta + serial ⇄ ISO es responsabilidad del lector y el escritor. +- **En anclajes `two-cell` no se serializan `width`/`height`** (spec §4.1): el tamaño lo define la + rejilla. El escritor de atributos ya lo contempla. +- **Las fórmulas conservan su dialecto**, no se traducen (riesgo R5). `FormulaDialect` ya está en + el IR. + +--- + +## Fase 4 — ODF (ODT y ODS) + +- `docsai-odf` es un esqueleto con la constante `FORMATS`; la regla de dependencia ya la impone + el compilador. +- El punto duro conocido es la **des-automatización de estilos**: los `office:automatic-styles` + de ODF representan formato directo, y hay que mapearlos a deltas del IR. El modelo + «referencia + delta» está preparado para recibirlos: `FontProps::minus()` es exactamente la + operación inversa. +- `detect()` ya reconoce ODF por `content.xml` + `mimetype`; sólo desempata `.odt` de `.ods` por + extensión, lo que habrá que afinar leyendo el `mimetype`. +- El modelo de imágenes tiene ya los conceptos que ODF necesita (`as-char`/`char`/`paragraph`/ + `page` → `Inline`/`Floating`, `fo:clip` → `CropRect`). + +--- + +## Fase 5 — DOC legado + +- `detect()` ya reconoce el contenedor OLE2 y desempata `.doc` de `.xls` por nombre; queda + sustituirlo por lectura del directorio CFB. +- La estrategia de dos niveles del análisis §1.3 sigue en pie: fallback a LibreOffice headless + y extractor nativo degradado. +- El `Warning::ImageGeometryDegraded` que necesita el extractor nativo ya existe y ya se usa + (para VML), así que el patrón está establecido. + +--- + +## Fases 6 a 9 — Producto, MCP y endurecimiento + +- **CLI (Fase 6)**: `convert` y `formats` existen con `--fidelity`, `--assets-dir`, `--json`, + `--strict`, `--verbose` y los códigos de salida de arquitectura §5. Faltan `inspect`, + `roundtrip`, `--style-map`, stdin/stdout con `-`, el procesado por lotes y `cargo-dist`. +- **MCP (Fase 7)**: `docsai-mcp` declara las cuatro tools. La regla de stdout limpio ya se + respeta en la CLI (`tracing` escribe siempre a stderr), así que el test automático de esa + garantía tiene sentido desde ya. +- **Endurecimiento (Fase 8)**: parte del trabajo está adelantado —topes de descompresión, límite + de profundidad XML, saneado de rutas, 900+ inputs corruptos en CI—. Lo que falta es `cargo-fuzz` + de verdad, la suite adversaria completa, los benchmarks con `criterion` y `cargo audit`/`deny`. + El corpus existente sirve de semilla para el fuzzing. + +--- + +## Deuda técnica conocida + +Ninguna de estas cosas bloquea nada, pero conviene tenerlas anotadas: + +| Punto | Dónde | Nota | +|---|---|---| +| Documentos reales anonimizados | Fase 1, tarea 10 | El corpus sintético cubre los rasgos uno a uno, pero no las rarezas del mundo real | +| Cuadros de texto como raw-block | `docx/drawing.rs` | El tipo del IR y la sintaxis de la spec existen; falta emitir y reconstruir | +| `w:lvlOverride` no modelado | `docx/numbering.rs` | Emite `Warning::Degraded` | +| Comentarios ignorados | `docx/body.rs` | Fuera de alcance v1; emite advertencia | +| Efectos DrawingML | `docx/drawing.rs` | Se detectan y se avisa, pero aún no se vuelcan a `effects_raw` | +| Árbol XML en memoria | `office/xml.rs` | Revisar para hojas grandes en la Fase 3 | +| Hash FNV para assets | `model/assets.rs` | Suficiente hoy; revisar si el nombre pasa a ser frontera de confianza | + +## Reglas que no conviene romper + +1. **No adelantar fases.** El orden del plan responde a dependencias reales. +2. **No cambiar la spec DocMark para hacer pasar un test.** Está congelada en v1.0; cualquier + cambio sube la versión del front matter y documenta la migración. +3. **Nada se degrada en silencio.** Toda pérdida es una `Warning` tipada. +4. **Los parsers nunca entran en pánico.** Hay un test que lo comprueba con 900+ entradas + corruptas; que siga pasando. +5. **El serializador es determinista.** `BTreeMap` siempre, `HashMap` nunca, en cualquier camino + que llegue a la salida. diff --git a/kb/README.md b/kb/README.md new file mode 100644 index 0000000..956271c --- /dev/null +++ b/kb/README.md @@ -0,0 +1,37 @@ +# Base de conocimiento de `docsai` + +Documentación de trabajo sobre **lo que hay construido**, cómo está organizado y qué hay que +tener en cuenta al abordar las fases siguientes. Complementa —no sustituye— la documentación de +diseño de [`docs/`](../docs/), que describe el proyecto completo tal como se concibió. + +Diferencia práctica entre ambas carpetas: + +| Carpeta | Responde a | +|---|---| +| [`docs/`](../docs/) | Qué se quiere construir y por qué: análisis, spec DocMark, arquitectura, plan por fases | +| `kb/` (esta) | Qué está construido hoy, cómo, y qué se sabe ya sobre lo que viene | + +## Índice + +| Documento | Contenido | +|---|---| +| [01 — Resumen de las fases 0 y 1](01-resumen-fases-0-1.md) | Qué se entregó, criterios de aceptación, qué quedó fuera y por qué | +| [02 — Estructura del proyecto](02-estructura-proyecto.md) | Crates, módulos, reglas de dependencia y por dónde entra cada cosa | +| [03 — Decisiones técnicas](03-decisiones-tecnicas.md) | Las decisiones no obvias, con su motivo y su coste | +| [04 — Consideraciones para las siguientes fases](04-siguientes-fases.md) | Lo que la Fase 2 y posteriores encontrarán ya resuelto, y lo que les espera | + +## Estado en una línea + +**Fases 0 y 1 cerradas**: `docsai convert x.docx -o x.dmk.md` funciona con estilos, listas, +tablas, imágenes (geometría completa), cabeceras, pies, notas al pie, campos y propiedades. La +vuelta (DocMark → `.docx`) es la Fase 2 y no existe todavía. + +## Comprobaciones rápidas + +```bash +cargo test --workspace # 164 tests +cargo clippy --workspace --all-targets -- -D warnings +cargo fmt --all -- --check +python3 corpus/generate.py --check # el corpus está al día +cargo run -p docsai-cli -- formats # matriz de soporte real del binario +```