From 4d620da1f4da9bc01968ad73eab492690a1f4b55 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 2 Aug 2026 07:57:14 +0000 Subject: [PATCH 1/4] docs: translate project docs to English and enforce English-only policy --- AGENTS.md | 257 +++++----- README.md | 151 +++--- corpus/README.md | 126 ++--- crates/docsai-docmark/src/attrs.rs | 172 +++++++ crates/docsai-docmark/src/error.rs | 54 ++ crates/docsai-docmark/src/escape.rs | 46 ++ crates/docsai-docmark/src/yaml.rs | 476 ++++++++++++++++++ crates/docsai-model/src/units.rs | 58 ++- ...rquitectura.md => _es_arquitectura_src.md} | 0 docs/architecture.md | 252 ++++++++++ ...plan-desarrollo.md => development-plan.md} | 0 ...on-docmark.md => docmark-specification.md} | 0 ...estrategia-docx.md => R1-docx-strategy.md} | 0 ...lisis-tecnico.md => technical-analysis.md} | 0 kb/01-phases-0-1-summary.md | 112 +++++ kb/01-resumen-fases-0-1.md | 112 ----- kb/02-estructura-proyecto.md | 155 ------ kb/02-project-structure.md | 155 ++++++ kb/03-decisiones-tecnicas.md | 177 ------- kb/03-technical-decisions.md | 175 +++++++ kb/04-next-phases.md | 160 ++++++ kb/04-siguientes-fases.md | 162 ------ kb/README.md | 42 +- 23 files changed, 1942 insertions(+), 900 deletions(-) create mode 100644 crates/docsai-docmark/src/error.rs create mode 100644 crates/docsai-docmark/src/yaml.rs rename docs/{arquitectura.md => _es_arquitectura_src.md} (100%) create mode 100644 docs/architecture.md rename docs/{plan-desarrollo.md => development-plan.md} (100%) rename docs/{especificacion-docmark.md => docmark-specification.md} (100%) rename docs/spikes/{R1-estrategia-docx.md => R1-docx-strategy.md} (100%) rename docs/{analisis-tecnico.md => technical-analysis.md} (100%) create mode 100644 kb/01-phases-0-1-summary.md delete mode 100644 kb/01-resumen-fases-0-1.md delete mode 100644 kb/02-estructura-proyecto.md create mode 100644 kb/02-project-structure.md delete mode 100644 kb/03-decisiones-tecnicas.md create mode 100644 kb/03-technical-decisions.md create mode 100644 kb/04-next-phases.md delete mode 100644 kb/04-siguientes-fases.md diff --git a/AGENTS.md b/AGENTS.md index 0b46dda..b389288 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,144 +1,157 @@ -# AGENTS.md — Guía para desarrolladores y agentes de IA - -Este fichero es la referencia operativa para cualquier persona o agente de IA que trabaje -en el repositorio `docsai`. Léelo completo antes de tocar código. - -## 1. Qué es este proyecto - -`docsai` es un binario Rust multiplataforma (Windows/Linux/macOS) que convierte documentos -Office (`.doc`, `.docx`, `.xls`, `.xlsx`) y LibreOffice (`.odt`, `.ods`) a un Markdown -extendido llamado **DocMark**, y de vuelta, con pérdida mínima de formato. Se invoca como -CLI o como servidor **MCP por stdio**. - -**Estado actual**: **Fases 0 y 1 cerradas**. Existe el workspace con los siete crates, el IR -(`docsai-model`), el lector `.docx` (`docsai-office`), el serializador DocMark -(`docsai-docmark`), la orquestación (`docsai-convert`) y la CLI con `convert` y `formats`. -`docsai-odf` y `docsai-mcp` son esqueletos que sólo fijan las reglas de dependencia. - -Lo siguiente es la **Fase 2**: parser DocMark, writer `.docx` y el comando `roundtrip`. - -## 2. Documentos que debes leer antes de implementar - -Orden de lectura obligatorio: - -1. `docs/plan-desarrollo.md` — el plan por fases. **Identifica en qué fase está el proyecto - antes de escribir nada.** No implementes elementos de fases futuras. -2. `docs/arquitectura.md` — estructura del workspace, modelo IR, contratos entre crates. -3. `docs/especificacion-docmark.md` — el formato Markdown extendido. Es un contrato: - cualquier cambio requiere subir la versión del campo `docmark` del front matter y - documentar la migración. -4. `docs/analisis-tecnico.md` — por qué se eligió cada librería. No sustituyas una - dependencia clave (calamine, docx-rs, rmcp, comrak…) sin dejar constancia escrita del - motivo en ese documento. -5. `kb/` — base de conocimiento de lo **ya construido**: resumen de las fases cerradas, - estructura real del código, decisiones técnicas con su motivo, y las consideraciones para - la fase que vayas a abordar. Empieza por `kb/README.md`. Si tu cambio altera la estructura - o toma una decisión técnica no obvia, actualízala en el mismo PR. - -## 3. Estructura del repositorio (objetivo) +# AGENTS.md — Guide for developers and AI agents + +This file is the operational reference for anyone or any AI agent working +on the `docsai` repository. Read it fully before touching code. + +## 1. What this project is + +`docsai` is a cross-platform Rust binary (Windows/Linux/macOS) that converts +Office documents (`.doc`, `.docx`, `.xls`, `.xlsx`) and LibreOffice documents +(`.odt`, `.ods`) to an extended Markdown called **DocMark**, and back, with +minimal format loss. It is invoked as a CLI or as an **MCP server over stdio**. + +**Current status**: **Phases 0 and 1 are closed**. The workspace with the seven +crates exists, along with the IR (`docsai-model`), the `.docx` reader +(`docsai-office`), the DocMark serializer (`docsai-docmark`), orchestration +(`docsai-convert`), and the CLI with `convert` and `formats`. `docsai-odf` and +`docsai-mcp` are skeletons that only fix the dependency rules. + +Next up is **Phase 2**: DocMark parser, `.docx` writer, and the `roundtrip` +command. + +## 2. Documents you must read before implementing + +Mandatory reading order: + +1. `docs/development-plan.md` — the phased plan. **Identify which phase the + project is in before writing anything.** Do not implement items from future + phases. +2. `docs/architecture.md` — workspace structure, IR model, contracts between + crates. +3. `docs/docmark-specification.md` — the extended Markdown format. It is a + contract: any change requires bumping the `docmark` field version in the + front matter and documenting the migration. +4. `docs/technical-analysis.md` — why each library was chosen. Do not replace a + key dependency (calamine, docx-rs, rmcp, comrak…) without leaving a written + record of the reason in that document. +5. `kb/` — knowledge base of what is **already built**: summary of closed + phases, real code structure, technical decisions with their rationale, and + considerations for the phase you are about to tackle. Start with + `kb/README.md`. If your change alters the structure or takes a non-obvious + technical decision, update it in the same PR. + +## 3. Repository structure (target) ``` docsai/ -├── Cargo.toml # workspace raíz +├── Cargo.toml # root workspace ├── crates/ -│ ├── docsai-model/ # IR: modelo de documento intermedio (sin I/O, sin deps pesadas) -│ ├── docsai-docmark/ # serializador + parser de DocMark (IR ⇄ Markdown extendido) -│ ├── docsai-office/ # lectores/escritores OOXML: docx, xlsx (+ xls, doc lectura) -│ ├── docsai-odf/ # lectores/escritores ODF: odt, ods -│ ├── docsai-convert/ # orquestación: pipelines, detección de formato, assets, informes de fidelidad -│ ├── docsai-cli/ # binario CLI (clap) -│ └── docsai-mcp/ # servidor MCP stdio (rmcp) -├── docs/ # documentación de diseño (este conjunto) -├── corpus/ # documentos de prueba versionados (ver §6) -└── tests/ # tests de integración y round-trip +│ ├── docsai-model/ # IR: intermediate document model (no I/O, no heavy deps) +│ ├── docsai-docmark/ # DocMark serializer + parser (IR ⇄ extended Markdown) +│ ├── docsai-office/ # OOXML readers/writers: docx, xlsx (+ xls, doc read) +│ ├── docsai-odf/ # ODF readers/writers: odt, ods +│ ├── docsai-convert/ # orchestration: pipelines, format detection, assets, fidelity reports +│ ├── docsai-cli/ # CLI binary (clap) +│ └── docsai-mcp/ # MCP stdio server (rmcp) +├── docs/ # design documentation (this set) +├── corpus/ # versioned test documents (see §6) +└── tests/ # integration and round-trip tests ``` -Reglas de dependencia entre crates (violarlas es un error de arquitectura): +Dependency rules between crates (violating them is an architecture error): -- `docsai-model` no depende de ningún otro crate del workspace. -- `docsai-docmark`, `docsai-office`, `docsai-odf` dependen **solo** de `docsai-model`. -- `docsai-convert` depende de los cuatro anteriores. `docsai-cli` y `docsai-mcp` - dependen solo de `docsai-convert` (y `docsai-model` para tipos). -- Ningún crate de formato importa a otro crate de formato. +- `docsai-model` depends on no other crate in the workspace. +- `docsai-docmark`, `docsai-office`, `docsai-odf` depend **only** on `docsai-model`. +- `docsai-convert` depends on the four above. `docsai-cli` and `docsai-mcp` + depend only on `docsai-convert` (and `docsai-model` for types). +- No format crate imports another format crate. -## 4. Comandos de trabajo +## 4. Working commands -Cuando exista el workspace, estos son los comandos canónicos (deben mantenerse verdes): +When the workspace exists, these are the canonical commands (they must stay green): ```bash cargo build --workspace -cargo test --workspace # unit + integración + round-trip +cargo test --workspace # unit + integration + round-trip cargo clippy --workspace --all-targets -- -D warnings cargo fmt --all -- --check cargo run -p docsai-cli -- convert corpus/docx/basic-styles.docx -o /tmp/out.dmk.md -python3 corpus/generate.py --check # el corpus es generado; CI comprueba que está al día +python3 corpus/generate.py --check # the corpus is generated; CI checks it is up to date ``` -Actualizar un golden es un acto deliberado, y su diff se revisa a mano: +Updating a golden is a deliberate act, and its diff is reviewed by hand: ```bash DOCSAI_UPDATE_GOLDENS=1 cargo test -p docsai-convert --test goldens ``` -CI (GitHub Actions) ejecuta la matriz `{ubuntu-latest, windows-latest, macos-latest}` × -`{stable}`. Un PR no se fusiona con CI en rojo. - -## 5. Convenciones de código - -- **Edición Rust 2021+ / toolchain stable**. Nada de `nightly` en el árbol principal. -- `rustfmt` por defecto y `clippy -D warnings`. Sin excepciones sin comentario `#[allow]` justificado. -- Errores: `thiserror` en las bibliotecas, `anyhow` solo en los binarios (`docsai-cli`, `docsai-mcp`). -- Logging: `tracing`. En el servidor MCP **jamás escribir en stdout** salvo el protocolo - (stdout es el canal JSON-RPC); logs siempre a stderr. -- Nombres de código, mensajes de commit y comentarios de código en **inglés**; - la documentación de `docs/` se mantiene en español. -- Commits: Conventional Commits (`feat:`, `fix:`, `docs:`, `test:`, `refactor:`, `chore:`), - con ámbito de crate cuando aplique: `feat(office): read numbering.xml`. -- `unsafe` prohibido salvo justificación documentada en el PR (no debería ser necesario). -- Toda función pública de los crates de biblioteca lleva doc-comment con ejemplo cuando sea razonable. - -## 6. Estrategia de pruebas (resumen; detalle en el plan §Fase 0 y §Fase 8) - -- **Corpus versionado** en `corpus/`: documentos pequeños creados a propósito, un rasgo - por fichero (`basic-styles.docx`, `nested-lists.docx`, `formulas-basic.xlsx`…). Nunca - añadir documentos con datos reales o privados. -- **Golden files**: cada documento del corpus tiene su DocMark esperado al lado - (`.expected.dmk.md`). Los tests comparan la salida con el golden; actualizar un golden - requiere revisar el diff a mano. -- **Round-trip tests**: `Office → DocMark → Office → DocMark` debe producir DocMark - idéntico en la segunda pasada (idempotencia). La comparación primera-vs-segunda pasada - del fichero Office se hace a nivel de IR normalizado, no de bytes. -- **Fuzzing** (`cargo-fuzz`) sobre los parsers de entrada a partir de la Fase 8; los - parsers nunca deben entrar en pánico con entrada corrupta: siempre `Err`, nunca `panic!`. - -## 7. Reglas específicas para agentes de IA - -1. **No amplíes el alcance.** Si la tarea pide la Fase N, no adelantes trabajo de la fase - N+1 "ya que estás". El plan define el orden por dependencias reales. -2. **No cambies la especificación DocMark para hacer pasar un test.** Si el formato no - puede representar algo, documenta la limitación y usa el mecanismo `raw-block` descrito - en la especificación. -3. **Nunca degrades la fidelidad en silencio.** Toda pérdida de información en una - conversión debe emitir una advertencia estructurada (ver `ConversionReport` en - `docs/arquitectura.md`). -4. **No añadas dependencias pesadas sin justificación.** El objetivo es un binario único y - razonablemente pequeño. Antes de añadir un crate: ¿está mantenido?, ¿es pure-Rust?, - ¿qué añade al tamaño del binario? Deja la justificación en el PR. -5. **Documenta al terminar.** Si tu cambio altera comportamiento visible (CLI, formato, - tools MCP), actualiza README.md y el documento de `docs/` correspondiente en el mismo PR. -6. **Verifica en las tres plataformas mentales.** Rutas con `std::path` (nunca concatenar - con `/`), finales de línea (el serializador DocMark emite siempre `\n`; el parser acepta - `\r\n`), y nada de dependencias de herramientas POSIX en el código de producción. -7. Trabaja en ramas y push a la rama que se te indique; no hagas push a `main`. - -## 8. Definición de "hecho" (Definition of Done) - -Una tarea/fase se considera terminada cuando: - -- [ ] Compila y pasa `cargo test --workspace` en las tres plataformas de CI. -- [ ] `clippy` y `fmt` limpios. -- [ ] Tests nuevos cubren el comportamiento añadido (incluidos casos de error). -- [ ] Golden files y corpus actualizados si aplica. -- [ ] Documentación actualizada (README, docs/, `--help` de la CLI). -- [ ] Los criterios de aceptación de la fase correspondiente en `docs/plan-desarrollo.md` - están marcados y verificados. +CI (GitHub Actions) runs the matrix `{ubuntu-latest, windows-latest, macos-latest}` × +`{stable}`. A PR is not merged with red CI. + +## 5. Code conventions + +- **Rust edition 2021+ / stable toolchain**. No `nightly` in the main tree. +- Default `rustfmt` and `clippy -D warnings`. No exceptions without a justified + `#[allow]` comment. +- Errors: `thiserror` in libraries, `anyhow` only in binaries (`docsai-cli`, + `docsai-mcp`). +- Logging: `tracing`. In the MCP server **never write to stdout** except for the + protocol (stdout is the JSON-RPC channel); logs always go to stderr. +- **Hard language rule: everything is English.** All documentation (`docs/`, + `kb/`, `README.md`, `AGENTS.md`), code identifiers, commit messages, and code + comments **must** be in English. Do not write Spanish (or any other language) + in those places. +- Commits: Conventional Commits (`feat:`, `fix:`, `docs:`, `test:`, `refactor:`, + `chore:`), with crate scope when applicable: `feat(office): read numbering.xml`. +- `unsafe` is forbidden unless justified in the PR documentation (it should not + be necessary). +- Every public function in library crates has a doc-comment with an example when + reasonable. + +## 6. Testing strategy (summary; detail in the plan §Phase 0 and §Phase 8) + +- **Versioned corpus** in `corpus/`: small documents created on purpose, one + trait per file (`basic-styles.docx`, `nested-lists.docx`, + `formulas-basic.xlsx`…). Never add documents with real or private data. +- **Golden files**: each corpus document has its expected DocMark beside it + (`.expected.dmk.md`). Tests compare output against the golden; updating a + golden requires reviewing the diff by hand. +- **Round-trip tests**: `Office → DocMark → Office → DocMark` must produce + identical DocMark on the second pass (idempotence). First-vs-second pass + comparison of the Office file is done at the normalized IR level, not bytes. +- **Fuzzing** (`cargo-fuzz`) on input parsers from Phase 8 onward; parsers must + never panic on corrupt input: always `Err`, never `panic!`. + +## 7. Specific rules for AI agents + +1. **Do not expand scope.** If the task asks for Phase N, do not advance Phase + N+1 work "while you are at it". The plan defines order by real dependencies. +2. **Do not change the DocMark specification to make a test pass.** If the + format cannot represent something, document the limitation and use the + `raw-block` mechanism described in the specification. +3. **Never degrade fidelity silently.** Every information loss in a conversion + must emit a structured warning (see `ConversionReport` in + `docs/architecture.md`). +4. **Do not add heavy dependencies without justification.** The goal is a single, + reasonably small binary. Before adding a crate: is it maintained? is it + pure-Rust? what does it add to binary size? Leave the justification in the PR. +5. **Document when you finish.** If your change alters visible behavior (CLI, + format, MCP tools), update README.md and the corresponding document under + `docs/` in the same PR. +6. **Verify against the three mental platforms.** Paths with `std::path` (never + concatenate with `/`), line endings (the DocMark serializer always emits + `\n`; the parser accepts `\r\n`), and no POSIX tool dependencies in production + code. +7. Work on branches and push to the branch you are told; do not push to `main`. + +## 8. Definition of Done + +A task/phase is considered finished when: + +- [ ] It compiles and passes `cargo test --workspace` on all three CI platforms. +- [ ] `clippy` and `fmt` are clean. +- [ ] New tests cover the added behavior (including error cases). +- [ ] Golden files and corpus are updated if applicable. +- [ ] Documentation is updated (README, docs/, CLI `--help`). +- [ ] The acceptance criteria for the corresponding phase in + `docs/development-plan.md` are marked and verified. diff --git a/README.md b/README.md index 72ab3cc..6d46462 100644 --- a/README.md +++ b/README.md @@ -1,103 +1,104 @@ # docsai -**Conversor bidireccional de documentos Office / LibreOffice ⇄ Markdown extendido, escrito en Rust.** +**Bidirectional converter between Office / LibreOffice documents and extended Markdown, written in Rust.** -`docsai` es un binario único y multiplataforma (Windows, Linux, macOS) que convierte documentos ofimáticos a un perfil de Markdown extendido — **DocMark** — diseñado para conservar la máxima información posible (estilos, imágenes, propiedades, fórmulas) y permitir la **conversión inversa con pérdida mínima de formato**. Puede usarse como herramienta CLI o como **servidor MCP (Model Context Protocol) por stdio**, para integrarse con asistentes de IA como Claude. +`docsai` is a single cross-platform binary (Windows, Linux, macOS) that converts office documents to an extended Markdown profile — **DocMark** — designed to keep as much information as possible (styles, images, properties, formulas) and to allow the **reverse conversion with minimal format loss**. It can be used as a CLI tool or as an **MCP (Model Context Protocol) server over stdio**, for integration with AI assistants such as Claude. -> **Estado del proyecto: Fases 0 y 1 completadas.** -> Ya se convierte `.docx` → DocMark con estilos, listas, tablas, imágenes (con toda su -> geometría), cabeceras, pies, notas al pie, campos y propiedades. La conversión inversa -> (DocMark → `.docx`) es la Fase 2 y todavía no existe; las hojas de cálculo llegan en la -> Fase 3. Ver [`docs/plan-desarrollo.md`](docs/plan-desarrollo.md). +> **Project status: Phases 0 and 1 completed.** +> `.docx` → DocMark already works with styles, lists, tables, images (full geometry), +> headers, footers, footnotes, fields and properties. The reverse conversion +> (DocMark → `.docx`) is Phase 2 and does not exist yet; spreadsheets arrive in +> Phase 3. See [`docs/development-plan.md`](docs/development-plan.md). ```bash -cargo run -p docsai-cli -- convert informe.docx -o informe.dmk.md +cargo run -p docsai-cli -- convert report.docx -o report.dmk.md cargo run -p docsai-cli -- formats ``` --- -## Formatos soportados (objetivo) +## Supported formats (target) -✅ = funciona hoy · 🕓 = planificado, con la fase que lo trae · ➖ = fuera de alcance +✅ = works today · 🕓 = planned, with the phase that lands it · ➖ = out of scope -| Formato | Extensión | Lectura | Escritura | Notas | +| Format | Extension | Read | Write | Notes | |---|---|---|---|---| -| Word OOXML | `.docx` | ✅ | 🕓 Fase 2 | Estilos, imágenes, tablas, listas, cabeceras/pies, notas al pie, campos, propiedades | -| Word binario | `.doc` | 🕓 Fase 5 | ➖ | Solo lectura (vía parser nativo o fallback LibreOffice headless) | -| Excel OOXML | `.xlsx` | 🕓 Fase 3 | 🕓 Fase 3 | Valores **y fórmulas**, formatos de número, celdas combinadas, imágenes ancladas | -| Excel binario | `.xls` | 🕓 Fase 3 | ➖ | Solo lectura (calamine) | -| OpenDocument Text | `.odt` | 🕓 Fase 4 | 🕓 Fase 4 | Equivalente libre de `.docx` | -| OpenDocument Spreadsheet | `.ods` | 🕓 Fase 4 | 🕓 Fase 4 | Equivalente libre de `.xlsx` | -| Markdown extendido | `.dmk.md` | 🕓 Fase 2 | ✅ | Formato pivote **DocMark** (superconjunto de CommonMark + GFM) | +| Word OOXML | `.docx` | ✅ | 🕓 Phase 2 | Styles, images, tables, lists, headers/footers, footnotes, fields, properties | +| Word binary | `.doc` | 🕓 Phase 5 | ➖ | Read only (native parser or LibreOffice headless fallback) | +| Excel OOXML | `.xlsx` | 🕓 Phase 3 | 🕓 Phase 3 | Values **and formulas**, number formats, merged cells, anchored images | +| Excel binary | `.xls` | 🕓 Phase 3 | ➖ | Read only (calamine) | +| OpenDocument Text | `.odt` | 🕓 Phase 4 | 🕓 Phase 4 | Free equivalent of `.docx` | +| OpenDocument Spreadsheet | `.ods` | 🕓 Phase 4 | 🕓 Phase 4 | Free equivalent of `.xlsx` | +| Extended Markdown | `.dmk.md` | 🕓 Phase 2 | ✅ | Pivot format **DocMark** (superset of CommonMark + GFM) | -`docsai formats` imprime esta misma matriz según lo que el binario realmente sabe hacer. +`docsai formats` prints this same matrix for what the binary can actually do. -La escritura de `.doc` y `.xls` (formatos binarios legados) queda fuera de alcance de forma deliberada: la ruta de salida recomendada hacia el ecosistema Microsoft es siempre OOXML (`.docx` / `.xlsx`). +Writing `.doc` and `.xls` (legacy binary formats) is deliberately out of scope: the recommended output path into the Microsoft ecosystem is always OOXML (`.docx` / `.xlsx`). -## ¿Qué es DocMark? +## What is DocMark? -DocMark es un perfil de Markdown extendido definido en este proyecto (ver [`docs/especificacion-docmark.md`](docs/especificacion-docmark.md)). Es **Markdown legible y editable a mano**, que se renderiza de forma razonable en GitHub o cualquier visor CommonMark, pero que añade capas de metadatos para no perder información: +DocMark is an extended Markdown profile defined in this project (see [`docs/docmark-specification.md`](docs/docmark-specification.md)). It is **human-readable and hand-editable Markdown** that renders reasonably on GitHub or any CommonMark viewer, but adds metadata layers so information is not lost: -- **Front matter YAML** con las propiedades del documento (título, autor, idioma…) y el **catálogo de estilos** original. -- **Atributos inline y de bloque** `{#id .clase clave="valor"}` (sintaxis compatible con Pandoc) para anclar estilos, dimensiones de imagen, propiedades de celda, etc. -- **Contenedores fenced** `::: {...}` para secciones, cuadros de texto, cabeceras y pies. -- **Tablas extendidas** con metadatos por celda (fórmulas, tipos, formatos de número, combinaciones) para hojas de cálculo. -- **Activos externos**: las imágenes se extraen a un directorio `assets/` (deduplicadas por hash de contenido) y se referencian con un modelo completo de atributos de geometría: tamaño mostrado y nativo, posición y anclaje (en línea, flotante con wrap y z-order, o anclada a celdas en hojas de cálculo), rotación, recorte, volteo, texto alternativo e hipervínculo. -- **Escotilla de fidelidad** (`raw-block`) para fragmentos sin representación Markdown posible, que se conservan de forma opaca y se restauran en la conversión inversa. +- **YAML front matter** with document properties (title, author, language…) and the original **style catalogue**. +- **Inline and block attributes** `{#id .class key="value"}` (Pandoc-compatible syntax) to attach styles, image dimensions, cell properties, and more. +- **Fenced containers** `::: {...}` for sections, text boxes, headers and footers. +- **Extended tables** with per-cell metadata (formulas, types, number formats, merges) for spreadsheets. +- **External assets**: images are extracted to an `assets/` directory (deduplicated by content hash) and referenced with a full geometry attribute model: display and native size, position and anchoring (inline, floating with wrap and z-order, or cell-anchored in spreadsheets), rotation, crop, flip, alternative text and hyperlink. +- **Fidelity hatch** (`raw-block`) for fragments with no Markdown representation, kept opaque and restored on the reverse conversion. -Ejemplo mínimo: +Minimal example: ```markdown --- docmark: "1.0" source-format: docx -title: "Informe Anual" +title: "Annual Report" styles: Heading1: { font: "Calibri Light", size: 16pt, color: "#2E74B5" } --- -# Informe Anual {.Heading1} +# Annual Report {.Heading1} -Texto con **negrita** y [color]{color="#FF0000"} personalizado. +Text with **bold** and [color]{color="#FF0000"} custom colour. -![Diagrama de ventas](assets/img-001.png){width=450px height=300px anchor=inline} +![Sales chart](assets/img-001.png){width=450px height=300px anchor=inline} ``` -## Uso (CLI) +## Usage (CLI) -Lo que funciona hoy: +What works today: ```bash -docsai convert informe.docx -o informe.dmk.md # extrae assets/ junto al .md -docsai convert informe.docx # a stdout, sin escribir medios al lado -docsai convert informe.docx --fidelity plain # CommonMark limpio, para LLM/RAG -docsai convert informe.docx -o out.md --json # informe de conversión en JSON -docsai formats # matriz de soporte de este binario +docsai convert report.docx -o report.dmk.md # extracts assets/ next to the .md +docsai convert report.docx # to stdout, without writing media beside it +docsai convert report.docx --fidelity plain # clean CommonMark, for LLM/RAG +docsai convert report.docx -o out.md --json # conversion report as JSON +docsai formats # support matrix for this binary ``` -Grados de fidelidad (`--fidelity`, spec §6): `full` (defecto, apto para ida y vuelta), -`standard` (Markdown rico sin catálogos ni raw-blocks) y `plain` (CommonMark+GFM puro). +Fidelity levels (`--fidelity`, spec §6): `full` (default, round-trip grade), +`standard` (rich Markdown without catalogues or raw-blocks) and `plain` (pure +CommonMark+GFM). -Códigos de salida: `0` correcto, `1` conversión con pérdidas (`--strict` incluye las advertencias -menores), `2` error de entrada, `3` formato no soportado. +Exit codes: `0` success, `1` conversion with losses (`--strict` also treats minor +warnings as failures), `2` input error, `3` unsupported format. -Planificado: +Planned: ```bash -docsai convert informe.dmk.md -o informe.docx # Fase 2 -docsai convert ventas.xlsx -o ventas.dmk.md # Fase 3 -docsai inspect informe.docx # Fase 6 -docsai roundtrip informe.docx # Fase 2 +docsai convert report.dmk.md -o report.docx # Phase 2 +docsai convert sales.xlsx -o sales.dmk.md # Phase 3 +docsai inspect report.docx # Phase 6 +docsai roundtrip report.docx # Phase 2 ``` -## Uso previsto (servidor MCP, Fase 7) +## Intended usage (MCP server, Phase 7) ```bash -docsai mcp # arranca el servidor MCP por stdio +docsai mcp # starts the MCP server over stdio ``` -Registro en un cliente MCP (p. ej. Claude Desktop / Claude Code): +Registration in an MCP client (e.g. Claude Desktop / Claude Code): ```json { @@ -107,41 +108,41 @@ Registro en un cliente MCP (p. ej. Claude Desktop / Claude Code): } ``` -Tools MCP previstas: `convert_to_markdown`, `convert_from_markdown`, `inspect_document`, `list_supported_formats`. Ver detalle en [`docs/arquitectura.md`](docs/arquitectura.md). +Planned MCP tools: `convert_to_markdown`, `convert_from_markdown`, `inspect_document`, `list_supported_formats`. Details in [`docs/architecture.md`](docs/architecture.md). -## Documentación del proyecto +## Project documentation -| Documento | Contenido | +| Document | Contents | |---|---| -| [`docs/analisis-tecnico.md`](docs/analisis-tecnico.md) | Análisis de formatos, librerías Rust evaluadas, proyectos open source previos (Pandoc, MarkItDown, Docling, mammoth…), decisiones y riesgos | -| [`docs/especificacion-docmark.md`](docs/especificacion-docmark.md) | Especificación del formato DocMark (Markdown extendido) v1.0-draft | -| [`docs/arquitectura.md`](docs/arquitectura.md) | Arquitectura del software: workspace de crates, modelo de documento intermedio (IR), CLI, servidor MCP | -| [`docs/plan-desarrollo.md`](docs/plan-desarrollo.md) | Plan de desarrollo detallado en 9 fases, con entregables, criterios de aceptación, estimaciones y estrategia de pruebas | -| [`AGENTS.md`](AGENTS.md) | Guía operativa para desarrolladores y agentes de IA que trabajen en este repositorio | -| [`corpus/README.md`](corpus/README.md) | El corpus de pruebas: qué aísla cada documento y cómo se regenera | -| [`docs/spikes/`](docs/spikes/) | Informes de los spikes de riesgo, con la decisión que cerró cada uno | -| [`kb/`](kb/) | Base de conocimiento: qué hay construido, cómo está estructurado, las decisiones técnicas y lo que espera a las fases siguientes | +| [`docs/technical-analysis.md`](docs/technical-analysis.md) | Format analysis, evaluated Rust libraries, prior open-source projects (Pandoc, MarkItDown, Docling, mammoth…), decisions and risks | +| [`docs/docmark-specification.md`](docs/docmark-specification.md) | DocMark format specification (extended Markdown) v1.0-draft | +| [`docs/architecture.md`](docs/architecture.md) | Software architecture: crate workspace, intermediate document model (IR), CLI, MCP server | +| [`docs/development-plan.md`](docs/development-plan.md) | Detailed development plan in 9 phases, with deliverables, acceptance criteria, estimates and testing strategy | +| [`AGENTS.md`](AGENTS.md) | Operational guide for developers and AI agents working in this repository | +| [`corpus/README.md`](corpus/README.md) | The test corpus: what each document isolates and how it is regenerated | +| [`docs/spikes/`](docs/spikes/) | Risk-spike reports, with the decision that closed each one | +| [`kb/`](kb/) | Knowledge base: what is built, how it is structured, technical decisions and what later phases will face | -## Desarrollo +## Development ```bash cargo test --workspace cargo clippy --workspace --all-targets -- -D warnings cargo fmt --all -- --check -python3 corpus/generate.py --check # el corpus es generado, no dibujado a mano +python3 corpus/generate.py --check # the corpus is generated, not hand-drawn ``` -Los golden files viven junto al corpus (`corpus/docx/*.expected.dmk.md`). Para actualizarlos: -`DOCSAI_UPDATE_GOLDENS=1 cargo test -p docsai-convert --test goldens`, y **se revisa el diff**. +Golden files live next to the corpus (`corpus/docx/*.expected.dmk.md`). To update them: +`DOCSAI_UPDATE_GOLDENS=1 cargo test -p docsai-convert --test goldens`, and **review the diff**. -## Principios de diseño +## Design principles -1. **Binario único, sin runtime externo**: pura biblioteca Rust siempre que sea posible; los fallbacks externos (LibreOffice headless para `.doc`) son opcionales y detectados en tiempo de ejecución, nunca requeridos. -2. **IR pivote**: todos los formatos convergen en un modelo de documento intermedio (inspirado en el AST de Pandoc y en DoclingDocument); los conversores nunca se hablan entre sí directamente. -3. **Fidelidad medible**: la pérdida de formato no se estima, se mide — el comando `roundtrip` y la suite de tests de ida y vuelta forman parte del producto. -4. **Markdown primero legible, después completo**: los metadatos extendidos degradan con elegancia; un visor Markdown normal muestra un documento útil aunque ignore los atributos. -5. **Los datos del usuario nunca se pierden en silencio**: lo que no se pueda representar se conserva en bloques raw o se reporta como advertencia explícita. +1. **Single binary, no external runtime**: pure Rust libraries whenever possible; external fallbacks (LibreOffice headless for `.doc`) are optional and detected at runtime, never required. +2. **Pivot IR**: every format converges on an intermediate document model (inspired by Pandoc's AST and DoclingDocument); converters never talk to each other directly. +3. **Measurable fidelity**: format loss is not estimated, it is measured — the `roundtrip` command and the round-trip test suite are part of the product. +4. **Markdown readable first, complete second**: extended metadata degrades gracefully; a normal Markdown viewer shows a useful document even if it ignores the attributes. +5. **User data is never lost silently**: what cannot be represented is kept in raw blocks or reported as an explicit warning. -## Licencia +## License -Ver [LICENSE](LICENSE). +See [LICENSE](LICENSE). diff --git a/corpus/README.md b/corpus/README.md index 7bd3bdb..a558e16 100644 --- a/corpus/README.md +++ b/corpus/README.md @@ -1,83 +1,87 @@ -# Corpus de pruebas +# Test corpus -Documentos de prueba versionados. **Un rasgo por fichero**: cuando un golden falla, el fichero -que falla señala el área del lector que se ha roto. +Versioned test documents. **One trait per file**: when a golden fails, the failing +file points at the reader area that broke. -Ningún documento contiene datos reales ni privados (`AGENTS.md` §6). +No document contains real or private data (`AGENTS.md` §6). -## Cómo se generan +## How they are generated -Todos los ficheros los produce `corpus/generate.py`, sin dependencias más allá de la biblioteca -estándar de Python 3: +Every file is produced by `corpus/generate.py`, with no dependencies beyond the +Python 3 standard library: ```bash -python3 corpus/generate.py # regenera todo -python3 corpus/generate.py --check # falla si el árbol está desfasado (lo ejecuta CI) +python3 corpus/generate.py # regenerate everything +python3 corpus/generate.py --check # fails if the tree is out of date (run by CI) ``` -Que el corpus sea **generado y no dibujado a mano** es deliberado: +That the corpus is **generated rather than hand-drawn** is deliberate: -- El XML de cada documento vive en el generador, donde se revisa en un `git diff` normal; un - `.docx` hecho con Word es una caja opaca en la revisión. -- Los paquetes se escriben con marca de tiempo y orden de miembros fijos, así que regenerar - produce archivos byte a byte idénticos y el repositorio no acumula ruido binario. -- Los medios (PNG, GIF, EMF) se sintetizan en Python puro, sin Pillow, para que el generador - funcione igual en las tres plataformas de CI. +- The XML of each document lives in the generator, where it is reviewed in a + normal `git diff`; a `.docx` made with Word is an opaque box in review. +- Packages are written with a fixed timestamp and member order, so regenerating + produces byte-identical files and the repository does not accumulate binary + noise. +- Media (PNG, GIF, EMF) are synthesised in pure Python, without Pillow, so the + generator works the same on the three CI platforms. -La contrapartida: son documentos *mínimos*, no documentos de Word reales. Los documentos reales -anonimizados que pide la Fase 1 (tarea 10) y los corpus de rendimiento y adversarios de la Fase 8 -se añadirán aparte; el test de rendimiento de 50 páginas sintetiza su propio documento en tiempo -de ejecución (`crates/docsai-convert/tests/goldens.rs`). +The trade-off: these are *minimal* documents, not real Word documents. The +anonymised real-world documents required by Phase 1 (task 10) and the +performance and adversarial corpora of Phase 8 will be added separately; the +50-page performance test synthesises its own document at runtime +(`crates/docsai-convert/tests/goldens.rs`). ## Golden files -Cada `docx/.docx` tiene al lado su DocMark esperado, `docx/.expected.dmk.md`. -Los comparan los tests de `crates/docsai-convert/tests/goldens.rs`. Para actualizarlos: +Each `docx/.docx` has its expected DocMark beside it, +`docx/.expected.dmk.md`. They are compared by the tests in +`crates/docsai-convert/tests/goldens.rs`. To update them: ```bash DOCSAI_UPDATE_GOLDENS=1 cargo test -p docsai-convert --test goldens ``` -El diff resultante **se revisa a mano** antes de confirmarlo: un golden actualizado sin mirar es -un test que ha dejado de comprobar nada. +The resulting diff **is reviewed by hand** before confirming: a golden updated +without looking is a test that has stopped checking anything. -## Documentos de texto (`docx/`) +## Text documents (`docx/`) -| Fichero | Rasgo que aísla | +| File | Trait it isolates | |---|---| -| `basic-text.docx` | Párrafos, salto de línea manual, párrafo vacío y los caracteres que Markdown escapa | -| `basic-styles.docx` | Negrita, cursiva, tachado, subrayado, color, resaltado, fuente y tamaño, sub/superíndice, hipervínculo, alineación y sangrías | -| `nested-lists.docx` | `numbering.xml` con tres niveles numerados y dos de viñetas; reconstrucción del árbol desde pares `(numId, ilvl)` | -| `table-simple.docx` | Tabla regular con estilo de tabla y rejilla | -| `table-merged.docx` | `gridSpan` y `vMerge` → `colspan`/`rowspan` y celdas absorbidas | -| `images-inline.docx` | Imágenes `wp:inline`: PNG con alt/título/nombre, GIF entre texto, EMF vectorial | -| `images-floating.docx` | `wp:anchor`: offsets relativos al margen con `wrapSquare`, alineación simbólica relativa a la página con `wrapTopAndBottom`, y marca de agua con `behindDoc` | -| `images-transformed.docx` | Rotación 45°, recorte `a:srcRect` con borde `a:ln`, volteo H+V y escala ≠ 100 % | -| `images-duplicated.docx` | El mismo mapa de bits en tres partes distintas del paquete con geometrías distintas: prueba la deduplicación del `AssetStore` | -| `images-vml.docx` | `w:pict` con VML heredado (documentos convertidos desde `.doc`) | -| `headers-footers.docx` | `sectPr` con cabecera por defecto y de primera página, pie con campos `PAGE`/`NUMPAGES`, dos columnas y `titlePg` | -| `footnotes.docx` | `footnotes.xml` con dos notas, una con formato dentro | -| `custom-styles.docx` | Estilo personalizado, estilo heredado con delta directo, estilo de carácter y propiedades personalizadas del documento | -| `fields-raw.docx` | Control de contenido `w:sdt`, campo complejo `TOC` y campo simple `DATE` | - -## Hojas de cálculo (`xlsx/`) - -Generadas en la Fase 0 para que el corpus esté completo; las consume la **Fase 3**, que es cuando -existirá el lector de `xlsx`. - -| Fichero | Rasgo que aísla | +| `basic-text.docx` | Paragraphs, manual line break, empty paragraph and characters that Markdown escapes | +| `basic-styles.docx` | Bold, italic, strike, underline, colour, highlight, font and size, sub/superscript, hyperlink, alignment and indents | +| `nested-lists.docx` | `numbering.xml` with three numbered levels and two bullet levels; tree rebuild from `(numId, ilvl)` pairs | +| `table-simple.docx` | Regular table with table style and grid | +| `table-merged.docx` | `gridSpan` and `vMerge` → `colspan`/`rowspan` and absorbed cells | +| `images-inline.docx` | `wp:inline` images: PNG with alt/title/name, GIF between text, EMF vector | +| `images-floating.docx` | `wp:anchor`: margin-relative offsets with `wrapSquare`, symbolic page-relative alignment with `wrapTopAndBottom`, and watermark with `behindDoc` | +| `images-transformed.docx` | 45° rotation, `a:srcRect` crop with `a:ln` border, H+V flip and scale ≠ 100 % | +| `images-duplicated.docx` | The same bitmap in three distinct package parts with different geometries: tests `AssetStore` deduplication | +| `images-vml.docx` | `w:pict` with legacy VML (documents converted from `.doc`) | +| `headers-footers.docx` | `sectPr` with default and first-page header, footer with `PAGE`/`NUMPAGES` fields, two columns and `titlePg` | +| `footnotes.docx` | `footnotes.xml` with two notes, one with inner formatting | +| `custom-styles.docx` | Custom style, inherited style with direct delta, character style and custom document properties | +| `fields-raw.docx` | `w:sdt` content control, complex `TOC` field and simple `DATE` field | + +## Spreadsheets (`xlsx/`) + +Generated in Phase 0 so the corpus is complete; consumed by **Phase 3**, when +the `xlsx` reader exists. + +| File | Trait it isolates | |---|---| -| `values-types.xlsx` | Los seis tipos de celda: entero, decimal, booleano, error, fecha (serial + `numFmt`) y cadena en línea | -| `formulas-basic.xlsx` | Fórmulas con valor cacheado, referencia entre celdas y un nombre definido | -| `formulas-shared.xlsx` | Fórmulas compartidas (`t="shared"`) y de matriz (`t="array"`) | -| `number-formats.xlsx` | Moneda, fecha, porcentaje y millares por `numFmtId` | -| `merged-cells.xlsx` | `mergeCells` horizontal y vertical, y ancho de columna personalizado | -| `images-anchored.xlsx` | Los tres anclajes de hoja: `twoCellAnchor`, `oneCellAnchor` y `absoluteAnchor` | - -## Añadir un documento - -1. Escribe una función `docx_()` en `generate.py` y añádela a `GENERATORS`. -2. Regenera (`python3 corpus/generate.py`) y añade la fila a la tabla de arriba. -3. Genera su golden y **revisa el diff**. -4. Si el rasgo aún no está implementado, el golden documentará la degradación actual (un - raw-block, por ejemplo). Eso es correcto: hace visible lo que falta. +| `values-types.xlsx` | The six cell types: integer, decimal, boolean, error, date (serial + `numFmt`) and inline string | +| `formulas-basic.xlsx` | Formulas with cached value, cell reference and a defined name | +| `formulas-shared.xlsx` | Shared formulas (`t="shared"`) and array formulas (`t="array"`) | +| `number-formats.xlsx` | Currency, date, percentage and thousands via `numFmtId` | +| `merged-cells.xlsx` | Horizontal and vertical `mergeCells`, and custom column width | +| `images-anchored.xlsx` | The three sheet anchors: `twoCellAnchor`, `oneCellAnchor` and `absoluteAnchor` | + +## Adding a document + +1. Write a `docx_()` function in `generate.py` and add it to `GENERATORS`. +2. Regenerate (`python3 corpus/generate.py`) and add the row to the table above. +3. Generate its golden and **review the diff**. +4. If the trait is not implemented yet, the golden will document the current + degradation (a raw-block, for example). That is correct: it makes the gap + visible. diff --git a/crates/docsai-docmark/src/attrs.rs b/crates/docsai-docmark/src/attrs.rs index 8823361..84ea3e8 100644 --- a/crates/docsai-docmark/src/attrs.rs +++ b/crates/docsai-docmark/src/attrs.rs @@ -99,6 +99,152 @@ impl Attrs { format!(" {}", self.render()) } } + + /// The id, if any (`#id`). + pub fn id_ref(&self) -> Option<&str> { + self.id.as_deref() + } + + /// Classes in insertion order. + pub fn classes(&self) -> &[String] { + &self.classes + } + + /// True when `class` is present. + pub fn has_class(&self, class: &str) -> bool { + self.classes.iter().any(|c| c == class) + } + + /// First class, commonly the style id. + pub fn first_class(&self) -> Option<&str> { + self.classes.first().map(String::as_str) + } + + /// Value of `key`, if present. + pub fn get(&self, key: &str) -> Option<&str> { + self.pairs.get(key).map(String::as_str) + } + + /// Removes and returns the value of `key`. + pub fn take(&mut self, key: &str) -> Option { + self.pairs.remove(key) + } + + /// All key/value pairs in sorted order. + pub fn pairs(&self) -> &BTreeMap { + &self.pairs + } + + /// Parses a Pandoc-style attribute block, with or without surrounding `{}`. + /// + /// ``` + /// use docsai_docmark::attrs::Attrs; + /// let a = Attrs::parse(r#"{#img-1 .Quote title="Figura 1" width=450px}"#).unwrap(); + /// assert_eq!(a.id_ref(), Some("img-1")); + /// assert!(a.has_class("Quote")); + /// assert_eq!(a.get("width"), Some("450px")); + /// assert_eq!(a.get("title"), Some("Figura 1")); + /// ``` + pub fn parse(input: &str) -> Option { + let input = input.trim(); + let inner = if input.starts_with('{') && input.ends_with('}') && input.len() >= 2 { + &input[1..input.len() - 1] + } else { + input + }; + let mut attrs = Attrs::new(); + let mut rest = inner.trim(); + while !rest.is_empty() { + rest = rest.trim_start(); + if rest.is_empty() { + break; + } + if let Some(stripped) = rest.strip_prefix('#') { + let (id, next) = take_token(stripped); + if id.is_empty() { + return None; + } + attrs.id = Some(id.to_string()); + rest = next; + continue; + } + if let Some(stripped) = rest.strip_prefix('.') { + let (class, next) = take_token(stripped); + if class.is_empty() { + return None; + } + attrs.class(class); + rest = next; + continue; + } + let (key, after_key) = take_ident(rest); + if key.is_empty() { + return None; + } + let after_key = after_key.trim_start(); + let Some(after_eq) = after_key.strip_prefix('=') else { + return None; + }; + let after_eq = after_eq.trim_start(); + let (value, next) = take_value(after_eq)?; + attrs.set(key, value); + rest = next; + } + Some(attrs) + } +} + +/// Identifier or class/id token: letters, digits, `_`, `-`, `.` (for class names). +fn take_token(input: &str) -> (&str, &str) { + let end = input + .char_indices() + .find(|(_, c)| !(c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.'))) + .map(|(i, _)| i) + .unwrap_or(input.len()); + (&input[..end], &input[end..]) +} + +fn take_ident(input: &str) -> (&str, &str) { + let end = input + .char_indices() + .find(|(_, c)| !(c.is_ascii_alphanumeric() || matches!(c, '-' | '_'))) + .map(|(i, _)| i) + .unwrap_or(input.len()); + (&input[..end], &input[end..]) +} + +fn take_value(input: &str) -> Option<(String, &str)> { + if let Some(rest) = input.strip_prefix('"') { + let mut out = String::new(); + let mut chars = rest.char_indices(); + while let Some((i, c)) = chars.next() { + match c { + '\\' => match chars.next() { + Some((_, '"')) => out.push('"'), + Some((_, '\\')) => out.push('\\'), + Some((_, 'n')) => out.push('\n'), + Some((_, other)) => { + out.push('\\'); + out.push(other); + } + None => out.push('\\'), + }, + '"' => return Some((out, &rest[i + 1..])), + other => out.push(other), + } + } + None + } else { + let end = input + .char_indices() + .find(|(_, c)| c.is_whitespace()) + .map(|(i, _)| i) + .unwrap_or(input.len()); + if end == 0 { + return None; + } + Some((input[..end].to_string(), &input[end..])) + } } #[cfg(test)] @@ -152,4 +298,30 @@ mod tests { a.set("k", "1").set("k", "2"); assert_eq!(a.render(), "{k=2}"); } + + #[test] + fn parse_round_trips_with_render() { + let samples = [ + r#"{#img-1 .Quote title="Figura 1" width=450px}"#, + "{#x .Alpha .Beta a=2 z=1}", + r#"{color=#FF0000 link="https://example.com/a b" name="Logo corporativo" width=450px}"#, + "{on=true}", + "{.Heading1}", + "{align=center indent-first-line=24px}", + ]; + for sample in samples { + let parsed = Attrs::parse(sample).unwrap_or_else(|| panic!("parse {sample}")); + assert_eq!(parsed.render(), sample, "render mismatch for {sample}"); + } + } + + #[test] + fn parse_getters_and_take() { + let mut a = Attrs::parse(r#"{#id .A .B k=v}"#).unwrap(); + assert_eq!(a.id_ref(), Some("id")); + assert_eq!(a.classes(), &["A".to_string(), "B".to_string()]); + assert_eq!(a.get("k"), Some("v")); + assert_eq!(a.take("k"), Some("v".into())); + assert_eq!(a.get("k"), None); + } } diff --git a/crates/docsai-docmark/src/error.rs b/crates/docsai-docmark/src/error.rs new file mode 100644 index 0000000..0ac82a5 --- /dev/null +++ b/crates/docsai-docmark/src/error.rs @@ -0,0 +1,54 @@ +//! Parser errors for DocMark → IR. + +use std::io; +use std::path::PathBuf; + +/// Something went wrong parsing a DocMark document. +#[derive(Debug, thiserror::Error)] +pub enum ParseError { + #[error("invalid front matter at line {line}: {message}")] + InvalidFrontMatter { line: usize, message: String }, + + #[error("unexpected content at line {line}: {message}")] + Unexpected { line: usize, message: String }, + + #[error("unsupported: {what}")] + Unsupported { what: String }, + + #[error("i/o error{path}: {source}", path = path_suffix(.path))] + Io { + path: Option, + #[source] + source: io::Error, + }, + + #[error("asset error: {0}")] + Asset(#[from] docsai_model::assets::AssetError), +} + +fn path_suffix(path: &Option) -> String { + match path { + Some(p) => format!(" on `{}`", p.display()), + None => String::new(), + } +} + +impl ParseError { + pub fn front_matter(line: usize, message: impl Into) -> Self { + ParseError::InvalidFrontMatter { + line, + message: message.into(), + } + } + + pub fn unexpected(line: usize, message: impl Into) -> Self { + ParseError::Unexpected { + line, + message: message.into(), + } + } + + pub fn io(path: Option, source: io::Error) -> Self { + ParseError::Io { path, source } + } +} diff --git a/crates/docsai-docmark/src/escape.rs b/crates/docsai-docmark/src/escape.rs index b0a6dde..a3774ff 100644 --- a/crates/docsai-docmark/src/escape.rs +++ b/crates/docsai-docmark/src/escape.rs @@ -71,6 +71,32 @@ pub fn escape_attr_value(value: &str) -> String { out } +/// Inverse of [`escape`] for block (and table-cell) content: each backslash +/// escapes the following character, and is dropped. +/// +/// ``` +/// use docsai_docmark::escape::{escape, unescape, TextContext}; +/// let raw = r"a*b_c[d]"; +/// let once = escape(raw, TextContext::Block); +/// assert_eq!(unescape(&once), raw); +/// ``` +pub fn unescape(text: &str) -> String { + let mut out = String::with_capacity(text.len()); + let mut chars = text.chars(); + while let Some(c) = chars.next() { + if c == '\\' { + if let Some(next) = chars.next() { + out.push(next); + } else { + out.push('\\'); + } + } else { + out.push(c); + } + } + out +} + /// True when a value can be written without quotes: a number or a simple /// identifier (spec §8). pub fn is_bare_value(value: &str) -> bool { @@ -139,4 +165,24 @@ mod tests { let twice = escape(&once, TextContext::Block); assert_eq!(twice, r"a\\\*b"); } + + #[test] + fn unescape_inverts_escape() { + let samples = [ + "*asterisco* _guion_ `code` [x] \\z", + "# titulo", + "- item", + "1. uno", + "a|b", + "a&b", + "coma, punto. dos: puntos; ¿eñe?", + r"\*asterisco\* \_guion\_", + ]; + for sample in samples { + let escaped = escape(sample, TextContext::Block); + assert_eq!(unescape(&escaped), sample, "failed on {sample:?}"); + } + assert_eq!(unescape(r"a\|b"), "a|b"); + assert_eq!(unescape("trailing\\"), "trailing\\"); + } } diff --git a/crates/docsai-docmark/src/yaml.rs b/crates/docsai-docmark/src/yaml.rs new file mode 100644 index 0000000..1ed1286 --- /dev/null +++ b/crates/docsai-docmark/src/yaml.rs @@ -0,0 +1,476 @@ +//! Small YAML subset used by DocMark front matter. +//! +//! Only the shapes the serialiser emits are accepted: scalars, flow maps, +//! nested block maps and simple sequences of flow maps. + +use std::collections::BTreeMap; + +#[derive(Debug, Clone, PartialEq)] +pub enum Value { + Null, + Bool(bool), + Number(f64), + String(String), + Seq(Vec), + Map(BTreeMap), +} + +impl Value { + pub fn as_str(&self) -> Option<&str> { + match self { + Value::String(s) => Some(s), + _ => None, + } + } + + pub fn as_bool(&self) -> Option { + match self { + Value::Bool(b) => Some(*b), + _ => None, + } + } + + pub fn as_f64(&self) -> Option { + match self { + Value::Number(n) => Some(*n), + Value::String(s) => s.parse().ok(), + _ => None, + } + } + + pub fn as_i64(&self) -> Option { + self.as_f64().map(|n| n as i64) + } + + pub fn as_u16(&self) -> Option { + self.as_f64().and_then(|n| { + if n >= 0.0 && n <= u16::MAX as f64 { + Some(n as u16) + } else { + None + } + }) + } + + pub fn as_u8(&self) -> Option { + self.as_f64().and_then(|n| { + if n >= 0.0 && n <= u8::MAX as f64 { + Some(n as u8) + } else { + None + } + }) + } + + pub fn as_map(&self) -> Option<&BTreeMap> { + match self { + Value::Map(m) => Some(m), + _ => None, + } + } + + pub fn as_seq(&self) -> Option<&[Value]> { + match self { + Value::Seq(s) => Some(s), + _ => None, + } + } + + pub fn get(&self, key: &str) -> Option<&Value> { + self.as_map()?.get(key) + } + + pub fn string_or_empty(&self) -> String { + match self { + Value::String(s) => s.clone(), + Value::Number(n) => crate::units::trim_display(*n), + Value::Bool(b) => b.to_string(), + _ => String::new(), + } + } +} + +/// Parses a YAML document body (without the `---` fences) into a map. +pub fn parse_document(text: &str) -> Result, String> { + let lines: Vec<&str> = text.lines().collect(); + let mut parser = Parser { + lines: &lines, + index: 0, + base_line: 1, + }; + parser.parse_block_map(0) +} + +struct Parser<'a> { + lines: &'a [&'a str], + index: usize, + base_line: usize, +} + +impl<'a> Parser<'a> { + fn line_no(&self) -> usize { + self.base_line + self.index + } + + fn peek(&self) -> Option<&'a str> { + self.lines.get(self.index).copied() + } + + fn bump(&mut self) { + self.index += 1; + } + + fn parse_block_map(&mut self, indent: usize) -> Result, String> { + let mut map = BTreeMap::new(); + while let Some(raw) = self.peek() { + if raw.trim().is_empty() { + self.bump(); + continue; + } + let current = leading_spaces(raw); + if current < indent { + break; + } + if current > indent { + return Err(format!( + "line {}: unexpected indent {current}, expected {indent}", + self.line_no() + )); + } + let content = &raw[indent..]; + let (key, rest) = split_key(content).ok_or_else(|| { + format!("line {}: expected mapping key", self.line_no()) + })?; + self.bump(); + let value = if rest.is_empty() { + self.parse_nested(indent + 2)? + } else { + parse_flow(rest.trim()).map_err(|e| format!("line {}: {e}", self.line_no() - 1))? + }; + map.insert(key, value); + } + Ok(map) + } + + fn parse_nested(&mut self, indent: usize) -> Result { + let Some(raw) = self.peek() else { + return Ok(Value::Null); + }; + if raw.trim().is_empty() { + self.bump(); + return self.parse_nested(indent); + } + let current = leading_spaces(raw); + if current < indent { + return Ok(Value::Null); + } + let content = raw.trim_start(); + if content.starts_with("- ") || content == "-" { + return self.parse_block_seq(indent).map(Value::Seq); + } + self.parse_block_map(indent).map(Value::Map) + } + + fn parse_block_seq(&mut self, indent: usize) -> Result, String> { + let mut items = Vec::new(); + while let Some(raw) = self.peek() { + if raw.trim().is_empty() { + self.bump(); + continue; + } + let current = leading_spaces(raw); + if current < indent { + break; + } + if current > indent { + return Err(format!( + "line {}: unexpected indent in sequence", + self.line_no() + )); + } + let content = raw[indent..].trim_start(); + let Some(rest) = content.strip_prefix('-') else { + break; + }; + let rest = rest.strip_prefix(' ').unwrap_or(rest).trim(); + self.bump(); + let value = if rest.is_empty() { + self.parse_nested(indent + 2)? + } else { + parse_flow(rest).map_err(|e| format!("line {}: {e}", self.line_no() - 1))? + }; + items.push(value); + } + Ok(items) + } +} + +fn leading_spaces(line: &str) -> usize { + line.chars().take_while(|c| *c == ' ').count() +} + +fn split_key(content: &str) -> Option<(String, &str)> { + let content = content.trim_end(); + if let Some(rest) = content.strip_prefix('"') { + let mut out = String::new(); + let mut chars = rest.chars(); + while let Some(c) = chars.next() { + match c { + '\\' => match chars.next() { + Some('"') => out.push('"'), + Some('\\') => out.push('\\'), + Some('n') => out.push('\n'), + Some(other) => { + out.push('\\'); + out.push(other); + } + None => out.push('\\'), + }, + '"' => { + let rest: String = chars.collect(); + let rest = rest.trim_start(); + let rest = rest.strip_prefix(':')?; + return Some((out, rest.trim_start())); + } + other => out.push(other), + } + } + return None; + } + let colon = content.find(':')?; + let key = content[..colon].trim(); + if key.is_empty() { + return None; + } + // Bare keys cannot contain spaces in our serialiser except when quoted. + Some((key.to_string(), content[colon + 1..].trim_start())) +} + +/// Parses a flow scalar, map or sequence. +pub fn parse_flow(input: &str) -> Result { + let input = input.trim(); + if input.is_empty() { + return Ok(Value::Null); + } + if input.starts_with('{') { + return parse_flow_map(input); + } + if input.starts_with('[') { + return parse_flow_seq(input); + } + Ok(parse_scalar(input)) +} + +fn parse_scalar(input: &str) -> Value { + let input = input.trim(); + if let Some(s) = unquote(input) { + return Value::String(s); + } + match input { + "true" | "True" | "TRUE" => return Value::Bool(true), + "false" | "False" | "FALSE" => return Value::Bool(false), + "null" | "Null" | "~" | "" => return Value::Null, + _ => {} + } + if let Ok(n) = input.parse::() { + if input.chars().all(|c| c.is_ascii_digit() || matches!(c, '-' | '+' | '.' | 'e' | 'E')) { + return Value::Number(n); + } + } + Value::String(input.to_string()) +} + +fn unquote(input: &str) -> Option { + let rest = input.strip_prefix('"')?; + if !input.ends_with('"') || input.len() < 2 { + return None; + } + let inner = &input[1..input.len() - 1]; + // Ensure the closing quote is not escaped. + let mut out = String::new(); + let mut chars = inner.chars(); + while let Some(c) = chars.next() { + match c { + '\\' => match chars.next() { + Some('"') => out.push('"'), + Some('\\') => out.push('\\'), + Some('n') => out.push('\n'), + Some('r') => out.push('\r'), + Some('t') => out.push('\t'), + Some(other) => { + out.push('\\'); + out.push(other); + } + None => out.push('\\'), + }, + other => out.push(other), + } + } + // Verify it was properly quoted by checking we consumed a matching form. + let _ = rest; + Some(out) +} + +fn parse_flow_map(input: &str) -> Result { + let inner = strip_wrappers(input, '{', '}')?; + let mut map = BTreeMap::new(); + if inner.trim().is_empty() { + return Ok(Value::Map(map)); + } + for item in split_flow_items(inner)? { + let (key, value) = split_flow_pair(item)?; + map.insert(key, parse_flow(value)?); + } + Ok(Value::Map(map)) +} + +fn parse_flow_seq(input: &str) -> Result { + let inner = strip_wrappers(input, '[', ']')?; + if inner.trim().is_empty() { + return Ok(Value::Seq(Vec::new())); + } + let mut items = Vec::new(); + for item in split_flow_items(inner)? { + items.push(parse_flow(item)?); + } + Ok(Value::Seq(items)) +} + +fn strip_wrappers(input: &str, open: char, close: char) -> Result<&str, String> { + let input = input.trim(); + if !input.starts_with(open) || !input.ends_with(close) { + return Err(format!("expected {open}...{close}")); + } + Ok(&input[open.len_utf8()..input.len() - close.len_utf8()]) +} + +fn split_flow_pair(item: &str) -> Result<(String, &str), String> { + let item = item.trim(); + if let Some(rest) = item.strip_prefix('"') { + // quoted key + let mut i = 0usize; + let bytes = rest.as_bytes(); + while i < bytes.len() { + if bytes[i] == b'\\' { + i += 2; + continue; + } + if bytes[i] == b'"' { + let key = unquote(&item[..i + 2]).ok_or_else(|| "bad quoted key".to_string())?; + let after = item[i + 2..].trim_start(); + let after = after + .strip_prefix(':') + .ok_or_else(|| "expected ':' after key".to_string())?; + return Ok((key, after.trim_start())); + } + i += 1; + } + return Err("unterminated quoted key".into()); + } + let colon = item + .find(':') + .ok_or_else(|| format!("expected key: value in `{item}`"))?; + Ok((item[..colon].trim().to_string(), item[colon + 1..].trim())) +} + +fn split_flow_items(input: &str) -> Result, String> { + let mut items = Vec::new(); + let mut start = 0usize; + let mut depth = 0i32; + let mut in_string = false; + let mut escape = false; + let chars: Vec<(usize, char)> = input.char_indices().collect(); + for (idx, &(i, c)) in chars.iter().enumerate() { + if in_string { + if escape { + escape = false; + } else if c == '\\' { + escape = true; + } else if c == '"' { + in_string = false; + } + continue; + } + match c { + '"' => in_string = true, + '{' | '[' => depth += 1, + '}' | ']' => depth -= 1, + ',' if depth == 0 => { + items.push(input[start..i].trim()); + start = i + c.len_utf8(); + } + _ => {} + } + let _ = idx; + } + let tail = input[start..].trim(); + if !tail.is_empty() { + items.push(tail); + } + Ok(items) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_flow_map_with_quoted_values() { + let v = parse_flow(r#"{ name: "Calibri Light", size: 16pt, color: "#2E74B5" }"#).unwrap(); + let map = v.as_map().unwrap(); + assert_eq!(map.get("name").and_then(|v| v.as_str()), Some("Calibri Light")); + assert_eq!(map.get("size").and_then(|v| v.as_str()), Some("16pt")); + assert_eq!(map.get("color").and_then(|v| v.as_str()), Some("#2E74B5")); + } + + #[test] + fn parses_nested_block_map() { + let text = "\ +docmark: \"1.0\" +page: + size: A4 + margins: { top: 70.85pt, bottom: 70.85pt } +styles: + Heading1: + type: paragraph + name: \"heading 1\" +"; + let map = parse_document(text).unwrap(); + assert_eq!(map.get("docmark").and_then(|v| v.as_str()), Some("1.0")); + assert_eq!( + map.get("page") + .and_then(|v| v.get("size")) + .and_then(|v| v.as_str()), + Some("A4") + ); + assert_eq!( + map.get("styles") + .and_then(|v| v.get("Heading1")) + .and_then(|v| v.get("name")) + .and_then(|v| v.as_str()), + Some("heading 1") + ); + } + + #[test] + fn parses_list_definition_sequence() { + let text = "\ +list-definitions: + L1: + levels: + - { format: decimal, text: \"%1.\", start: 1 } +"; + let map = parse_document(text).unwrap(); + let levels = map + .get("list-definitions") + .and_then(|v| v.get("L1")) + .and_then(|v| v.get("levels")) + .and_then(|v| v.as_seq()) + .unwrap(); + assert_eq!(levels.len(), 1); + assert_eq!( + levels[0].get("format").and_then(|v| v.as_str()), + Some("decimal") + ); + } +} diff --git a/crates/docsai-model/src/units.rs b/crates/docsai-model/src/units.rs index 98e92af..77ec1fa 100644 --- a/crates/docsai-model/src/units.rs +++ b/crates/docsai-model/src/units.rs @@ -144,6 +144,38 @@ impl Length { pub const fn is_zero(self) -> bool { self.0 == 0 } + + /// Parses a DocMark length (`450px`, `3.5cm`, `70.85pt`, `123emu`, or a bare + /// number treated as pixels at 96 dpi). + /// + /// ``` + /// use docsai_model::units::Length; + /// assert_eq!(Length::parse("450px"), Some(Length::from_px(450.0))); + /// assert_eq!(Length::parse("3.5cm"), Some(Length::from_cm(3.5))); + /// assert_eq!(Length::parse("70.85pt"), Some(Length::from_pt(70.85))); + /// assert_eq!(Length::parse("123emu"), Some(Length::from_emu(123))); + /// assert_eq!(Length::parse("96"), Some(Length::from_px(96.0))); + /// assert_eq!(Length::parse("nope"), None); + /// ``` + pub fn parse(text: &str) -> Option { + let text = text.trim(); + if text.is_empty() { + return None; + } + let split = text.find(|c: char| c.is_ascii_alphabetic()).unwrap_or(text.len()); + let (number, unit) = text.split_at(split); + let value: f64 = number.parse().ok()?; + match unit { + "" | "px" => Some(Length::from_px(value)), + "cm" => Some(Length::from_cm(value)), + "mm" => Some(Length::from_mm(value)), + "pt" => Some(Length::from_pt(value)), + "in" | "inch" => Some(Length::from_inch(value)), + "emu" => Some(Length::from_emu(value.round() as i64)), + "twip" | "twips" => Some(Length::from_twips(value.round() as i64)), + _ => None, + } + } } const fn div_round(value: i64, divisor: i64) -> i64 { @@ -292,20 +324,6 @@ mod tests { #[test] fn display_is_always_lossless() { - // Parsing back what Display wrote must give the very same EMU, or a - // round-trip would move margins and image sizes. - fn reparse(text: &str) -> Length { - let split = text.find(|c: char| c.is_alphabetic()).unwrap(); - let (number, unit) = text.split_at(split); - let value: f64 = number.parse().unwrap(); - match unit { - "px" => Length::from_px(value), - "cm" => Length::from_cm(value), - "pt" => Length::from_pt(value), - "emu" => Length::from_emu(value as i64), - other => panic!("unexpected unit {other}"), - } - } let samples = [ Length::from_twips(1417), // a Word margin: not a whole px or cm Length::from_twips(2500), // a table column width @@ -317,12 +335,22 @@ mod tests { ]; for length in samples { let text = length.to_string(); - assert_eq!(reparse(&text), length, "`{text}` did not round-trip"); + assert_eq!(Length::parse(&text), Some(length), "`{text}` did not round-trip"); } assert_eq!(Length::from_twips(1417).to_string(), "70.85pt"); assert_eq!(Length::from_emu(1).to_string(), "1emu"); } + #[test] + fn parse_accepts_bare_pixels_and_rejects_garbage() { + assert_eq!(Length::parse("450px"), Some(Length::from_px(450.0))); + assert_eq!(Length::parse(" 3.5cm "), Some(Length::from_cm(3.5))); + assert_eq!(Length::parse("123"), Some(Length::from_px(123.0))); + assert_eq!(Length::parse(""), None); + assert_eq!(Length::parse("cm"), None); + assert_eq!(Length::parse("12xy"), None); + } + #[test] fn trim_float_is_deterministic() { assert_eq!(trim_float(2.500, 3), "2.5"); diff --git a/docs/arquitectura.md b/docs/_es_arquitectura_src.md similarity index 100% rename from docs/arquitectura.md rename to docs/_es_arquitectura_src.md diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 0000000..b31bb1d --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,252 @@ +# docsai architecture + +Describes the software structure: crate workspace, intermediate document model (IR), +conversion pipelines, CLI and MCP server. Complements `technical-analysis.md` (the why) +and `development-plan.md` (the when). + +## 1. Overview + +``` + ┌────────────────────────── docsai-convert ──────────────────────────┐ + .docx ─┐ │ │ + .doc ─┤ readers (docsai-office / docsai-odf) writers │ + .odt ─┼──────────────► IR (docsai-model) ◄──────────────────────┐ │ + .xlsx ─┤ ▲ │ │ │ + .xls ─┤ │ ▼ │ │ + .ods ─┘ DocMark parser DocMark serializer .docx/.xlsx/.odt/.ods │ + (docsai-docmark) ▲ │ + ▲ │ │ │ + │ ▼ │ │ + .dmk.md file + assets/ ────────────────────┘ │ + └─────────────────────────────────────────────────────────────────────┘ + ▲ ▲ + docsai-cli (clap) docsai-mcp (rmcp, stdio) +``` + +Central rule: **no converter talks to another converter**. Everything goes through the IR. + +## 2. Crate workspace + +| Crate | Type | Responsibility | Internal dependencies | +|---|---|---|---| +| `docsai-model` | lib | IR + common types (`Style`, `ConversionReport`, units) | none | +| `docsai-docmark` | lib | DocMark serializer and parser (IR ⇄ .dmk.md + assets) | model | +| `docsai-office` | lib | docx/xlsx/xls/doc readers and docx/xlsx writers | model | +| `docsai-odf` | lib | odt/ods readers/writers | model | +| `docsai-convert` | lib | Orchestration: format detection, pipelines, asset management, LibreOffice fallback, reports | all of the above | +| `docsai-cli` | bin | `docsai` binary (convert/inspect/roundtrip/mcp subcommands) | convert | +| `docsai-mcp` | lib | MCP server implementation (CLI starts it with `docsai mcp`) | convert | + +Notes: +- A single distributed binary (`docsai`); `docsai-mcp` is a lib so the `mcp` subcommand + lives inside the same executable. +- Cargo features per format (`office-doc`, `odf`, …) to compile minimal variants. + +## 3. The intermediate model (IR) — `docsai-model` + +Two roots: + +```rust +pub enum Document { + Text(TextDocument), + Workbook(Workbook), +} + +pub struct TextDocument { + pub meta: DocumentMeta, // title, author, dates, custom props, language + pub styles: StyleCatalog, // named styles with inheritance (based_on) + pub list_defs: ListCatalog, + pub sections: Vec
, // page geometry + headers/footers + blocks +} + +pub enum Block { + Paragraph(Paragraph), // Vec + ParaProps (style_id + direct deltas) + Heading(Heading), // outline level + Paragraph + List(List), // already reconstructed tree (not numId/ilvl pairs) + Table(Table), // grid with spans, col-widths, style + Image(ImageRef), // AssetStore reference + geometry/anchor + TextBox(TextBox), + Raw(RawFragment), // fidelity hatch (source format + XML bytes) + // … +} + +pub enum Inline { + Text(String), + Styled(Vec, RunProps), // RunProps = optional style_id + deltas (bold, color…) + Link { target: String, content: Vec, props: RunProps }, + Footnote(Vec), + Field { kind: FieldKind, cached: String }, + Break(BreakKind), + ImageInline(ImageRef), +} + +pub struct Workbook { + pub meta: DocumentMeta, + pub styles: StyleCatalog, + pub defined_names: Vec, + pub sheets: Vec, // Sheet = sparse Cell grid + col/row props + merges +} // + panes + images: Vec (sheet-anchored) + +pub struct Cell { + pub value: CellValue, // Number | Text | Bool | DateTime | Error | Empty + pub formula: Option, // text + dialect (Ooxml | OpenFormula) + shared/array info + pub num_fmt: Option, + pub style_id: Option, +} +``` + +### 3.1 Normalized image model (`ImageRef` + `ImageGeometry`) + +Each input format has its own graphics model (DrawingML, VML, SpreadsheetDrawingML, +ODF `draw:frame`, Escher — see analysis §1.4). The IR normalizes them to a single model, +which DocMark spec §3.5/§4.1 serializes attribute by attribute: + +```rust +pub struct ImageRef { + pub asset: AssetId, // key in the AssetStore (content hash) + pub geometry: ImageGeometry, + pub alt: String, // alternative text (accessibility) + pub title: Option, + pub name: Option, // internal object name (docPr @name) + pub link: Option, // hyperlink on the image + pub external_src: Option, // linked image, not embedded + pub effects_raw: Option, // unmodeled DrawingML effects → associated raw-block +} + +pub struct ImageGeometry { + pub display_size: Size, // EMU; displayed size (≠ native size) + pub native_size_px: Option<(u32, u32)>, + pub dpi: Option, + pub anchor: Anchor, + pub rotation_deg: f32, + pub flip: Flip, // None | H | V | HV + pub crop: Option, // % of original per side + pub border: Option, + pub z_index: Option, +} + +pub enum Anchor { + Inline, // flows with text + Floating { relative_to: RelBase, // Page | Margin | Paragraph | Character + position: HVPos, // EMU offsets or symbolic alignment + wrap: WrapMode, wrap_side: WrapSide, + behind_text: bool }, + // Spreadsheets only: + SheetTwoCell { from: CellAnchor, to: CellAnchor, // CellAnchor = (col,row) + EMU offsets + move_with_cells: bool, size_with_cells: bool }, + SheetOneCell { from: CellAnchor }, // + fixed display_size + SheetAbsolute { pos: Point }, +} +``` + +Model rules: +- Readers **translate into this model, they do not bypass it**: an image attribute a reader + cannot map is degraded with a typed warning (`Warning::ImageGeometryDegraded`), and + unrepresentable effects go to raw-block via `effects_raw` — never silently discarded. +- Writers do the full reverse translation (e.g. `Anchor::Floating` → `wp:anchor` + with `positionH/V`, wrap and z-order; `SheetTwoCell` → `xdr:twoCellAnchor`). +- Cross conversion of anchors (text document ⇄ sheet) does not exist: each sheet `Anchor` + is only valid inside a `Workbook` (invariant checked by the IR validator). + +### 3.2 `AssetStore` + +Trait that abstracts media storage; `docsai-convert` provides the implementations +(`assets/` directory in CLI, memory/base64 in MCP): + +- **Content-hash deduplication**: `put(bytes) -> AssetId` returns the same id for the same + content; N appearances of a bitmap share a file, each `ImageRef` keeps its own geometry. +- **Deterministic names** `img-.` (extension derived from content sniffing, + not the source) → round-trips do not duplicate media and `assets/` diffs are stable. +- **Manifest**: the store records content-type, byte size and detected native dimensions + (`image` crate); `inspect --json` exposes it and the writer uses it to fill required + destination fields without re-reading files. +- **Security**: asset names are sanitized (never derived from internal document paths → + no path traversal); external linked images are never downloaded. + +IR principles: +- **Style = reference + delta**, never flattened format (see analysis §5.2). +- **No I/O dependency**: `docsai-model` knows nothing about ZIP or XML; `ImageRef`s point to an + abstract `AssetStore` (trait) materialized by `docsai-convert`. +- All types are `serde`-serializable → free `inspect --json` and simple debugging. +- Units normalized to EMU/twips internally with newtypes (`Length`), converting to + `pt/cm/px` only when serializing DocMark. + +## 4. Conversion contracts + +```rust +pub trait DocumentReader { + fn detect(path: &Path, sniff: &[u8]) -> DetectScore; // by content, not only extension + fn read(&self, input: &mut dyn ReadSeek, assets: &mut dyn AssetStore) + -> Result<(Document, ConversionReport), ReadError>; +} + +pub trait DocumentWriter { + fn write(&self, doc: &Document, assets: &dyn AssetStore, out: &mut dyn Write) + -> Result; +} + +pub struct ConversionReport { + pub warnings: Vec, // typed: UnsupportedElement { kind, location, action } + pub raw_blocks_emitted: u32, // Degraded { what, why } · AssetIssue { … } + pub stats: ConversionStats, // paragraphs, cells, images, formulas processed +} +``` + +- Readers **never panic** on corrupt input: always typed `Err`. +- `ConversionReport` flows to CLI (readable stderr / `--json`) and to the MCP response. +- The `roundtrip` command compares original IR vs IR after round-trip with a **structural + diff of normalized IR** and produces a fidelity metric (% nodes preserved per category). + +## 5. CLI (`docsai-cli`) + +``` +docsai convert [-o ] [--to ] [--fidelity full|standard|plain] + [--assets-dir ] [--style-map ] [--max-cells N] + [--use-loffice auto|never|require] [--json] +docsai inspect [--json] # metadata, styles, sheets, media, without converting +docsai roundtrip [--report ] [--json] +docsai formats # current support matrix +docsai mcp # MCP server over stdio +``` + +- Destination format inferred from `-o` extension, forceable with `--to`. +- stdin/stdout supported (`-` as name) for pipelines, except binary output formats on a + terminal (standard protection). +- Exit codes: 0 OK, 1 conversion with severe warnings (`--strict` makes them fatal), + 2 input error, 3 unsupported format. + +## 6. MCP server (`docsai-mcp`) + +Based on `rmcp` (official SDK), **stdio** transport. Logs to stderr exclusively. + +Tools exposed (v1): + +| Tool | Input | Output | +|---|---|---| +| `convert_to_markdown` | `path` (or `content_base64` + `filename`), `fidelity`, `assets` = `inline-base64`\|`files` | DocMark (text), assets, `report` | +| `convert_from_markdown` | `markdown`, `target_format`, optional assets | base64 file or written `path`, `report` | +| `inspect_document` | `path`/`content_base64` | structure JSON (same shape as `inspect --json`) | +| `list_supported_formats` | — | support matrix with status per direction | + +Decisions: +- Dual **path/base64** mode: local MCP clients (Claude Desktop/Code) pass paths; remote + ones can pass embedded content. Size limit configurable via environment variable. +- Large responses: DocMark as `text content`; binaries as base64 resource with correct MIME. +- The server is stateless; each tool call is an independent conversion (no persistent temp + files unless the client asks for `assets=files`). + +## 7. Cross-platform and distribution + +- No system dependencies on the main path (pure Rust). `soffice` is searched at runtime + in standard locations per OS only for `.doc`. +- CI: GitHub Actions, ubuntu/windows/macos matrix; release artifacts with `cargo-dist` + (tar.gz/zip + shell/powershell installers; optional: Homebrew tap, Scoop/winget, cargo-binstall). +- Targets: `x86_64-unknown-linux-gnu`, `aarch64-unknown-linux-gnu`, `x86_64-pc-windows-msvc`, + `aarch64-apple-darwin`, `x86_64-apple-darwin`. LTO + `strip` on release. + +## 8. Performance (indicative budgets) + +- 100-page docx → DocMark: < 1 s on ordinary hardware. +- 100k-cell xlsx: < 3 s and < 500 MB RAM (avoid cloning sharedStrings; use `Cow`). +- Readers stream where the crate allows (calamine is lazy per sheet). +- Benchmarks with `criterion` on the corpus from Phase 8; regressions > 20% block PRs. diff --git a/docs/plan-desarrollo.md b/docs/development-plan.md similarity index 100% rename from docs/plan-desarrollo.md rename to docs/development-plan.md diff --git a/docs/especificacion-docmark.md b/docs/docmark-specification.md similarity index 100% rename from docs/especificacion-docmark.md rename to docs/docmark-specification.md diff --git a/docs/spikes/R1-estrategia-docx.md b/docs/spikes/R1-docx-strategy.md similarity index 100% rename from docs/spikes/R1-estrategia-docx.md rename to docs/spikes/R1-docx-strategy.md diff --git a/docs/analisis-tecnico.md b/docs/technical-analysis.md similarity index 100% rename from docs/analisis-tecnico.md rename to docs/technical-analysis.md diff --git a/kb/01-phases-0-1-summary.md b/kb/01-phases-0-1-summary.md new file mode 100644 index 0000000..baa3855 --- /dev/null +++ b/kb/01-phases-0-1-summary.md @@ -0,0 +1,112 @@ +# 01 — Summary of phases 0 and 1 + +## What you can do today + +```bash +docsai convert report.docx -o report.dmk.md # extracts assets/ next to the .md +docsai convert report.docx # to stdout +docsai convert report.docx --fidelity plain # clean CommonMark for LLM/RAG +docsai convert report.docx -o out.md --json # conversion report as JSON +docsai formats # support matrix for this binary +``` + +A `.docx` goes in and a DocMark file comes out with: + +- **Document properties** (core, app, and custom) in the front matter. +- Full **style catalog**, with `w:docDefaults` and the `basedOn` chain, without flattening. +- **Paragraphs and runs**: bold, italic, strikethrough, underline (all seven styles), color, + highlight, font, size, sub/superscript, small caps, all caps, hyperlinks, line/page/column + breaks, and tabs. +- **Lists** rebuilt as a tree from flat `(numId, ilvl)` pairs, with their typographic definition + in `list-definitions`. +- **Tables** with `gridSpan` and `vMerge` resolved to `colspan`/`rowspan` and absorbed cells + marked; `::: {.table complex=true}` container for what GFM does not allow. +- **Images** with full geometry: displayed and native size, inline or floating anchor, + position base per axis, offsets or symbolic alignment, text wrapping and side, Z-order, + rotation, flips, crop, border, alt text, title, internal name, and hyperlink. + Both DrawingML and **legacy VML**. +- **Sections** split at each `sectPr`, with headers and footers by scope (default, first + page, even). +- **Footnotes** inserted at their reference point. +- **Simple and complex fields**, preserving the original instruction. +- **Raw-blocks** with the exact bytes of every unrecognized OOXML element. + +What you **cannot** do yet: reverse conversion (Phase 2), spreadsheets (Phase 3), ODF +(Phase 4), `.doc` (Phase 5), and the MCP server (Phase 7). + +## Phase 0 — Foundations + +| Plan task | Status | +|---|---| +| 1. Workspace with the 7 crates | ✅ | +| 2. `docsai-model` v1 (IR, units, images, `AssetStore`, validator) | ✅ | +| 3. Risk spike R1 | ✅ [`docs/spikes/R1-docx-strategy.md`](../docs/spikes/R1-docx-strategy.md) | +| 4. Freeze DocMark spec v1.0 | ✅ annex A resolved, changelog in §10 | +| 5. Initial corpus | ✅ 14 docx + 6 xlsx generated by `corpus/generate.py` | +| 6. CI GitHub Actions on 3 OSes | ✅ build, test, corpus up to date, fmt, clippy, rustdoc | +| 7. CLI skeleton | ✅ surpassed: the CLI already does full `convert` | + +**Acceptance criteria**: all four met and verified with tests +(`cargo test --workspace` green, IR JSON round-trip with proptest, spike documented with +signed decision, spec with no TODOs). + +## Phase 1 — DOCX reading → DocMark + +| Plan task | Status | +|---|---| +| 1. docx reader (ZIP, `document.xml`, relationships) | ✅ | +| 2. Styles with reference + delta resolution | ✅ | +| 3. Paragraphs and runs with all inline formatting from spec §3.2 | ✅ | +| 4. Lists from `numbering.xml` | ✅ | +| 5. Tables with spans, widths, and style | ✅ | +| 6. DrawingML + VML images, with hash-based deduplication | ✅ | +| 7. Sections, headers/footers, footnotes, fields, properties | ✅ | +| 8. DocMark serializer with fidelity modes | ✅ | +| 9. Raw-block + typed warning for the unrecognized | ✅ | +| 10. Corpus golden tests + 5 anonymized real documents | ⚠️ goldens yes; real documents **pending** | + +**Acceptance criteria**, all verified by tests: + +- ✅ Corpus goldens pass, including the three image cases with full geometry. +- ✅ Zero panics on synthetic corrupt corpus: 900+ inputs (truncated and byte-flipped) + return `Err` (`crates/docsai-office/tests/robustness.rs`). +- ✅ 50+ page document in under 1 s with fewer than 10 raw-blocks. +- ✅ `--fidelity plain` is clean CommonMark, verified by parsing it with comrak. + +## Metrics + +| | | +|---|---| +| Passing tests | 164 | +| Production code | ~9,500 lines of Rust | +| Test code | ~1,200 lines in integration files, plus `#[cfg(test)]` modules | +| Corpus documents | 14 docx + 6 xlsx, with their 14 goldens | +| Core external dependencies | `serde`, `thiserror`, `quick-xml`, `zip`, `tracing` | +| `unsafe` | Zero (`#![forbid(unsafe_code)]` in all library crates) | + +## Three defects the corpus uncovered + +All three were on the fidelity path, and none would have been seen without running real +documents through the pipeline: + +1. **Lossy length formatting.** A Word margin of 1417 twips was written as `2.499cm` + and re-reading it produced a different value: it would have shifted margins and images on + every round-trip. Now the first unit that represents the length **exactly** is chosen + (`px` → `cm` → `pt` → `emu`), and a test verifies it for each case. +2. **Empty paragraphs disappeared silently.** A blank line does not survive Markdown. + In `full` mode they are now written as `[]{.empty}`. +3. **Hyperlink style emitted twice**, inside and outside the tag. + +Additionally, the spike itself uncovered a bug in the corpus generator (`w:drawing` placed +outside a `w:r`, which is invalid OOXML) before it contaminated the goldens. + +## What was left out, and why + +| Element | Reason | +|---|---| +| **Anonymized real documents** (Phase 1, task 10) | Requires having real documents available. Does not block Phase 2; the synthetic corpus already covers traits one by one | +| **DrawingML text boxes** (`wps:txbx`) | Travel as raw-block. The `Block::TextBox` type exists in the IR and `::: {.textbox}` is in the spec, but emitting it without the writer that rebuilds it would advance Phase 2 work | +| **Validation in Word and LibreOffice** | Phase 2 criterion (`.docx` files are not generated yet), and there is no LibreOffice in the environment | +| **Comments and track changes** | Out of v1 scope per analysis §1.1. Revisions are taken as accepted and the count is reported | +| **`w:lvlOverride` in lists** | Not modeled in v1; a `Degraded` warning is emitted | +| **xlsx reader** | Phase 3. The corpus files already exist for then | diff --git a/kb/01-resumen-fases-0-1.md b/kb/01-resumen-fases-0-1.md deleted file mode 100644 index bf2d476..0000000 --- a/kb/01-resumen-fases-0-1.md +++ /dev/null @@ -1,112 +0,0 @@ -# 01 — Resumen de las fases 0 y 1 - -## Qué se puede hacer hoy - -```bash -docsai convert informe.docx -o informe.dmk.md # extrae assets/ junto al .md -docsai convert informe.docx # a stdout -docsai convert informe.docx --fidelity plain # CommonMark limpio para LLM/RAG -docsai convert informe.docx -o out.md --json # informe de conversión en JSON -docsai formats # matriz de soporte de este binario -``` - -Un `.docx` entra y sale un fichero DocMark con: - -- **Propiedades** del documento (core, app y personalizadas) en el front matter. -- **Catálogo de estilos** completo, con `w:docDefaults` y la cadena `basedOn`, sin aplanar. -- **Párrafos y runs**: negrita, cursiva, tachado, subrayado (los siete estilos), color, - resaltado, fuente, tamaño, sub/superíndice, versalitas, mayúsculas, hipervínculos, saltos de - línea, página y columna, y tabuladores. -- **Listas** reconstruidas en árbol desde los pares planos `(numId, ilvl)`, con su definición - tipográfica en `list-definitions`. -- **Tablas** con `gridSpan` y `vMerge` resueltos a `colspan`/`rowspan` y celdas absorbidas - marcadas; contenedor `::: {.table complex=true}` para lo que GFM no admite. -- **Imágenes** con la geometría entera: tamaño mostrado y nativo, anclaje en línea o flotante, - base de posición por eje, offsets o alineación simbólica, ajuste de texto y lado, orden Z, - rotación, volteos, recorte, borde, texto alternativo, título, nombre interno e hipervínculo. - Tanto DrawingML como **VML heredado**. -- **Secciones** partidas en cada `sectPr`, con cabeceras y pies por ámbito (defecto, primera - página, pares). -- **Notas al pie** insertadas en el punto de su referencia. -- **Campos** simples y complejos, conservando la instrucción original. -- **Raw-blocks** con los bytes exactos de todo elemento OOXML no reconocido. - -Lo que **no** se puede hacer: la conversión inversa (Fase 2), hojas de cálculo (Fase 3), ODF -(Fase 4), `.doc` (Fase 5) y el servidor MCP (Fase 7). - -## Fase 0 — Fundamentos - -| Tarea del plan | Estado | -|---|---| -| 1. Workspace con los 7 crates | ✅ | -| 2. `docsai-model` v1 (IR, unidades, imágenes, `AssetStore`, validador) | ✅ | -| 3. Spike de riesgo R1 | ✅ [`docs/spikes/R1-estrategia-docx.md`](../docs/spikes/R1-estrategia-docx.md) | -| 4. Congelar la spec DocMark v1.0 | ✅ anexo A resuelto, registro de cambios en §10 | -| 5. Corpus inicial | ✅ 14 docx + 6 xlsx generados por `corpus/generate.py` | -| 6. CI GitHub Actions en 3 SO | ✅ build, test, corpus al día, fmt, clippy, rustdoc | -| 7. Esqueleto de CLI | ✅ superado: la CLI ya hace `convert` completo | - -**Criterios de aceptación**: los cuatro cumplidos y verificados con tests -(`cargo test --workspace` verde, round-trip JSON del IR con proptest, spike documentado con -decisión firmada, spec sin TODO). - -## Fase 1 — Lectura DOCX → DocMark - -| Tarea del plan | Estado | -|---|---| -| 1. Reader docx (ZIP, `document.xml`, relaciones) | ✅ | -| 2. Estilos con resolución referencia + delta | ✅ | -| 3. Párrafos y runs con todo el formato inline de la spec §3.2 | ✅ | -| 4. Listas desde `numbering.xml` | ✅ | -| 5. Tablas con spans, anchos y estilo | ✅ | -| 6. Imágenes DrawingML + VML, con deduplicación por hash | ✅ | -| 7. Secciones, cabeceras/pies, notas al pie, campos, propiedades | ✅ | -| 8. Serializador DocMark con modos de fidelidad | ✅ | -| 9. Raw-block + advertencia tipada para lo no reconocido | ✅ | -| 10. Golden tests del corpus + 5 documentos reales anonimizados | ⚠️ goldens sí; documentos reales **pendientes** | - -**Criterios de aceptación**, todos verificados por tests: - -- ✅ Los goldens del corpus pasan, incluidos los tres de imágenes con geometría completa. -- ✅ Cero pánicos con corpus corrupto sintético: 900+ entradas (truncados y volteos de byte) - devuelven `Err` (`crates/docsai-office/tests/robustness.rs`). -- ✅ Documento de 50+ páginas en menos de 1 s con menos de 10 raw-blocks. -- ✅ `--fidelity plain` es CommonMark limpio, verificado parseándolo con comrak. - -## Métricas - -| | | -|---|---| -| Tests que pasan | 164 | -| Código de producción | ~9 500 líneas de Rust | -| Código de test | ~1 200 líneas en ficheros de integración, más los módulos `#[cfg(test)]` | -| Documentos del corpus | 14 docx + 6 xlsx, con sus 14 goldens | -| Dependencias externas del núcleo | `serde`, `thiserror`, `quick-xml`, `zip`, `tracing` | -| `unsafe` | Cero (`#![forbid(unsafe_code)]` en todos los crates de biblioteca) | - -## Tres defectos que el corpus destapó - -Los tres estaban en el camino de la fidelidad, y ninguno se habría visto sin pasar documentos -reales por el pipeline: - -1. **Formato de longitudes con pérdida.** Un margen de Word de 1417 twips se escribía `2.499cm` - y al releerlo daba un valor distinto: habría movido márgenes e imágenes en cada ida y vuelta. - Ahora se elige la primera unidad que representa la longitud de forma **exacta** - (`px` → `cm` → `pt` → `emu`), y hay un test que lo verifica para cada caso. -2. **Párrafos vacíos desaparecían en silencio.** Una línea en blanco no sobrevive a Markdown. - En modo `full` se escriben ahora como `[]{.empty}`. -3. **Estilo de hipervínculo emitido dos veces**, dentro y fuera de la etiqueta. - -Además, el propio spike destapó un error en el generador del corpus (`w:drawing` colocado fuera -de un `w:r`, que es OOXML inválido) antes de que contaminara los goldens. - -## Qué quedó fuera, y por qué - -| Elemento | Motivo | -|---|---| -| **Documentos reales anonimizados** (Fase 1, tarea 10) | Requiere disponer de documentos reales. No bloquea la Fase 2; el corpus sintético ya cubre los rasgos uno a uno | -| **Cuadros de texto DrawingML** (`wps:txbx`) | Viajan como raw-block. El tipo `Block::TextBox` existe en el IR y `::: {.textbox}` está en la spec, pero emitirlo sin el writer que lo reconstruya es adelantar trabajo de la Fase 2 | -| **Validación en Word y LibreOffice** | Criterio de la Fase 2 (aún no se generan `.docx`), y no hay LibreOffice en el entorno | -| **Comentarios y control de cambios** | Fuera de alcance de la v1 según el análisis §1.1. Las revisiones se toman como aceptadas y se reporta cuántas | -| **`w:lvlOverride` en listas** | No modelado en v1; se emite advertencia `Degraded` | -| **Lector xlsx** | Fase 3. Los ficheros del corpus ya existen para entonces | diff --git a/kb/02-estructura-proyecto.md b/kb/02-estructura-proyecto.md deleted file mode 100644 index 1bfc3fa..0000000 --- a/kb/02-estructura-proyecto.md +++ /dev/null @@ -1,155 +0,0 @@ -# 02 — Estructura del proyecto - -## Árbol - -``` -docsai/ -├── Cargo.toml # workspace: 7 miembros + dependencias compartidas -├── .github/workflows/ci.yml # build/test en 3 SO + fmt/clippy/rustdoc -├── crates/ -│ ├── docsai-model/ # el IR. Sin I/O, sin dependencias pesadas -│ ├── docsai-docmark/ # serializador DocMark (el parser es Fase 2) -│ ├── docsai-office/ # lector .docx (xlsx Fase 3, doc Fase 5) -│ ├── docsai-odf/ # esqueleto (Fase 4) -│ ├── docsai-convert/ # orquestación: detección, pipelines, assets -│ ├── docsai-cli/ # binario `docsai` -│ └── docsai-mcp/ # esqueleto (Fase 7) -├── corpus/ -│ ├── generate.py # genera TODOS los ficheros del corpus -│ ├── README.md # qué aísla cada documento -│ ├── docx/*.docx # 14 documentos + 14 *.expected.dmk.md (goldens) -│ └── xlsx/*.xlsx # 6 libros, para la Fase 3 -├── docs/ # documentación de diseño (español) -│ └── spikes/ # informes de spikes con su decisión -└── kb/ # esta base de conocimiento -``` - -## Los siete crates - -| Crate | Tipo | Líneas | Responsabilidad | Depende de | -|---|---|---|---|---| -| `docsai-model` | lib | ~3 500 | El IR y los tipos comunes | *(nada del workspace)* | -| `docsai-docmark` | lib | ~1 850 | IR ⇄ DocMark | model | -| `docsai-office` | lib | ~3 500 | Lectores/escritores OOXML y legados | model | -| `docsai-odf` | lib | 21 | Lectores/escritores ODF | model | -| `docsai-convert` | lib | ~500 | Detección, pipelines, gestión de assets | los cuatro anteriores | -| `docsai-cli` | bin | ~200 | El binario `docsai` | convert, model | -| `docsai-mcp` | lib | 26 | Servidor MCP por stdio | convert | - -**Regla de dependencia (`AGENTS.md` §3)**: ningún crate de formato importa a otro crate de -formato. `docsai-odf` y `docsai-mcp` existen ya, vacíos, precisamente para que el compilador -haga cumplir esa regla desde el primer día en lugar de descubrirla rota en la Fase 4. - -## `docsai-model` — el IR - -| Módulo | Contenido | -|---|---| -| `units` | `Length` (newtype sobre EMU), `Size`, `Point`, conversiones exactas y el formato de salida | -| `style` | `StyleCatalog`, `Style`, `FontProps`, `ParaProps`, `DocDefaults` y la resolución de la cascada | -| `list` | `ListCatalog`, `ListDef`, `ListLevel`, `NumFormat` | -| `text` | `TextDocument`, `Section`, `PageGeometry`, `Block`, `Inline`, `Paragraph`, `Table`, `RawFragment` | -| `sheet` | `Workbook`, `Sheet`, `Cell`, `CellRef` (A1), `Formula`, `NumFmt` — poblado en la Fase 3 | -| `image` | `ImageRef`, `ImageGeometry`, `Anchor` y su cortejo (wrap, crop, flip, bordes) | -| `assets` | `AssetStore` (trait), `MemoryAssetStore`, hash de contenido, sniffing y dimensiones | -| `report` | `ConversionReport`, `Warning` tipada, `Severity`, `ConversionStats` | -| `validate` | Invariantes: anclajes de hoja solo en `Workbook`, filas más anchas que la rejilla, anclajes `two-cell` invertidos | - -Tres principios que los tipos codifican, y que conviene no romper: - -1. **Estilo = referencia + delta.** Cada campo de `FontProps`/`ParaProps` es `Option`: `None` - significa *heredar*, no *desactivado*. `over()` mezcla la cascada; `minus()` calcula el delta - que hay que emitir (la «regla de economía» de la spec §3.1). -2. **Todas las longitudes en EMU.** Un solo entero, sin pérdidas, para OOXML (EMU/twips), - ODF (cm/in) y `.doc` (twips). -3. **El IR no conoce el I/O.** Las imágenes son un `AssetId` detrás de un trait. - -## `docsai-office` — el lector docx - -| Módulo | Responsabilidad | -|---|---| -| `xml` | Árbol XML sobre `quick-xml` con **spans de bytes** por nodo | -| `package` | ZIP, relaciones OPC, límites anti-bomba, saneado de nombres | -| `detect` | Detección de formato **por contenido**, no por extensión | -| `docx/format` | `w:rPr` y `w:pPr` → los tipos delta del IR | -| `docx/styles` | `styles.xml` → `StyleCatalog`; nivel de encabezado | -| `docx/numbering` | `numbering.xml` → `ListCatalog` (aplana la doble indirección) | -| `docx/drawing` | DrawingML y VML → `ImageRef`/`ImageGeometry` | -| `docx/body` | El recorrido del cuerpo: bloques, inlines, campos, tablas, listas | -| `docx/mod` | Ensamblaje: propiedades, secciones, cabeceras/pies, notas al pie | - -El detalle que lo sostiene todo: **cada nodo del árbol XML recuerda su rango de bytes en el -origen**. Un elemento no reconocido se conserva citando los bytes originales, no una -re-serialización, y por eso el raw-block es exacto. - -## `docsai-docmark` — el serializador - -| Módulo | Responsabilidad | -|---|---| -| `escape` | Tabla de escapado fija por contexto (bloque, celda de tabla, etiqueta de enlace) | -| `attrs` | Bloques `{#id .clase clave="valor"}` con el orden canónico de la spec §8 | -| `units` | Cómo se escribe una longitud, un porcentaje o un número | -| `frontmatter` | El YAML, escrito a mano para garantizar determinismo byte a byte | -| `writer` | El recorrido del IR: bloques, inlines, imágenes, tablas, contenedores | - -## Flujo de una conversión - -``` -fichero.docx - │ docsai-convert::read_document - ▼ -detect (por contenido) ──► docsai-office::read_docx - │ Package::open → partes en memoria, nombres saneados - │ Element::parse → árbol XML con spans - │ styles + numbering + footnotes - │ read_sections → bloques, con DirAssetStore recibiendo medios - ▼ - (Document, ConversionReport) - │ docsai_model::validate - ▼ - docsai-docmark::serialize - │ front matter + cuerpo, según --fidelity - ▼ - fichero.dmk.md + assets/img-. -``` - -## Dependencias externas - -Todo el núcleo se apoya en cinco crates: - -| Crate | Para qué | Dónde | -|---|---|---| -| `serde` | Serialización del IR | model y derivados | -| `thiserror` | Errores tipados de biblioteca | model, docmark, office, convert | -| `quick-xml` | Parsing XML | office | -| `zip` | Contenedores OOXML | office | -| `tracing` | Logs, siempre a stderr | office, convert, cli | - -Y en los binarios: `clap` (CLI), `anyhow` (errores de binario), `serde_json` (`--json`), -`tracing-subscriber`. En tests: `proptest` y `comrak`. - -Deliberadamente **ausentes**: `docx-rs` (descartado por el spike R1), `image` (basta leer la -cabecera del formato), `serde_yaml` (el front matter se escribe a mano y el crate no está -mantenido) e `insta` (los goldens son ficheros de texto revisables). Cada ausencia está -justificada en [`docs/analisis-tecnico.md`](../docs/analisis-tecnico.md) §4, como exige -`AGENTS.md` §2. - -## Dónde están los tests - -| Ubicación | Qué cubre | -|---|---| -| Módulos `#[cfg(test)]` de cada fichero | Unidades: conversiones, escapado, cascada de estilos, parsing de fragmentos XML | -| `crates/docsai-model/tests/json_roundtrip.rs` | Round-trip JSON del IR: documentos a mano con todos los nodos + proptest | -| `crates/docsai-office/tests/docx_images.rs` | Geometría de imágenes sobre el corpus real | -| `crates/docsai-office/tests/robustness.rs` | Los 900+ inputs corruptos, path traversal, ZIP sin documento | -| `crates/docsai-convert/tests/goldens.rs` | Goldens, determinismo, finales de línea, `plain` con comrak, rendimiento | - -## Convenciones que conviene respetar - -- Nombres de código, comentarios y mensajes de commit en **inglés**; la documentación de `docs/` - y `kb/` en **español** (`AGENTS.md` §5). -- `thiserror` en bibliotecas, `anyhow` sólo en binarios. -- Los parsers **nunca entran en pánico**: sin `unwrap`, `expect` ni indexación sin comprobar en - el camino de lectura. -- El serializador es **determinista**: mismo IR ⇒ mismos bytes. Cualquier iteración sobre un - mapa usa `BTreeMap`, nunca `HashMap`. -- Rutas con `std::path`, nunca concatenando `/`. diff --git a/kb/02-project-structure.md b/kb/02-project-structure.md new file mode 100644 index 0000000..097a93d --- /dev/null +++ b/kb/02-project-structure.md @@ -0,0 +1,155 @@ +# 02 — Project structure + +## Tree + +``` +docsai/ +├── Cargo.toml # workspace: 7 members + shared dependencies +├── .github/workflows/ci.yml # build/test on 3 OSes + fmt/clippy/rustdoc +├── crates/ +│ ├── docsai-model/ # the IR. No I/O, no heavy dependencies +│ ├── docsai-docmark/ # DocMark serializer (the parser is Phase 2) +│ ├── docsai-office/ # .docx reader (xlsx Phase 3, doc Phase 5) +│ ├── docsai-odf/ # skeleton (Phase 4) +│ ├── docsai-convert/ # orchestration: detection, pipelines, assets +│ ├── docsai-cli/ # `docsai` binary +│ └── docsai-mcp/ # skeleton (Phase 7) +├── corpus/ +│ ├── generate.py # generates ALL corpus files +│ ├── README.md # what each document isolates +│ ├── docx/*.docx # 14 documents + 14 *.expected.dmk.md (goldens) +│ └── xlsx/*.xlsx # 6 workbooks, for Phase 3 +├── docs/ # design documentation (English) +│ └── spikes/ # spike reports with their decision +└── kb/ # this knowledge base +``` + +## The seven crates + +| Crate | Type | Lines | Responsibility | Depends on | +|---|---|---|---|---| +| `docsai-model` | lib | ~3,500 | The IR and common types | *(nothing from the workspace)* | +| `docsai-docmark` | lib | ~1,850 | IR ⇄ DocMark | model | +| `docsai-office` | lib | ~3,500 | OOXML and legacy readers/writers | model | +| `docsai-odf` | lib | 21 | ODF readers/writers | model | +| `docsai-convert` | lib | ~500 | Detection, pipelines, asset management | the four above | +| `docsai-cli` | bin | ~200 | The `docsai` binary | convert, model | +| `docsai-mcp` | lib | 26 | MCP server over stdio | convert | + +**Dependency rule (`AGENTS.md` §3)**: no format crate imports another format crate. +`docsai-odf` and `docsai-mcp` already exist, empty, precisely so the compiler enforces that +rule from day one rather than discovering it broken in Phase 4. + +## `docsai-model` — the IR + +| Module | Contents | +|---|---| +| `units` | `Length` (newtype over EMU), `Size`, `Point`, exact conversions and output formatting | +| `style` | `StyleCatalog`, `Style`, `FontProps`, `ParaProps`, `DocDefaults` and cascade resolution | +| `list` | `ListCatalog`, `ListDef`, `ListLevel`, `NumFormat` | +| `text` | `TextDocument`, `Section`, `PageGeometry`, `Block`, `Inline`, `Paragraph`, `Table`, `RawFragment` | +| `sheet` | `Workbook`, `Sheet`, `Cell`, `CellRef` (A1), `Formula`, `NumFmt` — populated in Phase 3 | +| `image` | `ImageRef`, `ImageGeometry`, `Anchor` and related types (wrap, crop, flip, borders) | +| `assets` | `AssetStore` (trait), `MemoryAssetStore`, content hash, sniffing and dimensions | +| `report` | `ConversionReport`, typed `Warning`, `Severity`, `ConversionStats` | +| `validate` | Invariants: sheet anchors only in `Workbook`, rows wider than the grid, inverted `two-cell` anchors | + +Three principles encoded in the types, which should not be broken: + +1. **Style = reference + delta.** Every field of `FontProps`/`ParaProps` is `Option`: `None` + means *inherit*, not *disabled*. `over()` merges the cascade; `minus()` computes the delta + that must be emitted (the “economy rule” of spec §3.1). +2. **All lengths in EMU.** A single integer, lossless, for OOXML (EMU/twips), + ODF (cm/in), and `.doc` (twips). +3. **The IR does not know about I/O.** Images are an `AssetId` behind a trait. + +## `docsai-office` — the docx reader + +| Module | Responsibility | +|---|---| +| `xml` | XML tree over `quick-xml` with **byte spans** per node | +| `package` | ZIP, OPC relationships, anti-bomb limits, name sanitization | +| `detect` | Format detection **by content**, not by extension | +| `docx/format` | `w:rPr` and `w:pPr` → the IR delta types | +| `docx/styles` | `styles.xml` → `StyleCatalog`; heading level | +| `docx/numbering` | `numbering.xml` → `ListCatalog` (flattens the double indirection) | +| `docx/drawing` | DrawingML and VML → `ImageRef`/`ImageGeometry` | +| `docx/body` | Body traversal: blocks, inlines, fields, tables, lists | +| `docx/mod` | Assembly: properties, sections, headers/footers, footnotes | + +The detail that holds everything together: **every node in the XML tree remembers its byte +range in the source**. An unrecognized element is preserved by citing the original bytes, not a +re-serialization, and that is why the raw-block is exact. + +## `docsai-docmark` — the serializer + +| Module | Responsibility | +|---|---| +| `escape` | Fixed escape table by context (block, table cell, link label) | +| `attrs` | `{#id .class key="value"}` blocks with the canonical order from spec §8 | +| `units` | How a length, a percentage, or a number is written | +| `frontmatter` | The YAML, handwritten to guarantee byte-for-byte determinism | +| `writer` | IR traversal: blocks, inlines, images, tables, containers | + +## Conversion flow + +``` +file.docx + │ docsai-convert::read_document + ▼ +detect (by content) ──► docsai-office::read_docx + │ Package::open → parts in memory, sanitized names + │ Element::parse → XML tree with spans + │ styles + numbering + footnotes + │ read_sections → blocks, with DirAssetStore receiving media + ▼ + (Document, ConversionReport) + │ docsai_model::validate + ▼ + docsai-docmark::serialize + │ front matter + body, according to --fidelity + ▼ + file.dmk.md + assets/img-. +``` + +## External dependencies + +The entire core rests on five crates: + +| Crate | Purpose | Where | +|---|---|---| +| `serde` | IR serialization | model and derivatives | +| `thiserror` | Typed library errors | model, docmark, office, convert | +| `quick-xml` | XML parsing | office | +| `zip` | OOXML containers | office | +| `tracing` | Logs, always to stderr | office, convert, cli | + +And in the binaries: `clap` (CLI), `anyhow` (binary errors), `serde_json` (`--json`), +`tracing-subscriber`. In tests: `proptest` and `comrak`. + +Deliberately **absent**: `docx-rs` (rejected by spike R1), `image` (reading the format +header is enough), `serde_yaml` (front matter is handwritten and the crate is unmaintained), +and `insta` (goldens are reviewable text files). Each absence is justified in +[`docs/technical-analysis.md`](../docs/technical-analysis.md) §4, as required by +`AGENTS.md` §2. + +## Where the tests live + +| Location | What it covers | +|---|---| +| `#[cfg(test)]` modules in each file | Units: conversions, escaping, style cascade, XML fragment parsing | +| `crates/docsai-model/tests/json_roundtrip.rs` | IR JSON round-trip: hand-built documents with all nodes + proptest | +| `crates/docsai-office/tests/docx_images.rs` | Image geometry over the real corpus | +| `crates/docsai-office/tests/robustness.rs` | The 900+ corrupt inputs, path traversal, ZIP without document | +| `crates/docsai-convert/tests/goldens.rs` | Goldens, determinism, line endings, `plain` with comrak, performance | + +## Conventions worth respecting + +- Code names, comments, commit messages, and **all documentation in `docs/` and `kb/`** must be + in **English** (`AGENTS.md` §5). +- `thiserror` in libraries, `anyhow` only in binaries. +- Parsers **never panic**: no `unwrap`, `expect`, or unchecked indexing on the + read path. +- The serializer is **deterministic**: same IR ⇒ same bytes. Any iteration over a + map uses `BTreeMap`, never `HashMap`. +- Paths with `std::path`, never concatenating `/`. diff --git a/kb/03-decisiones-tecnicas.md b/kb/03-decisiones-tecnicas.md deleted file mode 100644 index cdb730a..0000000 --- a/kb/03-decisiones-tecnicas.md +++ /dev/null @@ -1,177 +0,0 @@ -# 03 — Decisiones técnicas - -Las decisiones que no se deducen del plan y que un futuro colaborador podría deshacer sin -querer. Cada una lleva su motivo y su coste. - -## 1. Parser docx propio, sin `docx-rs` - -**Decisión**: la lectura `.docx` se hace con `zip` + `quick-xml` propios. - -**Motivo** (spike R1, medido sobre el corpus, no supuesto): - -- `docx-rs` resuelve bien estilos, numeración y cabeceras/pies. Pero pierde **casi todo el modelo - de imagen**: ajuste de texto y lado, `behindDoc`, recorte, volteos, texto alternativo, título, - nombre del objeto, hipervínculo sobre la imagen y las imágenes enlazadas. La rotación existe - como campo pero devuelve `0` para un `rot="2700000"`, y el tipo es `u16`, que no puede - representar sesentamilésimas de grado ni valores negativos. -- Descarta `w:footnoteReference` y el `w:instr` de `w:fldSimple`. -- El VML (`w:pict`) se colapsa a un nodo genérico sin `r:id`, sin estilo y sin `alt`. -- No conserva elementos desconocidos, que es justo lo que el raw-block necesita. -- **204 pánicos de 903 entradas corruptas** (23 %), contra un criterio de aceptación que exige - siempre `Err`. - -Las imágenes son requisito de primera clase del proyecto y la mitad de los criterios de la -Fase 1. Lo que quedaba delegable en `docx-rs` era la parte mecánica; el complemento propio -habría sido el grueso de `document.xml` de todos modos, con dos parsers XML en el binario y dos -árboles del mismo documento que reconciliar. - -**Coste asumido**: más superficie propia que mantener. Se mitiga con el corpus, los goldens y la -captura genérica de elementos desconocidos, que hace *visible* lo que falta en vez de silencioso. - -**Detalle**: `docx-rs` sigue siendo candidato para el **writer** de la Fase 2. Es una decisión -independiente y aún abierta: escribir es mucho más simple que leer, porque el XML de salida lo -controlamos nosotros. - -## 2. Árbol XML con spans de bytes - -**Decisión**: `quick-xml` alimenta un árbol en memoria donde **cada nodo recuerda el rango de -bytes que ocupa en el origen**. - -**Motivo**: un raw-block tiene que conservar los bytes originales. Re-serializar el subárbol -daría XML equivalente pero no idéntico, y la re-inyección de la Fase 2 dejaría de ser exacta. Con -el rango basta cortar la cadena de origen. Añadido: un árbol es mucho más revisable que una -máquina de estados sobre eventos, y las partes OOXML caben en memoria de sobra. - -**Coste**: memoria proporcional al tamaño de la parte XML. Aceptable para documentos; habrá que -revisarlo en la Fase 3 para hojas de 100 k celdas, donde `calamine` ya hace streaming por hoja. - -## 3. Coincidencia por nombre local, no por espacio de nombres - -**Decisión**: los elementos se buscan por su nombre local (`p`, no `w:p`). - -**Motivo**: los prefijos OOXML son convencionales pero no obligatorios, y toda la navegación del -lector es contextual (hijos de un padre conocido), así que no hay ambigüedad. Un documento que -renombre sus prefijos se lee igual. - -**Excepción**: los atributos con significado distinto según el espacio de nombres —`r:embed` y -`r:link` junto a un `a:blip`— se buscan por nombre **cualificado** con `attr_qualified`. - -## 4. Formato de longitudes: exactitud antes que legibilidad - -**Decisión**: al escribir una longitud se elige la **primera unidad que la representa -exactamente**, en el orden `px` (96 dpi) → `cm` → `pt` → `emu`. - -**Motivo**: un margen de Word de 1417 twips no es un número redondo de centímetros. Escribirlo -`2.499cm` lo mueve 155 EMU en cada ida y vuelta, y esos errores se acumulan. Ahora sale -`70.85pt`, que es exacto. `emu` es la escotilla final y siempre existe. - -**Coste**: alguna longitud se lee peor (`85.05pt` en vez de «unos 3 cm»). Es el precio de que el -round-trip de la Fase 2 pueda aspirar al 95 % de fidelidad. - -## 5. `Document` externamente etiquetado en serde - -**Decisión**: `#[serde(rename_all = "kebab-case")]` sin `tag`, a diferencia de `Block` e -`Inline`, que sí son adyacentemente etiquetados. - -**Motivo**: un enum *internamente* etiquetado hace que serde almacene el contenido en un búfer -`Content` intermedio, y ese búfer **reescribe toda clave de mapa como cadena**. Eso rompe los -mapas con clave entera de `Workbook` (`cols`, `rows`). Es un fallo que sólo aparece al -deserializar y con un mensaje poco claro (`invalid type: string "0", expected u32`). - -**Relacionado**: `CellRef` implementa `Serialize`/`Deserialize` a mano como su referencia A1 -(`"B2"`), para poder ser clave de mapa y de paso dejar el `inspect --json` legible. - -## 6. `serde_json` con la feature `float_roundtrip` - -**Decisión**: activada en el workspace. - -**Motivo**: sin ella, el parser rápido de coma flotante de `serde_json` puede desplazar un valor -un ULP. Sobre celdas de hoja de cálculo eso es **corromper datos del usuario en silencio**. Se -detectó comparando IR antes y después de un round-trip JSON en el test de proptest. - -## 7. Hash no criptográfico para los assets - -**Decisión**: FNV-1a de 64 bits, mezclando la longitud, renderizado en 16 dígitos hex. - -**Motivo**: el id sólo tiene que ser estable y libre de colisiones en la práctica para los medios -de un documento. Evita meter `sha2` en `docsai-model`, que es el crate que debe permanecer -ligero. El trait `AssetStore` permite que la capa de conversión sustituya el hash sin tocar el -modelo. - -**Cuándo revisarlo**: si algún día el nombre del asset se usa como frontera de confianza. Hoy no -lo es. - -## 8. Sniffing de imágenes propio, sin el crate `image` - -**Decisión**: leer la cabecera de PNG, JPEG, GIF, BMP, TIFF, WebP, EMF y WMF a mano. - -**Motivo**: docsai **nunca recodifica** un mapa de bits; sólo necesita nombrarlo (extensión y -content-type) y medirlo (dimensiones nativas). Eso son unas decenas de líneas contra un árbol de -dependencias considerable en el binario final. - -**Detalle de seguridad**: la extensión sale del **contenido**, nunca del nombre que traiga el -documento. - -## 9. Front matter YAML escrito a mano - -**Decisión**: sin librería YAML. - -**Motivo**: la spec exige determinismo byte a byte, el esquema es pequeño y conocido, y -`serde_yaml` está sin mantenimiento activo. Escribirlo a mano da control total sobre el orden de -las claves y el entrecomillado. - -**Coste**: hay que mantener el emisor a mano. Cuando la Fase 2 escriba el parser, ambos lados -tendrán que moverse juntos; el test de idempotencia lo hará evidente si se desincronizan. - -## 10. Golden files como ficheros de texto, no snapshots de `insta` - -**Decisión**: `corpus/docx/.expected.dmk.md` junto al documento. - -**Motivo**: es lo que prescribe `AGENTS.md` §6, y un golden que es DocMark legible se revisa en -el diff como cualquier otro texto. Un `.snap` de `insta` añade un formato intermedio entre el -revisor y lo que el programa produce. - -Actualizar es deliberado: `DOCSAI_UPDATE_GOLDENS=1 cargo test -p docsai-convert --test goldens`, -y **se revisa el diff**. - -## 11. Corpus generado, no dibujado a mano - -**Decisión**: `corpus/generate.py` produce los 20 ficheros; CI comprueba que el árbol está al día. - -**Motivo**: el XML de cada documento vive en el generador, donde se revisa; un `.docx` hecho con -Word es opaco en la revisión. Los paquetes salen con marca de tiempo y orden de miembros fijos, -así que regenerar da archivos idénticos byte a byte y el repositorio no acumula ruido binario. -Los medios se sintetizan en Python puro para que funcione igual en las tres plataformas de CI. - -**Coste**: son documentos *mínimos*, no documentos de Word reales. Los reales anonimizados siguen -pendientes (Fase 1, tarea 10). La decisión valió su precio de inmediato: el spike destapó que el -generador colocaba `w:drawing` fuera de un `w:r`, que es OOXML inválido, antes de que eso -contaminara ningún golden. - -## 12. `w:sdt` se aplana en lugar de conservarse opaco - -**Decisión**: un control de contenido emite sus bloques hijos y una advertencia `Degraded`, en -vez de un raw-block con todo dentro. - -**Motivo**: meterlo entero en un raw-block escondería texto perfectamente legible detrás de un -bloque opaco, y en modo `plain` desaparecería del todo. El texto es lo que el usuario quiere ver. - -**Coste**: se pierden las propiedades del control (`w:alias`, binding de datos). Está reportado, -no es silencioso, y la Fase 2 tendrá que decidir si las necesita. - -## 13. La advertencia es parte de la salida, no un adorno - -Cada degradación emite una `Warning` **tipada** con severidad. La CLI las cuenta en stderr, las -detalla con `--verbose` y las serializa con `--json`; el código de salida 1 marca una conversión -que perdió algo. `--strict` sube el listón para que cuenten también las menores. - -Es la regla 3 de `AGENTS.md` hecha código: **nada se degrada en silencio**. - -## 14. Límites de seguridad desde el principio - -Aunque el endurecimiento es la Fase 8, tres cosas eran demasiado baratas como para posponerlas: - -- **Tope de descompresión**: 512 MB por paquete, 128 MB por parte. -- **Límite de profundidad XML**: 256 niveles. -- **Saneado de nombres de miembro**: un `word/media/../../evil.png` nunca llega a ser una parte - ni un asset. Hay un test dedicado. diff --git a/kb/03-technical-decisions.md b/kb/03-technical-decisions.md new file mode 100644 index 0000000..66583ce --- /dev/null +++ b/kb/03-technical-decisions.md @@ -0,0 +1,175 @@ +# 03 — Technical decisions + +The decisions that do not follow from the plan alone and that a future collaborator could undo +unintentionally. Each one carries its rationale and its cost. + +## 1. Custom docx parser, without `docx-rs` + +**Decision**: `.docx` reading is done with our own `zip` + `quick-xml`. + +**Rationale** (spike R1, measured on the corpus, not assumed): + +- `docx-rs` handles styles, numbering, and headers/footers well. But it loses **almost the entire + image model**: text wrapping and side, `behindDoc`, crop, flips, alt text, title, + object name, hyperlink on the image, and linked images. Rotation exists as a field but + returns `0` for `rot="2700000"`, and the type is `u16`, which cannot represent sixty-thousandths + of a degree or negative values. +- It drops `w:footnoteReference` and the `w:instr` of `w:fldSimple`. +- VML (`w:pict`) collapses to a generic node with no `r:id`, no style, and no `alt`. +- It does not preserve unknown elements, which is exactly what the raw-block needs. +- **204 panics out of 903 corrupt inputs** (23 %), against an acceptance criterion that requires + always `Err`. + +Images are a first-class requirement of the project and half of the Phase 1 criteria. What +remained delegable to `docx-rs` was the mechanical part; a custom complement would have been the +bulk of `document.xml` anyway, with two XML parsers in the binary and two trees of the same +document to reconcile. + +**Accepted cost**: more custom surface to maintain. Mitigated by the corpus, the goldens, and +generic capture of unknown elements, which makes what is missing *visible* rather than silent. + +**Detail**: `docx-rs` remains a candidate for the Phase 2 **writer**. That is an independent and +still open decision: writing is much simpler than reading, because we control the output XML. + +## 2. XML tree with byte spans + +**Decision**: `quick-xml` feeds an in-memory tree where **each node remembers the byte range it +occupies in the source**. + +**Rationale**: a raw-block must preserve the original bytes. Re-serializing the subtree would +yield equivalent but not identical XML, and Phase 2 re-injection would no longer be exact. With +the range it is enough to slice the source string. Bonus: a tree is far more reviewable than a +state machine over events, and OOXML parts fit comfortably in memory. + +**Cost**: memory proportional to the size of the XML part. Acceptable for documents; it will need +review in Phase 3 for sheets with 100k cells, where `calamine` already streams per sheet. + +## 3. Match by local name, not by namespace + +**Decision**: elements are looked up by their local name (`p`, not `w:p`). + +**Rationale**: OOXML prefixes are conventional but not mandatory, and all reader navigation is +contextual (children of a known parent), so there is no ambiguity. A document that renames its +prefixes is read the same way. + +**Exception**: attributes with different meaning depending on the namespace — `r:embed` and +`r:link` next to an `a:blip` — are looked up by **qualified** name with `attr_qualified`. + +## 4. Length formatting: exactness before readability + +**Decision**: when writing a length, choose the **first unit that represents it exactly**, in the +order `px` (96 dpi) → `cm` → `pt` → `emu`. + +**Rationale**: a Word margin of 1417 twips is not a round number of centimeters. Writing it as +`2.499cm` shifts it by 155 EMU on every round-trip, and those errors accumulate. Now it comes out +as `70.85pt`, which is exact. `emu` is the final escape hatch and always exists. + +**Cost**: some lengths read worse (`85.05pt` instead of “about 3 cm”). That is the price so that +the Phase 2 round-trip can aim for 95 % fidelity. + +## 5. Externally tagged `Document` in serde + +**Decision**: `#[serde(rename_all = "kebab-case")]` without `tag`, unlike `Block` and +`Inline`, which are adjacently tagged. + +**Rationale**: an *internally* tagged enum makes serde store the content in an intermediate +`Content` buffer, and that buffer **rewrites every map key as a string**. That breaks maps with +integer keys in `Workbook` (`cols`, `rows`). It is a failure that only appears on deserialize and +with an unclear message (`invalid type: string "0", expected u32`). + +**Related**: `CellRef` implements `Serialize`/`Deserialize` by hand as its A1 reference +(`"B2"`), so it can be a map key and also keep `inspect --json` readable. + +## 6. `serde_json` with the `float_roundtrip` feature + +**Decision**: enabled in the workspace. + +**Rationale**: without it, `serde_json`'s fast floating-point parser can shift a value by one ULP. +On spreadsheet cells that is **silently corrupting user data**. It was detected by comparing IR +before and after a JSON round-trip in the proptest test. + +## 7. Non-cryptographic hash for assets + +**Decision**: 64-bit FNV-1a, mixing in the length, rendered as 16 hex digits. + +**Rationale**: the id only needs to be stable and practically collision-free for the media of one +document. Avoids pulling `sha2` into `docsai-model`, which is the crate that must stay light. The +`AssetStore` trait lets the conversion layer replace the hash without touching the model. + +**When to revisit**: if someday the asset name is used as a trust boundary. Today it is not. + +## 8. Custom image sniffing, without the `image` crate + +**Decision**: read the header of PNG, JPEG, GIF, BMP, TIFF, WebP, EMF, and WMF by hand. + +**Rationale**: docsai **never re-encodes** a bitmap; it only needs to name it (extension and +content-type) and measure it (native dimensions). That is a few dozen lines against a considerable +dependency tree in the final binary. + +**Security detail**: the extension comes from the **content**, never from the name the document +carries. + +## 9. Handwritten YAML front matter + +**Decision**: no YAML library. + +**Rationale**: the spec requires byte-for-byte determinism, the schema is small and known, and +`serde_yaml` has no active maintenance. Writing it by hand gives full control over key order and +quoting. + +**Cost**: the emitter must be maintained by hand. When Phase 2 writes the parser, both sides will +have to move together; the idempotence test will make it obvious if they drift. + +## 10. Golden files as text files, not `insta` snapshots + +**Decision**: `corpus/docx/.expected.dmk.md` next to the document. + +**Rationale**: that is what `AGENTS.md` §6 prescribes, and a golden that is readable DocMark is +reviewed in the diff like any other text. An `insta` `.snap` adds an intermediate format between +the reviewer and what the program produces. + +Updating is deliberate: `DOCSAI_UPDATE_GOLDENS=1 cargo test -p docsai-convert --test goldens`, +and **the diff is reviewed**. + +## 11. Generated corpus, not hand-drawn + +**Decision**: `corpus/generate.py` produces the 20 files; CI checks that the tree is up to date. + +**Rationale**: each document's XML lives in the generator, where it is reviewed; a `.docx` made +with Word is opaque in review. Packages come out with fixed timestamps and member order, so +regenerating yields byte-identical files and the repository does not accumulate binary noise. +Media are synthesized in pure Python so it works the same on all three CI platforms. + +**Cost**: these are *minimal* documents, not real Word documents. Anonymized real ones remain +pending (Phase 1, task 10). The decision paid for itself immediately: the spike uncovered that the +generator placed `w:drawing` outside a `w:r`, which is invalid OOXML, before that contaminated any +golden. + +## 12. `w:sdt` is flattened instead of kept opaque + +**Decision**: a content control emits its child blocks and a `Degraded` warning, instead of a +raw-block with everything inside. + +**Rationale**: putting the whole thing in a raw-block would hide perfectly readable text behind an +opaque block, and in `plain` mode it would disappear entirely. The text is what the user wants to +see. + +**Cost**: control properties are lost (`w:alias`, data binding). It is reported, not silent, and +Phase 2 will have to decide whether it needs them. + +## 13. The warning is part of the output, not decoration + +Every degradation emits a **typed** `Warning` with severity. The CLI counts them on stderr, +details them with `--verbose`, and serializes them with `--json`; exit code 1 marks a conversion +that lost something. `--strict` raises the bar so minor ones also count. + +It is rule 3 of `AGENTS.md` made code: **nothing is degraded silently**. + +## 14. Security limits from the start + +Although hardening is Phase 8, three things were too cheap to postpone: + +- **Decompression cap**: 512 MB per package, 128 MB per part. +- **XML depth limit**: 256 levels. +- **Member name sanitization**: a `word/media/../../evil.png` never becomes a part or an asset. + There is a dedicated test. diff --git a/kb/04-next-phases.md b/kb/04-next-phases.md new file mode 100644 index 0000000..f41e63a --- /dev/null +++ b/kb/04-next-phases.md @@ -0,0 +1,160 @@ +# 04 — Considerations for the next phases + +What the following phases will already find solved, what awaits them, and the traps that are +already known. + +--- + +## Phase 2 — DocMark → DOCX writing + round-trip + +This is the immediate phase. It closes the cycle and builds the fidelity infrastructure. + +### What is already done for it + +| | | +|---|---| +| **The spec is frozen** | v1.0, with the changelog in §10. The parser has a fixed contract to work against, not a moving draft | +| **The serializer is deterministic** | The idempotence test `serialize(parse(md)) == md` already has half the equation guaranteed | +| **Lengths do not lose precision** | Every written length re-reads exactly. Without this, the fidelity metric would have an artificial floor | +| **The IR is complete** | `Block::TextBox`, `Workbook`, sheet anchors… already exist, even if the docx reader does not produce all of them | +| **The invariant validator** | Already invoked on every conversion; the parser inherits that safety net | +| **Raw-blocks are exact** | They store the original bytes, so `format=ooxml` re-injection is copy and paste | +| **Goldens are the reference** | 14 documents with their expected DocMark: the parser has 14 known inputs and their expected IR | + +### What needs to be built + +1. **DocMark parser** (`docsai-docmark`): comrak + custom layer for `{...}` attributes and fenced + divs `:::`. Comrak is already in the tree as a test dependency, so evaluation is done. +2. **docx writer** (`docsai-office`): IR → `document.xml`, `styles.xml`, `numbering.xml`, media and + properties. +3. **`roundtrip` command** with structural diff of the normalized IR and per-category metric. +4. **Serializer idempotence test** over all goldens. +5. **Property testing**: random IR → md → IR must be the identity. The arbitrary IR generator + **already exists** in `crates/docsai-model/tests/json_roundtrip.rs` and can be reused almost as-is. + +### Traps already identified + +- **Escaping must be reversible, not merely correct.** `escape()` escapes `\` first, so + unescaping is deterministic. The `escaping_is_idempotent_in_shape` test fixes that property; the + parser must be its exact inverse. +- **The three fidelity modes are not symmetric.** Only `full` is reversible. `standard` and + `plain` lose information on purpose, and `roundtrip` only makes sense over `full`. +- **`[]{.empty}`, `[]{.break kind=page}`, and `{.field ...}` are custom syntax** on top of + CommonMark: the parser needs them explicitly. They are in the spec §3.1, §3.2, and §10. +- **A list item never carries two `{...}` blocks.** `list=` goes inside the attribute block of the + first item. +- **The docx writer must put absorbed cells back.** The IR marks `covered=true` and + `colspan`/`rowspan` on the cell that opens the area; OOXML expects `w:gridSpan` and `w:vMerge`. +- **Text boxes are still raw-block.** If Phase 2 wants real `::: {.textbox}`, the reader must be + extended first — it is noted in the plan. +- **Decide whether `docx-rs` works as a writer.** Still open. Spike R1 only closed reading. +- **Validating in Word and LibreOffice** is an acceptance criterion and there is no LibreOffice in + the current CI environment: it must be planned for (manual checklist per release, or headless + `soffice` on the Linux runner). + +--- + +## Phase 3 — Spreadsheets (XLSX/XLS) + +### What is already done + +- **The xlsx corpus exists**: `values-types`, `formulas-basic`, `formulas-shared`, + `number-formats`, `merged-cells`, and `images-anchored` (all three anchors). Generated in Phase 0 + precisely so it would not have to be invented in a hurry. +- **The `Workbook` side of the IR is defined and tested**: `Sheet`, `Cell`, `CellValue`, `Formula` + with its dialect, `NumFmt`, `CellRef` with A1 notation both ways, `ColProps`, `RowProps`, + `Pane`, `DefinedName`. All with verified JSON round-trip. +- **The three sheet anchors** are in the image model and the validator already rejects using them + outside a `Workbook`. +- **`float_roundtrip`** is enabled: numeric values are not corrupted when going through JSON. + +### What needs to be decided and built + +1. **xlsx writer spike**: `umya-spreadsheet` versus `rust_xlsxwriter`. Criterion: which regenerates + styles + `numFmt` from the IR with higher fidelity. Document in `docs/spikes/`. +2. **Sheet serialization to DocMark** (spec §4): the syntax is specified but **not implemented**. + `docsai_docmark::serialize` today returns front matter and an explicit `Degraded` warning for + `Document::Workbook`; that is the entry point. +3. **Reading `xl/drawings/drawing*.xml`** with custom `quick-xml`: neither `calamine` nor `umya` + expose full geometry. The `docx/drawing.rs` module is the pattern to follow; the target model + is the same. + +### Traps already identified + +- **Streaming.** The in-memory XML tree is comfortable for documents but the budget for a 100k-cell + sheet is < 3 s and < 500 MB. Use `calamine` (lazy per sheet) for values and reserve the custom + tree for small parts (`styles.xml`, `drawing*.xml`). +- **Dates are serial numbers plus format.** The IR stores `CellValue::DateTime` in ISO-8601 so that + hand-editing DocMark cannot corrupt them; serial ⇄ ISO conversion both ways is the reader and + writer responsibility. +- **In `two-cell` anchors, `width`/`height` are not serialized** (spec §4.1): size is defined by the + grid. The attribute writer already accounts for this. +- **Formulas keep their dialect**; they are not translated (risk R5). `FormulaDialect` is already + in the IR. + +--- + +## Phase 4 — ODF (ODT and ODS) + +- `docsai-odf` is a skeleton with the `FORMATS` constant; the dependency rule is already enforced + by the compiler. +- The known hard point is **de-automatizing styles**: ODF `office:automatic-styles` represent + direct formatting and must be mapped to IR deltas. The “reference + delta” model is ready to + receive them: `FontProps::minus()` is exactly the inverse operation. +- `detect()` already recognizes ODF by `content.xml` + `mimetype`; it only disambiguates `.odt` + from `.ods` by extension, which will need refining by reading the `mimetype`. +- The image model already has the concepts ODF needs (`as-char`/`char`/`paragraph`/ + `page` → `Inline`/`Floating`, `fo:clip` → `CropRect`). + +--- + +## Phase 5 — Legacy DOC + +- `detect()` already recognizes the OLE2 container and disambiguates `.doc` from `.xls` by name; + that remains to be replaced by reading the CFB directory. +- The two-level strategy from analysis §1.3 still stands: fallback to headless LibreOffice and a + degraded native extractor. +- The `Warning::ImageGeometryDegraded` that the native extractor needs already exists and is + already used (for VML), so the pattern is established. + +--- + +## Phases 6 to 9 — Product, MCP, and hardening + +- **CLI (Phase 6)**: `convert` and `formats` exist with `--fidelity`, `--assets-dir`, `--json`, + `--strict`, `--verbose`, and the exit codes from architecture §5. Still missing: `inspect`, + `roundtrip`, `--style-map`, stdin/stdout with `-`, batch processing, and `cargo-dist`. +- **MCP (Phase 7)**: `docsai-mcp` declares the four tools. The clean-stdout rule is already + respected in the CLI (`tracing` always writes to stderr), so the automated test of that guarantee + already makes sense. +- **Hardening (Phase 8)**: part of the work is ahead of schedule — decompression caps, XML depth + limit, path sanitization, 900+ corrupt inputs in CI. Still missing: real `cargo-fuzz`, the full + adversarial suite, benchmarks with `criterion`, and `cargo audit`/`deny`. The existing corpus + serves as a seed for fuzzing. + +--- + +## Known technical debt + +None of these block anything, but they are worth noting: + +| Item | Where | Note | +|---|---|---| +| Anonymized real documents | Phase 1, task 10 | The synthetic corpus covers traits one by one, but not real-world oddities | +| Text boxes as raw-block | `docx/drawing.rs` | The IR type and spec syntax exist; emit and rebuild are missing | +| `w:lvlOverride` not modeled | `docx/numbering.rs` | Emits `Warning::Degraded` | +| Comments ignored | `docx/body.rs` | Out of v1 scope; emits a warning | +| DrawingML effects | `docx/drawing.rs` | Detected and warned, but not yet dumped to `effects_raw` | +| In-memory XML tree | `office/xml.rs` | Revisit for large sheets in Phase 3 | +| FNV hash for assets | `model/assets.rs` | Sufficient today; revisit if the name becomes a trust boundary | + +## Rules not to break + +1. **Do not advance phases.** The plan order reflects real dependencies. +2. **Do not change the DocMark spec to make a test pass.** It is frozen at v1.0; any change bumps + the front-matter version and documents the migration. +3. **Nothing is degraded silently.** Every loss is a typed `Warning`. +4. **Parsers never panic.** There is a test that checks this with 900+ corrupt inputs; keep it + passing. +5. **The serializer is deterministic.** `BTreeMap` always, `HashMap` never, on any path that + reaches the output. diff --git a/kb/04-siguientes-fases.md b/kb/04-siguientes-fases.md deleted file mode 100644 index f62811b..0000000 --- a/kb/04-siguientes-fases.md +++ /dev/null @@ -1,162 +0,0 @@ -# 04 — Consideraciones para las siguientes fases - -Lo que las fases siguientes van a encontrarse ya resuelto, lo que les espera, y las trampas que -ya se conocen. - ---- - -## Fase 2 — Escritura DocMark → DOCX + round-trip - -Es la fase inmediata. Cierra el ciclo y monta la infraestructura de fidelidad. - -### Lo que ya está hecho para ella - -| | | -|---|---| -| **La spec está congelada** | v1.0, con el registro de cambios en §10. El parser tiene un contrato fijo contra el que trabajar, no un borrador móvil | -| **El serializador es determinista** | El test de idempotencia `serialize(parse(md)) == md` tiene la mitad de la ecuación ya garantizada | -| **Las longitudes no pierden** | Toda longitud escrita se relee exacta. Sin esto, la métrica de fidelidad tendría un suelo artificial | -| **El IR es completo** | `Block::TextBox`, `Workbook`, anclajes de hoja… ya existen, aunque el lector docx no los produzca todos | -| **El validador de invariantes** | Ya se invoca en cada conversión; el parser hereda esa red | -| **Los raw-blocks son exactos** | Guardan los bytes originales, así que la re-inyección `format=ooxml` es copiar y pegar | -| **Los goldens son la referencia** | 14 documentos con su DocMark esperado: el parser tiene 14 entradas conocidas y su IR esperado | - -### Lo que hay que construir - -1. **Parser DocMark** (`docsai-docmark`): comrak + capa propia de atributos `{...}` y fenced divs - `:::`. Comrak ya está en el árbol como dependencia de test, así que la evaluación está hecha. -2. **Writer docx** (`docsai-office`): IR → `document.xml`, `styles.xml`, `numbering.xml`, medios y - propiedades. -3. **Comando `roundtrip`** con diff estructural del IR normalizado y métrica por categoría. -4. **Test de idempotencia** del serializador sobre todos los goldens. -5. **Property testing**: IR aleatorio → md → IR debe ser la identidad. El generador de IRs - arbitrarios **ya existe** en `crates/docsai-model/tests/json_roundtrip.rs` y se puede reutilizar - casi tal cual. - -### Trampas ya identificadas - -- **El escapado tiene que ser reversible, no sólo correcto.** `escape()` escapa `\` primero, de - modo que desescapar es determinista. El test `escaping_is_idempotent_in_shape` fija esa - propiedad; el parser tiene que ser su inverso exacto. -- **Los tres modos de fidelidad no son simétricos.** Sólo `full` es reversible. `standard` y - `plain` pierden a propósito, y el `roundtrip` sólo tiene sentido sobre `full`. -- **`[]{.empty}`, `[]{.break kind=page}` y `{.field ...}` son sintaxis propia** por encima de - CommonMark: el parser los necesita explícitamente. Están en la spec §3.1, §3.2 y §10. -- **Un ítem de lista nunca lleva dos bloques `{...}`.** `list=` va dentro del bloque de atributos - del primer ítem. -- **El writer docx tiene que volver a poner las celdas absorbidas.** El IR marca `covered=true` y - `colspan`/`rowspan` en la celda que abre el área; OOXML espera `w:gridSpan` y `w:vMerge`. -- **Los cuadros de texto siguen siendo raw-block.** Si la Fase 2 quiere `::: {.textbox}` de - verdad, hay que ampliar antes el lector — está anotado en el plan. -- **Decidir si `docx-rs` sirve como writer.** Sigue abierto. El spike R1 sólo cerró la lectura. -- **Validar en Word y LibreOffice** es criterio de aceptación y no hay LibreOffice en el entorno - de CI actual: hay que preverlo (checklist manual por release, o `soffice` headless en el runner - Linux). - ---- - -## Fase 3 — Hojas de cálculo (XLSX/XLS) - -### Lo que ya está hecho - -- **El corpus xlsx existe**: `values-types`, `formulas-basic`, `formulas-shared`, - `number-formats`, `merged-cells` e `images-anchored` (los tres anclajes). Generado en la Fase 0 - precisamente para no tener que inventarlo con prisa. -- **El lado `Workbook` del IR está definido y probado**: `Sheet`, `Cell`, `CellValue`, `Formula` - con su dialecto, `NumFmt`, `CellRef` con notación A1 en ambos sentidos, `ColProps`, `RowProps`, - `Pane`, `DefinedName`. Todo con round-trip JSON verificado. -- **Los tres anclajes de hoja** están en el modelo de imágenes y el validador ya rechaza usarlos - fuera de un `Workbook`. -- **`float_roundtrip`** está activado: los valores numéricos no se corrompen al pasar por JSON. - -### Lo que hay que decidir y construir - -1. **Spike del writer xlsx**: `umya-spreadsheet` contra `rust_xlsxwriter`. Criterio: cuál - regenera con mayor fidelidad estilos + `numFmt` desde el IR. Documentar en `docs/spikes/`. -2. **Serialización de hojas a DocMark** (spec §4): la sintaxis está especificada pero **no - implementada**. `docsai_docmark::serialize` devuelve hoy front matter y una advertencia - `Degraded` explícita para `Document::Workbook`; ese es el punto de entrada. -3. **Lectura de `xl/drawings/drawing*.xml`** propia con `quick-xml`: ni `calamine` ni `umya` - exponen la geometría completa. El módulo `docx/drawing.rs` es el patrón a imitar; el modelo de - destino es el mismo. - -### Trampas ya identificadas - -- **Streaming.** El árbol XML en memoria es cómodo para documentos pero el presupuesto de la hoja - de 100 k celdas es < 3 s y < 500 MB. Habrá que usar `calamine` (que es perezoso por hoja) para - valores y reservar el árbol propio para partes pequeñas (`styles.xml`, `drawing*.xml`). -- **Las fechas son números de serie más formato.** El IR guarda `CellValue::DateTime` en ISO-8601 - para que una edición a mano del DocMark no pueda corromperlas; la conversión de ida y vuelta - serial ⇄ ISO es responsabilidad del lector y el escritor. -- **En anclajes `two-cell` no se serializan `width`/`height`** (spec §4.1): el tamaño lo define la - rejilla. El escritor de atributos ya lo contempla. -- **Las fórmulas conservan su dialecto**, no se traducen (riesgo R5). `FormulaDialect` ya está en - el IR. - ---- - -## Fase 4 — ODF (ODT y ODS) - -- `docsai-odf` es un esqueleto con la constante `FORMATS`; la regla de dependencia ya la impone - el compilador. -- El punto duro conocido es la **des-automatización de estilos**: los `office:automatic-styles` - de ODF representan formato directo, y hay que mapearlos a deltas del IR. El modelo - «referencia + delta» está preparado para recibirlos: `FontProps::minus()` es exactamente la - operación inversa. -- `detect()` ya reconoce ODF por `content.xml` + `mimetype`; sólo desempata `.odt` de `.ods` por - extensión, lo que habrá que afinar leyendo el `mimetype`. -- El modelo de imágenes tiene ya los conceptos que ODF necesita (`as-char`/`char`/`paragraph`/ - `page` → `Inline`/`Floating`, `fo:clip` → `CropRect`). - ---- - -## Fase 5 — DOC legado - -- `detect()` ya reconoce el contenedor OLE2 y desempata `.doc` de `.xls` por nombre; queda - sustituirlo por lectura del directorio CFB. -- La estrategia de dos niveles del análisis §1.3 sigue en pie: fallback a LibreOffice headless - y extractor nativo degradado. -- El `Warning::ImageGeometryDegraded` que necesita el extractor nativo ya existe y ya se usa - (para VML), así que el patrón está establecido. - ---- - -## Fases 6 a 9 — Producto, MCP y endurecimiento - -- **CLI (Fase 6)**: `convert` y `formats` existen con `--fidelity`, `--assets-dir`, `--json`, - `--strict`, `--verbose` y los códigos de salida de arquitectura §5. Faltan `inspect`, - `roundtrip`, `--style-map`, stdin/stdout con `-`, el procesado por lotes y `cargo-dist`. -- **MCP (Fase 7)**: `docsai-mcp` declara las cuatro tools. La regla de stdout limpio ya se - respeta en la CLI (`tracing` escribe siempre a stderr), así que el test automático de esa - garantía tiene sentido desde ya. -- **Endurecimiento (Fase 8)**: parte del trabajo está adelantado —topes de descompresión, límite - de profundidad XML, saneado de rutas, 900+ inputs corruptos en CI—. Lo que falta es `cargo-fuzz` - de verdad, la suite adversaria completa, los benchmarks con `criterion` y `cargo audit`/`deny`. - El corpus existente sirve de semilla para el fuzzing. - ---- - -## Deuda técnica conocida - -Ninguna de estas cosas bloquea nada, pero conviene tenerlas anotadas: - -| Punto | Dónde | Nota | -|---|---|---| -| Documentos reales anonimizados | Fase 1, tarea 10 | El corpus sintético cubre los rasgos uno a uno, pero no las rarezas del mundo real | -| Cuadros de texto como raw-block | `docx/drawing.rs` | El tipo del IR y la sintaxis de la spec existen; falta emitir y reconstruir | -| `w:lvlOverride` no modelado | `docx/numbering.rs` | Emite `Warning::Degraded` | -| Comentarios ignorados | `docx/body.rs` | Fuera de alcance v1; emite advertencia | -| Efectos DrawingML | `docx/drawing.rs` | Se detectan y se avisa, pero aún no se vuelcan a `effects_raw` | -| Árbol XML en memoria | `office/xml.rs` | Revisar para hojas grandes en la Fase 3 | -| Hash FNV para assets | `model/assets.rs` | Suficiente hoy; revisar si el nombre pasa a ser frontera de confianza | - -## Reglas que no conviene romper - -1. **No adelantar fases.** El orden del plan responde a dependencias reales. -2. **No cambiar la spec DocMark para hacer pasar un test.** Está congelada en v1.0; cualquier - cambio sube la versión del front matter y documenta la migración. -3. **Nada se degrada en silencio.** Toda pérdida es una `Warning` tipada. -4. **Los parsers nunca entran en pánico.** Hay un test que lo comprueba con 900+ entradas - corruptas; que siga pasando. -5. **El serializador es determinista.** `BTreeMap` siempre, `HashMap` nunca, en cualquier camino - que llegue a la salida. diff --git a/kb/README.md b/kb/README.md index 956271c..9818d8c 100644 --- a/kb/README.md +++ b/kb/README.md @@ -1,37 +1,37 @@ -# Base de conocimiento de `docsai` +# Knowledge base for `docsai` -Documentación de trabajo sobre **lo que hay construido**, cómo está organizado y qué hay que -tener en cuenta al abordar las fases siguientes. Complementa —no sustituye— la documentación de -diseño de [`docs/`](../docs/), que describe el proyecto completo tal como se concibió. +Working documentation on **what is already built**, how it is organized, and what to keep in +mind when tackling the next phases. It complements — does not replace — the design documentation +in [`docs/`](../docs/), which describes the full project as originally conceived. -Diferencia práctica entre ambas carpetas: +Practical difference between the two folders: -| Carpeta | Responde a | +| Folder | Answers | |---|---| -| [`docs/`](../docs/) | Qué se quiere construir y por qué: análisis, spec DocMark, arquitectura, plan por fases | -| `kb/` (esta) | Qué está construido hoy, cómo, y qué se sabe ya sobre lo que viene | +| [`docs/`](../docs/) | What we want to build and why: analysis, DocMark spec, architecture, phased plan | +| `kb/` (this one) | What is built today, how, and what is already known about what comes next | -## Índice +## Index -| Documento | Contenido | +| Document | Contents | |---|---| -| [01 — Resumen de las fases 0 y 1](01-resumen-fases-0-1.md) | Qué se entregó, criterios de aceptación, qué quedó fuera y por qué | -| [02 — Estructura del proyecto](02-estructura-proyecto.md) | Crates, módulos, reglas de dependencia y por dónde entra cada cosa | -| [03 — Decisiones técnicas](03-decisiones-tecnicas.md) | Las decisiones no obvias, con su motivo y su coste | -| [04 — Consideraciones para las siguientes fases](04-siguientes-fases.md) | Lo que la Fase 2 y posteriores encontrarán ya resuelto, y lo que les espera | +| [01 — Summary of phases 0 and 1](01-phases-0-1-summary.md) | What was delivered, acceptance criteria, what was left out and why | +| [02 — Project structure](02-project-structure.md) | Crates, modules, dependency rules, and where each piece enters | +| [03 — Technical decisions](03-technical-decisions.md) | The non-obvious decisions, with their rationale and cost | +| [04 — Considerations for the next phases](04-next-phases.md) | What Phase 2 and later will already find solved, and what awaits them | -## Estado en una línea +## Status in one line -**Fases 0 y 1 cerradas**: `docsai convert x.docx -o x.dmk.md` funciona con estilos, listas, -tablas, imágenes (geometría completa), cabeceras, pies, notas al pie, campos y propiedades. La -vuelta (DocMark → `.docx`) es la Fase 2 y no existe todavía. +**Phases 0 and 1 closed**: `docsai convert x.docx -o x.dmk.md` works with styles, lists, +tables, images (full geometry), headers, footers, footnotes, fields, and properties. The reverse +path (DocMark → `.docx`) is Phase 2 and does not exist yet. -## Comprobaciones rápidas +## Quick checks ```bash cargo test --workspace # 164 tests cargo clippy --workspace --all-targets -- -D warnings cargo fmt --all -- --check -python3 corpus/generate.py --check # el corpus está al día -cargo run -p docsai-cli -- formats # matriz de soporte real del binario +python3 corpus/generate.py --check # corpus is up to date +cargo run -p docsai-cli -- formats # real support matrix of the binary ``` From f4ecc5cc3c328d46e176e7c6e822fd20cec2e411 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 2 Aug 2026 08:01:01 +0000 Subject: [PATCH 2/4] docs: finish English translations of design docs and DocMark spec --- crates/docsai-cli/src/main.rs | 2 +- crates/docsai-convert/src/assets.rs | 2 +- crates/docsai-convert/src/lib.rs | 20 +- crates/docsai-convert/tests/goldens.rs | 12 +- crates/docsai-docmark/src/attrs.rs | 2 +- crates/docsai-docmark/src/frontmatter.rs | 2 +- .../docsai-docmark/src/frontmatter_parse.rs | 376 ++++ crates/docsai-docmark/src/lib.rs | 26 +- crates/docsai-docmark/src/parser.rs | 1844 +++++++++++++++++ crates/docsai-docmark/src/units.rs | 2 +- crates/docsai-docmark/src/writer.rs | 4 +- crates/docsai-docmark/src/yaml.rs | 27 +- crates/docsai-mcp/src/lib.rs | 6 +- crates/docsai-model/src/assets.rs | 2 +- crates/docsai-model/src/image.rs | 4 +- crates/docsai-model/src/lib.rs | 2 +- crates/docsai-model/src/report.rs | 2 +- crates/docsai-model/src/sheet.rs | 2 +- crates/docsai-model/src/style.rs | 4 +- crates/docsai-model/src/units.rs | 2 +- crates/docsai-model/src/validate.rs | 2 +- crates/docsai-model/tests/json_roundtrip.rs | 6 +- crates/docsai-odf/src/lib.rs | 4 +- crates/docsai-office/src/detect.rs | 4 +- crates/docsai-office/src/docx/drawing.rs | 6 +- crates/docsai-office/src/docx/mod.rs | 6 +- crates/docsai-office/src/error.rs | 2 +- crates/docsai-office/src/lib.rs | 4 +- crates/docsai-office/src/package.rs | 4 +- crates/docsai-office/tests/docx_images.rs | 2 +- crates/docsai-office/tests/robustness.rs | 4 +- docs/_es_arquitectura_src.md | 254 --- docs/development-plan.md | 556 ++--- docs/docmark-specification.md | 491 +++-- docs/spikes/R1-docx-strategy.md | 221 +- docs/technical-analysis.md | 424 ++-- 36 files changed, 3162 insertions(+), 1171 deletions(-) create mode 100644 crates/docsai-docmark/src/frontmatter_parse.rs create mode 100644 crates/docsai-docmark/src/parser.rs delete mode 100644 docs/_es_arquitectura_src.md diff --git a/crates/docsai-cli/src/main.rs b/crates/docsai-cli/src/main.rs index 9fb8677..88bc667 100644 --- a/crates/docsai-cli/src/main.rs +++ b/crates/docsai-cli/src/main.rs @@ -8,7 +8,7 @@ use clap::{Parser, Subcommand}; use docsai_convert::{ConvertOptions, Fidelity, SUPPORT}; use docsai_model::Format; -/// Exit codes (arquitectura §5). +/// Exit codes (architecture §5). const EXIT_OK: u8 = 0; const EXIT_WARNINGS: u8 = 1; const EXIT_INPUT: u8 = 2; diff --git a/crates/docsai-convert/src/assets.rs b/crates/docsai-convert/src/assets.rs index f4361b2..0ca4e45 100644 --- a/crates/docsai-convert/src/assets.rs +++ b/crates/docsai-convert/src/assets.rs @@ -1,4 +1,4 @@ -//! [`AssetStore`] implementations (arquitectura §3.2). +//! [`AssetStore`] implementations (architecture §3.2). use std::collections::BTreeMap; use std::path::{Path, PathBuf}; diff --git a/crates/docsai-convert/src/lib.rs b/crates/docsai-convert/src/lib.rs index d61fc38..f06df08 100644 --- a/crates/docsai-convert/src/lib.rs +++ b/crates/docsai-convert/src/lib.rs @@ -1,7 +1,7 @@ //! Conversion orchestration: format detection, pipelines and asset management. //! //! This is the only crate that knows about more than one format; the readers -//! and writers never talk to each other (arquitectura §1). +//! and writers never talk to each other (architecture §1). #![forbid(unsafe_code)] @@ -43,7 +43,7 @@ pub struct FormatSupport { pub format: Format, pub read: bool, pub write: bool, - /// Phase of `docs/plan-desarrollo.md` that lands the missing direction. + /// Phase of `docs/development-plan.md` that lands the missing direction. pub note: &'static str, } @@ -53,43 +53,43 @@ pub const SUPPORT: &[FormatSupport] = &[ format: Format::Docx, read: true, write: false, - note: "writing arrives in Fase 2", + note: "writing arrives in Phase 2", }, FormatSupport { format: Format::Doc, read: false, write: false, - note: "reading arrives in Fase 5; writing is out of scope", + note: "reading arrives in Phase 5; writing is out of scope", }, FormatSupport { format: Format::Xlsx, read: false, write: false, - note: "arrives in Fase 3", + note: "arrives in Phase 3", }, FormatSupport { format: Format::Xls, read: false, write: false, - note: "reading arrives in Fase 3; writing is out of scope", + note: "reading arrives in Phase 3; writing is out of scope", }, FormatSupport { format: Format::Odt, read: false, write: false, - note: "arrives in Fase 4", + note: "arrives in Phase 4", }, FormatSupport { format: Format::Ods, read: false, write: false, - note: "arrives in Fase 4", + note: "arrives in Phase 4", }, FormatSupport { format: Format::DocMark, read: false, write: true, - note: "reading arrives in Fase 2", + note: "reading arrives in Phase 2", }, ]; @@ -120,7 +120,7 @@ mod tests { ); } assert!(can_read(Format::Docx)); - assert!(!can_write(Format::Docx), "docx writing is Fase 2"); + assert!(!can_write(Format::Docx), "docx writing is Phase 2"); assert!(can_write(Format::DocMark)); } diff --git a/crates/docsai-convert/tests/goldens.rs b/crates/docsai-convert/tests/goldens.rs index 2f100af..7334357 100644 --- a/crates/docsai-convert/tests/goldens.rs +++ b/crates/docsai-convert/tests/goldens.rs @@ -109,7 +109,7 @@ fn serialisation_is_deterministic() { assert_eq!( first, second, - "{} serialises differently on a second run", + "{} serializes differently on a second run", document.display() ); } @@ -137,8 +137,8 @@ fn every_document_uses_unix_line_endings_and_no_bom() { } } -/// Fase 1 acceptance criterion: *«La salida en `--fidelity plain` es CommonMark -/// limpio verificado con comrak»*. +/// Phase 1 acceptance criterion: *"`--fidelity plain` output is clean CommonMark +/// verified with comrak"*. #[test] fn plain_fidelity_is_clean_commonmark() { let mut options = comrak::Options::default(); @@ -204,8 +204,8 @@ fn standard_fidelity_drops_raw_blocks_and_says_so() { assert_eq!(dropped as u32, full_report.raw_blocks_emitted); } -/// Fase 1 acceptance criterion: *«Un docx real de 50+ páginas convierte en -/// < 1 s con < 10 raw-blocks»*. +/// Phase 1 acceptance criterion: *"A real 50+ page docx converts in +/// < 1 s with < 10 raw-blocks"*. #[test] fn a_fifty_page_document_converts_quickly_with_few_raw_blocks() { let path = std::env::temp_dir().join(format!("docsai-perf-{}.docx", std::process::id())); @@ -230,7 +230,7 @@ fn a_fifty_page_document_converts_quickly_with_few_raw_blocks() { report.raw_blocks_emitted ); - // The budget of arquitectura §8 is for an optimised build; a debug build + // The budget of architecture §8 is for an optimised build; a debug build // runs roughly an order of magnitude slower, so it gets a looser bound // rather than no bound at all. let budget = if cfg!(debug_assertions) { diff --git a/crates/docsai-docmark/src/attrs.rs b/crates/docsai-docmark/src/attrs.rs index 84ea3e8..062e503 100644 --- a/crates/docsai-docmark/src/attrs.rs +++ b/crates/docsai-docmark/src/attrs.rs @@ -2,7 +2,7 @@ //! //! The canonical order of spec §8 is enforced here, in one place: id first, //! then classes alphabetically, then keys alphabetically. That ordering is -//! what makes the serialiser deterministic and the golden diffs stable. +//! what makes the serializer deterministic and the golden diffs stable. use std::collections::BTreeMap; diff --git a/crates/docsai-docmark/src/frontmatter.rs b/crates/docsai-docmark/src/frontmatter.rs index 8063548..5b6a3c8 100644 --- a/crates/docsai-docmark/src/frontmatter.rs +++ b/crates/docsai-docmark/src/frontmatter.rs @@ -2,7 +2,7 @@ //! //! Written by hand rather than through a YAML library: the spec demands byte //! determinism, and the schema is small and fully known. Doing it here also -//! keeps a YAML serialiser out of the dependency tree. +//! keeps a YAML serializer out of the dependency tree. use docsai_model::list::{ListCatalog, NumFormat}; use docsai_model::sheet::Workbook; diff --git a/crates/docsai-docmark/src/frontmatter_parse.rs b/crates/docsai-docmark/src/frontmatter_parse.rs new file mode 100644 index 0000000..53bc3ad --- /dev/null +++ b/crates/docsai-docmark/src/frontmatter_parse.rs @@ -0,0 +1,376 @@ +//! Front matter → IR catalogues and metadata. + +use std::collections::BTreeMap; + +use docsai_model::list::{ListCatalog, ListDef, ListId, ListLevel, NumFormat}; +use docsai_model::style::{ + Align, DocDefaults, FontProps, LineHeight, ParaProps, Style, StyleCatalog, StyleId, StyleType, + Underline, VertAlign, +}; +use docsai_model::text::{DocumentMeta, Margins, Orientation, PageGeometry}; +use docsai_model::units::{Length, Size}; +use docsai_model::{Format, DOCMARK_VERSION}; + +use crate::error::ParseError; +use crate::yaml::{self, Value}; + +#[derive(Debug, Default)] +pub struct FrontMatter { + pub source_format: Format, + pub meta: DocumentMeta, + pub page: Option, + pub styles: StyleCatalog, + pub list_defs: ListCatalog, +} + +/// Parses the text between the opening and closing `---` fences. +pub fn parse(text: &str, start_line: usize) -> Result { + let map = yaml::parse_document(text).map_err(|message| { + // yaml errors already include line numbers relative to the body; + // shift them if needed. + let line = message + .strip_prefix("line ") + .and_then(|s| s.split(':').next()) + .and_then(|n| n.parse::().ok()) + .map(|n| start_line + n.saturating_sub(1)) + .unwrap_or(start_line); + ParseError::front_matter(line, message) + })?; + + let mut fm = FrontMatter { + source_format: Format::Docx, + ..Default::default() + }; + + if let Some(v) = map.get("docmark") { + let version = v.as_str().unwrap_or("").trim(); + if !version.is_empty() && version != DOCMARK_VERSION { + // Accept with a soft approach: still parse; future versions may + // need migration, but v1.0 is the only supported contract today. + if !version.starts_with('1') { + return Err(ParseError::front_matter( + start_line, + format!("unsupported docmark version `{version}`"), + )); + } + } + } + + if let Some(v) = map.get("source-format").and_then(|v| v.as_str()) { + fm.source_format = Format::parse(v).unwrap_or(Format::Docx); + } + + fm.meta = read_meta(&map); + if let Some(page) = map.get("page") { + fm.page = Some(read_page(page, start_line)?); + } + if let Some(defaults) = map.get("style-defaults") { + fm.styles.defaults = read_defaults(defaults); + } + if let Some(styles) = map.get("styles").and_then(|v| v.as_map()) { + for (id, value) in styles { + if let Some(style) = read_style(id, value) { + fm.styles.insert(style); + } + } + } + if let Some(lists) = map.get("list-definitions").and_then(|v| v.as_map()) { + for (id, value) in lists { + if let Some(def) = read_list_def(value) { + fm.list_defs.insert(ListId::new(id.clone()), def); + } + } + } + + Ok(fm) +} + +fn read_meta(map: &BTreeMap) -> DocumentMeta { + let text = |key: &str| map.get(key).and_then(|v| v.as_str()).map(str::to_string); + let mut meta = DocumentMeta { + title: text("title"), + author: text("author"), + last_modified_by: text("last-modified-by"), + subject: text("subject"), + keywords: text("keywords"), + description: text("description"), + created: map + .get("created") + .map(|v| v.string_or_empty()) + .filter(|s| !s.is_empty()), + modified: map + .get("modified") + .map(|v| v.string_or_empty()) + .filter(|s| !s.is_empty()), + language: text("language"), + application: text("application"), + custom: BTreeMap::new(), + }; + if let Some(custom) = map.get("custom-properties").and_then(|v| v.as_map()) { + for (k, v) in custom { + meta.custom.insert(k.clone(), v.string_or_empty()); + } + } + meta +} + +fn read_page(value: &Value, line: usize) -> Result { + let map = value.as_map().ok_or_else(|| { + ParseError::front_matter(line, "page must be a mapping") + })?; + let mut page = PageGeometry { + columns: 1, + ..Default::default() + }; + + if let Some(size) = map.get("size") { + if let Some(name) = size.as_str() { + page.size = paper_size(name).ok_or_else(|| { + ParseError::front_matter(line, format!("unknown paper size `{name}`")) + })?; + } else if let Some(m) = size.as_map() { + page.size = Size::new( + len_value(m.get("width")).unwrap_or(Length::ZERO), + len_value(m.get("height")).unwrap_or(Length::ZERO), + ); + } + } + + if let Some(margins) = map.get("margins").and_then(|v| v.as_map()) { + page.margins = Margins { + top: len_value(margins.get("top")).unwrap_or(Length::ZERO), + bottom: len_value(margins.get("bottom")).unwrap_or(Length::ZERO), + left: len_value(margins.get("left")).unwrap_or(Length::ZERO), + right: len_value(margins.get("right")).unwrap_or(Length::ZERO), + header: len_value(margins.get("header")).unwrap_or(Length::ZERO), + footer: len_value(margins.get("footer")).unwrap_or(Length::ZERO), + }; + } + + if let Some(orient) = map.get("orientation").and_then(|v| v.as_str()) { + page.orientation = match orient { + "landscape" => Orientation::Landscape, + _ => Orientation::Portrait, + }; + } + if let Some(cols) = map.get("columns").and_then(|v| v.as_u16()) { + page.columns = cols.max(1); + } + if map.get("title-page").and_then(|v| v.as_bool()) == Some(true) { + page.title_page = true; + } + Ok(page) +} + +fn paper_size(name: &str) -> Option { + // Portrait short × long in mm, matching PageGeometry::paper_name. + let (w_mm, h_mm) = match name { + "A3" => (297.0, 420.0), + "A4" => (210.0, 297.0), + "A5" => (148.0, 210.0), + "Letter" => (215.9, 279.4), + "Legal" => (215.9, 355.6), + "Tabloid" => (279.4, 431.8), + _ => return None, + }; + Some(Size::new(Length::from_mm(w_mm), Length::from_mm(h_mm))) +} + +fn read_defaults(value: &Value) -> DocDefaults { + let mut defaults = DocDefaults::default(); + if let Some(font) = value.get("font") { + defaults.font = read_font(font); + } + if let Some(para) = value.get("paragraph") { + defaults.paragraph = read_para(para); + } + defaults +} + +fn read_style(id: &str, value: &Value) -> Option