diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..0c86fc7 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,37 @@ +name: CI + +on: + pull_request: + push: + branches: + - main + +jobs: + test: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '18' + + - name: Setup pnpm + uses: pnpm/action-setup@v2 + with: + version: 8 + + - name: Install dependencies + run: pnpm install + + - name: Typecheck + run: pnpm typecheck + + - name: Test + run: pnpm test + + - name: Build + run: pnpm build diff --git a/SPEC.md b/SPEC.md index ca5df71..af67c44 100644 --- a/SPEC.md +++ b/SPEC.md @@ -1,8 +1,8 @@ # 🌿 MINT Format Specification -**Version:** 1.0.0 +**Version:** 1.1.0 **Status:** Stable -**Last Updated:** December 2024 +**Last Updated:** May 2026 --- @@ -13,7 +13,7 @@ ### 1.1 Design Philosophy - **Familiar** — Uses YAML for objects, Markdown for tables -- **Minimal** — No unnecessary syntax overhead +- **Minimal** — No unnecessary syntax overhead - **Readable** — Anyone can understand it instantly - **Efficient** — 40-50% fewer tokens than JSON @@ -29,6 +29,7 @@ ## 2. Document Structure A MINT document consists of statements. Each statement is either: + - A key-value pair - A table block @@ -43,6 +44,7 @@ active: true ``` Equivalent JSON: + ```json { "name": "Invoice Reconciliation", @@ -182,6 +184,7 @@ users: ``` Equivalent JSON: + ```json { "users": [ @@ -285,6 +288,7 @@ Comments are not preserved during round-trip. ### 9.1 When to Quote Strings must be quoted when they: + - Contain `|` or `:` (not in URLs) - Start/end with whitespace - Match keywords (`true`, `false`, `null`) @@ -293,13 +297,13 @@ Strings must be quoted when they: ### 9.2 Escape Sequences -| Sequence | Meaning | -|----------|---------| -| `\\` | Backslash | -| `\"` | Double quote | -| `\n` | Newline | -| `\r` | Carriage return | -| `\t` | Tab | +| Sequence | Meaning | +| -------- | --------------- | +| `\\` | Backslash | +| `\"` | Double quote | +| `\n` | Newline | +| `\r` | Carriage return | +| `\t` | Tab | --- @@ -307,16 +311,17 @@ Strings must be quoted when they: Optional symbols for additional compression: -| Symbol | Meaning | -|--------|---------| -| `āœ“` | completed, success, true | -| `āœ—` | failed, error, false | -| `ā³` | pending, waiting | -| `⚠` | warning | -| `?` | review, unknown | -| `-` | null, none | +| Symbol | Meaning | +| ------ | ------------------------ | +| `āœ“` | completed, success, true | +| `āœ—` | failed, error, false | +| `ā³` | pending, waiting | +| `⚠` | warning | +| `?` | review, unknown | +| `-` | null, none | Enable via encoder options: + ```typescript encode(data, { compact: true }); ``` @@ -327,25 +332,25 @@ encode(data, { compact: true }); ### 11.1 Parsing (MINT → JS) -| Input | Type | -|-------|------| -| `true` | boolean | -| `false` | boolean | -| `null`, `-` | null | -| `42`, `-3.14` | number | -| Everything else | string | +| Input | Type | +| --------------- | ------- | +| `true` | boolean | +| `false` | boolean | +| `null`, `-` | null | +| `42`, `-3.14` | number | +| Everything else | string | ### 11.2 Encoding (JS → MINT) -| JavaScript | MINT | -|------------|------| -| `string` | Unquoted or quoted | -| `number` | Decimal | -| `boolean` | `true`/`false` | -| `null` | `null` or `-` in tables | -| `Array` | Inline or table | -| `Object` | Nested structure | -| `Date` | ISO string (quoted) | +| JavaScript | MINT | +| ---------- | ----------------------- | +| `string` | Unquoted or quoted | +| `number` | Decimal | +| `boolean` | `true`/`false` | +| `null` | `null` or `-` in tables | +| `Array` | Inline or table | +| `Object` | Nested structure | +| `Date` | ISO string (quoted) | --- @@ -448,12 +453,15 @@ errors: ### 14.1 Levels **Level 1: Basic** + - Primitives, simple objects, inline arrays -**Level 2: Standard** +**Level 2: Standard** + - Tables, nested structures, quoting **Level 3: Full** + - Compact mode, comments, validation ### 14.2 Test Suite @@ -462,15 +470,57 @@ See `/tests/conformance/` for validation fixtures. --- +## 15. Documents + +MINT can represent a **parsed document** (e.g. the output of a PDF/DOCX parser such as Docling): a title, optional metadata, and a flat list of sections whose hierarchy is expressed by an integer heading level (Markdown-style). Each section holds ordered blocks: prose text, tables, lists, and figures. + +Headings use `§` rather than `#` (which denotes comments — see §8). Document notation is produced and consumed by the dedicated `encodeDocument` / `decodeDocument` functions and never interferes with general-purpose `encode` / `decode`. + +### 15.1 Grammar + +``` +@title Document title (optional, first line) +@meta Metadata block; indented `key: value` lines + key: value +§ Section heading; level is a positive integer + Any non-marker line is prose text +@table[: ] Optional caption for the table that follows +| h1 | h2 | Table rows (the first row is the header) +- Unordered list item +1. Ordered list item +@fig: "" p. Figure (caption and page both optional) +``` + +A prose line that begins with a reserved marker (`§`, `@`, `|`, `- `, or `.`) is prefixed with `\` on encode and unescaped on decode, keeping round-trips lossless. + +### 15.2 Example + +```mint +@title Returns & Refunds SOP +@meta + pages: 4 +§1 Overview +This SOP covers returns. +§2 Eligibility +@table: Return windows +| Category | Days | +| Apparel | 30 | +1. Inspect item +2. Issue refund +@fig: "Refund flow" p.3 +``` + +--- + ## Appendix A: Token Efficiency For an array of 10 objects with 5 fields: -| Format | Tokens | -|--------|--------| -| JSON | 250-300 | -| YAML | 180-220 | -| TOON | 120-150 | +| Format | Tokens | +| -------- | ----------- | +| JSON | 250-300 | +| YAML | 180-220 | +| TOON | 120-150 | | **MINT** | **140-170** | MINT trades ~15% vs TOON for significantly better readability. @@ -497,4 +547,4 @@ MINT trades ~15% vs TOON for significantly better readability. --- -*End of Specification* +_End of Specification_ diff --git a/package.json b/package.json index 356f80d..093fea7 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "mint-format", - "version": "1.0.2", + "version": "1.1.0", "private": true, "description": "🌿 MINT Format - Minimal Inference Notation for Tokens", "keywords": [ @@ -75,4 +75,4 @@ "build": "^0.1.4", "test": "^3.3.0" } -} \ No newline at end of file +} diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index e44bf45..144d180 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -8,7 +8,7 @@ import { readFileSync, writeFileSync, existsSync } from 'fs'; import { resolve, extname } from 'path'; -import { encode, decode, validate, estimateTokens } from '@q1k-oss/mint-format'; +import { encode, decode, validate } from '@q1k-oss/mint-format'; interface CliOptions { input?: string; diff --git a/packages/mint/dist/index.cjs b/packages/mint/dist/index.cjs index 8556734..e0af445 100644 --- a/packages/mint/dist/index.cjs +++ b/packages/mint/dist/index.cjs @@ -21,12 +21,186 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru var index_exports = {}; __export(index_exports, { decode: () => decode, + decodeDocument: () => decodeDocument, default: () => index_default, encode: () => encode, + encodeDocument: () => encodeDocument, estimateTokens: () => estimateTokens, validate: () => validate }); module.exports = __toCommonJS(index_exports); + +// src/document.ts +var RESERVED_LINE = /^(§|@|\||- |\\|\d+\. )/; +function escapeProse(line) { + return RESERVED_LINE.test(line) ? `\\${line}` : line; +} +function unescapeProse(line) { + return line.startsWith("\\") ? line.slice(1) : line; +} +function quote(value) { + return `"${value.replace(/\\/g, "\\\\").replace(/"/g, '\\"')}"`; +} +function unquote(value) { + const inner = value.slice(1, -1); + return inner.replace(/\\"/g, '"').replace(/\\\\/g, "\\"); +} +function renderTable(table, out) { + if (table.caption) out.push(`@table: ${table.caption}`); + const widths = table.headers.map( + (h, i) => Math.max(h.length, ...table.rows.map((r) => (r[i] ?? "").length)) + ); + const fmtRow = (cells) => `| ${cells.map((c, i) => (c ?? "").padEnd(widths[i])).join(" | ")} |`; + out.push(fmtRow(table.headers)); + for (const row of table.rows) out.push(fmtRow(table.headers.map((_, i) => row[i] ?? ""))); +} +function renderBlock(block, out) { + switch (block.type) { + case "text": + for (const line of block.text.split("\n")) out.push(escapeProse(line)); + break; + case "table": + renderTable(block.table, out); + break; + case "list": + block.list.items.forEach( + (item, i) => out.push(block.list.ordered ? `${i + 1}. ${item}` : `- ${item}`) + ); + break; + case "figure": { + const parts = []; + if (block.figure.caption) parts.push(quote(block.figure.caption)); + if (typeof block.figure.page === "number") parts.push(`p.${block.figure.page}`); + out.push(`@fig:${parts.length ? ` ${parts.join(" ")}` : ""}`); + break; + } + } +} +function encodeDocument(doc) { + const out = []; + if (doc.title) out.push(`@title ${doc.title}`); + if (doc.metadata && Object.keys(doc.metadata).length > 0) { + out.push("@meta"); + for (const [key, value] of Object.entries(doc.metadata)) { + out.push(` ${key}: ${value}`); + } + } + for (const section of doc.sections) { + out.push(`\xA7${section.level} ${section.heading}`); + for (const block of section.blocks) renderBlock(block, out); + } + return out.join("\n"); +} +function splitRow(line) { + return line.trim().split("|").slice(1, -1).map((c) => c.trim()); +} +function decodeDocument(input) { + const lines = input.replace(/\r\n/g, "\n").replace(/\r/g, "\n").split("\n"); + const doc = { sections: [] }; + let i = 0; + if (lines[i]?.startsWith("@title ")) { + doc.title = lines[i].slice("@title ".length); + i++; + } + if (lines[i] === "@meta") { + i++; + const meta = {}; + while (i < lines.length && /^\s+\S/.test(lines[i]) && !lines[i].startsWith("\xA7")) { + const trimmed = lines[i].trim(); + const colon = trimmed.indexOf(":"); + if (colon === -1) break; + const key = trimmed.slice(0, colon).trim(); + const raw = trimmed.slice(colon + 1).trim(); + meta[key] = /^-?\d+(\.\d+)?$/.test(raw) ? Number(raw) : raw; + i++; + } + doc.metadata = meta; + } + let current = null; + const state = { text: null, list: null, table: null, pendingTableCaption: void 0 }; + const flush = () => { + if (!current) return; + if (state.text) current.blocks.push({ type: "text", text: state.text.join("\n") }); + if (state.list) current.blocks.push({ type: "list", list: state.list }); + if (state.table) { + const [headers, ...rows] = state.table.rows; + current.blocks.push({ + type: "table", + table: { caption: state.table.caption, headers: headers ?? [], rows } + }); + } + state.text = null; + state.list = null; + state.table = null; + }; + for (; i < lines.length; i++) { + const line = lines[i]; + const headingMatch = /^§(\d+)\s(.*)$/.exec(line); + if (headingMatch) { + flush(); + current = { heading: headingMatch[2], level: Number(headingMatch[1]), blocks: [] }; + doc.sections.push(current); + continue; + } + if (!current) continue; + if (line.startsWith("@table")) { + flush(); + const colon = line.indexOf(":"); + state.pendingTableCaption = colon !== -1 ? line.slice(colon + 1).trim() || void 0 : void 0; + continue; + } + if (line.startsWith("|")) { + if (!state.table) { + flush(); + state.table = { caption: state.pendingTableCaption, rows: [] }; + state.pendingTableCaption = void 0; + } + state.table.rows.push(splitRow(line)); + continue; + } + if (line.startsWith("@fig:")) { + flush(); + const body = line.slice("@fig:".length).trim(); + const figure = {}; + const pageMatch = /\bp\.(\d+)\s*$/.exec(body); + let rest = body; + if (pageMatch) { + figure.page = Number(pageMatch[1]); + rest = body.slice(0, pageMatch.index).trim(); + } + if (rest.startsWith('"') && rest.endsWith('"')) figure.caption = unquote(rest); + else if (rest) figure.caption = rest; + current.blocks.push({ type: "figure", figure }); + continue; + } + const orderedMatch = /^(\d+)\.\s(.*)$/.exec(line); + if (orderedMatch) { + if (!state.list || !state.list.ordered) { + flush(); + state.list = { ordered: true, items: [] }; + } + state.list.items.push(orderedMatch[2]); + continue; + } + if (line.startsWith("- ")) { + if (!state.list || state.list.ordered) { + flush(); + state.list = { ordered: false, items: [] }; + } + state.list.items.push(line.slice(2)); + continue; + } + if (!state.text) { + flush(); + state.text = []; + } + state.text.push(unescapeProse(line)); + } + flush(); + return doc; +} + +// src/index.ts var STATUS_SYMBOLS = { completed: "\u2713", complete: "\u2713", @@ -470,11 +644,13 @@ function estimateTokens(data) { savingsPercent }; } -var index_default = { encode, decode, validate, estimateTokens }; +var index_default = { encode, decode, validate, estimateTokens, encodeDocument, decodeDocument }; // Annotate the CommonJS export names for ESM import in node: 0 && (module.exports = { decode, + decodeDocument, encode, + encodeDocument, estimateTokens, validate }); diff --git a/packages/mint/dist/index.d.cts b/packages/mint/dist/index.d.cts index 98ba007..fcc94fe 100644 --- a/packages/mint/dist/index.d.cts +++ b/packages/mint/dist/index.d.cts @@ -1,3 +1,81 @@ +/** + * 🌿 MINT Document — a first-class representation for parsed documents. + * + * Tools like Docling parse a PDF/DOCX into a hierarchical document: a title, + * nested sections (heading + prose), tables, lists, and figures with page + * references. MINT's object/table syntax can't express that hierarchy + * compactly, and `#` is reserved for comments — so headings need their own + * grammar. This module adds a Markdown-flavoured, token-efficient, lossless + * document encoding on top of MINT, exposed as explicit + * `encodeDocument` / `decodeDocument` functions so it never destabilises the + * general-purpose `encode` / `decode`. + * + * Grammar (line-oriented): + * @title document title (optional, first) + * @meta metadata block; indented `key: value` lines + * key: value + * § section heading; level is a positive integer + * any non-marker line is prose text + * @table[: ] optional caption for the table that follows + * | a | b | table rows (first row = headers) + * - unordered list item + * 1. ordered list item + * @fig: "" p. figure (caption and/or page optional) + * + * Prose lines that begin with a reserved marker (`§ @ | - ` or `.`) are + * prefixed with `\` on encode and unescaped on decode, keeping round-trips + * lossless. + * + * @packageDocumentation + */ +interface MintDocTable { + caption?: string; + headers: string[]; + rows: string[][]; +} +interface MintDocList { + ordered: boolean; + items: string[]; +} +interface MintDocFigure { + caption?: string; + /** 1-based page number the figure appears on, if known. */ + page?: number; +} +type MintDocBlock = { + type: 'text'; + text: string; +} | { + type: 'table'; + table: MintDocTable; +} | { + type: 'list'; + list: MintDocList; +} | { + type: 'figure'; + figure: MintDocFigure; +}; +interface MintSection { + heading: string; + /** Heading depth, 1-based. Hierarchy is implied by level, Markdown-style. */ + level: number; + blocks: MintDocBlock[]; +} +interface MintDocument { + title?: string; + metadata?: Record; + sections: MintSection[]; +} +/** + * Encode a parsed document into MINT document notation. + */ +declare function encodeDocument(doc: MintDocument): string; +/** + * Decode MINT document notation back into a MintDocument. Inverse of + * {@link encodeDocument}. + */ +declare function decodeDocument(input: string): MintDocument; + /** * 🌿 MINT Format - Minimal Inference Notation for Tokens * @@ -94,11 +172,14 @@ declare function validate(input: string): ValidationResult; * @returns Token estimates for JSON and MINT */ declare function estimateTokens(data: unknown): TokenEstimate; + declare const _default: { encode: typeof encode; decode: typeof decode; validate: typeof validate; estimateTokens: typeof estimateTokens; + encodeDocument: typeof encodeDocument; + decodeDocument: typeof decodeDocument; }; -export { type DecodeOptions, type EncodeOptions, type TokenEstimate, type ValidationError, type ValidationResult, decode, _default as default, encode, estimateTokens, validate }; +export { type DecodeOptions, type EncodeOptions, type MintDocBlock, type MintDocFigure, type MintDocList, type MintDocTable, type MintDocument, type MintSection, type TokenEstimate, type ValidationError, type ValidationResult, decode, decodeDocument, _default as default, encode, encodeDocument, estimateTokens, validate }; diff --git a/packages/mint/dist/index.d.ts b/packages/mint/dist/index.d.ts index 98ba007..fcc94fe 100644 --- a/packages/mint/dist/index.d.ts +++ b/packages/mint/dist/index.d.ts @@ -1,3 +1,81 @@ +/** + * 🌿 MINT Document — a first-class representation for parsed documents. + * + * Tools like Docling parse a PDF/DOCX into a hierarchical document: a title, + * nested sections (heading + prose), tables, lists, and figures with page + * references. MINT's object/table syntax can't express that hierarchy + * compactly, and `#` is reserved for comments — so headings need their own + * grammar. This module adds a Markdown-flavoured, token-efficient, lossless + * document encoding on top of MINT, exposed as explicit + * `encodeDocument` / `decodeDocument` functions so it never destabilises the + * general-purpose `encode` / `decode`. + * + * Grammar (line-oriented): + * @title document title (optional, first) + * @meta metadata block; indented `key: value` lines + * key: value + * § section heading; level is a positive integer + * any non-marker line is prose text + * @table[: ] optional caption for the table that follows + * | a | b | table rows (first row = headers) + * - unordered list item + * 1. ordered list item + * @fig: "" p. figure (caption and/or page optional) + * + * Prose lines that begin with a reserved marker (`§ @ | - ` or `.`) are + * prefixed with `\` on encode and unescaped on decode, keeping round-trips + * lossless. + * + * @packageDocumentation + */ +interface MintDocTable { + caption?: string; + headers: string[]; + rows: string[][]; +} +interface MintDocList { + ordered: boolean; + items: string[]; +} +interface MintDocFigure { + caption?: string; + /** 1-based page number the figure appears on, if known. */ + page?: number; +} +type MintDocBlock = { + type: 'text'; + text: string; +} | { + type: 'table'; + table: MintDocTable; +} | { + type: 'list'; + list: MintDocList; +} | { + type: 'figure'; + figure: MintDocFigure; +}; +interface MintSection { + heading: string; + /** Heading depth, 1-based. Hierarchy is implied by level, Markdown-style. */ + level: number; + blocks: MintDocBlock[]; +} +interface MintDocument { + title?: string; + metadata?: Record; + sections: MintSection[]; +} +/** + * Encode a parsed document into MINT document notation. + */ +declare function encodeDocument(doc: MintDocument): string; +/** + * Decode MINT document notation back into a MintDocument. Inverse of + * {@link encodeDocument}. + */ +declare function decodeDocument(input: string): MintDocument; + /** * 🌿 MINT Format - Minimal Inference Notation for Tokens * @@ -94,11 +172,14 @@ declare function validate(input: string): ValidationResult; * @returns Token estimates for JSON and MINT */ declare function estimateTokens(data: unknown): TokenEstimate; + declare const _default: { encode: typeof encode; decode: typeof decode; validate: typeof validate; estimateTokens: typeof estimateTokens; + encodeDocument: typeof encodeDocument; + decodeDocument: typeof decodeDocument; }; -export { type DecodeOptions, type EncodeOptions, type TokenEstimate, type ValidationError, type ValidationResult, decode, _default as default, encode, estimateTokens, validate }; +export { type DecodeOptions, type EncodeOptions, type MintDocBlock, type MintDocFigure, type MintDocList, type MintDocTable, type MintDocument, type MintSection, type TokenEstimate, type ValidationError, type ValidationResult, decode, decodeDocument, _default as default, encode, encodeDocument, estimateTokens, validate }; diff --git a/packages/mint/dist/index.js b/packages/mint/dist/index.js index 8fad02f..f8138e9 100644 --- a/packages/mint/dist/index.js +++ b/packages/mint/dist/index.js @@ -1,3 +1,173 @@ +// src/document.ts +var RESERVED_LINE = /^(§|@|\||- |\\|\d+\. )/; +function escapeProse(line) { + return RESERVED_LINE.test(line) ? `\\${line}` : line; +} +function unescapeProse(line) { + return line.startsWith("\\") ? line.slice(1) : line; +} +function quote(value) { + return `"${value.replace(/\\/g, "\\\\").replace(/"/g, '\\"')}"`; +} +function unquote(value) { + const inner = value.slice(1, -1); + return inner.replace(/\\"/g, '"').replace(/\\\\/g, "\\"); +} +function renderTable(table, out) { + if (table.caption) out.push(`@table: ${table.caption}`); + const widths = table.headers.map( + (h, i) => Math.max(h.length, ...table.rows.map((r) => (r[i] ?? "").length)) + ); + const fmtRow = (cells) => `| ${cells.map((c, i) => (c ?? "").padEnd(widths[i])).join(" | ")} |`; + out.push(fmtRow(table.headers)); + for (const row of table.rows) out.push(fmtRow(table.headers.map((_, i) => row[i] ?? ""))); +} +function renderBlock(block, out) { + switch (block.type) { + case "text": + for (const line of block.text.split("\n")) out.push(escapeProse(line)); + break; + case "table": + renderTable(block.table, out); + break; + case "list": + block.list.items.forEach( + (item, i) => out.push(block.list.ordered ? `${i + 1}. ${item}` : `- ${item}`) + ); + break; + case "figure": { + const parts = []; + if (block.figure.caption) parts.push(quote(block.figure.caption)); + if (typeof block.figure.page === "number") parts.push(`p.${block.figure.page}`); + out.push(`@fig:${parts.length ? ` ${parts.join(" ")}` : ""}`); + break; + } + } +} +function encodeDocument(doc) { + const out = []; + if (doc.title) out.push(`@title ${doc.title}`); + if (doc.metadata && Object.keys(doc.metadata).length > 0) { + out.push("@meta"); + for (const [key, value] of Object.entries(doc.metadata)) { + out.push(` ${key}: ${value}`); + } + } + for (const section of doc.sections) { + out.push(`\xA7${section.level} ${section.heading}`); + for (const block of section.blocks) renderBlock(block, out); + } + return out.join("\n"); +} +function splitRow(line) { + return line.trim().split("|").slice(1, -1).map((c) => c.trim()); +} +function decodeDocument(input) { + const lines = input.replace(/\r\n/g, "\n").replace(/\r/g, "\n").split("\n"); + const doc = { sections: [] }; + let i = 0; + if (lines[i]?.startsWith("@title ")) { + doc.title = lines[i].slice("@title ".length); + i++; + } + if (lines[i] === "@meta") { + i++; + const meta = {}; + while (i < lines.length && /^\s+\S/.test(lines[i]) && !lines[i].startsWith("\xA7")) { + const trimmed = lines[i].trim(); + const colon = trimmed.indexOf(":"); + if (colon === -1) break; + const key = trimmed.slice(0, colon).trim(); + const raw = trimmed.slice(colon + 1).trim(); + meta[key] = /^-?\d+(\.\d+)?$/.test(raw) ? Number(raw) : raw; + i++; + } + doc.metadata = meta; + } + let current = null; + const state = { text: null, list: null, table: null, pendingTableCaption: void 0 }; + const flush = () => { + if (!current) return; + if (state.text) current.blocks.push({ type: "text", text: state.text.join("\n") }); + if (state.list) current.blocks.push({ type: "list", list: state.list }); + if (state.table) { + const [headers, ...rows] = state.table.rows; + current.blocks.push({ + type: "table", + table: { caption: state.table.caption, headers: headers ?? [], rows } + }); + } + state.text = null; + state.list = null; + state.table = null; + }; + for (; i < lines.length; i++) { + const line = lines[i]; + const headingMatch = /^§(\d+)\s(.*)$/.exec(line); + if (headingMatch) { + flush(); + current = { heading: headingMatch[2], level: Number(headingMatch[1]), blocks: [] }; + doc.sections.push(current); + continue; + } + if (!current) continue; + if (line.startsWith("@table")) { + flush(); + const colon = line.indexOf(":"); + state.pendingTableCaption = colon !== -1 ? line.slice(colon + 1).trim() || void 0 : void 0; + continue; + } + if (line.startsWith("|")) { + if (!state.table) { + flush(); + state.table = { caption: state.pendingTableCaption, rows: [] }; + state.pendingTableCaption = void 0; + } + state.table.rows.push(splitRow(line)); + continue; + } + if (line.startsWith("@fig:")) { + flush(); + const body = line.slice("@fig:".length).trim(); + const figure = {}; + const pageMatch = /\bp\.(\d+)\s*$/.exec(body); + let rest = body; + if (pageMatch) { + figure.page = Number(pageMatch[1]); + rest = body.slice(0, pageMatch.index).trim(); + } + if (rest.startsWith('"') && rest.endsWith('"')) figure.caption = unquote(rest); + else if (rest) figure.caption = rest; + current.blocks.push({ type: "figure", figure }); + continue; + } + const orderedMatch = /^(\d+)\.\s(.*)$/.exec(line); + if (orderedMatch) { + if (!state.list || !state.list.ordered) { + flush(); + state.list = { ordered: true, items: [] }; + } + state.list.items.push(orderedMatch[2]); + continue; + } + if (line.startsWith("- ")) { + if (!state.list || state.list.ordered) { + flush(); + state.list = { ordered: false, items: [] }; + } + state.list.items.push(line.slice(2)); + continue; + } + if (!state.text) { + flush(); + state.text = []; + } + state.text.push(unescapeProse(line)); + } + flush(); + return doc; +} + // src/index.ts var STATUS_SYMBOLS = { completed: "\u2713", @@ -442,11 +612,13 @@ function estimateTokens(data) { savingsPercent }; } -var index_default = { encode, decode, validate, estimateTokens }; +var index_default = { encode, decode, validate, estimateTokens, encodeDocument, decodeDocument }; export { decode, + decodeDocument, index_default as default, encode, + encodeDocument, estimateTokens, validate }; diff --git a/packages/mint/package.json b/packages/mint/package.json index e0201c4..d316032 100644 --- a/packages/mint/package.json +++ b/packages/mint/package.json @@ -1,6 +1,6 @@ { "name": "@q1k-oss/mint-format", - "version": "1.0.2", + "version": "1.1.0", "description": "🌿 MINT Format encoder/decoder - Minimal Inference Notation for Tokens", "keywords": [ "mint", @@ -59,4 +59,4 @@ "engines": { "node": ">=16.0.0" } -} \ No newline at end of file +} diff --git a/packages/mint/src/document.test.ts b/packages/mint/src/document.test.ts new file mode 100644 index 0000000..04aa544 --- /dev/null +++ b/packages/mint/src/document.test.ts @@ -0,0 +1,139 @@ +import { describe, it, expect } from "vitest"; +import { validate } from "./index"; +import { encodeDocument, decodeDocument, type MintDocument } from "./document"; + +describe("🌿 MINT Document", () => { + const sample: MintDocument = { + title: "Returns & Refunds SOP", + metadata: { pages: 4, author: "Acme Ops" }, + sections: [ + { + heading: "Overview", + level: 1, + blocks: [ + { + type: "text", + text: "This SOP covers returns.\nIt applies to all stores.", + }, + ], + }, + { + heading: "Eligibility", + level: 2, + blocks: [ + { type: "text", text: "Items are eligible within 30 days." }, + { + type: "table", + table: { + caption: "Return windows", + headers: ["Category", "Days"], + rows: [ + ["Apparel", "30"], + ["Electronics", "14"], + ], + }, + }, + { + type: "list", + list: { + ordered: true, + items: ["Inspect item", "Issue refund", "Update inventory"], + }, + }, + { type: "figure", figure: { caption: "Refund flow", page: 3 } }, + ], + }, + ], + }; + + it("round-trips a full document losslessly", () => { + expect(decodeDocument(encodeDocument(sample))).toEqual(sample); + }); + + it("produces output that passes MINT validation", () => { + expect(validate(encodeDocument(sample)).valid).toBe(true); + }); + + it("renders headings with § level markers, not # (reserved for comments)", () => { + const out = encodeDocument(sample); + expect(out).toContain("§1 Overview"); + expect(out).toContain("§2 Eligibility"); + expect(out).not.toMatch(/^#/m); + }); + + it("renders tables with optional captions and MINT pipe syntax", () => { + const out = encodeDocument(sample); + expect(out).toContain("@table: Return windows"); + expect(out).toMatch(/\| Category\s+\| Days\s+\|/); + }); + + it("renders ordered vs unordered lists distinctly and round-trips", () => { + const doc: MintDocument = { + sections: [ + { + heading: "Lists", + level: 1, + blocks: [ + { + type: "list", + list: { ordered: false, items: ["alpha", "beta"] }, + }, + { type: "list", list: { ordered: true, items: ["one", "two"] } }, + ], + }, + ], + }; + const out = encodeDocument(doc); + expect(out).toContain("- alpha"); + expect(out).toContain("1. one"); + expect(decodeDocument(out)).toEqual(doc); + }); + + it("escapes prose lines that begin with reserved markers", () => { + const doc: MintDocument = { + sections: [ + { + heading: "Tricky", + level: 1, + blocks: [ + { + type: "text", + text: "| not a table\n- not a list\n§ not a heading\n@ not a tag", + }, + ], + }, + ], + }; + expect(decodeDocument(encodeDocument(doc))).toEqual(doc); + }); + + it("handles figures with only a caption or only a page", () => { + const doc: MintDocument = { + sections: [ + { + heading: "Figures", + level: 1, + blocks: [ + { type: "figure", figure: { caption: "Diagram only" } }, + { type: "figure", figure: { page: 7 } }, + { type: "figure", figure: {} }, + ], + }, + ], + }; + expect(decodeDocument(encodeDocument(doc))).toEqual(doc); + }); + + it("round-trips a document with no title or metadata", () => { + const doc: MintDocument = { + sections: [ + { + heading: "Body", + level: 1, + blocks: [{ type: "text", text: "Just text." }], + }, + ], + }; + expect(decodeDocument(encodeDocument(doc))).toEqual(doc); + }); +}); diff --git a/packages/mint/src/document.ts b/packages/mint/src/document.ts new file mode 100644 index 0000000..e7c2ddd --- /dev/null +++ b/packages/mint/src/document.ts @@ -0,0 +1,301 @@ +/** + * 🌿 MINT Document — a first-class representation for parsed documents. + * + * Tools like Docling parse a PDF/DOCX into a hierarchical document: a title, + * nested sections (heading + prose), tables, lists, and figures with page + * references. MINT's object/table syntax can't express that hierarchy + * compactly, and `#` is reserved for comments — so headings need their own + * grammar. This module adds a Markdown-flavoured, token-efficient, lossless + * document encoding on top of MINT, exposed as explicit + * `encodeDocument` / `decodeDocument` functions so it never destabilises the + * general-purpose `encode` / `decode`. + * + * Grammar (line-oriented): + * @title document title (optional, first) + * @meta metadata block; indented `key: value` lines + * key: value + * § section heading; level is a positive integer + * any non-marker line is prose text + * @table[: ] optional caption for the table that follows + * | a | b | table rows (first row = headers) + * - unordered list item + * 1. ordered list item + * @fig: "" p. figure (caption and/or page optional) + * + * Prose lines that begin with a reserved marker (`§ @ | - ` or `.`) are + * prefixed with `\` on encode and unescaped on decode, keeping round-trips + * lossless. + * + * @packageDocumentation + */ + +export interface MintDocTable { + caption?: string; + headers: string[]; + rows: string[][]; +} + +export interface MintDocList { + ordered: boolean; + items: string[]; +} + +export interface MintDocFigure { + caption?: string; + /** 1-based page number the figure appears on, if known. */ + page?: number; +} + +export type MintDocBlock = + | { type: "text"; text: string } + | { type: "table"; table: MintDocTable } + | { type: "list"; list: MintDocList } + | { type: "figure"; figure: MintDocFigure }; + +export interface MintSection { + heading: string; + /** Heading depth, 1-based. Hierarchy is implied by level, Markdown-style. */ + level: number; + blocks: MintDocBlock[]; +} + +export interface MintDocument { + title?: string; + metadata?: Record; + sections: MintSection[]; +} + +const RESERVED_LINE = /^(§|@|\||- |\\|\d+\. )/; + +function escapeProse(line: string): string { + return RESERVED_LINE.test(line) ? `\\${line}` : line; +} + +function unescapeProse(line: string): string { + return line.startsWith("\\") ? line.slice(1) : line; +} + +function quote(value: string): string { + return `"${value.replace(/\\/g, "\\\\").replace(/"/g, '\\"')}"`; +} + +function unquote(value: string): string { + const inner = value.slice(1, -1); + return inner.replace(/\\"/g, '"').replace(/\\\\/g, "\\"); +} + +function renderTable(table: MintDocTable, out: string[]): void { + if (table.caption) out.push(`@table: ${table.caption}`); + const widths = table.headers.map((h, i) => + Math.max(h.length, ...table.rows.map((r) => (r[i] ?? "").length)), + ); + const fmtRow = (cells: string[]) => + `| ${cells.map((c, i) => (c ?? "").padEnd(widths[i])).join(" | ")} |`; + out.push(fmtRow(table.headers)); + for (const row of table.rows) + out.push(fmtRow(table.headers.map((_, i) => row[i] ?? ""))); +} + +function renderBlock(block: MintDocBlock, out: string[]): void { + switch (block.type) { + case "text": + for (const line of block.text.split("\n")) out.push(escapeProse(line)); + break; + case "table": + renderTable(block.table, out); + break; + case "list": + block.list.items.forEach((item, i) => + out.push(block.list.ordered ? `${i + 1}. ${item}` : `- ${item}`), + ); + break; + case "figure": { + const parts: string[] = []; + if (block.figure.caption) parts.push(quote(block.figure.caption)); + if (typeof block.figure.page === "number") + parts.push(`p.${block.figure.page}`); + out.push(`@fig:${parts.length ? ` ${parts.join(" ")}` : ""}`); + break; + } + } +} + +/** + * Encode a parsed document into MINT document notation. + */ +export function encodeDocument(doc: MintDocument): string { + const out: string[] = []; + + if (doc.title) out.push(`@title ${doc.title}`); + + if (doc.metadata && Object.keys(doc.metadata).length > 0) { + out.push("@meta"); + for (const [key, value] of Object.entries(doc.metadata)) { + out.push(` ${key}: ${value}`); + } + } + + for (const section of doc.sections) { + out.push(`§${section.level} ${section.heading}`); + for (const block of section.blocks) renderBlock(block, out); + } + + return out.join("\n"); +} + +interface ParseState { + text: string[] | null; + list: { ordered: boolean; items: string[] } | null; + table: { caption?: string; rows: string[][] } | null; + pendingTableCaption: string | undefined; +} + +function splitRow(line: string): string[] { + return line + .trim() + .split("|") + .slice(1, -1) + .map((c) => c.trim()); +} + +/** + * Decode MINT document notation back into a MintDocument. Inverse of + * {@link encodeDocument}. + */ +export function decodeDocument(input: string): MintDocument { + const lines = input.replace(/\r\n/g, "\n").replace(/\r/g, "\n").split("\n"); + const doc: MintDocument = { sections: [] }; + + let i = 0; + + // Header: @title + @meta (only valid before the first section). + if (lines[i]?.startsWith("@title ")) { + doc.title = lines[i].slice("@title ".length); + i++; + } + if (lines[i] === "@meta") { + i++; + const meta: Record = {}; + while ( + i < lines.length && + /^\s+\S/.test(lines[i]) && + !lines[i].startsWith("§") + ) { + const trimmed = lines[i].trim(); + const colon = trimmed.indexOf(":"); + if (colon === -1) break; + const key = trimmed.slice(0, colon).trim(); + const raw = trimmed.slice(colon + 1).trim(); + meta[key] = /^-?\d+(\.\d+)?$/.test(raw) ? Number(raw) : raw; + i++; + } + doc.metadata = meta; + } + + let current: MintSection | null = null; + const state: ParseState = { + text: null, + list: null, + table: null, + pendingTableCaption: undefined, + }; + + const flush = () => { + if (!current) return; + if (state.text) + current.blocks.push({ type: "text", text: state.text.join("\n") }); + if (state.list) current.blocks.push({ type: "list", list: state.list }); + if (state.table) { + const [headers, ...rows] = state.table.rows; + current.blocks.push({ + type: "table", + table: { caption: state.table.caption, headers: headers ?? [], rows }, + }); + } + state.text = null; + state.list = null; + state.table = null; + }; + + for (; i < lines.length; i++) { + const line = lines[i]; + const headingMatch = /^§(\d+)\s(.*)$/.exec(line); + + if (headingMatch) { + flush(); + current = { + heading: headingMatch[2], + level: Number(headingMatch[1]), + blocks: [], + }; + doc.sections.push(current); + continue; + } + + if (!current) continue; // prose before any section is dropped (shouldn't happen) + + if (line.startsWith("@table")) { + flush(); + const colon = line.indexOf(":"); + state.pendingTableCaption = + colon !== -1 ? line.slice(colon + 1).trim() || undefined : undefined; + continue; + } + + if (line.startsWith("|")) { + if (!state.table) { + flush(); + state.table = { caption: state.pendingTableCaption, rows: [] }; + state.pendingTableCaption = undefined; + } + state.table.rows.push(splitRow(line)); + continue; + } + + if (line.startsWith("@fig:")) { + flush(); + const body = line.slice("@fig:".length).trim(); + const figure: MintDocFigure = {}; + const pageMatch = /\bp\.(\d+)\s*$/.exec(body); + let rest = body; + if (pageMatch) { + figure.page = Number(pageMatch[1]); + rest = body.slice(0, pageMatch.index).trim(); + } + if (rest.startsWith('"') && rest.endsWith('"')) + figure.caption = unquote(rest); + else if (rest) figure.caption = rest; + current.blocks.push({ type: "figure", figure }); + continue; + } + + const orderedMatch = /^(\d+)\.\s(.*)$/.exec(line); + if (orderedMatch) { + if (!state.list || !state.list.ordered) { + flush(); + state.list = { ordered: true, items: [] }; + } + state.list.items.push(orderedMatch[2]); + continue; + } + + if (line.startsWith("- ")) { + if (!state.list || state.list.ordered) { + flush(); + state.list = { ordered: false, items: [] }; + } + state.list.items.push(line.slice(2)); + continue; + } + + // Prose text (possibly escaped). + if (!state.text) { + flush(); + state.text = []; + } + state.text.push(unescapeProse(line)); + } + + flush(); + return doc; +} diff --git a/packages/mint/src/index.ts b/packages/mint/src/index.ts index ef75d7a..bd0a0d7 100644 --- a/packages/mint/src/index.ts +++ b/packages/mint/src/index.ts @@ -43,50 +43,57 @@ export interface TokenEstimate { // Status symbols for compact mode const STATUS_SYMBOLS: Record = { - completed: 'āœ“', - complete: 'āœ“', - success: 'āœ“', - done: 'āœ“', - passed: 'āœ“', - true: 'āœ“', - yes: 'āœ“', - failed: 'āœ—', - failure: 'āœ—', - error: 'āœ—', - rejected: 'āœ—', - false: 'āœ—', - no: 'āœ—', - pending: 'ā³', - waiting: 'ā³', - in_progress: 'ā³', - running: 'ā³', - warning: '⚠', - warn: '⚠', - review: '?', - unknown: '?', + completed: "āœ“", + complete: "āœ“", + success: "āœ“", + done: "āœ“", + passed: "āœ“", + true: "āœ“", + yes: "āœ“", + failed: "āœ—", + failure: "āœ—", + error: "āœ—", + rejected: "āœ—", + false: "āœ—", + no: "āœ—", + pending: "ā³", + waiting: "ā³", + in_progress: "ā³", + running: "ā³", + warning: "⚠", + warn: "⚠", + review: "?", + unknown: "?", }; const REVERSE_SYMBOLS: Record = { - 'āœ“': 'true', - 'āœ—': 'false', - 'ā³': 'pending', - '⚠': 'warning', - '?': 'unknown', + "āœ“": "true", + "āœ—": "false", + "ā³": "pending", + "⚠": "warning", + "?": "unknown", }; -type JsonValue = string | number | boolean | null | JsonValue[] | { [key: string]: JsonValue }; +type JsonValue = + | string + | number + | boolean + | null + | JsonValue[] + | { [key: string]: JsonValue }; /** * Check if a value needs quoting */ function needsQuoting(value: string): boolean { - if (value === '') return true; - if (value.startsWith(' ') || value.endsWith(' ')) return true; - if (value.includes('|') || value.includes('\n') || value.includes('\r')) return true; - if (value.includes(',')) return true; // Commas would be parsed as array separators + if (value === "") return true; + if (value.startsWith(" ") || value.endsWith(" ")) return true; + if (value.includes("|") || value.includes("\n") || value.includes("\r")) + return true; + if (value.includes(",")) return true; // Commas would be parsed as array separators if (/^-?\d+\.?\d*$/.test(value)) return true; - if (['true', 'false', 'null'].includes(value.toLowerCase())) return true; - if (value.includes(':') && !value.includes('://')) return true; + if (["true", "false", "null"].includes(value.toLowerCase())) return true; + if (value.includes(":") && !value.includes("://")) return true; if (value.includes('"')) return true; return false; } @@ -96,11 +103,11 @@ function needsQuoting(value: string): boolean { */ function escapeString(value: string): string { return value - .replace(/\\/g, '\\\\') + .replace(/\\/g, "\\\\") .replace(/"/g, '\\"') - .replace(/\n/g, '\\n') - .replace(/\r/g, '\\r') - .replace(/\t/g, '\\t'); + .replace(/\n/g, "\\n") + .replace(/\r/g, "\\r") + .replace(/\t/g, "\\t"); } /** @@ -108,36 +115,40 @@ function escapeString(value: string): string { */ function unescapeString(value: string): string { return value - .replace(/\\n/g, '\n') - .replace(/\\r/g, '\r') - .replace(/\\t/g, '\t') + .replace(/\\n/g, "\n") + .replace(/\\r/g, "\r") + .replace(/\\t/g, "\t") .replace(/\\"/g, '"') - .replace(/\\\\/g, '\\'); + .replace(/\\\\/g, "\\"); } /** * Format a primitive value */ -function formatPrimitive(value: JsonValue, options: EncodeOptions, inTable = false): string { +function formatPrimitive( + value: JsonValue, + options: EncodeOptions, + inTable = false, +): string { if (value === null || value === undefined) { - return inTable ? '-' : 'null'; + return inTable ? "-" : "null"; } - if (typeof value === 'boolean') { + if (typeof value === "boolean") { return String(value); } - if (typeof value === 'number') { - if (!isFinite(value)) return 'null'; + if (typeof value === "number") { + if (!isFinite(value)) return "null"; return String(value); } - if (typeof value === 'string') { + if (typeof value === "string") { if (options.compact && STATUS_SYMBOLS[value.toLowerCase()]) { return STATUS_SYMBOLS[value.toLowerCase()]; } - if (value === '' && inTable) return '-'; + if (value === "" && inTable) return "-"; if (needsQuoting(value)) { return `"${escapeString(value)}"`; } @@ -151,7 +162,12 @@ function formatPrimitive(value: JsonValue, options: EncodeOptions, inTable = fal * Check if a value is a primitive (not an object or array) */ function isPrimitive(value: JsonValue): boolean { - return value === null || typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean'; + return ( + value === null || + typeof value === "string" || + typeof value === "number" || + typeof value === "boolean" + ); } /** @@ -160,18 +176,23 @@ function isPrimitive(value: JsonValue): boolean { function isTableArray(arr: JsonValue[]): arr is Record[] { if (arr.length === 0) return false; - if (!arr.every((item) => item !== null && typeof item === 'object' && !Array.isArray(item))) { + if ( + !arr.every( + (item) => + item !== null && typeof item === "object" && !Array.isArray(item), + ) + ) { return false; } const firstKeys = Object.keys(arr[0] as Record) .sort() - .join(','); + .join(","); const sameKeys = arr.every((item) => { const keys = Object.keys(item as Record) .sort() - .join(','); + .join(","); return keys === firstKeys; }); @@ -210,20 +231,26 @@ function getColumnWidths(headers: string[], rows: string[][]): number[] { * Pad cell to width */ function padCell(value: string, width: number): string { - return value + ' '.repeat(Math.max(0, width - value.length)); + return value + " ".repeat(Math.max(0, width - value.length)); } /** * Encode array as table */ -function encodeTable(arr: Record[], options: EncodeOptions, indentLevel: number): string { - if (arr.length === 0) return '| |'; - - const indent = ' '.repeat(options.indent || 2); +function encodeTable( + arr: Record[], + options: EncodeOptions, + indentLevel: number, +): string { + if (arr.length === 0) return "| |"; + + const indent = " ".repeat(options.indent || 2); const baseIndent = indent.repeat(indentLevel); const headers = Object.keys(arr[0]); - const rows: string[][] = arr.map((obj) => headers.map((h) => formatPrimitive(obj[h], options, true))); + const rows: string[][] = arr.map((obj) => + headers.map((h) => formatPrimitive(obj[h], options, true)), + ); const widths = getColumnWidths(headers, rows); @@ -231,51 +258,55 @@ function encodeTable(arr: Record[], options: EncodeOptions, i // Header const headerCells = headers.map((h, i) => padCell(h, widths[i])); - lines.push(`${baseIndent}| ${headerCells.join(' | ')} |`); + lines.push(`${baseIndent}| ${headerCells.join(" | ")} |`); // Rows for (const row of rows) { const cells = row.map((cell, i) => padCell(cell, widths[i])); - lines.push(`${baseIndent}| ${cells.join(' | ')} |`); + lines.push(`${baseIndent}| ${cells.join(" | ")} |`); } - return lines.join('\n'); + return lines.join("\n"); } /** * Encode value recursively */ -function encodeValue(value: JsonValue, options: EncodeOptions, indentLevel: number): string { - const indent = ' '.repeat(options.indent || 2); +function encodeValue( + value: JsonValue, + options: EncodeOptions, + indentLevel: number, +): string { + const indent = " ".repeat(options.indent || 2); const baseIndent = indent.repeat(indentLevel); // Primitives - if (value === null || typeof value !== 'object') { + if (value === null || typeof value !== "object") { return formatPrimitive(value, options); } // Arrays if (Array.isArray(value)) { if (value.length === 0) { - return '[]'; + return "[]"; } if (isPrimitiveArray(value)) { - return value.map((v) => formatPrimitive(v, options)).join(', '); + return value.map((v) => formatPrimitive(v, options)).join(", "); } if (isTableArray(value)) { - return '\n' + encodeTable(value, options, indentLevel + 1); + return "\n" + encodeTable(value, options, indentLevel + 1); } // Mixed arrays const lines: string[] = []; for (const item of value) { - if (item === null || typeof item !== 'object') { + if (item === null || typeof item !== "object") { lines.push(`${baseIndent}${indent}- ${formatPrimitive(item, options)}`); } else { const encoded = encodeValue(item, options, indentLevel + 2); - if (encoded.includes('\n')) { + if (encoded.includes("\n")) { lines.push(`${baseIndent}${indent}-`); lines.push(encoded); } else { @@ -283,7 +314,7 @@ function encodeValue(value: JsonValue, options: EncodeOptions, indentLevel: numb } } } - return '\n' + lines.join('\n'); + return "\n" + lines.join("\n"); } // Objects @@ -291,7 +322,7 @@ function encodeValue(value: JsonValue, options: EncodeOptions, indentLevel: numb const keys = options.sortKeys ? Object.keys(obj).sort() : Object.keys(obj); if (keys.length === 0) { - return ''; + return ""; } const lines: string[] = []; @@ -301,21 +332,25 @@ function encodeValue(value: JsonValue, options: EncodeOptions, indentLevel: numb if (val === null) { lines.push(`${baseIndent}${key}: null`); - } else if (typeof val !== 'object') { + } else if (typeof val !== "object") { lines.push(`${baseIndent}${key}: ${formatPrimitive(val, options)}`); } else if (Array.isArray(val)) { if (val.length === 0) { lines.push(`${baseIndent}${key}: []`); } else if (isPrimitiveArray(val)) { - lines.push(`${baseIndent}${key}: ${val.map((v) => formatPrimitive(v, options)).join(', ')}`); + lines.push( + `${baseIndent}${key}: ${val.map((v) => formatPrimitive(v, options)).join(", ")}`, + ); } else { - lines.push(`${baseIndent}${key}:${encodeValue(val, options, indentLevel)}`); + lines.push( + `${baseIndent}${key}:${encodeValue(val, options, indentLevel)}`, + ); } } else { const nested = encodeValue(val, options, indentLevel + 1); - if (nested === '') { + if (nested === "") { lines.push(`${baseIndent}${key}:`); - } else if (nested.includes('\n')) { + } else if (nested.includes("\n")) { lines.push(`${baseIndent}${key}:`); lines.push(nested); } else { @@ -324,7 +359,7 @@ function encodeValue(value: JsonValue, options: EncodeOptions, indentLevel: numb } } - return lines.join('\n'); + return lines.join("\n"); } /** @@ -359,17 +394,17 @@ export function encode(value: unknown, options: EncodeOptions = {}): string { }; if (value === null || value === undefined) { - return 'null'; + return "null"; } - if (typeof value !== 'object') { + if (typeof value !== "object") { return formatPrimitive(value as JsonValue, opts); } if (Array.isArray(value)) { - if (value.length === 0) return '_: []'; + if (value.length === 0) return "_: []"; if (isPrimitiveArray(value)) { - return `_: ${value.map((v) => formatPrimitive(v, opts)).join(', ')}`; + return `_: ${value.map((v) => formatPrimitive(v, opts)).join(", ")}`; } if (isTableArray(value)) { return `_:\n${encodeTable(value, opts, 1)}`; @@ -386,12 +421,12 @@ export function encode(value: unknown, options: EncodeOptions = {}): string { function parsePrimitive(value: string): JsonValue { const trimmed = value.trim(); - if (trimmed === 'null' || trimmed === '-' || trimmed === '') { + if (trimmed === "null" || trimmed === "-" || trimmed === "") { return null; } - if (trimmed === 'true') return true; - if (trimmed === 'false') return false; + if (trimmed === "true") return true; + if (trimmed === "false") return false; if (REVERSE_SYMBOLS[trimmed]) { return REVERSE_SYMBOLS[trimmed]; @@ -415,19 +450,19 @@ function parsePrimitive(value: string): JsonValue { function parseTable( lines: string[], startIndex: number, - indent: number + indent: number, ): { value: JsonValue[]; endIndex: number } { const result: Record[] = []; let headers: string[] = []; let i = startIndex; const headerLine = lines[i].trim(); - if (!headerLine.startsWith('|')) { + if (!headerLine.startsWith("|")) { return { value: [], endIndex: startIndex }; } headers = headerLine - .split('|') + .split("|") .slice(1, -1) .map((h) => h.trim()); @@ -438,16 +473,16 @@ function parseTable( const lineIndent = line.length - line.trimStart().length; const trimmed = line.trim(); - if (!trimmed.startsWith('|')) { + if (!trimmed.startsWith("|")) { break; } - if (lineIndent < indent && trimmed !== '') { + if (lineIndent < indent && trimmed !== "") { break; } const cells = trimmed - .split('|') + .split("|") .slice(1, -1) .map((c) => parsePrimitive(c.trim())); @@ -472,7 +507,7 @@ function parseDocument( lines: string[], startIndex: number, baseIndent: number, - options: DecodeOptions + options: DecodeOptions, ): { value: JsonValue; endIndex: number } { const result: Record = {}; let i = startIndex; @@ -480,7 +515,7 @@ function parseDocument( while (i < lines.length) { const line = lines[i]; - if (line.trim() === '' || line.trim().startsWith('#')) { + if (line.trim() === "" || line.trim().startsWith("#")) { i++; continue; } @@ -501,12 +536,12 @@ function parseDocument( const trimmed = line.trim(); // Table rows at this level shouldn't happen in object context - if (trimmed.startsWith('|')) { + if (trimmed.startsWith("|")) { i++; continue; } - const colonIndex = trimmed.indexOf(':'); + const colonIndex = trimmed.indexOf(":"); if (colonIndex === -1) { i++; continue; @@ -515,13 +550,13 @@ function parseDocument( const key = trimmed.slice(0, colonIndex).trim(); const valueStr = trimmed.slice(colonIndex + 1).trim(); - if (valueStr === '' || valueStr === '[]') { + if (valueStr === "" || valueStr === "[]") { // Look ahead for nested content let foundNested = false; let nextIdx = i + 1; // Skip blank lines to find actual content - while (nextIdx < lines.length && lines[nextIdx].trim() === '') { + while (nextIdx < lines.length && lines[nextIdx].trim() === "") { nextIdx++; } @@ -530,15 +565,24 @@ function parseDocument( const nextIndent = nextLine.length - nextLine.trimStart().length; const nextTrimmed = nextLine.trim(); - if (nextIndent > baseIndent && nextTrimmed.startsWith('|')) { + if (nextIndent > baseIndent && nextTrimmed.startsWith("|")) { // Parse table const tableResult = parseTable(lines, nextIdx, nextIndent); result[key] = tableResult.value; i = tableResult.endIndex + 1; foundNested = true; - } else if (nextIndent > baseIndent && nextTrimmed !== '' && !nextTrimmed.startsWith('#')) { + } else if ( + nextIndent > baseIndent && + nextTrimmed !== "" && + !nextTrimmed.startsWith("#") + ) { // Parse nested object - const nestedResult = parseDocument(lines, nextIdx, nextIndent, options); + const nestedResult = parseDocument( + lines, + nextIdx, + nextIndent, + options, + ); result[key] = nestedResult.value; i = nestedResult.endIndex; foundNested = true; @@ -546,18 +590,18 @@ function parseDocument( } if (!foundNested) { - result[key] = valueStr === '[]' ? [] : {}; + result[key] = valueStr === "[]" ? [] : {}; i++; } } else if (valueStr.startsWith('"') && valueStr.endsWith('"')) { // Quoted string - parse as single value, don't split result[key] = parsePrimitive(valueStr); i++; - } else if (valueStr.includes(' | ')) { - result[key] = valueStr.split(' | ').map((v) => parsePrimitive(v.trim())); + } else if (valueStr.includes(" | ")) { + result[key] = valueStr.split(" | ").map((v) => parsePrimitive(v.trim())); i++; - } else if (valueStr.includes(', ')) { - result[key] = valueStr.split(', ').map((v) => parsePrimitive(v.trim())); + } else if (valueStr.includes(", ")) { + result[key] = valueStr.split(", ").map((v) => parsePrimitive(v.trim())); i++; } else { result[key] = parsePrimitive(valueStr); @@ -595,17 +639,18 @@ export function decode(input: string, options: DecodeOptions = {}): unknown { ...options, }; - const normalized = input.replace(/\r\n/g, '\n').replace(/\r/g, '\n'); - const lines = normalized.split('\n'); + const normalized = input.replace(/\r\n/g, "\n").replace(/\r/g, "\n"); + const lines = normalized.split("\n"); - if (lines.every((l) => l.trim() === '' || l.trim().startsWith('#'))) { + if (lines.every((l) => l.trim() === "" || l.trim().startsWith("#"))) { return {}; } let startIndex = 0; while ( startIndex < lines.length && - (lines[startIndex].trim() === '' || lines[startIndex].trim().startsWith('#')) + (lines[startIndex].trim() === "" || + lines[startIndex].trim().startsWith("#")) ) { startIndex++; } @@ -615,23 +660,23 @@ export function decode(input: string, options: DecodeOptions = {}): unknown { } const firstLine = lines[startIndex].trim(); - if (firstLine.startsWith('|')) { + if (firstLine.startsWith("|")) { const tableResult = parseTable(lines, startIndex, 0); return tableResult.value; } - if (firstLine.startsWith('_:')) { + if (firstLine.startsWith("_:")) { const valueStr = firstLine.slice(2).trim(); - if (valueStr === '' || valueStr === '[]') { + if (valueStr === "" || valueStr === "[]") { const nextLine = lines[startIndex + 1]; - if (nextLine && nextLine.trim().startsWith('|')) { + if (nextLine && nextLine.trim().startsWith("|")) { const tableResult = parseTable(lines, startIndex + 1, 2); return tableResult.value; } return []; } - if (valueStr.includes(', ')) { - return valueStr.split(', ').map((v) => parsePrimitive(v.trim())); + if (valueStr.includes(", ")) { + return valueStr.split(", ").map((v) => parsePrimitive(v.trim())); } return [parsePrimitive(valueStr)]; } @@ -648,7 +693,7 @@ export function decode(input: string, options: DecodeOptions = {}): unknown { */ export function validate(input: string): ValidationResult { const errors: ValidationError[] = []; - const lines = input.replace(/\r\n/g, '\n').replace(/\r/g, '\n').split('\n'); + const lines = input.replace(/\r\n/g, "\n").replace(/\r/g, "\n").split("\n"); let inTable = false; let tableColumns = 0; @@ -658,7 +703,7 @@ export function validate(input: string): ValidationResult { const lineNum = i + 1; const trimmed = line.trim(); - if (trimmed === '' || trimmed.startsWith('#')) { + if (trimmed === "" || trimmed.startsWith("#")) { continue; } @@ -673,8 +718,8 @@ export function validate(input: string): ValidationResult { }); } - if (trimmed.startsWith('|')) { - const pipes = trimmed.split('|').length - 1; + if (trimmed.startsWith("|")) { + const pipes = trimmed.split("|").length - 1; if (!inTable) { inTable = true; @@ -690,11 +735,11 @@ export function validate(input: string): ValidationResult { } } - if (!trimmed.endsWith('|')) { + if (!trimmed.endsWith("|")) { errors.push({ line: lineNum, column: trimmed.length, - message: 'Table row must end with |', + message: "Table row must end with |", context: line, }); } @@ -734,4 +779,24 @@ export function estimateTokens(data: unknown): TokenEstimate { }; } -export default { encode, decode, validate, estimateTokens }; \ No newline at end of file +export { + encodeDocument, + decodeDocument, + type MintDocument, + type MintSection, + type MintDocBlock, + type MintDocTable, + type MintDocList, + type MintDocFigure, +} from "./document"; + +import { encodeDocument, decodeDocument } from "./document"; + +export default { + encode, + decode, + validate, + estimateTokens, + encodeDocument, + decodeDocument, +};