diff --git a/.gitignore b/.gitignore index e6f64b4..9e15dc8 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ larkDocx2mdOutput/ .qoder .agents coverage +.claude diff --git a/CHANGELOG.md b/CHANGELOG.md index e8ec33b..a6609a5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## 0.8.1 + +### Fixed + +- **嵌套子块保留与未支持块回退**:飞书大纲/折叠场景下,任意叶子块(todo/heading/quote/equation/code/image 等)都可能携带 children,原实现只让容器型 parser 消费 children,导致这些子块被静默丢弃、导出文档章节内容缺失。本次让叶子块未自行消费的 children 统一挂到 `node.blocks`,序列化时在节点自身内容之后渲染;容器型 parser 标记 `consumesChildren` 避免重复挂载。未支持块类型不再整块丢弃,而是回退为 paragraph(尽量提取文本)并保留其嵌套子树,既无文本也无子块时才丢弃 + ## 0.8.0 ### Removed diff --git a/package.json b/package.json index d04984e..8fa47ec 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "lark-docx2md", - "version": "0.8.0", + "version": "0.8.1", "description": "Convert Lark/Feishu documents to Markdown", "type": "module", "main": "./dist/converter.js", diff --git a/src/md-ast/parser.ts b/src/md-ast/parser.ts index e9bd009..59716b2 100644 --- a/src/md-ast/parser.ts +++ b/src/md-ast/parser.ts @@ -1,10 +1,17 @@ import { Registry } from '../core/registry.js'; +import { createLogger } from '../lib/logger.js'; import type { DocInfo, DocxBlock, TextBody, TextElement } from '../lib/types.js'; import type { MdBlockNode, MdInlineNode } from './types.js'; +const logger = createLogger('parser'); + export interface BlockParser { blockType: number; + /** 容器型 parser(page/bullet/ordered/callout/quoteContainer/grid/table)自行消费 block.children, + * 置为 true;叶子型 parser 省略(false),由 Parser 框架统一解析 block.children 挂到 node.blocks。 */ + consumesChildren?: boolean; + parse (block: DocxBlock, ctx: ParserContext): MdBlockNode | null; } @@ -37,13 +44,45 @@ export class Parser { blockMap, parseBlock: (block: DocxBlock): MdBlockNode | null => { const bt = block.block_type; - if (bt === undefined || bt === null) return null; + if (bt === undefined || bt === null) { + logger.warn(`Dropped block ${block.block_id}: missing block_type`); + return null; + } const parser = this.registry.get(bt); - if (parser) { - return parser.parse(block, ctx); + if (!parser) { + // 未注册的块类型(如 Bitable/File/Mindnote/Iframe/Diagram 等): + // 回退为文案(尽量提取文本),并保留其嵌套子块——飞书任意块都可能挂 children, + // 若直接丢弃会把整棵子树一起丢掉。 + const text = extractBlockText(block); + const hasChildren = (block.children?.length ?? 0) > 0; + if (!text && !hasChildren) { + // GridColumn(25) 由 Grid parser 内部处理;此处为既无文本也无子块的未支持块 + logger.warn(`Dropped block ${block.block_id}: no parser for block_type=${bt} and no text/children (parent=${block.parent_id ?? '-'}) — ${previewBlock(block)}`); + return null; + } + logger.warn(`Fallback for block ${block.block_id}: no parser for block_type=${bt} (parent=${block.parent_id ?? '-'}) — rendered as paragraph; ${previewBlock(block)}`); + const node: MdBlockNode = { + type: 'paragraph', + children: text ? [{ type: 'text', content: text }] : [], + }; + if (hasChildren) { + const nested = ctx.parseChildren(block); + if (nested.length > 0) node.blocks = nested; + } + return node; + } + const node = parser.parse(block, ctx); + if (!node) { + logger.warn(`Dropped block ${block.block_id}: parser(block_type=${bt}) returned null — ${previewBlock(block)}`); + return null; } - // GridColumn(25) is handled inside Grid parser - return null; + // 框架统一处理:叶子型块(未自行消费 children)若携带嵌套子块,挂到 node.blocks, + // 序列化时在节点自身内容之后渲染(飞书大纲/折叠等场景下,任意块都可能嵌套内容)。 + if (!parser.consumesChildren && (block.children?.length ?? 0) > 0) { + const nested = ctx.parseChildren(block); + if (nested.length > 0) node.blocks = nested; + } + return node; }, parseText: (body: TextBody): MdInlineNode[] => { const inline = body.elements.length > 1; @@ -56,10 +95,12 @@ export class Parser { const children: MdBlockNode[] = []; for (const id of block.children ?? []) { const child = blockMap.get(id); - if (child) { - const node = ctx.parseBlock(child); - if (node) children.push(node); + if (!child) { + logger.warn(`Missing child block ${id} (parent=${block.block_id}, parent_type=${block.block_type}); referenced but absent from blockMap — expected under --filter-title, otherwise signals a pagination gap or API omission`); + continue; } + const node = ctx.parseBlock(child); + if (node) children.push(node); } return children; }, @@ -129,3 +170,37 @@ function parseTextRun (tr: NonNullable): MdInlineNode { return node; } + +// ─── Diagnostics ───────────────────────────────────────────────────────────── + +/** 提取块的文本预览,便于在日志中定位被丢弃的内容 */ +function previewBlock (block: DocxBlock): string { + const text = extractBlockText(block); + if (text) return `text="${text.slice(0, 80)}"`; + return `raw=${JSON.stringify(block).slice(0, 120)}`; +} + +const BLOCK_TEXT_FIELDS = ['text', 'heading1', 'heading2', 'heading3', 'heading4', 'heading5', 'heading6', 'heading7', 'heading8', 'heading9', 'bullet', 'ordered', 'code', 'quote', 'equation', 'todo', 'callout']; + +function extractBlockText (block: DocxBlock): string { + for (const f of BLOCK_TEXT_FIELDS) { + const t = extractBodyText((block as Record)[f]); + if (t) return t; + } + return ''; +} + +function extractBodyText (body: unknown): string { + if (!body || typeof body !== 'object') return ''; + const elements = (body as { elements?: unknown[] }).elements; + if (!Array.isArray(elements)) return ''; + return elements + .map((el) => { + if (el && typeof el === 'object' && 'text_run' in el) { + const c = (el as { text_run?: { content?: string } }).text_run?.content; + if (c) return c; + } + return ''; + }) + .join(''); +} diff --git a/src/md-ast/parsers/bullet.ts b/src/md-ast/parsers/bullet.ts index caacaae..2d9fa03 100644 --- a/src/md-ast/parsers/bullet.ts +++ b/src/md-ast/parsers/bullet.ts @@ -4,6 +4,7 @@ import type { MdBlockNode } from '../types.js'; export const bulletParser: BlockParser = { blockType: 12, + consumesChildren: true, parse (block: DocxBlock, ctx: ParserContext): MdBlockNode { return { type: 'bullet', diff --git a/src/md-ast/parsers/callout.ts b/src/md-ast/parsers/callout.ts index 4a5e3de..12a4b5c 100644 --- a/src/md-ast/parsers/callout.ts +++ b/src/md-ast/parsers/callout.ts @@ -4,6 +4,7 @@ import type { MdBlockNode } from '../types.js'; export const calloutParser: BlockParser = { blockType: 19, + consumesChildren: true, parse (block: DocxBlock, ctx: ParserContext): MdBlockNode { return { type: 'callout', diff --git a/src/md-ast/parsers/grid.ts b/src/md-ast/parsers/grid.ts index d3ac172..388d8c6 100644 --- a/src/md-ast/parsers/grid.ts +++ b/src/md-ast/parsers/grid.ts @@ -4,6 +4,7 @@ import type { MdBlockNode } from '../types.js'; export const gridParser: BlockParser = { blockType: 24, + consumesChildren: true, parse (block: DocxBlock, ctx: ParserContext): MdBlockNode { const children: MdBlockNode[] = []; for (const colId of block.children ?? []) { diff --git a/src/md-ast/parsers/ordered.ts b/src/md-ast/parsers/ordered.ts index bf8f58b..a98597a 100644 --- a/src/md-ast/parsers/ordered.ts +++ b/src/md-ast/parsers/ordered.ts @@ -4,6 +4,7 @@ import type { MdBlockNode } from '../types.js'; export const orderedParser: BlockParser = { blockType: 13, + consumesChildren: true, parse (block: DocxBlock, ctx: ParserContext): MdBlockNode { const parent = block.parent_id ? ctx.blockMap.get(block.parent_id) : undefined; let order = 1; diff --git a/src/md-ast/parsers/page.ts b/src/md-ast/parsers/page.ts index f6a2ce2..f469957 100644 --- a/src/md-ast/parsers/page.ts +++ b/src/md-ast/parsers/page.ts @@ -4,6 +4,7 @@ import type { MdBlockNode } from '../types.js'; export const pageParser: BlockParser = { blockType: 1, + consumesChildren: true, parse (block: DocxBlock, ctx: ParserContext): MdBlockNode { return { type: 'page', diff --git a/src/md-ast/parsers/quoteContainer.ts b/src/md-ast/parsers/quoteContainer.ts index 5c620ef..c87cdcc 100644 --- a/src/md-ast/parsers/quoteContainer.ts +++ b/src/md-ast/parsers/quoteContainer.ts @@ -4,6 +4,7 @@ import type { MdBlockNode } from '../types.js'; export const quoteContainerParser: BlockParser = { blockType: 34, + consumesChildren: true, parse (block: DocxBlock, ctx: ParserContext): MdBlockNode { return { type: 'quote', diff --git a/src/md-ast/parsers/table.ts b/src/md-ast/parsers/table.ts index b913c68..6abb538 100644 --- a/src/md-ast/parsers/table.ts +++ b/src/md-ast/parsers/table.ts @@ -4,6 +4,7 @@ import type { MdBlockNode, MdInlineNode, MdTableRow } from '../types.js'; export const tableParser: BlockParser = { blockType: 31, + consumesChildren: true, parse (block: DocxBlock, ctx: ParserContext): MdBlockNode { const t = block.table!; const cols = t.property.column_size; diff --git a/src/md-ast/serializer.ts b/src/md-ast/serializer.ts index 6e49a2b..691843d 100644 --- a/src/md-ast/serializer.ts +++ b/src/md-ast/serializer.ts @@ -33,10 +33,13 @@ export class MdSerializer { sourceType: options.sourceType ?? 'docx', serialize: (node: MdBlockNode, indent = 0): string => { const serializer = this.registry.get(node.type); - if (serializer) { - return serializer.serialize(node, ctx); + if (!serializer) return ''; + let s = serializer.serialize(node, ctx); + // 通用:渲染节点携带的嵌套子块(飞书大纲/折叠等场景下叶子块的内容),位于节点自身内容之后 + if (node.blocks?.length) { + for (const child of node.blocks) s += ctx.serialize(child, indent); } - return ''; + return s; }, serializeInline: (nodes: MdInlineNode[]): string => { return nodes.map(n => serializeInlineNode(n)).join(''); diff --git a/src/md-ast/transformer.ts b/src/md-ast/transformer.ts index 6cc1c7e..9e64b4a 100644 --- a/src/md-ast/transformer.ts +++ b/src/md-ast/transformer.ts @@ -347,53 +347,56 @@ function replaceInAst ( return; } - if (!hasBlockChildren(node)) return; - - const children = node.children as MdBlockNode[]; - for (let i = 0; i < children.length; i++) { - const child = children[i]!; - if (child.type === 'whiteboard') { - const replacement = whiteboardMap.get(child.token); - if (replacement) { - children[i] = replacement; - continue; + for (const arr of blockChildArrays(node)) { + for (let i = 0; i < arr.length; i++) { + const child = arr[i]!; + if (child.type === 'whiteboard') { + const replacement = whiteboardMap.get(child.token); + if (replacement) { + arr[i] = replacement; + continue; + } } - } - if (child.type === 'sheet') { - const replacement = sheetMap.get(child.token); - if (replacement) { - children[i] = replacement; - continue; + if (child.type === 'sheet') { + const replacement = sheetMap.get(child.token); + if (replacement) { + arr[i] = replacement; + continue; + } } - } - if (child.type === 'image') { - const newSrc = imageMap.get(child.src); - if (newSrc) { - (child as MdBlockNode & { src: string }).src = newSrc; + if (child.type === 'image') { + const newSrc = imageMap.get(child.src); + if (newSrc) { + (child as MdBlockNode & { src: string }).src = newSrc; + } } + replaceInAst(child, imageMap, whiteboardMap, sheetMap); } - replaceInAst(child, imageMap, whiteboardMap, sheetMap); } } function traverseBlockAst (node: MdBlockNode, visitor: (node: MdBlockNode) => void): void { visitor(node); - if (!hasBlockChildren(node)) return; - for (const child of (node.children as MdBlockNode[])) { - traverseBlockAst(child, visitor); + for (const arr of blockChildArrays(node)) { + for (const child of arr) { + traverseBlockAst(child, visitor); + } } } -function hasBlockChildren (node: MdBlockNode): node is Extract< - MdBlockNode, - { type: 'page' | 'bullet' | 'ordered' | 'callout' | 'quote' | 'grid' } -> { - return ( - node.type === 'page' || - node.type === 'bullet' || - node.type === 'ordered' || - node.type === 'callout' || - node.type === 'quote' || - node.type === 'grid' - ); +/** + * 返回节点持有的块级子节点数组(可变引用),供遍历/替换使用。 + * 容器型(page/bullet/ordered/callout/quote/grid)子块在 children; + * 任意节点都可能携带嵌套 blocks(飞书大纲/折叠等场景下叶子块的内容)。 + */ +function blockChildArrays (node: MdBlockNode): MdBlockNode[][] { + const arrs: MdBlockNode[][] = []; + if ( + node.type === 'page' || node.type === 'bullet' || node.type === 'ordered' || + node.type === 'callout' || node.type === 'quote' || node.type === 'grid' + ) { + arrs.push(node.children as MdBlockNode[]); + } + if (node.blocks) arrs.push(node.blocks); + return arrs; } diff --git a/src/md-ast/types.ts b/src/md-ast/types.ts index f1a1b84..e8ec0f5 100644 --- a/src/md-ast/types.ts +++ b/src/md-ast/types.ts @@ -2,24 +2,28 @@ import type { ResolvedSheet } from '../sheet/index.js'; // ─── 块级节点 ─────────────────────────────────────────────────────────────── +/** 任意块节点都可能携带「未被自身 parser 消费的嵌套子块」,对应飞书大纲/折叠等场景下 + * 叶子块所嵌套的内容;序列化时在节点自身内容之后渲染。 */ +interface NestedBlocks { blocks?: MdBlockNode[] } + export type MdBlockNode = - | { type: 'page'; title: MdInlineNode[]; children: MdBlockNode[] } - | { type: 'heading'; level: number; children: MdInlineNode[] } - | { type: 'paragraph'; children: MdInlineNode[] } - | { type: 'bullet'; text: MdInlineNode[]; children: MdBlockNode[] } - | { type: 'ordered'; order: number; text: MdInlineNode[]; children: MdBlockNode[] } - | { type: 'codeBlock'; lang: string; content: string } - | { type: 'todo'; checked: boolean; text: MdInlineNode[] } - | { type: 'callout'; children: MdBlockNode[] } - | { type: 'quote'; children: MdBlockNode[] } - | { type: 'divider' } - | { type: 'image'; alt: string; src: string } - | { type: 'whiteboard'; token: string } - | { type: 'sheet'; token: string } - | { type: 'sheetResolved'; title: string; sheets: ResolvedSheet[] } - | { type: 'table'; rows: MdTableRow[] } - | { type: 'grid'; children: MdBlockNode[] } - | { type: 'html'; content: string }; + | ({ type: 'page'; title: MdInlineNode[]; children: MdBlockNode[] } & NestedBlocks) + | ({ type: 'heading'; level: number; children: MdInlineNode[] } & NestedBlocks) + | ({ type: 'paragraph'; children: MdInlineNode[] } & NestedBlocks) + | ({ type: 'bullet'; text: MdInlineNode[]; children: MdBlockNode[] } & NestedBlocks) + | ({ type: 'ordered'; order: number; text: MdInlineNode[]; children: MdBlockNode[] } & NestedBlocks) + | ({ type: 'codeBlock'; lang: string; content: string } & NestedBlocks) + | ({ type: 'todo'; checked: boolean; text: MdInlineNode[] } & NestedBlocks) + | ({ type: 'callout'; children: MdBlockNode[] } & NestedBlocks) + | ({ type: 'quote'; children: MdBlockNode[] } & NestedBlocks) + | ({ type: 'divider' } & NestedBlocks) + | ({ type: 'image'; alt: string; src: string } & NestedBlocks) + | ({ type: 'whiteboard'; token: string } & NestedBlocks) + | ({ type: 'sheet'; token: string } & NestedBlocks) + | ({ type: 'sheetResolved'; title: string; sheets: ResolvedSheet[] } & NestedBlocks) + | ({ type: 'table'; rows: MdTableRow[] } & NestedBlocks) + | ({ type: 'grid'; children: MdBlockNode[] } & NestedBlocks) + | ({ type: 'html'; content: string } & NestedBlocks); export type MdTableRow = { cells: MdTableCell[] }; export type MdTableCell = { content: MdInlineNode[]; rowSpan?: number; colSpan?: number }; diff --git a/tests/md-ast/parser.test.ts b/tests/md-ast/parser.test.ts index 2bfe9c4..4a0789f 100644 --- a/tests/md-ast/parser.test.ts +++ b/tests/md-ast/parser.test.ts @@ -62,6 +62,48 @@ describe('Parser - end-to-end', () => { } }); + it('parses a heading carrying nested children (Feishu outline/folded heading)', () => { + const blocks: DocxBlock[] = [ + b({ block_id: 'root', block_type: 1, children: ['h'] }), + b({ block_id: 'h', block_type: 5, parent_id: 'root', heading3: { elements: [textRun('消息类型 msgType')] }, children: ['p1', 'h2'] }), + b({ block_id: 'p1', block_type: 2, parent_id: 'h', text: { elements: [textRun('intro text')] } }), + b({ block_id: 'h2', block_type: 6, parent_id: 'h', heading4: { elements: [textRun('子标题')] } }), + ]; + const ast = buildParser().parse(doc, blocks); + if (ast.type !== 'page') throw new Error(); + const h = ast.children[0]!; + expect(h).toMatchObject({ type: 'heading', level: 3 }); + if (h.type !== 'heading') throw new Error(); + expect(h.blocks).toHaveLength(2); + expect(h.blocks![0]).toMatchObject({ type: 'paragraph' }); + expect(h.blocks![1]).toMatchObject({ type: 'heading', level: 4 }); + + const ser = new MdSerializer(); + registerBuiltinSerializers(ser); + expect(ser.serialize(ast)).toBe('### 消息类型 msgType\n\nintro text\n\n#### 子标题\n\n'); + }); + + it('parses a leaf block (todo) carrying nested children (generalized outline)', () => { + const blocks: DocxBlock[] = [ + b({ block_id: 'root', block_type: 1, children: ['t'] }), + b({ block_id: 't', block_type: 17, parent_id: 'root', todo: { style: { done: false }, elements: [textRun('消息类型 msgType')] }, children: ['p1', 't2'] }), + b({ block_id: 'p1', block_type: 2, parent_id: 't', text: { elements: [textRun('intro')] } }), + b({ block_id: 't2', block_type: 17, parent_id: 't', todo: { style: { done: true }, elements: [textRun('sub todo')] } }), + ]; + const ast = buildParser().parse(doc, blocks); + if (ast.type !== 'page') throw new Error(); + const t = ast.children[0]!; + expect(t).toMatchObject({ type: 'todo', checked: false }); + if (t.type !== 'todo') throw new Error(); + expect(t.blocks).toHaveLength(2); + expect(t.blocks![0]).toMatchObject({ type: 'paragraph' }); + expect(t.blocks![1]).toMatchObject({ type: 'todo', checked: true }); + + const ser = new MdSerializer(); + registerBuiltinSerializers(ser); + expect(ser.serialize(ast)).toBe('- [ ] 消息类型 msgType\nintro\n\n- [x] sub todo\n'); + }); + it('parses inline styles (bold/italic/underline/strikethrough/inlineCode/link)', () => { const blocks: DocxBlock[] = [ b({ block_id: 'root', block_type: 1, children: ['p'] }), @@ -438,6 +480,27 @@ describe('Parser - end-to-end', () => { expect(ast.children).toHaveLength(0); }); + it('falls back an unknown block_type with text/children to a paragraph (preserves subtree)', () => { + const blocks: DocxBlock[] = [ + b({ block_id: 'root', block_type: 1, children: ['u'] }), + b({ block_id: 'u', block_type: 999, parent_id: 'root', text: { elements: [textRun('未支持块正文')] }, children: ['p1'] } as any), + b({ block_id: 'p1', block_type: 2, parent_id: 'u', text: { elements: [textRun('嵌套子块')] } }), + ]; + const ast = buildParser().parse(doc, blocks); + if (ast.type !== 'page') throw new Error(); + expect(ast.children).toHaveLength(1); + const u = ast.children[0]!; + expect(u).toMatchObject({ type: 'paragraph' }); + if (u.type !== 'paragraph') throw new Error(); + expect(u.children[0]).toMatchObject({ type: 'text', content: '未支持块正文' }); + expect(u.blocks).toHaveLength(1); + expect(u.blocks![0]).toMatchObject({ type: 'paragraph' }); + + const ser = new MdSerializer(); + registerBuiltinSerializers(ser); + expect(ser.serialize(ast)).toBe('未支持块正文\n\n嵌套子块\n\n'); + }); + // ─── Parser + Serializer integration ────────────────────────────────────── it('end-to-end: parse → serialize produces expected markdown', () => { const blocks: DocxBlock[] = [ diff --git a/tests/md-ast/serializer.test.ts b/tests/md-ast/serializer.test.ts index 61bf025..1216385 100644 --- a/tests/md-ast/serializer.test.ts +++ b/tests/md-ast/serializer.test.ts @@ -29,6 +29,19 @@ describe('MdSerializer', () => { const node: MdBlockNode = { type: 'heading', level: 2, children: [bold('Important')] }; expect(s.serialize(node)).toBe('## **Important**\n\n'); }); + + it('renders nested block children after the heading line', () => { + const node: MdBlockNode = { + type: 'heading', + level: 2, + children: [text('Section')], + blocks: [ + { type: 'paragraph', children: [text('Body')] }, + { type: 'heading', level: 3, children: [text('Sub')] }, + ], + }; + expect(s.serialize(node)).toBe('## Section\n\nBody\n\n### Sub\n\n'); + }); }); describe('paragraph', () => { @@ -85,6 +98,19 @@ describe('MdSerializer', () => { const node: MdBlockNode = { type: 'todo', checked: true, text: [text('done')] }; expect(s.serialize(node)).toBe('- [x] done\n'); }); + + it('renders nested block children after the todo line', () => { + const node: MdBlockNode = { + type: 'todo', + checked: false, + text: [text('Task')], + blocks: [ + { type: 'paragraph', children: [text('Detail')] }, + { type: 'todo', checked: true, text: [text('Sub')] }, + ], + }; + expect(s.serialize(node)).toBe('- [ ] Task\nDetail\n\n- [x] Sub\n'); + }); }); describe('divider', () => {