From bddef7a04b09b201378a02a6cbd6cc5906765d8d Mon Sep 17 00:00:00 2001 From: ocavue Date: Mon, 22 Jun 2026 00:03:17 +1000 Subject: [PATCH 1/2] fix(check-roundtrip): classify trailing-whitespace-only diffs as normalizing --- packages/core/src/converters/check-roundtrip.test.ts | 2 ++ packages/core/src/converters/check-roundtrip.ts | 11 +++++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/packages/core/src/converters/check-roundtrip.test.ts b/packages/core/src/converters/check-roundtrip.test.ts index 72bcbd75..381771fc 100644 --- a/packages/core/src/converters/check-roundtrip.test.ts +++ b/packages/core/src/converters/check-roundtrip.test.ts @@ -41,6 +41,8 @@ describe('checkRoundTrip', () => { it.each([ 'a\n\n\nb', // extra blank lines collapse to one '- a\n\n- b', // a loose list serializes tight + '- [ ] Asdf\n- [ ]\n- [ ] ', // a trailing space on an empty task is normalized away + 'trailing spaces ', // trailing whitespace is insignificant ])('reports normalizing for %j', (markdown) => { expect(checkRoundTrip(markdown)).toBe('normalizing') }) diff --git a/packages/core/src/converters/check-roundtrip.ts b/packages/core/src/converters/check-roundtrip.ts index 789dc6a9..6d33ee8c 100644 --- a/packages/core/src/converters/check-roundtrip.ts +++ b/packages/core/src/converters/check-roundtrip.ts @@ -4,7 +4,8 @@ import { docToMarkdown } from './pm-to-md.ts' /** * How faithfully markdown survives a parse-then-serialize round trip: * - `exact`: byte-identical (modulo the trailing newline). - * - `normalizing`: same non-blank lines, only blank-line layout differs. + * - `normalizing`: same non-blank lines ignoring trailing whitespace; only + * blank-line layout or insignificant trailing whitespace differs. * - `lossy`: a non-blank line changed, so content would be lost or altered. */ export type RoundTripFidelity = 'exact' | 'normalizing' | 'lossy' @@ -23,7 +24,13 @@ export function checkRoundTrip(markdown: string): RoundTripFidelity { if (trimTrailingNewlines(serialized) === trimTrailingNewlines(markdown)) return 'exact' const before = nonBlankLines(markdown) const after = nonBlankLines(serialized) - if (before.length === after.length && before.every((line, index) => line === after[index])) { + // Compare by trimEnd: trailing whitespace is insignificant in Markdown and the + // serializer normalizes it away, so a trailing-space-only difference is + // `normalizing`, not `lossy`. Leading indentation is structural and must match. + if ( + before.length === after.length && + before.every((line, index) => line.trimEnd() === after[index].trimEnd()) + ) { return 'normalizing' } return 'lossy' From c12d5f3b8fab90e78fea8917c0c17e4d93c9477c Mon Sep 17 00:00:00 2001 From: ocavue Date: Mon, 22 Jun 2026 00:03:34 +1000 Subject: [PATCH 2/2] fix(converter): keep list soft breaks via a hardBreak node --- .../src/converters/check-roundtrip.test.ts | 2 + packages/core/src/converters/md-to-pm.test.ts | 42 ++++++++++++++- packages/core/src/converters/md-to-pm.ts | 54 +++++++++++++++++-- packages/core/src/converters/pm-to-md.ts | 18 ++++--- .../core/src/converters/roundtrip.test.ts | 45 ++++++++++++++++ packages/core/src/extensions/extension.ts | 2 + packages/core/src/extensions/node-names.ts | 1 + 7 files changed, 154 insertions(+), 10 deletions(-) diff --git a/packages/core/src/converters/check-roundtrip.test.ts b/packages/core/src/converters/check-roundtrip.test.ts index 381771fc..13e9bdb9 100644 --- a/packages/core/src/converters/check-roundtrip.test.ts +++ b/packages/core/src/converters/check-roundtrip.test.ts @@ -34,6 +34,8 @@ describe('checkRoundTrip', () => { Hello ===== `, // setext heading keeps its text and underline length + '- x\n\n line one\n line two', // a list item's soft-wrapped paragraph keeps its indent + '- a\n - x\n\n line one\n line two', // nested list, same ])('reports exact for %j', (markdown) => { expect(checkRoundTrip(markdown)).toBe('exact') }) diff --git a/packages/core/src/converters/md-to-pm.test.ts b/packages/core/src/converters/md-to-pm.test.ts index 5343f648..e216e4ca 100644 --- a/packages/core/src/converters/md-to-pm.test.ts +++ b/packages/core/src/converters/md-to-pm.test.ts @@ -591,7 +591,47 @@ describe('markdownToDoc', () => { content: [ { type: 'blockquote', - content: [{ type: 'paragraph', content: [{ type: 'text', text: 'l1\nl2' }] }], + content: [ + { + type: 'paragraph', + content: [ + { type: 'text', text: 'l1' }, + { type: 'hardBreak' }, + { type: 'text', text: 'l2' }, + ], + }, + ], + }, + ], + }) + }) + + it('keeps a list item soft break as a dedented hardBreak', () => { + expect(markdownToDoc('- x\n\n line one\n line two').toJSON()).toEqual({ + type: 'doc', + attrs: { frontmatter: null }, + content: [ + { + type: 'list', + attrs: { + kind: 'bullet', + order: null, + checked: false, + collapsed: false, + marker: '-', + taskMarker: null, + }, + content: [ + { type: 'paragraph', content: [{ type: 'text', text: 'x' }] }, + { + type: 'paragraph', + content: [ + { type: 'text', text: 'line one' }, + { type: 'hardBreak' }, + { type: 'text', text: 'line two' }, + ], + }, + ], }, ], }) diff --git a/packages/core/src/converters/md-to-pm.ts b/packages/core/src/converters/md-to-pm.ts index d4ad543e..99a1503e 100644 --- a/packages/core/src/converters/md-to-pm.ts +++ b/packages/core/src/converters/md-to-pm.ts @@ -13,6 +13,8 @@ import { CHAR_LOWERCASE_X, CHAR_PLUS, CHAR_RIGHT_PARENTHESIS, + CHAR_SPACE, + CHAR_TAB, CHAR_UPPERCASE_X, isSpaceChar, } from '../unicode.ts' @@ -204,6 +206,51 @@ function countUnderlineChars(text: string, from: number, to: number): number { return count } +/** The number of leading characters before `from` on its own line (the container's content column). */ +function lineIndentWidth(text: string, from: number): number { + return from - (text.lastIndexOf('\n', from - 1) + 1) +} + +/** Drop up to `width` leading space/tab characters from a continuation line. */ +function stripIndent(line: string, width: number): string { + let index = 0 + while (index < width && index < line.length) { + const code = line.charCodeAt(index) + if (code !== CHAR_SPACE && code !== CHAR_TAB) break + index++ + } + return index === 0 ? line : line.slice(index) +} + +/** + * Build a paragraph from raw markdown content. A soft line break becomes a + * `hardBreak` node (rendered as `
`), never a bare `\n`: ProseMirror cannot + * hold a literal newline in a textblock, because a DOM re-read folds it to a + * space and silently drops the break. Continuation lines are dedented by + * `indentWidth` (the container's content column) so the serializer's own line + * prefix does not double the indent. + */ +function buildParagraph( + nodes: TypedNodeBuilders, + content: string, + indentWidth: number, +): ProseMirrorNode { + if (content === '') return nodes.paragraph() + if (!content.includes('\n')) return nodes.paragraph(content) + const lines = content.split('\n') + const children: Array = [] + for (let index = 0; index < lines.length; index++) { + if (index === 0) { + if (lines[index] !== '') children.push(lines[index]) + continue + } + children.push(nodes.hardBreak()) + const line = stripIndent(lines[index], indentWidth) + if (line !== '') children.push(line) + } + return nodes.paragraph(...children) +} + function convertParagraph( nodes: TypedNodeBuilders, cursor: TreeCursor, @@ -211,6 +258,7 @@ function convertParagraph( ): ProseMirrorNode { const from = cursor.from const to = cursor.to + const indentWidth = lineIndentWidth(text, from) // In block-only parsing a paragraph has no inline children, with one // exception: lezer leaves the lazy-continuation `QuoteMark`s of a multi-line // blockquote (`> l1\n> l2`) embedded in the paragraph's span. @@ -226,9 +274,9 @@ function convertParagraph( } while (cursor.nextSibling()) cursor.parent() content += text.slice(pos, to) - return content === '' ? nodes.paragraph() : nodes.paragraph(content) + return buildParagraph(nodes, content, indentWidth) } - return nodes.paragraph(text.slice(from, to)) + return buildParagraph(nodes, text.slice(from, to), indentWidth) } function convertBlockquote( @@ -320,7 +368,7 @@ function convertListItem( // Skip the single separating whitespace after `[ ]` / `[x]` if (isSpaceChar(text.charCodeAt(taskStart))) taskStart += 1 const taskText = text.slice(taskStart, taskEnd) - content.push(taskText === '' ? nodes.paragraph() : nodes.paragraph(taskText)) + content.push(buildParagraph(nodes, taskText, lineIndentWidth(text, taskStart))) continue } content.push(...convertBlock(nodes, cursor, text)) diff --git a/packages/core/src/converters/pm-to-md.ts b/packages/core/src/converters/pm-to-md.ts index 20dbc1f4..1678b722 100644 --- a/packages/core/src/converters/pm-to-md.ts +++ b/packages/core/src/converters/pm-to-md.ts @@ -289,17 +289,23 @@ function isTightItem(item: ProseMirrorNode): boolean { } /** - * Walk inline children writing text directly. The schema has no marks, so - * every inline child is currently a text node - but going through this - * loop instead of `node.textContent` avoids one intermediate string - * allocation per leaf block (paragraph / heading content). + * Walk inline children writing text directly. Inline content is a text node or a + * `hardBreak` (a soft line break); going through this loop instead of + * `node.textContent` avoids one intermediate string allocation per leaf block + * (paragraph / heading content). A `hardBreak` emits a `\n`; `MdOut.write` then + * re-applies the current line prefix, so a break inside a list item stays + * aligned to the item's continuation indent. */ function emitInlineChildren(node: ProseMirrorNode, out: MdOut): void { const count = node.childCount for (let i = 0; i < count; i++) { const child = node.child(i) - if (child.isText && child.text) out.write(child.text) - // Future inline node types (hardBreak, image, mention) go here. + if (child.isText && child.text) { + out.write(child.text) + } else if (child.type.name === ('hardBreak' satisfies NodeName)) { + out.write('\n') + } + // Future inline node types (image, mention) go here. } } diff --git a/packages/core/src/converters/roundtrip.test.ts b/packages/core/src/converters/roundtrip.test.ts index 86fc9664..38602eb4 100644 --- a/packages/core/src/converters/roundtrip.test.ts +++ b/packages/core/src/converters/roundtrip.test.ts @@ -650,3 +650,48 @@ describe('idempotency', () => { expect(roundtrip(once)).toBe(once) }) }) + +describe('soft line breaks in lists', () => { + it('keeps a tight list item soft break', () => { + expect(roundtrip('- line one\n line two')).toBe('- line one\n line two\n') + }) + + it('keeps a loose list item second paragraph', () => { + expect(roundtrip('- x\n\n line one\n line two')).toBe('- x\n\n line one\n line two\n') + }) + + it('keeps a nested list item soft break', () => { + expect(roundtrip('- a\n - x\n\n line one\n line two')).toBe( + '- a\n - x\n\n line one\n line two\n', + ) + }) + + it('keeps a multi-line task', () => { + expect(roundtrip('- [ ] line one\n line two')).toBe('- [ ] line one\n line two\n') + }) + + it('stays stable on a second pass', () => { + const once = roundtrip('- x\n\n line one\n line two') + expect(roundtrip(once)).toBe(once) + }) +}) + +describe('reflect daily-note regressions', () => { + // Both snippets come from real reflect-open daily notes that opened read-only + // because the round-trip guard (checkRoundTrip) classified them `lossy`. + + it('round-trips a nested multi-line paragraph (2026-04-06)', () => { + // A soft-wrapped paragraph inside a nested list item, now kept via hardBreak + // with the continuation line dedented so the indent does not double. + expect( + roundtrip('- [[Audio memos]]\n - first paragraph\n\n He is nice.\n LTV $1200.'), + ).toBe('- [[Audio memos]]\n - first paragraph\n\n He is nice.\n LTV $1200.\n') + }) + + it.fails('keeps a trailing space on an empty task (2026-06-15)', () => { + // The serializer still normalizes the trailing space away; checkRoundTrip now + // classifies this `normalizing`, not `lossy` (see check-roundtrip.test.ts), so + // the note opens editable instead of read-only. + expect(roundtrip('- [ ] Asdf\n- [ ]\n- [ ] ')).toBe('- [ ] Asdf\n- [ ]\n- [ ] \n') + }) +}) diff --git a/packages/core/src/extensions/extension.ts b/packages/core/src/extensions/extension.ts index 9556ab93..07d47b82 100644 --- a/packages/core/src/extensions/extension.ts +++ b/packages/core/src/extensions/extension.ts @@ -9,6 +9,7 @@ import { defineBlockquote } from '@prosekit/extensions/blockquote' import { defineCodeBlock } from '@prosekit/extensions/code-block' import { defineDoc } from '@prosekit/extensions/doc' import { defineGapCursor } from '@prosekit/extensions/gap-cursor' +import { defineHardBreak } from '@prosekit/extensions/hard-break' import { defineModClickPrevention } from '@prosekit/extensions/mod-click-prevention' import { defineParagraph } from '@prosekit/extensions/paragraph' import { defineText } from '@prosekit/extensions/text' @@ -33,6 +34,7 @@ function defineEditorExtensionImpl() { defineDoc(), defineDocFrontmatterAttr(), defineText(), + defineHardBreak(), defineBlockquote(), defineMeowdownList(), defineHeading(), diff --git a/packages/core/src/extensions/node-names.ts b/packages/core/src/extensions/node-names.ts index 66b0111b..24999a2c 100644 --- a/packages/core/src/extensions/node-names.ts +++ b/packages/core/src/extensions/node-names.ts @@ -4,6 +4,7 @@ export const NODE_NAMES = [ 'doc', 'text', + 'hardBreak', 'paragraph', 'heading', 'blockquote',