fix(editor): preserve soft line breaks

packages/core/src/converters/check-roundtrip.test.ts

@@ -30,23 +30,45 @@ describe('checkRoundTrip', () => {
       | --- | --- | --- |
       |  | b |  |
     `,
+
+    // setext heading keeps its text and underline length
     dedent`
       Hello
       =====
-    `, // setext heading keeps its text and underline length
+    `,
+
+    // a list item's soft-wrapped paragraph keeps its indent
+    dedent`
+      - x
+
+        line one
+        line two
+    `,
+    // nested list, same
+    dedent`
+      - a
+        - x
+
+          line one
+          line two
+    `,
   ])('reports exact for %j', (markdown) => {
     expect(checkRoundTrip(markdown)).toBe('exact')
   })
 
   it.each([
     'a\n\n\nb', // extra blank lines collapse to one
     '- a\n\n- b', // a loose list serializes tight
+    '- [ ] Asdf\n- [ ]\n- [ ] ', // a trailing space on an empty task is normalized away
+    'trailing spaces   ', // trailing whitespace is insignificant
   ])('reports normalizing for %j', (markdown) => {
     expect(checkRoundTrip(markdown)).toBe('normalizing')
   })
 
   it.each([
-    '# Hello #', // a closing ATX hash sequence is dropped
+    '# Hello #', // a closing ATX hash sequence is dropped (same line count, content differs)
+    '    indented', // an indented code block becomes a fence: the non-blank line count grows
+    '~~~\ntilde\n~~~', // a tilde fence becomes a backtick fence: same line count, content differs
   ])('reports lossy for %j', (markdown) => {
     expect(checkRoundTrip(markdown)).toBe('lossy')
   })

packages/core/src/converters/check-roundtrip.ts

@@ -4,7 +4,8 @@ import { docToMarkdown } from './pm-to-md.ts'
 /**
  * How faithfully markdown survives a parse-then-serialize round trip:
  * - `exact`: byte-identical (modulo the trailing newline).
- * - `normalizing`: same non-blank lines, only blank-line layout differs.
+ * - `normalizing`: same non-blank lines ignoring trailing whitespace; only
+ *   blank-line layout or insignificant trailing whitespace differs.
  * - `lossy`: a non-blank line changed, so content would be lost or altered.
  */
 export type RoundTripFidelity = 'exact' | 'normalizing' | 'lossy'
@@ -33,7 +34,13 @@ export function checkRoundTrip(
   if (trimTrailingNewlines(serialized) === trimTrailingNewlines(markdown)) return 'exact'
   const before = nonBlankLines(markdown)
   const after = nonBlankLines(serialized)
-  if (before.length === after.length && before.every((line, index) => line === after[index])) {
+  // Compare by trimEnd: trailing whitespace is insignificant in Markdown and the
+  // serializer normalizes it away, so a trailing-space-only difference is
+  // `normalizing`, not `lossy`. Leading indentation is structural and must match.
+  if (
+    before.length === after.length &&
+    before.every((line, index) => line.trimEnd() === after[index].trimEnd())
+  ) {
     return 'normalizing'
   }
   return 'lossy'

packages/core/src/converters/md-to-pm.test.ts

@@ -1,7 +1,7 @@
 import dedent from 'dedent'
 import { describe, expect, it } from 'vitest'
 
-import { markdownToDoc } from './md-to-pm.ts'
+import { dedentContinuation, markdownToDoc, measureContentColumn, sliceColumn } from './md-to-pm.ts'
 import { sampleContent, sampleContentMarkdown } from './sample-content.ts'
 
 function tableShape(markdown: string): Array<Array<{ type: string; text: string }>> {
@@ -597,6 +597,30 @@ describe('markdownToDoc', () => {
     })
   })
 
+  it('keeps a list item soft break as a dedented single text node', () => {
+    expect(markdownToDoc('- x\n\n  line one\n  line two').toJSON()).toEqual({
+      type: 'doc',
+      attrs: { frontmatter: null },
+      content: [
+        {
+          type: 'list',
+          attrs: {
+            kind: 'bullet',
+            order: null,
+            checked: false,
+            collapsed: false,
+            marker: '-',
+            taskMarker: null,
+          },
+          content: [
+            { type: 'paragraph', content: [{ type: 'text', text: 'x' }] },
+            { type: 'paragraph', content: [{ type: 'text', text: 'line one\nline two' }] },
+          ],
+        },
+      ],
+    })
+  })
+
   it('keeps YAML frontmatter as a doc attribute', () => {
     // The whole input is the frontmatter block, so the only content is the
     // empty paragraph the schema fills in (it serializes back to nothing).
@@ -642,3 +666,88 @@ describe('markdownToDoc', () => {
     expect(markdownToDoc('---\ntitle: x\n---').attrs.frontmatter).toBe(null)
   })
 })
+
+describe('measureContentColumn', () => {
+  it('is 0 at the document start', () => {
+    expect(measureContentColumn('hello', 0)).toBe(0)
+  })
+
+  it('is 0 at the start of a line', () => {
+    expect(measureContentColumn('abc\nx', 4)).toBe(0)
+  })
+
+  it('counts the characters before the position', () => {
+    expect(measureContentColumn('- item', 2)).toBe(2)
+  })
+
+  it('measures only the current line', () => {
+    expect(measureContentColumn('abc\n  x', 6)).toBe(2)
+  })
+
+  it('counts a tab from column 0 as 4', () => {
+    expect(measureContentColumn('\tx', 1)).toBe(4)
+  })
+
+  it('counts a tab to the next multiple of 4', () => {
+    expect(measureContentColumn('a\tx', 2)).toBe(4)
+    expect(measureContentColumn('ab\tx', 3)).toBe(4)
+    expect(measureContentColumn('abc\tx', 4)).toBe(4)
+  })
+
+  it('accumulates multiple tabs', () => {
+    expect(measureContentColumn('\t\tx', 2)).toBe(8)
+  })
+})
+
+describe('sliceColumn', () => {
+  it('drops leading spaces up to the column', () => {
+    expect(sliceColumn('  line', 2)).toBe('line')
+  })
+
+  it('stops at the first non-whitespace', () => {
+    expect(sliceColumn('  line', 8)).toBe('line')
+  })
+
+  it('keeps whitespace beyond the column', () => {
+    expect(sliceColumn('    deep', 2)).toBe('  deep')
+  })
+
+  it('advances a tab to the next multiple of 4', () => {
+    expect(sliceColumn('\tx', 4)).toBe('x')
+    expect(sliceColumn(' \tx', 4)).toBe('x')
+    expect(sliceColumn('  \tx', 4)).toBe('x')
+    expect(sliceColumn('   \tx', 4)).toBe('x')
+  })
+
+  it('stops once a tab reaches the column', () => {
+    expect(sliceColumn('\t\tx', 4)).toBe('\tx')
+  })
+
+  it('consumes a whole tab when the column falls mid-tab', () => {
+    expect(sliceColumn('\tx', 2)).toBe('x')
+  })
+
+  it('returns the line unchanged at column 0', () => {
+    expect(sliceColumn('  line', 0)).toBe('  line')
+  })
+})
+
+describe('dedentContinuation', () => {
+  it('returns single-line content unchanged', () => {
+    expect(dedentContinuation('hello', 2)).toBe('hello')
+  })
+
+  it('returns content unchanged at column 0', () => {
+    expect(dedentContinuation('a\n  b', 0)).toBe('a\n  b')
+  })
+
+  it('keeps the first line and dedents the rest', () => {
+    expect(dedentContinuation('one\n    two', 2)).toBe('one\n  two')
+  })
+
+  it('strips the full column from each continuation line', () => {
+    expect(dedentContinuation('line one\n  line two\n  line three', 2)).toBe(
+      'line one\nline two\nline three',
+    )
+  })
+})

packages/core/src/converters/md-to-pm.ts

@@ -13,6 +13,8 @@ import {
   CHAR_LOWERCASE_X,
   CHAR_PLUS,
   CHAR_RIGHT_PARENTHESIS,
+  CHAR_SPACE,
+  CHAR_TAB,
   CHAR_UPPERCASE_X,
   isSpaceChar,
 } from '../unicode.ts'
@@ -200,7 +202,10 @@ function convertHeading(
     cursor.parent()
   }
 
-  const content = text.slice(contentStart, contentEnd).trim()
+  // Dedent before trimming so a multi-line setext heading inside a container
+  // (rare) keeps its continuation lines aligned; trim then drops the outer ends.
+  const raw = text.slice(contentStart, contentEnd)
+  const content = dedentContinuation(raw, measureContentColumn(text, contentStart)).trim()
   // CommonMark setext underlines may be any length; keep the source count so
   // the round-trip is lossless. Fall back to 1 if lezer reported no run.
   const setextUnderline = isSetext
@@ -220,13 +225,77 @@ function countUnderlineChars(text: string, from: number, to: number): number {
   return count
 }
 
+/**
+ * The column at which content begins on the line containing `from` (i.e. the
+ * enclosing container's content column). Columns count a tab as a CommonMark
+ * tab stop of 4 (`4 - col % 4`), matching how lezer measures indentation.
+ */
+export function measureContentColumn(text: string, from: number): number {
+  const lineStart = text.lastIndexOf('\n', from - 1) + 1
+  let col = 0
+  for (let index = lineStart; index < from; index++) {
+    col += text.charCodeAt(index) === CHAR_TAB ? 4 - (col % 4) : 1
+  }
+  return col
+}
+
+/** Drop a line's leading whitespace up to `column`, counting a tab as `4 - col % 4` columns. */
+export function sliceColumn(line: string, column: number): string {
+  let col = 0
+  let index = 0
+  while (index < line.length && col < column) {
+    const code = line.charCodeAt(index)
+    if (code === CHAR_SPACE) col += 1
+    else if (code === CHAR_TAB) col += 4 - (col % 4)
+    else break
+    index++
+  }
+  return line.slice(index)
+}
+
+/**
+ * Strip a leaf block's structural continuation indent.
+ *
+ * lezer keeps the indent of a multi-line block's continuation lines inside the
+ * source span (its `scrub` pads each line's container prefix to equal-width
+ * whitespace to preserve positions). CommonMark and lezer require every
+ * continuation line to be indented to the same content `column`, which equals
+ * the block's first-line column. The first line is already past its indent, so
+ * only lines 2..n are dedented. Returns `content` untouched at column 0 (a
+ * top-level block) or when there is no continuation line.
+ */
+export function dedentContinuation(content: string, column: number): string {
+  if (column === 0 || !content.includes('\n')) return content
+  return content
+    .split('\n')
+    .map((line, index) => (index === 0 ? line : sliceColumn(line, column)))
+    .join('\n')
+}
+
+/**
+ * Build a paragraph from raw markdown content, dedenting continuation lines so
+ * the serializer's own line prefix does not double the indent. A soft line break
+ * stays a literal `\n` in a single text node; the paragraph spec's
+ * `whitespace: 'pre'` keeps a DOM re-read from folding it to a space.
+ */
+function buildParagraph(
+  nodes: TypedNodeBuilders,
+  content: string,
+  column: number,
+): ProseMirrorNode {
+  // An empty string adds no child (the builder skips falsy text), so this also
+  // covers the empty-paragraph case.
+  return nodes.paragraph(dedentContinuation(content, column))
+}
+
 function convertParagraph(
   nodes: TypedNodeBuilders,
   cursor: TreeCursor,
   text: string,
 ): ProseMirrorNode {
   const from = cursor.from
   const to = cursor.to
+  const column = measureContentColumn(text, from)
   // In block-only parsing a paragraph has no inline children, with one
   // exception: lezer leaves the lazy-continuation `QuoteMark`s of a multi-line
   // blockquote (`> l1\n> l2`) embedded in the paragraph's span.
@@ -242,9 +311,9 @@ function convertParagraph(
     } while (cursor.nextSibling())
     cursor.parent()
     content += text.slice(pos, to)
-    return content === '' ? nodes.paragraph() : nodes.paragraph(content)
+    return buildParagraph(nodes, content, column)
   }
-  return nodes.paragraph(text.slice(from, to))
+  return buildParagraph(nodes, text.slice(from, to), column)
 }
 
 function convertBlockquote(
@@ -336,7 +405,7 @@ function convertListItem(
         // Skip the single separating whitespace after `[ ]` / `[x]`
         if (isSpaceChar(text.charCodeAt(taskStart))) taskStart += 1
         const taskText = text.slice(taskStart, taskEnd)
-        content.push(taskText === '' ? nodes.paragraph() : nodes.paragraph(taskText))
+        content.push(buildParagraph(nodes, taskText, measureContentColumn(text, taskStart)))
         continue
       }
       content.push(...convertBlock(nodes, cursor, text))