From c180c4565122b00bf11fefcb51c070854442d88b Mon Sep 17 00:00:00 2001 From: Steve Dempsey Date: Tue, 10 Nov 2026 20:07:09 -0500 Subject: [PATCH] Keep paragraph spacing, headings, lists and tables in IM text A Bot message reached Telegram as one blob with no spacing. Blocks were rejoined with a single newline, so every blank line was thrown away, and headings, list markers and table pipes had no rule at all, so they arrived literally. Blocks now rejoin with a blank line, a heading becomes bold, list items are double spaced, and a Markdown table becomes an aligned pre grid with its separator row dropped. The four exports keep their names and order. The existing fenced-code test asserted the single-newline join, which is the behaviour being fixed, so its expectation moves to the blank line. --- ds-bot/src/host/im/format.js | 134 ++++++++++++++++++++++++++++++++-- ds-bot/test/im-format.test.js | 29 +++++++- 2 files changed, 154 insertions(+), 9 deletions(-) diff --git a/ds-bot/src/host/im/format.js b/ds-bot/src/host/im/format.js index 76e06ff1a2..835a22d2f0 100644 --- a/ds-bot/src/host/im/format.js +++ b/ds-bot/src/host/im/format.js @@ -1,6 +1,14 @@ // Outbound text for IM platforms. Bots write Markdown; each platform adapter turns it // into what the platform renders (Telegram HTML) or plain text, and cuts it at the // platform's per-message limit without breaking inside a code fence when it can help it. +// +// 2026-11-10: paragraph spacing, headings, lists and tables. +// The previous version dropped every blank line and rejoined blocks with a single +// newline, so paragraphs landed tight against each other and Steve got "big blobs of +// text with no spacing". Headings, list markers, table pipes and rules had no rule at +// all, so they arrived as literal `##`, `-` and `|`. Blocks are now rejoined with a +// blank line, headings become bold, list items are double spaced, and a Markdown table +// becomes an aligned monospace block. const escapeHtml = text => String(text).replace(/&/g, '&').replace(//g, '>') @@ -33,6 +41,98 @@ function inlinePlain(line) { .replace(/~~([^~]+)~~/g, '$1') } +const HEADING_RE = /^\s{0,3}#{1,6}\s+(.*?)\s*#*\s*$/ +const LIST_RE = /^\s{0,3}(?:[-*+]|\d{1,3}[.)])\s+/ +const RULE_RE = /^\s{0,3}(?:-{3,}|\*{3,}|_{3,})\s*$/ +const TABLE_ROW_RE = /^\s*\|.*\|\s*$/ +const TABLE_SEP_RE = /^\s*\|(?:\s*:?-{2,}:?\s*\|)+\s*$/ + +const isListItem = line => LIST_RE.test(line) +const isRule = line => RULE_RE.test(line) +const isTableRow = line => TABLE_ROW_RE.test(line) +const isTableSep = line => TABLE_SEP_RE.test(line) + +// A table row as plain cell strings. The leading and trailing pipe are framing, not cells. +function tableCells(line) { + const trimmed = line.trim().replace(/^\|/, '').replace(/\|$/, '') + return trimmed.split('|').map(cell => cell.trim()) +} + +// Aligned monospace grid. Telegram has no tables, so this is the only shape that keeps +// columns readable; it is wrapped in
 so the padding survives.
+function tableRows(lines) {
+  return lines.filter(line => !isTableSep(line)).map(tableCells)
+}
+
+function padRows(rows, toPlainFn) {
+  const plain = rows.map(cells => cells.map(cell => toPlainFn(cell)))
+  const width = []
+  for (const cells of plain) {
+    cells.forEach((cell, index) => {
+      width[index] = Math.max(width[index] ?? 0, Array.from(cell).length)
+    })
+  }
+  return plain.map(cells => cells
+    .map((cell, index) => {
+      const pad = width[index] - Array.from(cell).length
+      return cell + ' '.repeat(Math.max(0, pad))
+    })
+    .join('  ')
+    .replace(/\s+$/, ''))
+}
+
+// One text block, in order, with list items separated by a blank line.
+function renderLines(lines, inline, wrapHeading) {
+  const out = []
+  let previousWasListItem = false
+  for (const line of lines) {
+    const heading = HEADING_RE.exec(line)
+    if (heading !== null) {
+      out.push(wrapHeading(inline(heading[1])))
+      previousWasListItem = false
+      continue
+    }
+    if (isRule(line)) {
+      // Same wrapper as a heading, so the plain renderer never sees HTML tags.
+      out.push(wrapHeading('-'.repeat(12)))
+      previousWasListItem = false
+      continue
+    }
+    const item = isListItem(line)
+    if (item && previousWasListItem) out.push('')
+    out.push(inline(line))
+    previousWasListItem = item
+  }
+  return out.join('\n')
+}
+
+function renderHtmlLines(lines) {
+  return renderLines(lines, inlineHtml, inner => `${inner}`)
+}
+
+function renderPlainLines(lines) {
+  return renderLines(lines, inlinePlain, inner => inner)
+}
+
+// Split a block's lines into table runs and ordinary runs, so a table can become 
+// without dragging the paragraphs beside it into monospace.
+function splitRuns(lines) {
+  const runs = []
+  let at = 0
+  while (at < lines.length) {
+    if (isTableRow(lines[at])) {
+      const start = at
+      while (at < lines.length && isTableRow(lines[at])) at += 1
+      runs.push({ kind: 'table', lines: lines.slice(start, at) })
+      continue
+    }
+    const start = at
+    while (at < lines.length && !isTableRow(lines[at])) at += 1
+    runs.push({ kind: 'lines', lines: lines.slice(start, at) })
+  }
+  return runs
+}
+
 // One Markdown document as blocks: fenced code stays a block of its own so chunking
 // can keep it whole; the rest splits on blank lines.
 function blocks(markdown) {
@@ -63,18 +163,36 @@ function blocks(markdown) {
   return out
 }
 
-const renderHtml = block => (block.kind === 'code'
-  ? `
${escapeHtml(block.lines.join('\n'))}
` - : block.lines.map(inlineHtml).join('\n')) +const renderHtml = block => { + if (block.kind === 'code') { + return `
${escapeHtml(block.lines.join('\n'))}
` + } + return splitRuns(block.lines) + .map(run => (run.kind === 'table' + ? `
${escapeHtml(padRows(tableRows(run.lines), inlinePlain).join('\n'))}
` + : renderHtmlLines(run.lines))) + .join('\n\n') +} + +const renderPlain = block => { + if (block.kind === 'code') return block.lines.join('\n') + return splitRuns(block.lines) + .map(run => (run.kind === 'table' + ? padRows(tableRows(run.lines), inlinePlain).join('\n') + : renderPlainLines(run.lines))) + .join('\n\n') +} -const renderPlain = block => (block.kind === 'code' ? block.lines.join('\n') : block.lines.map(inlinePlain).join('\n')) +// Blocks are one paragraph each, so they are rejoined with a BLANK line. This is the +// line that stops paragraphs running together. +const BLOCK_JOIN = '\n\n' export function toTelegramHtml(markdown) { - return blocks(markdown).map(renderHtml).join('\n') + return blocks(markdown).map(renderHtml).join(BLOCK_JOIN) } export function toPlain(markdown) { - return blocks(markdown).map(renderPlain).join('\n') + return blocks(markdown).map(renderPlain).join(BLOCK_JOIN) } const points = text => Array.from(text).length @@ -171,10 +289,10 @@ export function chunkTelegram(markdown, limit = 4096) { for (const block of blocks(markdown)) { const blockHtml = renderHtml(block) const blockText = renderPlain(block) - const candidate = html === '' ? blockHtml : `${html}\n${blockHtml}` + const candidate = html === '' ? blockHtml : `${html}${BLOCK_JOIN}${blockHtml}` if (points(candidate) <= limit) { html = candidate - text = text === '' ? blockText : `${text}\n${blockText}` + text = text === '' ? blockText : `${text}${BLOCK_JOIN}${blockText}` continue } flush() diff --git a/ds-bot/test/im-format.test.js b/ds-bot/test/im-format.test.js index 245b4b974f..47f09eb479 100644 --- a/ds-bot/test/im-format.test.js +++ b/ds-bot/test/im-format.test.js @@ -8,7 +8,8 @@ test('Markdown becomes Telegram HTML', () => { assert.equal(toTelegramHtml('see [the docs](https://a.b/c)'), 'see the docs') assert.equal(toTelegramHtml('a < b & c > d'), 'a < b & c > d') const fenced = toTelegramHtml('before\n```js\nlet x = 1 < 2\n```\nafter') - assert.equal(fenced, 'before\n
let x = 1 < 2
\nafter') + // A fenced block is a block of its own, so it is separated by a blank line. + assert.equal(fenced, 'before\n\n
let x = 1 < 2
\n\nafter') assert.equal(toTelegramHtml('done ~~deal~~'), 'done deal') }) @@ -36,3 +37,29 @@ test('an oversized line is hard cut and stays under the limit', () => { assert.equal(chunks.length, 3) for (const chunk of chunks) assert.ok(chunk.html.length <= 4096) }) + +// A Telegram message arrived as one blob with no spacing: blocks were rejoined with a +// single newline, and headings, list markers and table pipes had no rule at all. +test('a blank line between paragraphs survives', () => { + assert.equal(toTelegramHtml('first para\n\nsecond para'), 'first para\n\nsecond para') + assert.equal(toPlain('first para\n\nsecond para'), 'first para\n\nsecond para') + // Every block boundary is one blank line, never a run of them. + assert.equal(toPlain('p1\n\np2\n\np3'), 'p1\n\np2\n\np3') +}) + +test('a heading becomes bold instead of a literal marker', () => { + assert.equal(toTelegramHtml('## Heading\ntext'), 'Heading\ntext') + assert.equal(toPlain('## Heading\ntext'), 'Heading\ntext') +}) + +test('list items are double spaced', () => { + assert.equal(toTelegramHtml('- one\n- two'), '- one\n\n- two') + assert.equal(toPlain('- one\n- two'), '- one\n\n- two') +}) + +test('a table becomes an aligned grid with the separator dropped', () => { + assert.equal(toTelegramHtml('| a | b |\n| --- | --- |\n| 1 | 2 |'), '
a  b\n1  2
') + assert.equal(toPlain('| a | b |\n| --- | --- |\n| 1 | 2 |'), 'a b\n1 2') + // Padding is to the widest cell, so the columns line up. + assert.equal(toPlain('| name | n |\n| --- | --- |\n| a | 22 |'), 'name n\na 22') +})