diff --git a/packages/ui/src/__tests__/markdown-body.test.ts b/packages/ui/src/__tests__/markdown-body.test.ts index c38ec386cd..f824394dc0 100644 --- a/packages/ui/src/__tests__/markdown-body.test.ts +++ b/packages/ui/src/__tests__/markdown-body.test.ts @@ -59,6 +59,50 @@ it('renders Markdown emphasis and LaTeX without exposing their source delimiters assert.doesNotMatch(markup, /\\\\\\\(/); }); +it('keeps URL, email, and Markdown markers atomic inside math', () => { + const markup = renderToStaticMarkup(createElement(LocaleProvider, { + locale: 'en', + children: createElement(MarkdownBody, { + text: [ + 'URL \\( \\texttt{https://example.com} \\)', + 'Email \\( \\text{person@example.com} \\)', + 'Markers \\( x \\left[y\\right] * z \\)', + ].join('\n\n'), + streaming: true, + settledText: [ + 'URL \\( \\texttt{https://example.com} \\)', + 'Email \\( \\text{person@example.com} \\)', + 'Markers \\( x \\left[y\\right] * z \\)', + ].join('\n\n'), + }), + })); + + assert.equal((markup.match(/class="maka-math maka-math-inline"/g) ?? []).length, 3); + assert.equal((markup.match(/class="katex"/g) ?? []).length, 3); + assert.doesNotMatch(markup, / { + const markup = renderToStaticMarkup(createElement(MarkdownBody, { + text: 'Empty \\( \\) end', + })); + + assert.doesNotMatch(markup, /class="maka-math/); + assert.doesNotMatch(markup, /\\\(|\\\)/); + assert.match(markup, /Empty \( \) end/); +}); + +it('keeps literal math transport syntax as prose', () => { + const literalToken = '\uE000MAKA_MATH:0:78\uE001'; + const markup = renderToStaticMarkup(createElement(MarkdownBody, { + text: `Literal ${literalToken} end`, + })); + + assert.doesNotMatch(markup, /class="maka-math/); + assert.match(markup, new RegExp(literalToken)); +}); + it('leaves LaTeX delimiters untouched inside inline and fenced code', () => { const markup = renderToStaticMarkup(createElement(MarkdownBody, { text: ['Use `\\( x + 1 \\)` literally.', '', '```tex', '\\( y + 2 \\)', '```'].join('\n'), @@ -69,6 +113,44 @@ it('leaves LaTeX delimiters untouched inside inline and fenced code', () => { assert.match(markup, /\\\( y \+ 2 \\\)/); }); +it('does not let an unmatched inline backtick hide later math', () => { + const markup = renderToStaticMarkup(createElement(MarkdownBody, { + text: 'Unmatched ` prose.\n\nMath \\(x + 1\\)', + })); + + assert.match(markup, /class="maka-math maka-math-inline"/); + assert.match(markup, /class="katex"/); +}); + +it('does not let an unmatched math delimiter hide a later formula', () => { + for (const text of [ + 'bad \\( then \\[x\\]', + 'bad $$ then \\(x\\)', + 'bad \\[ then \\(x\\)', + ]) { + const markup = renderToStaticMarkup(createElement(MarkdownBody, { text })); + assert.match(markup, /class="maka-math/); + assert.match(markup, /class="katex/); + } +}); + +it('lets a formula own backticks that occur inside its delimiters', () => { + for (const formula of ['\\(x ` y\\)', '\\(x \\text{`foo`}\\)']) { + const markup = renderToStaticMarkup(createElement(MarkdownBody, { text: formula })); + assert.match(markup, /class="maka-math maka-math-inline"/); + assert.match(markup, /class="katex"/); + } +}); + +it('keeps scanning after a malformed math transport prefix', () => { + const markup = renderToStaticMarkup(createElement(MarkdownBody, { + text: `bad \uE000MAKA_MATH:bad then \\(x\\)`, + })); + + assert.match(markup, /MAKA_MATH:bad/); + assert.match(markup, /class="maka-math maka-math-inline"/); +}); + it('renders display math while leaving ordinary currency alone', () => { const markup = renderToStaticMarkup(createElement(MarkdownBody, { text: 'Budget: $5 and $10. Range: $5–$10.\n\n\\[ x^2 + y^2 = z^2 \\]', @@ -99,21 +181,6 @@ it('does not treat shell variables, currency, or inline code as dollar-delimited assert.match(markup, /class="katex"/); }); -it('keeps raw and malformed internal-token lookalikes literal', () => { - for (const token of [ - '\uE000MAKAMATHIFFFFFFEND\uE001', - '\uE000MAKAMATH:0:0:\uE001', - '\uE000MAKAMATH:999:not-a-token:\uE001', - ]) { - const markup = renderToStaticMarkup(createElement(MarkdownBody, { - text: `Before ${token} after with \\( x + 1 \\).`, - })); - - assert.match(markup, new RegExp(token)); - assert.equal((markup.match(/class="maka-math maka-math-inline"/g) ?? []).length, 1); - } -}); - it('renders multiline display math outside code for both supported delimiters', () => { for (const [text, mathNode] of [ [['Before', '', '$$', 'E = mc^2', '$$', '', 'After'].join('\n'), ''], @@ -132,6 +199,26 @@ it('renders multiline display math outside code for both supported delimiters', } }); +it('keeps display math intact across Markdown-looking block boundaries', () => { + const bodies = [ + ['x + 1', '', 'y + 2'], + ['x + 1', '# heading-shaped'], + ['x + 1', '- list-shaped'], + ['x + 1', '| table | shaped |', '| --- | --- |'], + ]; + + for (const [open, close] of [['$$', '$$'], ['\\[', '\\]']]) { + for (const body of bodies) { + const markup = renderToStaticMarkup(createElement(MarkdownBody, { + text: ['Before', '', open, ...body, close, '', 'After'].join('\n'), + })); + + assert.match(markup, /class="maka-math maka-math-display"/); + assert.doesNotMatch(markup, / { const markup = renderToStaticMarkup(createElement(MarkdownBody, { text: ['$$', 'outside', '```tex', 'inside', '```', '$$'].join('\n'), @@ -409,15 +496,13 @@ it('shows only the restored prefix on its first streaming render', () => { assert.doesNotMatch(markup, /new delta/); }); -it('uses the same protected math registry for live and settled streaming text', () => { - const rawToken = '\uE000MAKAMATH:0:0:\uE001'; +it('renders settled math while keeping the live tail behind the display cursor', () => { const markup = renderToStaticMarkup(createElement(MarkdownBody, { - text: `Stable ${rawToken} \\( x + 1 \\) with a new delta`, + text: 'Stable \\( x + 1 \\) with a new delta', streaming: true, - settledText: `Stable ${rawToken} \\( x + 1 \\)`, + settledText: 'Stable \\( x + 1 \\)', })); - assert.match(markup, new RegExp(rawToken)); assert.match(markup, /class="maka-math maka-math-inline"/); assert.match(markup, /class="katex"/); assert.doesNotMatch(markup, /new delta/); diff --git a/packages/ui/src/__tests__/streaming-text.test.tsx b/packages/ui/src/__tests__/streaming-text.test.tsx index fa3e090cd7..fa2a97f2a3 100644 --- a/packages/ui/src/__tests__/streaming-text.test.tsx +++ b/packages/ui/src/__tests__/streaming-text.test.tsx @@ -23,6 +23,8 @@ import { useStreamingText } from '@astryxdesign/core'; import { act } from 'react'; import { createRoot } from 'react-dom/client'; import { parseHTML } from 'linkedom'; +import { LocaleProvider } from '../locale-context.js'; +import { MarkdownBody } from '../markdown-body.js'; const originalGlobals = { document: globalThis.document, @@ -46,6 +48,9 @@ function streamingRoot( requestAnimationFrame: (callback: FrameRequestCallback) => number = () => 1, ) { const { document, window } = parseHTML('
'); + Object.assign(window, { + getComputedStyle: () => ({ direction: 'ltr', writingMode: 'horizontal-tb' }), + }); Object.assign(globalThis, { document, window, @@ -109,3 +114,153 @@ test('never reveals half of a Unicode code point', async () => { await act(() => root.unmount()); }); + +test('keeps settled math stable while the live Markdown tail grows and flushes', async () => { + const frames: FrameRequestCallback[] = []; + const { container, root } = streamingRoot((callback) => { + frames.push(callback); + return frames.length; + }); + const settled = [ + 'Before', + '', + '\\( \\texttt{https://example.com} + \\text{person@example.com} \\)', + '', + '', + ].join('\n'); + + function render(text: string, streaming: boolean) { + return root.render( + + + , + ); + } + + await act(() => render(`${settled}growing`, true)); + const math = container.querySelector('.maka-math'); + assert.ok(math); + assert.ok(math.querySelector('.katex')); + assert.equal(container.querySelector('a'), null); + + const firstFrame = frames.shift(); + assert.ok(firstFrame); + await act(() => firstFrame(100)); + assert.equal(container.querySelector('.maka-math'), math); + + await act(() => render(`${settled}growing live tail`, true)); + const secondFrame = frames.shift(); + assert.ok(secondFrame); + await act(() => secondFrame(200)); + assert.equal(container.querySelector('.maka-math'), math); + assert.equal(container.querySelector('a'), null); + + await act(() => render(`${settled}final tail`, false)); + assert.ok(container.querySelector('.maka-math .katex')); + assert.equal(container.querySelector('a'), null); + assert.match(container.textContent ?? '', /final tail/); + + await act(() => root.unmount()); +}); + +test('never exposes math transport syntax as a formula crosses the display cursor', async () => { + const frames: FrameRequestCallback[] = []; + const { container, root } = streamingRoot((callback) => { + frames.push(callback); + return frames.length; + }); + const target = 'Before \\(x + 1\\) after'; + + await act(() => root.render( + + + , + )); + + for (let tick = 1; tick <= 8 && container.querySelector('.maka-math') === null; tick++) { + const frame = frames.shift(); + assert.ok(frame); + await act(() => frame(tick * 100)); + assert.doesNotMatch(container.textContent ?? '', /MAKA_MATH|\uE000|\uE001/); + } + + assert.ok(container.querySelector('.maka-math .katex')); + assert.match(container.textContent ?? '', /Before/); + await act(() => root.unmount()); +}); + +test('keeps a restored prefix inside math visible and handles a formula rewrite', async () => { + const frames: FrameRequestCallback[] = []; + const { container, root } = streamingRoot((callback) => { + frames.push(callback); + return frames.length; + }); + const first = 'Before \\(x + 1\\) after'; + const second = 'Before \\(x + 2\\) after'; + + await act(() => root.render( + + + , + )); + assert.match(container.textContent ?? '', /Before \(x/); + assert.doesNotMatch(container.textContent ?? '', /MAKA_MATH|\uE000|\uE001/); + + await act(() => root.render( + + + , + )); + assert.doesNotMatch(container.textContent ?? '', /MAKA_MATH|\uE000|\uE001/); + + for (let tick = 1; tick <= 8 && container.querySelector('.maka-math') === null; tick++) { + const frame = frames.shift(); + assert.ok(frame); + await act(() => frame(tick * 100)); + assert.doesNotMatch(container.textContent ?? '', /MAKA_MATH|\uE000|\uE001/); + } + assert.ok(container.querySelector('.maka-math .katex')); + assert.match(container.textContent ?? '', /2/); + + await act(() => root.unmount()); +}); + +test('keeps split fenced-code openers literal through final flush', async () => { + for (const { prefix, target } of [ + { prefix: '~~', target: '~~~ts\n\\(not math\\)\n~~~' }, + { prefix: '``', target: '```ts\n\\(not math\\)\n```' }, + ]) { + const frames: FrameRequestCallback[] = []; + const { container, root } = streamingRoot((callback) => { + frames.push(callback); + return frames.length; + }); + const render = (text: string, streaming: boolean) => root.render( + + + , + ); + + await act(() => render(prefix, true)); + await act(() => render(target, true)); + for (let tick = 1; tick <= 30 && frames.length > 0; tick++) { + const frame = frames.shift(); + assert.ok(frame); + await act(() => frame(tick * 100)); + } + await act(() => render(target, false)); + + assert.equal(container.querySelector('code')?.textContent, '\\(not math\\)'); + assert.equal(container.querySelector('.maka-math'), null); + assert.doesNotMatch(container.textContent ?? '', /MAKA_MATH|\uE000|\uE001/); + await act(() => root.unmount()); + } +}); diff --git a/packages/ui/src/markdown-body.tsx b/packages/ui/src/markdown-body.tsx index 1687ae12b9..e293b4b59e 100644 --- a/packages/ui/src/markdown-body.tsx +++ b/packages/ui/src/markdown-body.tsx @@ -29,7 +29,7 @@ * product-specific trust boundaries around that renderer. */ -import { useContext, type ReactNode } from 'react'; +import { useCallback, useContext, useRef, type ReactNode } from 'react'; import { Markdown as AstryxMarkdown, type MarkdownComponents, @@ -46,7 +46,11 @@ import { MakaUriContext } from './markdown.js'; import { useUiLocale } from './locale-context.js'; import { getSharedUiCopy } from './shared-ui-copy.js'; import { MermaidDiagram } from './mermaid-diagram.js'; -import { prepareMarkdownMath } from './markdown-math.js'; +import { + createMarkdownMathCache, + MARKDOWN_MATH_PLUGINS, + prepareMarkdownMath, +} from './markdown-math.js'; import { parseAttachmentResourceRef } from '@maka/core/attachments'; import { useAttachmentImageSource } from './attachment-image.js'; @@ -132,9 +136,12 @@ export function MarkdownBody(props: { settledText?: string; density?: 'default' | 'compact'; }) { - const prepared = prepareMarkdownMath(props.text, props.settledText); - const safeText = prepared.text; - const budgetedText = props.streaming ? safeText : applyMermaidRenderBudget(safeText); + const mathCache = useRef(createMarkdownMathCache()); + const transformMathSource = useCallback( + (source: string) => prepareMarkdownMath(source, mathCache.current), + [], + ); + const budgetedText = props.streaming ? props.text : applyMermaidRenderBudget(props.text); const density = props.density ?? 'default'; const components = props.streaming ? density === 'compact' @@ -175,9 +182,10 @@ export function MarkdownBody(props: { // the one combination neither half of the argument asks for. density={density} components={components} - inlinePlugins={[prepared.plugin]} + inlinePlugins={MARKDOWN_MATH_PLUGINS} isStreaming={props.streaming} - settledText={prepared.settledText} + settledText={props.settledText} + transformSource={transformMathSource} > {budgetedText} diff --git a/packages/ui/src/markdown-math.tsx b/packages/ui/src/markdown-math.tsx index eccd8cbb98..74f5e9f19c 100644 --- a/packages/ui/src/markdown-math.tsx +++ b/packages/ui/src/markdown-math.tsx @@ -20,177 +20,296 @@ import katex from 'katex'; import type { MarkdownInlinePlugin } from '@astryxdesign/core/Markdown'; -export interface PreparedMarkdownMath { +const TOKEN_START = '\uE000MAKA_MATH:'; +const TOKEN_END = '\uE001'; +const TOKEN_PATTERN = /\uE000MAKA_MATH:([012]):([0-9a-f]+)\uE001/g; +const LITERAL_TOKEN_PATTERN = /^\uE000MAKA_MATH:[012]:[0-9a-f]+\uE001/; + +/** + * Discardable derived state owned by one MarkdownBody mount. Capacity is two + * strings no larger than that component's currently displayed source and its + * transport form; a rewrite resets both and unmounting drops the cache. + */ +export interface MarkdownMathCache { + source: string; text: string; - settledText?: string; - plugin: MarkdownInlinePlugin; + safeSourceEnd: number; + safeTextEnd: number; } -interface MathTokenValue { - formula: string; - displayMode: boolean; +export function createMarkdownMathCache(): MarkdownMathCache { + return { source: '', text: '', safeSourceEnd: 0, safeTextEnd: 0 }; } export function prepareMarkdownMath( source: string, - settledSource?: string, -): PreparedMarkdownMath { - const registry = createMathTokenRegistry([source, settledSource]); - return { - text: protectMathOutsideCode(source, registry.register), - ...(settledSource === undefined - ? {} - : { settledText: protectMathOutsideCode(settledSource, registry.register) }), - plugin: { - pattern: registry.pattern, - render: (match, key) => { - const value = registry.values.get(match[0]); - if (!value) return match[0]; - const html = katex.renderToString(value.formula, { - displayMode: value.displayMode, - output: 'htmlAndMathml', - strict: 'warn', - throwOnError: false, - trust: false, - }); - return ( - - ); - }, - }, - }; -} + cache: MarkdownMathCache, +): string { + // The caller currently supplies a full string rather than an append token, + // so proving that a rewrite did not occur requires this prefix check. It + // keeps the JavaScript lexer on the changing tail; it does not make the + // full-string identity check itself incremental. + const extendsPrevious = source.startsWith(cache.source); + const sourceStart = extendsPrevious ? cache.safeSourceEnd : 0; + const textStart = extendsPrevious ? cache.safeTextEnd : 0; + const protectedTail = protectMarkdownMath( + source.slice(sourceStart), + sourceStart === 0 || source[sourceStart - 1] === '\n', + ); + const text = `${extendsPrevious ? cache.text.slice(0, textStart) : ''}${protectedTail.text}`; -function createMathTokenRegistry(sources: Array): { - pattern: RegExp; - register: (formula: string, displayMode: boolean) => string; - values: Map; -} { - let namespaceIndex = 0; - let namespace = ''; - do { - namespace = `\uE000MAKAMATH:${namespaceIndex}:`; - namespaceIndex += 1; - } while (sources.some((source) => source?.includes(namespace))); - - const tokenEnd = ':\uE001'; - const values = new Map(); - const tokensByValue = new Map(); - let nextTokenId = 0; - return { - pattern: new RegExp(`${escapeRegExp(namespace)}\\d+${escapeRegExp(tokenEnd)}`, 'g'), - register: (formula, displayMode) => { - const valueKey = JSON.stringify([displayMode, formula]); - const existingToken = tokensByValue.get(valueKey); - if (existingToken) return existingToken; - - const token = `${namespace}${nextTokenId}${tokenEnd}`; - nextTokenId += 1; - values.set(token, { formula, displayMode }); - tokensByValue.set(valueKey, token); - return token; - }, - values, - }; + cache.source = source; + cache.text = text; + cache.safeSourceEnd = sourceStart + protectedTail.safeSourceEnd; + cache.safeTextEnd = textStart + protectedTail.safeTextEnd; + return text; } -function protectMathOutsideCode( - source: string, - register: (formula: string, displayMode: boolean) => string, -): string { - const lines = source.split('\n'); - let fence: { character: string; length: number } | undefined; - let proseLines: string[] = []; - const protectedParts: string[] = []; - - const flushProse = () => { - if (proseLines.length === 0) return; - protectedParts.push(protectMathInProse(proseLines.join('\n'), register)); - proseLines = []; +export const MARKDOWN_MATH_PLUGINS = [{ + pattern: TOKEN_PATTERN, + render: (match, key) => { + const formula = decodeFormula(match[2] ?? ''); + if (match[1] === '2') return formula; + const displayMode = match[1] === '1'; + const html = katex.renderToString(formula, { + displayMode, + output: 'htmlAndMathml', + strict: 'warn', + throwOnError: false, + trust: false, + }); + return ( + + ); + }, +}] satisfies MarkdownInlinePlugin[]; + +function protectMarkdownMath(source: string, startsAtLineStart = true): { + text: string; + safeSourceEnd: number; + safeTextEnd: number; +} { + let text = ''; + let index = 0; + let safeSourceEnd = 0; + let safeTextEnd = 0; + let atLineStart = startsAtLineStart; + let canMarkSafe = true; + const markSafe = () => { + if (!canMarkSafe) return; + safeSourceEnd = index; + safeTextEnd = text.length; }; - for (const line of lines) { - const opening = /^( {0,3})(`{3,}|~{3,})/.exec(line); - if (opening) { - flushProse(); - const marker = opening[2] ?? ''; - const character = marker[0] ?? ''; - if (!fence) { - fence = { character, length: marker.length }; - } else if (character === fence.character && marker.length >= fence.length) { - fence = undefined; - } - protectedParts.push(line); - } else if (fence) { - protectedParts.push(line); - } else { - proseLines.push(line); + while (index < source.length) { + const fence = atLineStart ? readFence(source, index) : undefined; + if (fence?.kind === 'pending') { + text += source.slice(index); + break; + } + if (fence?.kind === 'match') { + text += source.slice(index, fence.end); + index = fence.end; + atLineStart = source[index - 1] === '\n'; + if (fence.closed) markSafe(); + else break; + continue; } - } - flushProse(); - - return protectedParts.join('\n'); -} -function protectMathInProse( - source: string, - register: (formula: string, displayMode: boolean) => string, -): string { - let output = ''; - let index = 0; + const literalToken = readLiteralToken(source, index); + if (literalToken?.kind === 'pending') { + text += source.slice(index); + break; + } + if (literalToken?.kind === 'match') { + text += transportToken(literalToken.source, '2'); + index = literalToken.end; + atLineStart = false; + markSafe(); + continue; + } - while (index < source.length) { if (source[index] === '`') { - const run = /^`+/.exec(source.slice(index))?.[0] ?? '`'; - const close = source.indexOf(run, index + run.length); - if (close >= 0) { - output += source.slice(index, close + run.length); - index = close + run.length; + let runEnd = index + 1; + while (source[runEnd] === '`') runEnd++; + const run = source.slice(index, runEnd); + const close = source.indexOf(run, runEnd); + if (close < 0) { + text += run; + index = runEnd; + atLineStart = false; + canMarkSafe = false; continue; } + const end = close + run.length; + text += source.slice(index, end); + index = end; + atLineStart = source[index - 1] === '\n'; + markSafe(); + continue; } const delimited = readDelimitedMath(source, index, '\\(', '\\)', false, false) ?? readDelimitedMath(source, index, '\\[', '\\]', true, true) ?? readDelimitedMath(source, index, '$$', '$$', true, true); - if (delimited) { - output += register(delimited.formula, delimited.displayMode); + if (delimited?.kind === 'pending') { + text += source.slice(index, delimited.end); index = delimited.end; + atLineStart = false; + canMarkSafe = false; + continue; + } + if (delimited?.kind === 'match') { + text += mathToken(delimited.formula, delimited.displayMode); + index = delimited.end; + atLineStart = false; + markSafe(); continue; } - output += source[index]; - index += 1; + const character = source[index] ?? ''; + text += character; + index++; + atLineStart = character === '\n'; + if ( + index < source.length || + (character !== '\\' && character !== '$' && character !== '`') + ) { + markSafe(); + } } - return output; + return { text, safeSourceEnd, safeTextEnd }; +} + +function readFence( + source: string, + index: number, +): + | { kind: 'match'; end: number; closed: boolean } + | { kind: 'pending' } + | undefined { + const tail = source.slice(index); + const opening = /^( {0,3})(`{3,}|~{3,})/.exec(tail); + if (!opening) { + return /^ {0,3}(?:`{1,2}|~{1,2})?$/.test(tail) + ? { kind: 'pending' } + : undefined; + } + const marker = opening[2] ?? ''; + let lineStart = source.indexOf('\n', index); + while (lineStart >= 0) { + lineStart++; + const candidate = /^( {0,3})(`{3,}|~{3,})/.exec(source.slice(lineStart)); + const candidateMarker = candidate?.[2] ?? ''; + if ( + candidateMarker[0] === marker[0] && + candidateMarker.length >= marker.length + ) { + const lineEnd = source.indexOf('\n', lineStart); + return { + kind: 'match', + end: lineEnd < 0 ? source.length : lineEnd + 1, + closed: true, + }; + } + lineStart = source.indexOf('\n', lineStart); + } + return { kind: 'match', end: source.length, closed: false }; +} + +function readLiteralToken( + source: string, + index: number, +): + | { kind: 'match'; source: string; end: number } + | { kind: 'pending' } + | undefined { + if (source[index] !== TOKEN_START[0]) return undefined; + if (!source.startsWith(TOKEN_START, index)) { + const tail = source.slice(index); + return tail.length < TOKEN_START.length && TOKEN_START.startsWith(tail) + ? { kind: 'pending' } + : undefined; + } + const tokenEnd = source.indexOf(TOKEN_END, index + TOKEN_START.length); + if (tokenEnd < 0) { + const payload = source.slice(index + TOKEN_START.length); + return /^(?:[012](?::[0-9a-f]*)?)?$/.test(payload) + ? { kind: 'pending' } + : undefined; + } + const candidate = source.slice(index, tokenEnd + TOKEN_END.length); + const match = LITERAL_TOKEN_PATTERN.exec(candidate); + if (!match) return undefined; + const token = match[0]; + return { kind: 'match', source: token, end: index + token.length }; } function readDelimitedMath( - line: string, + source: string, index: number, opening: string, closing: string, displayMode: boolean, allowNewlines: boolean, -): { formula: string; displayMode: boolean; end: number } | undefined { - if (!line.startsWith(opening, index)) return undefined; +): + | { kind: 'match'; formula: string; displayMode: boolean; end: number } + | { kind: 'pending'; end: number } + | undefined { + if (!source.startsWith(opening, index)) return undefined; const contentStart = index + opening.length; - const close = line.indexOf(closing, contentStart); - if (close < 0) return undefined; - if (!allowNewlines && line.slice(contentStart, close).includes('\n')) return undefined; - const formula = line.slice(contentStart, close).trim(); - if (!formula) return undefined; - return { formula, displayMode, end: close + closing.length }; + const close = source.indexOf(closing, contentStart); + + if (close < 0) { + if (!allowNewlines && source.indexOf('\n', contentStart) >= 0) return undefined; + if (findPendingFenceBoundary(source, contentStart) >= 0) return undefined; + return { kind: 'pending', end: contentStart }; + } + const rawFormula = source.slice(contentStart, close); + if (!allowNewlines && rawFormula.includes('\n')) return undefined; + if (/(?:^|\n) {0,3}(?:`{3,}|~{3,})/.test(rawFormula)) { + return undefined; + } + const formula = rawFormula.trim(); + if (formula === '') return undefined; + return { kind: 'match', formula, displayMode, end: close + closing.length }; +} + +function findPendingFenceBoundary(source: string, from: number): number { + const fenceMatch = /(?:^|\n) {0,3}(?:`{3,}|~{3,})/g; + fenceMatch.lastIndex = from; + const fence = fenceMatch.exec(source); + return fence ? fence.index + (source[fence.index] === '\n' ? 1 : 0) : -1; +} + +function mathToken(formula: string, displayMode: boolean): string { + return transportToken(formula, displayMode ? '1' : '0'); +} + +function transportToken(value: string, kind: '0' | '1' | '2'): string { + return `${TOKEN_START}${kind}:${encodeFormula(value)}${TOKEN_END}`; } -function escapeRegExp(value: string): string { - return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +function encodeFormula(formula: string): string { + let encoded = ''; + for (const byte of new TextEncoder().encode(formula)) { + encoded += byte.toString(16).padStart(2, '0'); + } + return encoded; +} + +function decodeFormula(encoded: string): string { + const bytes = new Uint8Array(encoded.length / 2); + for (let index = 0; index < bytes.length; index++) { + bytes[index] = Number.parseInt(encoded.slice(index * 2, index * 2 + 2), 16); + } + return new TextDecoder().decode(bytes); } diff --git a/patches/@astryxdesign+core+0.5.0.patch b/patches/@astryxdesign+core+0.5.0.patch index 0b7127ec3b..6a2d3c8c83 100644 --- a/patches/@astryxdesign+core+0.5.0.patch +++ b/patches/@astryxdesign+core+0.5.0.patch @@ -154,29 +154,34 @@ index 08b5f7d..94c623b 100644 ...(isOrdered && start != null && start !== 1 ? { start diff --git a/node_modules/@astryxdesign/core/dist/Markdown/Markdown.d.ts b/node_modules/@astryxdesign/core/dist/Markdown/Markdown.d.ts -index d9ad714..2b9c461 100644 +index d9ad714..91cde1e 100644 --- a/node_modules/@astryxdesign/core/dist/Markdown/Markdown.d.ts +++ b/node_modules/@astryxdesign/core/dist/Markdown/Markdown.d.ts -@@ -90,6 +90,8 @@ export interface MarkdownProps extends BaseProps { +@@ -90,6 +90,10 @@ export interface MarkdownProps extends BaseProps { */ headingLevelStart?: 1 | 2 | 3 | 4 | 5 | 6; isStreaming?: boolean; + /** Source text the host has authoritatively marked as already presented. */ + settledText?: string; ++ /** Transform displayed source immediately before Markdown parsing. */ ++ transformSource?: (source: string) => string; onLinkClick?: (href: string, event: React.MouseEvent) => void | false; /** * Citation sources keyed by ID. When provided, `[id]` and `【id】` markers -@@ -155,7 +157,7 @@ export interface MarkdownProps extends BaseProps { +@@ -155,8 +159,8 @@ export interface MarkdownProps extends BaseProps { * * ``` */ -export declare function Markdown({ ref, children, display, density, headingLevelStart, isStreaming, onLinkClick, sources, citationStyle, contentWidth, contentAlign, components, inlinePlugins, autolink, xstyle, className, style, 'data-testid': testId, }: MarkdownProps): React.ReactElement; -+export declare function Markdown({ ref, children, display, density, headingLevelStart, isStreaming, settledText, onLinkClick, sources, citationStyle, contentWidth, contentAlign, components, inlinePlugins, autolink, xstyle, className, style, 'data-testid': testId, }: MarkdownProps): React.ReactElement; ++export declare function Markdown({ ref, children, display, density, headingLevelStart, isStreaming, settledText, transformSource, onLinkClick, sources, citationStyle, contentWidth, contentAlign, components, inlinePlugins, autolink, xstyle, className, style, 'data-testid': testId, }: MarkdownProps): React.ReactElement; export declare namespace Markdown { var displayName: string; } +-//# sourceMappingURL=Markdown.d.ts.map +\ No newline at end of file ++//# sourceMappingURL=Markdown.d.ts.map diff --git a/node_modules/@astryxdesign/core/dist/Markdown/Markdown.js b/node_modules/@astryxdesign/core/dist/Markdown/Markdown.js -index 0fd8329..7e4ec57 100644 +index 0fd8329..a4a158a 100644 --- a/node_modules/@astryxdesign/core/dist/Markdown/Markdown.js +++ b/node_modules/@astryxdesign/core/dist/Markdown/Markdown.js @@ -8,7 +8,7 @@ @@ -188,15 +193,16 @@ index 0fd8329..7e4ec57 100644 import { Fragment } from 'react'; import * as stylex from '@stylexjs/stylex'; import "../theme/tokens.stylex.js"; -@@ -1048,6 +1048,7 @@ export function Markdown({ +@@ -1048,6 +1048,8 @@ export function Markdown({ density = 'default', headingLevelStart = 1, isStreaming = false, + settledText, ++ transformSource, onLinkClick, sources, citationStyle = 'label', -@@ -1072,7 +1073,9 @@ export function Markdown({ +@@ -1072,26 +1074,22 @@ export function Markdown({ // Smooth bursty streamed chunks into a steady character-by-character reveal. // When not streaming, the hook returns children unchanged (no-op). @@ -204,10 +210,42 @@ index 0fd8329..7e4ec57 100644 + const smoothedText = useStreamingText(children, isStreaming, { + settledText + }); ++ const parsedText = transformSource ? transformSource(smoothedText) : smoothedText; const incrementalStateRef = useRef(createIncrementalState()); - // Reset incremental cache when the autolink option toggles — cached - // settled blocks were parsed with the previous setting. -@@ -1139,17 +1142,25 @@ export function Markdown({ +- // Reset incremental cache when the autolink option toggles — cached +- // settled blocks were parsed with the previous setting. +- const prevAutolinkRef = useRef(autolink); +- if (prevAutolinkRef.current !== autolink) { +- incrementalStateRef.current = createIncrementalState(); +- prevAutolinkRef.current = autolink; +- } + const blocks = useMemo(() => { + if (display === 'inline') { + return []; + } + if (isStreaming) { + if (smoothedText === '') { + incrementalStateRef.current = createIncrementalState(); + return []; + } +- const input = trimStreamingArtifacts(smoothedText); ++ const input = trimStreamingArtifacts(parsedText); + return parseMarkdownIncremental(input, incrementalStateRef.current, parseOptions); + } +- return parseMarkdown(children, parseOptions); +- }, [display, smoothedText, children, isStreaming, parseOptions]); ++ return parseMarkdown(parsedText, parseOptions); ++ }, [display, parsedText, isStreaming, parseOptions]); +@@ -1120,6 +1119,6 @@ export function Markdown({ + if (display !== 'inline') { + return []; + } +- const input = isStreaming ? trimStreamingArtifacts(smoothedText) : children; ++ const input = isStreaming ? trimStreamingArtifacts(parsedText) : parsedText; + return parseInline(input, parseOptions); +- }, [display, smoothedText, children, isStreaming, parseOptions]); ++ }, [display, parsedText, isStreaming, parseOptions]); +@@ -1139,17 +1137,25 @@ export function Markdown({ const tickMs = tick != null ? Math.max(4, Math.round(tick / 10)) : 50; return Math.min(Math.ceil(duration / tickMs), 12); }, [token]); @@ -244,7 +282,7 @@ index 0fd8329..7e4ec57 100644 const cursor = { offset: 0, boundaries, -@@ -1174,9 +1185,6 @@ export function Markdown({ +@@ -1174,9 +1180,6 @@ export function Markdown({ children: inlineNodes.map((node, i) => renderInline(node, i, onLinkClick, cursor, citationCtx, LinkComponent, inlinePlugins, components)) }); @@ -254,7 +292,7 @@ index 0fd8329..7e4ec57 100644 return renderedInline; } const rendered = /*#__PURE__*/_jsx("div", { -@@ -1189,11 +1197,6 @@ export function Markdown({ +@@ -1189,11 +1192,6 @@ export function Markdown({ children: blocks.map((block, i) => renderBlock(block, i, blocks.length, density, headingLevelStart, onLinkClick, cursor, citationCtx, contentWidth ? typeof contentWidth === 'number' ? `${contentWidth}px` : contentWidth : null, contentAlign, LinkComponent, inlinePlugins, components, t, headingIdMap)) }); @@ -265,8 +303,9 @@ index 0fd8329..7e4ec57 100644 - prevInlineNodesRef.current = []; return rendered; } - Markdown.displayName = 'Markdown'; +-Markdown.displayName = 'Markdown'; \ No newline at end of file ++Markdown.displayName = 'Markdown'; diff --git a/node_modules/@astryxdesign/core/dist/SideNav/SideNavItem.d.ts b/node_modules/@astryxdesign/core/dist/SideNav/SideNavItem.d.ts index e7b8e00..fad2e72 100644 --- a/node_modules/@astryxdesign/core/dist/SideNav/SideNavItem.d.ts @@ -541,24 +580,27 @@ index ea918b4..d83af73 100644 import type React from 'react'; import {Fragment} from 'react'; import * as stylex from '@stylexjs/stylex'; -@@ -145,6 +145,8 @@ export interface MarkdownProps extends BaseProps { +@@ -145,6 +145,10 @@ export interface MarkdownProps extends BaseProps { */ headingLevelStart?: 1 | 2 | 3 | 4 | 5 | 6; isStreaming?: boolean; + /** Source text the host has authoritatively marked as already presented. */ + settledText?: string; ++ /** Transform displayed source immediately before Markdown parsing. */ ++ transformSource?: (source: string) => string; onLinkClick?: ( href: string, event: React.MouseEvent, -@@ -1610,6 +1612,7 @@ export function Markdown({ +@@ -1610,6 +1614,8 @@ export function Markdown({ density = 'default', headingLevelStart = 1, isStreaming = false, + settledText, ++ transformSource, onLinkClick, sources, citationStyle = 'label', -@@ -1638,7 +1641,9 @@ export function Markdown({ +@@ -1638,7 +1644,10 @@ export function Markdown({ // Smooth bursty streamed chunks into a steady character-by-character reveal. // When not streaming, the hook returns children unchanged (no-op). @@ -566,10 +608,44 @@ index ea918b4..d83af73 100644 + const smoothedText = useStreamingText(children, isStreaming, { + settledText, + }); ++ const parsedText = transformSource ? transformSource(smoothedText) : smoothedText; const incrementalStateRef = useRef( createIncrementalState(), -@@ -1713,20 +1718,43 @@ export function Markdown({ +@@ -1648,7 +1657,0 @@ export function Markdown({ +- // Reset incremental cache when the autolink option toggles — cached +- // settled blocks were parsed with the previous setting. +- const prevAutolinkRef = useRef(autolink); +- if (prevAutolinkRef.current !== autolink) { +- incrementalStateRef.current = createIncrementalState(); +- prevAutolinkRef.current = autolink; +- } +@@ -1660,12 +1662,12 @@ export function Markdown({ + incrementalStateRef.current = createIncrementalState(); + return []; + } +- const input = trimStreamingArtifacts(smoothedText); ++ const input = trimStreamingArtifacts(parsedText); + return parseMarkdownIncremental( + input, + incrementalStateRef.current, + parseOptions, + ); + } +- return parseMarkdown(children, parseOptions); +- }, [display, smoothedText, children, isStreaming, parseOptions]); ++ return parseMarkdown(parsedText, parseOptions); ++ }, [display, parsedText, isStreaming, parseOptions]); +@@ -1695,6 +1704,6 @@ export function Markdown({ + if (display !== 'inline') { + return []; + } +- const input = isStreaming ? trimStreamingArtifacts(smoothedText) : children; ++ const input = isStreaming ? trimStreamingArtifacts(parsedText) : parsedText; + return parseInline(input, parseOptions); +- }, [display, smoothedText, children, isStreaming, parseOptions]); ++ }, [display, parsedText, isStreaming, parseOptions]); +@@ -1713,20 +1722,43 @@ export function Markdown({ return Math.min(Math.ceil(duration / tickMs), 12); }, [token]); @@ -627,7 +703,7 @@ index ea918b4..d83af73 100644 const cursor: StreamingCursor = { offset: 0, -@@ -1766,10 +1794,6 @@ export function Markdown({ +@@ -1766,10 +1798,6 @@ export function Markdown({ ); @@ -638,7 +714,7 @@ index ea918b4..d83af73 100644 return renderedInline; } -@@ -1810,12 +1834,6 @@ export function Markdown({ +@@ -1810,12 +1838,6 @@ export function Markdown({ ); diff --git a/patches/README.md b/patches/README.md index 6ed332c9a4..8e76603c70 100644 --- a/patches/README.md +++ b/patches/README.md @@ -76,5 +76,8 @@ Streaming text and Markdown expose an explicit `settledText` seam so the renderer can verify and advance the exact prefix already presented without replaying it. The default remains progressive for a genuinely new stream, and rewritten or later text still reveals and fades from a parsed-visible boundary. +Markdown can also transform the displayed prefix immediately before its +existing incremental parser, so host syntax such as math stays behind the +streaming cursor without adding another parser or scheduler. Delete each hunk when the corresponding behavior ships in Astryx.