@@ -4,6 +4,7 @@ import langDiff from "@shikijs/langs/diff";
44import langJson from "@shikijs/langs/json" ;
55import langJsonc from "@shikijs/langs/jsonc" ;
66import langKotlin from "@shikijs/langs/kotlin" ;
7+ import langMarkdown from "@shikijs/langs/markdown" ;
78import langPython from "@shikijs/langs/python" ;
89import langRust from "@shikijs/langs/rust" ;
910import langShellscript from "@shikijs/langs/shellscript" ;
@@ -14,7 +15,7 @@ import langTsx from "@shikijs/langs/tsx";
1415import langYaml from "@shikijs/langs/yaml" ;
1516import themeGithubDark from "@shikijs/themes/github-dark" ;
1617import themeGithubLight from "@shikijs/themes/github-light" ;
17- import { useSyncExternalStore } from "react" ;
18+ import { useMemo , useSyncExternalStore } from "react" ;
1819import {
1920 createNativeEngine ,
2021 isNativeEngineAvailable ,
@@ -27,7 +28,7 @@ import {
2728 *
2829 * Grammars are imported per-language — NEVER `@shikijs/langs` wholesale
2930 * (7.7MB) or `html` (silently chains javascript+css, ~291KB). Core set
30- * ~470KB source, chosen by (a) sidecode's domain (TS/Swift/Kotlin) and
31+ * ~535KB source, chosen by (a) sidecode's domain (TS/Swift/Kotlin) and
3132 * (b) what LLMs actually write in fence info-strings. One tsx grammar
3233 * covers the whole JS family via aliases (superset syntax; saves the
3334 * 180KB-each typescript/javascript grammars). Unknown languages degrade
@@ -42,7 +43,12 @@ let highlighter: HighlighterCore | null = null;
4243let initStarted = false ;
4344const listeners = new Set < ( ) => void > ( ) ;
4445
45- function ensureHighlighter ( ) {
46+ /** Kick off engine + grammar loading. Idempotent. Called from the root
47+ * layout once the launch settles (the engine README recommends app-start
48+ * init; the grammar JSON.parse burst (~535KB source) belongs in the
49+ * launch quiet window, not the first session-enter navigation). The
50+ * useHighlighter subscribe path keeps a deferred call as fallback. */
51+ export function initHighlighter ( ) {
4652 if ( initStarted ) return ;
4753 initStarted = true ;
4854 if ( ! isNativeEngineAvailable ( ) ) {
@@ -66,6 +72,11 @@ function ensureHighlighter() {
6672 langJson ,
6773 langJsonc ,
6874 langDiff ,
75+ // 65KB standalone — its embedded-language list is LAZY (no chained
76+ // grammar imports, unlike html). Nested fences inside a ```markdown
77+ // block highlight for whatever grammars are loaded above; the rest
78+ // render plain.
79+ langMarkdown ,
6980 ] ,
7081 engine : createNativeEngine ( ) ,
7182 } )
@@ -82,7 +93,9 @@ export function useHighlighter(): HighlighterCore | null {
8293 return useSyncExternalStore (
8394 ( cb ) => {
8495 listeners . add ( cb ) ;
85- ensureHighlighter ( ) ;
96+ // Fallback only — the root layout already ran this at app launch in
97+ // any real flow; idempotent no-op here.
98+ initHighlighter ( ) ;
8699 return ( ) => listeners . delete ( cb ) ;
87100 } ,
88101 ( ) => highlighter ,
@@ -118,6 +131,8 @@ const LANG_ALIASES: Record<string, string> = {
118131 jsonc : "jsonc" ,
119132 diff : "diff" ,
120133 patch : "diff" ,
134+ markdown : "markdown" ,
135+ md : "markdown" ,
121136} ;
122137
123138/** Info-strings that MEAN plain text — skip highlight without the
@@ -161,3 +176,33 @@ export function tokenizeCode(
161176 theme : scheme === "dark" ? "github-dark" : "github-light" ,
162177 } ) ;
163178}
179+
180+ // ─── Tokenization hook ──────────────────────────────────────────────────────
181+
182+ /** Tokenized lines for a code block, or null while the engine loads /
183+ * for unsupported languages (null → caller renders plain text with
184+ * identical metrics, so the colored flip is layout-neutral).
185+ *
186+ * Deliberately SYNC on the render frame. A cached + time-sliced variant
187+ * was built and A/B-tested (2026-06-12) and removed as unneeded: the
188+ * drawer-settle gate in the session screen keeps mount bursts out of
189+ * animation windows, and per-block tokenize (~1-3.5ms measured) is within
190+ * budget at our session scale. Revisit (see
191+ * memory/project_transcript_chunked_markdown_plan) if profiling ever
192+ * shows tokenize back on a hot path. */
193+ export function useCodeTokens (
194+ code : string ,
195+ infoString : string ,
196+ scheme : "light" | "dark" ,
197+ onTokenizeMs ?: ( ms : number ) => void ,
198+ ) : TokenLines | null {
199+ const hl = useHighlighter ( ) ;
200+ const langId = resolveLang ( infoString ) ;
201+ return useMemo ( ( ) => {
202+ if ( hl === null || langId === null ) return null ;
203+ const t0 = performance . now ( ) ;
204+ const lines = tokenizeCode ( hl , code , langId , scheme ) ;
205+ onTokenizeMs ?.( performance . now ( ) - t0 ) ;
206+ return lines ;
207+ } , [ hl , langId , code , scheme , onTokenizeMs ] ) ;
208+ }
0 commit comments