From 9aa6ea7b8084fe44282217cb6b8fc735e05b57c2 Mon Sep 17 00:00:00 2001 From: HKLHaoBin Date: Sun, 4 Oct 2026 22:23:13 +0800 Subject: [PATCH 1/5] feat(capsule): move streaming transcript to bounded rail --- .../crates/openless-core/src/shared_types.rs | 5 + .../app/scripts/insert-text-motion-h5.mjs | 65 +- .../app/scripts/windows-ui-config.test.mjs | 249 +++- .../app/src-tauri/src/commands/dictation.rs | 25 + openless-all/app/src-tauri/src/coordinator.rs | 91 +- .../src/coordinator/capsule_focus.rs | 24 +- .../src-tauri/src/coordinator/hotkey_loops.rs | 1 + openless-all/app/src-tauri/src/lib.rs | 114 +- .../src-tauri/src/tauri_coordinator_host.rs | 1042 ++++++++++++++++- .../app/src-tauri/src/tauri_events.rs | 20 +- openless-all/app/src-tauri/tauri.conf.json | 17 + openless-all/app/src/App.tsx | 6 + openless-all/app/src/components/Capsule.tsx | 239 ++-- .../app/src/components/CapsuleStyles.css | 104 +- .../app/src/components/LiveTranscriptPill.tsx | 246 +++- .../app/src/components/TypelessCapsule.tsx | 122 +- openless-all/app/src/lib/backendEvent.test.ts | 287 ++++- openless-all/app/src/lib/backendEvent.ts | 183 ++- .../app/src/lib/capsuleLayout.test.ts | 126 +- openless-all/app/src/lib/capsuleLayout.ts | 150 ++- .../app/src/lib/insertTextAnimation.test.ts | 100 ++ .../app/src/lib/insertTextAnimation.ts | 199 ++++ openless-all/app/src/lib/ipc/dictation.ts | 13 + openless-all/app/src/lib/ipc/index.ts | 2 + openless-all/app/src/lib/types.ts | 19 + openless-all/app/src/main.tsx | 2 + 26 files changed, 3135 insertions(+), 316 deletions(-) diff --git a/openless-all/app/crates/openless-core/src/shared_types.rs b/openless-all/app/crates/openless-core/src/shared_types.rs index 9da6eb7a3..924fe5740 100644 --- a/openless-all/app/crates/openless-core/src/shared_types.rs +++ b/openless-all/app/crates/openless-core/src/shared_types.rs @@ -2558,6 +2558,11 @@ pub enum CapsuleStyle { #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct CapsulePayload { + /// Session generation that owns this capsule frame. Keeping this on the + /// payload lets a Recording -> Recording transition reset the transcript + /// without depending on the order of backend:event and capsule:state. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub session_id: Option, pub state: CapsuleState, pub level: f32, // 0..1 RMS pub elapsed_ms: u64, diff --git a/openless-all/app/scripts/insert-text-motion-h5.mjs b/openless-all/app/scripts/insert-text-motion-h5.mjs index 7d698f8ce..873245144 100644 --- a/openless-all/app/scripts/insert-text-motion-h5.mjs +++ b/openless-all/app/scripts/insert-text-motion-h5.mjs @@ -6,9 +6,11 @@ import { basename, join, resolve } from 'node:path'; import { createServer } from 'vite'; const out = process.env.OPENLESS_MOTION_ARTIFACT_DIR || join(tmpdir(), 'openless-motion-h5'); +const serverPort = Number(process.env.OPENLESS_MOTION_H5_PORT || 1438); +const chromePort = Number(process.env.OPENLESS_MOTION_CHROME_PORT || 9437); mkdirSync(out, { recursive: true }); const server = await createServer({ - server: { host: '127.0.0.1', port: 1438, strictPort: true, watch: null }, + server: { host: '127.0.0.1', port: serverPort, strictPort: true, watch: null }, }); const fixture = resolve(`.insert-motion-h5-${process.pid}-${Date.now()}.html`); writeFileSync( @@ -28,14 +30,14 @@ writeFileSync( `, ); await server.listen(); -const pageUrl = 'http://127.0.0.1:1438/' + basename(fixture); +const pageUrl = `http://127.0.0.1:${serverPort}/` + basename(fixture); const chrome = spawn( process.env.CHROME_PATH || 'C:/Program Files/Google/Chrome/Application/chrome.exe', [ '--headless=new', '--no-first-run', '--no-default-browser-check', - '--remote-debugging-port=9437', + `--remote-debugging-port=${chromePort}`, '--window-size=900,500', '--user-data-dir=' + join(tmpdir(), 'openless-motion-' + Date.now()), pageUrl, @@ -48,7 +50,7 @@ try { let target; for (let i = 0; i < 100; i++) { try { - target = (await (await fetch('http://127.0.0.1:9437/json/list')).json()).find( + target = (await (await fetch(`http://127.0.0.1:${chromePort}/json/list`)).json()).find( (t) => t.type === 'page' && t.url === pageUrl, ); } catch {} @@ -83,11 +85,14 @@ try { throw Error(r.exceptionDetails.exception?.description || r.exceptionDetails.text); return r.result.value; }; - for (let i = 0; i < 150; i++) { - if (await evaluate('Boolean(window.ready)')) break; - await sleep(100); - } - assert(await evaluate('Boolean(window.ready)'), 'React fixture ready'); + const waitForFixture = async () => { + for (let i = 0; i < 150; i++) { + if (await evaluate("Boolean(window.ready && typeof window.show === 'function')")) return; + await sleep(100); + } + assert(false, 'React fixture ready'); + }; + await waitForFixture(); await sleep(800); const samples = await evaluate(`new Promise(resolve=>{ const frames=[]; const start=performance.now();window.show('帮我查找一下'); @@ -166,8 +171,9 @@ try { await send('Emulation.setEmulatedMedia', { features: [{ name: 'prefers-reduced-motion', value: 'reduce' }], }); + await evaluate(`window.ready=false; window.show=undefined`); await send('Page.reload'); - await sleep(1000); + await waitForFixture(); await evaluate(`window.show('减少动态效果验证')`); await sleep(80); assert( @@ -177,8 +183,9 @@ try { 'reduced motion immediate', ); await send('Emulation.setEmulatedMedia', { features: [] }); + await evaluate(`window.ready=false; window.show=undefined`); await send('Page.reload'); - await sleep(900); + await waitForFixture(); await evaluate(`window.show('帮我查找一下十六号的天气')`); await sleep(1000); // Optional real browser filmstrip; timestamps preserve capture cadence. @@ -220,8 +227,9 @@ try { } const shot = await send('Page.captureScreenshot', { format: 'png' }); writeFileSync(join(out, 'settled.png'), Buffer.from(shot.data, 'base64')); + await evaluate(`window.ready=false; window.show=undefined`); await send('Page.navigate', { url: pageUrl }); - await sleep(1200); + await waitForFixture(); await evaluate(`window.show('10六号')`); await sleep(1000); const correction = await evaluate(`new Promise(resolve=>{ @@ -241,12 +249,19 @@ try { for (const enabled of [true, false]) { await send('Emulation.setDeviceMetricsOverride', { width: style === 'typeless' ? 206 : 460, - height: style === 'typeless' ? 57 : style === 'classic' ? 100 : 180, + height: + style === 'typeless' + ? enabled + ? 65 + : 57 + : style === 'classic' + ? 172 + : 228, deviceScaleFactor: 1, mobile: false, }); await send('Page.navigate', { - url: `http://127.0.0.1:1438/?window=capsule&os=win&style=${style}&insertDemo=1&transcript=${enabled ? 1 : 0}&fontSize=20`, + url: `http://127.0.0.1:${serverPort}/?window=capsule&os=win&style=${style}&insertDemo=1&transcript=${enabled ? 1 : 0}&translation=${style === 'typeless' && enabled ? 1 : 0}&fontSize=20`, }); // Production demo simulates five recognition batches ending after 2440ms. for (let attempt = 0; attempt < 150; attempt += 1) { @@ -261,7 +276,16 @@ try { const pills=[...document.querySelectorAll('.ol-live-transcript-pill')]; const chars=[...document.querySelectorAll('.ol-live-transcript-char')]; const rect=pills[0]?.getBoundingClientRect(); - return {pills:pills.length,chars:chars.length,width:rect?.width,font:chars[0]?parseFloat(getComputedStyle(chars[0]).fontSize):0,buttons:document.querySelectorAll('button').length}; + const rail=document.querySelector('.ol-capsule-transcript-rail'); + const body=${JSON.stringify(style)}==='typeless'?document.querySelector('.ol-capsule-waveform'): ${JSON.stringify(style)}==='classic'?document.querySelector('.ol-capsule-pill'):document.querySelector('canvas'); + const hint=document.querySelector('.ol-typeless-translation'); + const hintRect=hint?.getBoundingClientRect(); + const hintStyle=hint?getComputedStyle(hint):null; + const railRect=rail?.getBoundingClientRect(); + const bodyRect=body?.getBoundingClientRect(); + const translationHintVisible=Boolean(hint && hintStyle?.opacity !== '0' && hintRect && hintRect.width > 0 && hintRect.top >= -0.5 && hintRect.bottom <= innerHeight + 0.5); + const translationHintNonOverlapping=Boolean(!translationHintVisible || !hintRect || !bodyRect || (hintRect.bottom <= bodyRect.top + 0.5 && (!railRect || hintRect.top >= railRect.bottom - 0.5))); + return {pills:pills.length,chars:chars.length,width:rect?.width,font:chars[0]?parseFloat(getComputedStyle(chars[0]).fontSize):0,buttons:document.querySelectorAll('button').length,rail:Boolean(rail),body:Boolean(body),railOutsideBody:Boolean(rail && body && !body.contains(rail)),translationHintVisible,translationHintNonOverlapping}; })()`); if (result.pills !== (enabled ? 1 : 0)) { writeFileSync( @@ -272,7 +296,18 @@ try { assert.equal(result.pills, enabled ? 1 : 0, `${style} display preference`); if (enabled) { assert(result.chars > 0, `${style} original text`); + assert(result.chars <= 64, `${style} motion glyph count stays bounded`); + assert( + result.rail && result.body && result.railOutsideBody, + `${style} keeps the capsule body beside, not inside, the transcript rail`, + ); assert(result.width <= (style === 'typeless' ? 206 : 460), `${style} bounded width`); + if (style === 'typeless') { + assert(result.translationHintVisible, 'typeless translation hint stays inside the native viewport'); + if (enabled) { + assert(result.translationHintNonOverlapping, 'typeless translation hint has a dedicated non-overlapping row'); + } + } assert( Math.abs(result.font * (style === 'typeless' ? 0.447 : 1) - 20) < 0.1, `${style} visible font size`, diff --git a/openless-all/app/scripts/windows-ui-config.test.mjs b/openless-all/app/scripts/windows-ui-config.test.mjs index ebd4d3307..e0c0abeda 100644 --- a/openless-all/app/scripts/windows-ui-config.test.mjs +++ b/openless-all/app/scripts/windows-ui-config.test.mjs @@ -15,8 +15,14 @@ function assertMatch(source, pattern, name) { const raw = await readFile(new URL('../src-tauri/tauri.conf.json', import.meta.url), 'utf-8'); const config = JSON.parse(raw); const capsuleWindow = config.app.windows.find((window) => window.label === 'capsule'); +const capsuleRailWindow = config.app.windows.find((window) => window.label === 'capsule-rail'); const mainWindow = config.app.windows.find((window) => window.label === 'main'); const libRs = await readFile(new URL('../src-tauri/src/lib.rs', import.meta.url), 'utf-8'); +const dictationRs = await readFile(new URL('../src-tauri/src/commands/dictation.rs', import.meta.url), 'utf-8'); +const sharedTypesRs = await readFile( + new URL('../crates/openless-core/src/shared_types.rs', import.meta.url), + 'utf-8', +); // Contract check over the union of the capsule subsystem compiled into the binary; native // window manipulation belongs to the explicit Tauri Host. const coordinatorRs = @@ -28,6 +34,7 @@ const coordinatorRs = )) + '\n' + (await readFile(new URL('../src-tauri/src/tauri_coordinator_host.rs', import.meta.url), 'utf-8')); +const tauriEventsRs = await readFile(new URL('../src-tauri/src/tauri_events.rs', import.meta.url), 'utf-8'); const capsuleTsx = await readFile( new URL('../src/components/Capsule.tsx', import.meta.url), 'utf-8', @@ -36,6 +43,24 @@ const capsuleLayoutTs = await readFile( new URL('../src/lib/capsuleLayout.ts', import.meta.url), 'utf-8', ); +const capsuleStylesCss = await readFile( + new URL('../src/components/CapsuleStyles.css', import.meta.url), + 'utf-8', +); +const typelessTsx = await readFile( + new URL('../src/components/TypelessCapsule.tsx', import.meta.url), + 'utf-8', +); +const liveTranscriptTsx = await readFile( + new URL('../src/components/LiveTranscriptPill.tsx', import.meta.url), + 'utf-8', +); +const backendEventTs = await readFile( + new URL('../src/lib/backendEvent.ts', import.meta.url), + 'utf-8', +); +const appTsx = await readFile(new URL('../src/App.tsx', import.meta.url), 'utf-8'); +const mainTsx = await readFile(new URL('../src/main.tsx', import.meta.url), 'utf-8'); const windowChromeTsx = await readFile( new URL('../src/components/WindowChrome.tsx', import.meta.url), 'utf-8', @@ -53,6 +78,9 @@ if (!capsuleWindow) { if (!mainWindow) { throw new Error('main window config missing'); } +if (!capsuleRailWindow) { + throw new Error('capsule rail window config missing'); +} assertEqual(capsuleWindow.width, 460, 'windows capsule config keeps the shared bootstrap width'); assertEqual(capsuleWindow.height, 180, 'windows capsule config keeps the shared bootstrap height'); assertEqual(capsuleWindow.transparent, true, 'capsule window should keep transparent visuals'); @@ -61,6 +89,11 @@ assertEqual( true, 'capsule window should stay above the focused app while recording', ); +assertEqual(capsuleRailWindow.transparent, true, 'capsule rail should be transparent'); +assertEqual(capsuleRailWindow.visible, false, 'capsule rail should start hidden'); +assertEqual(capsuleRailWindow.focus, false, 'capsule rail should never take focus'); +assertEqual(capsuleRailWindow.skipTaskbar, true, 'capsule rail should stay out of the taskbar'); +assertEqual(capsuleRailWindow.url, 'index.html?window=capsule-rail&os=win', 'capsule rail route'); assertEqual(mainWindow.decorations, true, 'windows main window should keep native decorations'); assertEqual( mainWindow.visible, @@ -188,7 +221,7 @@ assertMatch( assertMatch( capsuleLayoutTs, - /const stage = getCapsulePillMetrics\(os\);[\s\S]*?width: stage\.width,[\s\S]*?height: style === 'siri' \? stage\.height : style === 'classic' \? 100 : 128,[\s\S]*?horizontalInset: 0,[\s\S]*?bottomInset: style === 'siri' \? 0 : 16,[\s\S]*?badgeGap: 8,[\s\S]*?boxSizing: 'border-box'/, + /const stage = getCapsulePillMetrics\(os\);[\s\S]*?width: stage\.width,[\s\S]*?height: style === 'siri' \? SIRI_HOST_HEIGHT : style === 'classic' \? CLASSIC_HOST_HEIGHT : 128,[\s\S]*?horizontalInset: 0,[\s\S]*?bottomInset: style === 'siri' \? 0 : 16,[\s\S]*?badgeGap: 8,[\s\S]*?boxSizing: 'border-box'/, 'capsule host metrics should preserve Siri and reserve compact surfaces for Classic and Typeless', ); @@ -201,7 +234,7 @@ if ( } if ( - !/return\s*\(\s*[\s\S]*?ol-typeless-translation[\s\S]*?
/, + 'typeless translation should occupy its own layout row between the rail and capsule', +); +assertMatch( + capsuleStylesCss, + /\.ol-typeless-translation-row \{[\s\S]*?flex: 0 0 20px;[\s\S]*?height: 20px;[\s\S]*?\.ol-typeless-translation \{[\s\S]*?position: static;/, + 'typeless translation row should prevent badge overlap and clipping', +); + +const conditionalReturn = capsuleTsx.indexOf('if (insertFallback)'); +const transcriptVisibilityHook = capsuleTsx.indexOf('setCapsuleTranscriptVisible(transcriptVisible)'); +if (conditionalReturn < 0 || transcriptVisibilityHook < 0 || transcriptVisibilityHook > conditionalReturn) { + throw new Error('transcript visibility hook must run before every conditional return'); +} +assertMatch( + capsuleTsx, + /p\.state === 'idle' \|\| p\.state === 'done' \|\| p\.state === 'cancelled' \|\| p\.state === 'error'[\s\S]*?setLocalAsrText\(''\)/, + 'terminal capsule states should clear the transcript visibility source', +); +if (capsuleTsx.slice(conditionalReturn).includes('useEffect(')) { + throw new Error('capsule must not call hooks after entering a conditional return branch'); +} + assertMatch( libRs, - /fn capsule_window_bounds_for_style\(style: types::CapsuleStyle\)[\s\S]*?types::CapsuleStyle::Typeless => 206\.0,[\s\S]*?CapsuleStyle::Siri \| types::CapsuleStyle::Classic => 460\.0,[\s\S]*?types::CapsuleStyle::Siri => 180\.0,[\s\S]*?types::CapsuleStyle::Classic => 100\.0,[\s\S]*?types::CapsuleStyle::Typeless => 57\.0,[\s\S]*?bottom_inset: 0\.0,/, - 'native capsule bounds should match the frontend dimensions (typeless = 1/5 area of the original 460x128)', + /fn capsule_window_bounds_for_style_with_transcript_and_translation\([\s\S]*?types::CapsuleStyle::Typeless => 206\.0,[\s\S]*?types::CapsuleStyle::Siri \| types::CapsuleStyle::Classic => 460\.0,[\s\S]*?types::CapsuleStyle::Siri => \{[\s\S]*?228\.0[\s\S]*?180\.0[\s\S]*?types::CapsuleStyle::Classic => \{[\s\S]*?172\.0[\s\S]*?100\.0[\s\S]*?types::CapsuleStyle::Typeless => \{[\s\S]*?65\.0[\s\S]*?57\.0[\s\S]*?bottom_inset: 0\.0,/, + 'native capsule bounds should match the frontend dimensions including the translation row', +); + +assertMatch( + coordinatorRs, + /transcript_visible: AtomicBool[\s\S]*?set_transcript_visible\([\s\S]*?position_capsule_bottom_center_with_style_and_transcript/, + 'capsule hit area should track whether the transcript rail is visible', +); +assertMatch( + capsuleTsx, + /setCapsuleTranscriptVisible\(transcriptVisible\)/, + 'frontend should sync transcript visibility to the native hit area', +); + +assertMatch( + coordinatorRs, + /capsule_control_hit_rect[\s\S]*?WM_NCHITTEST[\s\S]*?HTTRANSPARENT/, + 'capsule hit testing should cover X/Y and use a real native input region', +); +assertMatch( + coordinatorRs, + /control_left:[\s\S]*?control_right:[\s\S]*?control_top:[\s\S]*?control_bottom:/, + 'capsule hit-test state should store both X and Y bounds', +); +assertMatch(coordinatorRs, /CreateRoundRectRgn[\s\S]*?SetWindowRgn/, 'capsule input region should be native'); +assertMatch(coordinatorRs, /WM_NCDESTROY[\s\S]*?remove\([\s\S]*?SetWindowLongPtrW/, 'subclass teardown should restore the old WndProc and remove its table entry'); +assertMatch(coordinatorRs, /SetLastError[\s\S]*?GetLastError/, 'WndProc installation should detect SetWindowLongPtrW failure'); +assertMatch(coordinatorRs, /DeleteObject[\s\S]*?SetWindowRgn failed/, 'failed regions should release their HRGN'); +assertMatch( + coordinatorRs, + /windows_hit_test_uses_x_and_y_and_translation_bounds/, + 'hit-test regression should cover X/Y and translation height', +); +assertMatch( + coordinatorRs, + /HIT_TEST_MODE_CARD[\s\S]*?set_card_hit_test_mode[\s\S]*?restore_capsule_hit_test_mode/, + 'cards should own and restore the shared window hit-test mode', +); +assertMatch(coordinatorRs, /set_card_hit_test_mode[\s\S]*?configure_card_hit_test/, 'card mode configures full client area'); +assertMatch(coordinatorRs, /refresh_card_hit_test/, 'card resize refreshes native hit area'); +assertMatch(coordinatorRs, /restore_capsule_geometry/, 'card dismissal restores style-aware capsule geometry'); +if (/capsule_window_bounds\(false\)|position_capsule_bottom_center\(false\)/.test( + await readFile(new URL('../src-tauri/src/coordinator.rs', import.meta.url), 'utf-8'), +)) { + throw new Error('card dismissal must not restore a hard-coded capsule geometry'); +} +assertMatch( + coordinatorRs, + /hit_test_mode\.load\(Ordering::SeqCst\) == HIT_TEST_MODE_CARD[\s\S]*?hide_transcript_overlay[\s\S]*?return Ok\(\(\)\)/, + 'card-owned windows must ignore transcript visibility layout updates', +); +assertMatch( + coordinatorRs, + /emit_to\("capsule-rail", "capsule:state"/, + 'coordinator state events should reach the dedicated transcript rail', +); +assertMatch( + tauriEventsRs, + /get_webview_window\("capsule-rail"\)[\s\S]*?emit\("capsule:state"/, + 'direct dictation state events should reach the dedicated transcript rail', +); +assertMatch(mainTsx, /const isCapsuleRail = windowKind === 'capsule-rail'/, 'rail route should be selected'); +assertMatch(appTsx, /isCapsuleRail[\s\S]*?CapsuleTranscriptOverlay/, 'rail route should render the overlay'); +assertMatch( + liveTranscriptTsx, + /export function CapsuleTranscriptOverlay[\s\S]*?listen\('capsule:state'[\s\S]*?listen\('backend:event'[\s\S]*?prefs:changed/, + 'rail should own state, transcript, and preference subscriptions', +); +assertMatch( + liveTranscriptTsx, + /const replaySnapshot[\s\S]*?getCapsuleSnapshot\(\)[\s\S]*?await replaySnapshot\(preferences\.capsuleStyle\)/, + 'rail should replay the native snapshot with bounded retry after its listeners are ready', +); +assertMatch( + liveTranscriptTsx, + /snapshotRetryDelays = \[0, 16, 32, 64, 128, 256\][\s\S]*?if \(!snapshot\) continue/, + 'rail replay should retry transient empty snapshots for a bounded interval', +); +assertMatch(liveTranscriptTsx, /visibleCapsuleTranscript\(text, enabled, state, selectionPolish\)/, 'rail should apply the same selection-polish visibility predicate'); +assertMatch( + coordinatorRs, + /set_ignore_cursor_events\(true\)[\s\S]*?set_size\([\s\S]*?set_position\([\s\S]*?show_capsule_window_for_recording\(&self\.app, &rail, false\)/, + 'transcript rail should be positioned as its own click-through window', +); +assertMatch( + coordinatorRs, + /fn capsule_transcript_rail_position[\s\S]*?CapsuleStyle::Siri[\s\S]*?CapsuleStyle::Classic[\s\S]*?CapsuleStyle::Typeless/, + 'rail positioning should have one style-aware pure geometry function', +); +assertMatch( + coordinatorRs, + /capsule_transcript_rail_position\(\s*style,\s*translation_active,\s*visible,?\s*\)[\s\S]*?position\.top_offset/, + 'rail native positioning should use body-relative top offset and translation state', +); +assertMatch(coordinatorRs, /SW_SHOWNOACTIVATE[\s\S]*?SWP_NOACTIVATE/, 'rail show should use the no-activate path'); +assertMatch( + coordinatorRs, + /no_activate failed: Win32 handle is null/, + 'no-activate display should validate HWND and fall back when SetWindowPos fails', +); +assertMatch( + coordinatorRs, + /if let Err\(error\) = unsafe \{[\s\S]*?SetWindowPos\([\s\S]*?return false/, + 'no-activate display should return false when SetWindowPos fails', +); +assertMatch( + dictationRs, + /get_capsule_snapshot\([\s\S]*?Result, String>[\s\S]*?capsule_snapshot/, + 'rail snapshot command and implementation must use the same CapsuleSnapshot DTO', +); +assertMatch( + coordinatorRs, + /struct CapsuleSnapshot[\s\S]*?payload: CapsulePayload[\s\S]*?transcript: String[\s\S]*?sequence: u64[\s\S]*?revision: u64/, + 'native rail snapshot should include the session-bearing payload, transcript, and both watermarks', +); +assertMatch( + sharedTypesRs, + /struct CapsulePayload[\s\S]*?session_id: Option/, + 'capsule payload should carry the session generation used by the rail replay', +); +assertMatch( + coordinatorRs, + /struct CapsuleSnapshotState[\s\S]*?payload_revision: u64[\s\S]*?snapshot: Mutex[\s\S]*?capsule_snapshot\([\s\S]*?self\.capsule\.snapshot\.lock\(\)\.clone\(\)[\s\S]*?payload_revision: snapshot\.payload_revision/, + 'native rail snapshot should keep payload and payload commit revision under one lock', +); +assertMatch( + coordinatorRs, + /record_backend_event\([\s\S]*?payload_projection_pending[\s\S]*?snapshot\.revision = event\.sequence/, + 'native rail snapshot should advance relevant transcript/session revisions', +); +assertMatch( + coordinatorRs, + /snapshot\.pending_payload_revision = Some\(event\.sequence\)/, + 'native rail snapshot should keep revisions pending until a matching payload commit', +); +assertMatch( + coordinatorRs, + /pending_payload_revision: Option[\s\S]*?fn can_commit_capsule_payload\([\s\S]*?captured_revision == current_revision[\s\S]*?commit_capsule_payload\([\s\S]*?can_commit_capsule_payload\([\s\S]*?snapshot\.payload = Some\(committed\)[\s\S]*?snapshot\.payload_revision = current_revision/, + 'capsule payload commits must use strict captured/pending revisions and reject stale frames', +); +assertMatch( + tauriEventsRs, + /record_backend_event\(&event\)[\s\S]*?forward_legacy_event\(/, + 'backend event watermark must advance before queued capsule payload presentation', +); +assertMatch( + liveTranscriptTsx, + /applyTranscriptSnapshot[\s\S]*?const nextTranscript[\s\S]*?setText\(nextTranscript\.text\)/, + 'rail replay should reject snapshots older than the live event watermark', +); +assertMatch( + backendEventTs, + /payloadRevision[\s\S]*?if \(payloadRevision !== snapshot\.revision\) return state/, + 'rail replay should reject snapshots whose payload and transcript revisions differ', ); assertMatch( diff --git a/openless-all/app/src-tauri/src/commands/dictation.rs b/openless-all/app/src-tauri/src/commands/dictation.rs index 712be93c9..5e8cc54ef 100644 --- a/openless-all/app/src-tauri/src/commands/dictation.rs +++ b/openless-all/app/src-tauri/src/commands/dictation.rs @@ -7,6 +7,14 @@ pub async fn get_startup_snapshot( core.start().await.map_err(|error| error.to_string()) } +#[cfg(not(mobile))] +#[tauri::command] +pub async fn get_capsule_snapshot( + coord: CoordinatorState<'_>, +) -> Result, String> { + Ok(coord.tauri_host().capsule_snapshot()) +} + async fn ensure_core_started(core: &openless_core::OpenLessBackend) -> Result<(), String> { if !core.snapshot().running { core.start().await.map_err(|error| error.to_string())?; @@ -48,6 +56,23 @@ pub async fn cancel_dictation( Ok(()) } +#[cfg(not(mobile))] +#[tauri::command] +pub async fn set_capsule_transcript_visible( + coord: CoordinatorState<'_>, + visible: bool, +) -> Result<(), String> { + let host = coord.tauri_host(); + let Some(capsule) = host.capsule_window() else { + return Ok(()); + }; + host.run_on_main_thread(move || { + if let Err(error) = capsule.set_transcript_visible(visible) { + log::warn!("[capsule] transcript visibility layout failed: {error}"); + } + }) +} + #[tauri::command] pub async fn handle_window_hotkey_event( coord: CoordinatorState<'_>, diff --git a/openless-all/app/src-tauri/src/coordinator.rs b/openless-all/app/src-tauri/src/coordinator.rs index af35194d5..c42fd2216 100644 --- a/openless-all/app/src-tauri/src/coordinator.rs +++ b/openless-all/app/src-tauri/src/coordinator.rs @@ -117,12 +117,13 @@ fn show_vocab_suggestion_card(inner: &Arc) { return; } inner.vocab_card_visible.store(true, Ordering::SeqCst); - // The card must be clickable, so cursor passthrough must be disabled. + // The card owns the shared HWND: its full client area is interactive, + // while the capsule rail is hidden until the card is dismissed. // Android has no capsule window and tauri's set_ignore_cursor_events // does not exist there (same handling as capsule_focus.rs). #[cfg(not(mobile))] - if let Err(e) = capsule.set_cursor_passthrough(false) { - log::warn!("[vocab-card] set_ignore_cursor_events(false) failed: {e}"); + if let Err(e) = capsule.set_card_hit_test_mode() { + log::warn!("[vocab-card] enabling card hit testing failed: {e}"); } if let Err(e) = capsule.set_size(VOCAB_CARD_WIDTH, height) { log::warn!("[vocab-card] resize failed: {e}"); @@ -132,6 +133,9 @@ fn show_vocab_suggestion_card(inner: &Arc) { { log::warn!("[vocab-card] position failed: {e}"); } + if let Err(e) = capsule.refresh_card_hit_test() { + log::warn!("[vocab-card] refreshing card hit testing failed: {e}"); + } // Positioning ditto: the dedup cache in `maybe_position_capsule_bottom_center` // only tracks "monitor + translation state" and knows nothing about // this move. Without invalidation, the next recording would judge @@ -170,28 +174,14 @@ fn hide_vocab_suggestion_card(inner: &Arc) { // Passthrough must be restored, or the capsule keeps blocking that // area at the bottom of the screen. #[cfg(not(mobile))] - if let Err(e) = capsule.set_cursor_passthrough(true) { - log::warn!("[vocab-card] restoring cursor passthrough failed: {e}"); - } - // Size must be restored too — the card shrank the window to its own - // size, and without this the next capsule would be squeezed into a - // 320×108 window, effectively invisible. - let bounds = crate::capsule_window_bounds(false); - if let Err(e) = capsule.set_size(bounds.width, bounds.height) { - log::warn!("[vocab-card] restoring capsule size failed: {e}"); - } - // Position must be restored too — the card moved the window to the - // bottom-right while the capsule sits bottom-center. Restoring size - // alone would put the next recording capsule in the bottom-right - // corner. - // - // Both cache invalidation and this repositioning are needed: - // invalidation guarantees the next emit_capsule recomputes even if this - // repositioning fails; repositioning guarantees the window is already - // in the right place even if some path shows it without emit_capsule. - capsule.invalidate_layout(); - if let Err(e) = capsule.position_capsule_bottom_center(false) { - log::warn!("[vocab-card] restoring capsule position failed: {e}"); + if let Err(e) = capsule.restore_capsule_hit_test_mode() { + log::warn!("[vocab-card] restoring capsule hit testing failed: {e}"); + } + // On mobile there is no native hit-test mode to restore, but the capsule + // still needs the cached style/rail geometry after the card resized it. + #[cfg(mobile)] + if let Err(e) = capsule.restore_capsule_geometry() { + log::warn!("[vocab-card] restoring capsule geometry failed: {e}"); } }); } @@ -264,8 +254,8 @@ fn show_insert_fallback_card(inner: &Arc, text: String, reason: &'static presentation_id, }; #[cfg(not(mobile))] - if let Err(e) = capsule.set_cursor_passthrough(false) { - log::warn!("[fallback-card] set_ignore_cursor_events(false) failed: {e}"); + if let Err(e) = capsule.set_card_hit_test_mode() { + log::warn!("[fallback-card] enabling card hit testing failed: {e}"); } if let Err(e) = capsule.set_size(FALLBACK_CARD_WIDTH, FALLBACK_CARD_INITIAL_HEIGHT) { log::warn!("[fallback-card] resize failed: {e}"); @@ -275,6 +265,9 @@ fn show_insert_fallback_card(inner: &Arc, text: String, reason: &'static { log::warn!("[fallback-card] position failed: {e}"); } + if let Err(e) = capsule.refresh_card_hit_test() { + log::warn!("[fallback-card] refreshing card hit testing failed: {e}"); + } // Positioning ditto: the dedup cache in `maybe_position_capsule_bottom_center` // only tracks "monitor + translation state" and knows nothing about // this move. Without invalidation the next recording would judge @@ -320,6 +313,9 @@ fn report_insert_fallback_card_height( if let Err(e) = capsule.position_fallback_card(FALLBACK_CARD_WIDTH, height) { log::warn!("[fallback-card] measured position failed: {e}"); } + if let Err(e) = capsule.refresh_card_hit_test() { + log::warn!("[fallback-card] refreshing measured card hit testing failed: {e}"); + } }) } @@ -348,24 +344,14 @@ fn hide_insert_fallback_card(inner: &Arc) { // Passthrough must be restored, or the capsule keeps blocking that // screen area. #[cfg(not(mobile))] - if let Err(e) = capsule.set_cursor_passthrough(true) { - log::warn!("[fallback-card] restoring cursor passthrough failed: {e}"); - } - // Size must be restored too — the card shrank the window to its own - // size, and without this the next capsule would be squeezed into a - // card-sized window, effectively invisible. - let bounds = crate::capsule_window_bounds(false); - if let Err(e) = capsule.set_size(bounds.width, bounds.height) { - log::warn!("[fallback-card] restoring capsule size failed: {e}"); - } - // Position must be restored too — the card moved the window to the - // bottom-right while the capsule sits bottom-center. Restoring size - // alone would put the next recording capsule in the bottom-right - // corner. Both cache invalidation and this repositioning are needed; - // see `hide_vocab_suggestion_card` for rationale. - capsule.invalidate_layout(); - if let Err(e) = capsule.position_capsule_bottom_center(false) { - log::warn!("[fallback-card] restoring capsule position failed: {e}"); + if let Err(e) = capsule.restore_capsule_hit_test_mode() { + log::warn!("[fallback-card] restoring capsule hit testing failed: {e}"); + } + // On mobile there is no native hit-test mode to restore, but the capsule + // still needs the cached style/rail geometry after the card resized it. + #[cfg(mobile)] + if let Err(e) = capsule.restore_capsule_geometry() { + log::warn!("[fallback-card] restoring capsule geometry failed: {e}"); } if let Some(payload) = deferred_capsule { // During the card, QA / Selection Polish still advance capsule @@ -899,7 +885,8 @@ impl Coordinator { } pub fn present_core_capsule(&self, payload: CapsulePayload) { - let _ = self.present_core_capsule_if_current(payload, None); + let captured_revision = self.inner.host.capsule_revision(); + let _ = self.present_core_capsule_at_revision(payload, None, captured_revision); } /// Selection-voice claims the shared capsule from hotkey Start through @@ -931,12 +918,22 @@ impl Coordinator { &self, payload: CapsulePayload, expected_epoch: Option, + ) -> Option { + let captured_revision = self.inner.host.capsule_revision(); + self.present_core_capsule_at_revision(payload, expected_epoch, captured_revision) + } + + fn present_core_capsule_at_revision( + &self, + payload: CapsulePayload, + expected_epoch: Option, + captured_revision: u64, ) -> Option { let state = payload.state; // Core already owns this frame's translation, readiness, and session // ownership; the window layer must not rebuild it from stale late // state, or cold starts and fast switches lose real feedback. - let epoch = emit_core_capsule(&self.inner, payload, expected_epoch)?; + let epoch = emit_core_capsule(&self.inner, payload, expected_epoch, captured_revision)?; if let Some(delay_ms) = core_capsule_hide_delay(state) { let inner = Arc::clone(&self.inner); self.inner.host.spawn(async move { diff --git a/openless-all/app/src-tauri/src/coordinator/capsule_focus.rs b/openless-all/app/src-tauri/src/coordinator/capsule_focus.rs index 0f10d044f..6896a7e1c 100644 --- a/openless-all/app/src-tauri/src/coordinator/capsule_focus.rs +++ b/openless-all/app/src-tauri/src/coordinator/capsule_focus.rs @@ -195,6 +195,7 @@ fn emit_capsule_with_context_locked( ) -> u64 { let dictation = inner.backend.snapshot().dictation; let payload = CapsulePayload { + session_id: dictation.session_id.map(|session| session.to_string()), state, level, elapsed_ms, @@ -212,7 +213,8 @@ fn emit_capsule_with_context_locked( selection_polish, capsule_style: inner.host.cached_capsule_style(), }; - emit_capsule_payload_locked(inner, payload) + let captured_revision = inner.host.capsule_revision(); + emit_capsule_payload_locked(inner, payload, captured_revision) } /// Core feedback enters the same native display outlet wholesale, including warming, @@ -221,12 +223,13 @@ pub(super) fn emit_core_capsule( inner: &Arc, payload: CapsulePayload, expected_epoch: Option, + captured_revision: u64, ) -> Option { emit_capsule_at_epoch( &inner.capsule_event_lock, &inner.capsule_event_epoch, expected_epoch, - || emit_capsule_payload_locked(inner, payload), + || emit_capsule_payload_locked(inner, payload, captured_revision), ) } @@ -252,7 +255,11 @@ pub(super) fn hide_core_capsule_if_current(inner: &Arc, expected_epoch: u ); } -fn emit_capsule_payload_locked(inner: &Arc, payload: CapsulePayload) -> u64 { +fn emit_capsule_payload_locked( + inner: &Arc, + payload: CapsulePayload, + captured_revision: u64, +) -> u64 { let state = payload.state; let selection_polish = payload.selection_polish; // Advance the epoch on every payload. That way an old selection-polish terminal timer is @@ -291,6 +298,16 @@ fn emit_capsule_payload_locked(inner: &Arc, payload: CapsulePayload) -> u // Keep the latest full feedback during a fallback card, even before the window handle is // validated; a re-show must not regress to an old preparing state. defer_capsule_payload_if_fallback_active(inner, &payload); + // Commit the replay frame before queueing any native window work. The + // snapshot lock therefore has one revision for payload and transcript even + // when run_on_main_thread is delayed. + if !inner + .host + .commit_capsule_payload(&payload, captured_revision) + { + log::debug!("[capsule] skipped stale payload commit at revision {captured_revision}"); + return event_epoch; + } let Some(capsule) = inner.host.capsule_window() else { return event_epoch; }; @@ -456,6 +473,7 @@ mod tests { fn payload(state: CapsuleState) -> CapsulePayload { CapsulePayload { + session_id: None, state, level: 0.0, elapsed_ms: 0, diff --git a/openless-all/app/src-tauri/src/coordinator/hotkey_loops.rs b/openless-all/app/src-tauri/src/coordinator/hotkey_loops.rs index 6803ac60a..8ee061c3d 100644 --- a/openless-all/app/src-tauri/src/coordinator/hotkey_loops.rs +++ b/openless-all/app/src-tauri/src/coordinator/hotkey_loops.rs @@ -2816,6 +2816,7 @@ pub(crate) mod windows_less_computer_tests { fixture_coordinator(crate::types::HotkeyMode::Toggle, std::time::Duration::ZERO); coordinator.inner.host.begin_insert_fallback_card(); let expected = CapsulePayload { + session_id: None, state: CapsuleState::Recording, level: 0.0, elapsed_ms: 0, diff --git a/openless-all/app/src-tauri/src/lib.rs b/openless-all/app/src-tauri/src/lib.rs index 264e7063e..dd72f4700 100644 --- a/openless-all/app/src-tauri/src/lib.rs +++ b/openless-all/app/src-tauri/src/lib.rs @@ -213,6 +213,7 @@ macro_rules! app_invoke_handler_desktop { () => { tauri::generate_handler![ commands::get_startup_snapshot, + commands::get_capsule_snapshot, commands::get_settings, commands::get_settings_snapshot, commands::update_setting_fields, @@ -309,6 +310,7 @@ macro_rules! app_invoke_handler_desktop { commands::start_dictation, commands::stop_dictation, commands::cancel_dictation, + commands::set_capsule_transcript_visible, coding_agent::commands::coding_agent_detect, coding_agent::commands::coding_agent_detect_opencode, coding_agent::commands::coding_agent_detect_cli, @@ -3265,10 +3267,36 @@ pub(crate) fn position_capsule_bottom_center( /// following auto-hide settings. pub(crate) fn position_capsule_bottom_center_with_style( window: &tauri::WebviewWindow, - _translation_active: bool, + translation_active: bool, + style: types::CapsuleStyle, +) -> tauri::Result<()> { + position_capsule_bottom_center_with_style_and_transcript( + window, + translation_active, + style, + true, + ) +} + +pub(crate) fn position_capsule_bottom_center_with_style_and_transcript( + window: &tauri::WebviewWindow, + translation_active: bool, style: types::CapsuleStyle, + transcript_visible: bool, ) -> tauri::Result<()> { - let bounds = capsule_window_bounds_for_style(style); + // Windows renders the transcript in a separate click-through overlay HWND. The capsule + // window itself therefore reserves only the body height; keeping the rail in this window + // would make its transparent gutter part of the input region again. + let native_transcript_visible = if cfg!(target_os = "windows") { + false + } else { + transcript_visible + }; + let bounds = capsule_window_bounds_for_style_with_transcript_and_translation( + style, + native_transcript_visible, + translation_active, + ); const EDGE_GAP: f64 = 12.0; // Typeless hugs the work-area bottom edge (the work area already excludes Dock / taskbar). let bottom_gap = match style { @@ -3371,6 +3399,25 @@ fn capsule_window_bounds(translation_active: bool) -> CapsuleWindowBounds { } fn capsule_window_bounds_for_style(style: types::CapsuleStyle) -> CapsuleWindowBounds { + capsule_window_bounds_for_style_with_transcript(style, true) +} + +fn capsule_window_bounds_for_style_with_transcript( + style: types::CapsuleStyle, + transcript_visible: bool, +) -> CapsuleWindowBounds { + capsule_window_bounds_for_style_with_transcript_and_translation( + style, + transcript_visible, + false, + ) +} + +fn capsule_window_bounds_for_style_with_transcript_and_translation( + style: types::CapsuleStyle, + transcript_visible: bool, + translation_active: bool, +) -> CapsuleWindowBounds { CapsuleWindowBounds { // The typeless window area is 1/5 of the original size (460×128); the frontend // scales content in sync with CSS zoom — see CapsuleStyles.css and @@ -3380,9 +3427,30 @@ fn capsule_window_bounds_for_style(style: types::CapsuleStyle) -> CapsuleWindowB types::CapsuleStyle::Siri | types::CapsuleStyle::Classic => 460.0, }, height: match style { - types::CapsuleStyle::Siri => 180.0, - types::CapsuleStyle::Classic => 100.0, - types::CapsuleStyle::Typeless => 57.0, + // The rail sits above the 180px Siri stage: 40px rail + 8px gap. + types::CapsuleStyle::Siri => { + if transcript_visible { + 228.0 + } else { + 180.0 + } + } + // Classic keeps the 52px pill, 40px rail, 8px gap, 16px bottom inset, + // and enough headroom for the translating badge. + types::CapsuleStyle::Classic => { + if cfg!(target_os = "windows") || transcript_visible { + 172.0 + } else { + 100.0 + } + } + types::CapsuleStyle::Typeless => { + if translation_active { + 65.0 + } else { + 57.0 + } + } }, bottom_inset: 0.0, } @@ -3402,7 +3470,9 @@ fn capsule_height_for_qa() -> f64 { mod tests { use super::{ bottom_center_position, capsule_height_for_qa, capsule_visual_height, - capsule_window_bounds, capsule_window_bounds_for_style, clamp_to_monitor, + capsule_window_bounds, capsule_window_bounds_for_style, + capsule_window_bounds_for_style_with_transcript, + capsule_window_bounds_for_style_with_transcript_and_translation, clamp_to_monitor, frame_contains_point, frame_distance_to_point_squared, logical_monitor_frame, parse_tray_style_pack_menu_id, resolve_tray_style_pack_id, rotate_log_if_too_large, tray_style_menu_enabled, tray_style_pack_menu_entries, tray_style_pack_menu_id, @@ -3596,11 +3666,11 @@ mod tests { } #[test] - fn capsule_window_bounds_match_voice_orb_stage() { + fn capsule_window_bounds_reserve_the_transcript_rail() { let bounds = capsule_window_bounds(false); assert_eq!( (bounds.width, bounds.height, bounds.bottom_inset), - (460.0, 180.0, 0.0) + (460.0, 228.0, 0.0) ); } @@ -3636,7 +3706,7 @@ mod tests { let bounds = capsule_window_bounds(true); assert_eq!( (bounds.width, bounds.height, bounds.bottom_inset), - (460.0, 180.0, 0.0) + (460.0, 228.0, 0.0) ); } @@ -3646,6 +3716,32 @@ mod tests { assert_eq!((bounds.width, bounds.height), (206.0, 57.0)); } + #[test] + fn typeless_capsule_window_reserves_translation_badge_row() { + let bounds = capsule_window_bounds_for_style_with_transcript_and_translation( + CapsuleStyle::Typeless, + true, + true, + ); + assert_eq!((bounds.width, bounds.height), (206.0, 65.0)); + } + + #[test] + fn classic_capsule_window_keeps_external_rail_host_height_on_windows() { + let bounds = capsule_window_bounds_for_style_with_transcript(CapsuleStyle::Classic, false); + assert_eq!( + (bounds.width, bounds.height), + ( + 460.0, + if cfg!(target_os = "windows") { + 172.0 + } else { + 100.0 + } + ) + ); + } + #[test] fn capsule_visual_height_keeps_the_qa_anchor_reference() { assert_eq!(capsule_visual_height(true), 140.0); diff --git a/openless-all/app/src-tauri/src/tauri_coordinator_host.rs b/openless-all/app/src-tauri/src/tauri_coordinator_host.rs index 39d958398..27d306b95 100644 --- a/openless-all/app/src-tauri/src/tauri_coordinator_host.rs +++ b/openless-all/app/src-tauri/src/tauri_coordinator_host.rs @@ -4,10 +4,15 @@ //! [`tauri::AppHandle`] and keeps window, main-thread and managed-state access //! out of the coordinator's business paths. +#[cfg(target_os = "windows")] +use std::collections::HashMap; use std::future::Future; use std::sync::atomic::{AtomicBool, AtomicU64, AtomicU8, Ordering}; use std::sync::Arc; +#[cfg(target_os = "windows")] +use std::sync::OnceLock; +use openless_core::{BackendEvent, BackendEventKind, DictationPhase, SelectionVoicePhase}; use parking_lot::Mutex; use tauri::{AppHandle, Emitter, Manager}; @@ -18,6 +23,414 @@ static CAPSULE_FIRST_SHOW_LOGGED: AtomicBool = AtomicBool::new(false); static CAPSULE_NO_ACTIVATE_FALLBACK_WARNED: AtomicBool = AtomicBool::new(false); static CAPSULE_WINDOW_MISSING_LOGGED: AtomicBool = AtomicBool::new(false); +const HIT_TEST_MODE_CAPSULE: u8 = 0; +const HIT_TEST_MODE_CARD: u8 = 1; + +#[derive(Clone, Copy, Debug, PartialEq)] +struct CapsuleTranscriptRailPosition { + width: f64, + height: f64, + /// Rail top relative to the capsule body HWND top, in logical/native pixels. + top_offset: f64, + gap: f64, +} + +/// Keep the standalone rail attached to the actual capsule body content. The +/// capsule HWND contains transparent headroom for each style, so subtracting +/// rail height from the HWND top puts Classic and Typeless rails too high. +fn capsule_transcript_rail_position( + style: CapsuleStyle, + translation_active: bool, + transcript_visible: bool, +) -> Option { + if !transcript_visible { + return None; + } + match style { + CapsuleStyle::Siri => { + let body_top = 0.0; + let height = 40.0; + let gap = 8.0; + Some(CapsuleTranscriptRailPosition { + width: 460.0, + height, + top_offset: body_top - gap - height, + gap, + }) + } + CapsuleStyle::Classic => { + const HOST_HEIGHT: f64 = 172.0; + const BODY_BOTTOM_INSET: f64 = 16.0; + const BODY_HEIGHT: f64 = 52.0; + const RAIL_HEIGHT: f64 = 52.0; + const RAIL_GAP: f64 = 8.0; + const BADGE_HEIGHT: f64 = 22.0; + const BADGE_GAP: f64 = 8.0; + let body_top = HOST_HEIGHT - BODY_BOTTOM_INSET - BODY_HEIGHT; + let height = 52.0; + let gap = RAIL_GAP; + let external_badge_lane = translation_active; + let top_offset = body_top - RAIL_GAP - RAIL_HEIGHT; + if external_badge_lane { + top_offset -= BADGE_HEIGHT + BADGE_GAP; + } + Some(CapsuleTranscriptRailPosition { + width: 460.0, + height, + top_offset, + gap, + }) + } + CapsuleStyle::Typeless => { + const ZOOM: f64 = 0.447; + let host_height = if translation_active { 65.0 } else { 57.0 }; + let body_bottom = host_height; + let body_top = + body_bottom - (64.0 * ZOOM) - if translation_active { 20.0 * ZOOM } else { 0.0 }; + let height = 52.0 * ZOOM; + let gap = 0.0; + Some(CapsuleTranscriptRailPosition { + width: 206.0, + height, + top_offset: body_top - gap - height, + gap, + }) + } + } +} + +#[cfg(target_os = "windows")] +#[derive(Clone, Copy)] +struct CapsuleHitTestEntry { + previous_proc: isize, + control_left: i32, + control_right: i32, + control_top: i32, + control_bottom: i32, +} + +#[cfg(target_os = "windows")] +static CAPSULE_HIT_TEST_ENTRIES: OnceLock>> = + OnceLock::new(); + +#[cfg(target_os = "windows")] +fn capsule_hit_test_entries() -> &'static Mutex> { + CAPSULE_HIT_TEST_ENTRIES.get_or_init(|| Mutex::new(HashMap::new())) +} + +#[cfg(target_os = "windows")] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct CapsuleHitTestRect { + left: i32, + right: i32, + top: i32, + bottom: i32, +} + +#[cfg(target_os = "windows")] +fn capsule_control_hit_rect( + style: CapsuleStyle, + transcript_visible: bool, + translation_active: bool, + window_width: i32, + window_height: i32, +) -> Option { + if style == CapsuleStyle::Siri || window_width <= 0 || window_height <= 0 { + return None; + } + let ( + logical_host_width, + logical_host_height, + logical_body_width, + logical_body_height, + logical_bottom_inset, + ) = match style { + // Classic transcript is hosted by the separate click-through rail HWND on + // Windows. The capsule HWND therefore owns only its 100px body, regardless + // of whether the rail is visible. + CapsuleStyle::Classic => (460.0, 100.0, 196.0, 52.0, 16.0), + // Typeless content is scaled by zoom: 64px CSS × 0.447. The native window may be + // 57px or 65px high when the dedicated translation row is present. + CapsuleStyle::Typeless => ( + 206.0, + if translation_active { 65.0 } else { 57.0 }, + 232.0 * 0.447, + 64.0 * 0.447, + 0.0, + ), + CapsuleStyle::Siri => unreachable!(), + }; + let _ = transcript_visible; + let scale = + (window_width as f64 / logical_host_width).min(window_height as f64 / logical_host_height); + let body_width = (logical_body_width * scale).round() as i32; + let body_height = (logical_body_height * scale).round() as i32; + let bottom_inset = (logical_bottom_inset * scale).round() as i32; + let control_left = ((window_width as f64 - body_width as f64) / 2.0).round() as i32; + let control_bottom = window_height - bottom_inset; + Some(CapsuleHitTestRect { + left: control_left.max(0), + right: (control_left + body_width).min(window_width), + top: (control_bottom - body_height).max(0), + bottom: control_bottom.max(0), + }) +} + +#[cfg(target_os = "windows")] +fn capsule_card_hit_rect(window_width: i32, window_height: i32) -> CapsuleHitTestRect { + CapsuleHitTestRect { + left: 0, + right: window_width.max(0), + top: 0, + bottom: window_height.max(0), + } +} + +#[cfg(target_os = "windows")] +unsafe extern "system" fn capsule_hit_test_window_proc( + hwnd: windows::Win32::Foundation::HWND, + msg: u32, + wparam: windows::Win32::Foundation::WPARAM, + lparam: windows::Win32::Foundation::LPARAM, +) -> windows::Win32::Foundation::LRESULT { + use windows::Win32::Foundation::RECT; + use windows::Win32::UI::WindowsAndMessaging::{ + CallWindowProcW, DefWindowProcW, GetWindowRect, SetWindowLongPtrW, GWLP_WNDPROC, + HTTRANSPARENT, WM_NCDESTROY, WM_NCHITTEST, + }; + + if msg == WM_NCDESTROY { + let previous_proc = capsule_hit_test_entries() + .lock() + .remove(&(hwnd.0 as isize)) + .map(|entry| entry.previous_proc) + .unwrap_or_default(); + if previous_proc != 0 { + let previous: windows::Win32::UI::WindowsAndMessaging::WNDPROC = + std::mem::transmute(previous_proc); + let _ = SetWindowLongPtrW(hwnd, GWLP_WNDPROC, previous_proc); + return CallWindowProcW(previous, hwnd, msg, wparam, lparam); + } + return DefWindowProcW(hwnd, msg, wparam, lparam); + } + + if msg == WM_NCHITTEST { + let key = hwnd.0 as isize; + let entry = capsule_hit_test_entries().lock().get(&key).copied(); + if let Some(entry) = entry { + let screen_x = (lparam.0 as i32 & 0xffff) as u16 as i16 as i32; + let screen_y = ((lparam.0 >> 16) & 0xffff) as u16 as i16 as i32; + let mut rect = RECT::default(); + if GetWindowRect(hwnd, &mut rect).is_ok() { + let client_x = screen_x - rect.left; + let client_y = screen_y - rect.top; + if client_x < entry.control_left + || client_x >= entry.control_right + || client_y < entry.control_top + || client_y >= entry.control_bottom + { + return windows::Win32::Foundation::LRESULT(HTTRANSPARENT as isize); + } + } + } + } + + let previous_proc = capsule_hit_test_entries() + .lock() + .get(&(hwnd.0 as isize)) + .map(|entry| entry.previous_proc) + .unwrap_or_default(); + if previous_proc != 0 { + let previous: windows::Win32::UI::WindowsAndMessaging::WNDPROC = + std::mem::transmute(previous_proc); + CallWindowProcW(previous, hwnd, msg, wparam, lparam) + } else { + DefWindowProcW(hwnd, msg, wparam, lparam) + } +} + +#[cfg(target_os = "windows")] +fn install_capsule_hit_test_proc( + hwnd: windows::Win32::Foundation::HWND, + rect: CapsuleHitTestRect, +) -> tauri::Result<()> { + use windows::Win32::Foundation::{GetLastError, SetLastError, ERROR_SUCCESS}; + use windows::Win32::UI::WindowsAndMessaging::{SetWindowLongPtrW, GWLP_WNDPROC}; + + if hwnd.0.is_null() { + return Err(tauri::Error::Anyhow(anyhow::anyhow!( + "cannot install hit-test proc for a null HWND" + ))); + } + + let key = hwnd.0 as isize; + let mut entries = capsule_hit_test_entries().lock(); + if let Some(entry) = entries.get_mut(&key) { + entry.control_left = rect.left; + entry.control_right = rect.right; + entry.control_top = rect.top; + entry.control_bottom = rect.bottom; + return Ok(()); + } + unsafe { SetLastError(ERROR_SUCCESS) }; + let previous = unsafe { + SetWindowLongPtrW( + hwnd, + GWLP_WNDPROC, + capsule_hit_test_window_proc as usize as isize, + ) + }; + if previous == 0 && unsafe { GetLastError() } != ERROR_SUCCESS { + return Err(tauri::Error::Anyhow(anyhow::anyhow!( + "SetWindowLongPtrW(GWLP_WNDPROC) failed" + ))); + } + entries.insert( + key, + CapsuleHitTestEntry { + previous_proc: previous, + control_left: rect.left, + control_right: rect.right, + control_top: rect.top, + control_bottom: rect.bottom, + }, + ); + Ok(()) +} + +#[cfg(target_os = "windows")] +fn set_capsule_input_region( + hwnd: windows::Win32::Foundation::HWND, + rect: CapsuleHitTestRect, +) -> tauri::Result<()> { + use windows::Win32::Foundation::BOOL; + use windows::Win32::Graphics::Gdi::{CreateRoundRectRgn, DeleteObject, SetWindowRgn, HGDIOBJ}; + + if hwnd.0.is_null() { + return Err(tauri::Error::Anyhow(anyhow::anyhow!( + "cannot set input region for a null HWND" + ))); + } + + let radius = ((rect.bottom - rect.top) / 2).max(1); + let region = + unsafe { CreateRoundRectRgn(rect.left, rect.top, rect.right, rect.bottom, radius, radius) }; + if region.0.is_null() { + return Err(tauri::Error::Anyhow(anyhow::anyhow!( + "CreateRoundRectRgn returned a null region" + ))); + } + let result = unsafe { SetWindowRgn(hwnd, region, BOOL(1)) }; + if result == 0 { + unsafe { + let _ = DeleteObject(HGDIOBJ(region.0)); + } + return Err(tauri::Error::Anyhow(anyhow::anyhow!("SetWindowRgn failed"))); + } + Ok(()) +} + +#[cfg(target_os = "windows")] +fn clear_capsule_input_region(hwnd: windows::Win32::Foundation::HWND) { + use windows::Win32::Foundation::BOOL; + use windows::Win32::Graphics::Gdi::{SetWindowRgn, HRGN}; + + if hwnd.0.is_null() { + return; + } + unsafe { + let _ = SetWindowRgn(hwnd, HRGN::default(), BOOL(1)); + } +} + +#[cfg(target_os = "windows")] +fn configure_capsule_hit_test( + window: &tauri::WebviewWindow, + style: CapsuleStyle, + transcript_visible: bool, + translation_active: bool, +) -> tauri::Result<()> { + use raw_window_handle::{HasWindowHandle, RawWindowHandle}; + use windows::Win32::Foundation::RECT; + use windows::Win32::UI::WindowsAndMessaging::GetClientRect; + + let Ok(handle) = window.window_handle() else { + return Ok(()); + }; + let RawWindowHandle::Win32(raw) = handle.as_raw() else { + return Ok(()); + }; + let hwnd = windows::Win32::Foundation::HWND(raw.hwnd.get() as *mut _); + if hwnd.0.is_null() { + return Err(tauri::Error::Anyhow(anyhow::anyhow!( + "capsule window returned a null HWND" + ))); + } + let mut client = RECT::default(); + GetClientRect(hwnd, &mut client).map_err(|error| tauri::Error::Anyhow(error.into()))?; + let Some(rect) = capsule_control_hit_rect( + style, + transcript_visible, + translation_active, + client.right - client.left, + client.bottom - client.top, + ) else { + return Ok(()); + }; + set_capsule_input_region(hwnd, rect)?; + if let Err(error) = install_capsule_hit_test_proc(hwnd, rect) { + clear_capsule_input_region(hwnd); + return Err(error); + } + Ok(()) +} + +#[cfg(target_os = "windows")] +fn configure_card_hit_test( + window: &tauri::WebviewWindow, +) -> tauri::Result<()> { + use raw_window_handle::{HasWindowHandle, RawWindowHandle}; + use windows::Win32::Foundation::RECT; + use windows::Win32::UI::WindowsAndMessaging::GetClientRect; + + let Ok(handle) = window.window_handle() else { + return Ok(()); + }; + let RawWindowHandle::Win32(raw) = handle.as_raw() else { + return Ok(()); + }; + let hwnd = windows::Win32::Foundation::HWND(raw.hwnd.get() as *mut _); + if hwnd.0.is_null() { + return Err(tauri::Error::Anyhow(anyhow::anyhow!( + "card window returned a null HWND" + ))); + } + let mut client = RECT::default(); + GetClientRect(hwnd, &mut client).map_err(|error| tauri::Error::Anyhow(error.into()))?; + let rect = capsule_card_hit_rect(client.right - client.left, client.bottom - client.top); + clear_capsule_input_region(hwnd); + install_capsule_hit_test_proc(hwnd, rect) +} + +#[cfg(target_os = "windows")] +fn clear_capsule_input_region_for_window( + window: &tauri::WebviewWindow, +) -> tauri::Result<()> { + use raw_window_handle::{HasWindowHandle, RawWindowHandle}; + + let Ok(handle) = window.window_handle() else { + return Ok(()); + }; + let RawWindowHandle::Win32(raw) = handle.as_raw() else { + return Ok(()); + }; + let hwnd = windows::Win32::Foundation::HWND(raw.hwnd.get() as *mut _); + if !hwnd.0.is_null() { + clear_capsule_input_region(hwnd); + } + Ok(()) +} + #[derive(Clone, Copy, Debug, PartialEq, Eq)] enum CapsuleShowStrategy { NoActivate, @@ -91,9 +504,13 @@ fn show_capsule_window_no_activate( return false; }; let hwnd = HWND(raw.hwnd.get() as *mut _); + if hwnd.0.is_null() { + log::warn!("[capsule] no_activate failed: Win32 handle is null"); + return false; + } let _ = unsafe { ShowWindow(hwnd, SW_SHOWNOACTIVATE) }; - let _ = unsafe { + if let Err(error) = unsafe { SetWindowPos( hwnd, HWND_TOPMOST, @@ -103,7 +520,10 @@ fn show_capsule_window_no_activate( 0, SWP_NOMOVE | SWP_NOSIZE | SWP_NOACTIVATE | SWP_SHOWWINDOW, ) - }; + } { + log::warn!("[capsule] no_activate failed: SetWindowPos returned {error}"); + return false; + } true } @@ -235,6 +655,7 @@ fn capsule_window_action( #[derive(Clone, Copy, Debug, PartialEq, Eq)] struct CapsuleLayoutState { translation_active: bool, + transcript_visible: bool, style: CapsuleStyle, monitor_x: i32, monitor_y: i32, @@ -247,14 +668,48 @@ struct CapsuleLayoutState { scale_bits: u64, } +#[derive(Clone, Debug, Default)] +struct CapsuleSnapshotState { + payload: Option, + payload_revision: u64, + pending_payload_revision: Option, + session_id: Option, + sequence: u64, + revision: u64, + text: String, +} + +fn can_commit_capsule_payload( + current_revision: u64, + pending_revision: Option, + captured_revision: u64, +) -> bool { + captured_revision == current_revision + && pending_revision.map_or(true, |pending| pending == current_revision) +} + +#[derive(Clone, Debug, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct CapsuleSnapshot { + #[serde(flatten)] + pub(crate) payload: CapsulePayload, + pub(crate) transcript: String, + pub(crate) sequence: u64, + pub(crate) revision: u64, + pub(crate) payload_revision: u64, +} + struct CapsuleWindowState { layout: Mutex>, + capsule_visible: AtomicBool, cursor_passthrough: AtomicBool, + hit_test_mode: AtomicU8, + transcript_visible: AtomicBool, style: AtomicU8, fallback_card_visible: AtomicBool, fallback_presentation_id: AtomicU64, deferred_payload: Mutex>, - last_payload: Mutex>, + snapshot: Mutex, layout_watch_active: AtomicBool, layout_watch_epoch: AtomicU64, } @@ -263,12 +718,15 @@ impl Default for CapsuleWindowState { fn default() -> Self { Self { layout: Mutex::new(None), + capsule_visible: AtomicBool::new(false), cursor_passthrough: AtomicBool::new(true), + hit_test_mode: AtomicU8::new(HIT_TEST_MODE_CAPSULE), + transcript_visible: AtomicBool::new(false), style: AtomicU8::new(0), fallback_card_visible: AtomicBool::new(false), fallback_presentation_id: AtomicU64::new(0), deferred_payload: Mutex::new(None), - last_payload: Mutex::new(None), + snapshot: Mutex::new(CapsuleSnapshotState::default()), layout_watch_active: AtomicBool::new(false), layout_watch_epoch: AtomicU64::new(0), } @@ -346,6 +804,50 @@ impl TauriCapsuleWindow { self.app.get_webview_window("capsule") } + fn rail_window(&self) -> Option { + self.app.get_webview_window("capsule-rail") + } + + fn hide_transcript_overlay(&self) { + if let Some(window) = self.rail_window() { + let _ = window.hide(); + } + } + + #[cfg(target_os = "windows")] + fn position_transcript_overlay( + &self, + capsule: &tauri::WebviewWindow, + style: CapsuleStyle, + translation_active: bool, + visible: bool, + ) -> tauri::Result<()> { + let Some(rail) = self.rail_window() else { + return Ok(()); + }; + if !visible { + rail.hide()?; + return Ok(()); + } + let Some(position) = capsule_transcript_rail_position(style, translation_active, visible) + else { + rail.hide()?; + return Ok(()); + }; + let scale = capsule.scale_factor().unwrap_or(1.0); + let main_position = capsule.outer_position()?; + rail.set_ignore_cursor_events(true)?; + rail.set_size(tauri::LogicalSize::new(position.width, position.height))?; + rail.set_position(tauri::PhysicalPosition::new( + main_position.x, + main_position.y + (position.top_offset * scale).round() as i32, + ))?; + // The rail is a sibling overlay and must never steal focus from the + // application under the capsule, just like the main capsule HWND. + show_capsule_window_for_recording(&self.app, &rail, false); + Ok(()) + } + pub(crate) fn is_available_for(&self, state: CapsuleState) -> bool { let available = self.window().is_some(); if !available && !CAPSULE_WINDOW_MISSING_LOGGED.swap(true, Ordering::SeqCst) { @@ -379,6 +881,26 @@ impl TauriCapsuleWindow { #[cfg(not(mobile))] pub(crate) fn set_cursor_passthrough(&self, passthrough: bool) -> tauri::Result<()> { if let Some(window) = self.window() { + #[cfg(target_os = "windows")] + if passthrough { + clear_capsule_input_region_for_window(&window)?; + } else if self.state.hit_test_mode.load(Ordering::SeqCst) == HIT_TEST_MODE_CARD { + configure_card_hit_test(&window)?; + } else { + let translation_active = self + .state + .snapshot + .lock() + .payload + .as_ref() + .is_some_and(|payload| payload.translation); + configure_capsule_hit_test( + &window, + self.state.cached_style(), + self.state.transcript_visible.load(Ordering::SeqCst), + translation_active, + )?; + } window.set_ignore_cursor_events(passthrough)?; self.state .cursor_passthrough @@ -387,12 +909,69 @@ impl TauriCapsuleWindow { Ok(()) } + /// Rebuild the real client hit region after a card resize. The card owns the + /// shared HWND, so its whole current client area must be interactive; using + /// the previous capsule region would make Copy/Dismiss intermittently miss. + pub(crate) fn refresh_card_hit_test(&self) -> tauri::Result<()> { + if self.state.hit_test_mode.load(Ordering::SeqCst) != HIT_TEST_MODE_CARD { + return Ok(()); + } + #[cfg(target_os = "windows")] + if let Some(window) = self.window() { + configure_card_hit_test(&window)?; + } + Ok(()) + } + + #[cfg(not(mobile))] + pub(crate) fn set_card_hit_test_mode(&self) -> tauri::Result<()> { + self.state + .hit_test_mode + .store(HIT_TEST_MODE_CARD, Ordering::SeqCst); + self.hide_transcript_overlay(); + if let Some(window) = self.window() { + window.set_ignore_cursor_events(false)?; + self.state.cursor_passthrough.store(false, Ordering::SeqCst); + #[cfg(target_os = "windows")] + configure_card_hit_test(&window)?; + } + Ok(()) + } + + #[cfg(not(mobile))] + pub(crate) fn restore_capsule_hit_test_mode(&self) -> tauri::Result<()> { + self.state + .hit_test_mode + .store(HIT_TEST_MODE_CAPSULE, Ordering::SeqCst); + self.set_cursor_passthrough(true)?; + // Restoring a card must also restore the capsule geometry/rail state that + // was current before the card took ownership of the window. + self.invalidate_layout(); + if let Some(window) = self.window() { + let translation = self + .state + .snapshot + .lock() + .payload + .as_ref() + .is_some_and(|payload| payload.translation); + self.maybe_position_capsule_bottom_center( + &window, + translation, + self.state.cached_style(), + ); + } + Ok(()) + } + pub(crate) fn invalidate_layout(&self) { *self.state.layout.lock() = None; } pub(crate) fn hide(&self) -> tauri::Result<()> { + self.state.capsule_visible.store(false, Ordering::SeqCst); self.stop_layout_watch(); + self.hide_transcript_overlay(); if let Some(window) = self.window() { window.hide()?; } @@ -446,15 +1025,81 @@ impl TauriCapsuleWindow { pub(crate) fn position_capsule_bottom_center(&self, translation: bool) -> tauri::Result<()> { if let Some(window) = self.window() { - crate::position_capsule_bottom_center_with_style( + crate::position_capsule_bottom_center_with_style_and_transcript( &window, translation, self.state.cached_style(), + self.state.transcript_visible.load(Ordering::SeqCst), )?; } Ok(()) } + /// Restore the capsule window from the cached style and latest payload. + /// Card dismissal must not guess Siri geometry or discard the rail state. + pub(crate) fn restore_capsule_geometry(&self) -> tauri::Result<()> { + self.invalidate_layout(); + if let Some(window) = self.window() { + let translation = self + .state + .snapshot + .lock() + .payload + .as_ref() + .is_some_and(|payload| payload.translation); + self.maybe_position_capsule_bottom_center( + &window, + translation, + self.state.cached_style(), + ); + } + Ok(()) + } + + pub(crate) fn set_transcript_visible(&self, visible: bool) -> tauri::Result<()> { + if !self.state.capsule_visible.load(Ordering::SeqCst) { + self.state.transcript_visible.store(false, Ordering::SeqCst); + self.hide_transcript_overlay(); + return Ok(()); + } + self.state + .transcript_visible + .store(visible, Ordering::SeqCst); + if self.state.hit_test_mode.load(Ordering::SeqCst) == HIT_TEST_MODE_CARD { + // Card geometry owns this shared window until dismissal. The card webview may + // report transcriptVisible=false, but that must not restore capsule bounds. + self.hide_transcript_overlay(); + return Ok(()); + } + self.state.layout.lock().take(); + if let Some(window) = self.window() { + let translation = self + .state + .snapshot + .lock() + .payload + .as_ref() + .is_some_and(|payload| payload.translation); + self.maybe_position_capsule_bottom_center( + &window, + translation, + self.state.cached_style(), + ); + #[cfg(target_os = "windows")] + if self.state.hit_test_mode.load(Ordering::SeqCst) != HIT_TEST_MODE_CARD + && !self.state.cursor_passthrough.load(Ordering::SeqCst) + { + configure_capsule_hit_test( + &window, + self.state.cached_style(), + visible, + translation, + )?; + } + } + Ok(()) + } + fn layout_snapshot( &self, window: &tauri::WebviewWindow, @@ -466,6 +1111,7 @@ impl TauriCapsuleWindow { if let Some(mon) = crate::foreground_window_monitor() { return Some(CapsuleLayoutState { translation_active, + transcript_visible: self.state.transcript_visible.load(Ordering::SeqCst), style, monitor_x: mon.left, monitor_y: mon.top, @@ -484,6 +1130,7 @@ impl TauriCapsuleWindow { if let Some(mon) = crate::capsule_target_monitor(window) { return Some(CapsuleLayoutState { translation_active, + transcript_visible: self.state.transcript_visible.load(Ordering::SeqCst), style, monitor_x: mon.physical_x, monitor_y: mon.physical_y, @@ -500,6 +1147,7 @@ impl TauriCapsuleWindow { let monitor = window.current_monitor().ok().flatten()?; Some(CapsuleLayoutState { translation_active, + transcript_visible: self.state.transcript_visible.load(Ordering::SeqCst), style, monitor_x: monitor.position().x, monitor_y: monitor.position().y, @@ -522,17 +1170,49 @@ impl TauriCapsuleWindow { let Some(next) = self.layout_snapshot(window, translation_active, style) else { return; }; - if self.state.layout.lock().as_ref() == Some(&next) { - return; - } - if crate::position_capsule_bottom_center_with_style(window, translation_active, style) + let layout_changed = self.state.layout.lock().as_ref() != Some(&next); + if layout_changed + && crate::position_capsule_bottom_center_with_style_and_transcript( + window, + translation_active, + style, + self.state.transcript_visible.load(Ordering::SeqCst), + ) .is_ok() { *self.state.layout.lock() = Some(next); } + #[cfg(target_os = "windows")] + if self.state.capsule_visible.load(Ordering::SeqCst) { + if let Err(error) = self.position_transcript_overlay( + window, + style, + translation_active, + self.state.transcript_visible.load(Ordering::SeqCst), + ) { + log::warn!("[capsule] transcript overlay positioning failed: {error}"); + } + } else { + self.hide_transcript_overlay(); + } + #[cfg(target_os = "windows")] + if layout_changed + && self.state.hit_test_mode.load(Ordering::SeqCst) != HIT_TEST_MODE_CARD + && !self.state.cursor_passthrough.load(Ordering::SeqCst) + { + if let Err(error) = configure_capsule_hit_test( + window, + style, + self.state.transcript_visible.load(Ordering::SeqCst), + translation_active, + ) { + log::warn!("[capsule] configure client-band hit testing failed: {error}"); + } + } } pub(crate) fn show_for_recording(&self, reassert_spaces: bool) { + self.state.capsule_visible.store(true, Ordering::SeqCst); if let Some(window) = self.window() { show_capsule_window_for_recording(&self.app, &window, reassert_spaces); } @@ -574,7 +1254,7 @@ impl TauriCapsuleWindow { capsule.stop_layout_watch(); return; } - let payload = capsule.state.last_payload.lock().clone(); + let payload = capsule.state.snapshot.lock().payload.clone(); if let Some(payload) = payload { capsule.maybe_position_capsule_bottom_center( &window, @@ -615,7 +1295,7 @@ impl TauriCapsuleWindow { { return; } - let payload = self.state.last_payload.lock().clone(); + let payload = self.state.snapshot.lock().payload.clone(); if let Some(payload) = payload { let style = self.state.cached_style(); self.maybe_position_capsule_bottom_center(&window, payload.translation, style); @@ -631,7 +1311,6 @@ impl TauriCapsuleWindow { reassert_spaces: bool, ) { self.state.cache_style(style); - *self.state.last_payload.lock() = Some(payload.clone()); let Some(window) = self.window() else { return; }; @@ -646,6 +1325,17 @@ impl TauriCapsuleWindow { return; } + match action { + CapsuleWindowAction::ShowCapsule => { + self.state.capsule_visible.store(true, Ordering::SeqCst); + } + CapsuleWindowAction::HideCapsule => { + self.state.capsule_visible.store(false, Ordering::SeqCst); + self.hide_transcript_overlay(); + } + CapsuleWindowAction::PreserveFallbackCard => unreachable!(), + } + self.maybe_position_capsule_bottom_center(&window, payload.translation, style); self.update_cursor_for_payload(payload, style); @@ -675,7 +1365,7 @@ impl TauriCapsuleWindow { ); } hide_capsule_window_if_present(); - let _ = window.hide(); + let _ = self.hide(); } } } @@ -724,6 +1414,218 @@ impl TauriCoordinatorHost { self.capsule.cached_style() } + pub(crate) fn capsule_snapshot(&self) -> Option { + let snapshot = self.capsule.snapshot.lock().clone(); + if snapshot.pending_payload_revision.is_some() + || snapshot.payload_revision != snapshot.revision + { + // A rail ready replay must never observe a payload from one + // revision together with transcript/session data from another. + return None; + } + let mut payload = snapshot.payload?; + payload.session_id = snapshot.session_id.clone(); + Some(CapsuleSnapshot { + payload, + transcript: snapshot.text, + sequence: snapshot.sequence, + revision: snapshot.revision, + payload_revision: snapshot.payload_revision, + }) + } + + /// Commit the replay frame before the webview/main-thread work is queued. + /// The rail snapshot is an IPC replay source, so it must not wait for a + /// window callback to become internally consistent. + pub(crate) fn capsule_revision(&self) -> u64 { + self.capsule.snapshot.lock().revision + } + + pub(crate) fn commit_capsule_payload( + &self, + payload: &CapsulePayload, + captured_revision: u64, + ) -> bool { + let mut snapshot = self.capsule.snapshot.lock(); + let current_revision = snapshot.revision; + if !can_commit_capsule_payload( + current_revision, + snapshot.pending_payload_revision, + captured_revision, + ) { + return false; + } + let mut committed = payload.clone(); + if let Some(session_id) = snapshot.session_id.as_ref() { + if committed + .session_id + .as_ref() + .is_some_and(|payload_session| payload_session != session_id) + { + return false; + } + committed.session_id = Some(session_id.clone()); + } + snapshot.payload = Some(committed); + snapshot.payload_revision = current_revision; + snapshot.pending_payload_revision = None; + true + } + + fn payload_matches_backend_event(payload: &CapsulePayload, event: &BackendEventKind) -> bool { + match event { + BackendEventKind::DictationStateChanged(dictation) => { + let expected = match dictation.phase { + DictationPhase::Idle => CapsuleState::Idle, + DictationPhase::Starting | DictationPhase::Recording => CapsuleState::Recording, + DictationPhase::Transcribing => CapsuleState::Transcribing, + DictationPhase::Polishing | DictationPhase::Inserting => { + CapsuleState::Polishing + } + DictationPhase::Completed => CapsuleState::Done, + DictationPhase::Cancelled => CapsuleState::Cancelled, + DictationPhase::Failed => CapsuleState::Error, + }; + payload.state == expected + } + BackendEventKind::DictationCompleted(_) => payload.state == CapsuleState::Done, + BackendEventKind::BackendStopping => payload.state == CapsuleState::Idle, + BackendEventKind::SelectionVoiceLevel(_) => payload.state == CapsuleState::Recording, + BackendEventKind::SelectionVoiceStateChanged(selection) => { + let expected = match selection.phase { + SelectionVoicePhase::Idle + | SelectionVoicePhase::Completed + | SelectionVoicePhase::Cancelled + | SelectionVoicePhase::Failed => CapsuleState::Idle, + SelectionVoicePhase::Recording => CapsuleState::Recording, + SelectionVoicePhase::Processing + | SelectionVoicePhase::AwaitingIntent + | SelectionVoicePhase::Preview + | SelectionVoicePhase::Applying => CapsuleState::Polishing, + }; + payload.state == expected + } + // QA projections do not own the transcript session. They still use + // the same revision path, but their detailed phase is resolved by + // the async QA snapshot before presentation. + BackendEventKind::QaLevel(_) | BackendEventKind::QaState(_) => true, + _ => false, + } + } + + /// Keep a complete replay string for a rail that mounts after live events. + /// The animation layer remains bounded; this cache is intentionally the + /// authoritative text used only for native ready-handshake recovery. + pub(crate) fn record_backend_event(&self, event: &BackendEvent) { + let mut snapshot = self.capsule.snapshot.lock(); + if event.sequence <= snapshot.sequence { + return; + } + let event_session = event.session_id.map(|session| session.to_string()); + let payload_projection_pending = matches!( + &event.kind, + BackendEventKind::DictationStateChanged(_) + | BackendEventKind::TranscriptDelta(_) + | BackendEventKind::DictationCompleted(_) + | BackendEventKind::QaLevel(_) + | BackendEventKind::QaState(_) + | BackendEventKind::SelectionVoiceLevel(_) + | BackendEventKind::SelectionVoiceStateChanged(_) + | BackendEventKind::BackendStopping + ); + match &event.kind { + BackendEventKind::DictationStateChanged(dictation) + if dictation.phase == DictationPhase::Starting => + { + let session_id = dictation + .session_id + .map(|session| session.to_string()) + .or(event_session.clone()); + if session_id.is_some() && snapshot.session_id != session_id { + snapshot.session_id = session_id; + snapshot.text.clear(); + } else if snapshot.session_id.is_none() { + snapshot.session_id = session_id; + } + } + BackendEventKind::DictationStateChanged(dictation) + if dictation.phase == DictationPhase::Recording => + { + let session_id = dictation + .session_id + .map(|session| session.to_string()) + .or(event_session.clone()); + if session_id.is_some() && snapshot.session_id != session_id { + snapshot.session_id = session_id; + snapshot.text.clear(); + } else if snapshot.session_id.is_none() { + snapshot.session_id = session_id; + } + } + BackendEventKind::SelectionVoiceStateChanged(selection) + if selection.phase == SelectionVoicePhase::Recording => + { + let session_id = selection + .session_id + .map(|session| session.to_string()) + .or(event_session.clone()); + if session_id.is_some() && snapshot.session_id != session_id { + snapshot.session_id = session_id; + snapshot.text.clear(); + } else if snapshot.session_id.is_none() { + snapshot.session_id = session_id; + } + } + BackendEventKind::SelectionVoiceStateChanged(selection) => { + let session_id = selection + .session_id + .map(|session| session.to_string()) + .or(event_session.clone()); + if session_id.is_some() && snapshot.session_id != session_id { + snapshot.session_id = session_id; + snapshot.text.clear(); + } + } + BackendEventKind::TranscriptDelta(delta) + if event_session.is_none() || snapshot.session_id == event_session => + { + let offset = usize::try_from(delta.offset).ok(); + if let Some(offset) = offset { + let mut chars: Vec = snapshot.text.chars().collect(); + if offset <= chars.len() { + chars.truncate(offset); + chars.extend(delta.text.chars()); + snapshot.text = chars.into_iter().collect(); + if event_session.is_some() { + snapshot.session_id = event_session.clone(); + } + } + } + } + _ => {} + } + snapshot.sequence = event.sequence; + if payload_projection_pending { + snapshot.revision = event.sequence; + // Only an actual precommit carrying this exact event revision is + // coherent. In particular, an old Recording payload must not be + // promoted when the next Recording event is just a PCM update. + if snapshot.payload_revision == event.sequence + && snapshot.payload.as_ref().is_some_and(|payload| { + Self::payload_matches_backend_event(payload, &event.kind) + }) + { + snapshot.pending_payload_revision = None; + } else { + snapshot.pending_payload_revision = Some(event.sequence); + } + } else if snapshot.pending_payload_revision.is_none() { + // Transcript-only events keep the latest committed payload frame, + // but advance its observable revision with the text watermark. + snapshot.payload_revision = event.sequence; + } + } + pub(crate) fn cache_capsule_style(&self, style: CapsuleStyle) { self.capsule.cache_style(style); if let Some(capsule) = self.capsule_window() { @@ -871,6 +1773,7 @@ impl TauriCoordinatorHost { pub(crate) fn emit_capsule_state_to_capsule(&self, payload: &crate::types::CapsulePayload) { if let Some(app) = self.app() { let _ = app.emit_to("capsule", "capsule:state", payload); + let _ = app.emit_to("capsule-rail", "capsule:state", payload); } } @@ -945,6 +1848,15 @@ impl TauriCoordinatorHost { mod tests { use super::*; + #[test] + fn capsule_payload_commit_requires_the_current_pending_revision() { + assert!(can_commit_capsule_payload(7, None, 7)); + assert!(can_commit_capsule_payload(7, Some(7), 7)); + assert!(!can_commit_capsule_payload(7, Some(6), 7)); + assert!(!can_commit_capsule_payload(7, Some(7), 6)); + assert!(!can_commit_capsule_payload(7, None, 8)); + } + #[test] fn capsule_style_cache_preserves_every_wire_variant() { let state = CapsuleWindowState::default(); @@ -963,6 +1875,7 @@ mod tests { fn capsule_layout_cache_notices_work_area_and_style_changes() { let initial = CapsuleLayoutState { translation_active: false, + transcript_visible: false, style: CapsuleStyle::Classic, monitor_x: 0, monitor_y: 0, @@ -1035,4 +1948,107 @@ mod tests { CapsuleWindowAction::HideCapsule ); } + + #[test] + fn capsule_transcript_rail_position_uses_body_content_for_all_styles() { + let siri = capsule_transcript_rail_position(CapsuleStyle::Siri, false, true).unwrap(); + assert_eq!( + (siri.width, siri.height, siri.top_offset, siri.gap), + (460.0, 40.0, -48.0, 8.0) + ); + assert_eq!( + capsule_transcript_rail_position(CapsuleStyle::Siri, true, true), + Some(siri) + ); + + let classic = capsule_transcript_rail_position(CapsuleStyle::Classic, false, true).unwrap(); + assert_eq!( + ( + classic.width, + classic.height, + classic.top_offset, + classic.gap + ), + (460.0, 52.0, 44.0, 8.0) + ); + assert_eq!( + capsule_transcript_rail_position(CapsuleStyle::Classic, true, true), + Some(CapsuleTranscriptRailPosition { + width: 460.0, + height: 52.0, + top_offset: 14.0, + gap: 8.0, + }) + ); + + let typeless = + capsule_transcript_rail_position(CapsuleStyle::Typeless, false, true).unwrap(); + assert_eq!(typeless.width, 206.0); + assert_eq!(typeless.gap, 0.0); + assert!((typeless.height - 52.0 * 0.447).abs() < f64::EPSILON); + assert!((typeless.top_offset - (57.0 - 64.0 * 0.447 - 52.0 * 0.447)).abs() < f64::EPSILON); + + let typeless_translation = + capsule_transcript_rail_position(CapsuleStyle::Typeless, true, true).unwrap(); + assert!( + (typeless_translation.top_offset - (65.0 - 20.0 * 0.447 - 64.0 * 0.447 - 52.0 * 0.447)) + .abs() + < f64::EPSILON + ); + + for style in [ + CapsuleStyle::Siri, + CapsuleStyle::Classic, + CapsuleStyle::Typeless, + ] { + assert_eq!(capsule_transcript_rail_position(style, false, false), None); + assert_eq!(capsule_transcript_rail_position(style, true, false), None); + } + } + + #[cfg(target_os = "windows")] + #[test] + fn windows_hit_test_uses_x_and_y_and_translation_bounds() { + assert_eq!( + capsule_control_hit_rect(CapsuleStyle::Classic, true, false, 460, 100), + Some(CapsuleHitTestRect { + left: 132, + right: 328, + top: 32, + bottom: 84, + }) + ); + assert_eq!( + capsule_control_hit_rect(CapsuleStyle::Classic, false, false, 460, 100), + Some(CapsuleHitTestRect { + left: 132, + right: 328, + top: 32, + bottom: 84, + }) + ); + let typeless = capsule_control_hit_rect(CapsuleStyle::Typeless, true, true, 206, 65) + .expect("translation typeless hit rect"); + assert_eq!((typeless.left, typeless.right), (51, 155)); + assert_eq!((typeless.top, typeless.bottom), (36, 65)); + assert!(typeless.left > 0 && typeless.right < 206); + assert!(typeless.top > 0 && typeless.bottom <= 65); + let typeless_compact = + capsule_control_hit_rect(CapsuleStyle::Typeless, false, false, 206, 57) + .expect("compact typeless hit rect"); + assert_eq!((typeless_compact.top, typeless_compact.bottom), (28, 57)); + assert_eq!( + capsule_control_hit_rect(CapsuleStyle::Siri, false, false, 460, 180), + None + ); + assert_eq!( + capsule_card_hit_rect(320, 140), + CapsuleHitTestRect { + left: 0, + right: 320, + top: 0, + bottom: 140, + } + ); + } } diff --git a/openless-all/app/src-tauri/src/tauri_events.rs b/openless-all/app/src-tauri/src/tauri_events.rs index cfb928f9c..2307d67ec 100644 --- a/openless-all/app/src-tauri/src/tauri_events.rs +++ b/openless-all/app/src-tauri/src/tauri_events.rs @@ -34,11 +34,20 @@ pub fn start(app: AppHandle, backend: Arc) { match events.recv().await { Ok(event) => { let _ = app.emit("backend:event", &event); + if let Some(coordinator) = + app.try_state::>() + { + // Advance the transcript watermark before legacy + // capsule presentation. present_core_capsule then + // commits the matching payload under the same native + // snapshot lock before queueing the HWND work. + coordinator.tauri_host().record_backend_event(&event); + } forward_legacy_event( &app, &backend_for_events, event.session_id, - event.kind, + event.kind.clone(), &mut capsule_owners, ) .await; @@ -193,6 +202,7 @@ async fn forward_legacy_event( openless_core::DictationInsertStatus::NotRequested => "处理完成", }; coordinator.present_core_capsule(CapsulePayload { + session_id: Some(result.session_id.to_string()), state: CapsuleState::Done, level: 0.0, elapsed_ms: result.duration_ms, @@ -274,6 +284,7 @@ async fn forward_legacy_event( LessComputerVoicePhase::Idle => CapsuleState::Idle, }; coordinator.present_core_capsule(CapsulePayload { + session_id: Some(session_id.to_string()), state, level: *level, elapsed_ms: *elapsed_ms, @@ -342,6 +353,7 @@ async fn forward_legacy_event( if let Some(coordinator) = app.try_state::>() { if coordinator.selection_voice_accepts_level(&level.session_id) { coordinator.present_core_capsule(CapsulePayload { + session_id: Some(level.session_id.clone()), state: CapsuleState::Recording, level: level.level, elapsed_ms: level.elapsed_ms, @@ -462,6 +474,9 @@ fn emit_dictation_state( if let Some(capsule) = app.get_webview_window("capsule") { let _ = capsule.emit("capsule:state", &payload); } + if let Some(rail) = app.get_webview_window("capsule-rail") { + let _ = rail.emit("capsule:state", &payload); + } if let Some(main) = app.get_webview_window("main") { let _ = main.emit("capsule:state", &payload); } @@ -509,6 +524,7 @@ fn map_dictation_state( _ => snapshot.message, }; CapsulePayload { + session_id: snapshot.session_id.map(|session| session.to_string()), state, level: snapshot.level, elapsed_ms: snapshot.elapsed_ms, @@ -577,6 +593,7 @@ fn transcription_notice_payload( (0, false) }; Some(CapsulePayload { + session_id: Some(session_id.to_string()), state: CapsuleState::Transcribing, level: 0.0, elapsed_ms, @@ -682,6 +699,7 @@ fn qa_capsule_payload( finish_qa_transcription_notice(&mut owners.transcription_notice, session_id, kind); } Some(CapsulePayload { + session_id: session_id.map(|session| session.to_string()), state, level: if level.is_finite() { level.clamp(0.0, 1.0) diff --git a/openless-all/app/src-tauri/tauri.conf.json b/openless-all/app/src-tauri/tauri.conf.json index 5aea120b4..6d52e4c7c 100644 --- a/openless-all/app/src-tauri/tauri.conf.json +++ b/openless-all/app/src-tauri/tauri.conf.json @@ -46,6 +46,23 @@ "visible": false, "center": false, "acceptFirstMouse": true + }, + { + "label": "capsule-rail", + "url": "index.html?window=capsule-rail&os=win", + "title": "OpenLess Transcript Rail", + "width": 460, + "height": 52, + "decorations": false, + "transparent": true, + "shadow": false, + "alwaysOnTop": true, + "skipTaskbar": true, + "resizable": false, + "focus": false, + "visible": false, + "center": false, + "acceptFirstMouse": false } ], "security": { diff --git a/openless-all/app/src/App.tsx b/openless-all/app/src/App.tsx index d89726f6d..d74166094 100644 --- a/openless-all/app/src/App.tsx +++ b/openless-all/app/src/App.tsx @@ -1,5 +1,6 @@ import { lazy, Suspense, useEffect, useState } from 'react'; import { Capsule } from './components/Capsule'; +import { CapsuleTranscriptOverlay } from './components/LiveTranscriptPill'; import { CoreStartupScreen } from './components/CoreStartupScreen'; import { GlobalDownloadProgress } from './components/GlobalDownloadProgress'; import { detectOS, type OS } from './components/WindowChrome'; @@ -52,6 +53,7 @@ const LessComputerGlow = LESS_COMPUTER_BUNDLED interface AppProps { isCapsule: boolean; + isCapsuleRail: boolean; isQa: boolean; isSelectionVoiceIntent: boolean; isLessComputer: boolean; @@ -96,12 +98,16 @@ export function App(props: AppProps) { function ReadyApp({ isCapsule, + isCapsuleRail, isQa, isSelectionVoiceIntent, isLessComputer, isLessComputerGlow, forcedOs, }: AppProps) { + if (isCapsuleRail) { + return ; + } if (isCapsule) { return ; } diff --git a/openless-all/app/src/components/Capsule.tsx b/openless-all/app/src/components/Capsule.tsx index 46888b02d..bf9cf780e 100644 --- a/openless-all/app/src/components/Capsule.tsx +++ b/openless-all/app/src/components/Capsule.tsx @@ -5,14 +5,21 @@ import { detectOS, type OS } from './WindowChrome'; import { warmUpSiriShaders } from './SiriGL'; import { VoiceOrbStage } from './VoiceOrbStage'; import { TypelessCapsule } from './TypelessCapsule'; -import { LiveTranscriptPill } from './LiveTranscriptPill'; +import { CapsuleTranscriptRail } from './LiveTranscriptPill'; import { capsuleTranscriptFontSize, visibleCapsuleTranscript } from '../lib/capsuleTranscript'; import { getSettings } from '../lib/ipc/settings'; -import { cancelDictation, stopDictation } from '../lib/ipc/dictation'; +import { + cancelDictation, + setCapsuleTranscriptVisible, + stopDictation, +} from '../lib/ipc/dictation'; import { getCapsuleHostMetrics, getCapsuleMessageLayout, getCapsulePillMetrics, + getClassicCapsuleGeometry, + getClassicProcessingLabel, + getCapsuleTranscriptRailHeight, parseCapsuleStyle, } from '../lib/capsuleLayout'; import { isTauri } from '../lib/ipc'; @@ -28,6 +35,9 @@ import { VocabSuggestionCard } from './VocabSuggestionCard'; import { InsertFallbackCard } from './InsertFallbackCard'; import { applyTranscriptEvent, + beginTranscriptGeneration, + clearTranscriptText, + createTranscriptViewState, type BackendEvent, type TranscriptViewState, } from '../lib/backendEvent'; @@ -203,10 +213,10 @@ function classicPillMetrics(os: OS): ClassicPillMetrics { return os === 'win' ? CLASSIC_PILL_METRICS_WIN : CLASSIC_PILL_METRICS; } -function AudioBars({ level }: { level: number }) { +function AudioBars({ level, compact = false }: { level: number; compact?: boolean }) { const envelope = [0.55, 0.85, 1.0, 0.85, 0.55]; const base = 2; - const max = 24; + const max = compact ? 16 : 24; const voice = Math.min(1, Math.max(0, level)); const silenceGate = 0.012; const responseCeiling = 0.34; @@ -218,12 +228,13 @@ function AudioBars({ level }: { level: number }) { const visualVoice = Math.pow(easedVoice, 0.42); return (
@@ -232,7 +243,7 @@ function AudioBars({ level }: { level: number }) { key={i} style={{ display: 'inline-block', - width: 3, + width: compact ? 2 : 3, height: base + (max - base) * visualVoice * env, borderRadius: 999, background: 'var(--ol-blue)', @@ -336,7 +347,7 @@ interface ClassicPillProps { level: number; insertedChars: number; message?: string; - operating?: boolean; + operating: boolean; onCancel: () => void; onConfirm: () => void; } @@ -357,15 +368,14 @@ function ClassicPill({ const cancelEnabled = state === 'recording' || state === 'transcribing' || state === 'polishing'; const confirmEnabled = state === 'recording'; - // "thinking" shine speed: fast (0.9s/cycle) for the first 2s of transcribing/polishing - // (signals "stream just started"), then slow (2.4s) as the steady state. Also resets - // to fast on idle/done/other states so the next entry bursts from the start. + // Keep the original Classic processing treatment: the thinking/using label + // has its own entry and shine animation, while recording keeps the live bars. const [shineFast, setShineFast] = useState(true); useEffect(() => { if (state === 'transcribing' || state === 'polishing') { setShineFast(true); - const t = setTimeout(() => setShineFast(false), 2000); - return () => clearTimeout(t); + const timer = setTimeout(() => setShineFast(false), 2000); + return () => clearTimeout(timer); } setShineFast(true); return undefined; @@ -383,28 +393,22 @@ function ClassicPill({ style={{ display: 'inline-flex', alignItems: 'center', - // 4px side padding + outer gap already puts the "thinking" ↔ ✗/✓ visual - // spacing at ~4-5px. - padding: '0 4px', + gap: 5, + padding: '0 2px', width: '100%', maxWidth: metrics.textWidth, minWidth: 0, justifyContent: 'center', - // State-entry animation: an extra fade cue when going recording → polishing, - // easier to perceive than swapping the center content alone. animation: 'cap-state-enter 220ms var(--ol-motion-soft) both', }} > + - {t(operating ? 'capsule.using' : 'capsule.thinking')} + {t(getClassicProcessingLabel(operating))}
); @@ -514,7 +515,9 @@ interface ClassicCapsuleProps { message?: string; transcript?: string; transcriptFontSize?: number; - operating?: boolean; + transcriptVisible?: boolean; + transcriptInSameWindow?: boolean; + operating: boolean; translation: boolean; } @@ -531,11 +534,12 @@ function ClassicCapsule({ message, transcript, transcriptFontSize = 14, + transcriptVisible = false, + transcriptInSameWindow = true, operating, translation, }: ClassicCapsuleProps) { const { t } = useTranslation(); - const metrics = classicPillMetrics(os); const hostMetrics = getCapsuleHostMetrics(os, false, 'classic'); const onCancel = useCallback(() => { void cancelDictation(); @@ -544,8 +548,12 @@ function ClassicCapsule({ void stopDictation(); }, []); const liveText = transcript?.trim() ?? ''; - const recording = state === 'recording'; - const processing = state === 'transcribing' || state === 'polishing'; + const classicGeometry = getClassicCapsuleGeometry( + os, + transcriptVisible || Boolean(liveText), + translation, + transcriptInSameWindow, + ); return ( <> @@ -558,7 +566,7 @@ function ClassicCapsule({ style={{ position: 'absolute', left: '50%', - bottom: hostMetrics.bottomInset + metrics.height + hostMetrics.badgeGap, + bottom: classicGeometry.badgeBottomOffset, transform: 'translateX(-50%)', pointerEvents: 'none', }} @@ -593,24 +601,17 @@ function ClassicCapsule({ {t('capsule.translating')}
- {liveText ? ( - - ) : ( +
+ {liveText && ( + + )} - )} +
); } @@ -723,11 +724,7 @@ export function Capsule({ os: forcedOs }: CapsuleProps = {}) { : Number(new URLSearchParams(window.location.search).get('fontSize') || 14), ), ); - const transcriptViewRef = useRef({ - sessionId: null, - sequence: 0, - text: '', - }); + const transcriptViewRef = useRef(createTranscriptViewState()); const capsuleStateRef = useRef(preview.state); const [translation, setTranslation] = useState(preview.translation); const [selectionPolish, setSelectionPolish] = useState(preview.selectionPolish); @@ -737,6 +734,9 @@ export function Capsule({ os: forcedOs }: CapsuleProps = {}) { const stylePreferenceReadyRef = useRef(false); const isClassic = capsuleStyle === 'classic'; const isTypeless = capsuleStyle === 'typeless'; + // Windows keeps the transcript in the dedicated capsule-rail HWND. The browser + // preview and other platforms continue rendering the sibling rail in this tree. + const splitTranscriptRail = isTauri && os === 'win'; // Warming: the mic hasn't produced its first PCM frame yet. When true, the orb runs // its standby breathing form (see SiriGL warming). const [warming, setWarming] = useState(preview.warming); @@ -814,7 +814,14 @@ export function Capsule({ os: forcedOs }: CapsuleProps = {}) { // the previous capsule was still transcribing/polishing/done from // dictation (selection-voice Start used to keep the stale stream). if (p.state === 'recording' && previous !== 'recording') { - transcriptViewRef.current = { sessionId: null, sequence: 0, text: '' }; + transcriptViewRef.current = beginTranscriptGeneration(transcriptViewRef.current); + setLocalAsrText(''); + } + if (p.state === 'idle' || p.state === 'done' || p.state === 'cancelled' || p.state === 'error') { + // Terminal payloads must release the native transcript rail before the exit/card + // branch can leave the capsule webview. Keeping the old text here would make the + // enlarged native window stay interactive after the session has ended. + transcriptViewRef.current = clearTranscriptText(transcriptViewRef.current); setLocalAsrText(''); } setTranslation(p.translation === true); @@ -979,6 +986,43 @@ export function Capsule({ os: forcedOs }: CapsuleProps = {}) { }; }, [preview.insertDemo]); + // Derive these values before any conditional return. Hooks must be called for the same + // component shape while the capsule switches between the live pill, fallback card, + // suggestion card, and the hidden idle branch. + const renderedState: CapsuleState = state === 'idle' ? lastVisibleState : state; + const renderedSelectionPolish = state === 'idle' ? lastVisibleSelectionPolish : selectionPolish; + const renderedMessage = + state === 'idle' + ? lastVisibleMessage + : state === 'transcribing' && localAsrText + ? localAsrText + : message; + const liveTranscript = visibleCapsuleTranscript( + localAsrText, + transcriptEnabled, + renderedState, + renderedSelectionPolish, + ); + const showLiveTranscript = state !== 'idle' && liveTranscript.length > 0; + const transcriptVisible = showLiveTranscript && !insertFallback && suggestions.length === 0; + + useEffect(() => { + if (!isTauri) return; + void setCapsuleTranscriptVisible(transcriptVisible).catch((error) => + console.warn('[capsule] transcript visibility sync failed', error), + ); + }, [transcriptVisible]); + + // Always release the native hit-test region when the capsule webview is torn down. + useEffect(() => { + if (!isTauri) return; + return () => { + void setCapsuleTranscriptVisible(false).catch((error) => + console.warn('[capsule] transcript visibility cleanup failed', error), + ); + }; + }, []); + // Fallback card comes first: it pops the moment the session ends, while the capsule // is still rendering the Done/Error terminal state — and the session's outcome was // exactly "didn't insert", so letting the terminal state cover it would report a @@ -1000,24 +1044,6 @@ export function Capsule({ os: forcedOs }: CapsuleProps = {}) { return
; } - // During exit, render the last frame from lastVisibleState so idle isn't treated as - // a no-waveform state. - const renderedState: CapsuleState = state === 'idle' ? lastVisibleState : state; - const renderedSelectionPolish = state === 'idle' ? lastVisibleSelectionPolish : selectionPolish; - const renderedMessage = - state === 'idle' - ? lastVisibleMessage - : state === 'transcribing' && localAsrText - ? localAsrText - : message; - const liveTranscript = visibleCapsuleTranscript( - localAsrText, - transcriptEnabled, - renderedState, - renderedSelectionPolish, - ); - const showLiveTranscript = liveTranscript.length > 0; - return (
@@ -1062,7 +1091,7 @@ export function Capsule({ os: forcedOs }: CapsuleProps = {}) { level={leaving ? 0 : level} insertedChars={insertedCharsRef.current} message={renderedMessage} - transcript={liveTranscript} + transcript={splitTranscriptRail ? '' : liveTranscript} transcriptFontSize={transcriptFontSize} operating={operatingRef.current} translation={translation} @@ -1118,41 +1147,33 @@ export function Capsule({ os: forcedOs }: CapsuleProps = {}) { {t('capsule.translating')}
-
- -
- {showLiveTranscript && ( -
- + {showLiveTranscript && !splitTranscriptRail && ( + + )} +
+
- )} +
))} {renderedSelectionPolish && ( diff --git a/openless-all/app/src/components/CapsuleStyles.css b/openless-all/app/src/components/CapsuleStyles.css index 4a67c70e0..cc4ac372d 100644 --- a/openless-all/app/src/components/CapsuleStyles.css +++ b/openless-all/app/src/components/CapsuleStyles.css @@ -18,6 +18,93 @@ background: #ffffff; } +.ol-capsule-transcript-rail { + display: flex; + flex: 0 0 auto; + align-items: center; + justify-content: center; + width: 100%; + pointer-events: none; + position: relative; + z-index: 2; +} + +/* Windows owns this rail in a separate transparent WebView window. Its HWND is + always cursor-through, so the full transcript/gutter can never intercept a + click destined for another process. */ +.ol-capsule-transcript-overlay { + width: 460px; + height: 52px; + overflow: visible; + pointer-events: none; +} + +.ol-capsule-transcript-overlay .ol-capsule-transcript-rail { + height: 52px; +} + +.ol-capsule-transcript-overlay[data-style='siri'] { + height: 40px; +} + +.ol-capsule-transcript-overlay[data-style='siri'] .ol-capsule-transcript-rail { + height: 40px; +} + +.ol-capsule-transcript-overlay--typeless { + width: 460px; + height: 52px; + zoom: 0.447; +} + +.ol-classic-capsule-stack, +.ol-siri-capsule-stack, +.ol-typeless-capsule-stack { + display: flex; + flex: 0 0 auto; + flex-direction: column; + align-items: center; + justify-content: flex-end; + gap: 8px; +} + +.ol-classic-capsule-stack, +.ol-siri-capsule-stack, +.ol-typeless-capsule-stack { + width: 460px; +} + +.ol-siri-capsule-stack .ol-capsule-transcript-rail { + height: 40px; +} + +.ol-classic-capsule-stack .ol-capsule-transcript-rail, +.ol-typeless-capsule-stack .ol-capsule-transcript-rail { + height: 52px; +} + +.ol-typeless-capsule-stack { + gap: 0; +} + +.ol-typeless-translation-row { + display: flex; + flex: 0 0 20px; + align-items: center; + justify-content: center; + width: 100%; + height: 20px; + pointer-events: none; +} + +.ol-typeless-capsule-stage { + position: relative; + display: flex; + flex: 0 0 auto; + align-items: center; + justify-content: center; +} + .ol-typeless-capsule { position: relative; width: 176px; @@ -34,7 +121,8 @@ transition: width 220ms cubic-bezier(0.22, 1, 0.36, 1); } -.ol-typeless-capsule[data-recording='true'] { +.ol-typeless-capsule[data-recording='true'], +.ol-typeless-capsule[data-processing='true'] { width: 232px; } .ol-typeless-capsule[data-error='true'] { @@ -62,7 +150,13 @@ transform: translateX(-50%); } .ol-typeless-capsule[data-recording='true'] .ol-typeless-recording, -.ol-typeless-capsule:not([data-recording='true']) .ol-typeless-status { +.ol-typeless-capsule[data-processing='true'] .ol-typeless-recording { + opacity: 1; + pointer-events: auto; +} + +.ol-typeless-capsule:not([data-recording='true']):not([data-processing='true']) + .ol-typeless-status { opacity: 1; pointer-events: auto; } @@ -156,11 +250,9 @@ cursor: pointer; } .ol-typeless-translation { - position: absolute; - bottom: calc(100% + 7px); - left: 50%; - transform: translateX(-50%); + position: static; padding: 3px 10px; + box-sizing: border-box; white-space: nowrap; font-size: 10px; font-weight: 500; diff --git a/openless-all/app/src/components/LiveTranscriptPill.tsx b/openless-all/app/src/components/LiveTranscriptPill.tsx index 913dd03ae..c3613e7b6 100644 --- a/openless-all/app/src/components/LiveTranscriptPill.tsx +++ b/openless-all/app/src/components/LiveTranscriptPill.tsx @@ -1,19 +1,43 @@ import { animate, motion, useMotionValue, useReducedMotion, useTransform } from 'framer-motion'; -import { useEffect, useMemo, useState } from 'react'; +import { useEffect, useMemo, useRef, useState } from 'react'; import './CapsuleStyles.css'; import { INSERT_TEXT_MOTION, appendedBirthAdvance, clampInsertContentWidth, - diffInsertUnits, + diffInsertWindowUnits, firstBornIndex, planCapsuleInsertMotion, + INSERT_TEXT_RENDER_WINDOW, planCharDelays, prefixWidths, rightAnchoredPositions, + selectInsertRenderWindow, type InsertUnit, } from '../lib/insertTextAnimation'; import { Icon } from './Icon'; +import { + applyTranscriptEvent, + applyTranscriptSessionId, + applyTranscriptSnapshot, + beginTranscriptGeneration, + clearTranscriptText, + createTranscriptViewState, + shouldRetryTranscriptSnapshot, + type BackendEvent, + type TranscriptViewState, +} from '../lib/backendEvent'; +import { capsuleTranscriptFontSize, visibleCapsuleTranscript } from '../lib/capsuleTranscript'; +import { getCapsuleTranscriptRailHeight } from '../lib/capsuleLayout'; +import { getSettings } from '../lib/ipc/settings'; +import { getCapsuleSnapshot, isTauri } from '../lib/ipc'; +import type { + CapsulePayload, + CapsuleSnapshot, + CapsuleState, + CapsuleStyle, + UserPreferences, +} from '../lib/types'; export type LiveTranscriptTone = 'frost' | 'dark'; @@ -48,19 +72,194 @@ function getSansFont(): string { return token || 'system-ui, sans-serif'; } -function measureGlyphWidths(glyphs: string[], fontSize: number): number[] { - if (glyphs.length === 0) return []; +function measureGlyphWidth(glyph: string, fontSize: number): number { if (typeof document === 'undefined') { - return glyphs.map((glyph) => Math.max(fontSize * 0.92, glyph.length * fontSize)); + return Math.max(fontSize * 0.92, glyph.length * fontSize); } if (!measureCanvas) measureCanvas = document.createElement('canvas'); const ctx = measureCanvas.getContext('2d'); - if (!ctx) return glyphs.map(() => fontSize); + if (!ctx) return fontSize; ctx.font = `${FONT_WEIGHT} ${fontSize}px ${getSansFont()}`; - return glyphs.map((glyph) => { - const raw = ctx.measureText(glyph === ' ' ? '\u00a0' : glyph).width; - return Math.max(1, raw) + INSERT_TEXT_MOTION.glyphGapPx; - }); + const raw = ctx.measureText(glyph === ' ' ? '\u00a0' : glyph).width; + return Math.max(1, raw) + INSERT_TEXT_MOTION.glyphGapPx; +} + +export interface CapsuleTranscriptRailProps { + text: string; + tone?: LiveTranscriptTone; + stageWidth: number; + maxWidth: number; + height?: number; + fontSize?: number; +} + +/** Transcript is deliberately a sibling rail: the capsule body remains responsible for audio feedback and controls. */ +export function CapsuleTranscriptRail({ + text, + tone = 'frost', + stageWidth, + maxWidth, + height = DEFAULT_PILL_HEIGHT, + fontSize = FONT_SIZE, +}: CapsuleTranscriptRailProps) { + const liveText = text.trim(); + if (!liveText) return null; + return ( +
+ +
+ ); +} + +/** + * Windows renders the transcript in a separate, permanently click-through HWND. + * The capsule HWND can therefore keep a small, reliable native hit region for + * its controls while this rail never participates in cross-process hit testing. + */ +export function CapsuleTranscriptOverlay() { + const [state, setState] = useState('idle'); + const [style, setStyle] = useState('siri'); + const [text, setText] = useState(''); + const [enabled, setEnabled] = useState(true); + const [fontSize, setFontSize] = useState(14); + const [selectionPolish, setSelectionPolish] = useState(false); + const transcriptRef = useRef(createTranscriptViewState()); + const stateRef = useRef('idle'); + const snapshotRetryDelays = [0, 16, 32, 64, 128, 256] as const; + + useEffect(() => { + if (!isTauri) return; + let cancelled = false; + let unlisten: (() => void) | undefined; + (async () => { + const { listen } = await import('@tauri-apps/api/event'); + const stateHandle = await listen('capsule:state', (event) => { + const payload = event.payload; + const previousState = stateRef.current; + const previousTranscript = transcriptRef.current; + const sessionTranscript = applyTranscriptSessionId( + previousTranscript, + payload.sessionId, + ); + const sessionChanged = sessionTranscript !== previousTranscript; + transcriptRef.current = sessionTranscript; + stateRef.current = payload.state; + setState(payload.state); + if (payload.capsuleStyle) setStyle(payload.capsuleStyle); + setSelectionPolish(payload.selectionPolish === true); + if (payload.state === 'recording' && previousState !== 'recording' && !sessionChanged) { + transcriptRef.current = beginTranscriptGeneration(transcriptRef.current); + setText(transcriptRef.current.text); + } else if (sessionChanged) { + setText(sessionTranscript.text); + } + if (['idle', 'done', 'cancelled', 'error'].includes(payload.state)) { + transcriptRef.current = clearTranscriptText(transcriptRef.current); + setText(''); + } + }); + const transcriptHandle = await listen('backend:event', (event) => { + const next = applyTranscriptEvent(transcriptRef.current, event.payload); + transcriptRef.current = next; + setText(next.text); + }); + const preferenceHandle = await listen('prefs:changed', (event) => { + setEnabled(event.payload.capsuleTranscriptEnabled ?? true); + setFontSize(capsuleTranscriptFontSize(event.payload.capsuleTranscriptFontSize)); + if (event.payload.capsuleStyle) setStyle(event.payload.capsuleStyle); + }); + const applySnapshot = ( + snapshot: CapsuleSnapshot, + fallbackStyle: CapsuleStyle, + ): 'retry' | 'done' => { + const previousTranscript = transcriptRef.current; + if (shouldRetryTranscriptSnapshot(previousTranscript, snapshot)) return 'retry'; + const nextTranscript = applyTranscriptSnapshot(previousTranscript, snapshot); + if (nextTranscript === previousTranscript) return 'done'; + const sessionChanged = + snapshot.sessionId !== null && snapshot.sessionId !== previousTranscript.sessionId; + transcriptRef.current = nextTranscript; + setText(nextTranscript.text); + + const previousState = stateRef.current; + stateRef.current = snapshot.state; + setState(snapshot.state); + setStyle(snapshot.capsuleStyle ?? fallbackStyle); + setSelectionPolish(snapshot.selectionPolish === true); + if (snapshot.state === 'recording' && previousState !== 'recording' && !sessionChanged) { + transcriptRef.current = beginTranscriptGeneration(transcriptRef.current); + setText(transcriptRef.current.text); + } + if (['idle', 'done', 'cancelled', 'error'].includes(snapshot.state)) { + transcriptRef.current = clearTranscriptText(transcriptRef.current); + setText(''); + } + return 'done'; + }; + const replaySnapshot = async (fallbackStyle: CapsuleStyle) => { + for (const delayMs of snapshotRetryDelays) { + if (cancelled) return; + if (delayMs > 0) { + await new Promise((resolve) => window.setTimeout(resolve, delayMs)); + } + if (cancelled) return; + const snapshot = await getCapsuleSnapshot(); + if (cancelled) return; + if (!snapshot) continue; + if (applySnapshot(snapshot, fallbackStyle) === 'done') return; + } + }; + if (cancelled) { + stateHandle(); + transcriptHandle(); + preferenceHandle(); + } else { + unlisten = () => { + stateHandle(); + transcriptHandle(); + preferenceHandle(); + }; + const preferences = await getSettings(); + if (!cancelled) { + setEnabled(preferences.capsuleTranscriptEnabled ?? true); + setFontSize(capsuleTranscriptFontSize(preferences.capsuleTranscriptFontSize)); + setStyle(preferences.capsuleStyle); + await replaySnapshot(preferences.capsuleStyle); + } + } + })().catch((error) => console.warn('[capsule-rail] subscription failed', error)); + return () => { + cancelled = true; + unlisten?.(); + }; + }, []); + + const liveText = visibleCapsuleTranscript(text, enabled, state, selectionPolish); + const typeless = style === 'typeless'; + const railHeight = getCapsuleTranscriptRailHeight(style); + return ( +
+ +
+ ); } export function LiveTranscriptPill({ @@ -83,21 +282,26 @@ export function LiveTranscriptPill({ const hasControls = Boolean(onCancel || onConfirm); const padX = INSERT_TEXT_MOTION.padX + (hasControls ? controlSize + 6 : 0); const pillMinWidth = hasControls ? Math.max(minWidth, controlSize * 2 + 72) : minWidth; - const [units, setUnits] = useState(() => diffInsertUnits([], text)); + const [units, setUnits] = useState(() => diffInsertWindowUnits([], text)); const [seenText, setSeenText] = useState(text); if (text !== seenText) { setSeenText(text); - setUnits((prev) => diffInsertUnits(prev, text)); + setUnits((prev) => diffInsertWindowUnits(prev, text)); } - const widths = useMemo( + const renderWindow = useMemo( () => - measureGlyphWidths( - units.map((unit) => unit.text), - fontSize, + selectInsertRenderWindow( + units, + (glyph) => measureGlyphWidth(glyph, fontSize), + maxWidth, + padX, + INSERT_TEXT_RENDER_WINDOW, ), - [units, fontSize], + [units, fontSize, maxWidth, padX], ); + const renderUnits = renderWindow.units; + const widths = renderWindow.widths; const contentWidth = useMemo( () => clampInsertContentWidth( @@ -110,11 +314,11 @@ export function LiveTranscriptPill({ const positions = useMemo(() => rightAnchoredPositions(widths), [widths]); // Start appended glyphs beyond the old tail, so they never appear on top of // retained text while the displacement spring is still gathering speed. - const birthAdvance = appendedBirthAdvance(units, widths); - const originIndex = firstBornIndex(units); + const birthAdvance = appendedBirthAdvance(renderUnits, widths); + const originIndex = firstBornIndex(renderUnits); const starts = useMemo(() => prefixWidths(widths), [widths]); const originX = (starts[originIndex] ?? 0) + (widths[originIndex] ?? 0) / 2; - const insertedCount = units.filter((unit) => unit.born).length; + const insertedCount = renderUnits.filter((unit) => unit.born).length; const delays = useMemo( () => planCharDelays(widths, originIndex, insertedCount), [widths, originIndex, insertedCount], @@ -233,7 +437,7 @@ export function LiveTranscriptPill({ height, }} > - {units.map((unit, index) => ( + {renderUnits.map((unit, index) => ( void cancelDictation(), []); const confirm = useCallback(() => void stopDictation(), []); - if (liveText) { - return ( -
- {translation && {t('capsule.translating')}} - -
- ); - } - return ( -
- {translation && {t('capsule.translating')}} -
-
- - - -
-
- - {label} - - {processing && !preview && ( - - )} +
+ + + +
+
+ + {label} + +
+
diff --git a/openless-all/app/src/lib/backendEvent.test.ts b/openless-all/app/src/lib/backendEvent.test.ts index 2723a649c..1781c5853 100644 --- a/openless-all/app/src/lib/backendEvent.test.ts +++ b/openless-all/app/src/lib/backendEvent.test.ts @@ -1,4 +1,13 @@ -import { applyTranscriptEvent, type TranscriptViewState } from './backendEvent'; +import { + applyTranscriptEvent, + isTranscriptSnapshotCoherent, + applyTranscriptSnapshot, + shouldRetryTranscriptSnapshot, + beginTranscriptGeneration, + clearTranscriptText, + createTranscriptViewState, + type TranscriptViewState, +} from './backendEvent'; function assertState(actual: TranscriptViewState, text: string, sequence: number) { if (actual.text !== text || actual.sequence !== sequence) { @@ -8,7 +17,7 @@ function assertState(actual: TranscriptViewState, text: string, sequence: number } } -let state: TranscriptViewState = { sessionId: null, sequence: 0, text: '' }; +let state: TranscriptViewState = createTranscriptViewState(); state = applyTranscriptEvent(state, { sequence: 1, sessionId: 'a', @@ -86,8 +95,280 @@ state = applyTranscriptEvent(state, { }); assertState(state, '帮我写', 10); +// A UI recording edge must not reset the global watermark. Until the matching +// backend start arrives, delayed events from the old session are ignored. +state = clearTranscriptText(state); +state = beginTranscriptGeneration(state); +const generation = state.generation; +const watermark = state.sequence; +state = applyTranscriptEvent(state, { + sequence: 13, + sessionId: 'sv', + kind: { type: 'transcript_delta', payload: { text: '旧会话延迟', offset: 0, isFinal: false } }, +}); +assertState(state, '', watermark); +if (state.generation !== generation || !state.awaitingSessionStart) { + throw new Error('new generation must retain the watermark and await its session start'); +} +state = applyTranscriptEvent(state, { + sequence: 13, + sessionId: 'next', + kind: { type: 'dictation_state_changed', payload: { phase: 'starting', sessionId: 'next' } }, +}); +state = applyTranscriptEvent(state, { + sequence: 14, + sessionId: 'sv', + kind: { type: 'transcript_delta', payload: { text: '旧会话仍延迟', offset: 0, isFinal: false } }, +}); +assertState(state, '', 13); +state = applyTranscriptEvent(state, { + sequence: 14, + sessionId: 'next', + kind: { type: 'transcript_delta', payload: { text: '新会话', offset: 0, isFinal: false } }, +}); +assertState(state, '新会话', 14); + +// Tauri's real order is backend:event(starting) -> capsule:state(recording) -> +// backend:event(transcript_delta). The recording edge must reuse the session +// already established by the first event instead of reopening the gate. +let orderedState: TranscriptViewState = createTranscriptViewState(); +orderedState = applyTranscriptEvent(orderedState, { + sequence: 20, + sessionId: 'ordered-session', + kind: { + type: 'dictation_state_changed', + payload: { phase: 'starting', sessionId: 'ordered-session' }, + }, +}); +orderedState = beginTranscriptGeneration(orderedState); +if ( + orderedState.awaitingSessionStart || + orderedState.sessionId !== 'ordered-session' || + orderedState.generation !== 1 +) { + throw new Error('recording edge must reuse the session established by dictation starting'); +} +orderedState = applyTranscriptEvent(orderedState, { + sequence: 21, + sessionId: 'ordered-session', + kind: { type: 'transcript_delta', payload: { text: '首个字', offset: 0, isFinal: false } }, +}); +assertState(orderedState, '首个字', 21); +orderedState = applyTranscriptEvent(orderedState, { + sequence: 22, + sessionId: 'old-session', + kind: { type: 'transcript_delta', payload: { text: '旧延迟', offset: 0, isFinal: false } }, +}); +assertState(orderedState, '首个字', 21); + +// A delayed capsule:state(recording) can arrive after the backend has already +// announced starting and delivered the first delta. The same session must be +// reused without clearing the confirmed prefix. +let lateRecordingState: TranscriptViewState = createTranscriptViewState(); +lateRecordingState = applyTranscriptEvent(lateRecordingState, { + sequence: 30, + sessionId: 'late-recording-session', + kind: { + type: 'dictation_state_changed', + payload: { phase: 'starting', sessionId: 'late-recording-session' }, + }, +}); +lateRecordingState = applyTranscriptEvent(lateRecordingState, { + sequence: 31, + sessionId: 'late-recording-session', + kind: { type: 'transcript_delta', payload: { text: '已确认', offset: 0, isFinal: false } }, +}); +lateRecordingState = beginTranscriptGeneration(lateRecordingState); +assertState(lateRecordingState, '已确认', 31); +if (lateRecordingState.awaitingSessionStart || lateRecordingState.sessionId !== 'late-recording-session') { + throw new Error('late recording edge must preserve the already-started session transcript'); +} + +// The rail ready handshake can resolve after a newer live event. Its older +// snapshot must not roll that event back, while a current snapshot replays the +// complete transcript and leaves the session open for a later recording edge. +let replayState = applyTranscriptEvent(createTranscriptViewState(), { + sequence: 40, + sessionId: 'replay-session', + kind: { + type: 'dictation_state_changed', + payload: { phase: 'starting', sessionId: 'replay-session' }, + }, +}); +replayState = applyTranscriptEvent(replayState, { + sequence: 41, + sessionId: 'replay-session', + kind: { type: 'transcript_delta', payload: { text: '更新后的原文', offset: 0, isFinal: false } }, +}); +const staleReplay = applyTranscriptSnapshot(replayState, { + transcript: '旧快照', sessionId: 'replay-session', sequence: 40, revision: 40, +}); +assertState(staleReplay, '更新后的原文', 41); +const tornReplay = applyTranscriptSnapshot(replayState, { + transcript: '撕裂帧', sessionId: 'replay-session', sequence: 42, revision: 42, + payloadRevision: 41, +}); +assertState(tornReplay, '更新后的原文', 41); +if ( + !shouldRetryTranscriptSnapshot(replayState, { + transcript: '撕裂帧', sessionId: 'replay-session', sequence: 42, revision: 42, + payloadRevision: 41, + }) || + !isTranscriptSnapshotCoherent({ + transcript: '一致帧', sessionId: 'replay-session', sequence: 42, revision: 42, + payloadRevision: 42, + }) +) { + throw new Error('rail must retry a current torn snapshot and accept a coherent one'); +} +// Deterministic snapshot transaction ordering: +// 1. payload commit before the backend event is already coherent; +// 2. backend event before a queued payload is rejected only temporarily; +// 3. the matching payload commit at the same revision is accepted and must +// not permanently lose a legal first frame. +const payloadBeforeBackend = applyTranscriptSnapshot(replayState, { + transcript: 'payload先提交', sessionId: 'replay-session', sequence: 60, revision: 60, + payloadRevision: 60, +}); +assertState(payloadBeforeBackend, 'payload先提交', 60); +const backendBeforeQueuedPayload = applyTranscriptSnapshot(payloadBeforeBackend, { + transcript: 'backend先到', sessionId: 'replay-session', sequence: 61, revision: 61, + payloadRevision: 60, +}); +assertState(backendBeforeQueuedPayload, 'payload先提交', 60); +const queuedPayloadCommit = applyTranscriptSnapshot(backendBeforeQueuedPayload, { + transcript: 'backend先到', sessionId: 'replay-session', sequence: 61, revision: 61, + payloadRevision: 61, +}); +assertState(queuedPayloadCommit, 'backend先到', 61); + +// A Recording -> Recording state update is the first-PCM path. It must advance +// the watermark without treating the previous Recording payload as the new +// frame; the first delta after the update is still accepted for the same session. +let repeatedRecording = applyTranscriptEvent(createTranscriptViewState(), { + sequence: 70, + sessionId: 'pcm-session', + kind: { + type: 'dictation_state_changed', + payload: { phase: 'starting', sessionId: 'pcm-session' }, + }, +}); +repeatedRecording = applyTranscriptEvent(repeatedRecording, { + sequence: 71, + sessionId: 'pcm-session', + kind: { type: 'transcript_delta', payload: { text: '已有前缀', offset: 0, isFinal: false } }, +}); +repeatedRecording = applyTranscriptEvent(repeatedRecording, { + sequence: 72, + sessionId: 'pcm-session', + kind: { + type: 'dictation_state_changed', + payload: { phase: 'recording', sessionId: 'pcm-session' }, + }, +}); +assertState(repeatedRecording, '已有前缀', 72); +repeatedRecording = applyTranscriptEvent(repeatedRecording, { + sequence: 73, + sessionId: 'pcm-session', + kind: { type: 'transcript_delta', payload: { text: '首个 PCM 字', offset: 0, isFinal: false } }, +}); +assertState(repeatedRecording, '首个 PCM 字', 73); + +// SessionId, rather than a Recording -> Recording state edge, is the +// generation boundary. A new session arriving on the same state must clear +// the old text, and a delayed old-session delta must remain rejected. +let sameStateNewSession = applyTranscriptEvent(createTranscriptViewState(), { + sequence: 80, + sessionId: 'first-session', + kind: { + type: 'dictation_state_changed', + payload: { phase: 'recording', sessionId: 'first-session' }, + }, +}); +sameStateNewSession = applyTranscriptEvent(sameStateNewSession, { + sequence: 81, + sessionId: 'first-session', + kind: { type: 'transcript_delta', payload: { text: '旧文本', offset: 0, isFinal: false } }, +}); +const firstGeneration = sameStateNewSession.generation; +sameStateNewSession = applyTranscriptEvent(sameStateNewSession, { + sequence: 82, + sessionId: 'second-session', + kind: { + type: 'dictation_state_changed', + payload: { phase: 'recording', sessionId: 'second-session' }, + }, +}); +assertState(sameStateNewSession, '', 82); +if ( + sameStateNewSession.generation !== firstGeneration + 1 || + sameStateNewSession.sessionId !== 'second-session' +) { + throw new Error('Recording -> Recording with a new sessionId must start a new transcript generation'); +} +sameStateNewSession = applyTranscriptEvent(sameStateNewSession, { + sequence: 83, + sessionId: 'first-session', + kind: { type: 'transcript_delta', payload: { text: '延迟旧文本', offset: 0, isFinal: false } }, +}); +assertState(sameStateNewSession, '', 82); +sameStateNewSession = applyTranscriptEvent(sameStateNewSession, { + sequence: 84, + sessionId: 'second-session', + kind: { type: 'transcript_delta', payload: { text: '新文本', offset: 0, isFinal: false } }, +}); +assertState(sameStateNewSession, '新文本', 84); +const currentReplay = applyTranscriptSnapshot(replayState, { + transcript: '完整原文回放', sessionId: 'replay-session', sequence: 42, revision: 42, +}); +assertState(currentReplay, '完整原文回放', 42); +if ( + currentReplay.revision !== 42 || + !currentReplay.sessionStartObserved || + currentReplay.awaitingSessionStart +) { + throw new Error('current rail snapshot must keep its session eligible for the recording edge'); +} + +// Selection-voice can mount the rail midway through its session as well. Its +// state replay must advance the same session/revision watermark so subsequent +// deltas are accepted, while a delayed dictation delta is still rejected. +let selectionReplay = applyTranscriptEvent(createTranscriptViewState(), { + sequence: 50, + sessionId: 'selection-session', + kind: { + type: 'selection_voice_state_changed', + payload: { phase: 'recording', sessionId: 'selection-session' }, + }, +}); +if ( + selectionReplay.sessionId !== 'selection-session' || + selectionReplay.sequence !== 50 || + selectionReplay.revision !== 50 +) { + throw new Error('selection voice replay must advance session and revision watermarks'); +} +selectionReplay = applyTranscriptSnapshot(selectionReplay, { + transcript: '选区语音已回放', + sessionId: 'selection-session', + sequence: 51, + revision: 51, +}); +selectionReplay = applyTranscriptEvent(selectionReplay, { + sequence: 52, + sessionId: 'selection-session', + kind: { type: 'transcript_delta', payload: { text: '选区语音后续', offset: 0, isFinal: false } }, +}); +assertState(selectionReplay, '选区语音后续', 52); +selectionReplay = applyTranscriptEvent(selectionReplay, { + sequence: 53, + sessionId: 'dictation-session', + kind: { type: 'transcript_delta', payload: { text: '旧听写延迟', offset: 0, isFinal: false } }, +}); +assertState(selectionReplay, '选区语音后续', 52); + // Cloud ASR snapshots must retain all preceding words and apply corrections. -let cloudState: TranscriptViewState = { sessionId: null, sequence: 0, text: '' }; +let cloudState: TranscriptViewState = createTranscriptViewState(); for (const [index, text] of ['你', '你好', '您好', '您好。', '您好。世界'].entries()) { cloudState = applyTranscriptEvent(cloudState, { sequence: index + 1, sessionId: 'cloud', diff --git a/openless-all/app/src/lib/backendEvent.ts b/openless-all/app/src/lib/backendEvent.ts index 922141a30..3da72e1db 100644 --- a/openless-all/app/src/lib/backendEvent.ts +++ b/openless-all/app/src/lib/backendEvent.ts @@ -14,6 +14,132 @@ export interface TranscriptViewState { sessionId: string | null; sequence: number; text: string; + /** Native event revision used to reject a rail snapshot captured before a live event. */ + revision: number; + /** Generation increments at the UI recording edge without lowering the sequence watermark. */ + generation: number; + /** Reject deltas until the backend announces the matching session start. */ + awaitingSessionStart: boolean; + /** True only after the start event for the session that will own the next recording edge. */ + sessionStartObserved: boolean; +} + +export function createTranscriptViewState(): TranscriptViewState { + return { + sessionId: null, + sequence: 0, + text: '', + revision: 0, + generation: 0, + awaitingSessionStart: false, + sessionStartObserved: false, + }; +} + +/** + * Start a new UI transcript generation while retaining the global event watermark. + * A delayed delta from the previous session is rejected until a backend start event + * supplies the new session id, so clearing text never re-opens the old-session gate. + */ +export function beginTranscriptGeneration(state: TranscriptViewState): TranscriptViewState { + const reuseStartedSession = + state.sessionStartObserved && state.sessionId !== null && !state.awaitingSessionStart; + return { + ...state, + sessionId: reuseStartedSession ? state.sessionId : null, + // Tauri can deliver backend:event(starting) and transcript_delta before the + // capsule:state(recording) edge. Reusing that observed session must not + // replay the whole transcript as a new generation or erase its prefix. + text: reuseStartedSession ? state.text : '', + generation: state.generation + 1, + awaitingSessionStart: !reuseStartedSession, + sessionStartObserved: false, + }; +} + +export function clearTranscriptText(state: TranscriptViewState): TranscriptViewState { + return { ...state, text: '', sessionStartObserved: false }; +} + +export interface TranscriptSnapshot { + transcript: string; + sessionId: string | null; + sequence: number; + revision: number; + /** Native payload commit revision; absent only for legacy test/mock snapshots. */ + payloadRevision?: number; +} + +/** A snapshot is replayable only when its payload and text share one revision. */ +export function isTranscriptSnapshotCoherent(snapshot: TranscriptSnapshot): boolean { + return (snapshot.payloadRevision ?? snapshot.revision) === snapshot.revision; +} + +/** + * A torn snapshot whose watermark is at least as new as the live state may + * become coherent on the next native frame. Older torn snapshots are already + * superseded and should be ignored without spinning the ready handshake. + */ +export function shouldRetryTranscriptSnapshot( + state: TranscriptViewState, + snapshot: TranscriptSnapshot, +): boolean { + if (isTranscriptSnapshotCoherent(snapshot)) return false; + return ( + snapshot.revision >= state.revision && + !(snapshot.revision === state.revision && snapshot.sequence < state.sequence) + ); +} + +/** Apply a session watermark carried by capsule:state before backend:event. */ +export function applyTranscriptSessionId( + state: TranscriptViewState, + sessionId: string | null | undefined, +): TranscriptViewState { + if (!sessionId || sessionId === state.sessionId) return state; + return { + ...state, + sessionId, + text: '', + generation: state.generation + 1, + awaitingSessionStart: false, + sessionStartObserved: true, + }; +} + +/** + * Apply the native rail replay only when its two-part watermark is not older + * than the live event stream. Keeping this predicate beside event reduction + * makes the ready-handshake race deterministic and directly testable. + */ +export function applyTranscriptSnapshot( + state: TranscriptViewState, + snapshot: TranscriptSnapshot, +): TranscriptViewState { + const payloadRevision = snapshot.payloadRevision ?? snapshot.revision; + // Native payload and transcript are committed by different Tauri/main-thread + // callbacks. Never let a torn frame (new text + old idle/terminal payload, or + // the reverse) clear/replace the rail; the next ready replay/event will carry + // a matching commit revision. + if (payloadRevision !== snapshot.revision) return state; + const stale = + snapshot.revision < state.revision || + (snapshot.revision === state.revision && snapshot.sequence < state.sequence); + if (stale) return state; + const sessionChanged = + snapshot.sessionId !== null && snapshot.sessionId !== state.sessionId; + return { + ...state, + sessionId: snapshot.sessionId, + sequence: snapshot.sequence, + revision: snapshot.revision, + text: snapshot.transcript, + generation: sessionChanged ? state.generation + 1 : state.generation, + awaitingSessionStart: false, + // A later capsule:state(recording) edge may still need to reuse this + // session after the rail snapshot has replayed it. + sessionStartObserved: snapshot.sessionId !== null, + }; } export function applyTranscriptEvent( @@ -23,31 +149,74 @@ export function applyTranscriptEvent( if (event.sequence <= state.sequence) return state; if (event.kind.type === 'dictation_state_changed') { const payload = event.kind.payload as { phase?: string; sessionId?: string | null } | undefined; - if (payload?.phase === 'starting' && payload.sessionId) { - return { sessionId: payload.sessionId, sequence: event.sequence, text: '' }; + const sessionId = payload?.sessionId ?? event.sessionId; + const sessionChanged = Boolean(sessionId && sessionId !== state.sessionId); + if (sessionChanged) { + return { + ...state, + sessionId, + sequence: event.sequence, + revision: Math.max(state.revision, event.sequence), + text: '', + awaitingSessionStart: false, + sessionStartObserved: true, + // The normal starting -> capsule:state(recording) path increments at + // the UI recording edge. Do not count that same generation twice; + // Recording -> Recording with a different sessionId still increments + // below because it has no separate start edge to rely on. + generation: payload?.phase === 'starting' ? state.generation : state.generation + 1, + }; } - return { ...state, sequence: event.sequence }; + return { + ...state, + sessionId: sessionId ?? state.sessionId, + sequence: event.sequence, + revision: Math.max(state.revision, event.sequence), + awaitingSessionStart: payload?.phase === 'starting' ? false : state.awaitingSessionStart, + sessionStartObserved: payload?.phase === 'starting' ? true : state.sessionStartObserved, + }; } if (event.kind.type === 'selection_voice_state_changed') { const payload = event.kind.payload as { phase?: string; sessionId?: string | null } | undefined; - if (payload?.phase === 'recording') { + const sessionId = payload?.sessionId ?? event.sessionId; + const sessionChanged = Boolean(sessionId && sessionId !== state.sessionId); + if (sessionChanged) { return { - sessionId: payload.sessionId ?? event.sessionId, + ...state, + sessionId, sequence: event.sequence, + revision: Math.max(state.revision, event.sequence), text: '', + awaitingSessionStart: false, + sessionStartObserved: true, + generation: state.generation + 1, }; } - return { ...state, sequence: event.sequence }; + return { + ...state, + sessionId: sessionId ?? state.sessionId, + sequence: event.sequence, + revision: Math.max(state.revision, event.sequence), + awaitingSessionStart: + payload?.phase === 'recording' ? false : state.awaitingSessionStart, + sessionStartObserved: + payload?.phase === 'recording' ? true : state.sessionStartObserved, + }; + } + if (event.kind.type !== 'transcript_delta') { + return { ...state, sequence: event.sequence, revision: Math.max(state.revision, event.sequence) }; } - if (event.kind.type !== 'transcript_delta') return { ...state, sequence: event.sequence }; + if (state.awaitingSessionStart) return state; if (state.sessionId !== null && event.sessionId !== state.sessionId) return state; const delta = event.kind.payload as TranscriptDelta | undefined; if (!delta || !Number.isSafeInteger(delta.offset) || delta.offset < 0) return state; const current = Array.from(state.text); if (delta.offset > current.length) return state; return { + ...state, sessionId: event.sessionId, sequence: event.sequence, + revision: Math.max(state.revision, event.sequence), text: current.slice(0, delta.offset).join('') + delta.text, }; } diff --git a/openless-all/app/src/lib/capsuleLayout.test.ts b/openless-all/app/src/lib/capsuleLayout.test.ts index f33307f10..f81e19da4 100644 --- a/openless-all/app/src/lib/capsuleLayout.test.ts +++ b/openless-all/app/src/lib/capsuleLayout.test.ts @@ -2,6 +2,11 @@ import { getCapsuleHostMetrics, getCapsuleMessageLayout, getCapsulePillMetrics, + getCapsuleTranscriptRailHeight, + getCapsuleTranscriptRailPosition, + getClassicCapsuleGeometry, + getClassicProcessingLabel, + getClassicTranslationBadgeOffset, parseCapsuleStyle, } from './capsuleLayout.ts'; @@ -11,6 +16,12 @@ function assertEqual(actual: T, expected: T, name: string) { } } +function assertClose(actual: number, expected: number, name: string) { + if (Math.abs(actual - expected) > 1e-9) { + throw new Error(`${name}: expected ${expected}, got ${actual}`); + } +} + const winMetrics = getCapsulePillMetrics('win'); assertEqual(winMetrics.width, 460, 'windows voice orb stage uses the demo-scale width'); assertEqual(winMetrics.height, 180, 'windows voice orb stage uses the demo-scale height'); @@ -23,7 +34,11 @@ assertEqual( const winHost = getCapsuleHostMetrics('win', false); assertEqual(winHost.width, 460, 'windows voice orb host matches stage width'); -assertEqual(winHost.height, 180, 'windows voice orb host matches stage height'); +assertEqual( + winHost.height, + 228, + 'windows voice orb host reserves the transcript rail above the stage', +); assertEqual(winHost.horizontalInset, 0, 'windows voice orb host has no side button inset'); assertEqual(winHost.boxSizing, 'border-box', 'windows voice orb host keeps border-box sizing'); assertEqual( @@ -32,9 +47,9 @@ assertEqual( 'windows voice orb host keeps the visible stage width after reserving side insets', ); assertEqual( - winHost.height - winHost.bottomInset, + winHost.height - 48 - winHost.bottomInset, winMetrics.height, - 'windows voice orb host keeps the visible stage height after reserving bottom inset', + 'windows voice orb host keeps the visible stage height below the transcript rail', ); const winHostWithTranslation = getCapsuleHostMetrics('win', true); @@ -45,8 +60,8 @@ assertEqual( ); assertEqual( winHostWithTranslation.height, - 180, - 'windows translation voice orb keeps the same outer height', + 228, + 'windows translation voice orb keeps the transcript rail height', ); assertEqual( winHostWithTranslation.horizontalInset, @@ -67,7 +82,7 @@ assertEqual(macMetrics.boxSizing, 'border-box', 'mac voice orb stage keeps borde const macHost = getCapsuleHostMetrics('mac', false); assertEqual(macHost.width, 460, 'mac voice orb host matches stage width'); -assertEqual(macHost.height, 180, 'mac voice orb host matches stage height'); +assertEqual(macHost.height, 228, 'mac voice orb host reserves the transcript rail above the stage'); assertEqual(macHost.boxSizing, 'border-box', 'mac voice orb host keeps border-box sizing'); const winErrorLayout = getCapsuleMessageLayout('win', 'error'); @@ -85,14 +100,109 @@ assertEqual(macErrorLayout.allowWrap, false, 'mac error message stays nowrap'); for (const os of ['mac', 'win'] as const) { const classic = getCapsuleHostMetrics(os, false, 'classic'); const typeless = getCapsuleHostMetrics(os, true, 'typeless'); - assertEqual(classic.height, 100, `${os}: classic uses the compact native window`); - assertEqual(typeless.height, 57, `${os}: typeless window is 1/5 of the old 460x128 area`); + assertEqual( + classic.height, + 172, + `${os}: classic reserves the transcript rail and translating badge`, + ); + assertEqual(typeless.height, 65, `${os}: typeless translation host reserves a badge row`); assertEqual(typeless.width, 206, `${os}: typeless window keeps the 1/5 stage width`); assertEqual(typeless.bottomInset, 0, `${os}: typeless pill hugs the work-area bottom edge`); + assertEqual( + getCapsuleHostMetrics(os, false, 'typeless').height, + 57, + `${os}: typeless host stays compact when translation is hidden`, + ); } for (const style of ['siri', 'classic', 'typeless'] as const) { assertEqual(parseCapsuleStyle(style), style, `${style} is accepted from preferences and events`); } +assertEqual(getCapsuleTranscriptRailHeight('siri'), 40, 'Siri rail height matches native overlay'); +assertEqual(getCapsuleTranscriptRailHeight('classic'), 52, 'Classic rail height matches native overlay'); +assertEqual(getCapsuleTranscriptRailHeight('typeless'), 52, 'Typeless logical rail height matches CSS overlay'); +assertEqual( + getClassicTranslationBadgeOffset(true), + 60, + 'Classic translation badge clears the external 52px rail and its 8px gap', +); +assertEqual( + getClassicTranslationBadgeOffset(false), + 0, + 'Classic translation badge does not reserve a hidden rail', +); +assertEqual( + getClassicTranslationBadgeOffset(true, false), + 0, + 'Classic external rail uses its own badge lane instead of same-window offset', +); +const classicExternalGeometry = getClassicCapsuleGeometry('win', true, true, false); +assertEqual(classicExternalGeometry.hostHeight, 172, 'Classic external rail uses the 172px host'); +assertEqual(classicExternalGeometry.bodyTop, 104, 'Classic body is anchored at the bottom of the host'); +assertEqual(classicExternalGeometry.railTop, 14, 'Classic external rail clears the translation badge lane'); +assertEqual(classicExternalGeometry.badgeTop, 74, 'Classic external badge sits above the pill'); +assertEqual(classicExternalGeometry.badgeBottom, 96, 'Classic external badge leaves an 8px body gap'); +if ( + classicExternalGeometry.railTop! + classicExternalGeometry.railHeight > classicExternalGeometry.badgeTop || + classicExternalGeometry.badgeBottom > classicExternalGeometry.bodyTop +) { + throw new Error('Classic external rail and translation badge must not overlap'); +} +const classicSameWindowGeometry = getClassicCapsuleGeometry('win', true, true, true); +assertEqual(classicSameWindowGeometry.railTop, 44, 'Classic same-window rail keeps the body rail anchor'); +assertEqual(classicSameWindowGeometry.badgeBottom, 36, 'Classic same-window badge clears the rail'); +for (const os of ['mac', 'win'] as const) { + const external = getClassicCapsuleGeometry(os, true, true, false); + if (external.railTop! + external.railHeight > external.badgeTop || external.badgeBottom > external.bodyTop) { + throw new Error(`${os}: external Classic rail and badge overlap`); + } +} +assertEqual( + getClassicProcessingLabel(false), + 'capsule.thinking', + 'Classic processing uses thinking copy for ordinary dictation', +); +assertEqual( + getClassicProcessingLabel(true), + 'capsule.using', + 'Classic processing uses using copy for operating sessions', +); +assertEqual( + getCapsuleTranscriptRailPosition('siri', false, true)?.topOffset, + -48, + 'Siri rail sits above the 180px body with its 8px CSS gap', +); +assertEqual( + getCapsuleTranscriptRailPosition('siri', true, true)?.topOffset, + -48, + 'Siri translation does not move the full-height body rail anchor', +); +assertEqual( + getCapsuleTranscriptRailPosition('classic', false, true)?.topOffset, + 44, + 'Classic compact external rail anchors to the 172px body content top', +); +assertEqual( + getCapsuleTranscriptRailPosition('classic', true, true)?.topOffset, + 14, + 'Classic translation external rail clears the explicit badge lane', +); +const typelessCompactRail = getCapsuleTranscriptRailPosition('typeless', false, true); +const typelessTranslationRail = getCapsuleTranscriptRailPosition('typeless', true, true); +if (!typelessCompactRail || !typelessTranslationRail) throw new Error('typeless rail geometry missing'); +assertClose(typelessCompactRail.height, 52 * 0.447, 'Typeless rail uses the zoomed native height'); +assertClose(typelessCompactRail.topOffset, 57 - 64 * 0.447 - 52 * 0.447, 'Typeless compact body anchor'); +assertClose( + typelessTranslationRail.topOffset, + 65 - 20 * 0.447 - 64 * 0.447 - 52 * 0.447, + 'Typeless translation body anchor includes the translation row', +); +for (const style of ['siri', 'classic', 'typeless'] as const) { + assertEqual( + getCapsuleTranscriptRailPosition(style, false, false), + null, + `${style} hides the rail when transcript is not visible`, + ); +} assertEqual( parseCapsuleStyle('unknown'), undefined, diff --git a/openless-all/app/src/lib/capsuleLayout.ts b/openless-all/app/src/lib/capsuleLayout.ts index 92ccc9908..6cbd90d4b 100644 --- a/openless-all/app/src/lib/capsuleLayout.ts +++ b/openless-all/app/src/lib/capsuleLayout.ts @@ -30,12 +30,157 @@ export interface CapsuleMessageLayout { const VOICE_ORB_STAGE_WIDTH = 460; const VOICE_ORB_STAGE_HEIGHT = 180; const VOICE_ORB_TEXT_WIDTH = 400; +export const CAPSULE_TRANSCRIPT_RAIL_HEIGHT = 40; +export const CAPSULE_TRANSCRIPT_RAIL_GAP = 8; +export const CAPSULE_TRANSCRIPT_RAIL_HEIGHTS: Record = { + siri: 40, + classic: 52, + // Typeless is rendered in its 0.447 zoomed tree, but its CSS rail still owns + // the same 52px logical row as the standalone Windows overlay. + typeless: 52, +}; + +export const CLASSIC_CAPSULE_HOST_HEIGHT = 172; +export const CLASSIC_CAPSULE_BOTTOM_INSET = 16; +export const CLASSIC_CAPSULE_BADGE_HEIGHT = 22; +export const CLASSIC_CAPSULE_BADGE_GAP = 8; +export const CLASSIC_CAPSULE_RAIL_GAP = 8; + +export interface ClassicCapsuleGeometry { + hostHeight: number; + bodyTop: number; + bodyBottom: number; + badgeTop: number; + badgeBottom: number; + badgeBottomOffset: number; + railTop: number | null; + railHeight: number; +} + +/** + * Classic uses one 172px layout coordinate system on desktop. The body, badge + * lane, and a sibling rail each get an explicit vertical anchor so an external + * rail is not treated as a missing same-window rail. + */ +export function getClassicCapsuleGeometry( + os: OS, + transcriptVisible: boolean, + translationActive: boolean, + transcriptInSameWindow: boolean, +): ClassicCapsuleGeometry { + const pillHeight = os === 'win' ? 52 : 42; + const hostHeight = CLASSIC_CAPSULE_HOST_HEIGHT; + const bodyBottom = hostHeight - CLASSIC_CAPSULE_BOTTOM_INSET; + const bodyTop = bodyBottom - pillHeight; + const railHeight = getCapsuleTranscriptRailHeight('classic'); + const sameWindowRail = transcriptVisible && transcriptInSameWindow; + const externalBadgeLane = transcriptVisible && !transcriptInSameWindow && translationActive; + const railTop = transcriptVisible + ? bodyTop - + CLASSIC_CAPSULE_RAIL_GAP - + railHeight - + (externalBadgeLane ? CLASSIC_CAPSULE_BADGE_HEIGHT + CLASSIC_CAPSULE_BADGE_GAP : 0) + : null; + const badgeBottom = + bodyTop - + CLASSIC_CAPSULE_BADGE_GAP - + (sameWindowRail && translationActive ? railHeight + CLASSIC_CAPSULE_RAIL_GAP : 0); + const badgeTop = badgeBottom - CLASSIC_CAPSULE_BADGE_HEIGHT; + return { + hostHeight, + bodyTop, + bodyBottom, + badgeTop, + badgeBottom, + badgeBottomOffset: hostHeight - badgeBottom, + railTop, + railHeight, + }; +} + +export function getCapsuleTranscriptRailHeight(style: CapsuleStyle): number { + return CAPSULE_TRANSCRIPT_RAIL_HEIGHTS[style]; +} + +/** Classic's translating badge lives above the body and must also clear the + * external 52px rail when the rail is hosted in a sibling HWND. */ +export function getClassicTranslationBadgeOffset( + transcriptVisible: boolean, + transcriptInSameWindow = true, +): number { + const geometry = getClassicCapsuleGeometry('win', transcriptVisible, true, transcriptInSameWindow); + const baseBadgeBottomOffset = + CLASSIC_CAPSULE_HOST_HEIGHT - geometry.bodyTop + CLASSIC_CAPSULE_BADGE_GAP; + return geometry.badgeBottomOffset - baseBadgeBottomOffset; +} + +export function getClassicProcessingLabel(operating: boolean): 'capsule.thinking' | 'capsule.using' { + return operating ? 'capsule.using' : 'capsule.thinking'; +} + +export interface CapsuleTranscriptRailPosition { + width: number; + height: number; + /** Rail top relative to the capsule body HWND top, in logical/native pixels. */ + topOffset: number; + /** CSS gap between the rail and the capsule's actual body content. */ + gap: number; +} + +const TYPELESS_ZOOM = 0.447; +const TYPELESS_CAPSULE_HEIGHT = 64 * TYPELESS_ZOOM; +const TYPELESS_TRANSLATION_ROW_HEIGHT = 20 * TYPELESS_ZOOM; +const TYPELESS_RAIL_HEIGHT = 52 * TYPELESS_ZOOM; + +/** + * Position the standalone Windows rail against the actual capsule body, not + * against the transparent HWND edge. Keep this in sync with the native helper + * of the same name in tauri_coordinator_host.rs. + */ +export function getCapsuleTranscriptRailPosition( + style: CapsuleStyle, + translationActive: boolean, + transcriptVisible: boolean, +): CapsuleTranscriptRailPosition | null { + if (!transcriptVisible) return null; + if (style === 'siri') { + const bodyTop = 0; + const railHeight = 40; + const gap = 8; + return { width: 460, height: railHeight, topOffset: bodyTop - gap - railHeight, gap }; + } + if (style === 'classic') { + const geometry = getClassicCapsuleGeometry('win', true, translationActive, false); + const gap = CLASSIC_CAPSULE_RAIL_GAP; + return { + width: 460, + height: geometry.railHeight, + topOffset: geometry.railTop ?? 0, + gap, + }; + } + const hostHeight = translationActive ? 65 : 57; + const bodyBottom = hostHeight; + const bodyTop = bodyBottom - TYPELESS_CAPSULE_HEIGHT - (translationActive ? TYPELESS_TRANSLATION_ROW_HEIGHT : 0); + const gap = 0; + return { + width: 206, + height: TYPELESS_RAIL_HEIGHT, + topOffset: bodyTop - gap - TYPELESS_RAIL_HEIGHT, + gap, + }; +} + +const SIRI_HOST_HEIGHT = + VOICE_ORB_STAGE_HEIGHT + CAPSULE_TRANSCRIPT_RAIL_HEIGHT + CAPSULE_TRANSCRIPT_RAIL_GAP; +const CLASSIC_HOST_HEIGHT = CLASSIC_CAPSULE_HOST_HEIGHT; // The typeless window is 1/5 the area of the original size (460×128); content is scaled by the // zoom in CapsuleStyles.css, kept in sync with capsule_window_bounds_for_style in // src-tauri/src/lib.rs. const TYPELESS_STAGE_WIDTH = 206; const TYPELESS_STAGE_HEIGHT = 57; +const TYPELESS_TRANSLATION_HOST_HEIGHT = 65; export function parseCapsuleStyle(value: unknown): CapsuleStyle | undefined { return value === 'siri' || value === 'classic' || value === 'typeless' ? value : undefined; @@ -56,11 +201,10 @@ export function getCapsuleHostMetrics( translationActive: boolean, style: CapsuleStyle = 'siri', ): CapsuleHostMetrics { - void translationActive; if (style === 'typeless') { return { width: TYPELESS_STAGE_WIDTH, - height: TYPELESS_STAGE_HEIGHT, + height: translationActive ? TYPELESS_TRANSLATION_HOST_HEIGHT : TYPELESS_STAGE_HEIGHT, horizontalInset: 0, bottomInset: 0, badgeGap: 8, @@ -70,7 +214,7 @@ export function getCapsuleHostMetrics( const stage = getCapsulePillMetrics(os); return { width: stage.width, - height: style === 'siri' ? stage.height : style === 'classic' ? 100 : 128, + height: style === 'siri' ? SIRI_HOST_HEIGHT : style === 'classic' ? CLASSIC_HOST_HEIGHT : 128, horizontalInset: 0, bottomInset: style === 'siri' ? 0 : 16, badgeGap: 8, diff --git a/openless-all/app/src/lib/insertTextAnimation.test.ts b/openless-all/app/src/lib/insertTextAnimation.test.ts index 000d1ce52..efce54c4b 100644 --- a/openless-all/app/src/lib/insertTextAnimation.test.ts +++ b/openless-all/app/src/lib/insertTextAnimation.test.ts @@ -3,6 +3,7 @@ import { appendedBirthAdvance, clampInsertContentWidth, diffInsertUnits, + diffInsertWindowUnits, firstBornIndex, planCapsuleInsertMotion, planCharDelays, @@ -10,6 +11,8 @@ import { propagationDelayMs, resetInsertUnitKeys, rightAnchoredPositions, + selectInsertRenderWindow, + selectInsertTextWindow, segmentInsertUnits, type InsertUnit, } from './insertTextAnimation'; @@ -180,3 +183,100 @@ assertEqual( 32, 'append enters beyond old tail', ); + +const longUnits = diffInsertUnits([], 'abcdefghijklmnop'); +let measuredGlyphs = 0; +const boundedWindow = selectInsertRenderWindow( + longUnits, + () => { + measuredGlyphs += 1; + return 10; + }, + 58, + 8, + { maxUnits: 6, overflowBufferUnits: 2 }, +); +assertEqual( + boundedWindow.units.map((unit) => unit.text).join(''), + 'klmnop', + 'window keeps latest suffix', +); +assertEqual( + boundedWindow.units.length, + 6, + 'window includes only a bounded visible suffix and buffer', +); +assertEqual( + boundedWindow.widths.length, + boundedWindow.units.length, + 'window carries measured widths', +); +assertEqual(measuredGlyphs, boundedWindow.units.length, 'window does not measure discarded glyphs'); +assert(measuredGlyphs < longUnits.length, 'discarded glyphs are never measured'); +assert( + selectInsertRenderWindow(longUnits, () => 10, 58, 8, { maxUnits: 3 }).units.length <= 3, + 'hard max keeps motion nodes bounded even for wide glyphs', +); + +console.log('bounded insert render window passed'); + +const longTranscript = '0123456789'.repeat(200); +const textWindow = selectInsertTextWindow(longTranscript, { maxUnits: 8, overflowBufferUnits: 2 }); +assertEqual(textWindow, '0123456789', 'text window keeps the latest bounded graphemes'); + +let rollingUnits: InsertUnit[] = []; +for (let length = 1; length <= longTranscript.length; length += 1) { + const previousUnits = rollingUnits; + rollingUnits = diffInsertWindowUnits(previousUnits, longTranscript.slice(0, length), { + maxUnits: 8, + overflowBufferUnits: 2, + }); + assert(rollingUnits.length <= 10, 'rolling animation units stay bounded during long transcript'); + assertEqual( + rollingUnits.filter((unit) => unit.born).length, + 1, + `rolling append only births the newest glyph at length ${length}`, + ); + const previousByOffset = new Map( + previousUnits + .filter((unit) => unit.sourceOffset != null) + .map((unit) => [unit.sourceOffset as number, unit.key]), + ); + for (const unit of rollingUnits) { + if (unit.sourceOffset != null && previousByOffset.has(unit.sourceOffset)) { + assertEqual( + unit.key, + previousByOffset.get(unit.sourceOffset), + `rolling overlap reuses key at offset ${unit.sourceOffset}`, + ); + } + } +} +assertEqual( + rollingUnits.map((unit) => unit.text).join(''), + '0123456789', + 'rolling window ends at the current transcript tail', +); + +const correctionSource = 'a'.repeat(40) + 'weather today'; +let correctionUnits = diffInsertWindowUnits([], correctionSource, { maxUnits: 20 }); +const stableTailKey = correctionUnits[correctionUnits.length - 1]?.key; + correctionUnits = diffInsertWindowUnits(correctionUnits, correctionSource.replace('weather', 'whether'), { + maxUnits: 20, +}); +assertEqual( + correctionUnits[correctionUnits.length - 1]?.key, + stableTailKey, + 'window correction preserves keys for the unchanged visible suffix', +); +assertEqual( + correctionUnits.filter((unit) => unit.born).length, + 2, + 'window correction only births the changed graphemes', +); +assert( + correctionUnits.length <= 23, + 'window correction does not restore the discarded transcript history', +); + +console.log('bounded transcript diff passed'); diff --git a/openless-all/app/src/lib/insertTextAnimation.ts b/openless-all/app/src/lib/insertTextAnimation.ts index 06f46aa6d..230a3391a 100644 --- a/openless-all/app/src/lib/insertTextAnimation.ts +++ b/openless-all/app/src/lib/insertTextAnimation.ts @@ -35,6 +35,8 @@ export interface InsertUnit { key: string; text: string; born: boolean; + /** UTF-16 offset in the canonical transcript, when this unit came from a bounded window. */ + sourceOffset?: number; } let insertUnitSeq = 0; @@ -106,6 +108,203 @@ export function diffInsertUnits(prev: InsertUnit[], nextText: string): InsertUni return units; } +export const INSERT_TEXT_RENDER_WINDOW = { + maxUnits: 64, + overflowBufferUnits: 3, +} as const; + +export interface InsertTextWindowOptions { + maxUnits?: number; + overflowBufferUnits?: number; +} + +function normalizedInsertTextWindowLimit( + options: InsertTextWindowOptions = {}, +): { maxUnits: number; targetUnits: number } { + const maxUnits = Math.max(1, Math.floor(options.maxUnits ?? INSERT_TEXT_RENDER_WINDOW.maxUnits)); + const overflowBufferUnits = Math.max( + 0, + Math.floor(options.overflowBufferUnits ?? INSERT_TEXT_RENDER_WINDOW.overflowBufferUnits), + ); + return { maxUnits, targetUnits: maxUnits + overflowBufferUnits }; +} + +/** + * Return only the suffix needed to seed the bounded animation list. + * + * The probe starts near the end of the string and expands only when a grapheme is unusually + * large. This keeps the ordinary streaming path independent of the already-scrolled transcript; + * the full string remains available to the caller for recognition corrections and accessibility. + */ +export function selectInsertTextWindow( + text: string, + options: InsertTextWindowOptions = {}, +): string { + return selectInsertTextWindowSlice(text, options).text; +} + +interface InsertTextWindowGlyph { + text: string; + sourceOffset: number; +} + +interface InsertTextWindowSlice { + text: string; + glyphs: InsertTextWindowGlyph[]; +} + +function selectInsertTextWindowSlice( + text: string, + options: InsertTextWindowOptions = {}, +): InsertTextWindowSlice { + if (!text) return { text: '', glyphs: [] }; + const { targetUnits } = normalizedInsertTextWindowLimit(options); + let probeLength = Math.min(text.length, Math.max(32, targetUnits * 4)); + + while (true) { + const start = text.length - probeLength; + const segmented = segmentInsertUnits(text.slice(start)); + // A bounded slice can start in the middle of a grapheme. Drop that guard segment; when the + // slice starts at zero there is no preceding context and the first segment is trustworthy. + const safeSegments = start === 0 ? segmented : segmented.slice(1); + if (safeSegments.length >= targetUnits || start === 0) { + const windowSegments = safeSegments.slice(-targetUnits); + const safeStartOffset = start + (start === 0 ? 0 : segmented[0]?.length ?? 0); + const discardedUnits = safeSegments.length - windowSegments.length; + let windowStartOffset = safeStartOffset; + for (let i = 0; i < discardedUnits; i += 1) { + windowStartOffset += safeSegments[i].length; + } + const glyphs: InsertTextWindowGlyph[] = []; + let sourceOffset = windowStartOffset; + for (const glyph of windowSegments) { + glyphs.push({ text: glyph, sourceOffset }); + sourceOffset += glyph.length; + } + return { text: windowSegments.join(''), glyphs }; + } + const nextProbeLength = Math.min(text.length, Math.max(probeLength + 32, probeLength * 2)); + if (nextProbeLength === probeLength) { + const windowSegments = safeSegments.slice(-targetUnits); + const safeStartOffset = start + (start === 0 ? 0 : segmented[0]?.length ?? 0); + const discardedUnits = safeSegments.length - windowSegments.length; + let windowStartOffset = safeStartOffset; + for (let i = 0; i < discardedUnits; i += 1) { + windowStartOffset += safeSegments[i].length; + } + const glyphs: InsertTextWindowGlyph[] = []; + let sourceOffset = windowStartOffset; + for (const glyph of windowSegments) { + glyphs.push({ text: glyph, sourceOffset }); + sourceOffset += glyph.length; + } + return { text: windowSegments.join(''), glyphs }; + } + probeLength = nextProbeLength; + } +} + +/** + * Diff only the bounded suffix used by the motion track. The caller may keep the full nextText, + * but no complete-history InsertUnit array is created or compared here. + */ +export function diffInsertWindowUnits( + prev: InsertUnit[], + nextText: string, + options: InsertTextWindowOptions = {}, +): InsertUnit[] { + const nextWindow = selectInsertTextWindowSlice(nextText, options); + if (nextWindow.glyphs.length === 0) return []; + + // The suffix text is not a stable identity: once the window rolls, every glyph changes index + // even though all but the newest glyph are still on screen. Match the bounded overlap by its + // absolute UTF-16 offset first, so a scrolling deque reuses the old keys instead of replaying + // the born animation for the whole window. + const previousByOffset = new Map(); + for (const unit of prev) { + if (unit.sourceOffset != null) previousByOffset.set(unit.sourceOffset, unit); + } + let exactOverlap = 0; + for (const glyph of nextWindow.glyphs) { + const previous = previousByOffset.get(glyph.sourceOffset); + if (previous?.text === glyph.text) exactOverlap += 1; + } + + if (exactOverlap > 0 || prev.length === 0) { + return nextWindow.glyphs.map((glyph) => { + const previous = previousByOffset.get(glyph.sourceOffset); + if (previous?.text === glyph.text) { + return { ...previous, sourceOffset: glyph.sourceOffset, born: false }; + } + return { + key: allocInsertKey(), + text: glyph.text, + sourceOffset: glyph.sourceOffset, + born: true, + }; + }); + } + + // A recognition correction can change the UTF-16 length before the bounded suffix. In that + // case absolute offsets move together; the bounded local diff still preserves its common + // prefix/suffix without ever inspecting the discarded transcript history. + return diffInsertUnits(prev, nextWindow.text).map((unit, index) => ({ + ...unit, + sourceOffset: nextWindow.glyphs[index]?.sourceOffset, + })); +} + +export interface InsertRenderWindow { + units: InsertUnit[]; + widths: number[]; +} + +/** + * Select only the rightmost glyphs needed by the visual track. + * + * The input list is already a bounded animation history. A few extra units form a small left-side + * buffer for the mask/spring without allowing DOM growth. + */ +export function selectInsertRenderWindow( + units: InsertUnit[], + measureWidth: (text: string) => number, + maxWidth: number, + padX: number = INSERT_TEXT_MOTION.padX, + options: { + maxUnits?: number; + overflowBufferUnits?: number; + } = {}, +): InsertRenderWindow { + if (units.length === 0) return { units: [], widths: [] }; + + const innerWidth = Math.max(0, maxWidth - padX * 2); + const maxUnits = Math.max(1, Math.floor(options.maxUnits ?? INSERT_TEXT_RENDER_WINDOW.maxUnits)); + const overflowBufferUnits = Math.max( + 0, + Math.floor(options.overflowBufferUnits ?? INSERT_TEXT_RENDER_WINDOW.overflowBufferUnits), + ); + const selectedWidths: number[] = []; + let start = units.length; + let contentWidth = 0; + let bufferUnits = 0; + + for (let index = units.length - 1; index >= 0 && units.length - index <= maxUnits; index -= 1) { + const measured = measureWidth(units[index].text); + const width = Number.isFinite(measured) && measured > 0 ? measured : 1; + const stillVisible = contentWidth < innerWidth; + if (!stillVisible && bufferUnits >= overflowBufferUnits) break; + start = index; + selectedWidths.unshift(width); + contentWidth += width; + if (contentWidth >= innerWidth) bufferUnits += 1; + } + + return { + units: units.slice(start), + widths: selectedWidths, + }; +} + export function firstBornIndex(units: InsertUnit[]): number { const index = units.findIndex((unit) => unit.born); if (index >= 0) return index; diff --git a/openless-all/app/src/lib/ipc/dictation.ts b/openless-all/app/src/lib/ipc/dictation.ts index 1ed2bc890..619e7567f 100644 --- a/openless-all/app/src/lib/ipc/dictation.ts +++ b/openless-all/app/src/lib/ipc/dictation.ts @@ -1,4 +1,6 @@ +import { isDesktop } from './platform-exports'; import { invokeOrMock, platformCapabilities } from './shared'; +import type { CapsuleSnapshot } from '../types'; export function startDictation(): Promise { return invokeOrMock('start_dictation', undefined, () => undefined); @@ -12,6 +14,17 @@ export function cancelDictation(): Promise { return invokeOrMock('cancel_dictation', undefined, () => undefined); } +export function setCapsuleTranscriptVisible(visible: boolean): Promise { + if (!isDesktop()) return Promise.resolve(); + return invokeOrMock('set_capsule_transcript_visible', { visible }, () => undefined); +} + +/** Latest native capsule frame, used by the separate Windows rail after its webview is ready. */ +export function getCapsuleSnapshot(): Promise { + if (!isDesktop()) return Promise.resolve(null); + return invokeOrMock('get_capsule_snapshot', undefined, () => null); +} + export function handleWindowHotkeyEvent( eventType: 'keydown' | 'keyup', key: string, diff --git a/openless-all/app/src/lib/ipc/index.ts b/openless-all/app/src/lib/ipc/index.ts index 2114deb9d..72fa53984 100644 --- a/openless-all/app/src/lib/ipc/index.ts +++ b/openless-all/app/src/lib/ipc/index.ts @@ -96,6 +96,8 @@ export { startDictation, stopDictation, cancelDictation, + getCapsuleSnapshot, + setCapsuleTranscriptVisible, handleWindowHotkeyEvent, } from './dictation'; diff --git a/openless-all/app/src/lib/types.ts b/openless-all/app/src/lib/types.ts index d9cf11ab8..962cfa745 100644 --- a/openless-all/app/src/lib/types.ts +++ b/openless-all/app/src/lib/types.ts @@ -715,6 +715,8 @@ export type CapsuleState = export type CapsuleStyle = 'siri' | 'classic' | 'typeless'; export interface CapsulePayload { + /** Session generation that owns this capsule frame. */ + sessionId?: string | null; state: CapsuleState; level: number; // 0..1 RMS elapsedMs: number; @@ -743,6 +745,23 @@ export interface CapsulePayload { selectionPolish?: boolean; } +/** + * Native replay frame used when the standalone Windows transcript rail mounts + * after the capsule has already started. The payload fields are deliberately + * flattened so the rail can restore presentation state and transcript state + * from one revisioned snapshot. + */ +export interface CapsuleSnapshot extends CapsulePayload { + transcript: string; + sessionId: string | null; + /** Backend event watermark for rejecting delayed events from an old session. */ + sequence: number; + /** Monotonic native snapshot revision; currently aligned with the event watermark. */ + revision: number; + /** Revision at which the flattened payload was committed. Torn frames are rejected. */ + payloadRevision: number; +} + export interface CredentialsStatus { activeAsrProvider: string; activeLlmProvider: string; diff --git a/openless-all/app/src/main.tsx b/openless-all/app/src/main.tsx index 52448c213..1fb581cd1 100644 --- a/openless-all/app/src/main.tsx +++ b/openless-all/app/src/main.tsx @@ -16,6 +16,7 @@ import type { OS } from './components/WindowChrome'; const params = new URLSearchParams(window.location.search); const windowKind = params.get('window'); const isCapsule = windowKind === 'capsule'; +const isCapsuleRail = windowKind === 'capsule-rail'; const isQa = windowKind === 'qa'; const isSelectionVoiceIntent = windowKind === 'selection-voice-intent'; const isLessComputer = windowKind === 'less-computer'; @@ -37,6 +38,7 @@ const renderApp = () => { {isMainWindow && } Date: Sun, 4 Oct 2026 23:03:10 +0800 Subject: [PATCH 2/5] fix(capsule): allow rail offset adjustment --- openless-all/app/src-tauri/src/tauri_coordinator_host.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openless-all/app/src-tauri/src/tauri_coordinator_host.rs b/openless-all/app/src-tauri/src/tauri_coordinator_host.rs index 27d306b95..d1b6ecffe 100644 --- a/openless-all/app/src-tauri/src/tauri_coordinator_host.rs +++ b/openless-all/app/src-tauri/src/tauri_coordinator_host.rs @@ -70,7 +70,7 @@ fn capsule_transcript_rail_position( let height = 52.0; let gap = RAIL_GAP; let external_badge_lane = translation_active; - let top_offset = body_top - RAIL_GAP - RAIL_HEIGHT; + let mut top_offset = body_top - RAIL_GAP - RAIL_HEIGHT; if external_badge_lane { top_offset -= BADGE_HEIGHT + BADGE_GAP; } From 4a7feea53ee7cd1089d3d0fe9855b35d49920302 Mon Sep 17 00:00:00 2001 From: HKLHaoBin Date: Mon, 5 Oct 2026 00:48:36 +0800 Subject: [PATCH 3/5] fix(capsule): wrap Windows client bounds calls --- openless-all/app/src-tauri/src/tauri_coordinator_host.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/openless-all/app/src-tauri/src/tauri_coordinator_host.rs b/openless-all/app/src-tauri/src/tauri_coordinator_host.rs index d1b6ecffe..f5d63a2aa 100644 --- a/openless-all/app/src-tauri/src/tauri_coordinator_host.rs +++ b/openless-all/app/src-tauri/src/tauri_coordinator_host.rs @@ -367,7 +367,9 @@ fn configure_capsule_hit_test( ))); } let mut client = RECT::default(); - GetClientRect(hwnd, &mut client).map_err(|error| tauri::Error::Anyhow(error.into()))?; + unsafe { + GetClientRect(hwnd, &mut client).map_err(|error| tauri::Error::Anyhow(error.into()))?; + } let Some(rect) = capsule_control_hit_rect( style, transcript_visible, @@ -406,7 +408,9 @@ fn configure_card_hit_test( ))); } let mut client = RECT::default(); - GetClientRect(hwnd, &mut client).map_err(|error| tauri::Error::Anyhow(error.into()))?; + unsafe { + GetClientRect(hwnd, &mut client).map_err(|error| tauri::Error::Anyhow(error.into()))?; + } let rect = capsule_card_hit_rect(client.right - client.left, client.bottom - client.top); clear_capsule_input_region(hwnd); install_capsule_hit_test_proc(hwnd, rect) From 675bf95f6d88ef33d84d0d260a342f54b85daa4d Mon Sep 17 00:00:00 2001 From: HKLHaoBin Date: Mon, 5 Oct 2026 01:45:50 +0800 Subject: [PATCH 4/5] test(capsule): harden card restore contract --- .../macos-capsule-spaces-contract.test.mjs | 184 +++++++++++++++--- 1 file changed, 152 insertions(+), 32 deletions(-) diff --git a/openless-all/app/scripts/macos-capsule-spaces-contract.test.mjs b/openless-all/app/scripts/macos-capsule-spaces-contract.test.mjs index 0e5d51747..b8074b90b 100644 --- a/openless-all/app/scripts/macos-capsule-spaces-contract.test.mjs +++ b/openless-all/app/scripts/macos-capsule-spaces-contract.test.mjs @@ -105,20 +105,153 @@ assertMatch( const coordinatorRs = ( await readFile(new URL('../src-tauri/src/coordinator.rs', import.meta.url), 'utf-8') ).replace(/\r\n/g, '\n'); -assertMatch( - coordinatorHostRs, - /fn set_cursor_passthrough[\s\S]*?window\.set_ignore_cursor_events\(passthrough\)[\s\S]*?cursor_passthrough[\s\S]*?store\(passthrough, Ordering::SeqCst\)/, - 'the narrow capsule-window capability must update the Tauri window and its host-owned passthrough cache together', -); + +function extractBalancedBlock(source, openBrace) { + let depth = 0; + let inString = false; + let inChar = false; + let escaped = false; + let inLineComment = false; + let blockCommentDepth = 0; + let rawStringHashes = null; + for (let index = openBrace; index < source.length; index += 1) { + const char = source[index]; + const next = source[index + 1]; + if (inLineComment) { + if (char === '\n') inLineComment = false; + continue; + } + if (blockCommentDepth > 0) { + if (char === '/' && next === '*') { + blockCommentDepth += 1; + index += 1; + } else if (char === '*' && next === '/') { + blockCommentDepth -= 1; + index += 1; + } + continue; + } + if (rawStringHashes !== null) { + const terminator = `"${'#'.repeat(rawStringHashes)}`; + if (source.startsWith(terminator, index)) { + index += terminator.length - 1; + rawStringHashes = null; + } + continue; + } + if (inString || inChar) { + const terminator = inString ? '"' : "'"; + if (escaped) { + escaped = false; + } else if (char === '\\') { + escaped = true; + } else if (char === terminator) { + inString = false; + inChar = false; + } + continue; + } + if (char === '/' && next === '/') { + inLineComment = true; + index += 1; + continue; + } + if (char === '/' && next === '*') { + blockCommentDepth = 1; + index += 1; + continue; + } + if (char === 'r') { + let hashEnd = index + 1; + while (source[hashEnd] === '#') hashEnd += 1; + if (source[hashEnd] === '"') { + rawStringHashes = hashEnd - index - 1; + index = hashEnd; + continue; + } + } + if (char === '"') { + inString = true; + continue; + } + // Rust lifetimes start with a letter after the apostrophe; only enter the + // character-literal state when the closing apostrophe is locally evident. + if (char === "'" && (next === '\\' || source[index + 2] === "'")) { + inChar = true; + continue; + } + if (char === '{') depth += 1; + if (char === '}' && --depth === 0) { + return source.slice(openBrace, index + 1); + } + } + throw new Error('unterminated Rust block'); +} function extractFn(source, name) { - const match = source.match(new RegExp(`(?:pub\\(crate\\) )?fn ${name}[\\s\\S]*?\\n}\\n`)); + const signature = new RegExp( + `(?:pub\\([^)]*\\)\\s*)?(?:async\\s+)?fn\\s+${name}(?:<[^>{}]*>)?\\s*\\(`, + ); + const match = signature.exec(source); if (!match) { - throw new Error(`${name}: function not found in coordinator.rs`); + throw new Error(`${name}: function not found`); + } + const openBrace = source.indexOf('{', match.index); + if (openBrace === -1) { + throw new Error(`${name}: function body not found`); } - return match[0]; + const body = extractBalancedBlock(source, openBrace); + return source.slice(match.index, openBrace) + body; } +function extractCfgBlock(source, target) { + const marker = `#[cfg(target_os = "${target}")]`; + const markerIndex = source.indexOf(marker); + if (markerIndex === -1) { + throw new Error(`${target}: cfg block not found`); + } + const openBrace = source.indexOf('{', markerIndex + marker.length); + if (openBrace === -1) { + throw new Error(`${target}: cfg block body not found`); + } + return source.slice(markerIndex, openBrace) + extractBalancedBlock(source, openBrace); +} + +const setCursorPassthroughRs = extractFn(coordinatorHostRs, 'set_cursor_passthrough'); +assertMatch( + setCursorPassthroughRs, + /window\.set_ignore_cursor_events\(passthrough\)[\s\S]*?cursor_passthrough[\s\S]*?store\(passthrough, Ordering::SeqCst\)/, + 'the narrow capsule-window capability must update the Tauri window and its host-owned passthrough cache together', +); +const setCardHitTestModeRs = extractFn(coordinatorHostRs, 'set_card_hit_test_mode'); +const restoreCapsuleHitTestModeRs = extractFn(coordinatorHostRs, 'restore_capsule_hit_test_mode'); +assertMatch( + setCardHitTestModeRs, + /hit_test_mode[\s\S]*?store\(HIT_TEST_MODE_CARD, Ordering::SeqCst\)[\s\S]*?window\.set_ignore_cursor_events\(false\)[\s\S]*?cursor_passthrough[\s\S]*?store\(false, Ordering::SeqCst\)[\s\S]*?configure_card_hit_test/, + 'card mode must make the shared window interactive and keep its host-owned hit-test cache in sync', +); +assertMatch( + restoreCapsuleHitTestModeRs, + /hit_test_mode[\s\S]*?store\(HIT_TEST_MODE_CAPSULE, Ordering::SeqCst\)[\s\S]*?set_cursor_passthrough\(true\)[\s\S]*?invalidate_layout\(\)[\s\S]*?maybe_position_capsule_bottom_center/, + 'restoring capsule mode must restore passthrough and style-aware position through the host capability', +); +const maybePositionCapsuleRs = extractFn(coordinatorHostRs, 'maybe_position_capsule_bottom_center'); +assertMatch( + maybePositionCapsuleRs, + /position_capsule_bottom_center_with_style_and_transcript/, + 'capsule restore must route through the style-aware geometry helper', +); +const positionCapsuleRs = extractFn( + libRs, + 'position_capsule_bottom_center_with_style_and_transcript', +); +const macosPositionCapsuleRs = extractCfgBlock(positionCapsuleRs, 'macos'); +assertMatch( + macosPositionCapsuleRs, + /window\.set_size[\s\S]*?window\.set_position/, + 'macOS style-aware capsule restoration must update both window size and position', +); + // Showing a card = moving the shared window away; the dedup cache must be invalidated on the spot. for (const name of ['show_vocab_suggestion_card', 'show_insert_fallback_card']) { const body = extractFn(coordinatorRs, name); @@ -129,8 +262,8 @@ for (const name of ['show_vocab_suggestion_card', 'show_insert_fallback_card']) ); assertMatch( body, - /capsule\.set_cursor_passthrough\(false\)/, - `${name} must change cursor passthrough through the host capability that keeps its cache in sync`, + /capsule\.set_card_hit_test_mode\(\)/, + `${name} must enter card hit-test mode through the host capability that keeps its cache in sync`, ); } @@ -139,32 +272,19 @@ for (const name of ['hide_vocab_suggestion_card', 'hide_insert_fallback_card']) const body = extractFn(coordinatorRs, name); assertMatch( body, - /set_cursor_passthrough\(true\)/, - `${name} must restore cursor passthrough, or the capsule keeps blocking that strip of screen`, - ); - assertMatch( - body, - /capsule_window_bounds\(false\)[\s\S]*?set_size/, - `${name} must restore the capsule window size, or the next capsule is squeezed into a card-sized window`, - ); - assertMatch( - body, - /capsule\.invalidate_layout\(\)/, - `${name} must invalidate the capsule_layout dedup cache, or the next recording skips repositioning and the capsule stays bottom-right`, + /capsule\.restore_capsule_hit_test_mode\(\)/, + `${name} must restore passthrough and capsule geometry through the host capability`, ); assertMatch( body, - /position_capsule_bottom_center\(false\)/, - `${name} must move the capsule window back to bottom-center; restoring size alone leaves it in the card's bottom-right corner`, + /capsule\.hide\(\)[\s\S]*?capsule\.restore_capsule_hit_test_mode\(\)/, + `${name} must hide the window before the host restores its geometry`, ); - // Ordering invariant: size and position must change together; changing them while the window is - // still visible can composite a frame of "card stretched wide, still flying across half the - // screen". + // Ordering invariant: geometry restoration must happen after hiding, or the restore can + // composite a frame of "card stretched wide, still flying across half the screen". const hideAt = body.indexOf('capsule.hide()'); - const resizeAt = body.indexOf('set_size'); - if (hideAt === -1 || hideAt > resizeAt) { - throw new Error( - `${name} must hide the window before changing its geometry, or the restore animates on screen`, - ); + const restoreAt = body.indexOf('restore_capsule_hit_test_mode()'); + if (hideAt === -1 || hideAt > restoreAt) { + throw new Error(`${name} must hide the window before restoring its geometry`); } } From b0b0a27e7e7bb8099ad079691124ca1ae048b726 Mon Sep 17 00:00:00 2001 From: HKLHaoBin Date: Mon, 5 Oct 2026 03:48:07 +0800 Subject: [PATCH 5/5] test(capsule): allow stable floating point tolerance --- openless-all/app/src-tauri/src/tauri_coordinator_host.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/openless-all/app/src-tauri/src/tauri_coordinator_host.rs b/openless-all/app/src-tauri/src/tauri_coordinator_host.rs index f5d63a2aa..fe5a350de 100644 --- a/openless-all/app/src-tauri/src/tauri_coordinator_host.rs +++ b/openless-all/app/src-tauri/src/tauri_coordinator_host.rs @@ -1997,7 +1997,9 @@ mod tests { assert!( (typeless_translation.top_offset - (65.0 - 20.0 * 0.447 - 64.0 * 0.447 - 52.0 * 0.447)) .abs() - < f64::EPSILON + < 1e-12, + "unexpected typeless translation rail offset: {}", + typeless_translation.top_offset ); for style in [