diff --git a/docs/volcengine-setup.md b/docs/volcengine-setup.md index fd5635101..ff16dd6a2 100644 --- a/docs/volcengine-setup.md +++ b/docs/volcengine-setup.md @@ -29,3 +29,4 @@ - Agent Plan：`https://ark.cn-beijing.volces.com/api/plan/v3`；Coding Plan：`https://ark.cn-beijing.volces.com/api/coding/v3`。使用各自套餐的专属 API Key，按所选请求格式适配协议路径。 - 选择套餐后，通过“查看支持的模型”打开对应的 [Agent Plan 控制台](https://console.volcengine.com/ark/subscription/agent-plan)或 [Coding Plan 控制台](https://console.volcengine.com/ark/subscription/coding-plan)，复制支持的文本模型 ID，手动填写后执行“验证”。该按钮不拉取在线模型列表，也不验证密钥。 - 预设地址只读，已有自定义地址保持可编辑；新建自定义接口使用自定义供应商入口。普通火山方舟及其他供应商保留原有模型列表获取行为。 +- 方舟通道的协议设置提供“低延迟推理（fast）”开关；启用后请求会携带 `service_tier: "fast"`。该参数只对支持快速服务档位的模型有效，不支持的模型可能返回 400。 diff --git a/openless-all/app/crates/openless-core/src/api.rs b/openless-all/app/crates/openless-core/src/api.rs index a96c1cbe9..383eb8771 100644 --- a/openless-all/app/crates/openless-core/src/api.rs +++ b/openless-all/app/crates/openless-core/src/api.rs @@ -4390,13 +4390,34 @@ impl OpenLessBackend { Some(id.clone()), crate::llm_protocol::REQUEST_FORMAT_ACCOUNT, )?; + let service_tier_key = CredentialKey::new( + crate::CredentialNamespace::Llm, + Some(id.clone()), + crate::llm_protocol::SERVICE_TIER_ACCOUNT, + )?; let reset = kind == ChannelKind::Llm && previous.provider_type != provider_type; - let old_format = if reset { - let value = self.deps.credential_store.read(key.clone()).await?; + let (old_format, old_service_tier) = if reset { + let old_format = self.deps.credential_store.read(key.clone()).await?; + let old_service_tier = self + .deps + .credential_store + .read(service_tier_key.clone()) + .await?; self.deps.credential_store.remove(key.clone()).await?; - value + if let Err(error) = self + .deps + .credential_store + .remove(service_tier_key.clone()) + .await + { + if let Some(value) = old_format.clone() { + self.deps.credential_store.write(key.clone(), value).await?; + } + return Err(error); + } + (old_format, old_service_tier) } else { - None + (None, None) }; let result = self .deps @@ -4412,6 +4433,12 @@ impl OpenLessBackend { if let Some(value) = old_format { self.deps.credential_store.write(key, value).await?; } + if let Some(value) = old_service_tier { + self.deps + .credential_store + .write(service_tier_key, value) + .await?; + } } result?; self.refresh_and_publish_credentials().await.map(|_| ()) @@ -10298,7 +10325,7 @@ mod tests { } #[tokio::test] - async fn llm_protocol_mutations_reset_only_the_format_and_invalidate_tests() { + async fn llm_protocol_mutations_reset_protocol_overrides_and_invalidate_tests() { use crate::credentials::{CredentialNamespace, InMemoryCredentialStore, SecretValue}; use crate::llm_protocol::*; let backend = OpenLessBackend::new( @@ -10333,6 +10360,19 @@ mod tests { .set_credential(key(REQUEST_FORMAT_ACCOUNT), SecretValue::new("messages")) .await .unwrap(); + backend + .set_credential(key(SERVICE_TIER_ACCOUNT), SecretValue::new("fast")) + .await + .unwrap(); + assert_eq!( + backend + .read_credential(key(SERVICE_TIER_ACCOUNT)) + .await + .unwrap() + .unwrap() + .expose_secret(), + "fast" + ); backend .set_credential( key(crate::credentials::LLM_API_KEY_ACCOUNT), @@ -10379,6 +10419,11 @@ mod tests { .await .unwrap() .is_none()); + assert!(backend + .read_credential(key(SERVICE_TIER_ACCOUNT)) + .await + .unwrap() + .is_none()); assert_eq!( backend .read_credential(key(crate::credentials::LLM_API_KEY_ACCOUNT)) diff --git a/openless-all/app/crates/openless-core/src/cloud_sync_e2ee_documents/registry.rs b/openless-all/app/crates/openless-core/src/cloud_sync_e2ee_documents/registry.rs index 3eedc0809..d36eb1300 100644 --- a/openless-all/app/crates/openless-core/src/cloud_sync_e2ee_documents/registry.rs +++ b/openless-all/app/crates/openless-core/src/cloud_sync_e2ee_documents/registry.rs @@ -912,6 +912,7 @@ pub const LLM_ACCOUNTS: &[&str] = &[ "ark.messages_thinking", "ark.max_tokens", "ark.thinking_budget", + "ark.service_tier", ]; pub const OMNI_ACCOUNTS: &[&str] = &[ "omni.api_key", diff --git a/openless-all/app/crates/openless-core/src/credentials_legacy.rs b/openless-all/app/crates/openless-core/src/credentials_legacy.rs index 2193da9d5..6b034f247 100644 --- a/openless-all/app/crates/openless-core/src/credentials_legacy.rs +++ b/openless-all/app/crates/openless-core/src/credentials_legacy.rs @@ -109,6 +109,7 @@ struct LegacyEntry { messages_thinking: Option, max_tokens: Option, thinking_budget: Option, + service_tier: Option, } impl Default for LegacyEntry { @@ -141,6 +142,7 @@ impl Default for LegacyEntry { messages_thinking: None, max_tokens: None, thinking_budget: None, + service_tier: None, } } } @@ -169,6 +171,7 @@ impl LegacyEntry { &self.messages_thinking, &self.max_tokens, &self.thinking_budget, + &self.service_tier, ] .into_iter() .any(|value| value.as_deref().is_some_and(|value| !value.is_empty())) @@ -378,6 +381,7 @@ fn decode_entry( (MESSAGES_THINKING_ACCOUNT, entry.messages_thinking), (MAX_TOKENS_ACCOUNT, entry.max_tokens), (THINKING_BUDGET_ACCOUNT, entry.thinking_budget), + (SERVICE_TIER_ACCOUNT, entry.service_tier), ]); } if namespace == CredentialNamespace::Asr { diff --git a/openless-all/app/crates/openless-core/src/llm_protocol.rs b/openless-all/app/crates/openless-core/src/llm_protocol.rs index e09542507..001caf8ee 100644 --- a/openless-all/app/crates/openless-core/src/llm_protocol.rs +++ b/openless-all/app/crates/openless-core/src/llm_protocol.rs @@ -12,11 +12,13 @@ pub const REQUEST_FORMAT_ACCOUNT: &str = "ark.request_format"; pub const MESSAGES_THINKING_ACCOUNT: &str = "ark.messages_thinking"; pub const MAX_TOKENS_ACCOUNT: &str = "ark.max_tokens"; pub const THINKING_BUDGET_ACCOUNT: &str = "ark.thinking_budget"; -pub const CONFIG_ACCOUNTS: [&str; 4] = [ +pub const SERVICE_TIER_ACCOUNT: &str = "ark.service_tier"; +pub const CONFIG_ACCOUNTS: [&str; 5] = [ REQUEST_FORMAT_ACCOUNT, MESSAGES_THINKING_ACCOUNT, MAX_TOKENS_ACCOUNT, THINKING_BUDGET_ACCOUNT, + SERVICE_TIER_ACCOUNT, ]; #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] @@ -115,12 +117,21 @@ pub enum MessagesThinking { Budget, } +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum LlmServiceTier { + #[default] + Off, + Fast, +} + #[derive(Debug, Clone, PartialEq, Eq)] pub struct LlmProtocolConfig { pub format: LlmRequestFormat, pub messages_thinking: MessagesThinking, pub max_tokens: u32, pub thinking_budget: u32, + pub service_tier: LlmServiceTier, } impl Default for LlmProtocolConfig { @@ -130,6 +141,7 @@ impl Default for LlmProtocolConfig { messages_thinking: MessagesThinking::Adaptive, max_tokens: 8192, thinking_budget: 1024, + service_tier: LlmServiceTier::Off, } } } @@ -194,6 +206,12 @@ impl LlmProtocolConfig { return Err(config_error("llmThinkingBudgetInvalid")); } } + SERVICE_TIER_ACCOUNT => { + self.service_tier = match value { + "fast" => LlmServiceTier::Fast, + _ => return Err(config_error("llmServiceTierInvalid")), + } + } _ => return Err(config_error("llmRequestFormatInvalid")), } Ok(()) @@ -209,6 +227,12 @@ impl LlmProtocolConfig { } Ok(()) } + + pub(crate) fn apply_service_tier(&self, provider_id: &str, body: &mut Value) { + if provider_id.trim() == "ark" && self.service_tier == LlmServiceTier::Fast { + body["service_tier"] = json!("fast"); + } + } } fn positive_tokens(value: &str) -> Result { @@ -293,6 +317,9 @@ pub(crate) fn request_body( body["temperature"] = crate::polish::temperature_json(temperature); } } + config + .protocol + .apply_service_tier(&config.provider_id, &mut body); body } @@ -726,6 +753,36 @@ mod tests { } } + #[test] + fn service_tier_is_validated_and_applied_only_to_ark_requests() { + let mut protocol = LlmProtocolConfig::default(); + assert!(protocol.apply(SERVICE_TIER_ACCOUNT, "invalid").is_err()); + protocol.apply(SERVICE_TIER_ACCOUNT, "fast").unwrap(); + assert_eq!(protocol.service_tier, LlmServiceTier::Fast); + + let messages = vec![json!({"role":"user","content":"hi"})]; + for format in [LlmRequestFormat::Responses, LlmRequestFormat::Messages] { + let mut config = OpenAICompatibleConfig::new( + "ark", + "test", + "https://ark.cn-beijing.volces.com/api/v3", + "key", + "test-model", + ) + .with_protocol(LlmProtocolConfig { + format, + service_tier: LlmServiceTier::Fast, + ..Default::default() + }); + let body = request_body(&config, false, messages.clone()); + assert_eq!(body["service_tier"], "fast"); + + config.provider_id = "openai".into(); + let body = request_body(&config, false, messages.clone()); + assert!(body.get("service_tier").is_none()); + } + } + #[test] fn budgets_are_validated_and_non_streaming_extracts_only_complete_text() { let mut config = LlmProtocolConfig { diff --git a/openless-all/app/crates/openless-core/src/polish.rs b/openless-all/app/crates/openless-core/src/polish.rs index 725393e91..96fdab114 100644 --- a/openless-all/app/crates/openless-core/src/polish.rs +++ b/openless-all/app/crates/openless-core/src/polish.rs @@ -834,6 +834,9 @@ impl OpenAICompatibleLLMProvider { &self.config.model, self.config.thinking_enabled, ); + self.config + .protocol + .apply_service_tier(&self.config.provider_id, &mut body); body } @@ -2507,6 +2510,7 @@ mod tests { .into_iter() .flat_map(|format| { [ + ("ark", "/api/v3"), ("custom", "/gateway/v1"), ("opencode", "/zen/v1"), ("opencode", "/zen/go/v1"), @@ -2534,6 +2538,11 @@ mod tests { let split = request.windows(4).position(|w| w == b"\r\n\r\n").unwrap(); let headers = String::from_utf8_lossy(&request[..split]).to_ascii_lowercase(); let body: Value = serde_json::from_slice(&request[split + 4..]).unwrap(); + if preset == "ark" { + assert_eq!(body["service_tier"], "fast"); + } else { + assert!(body.get("service_tier").is_none()); + } if format == LlmRequestFormat::Responses { assert!(body.get("temperature").is_none()); } else { @@ -2626,6 +2635,7 @@ mod tests { .with_thinking_enabled(thinking_enabled) .with_protocol(LlmProtocolConfig { format, + service_tier: crate::llm_protocol::LlmServiceTier::Fast, ..Default::default() }); let provider = OpenAICompatibleLLMProvider::new(config); @@ -3364,6 +3374,27 @@ mod tests { ); } + #[test] + fn ark_chat_body_includes_fast_service_tier() { + let provider = OpenAICompatibleLLMProvider::new( + OpenAICompatibleConfig::new( + "ark", + "Ark", + "https://ark.cn-beijing.volces.com/api/v3", + "k", + "doubao-seed-2-1-lite-260915", + ) + .with_protocol(LlmProtocolConfig { + service_tier: crate::llm_protocol::LlmServiceTier::Fast, + ..Default::default() + }), + ); + + let body = provider.chat_body(false, vec![json!({ "role": "user", "content": "hi" })]); + + assert_eq!(body["service_tier"], "fast"); + } + #[test] fn openai_chat_body_adds_reasoning_effort_for_openai_reasoning_model() { let provider = OpenAICompatibleLLMProvider::new( diff --git a/openless-all/app/crates/openless-core/src/shared_types.rs b/openless-all/app/crates/openless-core/src/shared_types.rs index 9da6eb7a3..924fe5740 100644 --- a/openless-all/app/crates/openless-core/src/shared_types.rs +++ b/openless-all/app/crates/openless-core/src/shared_types.rs @@ -2558,6 +2558,11 @@ pub enum CapsuleStyle { #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct CapsulePayload { + /// Session generation that owns this capsule frame. Keeping this on the + /// payload lets a Recording -> Recording transition reset the transcript + /// without depending on the order of backend:event and capsule:state. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub session_id: Option, pub state: CapsuleState, pub level: f32, // 0..1 RMS pub elapsed_ms: u64, diff --git a/openless-all/app/scripts/insert-text-motion-h5.mjs b/openless-all/app/scripts/insert-text-motion-h5.mjs index 7d698f8ce..d53d91f3d 100644 --- a/openless-all/app/scripts/insert-text-motion-h5.mjs +++ b/openless-all/app/scripts/insert-text-motion-h5.mjs @@ -6,9 +6,11 @@ import { basename, join, resolve } from 'node:path'; import { createServer } from 'vite'; const out = process.env.OPENLESS_MOTION_ARTIFACT_DIR || join(tmpdir(), 'openless-motion-h5'); +const serverPort = Number(process.env.OPENLESS_MOTION_H5_PORT || 1438); +const chromePort = Number(process.env.OPENLESS_MOTION_CHROME_PORT || 9437); mkdirSync(out, { recursive: true }); const server = await createServer({ - server: { host: '127.0.0.1', port: 1438, strictPort: true, watch: null }, + server: { host: '127.0.0.1', port: serverPort, strictPort: true, watch: null }, }); const fixture = resolve(`.insert-motion-h5-${process.pid}-${Date.now()}.html`); writeFileSync( @@ -20,22 +22,40 @@ writeFileSync( import React from 'react'; import {createRoot} from 'react-dom/client'; import {flushSync} from 'react-dom'; - import {LiveTranscriptPill} from '/src/components/LiveTranscriptPill.tsx'; + import {mockIPC, mockWindows} from '@tauri-apps/api/mocks'; + let railSnapshot = null; + if (location.search.includes('native=1')) { + mockWindows('capsule-rail'); + mockIPC((cmd) => { + if (cmd === 'get_startup_snapshot') return {contractVersion:'2.0.0',backend:{running:true}}; + if (cmd === 'get_capsule_snapshot') return railSnapshot; + if (cmd === 'get_settings') return {capsuleStyle:railSnapshot.capsuleStyle,capsuleTranscriptEnabled:true,capsuleTranscriptFontSize:14}; + }, {shouldMockEvents:true}); + } + const {LiveTranscriptPill, CapsuleTranscriptOverlay}=await import('/src/components/LiveTranscriptPill.tsx'); + const {getCapsuleTranscriptRailPosition}=await import('/src/lib/capsuleLayout.ts'); + const {i18nReady}=await import('/src/i18n'); + await i18nReady; const root=createRoot(document.getElementById('root')); window.show=(text, controls=false, tone='frost')=>flushSync(()=>root.render(React.createElement(LiveTranscriptPill,{text,stageWidth:460,maxWidth:440,minWidth:72,tone,...(controls?{onCancel:()=>{},onConfirm:()=>{}}:{})}))); window.show('帮我'); + window.showRail=(style,text)=>{ + railSnapshot={state:'recording',capsuleStyle:style,translation:true,selectionPolish:false,transcript:text,sessionId:'test',sequence:1,revision:1,payloadRevision:1}; + root.render(React.createElement(CapsuleTranscriptOverlay,{key:style+text})); + return getCapsuleTranscriptRailPosition(style,true,Boolean(text)); + }; window.ready=true; `, ); await server.listen(); -const pageUrl = 'http://127.0.0.1:1438/' + basename(fixture); +const pageUrl = `http://127.0.0.1:${serverPort}/` + basename(fixture); const chrome = spawn( process.env.CHROME_PATH || 'C:/Program Files/Google/Chrome/Application/chrome.exe', [ '--headless=new', '--no-first-run', '--no-default-browser-check', - '--remote-debugging-port=9437', + `--remote-debugging-port=${chromePort}`, '--window-size=900,500', '--user-data-dir=' + join(tmpdir(), 'openless-motion-' + Date.now()), pageUrl, @@ -48,7 +68,7 @@ try { let target; for (let i = 0; i < 100; i++) { try { - target = (await (await fetch('http://127.0.0.1:9437/json/list')).json()).find( + target = (await (await fetch(`http://127.0.0.1:${chromePort}/json/list`)).json()).find( (t) => t.type === 'page' && t.url === pageUrl, ); } catch {} @@ -83,11 +103,14 @@ try { throw Error(r.exceptionDetails.exception?.description || r.exceptionDetails.text); return r.result.value; }; - for (let i = 0; i < 150; i++) { - if (await evaluate('Boolean(window.ready)')) break; - await sleep(100); - } - assert(await evaluate('Boolean(window.ready)'), 'React fixture ready'); + const waitForFixture = async () => { + for (let i = 0; i < 150; i++) { + if (await evaluate("Boolean(window.ready && typeof window.show === 'function')")) return; + await sleep(100); + } + assert(false, 'React fixture ready'); + }; + await waitForFixture(); await sleep(800); const samples = await evaluate(`new Promise(resolve=>{ const frames=[]; const start=performance.now();window.show('帮我查找一下'); @@ -166,8 +189,9 @@ try { await send('Emulation.setEmulatedMedia', { features: [{ name: 'prefers-reduced-motion', value: 'reduce' }], }); + await evaluate(`window.ready=false; window.show=undefined`); await send('Page.reload'); - await sleep(1000); + await waitForFixture(); await evaluate(`window.show('减少动态效果验证')`); await sleep(80); assert( @@ -177,8 +201,9 @@ try { 'reduced motion immediate', ); await send('Emulation.setEmulatedMedia', { features: [] }); + await evaluate(`window.ready=false; window.show=undefined`); await send('Page.reload'); - await sleep(900); + await waitForFixture(); await evaluate(`window.show('帮我查找一下十六号的天气')`); await sleep(1000); // Optional real browser filmstrip; timestamps preserve capture cadence. @@ -220,8 +245,9 @@ try { } const shot = await send('Page.captureScreenshot', { format: 'png' }); writeFileSync(join(out, 'settled.png'), Buffer.from(shot.data, 'base64')); + await evaluate(`window.ready=false; window.show=undefined`); await send('Page.navigate', { url: pageUrl }); - await sleep(1200); + await waitForFixture(); await evaluate(`window.show('10六号')`); await sleep(1000); const correction = await evaluate(`new Promise(resolve=>{ @@ -241,12 +267,12 @@ try { for (const enabled of [true, false]) { await send('Emulation.setDeviceMetricsOverride', { width: style === 'typeless' ? 206 : 460, - height: style === 'typeless' ? 57 : style === 'classic' ? 100 : 180, + height: style === 'typeless' ? (enabled ? 65 : 57) : style === 'classic' ? 172 : 228, deviceScaleFactor: 1, mobile: false, }); await send('Page.navigate', { - url: `http://127.0.0.1:1438/?window=capsule&os=win&style=${style}&insertDemo=1&transcript=${enabled ? 1 : 0}&fontSize=20`, + url: `http://127.0.0.1:${serverPort}/?window=capsule&os=win&style=${style}&insertDemo=1&transcript=${enabled ? 1 : 0}&translation=${style === 'typeless' && enabled ? 1 : 0}&fontSize=20`, }); // Production demo simulates five recognition batches ending after 2440ms. for (let attempt = 0; attempt < 150; attempt += 1) { @@ -261,7 +287,16 @@ try { const pills=[...document.querySelectorAll('.ol-live-transcript-pill')]; const chars=[...document.querySelectorAll('.ol-live-transcript-char')]; const rect=pills[0]?.getBoundingClientRect(); - return {pills:pills.length,chars:chars.length,width:rect?.width,font:chars[0]?parseFloat(getComputedStyle(chars[0]).fontSize):0,buttons:document.querySelectorAll('button').length}; + const rail=document.querySelector('.ol-capsule-transcript-rail'); + const body=${JSON.stringify(style)}==='typeless'?document.querySelector('.ol-capsule-waveform'): ${JSON.stringify(style)}==='classic'?document.querySelector('.ol-capsule-pill'):document.querySelector('canvas'); + const hint=document.querySelector('.ol-typeless-translation'); + const hintRect=hint?.getBoundingClientRect(); + const hintStyle=hint?getComputedStyle(hint):null; + const railRect=rail?.getBoundingClientRect(); + const bodyRect=body?.getBoundingClientRect(); + const translationHintVisible=Boolean(hint && hintStyle?.opacity !== '0' && hintRect && hintRect.width > 0 && hintRect.top >= -0.5 && hintRect.bottom <= innerHeight + 0.5); + const translationHintNonOverlapping=Boolean(!translationHintVisible || !hintRect || !bodyRect || (hintRect.bottom <= bodyRect.top + 0.5 && (!railRect || hintRect.top >= railRect.bottom - 0.5))); + return {pills:pills.length,chars:chars.length,width:rect?.width,font:chars[0]?parseFloat(getComputedStyle(chars[0]).fontSize):0,buttons:document.querySelectorAll('button').length,rail:Boolean(rail),body:Boolean(body),railOutsideBody:Boolean(rail && body && !body.contains(rail)),translationHintVisible,translationHintNonOverlapping}; })()`); if (result.pills !== (enabled ? 1 : 0)) { writeFileSync( @@ -272,7 +307,24 @@ try { assert.equal(result.pills, enabled ? 1 : 0, `${style} display preference`); if (enabled) { assert(result.chars > 0, `${style} original text`); + assert(result.chars <= 64, `${style} motion glyph count stays bounded`); + assert( + result.rail && result.body && result.railOutsideBody, + `${style} keeps the capsule body beside, not inside, the transcript rail`, + ); assert(result.width <= (style === 'typeless' ? 206 : 460), `${style} bounded width`); + if (style === 'typeless') { + assert( + result.translationHintVisible, + 'typeless translation hint stays inside the native viewport', + ); + if (enabled) { + assert( + result.translationHintNonOverlapping, + 'typeless translation hint has a dedicated non-overlapping row', + ); + } + } assert( Math.abs(result.font * (style === 'typeless' ? 0.447 : 1) - 20) < 0.1, `${style} visible font size`, @@ -286,6 +338,38 @@ try { ); } } + // Exercise the actual Windows overlay component; the browser capsule preview + // does not render this separate, click-through native window. + await send('Page.navigate', { url: pageUrl + '?native=1' }); + await waitForFixture(); + await evaluate("document.body.style.display='block'"); + for (const style of ['classic', 'typeless']) { + for (const text of ['', '需要翻译的原文']) { + const geometry = await evaluate( + `window.showRail(${JSON.stringify(style)},${JSON.stringify(text)})`, + ); + assert(geometry, `${style} translation has an overlay even without transcript`); + await send('Emulation.setDeviceMetricsOverride', { + width: Math.ceil(geometry.width), + height: Math.ceil(geometry.height), + deviceScaleFactor: 1, + mobile: false, + }); + await sleep(400); + const layout = await evaluate(`(() => { + const badge = document.querySelector('.ol-classic-translation, .ol-typeless-translation'); + const rail = document.querySelector('.ol-capsule-transcript-rail'); + const box = badge?.getBoundingClientRect(); + return {badgeTop:box?.top,badgeBottom:box?.bottom,railBottom:rail?.getBoundingClientRect().bottom}; + })()`); + assert( + layout.badgeTop >= -0.5 && layout.badgeBottom <= geometry.height + 0.5, + `${style} translation fits its native overlay: ${JSON.stringify(layout)}`, + ); + if (text) + assert(layout.railBottom <= layout.badgeTop + 0.5, `${style} rail clears translation`); + } + } console.log( JSON.stringify( { diff --git a/openless-all/app/scripts/macos-capsule-spaces-contract.test.mjs b/openless-all/app/scripts/macos-capsule-spaces-contract.test.mjs index 0e5d51747..b8074b90b 100644 --- a/openless-all/app/scripts/macos-capsule-spaces-contract.test.mjs +++ b/openless-all/app/scripts/macos-capsule-spaces-contract.test.mjs @@ -105,20 +105,153 @@ assertMatch( const coordinatorRs = ( await readFile(new URL('../src-tauri/src/coordinator.rs', import.meta.url), 'utf-8') ).replace(/\r\n/g, '\n'); -assertMatch( - coordinatorHostRs, - /fn set_cursor_passthrough[\s\S]*?window\.set_ignore_cursor_events\(passthrough\)[\s\S]*?cursor_passthrough[\s\S]*?store\(passthrough, Ordering::SeqCst\)/, - 'the narrow capsule-window capability must update the Tauri window and its host-owned passthrough cache together', -); + +function extractBalancedBlock(source, openBrace) { + let depth = 0; + let inString = false; + let inChar = false; + let escaped = false; + let inLineComment = false; + let blockCommentDepth = 0; + let rawStringHashes = null; + for (let index = openBrace; index < source.length; index += 1) { + const char = source[index]; + const next = source[index + 1]; + if (inLineComment) { + if (char === '\n') inLineComment = false; + continue; + } + if (blockCommentDepth > 0) { + if (char === '/' && next === '*') { + blockCommentDepth += 1; + index += 1; + } else if (char === '*' && next === '/') { + blockCommentDepth -= 1; + index += 1; + } + continue; + } + if (rawStringHashes !== null) { + const terminator = `"${'#'.repeat(rawStringHashes)}`; + if (source.startsWith(terminator, index)) { + index += terminator.length - 1; + rawStringHashes = null; + } + continue; + } + if (inString || inChar) { + const terminator = inString ? '"' : "'"; + if (escaped) { + escaped = false; + } else if (char === '\\') { + escaped = true; + } else if (char === terminator) { + inString = false; + inChar = false; + } + continue; + } + if (char === '/' && next === '/') { + inLineComment = true; + index += 1; + continue; + } + if (char === '/' && next === '*') { + blockCommentDepth = 1; + index += 1; + continue; + } + if (char === 'r') { + let hashEnd = index + 1; + while (source[hashEnd] === '#') hashEnd += 1; + if (source[hashEnd] === '"') { + rawStringHashes = hashEnd - index - 1; + index = hashEnd; + continue; + } + } + if (char === '"') { + inString = true; + continue; + } + // Rust lifetimes start with a letter after the apostrophe; only enter the + // character-literal state when the closing apostrophe is locally evident. + if (char === "'" && (next === '\\' || source[index + 2] === "'")) { + inChar = true; + continue; + } + if (char === '{') depth += 1; + if (char === '}' && --depth === 0) { + return source.slice(openBrace, index + 1); + } + } + throw new Error('unterminated Rust block'); +} function extractFn(source, name) { - const match = source.match(new RegExp(`(?:pub\\(crate\\) )?fn ${name}[\\s\\S]*?\\n}\\n`)); + const signature = new RegExp( + `(?:pub\\([^)]*\\)\\s*)?(?:async\\s+)?fn\\s+${name}(?:<[^>{}]*>)?\\s*\\(`, + ); + const match = signature.exec(source); if (!match) { - throw new Error(`${name}: function not found in coordinator.rs`); + throw new Error(`${name}: function not found`); + } + const openBrace = source.indexOf('{', match.index); + if (openBrace === -1) { + throw new Error(`${name}: function body not found`); } - return match[0]; + const body = extractBalancedBlock(source, openBrace); + return source.slice(match.index, openBrace) + body; } +function extractCfgBlock(source, target) { + const marker = `#[cfg(target_os = "${target}")]`; + const markerIndex = source.indexOf(marker); + if (markerIndex === -1) { + throw new Error(`${target}: cfg block not found`); + } + const openBrace = source.indexOf('{', markerIndex + marker.length); + if (openBrace === -1) { + throw new Error(`${target}: cfg block body not found`); + } + return source.slice(markerIndex, openBrace) + extractBalancedBlock(source, openBrace); +} + +const setCursorPassthroughRs = extractFn(coordinatorHostRs, 'set_cursor_passthrough'); +assertMatch( + setCursorPassthroughRs, + /window\.set_ignore_cursor_events\(passthrough\)[\s\S]*?cursor_passthrough[\s\S]*?store\(passthrough, Ordering::SeqCst\)/, + 'the narrow capsule-window capability must update the Tauri window and its host-owned passthrough cache together', +); +const setCardHitTestModeRs = extractFn(coordinatorHostRs, 'set_card_hit_test_mode'); +const restoreCapsuleHitTestModeRs = extractFn(coordinatorHostRs, 'restore_capsule_hit_test_mode'); +assertMatch( + setCardHitTestModeRs, + /hit_test_mode[\s\S]*?store\(HIT_TEST_MODE_CARD, Ordering::SeqCst\)[\s\S]*?window\.set_ignore_cursor_events\(false\)[\s\S]*?cursor_passthrough[\s\S]*?store\(false, Ordering::SeqCst\)[\s\S]*?configure_card_hit_test/, + 'card mode must make the shared window interactive and keep its host-owned hit-test cache in sync', +); +assertMatch( + restoreCapsuleHitTestModeRs, + /hit_test_mode[\s\S]*?store\(HIT_TEST_MODE_CAPSULE, Ordering::SeqCst\)[\s\S]*?set_cursor_passthrough\(true\)[\s\S]*?invalidate_layout\(\)[\s\S]*?maybe_position_capsule_bottom_center/, + 'restoring capsule mode must restore passthrough and style-aware position through the host capability', +); +const maybePositionCapsuleRs = extractFn(coordinatorHostRs, 'maybe_position_capsule_bottom_center'); +assertMatch( + maybePositionCapsuleRs, + /position_capsule_bottom_center_with_style_and_transcript/, + 'capsule restore must route through the style-aware geometry helper', +); +const positionCapsuleRs = extractFn( + libRs, + 'position_capsule_bottom_center_with_style_and_transcript', +); +const macosPositionCapsuleRs = extractCfgBlock(positionCapsuleRs, 'macos'); +assertMatch( + macosPositionCapsuleRs, + /window\.set_size[\s\S]*?window\.set_position/, + 'macOS style-aware capsule restoration must update both window size and position', +); + // Showing a card = moving the shared window away; the dedup cache must be invalidated on the spot. for (const name of ['show_vocab_suggestion_card', 'show_insert_fallback_card']) { const body = extractFn(coordinatorRs, name); @@ -129,8 +262,8 @@ for (const name of ['show_vocab_suggestion_card', 'show_insert_fallback_card']) ); assertMatch( body, - /capsule\.set_cursor_passthrough\(false\)/, - `${name} must change cursor passthrough through the host capability that keeps its cache in sync`, + /capsule\.set_card_hit_test_mode\(\)/, + `${name} must enter card hit-test mode through the host capability that keeps its cache in sync`, ); } @@ -139,32 +272,19 @@ for (const name of ['hide_vocab_suggestion_card', 'hide_insert_fallback_card']) const body = extractFn(coordinatorRs, name); assertMatch( body, - /set_cursor_passthrough\(true\)/, - `${name} must restore cursor passthrough, or the capsule keeps blocking that strip of screen`, - ); - assertMatch( - body, - /capsule_window_bounds\(false\)[\s\S]*?set_size/, - `${name} must restore the capsule window size, or the next capsule is squeezed into a card-sized window`, - ); - assertMatch( - body, - /capsule\.invalidate_layout\(\)/, - `${name} must invalidate the capsule_layout dedup cache, or the next recording skips repositioning and the capsule stays bottom-right`, + /capsule\.restore_capsule_hit_test_mode\(\)/, + `${name} must restore passthrough and capsule geometry through the host capability`, ); assertMatch( body, - /position_capsule_bottom_center\(false\)/, - `${name} must move the capsule window back to bottom-center; restoring size alone leaves it in the card's bottom-right corner`, + /capsule\.hide\(\)[\s\S]*?capsule\.restore_capsule_hit_test_mode\(\)/, + `${name} must hide the window before the host restores its geometry`, ); - // Ordering invariant: size and position must change together; changing them while the window is - // still visible can composite a frame of "card stretched wide, still flying across half the - // screen". + // Ordering invariant: geometry restoration must happen after hiding, or the restore can + // composite a frame of "card stretched wide, still flying across half the screen". const hideAt = body.indexOf('capsule.hide()'); - const resizeAt = body.indexOf('set_size'); - if (hideAt === -1 || hideAt > resizeAt) { - throw new Error( - `${name} must hide the window before changing its geometry, or the restore animates on screen`, - ); + const restoreAt = body.indexOf('restore_capsule_hit_test_mode()'); + if (hideAt === -1 || hideAt > restoreAt) { + throw new Error(`${name} must hide the window before restoring its geometry`); } } diff --git a/openless-all/app/scripts/windows-ui-config.test.mjs b/openless-all/app/scripts/windows-ui-config.test.mjs index ebd4d3307..040a37dfe 100644 --- a/openless-all/app/scripts/windows-ui-config.test.mjs +++ b/openless-all/app/scripts/windows-ui-config.test.mjs @@ -14,9 +14,23 @@ function assertMatch(source, pattern, name) { const raw = await readFile(new URL('../src-tauri/tauri.conf.json', import.meta.url), 'utf-8'); const config = JSON.parse(raw); -const capsuleWindow = config.app.windows.find((window) => window.label === 'capsule'); +const windowsConfig = JSON.parse( + await readFile(new URL('../src-tauri/tauri.windows.conf.json', import.meta.url), 'utf-8'), +); +// Tauri's JSON Merge Patch replaces arrays instead of merging windows by label. +const windows = windowsConfig.app?.windows ?? config.app.windows; +const capsuleWindow = windows.find((window) => window.label === 'capsule'); +const capsuleRailWindow = windows.find((window) => window.label === 'capsule-rail'); const mainWindow = config.app.windows.find((window) => window.label === 'main'); const libRs = await readFile(new URL('../src-tauri/src/lib.rs', import.meta.url), 'utf-8'); +const dictationRs = await readFile( + new URL('../src-tauri/src/commands/dictation.rs', import.meta.url), + 'utf-8', +); +const sharedTypesRs = await readFile( + new URL('../crates/openless-core/src/shared_types.rs', import.meta.url), + 'utf-8', +); // Contract check over the union of the capsule subsystem compiled into the binary; native // window manipulation belongs to the explicit Tauri Host. const coordinatorRs = @@ -27,7 +41,13 @@ const coordinatorRs = 'utf-8', )) + '\n' + + (await readFile(new URL('../src-tauri/src/capsule_snapshot.rs', import.meta.url), 'utf-8')) + + '\n' + (await readFile(new URL('../src-tauri/src/tauri_coordinator_host.rs', import.meta.url), 'utf-8')); +const tauriEventsRs = await readFile( + new URL('../src-tauri/src/tauri_events.rs', import.meta.url), + 'utf-8', +); const capsuleTsx = await readFile( new URL('../src/components/Capsule.tsx', import.meta.url), 'utf-8', @@ -36,6 +56,24 @@ const capsuleLayoutTs = await readFile( new URL('../src/lib/capsuleLayout.ts', import.meta.url), 'utf-8', ); +const capsuleStylesCss = await readFile( + new URL('../src/components/CapsuleStyles.css', import.meta.url), + 'utf-8', +); +const typelessTsx = await readFile( + new URL('../src/components/TypelessCapsule.tsx', import.meta.url), + 'utf-8', +); +const liveTranscriptTsx = await readFile( + new URL('../src/components/LiveTranscriptPill.tsx', import.meta.url), + 'utf-8', +); +const backendEventTs = await readFile( + new URL('../src/lib/backendEvent.ts', import.meta.url), + 'utf-8', +); +const appTsx = await readFile(new URL('../src/App.tsx', import.meta.url), 'utf-8'); +const mainTsx = await readFile(new URL('../src/main.tsx', import.meta.url), 'utf-8'); const windowChromeTsx = await readFile( new URL('../src/components/WindowChrome.tsx', import.meta.url), 'utf-8', @@ -53,6 +91,9 @@ if (!capsuleWindow) { if (!mainWindow) { throw new Error('main window config missing'); } +if (!capsuleRailWindow) { + throw new Error('capsule rail window config missing'); +} assertEqual(capsuleWindow.width, 460, 'windows capsule config keeps the shared bootstrap width'); assertEqual(capsuleWindow.height, 180, 'windows capsule config keeps the shared bootstrap height'); assertEqual(capsuleWindow.transparent, true, 'capsule window should keep transparent visuals'); @@ -61,6 +102,32 @@ assertEqual( true, 'capsule window should stay above the focused app while recording', ); +assertEqual(capsuleRailWindow.transparent, true, 'capsule rail should be transparent'); +assertEqual(capsuleRailWindow.visible, false, 'capsule rail should start hidden'); +assertEqual(capsuleRailWindow.focus, false, 'capsule rail should never take focus'); +assertEqual(capsuleRailWindow.skipTaskbar, true, 'capsule rail should stay out of the taskbar'); +assertEqual(capsuleRailWindow.url, 'index.html?window=capsule-rail&os=win', 'capsule rail route'); +assertEqual( + config.app.windows.some((window) => window.label === 'capsule-rail'), + false, + 'the Windows-only rail should not create unused webviews on other platforms', +); +const railCapability = JSON.parse( + await readFile(new URL('../src-tauri/capabilities/capsule-rail.json', import.meta.url), 'utf-8'), +); +assertEqual( + railCapability.platforms.includes('windows'), + true, + 'rail capability applies on Windows', +); +assertEqual( + railCapability.windows.includes(capsuleRailWindow.label), + true, + 'rail capability matches the effective window', +); +for (const permission of ['core:event:allow-listen', 'core:event:allow-unlisten']) { + assertEqual(railCapability.permissions.includes(permission), true, `rail requires ${permission}`); +} assertEqual(mainWindow.decorations, true, 'windows main window should keep native decorations'); assertEqual( mainWindow.visible, @@ -121,7 +188,11 @@ assertEqual(mainWindow.trafficLightPosition.x, 16, 'traffic lights should have a // In tao's inset_traffic_lights, y only scales the title-bar container (slope 1), so the // visual top inset ≈ y-14; with left=16 the measured left inset is 21.5px, and y=26 makes // the top inset equal (x==y does not). -assertEqual(mainWindow.trafficLightPosition.y, 26, 'traffic lights should have an equal visual top inset'); +assertEqual( + mainWindow.trafficLightPosition.y, + 26, + 'traffic lights should have an equal visual top inset', +); assertEqual(mainWindow.width, 1300, 'main window should use the reviewed default width'); assertEqual(mainWindow.height, 835, 'main window should use the reviewed default height'); assertEqual(mainWindow.resizable, true, 'users should still be able to resize the main window'); @@ -188,7 +259,7 @@ assertMatch( assertMatch( capsuleLayoutTs, - /const stage = getCapsulePillMetrics\(os\);[\s\S]*?width: stage\.width,[\s\S]*?height: style === 'siri' \? stage\.height : style === 'classic' \? 100 : 128,[\s\S]*?horizontalInset: 0,[\s\S]*?bottomInset: style === 'siri' \? 0 : 16,[\s\S]*?badgeGap: 8,[\s\S]*?boxSizing: 'border-box'/, + /const stage = getCapsulePillMetrics\(os\);[\s\S]*?width: stage\.width,[\s\S]*?height: style === 'siri' \? SIRI_HOST_HEIGHT : style === 'classic' \? CLASSIC_HOST_HEIGHT : 128,[\s\S]*?horizontalInset: 0,[\s\S]*?bottomInset: style === 'siri' \? 0 : 16,[\s\S]*?badgeGap: 8,[\s\S]*?boxSizing: 'border-box'/, 'capsule host metrics should preserve Siri and reserve compact surfaces for Classic and Typeless', ); @@ -201,7 +272,7 @@ if ( } if ( - !/return\s*\(\s*[\s\S]*?ol-typeless-translation[\s\S]*?
/, + 'typeless translation should occupy its own layout row between the rail and capsule', +); +assertMatch( + capsuleStylesCss, + /\.ol-typeless-translation-row \{[\s\S]*?flex: 0 0 20px;[\s\S]*?height: 20px;[\s\S]*?\.ol-typeless-translation \{[\s\S]*?position: static;/, + 'typeless translation row should prevent badge overlap and clipping', +); + +const conditionalReturn = capsuleTsx.indexOf('if (insertFallback)'); +const transcriptVisibilityHook = capsuleTsx.indexOf( + 'setCapsuleTranscriptVisible(transcriptVisible)', +); +if ( + conditionalReturn < 0 || + transcriptVisibilityHook < 0 || + transcriptVisibilityHook > conditionalReturn +) { + throw new Error('transcript visibility hook must run before every conditional return'); +} +assertMatch( + capsuleTsx, + /p\.state === 'idle'\s*\|\|\s*p\.state === 'done'\s*\|\|\s*p\.state === 'cancelled'\s*\|\|\s*p\.state === 'error'[\s\S]*?setLocalAsrText\(''\)/, + 'terminal capsule states should clear the transcript visibility source', +); +if (capsuleTsx.slice(conditionalReturn).includes('useEffect(')) { + throw new Error('capsule must not call hooks after entering a conditional return branch'); +} + assertMatch( libRs, - /fn capsule_window_bounds_for_style\(style: types::CapsuleStyle\)[\s\S]*?types::CapsuleStyle::Typeless => 206\.0,[\s\S]*?CapsuleStyle::Siri \| types::CapsuleStyle::Classic => 460\.0,[\s\S]*?types::CapsuleStyle::Siri => 180\.0,[\s\S]*?types::CapsuleStyle::Classic => 100\.0,[\s\S]*?types::CapsuleStyle::Typeless => 57\.0,[\s\S]*?bottom_inset: 0\.0,/, - 'native capsule bounds should match the frontend dimensions (typeless = 1/5 area of the original 460x128)', + /fn capsule_window_bounds_for_style_with_transcript_and_translation\([\s\S]*?types::CapsuleStyle::Typeless => 206\.0,[\s\S]*?types::CapsuleStyle::Siri \| types::CapsuleStyle::Classic => 460\.0,[\s\S]*?types::CapsuleStyle::Siri => \{[\s\S]*?228\.0[\s\S]*?180\.0[\s\S]*?types::CapsuleStyle::Classic => \{[\s\S]*?172\.0[\s\S]*?100\.0[\s\S]*?types::CapsuleStyle::Typeless => \{[\s\S]*?65\.0[\s\S]*?57\.0[\s\S]*?bottom_inset: 0\.0,/, + 'native capsule bounds should match the frontend dimensions including the translation row', +); + +assertMatch( + coordinatorRs, + /transcript_visible: AtomicBool[\s\S]*?set_transcript_visible\([\s\S]*?position_capsule_bottom_center_with_style_and_transcript/, + 'capsule hit area should track whether the transcript rail is visible', +); +assertMatch( + capsuleTsx, + /setCapsuleTranscriptVisible\(transcriptVisible\)/, + 'frontend should sync transcript visibility to the native hit area', +); + +assertMatch( + coordinatorRs, + /capsule_control_hit_rect[\s\S]*?WM_NCHITTEST[\s\S]*?HTTRANSPARENT/, + 'capsule hit testing should cover X/Y and use a real native input region', +); +assertMatch( + coordinatorRs, + /control_left:[\s\S]*?control_right:[\s\S]*?control_top:[\s\S]*?control_bottom:/, + 'capsule hit-test state should store both X and Y bounds', +); +assertMatch( + coordinatorRs, + /CreateRoundRectRgn[\s\S]*?SetWindowRgn/, + 'capsule input region should be native', +); +assertMatch( + coordinatorRs, + /WM_NCDESTROY[\s\S]*?remove\([\s\S]*?SetWindowLongPtrW/, + 'subclass teardown should restore the old WndProc and remove its table entry', +); +assertMatch( + coordinatorRs, + /SetLastError[\s\S]*?GetLastError/, + 'WndProc installation should detect SetWindowLongPtrW failure', +); +assertMatch( + coordinatorRs, + /DeleteObject[\s\S]*?SetWindowRgn failed/, + 'failed regions should release their HRGN', +); +assertMatch( + coordinatorRs, + /windows_hit_test_uses_x_and_y_and_translation_bounds/, + 'hit-test regression should cover X/Y and translation height', +); +assertMatch( + coordinatorRs, + /HIT_TEST_MODE_CARD[\s\S]*?set_card_hit_test_mode[\s\S]*?restore_capsule_hit_test_mode/, + 'cards should own and restore the shared window hit-test mode', +); +assertMatch( + coordinatorRs, + /set_card_hit_test_mode[\s\S]*?configure_card_hit_test/, + 'card mode configures full client area', +); +assertMatch(coordinatorRs, /refresh_card_hit_test/, 'card resize refreshes native hit area'); +assertMatch( + coordinatorRs, + /restore_capsule_geometry/, + 'card dismissal restores style-aware capsule geometry', +); +if ( + /capsule_window_bounds\(false\)|position_capsule_bottom_center\(false\)/.test( + await readFile(new URL('../src-tauri/src/coordinator.rs', import.meta.url), 'utf-8'), + ) +) { + throw new Error('card dismissal must not restore a hard-coded capsule geometry'); +} +assertMatch( + coordinatorRs, + /hit_test_mode\.load\(Ordering::SeqCst\) == HIT_TEST_MODE_CARD[\s\S]*?hide_transcript_overlay[\s\S]*?return Ok\(\(\)\)/, + 'card-owned windows must ignore transcript visibility layout updates', +); +assertMatch( + coordinatorRs, + /emit_to\("capsule-rail", "capsule:state"/, + 'coordinator state events should reach the dedicated transcript rail', +); +assertMatch( + tauriEventsRs, + /get_webview_window\("capsule-rail"\)[\s\S]*?emit\("capsule:state"/, + 'direct dictation state events should reach the dedicated transcript rail', +); +assertMatch( + mainTsx, + /const isCapsuleRail = windowKind === 'capsule-rail'/, + 'rail route should be selected', +); +assertMatch( + appTsx, + /isCapsuleRail[\s\S]*?CapsuleTranscriptOverlay/, + 'rail route should render the overlay', +); +assertMatch( + liveTranscriptTsx, + /export function CapsuleTranscriptOverlay[\s\S]*?listen\('capsule:state'[\s\S]*?listen\('backend:event'[\s\S]*?prefs:changed/, + 'rail should own state, transcript, and preference subscriptions', +); +assertMatch( + liveTranscriptTsx, + /const replaySnapshot[\s\S]*?getCapsuleSnapshot\(\)[\s\S]*?await replaySnapshot\(preferences\.capsuleStyle\)/, + 'rail should replay the native snapshot with bounded retry after its listeners are ready', +); +assertMatch( + liveTranscriptTsx, + /snapshotRetryDelays = \[0, 16, 32, 64, 128, 256\][\s\S]*?if \(!snapshot\) continue/, + 'rail replay should retry transient empty snapshots for a bounded interval', +); +assertMatch( + liveTranscriptTsx, + /visibleCapsuleTranscript\(text, enabled, state, selectionPolish\)/, + 'rail should apply the same selection-polish visibility predicate', +); +assertMatch( + coordinatorRs, + /set_ignore_cursor_events\(true\)[\s\S]*?set_size\([\s\S]*?set_position\([\s\S]*?show_capsule_window_for_recording\(&self\.app, &rail, false\)/, + 'transcript rail should be positioned as its own click-through window', +); +assertMatch( + coordinatorRs, + /fn capsule_transcript_rail_position[\s\S]*?CapsuleStyle::Siri[\s\S]*?CapsuleStyle::Classic[\s\S]*?CapsuleStyle::Typeless/, + 'rail positioning should have one style-aware pure geometry function', +); +assertMatch( + coordinatorRs, + /capsule_transcript_rail_position\(\s*style,\s*translation_active,\s*visible,?\s*\)[\s\S]*?position\.top_offset/, + 'rail native positioning should use body-relative top offset and translation state', +); +assertMatch( + coordinatorRs, + /SW_SHOWNOACTIVATE[\s\S]*?SWP_NOACTIVATE/, + 'rail show should use the no-activate path', +); +assertMatch( + coordinatorRs, + /no_activate failed: Win32 handle is null/, + 'no-activate display should validate HWND and fall back when SetWindowPos fails', +); +assertMatch( + coordinatorRs, + /if let Err\(error\) = unsafe \{[\s\S]*?SetWindowPos\([\s\S]*?return false/, + 'no-activate display should return false when SetWindowPos fails', +); +assertMatch( + dictationRs, + /get_capsule_snapshot\([\s\S]*?Result, String>[\s\S]*?capsule_snapshot/, + 'rail snapshot command and implementation must use the same CapsuleSnapshot DTO', +); +assertMatch( + coordinatorRs, + /struct CapsuleSnapshot[\s\S]*?payload: CapsulePayload[\s\S]*?transcript: String[\s\S]*?sequence: u64[\s\S]*?revision: u64/, + 'native rail snapshot should include the session-bearing payload, transcript, and both watermarks', +); +assertMatch( + sharedTypesRs, + /struct CapsulePayload[\s\S]*?session_id: Option/, + 'capsule payload should carry the session generation used by the rail replay', +); +assertMatch( + coordinatorRs, + /struct CapsuleSnapshotState[\s\S]*?payload_revision: u64[\s\S]*?snapshot: Mutex[\s\S]*?capsule_snapshot\([\s\S]*?self\.capsule\.snapshot\.lock\(\)\.clone\(\)[\s\S]*?payload_revision: snapshot\.payload_revision/, + 'native rail snapshot should keep payload and payload commit revision under one lock', +); +assertMatch( + coordinatorRs, + /record_backend_event\([\s\S]*?payload_projection_pending[\s\S]*?snapshot\.revision = event\.sequence/, + 'native rail snapshot should advance relevant transcript/session revisions', +); +assertMatch( + coordinatorRs, + /snapshot\.pending_payload_revision = Some\(event\.sequence\)/, + 'native rail snapshot should keep revisions pending until a matching payload commit', +); +assertMatch( + coordinatorRs, + /pending_payload_revision: Option[\s\S]*?fn can_commit_capsule_payload\([\s\S]*?captured_revision == current_revision[\s\S]*?commit_capsule_payload\([\s\S]*?can_commit_capsule_payload\([\s\S]*?snapshot\.payload = Some\(committed\)[\s\S]*?snapshot\.payload_revision = current_revision/, + 'capsule payload commits must use strict captured/pending revisions and reject stale frames', +); +assertMatch( + tauriEventsRs, + /record_backend_event\(&event\)[\s\S]*?forward_legacy_event\(/, + 'backend event watermark must advance before queued capsule payload presentation', +); +assertMatch( + liveTranscriptTsx, + /applyTranscriptSnapshot[\s\S]*?const nextTranscript[\s\S]*?setText\(nextTranscript\.text\)/, + 'rail replay should reject snapshots older than the live event watermark', +); +assertMatch( + backendEventTs, + /payloadRevision[\s\S]*?if \(payloadRevision !== snapshot\.revision\) return state/, + 'rail replay should reject snapshots whose payload and transcript revisions differ', ); assertMatch( diff --git a/openless-all/app/src-tauri/backend-tests/tests/capsule_snapshot.rs b/openless-all/app/src-tauri/backend-tests/tests/capsule_snapshot.rs new file mode 100644 index 000000000..6a417d964 --- /dev/null +++ b/openless-all/app/src-tauri/backend-tests/tests/capsule_snapshot.rs @@ -0,0 +1,2 @@ +#[path = "../../src/capsule_snapshot.rs"] +mod capsule_snapshot; diff --git a/openless-all/app/src-tauri/capabilities/capsule-rail.json b/openless-all/app/src-tauri/capabilities/capsule-rail.json new file mode 100644 index 000000000..1b6eff102 --- /dev/null +++ b/openless-all/app/src-tauri/capabilities/capsule-rail.json @@ -0,0 +1,8 @@ +{ + "$schema": "../gen/schemas/desktop-schema.json", + "identifier": "capsule-rail", + "description": "Receive capsule state, transcript and preference events in the Windows rail", + "platforms": ["windows"], + "windows": ["capsule-rail"], + "permissions": ["core:event:allow-listen", "core:event:allow-unlisten"] +} diff --git a/openless-all/app/src-tauri/src/capsule_snapshot.rs b/openless-all/app/src-tauri/src/capsule_snapshot.rs new file mode 100644 index 000000000..b93196039 --- /dev/null +++ b/openless-all/app/src-tauri/src/capsule_snapshot.rs @@ -0,0 +1,390 @@ +//! Replay state for the native transcript rail, independent of Tauri window handles. + +use openless_core::{ + BackendEvent, BackendEventKind, CapsulePayload, CapsuleState, DictationPhase, + LessComputerEvent, LessComputerEventKind, SelectionVoicePhase, +}; + +#[derive(Clone, Debug, Default)] +pub(super) struct CapsuleSnapshotState { + pub(super) payload: Option, + pub(super) payload_revision: u64, + pub(super) pending_payload_revision: Option, + // QA and Less Computer claim the transcript only after their existing + // business ownership checks actually present the matching capsule frame. + pending_session_id: Option, + pub(super) session_id: Option, + pub(super) sequence: u64, + pub(super) revision: u64, + pub(super) text: String, +} + +fn can_commit_capsule_payload( + current_revision: u64, + pending_revision: Option, + captured_revision: u64, +) -> bool { + captured_revision == current_revision + && pending_revision.map_or(true, |pending| pending == current_revision) +} + +fn payload_matches_backend_event(payload: &CapsulePayload, event: &BackendEventKind) -> bool { + match event { + BackendEventKind::DictationStateChanged(dictation) => { + let expected = match dictation.phase { + DictationPhase::Idle => CapsuleState::Idle, + DictationPhase::Starting | DictationPhase::Recording => CapsuleState::Recording, + DictationPhase::Transcribing => CapsuleState::Transcribing, + DictationPhase::Polishing | DictationPhase::Inserting => CapsuleState::Polishing, + DictationPhase::Completed => CapsuleState::Done, + DictationPhase::Cancelled => CapsuleState::Cancelled, + DictationPhase::Failed => CapsuleState::Error, + }; + payload.state == expected + } + BackendEventKind::DictationCompleted(_) => payload.state == CapsuleState::Done, + BackendEventKind::BackendStopping => payload.state == CapsuleState::Idle, + BackendEventKind::SelectionVoiceLevel(_) => payload.state == CapsuleState::Recording, + BackendEventKind::SelectionVoiceStateChanged(selection) => { + let expected = match selection.phase { + SelectionVoicePhase::Idle + | SelectionVoicePhase::Completed + | SelectionVoicePhase::Cancelled + | SelectionVoicePhase::Failed => CapsuleState::Idle, + SelectionVoicePhase::Recording => CapsuleState::Recording, + SelectionVoicePhase::Processing + | SelectionVoicePhase::AwaitingIntent + | SelectionVoicePhase::Preview + | SelectionVoicePhase::Applying => CapsuleState::Polishing, + }; + payload.state == expected + } + // QA projections do not own the transcript session. They still use + // the same revision path, but their detailed phase is resolved by + // the async QA snapshot before presentation. + BackendEventKind::QaLevel(_) | BackendEventKind::QaState(_) => true, + _ => false, + } +} + +impl CapsuleSnapshotState { + pub(super) fn commit_capsule_payload( + &mut self, + payload: &CapsulePayload, + captured_revision: u64, + ) -> bool { + let snapshot = self; + let current_revision = snapshot.revision; + if !can_commit_capsule_payload( + current_revision, + snapshot.pending_payload_revision, + captured_revision, + ) { + return false; + } + let mut committed = payload.clone(); + if snapshot.pending_session_id.is_some() + && committed.session_id != snapshot.pending_session_id + { + return false; + } + if let Some(session_id) = snapshot + .pending_session_id + .as_ref() + .or(snapshot.session_id.as_ref()) + { + if committed + .session_id + .as_ref() + .is_some_and(|payload_session| payload_session != session_id) + { + return false; + } + committed.session_id = Some(session_id.clone()); + } + if snapshot.session_id != committed.session_id { + snapshot.text.clear(); + snapshot.session_id = committed.session_id.clone(); + } + snapshot.payload = Some(committed); + snapshot.payload_revision = current_revision; + snapshot.pending_payload_revision = None; + snapshot.pending_session_id = None; + true + } + + pub(super) fn record_backend_event(&mut self, event: &BackendEvent) { + let snapshot = self; + if event.sequence <= snapshot.sequence { + return; + } + let event_session = event.session_id.map(|session| session.to_string()); + let pending_session_id = match &event.kind { + BackendEventKind::QaLevel(_) + | BackendEventKind::QaState(_) + | BackendEventKind::LessComputerEvent(LessComputerEvent { + kind: LessComputerEventKind::VoiceState { .. }, + .. + }) => event_session.clone(), + _ => None, + }; + let payload_projection_pending = matches!( + &event.kind, + BackendEventKind::DictationStateChanged(_) + | BackendEventKind::TranscriptDelta(_) + | BackendEventKind::DictationCompleted(_) + | BackendEventKind::QaLevel(_) + | BackendEventKind::QaState(_) + | BackendEventKind::SelectionVoiceLevel(_) + | BackendEventKind::SelectionVoiceStateChanged(_) + | BackendEventKind::BackendStopping + | BackendEventKind::LessComputerEvent(LessComputerEvent { + kind: LessComputerEventKind::VoiceState { .. }, + .. + }) + ); + match &event.kind { + BackendEventKind::DictationStateChanged(dictation) + if dictation.phase == DictationPhase::Starting => + { + let session_id = dictation + .session_id + .map(|session| session.to_string()) + .or(event_session.clone()); + if session_id.is_some() && snapshot.session_id != session_id { + snapshot.session_id = session_id; + snapshot.text.clear(); + } else if snapshot.session_id.is_none() { + snapshot.session_id = session_id; + } + } + BackendEventKind::DictationStateChanged(dictation) + if dictation.phase == DictationPhase::Recording => + { + let session_id = dictation + .session_id + .map(|session| session.to_string()) + .or(event_session.clone()); + if session_id.is_some() && snapshot.session_id != session_id { + snapshot.session_id = session_id; + snapshot.text.clear(); + } else if snapshot.session_id.is_none() { + snapshot.session_id = session_id; + } + } + BackendEventKind::SelectionVoiceStateChanged(selection) + if selection.phase == SelectionVoicePhase::Recording => + { + let session_id = selection + .session_id + .map(|session| session.to_string()) + .or(event_session.clone()); + if session_id.is_some() && snapshot.session_id != session_id { + snapshot.session_id = session_id; + snapshot.text.clear(); + } else if snapshot.session_id.is_none() { + snapshot.session_id = session_id; + } + } + BackendEventKind::SelectionVoiceStateChanged(selection) => { + let session_id = selection + .session_id + .map(|session| session.to_string()) + .or(event_session.clone()); + if session_id.is_some() && snapshot.session_id != session_id { + snapshot.session_id = session_id; + snapshot.text.clear(); + } + } + BackendEventKind::TranscriptDelta(delta) + if event_session.is_none() || snapshot.session_id == event_session => + { + let offset = usize::try_from(delta.offset).ok(); + if let Some(offset) = offset { + let mut chars: Vec = snapshot.text.chars().collect(); + if offset <= chars.len() { + chars.truncate(offset); + chars.extend(delta.text.chars()); + snapshot.text = chars.into_iter().collect(); + if event_session.is_some() { + snapshot.session_id = event_session.clone(); + } + } + } + } + _ => {} + } + snapshot.sequence = event.sequence; + if payload_projection_pending { + snapshot.revision = event.sequence; + // Only an actual precommit carrying this exact event revision is + // coherent. In particular, an old Recording payload must not be + // promoted when the next Recording event is just a PCM update. + if snapshot.payload_revision == event.sequence + && snapshot + .payload + .as_ref() + .is_some_and(|payload| payload_matches_backend_event(payload, &event.kind)) + { + snapshot.pending_payload_revision = None; + snapshot.pending_session_id = None; + } else { + snapshot.pending_payload_revision = Some(event.sequence); + snapshot.pending_session_id = pending_session_id; + } + } else if snapshot.pending_payload_revision.is_none() { + // Unrelated events keep the committed payload but advance both + // watermarks, matching the frontend's global event revision. + snapshot.revision = event.sequence; + snapshot.payload_revision = event.sequence; + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use openless_core::{ + CapsuleStyle, DictationStateSnapshot, LessComputerEvent, LessComputerEventKind, + LessComputerVoicePhase, QaRecordingLevel, QaStateEvent, QaStateKind, SessionId, + }; + + fn payload(session_id: SessionId, state: CapsuleState) -> CapsulePayload { + CapsulePayload { + session_id: Some(session_id.to_string()), + state, + level: 0.0, + elapsed_ms: 0, + message: None, + inserted_chars: None, + translation: false, + operating: false, + warming: false, + capsule_style: CapsuleStyle::Classic, + selection_polish: false, + } + } + + fn completed_dictation() -> (CapsuleSnapshotState, SessionId) { + let mut snapshot = CapsuleSnapshotState::default(); + let session_id = SessionId::new(); + for (sequence, phase, state) in [ + (1, DictationPhase::Starting, CapsuleState::Recording), + (2, DictationPhase::Idle, CapsuleState::Idle), + ] { + snapshot.record_backend_event(&BackendEvent { + sequence, + session_id: Some(session_id), + kind: BackendEventKind::DictationStateChanged(DictationStateSnapshot { + phase, + session_id: (phase != DictationPhase::Idle).then_some(session_id), + ..Default::default() + }), + }); + assert!(snapshot.commit_capsule_payload(&payload(session_id, state), sequence)); + } + snapshot.text = "previous transcript".into(); + (snapshot, session_id) + } + + #[test] + fn capsule_payload_commit_requires_the_current_pending_revision() { + assert!(can_commit_capsule_payload(7, None, 7)); + assert!(can_commit_capsule_payload(7, Some(7), 7)); + assert!(!can_commit_capsule_payload(7, Some(6), 7)); + assert!(!can_commit_capsule_payload(7, Some(7), 6)); + assert!(!can_commit_capsule_payload(7, None, 8)); + } + + #[test] + fn qa_and_less_computer_can_take_over_after_dictation() { + let next = SessionId::new(); + for kind in [ + BackendEventKind::QaState(QaStateEvent::simple(QaStateKind::Recording)), + BackendEventKind::QaLevel(QaRecordingLevel { + session_id: next.to_string(), + level: 0.2, + }), + BackendEventKind::LessComputerEvent(LessComputerEvent { + seq: None, + kind: LessComputerEventKind::VoiceState { + session_id: next, + phase: LessComputerVoicePhase::Recording, + level: 0.2, + elapsed_ms: 1, + mode: Default::default(), + transcript: String::new(), + outcome: None, + }, + }), + ] { + let (mut snapshot, previous) = completed_dictation(); + snapshot.record_backend_event(&BackendEvent { + sequence: 3, + session_id: Some(next), + kind, + }); + // Merely observing another domain's event must not claim the rail: + // forward_legacy_event can still reject a superseded/background owner. + assert_eq!(snapshot.session_id, Some(previous.to_string())); + assert_eq!(snapshot.text, "previous transcript"); + let mut anonymous = payload(next, CapsuleState::Idle); + anonymous.session_id = None; + assert!(!snapshot.commit_capsule_payload(&anonymous, 3)); + assert!( + !snapshot.commit_capsule_payload(&payload(previous, CapsuleState::Recording), 3) + ); + assert!(snapshot.commit_capsule_payload(&payload(next, CapsuleState::Recording), 3)); + assert_eq!(snapshot.session_id, Some(next.to_string())); + assert!(snapshot.text.is_empty()); + assert_eq!(snapshot.payload_revision, snapshot.revision); + assert!(snapshot.pending_payload_revision.is_none()); + assert!( + !snapshot.commit_capsule_payload(&payload(previous, CapsuleState::Recording), 3) + ); + assert!(!snapshot.commit_capsule_payload(&payload(next, CapsuleState::Idle), 2)); + } + } + + #[test] + fn unrelated_events_keep_the_committed_replay_frame_coherent() { + let mut state = CapsuleSnapshotState { + sequence: 1, + revision: 1, + payload_revision: 1, + text: "已有原文".into(), + ..Default::default() + }; + state.record_backend_event(&BackendEvent { + sequence: 2, + session_id: None, + kind: BackendEventKind::BackendStarted, + }); + assert_eq!(state.sequence, 2); + assert_eq!( + state.revision, state.payload_revision, + "ready replay must not reject an unrelated event as a torn frame" + ); + assert_eq!(state.revision, 2); + assert_eq!(state.text, "已有原文"); + } + + #[test] + fn unrelated_events_do_not_commit_a_pending_payload() { + let mut state = CapsuleSnapshotState { + sequence: 1, + revision: 1, + payload_revision: 0, + pending_payload_revision: Some(1), + ..Default::default() + }; + state.record_backend_event(&BackendEvent { + sequence: 2, + session_id: None, + kind: BackendEventKind::BackendStarted, + }); + assert_eq!(state.pending_payload_revision, Some(1)); + assert_eq!(state.revision, 1); + assert_eq!(state.payload_revision, 0); + } +} diff --git a/openless-all/app/src-tauri/src/commands/dictation.rs b/openless-all/app/src-tauri/src/commands/dictation.rs index 712be93c9..5e8cc54ef 100644 --- a/openless-all/app/src-tauri/src/commands/dictation.rs +++ b/openless-all/app/src-tauri/src/commands/dictation.rs @@ -7,6 +7,14 @@ pub async fn get_startup_snapshot( core.start().await.map_err(|error| error.to_string()) } +#[cfg(not(mobile))] +#[tauri::command] +pub async fn get_capsule_snapshot( + coord: CoordinatorState<'_>, +) -> Result, String> { + Ok(coord.tauri_host().capsule_snapshot()) +} + async fn ensure_core_started(core: &openless_core::OpenLessBackend) -> Result<(), String> { if !core.snapshot().running { core.start().await.map_err(|error| error.to_string())?; @@ -48,6 +56,23 @@ pub async fn cancel_dictation( Ok(()) } +#[cfg(not(mobile))] +#[tauri::command] +pub async fn set_capsule_transcript_visible( + coord: CoordinatorState<'_>, + visible: bool, +) -> Result<(), String> { + let host = coord.tauri_host(); + let Some(capsule) = host.capsule_window() else { + return Ok(()); + }; + host.run_on_main_thread(move || { + if let Err(error) = capsule.set_transcript_visible(visible) { + log::warn!("[capsule] transcript visibility layout failed: {error}"); + } + }) +} + #[tauri::command] pub async fn handle_window_hotkey_event( coord: CoordinatorState<'_>, diff --git a/openless-all/app/src-tauri/src/coordinator.rs b/openless-all/app/src-tauri/src/coordinator.rs index af35194d5..c42fd2216 100644 --- a/openless-all/app/src-tauri/src/coordinator.rs +++ b/openless-all/app/src-tauri/src/coordinator.rs @@ -117,12 +117,13 @@ fn show_vocab_suggestion_card(inner: &Arc) { return; } inner.vocab_card_visible.store(true, Ordering::SeqCst); - // The card must be clickable, so cursor passthrough must be disabled. + // The card owns the shared HWND: its full client area is interactive, + // while the capsule rail is hidden until the card is dismissed. // Android has no capsule window and tauri's set_ignore_cursor_events // does not exist there (same handling as capsule_focus.rs). #[cfg(not(mobile))] - if let Err(e) = capsule.set_cursor_passthrough(false) { - log::warn!("[vocab-card] set_ignore_cursor_events(false) failed: {e}"); + if let Err(e) = capsule.set_card_hit_test_mode() { + log::warn!("[vocab-card] enabling card hit testing failed: {e}"); } if let Err(e) = capsule.set_size(VOCAB_CARD_WIDTH, height) { log::warn!("[vocab-card] resize failed: {e}"); @@ -132,6 +133,9 @@ fn show_vocab_suggestion_card(inner: &Arc) { { log::warn!("[vocab-card] position failed: {e}"); } + if let Err(e) = capsule.refresh_card_hit_test() { + log::warn!("[vocab-card] refreshing card hit testing failed: {e}"); + } // Positioning ditto: the dedup cache in `maybe_position_capsule_bottom_center` // only tracks "monitor + translation state" and knows nothing about // this move. Without invalidation, the next recording would judge @@ -170,28 +174,14 @@ fn hide_vocab_suggestion_card(inner: &Arc) { // Passthrough must be restored, or the capsule keeps blocking that // area at the bottom of the screen. #[cfg(not(mobile))] - if let Err(e) = capsule.set_cursor_passthrough(true) { - log::warn!("[vocab-card] restoring cursor passthrough failed: {e}"); - } - // Size must be restored too — the card shrank the window to its own - // size, and without this the next capsule would be squeezed into a - // 320×108 window, effectively invisible. - let bounds = crate::capsule_window_bounds(false); - if let Err(e) = capsule.set_size(bounds.width, bounds.height) { - log::warn!("[vocab-card] restoring capsule size failed: {e}"); - } - // Position must be restored too — the card moved the window to the - // bottom-right while the capsule sits bottom-center. Restoring size - // alone would put the next recording capsule in the bottom-right - // corner. - // - // Both cache invalidation and this repositioning are needed: - // invalidation guarantees the next emit_capsule recomputes even if this - // repositioning fails; repositioning guarantees the window is already - // in the right place even if some path shows it without emit_capsule. - capsule.invalidate_layout(); - if let Err(e) = capsule.position_capsule_bottom_center(false) { - log::warn!("[vocab-card] restoring capsule position failed: {e}"); + if let Err(e) = capsule.restore_capsule_hit_test_mode() { + log::warn!("[vocab-card] restoring capsule hit testing failed: {e}"); + } + // On mobile there is no native hit-test mode to restore, but the capsule + // still needs the cached style/rail geometry after the card resized it. + #[cfg(mobile)] + if let Err(e) = capsule.restore_capsule_geometry() { + log::warn!("[vocab-card] restoring capsule geometry failed: {e}"); } }); } @@ -264,8 +254,8 @@ fn show_insert_fallback_card(inner: &Arc, text: String, reason: &'static presentation_id, }; #[cfg(not(mobile))] - if let Err(e) = capsule.set_cursor_passthrough(false) { - log::warn!("[fallback-card] set_ignore_cursor_events(false) failed: {e}"); + if let Err(e) = capsule.set_card_hit_test_mode() { + log::warn!("[fallback-card] enabling card hit testing failed: {e}"); } if let Err(e) = capsule.set_size(FALLBACK_CARD_WIDTH, FALLBACK_CARD_INITIAL_HEIGHT) { log::warn!("[fallback-card] resize failed: {e}"); @@ -275,6 +265,9 @@ fn show_insert_fallback_card(inner: &Arc, text: String, reason: &'static { log::warn!("[fallback-card] position failed: {e}"); } + if let Err(e) = capsule.refresh_card_hit_test() { + log::warn!("[fallback-card] refreshing card hit testing failed: {e}"); + } // Positioning ditto: the dedup cache in `maybe_position_capsule_bottom_center` // only tracks "monitor + translation state" and knows nothing about // this move. Without invalidation the next recording would judge @@ -320,6 +313,9 @@ fn report_insert_fallback_card_height( if let Err(e) = capsule.position_fallback_card(FALLBACK_CARD_WIDTH, height) { log::warn!("[fallback-card] measured position failed: {e}"); } + if let Err(e) = capsule.refresh_card_hit_test() { + log::warn!("[fallback-card] refreshing measured card hit testing failed: {e}"); + } }) } @@ -348,24 +344,14 @@ fn hide_insert_fallback_card(inner: &Arc) { // Passthrough must be restored, or the capsule keeps blocking that // screen area. #[cfg(not(mobile))] - if let Err(e) = capsule.set_cursor_passthrough(true) { - log::warn!("[fallback-card] restoring cursor passthrough failed: {e}"); - } - // Size must be restored too — the card shrank the window to its own - // size, and without this the next capsule would be squeezed into a - // card-sized window, effectively invisible. - let bounds = crate::capsule_window_bounds(false); - if let Err(e) = capsule.set_size(bounds.width, bounds.height) { - log::warn!("[fallback-card] restoring capsule size failed: {e}"); - } - // Position must be restored too — the card moved the window to the - // bottom-right while the capsule sits bottom-center. Restoring size - // alone would put the next recording capsule in the bottom-right - // corner. Both cache invalidation and this repositioning are needed; - // see `hide_vocab_suggestion_card` for rationale. - capsule.invalidate_layout(); - if let Err(e) = capsule.position_capsule_bottom_center(false) { - log::warn!("[fallback-card] restoring capsule position failed: {e}"); + if let Err(e) = capsule.restore_capsule_hit_test_mode() { + log::warn!("[fallback-card] restoring capsule hit testing failed: {e}"); + } + // On mobile there is no native hit-test mode to restore, but the capsule + // still needs the cached style/rail geometry after the card resized it. + #[cfg(mobile)] + if let Err(e) = capsule.restore_capsule_geometry() { + log::warn!("[fallback-card] restoring capsule geometry failed: {e}"); } if let Some(payload) = deferred_capsule { // During the card, QA / Selection Polish still advance capsule @@ -899,7 +885,8 @@ impl Coordinator { } pub fn present_core_capsule(&self, payload: CapsulePayload) { - let _ = self.present_core_capsule_if_current(payload, None); + let captured_revision = self.inner.host.capsule_revision(); + let _ = self.present_core_capsule_at_revision(payload, None, captured_revision); } /// Selection-voice claims the shared capsule from hotkey Start through @@ -931,12 +918,22 @@ impl Coordinator { &self, payload: CapsulePayload, expected_epoch: Option, + ) -> Option { + let captured_revision = self.inner.host.capsule_revision(); + self.present_core_capsule_at_revision(payload, expected_epoch, captured_revision) + } + + fn present_core_capsule_at_revision( + &self, + payload: CapsulePayload, + expected_epoch: Option, + captured_revision: u64, ) -> Option { let state = payload.state; // Core already owns this frame's translation, readiness, and session // ownership; the window layer must not rebuild it from stale late // state, or cold starts and fast switches lose real feedback. - let epoch = emit_core_capsule(&self.inner, payload, expected_epoch)?; + let epoch = emit_core_capsule(&self.inner, payload, expected_epoch, captured_revision)?; if let Some(delay_ms) = core_capsule_hide_delay(state) { let inner = Arc::clone(&self.inner); self.inner.host.spawn(async move { diff --git a/openless-all/app/src-tauri/src/coordinator/capsule_focus.rs b/openless-all/app/src-tauri/src/coordinator/capsule_focus.rs index 0f10d044f..6896a7e1c 100644 --- a/openless-all/app/src-tauri/src/coordinator/capsule_focus.rs +++ b/openless-all/app/src-tauri/src/coordinator/capsule_focus.rs @@ -195,6 +195,7 @@ fn emit_capsule_with_context_locked( ) -> u64 { let dictation = inner.backend.snapshot().dictation; let payload = CapsulePayload { + session_id: dictation.session_id.map(|session| session.to_string()), state, level, elapsed_ms, @@ -212,7 +213,8 @@ fn emit_capsule_with_context_locked( selection_polish, capsule_style: inner.host.cached_capsule_style(), }; - emit_capsule_payload_locked(inner, payload) + let captured_revision = inner.host.capsule_revision(); + emit_capsule_payload_locked(inner, payload, captured_revision) } /// Core feedback enters the same native display outlet wholesale, including warming, @@ -221,12 +223,13 @@ pub(super) fn emit_core_capsule( inner: &Arc, payload: CapsulePayload, expected_epoch: Option, + captured_revision: u64, ) -> Option { emit_capsule_at_epoch( &inner.capsule_event_lock, &inner.capsule_event_epoch, expected_epoch, - || emit_capsule_payload_locked(inner, payload), + || emit_capsule_payload_locked(inner, payload, captured_revision), ) } @@ -252,7 +255,11 @@ pub(super) fn hide_core_capsule_if_current(inner: &Arc, expected_epoch: u ); } -fn emit_capsule_payload_locked(inner: &Arc, payload: CapsulePayload) -> u64 { +fn emit_capsule_payload_locked( + inner: &Arc, + payload: CapsulePayload, + captured_revision: u64, +) -> u64 { let state = payload.state; let selection_polish = payload.selection_polish; // Advance the epoch on every payload. That way an old selection-polish terminal timer is @@ -291,6 +298,16 @@ fn emit_capsule_payload_locked(inner: &Arc, payload: CapsulePayload) -> u // Keep the latest full feedback during a fallback card, even before the window handle is // validated; a re-show must not regress to an old preparing state. defer_capsule_payload_if_fallback_active(inner, &payload); + // Commit the replay frame before queueing any native window work. The + // snapshot lock therefore has one revision for payload and transcript even + // when run_on_main_thread is delayed. + if !inner + .host + .commit_capsule_payload(&payload, captured_revision) + { + log::debug!("[capsule] skipped stale payload commit at revision {captured_revision}"); + return event_epoch; + } let Some(capsule) = inner.host.capsule_window() else { return event_epoch; }; @@ -456,6 +473,7 @@ mod tests { fn payload(state: CapsuleState) -> CapsulePayload { CapsulePayload { + session_id: None, state, level: 0.0, elapsed_ms: 0, diff --git a/openless-all/app/src-tauri/src/coordinator/hotkey_loops.rs b/openless-all/app/src-tauri/src/coordinator/hotkey_loops.rs index 6803ac60a..8ee061c3d 100644 --- a/openless-all/app/src-tauri/src/coordinator/hotkey_loops.rs +++ b/openless-all/app/src-tauri/src/coordinator/hotkey_loops.rs @@ -2816,6 +2816,7 @@ pub(crate) mod windows_less_computer_tests { fixture_coordinator(crate::types::HotkeyMode::Toggle, std::time::Duration::ZERO); coordinator.inner.host.begin_insert_fallback_card(); let expected = CapsulePayload { + session_id: None, state: CapsuleState::Recording, level: 0.0, elapsed_ms: 0, diff --git a/openless-all/app/src-tauri/src/lib.rs b/openless-all/app/src-tauri/src/lib.rs index 264e7063e..dd72f4700 100644 --- a/openless-all/app/src-tauri/src/lib.rs +++ b/openless-all/app/src-tauri/src/lib.rs @@ -213,6 +213,7 @@ macro_rules! app_invoke_handler_desktop { () => { tauri::generate_handler![ commands::get_startup_snapshot, + commands::get_capsule_snapshot, commands::get_settings, commands::get_settings_snapshot, commands::update_setting_fields, @@ -309,6 +310,7 @@ macro_rules! app_invoke_handler_desktop { commands::start_dictation, commands::stop_dictation, commands::cancel_dictation, + commands::set_capsule_transcript_visible, coding_agent::commands::coding_agent_detect, coding_agent::commands::coding_agent_detect_opencode, coding_agent::commands::coding_agent_detect_cli, @@ -3265,10 +3267,36 @@ pub(crate) fn position_capsule_bottom_center( /// following auto-hide settings. pub(crate) fn position_capsule_bottom_center_with_style( window: &tauri::WebviewWindow, - _translation_active: bool, + translation_active: bool, + style: types::CapsuleStyle, +) -> tauri::Result<()> { + position_capsule_bottom_center_with_style_and_transcript( + window, + translation_active, + style, + true, + ) +} + +pub(crate) fn position_capsule_bottom_center_with_style_and_transcript( + window: &tauri::WebviewWindow, + translation_active: bool, style: types::CapsuleStyle, + transcript_visible: bool, ) -> tauri::Result<()> { - let bounds = capsule_window_bounds_for_style(style); + // Windows renders the transcript in a separate click-through overlay HWND. The capsule + // window itself therefore reserves only the body height; keeping the rail in this window + // would make its transparent gutter part of the input region again. + let native_transcript_visible = if cfg!(target_os = "windows") { + false + } else { + transcript_visible + }; + let bounds = capsule_window_bounds_for_style_with_transcript_and_translation( + style, + native_transcript_visible, + translation_active, + ); const EDGE_GAP: f64 = 12.0; // Typeless hugs the work-area bottom edge (the work area already excludes Dock / taskbar). let bottom_gap = match style { @@ -3371,6 +3399,25 @@ fn capsule_window_bounds(translation_active: bool) -> CapsuleWindowBounds { } fn capsule_window_bounds_for_style(style: types::CapsuleStyle) -> CapsuleWindowBounds { + capsule_window_bounds_for_style_with_transcript(style, true) +} + +fn capsule_window_bounds_for_style_with_transcript( + style: types::CapsuleStyle, + transcript_visible: bool, +) -> CapsuleWindowBounds { + capsule_window_bounds_for_style_with_transcript_and_translation( + style, + transcript_visible, + false, + ) +} + +fn capsule_window_bounds_for_style_with_transcript_and_translation( + style: types::CapsuleStyle, + transcript_visible: bool, + translation_active: bool, +) -> CapsuleWindowBounds { CapsuleWindowBounds { // The typeless window area is 1/5 of the original size (460×128); the frontend // scales content in sync with CSS zoom — see CapsuleStyles.css and @@ -3380,9 +3427,30 @@ fn capsule_window_bounds_for_style(style: types::CapsuleStyle) -> CapsuleWindowB types::CapsuleStyle::Siri | types::CapsuleStyle::Classic => 460.0, }, height: match style { - types::CapsuleStyle::Siri => 180.0, - types::CapsuleStyle::Classic => 100.0, - types::CapsuleStyle::Typeless => 57.0, + // The rail sits above the 180px Siri stage: 40px rail + 8px gap. + types::CapsuleStyle::Siri => { + if transcript_visible { + 228.0 + } else { + 180.0 + } + } + // Classic keeps the 52px pill, 40px rail, 8px gap, 16px bottom inset, + // and enough headroom for the translating badge. + types::CapsuleStyle::Classic => { + if cfg!(target_os = "windows") || transcript_visible { + 172.0 + } else { + 100.0 + } + } + types::CapsuleStyle::Typeless => { + if translation_active { + 65.0 + } else { + 57.0 + } + } }, bottom_inset: 0.0, } @@ -3402,7 +3470,9 @@ fn capsule_height_for_qa() -> f64 { mod tests { use super::{ bottom_center_position, capsule_height_for_qa, capsule_visual_height, - capsule_window_bounds, capsule_window_bounds_for_style, clamp_to_monitor, + capsule_window_bounds, capsule_window_bounds_for_style, + capsule_window_bounds_for_style_with_transcript, + capsule_window_bounds_for_style_with_transcript_and_translation, clamp_to_monitor, frame_contains_point, frame_distance_to_point_squared, logical_monitor_frame, parse_tray_style_pack_menu_id, resolve_tray_style_pack_id, rotate_log_if_too_large, tray_style_menu_enabled, tray_style_pack_menu_entries, tray_style_pack_menu_id, @@ -3596,11 +3666,11 @@ mod tests { } #[test] - fn capsule_window_bounds_match_voice_orb_stage() { + fn capsule_window_bounds_reserve_the_transcript_rail() { let bounds = capsule_window_bounds(false); assert_eq!( (bounds.width, bounds.height, bounds.bottom_inset), - (460.0, 180.0, 0.0) + (460.0, 228.0, 0.0) ); } @@ -3636,7 +3706,7 @@ mod tests { let bounds = capsule_window_bounds(true); assert_eq!( (bounds.width, bounds.height, bounds.bottom_inset), - (460.0, 180.0, 0.0) + (460.0, 228.0, 0.0) ); } @@ -3646,6 +3716,32 @@ mod tests { assert_eq!((bounds.width, bounds.height), (206.0, 57.0)); } + #[test] + fn typeless_capsule_window_reserves_translation_badge_row() { + let bounds = capsule_window_bounds_for_style_with_transcript_and_translation( + CapsuleStyle::Typeless, + true, + true, + ); + assert_eq!((bounds.width, bounds.height), (206.0, 65.0)); + } + + #[test] + fn classic_capsule_window_keeps_external_rail_host_height_on_windows() { + let bounds = capsule_window_bounds_for_style_with_transcript(CapsuleStyle::Classic, false); + assert_eq!( + (bounds.width, bounds.height), + ( + 460.0, + if cfg!(target_os = "windows") { + 172.0 + } else { + 100.0 + } + ) + ); + } + #[test] fn capsule_visual_height_keeps_the_qa_anchor_reference() { assert_eq!(capsule_visual_height(true), 140.0); diff --git a/openless-all/app/src-tauri/src/persistence/credentials.rs b/openless-all/app/src-tauri/src/persistence/credentials.rs index 5ef7ef794..5124adbc7 100644 --- a/openless-all/app/src-tauri/src/persistence/credentials.rs +++ b/openless-all/app/src-tauri/src/persistence/credentials.rs @@ -955,6 +955,8 @@ struct CredsLlmEntry { maxTokens: Option, #[serde(skip_serializing_if = "Option::is_none")] thinkingBudget: Option, + #[serde(skip_serializing_if = "Option::is_none")] + serviceTier: Option, } impl CredsLlmEntry { @@ -965,6 +967,7 @@ impl CredsLlmEntry { MESSAGES_THINKING_ACCOUNT => Ok(&mut self.messagesThinking), MAX_TOKENS_ACCOUNT => Ok(&mut self.maxTokens), THINKING_BUDGET_ACCOUNT => Ok(&mut self.thinkingBudget), + SERVICE_TIER_ACCOUNT => Ok(&mut self.serviceTier), _ => anyhow::bail!("unsupported LLM protocol option"), } } @@ -990,6 +993,7 @@ impl CredsLlmEntry { && self.messagesThinking.is_none() && self.maxTokens.is_none() && self.thinkingBudget.is_none() + && self.serviceTier.is_none() && self .extraHeaders .as_ref() @@ -3116,7 +3120,8 @@ fn export_sync_credentials_root( )); let mut values = accounts!(entry, "ark.api_key"=>apiKey, "ark.endpoint"=>baseURL, "ark.model_id"=>model, "ark.request_format"=>requestFormat, "ark.messages_thinking"=>messagesThinking, - "ark.max_tokens"=>maxTokens, "ark.thinking_budget"=>thinkingBudget); + "ark.max_tokens"=>maxTokens, "ark.thinking_budget"=>thinkingBudget, + "ark.service_tier"=>serviceTier); if let Some(value) = entry.temperature { values.insert("ark.temperature".into(), value.to_string()); } @@ -3385,6 +3390,7 @@ fn apply_sync_credentials_root( "ark.messages_thinking", "ark.max_tokens", "ark.thinking_budget", + "ark.service_tier", ] { if let Some(value) = values.get(key) { protocol.apply(key, value)?; @@ -3417,6 +3423,7 @@ fn apply_sync_credentials_root( messagesThinking: value("ark.messages_thinking"), maxTokens: value("ark.max_tokens"), thinkingBudget: value("ark.thinking_budget"), + serviceTier: value("ark.service_tier"), }, ); if channel.active { @@ -4273,7 +4280,7 @@ mod tests { use openless_core::llm_protocol::*; let mut root = CredsRoot::default(); let entry = root.providers.llm.entry("channel-b".into()).or_default(); - let values = ["messages", "budget", "8192", "2048"]; + let values = ["messages", "budget", "8192", "2048", "fast"]; for (account, value) in CONFIG_ACCOUNTS.into_iter().zip(values) { *entry.protocol_option(account).unwrap() = Some(value.into()); } @@ -5645,6 +5652,7 @@ mod encrypted_sync_tests { messagesThinking: Some("adaptive".into()), maxTokens: Some("4096".into()), thinkingBudget: None, + serviceTier: Some("fast".into()), }, ); root.omni.active = "custom".into(); diff --git a/openless-all/app/src-tauri/src/tauri_coordinator_host.rs b/openless-all/app/src-tauri/src/tauri_coordinator_host.rs index 39d958398..e03c50f24 100644 --- a/openless-all/app/src-tauri/src/tauri_coordinator_host.rs +++ b/openless-all/app/src-tauri/src/tauri_coordinator_host.rs @@ -4,10 +4,18 @@ //! [`tauri::AppHandle`] and keeps window, main-thread and managed-state access //! out of the coordinator's business paths. +#[cfg(target_os = "windows")] +use std::collections::HashMap; use std::future::Future; use std::sync::atomic::{AtomicBool, AtomicU64, AtomicU8, Ordering}; use std::sync::Arc; +#[cfg(target_os = "windows")] +use std::sync::OnceLock; +#[path = "capsule_snapshot.rs"] +mod capsule_snapshot; +use capsule_snapshot::CapsuleSnapshotState; +use openless_core::BackendEvent; use parking_lot::Mutex; use tauri::{AppHandle, Emitter, Manager}; @@ -18,6 +26,420 @@ static CAPSULE_FIRST_SHOW_LOGGED: AtomicBool = AtomicBool::new(false); static CAPSULE_NO_ACTIVATE_FALLBACK_WARNED: AtomicBool = AtomicBool::new(false); static CAPSULE_WINDOW_MISSING_LOGGED: AtomicBool = AtomicBool::new(false); +const HIT_TEST_MODE_CAPSULE: u8 = 0; +const HIT_TEST_MODE_CARD: u8 = 1; + +#[derive(Clone, Copy, Debug, PartialEq)] +struct CapsuleTranscriptRailPosition { + width: f64, + height: f64, + /// Rail top relative to the capsule body HWND top, in logical/native pixels. + top_offset: f64, + gap: f64, +} + +/// Keep the standalone rail attached to the actual capsule body content. The +/// capsule HWND contains transparent headroom for each style, so subtracting +/// rail height from the HWND top puts Classic and Typeless rails too high. +fn capsule_transcript_rail_position( + style: CapsuleStyle, + translation_active: bool, + transcript_visible: bool, +) -> Option { + if !transcript_visible && (!translation_active || style == CapsuleStyle::Siri) { + return None; + } + match style { + CapsuleStyle::Siri => { + let body_top = 0.0; + let height = 40.0; + let gap = 8.0; + Some(CapsuleTranscriptRailPosition { + width: 460.0, + height, + top_offset: body_top - gap - height, + gap, + }) + } + CapsuleStyle::Classic => { + const HOST_HEIGHT: f64 = 172.0; + const BODY_BOTTOM_INSET: f64 = 16.0; + const BODY_HEIGHT: f64 = 52.0; + const RAIL_HEIGHT: f64 = 52.0; + const RAIL_GAP: f64 = 8.0; + const BADGE_HEIGHT: f64 = 22.0; + const BADGE_GAP: f64 = 8.0; + let body_top = HOST_HEIGHT - BODY_BOTTOM_INSET - BODY_HEIGHT; + let gap = RAIL_GAP; + let height = if transcript_visible { RAIL_HEIGHT } else { 0.0 } + + if translation_active { + BADGE_HEIGHT + if transcript_visible { BADGE_GAP } else { 0.0 } + } else { + 0.0 + }; + let top_offset = body_top - RAIL_GAP - height; + Some(CapsuleTranscriptRailPosition { + width: 460.0, + height, + top_offset, + gap, + }) + } + CapsuleStyle::Typeless => { + const ZOOM: f64 = 0.447; + let host_height = if translation_active { 65.0 } else { 57.0 }; + let body_bottom = host_height; + let body_top = + body_bottom - (64.0 * ZOOM) - if translation_active { 20.0 * ZOOM } else { 0.0 }; + let rail_height = if transcript_visible { 52.0 * ZOOM } else { 0.0 }; + let height = rail_height + if translation_active { 20.0 * ZOOM } else { 0.0 }; + let gap = 0.0; + Some(CapsuleTranscriptRailPosition { + width: 206.0, + height, + top_offset: body_top - gap - rail_height, + gap, + }) + } + } +} + +#[cfg(target_os = "windows")] +#[derive(Clone, Copy)] +struct CapsuleHitTestEntry { + previous_proc: isize, + control_left: i32, + control_right: i32, + control_top: i32, + control_bottom: i32, +} + +#[cfg(target_os = "windows")] +static CAPSULE_HIT_TEST_ENTRIES: OnceLock>> = + OnceLock::new(); + +#[cfg(target_os = "windows")] +fn capsule_hit_test_entries() -> &'static Mutex> { + CAPSULE_HIT_TEST_ENTRIES.get_or_init(|| Mutex::new(HashMap::new())) +} + +#[cfg(target_os = "windows")] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct CapsuleHitTestRect { + left: i32, + right: i32, + top: i32, + bottom: i32, +} + +#[cfg(target_os = "windows")] +fn capsule_control_hit_rect( + style: CapsuleStyle, + transcript_visible: bool, + translation_active: bool, + window_width: i32, + window_height: i32, +) -> Option { + if style == CapsuleStyle::Siri || window_width <= 0 || window_height <= 0 { + return None; + } + let ( + logical_host_width, + logical_host_height, + logical_body_width, + logical_body_height, + logical_bottom_inset, + ) = match style { + // Classic transcript is hosted by the separate click-through rail HWND on + // Windows. The capsule HWND therefore owns only its 172px host, regardless + // of whether the rail is visible. + CapsuleStyle::Classic => (460.0, 172.0, 196.0, 52.0, 16.0), + // Typeless content is scaled by zoom: 64px CSS × 0.447. The native window may be + // 57px or 65px high when the dedicated translation row is present. + CapsuleStyle::Typeless => ( + 206.0, + if translation_active { 65.0 } else { 57.0 }, + 232.0 * 0.447, + 64.0 * 0.447, + 0.0, + ), + CapsuleStyle::Siri => unreachable!(), + }; + let _ = transcript_visible; + let scale = + (window_width as f64 / logical_host_width).min(window_height as f64 / logical_host_height); + let body_width = (logical_body_width * scale).round() as i32; + let body_height = (logical_body_height * scale).round() as i32; + let bottom_inset = (logical_bottom_inset * scale).round() as i32; + let control_left = ((window_width as f64 - body_width as f64) / 2.0).round() as i32; + let control_bottom = window_height - bottom_inset; + Some(CapsuleHitTestRect { + left: control_left.max(0), + right: (control_left + body_width).min(window_width), + top: (control_bottom - body_height).max(0), + bottom: control_bottom.max(0), + }) +} + +#[cfg(target_os = "windows")] +fn capsule_card_hit_rect(window_width: i32, window_height: i32) -> CapsuleHitTestRect { + CapsuleHitTestRect { + left: 0, + right: window_width.max(0), + top: 0, + bottom: window_height.max(0), + } +} + +#[cfg(target_os = "windows")] +unsafe extern "system" fn capsule_hit_test_window_proc( + hwnd: windows::Win32::Foundation::HWND, + msg: u32, + wparam: windows::Win32::Foundation::WPARAM, + lparam: windows::Win32::Foundation::LPARAM, +) -> windows::Win32::Foundation::LRESULT { + use windows::Win32::Foundation::RECT; + use windows::Win32::UI::WindowsAndMessaging::{ + CallWindowProcW, DefWindowProcW, GetWindowRect, SetWindowLongPtrW, GWLP_WNDPROC, + HTTRANSPARENT, WM_NCDESTROY, WM_NCHITTEST, + }; + + if msg == WM_NCDESTROY { + let previous_proc = capsule_hit_test_entries() + .lock() + .remove(&(hwnd.0 as isize)) + .map(|entry| entry.previous_proc) + .unwrap_or_default(); + if previous_proc != 0 { + let previous: windows::Win32::UI::WindowsAndMessaging::WNDPROC = + std::mem::transmute(previous_proc); + let _ = SetWindowLongPtrW(hwnd, GWLP_WNDPROC, previous_proc); + return CallWindowProcW(previous, hwnd, msg, wparam, lparam); + } + return DefWindowProcW(hwnd, msg, wparam, lparam); + } + + if msg == WM_NCHITTEST { + let key = hwnd.0 as isize; + let entry = capsule_hit_test_entries().lock().get(&key).copied(); + if let Some(entry) = entry { + let screen_x = (lparam.0 as i32 & 0xffff) as u16 as i16 as i32; + let screen_y = ((lparam.0 >> 16) & 0xffff) as u16 as i16 as i32; + let mut rect = RECT::default(); + if GetWindowRect(hwnd, &mut rect).is_ok() { + let client_x = screen_x - rect.left; + let client_y = screen_y - rect.top; + if client_x < entry.control_left + || client_x >= entry.control_right + || client_y < entry.control_top + || client_y >= entry.control_bottom + { + return windows::Win32::Foundation::LRESULT(HTTRANSPARENT as isize); + } + } + } + } + + let previous_proc = capsule_hit_test_entries() + .lock() + .get(&(hwnd.0 as isize)) + .map(|entry| entry.previous_proc) + .unwrap_or_default(); + if previous_proc != 0 { + let previous: windows::Win32::UI::WindowsAndMessaging::WNDPROC = + std::mem::transmute(previous_proc); + CallWindowProcW(previous, hwnd, msg, wparam, lparam) + } else { + DefWindowProcW(hwnd, msg, wparam, lparam) + } +} + +#[cfg(target_os = "windows")] +fn install_capsule_hit_test_proc( + hwnd: windows::Win32::Foundation::HWND, + rect: CapsuleHitTestRect, +) -> tauri::Result<()> { + use windows::Win32::Foundation::{GetLastError, SetLastError, ERROR_SUCCESS}; + use windows::Win32::UI::WindowsAndMessaging::{SetWindowLongPtrW, GWLP_WNDPROC}; + + if hwnd.0.is_null() { + return Err(tauri::Error::Anyhow(anyhow::anyhow!( + "cannot install hit-test proc for a null HWND" + ))); + } + + let key = hwnd.0 as isize; + let mut entries = capsule_hit_test_entries().lock(); + if let Some(entry) = entries.get_mut(&key) { + entry.control_left = rect.left; + entry.control_right = rect.right; + entry.control_top = rect.top; + entry.control_bottom = rect.bottom; + return Ok(()); + } + unsafe { SetLastError(ERROR_SUCCESS) }; + let previous = unsafe { + SetWindowLongPtrW( + hwnd, + GWLP_WNDPROC, + capsule_hit_test_window_proc as usize as isize, + ) + }; + if previous == 0 && unsafe { GetLastError() } != ERROR_SUCCESS { + return Err(tauri::Error::Anyhow(anyhow::anyhow!( + "SetWindowLongPtrW(GWLP_WNDPROC) failed" + ))); + } + entries.insert( + key, + CapsuleHitTestEntry { + previous_proc: previous, + control_left: rect.left, + control_right: rect.right, + control_top: rect.top, + control_bottom: rect.bottom, + }, + ); + Ok(()) +} + +#[cfg(target_os = "windows")] +fn set_capsule_input_region( + hwnd: windows::Win32::Foundation::HWND, + rect: CapsuleHitTestRect, +) -> tauri::Result<()> { + use windows::Win32::Foundation::BOOL; + use windows::Win32::Graphics::Gdi::{CreateRoundRectRgn, DeleteObject, SetWindowRgn, HGDIOBJ}; + + if hwnd.0.is_null() { + return Err(tauri::Error::Anyhow(anyhow::anyhow!( + "cannot set input region for a null HWND" + ))); + } + + let radius = ((rect.bottom - rect.top) / 2).max(1); + let region = + unsafe { CreateRoundRectRgn(rect.left, rect.top, rect.right, rect.bottom, radius, radius) }; + if region.0.is_null() { + return Err(tauri::Error::Anyhow(anyhow::anyhow!( + "CreateRoundRectRgn returned a null region" + ))); + } + let result = unsafe { SetWindowRgn(hwnd, region, BOOL(1)) }; + if result == 0 { + unsafe { + let _ = DeleteObject(HGDIOBJ(region.0)); + } + return Err(tauri::Error::Anyhow(anyhow::anyhow!("SetWindowRgn failed"))); + } + Ok(()) +} + +#[cfg(target_os = "windows")] +fn clear_capsule_input_region(hwnd: windows::Win32::Foundation::HWND) { + use windows::Win32::Foundation::BOOL; + use windows::Win32::Graphics::Gdi::{SetWindowRgn, HRGN}; + + if hwnd.0.is_null() { + return; + } + unsafe { + let _ = SetWindowRgn(hwnd, HRGN::default(), BOOL(1)); + } +} + +#[cfg(target_os = "windows")] +fn configure_capsule_hit_test( + window: &tauri::WebviewWindow, + style: CapsuleStyle, + transcript_visible: bool, + translation_active: bool, +) -> tauri::Result<()> { + use raw_window_handle::{HasWindowHandle, RawWindowHandle}; + use windows::Win32::Foundation::RECT; + use windows::Win32::UI::WindowsAndMessaging::GetClientRect; + + let Ok(handle) = window.window_handle() else { + return Ok(()); + }; + let RawWindowHandle::Win32(raw) = handle.as_raw() else { + return Ok(()); + }; + let hwnd = windows::Win32::Foundation::HWND(raw.hwnd.get() as *mut _); + if hwnd.0.is_null() { + return Err(tauri::Error::Anyhow(anyhow::anyhow!( + "capsule window returned a null HWND" + ))); + } + let mut client = RECT::default(); + unsafe { + GetClientRect(hwnd, &mut client).map_err(|error| tauri::Error::Anyhow(error.into()))?; + } + let Some(rect) = capsule_control_hit_rect( + style, + transcript_visible, + translation_active, + client.right - client.left, + client.bottom - client.top, + ) else { + return Ok(()); + }; + set_capsule_input_region(hwnd, rect)?; + if let Err(error) = install_capsule_hit_test_proc(hwnd, rect) { + clear_capsule_input_region(hwnd); + return Err(error); + } + Ok(()) +} + +#[cfg(target_os = "windows")] +fn configure_card_hit_test( + window: &tauri::WebviewWindow, +) -> tauri::Result<()> { + use raw_window_handle::{HasWindowHandle, RawWindowHandle}; + use windows::Win32::Foundation::RECT; + use windows::Win32::UI::WindowsAndMessaging::GetClientRect; + + let Ok(handle) = window.window_handle() else { + return Ok(()); + }; + let RawWindowHandle::Win32(raw) = handle.as_raw() else { + return Ok(()); + }; + let hwnd = windows::Win32::Foundation::HWND(raw.hwnd.get() as *mut _); + if hwnd.0.is_null() { + return Err(tauri::Error::Anyhow(anyhow::anyhow!( + "card window returned a null HWND" + ))); + } + let mut client = RECT::default(); + unsafe { + GetClientRect(hwnd, &mut client).map_err(|error| tauri::Error::Anyhow(error.into()))?; + } + let rect = capsule_card_hit_rect(client.right - client.left, client.bottom - client.top); + clear_capsule_input_region(hwnd); + install_capsule_hit_test_proc(hwnd, rect) +} + +#[cfg(target_os = "windows")] +fn clear_capsule_input_region_for_window( + window: &tauri::WebviewWindow, +) -> tauri::Result<()> { + use raw_window_handle::{HasWindowHandle, RawWindowHandle}; + + let Ok(handle) = window.window_handle() else { + return Ok(()); + }; + let RawWindowHandle::Win32(raw) = handle.as_raw() else { + return Ok(()); + }; + let hwnd = windows::Win32::Foundation::HWND(raw.hwnd.get() as *mut _); + if !hwnd.0.is_null() { + clear_capsule_input_region(hwnd); + } + Ok(()) +} + #[derive(Clone, Copy, Debug, PartialEq, Eq)] enum CapsuleShowStrategy { NoActivate, @@ -91,9 +513,13 @@ fn show_capsule_window_no_activate( return false; }; let hwnd = HWND(raw.hwnd.get() as *mut _); + if hwnd.0.is_null() { + log::warn!("[capsule] no_activate failed: Win32 handle is null"); + return false; + } let _ = unsafe { ShowWindow(hwnd, SW_SHOWNOACTIVATE) }; - let _ = unsafe { + if let Err(error) = unsafe { SetWindowPos( hwnd, HWND_TOPMOST, @@ -103,7 +529,10 @@ fn show_capsule_window_no_activate( 0, SWP_NOMOVE | SWP_NOSIZE | SWP_NOACTIVATE | SWP_SHOWWINDOW, ) - }; + } { + log::warn!("[capsule] no_activate failed: SetWindowPos returned {error}"); + return false; + } true } @@ -235,6 +664,7 @@ fn capsule_window_action( #[derive(Clone, Copy, Debug, PartialEq, Eq)] struct CapsuleLayoutState { translation_active: bool, + transcript_visible: bool, style: CapsuleStyle, monitor_x: i32, monitor_y: i32, @@ -247,14 +677,28 @@ struct CapsuleLayoutState { scale_bits: u64, } +#[derive(Clone, Debug, serde::Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct CapsuleSnapshot { + #[serde(flatten)] + pub(crate) payload: CapsulePayload, + pub(crate) transcript: String, + pub(crate) sequence: u64, + pub(crate) revision: u64, + pub(crate) payload_revision: u64, +} + struct CapsuleWindowState { layout: Mutex>, + capsule_visible: AtomicBool, cursor_passthrough: AtomicBool, + hit_test_mode: AtomicU8, + transcript_visible: AtomicBool, style: AtomicU8, fallback_card_visible: AtomicBool, fallback_presentation_id: AtomicU64, deferred_payload: Mutex>, - last_payload: Mutex>, + snapshot: Mutex, layout_watch_active: AtomicBool, layout_watch_epoch: AtomicU64, } @@ -263,12 +707,15 @@ impl Default for CapsuleWindowState { fn default() -> Self { Self { layout: Mutex::new(None), + capsule_visible: AtomicBool::new(false), cursor_passthrough: AtomicBool::new(true), + hit_test_mode: AtomicU8::new(HIT_TEST_MODE_CAPSULE), + transcript_visible: AtomicBool::new(false), style: AtomicU8::new(0), fallback_card_visible: AtomicBool::new(false), fallback_presentation_id: AtomicU64::new(0), deferred_payload: Mutex::new(None), - last_payload: Mutex::new(None), + snapshot: Mutex::new(CapsuleSnapshotState::default()), layout_watch_active: AtomicBool::new(false), layout_watch_epoch: AtomicU64::new(0), } @@ -346,6 +793,46 @@ impl TauriCapsuleWindow { self.app.get_webview_window("capsule") } + fn rail_window(&self) -> Option { + self.app.get_webview_window("capsule-rail") + } + + fn hide_transcript_overlay(&self) { + if let Some(window) = self.rail_window() { + let _ = window.hide(); + } + } + + #[cfg(target_os = "windows")] + fn position_transcript_overlay( + &self, + capsule: &tauri::WebviewWindow, + style: CapsuleStyle, + translation_active: bool, + visible: bool, + ) -> tauri::Result<()> { + let Some(rail) = self.rail_window() else { + return Ok(()); + }; + let Some(position) = capsule_transcript_rail_position(style, translation_active, visible) + else { + rail.hide()?; + return Ok(()); + }; + let scale = capsule.scale_factor().unwrap_or(1.0); + let main_position = capsule.outer_position()?; + rail.set_ignore_cursor_events(true)?; + rail.set_size(tauri::LogicalSize::new(position.width, position.height))?; + rail.set_position(tauri::PhysicalPosition::new( + main_position.x, + main_position.y + (position.top_offset * scale).round() as i32, + ))?; + // The rail is a sibling overlay and must never steal focus from the + // application under the capsule, just like the main capsule HWND. + show_capsule_window_for_recording(&self.app, &rail, false); + Ok(()) + } + pub(crate) fn is_available_for(&self, state: CapsuleState) -> bool { let available = self.window().is_some(); if !available && !CAPSULE_WINDOW_MISSING_LOGGED.swap(true, Ordering::SeqCst) { @@ -379,6 +866,26 @@ impl TauriCapsuleWindow { #[cfg(not(mobile))] pub(crate) fn set_cursor_passthrough(&self, passthrough: bool) -> tauri::Result<()> { if let Some(window) = self.window() { + #[cfg(target_os = "windows")] + if passthrough { + clear_capsule_input_region_for_window(&window)?; + } else if self.state.hit_test_mode.load(Ordering::SeqCst) == HIT_TEST_MODE_CARD { + configure_card_hit_test(&window)?; + } else { + let translation_active = self + .state + .snapshot + .lock() + .payload + .as_ref() + .is_some_and(|payload| payload.translation); + configure_capsule_hit_test( + &window, + self.state.cached_style(), + self.state.transcript_visible.load(Ordering::SeqCst), + translation_active, + )?; + } window.set_ignore_cursor_events(passthrough)?; self.state .cursor_passthrough @@ -387,12 +894,69 @@ impl TauriCapsuleWindow { Ok(()) } + /// Rebuild the real client hit region after a card resize. The card owns the + /// shared HWND, so its whole current client area must be interactive; using + /// the previous capsule region would make Copy/Dismiss intermittently miss. + pub(crate) fn refresh_card_hit_test(&self) -> tauri::Result<()> { + if self.state.hit_test_mode.load(Ordering::SeqCst) != HIT_TEST_MODE_CARD { + return Ok(()); + } + #[cfg(target_os = "windows")] + if let Some(window) = self.window() { + configure_card_hit_test(&window)?; + } + Ok(()) + } + + #[cfg(not(mobile))] + pub(crate) fn set_card_hit_test_mode(&self) -> tauri::Result<()> { + self.state + .hit_test_mode + .store(HIT_TEST_MODE_CARD, Ordering::SeqCst); + self.hide_transcript_overlay(); + if let Some(window) = self.window() { + window.set_ignore_cursor_events(false)?; + self.state.cursor_passthrough.store(false, Ordering::SeqCst); + #[cfg(target_os = "windows")] + configure_card_hit_test(&window)?; + } + Ok(()) + } + + #[cfg(not(mobile))] + pub(crate) fn restore_capsule_hit_test_mode(&self) -> tauri::Result<()> { + self.state + .hit_test_mode + .store(HIT_TEST_MODE_CAPSULE, Ordering::SeqCst); + self.set_cursor_passthrough(true)?; + // Restoring a card must also restore the capsule geometry/rail state that + // was current before the card took ownership of the window. + self.invalidate_layout(); + if let Some(window) = self.window() { + let translation = self + .state + .snapshot + .lock() + .payload + .as_ref() + .is_some_and(|payload| payload.translation); + self.maybe_position_capsule_bottom_center( + &window, + translation, + self.state.cached_style(), + ); + } + Ok(()) + } + pub(crate) fn invalidate_layout(&self) { *self.state.layout.lock() = None; } pub(crate) fn hide(&self) -> tauri::Result<()> { + self.state.capsule_visible.store(false, Ordering::SeqCst); self.stop_layout_watch(); + self.hide_transcript_overlay(); if let Some(window) = self.window() { window.hide()?; } @@ -446,15 +1010,81 @@ impl TauriCapsuleWindow { pub(crate) fn position_capsule_bottom_center(&self, translation: bool) -> tauri::Result<()> { if let Some(window) = self.window() { - crate::position_capsule_bottom_center_with_style( + crate::position_capsule_bottom_center_with_style_and_transcript( &window, translation, self.state.cached_style(), + self.state.transcript_visible.load(Ordering::SeqCst), )?; } Ok(()) } + /// Restore the capsule window from the cached style and latest payload. + /// Card dismissal must not guess Siri geometry or discard the rail state. + pub(crate) fn restore_capsule_geometry(&self) -> tauri::Result<()> { + self.invalidate_layout(); + if let Some(window) = self.window() { + let translation = self + .state + .snapshot + .lock() + .payload + .as_ref() + .is_some_and(|payload| payload.translation); + self.maybe_position_capsule_bottom_center( + &window, + translation, + self.state.cached_style(), + ); + } + Ok(()) + } + + pub(crate) fn set_transcript_visible(&self, visible: bool) -> tauri::Result<()> { + if !self.state.capsule_visible.load(Ordering::SeqCst) { + self.state.transcript_visible.store(false, Ordering::SeqCst); + self.hide_transcript_overlay(); + return Ok(()); + } + self.state + .transcript_visible + .store(visible, Ordering::SeqCst); + if self.state.hit_test_mode.load(Ordering::SeqCst) == HIT_TEST_MODE_CARD { + // Card geometry owns this shared window until dismissal. The card webview may + // report transcriptVisible=false, but that must not restore capsule bounds. + self.hide_transcript_overlay(); + return Ok(()); + } + self.state.layout.lock().take(); + if let Some(window) = self.window() { + let translation = self + .state + .snapshot + .lock() + .payload + .as_ref() + .is_some_and(|payload| payload.translation); + self.maybe_position_capsule_bottom_center( + &window, + translation, + self.state.cached_style(), + ); + #[cfg(target_os = "windows")] + if self.state.hit_test_mode.load(Ordering::SeqCst) != HIT_TEST_MODE_CARD + && !self.state.cursor_passthrough.load(Ordering::SeqCst) + { + configure_capsule_hit_test( + &window, + self.state.cached_style(), + visible, + translation, + )?; + } + } + Ok(()) + } + fn layout_snapshot( &self, window: &tauri::WebviewWindow, @@ -466,6 +1096,7 @@ impl TauriCapsuleWindow { if let Some(mon) = crate::foreground_window_monitor() { return Some(CapsuleLayoutState { translation_active, + transcript_visible: self.state.transcript_visible.load(Ordering::SeqCst), style, monitor_x: mon.left, monitor_y: mon.top, @@ -484,6 +1115,7 @@ impl TauriCapsuleWindow { if let Some(mon) = crate::capsule_target_monitor(window) { return Some(CapsuleLayoutState { translation_active, + transcript_visible: self.state.transcript_visible.load(Ordering::SeqCst), style, monitor_x: mon.physical_x, monitor_y: mon.physical_y, @@ -500,6 +1132,7 @@ impl TauriCapsuleWindow { let monitor = window.current_monitor().ok().flatten()?; Some(CapsuleLayoutState { translation_active, + transcript_visible: self.state.transcript_visible.load(Ordering::SeqCst), style, monitor_x: monitor.position().x, monitor_y: monitor.position().y, @@ -522,17 +1155,49 @@ impl TauriCapsuleWindow { let Some(next) = self.layout_snapshot(window, translation_active, style) else { return; }; - if self.state.layout.lock().as_ref() == Some(&next) { - return; - } - if crate::position_capsule_bottom_center_with_style(window, translation_active, style) + let layout_changed = self.state.layout.lock().as_ref() != Some(&next); + if layout_changed + && crate::position_capsule_bottom_center_with_style_and_transcript( + window, + translation_active, + style, + self.state.transcript_visible.load(Ordering::SeqCst), + ) .is_ok() { *self.state.layout.lock() = Some(next); } + #[cfg(target_os = "windows")] + if self.state.capsule_visible.load(Ordering::SeqCst) { + if let Err(error) = self.position_transcript_overlay( + window, + style, + translation_active, + self.state.transcript_visible.load(Ordering::SeqCst), + ) { + log::warn!("[capsule] transcript overlay positioning failed: {error}"); + } + } else { + self.hide_transcript_overlay(); + } + #[cfg(target_os = "windows")] + if layout_changed + && self.state.hit_test_mode.load(Ordering::SeqCst) != HIT_TEST_MODE_CARD + && !self.state.cursor_passthrough.load(Ordering::SeqCst) + { + if let Err(error) = configure_capsule_hit_test( + window, + style, + self.state.transcript_visible.load(Ordering::SeqCst), + translation_active, + ) { + log::warn!("[capsule] configure client-band hit testing failed: {error}"); + } + } } pub(crate) fn show_for_recording(&self, reassert_spaces: bool) { + self.state.capsule_visible.store(true, Ordering::SeqCst); if let Some(window) = self.window() { show_capsule_window_for_recording(&self.app, &window, reassert_spaces); } @@ -574,7 +1239,7 @@ impl TauriCapsuleWindow { capsule.stop_layout_watch(); return; } - let payload = capsule.state.last_payload.lock().clone(); + let payload = capsule.state.snapshot.lock().payload.clone(); if let Some(payload) = payload { capsule.maybe_position_capsule_bottom_center( &window, @@ -615,7 +1280,7 @@ impl TauriCapsuleWindow { { return; } - let payload = self.state.last_payload.lock().clone(); + let payload = self.state.snapshot.lock().payload.clone(); if let Some(payload) = payload { let style = self.state.cached_style(); self.maybe_position_capsule_bottom_center(&window, payload.translation, style); @@ -631,7 +1296,6 @@ impl TauriCapsuleWindow { reassert_spaces: bool, ) { self.state.cache_style(style); - *self.state.last_payload.lock() = Some(payload.clone()); let Some(window) = self.window() else { return; }; @@ -646,6 +1310,17 @@ impl TauriCapsuleWindow { return; } + match action { + CapsuleWindowAction::ShowCapsule => { + self.state.capsule_visible.store(true, Ordering::SeqCst); + } + CapsuleWindowAction::HideCapsule => { + self.state.capsule_visible.store(false, Ordering::SeqCst); + self.hide_transcript_overlay(); + } + CapsuleWindowAction::PreserveFallbackCard => unreachable!(), + } + self.maybe_position_capsule_bottom_center(&window, payload.translation, style); self.update_cursor_for_payload(payload, style); @@ -675,7 +1350,7 @@ impl TauriCapsuleWindow { ); } hide_capsule_window_if_present(); - let _ = window.hide(); + let _ = self.hide(); } } } @@ -724,6 +1399,49 @@ impl TauriCoordinatorHost { self.capsule.cached_style() } + pub(crate) fn capsule_snapshot(&self) -> Option { + let snapshot = self.capsule.snapshot.lock().clone(); + if snapshot.pending_payload_revision.is_some() + || snapshot.payload_revision != snapshot.revision + { + // A rail ready replay must never observe a payload from one + // revision together with transcript/session data from another. + return None; + } + let mut payload = snapshot.payload?; + payload.session_id = snapshot.session_id.clone(); + Some(CapsuleSnapshot { + payload, + transcript: snapshot.text, + sequence: snapshot.sequence, + revision: snapshot.revision, + payload_revision: snapshot.payload_revision, + }) + } + + /// Commit the replay frame before the webview/main-thread work is queued. + /// The rail snapshot is an IPC replay source, so it must not wait for a + /// window callback to become internally consistent. + pub(crate) fn capsule_revision(&self) -> u64 { + self.capsule.snapshot.lock().revision + } + + pub(crate) fn commit_capsule_payload( + &self, + payload: &CapsulePayload, + captured_revision: u64, + ) -> bool { + self.capsule + .snapshot + .lock() + .commit_capsule_payload(payload, captured_revision) + } + + /// Update replay state without waiting for the native window callback. + pub(crate) fn record_backend_event(&self, event: &BackendEvent) { + self.capsule.snapshot.lock().record_backend_event(event); + } + pub(crate) fn cache_capsule_style(&self, style: CapsuleStyle) { self.capsule.cache_style(style); if let Some(capsule) = self.capsule_window() { @@ -871,6 +1589,7 @@ impl TauriCoordinatorHost { pub(crate) fn emit_capsule_state_to_capsule(&self, payload: &crate::types::CapsulePayload) { if let Some(app) = self.app() { let _ = app.emit_to("capsule", "capsule:state", payload); + let _ = app.emit_to("capsule-rail", "capsule:state", payload); } } @@ -963,6 +1682,7 @@ mod tests { fn capsule_layout_cache_notices_work_area_and_style_changes() { let initial = CapsuleLayoutState { translation_active: false, + transcript_visible: false, style: CapsuleStyle::Classic, monitor_x: 0, monitor_y: 0, @@ -1035,4 +1755,122 @@ mod tests { CapsuleWindowAction::HideCapsule ); } + + #[test] + fn capsule_transcript_rail_position_uses_body_content_for_all_styles() { + let siri = capsule_transcript_rail_position(CapsuleStyle::Siri, false, true).unwrap(); + assert_eq!( + (siri.width, siri.height, siri.top_offset, siri.gap), + (460.0, 40.0, -48.0, 8.0) + ); + assert_eq!( + capsule_transcript_rail_position(CapsuleStyle::Siri, true, true), + Some(siri) + ); + + let classic = capsule_transcript_rail_position(CapsuleStyle::Classic, false, true).unwrap(); + assert_eq!( + ( + classic.width, + classic.height, + classic.top_offset, + classic.gap + ), + (460.0, 52.0, 44.0, 8.0) + ); + assert_eq!( + capsule_transcript_rail_position(CapsuleStyle::Classic, true, true), + Some(CapsuleTranscriptRailPosition { + width: 460.0, + height: 82.0, + top_offset: 14.0, + gap: 8.0, + }) + ); + + let typeless = + capsule_transcript_rail_position(CapsuleStyle::Typeless, false, true).unwrap(); + assert_eq!(typeless.width, 206.0); + assert_eq!(typeless.gap, 0.0); + assert!((typeless.height - 52.0 * 0.447).abs() < f64::EPSILON); + assert!((typeless.top_offset - (57.0 - 64.0 * 0.447 - 52.0 * 0.447)).abs() < f64::EPSILON); + + let typeless_translation = + capsule_transcript_rail_position(CapsuleStyle::Typeless, true, true).unwrap(); + assert!( + (typeless_translation.top_offset - (65.0 - 20.0 * 0.447 - 64.0 * 0.447 - 52.0 * 0.447)) + .abs() + < 1e-12, + "unexpected typeless translation rail offset: {}", + typeless_translation.top_offset + ); + + for style in [ + CapsuleStyle::Siri, + CapsuleStyle::Classic, + CapsuleStyle::Typeless, + ] { + assert_eq!(capsule_transcript_rail_position(style, false, false), None); + } + let classic_badge = + capsule_transcript_rail_position(CapsuleStyle::Classic, true, false).unwrap(); + assert_eq!( + (classic_badge.height, classic_badge.top_offset), + (22.0, 74.0) + ); + let typeless_badge = + capsule_transcript_rail_position(CapsuleStyle::Typeless, true, false).unwrap(); + assert!((typeless_badge.height - 20.0 * 0.447).abs() < 1e-12); + assert!((typeless_translation.height - 72.0 * 0.447).abs() < 1e-12); + assert_eq!( + capsule_transcript_rail_position(CapsuleStyle::Siri, true, false), + None + ); + } + + #[cfg(target_os = "windows")] + #[test] + fn windows_hit_test_uses_x_and_y_and_translation_bounds() { + assert_eq!( + capsule_control_hit_rect(CapsuleStyle::Classic, true, false, 460, 172), + Some(CapsuleHitTestRect { + left: 132, + right: 328, + top: 104, + bottom: 156, + }) + ); + assert_eq!( + capsule_control_hit_rect(CapsuleStyle::Classic, false, false, 460, 172), + Some(CapsuleHitTestRect { + left: 132, + right: 328, + top: 104, + bottom: 156, + }) + ); + let typeless = capsule_control_hit_rect(CapsuleStyle::Typeless, true, true, 206, 65) + .expect("translation typeless hit rect"); + assert_eq!((typeless.left, typeless.right), (51, 155)); + assert_eq!((typeless.top, typeless.bottom), (36, 65)); + assert!(typeless.left > 0 && typeless.right < 206); + assert!(typeless.top > 0 && typeless.bottom <= 65); + let typeless_compact = + capsule_control_hit_rect(CapsuleStyle::Typeless, false, false, 206, 57) + .expect("compact typeless hit rect"); + assert_eq!((typeless_compact.top, typeless_compact.bottom), (28, 57)); + assert_eq!( + capsule_control_hit_rect(CapsuleStyle::Siri, false, false, 460, 180), + None + ); + assert_eq!( + capsule_card_hit_rect(320, 140), + CapsuleHitTestRect { + left: 0, + right: 320, + top: 0, + bottom: 140, + } + ); + } } diff --git a/openless-all/app/src-tauri/src/tauri_events.rs b/openless-all/app/src-tauri/src/tauri_events.rs index cfb928f9c..2307d67ec 100644 --- a/openless-all/app/src-tauri/src/tauri_events.rs +++ b/openless-all/app/src-tauri/src/tauri_events.rs @@ -34,11 +34,20 @@ pub fn start(app: AppHandle, backend: Arc) { match events.recv().await { Ok(event) => { let _ = app.emit("backend:event", &event); + if let Some(coordinator) = + app.try_state::>() + { + // Advance the transcript watermark before legacy + // capsule presentation. present_core_capsule then + // commits the matching payload under the same native + // snapshot lock before queueing the HWND work. + coordinator.tauri_host().record_backend_event(&event); + } forward_legacy_event( &app, &backend_for_events, event.session_id, - event.kind, + event.kind.clone(), &mut capsule_owners, ) .await; @@ -193,6 +202,7 @@ async fn forward_legacy_event( openless_core::DictationInsertStatus::NotRequested => "处理完成", }; coordinator.present_core_capsule(CapsulePayload { + session_id: Some(result.session_id.to_string()), state: CapsuleState::Done, level: 0.0, elapsed_ms: result.duration_ms, @@ -274,6 +284,7 @@ async fn forward_legacy_event( LessComputerVoicePhase::Idle => CapsuleState::Idle, }; coordinator.present_core_capsule(CapsulePayload { + session_id: Some(session_id.to_string()), state, level: *level, elapsed_ms: *elapsed_ms, @@ -342,6 +353,7 @@ async fn forward_legacy_event( if let Some(coordinator) = app.try_state::>() { if coordinator.selection_voice_accepts_level(&level.session_id) { coordinator.present_core_capsule(CapsulePayload { + session_id: Some(level.session_id.clone()), state: CapsuleState::Recording, level: level.level, elapsed_ms: level.elapsed_ms, @@ -462,6 +474,9 @@ fn emit_dictation_state( if let Some(capsule) = app.get_webview_window("capsule") { let _ = capsule.emit("capsule:state", &payload); } + if let Some(rail) = app.get_webview_window("capsule-rail") { + let _ = rail.emit("capsule:state", &payload); + } if let Some(main) = app.get_webview_window("main") { let _ = main.emit("capsule:state", &payload); } @@ -509,6 +524,7 @@ fn map_dictation_state( _ => snapshot.message, }; CapsulePayload { + session_id: snapshot.session_id.map(|session| session.to_string()), state, level: snapshot.level, elapsed_ms: snapshot.elapsed_ms, @@ -577,6 +593,7 @@ fn transcription_notice_payload( (0, false) }; Some(CapsulePayload { + session_id: Some(session_id.to_string()), state: CapsuleState::Transcribing, level: 0.0, elapsed_ms, @@ -682,6 +699,7 @@ fn qa_capsule_payload( finish_qa_transcription_notice(&mut owners.transcription_notice, session_id, kind); } Some(CapsulePayload { + session_id: session_id.map(|session| session.to_string()), state, level: if level.is_finite() { level.clamp(0.0, 1.0) diff --git a/openless-all/app/src-tauri/tauri.windows.conf.json b/openless-all/app/src-tauri/tauri.windows.conf.json index 5c87f3f01..5f99f62ad 100644 --- a/openless-all/app/src-tauri/tauri.windows.conf.json +++ b/openless-all/app/src-tauri/tauri.windows.conf.json @@ -38,6 +38,23 @@ "visible": false, "center": false, "acceptFirstMouse": true + }, + { + "label": "capsule-rail", + "url": "index.html?window=capsule-rail&os=win", + "title": "OpenLess Transcript Rail", + "width": 460, + "height": 52, + "decorations": false, + "transparent": true, + "shadow": false, + "alwaysOnTop": true, + "skipTaskbar": true, + "resizable": false, + "focus": false, + "visible": false, + "center": false, + "acceptFirstMouse": false } ] } diff --git a/openless-all/app/src/App.tsx b/openless-all/app/src/App.tsx index d89726f6d..d74166094 100644 --- a/openless-all/app/src/App.tsx +++ b/openless-all/app/src/App.tsx @@ -1,5 +1,6 @@ import { lazy, Suspense, useEffect, useState } from 'react'; import { Capsule } from './components/Capsule'; +import { CapsuleTranscriptOverlay } from './components/LiveTranscriptPill'; import { CoreStartupScreen } from './components/CoreStartupScreen'; import { GlobalDownloadProgress } from './components/GlobalDownloadProgress'; import { detectOS, type OS } from './components/WindowChrome'; @@ -52,6 +53,7 @@ const LessComputerGlow = LESS_COMPUTER_BUNDLED interface AppProps { isCapsule: boolean; + isCapsuleRail: boolean; isQa: boolean; isSelectionVoiceIntent: boolean; isLessComputer: boolean; @@ -96,12 +98,16 @@ export function App(props: AppProps) { function ReadyApp({ isCapsule, + isCapsuleRail, isQa, isSelectionVoiceIntent, isLessComputer, isLessComputerGlow, forcedOs, }: AppProps) { + if (isCapsuleRail) { + return ; + } if (isCapsule) { return ; } diff --git a/openless-all/app/src/components/Capsule.tsx b/openless-all/app/src/components/Capsule.tsx index 46888b02d..88200cb29 100644 --- a/openless-all/app/src/components/Capsule.tsx +++ b/openless-all/app/src/components/Capsule.tsx @@ -5,14 +5,17 @@ import { detectOS, type OS } from './WindowChrome'; import { warmUpSiriShaders } from './SiriGL'; import { VoiceOrbStage } from './VoiceOrbStage'; import { TypelessCapsule } from './TypelessCapsule'; -import { LiveTranscriptPill } from './LiveTranscriptPill'; +import { CapsuleTranscriptRail, CapsuleTranslationBadge } from './LiveTranscriptPill'; import { capsuleTranscriptFontSize, visibleCapsuleTranscript } from '../lib/capsuleTranscript'; import { getSettings } from '../lib/ipc/settings'; -import { cancelDictation, stopDictation } from '../lib/ipc/dictation'; +import { cancelDictation, setCapsuleTranscriptVisible, stopDictation } from '../lib/ipc/dictation'; import { getCapsuleHostMetrics, getCapsuleMessageLayout, getCapsulePillMetrics, + getClassicCapsuleGeometry, + getClassicProcessingLabel, + getCapsuleTranscriptRailHeight, parseCapsuleStyle, } from '../lib/capsuleLayout'; import { isTauri } from '../lib/ipc'; @@ -28,6 +31,9 @@ import { VocabSuggestionCard } from './VocabSuggestionCard'; import { InsertFallbackCard } from './InsertFallbackCard'; import { applyTranscriptEvent, + beginTranscriptGeneration, + clearTranscriptText, + createTranscriptViewState, type BackendEvent, type TranscriptViewState, } from '../lib/backendEvent'; @@ -203,10 +209,10 @@ function classicPillMetrics(os: OS): ClassicPillMetrics { return os === 'win' ? CLASSIC_PILL_METRICS_WIN : CLASSIC_PILL_METRICS; } -function AudioBars({ level }: { level: number }) { +function AudioBars({ level, compact = false }: { level: number; compact?: boolean }) { const envelope = [0.55, 0.85, 1.0, 0.85, 0.55]; const base = 2; - const max = 24; + const max = compact ? 16 : 24; const voice = Math.min(1, Math.max(0, level)); const silenceGate = 0.012; const responseCeiling = 0.34; @@ -218,12 +224,13 @@ function AudioBars({ level }: { level: number }) { const visualVoice = Math.pow(easedVoice, 0.42); return (
@@ -232,7 +239,7 @@ function AudioBars({ level }: { level: number }) { key={i} style={{ display: 'inline-block', - width: 3, + width: compact ? 2 : 3, height: base + (max - base) * visualVoice * env, borderRadius: 999, background: 'var(--ol-blue)', @@ -336,7 +343,7 @@ interface ClassicPillProps { level: number; insertedChars: number; message?: string; - operating?: boolean; + operating: boolean; onCancel: () => void; onConfirm: () => void; } @@ -357,15 +364,14 @@ function ClassicPill({ const cancelEnabled = state === 'recording' || state === 'transcribing' || state === 'polishing'; const confirmEnabled = state === 'recording'; - // "thinking" shine speed: fast (0.9s/cycle) for the first 2s of transcribing/polishing - // (signals "stream just started"), then slow (2.4s) as the steady state. Also resets - // to fast on idle/done/other states so the next entry bursts from the start. + // Keep the original Classic processing treatment: the thinking/using label + // has its own entry and shine animation, while recording keeps the live bars. const [shineFast, setShineFast] = useState(true); useEffect(() => { if (state === 'transcribing' || state === 'polishing') { setShineFast(true); - const t = setTimeout(() => setShineFast(false), 2000); - return () => clearTimeout(t); + const timer = setTimeout(() => setShineFast(false), 2000); + return () => clearTimeout(timer); } setShineFast(true); return undefined; @@ -383,28 +389,22 @@ function ClassicPill({ style={{ display: 'inline-flex', alignItems: 'center', - // 4px side padding + outer gap already puts the "thinking" ↔ ✗/✓ visual - // spacing at ~4-5px. - padding: '0 4px', + gap: 5, + padding: '0 2px', width: '100%', maxWidth: metrics.textWidth, minWidth: 0, justifyContent: 'center', - // State-entry animation: an extra fade cue when going recording → polishing, - // easier to perceive than swapping the center content alone. animation: 'cap-state-enter 220ms var(--ol-motion-soft) both', }} > + - {t(operating ? 'capsule.using' : 'capsule.thinking')} + {t(getClassicProcessingLabel(operating))}
); @@ -514,7 +511,9 @@ interface ClassicCapsuleProps { message?: string; transcript?: string; transcriptFontSize?: number; - operating?: boolean; + transcriptVisible?: boolean; + transcriptInSameWindow?: boolean; + operating: boolean; translation: boolean; } @@ -531,11 +530,11 @@ function ClassicCapsule({ message, transcript, transcriptFontSize = 14, + transcriptVisible = false, + transcriptInSameWindow = true, operating, translation, }: ClassicCapsuleProps) { - const { t } = useTranslation(); - const metrics = classicPillMetrics(os); const hostMetrics = getCapsuleHostMetrics(os, false, 'classic'); const onCancel = useCallback(() => { void cancelDictation(); @@ -544,8 +543,12 @@ function ClassicCapsule({ void stopDictation(); }, []); const liveText = transcript?.trim() ?? ''; - const recording = state === 'recording'; - const processing = state === 'transcribing' || state === 'polishing'; + const classicGeometry = getClassicCapsuleGeometry( + os, + transcriptVisible || Boolean(liveText), + translation, + transcriptInSameWindow, + ); return ( <> @@ -554,63 +557,30 @@ function ClassicCapsule({ (translateX(-50%)) with no animation; the inner layer only does vertical shift + fade. This avoids conflicting with translateX(-50%) and avoids keyframe vs inline transform overrides causing visual jumps. */} -
+ {transcriptInSameWindow && (
- - {t('capsule.translating')} +
-
- {liveText ? ( - - ) : ( + )} +
+ {liveText && ( + + )} - )} +
); } @@ -723,11 +693,7 @@ export function Capsule({ os: forcedOs }: CapsuleProps = {}) { : Number(new URLSearchParams(window.location.search).get('fontSize') || 14), ), ); - const transcriptViewRef = useRef({ - sessionId: null, - sequence: 0, - text: '', - }); + const transcriptViewRef = useRef(createTranscriptViewState()); const capsuleStateRef = useRef(preview.state); const [translation, setTranslation] = useState(preview.translation); const [selectionPolish, setSelectionPolish] = useState(preview.selectionPolish); @@ -737,6 +703,9 @@ export function Capsule({ os: forcedOs }: CapsuleProps = {}) { const stylePreferenceReadyRef = useRef(false); const isClassic = capsuleStyle === 'classic'; const isTypeless = capsuleStyle === 'typeless'; + // Windows keeps the transcript in the dedicated capsule-rail HWND. The browser + // preview and other platforms continue rendering the sibling rail in this tree. + const splitTranscriptRail = isTauri && os === 'win'; // Warming: the mic hasn't produced its first PCM frame yet. When true, the orb runs // its standby breathing form (see SiriGL warming). const [warming, setWarming] = useState(preview.warming); @@ -814,7 +783,19 @@ export function Capsule({ os: forcedOs }: CapsuleProps = {}) { // the previous capsule was still transcribing/polishing/done from // dictation (selection-voice Start used to keep the stale stream). if (p.state === 'recording' && previous !== 'recording') { - transcriptViewRef.current = { sessionId: null, sequence: 0, text: '' }; + transcriptViewRef.current = beginTranscriptGeneration(transcriptViewRef.current); + setLocalAsrText(''); + } + if ( + p.state === 'idle' || + p.state === 'done' || + p.state === 'cancelled' || + p.state === 'error' + ) { + // Terminal payloads must release the native transcript rail before the exit/card + // branch can leave the capsule webview. Keeping the old text here would make the + // enlarged native window stay interactive after the session has ended. + transcriptViewRef.current = clearTranscriptText(transcriptViewRef.current); setLocalAsrText(''); } setTranslation(p.translation === true); @@ -979,6 +960,43 @@ export function Capsule({ os: forcedOs }: CapsuleProps = {}) { }; }, [preview.insertDemo]); + // Derive these values before any conditional return. Hooks must be called for the same + // component shape while the capsule switches between the live pill, fallback card, + // suggestion card, and the hidden idle branch. + const renderedState: CapsuleState = state === 'idle' ? lastVisibleState : state; + const renderedSelectionPolish = state === 'idle' ? lastVisibleSelectionPolish : selectionPolish; + const renderedMessage = + state === 'idle' + ? lastVisibleMessage + : state === 'transcribing' && localAsrText + ? localAsrText + : message; + const liveTranscript = visibleCapsuleTranscript( + localAsrText, + transcriptEnabled, + renderedState, + renderedSelectionPolish, + ); + const showLiveTranscript = state !== 'idle' && liveTranscript.length > 0; + const transcriptVisible = showLiveTranscript && !insertFallback && suggestions.length === 0; + + useEffect(() => { + if (!isTauri) return; + void setCapsuleTranscriptVisible(transcriptVisible).catch((error) => + console.warn('[capsule] transcript visibility sync failed', error), + ); + }, [transcriptVisible]); + + // Always release the native hit-test region when the capsule webview is torn down. + useEffect(() => { + if (!isTauri) return; + return () => { + void setCapsuleTranscriptVisible(false).catch((error) => + console.warn('[capsule] transcript visibility cleanup failed', error), + ); + }; + }, []); + // Fallback card comes first: it pops the moment the session ends, while the capsule // is still rendering the Done/Error terminal state — and the session's outcome was // exactly "didn't insert", so letting the terminal state cover it would report a @@ -1000,24 +1018,6 @@ export function Capsule({ os: forcedOs }: CapsuleProps = {}) { return
; } - // During exit, render the last frame from lastVisibleState so idle isn't treated as - // a no-waveform state. - const renderedState: CapsuleState = state === 'idle' ? lastVisibleState : state; - const renderedSelectionPolish = state === 'idle' ? lastVisibleSelectionPolish : selectionPolish; - const renderedMessage = - state === 'idle' - ? lastVisibleMessage - : state === 'transcribing' && localAsrText - ? localAsrText - : message; - const liveTranscript = visibleCapsuleTranscript( - localAsrText, - transcriptEnabled, - renderedState, - renderedSelectionPolish, - ); - const showLiveTranscript = liveTranscript.length > 0; - return (
@@ -1062,10 +1065,11 @@ export function Capsule({ os: forcedOs }: CapsuleProps = {}) { level={leaving ? 0 : level} insertedChars={insertedCharsRef.current} message={renderedMessage} - transcript={liveTranscript} + transcript={splitTranscriptRail ? '' : liveTranscript} transcriptFontSize={transcriptFontSize} operating={operatingRef.current} translation={translation} + translationInSameWindow={!splitTranscriptRail} warming={!leaving && warming} /> ) : ( @@ -1118,41 +1122,33 @@ export function Capsule({ os: forcedOs }: CapsuleProps = {}) { {t('capsule.translating')}
-
- -
- {showLiveTranscript && ( -
- + {showLiveTranscript && !splitTranscriptRail && ( + + )} +
+
- )} +
))} {renderedSelectionPolish && ( diff --git a/openless-all/app/src/components/CapsuleStyles.css b/openless-all/app/src/components/CapsuleStyles.css index 4a67c70e0..f3f3988bf 100644 --- a/openless-all/app/src/components/CapsuleStyles.css +++ b/openless-all/app/src/components/CapsuleStyles.css @@ -18,6 +18,135 @@ background: #ffffff; } +.ol-capsule-transcript-rail { + display: flex; + flex: 0 0 auto; + align-items: center; + justify-content: center; + width: 100%; + pointer-events: none; + position: relative; + z-index: 2; +} + +/* Windows owns this rail in a separate transparent WebView window. Its HWND is + always cursor-through, so the full transcript/gutter can never intercept a + click destined for another process. */ +.ol-capsule-transcript-overlay { + display: flex; + flex-direction: column; + align-items: center; + width: 460px; + overflow: visible; + pointer-events: none; +} + +.ol-capsule-transcript-overlay[data-style='classic'] { + gap: 8px; +} + +.ol-classic-translation { + display: inline-flex; + flex: 0 0 22px; + height: 22px; + box-sizing: border-box; + align-items: center; + gap: 5px; + padding: 0 10px; + border-radius: 999px; + font-size: 10.5px; + font-weight: 600; + color: var(--ol-blue); + background: var(--ol-capsule-badge-bg); + border: 0.5px solid var(--ol-capsule-badge-border); + box-shadow: + 0 4px 12px -4px rgb(37 99 235 / 25%), + 0 0 0 0.5px rgb(0 0 0 / 4%); + letter-spacing: 0.02em; + white-space: nowrap; + transform-origin: center bottom; + transition: + opacity 0.24s ease-out, + transform 0.34s cubic-bezier(0.2, 0.9, 0.3, 1.1); +} + +.ol-classic-translation[data-visible='false'] { + opacity: 0; + transform: translateY(40px) scale(0.88); +} + +.ol-classic-translation > span { + width: 5px; + height: 5px; + border-radius: 999px; + background: var(--ol-blue); +} + +.ol-capsule-transcript-overlay .ol-capsule-transcript-rail { + height: 52px; +} + +.ol-capsule-transcript-overlay[data-style='siri'] { + height: 40px; +} + +.ol-capsule-transcript-overlay[data-style='siri'] .ol-capsule-transcript-rail { + height: 40px; +} + +.ol-capsule-transcript-overlay--typeless { + width: 460px; + zoom: 0.447; +} + +.ol-classic-capsule-stack, +.ol-siri-capsule-stack, +.ol-typeless-capsule-stack { + display: flex; + flex: 0 0 auto; + flex-direction: column; + align-items: center; + justify-content: flex-end; + gap: 8px; +} + +.ol-classic-capsule-stack, +.ol-siri-capsule-stack, +.ol-typeless-capsule-stack { + width: 460px; +} + +.ol-siri-capsule-stack .ol-capsule-transcript-rail { + height: 40px; +} + +.ol-classic-capsule-stack .ol-capsule-transcript-rail, +.ol-typeless-capsule-stack .ol-capsule-transcript-rail { + height: 52px; +} + +.ol-typeless-capsule-stack { + gap: 0; +} + +.ol-typeless-translation-row { + display: flex; + flex: 0 0 20px; + align-items: center; + justify-content: center; + width: 100%; + height: 20px; + pointer-events: none; +} + +.ol-typeless-capsule-stage { + position: relative; + display: flex; + flex: 0 0 auto; + align-items: center; + justify-content: center; +} + .ol-typeless-capsule { position: relative; width: 176px; @@ -34,7 +163,8 @@ transition: width 220ms cubic-bezier(0.22, 1, 0.36, 1); } -.ol-typeless-capsule[data-recording='true'] { +.ol-typeless-capsule[data-recording='true'], +.ol-typeless-capsule[data-processing='true'] { width: 232px; } .ol-typeless-capsule[data-error='true'] { @@ -62,7 +192,13 @@ transform: translateX(-50%); } .ol-typeless-capsule[data-recording='true'] .ol-typeless-recording, -.ol-typeless-capsule:not([data-recording='true']) .ol-typeless-status { +.ol-typeless-capsule[data-processing='true'] .ol-typeless-recording { + opacity: 1; + pointer-events: auto; +} + +.ol-typeless-capsule:not([data-recording='true']):not([data-processing='true']) + .ol-typeless-status { opacity: 1; pointer-events: auto; } @@ -156,11 +292,12 @@ cursor: pointer; } .ol-typeless-translation { - position: absolute; - bottom: calc(100% + 7px); - left: 50%; - transform: translateX(-50%); - padding: 3px 10px; + position: static; + display: flex; + align-items: center; + height: 20px; + padding: 0 10px; + box-sizing: border-box; white-space: nowrap; font-size: 10px; font-weight: 500; diff --git a/openless-all/app/src/components/LiveTranscriptPill.tsx b/openless-all/app/src/components/LiveTranscriptPill.tsx index 913dd03ae..482a6752f 100644 --- a/openless-all/app/src/components/LiveTranscriptPill.tsx +++ b/openless-all/app/src/components/LiveTranscriptPill.tsx @@ -1,22 +1,57 @@ import { animate, motion, useMotionValue, useReducedMotion, useTransform } from 'framer-motion'; -import { useEffect, useMemo, useState } from 'react'; +import { useEffect, useMemo, useRef, useState } from 'react'; +import { useTranslation } from 'react-i18next'; import './CapsuleStyles.css'; import { INSERT_TEXT_MOTION, appendedBirthAdvance, clampInsertContentWidth, - diffInsertUnits, + diffInsertWindowUnits, firstBornIndex, planCapsuleInsertMotion, + INSERT_TEXT_RENDER_WINDOW, planCharDelays, prefixWidths, rightAnchoredPositions, + selectInsertRenderWindow, type InsertUnit, } from '../lib/insertTextAnimation'; import { Icon } from './Icon'; +import { + applyTranscriptEvent, + applyTranscriptSessionId, + applyTranscriptSnapshot, + beginTranscriptGeneration, + clearTranscriptText, + createTranscriptViewState, + shouldRetryTranscriptSnapshot, + type BackendEvent, + type TranscriptViewState, +} from '../lib/backendEvent'; +import { capsuleTranscriptFontSize, visibleCapsuleTranscript } from '../lib/capsuleTranscript'; +import { getCapsuleTranscriptRailHeight } from '../lib/capsuleLayout'; +import { getSettings } from '../lib/ipc/settings'; +import { getCapsuleSnapshot, isTauri } from '../lib/ipc'; +import type { + CapsulePayload, + CapsuleSnapshot, + CapsuleState, + CapsuleStyle, + UserPreferences, +} from '../lib/types'; export type LiveTranscriptTone = 'frost' | 'dark'; +export function CapsuleTranslationBadge({ visible = true }: { visible?: boolean }) { + const { t } = useTranslation(); + return ( +
+ + {t('capsule.translating')} +
+ ); +} + export interface LiveTranscriptPillProps { text: string; tone?: LiveTranscriptTone; @@ -48,19 +83,201 @@ function getSansFont(): string { return token || 'system-ui, sans-serif'; } -function measureGlyphWidths(glyphs: string[], fontSize: number): number[] { - if (glyphs.length === 0) return []; +function measureGlyphWidth(glyph: string, fontSize: number): number { if (typeof document === 'undefined') { - return glyphs.map((glyph) => Math.max(fontSize * 0.92, glyph.length * fontSize)); + return Math.max(fontSize * 0.92, glyph.length * fontSize); } if (!measureCanvas) measureCanvas = document.createElement('canvas'); const ctx = measureCanvas.getContext('2d'); - if (!ctx) return glyphs.map(() => fontSize); + if (!ctx) return fontSize; ctx.font = `${FONT_WEIGHT} ${fontSize}px ${getSansFont()}`; - return glyphs.map((glyph) => { - const raw = ctx.measureText(glyph === ' ' ? '\u00a0' : glyph).width; - return Math.max(1, raw) + INSERT_TEXT_MOTION.glyphGapPx; - }); + const raw = ctx.measureText(glyph === ' ' ? '\u00a0' : glyph).width; + return Math.max(1, raw) + INSERT_TEXT_MOTION.glyphGapPx; +} + +export interface CapsuleTranscriptRailProps { + text: string; + tone?: LiveTranscriptTone; + stageWidth: number; + maxWidth: number; + height?: number; + fontSize?: number; +} + +/** Transcript is deliberately a sibling rail: the capsule body remains responsible for audio feedback and controls. */ +export function CapsuleTranscriptRail({ + text, + tone = 'frost', + stageWidth, + maxWidth, + height = DEFAULT_PILL_HEIGHT, + fontSize = FONT_SIZE, +}: CapsuleTranscriptRailProps) { + const liveText = text.trim(); + if (!liveText) return null; + return ( +
+ +
+ ); +} + +/** + * Windows renders the transcript in a separate, permanently click-through HWND. + * The capsule HWND can therefore keep a small, reliable native hit region for + * its controls while this rail never participates in cross-process hit testing. + */ +export function CapsuleTranscriptOverlay() { + const { t } = useTranslation(); + const [state, setState] = useState('idle'); + const [style, setStyle] = useState('siri'); + const [text, setText] = useState(''); + const [enabled, setEnabled] = useState(true); + const [fontSize, setFontSize] = useState(14); + const [selectionPolish, setSelectionPolish] = useState(false); + const [translation, setTranslation] = useState(false); + const transcriptRef = useRef(createTranscriptViewState()); + const stateRef = useRef('idle'); + const snapshotRetryDelays = [0, 16, 32, 64, 128, 256] as const; + + useEffect(() => { + if (!isTauri) return; + let cancelled = false; + let unlisten: (() => void) | undefined; + (async () => { + const { listen } = await import('@tauri-apps/api/event'); + const stateHandle = await listen('capsule:state', (event) => { + const payload = event.payload; + const previousState = stateRef.current; + const previousTranscript = transcriptRef.current; + const sessionTranscript = applyTranscriptSessionId(previousTranscript, payload.sessionId); + const sessionChanged = sessionTranscript !== previousTranscript; + transcriptRef.current = sessionTranscript; + stateRef.current = payload.state; + setState(payload.state); + if (payload.capsuleStyle) setStyle(payload.capsuleStyle); + setSelectionPolish(payload.selectionPolish === true); + setTranslation(payload.translation === true); + if (payload.state === 'recording' && previousState !== 'recording' && !sessionChanged) { + transcriptRef.current = beginTranscriptGeneration(transcriptRef.current); + setText(transcriptRef.current.text); + } else if (sessionChanged) { + setText(sessionTranscript.text); + } + if (['idle', 'done', 'cancelled', 'error'].includes(payload.state)) { + transcriptRef.current = clearTranscriptText(transcriptRef.current); + setText(''); + } + }); + const transcriptHandle = await listen('backend:event', (event) => { + const next = applyTranscriptEvent(transcriptRef.current, event.payload); + transcriptRef.current = next; + setText(next.text); + }); + const preferenceHandle = await listen('prefs:changed', (event) => { + setEnabled(event.payload.capsuleTranscriptEnabled ?? true); + setFontSize(capsuleTranscriptFontSize(event.payload.capsuleTranscriptFontSize)); + if (event.payload.capsuleStyle) setStyle(event.payload.capsuleStyle); + }); + const applySnapshot = ( + snapshot: CapsuleSnapshot, + fallbackStyle: CapsuleStyle, + ): 'retry' | 'done' => { + const previousTranscript = transcriptRef.current; + if (shouldRetryTranscriptSnapshot(previousTranscript, snapshot)) return 'retry'; + const nextTranscript = applyTranscriptSnapshot(previousTranscript, snapshot); + if (nextTranscript === previousTranscript) return 'done'; + const sessionChanged = + snapshot.sessionId !== null && snapshot.sessionId !== previousTranscript.sessionId; + transcriptRef.current = nextTranscript; + setText(nextTranscript.text); + + const previousState = stateRef.current; + stateRef.current = snapshot.state; + setState(snapshot.state); + setStyle(snapshot.capsuleStyle ?? fallbackStyle); + setSelectionPolish(snapshot.selectionPolish === true); + setTranslation(snapshot.translation === true); + if (snapshot.state === 'recording' && previousState !== 'recording' && !sessionChanged) { + transcriptRef.current = beginTranscriptGeneration(transcriptRef.current); + setText(transcriptRef.current.text); + } + if (['idle', 'done', 'cancelled', 'error'].includes(snapshot.state)) { + transcriptRef.current = clearTranscriptText(transcriptRef.current); + setText(''); + } + return 'done'; + }; + const replaySnapshot = async (fallbackStyle: CapsuleStyle) => { + for (const delayMs of snapshotRetryDelays) { + if (cancelled) return; + if (delayMs > 0) { + await new Promise((resolve) => window.setTimeout(resolve, delayMs)); + } + if (cancelled) return; + const snapshot = await getCapsuleSnapshot(); + if (cancelled) return; + if (!snapshot) continue; + if (applySnapshot(snapshot, fallbackStyle) === 'done') return; + } + }; + if (cancelled) { + stateHandle(); + transcriptHandle(); + preferenceHandle(); + } else { + unlisten = () => { + stateHandle(); + transcriptHandle(); + preferenceHandle(); + }; + const preferences = await getSettings(); + if (!cancelled) { + setEnabled(preferences.capsuleTranscriptEnabled ?? true); + setFontSize(capsuleTranscriptFontSize(preferences.capsuleTranscriptFontSize)); + setStyle(preferences.capsuleStyle); + await replaySnapshot(preferences.capsuleStyle); + } + } + })().catch((error) => console.warn('[capsule-rail] subscription failed', error)); + return () => { + cancelled = true; + unlisten?.(); + }; + }, []); + + const liveText = visibleCapsuleTranscript(text, enabled, state, selectionPolish); + const typeless = style === 'typeless'; + const railHeight = getCapsuleTranscriptRailHeight(style); + return ( +
+ + {translation && !selectionPolish && style === 'classic' && } + {translation && !selectionPolish && typeless && ( +
+ {t('capsule.translating')} +
+ )} +
+ ); } export function LiveTranscriptPill({ @@ -83,21 +300,26 @@ export function LiveTranscriptPill({ const hasControls = Boolean(onCancel || onConfirm); const padX = INSERT_TEXT_MOTION.padX + (hasControls ? controlSize + 6 : 0); const pillMinWidth = hasControls ? Math.max(minWidth, controlSize * 2 + 72) : minWidth; - const [units, setUnits] = useState(() => diffInsertUnits([], text)); + const [units, setUnits] = useState(() => diffInsertWindowUnits([], text)); const [seenText, setSeenText] = useState(text); if (text !== seenText) { setSeenText(text); - setUnits((prev) => diffInsertUnits(prev, text)); + setUnits((prev) => diffInsertWindowUnits(prev, text)); } - const widths = useMemo( + const renderWindow = useMemo( () => - measureGlyphWidths( - units.map((unit) => unit.text), - fontSize, + selectInsertRenderWindow( + units, + (glyph) => measureGlyphWidth(glyph, fontSize), + maxWidth, + padX, + INSERT_TEXT_RENDER_WINDOW, ), - [units, fontSize], + [units, fontSize, maxWidth, padX], ); + const renderUnits = renderWindow.units; + const widths = renderWindow.widths; const contentWidth = useMemo( () => clampInsertContentWidth( @@ -110,11 +332,11 @@ export function LiveTranscriptPill({ const positions = useMemo(() => rightAnchoredPositions(widths), [widths]); // Start appended glyphs beyond the old tail, so they never appear on top of // retained text while the displacement spring is still gathering speed. - const birthAdvance = appendedBirthAdvance(units, widths); - const originIndex = firstBornIndex(units); + const birthAdvance = appendedBirthAdvance(renderUnits, widths); + const originIndex = firstBornIndex(renderUnits); const starts = useMemo(() => prefixWidths(widths), [widths]); const originX = (starts[originIndex] ?? 0) + (widths[originIndex] ?? 0) / 2; - const insertedCount = units.filter((unit) => unit.born).length; + const insertedCount = renderUnits.filter((unit) => unit.born).length; const delays = useMemo( () => planCharDelays(widths, originIndex, insertedCount), [widths, originIndex, insertedCount], @@ -233,7 +455,7 @@ export function LiveTranscriptPill({ height, }} > - {units.map((unit, index) => ( + {renderUnits.map((unit, index) => ( void cancelDictation(), []); const confirm = useCallback(() => void stopDictation(), []); - if (liveText) { - return ( -
- {translation && {t('capsule.translating')}} - -
- ); - } - return ( -
- {translation && {t('capsule.translating')}} -
-
- - - -
-
- - {label} - - {processing && !preview && ( - - )} +
+ + + +
+
+ + {label} + +
+
diff --git a/openless-all/app/src/i18n/de.ts b/openless-all/app/src/i18n/de.ts index ed9c03e2e..7b5b4f6c3 100644 --- a/openless-all/app/src/i18n/de.ts +++ b/openless-all/app/src/i18n/de.ts @@ -1794,6 +1794,9 @@ export const de: typeof zhCN = { thinkingBudget: 'Festes Budget', maxTokensLabel: 'Maximale Ausgabetokens', thinkingBudgetLabel: 'Tokenbudget für das Denken', + serviceTierLabel: 'Ark-Inferenz mit niedriger Latenz (fast)', + serviceTierHint: + 'Nur unterstützte Ark-Modelle können diese Option verwenden; andere Modelle geben möglicherweise 400 zurück.', responsesThinkingHint: 'Bei einigen Modellen lässt sich das Denken nur reduzieren, nicht abschalten. Anfragen mit Denken senden keinen Temperaturparameter.', messagesThinkingHint: @@ -1803,6 +1806,8 @@ export const de: typeof zhCN = { llmTokenLimitInvalid: 'Tokenlimits müssen positive ganze Zahlen sein.', llmThinkingBudgetInvalid: 'Das Denkbudget muss mindestens 1024 betragen und im festen Modus unter der Ausgabegrenze liegen.', + llmServiceTierInvalid: + 'Ungültige Option für Inferenz mit niedriger Latenz. Wähle sie erneut.', llmResponseIncomplete: 'Die Antwort ist unvollständig oder hat die Ausgabegrenze erreicht. Bereits ausgegebener Text bleibt erhalten.', llmProtocolHeaderConflict: diff --git a/openless-all/app/src/i18n/en.ts b/openless-all/app/src/i18n/en.ts index 970ab81bc..5f2f32308 100644 --- a/openless-all/app/src/i18n/en.ts +++ b/openless-all/app/src/i18n/en.ts @@ -1762,6 +1762,9 @@ export const en: typeof zhCN = { thinkingBudget: 'Fixed budget', maxTokensLabel: 'Maximum output tokens', thinkingBudgetLabel: 'Thinking budget tokens', + serviceTierLabel: 'Ark low-latency inference (fast)', + serviceTierHint: + 'Only supported Ark models can use this option; unsupported models may return 400.', responsesThinkingHint: 'Some models can only reduce thinking, not turn it off. Reasoning requests omit temperature.', messagesThinkingHint: @@ -1771,6 +1774,7 @@ export const en: typeof zhCN = { llmTokenLimitInvalid: 'Token limits must be positive integers.', llmThinkingBudgetInvalid: 'Thinking budget must be at least 1024 and below the output limit in fixed-budget mode.', + llmServiceTierInvalid: 'Invalid low-latency inference option. Select it again.', llmResponseIncomplete: 'The response was incomplete or reached its output limit. Already emitted text is retained.', llmProtocolHeaderConflict: diff --git a/openless-all/app/src/i18n/es.ts b/openless-all/app/src/i18n/es.ts index c3ecec806..a73064033 100644 --- a/openless-all/app/src/i18n/es.ts +++ b/openless-all/app/src/i18n/es.ts @@ -1783,6 +1783,9 @@ export const es: typeof zhCN = { thinkingBudget: 'Presupuesto fijo', maxTokensLabel: 'Máximo de tokens de salida', thinkingBudgetLabel: 'Presupuesto de tokens de razonamiento', + serviceTierLabel: 'Inferencia de baja latencia de Ark (fast)', + serviceTierHint: + 'Solo los modelos Ark compatibles pueden usar esta opción; otros pueden devolver 400.', responsesThinkingHint: 'Algunos modelos solo permiten reducir el razonamiento, no desactivarlo. Las solicitudes de razonamiento omiten la temperatura.', messagesThinkingHint: @@ -1792,6 +1795,8 @@ export const es: typeof zhCN = { llmTokenLimitInvalid: 'Los límites de tokens deben ser números enteros positivos.', llmThinkingBudgetInvalid: 'El presupuesto de razonamiento debe ser al menos 1024 y, en modo fijo, inferior al límite de salida.', + llmServiceTierInvalid: + 'Opción de inferencia de baja latencia no válida. Selecciónala de nuevo.', llmResponseIncomplete: 'La respuesta no se completó o alcanzó el límite de salida. Se conserva el texto ya mostrado.', llmProtocolHeaderConflict: diff --git a/openless-all/app/src/i18n/fr.ts b/openless-all/app/src/i18n/fr.ts index c9b2bc894..aabf00a1a 100644 --- a/openless-all/app/src/i18n/fr.ts +++ b/openless-all/app/src/i18n/fr.ts @@ -1810,6 +1810,9 @@ export const fr: typeof zhCN = { thinkingBudget: 'Budget fixe', maxTokensLabel: 'Nombre maximal de jetons en sortie', thinkingBudgetLabel: 'Budget de jetons de raisonnement', + serviceTierLabel: 'Inférence Ark à faible latence (fast)', + serviceTierHint: + 'Seuls les modèles Ark compatibles peuvent utiliser cette option ; les autres peuvent renvoyer 400.', responsesThinkingHint: 'Certains modèles permettent seulement de réduire le raisonnement, pas de le désactiver. Les requêtes de raisonnement omettent la température.', messagesThinkingHint: @@ -1821,6 +1824,8 @@ export const fr: typeof zhCN = { llmTokenLimitInvalid: 'Les limites de jetons doivent être des entiers positifs.', llmThinkingBudgetInvalid: 'Le budget de raisonnement doit être au moins de 1024 et, en mode fixe, inférieur à la limite de sortie.', + llmServiceTierInvalid: + 'Option d’inférence à faible latence non valide. Sélectionnez-la à nouveau.', llmResponseIncomplete: 'La réponse est incomplète ou a atteint la limite de sortie. Le texte déjà affiché est conservé.', llmProtocolHeaderConflict: diff --git a/openless-all/app/src/i18n/ja.ts b/openless-all/app/src/i18n/ja.ts index 64e9e0363..21a2a58a7 100644 --- a/openless-all/app/src/i18n/ja.ts +++ b/openless-all/app/src/i18n/ja.ts @@ -1749,6 +1749,9 @@ export const ja: typeof zhCN = { thinkingBudget: '固定予算', maxTokensLabel: '最大出力トークン数', thinkingBudgetLabel: '思考トークン予算', + serviceTierLabel: 'Ark 低レイテンシ推論（fast）', + serviceTierHint: + '対応する Ark モデルでのみ利用できます。非対応モデルでは 400 が返る場合があります。', responsesThinkingHint: '一部のモデルでは思考を軽減できますが、完全には無効にできません。推論リクエストでは温度を送信しません。', messagesThinkingHint: @@ -1757,6 +1760,7 @@ export const ja: typeof zhCN = { llmThinkingModeInvalid: '思考方式が無効です。選択し直してください。', llmTokenLimitInvalid: 'トークン上限は正の整数にしてください。', llmThinkingBudgetInvalid: '思考予算は1024以上、固定予算では最大出力未満にしてください。', + llmServiceTierInvalid: '低レイテンシ推論の設定が無効です。もう一度選択してください。', llmResponseIncomplete: '応答が未完了か出力上限に達しました。出力済みテキストは保持されます。', llmProtocolHeaderConflict: 'Messages の認証とバージョンヘッダーは自動設定されます。追加ヘッダーから x-api-key と anthropic-version を削除してください。', diff --git a/openless-all/app/src/i18n/ko.ts b/openless-all/app/src/i18n/ko.ts index 371cc7e2d..7a1c3310c 100644 --- a/openless-all/app/src/i18n/ko.ts +++ b/openless-all/app/src/i18n/ko.ts @@ -1736,6 +1736,9 @@ export const ko: typeof zhCN = { thinkingBudget: '고정 예산', maxTokensLabel: '최대 출력 토큰', thinkingBudgetLabel: '사고 토큰 예산', + serviceTierLabel: 'Ark 저지연 추론(fast)', + serviceTierHint: + '지원되는 Ark 모델에서만 사용할 수 있으며, 지원되지 않는 모델은 400을 반환할 수 있습니다.', responsesThinkingHint: '일부 모델은 사고를 줄일 수만 있으며 완전히 끌 수 없습니다. 추론 요청에는 온도를 보내지 않습니다.', messagesThinkingHint: @@ -1745,6 +1748,7 @@ export const ko: typeof zhCN = { llmTokenLimitInvalid: '토큰 한도는 양의 정수여야 합니다.', llmThinkingBudgetInvalid: '사고 예산은 1024 이상이며 고정 예산 모드에서는 출력 한도보다 작아야 합니다.', + llmServiceTierInvalid: '저지연 추론 옵션이 잘못되었습니다. 다시 선택하세요.', llmResponseIncomplete: '응답이 완료되지 않았거나 출력 한도에 도달했습니다. 이미 출력된 텍스트는 유지됩니다.', llmProtocolHeaderConflict: diff --git a/openless-all/app/src/i18n/zh-CN.ts b/openless-all/app/src/i18n/zh-CN.ts index ee582d872..46e92b37f 100644 --- a/openless-all/app/src/i18n/zh-CN.ts +++ b/openless-all/app/src/i18n/zh-CN.ts @@ -1674,6 +1674,8 @@ export const zhCN = { thinkingBudget: '固定预算', maxTokensLabel: '最大输出 tokens', thinkingBudgetLabel: '思考预算 tokens', + serviceTierLabel: '方舟低延迟推理（fast）', + serviceTierHint: '仅适用于支持低延迟的方舟模型；不支持的模型可能返回 400。', responsesThinkingHint: '部分模型只能降低思考，不能完全关闭。推理请求不发送温度参数。', messagesThinkingHint: '旧模型或兼容网关可能需要固定预算；思考预算必须小于最大输出。开启思考时不发送温度参数。', @@ -1681,6 +1683,7 @@ export const zhCN = { llmThinkingModeInvalid: '思考方式无效，请重新选择。', llmTokenLimitInvalid: 'Token 上限必须为正整数。', llmThinkingBudgetInvalid: '思考预算至少为 1024，且固定预算必须小于最大输出。', + llmServiceTierInvalid: '低延迟推理选项无效，请重新选择。', llmResponseIncomplete: '响应未完整结束或达到输出上限；已输出正文会保留。', llmProtocolHeaderConflict: 'Messages 已自动设置鉴权和版本请求头，请移除额外 Headers 中的 x-api-key 和 anthropic-version。', diff --git a/openless-all/app/src/i18n/zh-TW.ts b/openless-all/app/src/i18n/zh-TW.ts index 5b7cc7b3e..77fce38cd 100644 --- a/openless-all/app/src/i18n/zh-TW.ts +++ b/openless-all/app/src/i18n/zh-TW.ts @@ -1675,6 +1675,8 @@ export const zhTW: typeof zhCN = { thinkingBudget: '固定預算', maxTokensLabel: '最大輸出 tokens', thinkingBudgetLabel: '思考預算 tokens', + serviceTierLabel: '方舟低延遲推理（fast）', + serviceTierHint: '僅適用於支援低延遲的方舟模型；不支援的模型可能回傳 400。', responsesThinkingHint: '部分模型只能降低思考，無法完全關閉。推理請求不傳送溫度引數。', messagesThinkingHint: '舊模型或相容閘道可能需要固定預算；思考預算必須小於最大輸出。開啟思考時不傳送溫度引數。', @@ -1682,6 +1684,7 @@ export const zhTW: typeof zhCN = { llmThinkingModeInvalid: '思考方式無效，請重新選擇。', llmTokenLimitInvalid: 'Token 上限必須為正整數。', llmThinkingBudgetInvalid: '思考預算至少為 1024，且固定預算必須小於最大輸出。', + llmServiceTierInvalid: '低延遲推理選項無效，請重新選擇。', llmResponseIncomplete: '回應未完整結束或達到輸出上限；已輸出正文會保留。', llmProtocolHeaderConflict: 'Messages 已自動設定驗證與版本標頭，請移除額外 Headers 中的 x-api-key 與 anthropic-version。', diff --git a/openless-all/app/src/lib/backendEvent.test.ts b/openless-all/app/src/lib/backendEvent.test.ts index 2723a649c..1781c5853 100644 --- a/openless-all/app/src/lib/backendEvent.test.ts +++ b/openless-all/app/src/lib/backendEvent.test.ts @@ -1,4 +1,13 @@ -import { applyTranscriptEvent, type TranscriptViewState } from './backendEvent'; +import { + applyTranscriptEvent, + isTranscriptSnapshotCoherent, + applyTranscriptSnapshot, + shouldRetryTranscriptSnapshot, + beginTranscriptGeneration, + clearTranscriptText, + createTranscriptViewState, + type TranscriptViewState, +} from './backendEvent'; function assertState(actual: TranscriptViewState, text: string, sequence: number) { if (actual.text !== text || actual.sequence !== sequence) { @@ -8,7 +17,7 @@ function assertState(actual: TranscriptViewState, text: string, sequence: number } } -let state: TranscriptViewState = { sessionId: null, sequence: 0, text: '' }; +let state: TranscriptViewState = createTranscriptViewState(); state = applyTranscriptEvent(state, { sequence: 1, sessionId: 'a', @@ -86,8 +95,280 @@ state = applyTranscriptEvent(state, { }); assertState(state, '帮我写', 10); +// A UI recording edge must not reset the global watermark. Until the matching +// backend start arrives, delayed events from the old session are ignored. +state = clearTranscriptText(state); +state = beginTranscriptGeneration(state); +const generation = state.generation; +const watermark = state.sequence; +state = applyTranscriptEvent(state, { + sequence: 13, + sessionId: 'sv', + kind: { type: 'transcript_delta', payload: { text: '旧会话延迟', offset: 0, isFinal: false } }, +}); +assertState(state, '', watermark); +if (state.generation !== generation || !state.awaitingSessionStart) { + throw new Error('new generation must retain the watermark and await its session start'); +} +state = applyTranscriptEvent(state, { + sequence: 13, + sessionId: 'next', + kind: { type: 'dictation_state_changed', payload: { phase: 'starting', sessionId: 'next' } }, +}); +state = applyTranscriptEvent(state, { + sequence: 14, + sessionId: 'sv', + kind: { type: 'transcript_delta', payload: { text: '旧会话仍延迟', offset: 0, isFinal: false } }, +}); +assertState(state, '', 13); +state = applyTranscriptEvent(state, { + sequence: 14, + sessionId: 'next', + kind: { type: 'transcript_delta', payload: { text: '新会话', offset: 0, isFinal: false } }, +}); +assertState(state, '新会话', 14); + +// Tauri's real order is backend:event(starting) -> capsule:state(recording) -> +// backend:event(transcript_delta). The recording edge must reuse the session +// already established by the first event instead of reopening the gate. +let orderedState: TranscriptViewState = createTranscriptViewState(); +orderedState = applyTranscriptEvent(orderedState, { + sequence: 20, + sessionId: 'ordered-session', + kind: { + type: 'dictation_state_changed', + payload: { phase: 'starting', sessionId: 'ordered-session' }, + }, +}); +orderedState = beginTranscriptGeneration(orderedState); +if ( + orderedState.awaitingSessionStart || + orderedState.sessionId !== 'ordered-session' || + orderedState.generation !== 1 +) { + throw new Error('recording edge must reuse the session established by dictation starting'); +} +orderedState = applyTranscriptEvent(orderedState, { + sequence: 21, + sessionId: 'ordered-session', + kind: { type: 'transcript_delta', payload: { text: '首个字', offset: 0, isFinal: false } }, +}); +assertState(orderedState, '首个字', 21); +orderedState = applyTranscriptEvent(orderedState, { + sequence: 22, + sessionId: 'old-session', + kind: { type: 'transcript_delta', payload: { text: '旧延迟', offset: 0, isFinal: false } }, +}); +assertState(orderedState, '首个字', 21); + +// A delayed capsule:state(recording) can arrive after the backend has already +// announced starting and delivered the first delta. The same session must be +// reused without clearing the confirmed prefix. +let lateRecordingState: TranscriptViewState = createTranscriptViewState(); +lateRecordingState = applyTranscriptEvent(lateRecordingState, { + sequence: 30, + sessionId: 'late-recording-session', + kind: { + type: 'dictation_state_changed', + payload: { phase: 'starting', sessionId: 'late-recording-session' }, + }, +}); +lateRecordingState = applyTranscriptEvent(lateRecordingState, { + sequence: 31, + sessionId: 'late-recording-session', + kind: { type: 'transcript_delta', payload: { text: '已确认', offset: 0, isFinal: false } }, +}); +lateRecordingState = beginTranscriptGeneration(lateRecordingState); +assertState(lateRecordingState, '已确认', 31); +if (lateRecordingState.awaitingSessionStart || lateRecordingState.sessionId !== 'late-recording-session') { + throw new Error('late recording edge must preserve the already-started session transcript'); +} + +// The rail ready handshake can resolve after a newer live event. Its older +// snapshot must not roll that event back, while a current snapshot replays the +// complete transcript and leaves the session open for a later recording edge. +let replayState = applyTranscriptEvent(createTranscriptViewState(), { + sequence: 40, + sessionId: 'replay-session', + kind: { + type: 'dictation_state_changed', + payload: { phase: 'starting', sessionId: 'replay-session' }, + }, +}); +replayState = applyTranscriptEvent(replayState, { + sequence: 41, + sessionId: 'replay-session', + kind: { type: 'transcript_delta', payload: { text: '更新后的原文', offset: 0, isFinal: false } }, +}); +const staleReplay = applyTranscriptSnapshot(replayState, { + transcript: '旧快照', sessionId: 'replay-session', sequence: 40, revision: 40, +}); +assertState(staleReplay, '更新后的原文', 41); +const tornReplay = applyTranscriptSnapshot(replayState, { + transcript: '撕裂帧', sessionId: 'replay-session', sequence: 42, revision: 42, + payloadRevision: 41, +}); +assertState(tornReplay, '更新后的原文', 41); +if ( + !shouldRetryTranscriptSnapshot(replayState, { + transcript: '撕裂帧', sessionId: 'replay-session', sequence: 42, revision: 42, + payloadRevision: 41, + }) || + !isTranscriptSnapshotCoherent({ + transcript: '一致帧', sessionId: 'replay-session', sequence: 42, revision: 42, + payloadRevision: 42, + }) +) { + throw new Error('rail must retry a current torn snapshot and accept a coherent one'); +} +// Deterministic snapshot transaction ordering: +// 1. payload commit before the backend event is already coherent; +// 2. backend event before a queued payload is rejected only temporarily; +// 3. the matching payload commit at the same revision is accepted and must +// not permanently lose a legal first frame. +const payloadBeforeBackend = applyTranscriptSnapshot(replayState, { + transcript: 'payload先提交', sessionId: 'replay-session', sequence: 60, revision: 60, + payloadRevision: 60, +}); +assertState(payloadBeforeBackend, 'payload先提交', 60); +const backendBeforeQueuedPayload = applyTranscriptSnapshot(payloadBeforeBackend, { + transcript: 'backend先到', sessionId: 'replay-session', sequence: 61, revision: 61, + payloadRevision: 60, +}); +assertState(backendBeforeQueuedPayload, 'payload先提交', 60); +const queuedPayloadCommit = applyTranscriptSnapshot(backendBeforeQueuedPayload, { + transcript: 'backend先到', sessionId: 'replay-session', sequence: 61, revision: 61, + payloadRevision: 61, +}); +assertState(queuedPayloadCommit, 'backend先到', 61); + +// A Recording -> Recording state update is the first-PCM path. It must advance +// the watermark without treating the previous Recording payload as the new +// frame; the first delta after the update is still accepted for the same session. +let repeatedRecording = applyTranscriptEvent(createTranscriptViewState(), { + sequence: 70, + sessionId: 'pcm-session', + kind: { + type: 'dictation_state_changed', + payload: { phase: 'starting', sessionId: 'pcm-session' }, + }, +}); +repeatedRecording = applyTranscriptEvent(repeatedRecording, { + sequence: 71, + sessionId: 'pcm-session', + kind: { type: 'transcript_delta', payload: { text: '已有前缀', offset: 0, isFinal: false } }, +}); +repeatedRecording = applyTranscriptEvent(repeatedRecording, { + sequence: 72, + sessionId: 'pcm-session', + kind: { + type: 'dictation_state_changed', + payload: { phase: 'recording', sessionId: 'pcm-session' }, + }, +}); +assertState(repeatedRecording, '已有前缀', 72); +repeatedRecording = applyTranscriptEvent(repeatedRecording, { + sequence: 73, + sessionId: 'pcm-session', + kind: { type: 'transcript_delta', payload: { text: '首个 PCM 字', offset: 0, isFinal: false } }, +}); +assertState(repeatedRecording, '首个 PCM 字', 73); + +// SessionId, rather than a Recording -> Recording state edge, is the +// generation boundary. A new session arriving on the same state must clear +// the old text, and a delayed old-session delta must remain rejected. +let sameStateNewSession = applyTranscriptEvent(createTranscriptViewState(), { + sequence: 80, + sessionId: 'first-session', + kind: { + type: 'dictation_state_changed', + payload: { phase: 'recording', sessionId: 'first-session' }, + }, +}); +sameStateNewSession = applyTranscriptEvent(sameStateNewSession, { + sequence: 81, + sessionId: 'first-session', + kind: { type: 'transcript_delta', payload: { text: '旧文本', offset: 0, isFinal: false } }, +}); +const firstGeneration = sameStateNewSession.generation; +sameStateNewSession = applyTranscriptEvent(sameStateNewSession, { + sequence: 82, + sessionId: 'second-session', + kind: { + type: 'dictation_state_changed', + payload: { phase: 'recording', sessionId: 'second-session' }, + }, +}); +assertState(sameStateNewSession, '', 82); +if ( + sameStateNewSession.generation !== firstGeneration + 1 || + sameStateNewSession.sessionId !== 'second-session' +) { + throw new Error('Recording -> Recording with a new sessionId must start a new transcript generation'); +} +sameStateNewSession = applyTranscriptEvent(sameStateNewSession, { + sequence: 83, + sessionId: 'first-session', + kind: { type: 'transcript_delta', payload: { text: '延迟旧文本', offset: 0, isFinal: false } }, +}); +assertState(sameStateNewSession, '', 82); +sameStateNewSession = applyTranscriptEvent(sameStateNewSession, { + sequence: 84, + sessionId: 'second-session', + kind: { type: 'transcript_delta', payload: { text: '新文本', offset: 0, isFinal: false } }, +}); +assertState(sameStateNewSession, '新文本', 84); +const currentReplay = applyTranscriptSnapshot(replayState, { + transcript: '完整原文回放', sessionId: 'replay-session', sequence: 42, revision: 42, +}); +assertState(currentReplay, '完整原文回放', 42); +if ( + currentReplay.revision !== 42 || + !currentReplay.sessionStartObserved || + currentReplay.awaitingSessionStart +) { + throw new Error('current rail snapshot must keep its session eligible for the recording edge'); +} + +// Selection-voice can mount the rail midway through its session as well. Its +// state replay must advance the same session/revision watermark so subsequent +// deltas are accepted, while a delayed dictation delta is still rejected. +let selectionReplay = applyTranscriptEvent(createTranscriptViewState(), { + sequence: 50, + sessionId: 'selection-session', + kind: { + type: 'selection_voice_state_changed', + payload: { phase: 'recording', sessionId: 'selection-session' }, + }, +}); +if ( + selectionReplay.sessionId !== 'selection-session' || + selectionReplay.sequence !== 50 || + selectionReplay.revision !== 50 +) { + throw new Error('selection voice replay must advance session and revision watermarks'); +} +selectionReplay = applyTranscriptSnapshot(selectionReplay, { + transcript: '选区语音已回放', + sessionId: 'selection-session', + sequence: 51, + revision: 51, +}); +selectionReplay = applyTranscriptEvent(selectionReplay, { + sequence: 52, + sessionId: 'selection-session', + kind: { type: 'transcript_delta', payload: { text: '选区语音后续', offset: 0, isFinal: false } }, +}); +assertState(selectionReplay, '选区语音后续', 52); +selectionReplay = applyTranscriptEvent(selectionReplay, { + sequence: 53, + sessionId: 'dictation-session', + kind: { type: 'transcript_delta', payload: { text: '旧听写延迟', offset: 0, isFinal: false } }, +}); +assertState(selectionReplay, '选区语音后续', 52); + // Cloud ASR snapshots must retain all preceding words and apply corrections. -let cloudState: TranscriptViewState = { sessionId: null, sequence: 0, text: '' }; +let cloudState: TranscriptViewState = createTranscriptViewState(); for (const [index, text] of ['你', '你好', '您好', '您好。', '您好。世界'].entries()) { cloudState = applyTranscriptEvent(cloudState, { sequence: index + 1, sessionId: 'cloud', diff --git a/openless-all/app/src/lib/backendEvent.ts b/openless-all/app/src/lib/backendEvent.ts index 922141a30..3da72e1db 100644 --- a/openless-all/app/src/lib/backendEvent.ts +++ b/openless-all/app/src/lib/backendEvent.ts @@ -14,6 +14,132 @@ export interface TranscriptViewState { sessionId: string | null; sequence: number; text: string; + /** Native event revision used to reject a rail snapshot captured before a live event. */ + revision: number; + /** Generation increments at the UI recording edge without lowering the sequence watermark. */ + generation: number; + /** Reject deltas until the backend announces the matching session start. */ + awaitingSessionStart: boolean; + /** True only after the start event for the session that will own the next recording edge. */ + sessionStartObserved: boolean; +} + +export function createTranscriptViewState(): TranscriptViewState { + return { + sessionId: null, + sequence: 0, + text: '', + revision: 0, + generation: 0, + awaitingSessionStart: false, + sessionStartObserved: false, + }; +} + +/** + * Start a new UI transcript generation while retaining the global event watermark. + * A delayed delta from the previous session is rejected until a backend start event + * supplies the new session id, so clearing text never re-opens the old-session gate. + */ +export function beginTranscriptGeneration(state: TranscriptViewState): TranscriptViewState { + const reuseStartedSession = + state.sessionStartObserved && state.sessionId !== null && !state.awaitingSessionStart; + return { + ...state, + sessionId: reuseStartedSession ? state.sessionId : null, + // Tauri can deliver backend:event(starting) and transcript_delta before the + // capsule:state(recording) edge. Reusing that observed session must not + // replay the whole transcript as a new generation or erase its prefix. + text: reuseStartedSession ? state.text : '', + generation: state.generation + 1, + awaitingSessionStart: !reuseStartedSession, + sessionStartObserved: false, + }; +} + +export function clearTranscriptText(state: TranscriptViewState): TranscriptViewState { + return { ...state, text: '', sessionStartObserved: false }; +} + +export interface TranscriptSnapshot { + transcript: string; + sessionId: string | null; + sequence: number; + revision: number; + /** Native payload commit revision; absent only for legacy test/mock snapshots. */ + payloadRevision?: number; +} + +/** A snapshot is replayable only when its payload and text share one revision. */ +export function isTranscriptSnapshotCoherent(snapshot: TranscriptSnapshot): boolean { + return (snapshot.payloadRevision ?? snapshot.revision) === snapshot.revision; +} + +/** + * A torn snapshot whose watermark is at least as new as the live state may + * become coherent on the next native frame. Older torn snapshots are already + * superseded and should be ignored without spinning the ready handshake. + */ +export function shouldRetryTranscriptSnapshot( + state: TranscriptViewState, + snapshot: TranscriptSnapshot, +): boolean { + if (isTranscriptSnapshotCoherent(snapshot)) return false; + return ( + snapshot.revision >= state.revision && + !(snapshot.revision === state.revision && snapshot.sequence < state.sequence) + ); +} + +/** Apply a session watermark carried by capsule:state before backend:event. */ +export function applyTranscriptSessionId( + state: TranscriptViewState, + sessionId: string | null | undefined, +): TranscriptViewState { + if (!sessionId || sessionId === state.sessionId) return state; + return { + ...state, + sessionId, + text: '', + generation: state.generation + 1, + awaitingSessionStart: false, + sessionStartObserved: true, + }; +} + +/** + * Apply the native rail replay only when its two-part watermark is not older + * than the live event stream. Keeping this predicate beside event reduction + * makes the ready-handshake race deterministic and directly testable. + */ +export function applyTranscriptSnapshot( + state: TranscriptViewState, + snapshot: TranscriptSnapshot, +): TranscriptViewState { + const payloadRevision = snapshot.payloadRevision ?? snapshot.revision; + // Native payload and transcript are committed by different Tauri/main-thread + // callbacks. Never let a torn frame (new text + old idle/terminal payload, or + // the reverse) clear/replace the rail; the next ready replay/event will carry + // a matching commit revision. + if (payloadRevision !== snapshot.revision) return state; + const stale = + snapshot.revision < state.revision || + (snapshot.revision === state.revision && snapshot.sequence < state.sequence); + if (stale) return state; + const sessionChanged = + snapshot.sessionId !== null && snapshot.sessionId !== state.sessionId; + return { + ...state, + sessionId: snapshot.sessionId, + sequence: snapshot.sequence, + revision: snapshot.revision, + text: snapshot.transcript, + generation: sessionChanged ? state.generation + 1 : state.generation, + awaitingSessionStart: false, + // A later capsule:state(recording) edge may still need to reuse this + // session after the rail snapshot has replayed it. + sessionStartObserved: snapshot.sessionId !== null, + }; } export function applyTranscriptEvent( @@ -23,31 +149,74 @@ export function applyTranscriptEvent( if (event.sequence <= state.sequence) return state; if (event.kind.type === 'dictation_state_changed') { const payload = event.kind.payload as { phase?: string; sessionId?: string | null } | undefined; - if (payload?.phase === 'starting' && payload.sessionId) { - return { sessionId: payload.sessionId, sequence: event.sequence, text: '' }; + const sessionId = payload?.sessionId ?? event.sessionId; + const sessionChanged = Boolean(sessionId && sessionId !== state.sessionId); + if (sessionChanged) { + return { + ...state, + sessionId, + sequence: event.sequence, + revision: Math.max(state.revision, event.sequence), + text: '', + awaitingSessionStart: false, + sessionStartObserved: true, + // The normal starting -> capsule:state(recording) path increments at + // the UI recording edge. Do not count that same generation twice; + // Recording -> Recording with a different sessionId still increments + // below because it has no separate start edge to rely on. + generation: payload?.phase === 'starting' ? state.generation : state.generation + 1, + }; } - return { ...state, sequence: event.sequence }; + return { + ...state, + sessionId: sessionId ?? state.sessionId, + sequence: event.sequence, + revision: Math.max(state.revision, event.sequence), + awaitingSessionStart: payload?.phase === 'starting' ? false : state.awaitingSessionStart, + sessionStartObserved: payload?.phase === 'starting' ? true : state.sessionStartObserved, + }; } if (event.kind.type === 'selection_voice_state_changed') { const payload = event.kind.payload as { phase?: string; sessionId?: string | null } | undefined; - if (payload?.phase === 'recording') { + const sessionId = payload?.sessionId ?? event.sessionId; + const sessionChanged = Boolean(sessionId && sessionId !== state.sessionId); + if (sessionChanged) { return { - sessionId: payload.sessionId ?? event.sessionId, + ...state, + sessionId, sequence: event.sequence, + revision: Math.max(state.revision, event.sequence), text: '', + awaitingSessionStart: false, + sessionStartObserved: true, + generation: state.generation + 1, }; } - return { ...state, sequence: event.sequence }; + return { + ...state, + sessionId: sessionId ?? state.sessionId, + sequence: event.sequence, + revision: Math.max(state.revision, event.sequence), + awaitingSessionStart: + payload?.phase === 'recording' ? false : state.awaitingSessionStart, + sessionStartObserved: + payload?.phase === 'recording' ? true : state.sessionStartObserved, + }; + } + if (event.kind.type !== 'transcript_delta') { + return { ...state, sequence: event.sequence, revision: Math.max(state.revision, event.sequence) }; } - if (event.kind.type !== 'transcript_delta') return { ...state, sequence: event.sequence }; + if (state.awaitingSessionStart) return state; if (state.sessionId !== null && event.sessionId !== state.sessionId) return state; const delta = event.kind.payload as TranscriptDelta | undefined; if (!delta || !Number.isSafeInteger(delta.offset) || delta.offset < 0) return state; const current = Array.from(state.text); if (delta.offset > current.length) return state; return { + ...state, sessionId: event.sessionId, sequence: event.sequence, + revision: Math.max(state.revision, event.sequence), text: current.slice(0, delta.offset).join('') + delta.text, }; } diff --git a/openless-all/app/src/lib/capsuleLayout.test.ts b/openless-all/app/src/lib/capsuleLayout.test.ts index f33307f10..299d18b59 100644 --- a/openless-all/app/src/lib/capsuleLayout.test.ts +++ b/openless-all/app/src/lib/capsuleLayout.test.ts @@ -2,6 +2,11 @@ import { getCapsuleHostMetrics, getCapsuleMessageLayout, getCapsulePillMetrics, + getCapsuleTranscriptRailHeight, + getCapsuleTranscriptRailPosition, + getClassicCapsuleGeometry, + getClassicProcessingLabel, + getClassicTranslationBadgeOffset, parseCapsuleStyle, } from './capsuleLayout.ts'; @@ -11,6 +16,12 @@ function assertEqual(actual: T, expected: T, name: string) { } } +function assertClose(actual: number, expected: number, name: string) { + if (Math.abs(actual - expected) > 1e-9) { + throw new Error(`${name}: expected ${expected}, got ${actual}`); + } +} + const winMetrics = getCapsulePillMetrics('win'); assertEqual(winMetrics.width, 460, 'windows voice orb stage uses the demo-scale width'); assertEqual(winMetrics.height, 180, 'windows voice orb stage uses the demo-scale height'); @@ -23,7 +34,11 @@ assertEqual( const winHost = getCapsuleHostMetrics('win', false); assertEqual(winHost.width, 460, 'windows voice orb host matches stage width'); -assertEqual(winHost.height, 180, 'windows voice orb host matches stage height'); +assertEqual( + winHost.height, + 228, + 'windows voice orb host reserves the transcript rail above the stage', +); assertEqual(winHost.horizontalInset, 0, 'windows voice orb host has no side button inset'); assertEqual(winHost.boxSizing, 'border-box', 'windows voice orb host keeps border-box sizing'); assertEqual( @@ -32,9 +47,9 @@ assertEqual( 'windows voice orb host keeps the visible stage width after reserving side insets', ); assertEqual( - winHost.height - winHost.bottomInset, + winHost.height - 48 - winHost.bottomInset, winMetrics.height, - 'windows voice orb host keeps the visible stage height after reserving bottom inset', + 'windows voice orb host keeps the visible stage height below the transcript rail', ); const winHostWithTranslation = getCapsuleHostMetrics('win', true); @@ -45,8 +60,8 @@ assertEqual( ); assertEqual( winHostWithTranslation.height, - 180, - 'windows translation voice orb keeps the same outer height', + 228, + 'windows translation voice orb keeps the transcript rail height', ); assertEqual( winHostWithTranslation.horizontalInset, @@ -67,7 +82,7 @@ assertEqual(macMetrics.boxSizing, 'border-box', 'mac voice orb stage keeps borde const macHost = getCapsuleHostMetrics('mac', false); assertEqual(macHost.width, 460, 'mac voice orb host matches stage width'); -assertEqual(macHost.height, 180, 'mac voice orb host matches stage height'); +assertEqual(macHost.height, 228, 'mac voice orb host reserves the transcript rail above the stage'); assertEqual(macHost.boxSizing, 'border-box', 'mac voice orb host keeps border-box sizing'); const winErrorLayout = getCapsuleMessageLayout('win', 'error'); @@ -85,14 +100,162 @@ assertEqual(macErrorLayout.allowWrap, false, 'mac error message stays nowrap'); for (const os of ['mac', 'win'] as const) { const classic = getCapsuleHostMetrics(os, false, 'classic'); const typeless = getCapsuleHostMetrics(os, true, 'typeless'); - assertEqual(classic.height, 100, `${os}: classic uses the compact native window`); - assertEqual(typeless.height, 57, `${os}: typeless window is 1/5 of the old 460x128 area`); + assertEqual( + classic.height, + 172, + `${os}: classic reserves the transcript rail and translating badge`, + ); + assertEqual(typeless.height, 65, `${os}: typeless translation host reserves a badge row`); assertEqual(typeless.width, 206, `${os}: typeless window keeps the 1/5 stage width`); assertEqual(typeless.bottomInset, 0, `${os}: typeless pill hugs the work-area bottom edge`); + assertEqual( + getCapsuleHostMetrics(os, false, 'typeless').height, + 57, + `${os}: typeless host stays compact when translation is hidden`, + ); } for (const style of ['siri', 'classic', 'typeless'] as const) { assertEqual(parseCapsuleStyle(style), style, `${style} is accepted from preferences and events`); } +assertEqual(getCapsuleTranscriptRailHeight('siri'), 40, 'Siri rail height matches native overlay'); +assertEqual( + getCapsuleTranscriptRailHeight('classic'), + 52, + 'Classic rail height matches native overlay', +); +assertEqual( + getCapsuleTranscriptRailHeight('typeless'), + 52, + 'Typeless logical rail height matches CSS overlay', +); +assertEqual( + getClassicTranslationBadgeOffset(true), + 60, + 'Classic translation badge clears the external 52px rail and its 8px gap', +); +assertEqual( + getClassicTranslationBadgeOffset(false), + 0, + 'Classic translation badge does not reserve a hidden rail', +); +assertEqual( + getClassicTranslationBadgeOffset(true, false), + 0, + 'Classic external rail uses its own badge lane instead of same-window offset', +); +const classicExternalGeometry = getClassicCapsuleGeometry('win', true, true, false); +assertEqual(classicExternalGeometry.hostHeight, 172, 'Classic external rail uses the 172px host'); +assertEqual( + classicExternalGeometry.bodyTop, + 104, + 'Classic body is anchored at the bottom of the host', +); +assertEqual( + classicExternalGeometry.railTop, + 14, + 'Classic external rail clears the translation badge lane', +); +assertEqual(classicExternalGeometry.badgeTop, 74, 'Classic external badge sits above the pill'); +assertEqual( + classicExternalGeometry.badgeBottom, + 96, + 'Classic external badge leaves an 8px body gap', +); +if ( + classicExternalGeometry.railTop! + classicExternalGeometry.railHeight > + classicExternalGeometry.badgeTop || + classicExternalGeometry.badgeBottom > classicExternalGeometry.bodyTop +) { + throw new Error('Classic external rail and translation badge must not overlap'); +} +const classicSameWindowGeometry = getClassicCapsuleGeometry('win', true, true, true); +assertEqual( + classicSameWindowGeometry.railTop, + 44, + 'Classic same-window rail keeps the body rail anchor', +); +assertEqual(classicSameWindowGeometry.badgeBottom, 36, 'Classic same-window badge clears the rail'); +for (const os of ['mac', 'win'] as const) { + const external = getClassicCapsuleGeometry(os, true, true, false); + if ( + external.railTop! + external.railHeight > external.badgeTop || + external.badgeBottom > external.bodyTop + ) { + throw new Error(`${os}: external Classic rail and badge overlap`); + } +} +assertEqual( + getClassicProcessingLabel(false), + 'capsule.thinking', + 'Classic processing uses thinking copy for ordinary dictation', +); +assertEqual( + getClassicProcessingLabel(true), + 'capsule.using', + 'Classic processing uses using copy for operating sessions', +); +assertEqual( + getCapsuleTranscriptRailPosition('siri', false, true)?.topOffset, + -48, + 'Siri rail sits above the 180px body with its 8px CSS gap', +); +assertEqual( + getCapsuleTranscriptRailPosition('siri', true, true)?.topOffset, + -48, + 'Siri translation does not move the full-height body rail anchor', +); +assertEqual( + getCapsuleTranscriptRailPosition('classic', false, true)?.topOffset, + 44, + 'Classic compact external rail anchors to the 172px body content top', +); +assertEqual( + getCapsuleTranscriptRailPosition('classic', true, true)?.topOffset, + 14, + 'Classic translation external rail clears the explicit badge lane', +); +const typelessCompactRail = getCapsuleTranscriptRailPosition('typeless', false, true); +const typelessTranslationRail = getCapsuleTranscriptRailPosition('typeless', true, true); +assertEqual( + getCapsuleTranscriptRailPosition('classic', true, false)?.height, + 22, + 'Classic translation remains in the click-through overlay with transcript disabled', +); +assertEqual( + getCapsuleTranscriptRailPosition('classic', true, true)?.height, + 82, + 'Classic overlay includes both the rail and the translation badge', +); +assertClose( + getCapsuleTranscriptRailPosition('typeless', true, false)?.height ?? 0, + 20 * 0.447, + 'Typeless translation remains outside the clipped button window', +); +assertClose( + typelessTranslationRail?.height ?? 0, + (52 + 20) * 0.447, + 'Typeless overlay includes both text and its translation row', +); +if (!typelessCompactRail || !typelessTranslationRail) + throw new Error('typeless rail geometry missing'); +assertClose(typelessCompactRail.height, 52 * 0.447, 'Typeless rail uses the zoomed native height'); +assertClose( + typelessCompactRail.topOffset, + 57 - 64 * 0.447 - 52 * 0.447, + 'Typeless compact body anchor', +); +assertClose( + typelessTranslationRail.topOffset, + 65 - 20 * 0.447 - 64 * 0.447 - 52 * 0.447, + 'Typeless translation body anchor includes the translation row', +); +for (const style of ['siri', 'classic', 'typeless'] as const) { + assertEqual( + getCapsuleTranscriptRailPosition(style, false, false), + null, + `${style} hides the rail when transcript is not visible`, + ); +} assertEqual( parseCapsuleStyle('unknown'), undefined, diff --git a/openless-all/app/src/lib/capsuleLayout.ts b/openless-all/app/src/lib/capsuleLayout.ts index 92ccc9908..db41f4a29 100644 --- a/openless-all/app/src/lib/capsuleLayout.ts +++ b/openless-all/app/src/lib/capsuleLayout.ts @@ -30,12 +30,173 @@ export interface CapsuleMessageLayout { const VOICE_ORB_STAGE_WIDTH = 460; const VOICE_ORB_STAGE_HEIGHT = 180; const VOICE_ORB_TEXT_WIDTH = 400; +export const CAPSULE_TRANSCRIPT_RAIL_HEIGHT = 40; +export const CAPSULE_TRANSCRIPT_RAIL_GAP = 8; +export const CAPSULE_TRANSCRIPT_RAIL_HEIGHTS: Record = { + siri: 40, + classic: 52, + // Typeless is rendered in its 0.447 zoomed tree, but its CSS rail still owns + // the same 52px logical row as the standalone Windows overlay. + typeless: 52, +}; + +export const CLASSIC_CAPSULE_HOST_HEIGHT = 172; +export const CLASSIC_CAPSULE_BOTTOM_INSET = 16; +export const CLASSIC_CAPSULE_BADGE_HEIGHT = 22; +export const CLASSIC_CAPSULE_BADGE_GAP = 8; +export const CLASSIC_CAPSULE_RAIL_GAP = 8; + +export interface ClassicCapsuleGeometry { + hostHeight: number; + bodyTop: number; + bodyBottom: number; + badgeTop: number; + badgeBottom: number; + badgeBottomOffset: number; + railTop: number | null; + railHeight: number; +} + +/** + * Classic uses one 172px layout coordinate system on desktop. The body, badge + * lane, and a sibling rail each get an explicit vertical anchor so an external + * rail is not treated as a missing same-window rail. + */ +export function getClassicCapsuleGeometry( + os: OS, + transcriptVisible: boolean, + translationActive: boolean, + transcriptInSameWindow: boolean, +): ClassicCapsuleGeometry { + const pillHeight = os === 'win' ? 52 : 42; + const hostHeight = CLASSIC_CAPSULE_HOST_HEIGHT; + const bodyBottom = hostHeight - CLASSIC_CAPSULE_BOTTOM_INSET; + const bodyTop = bodyBottom - pillHeight; + const railHeight = getCapsuleTranscriptRailHeight('classic'); + const sameWindowRail = transcriptVisible && transcriptInSameWindow; + const externalBadgeLane = transcriptVisible && !transcriptInSameWindow && translationActive; + const railTop = transcriptVisible + ? bodyTop - + CLASSIC_CAPSULE_RAIL_GAP - + railHeight - + (externalBadgeLane ? CLASSIC_CAPSULE_BADGE_HEIGHT + CLASSIC_CAPSULE_BADGE_GAP : 0) + : null; + const badgeBottom = + bodyTop - + CLASSIC_CAPSULE_BADGE_GAP - + (sameWindowRail && translationActive ? railHeight + CLASSIC_CAPSULE_RAIL_GAP : 0); + const badgeTop = badgeBottom - CLASSIC_CAPSULE_BADGE_HEIGHT; + return { + hostHeight, + bodyTop, + bodyBottom, + badgeTop, + badgeBottom, + badgeBottomOffset: hostHeight - badgeBottom, + railTop, + railHeight, + }; +} + +export function getCapsuleTranscriptRailHeight(style: CapsuleStyle): number { + return CAPSULE_TRANSCRIPT_RAIL_HEIGHTS[style]; +} + +/** Classic's translating badge lives above the body and must also clear the + * external 52px rail when the rail is hosted in a sibling HWND. */ +export function getClassicTranslationBadgeOffset( + transcriptVisible: boolean, + transcriptInSameWindow = true, +): number { + const geometry = getClassicCapsuleGeometry( + 'win', + transcriptVisible, + true, + transcriptInSameWindow, + ); + const baseBadgeBottomOffset = + CLASSIC_CAPSULE_HOST_HEIGHT - geometry.bodyTop + CLASSIC_CAPSULE_BADGE_GAP; + return geometry.badgeBottomOffset - baseBadgeBottomOffset; +} + +export function getClassicProcessingLabel( + operating: boolean, +): 'capsule.thinking' | 'capsule.using' { + return operating ? 'capsule.using' : 'capsule.thinking'; +} + +export interface CapsuleTranscriptRailPosition { + width: number; + height: number; + /** Rail top relative to the capsule body HWND top, in logical/native pixels. */ + topOffset: number; + /** CSS gap between the rail and the capsule's actual body content. */ + gap: number; +} + +const TYPELESS_ZOOM = 0.447; +const TYPELESS_CAPSULE_HEIGHT = 64 * TYPELESS_ZOOM; +const TYPELESS_TRANSLATION_ROW_HEIGHT = 20 * TYPELESS_ZOOM; +const TYPELESS_RAIL_HEIGHT = 52 * TYPELESS_ZOOM; + +/** + * Position the standalone Windows rail against the actual capsule body, not + * against the transparent HWND edge. Keep this in sync with the native helper + * of the same name in tauri_coordinator_host.rs. + */ +export function getCapsuleTranscriptRailPosition( + style: CapsuleStyle, + translationActive: boolean, + transcriptVisible: boolean, +): CapsuleTranscriptRailPosition | null { + if (!transcriptVisible && (!translationActive || style === 'siri')) return null; + if (style === 'siri') { + const bodyTop = 0; + const railHeight = 40; + const gap = 8; + return { width: 460, height: railHeight, topOffset: bodyTop - gap - railHeight, gap }; + } + if (style === 'classic') { + const geometry = getClassicCapsuleGeometry('win', transcriptVisible, translationActive, false); + const gap = CLASSIC_CAPSULE_RAIL_GAP; + return { + width: 460, + height: + (transcriptVisible ? geometry.railHeight : 0) + + (translationActive + ? CLASSIC_CAPSULE_BADGE_HEIGHT + (transcriptVisible ? CLASSIC_CAPSULE_BADGE_GAP : 0) + : 0), + topOffset: geometry.railTop ?? geometry.badgeTop, + gap, + }; + } + const hostHeight = translationActive ? 65 : 57; + const bodyBottom = hostHeight; + const bodyTop = + bodyBottom - + TYPELESS_CAPSULE_HEIGHT - + (translationActive ? TYPELESS_TRANSLATION_ROW_HEIGHT : 0); + const gap = 0; + return { + width: 206, + height: + (transcriptVisible ? TYPELESS_RAIL_HEIGHT : 0) + + (translationActive ? TYPELESS_TRANSLATION_ROW_HEIGHT : 0), + topOffset: bodyTop - gap - (transcriptVisible ? TYPELESS_RAIL_HEIGHT : 0), + gap, + }; +} + +const SIRI_HOST_HEIGHT = + VOICE_ORB_STAGE_HEIGHT + CAPSULE_TRANSCRIPT_RAIL_HEIGHT + CAPSULE_TRANSCRIPT_RAIL_GAP; +const CLASSIC_HOST_HEIGHT = CLASSIC_CAPSULE_HOST_HEIGHT; // The typeless window is 1/5 the area of the original size (460×128); content is scaled by the // zoom in CapsuleStyles.css, kept in sync with capsule_window_bounds_for_style in // src-tauri/src/lib.rs. const TYPELESS_STAGE_WIDTH = 206; const TYPELESS_STAGE_HEIGHT = 57; +const TYPELESS_TRANSLATION_HOST_HEIGHT = 65; export function parseCapsuleStyle(value: unknown): CapsuleStyle | undefined { return value === 'siri' || value === 'classic' || value === 'typeless' ? value : undefined; @@ -56,11 +217,10 @@ export function getCapsuleHostMetrics( translationActive: boolean, style: CapsuleStyle = 'siri', ): CapsuleHostMetrics { - void translationActive; if (style === 'typeless') { return { width: TYPELESS_STAGE_WIDTH, - height: TYPELESS_STAGE_HEIGHT, + height: translationActive ? TYPELESS_TRANSLATION_HOST_HEIGHT : TYPELESS_STAGE_HEIGHT, horizontalInset: 0, bottomInset: 0, badgeGap: 8, @@ -70,7 +230,7 @@ export function getCapsuleHostMetrics( const stage = getCapsulePillMetrics(os); return { width: stage.width, - height: style === 'siri' ? stage.height : style === 'classic' ? 100 : 128, + height: style === 'siri' ? SIRI_HOST_HEIGHT : style === 'classic' ? CLASSIC_HOST_HEIGHT : 128, horizontalInset: 0, bottomInset: style === 'siri' ? 0 : 16, badgeGap: 8, diff --git a/openless-all/app/src/lib/insertTextAnimation.test.ts b/openless-all/app/src/lib/insertTextAnimation.test.ts index 000d1ce52..efce54c4b 100644 --- a/openless-all/app/src/lib/insertTextAnimation.test.ts +++ b/openless-all/app/src/lib/insertTextAnimation.test.ts @@ -3,6 +3,7 @@ import { appendedBirthAdvance, clampInsertContentWidth, diffInsertUnits, + diffInsertWindowUnits, firstBornIndex, planCapsuleInsertMotion, planCharDelays, @@ -10,6 +11,8 @@ import { propagationDelayMs, resetInsertUnitKeys, rightAnchoredPositions, + selectInsertRenderWindow, + selectInsertTextWindow, segmentInsertUnits, type InsertUnit, } from './insertTextAnimation'; @@ -180,3 +183,100 @@ assertEqual( 32, 'append enters beyond old tail', ); + +const longUnits = diffInsertUnits([], 'abcdefghijklmnop'); +let measuredGlyphs = 0; +const boundedWindow = selectInsertRenderWindow( + longUnits, + () => { + measuredGlyphs += 1; + return 10; + }, + 58, + 8, + { maxUnits: 6, overflowBufferUnits: 2 }, +); +assertEqual( + boundedWindow.units.map((unit) => unit.text).join(''), + 'klmnop', + 'window keeps latest suffix', +); +assertEqual( + boundedWindow.units.length, + 6, + 'window includes only a bounded visible suffix and buffer', +); +assertEqual( + boundedWindow.widths.length, + boundedWindow.units.length, + 'window carries measured widths', +); +assertEqual(measuredGlyphs, boundedWindow.units.length, 'window does not measure discarded glyphs'); +assert(measuredGlyphs < longUnits.length, 'discarded glyphs are never measured'); +assert( + selectInsertRenderWindow(longUnits, () => 10, 58, 8, { maxUnits: 3 }).units.length <= 3, + 'hard max keeps motion nodes bounded even for wide glyphs', +); + +console.log('bounded insert render window passed'); + +const longTranscript = '0123456789'.repeat(200); +const textWindow = selectInsertTextWindow(longTranscript, { maxUnits: 8, overflowBufferUnits: 2 }); +assertEqual(textWindow, '0123456789', 'text window keeps the latest bounded graphemes'); + +let rollingUnits: InsertUnit[] = []; +for (let length = 1; length <= longTranscript.length; length += 1) { + const previousUnits = rollingUnits; + rollingUnits = diffInsertWindowUnits(previousUnits, longTranscript.slice(0, length), { + maxUnits: 8, + overflowBufferUnits: 2, + }); + assert(rollingUnits.length <= 10, 'rolling animation units stay bounded during long transcript'); + assertEqual( + rollingUnits.filter((unit) => unit.born).length, + 1, + `rolling append only births the newest glyph at length ${length}`, + ); + const previousByOffset = new Map( + previousUnits + .filter((unit) => unit.sourceOffset != null) + .map((unit) => [unit.sourceOffset as number, unit.key]), + ); + for (const unit of rollingUnits) { + if (unit.sourceOffset != null && previousByOffset.has(unit.sourceOffset)) { + assertEqual( + unit.key, + previousByOffset.get(unit.sourceOffset), + `rolling overlap reuses key at offset ${unit.sourceOffset}`, + ); + } + } +} +assertEqual( + rollingUnits.map((unit) => unit.text).join(''), + '0123456789', + 'rolling window ends at the current transcript tail', +); + +const correctionSource = 'a'.repeat(40) + 'weather today'; +let correctionUnits = diffInsertWindowUnits([], correctionSource, { maxUnits: 20 }); +const stableTailKey = correctionUnits[correctionUnits.length - 1]?.key; + correctionUnits = diffInsertWindowUnits(correctionUnits, correctionSource.replace('weather', 'whether'), { + maxUnits: 20, +}); +assertEqual( + correctionUnits[correctionUnits.length - 1]?.key, + stableTailKey, + 'window correction preserves keys for the unchanged visible suffix', +); +assertEqual( + correctionUnits.filter((unit) => unit.born).length, + 2, + 'window correction only births the changed graphemes', +); +assert( + correctionUnits.length <= 23, + 'window correction does not restore the discarded transcript history', +); + +console.log('bounded transcript diff passed'); diff --git a/openless-all/app/src/lib/insertTextAnimation.ts b/openless-all/app/src/lib/insertTextAnimation.ts index 06f46aa6d..230a3391a 100644 --- a/openless-all/app/src/lib/insertTextAnimation.ts +++ b/openless-all/app/src/lib/insertTextAnimation.ts @@ -35,6 +35,8 @@ export interface InsertUnit { key: string; text: string; born: boolean; + /** UTF-16 offset in the canonical transcript, when this unit came from a bounded window. */ + sourceOffset?: number; } let insertUnitSeq = 0; @@ -106,6 +108,203 @@ export function diffInsertUnits(prev: InsertUnit[], nextText: string): InsertUni return units; } +export const INSERT_TEXT_RENDER_WINDOW = { + maxUnits: 64, + overflowBufferUnits: 3, +} as const; + +export interface InsertTextWindowOptions { + maxUnits?: number; + overflowBufferUnits?: number; +} + +function normalizedInsertTextWindowLimit( + options: InsertTextWindowOptions = {}, +): { maxUnits: number; targetUnits: number } { + const maxUnits = Math.max(1, Math.floor(options.maxUnits ?? INSERT_TEXT_RENDER_WINDOW.maxUnits)); + const overflowBufferUnits = Math.max( + 0, + Math.floor(options.overflowBufferUnits ?? INSERT_TEXT_RENDER_WINDOW.overflowBufferUnits), + ); + return { maxUnits, targetUnits: maxUnits + overflowBufferUnits }; +} + +/** + * Return only the suffix needed to seed the bounded animation list. + * + * The probe starts near the end of the string and expands only when a grapheme is unusually + * large. This keeps the ordinary streaming path independent of the already-scrolled transcript; + * the full string remains available to the caller for recognition corrections and accessibility. + */ +export function selectInsertTextWindow( + text: string, + options: InsertTextWindowOptions = {}, +): string { + return selectInsertTextWindowSlice(text, options).text; +} + +interface InsertTextWindowGlyph { + text: string; + sourceOffset: number; +} + +interface InsertTextWindowSlice { + text: string; + glyphs: InsertTextWindowGlyph[]; +} + +function selectInsertTextWindowSlice( + text: string, + options: InsertTextWindowOptions = {}, +): InsertTextWindowSlice { + if (!text) return { text: '', glyphs: [] }; + const { targetUnits } = normalizedInsertTextWindowLimit(options); + let probeLength = Math.min(text.length, Math.max(32, targetUnits * 4)); + + while (true) { + const start = text.length - probeLength; + const segmented = segmentInsertUnits(text.slice(start)); + // A bounded slice can start in the middle of a grapheme. Drop that guard segment; when the + // slice starts at zero there is no preceding context and the first segment is trustworthy. + const safeSegments = start === 0 ? segmented : segmented.slice(1); + if (safeSegments.length >= targetUnits || start === 0) { + const windowSegments = safeSegments.slice(-targetUnits); + const safeStartOffset = start + (start === 0 ? 0 : segmented[0]?.length ?? 0); + const discardedUnits = safeSegments.length - windowSegments.length; + let windowStartOffset = safeStartOffset; + for (let i = 0; i < discardedUnits; i += 1) { + windowStartOffset += safeSegments[i].length; + } + const glyphs: InsertTextWindowGlyph[] = []; + let sourceOffset = windowStartOffset; + for (const glyph of windowSegments) { + glyphs.push({ text: glyph, sourceOffset }); + sourceOffset += glyph.length; + } + return { text: windowSegments.join(''), glyphs }; + } + const nextProbeLength = Math.min(text.length, Math.max(probeLength + 32, probeLength * 2)); + if (nextProbeLength === probeLength) { + const windowSegments = safeSegments.slice(-targetUnits); + const safeStartOffset = start + (start === 0 ? 0 : segmented[0]?.length ?? 0); + const discardedUnits = safeSegments.length - windowSegments.length; + let windowStartOffset = safeStartOffset; + for (let i = 0; i < discardedUnits; i += 1) { + windowStartOffset += safeSegments[i].length; + } + const glyphs: InsertTextWindowGlyph[] = []; + let sourceOffset = windowStartOffset; + for (const glyph of windowSegments) { + glyphs.push({ text: glyph, sourceOffset }); + sourceOffset += glyph.length; + } + return { text: windowSegments.join(''), glyphs }; + } + probeLength = nextProbeLength; + } +} + +/** + * Diff only the bounded suffix used by the motion track. The caller may keep the full nextText, + * but no complete-history InsertUnit array is created or compared here. + */ +export function diffInsertWindowUnits( + prev: InsertUnit[], + nextText: string, + options: InsertTextWindowOptions = {}, +): InsertUnit[] { + const nextWindow = selectInsertTextWindowSlice(nextText, options); + if (nextWindow.glyphs.length === 0) return []; + + // The suffix text is not a stable identity: once the window rolls, every glyph changes index + // even though all but the newest glyph are still on screen. Match the bounded overlap by its + // absolute UTF-16 offset first, so a scrolling deque reuses the old keys instead of replaying + // the born animation for the whole window. + const previousByOffset = new Map(); + for (const unit of prev) { + if (unit.sourceOffset != null) previousByOffset.set(unit.sourceOffset, unit); + } + let exactOverlap = 0; + for (const glyph of nextWindow.glyphs) { + const previous = previousByOffset.get(glyph.sourceOffset); + if (previous?.text === glyph.text) exactOverlap += 1; + } + + if (exactOverlap > 0 || prev.length === 0) { + return nextWindow.glyphs.map((glyph) => { + const previous = previousByOffset.get(glyph.sourceOffset); + if (previous?.text === glyph.text) { + return { ...previous, sourceOffset: glyph.sourceOffset, born: false }; + } + return { + key: allocInsertKey(), + text: glyph.text, + sourceOffset: glyph.sourceOffset, + born: true, + }; + }); + } + + // A recognition correction can change the UTF-16 length before the bounded suffix. In that + // case absolute offsets move together; the bounded local diff still preserves its common + // prefix/suffix without ever inspecting the discarded transcript history. + return diffInsertUnits(prev, nextWindow.text).map((unit, index) => ({ + ...unit, + sourceOffset: nextWindow.glyphs[index]?.sourceOffset, + })); +} + +export interface InsertRenderWindow { + units: InsertUnit[]; + widths: number[]; +} + +/** + * Select only the rightmost glyphs needed by the visual track. + * + * The input list is already a bounded animation history. A few extra units form a small left-side + * buffer for the mask/spring without allowing DOM growth. + */ +export function selectInsertRenderWindow( + units: InsertUnit[], + measureWidth: (text: string) => number, + maxWidth: number, + padX: number = INSERT_TEXT_MOTION.padX, + options: { + maxUnits?: number; + overflowBufferUnits?: number; + } = {}, +): InsertRenderWindow { + if (units.length === 0) return { units: [], widths: [] }; + + const innerWidth = Math.max(0, maxWidth - padX * 2); + const maxUnits = Math.max(1, Math.floor(options.maxUnits ?? INSERT_TEXT_RENDER_WINDOW.maxUnits)); + const overflowBufferUnits = Math.max( + 0, + Math.floor(options.overflowBufferUnits ?? INSERT_TEXT_RENDER_WINDOW.overflowBufferUnits), + ); + const selectedWidths: number[] = []; + let start = units.length; + let contentWidth = 0; + let bufferUnits = 0; + + for (let index = units.length - 1; index >= 0 && units.length - index <= maxUnits; index -= 1) { + const measured = measureWidth(units[index].text); + const width = Number.isFinite(measured) && measured > 0 ? measured : 1; + const stillVisible = contentWidth < innerWidth; + if (!stillVisible && bufferUnits >= overflowBufferUnits) break; + start = index; + selectedWidths.unshift(width); + contentWidth += width; + if (contentWidth >= innerWidth) bufferUnits += 1; + } + + return { + units: units.slice(start), + widths: selectedWidths, + }; +} + export function firstBornIndex(units: InsertUnit[]): number { const index = units.findIndex((unit) => unit.born); if (index >= 0) return index; diff --git a/openless-all/app/src/lib/ipc/channels.ts b/openless-all/app/src/lib/ipc/channels.ts index 64aee51af..858fcea26 100644 --- a/openless-all/app/src/lib/ipc/channels.ts +++ b/openless-all/app/src/lib/ipc/channels.ts @@ -132,7 +132,10 @@ export function setChannelProviderType( if (channel && channel.providerType !== providerType) { channel.providerType = providerType; channel.lastTest = null; - if (kind === 'llm') mockCredentialValues.delete(`${id}:ark.request_format`); + if (kind === 'llm') { + mockCredentialValues.delete(`${id}:ark.request_format`); + mockCredentialValues.delete(`${id}:ark.service_tier`); + } } }); } diff --git a/openless-all/app/src/lib/ipc/dictation.ts b/openless-all/app/src/lib/ipc/dictation.ts index 1ed2bc890..619e7567f 100644 --- a/openless-all/app/src/lib/ipc/dictation.ts +++ b/openless-all/app/src/lib/ipc/dictation.ts @@ -1,4 +1,6 @@ +import { isDesktop } from './platform-exports'; import { invokeOrMock, platformCapabilities } from './shared'; +import type { CapsuleSnapshot } from '../types'; export function startDictation(): Promise { return invokeOrMock('start_dictation', undefined, () => undefined); @@ -12,6 +14,17 @@ export function cancelDictation(): Promise { return invokeOrMock('cancel_dictation', undefined, () => undefined); } +export function setCapsuleTranscriptVisible(visible: boolean): Promise { + if (!isDesktop()) return Promise.resolve(); + return invokeOrMock('set_capsule_transcript_visible', { visible }, () => undefined); +} + +/** Latest native capsule frame, used by the separate Windows rail after its webview is ready. */ +export function getCapsuleSnapshot(): Promise { + if (!isDesktop()) return Promise.resolve(null); + return invokeOrMock('get_capsule_snapshot', undefined, () => null); +} + export function handleWindowHotkeyEvent( eventType: 'keydown' | 'keyup', key: string, diff --git a/openless-all/app/src/lib/ipc/index.ts b/openless-all/app/src/lib/ipc/index.ts index 2114deb9d..72fa53984 100644 --- a/openless-all/app/src/lib/ipc/index.ts +++ b/openless-all/app/src/lib/ipc/index.ts @@ -96,6 +96,8 @@ export { startDictation, stopDictation, cancelDictation, + getCapsuleSnapshot, + setCapsuleTranscriptVisible, handleWindowHotkeyEvent, } from './dictation'; diff --git a/openless-all/app/src/lib/types.ts b/openless-all/app/src/lib/types.ts index d9cf11ab8..962cfa745 100644 --- a/openless-all/app/src/lib/types.ts +++ b/openless-all/app/src/lib/types.ts @@ -715,6 +715,8 @@ export type CapsuleState = export type CapsuleStyle = 'siri' | 'classic' | 'typeless'; export interface CapsulePayload { + /** Session generation that owns this capsule frame. */ + sessionId?: string | null; state: CapsuleState; level: number; // 0..1 RMS elapsedMs: number; @@ -743,6 +745,23 @@ export interface CapsulePayload { selectionPolish?: boolean; } +/** + * Native replay frame used when the standalone Windows transcript rail mounts + * after the capsule has already started. The payload fields are deliberately + * flattened so the rail can restore presentation state and transcript state + * from one revisioned snapshot. + */ +export interface CapsuleSnapshot extends CapsulePayload { + transcript: string; + sessionId: string | null; + /** Backend event watermark for rejecting delayed events from an old session. */ + sequence: number; + /** Monotonic native snapshot revision; currently aligned with the event watermark. */ + revision: number; + /** Revision at which the flattened payload was committed. Torn frames are rejected. */ + payloadRevision: number; +} + export interface CredentialsStatus { activeAsrProvider: string; activeLlmProvider: string; diff --git a/openless-all/app/src/main.tsx b/openless-all/app/src/main.tsx index 52448c213..1fb581cd1 100644 --- a/openless-all/app/src/main.tsx +++ b/openless-all/app/src/main.tsx @@ -16,6 +16,7 @@ import type { OS } from './components/WindowChrome'; const params = new URLSearchParams(window.location.search); const windowKind = params.get('window'); const isCapsule = windowKind === 'capsule'; +const isCapsuleRail = windowKind === 'capsule-rail'; const isQa = windowKind === 'qa'; const isSelectionVoiceIntent = windowKind === 'selection-voice-intent'; const isLessComputer = windowKind === 'less-computer'; @@ -37,6 +38,7 @@ const renderApp = () => { {isMainWindow && } c.id === first)?.lastTest === null, 'Credential mutation invalidates old validation', ); +await setChannelProviderType('llm', first, 'ark'); +await setCredential('ark.service_tier', 'fast', first); +assert( + (await readCredential('ark.service_tier', first)) === 'fast', + 'ARK low-latency option must survive reload', +); await setChannelProviderType('llm', first, 'custom_responses'); assert( (await readCredential('ark.request_format', first)) === null, 'Changing preset resets the format override', ); +assert( + (await readCredential('ark.service_tier', first)) === null, + 'Changing preset clears the ARK-only low-latency option', +); assert( (await readCredential('ark.api_key', first)) === 'fixture-key', 'Changing preset preserves the key', diff --git a/openless-all/app/src/pages/settings/LlmProtocolFields.tsx b/openless-all/app/src/pages/settings/LlmProtocolFields.tsx index a7a418f6d..7bafc16bc 100644 --- a/openless-all/app/src/pages/settings/LlmProtocolFields.tsx +++ b/openless-all/app/src/pages/settings/LlmProtocolFields.tsx @@ -5,22 +5,24 @@ import { readCredential, setCredential } from '../../lib/ipc'; import type { LlmRequestFormat } from '../../lib/ipc/providers'; import { emitSaved } from '../../lib/savedEvent'; import { Btn } from '../_atoms'; -import { SettingRow, inputStyle } from './shared'; +import { SettingRow, Toggle, inputStyle } from './shared'; import { ProviderFormContext } from './ProviderForm'; -const accounts = [ +const protocolAccounts = [ 'ark.request_format', 'ark.messages_thinking', 'ark.max_tokens', 'ark.thinking_budget', ] as const; -type Account = (typeof accounts)[number]; +const serviceTierAccount = 'ark.service_tier' as const; +type Account = (typeof protocolAccounts)[number] | typeof serviceTierAccount; export type ProtocolValues = Record; const emptyValues: ProtocolValues = { 'ark.request_format': '', 'ark.messages_thinking': '', 'ark.max_tokens': '', 'ark.thinking_budget': '', + 'ark.service_tier': '', }; /** Immediate UI hint; Core still validates the same rules for what is stored and sent. */ @@ -29,6 +31,9 @@ export function protocolValidationError( defaultFormat: LlmRequestFormat, ): string | null { const format = values['ark.request_format'] || defaultFormat; + if (values['ark.service_tier'] && values['ark.service_tier'] !== 'fast') { + return 'llmServiceTierInvalid'; + } if (!['chat_completions', 'responses', 'messages'].includes(format)) return 'llmRequestFormatInvalid'; const mode = values['ark.messages_thinking'] || 'adaptive'; @@ -51,6 +56,7 @@ export function protocolValidationError( export function LlmProtocolFields({ channelId, + providerType, defaultFormat, formats, onUserMutation, @@ -58,6 +64,7 @@ export function LlmProtocolFields({ onSaved, }: { channelId: string; + providerType: string; defaultFormat: LlmRequestFormat; formats: LlmRequestFormat[]; onUserMutation: () => void; @@ -71,6 +78,8 @@ export function LlmProtocolFields({ const [loaded, setLoaded] = useState(false); const [saving, setSaving] = useState(false); const [error, setError] = useState<'read' | 'save' | null>(null); + const arkServiceTier = providerType === 'ark'; + const accounts = arkServiceTier ? [...protocolAccounts, serviceTierAccount] : protocolAccounts; const mounted = useRef(true); const writing = useRef(false); const pendingWrite = useRef>(Promise.resolve(true)); @@ -83,9 +92,10 @@ export function LlmProtocolFields({ Promise.all(accounts.map((account) => readCredential(account, channelId))) .then((result) => { if (!mounted.current) return; - const next = Object.fromEntries( - accounts.map((account, index) => [account, result[index] ?? '']), - ) as ProtocolValues; + const next = { + ...emptyValues, + ...Object.fromEntries(accounts.map((account, index) => [account, result[index] ?? ''])), + } as ProtocolValues; setValues(next); setSaved(next); setLoaded(true); @@ -96,7 +106,7 @@ export function LlmProtocolFields({ return () => { mounted.current = false; }; - }, [channelId]); + }, [channelId, providerType]); useEffect(() => { onBlockedChange( @@ -192,6 +202,19 @@ export function LlmProtocolFields({ {t('settings.providers.responsesThinkingHint')}

)} + {arkServiceTier && ( + + change('ark.service_tier', enabled ? 'fast' : '', true)} + /> + + )} {format === 'messages' && ( <> diff --git a/openless-all/app/src/pages/settings/ProvidersSection.tsx b/openless-all/app/src/pages/settings/ProvidersSection.tsx index 15b589c60..6e893b382 100644 --- a/openless-all/app/src/pages/settings/ProvidersSection.tsx +++ b/openless-all/app/src/pages/settings/ProvidersSection.tsx @@ -319,6 +319,7 @@ export function ChannelCredentialFields({ {!!descriptor.supportedRequestFormats?.length && descriptor.defaultRequestFormat && (