From 74e8aa601df3c5937ceb026a09aa5b715af9a1b0 Mon Sep 17 00:00:00 2001 From: Luca Trisiello Date: Mon, 10 Aug 2026 17:56:30 +0200 Subject: [PATCH 1/5] chore(deps): patch four high-severity advisories in the dev toolchain MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit npm audit fix, no --force needed: vite 6.4.1 -> 6.4.3, plus transitive bumps to postcss, nanoid, and picomatch. All within the ranges package.json already declares, so only the lockfile moves. The one that mattered here is the Vite dev server's arbitrary file read via WebSocket (GHSA-p9ff-h696-f583) — this project's documented workflow runs that dev server. Co-Authored-By: Claude Opus 5 --- frontend/package-lock.json | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 1f459602..b74e0ce7 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -948,9 +948,9 @@ "license": "MIT" }, "node_modules/nanoid": { - "version": "3.3.11", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.11.tgz", - "integrity": "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w==", + "version": "3.3.18", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.18.tgz", + "integrity": "sha512-DTg4MJbGMWkfi6VZFdNt2/caMbQy4Ou+Op/hJQvGEWcnVfoA1QA+xzRKAzw9jD6+GVOOeYr/mIcuDSdug6F6+w==", "dev": true, "funding": [ { @@ -974,9 +974,9 @@ "license": "ISC" }, "node_modules/picomatch": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.3.tgz", - "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.5.tgz", + "integrity": "sha512-RvwwcruNjI1ncT5xRakeyS9Lf8lcItv34KD+aif+VH9kduAyfYBipGh12274xtenIPZ119/R9BdTBa8gAwSh0A==", "dev": true, "license": "MIT", "engines": { @@ -987,9 +987,9 @@ } }, "node_modules/postcss": { - "version": "8.5.8", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.8.tgz", - "integrity": "sha512-OW/rX8O/jXnm82Ey1k44pObPtdblfiuWnrd8X7GJ7emImCOstunGbXUpp7HdBrFQX6rJzn3sPT397Wp5aCwCHg==", + "version": "8.5.26", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.26.tgz", + "integrity": "sha512-u82N74LFzG8ca+dD8puPnplTXoGH4fTPpVGuIbt36G3qvNlkvfD0lEAZSxaly3KX8TS/L1A1gsCEmvKmBcVbkQ==", "dev": true, "funding": [ { @@ -1007,7 +1007,7 @@ ], "license": "MIT", "dependencies": { - "nanoid": "^3.3.11", + "nanoid": "^3.3.17", "picocolors": "^1.1.1", "source-map-js": "^1.2.1" }, @@ -1108,9 +1108,9 @@ } }, "node_modules/vite": { - "version": "6.4.1", - "resolved": "https://registry.npmjs.org/vite/-/vite-6.4.1.tgz", - "integrity": "sha512-+Oxm7q9hDoLMyJOYfUYBuHQo+dkAloi33apOPP56pzj+vsdJDzr+j1NISE5pyaAuKL4A3UD34qd0lx5+kfKp2g==", + "version": "6.4.3", + "resolved": "https://registry.npmjs.org/vite/-/vite-6.4.3.tgz", + "integrity": "sha512-NTKlcQjlAK7MlQoyb6LgaqHc8sso/pVyUJYWMws3jg21uTJw/LddqIFPcPqP6PzpgbIcZyKI85sFE4HBrQDA8A==", "dev": true, "license": "MIT", "dependencies": { From 61603c3be729a7775314600fef7a4aa26ce677c3 Mon Sep 17 00:00:00 2001 From: Luca Trisiello Date: Mon, 10 Aug 2026 17:57:06 +0200 Subject: [PATCH 2/5] fix: deliver the reply when TTS produces no audio MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Nine call sites sent JARVIS's reply only inside `if audio:`, so whenever Fish Audio was unconfigured, rate-limited, or failing, the reply was dropped with no trace in the UI. The startup greeting and every proactive notification — research complete, build finished, project connected — went straight to nothing. Each site now falls back to a "text" message, which the client can surface. Three of those sites also passed the raw `bytes` from synthesize_speech straight to send_json as `"data": audio`, without base64. Bytes are not JSON-serializable, so the send raised — and in two cases the exception was swallowed by a bare `except Exception: pass`. Those paths ("Fix Yourself" and the calendar/mail lookups) delivered neither audio nor text even with a working Fish key, and did so silently. Co-Authored-By: Claude Opus 5 --- server.py | 59 +++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 40 insertions(+), 19 deletions(-) diff --git a/server.py b/server.py index f08e7370..9908d2a6 100644 --- a/server.py +++ b/server.py @@ -900,11 +900,15 @@ async def _execute_research(target: str, ws=None): try: notify_text = f"Research is complete, sir. Report is open in your browser." audio = await synthesize_speech(notify_text) + await ws.send_json({"type": "status", "state": "speaking"}) if audio: - await ws.send_json({"type": "status", "state": "speaking"}) await ws.send_json({"type": "audio", "data": base64.b64encode(audio).decode(), "text": notify_text}) - await ws.send_json({"type": "status", "state": "idle"}) - log.info(f"JARVIS: {notify_text}") + else: + # No voice available — the reply still has to reach the client, + # which shows it on screen. + await ws.send_json({"type": "text", "text": notify_text}) + await ws.send_json({"type": "status", "state": "idle"}) + log.info(f"JARVIS: {notify_text}") except Exception: pass # WebSocket might be gone @@ -915,6 +919,8 @@ async def _execute_research(target: str, ws=None): audio = await synthesize_speech("Research timed out, sir. It was taking too long.") if audio: await ws.send_json({"type": "audio", "data": base64.b64encode(audio).decode(), "text": "Research timed out, sir."}) + else: + await ws.send_json({"type": "text", "text": "Research timed out, sir."}) except Exception: pass except Exception as e: @@ -982,10 +988,13 @@ async def _execute_prompt_project(project_name: str, prompt: str, work_session: if not project_dir: msg = f"Couldn't find the {project_name} project directory, sir." audio = await synthesize_speech(msg) - if audio and ws: + if ws: try: await ws.send_json({"type": "status", "state": "speaking"}) - await ws.send_json({"type": "audio", "data": base64.b64encode(audio).decode(), "text": msg}) + if audio: + await ws.send_json({"type": "audio", "data": base64.b64encode(audio).decode(), "text": msg}) + else: + await ws.send_json({"type": "text", "text": msg}) except Exception: pass return @@ -1078,9 +1087,12 @@ async def _execute_prompt_project(project_name: str, prompt: str, work_session: try: msg = f"Had trouble connecting to {project_name}, sir." audio = await synthesize_speech(msg) - if audio and ws: + if ws: await ws.send_json({"type": "status", "state": "speaking"}) - await ws.send_json({"type": "audio", "data": base64.b64encode(audio).decode(), "text": msg}) + if audio: + await ws.send_json({"type": "audio", "data": base64.b64encode(audio).decode(), "text": msg}) + else: + await ws.send_json({"type": "text", "text": msg}) except Exception: pass @@ -1106,11 +1118,13 @@ async def self_work_and_notify(session: WorkSession, prompt: str, ws): try: audio = await synthesize_speech(msg) + await ws.send_json({"type": "status", "state": "speaking"}) if audio: - await ws.send_json({"type": "status", "state": "speaking"}) await ws.send_json({"type": "audio", "data": base64.b64encode(audio).decode(), "text": msg}) - await ws.send_json({"type": "status", "state": "idle"}) - log.info(f"JARVIS: {msg}") + else: + await ws.send_json({"type": "text", "text": msg}) + await ws.send_json({"type": "status", "state": "idle"}) + log.info(f"JARVIS: {msg}") except Exception: pass except Exception as e: @@ -1698,7 +1712,7 @@ async def _lookup_and_report(lookup_type: str, lookup_fn, ws, history: list[dict try: await ws.send_json({"type": "status", "state": "speaking"}) if audio: - await ws.send_json({"type": "audio", "data": audio, "text": result_text}) + await ws.send_json({"type": "audio", "data": base64.b64encode(audio).decode(), "text": result_text}) else: await ws.send_json({"type": "text", "text": result_text}) await ws.send_json({"type": "status", "state": "idle"}) @@ -1718,7 +1732,9 @@ async def _lookup_and_report(lookup_type: str, lookup_fn, ws, history: list[dict audio = await synthesize_speech(fallback) await ws.send_json({"type": "status", "state": "speaking"}) if audio: - await ws.send_json({"type": "audio", "data": audio, "text": fallback}) + await ws.send_json({"type": "audio", "data": base64.b64encode(audio).decode(), "text": fallback}) + else: + await ws.send_json({"type": "text", "text": fallback}) await ws.send_json({"type": "status", "state": "idle"}) except Exception: pass @@ -1994,13 +2010,15 @@ async def voice_handler(ws: WebSocket): async def _send_greeting(): try: audio_bytes = await synthesize_speech(greeting) + await ws.send_json({"type": "status", "state": "speaking"}) if audio_bytes: encoded = base64.b64encode(audio_bytes).decode() - await ws.send_json({"type": "status", "state": "speaking"}) await ws.send_json({"type": "audio", "data": encoded, "text": greeting}) - history.append({"role": "assistant", "content": greeting}) - log.info(f"JARVIS: {greeting}") - await ws.send_json({"type": "status", "state": "idle"}) + else: + await ws.send_json({"type": "text", "text": greeting}) + history.append({"role": "assistant", "content": greeting}) + log.info(f"JARVIS: {greeting}") + await ws.send_json({"type": "status", "state": "idle"}) except Exception as e: log.warning(f"Greeting failed: {e}") @@ -2027,7 +2045,7 @@ async def _send_greeting(): await ws.send_json({"type": "status", "state": "speaking"}) audio = await synthesize_speech(tts) if audio: - await ws.send_json({"type": "audio", "data": audio, "text": response_text}) + await ws.send_json({"type": "audio", "data": base64.b64encode(audio).decode(), "text": response_text}) else: await ws.send_json({"type": "text", "text": response_text}) continue @@ -2358,10 +2376,13 @@ async def _read_and_report(search_term, _ws): else: msg = f"Couldn't find a note matching '{search_term}', sir." audio = await synthesize_speech(strip_markdown_for_tts(msg)) - if audio and _ws: + if _ws: try: await _ws.send_json({"type": "status", "state": "speaking"}) - await _ws.send_json({"type": "audio", "data": base64.b64encode(audio).decode(), "text": msg}) + if audio: + await _ws.send_json({"type": "audio", "data": base64.b64encode(audio).decode(), "text": msg}) + else: + await _ws.send_json({"type": "text", "text": msg}) except Exception: pass asyncio.create_task(_read_and_report(embedded_action["target"].strip(), ws)) From 9eb2ad6d7e29f56eab029edaa627429700010c58 Mon Sep 17 00:00:00 2001 From: Luca Trisiello Date: Mon, 10 Aug 2026 17:57:18 +0200 Subject: [PATCH 3/5] feat: show JARVIS's replies on screen MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The client's stated "text fallback when TTS fails" only ran console.log, so a reply that arrived without audio reached the devtools console and nowhere else. With no Fish Audio key the interface was not merely silent but blank — the orb reacted to nothing and said nothing. Add a caption below the orb, in the existing palette. It is shown whether or not audio arrived, so it doubles as subtitles for a working voice. It holds for 4s plus 60ms per character, capped at 20s. Reading time rather than a fixed delay, because without a voice the caption is the entire response and vanishing mid-sentence loses it. Co-Authored-By: Claude Opus 5 --- frontend/index.html | 1 + frontend/src/main.ts | 25 +++++++++++++++++++++++++ frontend/src/style.css | 21 +++++++++++++++++++++ 3 files changed, 47 insertions(+) diff --git a/frontend/index.html b/frontend/index.html index 9440bda4..2d6aba1b 100644 --- a/frontend/index.html +++ b/frontend/index.html @@ -32,6 +32,7 @@ +
JARVIS
diff --git a/frontend/src/main.ts b/frontend/src/main.ts index ca5d1864..b611747d 100644 --- a/frontend/src/main.ts +++ b/frontend/src/main.ts @@ -21,6 +21,7 @@ let isMuted = false; const statusEl = document.getElementById("status-text")!; const errorEl = document.getElementById("error-text")!; +const captionEl = document.getElementById("jarvis-caption")!; function showError(msg: string) { errorEl.textContent = msg; @@ -30,6 +31,28 @@ function showError(msg: string) { }, 5000); } +let captionTimer: number | undefined; + +/** + * Put what JARVIS said on screen. + * + * Shown whether or not TTS produced audio: with a voice it reads as + * subtitles, and without one it is the only way to see the reply — the + * response used to reach nothing but the devtools console. + */ +function showCaption(text: unknown) { + if (typeof text !== "string" || !text.trim()) return; + clearTimeout(captionTimer); + captionEl.textContent = text; + captionEl.style.opacity = "1"; + // Hold long enough to read the line, since a caption may be all the user + // gets. Replies are a sentence or two, so length is a fair proxy. + const holdMs = Math.min(20000, 4000 + text.length * 60); + captionTimer = window.setTimeout(() => { + captionEl.style.opacity = "0"; + }, holdMs); +} + function updateStatus(state: State) { const labels: Record = { idle: "", @@ -123,6 +146,7 @@ socket.onMessage((msg) => { } // Log text for debugging if (msg.text) console.log("[JARVIS]", msg.text); + showCaption(msg.text); } else if (type === "status") { const state = msg.state as string; if (state === "thinking" && currentState !== "thinking") { @@ -137,6 +161,7 @@ socket.onMessage((msg) => { } else if (type === "text") { // Text fallback when TTS fails console.log("[JARVIS]", msg.text); + showCaption(msg.text); } else if (type === "task_spawned") { console.log("[task]", "spawned:", msg.task_id, msg.prompt); } else if (type === "task_complete") { diff --git a/frontend/src/style.css b/frontend/src/style.css index 899901c7..97380461 100644 --- a/frontend/src/style.css +++ b/frontend/src/style.css @@ -35,6 +35,27 @@ html, body { z-index: 10; } +/* What JARVIS just said. Sits above the status line, and carries the + conversation on its own when TTS is unavailable. */ +#jarvis-caption { + position: fixed; + bottom: 80px; + left: 50%; + transform: translateX(-50%); + width: min(640px, calc(100% - 48px)); + color: rgba(219, 238, 252, 0.92); + font-size: 17px; + line-height: 1.55; + font-weight: 300; + letter-spacing: 0.3px; + text-align: center; + text-shadow: 0 2px 12px rgba(0, 0, 0, 0.85); + transition: opacity 0.45s ease; + pointer-events: none; + z-index: 10; + opacity: 0; +} + #jarvis-label { position: fixed; bottom: 16px; From e35537394c2713ce504db22f140805ac5d64cd7f Mon Sep 17 00:00:00 2001 From: Luca Trisiello Date: Mon, 10 Aug 2026 17:57:33 +0200 Subject: [PATCH 4/5] feat: make the spoken language configurable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit recognition.lang was hardcoded to en-US, so anyone speaking another language was transcribed phonetically into English nonsense — "come stai" arrived as "comic Style". Add a Spoken Language selector to the settings panel, persisted as SPEECH_LANG and read live, so the choice applies without a restart. voice.ts rebuilds the recognition object on a language change rather than reassigning .lang. Chrome reads that property at construction and ignores it on a session that has already run, so the reassignment silently kept the old language. The retired session's onend checks whether it has been superseded, or it would restart and race the new one for the microphone. main.ts resolves the language before opening the microphone, so the first session is built correctly and the fragile switch path is reserved for live changes from the panel. The system prompt is English and would answer an Italian question in English, so a non-English selection appends a language directive. Kept to a bare directive deliberately: wordings that also discussed action selection measurably weakened the language adherence they were added to enforce. Known limitation: action routing is less reliable outside English. On one repeated question, English chose the right action 4/4 while Italian misrouted to a screen capture 4/4. The 46 hardcoded English strings in server.py ("Right away, sir.") also stay English — translating them is an i18n project, not a setting. Co-Authored-By: Claude Opus 5 --- frontend/src/main.ts | 32 ++++++++++- frontend/src/settings.ts | 30 +++++++++- frontend/src/voice.ts | 115 ++++++++++++++++++++++++++++----------- server.py | 42 +++++++++++++- 4 files changed, 182 insertions(+), 37 deletions(-) diff --git a/frontend/src/main.ts b/frontend/src/main.ts index b611747d..4619a23d 100644 --- a/frontend/src/main.ts +++ b/frontend/src/main.ts @@ -8,7 +8,7 @@ import { createOrb, type OrbState } from "./orb"; import { createVoiceInput, createAudioPlayer } from "./voice"; import { createSocket } from "./ws"; -import { openSettings, checkFirstTimeSetup } from "./settings"; +import { openSettings, checkFirstTimeSetup, SPEECH_LANG_EVENT } from "./settings"; import "./style.css"; // --------------------------------------------------------------------------- @@ -173,12 +173,40 @@ socket.onMessage((msg) => { // Kick off // --------------------------------------------------------------------------- +/** + * Read the speech language chosen in the settings panel. + * + * Resolved before the microphone opens so the first session is built in the + * right language: switching a live session means tearing it down and racing + * for the microphone, which is worth avoiding on every page load. Bounded, so + * an unreachable server costs a moment rather than the microphone. + */ +async function fetchSpeechLanguage(): Promise { + try { + const res = await fetch("/api/settings/preferences", { + signal: AbortSignal.timeout(2000), + }); + const prefs = (await res.json()) as { speech_lang?: string }; + return prefs.speech_lang || null; + } catch { + return null; // Server not ready — keep the default language. + } +} + // Start listening after a brief delay for the orb to render -setTimeout(() => { +setTimeout(async () => { + const lang = await fetchSpeechLanguage(); + if (lang) voiceInput.setLanguage(lang); voiceInput.start(); transition("listening"); }, 1000); +// Saving the setting re-tunes the running session — no reload needed. +document.addEventListener(SPEECH_LANG_EVENT, (e) => { + const lang = (e as CustomEvent).detail; + if (lang) voiceInput.setLanguage(lang); +}); + // Resume AudioContext on ANY user interaction (browser autoplay policy) function ensureAudioContext() { const ctx = audioPlayer.getAnalyser().context as AudioContext; diff --git a/frontend/src/settings.ts b/frontend/src/settings.ts index 7e945ef7..4f4131b5 100644 --- a/frontend/src/settings.ts +++ b/frontend/src/settings.ts @@ -30,8 +30,16 @@ interface PreferencesResponse { user_name: string; honorific: string; calendar_accounts: string; + speech_lang: string; } +/** + * Event fired when the speech language changes, so the live recognition + * session picks it up without a reload. Used instead of importing the voice + * input directly, which would make settings and main circular. + */ +export const SPEECH_LANG_EVENT = "jarvis:speech-lang"; + // --------------------------------------------------------------------------- // State // --------------------------------------------------------------------------- @@ -143,6 +151,22 @@ function buildPanelHTML(): string { +
+ + +
+
@@ -242,9 +266,11 @@ async function loadPreferences() { const nameEl = document.getElementById("input-user-name") as HTMLInputElement; const honEl = document.getElementById("input-honorific") as HTMLSelectElement; const calEl = document.getElementById("input-calendar-accounts") as HTMLTextAreaElement; + const langEl = document.getElementById("input-speech-lang") as HTMLSelectElement; if (nameEl) nameEl.value = prefs.user_name || ""; if (honEl) honEl.value = prefs.honorific || "sir"; if (calEl) calEl.value = prefs.calendar_accounts || "auto"; + if (langEl) langEl.value = prefs.speech_lang || "en-US"; } catch (e) { console.error("[settings] failed to load preferences:", e); } @@ -306,7 +332,9 @@ function wireEvents() { const user_name = (document.getElementById("input-user-name") as HTMLInputElement).value.trim(); const honorific = (document.getElementById("input-honorific") as HTMLSelectElement).value; const calendar_accounts = (document.getElementById("input-calendar-accounts") as HTMLTextAreaElement).value.trim(); - await apiPost("/api/settings/preferences", { user_name, honorific, calendar_accounts }); + const speech_lang = (document.getElementById("input-speech-lang") as HTMLSelectElement).value; + await apiPost("/api/settings/preferences", { user_name, honorific, calendar_accounts, speech_lang }); + document.dispatchEvent(new CustomEvent(SPEECH_LANG_EVENT, { detail: speech_lang })); await loadStatus(); }); diff --git a/frontend/src/voice.ts b/frontend/src/voice.ts index 8ca5e0a8..158280e3 100644 --- a/frontend/src/voice.ts +++ b/frontend/src/voice.ts @@ -11,8 +11,12 @@ export interface VoiceInput { stop(): void; pause(): void; resume(): void; + setLanguage(lang: string): void; } +/** Fallback until the configured language arrives from the server. */ +export const DEFAULT_SPEECH_LANG = "en-US"; + // eslint-disable-next-line @typescript-eslint/no-explicit-any declare const webkitSpeechRecognition: any; @@ -24,48 +28,65 @@ export function createVoiceInput( const SR = (window as any).SpeechRecognition || (typeof webkitSpeechRecognition !== "undefined" ? webkitSpeechRecognition : null); if (!SR) { onError("Speech recognition not supported in this browser"); - return { start() {}, stop() {}, pause() {}, resume() {} }; + return { start() {}, stop() {}, pause() {}, resume() {}, setLanguage() {} }; } - const recognition = new SR(); - recognition.continuous = true; - recognition.interimResults = true; - recognition.lang = "en-US"; - let shouldListen = false; let paused = false; + let currentLang = DEFAULT_SPEECH_LANG; + + /** + * Build a recognition session. + * + * Chrome reads `lang` when the object is constructed and ignores later + * assignment on a session that has already run, so switching language means + * building a new object rather than re-tagging this one. + */ + function buildRecognition(): any { + const r = new SR(); + r.continuous = true; + r.interimResults = true; + r.lang = currentLang; + + r.onresult = (event: any) => { + for (let i = event.resultIndex; i < event.results.length; i++) { + if (event.results[i].isFinal) { + const text = event.results[i][0].transcript.trim(); + if (text) onTranscript(text); + } + } + }; - recognition.onresult = (event: any) => { - for (let i = event.resultIndex; i < event.results.length; i++) { - if (event.results[i].isFinal) { - const text = event.results[i][0].transcript.trim(); - if (text) onTranscript(text); + r.onend = () => { + // A session replaced by a language switch must stay down, or it would + // race the new one for the microphone and keep the old language alive. + if (r !== recognition) return; + if (shouldListen && !paused) { + try { + r.start(); + } catch { + // Already started + } } - } - }; + }; - recognition.onend = () => { - if (shouldListen && !paused) { - try { - recognition.start(); - } catch { - // Already started + r.onerror = (event: any) => { + if (event.error === "not-allowed") { + onError("Microphone access denied. Please allow microphone access."); + shouldListen = false; + } else if (event.error === "no-speech") { + // Normal, just restart + } else if (event.error === "aborted") { + // Expected during pause + } else { + console.warn("[voice] recognition error:", event.error); } - } - }; + }; - recognition.onerror = (event: any) => { - if (event.error === "not-allowed") { - onError("Microphone access denied. Please allow microphone access."); - shouldListen = false; - } else if (event.error === "no-speech") { - // Normal, just restart - } else if (event.error === "aborted") { - // Expected during pause - } else { - console.warn("[voice] recognition error:", event.error); - } - }; + return r; + } + + let recognition = buildRecognition(); return { start() { @@ -96,6 +117,34 @@ export function createVoiceInput( } } }, + setLanguage(lang: string) { + if (!lang || lang === currentLang) return; + currentLang = lang; + + const previous = recognition; + recognition = buildRecognition(); + // Retires the old session: its onend now sees itself superseded. + try { + previous.stop(); + } catch { + // Wasn't running. + } + + if (shouldListen && !paused) { + // Let the retired session release the microphone before claiming it, + // otherwise Chrome rejects the new start outright. + setTimeout(() => { + if (recognition !== previous && shouldListen && !paused) { + try { + recognition.start(); + } catch { + // Already started + } + } + }, 250); + } + console.log("[voice] language set to", lang); + }, }; } diff --git a/server.py b/server.py index 9908d2a6..9bd975db 100644 --- a/server.py +++ b/server.py @@ -65,6 +65,29 @@ FISH_VOICE_ID = os.getenv("FISH_VOICE_ID", "612b878b113047d9a770c069c8b4fdfe") # JARVIS (MCU) FISH_API_URL = "https://api.fish.audio/v1/tts" USER_NAME = os.getenv("USER_NAME", "sir") +# BCP-47 tag the browser's speech recognition listens in, and the language +# JARVIS is told to reply in. Read per-request rather than cached, so changing +# it in the settings panel takes effect without a restart. +DEFAULT_SPEECH_LANG = "en-US" +# Offered in the settings panel, mapped to the name used to tell JARVIS which +# language to answer in. +SPEECH_LANGUAGES = { + "en-US": "English", + "en-GB": "English", + "it-IT": "Italian", + "es-ES": "Spanish", + "fr-FR": "French", + "de-DE": "German", + "pt-BR": "Portuguese", + "nl-NL": "Dutch", + "ja-JP": "Japanese", + "zh-CN": "Chinese", +} + + +def _current_speech_lang() -> str: + """The configured speech language, read live so the panel needs no restart.""" + return os.getenv("SPEECH_LANG", DEFAULT_SPEECH_LANG) or DEFAULT_SPEECH_LANG PROJECT_DIR = os.path.dirname(os.path.abspath(__file__)) _SKIP_PERMISSIONS = os.getenv("JARVIS_SKIP_PERMISSIONS", "true").lower() not in ("0", "false", "no") @@ -1211,6 +1234,20 @@ async def generate_response( user_name=USER_NAME, project_dir=PROJECT_DIR, ) + # The prompt above is written in English and would answer an Italian + # question in English, which makes the speech-language setting only half + # work. Kept to a bare language directive on purpose: wording that also + # discussed action selection measurably weakened the language adherence it + # was appended to enforce. Action routing is less reliable outside English + # regardless — see the non-English caveat in the README. + speech_lang = _current_speech_lang() + if not speech_lang.startswith("en"): + language = SPEECH_LANGUAGES.get(speech_lang, speech_lang) + system += ( + f"\n\nLANGUAGE: The user speaks {language}. Write every spoken reply in {language}, " + f"keeping the same dry, economical butler voice." + ) + if lookup_status: system += f"\n\nACTIVE LOOKUPS:\n{lookup_status}\nIf asked about progress, report this status." @@ -2509,10 +2546,11 @@ class PreferencesUpdate(BaseModel): user_name: str = "" honorific: str = "sir" calendar_accounts: str = "auto" + speech_lang: str = DEFAULT_SPEECH_LANG @app.post("/api/settings/keys") async def api_settings_keys(body: KeyUpdate): - allowed = {"ANTHROPIC_API_KEY", "FISH_API_KEY", "FISH_VOICE_ID", "USER_NAME", "HONORIFIC", "CALENDAR_ACCOUNTS"} + allowed = {"ANTHROPIC_API_KEY", "FISH_API_KEY", "FISH_VOICE_ID", "USER_NAME", "HONORIFIC", "CALENDAR_ACCOUNTS", "SPEECH_LANG"} if body.key_name not in allowed: return JSONResponse({"success": False, "error": "Invalid key name"}, status_code=400) _write_env_key(body.key_name, body.key_value) @@ -2592,6 +2630,7 @@ async def api_get_preferences(): "user_name": env_dict.get("USER_NAME", ""), "honorific": env_dict.get("HONORIFIC", "sir"), "calendar_accounts": env_dict.get("CALENDAR_ACCOUNTS", "auto"), + "speech_lang": env_dict.get("SPEECH_LANG", DEFAULT_SPEECH_LANG) or DEFAULT_SPEECH_LANG, } @app.post("/api/settings/preferences") @@ -2599,6 +2638,7 @@ async def api_save_preferences(body: PreferencesUpdate): _write_env_key("USER_NAME", body.user_name) _write_env_key("HONORIFIC", body.honorific) _write_env_key("CALENDAR_ACCOUNTS", body.calendar_accounts) + _write_env_key("SPEECH_LANG", body.speech_lang or DEFAULT_SPEECH_LANG) return {"success": True} # --------------------------------------------------------------------------- From 6e3488e4dfbc1a006d3ecd38869f74b3b70c8af3 Mon Sep 17 00:00:00 2001 From: Luca Trisiello Date: Mon, 10 Aug 2026 17:57:50 +0200 Subject: [PATCH 5/5] feat: accumulate speech fragments into one utterance Chrome closes a recognition segment at every pause, and the client sent each "final" result the instant it arrived. A sentence spoken with any hesitation reached JARVIS as several questions: he answered half a thought, then received the rest as a new one. The logs show single words like "hey" arriving alone. Collect fragments and send once the speaker has actually stopped, after a 1s gap. Two edges that would otherwise bite: - A 6s ceiling. Steady dictation never produces the gap, so without a cap the timer re-arms forever and the reply never comes. - Muting discards what is pending, so half a sentence cannot arrive a second after the user silenced him. Barge-in stays immediate: the audio is cut on the first fragment, not at flush, since interrupting is what the user wanted the moment they spoke. Costs 1s before JARVIS starts thinking. That is the price of knowing the sentence ended; UTTERANCE_GAP_MS trades it back for more split sentences. Co-Authored-By: Claude Opus 5 --- frontend/src/main.ts | 56 ++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 52 insertions(+), 4 deletions(-) diff --git a/frontend/src/main.ts b/frontend/src/main.ts index 4619a23d..e6b08f2d 100644 --- a/frontend/src/main.ts +++ b/frontend/src/main.ts @@ -103,13 +103,59 @@ function transition(newState: State) { // Voice input // --------------------------------------------------------------------------- +/** + * Fragment accumulation. + * + * Chrome closes a recognition segment at every pause, so a sentence spoken + * with any hesitation arrives as several "final" results. Sending each one + * immediately made JARVIS answer half a thought and then receive the rest as + * a separate question. Instead, collect the pieces and send once the speaker + * has actually stopped. + */ +const UTTERANCE_GAP_MS = 1000; +// A pause never arrives while someone is dictating steadily, so cap the wait +// rather than let a long sentence hold the whole conversation open. +const UTTERANCE_MAX_MS = 6000; + +let pendingFragments: string[] = []; +let gapTimer: number | undefined; +let maxTimer: number | undefined; + +function discardUtterance() { + clearTimeout(gapTimer); + clearTimeout(maxTimer); + gapTimer = undefined; + maxTimer = undefined; + pendingFragments = []; +} + +function flushUtterance() { + clearTimeout(gapTimer); + clearTimeout(maxTimer); + gapTimer = undefined; + maxTimer = undefined; + + const text = pendingFragments.join(" ").replace(/\s+/g, " ").trim(); + pendingFragments = []; + if (!text) return; + + socket.send({ type: "transcript", text, isFinal: true }); + transition("thinking"); +} + const voiceInput = createVoiceInput( (text: string) => { - // Cancel any current JARVIS response before sending new input + // Cancel any current JARVIS response before sending new input. Done on the + // first fragment, not at flush time: cutting him off is what the user + // wanted the moment they started talking. audioPlayer.stop(); - // User spoke — send transcript - socket.send({ type: "transcript", text, isFinal: true }); - transition("thinking"); + + pendingFragments.push(text); + clearTimeout(gapTimer); + gapTimer = window.setTimeout(flushUtterance, UTTERANCE_GAP_MS); + if (maxTimer === undefined) { + maxTimer = window.setTimeout(flushUtterance, UTTERANCE_MAX_MS); + } }, (msg: string) => { showError(msg); @@ -236,6 +282,8 @@ btnMute.addEventListener("click", (e) => { isMuted = !isMuted; btnMute.classList.toggle("muted", isMuted); if (isMuted) { + // Half-spoken words must not arrive a second after the user silenced him. + discardUtterance(); voiceInput.pause(); transition("idle"); } else {