From 74e8aa601df3c5937ceb026a09aa5b715af9a1b0 Mon Sep 17 00:00:00 2001 From: Luca Trisiello Date: Mon, 10 Aug 2026 17:56:30 +0200 Subject: [PATCH 01/10] chore(deps): patch four high-severity advisories in the dev toolchain MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit npm audit fix, no --force needed: vite 6.4.1 -> 6.4.3, plus transitive bumps to postcss, nanoid, and picomatch. All within the ranges package.json already declares, so only the lockfile moves. The one that mattered here is the Vite dev server's arbitrary file read via WebSocket (GHSA-p9ff-h696-f583) — this project's documented workflow runs that dev server. Co-Authored-By: Claude Opus 5 --- frontend/package-lock.json | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 1f459602..b74e0ce7 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -948,9 +948,9 @@ "license": "MIT" }, "node_modules/nanoid": { - "version": "3.3.11", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.11.tgz", - "integrity": "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w==", + "version": "3.3.18", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.18.tgz", + "integrity": "sha512-DTg4MJbGMWkfi6VZFdNt2/caMbQy4Ou+Op/hJQvGEWcnVfoA1QA+xzRKAzw9jD6+GVOOeYr/mIcuDSdug6F6+w==", "dev": true, "funding": [ { @@ -974,9 +974,9 @@ "license": "ISC" }, "node_modules/picomatch": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.3.tgz", - "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.5.tgz", + "integrity": "sha512-RvwwcruNjI1ncT5xRakeyS9Lf8lcItv34KD+aif+VH9kduAyfYBipGh12274xtenIPZ119/R9BdTBa8gAwSh0A==", "dev": true, "license": "MIT", "engines": { @@ -987,9 +987,9 @@ } }, "node_modules/postcss": { - "version": "8.5.8", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.8.tgz", - "integrity": "sha512-OW/rX8O/jXnm82Ey1k44pObPtdblfiuWnrd8X7GJ7emImCOstunGbXUpp7HdBrFQX6rJzn3sPT397Wp5aCwCHg==", + "version": "8.5.26", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.26.tgz", + "integrity": "sha512-u82N74LFzG8ca+dD8puPnplTXoGH4fTPpVGuIbt36G3qvNlkvfD0lEAZSxaly3KX8TS/L1A1gsCEmvKmBcVbkQ==", "dev": true, "funding": [ { @@ -1007,7 +1007,7 @@ ], "license": "MIT", "dependencies": { - "nanoid": "^3.3.11", + "nanoid": "^3.3.17", "picocolors": "^1.1.1", "source-map-js": "^1.2.1" }, @@ -1108,9 +1108,9 @@ } }, "node_modules/vite": { - "version": "6.4.1", - "resolved": "https://registry.npmjs.org/vite/-/vite-6.4.1.tgz", - "integrity": "sha512-+Oxm7q9hDoLMyJOYfUYBuHQo+dkAloi33apOPP56pzj+vsdJDzr+j1NISE5pyaAuKL4A3UD34qd0lx5+kfKp2g==", + "version": "6.4.3", + "resolved": "https://registry.npmjs.org/vite/-/vite-6.4.3.tgz", + "integrity": "sha512-NTKlcQjlAK7MlQoyb6LgaqHc8sso/pVyUJYWMws3jg21uTJw/LddqIFPcPqP6PzpgbIcZyKI85sFE4HBrQDA8A==", "dev": true, "license": "MIT", "dependencies": { From 61603c3be729a7775314600fef7a4aa26ce677c3 Mon Sep 17 00:00:00 2001 From: Luca Trisiello Date: Mon, 10 Aug 2026 17:57:06 +0200 Subject: [PATCH 02/10] fix: deliver the reply when TTS produces no audio MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Nine call sites sent JARVIS's reply only inside `if audio:`, so whenever Fish Audio was unconfigured, rate-limited, or failing, the reply was dropped with no trace in the UI. The startup greeting and every proactive notification — research complete, build finished, project connected — went straight to nothing. Each site now falls back to a "text" message, which the client can surface. Three of those sites also passed the raw `bytes` from synthesize_speech straight to send_json as `"data": audio`, without base64. Bytes are not JSON-serializable, so the send raised — and in two cases the exception was swallowed by a bare `except Exception: pass`. Those paths ("Fix Yourself" and the calendar/mail lookups) delivered neither audio nor text even with a working Fish key, and did so silently. Co-Authored-By: Claude Opus 5 --- server.py | 59 +++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 40 insertions(+), 19 deletions(-) diff --git a/server.py b/server.py index f08e7370..9908d2a6 100644 --- a/server.py +++ b/server.py @@ -900,11 +900,15 @@ async def _execute_research(target: str, ws=None): try: notify_text = f"Research is complete, sir. Report is open in your browser." audio = await synthesize_speech(notify_text) + await ws.send_json({"type": "status", "state": "speaking"}) if audio: - await ws.send_json({"type": "status", "state": "speaking"}) await ws.send_json({"type": "audio", "data": base64.b64encode(audio).decode(), "text": notify_text}) - await ws.send_json({"type": "status", "state": "idle"}) - log.info(f"JARVIS: {notify_text}") + else: + # No voice available — the reply still has to reach the client, + # which shows it on screen. + await ws.send_json({"type": "text", "text": notify_text}) + await ws.send_json({"type": "status", "state": "idle"}) + log.info(f"JARVIS: {notify_text}") except Exception: pass # WebSocket might be gone @@ -915,6 +919,8 @@ async def _execute_research(target: str, ws=None): audio = await synthesize_speech("Research timed out, sir. It was taking too long.") if audio: await ws.send_json({"type": "audio", "data": base64.b64encode(audio).decode(), "text": "Research timed out, sir."}) + else: + await ws.send_json({"type": "text", "text": "Research timed out, sir."}) except Exception: pass except Exception as e: @@ -982,10 +988,13 @@ async def _execute_prompt_project(project_name: str, prompt: str, work_session: if not project_dir: msg = f"Couldn't find the {project_name} project directory, sir." audio = await synthesize_speech(msg) - if audio and ws: + if ws: try: await ws.send_json({"type": "status", "state": "speaking"}) - await ws.send_json({"type": "audio", "data": base64.b64encode(audio).decode(), "text": msg}) + if audio: + await ws.send_json({"type": "audio", "data": base64.b64encode(audio).decode(), "text": msg}) + else: + await ws.send_json({"type": "text", "text": msg}) except Exception: pass return @@ -1078,9 +1087,12 @@ async def _execute_prompt_project(project_name: str, prompt: str, work_session: try: msg = f"Had trouble connecting to {project_name}, sir." audio = await synthesize_speech(msg) - if audio and ws: + if ws: await ws.send_json({"type": "status", "state": "speaking"}) - await ws.send_json({"type": "audio", "data": base64.b64encode(audio).decode(), "text": msg}) + if audio: + await ws.send_json({"type": "audio", "data": base64.b64encode(audio).decode(), "text": msg}) + else: + await ws.send_json({"type": "text", "text": msg}) except Exception: pass @@ -1106,11 +1118,13 @@ async def self_work_and_notify(session: WorkSession, prompt: str, ws): try: audio = await synthesize_speech(msg) + await ws.send_json({"type": "status", "state": "speaking"}) if audio: - await ws.send_json({"type": "status", "state": "speaking"}) await ws.send_json({"type": "audio", "data": base64.b64encode(audio).decode(), "text": msg}) - await ws.send_json({"type": "status", "state": "idle"}) - log.info(f"JARVIS: {msg}") + else: + await ws.send_json({"type": "text", "text": msg}) + await ws.send_json({"type": "status", "state": "idle"}) + log.info(f"JARVIS: {msg}") except Exception: pass except Exception as e: @@ -1698,7 +1712,7 @@ async def _lookup_and_report(lookup_type: str, lookup_fn, ws, history: list[dict try: await ws.send_json({"type": "status", "state": "speaking"}) if audio: - await ws.send_json({"type": "audio", "data": audio, "text": result_text}) + await ws.send_json({"type": "audio", "data": base64.b64encode(audio).decode(), "text": result_text}) else: await ws.send_json({"type": "text", "text": result_text}) await ws.send_json({"type": "status", "state": "idle"}) @@ -1718,7 +1732,9 @@ async def _lookup_and_report(lookup_type: str, lookup_fn, ws, history: list[dict audio = await synthesize_speech(fallback) await ws.send_json({"type": "status", "state": "speaking"}) if audio: - await ws.send_json({"type": "audio", "data": audio, "text": fallback}) + await ws.send_json({"type": "audio", "data": base64.b64encode(audio).decode(), "text": fallback}) + else: + await ws.send_json({"type": "text", "text": fallback}) await ws.send_json({"type": "status", "state": "idle"}) except Exception: pass @@ -1994,13 +2010,15 @@ async def voice_handler(ws: WebSocket): async def _send_greeting(): try: audio_bytes = await synthesize_speech(greeting) + await ws.send_json({"type": "status", "state": "speaking"}) if audio_bytes: encoded = base64.b64encode(audio_bytes).decode() - await ws.send_json({"type": "status", "state": "speaking"}) await ws.send_json({"type": "audio", "data": encoded, "text": greeting}) - history.append({"role": "assistant", "content": greeting}) - log.info(f"JARVIS: {greeting}") - await ws.send_json({"type": "status", "state": "idle"}) + else: + await ws.send_json({"type": "text", "text": greeting}) + history.append({"role": "assistant", "content": greeting}) + log.info(f"JARVIS: {greeting}") + await ws.send_json({"type": "status", "state": "idle"}) except Exception as e: log.warning(f"Greeting failed: {e}") @@ -2027,7 +2045,7 @@ async def _send_greeting(): await ws.send_json({"type": "status", "state": "speaking"}) audio = await synthesize_speech(tts) if audio: - await ws.send_json({"type": "audio", "data": audio, "text": response_text}) + await ws.send_json({"type": "audio", "data": base64.b64encode(audio).decode(), "text": response_text}) else: await ws.send_json({"type": "text", "text": response_text}) continue @@ -2358,10 +2376,13 @@ async def _read_and_report(search_term, _ws): else: msg = f"Couldn't find a note matching '{search_term}', sir." audio = await synthesize_speech(strip_markdown_for_tts(msg)) - if audio and _ws: + if _ws: try: await _ws.send_json({"type": "status", "state": "speaking"}) - await _ws.send_json({"type": "audio", "data": base64.b64encode(audio).decode(), "text": msg}) + if audio: + await _ws.send_json({"type": "audio", "data": base64.b64encode(audio).decode(), "text": msg}) + else: + await _ws.send_json({"type": "text", "text": msg}) except Exception: pass asyncio.create_task(_read_and_report(embedded_action["target"].strip(), ws)) From 9eb2ad6d7e29f56eab029edaa627429700010c58 Mon Sep 17 00:00:00 2001 From: Luca Trisiello Date: Mon, 10 Aug 2026 17:57:18 +0200 Subject: [PATCH 03/10] feat: show JARVIS's replies on screen MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The client's stated "text fallback when TTS fails" only ran console.log, so a reply that arrived without audio reached the devtools console and nowhere else. With no Fish Audio key the interface was not merely silent but blank — the orb reacted to nothing and said nothing. Add a caption below the orb, in the existing palette. It is shown whether or not audio arrived, so it doubles as subtitles for a working voice. It holds for 4s plus 60ms per character, capped at 20s. Reading time rather than a fixed delay, because without a voice the caption is the entire response and vanishing mid-sentence loses it. Co-Authored-By: Claude Opus 5 --- frontend/index.html | 1 + frontend/src/main.ts | 25 +++++++++++++++++++++++++ frontend/src/style.css | 21 +++++++++++++++++++++ 3 files changed, 47 insertions(+) diff --git a/frontend/index.html b/frontend/index.html index 9440bda4..2d6aba1b 100644 --- a/frontend/index.html +++ b/frontend/index.html @@ -32,6 +32,7 @@ +
JARVIS
diff --git a/frontend/src/main.ts b/frontend/src/main.ts index ca5d1864..b611747d 100644 --- a/frontend/src/main.ts +++ b/frontend/src/main.ts @@ -21,6 +21,7 @@ let isMuted = false; const statusEl = document.getElementById("status-text")!; const errorEl = document.getElementById("error-text")!; +const captionEl = document.getElementById("jarvis-caption")!; function showError(msg: string) { errorEl.textContent = msg; @@ -30,6 +31,28 @@ function showError(msg: string) { }, 5000); } +let captionTimer: number | undefined; + +/** + * Put what JARVIS said on screen. + * + * Shown whether or not TTS produced audio: with a voice it reads as + * subtitles, and without one it is the only way to see the reply — the + * response used to reach nothing but the devtools console. + */ +function showCaption(text: unknown) { + if (typeof text !== "string" || !text.trim()) return; + clearTimeout(captionTimer); + captionEl.textContent = text; + captionEl.style.opacity = "1"; + // Hold long enough to read the line, since a caption may be all the user + // gets. Replies are a sentence or two, so length is a fair proxy. + const holdMs = Math.min(20000, 4000 + text.length * 60); + captionTimer = window.setTimeout(() => { + captionEl.style.opacity = "0"; + }, holdMs); +} + function updateStatus(state: State) { const labels: Record = { idle: "", @@ -123,6 +146,7 @@ socket.onMessage((msg) => { } // Log text for debugging if (msg.text) console.log("[JARVIS]", msg.text); + showCaption(msg.text); } else if (type === "status") { const state = msg.state as string; if (state === "thinking" && currentState !== "thinking") { @@ -137,6 +161,7 @@ socket.onMessage((msg) => { } else if (type === "text") { // Text fallback when TTS fails console.log("[JARVIS]", msg.text); + showCaption(msg.text); } else if (type === "task_spawned") { console.log("[task]", "spawned:", msg.task_id, msg.prompt); } else if (type === "task_complete") { diff --git a/frontend/src/style.css b/frontend/src/style.css index 899901c7..97380461 100644 --- a/frontend/src/style.css +++ b/frontend/src/style.css @@ -35,6 +35,27 @@ html, body { z-index: 10; } +/* What JARVIS just said. Sits above the status line, and carries the + conversation on its own when TTS is unavailable. */ +#jarvis-caption { + position: fixed; + bottom: 80px; + left: 50%; + transform: translateX(-50%); + width: min(640px, calc(100% - 48px)); + color: rgba(219, 238, 252, 0.92); + font-size: 17px; + line-height: 1.55; + font-weight: 300; + letter-spacing: 0.3px; + text-align: center; + text-shadow: 0 2px 12px rgba(0, 0, 0, 0.85); + transition: opacity 0.45s ease; + pointer-events: none; + z-index: 10; + opacity: 0; +} + #jarvis-label { position: fixed; bottom: 16px; From e35537394c2713ce504db22f140805ac5d64cd7f Mon Sep 17 00:00:00 2001 From: Luca Trisiello Date: Mon, 10 Aug 2026 17:57:33 +0200 Subject: [PATCH 04/10] feat: make the spoken language configurable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit recognition.lang was hardcoded to en-US, so anyone speaking another language was transcribed phonetically into English nonsense — "come stai" arrived as "comic Style". Add a Spoken Language selector to the settings panel, persisted as SPEECH_LANG and read live, so the choice applies without a restart. voice.ts rebuilds the recognition object on a language change rather than reassigning .lang. Chrome reads that property at construction and ignores it on a session that has already run, so the reassignment silently kept the old language. The retired session's onend checks whether it has been superseded, or it would restart and race the new one for the microphone. main.ts resolves the language before opening the microphone, so the first session is built correctly and the fragile switch path is reserved for live changes from the panel. The system prompt is English and would answer an Italian question in English, so a non-English selection appends a language directive. Kept to a bare directive deliberately: wordings that also discussed action selection measurably weakened the language adherence they were added to enforce. Known limitation: action routing is less reliable outside English. On one repeated question, English chose the right action 4/4 while Italian misrouted to a screen capture 4/4. The 46 hardcoded English strings in server.py ("Right away, sir.") also stay English — translating them is an i18n project, not a setting. Co-Authored-By: Claude Opus 5 --- frontend/src/main.ts | 32 ++++++++++- frontend/src/settings.ts | 30 +++++++++- frontend/src/voice.ts | 115 ++++++++++++++++++++++++++++----------- server.py | 42 +++++++++++++- 4 files changed, 182 insertions(+), 37 deletions(-) diff --git a/frontend/src/main.ts b/frontend/src/main.ts index b611747d..4619a23d 100644 --- a/frontend/src/main.ts +++ b/frontend/src/main.ts @@ -8,7 +8,7 @@ import { createOrb, type OrbState } from "./orb"; import { createVoiceInput, createAudioPlayer } from "./voice"; import { createSocket } from "./ws"; -import { openSettings, checkFirstTimeSetup } from "./settings"; +import { openSettings, checkFirstTimeSetup, SPEECH_LANG_EVENT } from "./settings"; import "./style.css"; // --------------------------------------------------------------------------- @@ -173,12 +173,40 @@ socket.onMessage((msg) => { // Kick off // --------------------------------------------------------------------------- +/** + * Read the speech language chosen in the settings panel. + * + * Resolved before the microphone opens so the first session is built in the + * right language: switching a live session means tearing it down and racing + * for the microphone, which is worth avoiding on every page load. Bounded, so + * an unreachable server costs a moment rather than the microphone. + */ +async function fetchSpeechLanguage(): Promise { + try { + const res = await fetch("/api/settings/preferences", { + signal: AbortSignal.timeout(2000), + }); + const prefs = (await res.json()) as { speech_lang?: string }; + return prefs.speech_lang || null; + } catch { + return null; // Server not ready — keep the default language. + } +} + // Start listening after a brief delay for the orb to render -setTimeout(() => { +setTimeout(async () => { + const lang = await fetchSpeechLanguage(); + if (lang) voiceInput.setLanguage(lang); voiceInput.start(); transition("listening"); }, 1000); +// Saving the setting re-tunes the running session — no reload needed. +document.addEventListener(SPEECH_LANG_EVENT, (e) => { + const lang = (e as CustomEvent).detail; + if (lang) voiceInput.setLanguage(lang); +}); + // Resume AudioContext on ANY user interaction (browser autoplay policy) function ensureAudioContext() { const ctx = audioPlayer.getAnalyser().context as AudioContext; diff --git a/frontend/src/settings.ts b/frontend/src/settings.ts index 7e945ef7..4f4131b5 100644 --- a/frontend/src/settings.ts +++ b/frontend/src/settings.ts @@ -30,8 +30,16 @@ interface PreferencesResponse { user_name: string; honorific: string; calendar_accounts: string; + speech_lang: string; } +/** + * Event fired when the speech language changes, so the live recognition + * session picks it up without a reload. Used instead of importing the voice + * input directly, which would make settings and main circular. + */ +export const SPEECH_LANG_EVENT = "jarvis:speech-lang"; + // --------------------------------------------------------------------------- // State // --------------------------------------------------------------------------- @@ -143,6 +151,22 @@ function buildPanelHTML(): string { +
+ + +
+
@@ -242,9 +266,11 @@ async function loadPreferences() { const nameEl = document.getElementById("input-user-name") as HTMLInputElement; const honEl = document.getElementById("input-honorific") as HTMLSelectElement; const calEl = document.getElementById("input-calendar-accounts") as HTMLTextAreaElement; + const langEl = document.getElementById("input-speech-lang") as HTMLSelectElement; if (nameEl) nameEl.value = prefs.user_name || ""; if (honEl) honEl.value = prefs.honorific || "sir"; if (calEl) calEl.value = prefs.calendar_accounts || "auto"; + if (langEl) langEl.value = prefs.speech_lang || "en-US"; } catch (e) { console.error("[settings] failed to load preferences:", e); } @@ -306,7 +332,9 @@ function wireEvents() { const user_name = (document.getElementById("input-user-name") as HTMLInputElement).value.trim(); const honorific = (document.getElementById("input-honorific") as HTMLSelectElement).value; const calendar_accounts = (document.getElementById("input-calendar-accounts") as HTMLTextAreaElement).value.trim(); - await apiPost("/api/settings/preferences", { user_name, honorific, calendar_accounts }); + const speech_lang = (document.getElementById("input-speech-lang") as HTMLSelectElement).value; + await apiPost("/api/settings/preferences", { user_name, honorific, calendar_accounts, speech_lang }); + document.dispatchEvent(new CustomEvent(SPEECH_LANG_EVENT, { detail: speech_lang })); await loadStatus(); }); diff --git a/frontend/src/voice.ts b/frontend/src/voice.ts index 8ca5e0a8..158280e3 100644 --- a/frontend/src/voice.ts +++ b/frontend/src/voice.ts @@ -11,8 +11,12 @@ export interface VoiceInput { stop(): void; pause(): void; resume(): void; + setLanguage(lang: string): void; } +/** Fallback until the configured language arrives from the server. */ +export const DEFAULT_SPEECH_LANG = "en-US"; + // eslint-disable-next-line @typescript-eslint/no-explicit-any declare const webkitSpeechRecognition: any; @@ -24,48 +28,65 @@ export function createVoiceInput( const SR = (window as any).SpeechRecognition || (typeof webkitSpeechRecognition !== "undefined" ? webkitSpeechRecognition : null); if (!SR) { onError("Speech recognition not supported in this browser"); - return { start() {}, stop() {}, pause() {}, resume() {} }; + return { start() {}, stop() {}, pause() {}, resume() {}, setLanguage() {} }; } - const recognition = new SR(); - recognition.continuous = true; - recognition.interimResults = true; - recognition.lang = "en-US"; - let shouldListen = false; let paused = false; + let currentLang = DEFAULT_SPEECH_LANG; + + /** + * Build a recognition session. + * + * Chrome reads `lang` when the object is constructed and ignores later + * assignment on a session that has already run, so switching language means + * building a new object rather than re-tagging this one. + */ + function buildRecognition(): any { + const r = new SR(); + r.continuous = true; + r.interimResults = true; + r.lang = currentLang; + + r.onresult = (event: any) => { + for (let i = event.resultIndex; i < event.results.length; i++) { + if (event.results[i].isFinal) { + const text = event.results[i][0].transcript.trim(); + if (text) onTranscript(text); + } + } + }; - recognition.onresult = (event: any) => { - for (let i = event.resultIndex; i < event.results.length; i++) { - if (event.results[i].isFinal) { - const text = event.results[i][0].transcript.trim(); - if (text) onTranscript(text); + r.onend = () => { + // A session replaced by a language switch must stay down, or it would + // race the new one for the microphone and keep the old language alive. + if (r !== recognition) return; + if (shouldListen && !paused) { + try { + r.start(); + } catch { + // Already started + } } - } - }; + }; - recognition.onend = () => { - if (shouldListen && !paused) { - try { - recognition.start(); - } catch { - // Already started + r.onerror = (event: any) => { + if (event.error === "not-allowed") { + onError("Microphone access denied. Please allow microphone access."); + shouldListen = false; + } else if (event.error === "no-speech") { + // Normal, just restart + } else if (event.error === "aborted") { + // Expected during pause + } else { + console.warn("[voice] recognition error:", event.error); } - } - }; + }; - recognition.onerror = (event: any) => { - if (event.error === "not-allowed") { - onError("Microphone access denied. Please allow microphone access."); - shouldListen = false; - } else if (event.error === "no-speech") { - // Normal, just restart - } else if (event.error === "aborted") { - // Expected during pause - } else { - console.warn("[voice] recognition error:", event.error); - } - }; + return r; + } + + let recognition = buildRecognition(); return { start() { @@ -96,6 +117,34 @@ export function createVoiceInput( } } }, + setLanguage(lang: string) { + if (!lang || lang === currentLang) return; + currentLang = lang; + + const previous = recognition; + recognition = buildRecognition(); + // Retires the old session: its onend now sees itself superseded. + try { + previous.stop(); + } catch { + // Wasn't running. + } + + if (shouldListen && !paused) { + // Let the retired session release the microphone before claiming it, + // otherwise Chrome rejects the new start outright. + setTimeout(() => { + if (recognition !== previous && shouldListen && !paused) { + try { + recognition.start(); + } catch { + // Already started + } + } + }, 250); + } + console.log("[voice] language set to", lang); + }, }; } diff --git a/server.py b/server.py index 9908d2a6..9bd975db 100644 --- a/server.py +++ b/server.py @@ -65,6 +65,29 @@ FISH_VOICE_ID = os.getenv("FISH_VOICE_ID", "612b878b113047d9a770c069c8b4fdfe") # JARVIS (MCU) FISH_API_URL = "https://api.fish.audio/v1/tts" USER_NAME = os.getenv("USER_NAME", "sir") +# BCP-47 tag the browser's speech recognition listens in, and the language +# JARVIS is told to reply in. Read per-request rather than cached, so changing +# it in the settings panel takes effect without a restart. +DEFAULT_SPEECH_LANG = "en-US" +# Offered in the settings panel, mapped to the name used to tell JARVIS which +# language to answer in. +SPEECH_LANGUAGES = { + "en-US": "English", + "en-GB": "English", + "it-IT": "Italian", + "es-ES": "Spanish", + "fr-FR": "French", + "de-DE": "German", + "pt-BR": "Portuguese", + "nl-NL": "Dutch", + "ja-JP": "Japanese", + "zh-CN": "Chinese", +} + + +def _current_speech_lang() -> str: + """The configured speech language, read live so the panel needs no restart.""" + return os.getenv("SPEECH_LANG", DEFAULT_SPEECH_LANG) or DEFAULT_SPEECH_LANG PROJECT_DIR = os.path.dirname(os.path.abspath(__file__)) _SKIP_PERMISSIONS = os.getenv("JARVIS_SKIP_PERMISSIONS", "true").lower() not in ("0", "false", "no") @@ -1211,6 +1234,20 @@ async def generate_response( user_name=USER_NAME, project_dir=PROJECT_DIR, ) + # The prompt above is written in English and would answer an Italian + # question in English, which makes the speech-language setting only half + # work. Kept to a bare language directive on purpose: wording that also + # discussed action selection measurably weakened the language adherence it + # was appended to enforce. Action routing is less reliable outside English + # regardless — see the non-English caveat in the README. + speech_lang = _current_speech_lang() + if not speech_lang.startswith("en"): + language = SPEECH_LANGUAGES.get(speech_lang, speech_lang) + system += ( + f"\n\nLANGUAGE: The user speaks {language}. Write every spoken reply in {language}, " + f"keeping the same dry, economical butler voice." + ) + if lookup_status: system += f"\n\nACTIVE LOOKUPS:\n{lookup_status}\nIf asked about progress, report this status." @@ -2509,10 +2546,11 @@ class PreferencesUpdate(BaseModel): user_name: str = "" honorific: str = "sir" calendar_accounts: str = "auto" + speech_lang: str = DEFAULT_SPEECH_LANG @app.post("/api/settings/keys") async def api_settings_keys(body: KeyUpdate): - allowed = {"ANTHROPIC_API_KEY", "FISH_API_KEY", "FISH_VOICE_ID", "USER_NAME", "HONORIFIC", "CALENDAR_ACCOUNTS"} + allowed = {"ANTHROPIC_API_KEY", "FISH_API_KEY", "FISH_VOICE_ID", "USER_NAME", "HONORIFIC", "CALENDAR_ACCOUNTS", "SPEECH_LANG"} if body.key_name not in allowed: return JSONResponse({"success": False, "error": "Invalid key name"}, status_code=400) _write_env_key(body.key_name, body.key_value) @@ -2592,6 +2630,7 @@ async def api_get_preferences(): "user_name": env_dict.get("USER_NAME", ""), "honorific": env_dict.get("HONORIFIC", "sir"), "calendar_accounts": env_dict.get("CALENDAR_ACCOUNTS", "auto"), + "speech_lang": env_dict.get("SPEECH_LANG", DEFAULT_SPEECH_LANG) or DEFAULT_SPEECH_LANG, } @app.post("/api/settings/preferences") @@ -2599,6 +2638,7 @@ async def api_save_preferences(body: PreferencesUpdate): _write_env_key("USER_NAME", body.user_name) _write_env_key("HONORIFIC", body.honorific) _write_env_key("CALENDAR_ACCOUNTS", body.calendar_accounts) + _write_env_key("SPEECH_LANG", body.speech_lang or DEFAULT_SPEECH_LANG) return {"success": True} # --------------------------------------------------------------------------- From 6e3488e4dfbc1a006d3ecd38869f74b3b70c8af3 Mon Sep 17 00:00:00 2001 From: Luca Trisiello Date: Mon, 10 Aug 2026 17:57:50 +0200 Subject: [PATCH 05/10] feat: accumulate speech fragments into one utterance Chrome closes a recognition segment at every pause, and the client sent each "final" result the instant it arrived. A sentence spoken with any hesitation reached JARVIS as several questions: he answered half a thought, then received the rest as a new one. The logs show single words like "hey" arriving alone. Collect fragments and send once the speaker has actually stopped, after a 1s gap. Two edges that would otherwise bite: - A 6s ceiling. Steady dictation never produces the gap, so without a cap the timer re-arms forever and the reply never comes. - Muting discards what is pending, so half a sentence cannot arrive a second after the user silenced him. Barge-in stays immediate: the audio is cut on the first fragment, not at flush, since interrupting is what the user wanted the moment they spoke. Costs 1s before JARVIS starts thinking. That is the price of knowing the sentence ended; UTTERANCE_GAP_MS trades it back for more split sentences. Co-Authored-By: Claude Opus 5 --- frontend/src/main.ts | 56 ++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 52 insertions(+), 4 deletions(-) diff --git a/frontend/src/main.ts b/frontend/src/main.ts index 4619a23d..e6b08f2d 100644 --- a/frontend/src/main.ts +++ b/frontend/src/main.ts @@ -103,13 +103,59 @@ function transition(newState: State) { // Voice input // --------------------------------------------------------------------------- +/** + * Fragment accumulation. + * + * Chrome closes a recognition segment at every pause, so a sentence spoken + * with any hesitation arrives as several "final" results. Sending each one + * immediately made JARVIS answer half a thought and then receive the rest as + * a separate question. Instead, collect the pieces and send once the speaker + * has actually stopped. + */ +const UTTERANCE_GAP_MS = 1000; +// A pause never arrives while someone is dictating steadily, so cap the wait +// rather than let a long sentence hold the whole conversation open. +const UTTERANCE_MAX_MS = 6000; + +let pendingFragments: string[] = []; +let gapTimer: number | undefined; +let maxTimer: number | undefined; + +function discardUtterance() { + clearTimeout(gapTimer); + clearTimeout(maxTimer); + gapTimer = undefined; + maxTimer = undefined; + pendingFragments = []; +} + +function flushUtterance() { + clearTimeout(gapTimer); + clearTimeout(maxTimer); + gapTimer = undefined; + maxTimer = undefined; + + const text = pendingFragments.join(" ").replace(/\s+/g, " ").trim(); + pendingFragments = []; + if (!text) return; + + socket.send({ type: "transcript", text, isFinal: true }); + transition("thinking"); +} + const voiceInput = createVoiceInput( (text: string) => { - // Cancel any current JARVIS response before sending new input + // Cancel any current JARVIS response before sending new input. Done on the + // first fragment, not at flush time: cutting him off is what the user + // wanted the moment they started talking. audioPlayer.stop(); - // User spoke — send transcript - socket.send({ type: "transcript", text, isFinal: true }); - transition("thinking"); + + pendingFragments.push(text); + clearTimeout(gapTimer); + gapTimer = window.setTimeout(flushUtterance, UTTERANCE_GAP_MS); + if (maxTimer === undefined) { + maxTimer = window.setTimeout(flushUtterance, UTTERANCE_MAX_MS); + } }, (msg: string) => { showError(msg); @@ -236,6 +282,8 @@ btnMute.addEventListener("click", (e) => { isMuted = !isMuted; btnMute.classList.toggle("muted", isMuted); if (isMuted) { + // Half-spoken words must not arrive a second after the user silenced him. + discardUtterance(); voiceInput.pause(); transition("idle"); } else { From 7559472aa0c9393479981f3064827772612a463a Mon Sep 17 00:00:00 2001 From: Luca Trisiello Date: Mon, 10 Aug 2026 19:44:32 +0200 Subject: [PATCH 06/10] fix: address the user the way he asked, and greet him in his language MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three complaints from one Italian session, three separate causes. **He was called the wrong thing.** HONORIFIC was written to .env, served by the preferences API and offered in the settings panel — and never read. The prompt hardcoded `Address {user_name} as "sir"`, so the dropdown did nothing. Worse, the model translated that "sir" freely and settled on "signora": it addressed him with a feminine honorific, five times out of five, while his configuration said otherwise. The honorific is now a prompt variable, and the language directive states it must be reproduced letter for letter rather than translated or inflected. Placing that in the language block matters — the same instruction in the personality section was ignored 5/5, and obeyed 5/5 at the end. Measured, both ways round: "sir" and "signore" now each hold 5/5. The panel field becomes free text with suggestions. It offered sir, ma'am and none, so "signore" was not expressible in the first place. **The greeting arrived in English.** It was hardcoded, so every Italian session opened with "Good evening, sir." and switched language on the next sentence. It is the one line a user hears every single time. Now built from a small table for the languages the panel offers, falling back to English. **It could not hear its own name.** An Italian recogniser mangles an English name: the logs show "arbiss", "e gli arbis" and "hey Yaris" where he plainly said JARVIS. Added as corrections, taken from the transcripts rather than invented. "yaris" is handled only after a term of address. It is also a very common Toyota, and a bare rule rewrote "la mia auto Yaris" into a summons. Verified in both directions, including that "arbitro", "arbitri" and "arbitrario" survive untouched. Co-Authored-By: Claude Opus 5 --- frontend/src/settings.ts | 17 +++++++----- server.py | 58 +++++++++++++++++++++++++++++++++------- 2 files changed, 59 insertions(+), 16 deletions(-) diff --git a/frontend/src/settings.ts b/frontend/src/settings.ts index 4f4131b5..24ec0911 100644 --- a/frontend/src/settings.ts +++ b/frontend/src/settings.ts @@ -144,11 +144,14 @@ function buildPanelHTML(): string {
- + + + + + + + +
@@ -264,7 +267,7 @@ async function loadPreferences() { try { const prefs = await apiGet("/api/settings/preferences"); const nameEl = document.getElementById("input-user-name") as HTMLInputElement; - const honEl = document.getElementById("input-honorific") as HTMLSelectElement; + const honEl = document.getElementById("input-honorific") as HTMLInputElement; const calEl = document.getElementById("input-calendar-accounts") as HTMLTextAreaElement; const langEl = document.getElementById("input-speech-lang") as HTMLSelectElement; if (nameEl) nameEl.value = prefs.user_name || ""; @@ -330,7 +333,7 @@ function wireEvents() { // Save preferences document.getElementById("btn-save-prefs")?.addEventListener("click", async () => { const user_name = (document.getElementById("input-user-name") as HTMLInputElement).value.trim(); - const honorific = (document.getElementById("input-honorific") as HTMLSelectElement).value; + const honorific = (document.getElementById("input-honorific") as HTMLInputElement).value; const calendar_accounts = (document.getElementById("input-calendar-accounts") as HTMLTextAreaElement).value.trim(); const speech_lang = (document.getElementById("input-speech-lang") as HTMLSelectElement).value; await apiPost("/api/settings/preferences", { user_name, honorific, calendar_accounts, speech_lang }); diff --git a/server.py b/server.py index 9bd975db..ded5c381 100644 --- a/server.py +++ b/server.py @@ -88,6 +88,35 @@ def _current_speech_lang() -> str: """The configured speech language, read live so the panel needs no restart.""" return os.getenv("SPEECH_LANG", DEFAULT_SPEECH_LANG) or DEFAULT_SPEECH_LANG + + +def _current_honorific() -> str: + """How the user has asked to be addressed. Read live, like the language.""" + return os.getenv("HONORIFIC", "sir").strip() or "sir" + + +# Spoken at the top of a session, so it is the one line a user hears every +# single time. Left in English it announced the wrong language before JARVIS +# had said anything else. +GREETINGS = { + "en": ("Good morning", "Good afternoon", "Good evening"), + "it": ("Buongiorno", "Buon pomeriggio", "Buonasera"), + "es": ("Buenos días", "Buenas tardes", "Buenas noches"), + "fr": ("Bonjour", "Bon après-midi", "Bonsoir"), + "de": ("Guten Morgen", "Guten Tag", "Guten Abend"), + "pt": ("Bom dia", "Boa tarde", "Boa noite"), + "nl": ("Goedemorgen", "Goedemiddag", "Goedenavond"), + "ja": ("おはようございます", "こんにちは", "こんばんは"), + "zh": ("早上好", "下午好", "晚上好"), +} + + +def build_greeting(hour: int) -> str: + """The time-of-day greeting, in the configured language.""" + slot = 0 if hour < 12 else (1 if hour < 17 else 2) + lang = _current_speech_lang().split("-")[0].lower() + words = GREETINGS.get(lang, GREETINGS["en"]) + return f"{words[slot]}, {_current_honorific()}." PROJECT_DIR = os.path.dirname(os.path.abspath(__file__)) _SKIP_PERMISSIONS = os.getenv("JARVIS_SKIP_PERMISSIONS", "true").lower() not in ("0", "false", "no") @@ -98,7 +127,7 @@ def _current_speech_lang() -> str: VOICE & PERSONALITY: - British butler elegance with understated dry wit -- Address {user_name} as "sir" naturally — not every sentence, but regularly +- Address {user_name} as "{honorific}" naturally — not every sentence, but regularly. Use that word exactly as written, in every language: it is how he has asked to be addressed, not a word to translate or inflect. The examples below happen to say "sir"; say "{honorific}" instead. - Never say "How can I help you?" or "Is there anything else?" — just act - Deliver bad news calmly, like reporting weather: "We have a slight problem, sir." - Your humor is observational, never jokes: state facts and let implications land @@ -719,6 +748,18 @@ def format_projects_for_prompt(projects: list[dict]) -> str: r"\bquad\b": "Claude", r"\btravis\b": "JARVIS", r"\bjarves\b": "JARVIS", + # Non-English recognisers hear an English name badly. These are the exact + # forms observed in the logs, not guesses: an Italian session produced + # "arbiss", "e gli arbis" and "hey Yaris" while the user was plainly + # saying JARVIS. Matched as whole words so ordinary vocabulary is safe. + r"\bgli arbis\b": "JARVIS", + r"\barbiss?\b": "JARVIS", + r"\bgiarvis\b": "JARVIS", + r"\bjarvi\b": "JARVIS", + # "yaris" is a Toyota, and a common one — correcting it outright would + # rewrite "la mia auto Yaris". Only fix it where the sentence is plainly + # addressing someone. + r"\b(hey|hi|ehi|ciao|ok|senti)\s+yaris\b": r"\1 JARVIS", } @@ -1232,6 +1273,7 @@ async def generate_response( dispatch_context=dispatch_registry.format_for_prompt(), known_projects=format_projects_for_prompt(projects), user_name=USER_NAME, + honorific=_current_honorific(), project_dir=PROJECT_DIR, ) # The prompt above is written in English and would answer an Italian @@ -1243,9 +1285,13 @@ async def generate_response( speech_lang = _current_speech_lang() if not speech_lang.startswith("en"): language = SPEECH_LANGUAGES.get(speech_lang, speech_lang) + honorific = _current_honorific() system += ( f"\n\nLANGUAGE: The user speaks {language}. Write every spoken reply in {language}, " - f"keeping the same dry, economical butler voice." + f"keeping the same dry, economical butler voice. One exception: address him as " + f'"{honorific}" — that exact word, letter for letter. It is his chosen form of ' + f"address, not vocabulary to translate into {language} or to inflect for gender. " + f'Writing anything other than "{honorific}" is an error.' ) if lookup_status: @@ -2030,13 +2076,7 @@ async def voice_handler(ws: WebSocket): try: # ── Greeting — always start in conversation mode ── now = datetime.now() - hour = now.hour - if hour < 12: - greeting = "Good morning, sir." - elif hour < 17: - greeting = "Good afternoon, sir." - else: - greeting = "Good evening, sir." + greeting = build_greeting(now.hour) global _last_greeting_time should_greet = (time.time() - _last_greeting_time) > 60 From 3bfc0d31743307d53e67c17e91354e8ff377ebac Mon Sep 17 00:00:00 2001 From: Luca Trisiello Date: Tue, 11 Aug 2026 04:41:37 +0200 Subject: [PATCH 07/10] feat: speak with the macOS voices when Fish Audio is not configured MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Without a Fish Audio key JARVIS was mute — the whole voice half of a voice assistant, gated behind a paid signup. macOS already ships voices for every language the panel offers, so it can talk on a fresh install instead. TTS_BACKEND picks the synthesiser: auto (default) uses Fish when its key is set and the macOS voices otherwise, and fish/macos/none force the choice. MACOS_VOICE names a voice; empty resolves the best installed one for the configured language. The panel gains both, plus a preview button — choosing blind from seventy-odd voices is not a choice. Voices are discovered from `say -v ?` rather than hardcoded, since what is installed varies by machine and macOS version. Output is WAV: the `say` default is AIFF-C, which Chrome does not reliably decode. Two defects found by testing rather than by reading: The voice list parser keyed on the run of spaces before the locale tag, so it saw 73 voices and one Italian. Classic voices are padded into a column ("Alice it_IT #"), newer ones carry a parenthesised language and get a single space ("Eddy (Italiano (Italia)) it_IT #"). Anchored on the "#" instead: 183 voices, nine Italian. "auto" also resolved to Fish on an untouched install, because the placeholder in .env.example is a non-empty string and passed a truth test. Every line went to Fish for a 401 rather than falling back — precisely the case this feature exists for. Both that check and the settings panel's now share one _fish_configured helper. `say` interprets [[...]] as embedded commands, [[volm 0]] among them, so those are stripped: the text arrives from a speech transcript by way of an LLM. Arguments go straight to exec, never a shell. Co-Authored-By: Claude Opus 5 --- .env.example | 11 +++ frontend/src/settings.ts | 100 +++++++++++++++++++++++++- server.py | 84 ++++++++++++++++++++-- tts.py | 149 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 336 insertions(+), 8 deletions(-) create mode 100644 tts.py diff --git a/.env.example b/.env.example index 51090741..4fabc917 100644 --- a/.env.example +++ b/.env.example @@ -2,6 +2,17 @@ ANTHROPIC_API_KEY=your-anthropic-api-key-here FISH_API_KEY=your-fish-audio-api-key-here +# Optional: which speech synthesiser to use. +# auto (default) — Fish Audio when its key is set, otherwise macOS voices +# fish — always Fish Audio +# macos — always the built-in macOS voices: free, offline, no key +# none — silent; replies appear as on-screen captions only +# TTS_BACKEND=auto + +# Optional: which macOS voice to speak with (see `say -v ?`). +# Empty picks the best installed voice for SPEECH_LANG. +# MACOS_VOICE=Alice + # Optional: Fish Audio voice model (defaults to JARVIS MCU voice) # FISH_VOICE_ID=612b878b113047d9a770c069c8b4fdfe diff --git a/frontend/src/settings.ts b/frontend/src/settings.ts index 24ec0911..9b60daca 100644 --- a/frontend/src/settings.ts +++ b/frontend/src/settings.ts @@ -31,6 +31,14 @@ interface PreferencesResponse { honorific: string; calendar_accounts: string; speech_lang: string; + tts_backend: string; + macos_voice: string; +} + +interface VoicesResponse { + voices: { name: string; lang: string }[]; + matching_language: number; + resolved: string | null; } /** @@ -170,6 +178,24 @@ function buildPanelHTML(): string {
+
+ + +
+ +
+ +
+ + +
+
+
@@ -274,11 +300,47 @@ async function loadPreferences() { if (honEl) honEl.value = prefs.honorific || "sir"; if (calEl) calEl.value = prefs.calendar_accounts || "auto"; if (langEl) langEl.value = prefs.speech_lang || "en-US"; + + const backendEl = document.getElementById("input-tts-backend") as HTMLSelectElement; + if (backendEl) backendEl.value = prefs.tts_backend || "auto"; + await loadVoices(prefs.macos_voice || ""); + updateVoiceFieldVisibility(); } catch (e) { console.error("[settings] failed to load preferences:", e); } } +/** Only offer the voice picker when a macOS voice could actually be used. */ +function updateVoiceFieldVisibility() { + const backend = (document.getElementById("input-tts-backend") as HTMLSelectElement)?.value; + const field = document.getElementById("field-macos-voice"); + if (field) field.style.display = backend === "fish" || backend === "none" ? "none" : ""; +} + +async function loadVoices(selected: string) { + const voiceEl = document.getElementById("input-macos-voice") as HTMLSelectElement; + if (!voiceEl) return; + try { + const data = await apiGet("/api/settings/voices"); + // "Automatic" first, labelled with what that currently resolves to, so the + // default is not a mystery. + const auto = data.resolved ? `Automatic (${data.resolved})` : "Automatic"; + voiceEl.innerHTML = + `` + + data.voices + .map((v, i) => { + // Voices for the current language are listed first; mark where the + // rest begin rather than silently mixing them together. + const divider = i === data.matching_language && i > 0 ? "— other languages — " : ""; + return ``; + }) + .join(""); + voiceEl.value = selected; + } catch (e) { + console.error("[settings] failed to load voices:", e); + } +} + function wireEvents() { // Close document.getElementById("settings-close")?.addEventListener("click", closeSettings); @@ -336,11 +398,47 @@ function wireEvents() { const honorific = (document.getElementById("input-honorific") as HTMLInputElement).value; const calendar_accounts = (document.getElementById("input-calendar-accounts") as HTMLTextAreaElement).value.trim(); const speech_lang = (document.getElementById("input-speech-lang") as HTMLSelectElement).value; - await apiPost("/api/settings/preferences", { user_name, honorific, calendar_accounts, speech_lang }); + const tts_backend = (document.getElementById("input-tts-backend") as HTMLSelectElement).value; + const macos_voice = (document.getElementById("input-macos-voice") as HTMLSelectElement).value; + await apiPost("/api/settings/preferences", { + user_name, honorific, calendar_accounts, speech_lang, tts_backend, macos_voice, + }); document.dispatchEvent(new CustomEvent(SPEECH_LANG_EVENT, { detail: speech_lang })); await loadStatus(); }); + // Voice backend — show or hide the voice picker to match + document.getElementById("input-tts-backend")?.addEventListener("change", updateVoiceFieldVisibility); + + // Audition a voice before committing to it + document.getElementById("btn-preview-voice")?.addEventListener("click", async (e) => { + e.stopPropagation(); + const btn = e.currentTarget as HTMLButtonElement; + const voice = (document.getElementById("input-macos-voice") as HTMLSelectElement).value; + const original = btn.textContent; + btn.textContent = "..."; + btn.disabled = true; + try { + const res = await apiGet<{ audio: string | null }>( + `/api/tts-test?voice=${encodeURIComponent(voice)}` + ); + if (res.audio) { + const bytes = Uint8Array.from(atob(res.audio), (c) => c.charCodeAt(0)); + const ctx = new AudioContext(); + const buffer = await ctx.decodeAudioData(bytes.buffer); + const src = ctx.createBufferSource(); + src.buffer = buffer; + src.connect(ctx.destination); + src.start(); + } + } catch (err) { + console.error("[settings] voice preview failed:", err); + } finally { + btn.textContent = original; + btn.disabled = false; + } + }); + // Setup next button document.getElementById("btn-setup-next")?.addEventListener("click", advanceSetup); } diff --git a/server.py b/server.py index ded5c381..8d8a0f8f 100644 --- a/server.py +++ b/server.py @@ -50,6 +50,7 @@ format_tasks_for_voice, extract_memories, get_important_memories, ) from notes_access import get_recent_notes, read_note, search_notes_apple, create_apple_note +from tts import list_voices, resolve_voice, speak as macos_speak from dispatch_registry import DispatchRegistry from planner import TaskPlanner, detect_planning_mode, BYPASS_PHRASES @@ -1200,11 +1201,47 @@ async def self_work_and_notify(session: WorkSession, prompt: str, ws): # --------------------------------------------------------------------------- -# TTS (Fish Audio) +# TTS (Fish Audio, or the macOS synthesiser) # --------------------------------------------------------------------------- +_FISH_KEY_PLACEHOLDER = "your-fish-audio-api-key-here" + + +def _fish_configured() -> bool: + """Whether Fish Audio has a real key. + + The placeholder shipped in .env.example is a non-empty string, so a plain + truth test counts an untouched install as configured — which sent every + line to Fish for a 401 instead of falling back. + """ + key = (FISH_API_KEY or "").strip() + return bool(key) and key != _FISH_KEY_PLACEHOLDER + + +def _current_tts_backend() -> str: + """Which synthesiser to use: auto, fish, macos or none. + + "auto" prefers Fish Audio when a key is configured and falls back to the + macOS voices otherwise — so a fresh install talks instead of sitting mute, + which is what an unconfigured JARVIS used to do. + """ + choice = os.getenv("TTS_BACKEND", "auto").strip().lower() or "auto" + if choice != "auto": + return choice + return "fish" if _fish_configured() else "macos" + + async def synthesize_speech(text: str) -> Optional[bytes]: - """Generate speech audio from text using Fish Audio TTS.""" + """Generate speech audio for a line of JARVIS dialogue.""" + backend = _current_tts_backend() + + if backend == "none": + return None + + if backend == "macos": + voice = await resolve_voice(_current_speech_lang(), os.getenv("MACOS_VOICE", "")) + return await macos_speak(text, voice) + if not FISH_API_KEY: log.warning("FISH_API_KEY not set, skipping TTS") return None @@ -1540,9 +1577,17 @@ async def health(): @app.get("/api/tts-test") -async def tts_test(): - """Generate a test audio clip for debugging.""" - audio = await synthesize_speech("Testing audio, sir.") +async def tts_test(voice: str = ""): + """Generate a test audio clip. + + A voice can be named to audition it before saving, since picking one from + a list of seventy is otherwise guesswork. + """ + line = f"{build_greeting(datetime.now().hour)} All systems are operational." + if voice: + audio = await macos_speak(line, voice) + else: + audio = await synthesize_speech(line) if audio: return {"audio": base64.b64encode(audio).decode()} return {"audio": None, "error": "TTS failed"} @@ -2587,10 +2632,12 @@ class PreferencesUpdate(BaseModel): honorific: str = "sir" calendar_accounts: str = "auto" speech_lang: str = DEFAULT_SPEECH_LANG + tts_backend: str = "auto" + macos_voice: str = "" @app.post("/api/settings/keys") async def api_settings_keys(body: KeyUpdate): - allowed = {"ANTHROPIC_API_KEY", "FISH_API_KEY", "FISH_VOICE_ID", "USER_NAME", "HONORIFIC", "CALENDAR_ACCOUNTS", "SPEECH_LANG"} + allowed = {"ANTHROPIC_API_KEY", "FISH_API_KEY", "FISH_VOICE_ID", "USER_NAME", "HONORIFIC", "CALENDAR_ACCOUNTS", "SPEECH_LANG", "TTS_BACKEND", "MACOS_VOICE"} if body.key_name not in allowed: return JSONResponse({"success": False, "error": "Invalid key name"}, status_code=400) _write_env_key(body.key_name, body.key_value) @@ -2657,12 +2704,31 @@ async def api_settings_status(): "uptime_seconds": int(time.time() - _session_start), "env_keys_set": { "anthropic": bool(env_dict.get("ANTHROPIC_API_KEY", "").strip() and env_dict.get("ANTHROPIC_API_KEY", "") != "your-anthropic-api-key-here"), - "fish_audio": bool(env_dict.get("FISH_API_KEY", "").strip() and env_dict.get("FISH_API_KEY", "") != "your-fish-audio-api-key-here"), + "fish_audio": bool(env_dict.get("FISH_API_KEY", "").strip() and env_dict.get("FISH_API_KEY", "") != _FISH_KEY_PLACEHOLDER), "fish_voice_id": bool(env_dict.get("FISH_VOICE_ID", "").strip()), "user_name": env_dict.get("USER_NAME", ""), }, } +@app.get("/api/settings/voices") +async def api_settings_voices(): + """Installed macOS voices, so the panel can offer real choices. + + Voices for the configured language come first — that is what the user is + almost certainly picking from — with the rest kept for anyone who wants a + voice from another language. + """ + voices = await list_voices() + lang = _current_speech_lang().split("-")[0].lower() + preferred = [v for v in voices if v["lang"].lower().startswith(lang)] + others = [v for v in voices if not v["lang"].lower().startswith(lang)] + return { + "voices": preferred + others, + "matching_language": len(preferred), + "resolved": await resolve_voice(_current_speech_lang(), os.getenv("MACOS_VOICE", "")), + } + + @app.get("/api/settings/preferences") async def api_get_preferences(): _, env_dict = _read_env() @@ -2671,6 +2737,8 @@ async def api_get_preferences(): "honorific": env_dict.get("HONORIFIC", "sir"), "calendar_accounts": env_dict.get("CALENDAR_ACCOUNTS", "auto"), "speech_lang": env_dict.get("SPEECH_LANG", DEFAULT_SPEECH_LANG) or DEFAULT_SPEECH_LANG, + "tts_backend": env_dict.get("TTS_BACKEND", "auto") or "auto", + "macos_voice": env_dict.get("MACOS_VOICE", ""), } @app.post("/api/settings/preferences") @@ -2679,6 +2747,8 @@ async def api_save_preferences(body: PreferencesUpdate): _write_env_key("HONORIFIC", body.honorific) _write_env_key("CALENDAR_ACCOUNTS", body.calendar_accounts) _write_env_key("SPEECH_LANG", body.speech_lang or DEFAULT_SPEECH_LANG) + _write_env_key("TTS_BACKEND", body.tts_backend or "auto") + _write_env_key("MACOS_VOICE", body.macos_voice) return {"success": True} # --------------------------------------------------------------------------- diff --git a/tts.py b/tts.py new file mode 100644 index 00000000..a0d5c0a0 --- /dev/null +++ b/tts.py @@ -0,0 +1,149 @@ +""" +JARVIS macOS speech synthesis — a free, offline alternative to Fish Audio. + +macOS ships voices for every language JARVIS offers, so a user without a Fish +Audio key still gets a spoken assistant rather than a silent one. The voice is +plainer than the Fish JARVIS model; the trade is availability and cost. + +Voices are discovered from `say -v ?` at runtime rather than hardcoded, since +which ones are installed varies by machine and macOS version. +""" + +import asyncio +import logging +import re +import tempfile +from pathlib import Path + +log = logging.getLogger("jarvis.tts") + +# WAV, which every browser's decodeAudioData accepts. The `say` default is +# AIFF-C, which Chrome does not reliably decode. +_DATA_FORMAT = "LEI16@22050" + +# Preferred voice per language, used when installed. Everything else falls back +# to the first voice matching the language. Daniel is the British voice, which +# suits the butler better than the American default. +_PREFERRED = { + "en": ["Daniel", "Oliver", "Serena"], + "it": ["Alice", "Federica"], + "es": ["Monica", "Mónica", "Jorge"], + "fr": ["Thomas", "Audrey"], + "de": ["Anna", "Markus"], + "pt": ["Luciana", "Joana"], + "nl": ["Xander", "Claire"], + "ja": ["Kyoko", "Otoya"], + "zh": ["Tingting", "Tingting"], +} + +_voice_cache: list[dict] | None = None + +# `say` interprets [[...]] as embedded speech commands — [[volm 0]] silences it, +# among others. The text reaching here came from a speech transcript by way of +# an LLM, so strip the brackets rather than trust that none appear. +_SAY_COMMAND = re.compile(r"\[\[.*?\]\]") + + +def _parse_voice_line(line: str) -> dict | None: + """Parse one `say -v ?` row: name, locale tag, then a `#` sample phrase. + + Anchored on the `#`, because the column spacing is not consistent: classic + voices are padded into a column ("Alice it_IT # ...") while + newer ones carry a parenthesised language in the name and get a single + space ("Eddy (Italiano (Italia)) it_IT # ..."). Keying on run length + silently dropped every voice of the second kind. + """ + match = re.match(r"^(.+?)\s+([a-z]{2}_[A-Z]{2})\s+#", line) + if not match: + return None + return {"name": match.group(1).strip(), "lang": match.group(2)} + + +async def list_voices(refresh: bool = False) -> list[dict]: + """Every installed macOS voice, as {"name", "lang"}.""" + global _voice_cache + if _voice_cache is not None and not refresh: + return _voice_cache + try: + proc = await asyncio.create_subprocess_exec( + "say", "-v", "?", + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + stdout, _ = await asyncio.wait_for(proc.communicate(), timeout=10) + except Exception as e: + log.warning(f"Could not list voices: {e}") + return [] + + voices = [] + for line in stdout.decode(errors="replace").split("\n"): + parsed = _parse_voice_line(line) + if parsed: + voices.append(parsed) + _voice_cache = voices + return voices + + +async def resolve_voice(speech_lang: str, configured: str = "") -> str | None: + """Pick the voice to speak with, or None when nothing matches. + + An explicitly configured voice wins outright — including one whose language + differs from the interface, which is a legitimate thing to want. + """ + if configured.strip(): + return configured.strip() + + voices = await list_voices() + if not voices: + return None + + lang = speech_lang.split("-")[0].lower() + matching = [v for v in voices if v["lang"].lower().startswith(lang)] + if not matching: + return None + + names = {v["name"] for v in matching} + for preferred in _PREFERRED.get(lang, []): + if preferred in names: + return preferred + return matching[0]["name"] + + +async def speak(text: str, voice: str | None = None, timeout: float = 30) -> bytes | None: + """Render text to WAV bytes with the macOS speech synthesiser.""" + clean = _SAY_COMMAND.sub("", text).strip() + if not clean: + return None + + with tempfile.TemporaryDirectory() as tmp: + out = Path(tmp) / "speech.wav" + # Arguments are passed directly to exec, never through a shell, so the + # text cannot break out into a command. + args = ["say", "--data-format=" + _DATA_FORMAT, "-o", str(out)] + if voice: + args += ["-v", voice] + args += ["--", clean] + + try: + proc = await asyncio.create_subprocess_exec( + *args, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + _, stderr = await asyncio.wait_for(proc.communicate(), timeout=timeout) + if proc.returncode != 0: + log.warning(f"say failed: {stderr.decode(errors='replace')[:200]}") + return None + if not out.exists(): + log.warning("say produced no output file") + return None + return out.read_bytes() + except asyncio.TimeoutError: + log.warning("say timed out") + return None + except FileNotFoundError: + log.warning("say is unavailable — macOS TTS needs macOS") + return None + except Exception as e: + log.warning(f"say error: {e}") + return None From 633b13d132d6e1fc8529e3fb28efdda0dc111b28 Mon Sep 17 00:00:00 2001 From: Luca Trisiello Date: Tue, 11 Aug 2026 04:59:04 +0200 Subject: [PATCH 08/10] feat: translate the phrases JARVIS says in code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Acknowledgements, lookup results and error lines are built in Python, not written by the model, so they stayed English however the language was set. A single reply arrived as "Checking your calendar now, sir." followed by fluent Italian — the language setting looked half-finished because half of what JARVIS says never passed through it. Rather than ship hand-written translations for ten languages — inventing Japanese butler register is not something to do blind — the English catalogue is translated once per language by the model JARVIS already talks through, and cached on disk under data/. One Haiku call per language, ever. The cache is keyed by a hash of the catalogue, so editing a phrase re-translates instead of serving a stale set. Placeholders are named, never positional, because word order is exactly what changes between languages; the translator is told to reorder them freely but never rename or drop one. Translations are validated on the way out and again on the way in. A phrase that lost a placeholder does not raise — str.format ignores surplus arguments — so it would quietly drop the project name or the message count rather than fail visibly. Validating only at translation time would have left a hand-edited or truncated cache to do that silently, so load re-checks and falls back to English per phrase. Warm-up is backgrounded and English is used until it lands: a cold cache must not delay the first answer, and English is what that answer would have been anyway. Co-Authored-By: Claude Opus 5 --- calendar_access.py | 8 +- mail_access.py | 8 +- phrases.py | 243 +++++++++++++++++++++++++++++++++++++++++++++ server.py | 81 +++++++++------ 4 files changed, 303 insertions(+), 37 deletions(-) create mode 100644 phrases.py diff --git a/calendar_access.py b/calendar_access.py index c91d0903..8695fbcf 100644 --- a/calendar_access.py +++ b/calendar_access.py @@ -7,6 +7,8 @@ import asyncio import logging + +from phrases import phrase import os import time as _time from datetime import datetime, timedelta @@ -245,14 +247,14 @@ def format_events_for_context(events: list[dict]) -> str: def format_schedule_summary(events: list[dict]) -> str: """Format a brief voice-friendly summary of the schedule.""" if not events: - return "Your schedule is clear today, sir." + return phrase("calendar.clear") count = len(events) if count == 1: evt = events[0] if evt.get("all_day"): - return f"You have one all-day event: {evt['title']}." - return f"You have one event: {evt['title']} at {evt['start']}." + return phrase("calendar.one_all_day", title=evt["title"]) + return phrase("calendar.one_event", title=evt["title"], time=evt["start"]) summaries = [] for evt in events[:5]: diff --git a/mail_access.py b/mail_access.py index 03a88fe5..410d66cf 100644 --- a/mail_access.py +++ b/mail_access.py @@ -10,6 +10,8 @@ import asyncio import logging + +from phrases import phrase from datetime import datetime log = logging.getLogger("jarvis.mail") @@ -348,7 +350,7 @@ def format_unread_summary(unread: dict) -> str: """Format unread counts for voice.""" total = unread["total"] if total == 0: - return "Inbox is clear, sir. No unread messages." + return phrase("mail.clear") parts = [] for acct, count in unread["accounts"].items(): @@ -356,9 +358,9 @@ def format_unread_summary(unread: dict) -> str: parts.append(f"{count} in {acct}") if len(parts) == 1: - return f"You have {total} unread {'message' if total == 1 else 'messages'} — {parts[0]}." + return phrase("mail.one_account", total=total, detail=parts[0]) elif parts: - return f"You have {total} unread messages: {', '.join(parts)}." + return phrase("mail.many_accounts", total=total, detail=", ".join(parts)) else: return f"You have {total} unread {'message' if total == 1 else 'messages'}." diff --git a/phrases.py b/phrases.py new file mode 100644 index 00000000..06ac0792 --- /dev/null +++ b/phrases.py @@ -0,0 +1,243 @@ +""" +JARVIS fixed phrases — the lines JARVIS says that no LLM wrote. + +Acknowledgements, lookup results and error lines are built in code, so they +stayed English no matter which language the user had configured: a reply would +arrive as "Checking your calendar now, sir." followed by fluent Italian. + +Rather than ship hand-written translations for every language the panel offers +— inventing Japanese butler register is not something to do blind — the +English catalogue below is translated once per language by the same model +JARVIS already talks through, and cached on disk. The cache is keyed by a hash +of the catalogue, so editing any phrase re-translates rather than serving a +stale set. + +Until a translation exists the English original is used, which keeps a first +run talking rather than silent. +""" + +import asyncio +import hashlib +import json +import logging +import os +from pathlib import Path + +log = logging.getLogger("jarvis.phrases") + +_CACHE_DIR = Path(__file__).parent / "data" + +# Placeholders are named, never positional, so a translator can reorder them +# freely — word order is exactly what changes between languages. +CATALOGUE: dict[str, str] = { + # Acknowledgements — spoken while something slower happens + "ack.generic": "Right away, {honorific}.", + "ack.on_it": "On it, {honorific}.", + "ack.understood": "Understood, {honorific}.", + "ack.cancelled": "Cancelled, {honorific}.", + "ack.looking_into": "Looking into that now, {honorific}.", + "ack.taking_look": "Taking a look now, {honorific}.", + "ack.checking_calendar": "Checking your calendar now, {honorific}.", + "ack.checking_mail": "Checking your inbox now, {honorific}.", + "ack.building": "Building it now, {honorific}.", + "ack.connecting": "Connecting to {project} now, {honorific}.", + + # Modes + "mode.work_self": "Work mode active in my own repo, {honorific}. Tell me what needs fixing.", + "mode.back_to_chat": "Back to conversation mode, {honorific}.", + "mode.already_chat": "Already in conversation mode, {honorific}.", + + # Build and project status + "build.none_recent": "No recent builds on record, {honorific}.", + "build.still_working": "Still working on {project}, {honorific}. Been at it for {seconds} seconds.", + "build.problems": "{project} ran into problems, {honorific}.", + "build.status": "{project} is {status}, {honorific}.", + "build.finished": "{honorific}, {project} finished. Here's the gist: {summary}", + "build.done": "{honorific}, {project} is done. {summary}", + "build.issue": "{honorific}, I ran into an issue with {project}. {detail}", + "build.no_directory": "Couldn't find the {project} project directory, {honorific}.", + "build.connect_trouble": "Had trouble connecting to {project}, {honorific}.", + "build.work_complete": "Work is complete, {honorific}.", + + # Research + "research.complete": "Research is complete, {honorific}. The report is open in your browser.", + "research.timed_out": "Research timed out, {honorific}.", + + # Lookups + "lookup.slow": "That {kind} check is taking too long, {honorific}. The data may still be syncing.", + + # Calendar + "calendar.clear": "Your schedule is clear today, {honorific}.", + "calendar.one_all_day": "You have one all-day event: {title}.", + "calendar.one_event": "You have one event: {title} at {time}.", + + # Mail + "mail.clear": "Inbox is clear, {honorific}. No unread messages.", + "mail.one_account": "You have {total} unread in {detail}.", + "mail.many_accounts": "You have {total} unread messages: {detail}.", + + # Notes + "note.says": "{honorific}, your note '{title}' says: {body}", + "note.not_found": "Couldn't find a note matching '{query}', {honorific}.", + + # Failures + "error.generic": "Something went wrong, {honorific}.", +} + +_active_lang = "en" +_active: dict[str, str] = {} + + +def _catalogue_hash() -> str: + blob = json.dumps(CATALOGUE, sort_keys=True, ensure_ascii=False) + return hashlib.sha256(blob.encode("utf-8")).hexdigest()[:16] + + +def _cache_path(lang: str) -> Path: + return _CACHE_DIR / f"phrases-{lang}.json" + + +def phrase(key: str, **kwargs) -> str: + """The phrase for `key`, in the active language, with placeholders filled. + + Falls back to English whenever a translation is missing or malformed, so a + bad translation degrades to the original rather than to a crash. + """ + template = _active.get(key) or CATALOGUE.get(key, "") + if not template: + log.warning(f"Unknown phrase key: {key}") + return "" + kwargs.setdefault("honorific", os.getenv("HONORIFIC", "sir").strip() or "sir") + try: + return template.format(**kwargs) + except (KeyError, IndexError) as e: + # A translation that dropped or renamed a placeholder must not take the + # line down with it. + log.warning(f"Phrase {key} failed to format ({e}); using English") + try: + return CATALOGUE[key].format(**kwargs) + except Exception: + return CATALOGUE.get(key, "") + + +def load_cached(lang: str) -> bool: + """Load a cached translation for `lang`. Returns whether one was usable.""" + global _active, _active_lang + _active_lang = lang + if lang.startswith("en"): + _active = {} + return True + + path = _cache_path(lang) + if not path.exists(): + _active = {} + return False + try: + data = json.loads(path.read_text(encoding="utf-8")) + except Exception as e: + log.warning(f"Could not read phrase cache for {lang}: {e}") + _active = {} + return False + + if data.get("source_hash") != _catalogue_hash(): + log.info(f"Phrase cache for {lang} is stale — will re-translate") + _active = {} + return False + + # Validate on the way in as well as on the way out. A phrase missing a + # placeholder does not raise — str.format ignores surplus arguments — so it + # would quietly drop the project name or the count instead of failing + # loudly. Cache files can be edited, copied between machines, or truncated. + _active = {} + for key, text in (data.get("phrases") or {}).items(): + english = CATALOGUE.get(key) + if english is None or not isinstance(text, str): + continue + if _placeholders(text) != _placeholders(english): + log.warning(f"Cached phrase {key} has the wrong placeholders; using English") + continue + _active[key] = text + return bool(_active) + + +async def translate(lang: str, language_name: str, client) -> bool: + """Translate the catalogue into `lang` and cache it. Returns success.""" + global _active + if lang.startswith("en"): + _active = {} + return True + + prompt = ( + f"Translate these interface strings into {language_name}.\n\n" + "They are spoken aloud by JARVIS, a British butler AI: dry, economical, " + "never chatty. Match that register in the target language.\n\n" + "Rules:\n" + "- Return ONLY a JSON object with exactly the same keys.\n" + "- Keep every {placeholder} verbatim, including the braces. Reorder them " + "freely to suit the language, but never rename, drop or add one.\n" + "- {honorific} is the user's chosen form of address. Leave it as the " + "placeholder; do not replace it with a word.\n" + "- Keep them short. These are spoken, not written.\n\n" + f"{json.dumps(CATALOGUE, ensure_ascii=False, indent=1)}" + ) + + try: + response = await client.messages.create( + model="claude-haiku-4-5-20251001", + max_tokens=4000, + messages=[{"role": "user", "content": prompt}], + ) + raw = response.content[0].text.strip() + if raw.startswith("```"): + raw = raw.split("```")[1] + if raw.startswith("json"): + raw = raw[4:] + translated = json.loads(raw) + except Exception as e: + log.warning(f"Phrase translation for {lang} failed: {e}") + return False + + # Keep only keys we asked for, and only where every placeholder survived. + # A phrase that lost one would raise at format time on some future call + # with no way to see it coming. + clean = {} + for key, english in CATALOGUE.items(): + candidate = translated.get(key) + if not isinstance(candidate, str) or not candidate.strip(): + continue + expected = _placeholders(english) + if _placeholders(candidate) != expected: + log.warning(f"Phrase {key} lost a placeholder in {lang}; keeping English") + continue + clean[key] = candidate.strip() + + if not clean: + return False + + _active = clean + try: + _CACHE_DIR.mkdir(parents=True, exist_ok=True) + _cache_path(lang).write_text( + json.dumps({"source_hash": _catalogue_hash(), "phrases": clean}, + ensure_ascii=False, indent=1), + encoding="utf-8", + ) + except OSError as e: + log.warning(f"Could not cache phrases for {lang}: {e}") + + log.info(f"Translated {len(clean)}/{len(CATALOGUE)} phrases into {language_name}") + return True + + +def _placeholders(text: str) -> set[str]: + import re + return set(re.findall(r"\{(\w+)\}", text)) + + +async def ensure_language(lang: str, language_name: str, client) -> None: + """Make `lang` the active language, translating it if not already cached.""" + if load_cached(lang): + return + if client is None: + return + await translate(lang, language_name, client) diff --git a/server.py b/server.py index 8d8a0f8f..661c0377 100644 --- a/server.py +++ b/server.py @@ -50,6 +50,7 @@ format_tasks_for_voice, extract_memories, get_important_memories, ) from notes_access import get_recent_notes, read_note, search_notes_apple, create_apple_note +from phrases import phrase, ensure_language from tts import list_voices, resolve_voice, speak as macos_speak from dispatch_registry import DispatchRegistry from planner import TaskPlanner, detect_planning_mode, BYPASS_PHRASES @@ -91,6 +92,16 @@ def _current_speech_lang() -> str: return os.getenv("SPEECH_LANG", DEFAULT_SPEECH_LANG) or DEFAULT_SPEECH_LANG +async def _refresh_phrases() -> None: + """Make the fixed phrases match the configured language.""" + lang = _current_speech_lang() + name = SPEECH_LANGUAGES.get(lang, lang) + try: + await ensure_language(lang, name, anthropic_client) + except Exception as e: + log.warning(f"Could not prepare phrases for {lang}: {e}") + + def _current_honorific() -> str: """How the user has asked to be addressed. Read live, like the language.""" return os.getenv("HONORIFIC", "sir").strip() or "sir" @@ -963,7 +974,7 @@ async def _execute_research(target: str, ws=None): # Notify via voice if WebSocket still connected if ws: try: - notify_text = f"Research is complete, sir. Report is open in your browser." + notify_text = phrase("research.complete") audio = await synthesize_speech(notify_text) await ws.send_json({"type": "status", "state": "speaking"}) if audio: @@ -983,9 +994,9 @@ async def _execute_research(target: str, ws=None): try: audio = await synthesize_speech("Research timed out, sir. It was taking too long.") if audio: - await ws.send_json({"type": "audio", "data": base64.b64encode(audio).decode(), "text": "Research timed out, sir."}) + await ws.send_json({"type": "audio", "data": base64.b64encode(audio).decode(), "text": phrase("research.timed_out")}) else: - await ws.send_json({"type": "text", "text": "Research timed out, sir."}) + await ws.send_json({"type": "text", "text": phrase("research.timed_out")}) except Exception: pass except Exception as e: @@ -1051,7 +1062,7 @@ async def _execute_prompt_project(project_name: str, prompt: str, work_session: dispatch_id = dispatch_registry.register(project_name, project_dir or "", prompt) if not project_dir: - msg = f"Couldn't find the {project_name} project directory, sir." + msg = phrase("build.no_directory", project=project_name) audio = await synthesize_speech(msg) if ws: try: @@ -1095,7 +1106,7 @@ async def _execute_prompt_project(project_name: str, prompt: str, work_session: if not full_response or full_response.startswith("Hit a problem") or full_response.startswith("That's taking"): dispatch_registry.update_status(dispatch_id, "failed" if full_response else "timeout", response=full_response or "") - msg = f"Sir, I ran into an issue with {project_name}. {full_response[:150] if full_response else 'No response received.'}" + msg = phrase("build.issue", project=project_name, detail=full_response[:150] if full_response else "No response received.") else: # Summarize via Haiku — don't read word for word if anthropic_client: @@ -1117,9 +1128,9 @@ async def _execute_prompt_project(project_name: str, prompt: str, work_session: ) msg = summary.content[0].text except Exception: - msg = f"Sir, {project_name} finished. Here's the gist: {full_response[:200]}" + msg = phrase("build.finished", project=project_name, summary=full_response[:200]) else: - msg = f"Sir, {project_name} is done. {full_response[:200]}" + msg = phrase("build.done", project=project_name, summary=full_response[:200]) # Speak the result — skip if user has spoken recently to avoid audio collision log.info(f"Dispatch summary for {project_name}: {msg[:100]}") @@ -1150,7 +1161,7 @@ async def _execute_prompt_project(project_name: str, prompt: str, work_session: except Exception as e: log.error(f"Prompt project failed: {e}", exc_info=True) try: - msg = f"Had trouble connecting to {project_name}, sir." + msg = phrase("build.connect_trouble", project=project_name) audio = await synthesize_speech(msg) if ws: await ws.send_json({"type": "status", "state": "speaking"}) @@ -1179,7 +1190,7 @@ async def self_work_and_notify(session: WorkSession, prompt: str, ws): ) msg = summary.content[0].text except Exception: - msg = "Work is complete, sir." + msg = phrase("build.work_complete") try: audio = await synthesize_speech(msg) @@ -1549,6 +1560,12 @@ async def lifespan(application: FastAPI): anthropic_client = anthropic.AsyncAnthropic(api_key=ANTHROPIC_API_KEY) else: log.warning("ANTHROPIC_API_KEY not set — LLM features disabled") + + # Warm the fixed phrases for the configured language. Backgrounded: a + # translation costs one Haiku call on a cold cache, and JARVIS should not + # wait on it to start answering — until it lands, the English originals + # are used, which is what happened before this existed anyway. + asyncio.create_task(_refresh_phrases()) cached_projects = [] # Start context refresh in a separate thread (never touches event loop) @@ -1856,7 +1873,7 @@ async def _lookup_and_report(lookup_type: str, lookup_fn, ws, history: list[dict except asyncio.TimeoutError: _active_lookups[lookup_id]["status"] = "timeout" try: - fallback = f"That {lookup_type} check is taking too long, sir. The data may still be syncing." + fallback = phrase("lookup.slow", kind=lookup_type) audio = await synthesize_speech(fallback) await ws.send_json({"type": "status", "state": "speaking"}) if audio: @@ -2162,7 +2179,7 @@ async def _send_greeting(): if msg.get("type") == "fix_self": jarvis_dir = str(Path(__file__).parent) await work_session.start(jarvis_dir) - response_text = "Work mode active in my own repo, sir. Tell me what needs fixing." + response_text = phrase("mode.work_self") tts = strip_markdown_for_tts(response_text) await ws.send_json({"type": "status", "state": "speaking"}) audio = await synthesize_speech(tts) @@ -2226,7 +2243,7 @@ async def _send_greeting(): did = dispatch_registry.register(name, path, prompt[:200]) asyncio.create_task(_execute_prompt_project(name, prompt, work_session, ws, dispatch_id=did, history=history, voice_state=voice_state)) planner.reset() - response_text = "Building it now, sir." + response_text = phrase("ack.building") elif planner.active_plan and planner.active_plan.confirmed is False and planner.active_plan.current_question_index >= len(planner.active_plan.pending_questions): # Confirmation phase result = await planner.handle_confirmation(user_text) @@ -2239,10 +2256,10 @@ async def _send_greeting(): did = dispatch_registry.register(name, path, prompt[:200]) asyncio.create_task(_execute_prompt_project(name, prompt, work_session, ws, dispatch_id=did, history=history, voice_state=voice_state)) planner.reset() - response_text = "On it, sir." + response_text = phrase("ack.on_it") elif result["cancelled"]: planner.reset() - response_text = "Cancelled, sir." + response_text = phrase("ack.cancelled") else: response_text = result.get("modification_question", "How shall I adjust the plan, sir?") else: @@ -2255,9 +2272,9 @@ async def _send_greeting(): elif any(w in t_lower for w in ["quit work mode", "exit work mode", "go back to chat", "regular mode", "stop working"]): if work_session.active: await work_session.stop() - response_text = "Back to conversation mode, sir." + response_text = phrase("mode.back_to_chat") else: - response_text = "Already in conversation mode, sir." + response_text = phrase("mode.already_chat") # ── WORK MODE: speech → claude -p → Haiku summary → JARVIS voice ── elif work_session.active: @@ -2332,37 +2349,37 @@ async def _send_greeting(): elif action["action"] == "show_recent": response_text = await handle_show_recent() elif action["action"] == "describe_screen": - response_text = "Taking a look now, sir." + response_text = phrase("ack.taking_look") asyncio.create_task(_lookup_and_report("screen", _do_screen_lookup, ws, history=history, voice_state=voice_state)) elif action["action"] == "check_calendar": - response_text = "Checking your calendar now, sir." + response_text = phrase("ack.checking_calendar") asyncio.create_task(_lookup_and_report("calendar", _do_calendar_lookup, ws, history=history, voice_state=voice_state)) elif action["action"] == "check_mail": - response_text = "Checking your inbox now, sir." + response_text = phrase("ack.checking_mail") asyncio.create_task(_lookup_and_report("mail", _do_mail_lookup, ws, history=history, voice_state=voice_state)) elif action["action"] == "check_dispatch": recent = dispatch_registry.get_most_recent() if not recent: - response_text = "No recent builds on record, sir." + response_text = phrase("build.none_recent") else: name = recent["project_name"] status = recent["status"] if status == "building" or status == "pending": elapsed = int(time.time() - recent["updated_at"]) - response_text = f"Still working on {name}, sir. Been at it for {elapsed} seconds." + response_text = phrase("build.still_working", project=name, seconds=elapsed) elif status == "completed": response_text = recent.get("summary") or f"{name} is complete, sir." elif status in ("failed", "timeout"): - response_text = f"{name} ran into problems, sir." + response_text = phrase("build.problems", project=name) else: - response_text = f"{name} is {status}, sir." + response_text = phrase("build.status", project=name, status=status) elif action["action"] == "check_tasks": tasks = get_open_tasks() response_text = format_tasks_for_voice(tasks) elif action["action"] == "check_usage": response_text = get_usage_summary() else: - response_text = "Understood, sir." + response_text = phrase("ack.understood") else: if not anthropic_client: response_text = "API key not configured." @@ -2384,13 +2401,13 @@ async def _send_greeting(): action_type = embedded_action["action"] if action_type == "prompt_project": proj = embedded_action["target"].split("|||")[0].strip() - response_text = f"Connecting to {proj} now, sir." + response_text = phrase("ack.connecting", project=proj) elif action_type == "build": - response_text = "On it, sir." + response_text = phrase("ack.on_it") elif action_type == "research": - response_text = "Looking into that now, sir." + response_text = phrase("ack.looking_into") else: - response_text = "Right away, sir." + response_text = phrase("ack.generic") if embedded_action["action"] == "build": # Build in background — JARVIS stays conversational @@ -2494,9 +2511,9 @@ async def _send_greeting(): async def _read_and_report(search_term, _ws): note = await read_note(search_term) if note: - msg = f"Sir, your note '{note['title']}' says: {note['body'][:200]}" + msg = phrase("note.says", title=note["title"], body=note["body"][:200]) else: - msg = f"Couldn't find a note matching '{search_term}', sir." + msg = phrase("note.not_found", query=search_term) audio = await synthesize_speech(strip_markdown_for_tts(msg)) if _ws: try: @@ -2554,7 +2571,7 @@ async def _do_summary(): except Exception as e: log.error(f"Error: {e}", exc_info=True) try: - fallback = "Something went wrong, sir." + fallback = phrase("error.generic") audio = await synthesize_speech(fallback) if audio: await ws.send_json({"type": "audio", "data": base64.b64encode(audio).decode(), "text": fallback}) @@ -2749,6 +2766,8 @@ async def api_save_preferences(body: PreferencesUpdate): _write_env_key("SPEECH_LANG", body.speech_lang or DEFAULT_SPEECH_LANG) _write_env_key("TTS_BACKEND", body.tts_backend or "auto") _write_env_key("MACOS_VOICE", body.macos_voice) + # A new language needs its own set of fixed phrases. + asyncio.create_task(_refresh_phrases()) return {"success": True} # --------------------------------------------------------------------------- From 4e55402b6ac8ff11f9323e51949cebd9c3d8a012 Mon Sep 17 00:00:00 2001 From: Luca Trisiello Date: Tue, 11 Aug 2026 05:20:04 +0200 Subject: [PATCH 09/10] chore: ignore the generated phrase-translation cache data/phrases-.json is written at runtime by the translator and is keyed by a hash of the catalogue, so a committed copy would be both machine-specific and stale the moment a phrase changes. Co-Authored-By: Claude Opus 5 --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 9259a7df..f82d6083 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,7 @@ __pycache__/ *.db data/*.jsonl data/active_session.json +data/phrases-*.json data/.jarvis_output.txt # Certificates From cb78b7ea6292b65486abc983b6994afdd228cc90 Mon Sep 17 00:00:00 2001 From: Luca Trisiello Date: Tue, 11 Aug 2026 05:24:36 +0200 Subject: [PATCH 10/10] fix: move deep research to Opus 5 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit handle_research was pinned to claude-opus-4-6, three Opus generations behind. Swapping the model string alone would have broken it. Thinking is on by default on Opus 5, unlike 4.6 where omitting the parameter meant no thinking, so the response now arrives as ['thinking', 'text'] and content[0].text raises AttributeError — a ThinkingBlock has no .text. Verified against the live API rather than inferred. It now joins the text blocks and ignores the reasoning. max_tokens covers thinking and answer together on Opus 5, so 2000 — sized for prose on a non-thinking model — truncated the report. Raised to 16000, the ceiling before SDK HTTP timeouts constrain a non-streaming request. Opus 5's classifiers can decline a request, returning a successful HTTP 200 with stop_reason "refusal" and empty content; indexing content[0] would crash instead of reporting back. Checked first, and fallbacks="default" lets the API re-run a declined request elsewhere, routing by refusal category rather than pinning a model that will itself need migrating. Note handle_research has no callers: [ACTION:RESEARCH] dispatches to _execute_research, which spawns Claude Code. This changes no runtime behaviour today — it fixes code that would fail the moment anything reached it. Co-Authored-By: Claude Opus 5 --- phrases.py | 2 ++ server.py | 28 ++++++++++++++++++++++++---- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/phrases.py b/phrases.py index 06ac0792..25e765be 100644 --- a/phrases.py +++ b/phrases.py @@ -62,6 +62,8 @@ # Research "research.complete": "Research is complete, {honorific}. The report is open in your browser.", "research.timed_out": "Research timed out, {honorific}.", + "research.declined": "I'm not able to research that one, {honorific}.", + "research.empty": "That research came back empty, {honorific}.", # Lookups "lookup.slow": "That {kind} check is taking too long, {honorific}. The data may still be syncing.", diff --git a/server.py b/server.py index 661c0377..61fe0ee9 100644 --- a/server.py +++ b/server.py @@ -2015,13 +2015,33 @@ async def handle_browse(text: str, target: str) -> str: async def handle_research(text: str, target: str, client: anthropic.AsyncAnthropic) -> str: """Deep research with Opus — write results to HTML, open in browser.""" try: - research_response = await client.messages.create( - model="claude-opus-4-6", - max_tokens=2000, + research_response = await client.beta.messages.create( + model="claude-opus-5", + # Thinking is on by default on Opus 5 and shares this budget with the + # answer, so a limit sized for prose alone truncates the report. + max_tokens=16000, + # Opus 5's safety classifiers can decline a request outright. Letting + # the API re-run it elsewhere turns a dead end into an answer, and + # "default" routes by refusal category rather than pinning a model + # that will eventually be retired. + betas=["server-side-fallback-2026-07-01"], + fallbacks="default", system=f"You are JARVIS, researching a topic for {USER_NAME}. Be thorough, organized, and cite sources where possible.", messages=[{"role": "user", "content": f"Research this thoroughly:\n\n{target}"}], ) - research_text = research_response.content[0].text + + if research_response.stop_reason == "refusal": + log.warning(f"Research declined for: {target[:80]}") + return phrase("research.declined") + + # Take the text blocks rather than content[0]: with thinking on, the + # first block is the reasoning, which carries no .text at all. + research_text = "\n".join( + b.text for b in research_response.content if b.type == "text" + ).strip() + if not research_text: + log.warning("Research returned no text content") + return phrase("research.empty") import html as _html html_content = f"""