diff --git a/.env.example b/.env.example
index 51090741..4fabc917 100644
--- a/.env.example
+++ b/.env.example
@@ -2,6 +2,17 @@
ANTHROPIC_API_KEY=your-anthropic-api-key-here
FISH_API_KEY=your-fish-audio-api-key-here
+# Optional: which speech synthesiser to use.
+# auto (default) — Fish Audio when its key is set, otherwise macOS voices
+# fish — always Fish Audio
+# macos — always the built-in macOS voices: free, offline, no key
+# none — silent; replies appear as on-screen captions only
+# TTS_BACKEND=auto
+
+# Optional: which macOS voice to speak with (see `say -v ?`).
+# Empty picks the best installed voice for SPEECH_LANG.
+# MACOS_VOICE=Alice
+
# Optional: Fish Audio voice model (defaults to JARVIS MCU voice)
# FISH_VOICE_ID=612b878b113047d9a770c069c8b4fdfe
diff --git a/.gitignore b/.gitignore
index 9259a7df..f82d6083 100644
--- a/.gitignore
+++ b/.gitignore
@@ -17,6 +17,7 @@ __pycache__/
*.db
data/*.jsonl
data/active_session.json
+data/phrases-*.json
data/.jarvis_output.txt
# Certificates
diff --git a/calendar_access.py b/calendar_access.py
index c91d0903..8695fbcf 100644
--- a/calendar_access.py
+++ b/calendar_access.py
@@ -7,6 +7,8 @@
import asyncio
import logging
+
+from phrases import phrase
import os
import time as _time
from datetime import datetime, timedelta
@@ -245,14 +247,14 @@ def format_events_for_context(events: list[dict]) -> str:
def format_schedule_summary(events: list[dict]) -> str:
"""Format a brief voice-friendly summary of the schedule."""
if not events:
- return "Your schedule is clear today, sir."
+ return phrase("calendar.clear")
count = len(events)
if count == 1:
evt = events[0]
if evt.get("all_day"):
- return f"You have one all-day event: {evt['title']}."
- return f"You have one event: {evt['title']} at {evt['start']}."
+ return phrase("calendar.one_all_day", title=evt["title"])
+ return phrase("calendar.one_event", title=evt["title"], time=evt["start"])
summaries = []
for evt in events[:5]:
diff --git a/frontend/index.html b/frontend/index.html
index 9440bda4..2d6aba1b 100644
--- a/frontend/index.html
+++ b/frontend/index.html
@@ -32,6 +32,7 @@
+
JARVIS
diff --git a/frontend/package-lock.json b/frontend/package-lock.json
index 1f459602..b74e0ce7 100644
--- a/frontend/package-lock.json
+++ b/frontend/package-lock.json
@@ -948,9 +948,9 @@
"license": "MIT"
},
"node_modules/nanoid": {
- "version": "3.3.11",
- "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.11.tgz",
- "integrity": "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w==",
+ "version": "3.3.18",
+ "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.18.tgz",
+ "integrity": "sha512-DTg4MJbGMWkfi6VZFdNt2/caMbQy4Ou+Op/hJQvGEWcnVfoA1QA+xzRKAzw9jD6+GVOOeYr/mIcuDSdug6F6+w==",
"dev": true,
"funding": [
{
@@ -974,9 +974,9 @@
"license": "ISC"
},
"node_modules/picomatch": {
- "version": "4.0.3",
- "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.3.tgz",
- "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==",
+ "version": "4.0.5",
+ "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.5.tgz",
+ "integrity": "sha512-RvwwcruNjI1ncT5xRakeyS9Lf8lcItv34KD+aif+VH9kduAyfYBipGh12274xtenIPZ119/R9BdTBa8gAwSh0A==",
"dev": true,
"license": "MIT",
"engines": {
@@ -987,9 +987,9 @@
}
},
"node_modules/postcss": {
- "version": "8.5.8",
- "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.8.tgz",
- "integrity": "sha512-OW/rX8O/jXnm82Ey1k44pObPtdblfiuWnrd8X7GJ7emImCOstunGbXUpp7HdBrFQX6rJzn3sPT397Wp5aCwCHg==",
+ "version": "8.5.26",
+ "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.26.tgz",
+ "integrity": "sha512-u82N74LFzG8ca+dD8puPnplTXoGH4fTPpVGuIbt36G3qvNlkvfD0lEAZSxaly3KX8TS/L1A1gsCEmvKmBcVbkQ==",
"dev": true,
"funding": [
{
@@ -1007,7 +1007,7 @@
],
"license": "MIT",
"dependencies": {
- "nanoid": "^3.3.11",
+ "nanoid": "^3.3.17",
"picocolors": "^1.1.1",
"source-map-js": "^1.2.1"
},
@@ -1108,9 +1108,9 @@
}
},
"node_modules/vite": {
- "version": "6.4.1",
- "resolved": "https://registry.npmjs.org/vite/-/vite-6.4.1.tgz",
- "integrity": "sha512-+Oxm7q9hDoLMyJOYfUYBuHQo+dkAloi33apOPP56pzj+vsdJDzr+j1NISE5pyaAuKL4A3UD34qd0lx5+kfKp2g==",
+ "version": "6.4.3",
+ "resolved": "https://registry.npmjs.org/vite/-/vite-6.4.3.tgz",
+ "integrity": "sha512-NTKlcQjlAK7MlQoyb6LgaqHc8sso/pVyUJYWMws3jg21uTJw/LddqIFPcPqP6PzpgbIcZyKI85sFE4HBrQDA8A==",
"dev": true,
"license": "MIT",
"dependencies": {
diff --git a/frontend/src/main.ts b/frontend/src/main.ts
index ca5d1864..e6b08f2d 100644
--- a/frontend/src/main.ts
+++ b/frontend/src/main.ts
@@ -8,7 +8,7 @@
import { createOrb, type OrbState } from "./orb";
import { createVoiceInput, createAudioPlayer } from "./voice";
import { createSocket } from "./ws";
-import { openSettings, checkFirstTimeSetup } from "./settings";
+import { openSettings, checkFirstTimeSetup, SPEECH_LANG_EVENT } from "./settings";
import "./style.css";
// ---------------------------------------------------------------------------
@@ -21,6 +21,7 @@ let isMuted = false;
const statusEl = document.getElementById("status-text")!;
const errorEl = document.getElementById("error-text")!;
+const captionEl = document.getElementById("jarvis-caption")!;
function showError(msg: string) {
errorEl.textContent = msg;
@@ -30,6 +31,28 @@ function showError(msg: string) {
}, 5000);
}
+let captionTimer: number | undefined;
+
+/**
+ * Put what JARVIS said on screen.
+ *
+ * Shown whether or not TTS produced audio: with a voice it reads as
+ * subtitles, and without one it is the only way to see the reply — the
+ * response used to reach nothing but the devtools console.
+ */
+function showCaption(text: unknown) {
+ if (typeof text !== "string" || !text.trim()) return;
+ clearTimeout(captionTimer);
+ captionEl.textContent = text;
+ captionEl.style.opacity = "1";
+ // Hold long enough to read the line, since a caption may be all the user
+ // gets. Replies are a sentence or two, so length is a fair proxy.
+ const holdMs = Math.min(20000, 4000 + text.length * 60);
+ captionTimer = window.setTimeout(() => {
+ captionEl.style.opacity = "0";
+ }, holdMs);
+}
+
function updateStatus(state: State) {
const labels: Record = {
idle: "",
@@ -80,13 +103,59 @@ function transition(newState: State) {
// Voice input
// ---------------------------------------------------------------------------
+/**
+ * Fragment accumulation.
+ *
+ * Chrome closes a recognition segment at every pause, so a sentence spoken
+ * with any hesitation arrives as several "final" results. Sending each one
+ * immediately made JARVIS answer half a thought and then receive the rest as
+ * a separate question. Instead, collect the pieces and send once the speaker
+ * has actually stopped.
+ */
+const UTTERANCE_GAP_MS = 1000;
+// A pause never arrives while someone is dictating steadily, so cap the wait
+// rather than let a long sentence hold the whole conversation open.
+const UTTERANCE_MAX_MS = 6000;
+
+let pendingFragments: string[] = [];
+let gapTimer: number | undefined;
+let maxTimer: number | undefined;
+
+function discardUtterance() {
+ clearTimeout(gapTimer);
+ clearTimeout(maxTimer);
+ gapTimer = undefined;
+ maxTimer = undefined;
+ pendingFragments = [];
+}
+
+function flushUtterance() {
+ clearTimeout(gapTimer);
+ clearTimeout(maxTimer);
+ gapTimer = undefined;
+ maxTimer = undefined;
+
+ const text = pendingFragments.join(" ").replace(/\s+/g, " ").trim();
+ pendingFragments = [];
+ if (!text) return;
+
+ socket.send({ type: "transcript", text, isFinal: true });
+ transition("thinking");
+}
+
const voiceInput = createVoiceInput(
(text: string) => {
- // Cancel any current JARVIS response before sending new input
+ // Cancel any current JARVIS response before sending new input. Done on the
+ // first fragment, not at flush time: cutting him off is what the user
+ // wanted the moment they started talking.
audioPlayer.stop();
- // User spoke — send transcript
- socket.send({ type: "transcript", text, isFinal: true });
- transition("thinking");
+
+ pendingFragments.push(text);
+ clearTimeout(gapTimer);
+ gapTimer = window.setTimeout(flushUtterance, UTTERANCE_GAP_MS);
+ if (maxTimer === undefined) {
+ maxTimer = window.setTimeout(flushUtterance, UTTERANCE_MAX_MS);
+ }
},
(msg: string) => {
showError(msg);
@@ -123,6 +192,7 @@ socket.onMessage((msg) => {
}
// Log text for debugging
if (msg.text) console.log("[JARVIS]", msg.text);
+ showCaption(msg.text);
} else if (type === "status") {
const state = msg.state as string;
if (state === "thinking" && currentState !== "thinking") {
@@ -137,6 +207,7 @@ socket.onMessage((msg) => {
} else if (type === "text") {
// Text fallback when TTS fails
console.log("[JARVIS]", msg.text);
+ showCaption(msg.text);
} else if (type === "task_spawned") {
console.log("[task]", "spawned:", msg.task_id, msg.prompt);
} else if (type === "task_complete") {
@@ -148,12 +219,40 @@ socket.onMessage((msg) => {
// Kick off
// ---------------------------------------------------------------------------
+/**
+ * Read the speech language chosen in the settings panel.
+ *
+ * Resolved before the microphone opens so the first session is built in the
+ * right language: switching a live session means tearing it down and racing
+ * for the microphone, which is worth avoiding on every page load. Bounded, so
+ * an unreachable server costs a moment rather than the microphone.
+ */
+async function fetchSpeechLanguage(): Promise {
+ try {
+ const res = await fetch("/api/settings/preferences", {
+ signal: AbortSignal.timeout(2000),
+ });
+ const prefs = (await res.json()) as { speech_lang?: string };
+ return prefs.speech_lang || null;
+ } catch {
+ return null; // Server not ready — keep the default language.
+ }
+}
+
// Start listening after a brief delay for the orb to render
-setTimeout(() => {
+setTimeout(async () => {
+ const lang = await fetchSpeechLanguage();
+ if (lang) voiceInput.setLanguage(lang);
voiceInput.start();
transition("listening");
}, 1000);
+// Saving the setting re-tunes the running session — no reload needed.
+document.addEventListener(SPEECH_LANG_EVENT, (e) => {
+ const lang = (e as CustomEvent).detail;
+ if (lang) voiceInput.setLanguage(lang);
+});
+
// Resume AudioContext on ANY user interaction (browser autoplay policy)
function ensureAudioContext() {
const ctx = audioPlayer.getAnalyser().context as AudioContext;
@@ -183,6 +282,8 @@ btnMute.addEventListener("click", (e) => {
isMuted = !isMuted;
btnMute.classList.toggle("muted", isMuted);
if (isMuted) {
+ // Half-spoken words must not arrive a second after the user silenced him.
+ discardUtterance();
voiceInput.pause();
transition("idle");
} else {
diff --git a/frontend/src/settings.ts b/frontend/src/settings.ts
index 7e945ef7..9b60daca 100644
--- a/frontend/src/settings.ts
+++ b/frontend/src/settings.ts
@@ -30,8 +30,24 @@ interface PreferencesResponse {
user_name: string;
honorific: string;
calendar_accounts: string;
+ speech_lang: string;
+ tts_backend: string;
+ macos_voice: string;
}
+interface VoicesResponse {
+ voices: { name: string; lang: string }[];
+ matching_language: number;
+ resolved: string | null;
+}
+
+/**
+ * Event fired when the speech language changes, so the live recognition
+ * session picks it up without a reload. Used instead of importing the voice
+ * input directly, which would make settings and main circular.
+ */
+export const SPEECH_LANG_EVENT = "jarvis:speech-lang";
+
// ---------------------------------------------------------------------------
// State
// ---------------------------------------------------------------------------
@@ -136,13 +152,50 @@ function buildPanelHTML(): string {
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -240,16 +293,54 @@ async function loadPreferences() {
try {
const prefs = await apiGet("/api/settings/preferences");
const nameEl = document.getElementById("input-user-name") as HTMLInputElement;
- const honEl = document.getElementById("input-honorific") as HTMLSelectElement;
+ const honEl = document.getElementById("input-honorific") as HTMLInputElement;
const calEl = document.getElementById("input-calendar-accounts") as HTMLTextAreaElement;
+ const langEl = document.getElementById("input-speech-lang") as HTMLSelectElement;
if (nameEl) nameEl.value = prefs.user_name || "";
if (honEl) honEl.value = prefs.honorific || "sir";
if (calEl) calEl.value = prefs.calendar_accounts || "auto";
+ if (langEl) langEl.value = prefs.speech_lang || "en-US";
+
+ const backendEl = document.getElementById("input-tts-backend") as HTMLSelectElement;
+ if (backendEl) backendEl.value = prefs.tts_backend || "auto";
+ await loadVoices(prefs.macos_voice || "");
+ updateVoiceFieldVisibility();
} catch (e) {
console.error("[settings] failed to load preferences:", e);
}
}
+/** Only offer the voice picker when a macOS voice could actually be used. */
+function updateVoiceFieldVisibility() {
+ const backend = (document.getElementById("input-tts-backend") as HTMLSelectElement)?.value;
+ const field = document.getElementById("field-macos-voice");
+ if (field) field.style.display = backend === "fish" || backend === "none" ? "none" : "";
+}
+
+async function loadVoices(selected: string) {
+ const voiceEl = document.getElementById("input-macos-voice") as HTMLSelectElement;
+ if (!voiceEl) return;
+ try {
+ const data = await apiGet("/api/settings/voices");
+ // "Automatic" first, labelled with what that currently resolves to, so the
+ // default is not a mystery.
+ const auto = data.resolved ? `Automatic (${data.resolved})` : "Automatic";
+ voiceEl.innerHTML =
+ `` +
+ data.voices
+ .map((v, i) => {
+ // Voices for the current language are listed first; mark where the
+ // rest begin rather than silently mixing them together.
+ const divider = i === data.matching_language && i > 0 ? "— other languages — " : "";
+ return ``;
+ })
+ .join("");
+ voiceEl.value = selected;
+ } catch (e) {
+ console.error("[settings] failed to load voices:", e);
+ }
+}
+
function wireEvents() {
// Close
document.getElementById("settings-close")?.addEventListener("click", closeSettings);
@@ -304,12 +395,50 @@ function wireEvents() {
// Save preferences
document.getElementById("btn-save-prefs")?.addEventListener("click", async () => {
const user_name = (document.getElementById("input-user-name") as HTMLInputElement).value.trim();
- const honorific = (document.getElementById("input-honorific") as HTMLSelectElement).value;
+ const honorific = (document.getElementById("input-honorific") as HTMLInputElement).value;
const calendar_accounts = (document.getElementById("input-calendar-accounts") as HTMLTextAreaElement).value.trim();
- await apiPost("/api/settings/preferences", { user_name, honorific, calendar_accounts });
+ const speech_lang = (document.getElementById("input-speech-lang") as HTMLSelectElement).value;
+ const tts_backend = (document.getElementById("input-tts-backend") as HTMLSelectElement).value;
+ const macos_voice = (document.getElementById("input-macos-voice") as HTMLSelectElement).value;
+ await apiPost("/api/settings/preferences", {
+ user_name, honorific, calendar_accounts, speech_lang, tts_backend, macos_voice,
+ });
+ document.dispatchEvent(new CustomEvent(SPEECH_LANG_EVENT, { detail: speech_lang }));
await loadStatus();
});
+ // Voice backend — show or hide the voice picker to match
+ document.getElementById("input-tts-backend")?.addEventListener("change", updateVoiceFieldVisibility);
+
+ // Audition a voice before committing to it
+ document.getElementById("btn-preview-voice")?.addEventListener("click", async (e) => {
+ e.stopPropagation();
+ const btn = e.currentTarget as HTMLButtonElement;
+ const voice = (document.getElementById("input-macos-voice") as HTMLSelectElement).value;
+ const original = btn.textContent;
+ btn.textContent = "...";
+ btn.disabled = true;
+ try {
+ const res = await apiGet<{ audio: string | null }>(
+ `/api/tts-test?voice=${encodeURIComponent(voice)}`
+ );
+ if (res.audio) {
+ const bytes = Uint8Array.from(atob(res.audio), (c) => c.charCodeAt(0));
+ const ctx = new AudioContext();
+ const buffer = await ctx.decodeAudioData(bytes.buffer);
+ const src = ctx.createBufferSource();
+ src.buffer = buffer;
+ src.connect(ctx.destination);
+ src.start();
+ }
+ } catch (err) {
+ console.error("[settings] voice preview failed:", err);
+ } finally {
+ btn.textContent = original;
+ btn.disabled = false;
+ }
+ });
+
// Setup next button
document.getElementById("btn-setup-next")?.addEventListener("click", advanceSetup);
}
diff --git a/frontend/src/style.css b/frontend/src/style.css
index 899901c7..97380461 100644
--- a/frontend/src/style.css
+++ b/frontend/src/style.css
@@ -35,6 +35,27 @@ html, body {
z-index: 10;
}
+/* What JARVIS just said. Sits above the status line, and carries the
+ conversation on its own when TTS is unavailable. */
+#jarvis-caption {
+ position: fixed;
+ bottom: 80px;
+ left: 50%;
+ transform: translateX(-50%);
+ width: min(640px, calc(100% - 48px));
+ color: rgba(219, 238, 252, 0.92);
+ font-size: 17px;
+ line-height: 1.55;
+ font-weight: 300;
+ letter-spacing: 0.3px;
+ text-align: center;
+ text-shadow: 0 2px 12px rgba(0, 0, 0, 0.85);
+ transition: opacity 0.45s ease;
+ pointer-events: none;
+ z-index: 10;
+ opacity: 0;
+}
+
#jarvis-label {
position: fixed;
bottom: 16px;
diff --git a/frontend/src/voice.ts b/frontend/src/voice.ts
index 8ca5e0a8..158280e3 100644
--- a/frontend/src/voice.ts
+++ b/frontend/src/voice.ts
@@ -11,8 +11,12 @@ export interface VoiceInput {
stop(): void;
pause(): void;
resume(): void;
+ setLanguage(lang: string): void;
}
+/** Fallback until the configured language arrives from the server. */
+export const DEFAULT_SPEECH_LANG = "en-US";
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
declare const webkitSpeechRecognition: any;
@@ -24,48 +28,65 @@ export function createVoiceInput(
const SR = (window as any).SpeechRecognition || (typeof webkitSpeechRecognition !== "undefined" ? webkitSpeechRecognition : null);
if (!SR) {
onError("Speech recognition not supported in this browser");
- return { start() {}, stop() {}, pause() {}, resume() {} };
+ return { start() {}, stop() {}, pause() {}, resume() {}, setLanguage() {} };
}
- const recognition = new SR();
- recognition.continuous = true;
- recognition.interimResults = true;
- recognition.lang = "en-US";
-
let shouldListen = false;
let paused = false;
+ let currentLang = DEFAULT_SPEECH_LANG;
+
+ /**
+ * Build a recognition session.
+ *
+ * Chrome reads `lang` when the object is constructed and ignores later
+ * assignment on a session that has already run, so switching language means
+ * building a new object rather than re-tagging this one.
+ */
+ function buildRecognition(): any {
+ const r = new SR();
+ r.continuous = true;
+ r.interimResults = true;
+ r.lang = currentLang;
+
+ r.onresult = (event: any) => {
+ for (let i = event.resultIndex; i < event.results.length; i++) {
+ if (event.results[i].isFinal) {
+ const text = event.results[i][0].transcript.trim();
+ if (text) onTranscript(text);
+ }
+ }
+ };
- recognition.onresult = (event: any) => {
- for (let i = event.resultIndex; i < event.results.length; i++) {
- if (event.results[i].isFinal) {
- const text = event.results[i][0].transcript.trim();
- if (text) onTranscript(text);
+ r.onend = () => {
+ // A session replaced by a language switch must stay down, or it would
+ // race the new one for the microphone and keep the old language alive.
+ if (r !== recognition) return;
+ if (shouldListen && !paused) {
+ try {
+ r.start();
+ } catch {
+ // Already started
+ }
}
- }
- };
+ };
- recognition.onend = () => {
- if (shouldListen && !paused) {
- try {
- recognition.start();
- } catch {
- // Already started
+ r.onerror = (event: any) => {
+ if (event.error === "not-allowed") {
+ onError("Microphone access denied. Please allow microphone access.");
+ shouldListen = false;
+ } else if (event.error === "no-speech") {
+ // Normal, just restart
+ } else if (event.error === "aborted") {
+ // Expected during pause
+ } else {
+ console.warn("[voice] recognition error:", event.error);
}
- }
- };
+ };
- recognition.onerror = (event: any) => {
- if (event.error === "not-allowed") {
- onError("Microphone access denied. Please allow microphone access.");
- shouldListen = false;
- } else if (event.error === "no-speech") {
- // Normal, just restart
- } else if (event.error === "aborted") {
- // Expected during pause
- } else {
- console.warn("[voice] recognition error:", event.error);
- }
- };
+ return r;
+ }
+
+ let recognition = buildRecognition();
return {
start() {
@@ -96,6 +117,34 @@ export function createVoiceInput(
}
}
},
+ setLanguage(lang: string) {
+ if (!lang || lang === currentLang) return;
+ currentLang = lang;
+
+ const previous = recognition;
+ recognition = buildRecognition();
+ // Retires the old session: its onend now sees itself superseded.
+ try {
+ previous.stop();
+ } catch {
+ // Wasn't running.
+ }
+
+ if (shouldListen && !paused) {
+ // Let the retired session release the microphone before claiming it,
+ // otherwise Chrome rejects the new start outright.
+ setTimeout(() => {
+ if (recognition !== previous && shouldListen && !paused) {
+ try {
+ recognition.start();
+ } catch {
+ // Already started
+ }
+ }
+ }, 250);
+ }
+ console.log("[voice] language set to", lang);
+ },
};
}
diff --git a/mail_access.py b/mail_access.py
index 03a88fe5..410d66cf 100644
--- a/mail_access.py
+++ b/mail_access.py
@@ -10,6 +10,8 @@
import asyncio
import logging
+
+from phrases import phrase
from datetime import datetime
log = logging.getLogger("jarvis.mail")
@@ -348,7 +350,7 @@ def format_unread_summary(unread: dict) -> str:
"""Format unread counts for voice."""
total = unread["total"]
if total == 0:
- return "Inbox is clear, sir. No unread messages."
+ return phrase("mail.clear")
parts = []
for acct, count in unread["accounts"].items():
@@ -356,9 +358,9 @@ def format_unread_summary(unread: dict) -> str:
parts.append(f"{count} in {acct}")
if len(parts) == 1:
- return f"You have {total} unread {'message' if total == 1 else 'messages'} — {parts[0]}."
+ return phrase("mail.one_account", total=total, detail=parts[0])
elif parts:
- return f"You have {total} unread messages: {', '.join(parts)}."
+ return phrase("mail.many_accounts", total=total, detail=", ".join(parts))
else:
return f"You have {total} unread {'message' if total == 1 else 'messages'}."
diff --git a/phrases.py b/phrases.py
new file mode 100644
index 00000000..25e765be
--- /dev/null
+++ b/phrases.py
@@ -0,0 +1,245 @@
+"""
+JARVIS fixed phrases — the lines JARVIS says that no LLM wrote.
+
+Acknowledgements, lookup results and error lines are built in code, so they
+stayed English no matter which language the user had configured: a reply would
+arrive as "Checking your calendar now, sir." followed by fluent Italian.
+
+Rather than ship hand-written translations for every language the panel offers
+— inventing Japanese butler register is not something to do blind — the
+English catalogue below is translated once per language by the same model
+JARVIS already talks through, and cached on disk. The cache is keyed by a hash
+of the catalogue, so editing any phrase re-translates rather than serving a
+stale set.
+
+Until a translation exists the English original is used, which keeps a first
+run talking rather than silent.
+"""
+
+import asyncio
+import hashlib
+import json
+import logging
+import os
+from pathlib import Path
+
+log = logging.getLogger("jarvis.phrases")
+
+_CACHE_DIR = Path(__file__).parent / "data"
+
+# Placeholders are named, never positional, so a translator can reorder them
+# freely — word order is exactly what changes between languages.
+CATALOGUE: dict[str, str] = {
+ # Acknowledgements — spoken while something slower happens
+ "ack.generic": "Right away, {honorific}.",
+ "ack.on_it": "On it, {honorific}.",
+ "ack.understood": "Understood, {honorific}.",
+ "ack.cancelled": "Cancelled, {honorific}.",
+ "ack.looking_into": "Looking into that now, {honorific}.",
+ "ack.taking_look": "Taking a look now, {honorific}.",
+ "ack.checking_calendar": "Checking your calendar now, {honorific}.",
+ "ack.checking_mail": "Checking your inbox now, {honorific}.",
+ "ack.building": "Building it now, {honorific}.",
+ "ack.connecting": "Connecting to {project} now, {honorific}.",
+
+ # Modes
+ "mode.work_self": "Work mode active in my own repo, {honorific}. Tell me what needs fixing.",
+ "mode.back_to_chat": "Back to conversation mode, {honorific}.",
+ "mode.already_chat": "Already in conversation mode, {honorific}.",
+
+ # Build and project status
+ "build.none_recent": "No recent builds on record, {honorific}.",
+ "build.still_working": "Still working on {project}, {honorific}. Been at it for {seconds} seconds.",
+ "build.problems": "{project} ran into problems, {honorific}.",
+ "build.status": "{project} is {status}, {honorific}.",
+ "build.finished": "{honorific}, {project} finished. Here's the gist: {summary}",
+ "build.done": "{honorific}, {project} is done. {summary}",
+ "build.issue": "{honorific}, I ran into an issue with {project}. {detail}",
+ "build.no_directory": "Couldn't find the {project} project directory, {honorific}.",
+ "build.connect_trouble": "Had trouble connecting to {project}, {honorific}.",
+ "build.work_complete": "Work is complete, {honorific}.",
+
+ # Research
+ "research.complete": "Research is complete, {honorific}. The report is open in your browser.",
+ "research.timed_out": "Research timed out, {honorific}.",
+ "research.declined": "I'm not able to research that one, {honorific}.",
+ "research.empty": "That research came back empty, {honorific}.",
+
+ # Lookups
+ "lookup.slow": "That {kind} check is taking too long, {honorific}. The data may still be syncing.",
+
+ # Calendar
+ "calendar.clear": "Your schedule is clear today, {honorific}.",
+ "calendar.one_all_day": "You have one all-day event: {title}.",
+ "calendar.one_event": "You have one event: {title} at {time}.",
+
+ # Mail
+ "mail.clear": "Inbox is clear, {honorific}. No unread messages.",
+ "mail.one_account": "You have {total} unread in {detail}.",
+ "mail.many_accounts": "You have {total} unread messages: {detail}.",
+
+ # Notes
+ "note.says": "{honorific}, your note '{title}' says: {body}",
+ "note.not_found": "Couldn't find a note matching '{query}', {honorific}.",
+
+ # Failures
+ "error.generic": "Something went wrong, {honorific}.",
+}
+
+_active_lang = "en"
+_active: dict[str, str] = {}
+
+
+def _catalogue_hash() -> str:
+ blob = json.dumps(CATALOGUE, sort_keys=True, ensure_ascii=False)
+ return hashlib.sha256(blob.encode("utf-8")).hexdigest()[:16]
+
+
+def _cache_path(lang: str) -> Path:
+ return _CACHE_DIR / f"phrases-{lang}.json"
+
+
+def phrase(key: str, **kwargs) -> str:
+ """The phrase for `key`, in the active language, with placeholders filled.
+
+ Falls back to English whenever a translation is missing or malformed, so a
+ bad translation degrades to the original rather than to a crash.
+ """
+ template = _active.get(key) or CATALOGUE.get(key, "")
+ if not template:
+ log.warning(f"Unknown phrase key: {key}")
+ return ""
+ kwargs.setdefault("honorific", os.getenv("HONORIFIC", "sir").strip() or "sir")
+ try:
+ return template.format(**kwargs)
+ except (KeyError, IndexError) as e:
+ # A translation that dropped or renamed a placeholder must not take the
+ # line down with it.
+ log.warning(f"Phrase {key} failed to format ({e}); using English")
+ try:
+ return CATALOGUE[key].format(**kwargs)
+ except Exception:
+ return CATALOGUE.get(key, "")
+
+
+def load_cached(lang: str) -> bool:
+ """Load a cached translation for `lang`. Returns whether one was usable."""
+ global _active, _active_lang
+ _active_lang = lang
+ if lang.startswith("en"):
+ _active = {}
+ return True
+
+ path = _cache_path(lang)
+ if not path.exists():
+ _active = {}
+ return False
+ try:
+ data = json.loads(path.read_text(encoding="utf-8"))
+ except Exception as e:
+ log.warning(f"Could not read phrase cache for {lang}: {e}")
+ _active = {}
+ return False
+
+ if data.get("source_hash") != _catalogue_hash():
+ log.info(f"Phrase cache for {lang} is stale — will re-translate")
+ _active = {}
+ return False
+
+ # Validate on the way in as well as on the way out. A phrase missing a
+ # placeholder does not raise — str.format ignores surplus arguments — so it
+ # would quietly drop the project name or the count instead of failing
+ # loudly. Cache files can be edited, copied between machines, or truncated.
+ _active = {}
+ for key, text in (data.get("phrases") or {}).items():
+ english = CATALOGUE.get(key)
+ if english is None or not isinstance(text, str):
+ continue
+ if _placeholders(text) != _placeholders(english):
+ log.warning(f"Cached phrase {key} has the wrong placeholders; using English")
+ continue
+ _active[key] = text
+ return bool(_active)
+
+
+async def translate(lang: str, language_name: str, client) -> bool:
+ """Translate the catalogue into `lang` and cache it. Returns success."""
+ global _active
+ if lang.startswith("en"):
+ _active = {}
+ return True
+
+ prompt = (
+ f"Translate these interface strings into {language_name}.\n\n"
+ "They are spoken aloud by JARVIS, a British butler AI: dry, economical, "
+ "never chatty. Match that register in the target language.\n\n"
+ "Rules:\n"
+ "- Return ONLY a JSON object with exactly the same keys.\n"
+ "- Keep every {placeholder} verbatim, including the braces. Reorder them "
+ "freely to suit the language, but never rename, drop or add one.\n"
+ "- {honorific} is the user's chosen form of address. Leave it as the "
+ "placeholder; do not replace it with a word.\n"
+ "- Keep them short. These are spoken, not written.\n\n"
+ f"{json.dumps(CATALOGUE, ensure_ascii=False, indent=1)}"
+ )
+
+ try:
+ response = await client.messages.create(
+ model="claude-haiku-4-5-20251001",
+ max_tokens=4000,
+ messages=[{"role": "user", "content": prompt}],
+ )
+ raw = response.content[0].text.strip()
+ if raw.startswith("```"):
+ raw = raw.split("```")[1]
+ if raw.startswith("json"):
+ raw = raw[4:]
+ translated = json.loads(raw)
+ except Exception as e:
+ log.warning(f"Phrase translation for {lang} failed: {e}")
+ return False
+
+ # Keep only keys we asked for, and only where every placeholder survived.
+ # A phrase that lost one would raise at format time on some future call
+ # with no way to see it coming.
+ clean = {}
+ for key, english in CATALOGUE.items():
+ candidate = translated.get(key)
+ if not isinstance(candidate, str) or not candidate.strip():
+ continue
+ expected = _placeholders(english)
+ if _placeholders(candidate) != expected:
+ log.warning(f"Phrase {key} lost a placeholder in {lang}; keeping English")
+ continue
+ clean[key] = candidate.strip()
+
+ if not clean:
+ return False
+
+ _active = clean
+ try:
+ _CACHE_DIR.mkdir(parents=True, exist_ok=True)
+ _cache_path(lang).write_text(
+ json.dumps({"source_hash": _catalogue_hash(), "phrases": clean},
+ ensure_ascii=False, indent=1),
+ encoding="utf-8",
+ )
+ except OSError as e:
+ log.warning(f"Could not cache phrases for {lang}: {e}")
+
+ log.info(f"Translated {len(clean)}/{len(CATALOGUE)} phrases into {language_name}")
+ return True
+
+
+def _placeholders(text: str) -> set[str]:
+ import re
+ return set(re.findall(r"\{(\w+)\}", text))
+
+
+async def ensure_language(lang: str, language_name: str, client) -> None:
+ """Make `lang` the active language, translating it if not already cached."""
+ if load_cached(lang):
+ return
+ if client is None:
+ return
+ await translate(lang, language_name, client)
diff --git a/server.py b/server.py
index f08e7370..61fe0ee9 100644
--- a/server.py
+++ b/server.py
@@ -50,6 +50,8 @@
format_tasks_for_voice, extract_memories, get_important_memories,
)
from notes_access import get_recent_notes, read_note, search_notes_apple, create_apple_note
+from phrases import phrase, ensure_language
+from tts import list_voices, resolve_voice, speak as macos_speak
from dispatch_registry import DispatchRegistry
from planner import TaskPlanner, detect_planning_mode, BYPASS_PHRASES
@@ -65,6 +67,68 @@
FISH_VOICE_ID = os.getenv("FISH_VOICE_ID", "612b878b113047d9a770c069c8b4fdfe") # JARVIS (MCU)
FISH_API_URL = "https://api.fish.audio/v1/tts"
USER_NAME = os.getenv("USER_NAME", "sir")
+# BCP-47 tag the browser's speech recognition listens in, and the language
+# JARVIS is told to reply in. Read per-request rather than cached, so changing
+# it in the settings panel takes effect without a restart.
+DEFAULT_SPEECH_LANG = "en-US"
+# Offered in the settings panel, mapped to the name used to tell JARVIS which
+# language to answer in.
+SPEECH_LANGUAGES = {
+ "en-US": "English",
+ "en-GB": "English",
+ "it-IT": "Italian",
+ "es-ES": "Spanish",
+ "fr-FR": "French",
+ "de-DE": "German",
+ "pt-BR": "Portuguese",
+ "nl-NL": "Dutch",
+ "ja-JP": "Japanese",
+ "zh-CN": "Chinese",
+}
+
+
+def _current_speech_lang() -> str:
+ """The configured speech language, read live so the panel needs no restart."""
+ return os.getenv("SPEECH_LANG", DEFAULT_SPEECH_LANG) or DEFAULT_SPEECH_LANG
+
+
+async def _refresh_phrases() -> None:
+ """Make the fixed phrases match the configured language."""
+ lang = _current_speech_lang()
+ name = SPEECH_LANGUAGES.get(lang, lang)
+ try:
+ await ensure_language(lang, name, anthropic_client)
+ except Exception as e:
+ log.warning(f"Could not prepare phrases for {lang}: {e}")
+
+
+def _current_honorific() -> str:
+ """How the user has asked to be addressed. Read live, like the language."""
+ return os.getenv("HONORIFIC", "sir").strip() or "sir"
+
+
+# Spoken at the top of a session, so it is the one line a user hears every
+# single time. Left in English it announced the wrong language before JARVIS
+# had said anything else.
+GREETINGS = {
+ "en": ("Good morning", "Good afternoon", "Good evening"),
+ "it": ("Buongiorno", "Buon pomeriggio", "Buonasera"),
+ "es": ("Buenos días", "Buenas tardes", "Buenas noches"),
+ "fr": ("Bonjour", "Bon après-midi", "Bonsoir"),
+ "de": ("Guten Morgen", "Guten Tag", "Guten Abend"),
+ "pt": ("Bom dia", "Boa tarde", "Boa noite"),
+ "nl": ("Goedemorgen", "Goedemiddag", "Goedenavond"),
+ "ja": ("おはようございます", "こんにちは", "こんばんは"),
+ "zh": ("早上好", "下午好", "晚上好"),
+}
+
+
+def build_greeting(hour: int) -> str:
+ """The time-of-day greeting, in the configured language."""
+ slot = 0 if hour < 12 else (1 if hour < 17 else 2)
+ lang = _current_speech_lang().split("-")[0].lower()
+ words = GREETINGS.get(lang, GREETINGS["en"])
+ return f"{words[slot]}, {_current_honorific()}."
PROJECT_DIR = os.path.dirname(os.path.abspath(__file__))
_SKIP_PERMISSIONS = os.getenv("JARVIS_SKIP_PERMISSIONS", "true").lower() not in ("0", "false", "no")
@@ -75,7 +139,7 @@
VOICE & PERSONALITY:
- British butler elegance with understated dry wit
-- Address {user_name} as "sir" naturally — not every sentence, but regularly
+- Address {user_name} as "{honorific}" naturally — not every sentence, but regularly. Use that word exactly as written, in every language: it is how he has asked to be addressed, not a word to translate or inflect. The examples below happen to say "sir"; say "{honorific}" instead.
- Never say "How can I help you?" or "Is there anything else?" — just act
- Deliver bad news calmly, like reporting weather: "We have a slight problem, sir."
- Your humor is observational, never jokes: state facts and let implications land
@@ -696,6 +760,18 @@ def format_projects_for_prompt(projects: list[dict]) -> str:
r"\bquad\b": "Claude",
r"\btravis\b": "JARVIS",
r"\bjarves\b": "JARVIS",
+ # Non-English recognisers hear an English name badly. These are the exact
+ # forms observed in the logs, not guesses: an Italian session produced
+ # "arbiss", "e gli arbis" and "hey Yaris" while the user was plainly
+ # saying JARVIS. Matched as whole words so ordinary vocabulary is safe.
+ r"\bgli arbis\b": "JARVIS",
+ r"\barbiss?\b": "JARVIS",
+ r"\bgiarvis\b": "JARVIS",
+ r"\bjarvi\b": "JARVIS",
+ # "yaris" is a Toyota, and a common one — correcting it outright would
+ # rewrite "la mia auto Yaris". Only fix it where the sentence is plainly
+ # addressing someone.
+ r"\b(hey|hi|ehi|ciao|ok|senti)\s+yaris\b": r"\1 JARVIS",
}
@@ -898,13 +974,17 @@ async def _execute_research(target: str, ws=None):
# Notify via voice if WebSocket still connected
if ws:
try:
- notify_text = f"Research is complete, sir. Report is open in your browser."
+ notify_text = phrase("research.complete")
audio = await synthesize_speech(notify_text)
+ await ws.send_json({"type": "status", "state": "speaking"})
if audio:
- await ws.send_json({"type": "status", "state": "speaking"})
await ws.send_json({"type": "audio", "data": base64.b64encode(audio).decode(), "text": notify_text})
- await ws.send_json({"type": "status", "state": "idle"})
- log.info(f"JARVIS: {notify_text}")
+ else:
+ # No voice available — the reply still has to reach the client,
+ # which shows it on screen.
+ await ws.send_json({"type": "text", "text": notify_text})
+ await ws.send_json({"type": "status", "state": "idle"})
+ log.info(f"JARVIS: {notify_text}")
except Exception:
pass # WebSocket might be gone
@@ -914,7 +994,9 @@ async def _execute_research(target: str, ws=None):
try:
audio = await synthesize_speech("Research timed out, sir. It was taking too long.")
if audio:
- await ws.send_json({"type": "audio", "data": base64.b64encode(audio).decode(), "text": "Research timed out, sir."})
+ await ws.send_json({"type": "audio", "data": base64.b64encode(audio).decode(), "text": phrase("research.timed_out")})
+ else:
+ await ws.send_json({"type": "text", "text": phrase("research.timed_out")})
except Exception:
pass
except Exception as e:
@@ -980,12 +1062,15 @@ async def _execute_prompt_project(project_name: str, prompt: str, work_session:
dispatch_id = dispatch_registry.register(project_name, project_dir or "", prompt)
if not project_dir:
- msg = f"Couldn't find the {project_name} project directory, sir."
+ msg = phrase("build.no_directory", project=project_name)
audio = await synthesize_speech(msg)
- if audio and ws:
+ if ws:
try:
await ws.send_json({"type": "status", "state": "speaking"})
- await ws.send_json({"type": "audio", "data": base64.b64encode(audio).decode(), "text": msg})
+ if audio:
+ await ws.send_json({"type": "audio", "data": base64.b64encode(audio).decode(), "text": msg})
+ else:
+ await ws.send_json({"type": "text", "text": msg})
except Exception:
pass
return
@@ -1021,7 +1106,7 @@ async def _execute_prompt_project(project_name: str, prompt: str, work_session:
if not full_response or full_response.startswith("Hit a problem") or full_response.startswith("That's taking"):
dispatch_registry.update_status(dispatch_id, "failed" if full_response else "timeout", response=full_response or "")
- msg = f"Sir, I ran into an issue with {project_name}. {full_response[:150] if full_response else 'No response received.'}"
+ msg = phrase("build.issue", project=project_name, detail=full_response[:150] if full_response else "No response received.")
else:
# Summarize via Haiku — don't read word for word
if anthropic_client:
@@ -1043,9 +1128,9 @@ async def _execute_prompt_project(project_name: str, prompt: str, work_session:
)
msg = summary.content[0].text
except Exception:
- msg = f"Sir, {project_name} finished. Here's the gist: {full_response[:200]}"
+ msg = phrase("build.finished", project=project_name, summary=full_response[:200])
else:
- msg = f"Sir, {project_name} is done. {full_response[:200]}"
+ msg = phrase("build.done", project=project_name, summary=full_response[:200])
# Speak the result — skip if user has spoken recently to avoid audio collision
log.info(f"Dispatch summary for {project_name}: {msg[:100]}")
@@ -1076,11 +1161,14 @@ async def _execute_prompt_project(project_name: str, prompt: str, work_session:
except Exception as e:
log.error(f"Prompt project failed: {e}", exc_info=True)
try:
- msg = f"Had trouble connecting to {project_name}, sir."
+ msg = phrase("build.connect_trouble", project=project_name)
audio = await synthesize_speech(msg)
- if audio and ws:
+ if ws:
await ws.send_json({"type": "status", "state": "speaking"})
- await ws.send_json({"type": "audio", "data": base64.b64encode(audio).decode(), "text": msg})
+ if audio:
+ await ws.send_json({"type": "audio", "data": base64.b64encode(audio).decode(), "text": msg})
+ else:
+ await ws.send_json({"type": "text", "text": msg})
except Exception:
pass
@@ -1102,15 +1190,17 @@ async def self_work_and_notify(session: WorkSession, prompt: str, ws):
)
msg = summary.content[0].text
except Exception:
- msg = "Work is complete, sir."
+ msg = phrase("build.work_complete")
try:
audio = await synthesize_speech(msg)
+ await ws.send_json({"type": "status", "state": "speaking"})
if audio:
- await ws.send_json({"type": "status", "state": "speaking"})
await ws.send_json({"type": "audio", "data": base64.b64encode(audio).decode(), "text": msg})
- await ws.send_json({"type": "status", "state": "idle"})
- log.info(f"JARVIS: {msg}")
+ else:
+ await ws.send_json({"type": "text", "text": msg})
+ await ws.send_json({"type": "status", "state": "idle"})
+ log.info(f"JARVIS: {msg}")
except Exception:
pass
except Exception as e:
@@ -1122,11 +1212,47 @@ async def self_work_and_notify(session: WorkSession, prompt: str, ws):
# ---------------------------------------------------------------------------
-# TTS (Fish Audio)
+# TTS (Fish Audio, or the macOS synthesiser)
# ---------------------------------------------------------------------------
+_FISH_KEY_PLACEHOLDER = "your-fish-audio-api-key-here"
+
+
+def _fish_configured() -> bool:
+ """Whether Fish Audio has a real key.
+
+ The placeholder shipped in .env.example is a non-empty string, so a plain
+ truth test counts an untouched install as configured — which sent every
+ line to Fish for a 401 instead of falling back.
+ """
+ key = (FISH_API_KEY or "").strip()
+ return bool(key) and key != _FISH_KEY_PLACEHOLDER
+
+
+def _current_tts_backend() -> str:
+ """Which synthesiser to use: auto, fish, macos or none.
+
+ "auto" prefers Fish Audio when a key is configured and falls back to the
+ macOS voices otherwise — so a fresh install talks instead of sitting mute,
+ which is what an unconfigured JARVIS used to do.
+ """
+ choice = os.getenv("TTS_BACKEND", "auto").strip().lower() or "auto"
+ if choice != "auto":
+ return choice
+ return "fish" if _fish_configured() else "macos"
+
+
async def synthesize_speech(text: str) -> Optional[bytes]:
- """Generate speech audio from text using Fish Audio TTS."""
+ """Generate speech audio for a line of JARVIS dialogue."""
+ backend = _current_tts_backend()
+
+ if backend == "none":
+ return None
+
+ if backend == "macos":
+ voice = await resolve_voice(_current_speech_lang(), os.getenv("MACOS_VOICE", ""))
+ return await macos_speak(text, voice)
+
if not FISH_API_KEY:
log.warning("FISH_API_KEY not set, skipping TTS")
return None
@@ -1195,8 +1321,27 @@ async def generate_response(
dispatch_context=dispatch_registry.format_for_prompt(),
known_projects=format_projects_for_prompt(projects),
user_name=USER_NAME,
+ honorific=_current_honorific(),
project_dir=PROJECT_DIR,
)
+ # The prompt above is written in English and would answer an Italian
+ # question in English, which makes the speech-language setting only half
+ # work. Kept to a bare language directive on purpose: wording that also
+ # discussed action selection measurably weakened the language adherence it
+ # was appended to enforce. Action routing is less reliable outside English
+ # regardless — see the non-English caveat in the README.
+ speech_lang = _current_speech_lang()
+ if not speech_lang.startswith("en"):
+ language = SPEECH_LANGUAGES.get(speech_lang, speech_lang)
+ honorific = _current_honorific()
+ system += (
+ f"\n\nLANGUAGE: The user speaks {language}. Write every spoken reply in {language}, "
+ f"keeping the same dry, economical butler voice. One exception: address him as "
+ f'"{honorific}" — that exact word, letter for letter. It is his chosen form of '
+ f"address, not vocabulary to translate into {language} or to inflect for gender. "
+ f'Writing anything other than "{honorific}" is an error.'
+ )
+
if lookup_status:
system += f"\n\nACTIVE LOOKUPS:\n{lookup_status}\nIf asked about progress, report this status."
@@ -1415,6 +1560,12 @@ async def lifespan(application: FastAPI):
anthropic_client = anthropic.AsyncAnthropic(api_key=ANTHROPIC_API_KEY)
else:
log.warning("ANTHROPIC_API_KEY not set — LLM features disabled")
+
+ # Warm the fixed phrases for the configured language. Backgrounded: a
+ # translation costs one Haiku call on a cold cache, and JARVIS should not
+ # wait on it to start answering — until it lands, the English originals
+ # are used, which is what happened before this existed anyway.
+ asyncio.create_task(_refresh_phrases())
cached_projects = []
# Start context refresh in a separate thread (never touches event loop)
@@ -1443,9 +1594,17 @@ async def health():
@app.get("/api/tts-test")
-async def tts_test():
- """Generate a test audio clip for debugging."""
- audio = await synthesize_speech("Testing audio, sir.")
+async def tts_test(voice: str = ""):
+ """Generate a test audio clip.
+
+ A voice can be named to audition it before saving, since picking one from
+ a list of seventy is otherwise guesswork.
+ """
+ line = f"{build_greeting(datetime.now().hour)} All systems are operational."
+ if voice:
+ audio = await macos_speak(line, voice)
+ else:
+ audio = await synthesize_speech(line)
if audio:
return {"audio": base64.b64encode(audio).decode()}
return {"audio": None, "error": "TTS failed"}
@@ -1698,7 +1857,7 @@ async def _lookup_and_report(lookup_type: str, lookup_fn, ws, history: list[dict
try:
await ws.send_json({"type": "status", "state": "speaking"})
if audio:
- await ws.send_json({"type": "audio", "data": audio, "text": result_text})
+ await ws.send_json({"type": "audio", "data": base64.b64encode(audio).decode(), "text": result_text})
else:
await ws.send_json({"type": "text", "text": result_text})
await ws.send_json({"type": "status", "state": "idle"})
@@ -1714,11 +1873,13 @@ async def _lookup_and_report(lookup_type: str, lookup_fn, ws, history: list[dict
except asyncio.TimeoutError:
_active_lookups[lookup_id]["status"] = "timeout"
try:
- fallback = f"That {lookup_type} check is taking too long, sir. The data may still be syncing."
+ fallback = phrase("lookup.slow", kind=lookup_type)
audio = await synthesize_speech(fallback)
await ws.send_json({"type": "status", "state": "speaking"})
if audio:
- await ws.send_json({"type": "audio", "data": audio, "text": fallback})
+ await ws.send_json({"type": "audio", "data": base64.b64encode(audio).decode(), "text": fallback})
+ else:
+ await ws.send_json({"type": "text", "text": fallback})
await ws.send_json({"type": "status", "state": "idle"})
except Exception:
pass
@@ -1854,13 +2015,33 @@ async def handle_browse(text: str, target: str) -> str:
async def handle_research(text: str, target: str, client: anthropic.AsyncAnthropic) -> str:
"""Deep research with Opus — write results to HTML, open in browser."""
try:
- research_response = await client.messages.create(
- model="claude-opus-4-6",
- max_tokens=2000,
+ research_response = await client.beta.messages.create(
+ model="claude-opus-5",
+ # Thinking is on by default on Opus 5 and shares this budget with the
+ # answer, so a limit sized for prose alone truncates the report.
+ max_tokens=16000,
+ # Opus 5's safety classifiers can decline a request outright. Letting
+ # the API re-run it elsewhere turns a dead end into an answer, and
+ # "default" routes by refusal category rather than pinning a model
+ # that will eventually be retired.
+ betas=["server-side-fallback-2026-07-01"],
+ fallbacks="default",
system=f"You are JARVIS, researching a topic for {USER_NAME}. Be thorough, organized, and cite sources where possible.",
messages=[{"role": "user", "content": f"Research this thoroughly:\n\n{target}"}],
)
- research_text = research_response.content[0].text
+
+ if research_response.stop_reason == "refusal":
+ log.warning(f"Research declined for: {target[:80]}")
+ return phrase("research.declined")
+
+ # Take the text blocks rather than content[0]: with thinking on, the
+ # first block is the reasoning, which carries no .text at all.
+ research_text = "\n".join(
+ b.text for b in research_response.content if b.type == "text"
+ ).strip()
+ if not research_text:
+ log.warning("Research returned no text content")
+ return phrase("research.empty")
import html as _html
html_content = f"""
@@ -1977,13 +2158,7 @@ async def voice_handler(ws: WebSocket):
try:
# ── Greeting — always start in conversation mode ──
now = datetime.now()
- hour = now.hour
- if hour < 12:
- greeting = "Good morning, sir."
- elif hour < 17:
- greeting = "Good afternoon, sir."
- else:
- greeting = "Good evening, sir."
+ greeting = build_greeting(now.hour)
global _last_greeting_time
should_greet = (time.time() - _last_greeting_time) > 60
@@ -1994,13 +2169,15 @@ async def voice_handler(ws: WebSocket):
async def _send_greeting():
try:
audio_bytes = await synthesize_speech(greeting)
+ await ws.send_json({"type": "status", "state": "speaking"})
if audio_bytes:
encoded = base64.b64encode(audio_bytes).decode()
- await ws.send_json({"type": "status", "state": "speaking"})
await ws.send_json({"type": "audio", "data": encoded, "text": greeting})
- history.append({"role": "assistant", "content": greeting})
- log.info(f"JARVIS: {greeting}")
- await ws.send_json({"type": "status", "state": "idle"})
+ else:
+ await ws.send_json({"type": "text", "text": greeting})
+ history.append({"role": "assistant", "content": greeting})
+ log.info(f"JARVIS: {greeting}")
+ await ws.send_json({"type": "status", "state": "idle"})
except Exception as e:
log.warning(f"Greeting failed: {e}")
@@ -2022,12 +2199,12 @@ async def _send_greeting():
if msg.get("type") == "fix_self":
jarvis_dir = str(Path(__file__).parent)
await work_session.start(jarvis_dir)
- response_text = "Work mode active in my own repo, sir. Tell me what needs fixing."
+ response_text = phrase("mode.work_self")
tts = strip_markdown_for_tts(response_text)
await ws.send_json({"type": "status", "state": "speaking"})
audio = await synthesize_speech(tts)
if audio:
- await ws.send_json({"type": "audio", "data": audio, "text": response_text})
+ await ws.send_json({"type": "audio", "data": base64.b64encode(audio).decode(), "text": response_text})
else:
await ws.send_json({"type": "text", "text": response_text})
continue
@@ -2086,7 +2263,7 @@ async def _send_greeting():
did = dispatch_registry.register(name, path, prompt[:200])
asyncio.create_task(_execute_prompt_project(name, prompt, work_session, ws, dispatch_id=did, history=history, voice_state=voice_state))
planner.reset()
- response_text = "Building it now, sir."
+ response_text = phrase("ack.building")
elif planner.active_plan and planner.active_plan.confirmed is False and planner.active_plan.current_question_index >= len(planner.active_plan.pending_questions):
# Confirmation phase
result = await planner.handle_confirmation(user_text)
@@ -2099,10 +2276,10 @@ async def _send_greeting():
did = dispatch_registry.register(name, path, prompt[:200])
asyncio.create_task(_execute_prompt_project(name, prompt, work_session, ws, dispatch_id=did, history=history, voice_state=voice_state))
planner.reset()
- response_text = "On it, sir."
+ response_text = phrase("ack.on_it")
elif result["cancelled"]:
planner.reset()
- response_text = "Cancelled, sir."
+ response_text = phrase("ack.cancelled")
else:
response_text = result.get("modification_question", "How shall I adjust the plan, sir?")
else:
@@ -2115,9 +2292,9 @@ async def _send_greeting():
elif any(w in t_lower for w in ["quit work mode", "exit work mode", "go back to chat", "regular mode", "stop working"]):
if work_session.active:
await work_session.stop()
- response_text = "Back to conversation mode, sir."
+ response_text = phrase("mode.back_to_chat")
else:
- response_text = "Already in conversation mode, sir."
+ response_text = phrase("mode.already_chat")
# ── WORK MODE: speech → claude -p → Haiku summary → JARVIS voice ──
elif work_session.active:
@@ -2192,37 +2369,37 @@ async def _send_greeting():
elif action["action"] == "show_recent":
response_text = await handle_show_recent()
elif action["action"] == "describe_screen":
- response_text = "Taking a look now, sir."
+ response_text = phrase("ack.taking_look")
asyncio.create_task(_lookup_and_report("screen", _do_screen_lookup, ws, history=history, voice_state=voice_state))
elif action["action"] == "check_calendar":
- response_text = "Checking your calendar now, sir."
+ response_text = phrase("ack.checking_calendar")
asyncio.create_task(_lookup_and_report("calendar", _do_calendar_lookup, ws, history=history, voice_state=voice_state))
elif action["action"] == "check_mail":
- response_text = "Checking your inbox now, sir."
+ response_text = phrase("ack.checking_mail")
asyncio.create_task(_lookup_and_report("mail", _do_mail_lookup, ws, history=history, voice_state=voice_state))
elif action["action"] == "check_dispatch":
recent = dispatch_registry.get_most_recent()
if not recent:
- response_text = "No recent builds on record, sir."
+ response_text = phrase("build.none_recent")
else:
name = recent["project_name"]
status = recent["status"]
if status == "building" or status == "pending":
elapsed = int(time.time() - recent["updated_at"])
- response_text = f"Still working on {name}, sir. Been at it for {elapsed} seconds."
+ response_text = phrase("build.still_working", project=name, seconds=elapsed)
elif status == "completed":
response_text = recent.get("summary") or f"{name} is complete, sir."
elif status in ("failed", "timeout"):
- response_text = f"{name} ran into problems, sir."
+ response_text = phrase("build.problems", project=name)
else:
- response_text = f"{name} is {status}, sir."
+ response_text = phrase("build.status", project=name, status=status)
elif action["action"] == "check_tasks":
tasks = get_open_tasks()
response_text = format_tasks_for_voice(tasks)
elif action["action"] == "check_usage":
response_text = get_usage_summary()
else:
- response_text = "Understood, sir."
+ response_text = phrase("ack.understood")
else:
if not anthropic_client:
response_text = "API key not configured."
@@ -2244,13 +2421,13 @@ async def _send_greeting():
action_type = embedded_action["action"]
if action_type == "prompt_project":
proj = embedded_action["target"].split("|||")[0].strip()
- response_text = f"Connecting to {proj} now, sir."
+ response_text = phrase("ack.connecting", project=proj)
elif action_type == "build":
- response_text = "On it, sir."
+ response_text = phrase("ack.on_it")
elif action_type == "research":
- response_text = "Looking into that now, sir."
+ response_text = phrase("ack.looking_into")
else:
- response_text = "Right away, sir."
+ response_text = phrase("ack.generic")
if embedded_action["action"] == "build":
# Build in background — JARVIS stays conversational
@@ -2354,14 +2531,17 @@ async def _send_greeting():
async def _read_and_report(search_term, _ws):
note = await read_note(search_term)
if note:
- msg = f"Sir, your note '{note['title']}' says: {note['body'][:200]}"
+ msg = phrase("note.says", title=note["title"], body=note["body"][:200])
else:
- msg = f"Couldn't find a note matching '{search_term}', sir."
+ msg = phrase("note.not_found", query=search_term)
audio = await synthesize_speech(strip_markdown_for_tts(msg))
- if audio and _ws:
+ if _ws:
try:
await _ws.send_json({"type": "status", "state": "speaking"})
- await _ws.send_json({"type": "audio", "data": base64.b64encode(audio).decode(), "text": msg})
+ if audio:
+ await _ws.send_json({"type": "audio", "data": base64.b64encode(audio).decode(), "text": msg})
+ else:
+ await _ws.send_json({"type": "text", "text": msg})
except Exception:
pass
asyncio.create_task(_read_and_report(embedded_action["target"].strip(), ws))
@@ -2411,7 +2591,7 @@ async def _do_summary():
except Exception as e:
log.error(f"Error: {e}", exc_info=True)
try:
- fallback = "Something went wrong, sir."
+ fallback = phrase("error.generic")
audio = await synthesize_speech(fallback)
if audio:
await ws.send_json({"type": "audio", "data": base64.b64encode(audio).decode(), "text": fallback})
@@ -2488,10 +2668,13 @@ class PreferencesUpdate(BaseModel):
user_name: str = ""
honorific: str = "sir"
calendar_accounts: str = "auto"
+ speech_lang: str = DEFAULT_SPEECH_LANG
+ tts_backend: str = "auto"
+ macos_voice: str = ""
@app.post("/api/settings/keys")
async def api_settings_keys(body: KeyUpdate):
- allowed = {"ANTHROPIC_API_KEY", "FISH_API_KEY", "FISH_VOICE_ID", "USER_NAME", "HONORIFIC", "CALENDAR_ACCOUNTS"}
+ allowed = {"ANTHROPIC_API_KEY", "FISH_API_KEY", "FISH_VOICE_ID", "USER_NAME", "HONORIFIC", "CALENDAR_ACCOUNTS", "SPEECH_LANG", "TTS_BACKEND", "MACOS_VOICE"}
if body.key_name not in allowed:
return JSONResponse({"success": False, "error": "Invalid key name"}, status_code=400)
_write_env_key(body.key_name, body.key_value)
@@ -2558,12 +2741,31 @@ async def api_settings_status():
"uptime_seconds": int(time.time() - _session_start),
"env_keys_set": {
"anthropic": bool(env_dict.get("ANTHROPIC_API_KEY", "").strip() and env_dict.get("ANTHROPIC_API_KEY", "") != "your-anthropic-api-key-here"),
- "fish_audio": bool(env_dict.get("FISH_API_KEY", "").strip() and env_dict.get("FISH_API_KEY", "") != "your-fish-audio-api-key-here"),
+ "fish_audio": bool(env_dict.get("FISH_API_KEY", "").strip() and env_dict.get("FISH_API_KEY", "") != _FISH_KEY_PLACEHOLDER),
"fish_voice_id": bool(env_dict.get("FISH_VOICE_ID", "").strip()),
"user_name": env_dict.get("USER_NAME", ""),
},
}
+@app.get("/api/settings/voices")
+async def api_settings_voices():
+ """Installed macOS voices, so the panel can offer real choices.
+
+ Voices for the configured language come first — that is what the user is
+ almost certainly picking from — with the rest kept for anyone who wants a
+ voice from another language.
+ """
+ voices = await list_voices()
+ lang = _current_speech_lang().split("-")[0].lower()
+ preferred = [v for v in voices if v["lang"].lower().startswith(lang)]
+ others = [v for v in voices if not v["lang"].lower().startswith(lang)]
+ return {
+ "voices": preferred + others,
+ "matching_language": len(preferred),
+ "resolved": await resolve_voice(_current_speech_lang(), os.getenv("MACOS_VOICE", "")),
+ }
+
+
@app.get("/api/settings/preferences")
async def api_get_preferences():
_, env_dict = _read_env()
@@ -2571,6 +2773,9 @@ async def api_get_preferences():
"user_name": env_dict.get("USER_NAME", ""),
"honorific": env_dict.get("HONORIFIC", "sir"),
"calendar_accounts": env_dict.get("CALENDAR_ACCOUNTS", "auto"),
+ "speech_lang": env_dict.get("SPEECH_LANG", DEFAULT_SPEECH_LANG) or DEFAULT_SPEECH_LANG,
+ "tts_backend": env_dict.get("TTS_BACKEND", "auto") or "auto",
+ "macos_voice": env_dict.get("MACOS_VOICE", ""),
}
@app.post("/api/settings/preferences")
@@ -2578,6 +2783,11 @@ async def api_save_preferences(body: PreferencesUpdate):
_write_env_key("USER_NAME", body.user_name)
_write_env_key("HONORIFIC", body.honorific)
_write_env_key("CALENDAR_ACCOUNTS", body.calendar_accounts)
+ _write_env_key("SPEECH_LANG", body.speech_lang or DEFAULT_SPEECH_LANG)
+ _write_env_key("TTS_BACKEND", body.tts_backend or "auto")
+ _write_env_key("MACOS_VOICE", body.macos_voice)
+ # A new language needs its own set of fixed phrases.
+ asyncio.create_task(_refresh_phrases())
return {"success": True}
# ---------------------------------------------------------------------------
diff --git a/tts.py b/tts.py
new file mode 100644
index 00000000..a0d5c0a0
--- /dev/null
+++ b/tts.py
@@ -0,0 +1,149 @@
+"""
+JARVIS macOS speech synthesis — a free, offline alternative to Fish Audio.
+
+macOS ships voices for every language JARVIS offers, so a user without a Fish
+Audio key still gets a spoken assistant rather than a silent one. The voice is
+plainer than the Fish JARVIS model; the trade is availability and cost.
+
+Voices are discovered from `say -v ?` at runtime rather than hardcoded, since
+which ones are installed varies by machine and macOS version.
+"""
+
+import asyncio
+import logging
+import re
+import tempfile
+from pathlib import Path
+
+log = logging.getLogger("jarvis.tts")
+
+# WAV, which every browser's decodeAudioData accepts. The `say` default is
+# AIFF-C, which Chrome does not reliably decode.
+_DATA_FORMAT = "LEI16@22050"
+
+# Preferred voice per language, used when installed. Everything else falls back
+# to the first voice matching the language. Daniel is the British voice, which
+# suits the butler better than the American default.
+_PREFERRED = {
+ "en": ["Daniel", "Oliver", "Serena"],
+ "it": ["Alice", "Federica"],
+ "es": ["Monica", "Mónica", "Jorge"],
+ "fr": ["Thomas", "Audrey"],
+ "de": ["Anna", "Markus"],
+ "pt": ["Luciana", "Joana"],
+ "nl": ["Xander", "Claire"],
+ "ja": ["Kyoko", "Otoya"],
+ "zh": ["Tingting", "Tingting"],
+}
+
+_voice_cache: list[dict] | None = None
+
+# `say` interprets [[...]] as embedded speech commands — [[volm 0]] silences it,
+# among others. The text reaching here came from a speech transcript by way of
+# an LLM, so strip the brackets rather than trust that none appear.
+_SAY_COMMAND = re.compile(r"\[\[.*?\]\]")
+
+
+def _parse_voice_line(line: str) -> dict | None:
+ """Parse one `say -v ?` row: name, locale tag, then a `#` sample phrase.
+
+ Anchored on the `#`, because the column spacing is not consistent: classic
+ voices are padded into a column ("Alice it_IT # ...") while
+ newer ones carry a parenthesised language in the name and get a single
+ space ("Eddy (Italiano (Italia)) it_IT # ..."). Keying on run length
+ silently dropped every voice of the second kind.
+ """
+ match = re.match(r"^(.+?)\s+([a-z]{2}_[A-Z]{2})\s+#", line)
+ if not match:
+ return None
+ return {"name": match.group(1).strip(), "lang": match.group(2)}
+
+
+async def list_voices(refresh: bool = False) -> list[dict]:
+ """Every installed macOS voice, as {"name", "lang"}."""
+ global _voice_cache
+ if _voice_cache is not None and not refresh:
+ return _voice_cache
+ try:
+ proc = await asyncio.create_subprocess_exec(
+ "say", "-v", "?",
+ stdout=asyncio.subprocess.PIPE,
+ stderr=asyncio.subprocess.PIPE,
+ )
+ stdout, _ = await asyncio.wait_for(proc.communicate(), timeout=10)
+ except Exception as e:
+ log.warning(f"Could not list voices: {e}")
+ return []
+
+ voices = []
+ for line in stdout.decode(errors="replace").split("\n"):
+ parsed = _parse_voice_line(line)
+ if parsed:
+ voices.append(parsed)
+ _voice_cache = voices
+ return voices
+
+
+async def resolve_voice(speech_lang: str, configured: str = "") -> str | None:
+ """Pick the voice to speak with, or None when nothing matches.
+
+ An explicitly configured voice wins outright — including one whose language
+ differs from the interface, which is a legitimate thing to want.
+ """
+ if configured.strip():
+ return configured.strip()
+
+ voices = await list_voices()
+ if not voices:
+ return None
+
+ lang = speech_lang.split("-")[0].lower()
+ matching = [v for v in voices if v["lang"].lower().startswith(lang)]
+ if not matching:
+ return None
+
+ names = {v["name"] for v in matching}
+ for preferred in _PREFERRED.get(lang, []):
+ if preferred in names:
+ return preferred
+ return matching[0]["name"]
+
+
+async def speak(text: str, voice: str | None = None, timeout: float = 30) -> bytes | None:
+ """Render text to WAV bytes with the macOS speech synthesiser."""
+ clean = _SAY_COMMAND.sub("", text).strip()
+ if not clean:
+ return None
+
+ with tempfile.TemporaryDirectory() as tmp:
+ out = Path(tmp) / "speech.wav"
+ # Arguments are passed directly to exec, never through a shell, so the
+ # text cannot break out into a command.
+ args = ["say", "--data-format=" + _DATA_FORMAT, "-o", str(out)]
+ if voice:
+ args += ["-v", voice]
+ args += ["--", clean]
+
+ try:
+ proc = await asyncio.create_subprocess_exec(
+ *args,
+ stdout=asyncio.subprocess.PIPE,
+ stderr=asyncio.subprocess.PIPE,
+ )
+ _, stderr = await asyncio.wait_for(proc.communicate(), timeout=timeout)
+ if proc.returncode != 0:
+ log.warning(f"say failed: {stderr.decode(errors='replace')[:200]}")
+ return None
+ if not out.exists():
+ log.warning("say produced no output file")
+ return None
+ return out.read_bytes()
+ except asyncio.TimeoutError:
+ log.warning("say timed out")
+ return None
+ except FileNotFoundError:
+ log.warning("say is unavailable — macOS TTS needs macOS")
+ return None
+ except Exception as e:
+ log.warning(f"say error: {e}")
+ return None