diff --git a/frontend/package-lock.json b/frontend/package-lock.json
index 1f459602..b74e0ce7 100644
--- a/frontend/package-lock.json
+++ b/frontend/package-lock.json
@@ -948,9 +948,9 @@
"license": "MIT"
},
"node_modules/nanoid": {
- "version": "3.3.11",
- "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.11.tgz",
- "integrity": "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w==",
+ "version": "3.3.18",
+ "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.18.tgz",
+ "integrity": "sha512-DTg4MJbGMWkfi6VZFdNt2/caMbQy4Ou+Op/hJQvGEWcnVfoA1QA+xzRKAzw9jD6+GVOOeYr/mIcuDSdug6F6+w==",
"dev": true,
"funding": [
{
@@ -974,9 +974,9 @@
"license": "ISC"
},
"node_modules/picomatch": {
- "version": "4.0.3",
- "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.3.tgz",
- "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==",
+ "version": "4.0.5",
+ "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.5.tgz",
+ "integrity": "sha512-RvwwcruNjI1ncT5xRakeyS9Lf8lcItv34KD+aif+VH9kduAyfYBipGh12274xtenIPZ119/R9BdTBa8gAwSh0A==",
"dev": true,
"license": "MIT",
"engines": {
@@ -987,9 +987,9 @@
}
},
"node_modules/postcss": {
- "version": "8.5.8",
- "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.8.tgz",
- "integrity": "sha512-OW/rX8O/jXnm82Ey1k44pObPtdblfiuWnrd8X7GJ7emImCOstunGbXUpp7HdBrFQX6rJzn3sPT397Wp5aCwCHg==",
+ "version": "8.5.26",
+ "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.26.tgz",
+ "integrity": "sha512-u82N74LFzG8ca+dD8puPnplTXoGH4fTPpVGuIbt36G3qvNlkvfD0lEAZSxaly3KX8TS/L1A1gsCEmvKmBcVbkQ==",
"dev": true,
"funding": [
{
@@ -1007,7 +1007,7 @@
],
"license": "MIT",
"dependencies": {
- "nanoid": "^3.3.11",
+ "nanoid": "^3.3.17",
"picocolors": "^1.1.1",
"source-map-js": "^1.2.1"
},
@@ -1108,9 +1108,9 @@
}
},
"node_modules/vite": {
- "version": "6.4.1",
- "resolved": "https://registry.npmjs.org/vite/-/vite-6.4.1.tgz",
- "integrity": "sha512-+Oxm7q9hDoLMyJOYfUYBuHQo+dkAloi33apOPP56pzj+vsdJDzr+j1NISE5pyaAuKL4A3UD34qd0lx5+kfKp2g==",
+ "version": "6.4.3",
+ "resolved": "https://registry.npmjs.org/vite/-/vite-6.4.3.tgz",
+ "integrity": "sha512-NTKlcQjlAK7MlQoyb6LgaqHc8sso/pVyUJYWMws3jg21uTJw/LddqIFPcPqP6PzpgbIcZyKI85sFE4HBrQDA8A==",
"dev": true,
"license": "MIT",
"dependencies": {
diff --git a/frontend/src/main.ts b/frontend/src/main.ts
index ca5d1864..e6b08f2d 100644
--- a/frontend/src/main.ts
+++ b/frontend/src/main.ts
@@ -8,7 +8,7 @@
import { createOrb, type OrbState } from "./orb";
import { createVoiceInput, createAudioPlayer } from "./voice";
import { createSocket } from "./ws";
-import { openSettings, checkFirstTimeSetup } from "./settings";
+import { openSettings, checkFirstTimeSetup, SPEECH_LANG_EVENT } from "./settings";
import "./style.css";
// ---------------------------------------------------------------------------
@@ -21,6 +21,7 @@ let isMuted = false;
const statusEl = document.getElementById("status-text")!;
const errorEl = document.getElementById("error-text")!;
+const captionEl = document.getElementById("jarvis-caption")!;
function showError(msg: string) {
errorEl.textContent = msg;
@@ -30,6 +31,28 @@ function showError(msg: string) {
}, 5000);
}
+let captionTimer: number | undefined;
+
+/**
+ * Put what JARVIS said on screen.
+ *
+ * Shown whether or not TTS produced audio: with a voice it reads as
+ * subtitles, and without one it is the only way to see the reply — the
+ * response used to reach nothing but the devtools console.
+ */
+function showCaption(text: unknown) {
+ if (typeof text !== "string" || !text.trim()) return;
+ clearTimeout(captionTimer);
+ captionEl.textContent = text;
+ captionEl.style.opacity = "1";
+ // Hold long enough to read the line, since a caption may be all the user
+ // gets. Replies are a sentence or two, so length is a fair proxy.
+ const holdMs = Math.min(20000, 4000 + text.length * 60);
+ captionTimer = window.setTimeout(() => {
+ captionEl.style.opacity = "0";
+ }, holdMs);
+}
+
function updateStatus(state: State) {
const labels: Record = {
idle: "",
@@ -80,13 +103,59 @@ function transition(newState: State) {
// Voice input
// ---------------------------------------------------------------------------
+/**
+ * Fragment accumulation.
+ *
+ * Chrome closes a recognition segment at every pause, so a sentence spoken
+ * with any hesitation arrives as several "final" results. Sending each one
+ * immediately made JARVIS answer half a thought and then receive the rest as
+ * a separate question. Instead, collect the pieces and send once the speaker
+ * has actually stopped.
+ */
+const UTTERANCE_GAP_MS = 1000;
+// A pause never arrives while someone is dictating steadily, so cap the wait
+// rather than let a long sentence hold the whole conversation open.
+const UTTERANCE_MAX_MS = 6000;
+
+let pendingFragments: string[] = [];
+let gapTimer: number | undefined;
+let maxTimer: number | undefined;
+
+function discardUtterance() {
+ clearTimeout(gapTimer);
+ clearTimeout(maxTimer);
+ gapTimer = undefined;
+ maxTimer = undefined;
+ pendingFragments = [];
+}
+
+function flushUtterance() {
+ clearTimeout(gapTimer);
+ clearTimeout(maxTimer);
+ gapTimer = undefined;
+ maxTimer = undefined;
+
+ const text = pendingFragments.join(" ").replace(/\s+/g, " ").trim();
+ pendingFragments = [];
+ if (!text) return;
+
+ socket.send({ type: "transcript", text, isFinal: true });
+ transition("thinking");
+}
+
const voiceInput = createVoiceInput(
(text: string) => {
- // Cancel any current JARVIS response before sending new input
+ // Cancel any current JARVIS response before sending new input. Done on the
+ // first fragment, not at flush time: cutting him off is what the user
+ // wanted the moment they started talking.
audioPlayer.stop();
- // User spoke — send transcript
- socket.send({ type: "transcript", text, isFinal: true });
- transition("thinking");
+
+ pendingFragments.push(text);
+ clearTimeout(gapTimer);
+ gapTimer = window.setTimeout(flushUtterance, UTTERANCE_GAP_MS);
+ if (maxTimer === undefined) {
+ maxTimer = window.setTimeout(flushUtterance, UTTERANCE_MAX_MS);
+ }
},
(msg: string) => {
showError(msg);
@@ -123,6 +192,7 @@ socket.onMessage((msg) => {
}
// Log text for debugging
if (msg.text) console.log("[JARVIS]", msg.text);
+ showCaption(msg.text);
} else if (type === "status") {
const state = msg.state as string;
if (state === "thinking" && currentState !== "thinking") {
@@ -137,6 +207,7 @@ socket.onMessage((msg) => {
} else if (type === "text") {
// Text fallback when TTS fails
console.log("[JARVIS]", msg.text);
+ showCaption(msg.text);
} else if (type === "task_spawned") {
console.log("[task]", "spawned:", msg.task_id, msg.prompt);
} else if (type === "task_complete") {
@@ -148,12 +219,40 @@ socket.onMessage((msg) => {
// Kick off
// ---------------------------------------------------------------------------
+/**
+ * Read the speech language chosen in the settings panel.
+ *
+ * Resolved before the microphone opens so the first session is built in the
+ * right language: switching a live session means tearing it down and racing
+ * for the microphone, which is worth avoiding on every page load. Bounded, so
+ * an unreachable server costs a moment rather than the microphone.
+ */
+async function fetchSpeechLanguage(): Promise {
+ try {
+ const res = await fetch("/api/settings/preferences", {
+ signal: AbortSignal.timeout(2000),
+ });
+ const prefs = (await res.json()) as { speech_lang?: string };
+ return prefs.speech_lang || null;
+ } catch {
+ return null; // Server not ready — keep the default language.
+ }
+}
+
// Start listening after a brief delay for the orb to render
-setTimeout(() => {
+setTimeout(async () => {
+ const lang = await fetchSpeechLanguage();
+ if (lang) voiceInput.setLanguage(lang);
voiceInput.start();
transition("listening");
}, 1000);
+// Saving the setting re-tunes the running session — no reload needed.
+document.addEventListener(SPEECH_LANG_EVENT, (e) => {
+ const lang = (e as CustomEvent).detail;
+ if (lang) voiceInput.setLanguage(lang);
+});
+
// Resume AudioContext on ANY user interaction (browser autoplay policy)
function ensureAudioContext() {
const ctx = audioPlayer.getAnalyser().context as AudioContext;
@@ -183,6 +282,8 @@ btnMute.addEventListener("click", (e) => {
isMuted = !isMuted;
btnMute.classList.toggle("muted", isMuted);
if (isMuted) {
+ // Half-spoken words must not arrive a second after the user silenced him.
+ discardUtterance();
voiceInput.pause();
transition("idle");
} else {
diff --git a/frontend/src/settings.ts b/frontend/src/settings.ts
index 7e945ef7..4f4131b5 100644
--- a/frontend/src/settings.ts
+++ b/frontend/src/settings.ts
@@ -30,8 +30,16 @@ interface PreferencesResponse {
user_name: string;
honorific: string;
calendar_accounts: string;
+ speech_lang: string;
}
+/**
+ * Event fired when the speech language changes, so the live recognition
+ * session picks it up without a reload. Used instead of importing the voice
+ * input directly, which would make settings and main circular.
+ */
+export const SPEECH_LANG_EVENT = "jarvis:speech-lang";
+
// ---------------------------------------------------------------------------
// State
// ---------------------------------------------------------------------------
@@ -143,6 +151,22 @@ function buildPanelHTML(): string {
+
+
+
+
+
@@ -242,9 +266,11 @@ async function loadPreferences() {
const nameEl = document.getElementById("input-user-name") as HTMLInputElement;
const honEl = document.getElementById("input-honorific") as HTMLSelectElement;
const calEl = document.getElementById("input-calendar-accounts") as HTMLTextAreaElement;
+ const langEl = document.getElementById("input-speech-lang") as HTMLSelectElement;
if (nameEl) nameEl.value = prefs.user_name || "";
if (honEl) honEl.value = prefs.honorific || "sir";
if (calEl) calEl.value = prefs.calendar_accounts || "auto";
+ if (langEl) langEl.value = prefs.speech_lang || "en-US";
} catch (e) {
console.error("[settings] failed to load preferences:", e);
}
@@ -306,7 +332,9 @@ function wireEvents() {
const user_name = (document.getElementById("input-user-name") as HTMLInputElement).value.trim();
const honorific = (document.getElementById("input-honorific") as HTMLSelectElement).value;
const calendar_accounts = (document.getElementById("input-calendar-accounts") as HTMLTextAreaElement).value.trim();
- await apiPost("/api/settings/preferences", { user_name, honorific, calendar_accounts });
+ const speech_lang = (document.getElementById("input-speech-lang") as HTMLSelectElement).value;
+ await apiPost("/api/settings/preferences", { user_name, honorific, calendar_accounts, speech_lang });
+ document.dispatchEvent(new CustomEvent(SPEECH_LANG_EVENT, { detail: speech_lang }));
await loadStatus();
});
diff --git a/frontend/src/style.css b/frontend/src/style.css
index 899901c7..97380461 100644
--- a/frontend/src/style.css
+++ b/frontend/src/style.css
@@ -35,6 +35,27 @@ html, body {
z-index: 10;
}
+/* What JARVIS just said. Sits above the status line, and carries the
+ conversation on its own when TTS is unavailable. */
+#jarvis-caption {
+ position: fixed;
+ bottom: 80px;
+ left: 50%;
+ transform: translateX(-50%);
+ width: min(640px, calc(100% - 48px));
+ color: rgba(219, 238, 252, 0.92);
+ font-size: 17px;
+ line-height: 1.55;
+ font-weight: 300;
+ letter-spacing: 0.3px;
+ text-align: center;
+ text-shadow: 0 2px 12px rgba(0, 0, 0, 0.85);
+ transition: opacity 0.45s ease;
+ pointer-events: none;
+ z-index: 10;
+ opacity: 0;
+}
+
#jarvis-label {
position: fixed;
bottom: 16px;
diff --git a/frontend/src/voice.ts b/frontend/src/voice.ts
index 8ca5e0a8..158280e3 100644
--- a/frontend/src/voice.ts
+++ b/frontend/src/voice.ts
@@ -11,8 +11,12 @@ export interface VoiceInput {
stop(): void;
pause(): void;
resume(): void;
+ setLanguage(lang: string): void;
}
+/** Fallback until the configured language arrives from the server. */
+export const DEFAULT_SPEECH_LANG = "en-US";
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
declare const webkitSpeechRecognition: any;
@@ -24,48 +28,65 @@ export function createVoiceInput(
const SR = (window as any).SpeechRecognition || (typeof webkitSpeechRecognition !== "undefined" ? webkitSpeechRecognition : null);
if (!SR) {
onError("Speech recognition not supported in this browser");
- return { start() {}, stop() {}, pause() {}, resume() {} };
+ return { start() {}, stop() {}, pause() {}, resume() {}, setLanguage() {} };
}
- const recognition = new SR();
- recognition.continuous = true;
- recognition.interimResults = true;
- recognition.lang = "en-US";
-
let shouldListen = false;
let paused = false;
+ let currentLang = DEFAULT_SPEECH_LANG;
+
+ /**
+ * Build a recognition session.
+ *
+ * Chrome reads `lang` when the object is constructed and ignores later
+ * assignment on a session that has already run, so switching language means
+ * building a new object rather than re-tagging this one.
+ */
+ function buildRecognition(): any {
+ const r = new SR();
+ r.continuous = true;
+ r.interimResults = true;
+ r.lang = currentLang;
+
+ r.onresult = (event: any) => {
+ for (let i = event.resultIndex; i < event.results.length; i++) {
+ if (event.results[i].isFinal) {
+ const text = event.results[i][0].transcript.trim();
+ if (text) onTranscript(text);
+ }
+ }
+ };
- recognition.onresult = (event: any) => {
- for (let i = event.resultIndex; i < event.results.length; i++) {
- if (event.results[i].isFinal) {
- const text = event.results[i][0].transcript.trim();
- if (text) onTranscript(text);
+ r.onend = () => {
+ // A session replaced by a language switch must stay down, or it would
+ // race the new one for the microphone and keep the old language alive.
+ if (r !== recognition) return;
+ if (shouldListen && !paused) {
+ try {
+ r.start();
+ } catch {
+ // Already started
+ }
}
- }
- };
+ };
- recognition.onend = () => {
- if (shouldListen && !paused) {
- try {
- recognition.start();
- } catch {
- // Already started
+ r.onerror = (event: any) => {
+ if (event.error === "not-allowed") {
+ onError("Microphone access denied. Please allow microphone access.");
+ shouldListen = false;
+ } else if (event.error === "no-speech") {
+ // Normal, just restart
+ } else if (event.error === "aborted") {
+ // Expected during pause
+ } else {
+ console.warn("[voice] recognition error:", event.error);
}
- }
- };
+ };
- recognition.onerror = (event: any) => {
- if (event.error === "not-allowed") {
- onError("Microphone access denied. Please allow microphone access.");
- shouldListen = false;
- } else if (event.error === "no-speech") {
- // Normal, just restart
- } else if (event.error === "aborted") {
- // Expected during pause
- } else {
- console.warn("[voice] recognition error:", event.error);
- }
- };
+ return r;
+ }
+
+ let recognition = buildRecognition();
return {
start() {
@@ -96,6 +117,34 @@ export function createVoiceInput(
}
}
},
+ setLanguage(lang: string) {
+ if (!lang || lang === currentLang) return;
+ currentLang = lang;
+
+ const previous = recognition;
+ recognition = buildRecognition();
+ // Retires the old session: its onend now sees itself superseded.
+ try {
+ previous.stop();
+ } catch {
+ // Wasn't running.
+ }
+
+ if (shouldListen && !paused) {
+ // Let the retired session release the microphone before claiming it,
+ // otherwise Chrome rejects the new start outright.
+ setTimeout(() => {
+ if (recognition !== previous && shouldListen && !paused) {
+ try {
+ recognition.start();
+ } catch {
+ // Already started
+ }
+ }
+ }, 250);
+ }
+ console.log("[voice] language set to", lang);
+ },
};
}
diff --git a/server.py b/server.py
index f08e7370..9bd975db 100644
--- a/server.py
+++ b/server.py
@@ -65,6 +65,29 @@
FISH_VOICE_ID = os.getenv("FISH_VOICE_ID", "612b878b113047d9a770c069c8b4fdfe") # JARVIS (MCU)
FISH_API_URL = "https://api.fish.audio/v1/tts"
USER_NAME = os.getenv("USER_NAME", "sir")
+# BCP-47 tag the browser's speech recognition listens in, and the language
+# JARVIS is told to reply in. Read per-request rather than cached, so changing
+# it in the settings panel takes effect without a restart.
+DEFAULT_SPEECH_LANG = "en-US"
+# Offered in the settings panel, mapped to the name used to tell JARVIS which
+# language to answer in.
+SPEECH_LANGUAGES = {
+ "en-US": "English",
+ "en-GB": "English",
+ "it-IT": "Italian",
+ "es-ES": "Spanish",
+ "fr-FR": "French",
+ "de-DE": "German",
+ "pt-BR": "Portuguese",
+ "nl-NL": "Dutch",
+ "ja-JP": "Japanese",
+ "zh-CN": "Chinese",
+}
+
+
+def _current_speech_lang() -> str:
+ """The configured speech language, read live so the panel needs no restart."""
+ return os.getenv("SPEECH_LANG", DEFAULT_SPEECH_LANG) or DEFAULT_SPEECH_LANG
PROJECT_DIR = os.path.dirname(os.path.abspath(__file__))
_SKIP_PERMISSIONS = os.getenv("JARVIS_SKIP_PERMISSIONS", "true").lower() not in ("0", "false", "no")
@@ -900,11 +923,15 @@ async def _execute_research(target: str, ws=None):
try:
notify_text = f"Research is complete, sir. Report is open in your browser."
audio = await synthesize_speech(notify_text)
+ await ws.send_json({"type": "status", "state": "speaking"})
if audio:
- await ws.send_json({"type": "status", "state": "speaking"})
await ws.send_json({"type": "audio", "data": base64.b64encode(audio).decode(), "text": notify_text})
- await ws.send_json({"type": "status", "state": "idle"})
- log.info(f"JARVIS: {notify_text}")
+ else:
+ # No voice available — the reply still has to reach the client,
+ # which shows it on screen.
+ await ws.send_json({"type": "text", "text": notify_text})
+ await ws.send_json({"type": "status", "state": "idle"})
+ log.info(f"JARVIS: {notify_text}")
except Exception:
pass # WebSocket might be gone
@@ -915,6 +942,8 @@ async def _execute_research(target: str, ws=None):
audio = await synthesize_speech("Research timed out, sir. It was taking too long.")
if audio:
await ws.send_json({"type": "audio", "data": base64.b64encode(audio).decode(), "text": "Research timed out, sir."})
+ else:
+ await ws.send_json({"type": "text", "text": "Research timed out, sir."})
except Exception:
pass
except Exception as e:
@@ -982,10 +1011,13 @@ async def _execute_prompt_project(project_name: str, prompt: str, work_session:
if not project_dir:
msg = f"Couldn't find the {project_name} project directory, sir."
audio = await synthesize_speech(msg)
- if audio and ws:
+ if ws:
try:
await ws.send_json({"type": "status", "state": "speaking"})
- await ws.send_json({"type": "audio", "data": base64.b64encode(audio).decode(), "text": msg})
+ if audio:
+ await ws.send_json({"type": "audio", "data": base64.b64encode(audio).decode(), "text": msg})
+ else:
+ await ws.send_json({"type": "text", "text": msg})
except Exception:
pass
return
@@ -1078,9 +1110,12 @@ async def _execute_prompt_project(project_name: str, prompt: str, work_session:
try:
msg = f"Had trouble connecting to {project_name}, sir."
audio = await synthesize_speech(msg)
- if audio and ws:
+ if ws:
await ws.send_json({"type": "status", "state": "speaking"})
- await ws.send_json({"type": "audio", "data": base64.b64encode(audio).decode(), "text": msg})
+ if audio:
+ await ws.send_json({"type": "audio", "data": base64.b64encode(audio).decode(), "text": msg})
+ else:
+ await ws.send_json({"type": "text", "text": msg})
except Exception:
pass
@@ -1106,11 +1141,13 @@ async def self_work_and_notify(session: WorkSession, prompt: str, ws):
try:
audio = await synthesize_speech(msg)
+ await ws.send_json({"type": "status", "state": "speaking"})
if audio:
- await ws.send_json({"type": "status", "state": "speaking"})
await ws.send_json({"type": "audio", "data": base64.b64encode(audio).decode(), "text": msg})
- await ws.send_json({"type": "status", "state": "idle"})
- log.info(f"JARVIS: {msg}")
+ else:
+ await ws.send_json({"type": "text", "text": msg})
+ await ws.send_json({"type": "status", "state": "idle"})
+ log.info(f"JARVIS: {msg}")
except Exception:
pass
except Exception as e:
@@ -1197,6 +1234,20 @@ async def generate_response(
user_name=USER_NAME,
project_dir=PROJECT_DIR,
)
+ # The prompt above is written in English and would answer an Italian
+ # question in English, which makes the speech-language setting only half
+ # work. Kept to a bare language directive on purpose: wording that also
+ # discussed action selection measurably weakened the language adherence it
+ # was appended to enforce. Action routing is less reliable outside English
+ # regardless — see the non-English caveat in the README.
+ speech_lang = _current_speech_lang()
+ if not speech_lang.startswith("en"):
+ language = SPEECH_LANGUAGES.get(speech_lang, speech_lang)
+ system += (
+ f"\n\nLANGUAGE: The user speaks {language}. Write every spoken reply in {language}, "
+ f"keeping the same dry, economical butler voice."
+ )
+
if lookup_status:
system += f"\n\nACTIVE LOOKUPS:\n{lookup_status}\nIf asked about progress, report this status."
@@ -1698,7 +1749,7 @@ async def _lookup_and_report(lookup_type: str, lookup_fn, ws, history: list[dict
try:
await ws.send_json({"type": "status", "state": "speaking"})
if audio:
- await ws.send_json({"type": "audio", "data": audio, "text": result_text})
+ await ws.send_json({"type": "audio", "data": base64.b64encode(audio).decode(), "text": result_text})
else:
await ws.send_json({"type": "text", "text": result_text})
await ws.send_json({"type": "status", "state": "idle"})
@@ -1718,7 +1769,9 @@ async def _lookup_and_report(lookup_type: str, lookup_fn, ws, history: list[dict
audio = await synthesize_speech(fallback)
await ws.send_json({"type": "status", "state": "speaking"})
if audio:
- await ws.send_json({"type": "audio", "data": audio, "text": fallback})
+ await ws.send_json({"type": "audio", "data": base64.b64encode(audio).decode(), "text": fallback})
+ else:
+ await ws.send_json({"type": "text", "text": fallback})
await ws.send_json({"type": "status", "state": "idle"})
except Exception:
pass
@@ -1994,13 +2047,15 @@ async def voice_handler(ws: WebSocket):
async def _send_greeting():
try:
audio_bytes = await synthesize_speech(greeting)
+ await ws.send_json({"type": "status", "state": "speaking"})
if audio_bytes:
encoded = base64.b64encode(audio_bytes).decode()
- await ws.send_json({"type": "status", "state": "speaking"})
await ws.send_json({"type": "audio", "data": encoded, "text": greeting})
- history.append({"role": "assistant", "content": greeting})
- log.info(f"JARVIS: {greeting}")
- await ws.send_json({"type": "status", "state": "idle"})
+ else:
+ await ws.send_json({"type": "text", "text": greeting})
+ history.append({"role": "assistant", "content": greeting})
+ log.info(f"JARVIS: {greeting}")
+ await ws.send_json({"type": "status", "state": "idle"})
except Exception as e:
log.warning(f"Greeting failed: {e}")
@@ -2027,7 +2082,7 @@ async def _send_greeting():
await ws.send_json({"type": "status", "state": "speaking"})
audio = await synthesize_speech(tts)
if audio:
- await ws.send_json({"type": "audio", "data": audio, "text": response_text})
+ await ws.send_json({"type": "audio", "data": base64.b64encode(audio).decode(), "text": response_text})
else:
await ws.send_json({"type": "text", "text": response_text})
continue
@@ -2358,10 +2413,13 @@ async def _read_and_report(search_term, _ws):
else:
msg = f"Couldn't find a note matching '{search_term}', sir."
audio = await synthesize_speech(strip_markdown_for_tts(msg))
- if audio and _ws:
+ if _ws:
try:
await _ws.send_json({"type": "status", "state": "speaking"})
- await _ws.send_json({"type": "audio", "data": base64.b64encode(audio).decode(), "text": msg})
+ if audio:
+ await _ws.send_json({"type": "audio", "data": base64.b64encode(audio).decode(), "text": msg})
+ else:
+ await _ws.send_json({"type": "text", "text": msg})
except Exception:
pass
asyncio.create_task(_read_and_report(embedded_action["target"].strip(), ws))
@@ -2488,10 +2546,11 @@ class PreferencesUpdate(BaseModel):
user_name: str = ""
honorific: str = "sir"
calendar_accounts: str = "auto"
+ speech_lang: str = DEFAULT_SPEECH_LANG
@app.post("/api/settings/keys")
async def api_settings_keys(body: KeyUpdate):
- allowed = {"ANTHROPIC_API_KEY", "FISH_API_KEY", "FISH_VOICE_ID", "USER_NAME", "HONORIFIC", "CALENDAR_ACCOUNTS"}
+ allowed = {"ANTHROPIC_API_KEY", "FISH_API_KEY", "FISH_VOICE_ID", "USER_NAME", "HONORIFIC", "CALENDAR_ACCOUNTS", "SPEECH_LANG"}
if body.key_name not in allowed:
return JSONResponse({"success": False, "error": "Invalid key name"}, status_code=400)
_write_env_key(body.key_name, body.key_value)
@@ -2571,6 +2630,7 @@ async def api_get_preferences():
"user_name": env_dict.get("USER_NAME", ""),
"honorific": env_dict.get("HONORIFIC", "sir"),
"calendar_accounts": env_dict.get("CALENDAR_ACCOUNTS", "auto"),
+ "speech_lang": env_dict.get("SPEECH_LANG", DEFAULT_SPEECH_LANG) or DEFAULT_SPEECH_LANG,
}
@app.post("/api/settings/preferences")
@@ -2578,6 +2638,7 @@ async def api_save_preferences(body: PreferencesUpdate):
_write_env_key("USER_NAME", body.user_name)
_write_env_key("HONORIFIC", body.honorific)
_write_env_key("CALENDAR_ACCOUNTS", body.calendar_accounts)
+ _write_env_key("SPEECH_LANG", body.speech_lang or DEFAULT_SPEECH_LANG)
return {"success": True}
# ---------------------------------------------------------------------------