Voice UX: replies that arrive, in the language you speak

frontend/index.html

@@ -32,6 +32,7 @@
       <button id="btn-fix-self">Fix Yourself</button>
     </div>
 
+    <div id="jarvis-caption"></div>
     <div id="status-text"></div>
     <div id="jarvis-label">JARVIS</div>
     <div id="error-text"></div>

frontend/src/main.ts

@@ -8,7 +8,7 @@
 import { createOrb, type OrbState } from "./orb";
 import { createVoiceInput, createAudioPlayer } from "./voice";
 import { createSocket } from "./ws";
-import { openSettings, checkFirstTimeSetup } from "./settings";
+import { openSettings, checkFirstTimeSetup, SPEECH_LANG_EVENT } from "./settings";
 import "./style.css";
 
 // ---------------------------------------------------------------------------
@@ -21,6 +21,7 @@ let isMuted = false;
 
 const statusEl = document.getElementById("status-text")!;
 const errorEl = document.getElementById("error-text")!;
+const captionEl = document.getElementById("jarvis-caption")!;
 
 function showError(msg: string) {
   errorEl.textContent = msg;
@@ -30,6 +31,28 @@ function showError(msg: string) {
   }, 5000);
 }
 
+let captionTimer: number | undefined;
+
+/**
+ * Put what JARVIS said on screen.
+ *
+ * Shown whether or not TTS produced audio: with a voice it reads as
+ * subtitles, and without one it is the only way to see the reply — the
+ * response used to reach nothing but the devtools console.
+ */
+function showCaption(text: unknown) {
+  if (typeof text !== "string" || !text.trim()) return;
+  clearTimeout(captionTimer);
+  captionEl.textContent = text;
+  captionEl.style.opacity = "1";
+  // Hold long enough to read the line, since a caption may be all the user
+  // gets. Replies are a sentence or two, so length is a fair proxy.
+  const holdMs = Math.min(20000, 4000 + text.length * 60);
+  captionTimer = window.setTimeout(() => {
+    captionEl.style.opacity = "0";
+  }, holdMs);
+}
+
 function updateStatus(state: State) {
   const labels: Record<State, string> = {
     idle: "",
@@ -80,13 +103,59 @@ function transition(newState: State) {
 // Voice input
 // ---------------------------------------------------------------------------
 
+/**
+ * Fragment accumulation.
+ *
+ * Chrome closes a recognition segment at every pause, so a sentence spoken
+ * with any hesitation arrives as several "final" results. Sending each one
+ * immediately made JARVIS answer half a thought and then receive the rest as
+ * a separate question. Instead, collect the pieces and send once the speaker
+ * has actually stopped.
+ */
+const UTTERANCE_GAP_MS = 1000;
+// A pause never arrives while someone is dictating steadily, so cap the wait
+// rather than let a long sentence hold the whole conversation open.
+const UTTERANCE_MAX_MS = 6000;
+
+let pendingFragments: string[] = [];
+let gapTimer: number | undefined;
+let maxTimer: number | undefined;
+
+function discardUtterance() {
+  clearTimeout(gapTimer);
+  clearTimeout(maxTimer);
+  gapTimer = undefined;
+  maxTimer = undefined;
+  pendingFragments = [];
+}
+
+function flushUtterance() {
+  clearTimeout(gapTimer);
+  clearTimeout(maxTimer);
+  gapTimer = undefined;
+  maxTimer = undefined;
+
+  const text = pendingFragments.join(" ").replace(/\s+/g, " ").trim();
+  pendingFragments = [];
+  if (!text) return;
+
+  socket.send({ type: "transcript", text, isFinal: true });
+  transition("thinking");
+}
+
 const voiceInput = createVoiceInput(
   (text: string) => {
-    // Cancel any current JARVIS response before sending new input
+    // Cancel any current JARVIS response before sending new input. Done on the
+    // first fragment, not at flush time: cutting him off is what the user
+    // wanted the moment they started talking.
     audioPlayer.stop();
-    // User spoke — send transcript
-    socket.send({ type: "transcript", text, isFinal: true });
-    transition("thinking");
+
+    pendingFragments.push(text);
+    clearTimeout(gapTimer);
+    gapTimer = window.setTimeout(flushUtterance, UTTERANCE_GAP_MS);
+    if (maxTimer === undefined) {
+      maxTimer = window.setTimeout(flushUtterance, UTTERANCE_MAX_MS);
+    }
   },
   (msg: string) => {
     showError(msg);
@@ -123,6 +192,7 @@ socket.onMessage((msg) => {
     }
     // Log text for debugging
     if (msg.text) console.log("[JARVIS]", msg.text);
+    showCaption(msg.text);
   } else if (type === "status") {
     const state = msg.state as string;
     if (state === "thinking" && currentState !== "thinking") {
@@ -137,6 +207,7 @@ socket.onMessage((msg) => {
   } else if (type === "text") {
     // Text fallback when TTS fails
     console.log("[JARVIS]", msg.text);
+    showCaption(msg.text);
   } else if (type === "task_spawned") {
     console.log("[task]", "spawned:", msg.task_id, msg.prompt);
   } else if (type === "task_complete") {
@@ -148,12 +219,40 @@ socket.onMessage((msg) => {
 // Kick off
 // ---------------------------------------------------------------------------
 
+/**
+ * Read the speech language chosen in the settings panel.
+ *
+ * Resolved before the microphone opens so the first session is built in the
+ * right language: switching a live session means tearing it down and racing
+ * for the microphone, which is worth avoiding on every page load. Bounded, so
+ * an unreachable server costs a moment rather than the microphone.
+ */
+async function fetchSpeechLanguage(): Promise<string | null> {
+  try {
+    const res = await fetch("/api/settings/preferences", {
+      signal: AbortSignal.timeout(2000),
+    });
+    const prefs = (await res.json()) as { speech_lang?: string };
+    return prefs.speech_lang || null;
+  } catch {
+    return null; // Server not ready — keep the default language.
+  }
+}
+
 // Start listening after a brief delay for the orb to render
-setTimeout(() => {
+setTimeout(async () => {
+  const lang = await fetchSpeechLanguage();
+  if (lang) voiceInput.setLanguage(lang);
   voiceInput.start();
   transition("listening");
 }, 1000);
 
+// Saving the setting re-tunes the running session — no reload needed.
+document.addEventListener(SPEECH_LANG_EVENT, (e) => {
+  const lang = (e as CustomEvent<string>).detail;
+  if (lang) voiceInput.setLanguage(lang);
+});
+
 // Resume AudioContext on ANY user interaction (browser autoplay policy)
 function ensureAudioContext() {
   const ctx = audioPlayer.getAnalyser().context as AudioContext;
@@ -183,6 +282,8 @@ btnMute.addEventListener("click", (e) => {
   isMuted = !isMuted;
   btnMute.classList.toggle("muted", isMuted);
   if (isMuted) {
+    // Half-spoken words must not arrive a second after the user silenced him.
+    discardUtterance();
     voiceInput.pause();
     transition("idle");
   } else {

frontend/src/settings.ts

@@ -30,8 +30,16 @@ interface PreferencesResponse {
   user_name: string;
   honorific: string;
   calendar_accounts: string;
+  speech_lang: string;
 }
 
+/**
+ * Event fired when the speech language changes, so the live recognition
+ * session picks it up without a reload. Used instead of importing the voice
+ * input directly, which would make settings and main circular.
+ */
+export const SPEECH_LANG_EVENT = "jarvis:speech-lang";
+
 // ---------------------------------------------------------------------------
 // State
 // ---------------------------------------------------------------------------
@@ -143,6 +151,22 @@ function buildPanelHTML(): string {
             </select>
           </div>
 
+          <div class="settings-field">
+            <label>Spoken Language</label>
+            <select id="input-speech-lang">
+              <option value="en-US">English (US)</option>
+              <option value="en-GB">English (UK)</option>
+              <option value="it-IT">Italiano</option>
+              <option value="es-ES">Español</option>
+              <option value="fr-FR">Français</option>
+              <option value="de-DE">Deutsch</option>
+              <option value="pt-BR">Português (Brasil)</option>
+              <option value="nl-NL">Nederlands</option>
+              <option value="ja-JP">日本語</option>
+              <option value="zh-CN">中文</option>
+            </select>
+          </div>
+
           <div class="settings-field">
             <label>Calendar Accounts</label>
             <textarea id="input-calendar-accounts" rows="2" placeholder="auto (or comma-separated emails)"></textarea>
@@ -242,9 +266,11 @@ async function loadPreferences() {
     const nameEl = document.getElementById("input-user-name") as HTMLInputElement;
     const honEl = document.getElementById("input-honorific") as HTMLSelectElement;
     const calEl = document.getElementById("input-calendar-accounts") as HTMLTextAreaElement;
+    const langEl = document.getElementById("input-speech-lang") as HTMLSelectElement;
     if (nameEl) nameEl.value = prefs.user_name || "";
     if (honEl) honEl.value = prefs.honorific || "sir";
     if (calEl) calEl.value = prefs.calendar_accounts || "auto";
+    if (langEl) langEl.value = prefs.speech_lang || "en-US";
   } catch (e) {
     console.error("[settings] failed to load preferences:", e);
   }
@@ -306,7 +332,9 @@ function wireEvents() {
     const user_name = (document.getElementById("input-user-name") as HTMLInputElement).value.trim();
     const honorific = (document.getElementById("input-honorific") as HTMLSelectElement).value;
     const calendar_accounts = (document.getElementById("input-calendar-accounts") as HTMLTextAreaElement).value.trim();
-    await apiPost("/api/settings/preferences", { user_name, honorific, calendar_accounts });
+    const speech_lang = (document.getElementById("input-speech-lang") as HTMLSelectElement).value;
+    await apiPost("/api/settings/preferences", { user_name, honorific, calendar_accounts, speech_lang });
+    document.dispatchEvent(new CustomEvent(SPEECH_LANG_EVENT, { detail: speech_lang }));
     await loadStatus();
   });
 

frontend/src/style.css

@@ -35,6 +35,27 @@ html, body {
   z-index: 10;
 }
 
+/* What JARVIS just said. Sits above the status line, and carries the
+   conversation on its own when TTS is unavailable. */
+#jarvis-caption {
+  position: fixed;
+  bottom: 80px;
+  left: 50%;
+  transform: translateX(-50%);
+  width: min(640px, calc(100% - 48px));
+  color: rgba(219, 238, 252, 0.92);
+  font-size: 17px;
+  line-height: 1.55;
+  font-weight: 300;
+  letter-spacing: 0.3px;
+  text-align: center;
+  text-shadow: 0 2px 12px rgba(0, 0, 0, 0.85);
+  transition: opacity 0.45s ease;
+  pointer-events: none;
+  z-index: 10;
+  opacity: 0;
+}
+
 #jarvis-label {
   position: fixed;
   bottom: 16px;