From 6386fc049f2ee729d06c7bb144193e8aa1fd71c4 Mon Sep 17 00:00:00 2001 From: Chenyme <118253778+chenyme@users.noreply.github.com> Date: Fri, 7 Aug 2026 16:19:42 +0800 Subject: [PATCH] fix: stabilize browser speech input on Android --- .../chat/components/sections/chat-input.tsx | 19 +- .../chat/hooks/use-chat-speech-input.ts | 166 ++++++++++++++---- frontend/i18n/messages/en-US/chat.json | 13 ++ frontend/i18n/messages/zh-CN/chat.json | 13 ++ 4 files changed, 171 insertions(+), 40 deletions(-) diff --git a/frontend/features/chat/components/sections/chat-input.tsx b/frontend/features/chat/components/sections/chat-input.tsx index e8a0f4595..eba956ffc 100644 --- a/frontend/features/chat/components/sections/chat-input.tsx +++ b/frontend/features/chat/components/sections/chat-input.tsx @@ -4,7 +4,7 @@ import * as React from "react"; import dynamic from "next/dynamic"; import { Box, CornerDownRight, Eye, EyeOff, Film, Image, ImageOff, ImagePlus, LoaderCircle, PencilLine, Trash2 } from "lucide-react"; import { AnimatePresence, motion } from "motion/react"; -import { useTranslations } from "next-intl"; +import { useLocale, useTranslations } from "next-intl"; import { toast } from "sonner"; import { AudioLines } from "@/components/animate-ui/icons/audio-lines"; @@ -24,7 +24,10 @@ import { formatClipboardMarkdownPaste, resolveClipboardMarkdownPaste, } from "@/features/chat/utils/markdown-paste"; -import { useChatSpeechInput } from "@/features/chat/hooks/use-chat-speech-input"; +import { + useChatSpeechInput, + type SpeechInputErrorCode, +} from "@/features/chat/hooks/use-chat-speech-input"; import { useMarkdownPreviewSync } from "@/features/chat/hooks/use-markdown-preview-sync"; import { useChatMentionMenu, @@ -279,17 +282,27 @@ function ChatInputComponent({ const tChat = useTranslations("chat"); const tComposer = useTranslations("chat.composer"); const tFileStatus = useTranslations("files.status"); + const locale = useLocale(); const [isBlocksHovered, setIsBlocksHovered] = React.useState(false); const [isVoiceHovered, setIsVoiceHovered] = React.useState(false); const [toolsMenuHovered, setToolsMenuHovered] = React.useState(false); const [toolsMenuOpen, setToolsMenuOpen] = React.useState(false); const [editingQueuedMessageID, setEditingQueuedMessageID] = React.useState(null); const [editingQueuedMessageContent, setEditingQueuedMessageContent] = React.useState(""); + const handleSpeechInputError = React.useCallback((error: SpeechInputErrorCode) => { + toast.error(tComposer("voiceErrorTitle"), { + id: "chat-speech-input-error", + description: tComposer(`voiceErrors.${error}`), + }); + }, [tComposer]); const speechInput = useChatSpeechInput({ draft, + language: locale, listeningPlaceholder: tComposer("voiceListeningPlaceholder"), onDraftChange, + onError: handleSpeechInputError, placeholder: tComposer("inputPlaceholder"), + startingPlaceholder: tComposer("voiceStartingPlaceholder"), }); const [hoveredTool, setHoveredTool] = React.useState<"upload" | "screenshot" | null>(null); const [ragWarnDismissed, setRagWarnDismissed] = React.useState(false); @@ -1056,6 +1069,8 @@ function ChatInputComponent({ strokeWidth={1.4} animate="default-loop" /> + ) : speechInput.status === "starting" ? ( + ) : speechInput.active ? ( > = { + "audio-capture": "audioUnavailable", + "language-not-supported": "languageUnsupported", + network: "network", + "not-allowed": "permissionDenied", + "service-not-allowed": "serviceUnavailable", +}; type UseChatSpeechInputParams = { draft: string; + language: string; listeningPlaceholder: string; onDraftChange: (value: string) => void; + onError: (error: SpeechInputErrorCode) => void; placeholder: string; + startingPlaceholder: string; }; type UseChatSpeechInputState = { @@ -62,20 +85,31 @@ type UseChatSpeechInputState = { export function useChatSpeechInput({ draft, + language, listeningPlaceholder, onDraftChange, + onError, placeholder, + startingPlaceholder, }: UseChatSpeechInputParams): UseChatSpeechInputState { const [supported, setSupported] = React.useState(false); const [status, setStatus] = React.useState("idle"); const recognitionRef = React.useRef(null); const draftRef = React.useRef(draft); const baseDraftRef = React.useRef(""); + const renderedDraftRef = React.useRef(""); const cancelledRef = React.useRef(false); + const emptyRestartCountRef = React.useRef(0); + const sessionHadResultRef = React.useRef(false); + const recoverableErrorRef = React.useRef<"aborted" | "no-speech" | null>(null); const restartTimerRef = React.useRef(null); const active = status !== "idle"; - const resolvedPlaceholder = active ? listeningPlaceholder : placeholder; + const resolvedPlaceholder = status === "starting" + ? startingPlaceholder + : status === "listening" + ? listeningPlaceholder + : placeholder; React.useEffect(() => { draftRef.current = draft; @@ -83,7 +117,8 @@ export function useChatSpeechInput({ React.useEffect(() => { const browserWindow = window as BrowserWindowWithSpeechRecognition; - setSupported(Boolean(browserWindow.SpeechRecognition ?? browserWindow.webkitSpeechRecognition)); + const RecognitionConstructor = browserWindow.SpeechRecognition ?? browserWindow.webkitSpeechRecognition; + setSupported(window.isSecureContext && Boolean(RecognitionConstructor)); return () => { if (restartTimerRef.current !== null) { @@ -91,19 +126,22 @@ export function useChatSpeechInput({ restartTimerRef.current = null; } cancelledRef.current = true; - recognitionRef.current?.stop(); + const recognition = recognitionRef.current; recognitionRef.current = null; + recognition?.stop(); }; }, []); const commitTranscript = React.useCallback( (finalTranscript: string, interimTranscript: string) => { - const fragments = [ + const nextDraft = [ baseDraftRef.current, finalTranscript.trim(), interimTranscript.trim(), - ].filter(Boolean); - onDraftChange(fragments.join(" ")); + ].filter(Boolean).join(" "); + renderedDraftRef.current = nextDraft; + draftRef.current = nextDraft; + onDraftChange(nextDraft); }, [onDraftChange], ); @@ -114,12 +152,15 @@ export function useChatSpeechInput({ window.clearTimeout(restartTimerRef.current); restartTimerRef.current = null; } - recognitionRef.current?.stop(); + const recognition = recognitionRef.current; + recognitionRef.current = null; + recognition?.stop(); setStatus("idle"); }, []); const toggle = React.useCallback(() => { if (!supported) { + onError("unavailable"); return; } if (active) { @@ -131,22 +172,57 @@ export function useChatSpeechInput({ const RecognitionConstructor = browserWindow.SpeechRecognition ?? browserWindow.webkitSpeechRecognition; if (!RecognitionConstructor) { setSupported(false); + onError("unavailable"); return; } cancelledRef.current = false; + emptyRestartCountRef.current = 0; baseDraftRef.current = draftRef.current.trimEnd(); + renderedDraftRef.current = baseDraftRef.current; + + const failStart = () => { + cancelledRef.current = true; + recognitionRef.current = null; + setStatus("idle"); + onError("startFailed"); + }; const startRecognition = () => { - const recognition = new RecognitionConstructor(); - recognition.continuous = true; + let recognition: BrowserSpeechRecognition; + try { + recognition = new RecognitionConstructor(); + } catch { + failStart(); + return; + } + + sessionHadResultRef.current = false; + recoverableErrorRef.current = null; + recognition.continuous = false; recognition.interimResults = true; - recognition.lang = navigator.language || "zh-CN"; + recognition.lang = language; + + const finishWithError = (error: SpeechInputErrorCode) => { + if (recognitionRef.current !== recognition) { + return; + } + cancelledRef.current = true; + recognitionRef.current = null; + setStatus("idle"); + onError(error); + }; + recognition.onstart = () => { - setStatus("listening"); + if (!cancelledRef.current && recognitionRef.current === recognition) { + setStatus("listening"); + } }; recognition.onresult = (event) => { - setStatus("listening"); + if (cancelledRef.current || recognitionRef.current !== recognition) { + return; + } + const finalTranscripts: string[] = []; const interimTranscripts: string[] = []; for (let resultIndex = 0; resultIndex < event.results.length; resultIndex += 1) { @@ -164,54 +240,68 @@ export function useChatSpeechInput({ interimTranscripts.push(transcript); } } + if (finalTranscripts.length === 0 && interimTranscripts.length === 0) { + return; + } + + sessionHadResultRef.current = true; + emptyRestartCountRef.current = 0; + recoverableErrorRef.current = null; + setStatus("listening"); commitTranscript(finalTranscripts.join(" "), interimTranscripts.join(" ")); }; recognition.onerror = (event) => { - if (cancelledRef.current) { - setStatus("idle"); + if (cancelledRef.current || recognitionRef.current !== recognition) { return; } if (event.error === "no-speech" || event.error === "aborted") { - setStatus("listening"); + recoverableErrorRef.current = event.error; + setStatus("starting"); return; } - cancelledRef.current = true; - recognitionRef.current = null; - setStatus("idle"); + + finishWithError(SPEECH_RECOGNITION_ERROR_CODES[event.error] ?? "unavailable"); }; recognition.onend = () => { - if (!cancelledRef.current) { - if (recognitionRef.current !== recognition) { - return; - } - setStatus("listening"); - restartTimerRef.current = window.setTimeout(() => { - restartTimerRef.current = null; - if (cancelledRef.current || recognitionRef.current !== recognition) { - return; - } - startRecognition(); - }, 180); + if (recognitionRef.current !== recognition) { return; } - if (recognitionRef.current === recognition) { + if (cancelledRef.current) { recognitionRef.current = null; + setStatus("idle"); + return; } - setStatus("idle"); + + baseDraftRef.current = renderedDraftRef.current.trimEnd(); + if (!sessionHadResultRef.current) { + emptyRestartCountRef.current += 1; + if (emptyRestartCountRef.current > MAX_EMPTY_RESTARTS) { + finishWithError(recoverableErrorRef.current === "aborted" ? "interrupted" : "noSpeech"); + return; + } + } + + setStatus("starting"); + restartTimerRef.current = window.setTimeout(() => { + restartTimerRef.current = null; + if (cancelledRef.current || recognitionRef.current !== recognition) { + return; + } + startRecognition(); + }, RESTART_DELAY_MS); }; recognitionRef.current = recognition; + setStatus("starting"); try { recognition.start(); - setStatus("listening"); } catch { - recognitionRef.current = null; - setStatus("idle"); + failStart(); } }; startRecognition(); - }, [active, commitTranscript, stop, supported]); + }, [active, commitTranscript, language, onError, stop, supported]); return { supported, diff --git a/frontend/i18n/messages/en-US/chat.json b/frontend/i18n/messages/en-US/chat.json index e9633258a..36fda7263 100644 --- a/frontend/i18n/messages/en-US/chat.json +++ b/frontend/i18n/messages/en-US/chat.json @@ -347,6 +347,7 @@ "screenshot": "Screenshot", "openTools": "Open tools", "inputPlaceholder": "Message, or use / for context, @ for resources", + "voiceStartingPlaceholder": "Starting voice input…", "voiceListeningPlaceholder": "Receiving voice input…", "modelOptions": "Parameters", "mcpTools": "MCP tools", @@ -431,6 +432,18 @@ "voiceInput": "Voice input", "cancelVoiceInput": "Cancel voice input", "voiceUnsupported": "Voice input is not supported by this browser", + "voiceErrorTitle": "Voice input stopped", + "voiceErrors": { + "audioUnavailable": "No available microphone was found. Check the microphone permission and device.", + "interrupted": "Speech recognition was repeatedly interrupted. Please try again.", + "languageUnsupported": "The browser speech service does not support the current language.", + "network": "The browser speech service could not be reached. Check the network and try again.", + "noSpeech": "No speech was detected. Please try again and speak closer to the microphone.", + "permissionDenied": "Microphone permission was denied. Allow microphone access and try again.", + "serviceUnavailable": "The browser speech service is unavailable on this device.", + "startFailed": "Speech recognition could not be started. Please try again.", + "unavailable": "Voice input is unavailable in the current browser environment." + }, "booleanOn": "On", "booleanOff": "Off", "tool": "Tool {id}", diff --git a/frontend/i18n/messages/zh-CN/chat.json b/frontend/i18n/messages/zh-CN/chat.json index 733dd293c..13930941b 100644 --- a/frontend/i18n/messages/zh-CN/chat.json +++ b/frontend/i18n/messages/zh-CN/chat.json @@ -347,6 +347,7 @@ "screenshot": "屏幕截图", "openTools": "打开工具菜单", "inputPlaceholder": "输入消息，或用 / 添加上下文，@ 添加资源", + "voiceStartingPlaceholder": "正在启动语音输入…", "voiceListeningPlaceholder": "正在接收语音…", "modelOptions": "模型配置", "mcpTools": "MCP 工具", @@ -431,6 +432,18 @@ "voiceInput": "语音输入", "cancelVoiceInput": "取消语音输入", "voiceUnsupported": "当前浏览器不支持语音输入", + "voiceErrorTitle": "语音输入已停止", + "voiceErrors": { + "audioUnavailable": "未找到可用的麦克风，请检查麦克风权限和设备。", + "interrupted": "语音识别连续被中断，请重试。", + "languageUnsupported": "浏览器语音服务不支持当前语言。", + "network": "无法连接浏览器语音服务，请检查网络后重试。", + "noSpeech": "未检测到语音，请靠近麦克风后重试。", + "permissionDenied": "麦克风权限被拒绝，请允许访问后重试。", + "serviceUnavailable": "当前设备无法使用浏览器语音服务。", + "startFailed": "无法启动语音识别，请重试。", + "unavailable": "当前浏览器环境无法使用语音输入。" + }, "booleanOn": "开启", "booleanOff": "关闭", "tool": "工具 {id}",