From e62ca4362df3af406ac3d1826fe1eaa88e09be14 Mon Sep 17 00:00:00 2001 From: octo-patch <266937838+octo-patch@users.noreply.github.com> Date: Thu, 13 Aug 2026 05:53:15 +0800 Subject: [PATCH] feat: add MiniMax TTS backend --- backend/core/voice_service.py | 152 ++++++++++++++++++++++++- backend/tests/test_unit_minimax_tts.py | 114 +++++++++++++++++++ 2 files changed, 260 insertions(+), 6 deletions(-) create mode 100644 backend/tests/test_unit_minimax_tts.py diff --git a/backend/core/voice_service.py b/backend/core/voice_service.py index f138873c..3cd89b9e 100644 --- a/backend/core/voice_service.py +++ b/backend/core/voice_service.py @@ -13,6 +13,7 @@ from dataclasses import dataclass, field from typing import Any, AsyncIterator, TypedDict +import httpx import websockets from PIL import Image, ImageDraw @@ -94,9 +95,28 @@ def _env_bool(name: str, default: bool) -> bool: VOICE_DASHSCOPE_API_KEY = _env_str("VOICE_DASHSCOPE_API_KEY", "") VOICE_STT_API_KEY = _env_str("VOICE_STT_API_KEY", "") VOICE_TTS_API_KEY = _env_str("VOICE_TTS_API_KEY", "") +VOICE_TTS_PROVIDER = _env_str("VOICE_TTS_PROVIDER", "") +VOICE_MINIMAX_TTS_MODEL = _env_str("VOICE_MINIMAX_TTS_MODEL", "speech-2.8-hd") +VOICE_MINIMAX_TTS_REGION = _env_str("VOICE_MINIMAX_TTS_REGION", "global_en") +VOICE_MINIMAX_TTS_VOICE = _env_str("VOICE_MINIMAX_TTS_VOICE", "") VOICE_OPUS_FRAME_DURATION_MS = 60 VOICE_OPUS_FRAME_SIZE = 16000 * VOICE_OPUS_FRAME_DURATION_MS // 1000 +MINIMAX_TTS_MODELS = ( + "speech-2.8-hd", + "speech-2.8-turbo", + "speech-2.6-hd", + "speech-2.6-turbo", + "speech-02-hd", + "speech-02-turbo", + "speech-01-hd", + "speech-01-turbo", +) +MINIMAX_TTS_ENDPOINTS = { + "global_en": "https://api.minimax.io/v1/t2a_v2", + "cn_zh": "https://api.minimaxi.com/v1/t2a_v2", +} + class VoiceTurn(TypedDict): created_at: float @@ -116,6 +136,10 @@ class VoiceRuntimeSettings: llm_base_url: str | None = None stt_api_key: str | None = None tts_api_key: str | None = None + tts_provider: str = "default" + tts_model: str = VOICE_MINIMAX_TTS_MODEL + tts_region: str = VOICE_MINIMAX_TTS_REGION + tts_voice: str = VOICE_MINIMAX_TTS_VOICE @classmethod def from_llm( @@ -129,6 +153,10 @@ def from_llm( shared_voice_api_key = VOICE_DASHSCOPE_API_KEY or _env_str("DASHSCOPE_API_KEY", "") or None env_stt_key = VOICE_STT_API_KEY or shared_voice_api_key env_tts_key = VOICE_TTS_API_KEY or shared_voice_api_key + configured_tts_provider = VOICE_TTS_PROVIDER.strip().lower() + use_minimax_tts = configured_tts_provider == "minimax" or ( + not configured_tts_provider and llm_provider == "minimax" + ) user_aliyun_key = ( str(llm_api_key).strip() if llm_provider == "aliyun" and llm_api_key and str(llm_api_key).strip() @@ -141,6 +169,9 @@ def from_llm( else: stt_api_key = env_stt_key tts_api_key = env_tts_key + if use_minimax_tts: + user_minimax_key = str(llm_api_key).strip() if llm_provider == "minimax" and llm_api_key else "" + tts_api_key = user_minimax_key or VOICE_TTS_API_KEY or _env_str("MINIMAX_API_KEY", "") or None return cls( llm_provider=llm_provider, llm_model=llm_model, @@ -148,6 +179,7 @@ def from_llm( llm_base_url=llm_base_url, stt_api_key=stt_api_key, tts_api_key=tts_api_key, + tts_provider="minimax" if use_minimax_tts else "default", ) @@ -204,7 +236,7 @@ class VoiceWsSessionState: active_turn_id: str | None = None active_turn_transcript: str = "" turn_metrics: "VoiceWsTurnMetrics | None" = None - pending_tts_bridge: "_StreamingTtsBridge | None" = None + pending_tts_bridge: "_StreamingTtsBridge | _MiniMaxTtsBridge | None" = None _auto_commit_task: asyncio.Task[None] | None = None @@ -537,6 +569,15 @@ def _dashscope_tts_ws_url() -> str: return VOICE_STREAMING_TTS_WS_URL +def _minimax_tts_url(region: str) -> str: + normalized = region.strip().lower() + try: + return MINIMAX_TTS_ENDPOINTS[normalized] + except KeyError as exc: + supported = ", ".join(MINIMAX_TTS_ENDPOINTS) + raise ValueError(f"Unsupported MiniMax TTS region: {region}. Expected one of: {supported}") from exc + + async def _transcribe_pcm_bytes( pcm_bytes: bytes, *, @@ -910,7 +951,7 @@ def _split_delta_tts_segments(buffer: str, *, final: bool, idle_break: bool) -> async def _synthesize_reply_pcm(reply_text: str, *, settings: VoiceRuntimeSettings) -> bytes: started_at = time.perf_counter() - bridge = _StreamingTtsBridge(settings=settings, finish_delay_ms=VOICE_PROMPT_TTS_FINISH_DELAY_MS) + bridge = _create_tts_bridge(settings=settings, finish_delay_ms=VOICE_PROMPT_TTS_FINISH_DELAY_MS) bridge.start() bridge.feed_text(reply_text) bridge.finish() @@ -1138,15 +1179,114 @@ async def _receiver() -> None: self._loop.call_soon_threadsafe(self._audio_queue.put_nowait, None) +async def _synthesize_minimax_pcm(text: str, *, settings: VoiceRuntimeSettings) -> bytes: + if settings.tts_model not in MINIMAX_TTS_MODELS: + raise ValueError(f"Unsupported MiniMax TTS model: {settings.tts_model}") + if not settings.tts_api_key: + raise RuntimeError("Missing MiniMax API key") + + request_body: dict[str, Any] = { + "model": settings.tts_model, + "text": text, + "stream": False, + "output_format": "hex", + "audio_setting": { + "format": "pcm", + "sample_rate": VOICE_STREAMING_TTS_SAMPLE_RATE, + }, + } + if settings.tts_voice: + request_body["voice_setting"] = {"voice_id": settings.tts_voice} + + async with httpx.AsyncClient(timeout=180) as client: + response = await client.post( + _minimax_tts_url(settings.tts_region), + headers={"Authorization": f"Bearer {settings.tts_api_key}"}, + json=request_body, + ) + response.raise_for_status() + payload = response.json() + + base_resp = payload.get("base_resp") or {} + if base_resp.get("status_code") != 0: + message = base_resp.get("status_msg") or "MiniMax TTS request failed" + raise RuntimeError(message) + audio_hex = str((payload.get("data") or {}).get("audio") or "") + if not audio_hex: + raise RuntimeError("MiniMax TTS response did not include audio") + try: + return bytes.fromhex(audio_hex) + except ValueError as exc: + raise RuntimeError("MiniMax TTS response included invalid hex audio") from exc + + +class _MiniMaxTtsBridge: + """Buffer text segments and synthesize them with the MiniMax HTTP API.""" + + def __init__(self, *, settings: VoiceRuntimeSettings, finish_delay_ms: int = 0) -> None: + self._settings = settings + self._finish_delay_ms = max(0, finish_delay_ms) + self._text_queue: asyncio.Queue[str | None] = asyncio.Queue() + self._started_at = time.perf_counter() + self._first_audio_at = 0.0 + + def start(self) -> None: + return + + def feed_text(self, text: str) -> None: + self._text_queue.put_nowait(text) + + def finish(self) -> None: + self._text_queue.put_nowait(None) + + async def iter_audio(self) -> AsyncIterator[bytes]: + text_parts: list[str] = [] + while True: + text = await self._text_queue.get() + if text is None: + break + normalized = _normalize_tts_stream_text(text) + if normalized: + text_parts.append(normalized) + if not text_parts: + return + if self._finish_delay_ms: + await asyncio.sleep(self._finish_delay_ms / 1000) + audio = await _synthesize_minimax_pcm(" ".join(text_parts), settings=self._settings) + self._first_audio_at = time.perf_counter() + yield audio + + @property + def first_audio_delay_ms(self) -> int: + if self._first_audio_at <= 0: + return -1 + return int((self._first_audio_at - self._started_at) * 1000) + + +def _create_tts_bridge( + *, + settings: VoiceRuntimeSettings, + finish_delay_ms: int = 0, +) -> _StreamingTtsBridge | _MiniMaxTtsBridge: + if settings.tts_provider == "minimax": + return _MiniMaxTtsBridge(settings=settings, finish_delay_ms=finish_delay_ms) + return _StreamingTtsBridge(settings=settings, finish_delay_ms=finish_delay_ms) + + async def synthesize_prompt_pcm(text: str, settings: VoiceRuntimeSettings | None = None) -> bytes: effective_settings = settings or VoiceRuntimeSettings.from_llm( llm_provider="aliyun", llm_model="qwen3-coder-480b-a35b-instruct", ) + use_minimax_tts = effective_settings.tts_provider == "minimax" + tts_model = effective_settings.tts_model if use_minimax_tts else VOICE_STREAMING_TTS_MODEL + tts_voice = effective_settings.tts_voice if use_minimax_tts else VOICE_STREAMING_TTS_VOICE cache_key = "|".join( [ - VOICE_STREAMING_TTS_MODEL, - VOICE_STREAMING_TTS_VOICE, + effective_settings.tts_provider, + tts_model, + effective_settings.tts_region, + tts_voice, text, ] ) @@ -1621,7 +1761,7 @@ async def _feed_tts_segment(segment: str) -> None: tts_bridge = session.pending_tts_bridge session.pending_tts_bridge = None else: - tts_bridge = _StreamingTtsBridge(settings=session.settings) + tts_bridge = _create_tts_bridge(settings=session.settings) tts_bridge.start() llm_done = asyncio.Event() @@ -2114,7 +2254,7 @@ async def commit_voice_ws_audio(session: VoiceWsSessionState) -> None: if should_warmup: _start_voice_ws_generation(session, transcript=speculative_transcript, turn_id=turn_id, warmup=True) if VOICE_STREAMING_TTS_ENABLED and session.pending_tts_bridge is None: - session.pending_tts_bridge = _StreamingTtsBridge(settings=session.settings) + session.pending_tts_bridge = _create_tts_bridge(settings=session.settings) session.pending_tts_bridge.start() transcript = await session.asr_bridge.commit() old_asr_bridge = session.asr_bridge diff --git a/backend/tests/test_unit_minimax_tts.py b/backend/tests/test_unit_minimax_tts.py new file mode 100644 index 00000000..e49e7dca --- /dev/null +++ b/backend/tests/test_unit_minimax_tts.py @@ -0,0 +1,114 @@ +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from core.voice_service import ( + MINIMAX_TTS_ENDPOINTS, + MINIMAX_TTS_MODELS, + VoiceRuntimeSettings, + _MiniMaxTtsBridge, + _create_tts_bridge, + _synthesize_minimax_pcm, +) + + +def test_minimax_provider_selects_tts_backend(monkeypatch): + monkeypatch.setenv("MINIMAX_API_KEY", "env-minimax-key") + + settings = VoiceRuntimeSettings.from_llm( + llm_provider="minimax", + llm_model="MiniMax-M3", + ) + + assert settings.tts_provider == "minimax" + assert settings.tts_api_key == "env-minimax-key" + assert settings.tts_model == "speech-2.8-hd" + assert set(MINIMAX_TTS_ENDPOINTS) == {"global_en", "cn_zh"} + assert len(MINIMAX_TTS_MODELS) == 8 + + +@pytest.mark.asyncio +async def test_minimax_tts_request_and_hex_response(): + response = MagicMock() + response.json.return_value = { + "data": {"audio": "000102ff"}, + "base_resp": {"status_code": 0}, + } + response.raise_for_status = MagicMock() + client = AsyncMock() + client.post.return_value = response + context_manager = AsyncMock() + context_manager.__aenter__.return_value = client + + settings = VoiceRuntimeSettings( + llm_provider="minimax", + llm_model="MiniMax-M3", + tts_api_key="test-key", + tts_provider="minimax", + tts_model="speech-2.8-turbo", + tts_region="cn_zh", + tts_voice="test-voice", + ) + with patch("core.voice_service.httpx.AsyncClient", return_value=context_manager): + audio = await _synthesize_minimax_pcm("Hello", settings=settings) + + assert audio == b"\x00\x01\x02\xff" + response.raise_for_status.assert_called_once_with() + call = client.post.call_args + assert call.args[0] == "https://api.minimaxi.com/v1/t2a_v2" + assert call.kwargs["headers"] == {"Authorization": "Bearer test-key"} + assert call.kwargs["json"] == { + "model": "speech-2.8-turbo", + "text": "Hello", + "stream": False, + "output_format": "hex", + "voice_setting": {"voice_id": "test-voice"}, + "audio_setting": {"format": "pcm", "sample_rate": 16000}, + } + + +@pytest.mark.asyncio +async def test_minimax_tts_reports_api_error(): + response = MagicMock() + response.json.return_value = { + "data": {"audio": ""}, + "base_resp": {"status_code": 1001, "status_msg": "invalid request"}, + } + client = AsyncMock() + client.post.return_value = response + context_manager = AsyncMock() + context_manager.__aenter__.return_value = client + settings = VoiceRuntimeSettings( + llm_provider="minimax", + llm_model="MiniMax-M3", + tts_api_key="test-key", + tts_provider="minimax", + ) + + with ( + patch("core.voice_service.httpx.AsyncClient", return_value=context_manager), + pytest.raises(RuntimeError, match="invalid request"), + ): + await _synthesize_minimax_pcm("Hello", settings=settings) + + +@pytest.mark.asyncio +async def test_minimax_bridge_collects_streamed_text(): + settings = VoiceRuntimeSettings( + llm_provider="minimax", + llm_model="MiniMax-M3", + tts_api_key="test-key", + tts_provider="minimax", + ) + bridge = _create_tts_bridge(settings=settings) + assert isinstance(bridge, _MiniMaxTtsBridge) + bridge.start() + bridge.feed_text("Hello") + bridge.feed_text(" world") + bridge.finish() + + with patch("core.voice_service._synthesize_minimax_pcm", new=AsyncMock(return_value=b"pcm")) as synthesize: + chunks = [chunk async for chunk in bridge.iter_audio()] + + assert chunks == [b"pcm"] + synthesize.assert_awaited_once_with("Hello world", settings=settings)