From 04e9ba2fb9c9b6bfe8b48219cf7536c6b053c2ea Mon Sep 17 00:00:00 2001
From: buyun14 <2135481470@qq.com>
Date: Tue, 1 Sep 2026 16:23:25 +0800
Subject: [PATCH] feat: add STT latency and RTF tracking metrics
Track speech-to-text call timing and real-time factor (RTF) in both the pipeline preprocess stage and the web chat live chat service, and surface the metrics in the Live Mode dashboard.
- preprocess_stage: probe audio duration via get_media_duration, measure STT duration, log provider/model/duration/RTF, and record into event.trace for observability.
- live_chat_service: emit stt_total_time / stt_audio_duration / stt_rtf over the live chat websocket.
- LiveMode.vue: render the new STT latency metrics.
---
.../core/pipeline/preprocess_stage/stage.py | 46 +++++++++++++++++--
.../dashboard/services/live_chat_service.py | 30 ++++++++++++
dashboard/src/components/chat/LiveMode.vue | 8 ++++
3 files changed, 81 insertions(+), 3 deletions(-)
diff --git a/astrbot/core/pipeline/preprocess_stage/stage.py b/astrbot/core/pipeline/preprocess_stage/stage.py
index a090ccb197..36d4e29160 100644
--- a/astrbot/core/pipeline/preprocess_stage/stage.py
+++ b/astrbot/core/pipeline/preprocess_stage/stage.py
@@ -1,5 +1,6 @@
import asyncio
import random
+import time
import traceback
from collections.abc import AsyncGenerator
from pathlib import Path
@@ -13,6 +14,7 @@
ensure_jpeg,
ensure_wav,
file_uri_to_path,
+ get_media_duration,
is_file_uri,
)
@@ -182,20 +184,29 @@ async def process(
async def _stt_record(record_comp: Record, is_reply: bool = False):
"""对单个 Record 组件执行语音转文本，成功返回 Plain，失败返回 None。"""
prefix = "referenced " if is_reply else ""
+ suffix = " (referenced message)" if is_reply else ""
try:
path = await record_comp.convert_to_file_path()
except Exception as e:
logger.warning(f"Failed to resolve the {prefix}voice path: {e}")
return None
+ # 探测音频时长（毫秒），用于计算实时率 RTF；失败时为 None
+ try:
+ audio_duration_ms = await get_media_duration(path)
+ except Exception:
+ audio_duration_ms = None
+
+ stt_start = time.time()
+ status = "failed"
retry = 5
for i in range(retry):
try:
result = await stt_provider.get_text(audio_url=path)
if result:
- suffix = " (referenced message)" if is_reply else ""
+ status = "success"
logger.info(f"Speech-to-text{suffix} result: " + result)
- return Plain(result)
+ break
break
except FileNotFoundError:
# napcat workaround: file may not be ready immediately
@@ -206,9 +217,38 @@ async def _stt_record(record_comp: Record, is_reply: bool = False):
continue
except BaseException as e:
logger.error(traceback.format_exc())
- suffix = " (referenced message)" if is_reply else ""
logger.error(f"Speech-to-text{suffix} failed: {e}")
break
+
+ stt_duration = time.time() - stt_start
+ rtf = (
+ stt_duration / (audio_duration_ms / 1000.0)
+ if audio_duration_ms
+ else None
+ )
+
+ provider_type = getattr(stt_provider.meta(), "type", "")
+ provider_model = stt_provider.get_model()
+ logger.info(
+ f"Speech-to-text{suffix} stats: provider={provider_type} "
+ f"model={provider_model} duration={stt_duration * 1000:.0f}ms "
+ f"audio={audio_duration_ms if audio_duration_ms is not None else 'N/A'}ms "
+ f"rtf={rtf if rtf is not None else 'N/A'} status={status}"
+ )
+ event.trace.record(
+ "stt",
+ provider=provider_type,
+ model=provider_model,
+ status=status,
+ referenced=is_reply,
+ audio_duration_ms=audio_duration_ms,
+ stt_duration=stt_duration,
+ rtf=rtf,
+ audio=Path(path).name,
+ )
+
+ if status == "success":
+ return Plain(result)
return None
message_chain = event.get_messages()
diff --git a/astrbot/dashboard/services/live_chat_service.py b/astrbot/dashboard/services/live_chat_service.py
index 16b7eed0ad..7d6359aeaf 100644
--- a/astrbot/dashboard/services/live_chat_service.py
+++ b/astrbot/dashboard/services/live_chat_service.py
@@ -29,6 +29,7 @@
from astrbot.core.platform.sources.webchat.webchat_queue_mgr import webchat_queue_mgr
from astrbot.core.utils.astrbot_path import get_astrbot_data_path, get_astrbot_temp_path
from astrbot.core.utils.datetime_utils import generate_timestamp_id, to_utc_isoformat
+from astrbot.core.utils.media_utils import get_media_duration
from astrbot.dashboard.services.chat_service import (
BotMessageAccumulator,
build_bot_history_content,
@@ -884,7 +885,36 @@ async def process_audio(
await send_json({"t": "metrics", "data": {"stt": stt_provider.meta().type}})
+ stt_start = time.time()
user_text = await stt_provider.get_text(audio_path)
+ stt_duration = time.time() - stt_start
+
+ # 探测音频时长（毫秒）并计算实时率（RTF），探测失败时优雅降级
+ try:
+ audio_duration_ms = await get_media_duration(audio_path)
+ except Exception:
+ audio_duration_ms = None
+ audio_duration_sec = (
+ audio_duration_ms / 1000.0 if audio_duration_ms else None
+ )
+ rtf = stt_duration / audio_duration_sec if audio_duration_sec else None
+ logger.info(
+ f"[Live Chat] STT stats: provider={stt_provider.meta().type} "
+ f"model={stt_provider.get_model()} duration={stt_duration * 1000:.0f}ms "
+ f"audio={audio_duration_ms if audio_duration_ms is not None else 'N/A'}ms "
+ f"rtf={rtf if rtf is not None else 'N/A'}"
+ )
+ await send_json(
+ {
+ "t": "metrics",
+ "data": {
+ "stt_total_time": stt_duration,
+ "stt_audio_duration": audio_duration_sec,
+ "stt_rtf": rtf,
+ },
+ }
+ )
+
if not user_text:
logger.warning("[Live Chat] STT 识别结果为空")
return
diff --git a/dashboard/src/components/chat/LiveMode.vue b/dashboard/src/components/chat/LiveMode.vue
index 654ffb49f4..a0356f9461 100644
--- a/dashboard/src/components/chat/LiveMode.vue
+++ b/dashboard/src/components/chat/LiveMode.vue
@@ -44,6 +44,11 @@
1000).toFixed(0) }}ms
Speak -> End: {{ (metrics.wav_to_tts_total_time *
1000).toFixed(0) }}ms
+ STT Total Latency: {{ (metrics.stt_total_time *
+ 1000).toFixed(0) }}ms
+ STT Audio Duration: {{ (metrics.stt_audio_duration *
+ 1000).toFixed(0) }}ms
+ STT RTF: {{ metrics.stt_rtf.toFixed(2) }}
STT Provider: {{ metrics.stt }}
TTS Provider: {{ metrics.tts }}
Chat Model: {{ metrics.chat_model }}
@@ -108,6 +113,9 @@ interface LiveMetrics {
tts_first_frame_time?: number;
tts_total_time?: number;
wav_to_tts_total_time?: number;
+ stt_total_time?: number;
+ stt_audio_duration?: number;
+ stt_rtf?: number;
stt?: string;
tts?: string;
chat_model?: string;