From d344799c5f3b4d71eb28c3570561a19cfd1e376a Mon Sep 17 00:00:00 2001 From: jedzill4 Date: Wed, 17 Jun 2026 00:56:07 -0400 Subject: [PATCH 01/22] feat(asr): replace TRANSCRIBE_WS_URI with TRANSCRIBE_BASE_URL/API_KEY --- aymurai/settings.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/aymurai/settings.py b/aymurai/settings.py index e141afdc..5a97cff6 100644 --- a/aymurai/settings.py +++ b/aymurai/settings.py @@ -65,9 +65,10 @@ def assemble_cors_origins(cls, v) -> list[str]: SQLALCHEMY_DATABASE_URI: str = "sqlite:////resources/cache/sqlite/database.db" ########################################################################## - # ASR Config + # ASR Config (coro — OpenAI-compatible server) ########################################################################## - TRANSCRIBE_WS_URI: str | None = None + TRANSCRIBE_BASE_URL: str | None = None + TRANSCRIBE_API_KEY: str = "not-needed" ########################################################################## # Disambiguation Config From 979b4bb1619578c10fc6c436f61b238b441205e2 Mon Sep 17 00:00:00 2001 From: jedzill4 Date: Wed, 17 Jun 2026 00:56:39 -0400 Subject: [PATCH 02/22] feat(asr): add coro boundary schema, remove WLK websocket schemas --- aymurai/api/meta/asr/coro.py | 62 ++++++++++++++++ aymurai/api/meta/asr/websocket.py | 115 ------------------------------ 2 files changed, 62 insertions(+), 115 deletions(-) create mode 100644 aymurai/api/meta/asr/coro.py delete mode 100644 aymurai/api/meta/asr/websocket.py diff --git a/aymurai/api/meta/asr/coro.py b/aymurai/api/meta/asr/coro.py new file mode 100644 index 00000000..31db6df3 --- /dev/null +++ b/aymurai/api/meta/asr/coro.py @@ -0,0 +1,62 @@ +import re +from datetime import timedelta + +from pydantic import BaseModel, ConfigDict + +ISO8601_DURATION_RE = re.compile( + r"^PT(?:(?P\d+(?:\.\d+)?)H)?" + r"(?:(?P\d+(?:\.\d+)?)M)?" + r"(?:(?P\d+(?:\.\d+)?)S)?$" +) + + +def _parse_hhmmss(value: str | int | float | timedelta) -> timedelta: + """ + Parse a time value in HH:MM:SS format, ISO 8601 duration (PT#H#M#S), or seconds. + + Args: + value (str | int | float | timedelta): The time value to parse. + + Raises: + ValueError: If the value is not a valid time format. + + Returns: + timedelta: The parsed time value. + """ + if isinstance(value, timedelta): + return value + + if isinstance(value, (int, float)): + return timedelta(seconds=float(value)) + + if not isinstance(value, str): + raise ValueError("Invalid time format") + + if value.startswith("PT"): + match = ISO8601_DURATION_RE.match(value) + if match is None: + raise ValueError("Expected ISO 8601 duration format (PT#H#M#S)") + hours = float(match.group("hours") or 0) + minutes = float(match.group("minutes") or 0) + seconds = float(match.group("seconds") or 0) + return timedelta(hours=hours, minutes=minutes, seconds=seconds) + + parts = value.split(":") + if len(parts) != 3: + raise ValueError("Expected HH:MM:SS format") + + hours_str, minutes_str, seconds_str = parts + return timedelta( + hours=int(hours_str), minutes=int(minutes_str), seconds=float(seconds_str) + ) + + +class CoroSegment(BaseModel): + """A speaker-attributed transcription segment from the coro done-frame.""" + + model_config = ConfigDict(extra="ignore") + + start: float + end: float + text: str + speaker: str diff --git a/aymurai/api/meta/asr/websocket.py b/aymurai/api/meta/asr/websocket.py deleted file mode 100644 index c1e4a46e..00000000 --- a/aymurai/api/meta/asr/websocket.py +++ /dev/null @@ -1,115 +0,0 @@ -import re -from datetime import timedelta -from typing import ClassVar, Literal - -from pydantic import BaseModel, ConfigDict, field_validator - -ISO8601_DURATION_RE = re.compile( - r"^PT(?:(?P\d+(?:\.\d+)?)H)?" - r"(?:(?P\d+(?:\.\d+)?)M)?" - r"(?:(?P\d+(?:\.\d+)?)S)?$" -) - - -def _parse_hhmmss(value: str | int | float | timedelta) -> timedelta: - """ - Parse a time value in HH:MM:SS format, ISO 8601 duration format (PT#H#M#S), or as a number of seconds. - - Args: - value (str | int | float | timedelta): The time value to parse. - - Raises: - ValueError: If the time value is not a valid format, or if the ISO 8601 duration format is invalid. - - Returns: - timedelta: The parsed time value as a timedelta object. - """ - if isinstance(value, timedelta): - return value - - if isinstance(value, (int, float)): - return timedelta(seconds=float(value)) - - if not isinstance(value, str): - raise ValueError("Invalid time format") - - if value.startswith("PT"): - match = ISO8601_DURATION_RE.match(value) - if match is None: - raise ValueError("Expected ISO 8601 duration format (PT#H#M#S)") - - hours = float(match.group("hours") or 0) - minutes = float(match.group("minutes") or 0) - seconds = float(match.group("seconds") or 0) - - return timedelta(hours=hours, minutes=minutes, seconds=seconds) - - parts = value.split(":") - if len(parts) != 3: - raise ValueError("Expected HH:MM:SS format") - - hours_str, minutes_str, seconds_str = parts - hours = int(hours_str) - minutes = int(minutes_str) - seconds = float(seconds_str) - return timedelta(hours=hours, minutes=minutes, seconds=seconds) - - -class WLKMessageConfig(BaseModel): - type: Literal["config"] - useAudioWorklet: bool - - -class WLKMessageTranscriptionLine(BaseModel): - speaker: int - text: str - start: timedelta - end: timedelta - detected_language: str | None = None - - @field_validator("text", mode="before") - @classmethod - def normalize_text(cls, value: str | None) -> str: - if value is None: - return "" - return value - - @field_validator("start", "end", mode="before") - @classmethod - def parse_hhmmss(cls, value: str | int | float | timedelta) -> timedelta: - return _parse_hhmmss(value) - - -class WLKMessageStatus(BaseModel): - status: str - lines: list[WLKMessageTranscriptionLine] - buffer_transcription: str - buffer_diarization: str - buffer_translation: str - remaining_time_transcription: float - remaining_time_diarization: float - speaker_ids: dict[str, str] | None = None - - -class WLKMessageReadyToStopMessage(BaseModel): - type: Literal["ready_to_stop"] - - -WLKMessageRawResponse = ( - WLKMessageConfig | WLKMessageStatus | WLKMessageReadyToStopMessage -) - - -class TranscriptionItem(BaseModel): - model_config: ClassVar[ConfigDict] = ConfigDict(from_attributes=True) - - speaker_no: int - speaker_name: str | None = None - start: timedelta - end: timedelta - text: str - - @field_validator("start", "end", mode="before") - @classmethod - def parse_hhmmss(cls, value: str | int | float | timedelta) -> timedelta: - return _parse_hhmmss(value) From 9b0dacea70b2c3211fe579453982c04a3b26cae2 Mon Sep 17 00:00:00 2001 From: jedzill4 Date: Wed, 17 Jun 2026 00:57:20 -0400 Subject: [PATCH 03/22] refactor(asr): fold TranscriptionItem into ASRParagraph --- aymurai/meta/api_interfaces.py | 34 ++++++++++++++++++++++++++++------ 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/aymurai/meta/api_interfaces.py b/aymurai/meta/api_interfaces.py index a576df9a..67c78e82 100644 --- a/aymurai/meta/api_interfaces.py +++ b/aymurai/meta/api_interfaces.py @@ -3,11 +3,20 @@ import uuid from datetime import timedelta from functools import cached_property -from typing import TYPE_CHECKING, Literal - -from pydantic import UUID4, UUID5, BaseModel, Field, RootModel, computed_field - -from aymurai.api.meta.asr.websocket import TranscriptionItem +from typing import TYPE_CHECKING, ClassVar, Literal + +from pydantic import ( + UUID4, + UUID5, + BaseModel, + ConfigDict, + Field, + RootModel, + computed_field, + field_validator, +) + +from aymurai.api.meta.asr.coro import _parse_hhmmss from aymurai.database.utils import text_to_uuid from aymurai.meta.entities import EntityAttributes @@ -90,7 +99,20 @@ class Document(BaseModel): footer: list[str] | None = None -class ASRParagraph(TranscriptionItem): +class ASRParagraph(BaseModel): + model_config: ClassVar[ConfigDict] = ConfigDict(from_attributes=True) + + speaker_no: int + speaker_name: str | None = None + start: timedelta + end: timedelta + text: str + + @field_validator("start", "end", mode="before") + @classmethod + def parse_hhmmss(cls, value: str | int | float | timedelta) -> timedelta: + return _parse_hhmmss(value) + @computed_field @property def paragraph_id(self) -> UUID: From 93e1bdf0b17fdd4100a37c9bb90ed600f0cc9e86 Mon Sep 17 00:00:00 2001 From: jedzill4 Date: Wed, 17 Jun 2026 01:02:31 -0400 Subject: [PATCH 04/22] docs(asr): add ASRParagraph docstring --- aymurai/meta/api_interfaces.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/aymurai/meta/api_interfaces.py b/aymurai/meta/api_interfaces.py index 67c78e82..fe32211b 100644 --- a/aymurai/meta/api_interfaces.py +++ b/aymurai/meta/api_interfaces.py @@ -100,6 +100,8 @@ class Document(BaseModel): class ASRParagraph(BaseModel): + """A speaker-attributed transcription paragraph (persisted and anonymized).""" + model_config: ClassVar[ConfigDict] = ConfigDict(from_attributes=True) speaker_no: int From ce2f21c73cacdecaf1d893eaff4658bb704b2d01 Mon Sep 17 00:00:00 2001 From: jedzill4 Date: Wed, 17 Jun 2026 01:05:37 -0400 Subject: [PATCH 05/22] feat(asr): rewrite client to consume coro over openai SDK SSE --- aymurai/audio/asr_client.py | 202 +++++++--------------------------- test/audio/test_asr_client.py | 73 ++++++++++++ 2 files changed, 112 insertions(+), 163 deletions(-) create mode 100644 test/audio/test_asr_client.py diff --git a/aymurai/audio/asr_client.py b/aymurai/audio/asr_client.py index e045fb14..da682e2f 100644 --- a/aymurai/audio/asr_client.py +++ b/aymurai/audio/asr_client.py @@ -1,186 +1,62 @@ -import asyncio -import contextlib -import io import json -from pathlib import Path -import librosa -import numpy as np -import websockets -from pydantic import TypeAdapter, ValidationError +from openai import APIConnectionError, APIError, AsyncOpenAI -from aymurai.api.meta.asr.websocket import ( - WLKMessageRawResponse, - WLKMessageReadyToStopMessage, - WLKMessageStatus, -) +from aymurai.api.meta.asr.coro import CoroSegment from aymurai.logger import get_logger from aymurai.settings import settings logger = get_logger(__name__) -SAMPLE_RATE_HZ = 16000 -CHUNK_SECONDS = 1 -CHUNK_SAMPLES = SAMPLE_RATE_HZ * CHUNK_SECONDS -MAX_WS_LOG_CHARS = 2000 -ASR_RAW_RESPONSE_ADAPTER = TypeAdapter(WLKMessageRawResponse) +DONE_EVENT_TYPE = "transcript.text.done" -async def _stream_audio_bytes( +async def transcribe_audio_bytes( payload: bytes, - websocket: websockets.ClientConnection, -) -> int: + filename: str, + content_type: str, +) -> list[CoroSegment]: """ - Streams audio bytes to a WebSocket connection in chunks. + Transcribe audio by streaming it to the coro server over SSE. - Args: - payload (bytes): The audio data to be streamed. - websocket (websockets.ClientConnection): The WebSocket connection to stream the audio data to. - - Returns: - int: The total number of bytes sent to the WebSocket. - """ - audio, _ = librosa.load(io.BytesIO(payload), sr=SAMPLE_RATE_HZ, mono=True) - total_bytes = 0 - for i in range(0, len(audio), CHUNK_SAMPLES): - chunk = audio[i : i + CHUNK_SAMPLES] - if len(chunk) == 0: - continue - chunk_int16 = (chunk * 32768).astype(np.int16) - data = chunk_int16.tobytes() - total_bytes += len(data) - await websocket.send(data) - return total_bytes - - -def _parse_ws_message(message: str | bytes) -> WLKMessageRawResponse | None: - """ - Parses a WebSocket message into a WLKMessageRawResponse object. - - Args: - message (str | bytes): The WebSocket message to be parsed. - - Returns: - WLKMessageRawResponse | None: The parsed WLKMessageRawResponse object, or None if parsing fails. - """ - if isinstance(message, bytes): - message = message.decode("utf-8", errors="replace") - - payload_preview = message - if len(payload_preview) > MAX_WS_LOG_CHARS: - payload_preview = ( - f"{payload_preview[:MAX_WS_LOG_CHARS]}" - f"...[truncated {len(payload_preview) - MAX_WS_LOG_CHARS} chars]" - ) - - try: - parsed = json.loads(message) - except json.JSONDecodeError: - logger.warning("received non-json websocket payload: %s", payload_preview) - return None - - try: - return ASR_RAW_RESPONSE_ADAPTER.validate_python(parsed) - except ValidationError as exc: - logger.warning( - "received unrecognized websocket payload: %s; payload=%s", - exc, - payload_preview, - ) - return None - - -async def _receive_updates( - websocket: websockets.ClientConnection, -) -> WLKMessageStatus | None: - """ - Receives updates from the WebSocket connection and returns the last active transcription status. + Sends the audio to coro's OpenAI-compatible ``/v1/audio/transcriptions`` + endpoint with ``stream=True`` and parses the ``transcript.text.done`` frame. Args: - websocket (websockets.ClientConnection): The WebSocket connection to receive updates from. - - Returns: - WLKMessageStatus | None: The last active transcription status, - or None if no active transcription was received. - """ - last_active_transcription: WLKMessageStatus | None = None - while True: - try: - msg = await websocket.recv() - except websockets.exceptions.ConnectionClosedOK: - logger.info("connection closed normally") - break - except websockets.exceptions.WebSocketException as exc: - logger.error("websocket error while receiving updates: %s", exc) - break - - parsed = _parse_ws_message(msg) - match parsed: - case None: - continue - case WLKMessageStatus(status="active_transcription") as message: - last_active_transcription = message - case WLKMessageReadyToStopMessage(): - break - - return last_active_transcription - - -async def transcribe_audio_bytes(payload: bytes) -> WLKMessageStatus | None: - """ - Transcribes audio bytes by streaming them to a WebSocket transcription service and receiving updates. - - Args: - payload (bytes): The audio data to be transcribed. + payload (bytes): The raw audio file bytes (any ffmpeg-decodable format). + filename (str): The uploaded file name (used for the multipart part). + content_type (str): The uploaded file MIME type. Raises: - RuntimeError: If there is an error with the transcription service. + RuntimeError: If the base URL is unset, the service errors, or no done + frame is received. Returns: - WLKMessageStatus | None: The last active transcription status received from the transcription service, - or None if no active transcription was received. + list[CoroSegment]: The speaker-attributed transcription segments. """ - ws_uri = settings.TRANSCRIBE_WS_URI - - if not ws_uri: - raise RuntimeError("TRANSCRIBE_WS_URI is not configured") + base_url = settings.TRANSCRIBE_BASE_URL + if not base_url: + raise RuntimeError("TRANSCRIBE_BASE_URL is not configured") - logger.info("streaming audio for transcription") + client = AsyncOpenAI(base_url=base_url, api_key=settings.TRANSCRIBE_API_KEY) + logger.info("streaming audio to coro for transcription") try: - async with websockets.connect(ws_uri) as websocket: - receive_task = asyncio.create_task(_receive_updates(websocket)) - try: - total_bytes = await _stream_audio_bytes(payload, websocket) - await websocket.send(b"") - logger.info("sent %s bytes to transcription service", total_bytes) - last_active_transcription = await receive_task - except Exception: - if not receive_task.done(): - receive_task.cancel() - with contextlib.suppress(asyncio.CancelledError): - await receive_task - raise - except websockets.exceptions.WebSocketException as exc: - logger.error("websocket error during transcription: %s", exc) - raise RuntimeError("Transcription service websocket error") from exc - except Exception as exc: - logger.error("unexpected error during transcription: %s", exc) - raise RuntimeError("Unexpected error during transcription") from exc - - return last_active_transcription - - -def transcribe_audio_path(path: Path) -> WLKMessageStatus | None: - """ - Transcribes an audio file at the given path by reading its bytes and sending them to the transcription service. - - Args: - path (Path): The path to the audio file to be transcribed. - - Returns: - WLKMessageStatus | None: The last active transcription status received from the transcription service, - or None if no active transcription was received. - """ - payload = path.read_bytes() - return asyncio.run(transcribe_audio_bytes(payload)) + stream = await client.audio.transcriptions.create( + file=(filename, payload, content_type), + model="whisper-1", + language="es", + stream=True, + ) + async for event in stream: + if event.type == DONE_EVENT_TYPE: + data = json.loads(event.text) + return [ + CoroSegment.model_validate(segment) + for segment in data.get("segments", []) + ] + except (APIError, APIConnectionError) as exc: + logger.error("coro transcription error: %s", exc) + raise RuntimeError("Transcription service error") from exc + + raise RuntimeError("coro stream ended without a done frame") diff --git a/test/audio/test_asr_client.py b/test/audio/test_asr_client.py new file mode 100644 index 00000000..10cc82d3 --- /dev/null +++ b/test/audio/test_asr_client.py @@ -0,0 +1,73 @@ +import json +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from aymurai.api.meta.asr.coro import CoroSegment +from aymurai.audio.asr_client import transcribe_audio_bytes +from aymurai.settings import settings + + +class _FakeEvent: + def __init__(self, type: str, text: str | None = None): + self.type = type + self.text = text + + +class _FakeStream: + def __init__(self, events): + self._events = events + + def __aiter__(self): + async def _gen(): + for event in self._events: + yield event + + return _gen() + + +def _patched_client(events): + create = AsyncMock(return_value=_FakeStream(events)) + client = MagicMock() + client.audio.transcriptions.create = create + return patch("aymurai.audio.asr_client.AsyncOpenAI", return_value=client) + + +@pytest.mark.asyncio +async def test_should_return_segments_when_done_frame_received(monkeypatch): + monkeypatch.setattr(settings, "TRANSCRIBE_BASE_URL", "http://coro.local/v1") + done_text = json.dumps( + { + "segments": [ + {"start": 0.0, "end": 1.0, "text": "Hola", "speaker": "1"}, + {"start": 1.0, "end": 2.0, "text": "mundo", "speaker": "2"}, + ] + } + ) + events = [ + _FakeEvent("transcript.text.delta", None), + _FakeEvent("transcript.text.done", done_text), + ] + with _patched_client(events): + result = await transcribe_audio_bytes(b"audio", "sample.wav", "audio/wav") + + assert result == [ + CoroSegment(start=0.0, end=1.0, text="Hola", speaker="1"), + CoroSegment(start=1.0, end=2.0, text="mundo", speaker="2"), + ] + + +@pytest.mark.asyncio +async def test_should_raise_when_stream_has_no_done_frame(monkeypatch): + monkeypatch.setattr(settings, "TRANSCRIBE_BASE_URL", "http://coro.local/v1") + events = [_FakeEvent("transcript.text.delta", None)] + with _patched_client(events): + with pytest.raises(RuntimeError, match="done frame"): + await transcribe_audio_bytes(b"audio", "sample.wav", "audio/wav") + + +@pytest.mark.asyncio +async def test_should_raise_when_base_url_not_configured(monkeypatch): + monkeypatch.setattr(settings, "TRANSCRIBE_BASE_URL", None) + with pytest.raises(RuntimeError, match="not configured"): + await transcribe_audio_bytes(b"audio", "sample.wav", "audio/wav") From 46dda72ff7dc11dba378ffa8067a8da44b3ac6cf Mon Sep 17 00:00:00 2001 From: jedzill4 Date: Wed, 17 Jun 2026 01:08:03 -0400 Subject: [PATCH 06/22] test(asr): drive async client test with asyncio.run (repo has no pytest-asyncio) --- test/audio/test_asr_client.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/test/audio/test_asr_client.py b/test/audio/test_asr_client.py index 10cc82d3..3f2145f3 100644 --- a/test/audio/test_asr_client.py +++ b/test/audio/test_asr_client.py @@ -1,3 +1,4 @@ +import asyncio import json from unittest.mock import AsyncMock, MagicMock, patch @@ -33,8 +34,7 @@ def _patched_client(events): return patch("aymurai.audio.asr_client.AsyncOpenAI", return_value=client) -@pytest.mark.asyncio -async def test_should_return_segments_when_done_frame_received(monkeypatch): +def test_should_return_segments_when_done_frame_received(monkeypatch): monkeypatch.setattr(settings, "TRANSCRIBE_BASE_URL", "http://coro.local/v1") done_text = json.dumps( { @@ -49,7 +49,9 @@ async def test_should_return_segments_when_done_frame_received(monkeypatch): _FakeEvent("transcript.text.done", done_text), ] with _patched_client(events): - result = await transcribe_audio_bytes(b"audio", "sample.wav", "audio/wav") + result = asyncio.run( + transcribe_audio_bytes(b"audio", "sample.wav", "audio/wav") + ) assert result == [ CoroSegment(start=0.0, end=1.0, text="Hola", speaker="1"), @@ -57,17 +59,15 @@ async def test_should_return_segments_when_done_frame_received(monkeypatch): ] -@pytest.mark.asyncio -async def test_should_raise_when_stream_has_no_done_frame(monkeypatch): +def test_should_raise_when_stream_has_no_done_frame(monkeypatch): monkeypatch.setattr(settings, "TRANSCRIBE_BASE_URL", "http://coro.local/v1") events = [_FakeEvent("transcript.text.delta", None)] with _patched_client(events): with pytest.raises(RuntimeError, match="done frame"): - await transcribe_audio_bytes(b"audio", "sample.wav", "audio/wav") + asyncio.run(transcribe_audio_bytes(b"audio", "sample.wav", "audio/wav")) -@pytest.mark.asyncio -async def test_should_raise_when_base_url_not_configured(monkeypatch): +def test_should_raise_when_base_url_not_configured(monkeypatch): monkeypatch.setattr(settings, "TRANSCRIBE_BASE_URL", None) with pytest.raises(RuntimeError, match="not configured"): - await transcribe_audio_bytes(b"audio", "sample.wav", "audio/wav") + asyncio.run(transcribe_audio_bytes(b"audio", "sample.wav", "audio/wav")) From cb5f78bbe212fbaaee4a30fa06c33172fd59c49a Mon Sep 17 00:00:00 2001 From: jedzill4 Date: Wed, 17 Jun 2026 01:10:59 -0400 Subject: [PATCH 07/22] feat(asr): map coro segments to ASRParagraph in transcribe router --- .../api/endpoints/routers/asr/transcribe.py | 56 ++++++++++--------- 1 file changed, 29 insertions(+), 27 deletions(-) diff --git a/aymurai/api/endpoints/routers/asr/transcribe.py b/aymurai/api/endpoints/routers/asr/transcribe.py index 53a29f78..3c0dce78 100644 --- a/aymurai/api/endpoints/routers/asr/transcribe.py +++ b/aymurai/api/endpoints/routers/asr/transcribe.py @@ -28,61 +28,59 @@ logger = get_logger(__name__) -def get_transcribe_ws_uri() -> str: +def get_transcribe_base_url() -> str: """ - Get the WebSocket URI for the transcription service from settings. + Get the coro base URL for the transcription service from settings. Raises: - ConfigurationError: If the WebSocket URI is not configured in settings. + ConfigurationError: If the base URL is not configured in settings. Returns: - str: The WebSocket URI for the transcription service. + str: The coro base URL (e.g. http://localhost:8000/v1). """ - ws_uri = settings.TRANSCRIBE_WS_URI - if not ws_uri: - raise ConfigurationError(detail="TRANSCRIBE_WS_URI is not configured") - return ws_uri + base_url = settings.TRANSCRIBE_BASE_URL + if not base_url: + raise ConfigurationError(detail="TRANSCRIBE_BASE_URL is not configured") + return base_url async def _transcribe_audio_bytes_with_error_handling( data: bytes, + filename: str, + content_type: str, ) -> list[ASRParagraph]: """ - Transcribes audio bytes into a list of ASRParagraph objects. + Transcribe audio bytes into a list of ASRParagraph objects via coro. Args: data (bytes): The audio data to be transcribed. + filename (str): The uploaded file name. + content_type (str): The uploaded file MIME type. Raises: - UpstreamServiceError: If there is an error with the upstream transcription service. + UpstreamServiceError: If the coro service errors or returns no result. AymuraiAPIException: If there is an unexpected error during transcription. Returns: - list[ASRParagraph]: A list of ASRParagraph objects representing the transcribed audio. + list[ASRParagraph]: The transcribed paragraphs. """ try: - status = await transcribe_audio_bytes(data) + segments = await transcribe_audio_bytes(data, filename, content_type) except RuntimeError as exc: - message = str(exc) - if "websocket" in message.lower(): - raise UpstreamServiceError(detail=message) from exc - raise AymuraiAPIException(detail=message) from exc + raise UpstreamServiceError(detail=str(exc)) from exc except Exception as exc: raise AymuraiAPIException( detail="Unexpected error during transcription" ) from exc - if not status: - raise AymuraiAPIException(detail="No transcription result received") - return [ ASRParagraph( - speaker_no=line.speaker, - start=line.start, - end=line.end, - text=line.text, + speaker_no=int(segment.speaker), + start=segment.start, + end=segment.end, + text=segment.text, ) - for line in status.lines + for segment in segments ] @@ -90,7 +88,7 @@ async def _transcribe_audio_bytes_with_error_handling( async def transcribe( file: UploadFile, use_cache: bool = True, - ws_uri: str = Depends(get_transcribe_ws_uri), + base_url: str = Depends(get_transcribe_base_url), session: Session = Depends(get_session), ) -> ASRDocument: """ @@ -99,7 +97,7 @@ async def transcribe( Args: file (UploadFile): The audio file to be transcribed. use_cache (bool, optional): Whether to use cached transcription results. Defaults to True. - ws_uri (str, optional): The WebSocket URI for the transcription service. Defaults to Depends(get_transcribe_ws_uri). + base_url (str, optional): The coro base URL for the transcription service. Defaults to Depends(get_transcribe_base_url). session (Session, optional): The database session. Defaults to Depends(get_session). Returns: @@ -120,7 +118,11 @@ async def transcribe( ) return cached_document - transcription_items = await _transcribe_audio_bytes_with_error_handling(data) + transcription_items = await _transcribe_audio_bytes_with_error_handling( + data, + file.filename or str(document_id), + file.content_type or "application/octet-stream", + ) document = ASRDocument(document_id=document_id, document=transcription_items) audio_transcription_create_or_update( transcription_id=document_id, From 3e70e729dd74eb3541706dd4ffd47667aad92d39 Mon Sep 17 00:00:00 2001 From: jedzill4 Date: Wed, 17 Jun 2026 01:14:03 -0400 Subject: [PATCH 08/22] test(asr): update transcribe tests for coro client --- test/api/endpoints/routers/asr/conftest.py | 6 ++--- .../endpoints/routers/asr/test_transcribe.py | 27 ++++--------------- 2 files changed, 8 insertions(+), 25 deletions(-) diff --git a/test/api/endpoints/routers/asr/conftest.py b/test/api/endpoints/routers/asr/conftest.py index 270c0bdc..f61c3d4e 100644 --- a/test/api/endpoints/routers/asr/conftest.py +++ b/test/api/endpoints/routers/asr/conftest.py @@ -20,10 +20,10 @@ def _override_get_session(): yield session app.dependency_overrides[get_session] = _override_get_session - original_ws_uri = settings.TRANSCRIBE_WS_URI - settings.TRANSCRIBE_WS_URI = "ws://test-transcribe.local/ws" + original_base_url = settings.TRANSCRIBE_BASE_URL + settings.TRANSCRIBE_BASE_URL = "http://test-coro.local/v1" with TestClient(app) as client: yield client, sqlite_engine - settings.TRANSCRIBE_WS_URI = original_ws_uri + settings.TRANSCRIBE_BASE_URL = original_base_url diff --git a/test/api/endpoints/routers/asr/test_transcribe.py b/test/api/endpoints/routers/asr/test_transcribe.py index 6fd489ed..024bc1e2 100644 --- a/test/api/endpoints/routers/asr/test_transcribe.py +++ b/test/api/endpoints/routers/asr/test_transcribe.py @@ -4,10 +4,7 @@ from sqlmodel import Session -from aymurai.api.meta.asr.websocket import ( - WLKMessageStatus, - WLKMessageTranscriptionLine, -) +from aymurai.api.meta.asr.coro import CoroSegment from aymurai.database.meta.audio_transcription import AudioTranscription from aymurai.database.utils import data_to_uuid from aymurai.meta.api_interfaces import ASRDocument, ASRParagraph @@ -22,27 +19,13 @@ def test_should_transcribe_and_persist_document_when_service_returns_paragraphs( audio_bytes = make_wav_bytes() document_id = data_to_uuid(audio_bytes) - fake_status = WLKMessageStatus( - status="active_transcription", - lines=[ - WLKMessageTranscriptionLine( - speaker=1, - text="Hola mundo", - start=timedelta(seconds=0), - end=timedelta(seconds=1), - ) - ], - buffer_transcription="", - buffer_diarization="", - buffer_translation="", - remaining_time_transcription=0.0, - remaining_time_diarization=0.0, - speaker_ids={}, - ) + fake_segments = [ + CoroSegment(start=0.0, end=1.0, text="Hola mundo", speaker="1"), + ] with patch( "aymurai.api.endpoints.routers.asr.transcribe.transcribe_audio_bytes", - new=AsyncMock(return_value=fake_status), + new=AsyncMock(return_value=fake_segments), ): response = client.post( "/asr/transcribe?use_cache=false", From aeeca23ab1701ef7d6704e9ad7e9e9eef3ac5e77 Mon Sep 17 00:00:00 2001 From: jedzill4 Date: Wed, 17 Jun 2026 01:14:26 -0400 Subject: [PATCH 09/22] test(asr): gate integration test on TRANSCRIBE_BASE_URL --- test/integration/test_asr_pipeline.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/integration/test_asr_pipeline.py b/test/integration/test_asr_pipeline.py index 21ff6a34..7857ee88 100644 --- a/test/integration/test_asr_pipeline.py +++ b/test/integration/test_asr_pipeline.py @@ -19,8 +19,8 @@ def test_should_anonymize_audio_document_when_running_real_asr_and_anonymizer_en isolated_diskcache, integration_audio_bytes, ): - if not settings.TRANSCRIBE_WS_URI: - pytest.skip("TRANSCRIBE_WS_URI is required for real ASR integration test") + if not settings.TRANSCRIBE_BASE_URL: + pytest.skip("TRANSCRIBE_BASE_URL is required for real ASR integration test") if shutil.which(settings.LIBREOFFICE_BIN) is None: pytest.skip("LibreOffice binary is required for /anonymizer/anonymize-document") From 6613bfab5df8c8ce7019135325c1ab6988007543 Mon Sep 17 00:00:00 2001 From: jedzill4 Date: Wed, 17 Jun 2026 01:16:30 -0400 Subject: [PATCH 10/22] build(asr): add openai dep (1.x, capped by marker-pdf), drop librosa --- pyproject.toml | 2 +- uv.lock | 147 +------------------------------------------------ 2 files changed, 3 insertions(+), 146 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index d0421eb6..f43d3c00 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,7 +53,6 @@ classifiers = [ dependencies = [ "requests>=2.32.3", "numpy<2.0.0", - "librosa>=0.10.2", "more-itertools>=10.5.0", "odfpy>=1.4.1", "joblib>=1.4.2", @@ -86,6 +85,7 @@ dependencies = [ "pypandoc>=1.15", "python-docx>=1.2.0", "langextract[openai]==1.1.0", + "openai>=1.65.2", "ollama==0.6.1", "tiktoken==0.12.0", "marker-pdf==1.10.1", diff --git a/uv.lock b/uv.lock index b41e1967..0ba99b3b 100644 --- a/uv.lock +++ b/uv.lock @@ -283,15 +283,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3a/2a/7cc015f5b9f5db42b7d48157e23356022889fc354a2813c15934b7cb5c0e/attrs-25.4.0-py3-none-any.whl", hash = "sha256:adcf7e2a1fb3b36ac48d97835bb6d8ade15b8dcce26aba8bf1d14847b57a3373", size = 67615, upload-time = "2025-10-06T13:54:43.17Z" }, ] -[[package]] -name = "audioread" -version = "3.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a1/4a/874ecf9b472f998130c2b5e145dcdb9f6131e84786111489103b66772143/audioread-3.1.0.tar.gz", hash = "sha256:1c4ab2f2972764c896a8ac61ac53e261c8d29f0c6ccd652f84e18f08a4cab190", size = 20082, upload-time = "2025-10-26T19:44:13.484Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7e/16/fbe8e1e185a45042f7cd3a282def5bb8d95bb69ab9e9ef6a5368aa17e426/audioread-3.1.0-py3-none-any.whl", hash = "sha256:b30d1df6c5d3de5dcef0fb0e256f6ea17bdcf5f979408df0297d8a408e2971b4", size = 23143, upload-time = "2025-10-26T19:44:12.016Z" }, -] - [[package]] name = "aymurai" source = { editable = "." } @@ -308,12 +299,12 @@ dependencies = [ { name = "jiwer" }, { name = "joblib" }, { name = "langextract", extra = ["openai"] }, - { name = "librosa" }, { name = "marker-pdf" }, { name = "more-itertools" }, { name = "numpy" }, { name = "odfpy" }, { name = "ollama" }, + { name = "openai" }, { name = "psutil" }, { name = "pydantic" }, { name = "pydantic-settings" }, @@ -374,12 +365,12 @@ requires-dist = [ { name = "jiwer", specifier = "==3.0.5" }, { name = "joblib", specifier = ">=1.4.2" }, { name = "langextract", extras = ["openai"], specifier = "==1.1.0" }, - { name = "librosa", specifier = ">=0.10.2" }, { name = "marker-pdf", specifier = "==1.10.1" }, { name = "more-itertools", specifier = ">=10.5.0" }, { name = "numpy", specifier = "<2.0.0" }, { name = "odfpy", specifier = ">=1.4.1" }, { name = "ollama", specifier = "==0.6.1" }, + { name = "openai", specifier = ">=1.65.2" }, { name = "psutil", specifier = "==6.1.0" }, { name = "pydantic", specifier = ">=2.10.4" }, { name = "pydantic-settings", specifier = ">=2.7.0" }, @@ -2186,42 +2177,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/82/3d/14ce75ef66813643812f3093ab17e46d3a206942ce7376d31ec2d36229e7/lark-1.3.1-py3-none-any.whl", hash = "sha256:c629b661023a014c37da873b4ff58a817398d12635d3bbb2c5a03be7fe5d1e12", size = 113151, upload-time = "2025-10-27T18:25:54.882Z" }, ] -[[package]] -name = "lazy-loader" -version = "0.4" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "packaging" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/6f/6b/c875b30a1ba490860c93da4cabf479e03f584eba06fe5963f6f6644653d8/lazy_loader-0.4.tar.gz", hash = "sha256:47c75182589b91a4e1a85a136c074285a5ad4d9f39c63e0d7fb76391c4574cd1", size = 15431, upload-time = "2024-04-05T13:03:12.261Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/83/60/d497a310bde3f01cb805196ac61b7ad6dc5dcf8dce66634dc34364b20b4f/lazy_loader-0.4-py3-none-any.whl", hash = "sha256:342aa8e14d543a154047afb4ba8ef17f5563baad3fc610d7b15b213b0f119efc", size = 12097, upload-time = "2024-04-05T13:03:10.514Z" }, -] - -[[package]] -name = "librosa" -version = "0.11.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "audioread" }, - { name = "decorator" }, - { name = "joblib" }, - { name = "lazy-loader" }, - { name = "msgpack" }, - { name = "numba" }, - { name = "numpy" }, - { name = "pooch" }, - { name = "scikit-learn" }, - { name = "scipy" }, - { name = "soundfile" }, - { name = "soxr" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/64/36/360b5aafa0238e29758729e9486c6ed92a6f37fa403b7875e06c115cdf4a/librosa-0.11.0.tar.gz", hash = "sha256:f5ed951ca189b375bbe2e33b2abd7e040ceeee302b9bbaeeffdfddb8d0ace908", size = 327001, upload-time = "2025-03-11T15:09:54.884Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b5/ba/c63c5786dfee4c3417094c4b00966e61e4a63efecee22cb7b4c0387dda83/librosa-0.11.0-py3-none-any.whl", hash = "sha256:0b6415c4fd68bff4c29288abe67c6d80b587e0e1e2cfb0aad23e4559504a7fa1", size = 260749, upload-time = "2025-03-11T15:09:52.982Z" }, -] - [[package]] name = "lightning-utilities" version = "0.3.0" @@ -2257,18 +2212,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ea/53/aa31e4d057b3746b3c323ca993003d6cf15ef987e7fe7ceb53681695ae87/litellm-1.80.0-py3-none-any.whl", hash = "sha256:fd0009758f4772257048d74bf79bb64318859adb4ea49a8b66fdbc718cd80b6e", size = 10492975, upload-time = "2025-11-16T00:03:49.182Z" }, ] -[[package]] -name = "llvmlite" -version = "0.46.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/74/cd/08ae687ba099c7e3d21fe2ea536500563ef1943c5105bf6ab4ee3829f68e/llvmlite-0.46.0.tar.gz", hash = "sha256:227c9fd6d09dce2783c18b754b7cd9d9b3b3515210c46acc2d3c5badd9870ceb", size = 193456, upload-time = "2025-12-08T18:15:36.295Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3d/a4/3959e1c61c5ca9db7921e5fd115b344c29b9d57a5dadd87bef97963ca1a5/llvmlite-0.46.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4323177e936d61ae0f73e653e2e614284d97d14d5dd12579adc92b6c2b0597b0", size = 37232766, upload-time = "2025-12-08T18:14:34.765Z" }, - { url = "https://files.pythonhosted.org/packages/c2/a5/a4d916f1015106e1da876028606a8e87fd5d5c840f98c87bc2d5153b6a2f/llvmlite-0.46.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0a2d461cb89537b7c20feb04c46c32e12d5ad4f0896c9dfc0f60336219ff248e", size = 56275176, upload-time = "2025-12-08T18:14:37.944Z" }, - { url = "https://files.pythonhosted.org/packages/79/7f/a7f2028805dac8c1a6fae7bda4e739b7ebbcd45b29e15bf6d21556fcd3d5/llvmlite-0.46.0-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b1f6595a35b7b39c3518b85a28bf18f45e075264e4b2dce3f0c2a4f232b4a910", size = 55128629, upload-time = "2025-12-08T18:14:41.674Z" }, - { url = "https://files.pythonhosted.org/packages/b2/bc/4689e1ba0c073c196b594471eb21be0aa51d9e64b911728aa13cd85ef0ae/llvmlite-0.46.0-cp310-cp310-win_amd64.whl", hash = "sha256:e7a34d4aa6f9a97ee006b504be6d2b8cb7f755b80ab2f344dda1ef992f828559", size = 38138651, upload-time = "2025-12-08T18:14:45.845Z" }, -] - [[package]] name = "lxml" version = "6.0.2" @@ -2580,22 +2523,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/43/e3/7d92a15f894aa0c9c4b49b8ee9ac9850d6e63b03c9c32c0367a13ae62209/mpmath-1.3.0-py3-none-any.whl", hash = "sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c", size = 536198, upload-time = "2023-03-07T16:47:09.197Z" }, ] -[[package]] -name = "msgpack" -version = "1.1.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/4d/f2/bfb55a6236ed8725a96b0aa3acbd0ec17588e6a2c3b62a93eb513ed8783f/msgpack-1.1.2.tar.gz", hash = "sha256:3b60763c1373dd60f398488069bcdc703cd08a711477b5d480eecc9f9626f47e", size = 173581, upload-time = "2025-10-08T09:15:56.596Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f5/a2/3b68a9e769db68668b25c6108444a35f9bd163bb848c0650d516761a59c0/msgpack-1.1.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0051fffef5a37ca2cd16978ae4f0aef92f164df86823871b5162812bebecd8e2", size = 81318, upload-time = "2025-10-08T09:14:38.722Z" }, - { url = "https://files.pythonhosted.org/packages/5b/e1/2b720cc341325c00be44e1ed59e7cfeae2678329fbf5aa68f5bda57fe728/msgpack-1.1.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a605409040f2da88676e9c9e5853b3449ba8011973616189ea5ee55ddbc5bc87", size = 83786, upload-time = "2025-10-08T09:14:40.082Z" }, - { url = "https://files.pythonhosted.org/packages/71/e5/c2241de64bfceac456b140737812a2ab310b10538a7b34a1d393b748e095/msgpack-1.1.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8b696e83c9f1532b4af884045ba7f3aa741a63b2bc22617293a2c6a7c645f251", size = 398240, upload-time = "2025-10-08T09:14:41.151Z" }, - { url = "https://files.pythonhosted.org/packages/b7/09/2a06956383c0fdebaef5aa9246e2356776f12ea6f2a44bd1368abf0e46c4/msgpack-1.1.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:365c0bbe981a27d8932da71af63ef86acc59ed5c01ad929e09a0b88c6294e28a", size = 406070, upload-time = "2025-10-08T09:14:42.821Z" }, - { url = "https://files.pythonhosted.org/packages/0e/74/2957703f0e1ef20637d6aead4fbb314330c26f39aa046b348c7edcf6ca6b/msgpack-1.1.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:41d1a5d875680166d3ac5c38573896453bbbea7092936d2e107214daf43b1d4f", size = 393403, upload-time = "2025-10-08T09:14:44.38Z" }, - { url = "https://files.pythonhosted.org/packages/a5/09/3bfc12aa90f77b37322fc33e7a8a7c29ba7c8edeadfa27664451801b9860/msgpack-1.1.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:354e81bcdebaab427c3df4281187edc765d5d76bfb3a7c125af9da7a27e8458f", size = 398947, upload-time = "2025-10-08T09:14:45.56Z" }, - { url = "https://files.pythonhosted.org/packages/4b/4f/05fcebd3b4977cb3d840f7ef6b77c51f8582086de5e642f3fefee35c86fc/msgpack-1.1.2-cp310-cp310-win32.whl", hash = "sha256:e64c8d2f5e5d5fda7b842f55dec6133260ea8f53c4257d64494c534f306bf7a9", size = 64769, upload-time = "2025-10-08T09:14:47.334Z" }, - { url = "https://files.pythonhosted.org/packages/d0/3e/b4547e3a34210956382eed1c85935fff7e0f9b98be3106b3745d7dec9c5e/msgpack-1.1.2-cp310-cp310-win_amd64.whl", hash = "sha256:db6192777d943bdaaafb6ba66d44bf65aa0e9c5616fa1d2da9bb08828c6b39aa", size = 71293, upload-time = "2025-10-08T09:14:48.665Z" }, -] - [[package]] name = "multidict" version = "6.7.1" @@ -2774,22 +2701,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f9/33/bd5b9137445ea4b680023eb0469b2bb969d61303dedb2aac6560ff3d14a1/notebook_shim-0.2.4-py3-none-any.whl", hash = "sha256:411a5be4e9dc882a074ccbcae671eda64cceb068767e9a3419096986560e1cef", size = 13307, upload-time = "2024-02-14T23:35:16.286Z" }, ] -[[package]] -name = "numba" -version = "0.64.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "llvmlite" }, - { name = "numpy" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/23/c9/a0fb41787d01d621046138da30f6c2100d80857bf34b3390dd68040f27a3/numba-0.64.0.tar.gz", hash = "sha256:95e7300af648baa3308127b1955b52ce6d11889d16e8cfe637b4f85d2fca52b1", size = 2765679, upload-time = "2026-02-18T18:41:20.974Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/4c/5e/604fed821cd7e3426bb3bc99a7ed6ac0bcb489f4cd93052256437d082f95/numba-0.64.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:cc09b79440952e3098eeebea4bf6e8d2355fb7f12734fcd9fc5039f0dca90727", size = 2683250, upload-time = "2026-02-18T18:40:45.829Z" }, - { url = "https://files.pythonhosted.org/packages/4f/9f/9275a723d050b5f1a9b1c7fb7dbfce324fef301a8e50c5f88338569db06c/numba-0.64.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1afe3a80b8c2f376b211fb7a49e536ef9eafc92436afc95a2f41ea5392f8cc65", size = 3742168, upload-time = "2026-02-18T18:40:48.066Z" }, - { url = "https://files.pythonhosted.org/packages/e2/d1/97ca7dddaa36b16f4c46319bdb6b4913ba15d0245317d0d8ccde7b2d7d92/numba-0.64.0-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:23804194b93b8cd416c6444b5fbc4956082a45fed2d25436ef49c594666e7f7e", size = 3449103, upload-time = "2026-02-18T18:40:49.905Z" }, - { url = "https://files.pythonhosted.org/packages/52/0a/b9e137ad78415373e3353564500e8bf29dbce3c0d73633bb384d4e5d7537/numba-0.64.0-cp310-cp310-win_amd64.whl", hash = "sha256:e2a9fe998bb2cf848960b34db02c2c3b5e02cf82c07a26d9eef3494069740278", size = 2749950, upload-time = "2026-02-18T18:40:51.536Z" }, -] - [[package]] name = "numpy" version = "1.26.4" @@ -3258,20 +3169,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, ] -[[package]] -name = "pooch" -version = "1.9.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "packaging" }, - { name = "platformdirs" }, - { name = "requests" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/83/43/85ef45e8b36c6a48546af7b266592dc32d7f67837a6514d111bced6d7d75/pooch-1.9.0.tar.gz", hash = "sha256:de46729579b9857ffd3e741987a2f6d5e0e03219892c167c6578c0091fb511ed", size = 61788, upload-time = "2026-01-30T19:15:09.649Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2a/2d/d4bf65e47cea8ff2c794a600c4fd1273a7902f268757c531e0ee9f18aa58/pooch-1.9.0-py3-none-any.whl", hash = "sha256:f265597baa9f760d25ceb29d0beb8186c243d6607b0f60b83ecf14078dbc703b", size = 67175, upload-time = "2026-01-30T19:15:08.36Z" }, -] - [[package]] name = "pptree" version = "3.1" @@ -4277,25 +4174,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0", size = 29575, upload-time = "2021-05-16T22:03:41.177Z" }, ] -[[package]] -name = "soundfile" -version = "0.13.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cffi" }, - { name = "numpy" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/e1/41/9b873a8c055582859b239be17902a85339bec6a30ad162f98c9b0288a2cc/soundfile-0.13.1.tar.gz", hash = "sha256:b2c68dab1e30297317080a5b43df57e302584c49e2942defdde0acccc53f0e5b", size = 46156, upload-time = "2025-01-25T09:17:04.831Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/64/28/e2a36573ccbcf3d57c00626a21fe51989380636e821b341d36ccca0c1c3a/soundfile-0.13.1-py2.py3-none-any.whl", hash = "sha256:a23c717560da2cf4c7b5ae1142514e0fd82d6bbd9dfc93a50423447142f2c445", size = 25751, upload-time = "2025-01-25T09:16:44.235Z" }, - { url = "https://files.pythonhosted.org/packages/ea/ab/73e97a5b3cc46bba7ff8650a1504348fa1863a6f9d57d7001c6b67c5f20e/soundfile-0.13.1-py2.py3-none-macosx_10_9_x86_64.whl", hash = "sha256:82dc664d19831933fe59adad199bf3945ad06d84bc111a5b4c0d3089a5b9ec33", size = 1142250, upload-time = "2025-01-25T09:16:47.583Z" }, - { url = "https://files.pythonhosted.org/packages/a0/e5/58fd1a8d7b26fc113af244f966ee3aecf03cb9293cb935daaddc1e455e18/soundfile-0.13.1-py2.py3-none-macosx_11_0_arm64.whl", hash = "sha256:743f12c12c4054921e15736c6be09ac26b3b3d603aef6fd69f9dde68748f2593", size = 1101406, upload-time = "2025-01-25T09:16:49.662Z" }, - { url = "https://files.pythonhosted.org/packages/58/ae/c0e4a53d77cf6e9a04179535766b3321b0b9ced5f70522e4caf9329f0046/soundfile-0.13.1-py2.py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:9c9e855f5a4d06ce4213f31918653ab7de0c5a8d8107cd2427e44b42df547deb", size = 1235729, upload-time = "2025-01-25T09:16:53.018Z" }, - { url = "https://files.pythonhosted.org/packages/57/5e/70bdd9579b35003a489fc850b5047beeda26328053ebadc1fb60f320f7db/soundfile-0.13.1-py2.py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:03267c4e493315294834a0870f31dbb3b28a95561b80b134f0bd3cf2d5f0e618", size = 1313646, upload-time = "2025-01-25T09:16:54.872Z" }, - { url = "https://files.pythonhosted.org/packages/fe/df/8c11dc4dfceda14e3003bb81a0d0edcaaf0796dd7b4f826ea3e532146bba/soundfile-0.13.1-py2.py3-none-win32.whl", hash = "sha256:c734564fab7c5ddf8e9be5bf70bab68042cd17e9c214c06e365e20d64f9a69d5", size = 899881, upload-time = "2025-01-25T09:16:56.663Z" }, - { url = "https://files.pythonhosted.org/packages/14/e9/6b761de83277f2f02ded7e7ea6f07828ec78e4b229b80e4ca55dd205b9dc/soundfile-0.13.1-py2.py3-none-win_amd64.whl", hash = "sha256:1e70a05a0626524a69e9f0f4dd2ec174b4e9567f4d8b6c11d38b5c289be36ee9", size = 1019162, upload-time = "2025-01-25T09:16:59.573Z" }, -] - [[package]] name = "soupsieve" version = "2.8.3" @@ -4305,27 +4183,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/46/2c/1462b1d0a634697ae9e55b3cecdcb64788e8b7d63f54d923fcd0bb140aed/soupsieve-2.8.3-py3-none-any.whl", hash = "sha256:ed64f2ba4eebeab06cc4962affce381647455978ffc1e36bb79a545b91f45a95", size = 37016, upload-time = "2026-01-20T04:27:01.012Z" }, ] -[[package]] -name = "soxr" -version = "1.0.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/42/7e/f4b461944662ad75036df65277d6130f9411002bfb79e9df7dff40a31db9/soxr-1.0.0.tar.gz", hash = "sha256:e07ee6c1d659bc6957034f4800c60cb8b98de798823e34d2a2bba1caa85a4509", size = 171415, upload-time = "2025-09-07T13:22:21.317Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1e/a7/11c36d71595b52fe84a220040ace679035953acf06b83bf2c7117c565d2c/soxr-1.0.0-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:b876a3156f67c76aef0cff1084eaf4088d9ca584bb569cb993f89a52ec5f399f", size = 206459, upload-time = "2025-09-07T13:21:46.904Z" }, - { url = "https://files.pythonhosted.org/packages/43/5e/8962f2aeea7777d2a6e65a24a2b83c6aea1a28badeda027fd328f7f03bb7/soxr-1.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4d3b957a7b0cc19ae6aa45d40b2181474e53a8dd00efd7bce6bcf4e60e020892", size = 164808, upload-time = "2025-09-07T13:21:48.83Z" }, - { url = "https://files.pythonhosted.org/packages/fc/91/00384166f110a3888ea8efd44523ba7168dd2dc39e3e43c931cc2d069fa9/soxr-1.0.0-cp310-cp310-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b89685faedebc45af71f08f9957b61cc6143bc94ba43fe38e97067f81e272969", size = 208586, upload-time = "2025-09-07T13:21:50.341Z" }, - { url = "https://files.pythonhosted.org/packages/75/34/e18f1003e242aabed44ed8902534814d3e64209e4d1d874f5b9b67d73cde/soxr-1.0.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d255741b2f0084fd02d4a2ddd77cd495be9e7e7b6f9dba1c9494f86afefac65b", size = 242310, upload-time = "2025-09-07T13:21:51.56Z" }, - { url = "https://files.pythonhosted.org/packages/61/9c/a1c5ed106b40cc1e2e12cd58831b7f1b61c5fbdb8eceeca4b3a0b0dbef6c/soxr-1.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:158a4a9055958c4b95ef91dbbe280cabb00946b5423b25a9b0ce31bd9e0a271e", size = 173561, upload-time = "2025-09-07T13:21:53.03Z" }, - { url = "https://files.pythonhosted.org/packages/c5/c7/f92b81f1a151c13afb114f57799b86da9330bec844ea5a0d3fe6a8732678/soxr-1.0.0-cp312-abi3-macosx_10_14_x86_64.whl", hash = "sha256:abecf4e39017f3fadb5e051637c272ae5778d838e5c3926a35db36a53e3a607f", size = 205508, upload-time = "2025-09-07T13:22:01.252Z" }, - { url = "https://files.pythonhosted.org/packages/ff/1d/c945fea9d83ea1f2be9d116b3674dbaef26ed090374a77c394b31e3b083b/soxr-1.0.0-cp312-abi3-macosx_11_0_arm64.whl", hash = "sha256:e973d487ee46aa8023ca00a139db6e09af053a37a032fe22f9ff0cc2e19c94b4", size = 163568, upload-time = "2025-09-07T13:22:03.558Z" }, - { url = "https://files.pythonhosted.org/packages/b5/80/10640970998a1d2199bef6c4d92205f36968cddaf3e4d0e9fe35ddd405bd/soxr-1.0.0-cp312-abi3-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e8ce273cca101aff3d8c387db5a5a41001ba76ef1837883438d3c652507a9ccc", size = 204707, upload-time = "2025-09-07T13:22:05.125Z" }, - { url = "https://files.pythonhosted.org/packages/b1/87/2726603c13c2126cb8ded9e57381b7377f4f0df6ba4408e1af5ddbfdc3dd/soxr-1.0.0-cp312-abi3-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e8f2a69686f2856d37823bbb7b78c3d44904f311fe70ba49b893af11d6b6047b", size = 238032, upload-time = "2025-09-07T13:22:06.428Z" }, - { url = "https://files.pythonhosted.org/packages/ce/04/530252227f4d0721a5524a936336485dfb429bb206a66baf8e470384f4a2/soxr-1.0.0-cp312-abi3-win_amd64.whl", hash = "sha256:2a3b77b115ae7c478eecdbd060ed4f61beda542dfb70639177ac263aceda42a2", size = 172070, upload-time = "2025-09-07T13:22:07.62Z" }, -] - [[package]] name = "sqlalchemy" version = "2.0.48" From 8570bb7121c6fb955c84f61188541d549ce06b93 Mon Sep 17 00:00:00 2001 From: jedzill4 Date: Wed, 17 Jun 2026 01:18:27 -0400 Subject: [PATCH 11/22] chore(asr): drop whisperlivekit compose service (coro runs via uv tool) ASR now runs as a host process via scripts/run-coro.sh; .env points at TRANSCRIBE_BASE_URL. Includes pre-existing compose YAML formatting normalization already present in the working tree on this branch. --- docker-compose.yml | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index eb8d8fe0..92a247cc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -48,7 +48,10 @@ services: [ "/bin/sh", "-c", - "pip install --no-cache-dir psycopg2-binary boto3 && mlflow server --host 0.0.0.0 --port 5000 --backend-store-uri postgresql://mlflow:mlflow@postgres:5432/mlflow --default-artifact-root ${MLFLOW_ARTIFACT_ROOT}", + "pip install --no-cache-dir psycopg2-binary boto3 && mlflow + server --host 0.0.0.0 --port 5000 --backend-store-uri + postgresql://mlflow:mlflow@postgres:5432/mlflow + --default-artifact-root ${MLFLOW_ARTIFACT_ROOT}", ] depends_on: postgres: @@ -61,7 +64,8 @@ services: "CMD", "python", "-c", - "import urllib.request; urllib.request.urlopen('http://localhost:5000')", + "import urllib.request; + urllib.request.urlopen('http://localhost:5000')", ] interval: 10s timeout: 5s @@ -79,7 +83,7 @@ services: volumes: - postgres-data:/var/lib/postgresql/data healthcheck: - test: ["CMD-SHELL", "pg_isready -U mlflow"] + test: [ "CMD-SHELL", "pg_isready -U mlflow" ] interval: 10s timeout: 5s retries: 6 @@ -93,7 +97,7 @@ services: environment: MINIO_ROOT_USER: ${AWS_ACCESS_KEY_ID} MINIO_ROOT_PASSWORD: ${AWS_SECRET_ACCESS_KEY} - command: ["server", "/data", "--console-address", ":9001"] + command: [ "server", "/data", "--console-address", ":9001" ] volumes: - minio-data:/data # Healthcheck removed; minio image lacks wget and minio-mc already waits for readiness. @@ -108,7 +112,10 @@ services: [ "/bin/sh", "-c", - "until (/usr/bin/mc alias set local http://minio:9000 ${AWS_ACCESS_KEY_ID} ${AWS_SECRET_ACCESS_KEY}) do sleep 1; done; /usr/bin/mc mb -p local/mlflow; /usr/bin/mc anonymous set none local/mlflow; exit 0;", + "until (/usr/bin/mc alias set local http://minio:9000 + ${AWS_ACCESS_KEY_ID} ${AWS_SECRET_ACCESS_KEY}) do sleep 1; + done; /usr/bin/mc mb -p local/mlflow; /usr/bin/mc anonymous + set none local/mlflow; exit 0;", ] # CPU-friendly API From 1e44ae8a698c76f7a3f15dbc76ba1242a027f3ed Mon Sep 17 00:00:00 2001 From: jedzill4 Date: Wed, 17 Jun 2026 01:19:13 -0400 Subject: [PATCH 12/22] docs(asr): add run-coro.sh (cpu/gpu) and README instructions --- README.md | 22 ++++++++++++++++++++++ scripts/run-coro.sh | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+) create mode 100755 scripts/run-coro.sh diff --git a/README.md b/README.md index d140e5ad..87c2b3a9 100644 --- a/README.md +++ b/README.md @@ -61,6 +61,28 @@ docker load -i aymurai-api.tar For more information on Docker deployment, refer to the [Docker documentation](https://docs.docker.com/). If you need further assistance, feel free to contact us at [aymurai@datagenero.org](mailto:aymurai@datagenero.org). +## Running the ASR server (coro) +Audio transcription is handled by [coro](https://github.com/collectiveai-team/coro), an OpenAI-compatible ASR + speaker-diarization server. It runs as an isolated host process via `uv tool` (no dependency conflicts with the API), and the API talks to it over HTTP/SSE. + +Prerequisites (both modes): `ffmpeg` on the host, [`uv`](https://docs.astral.sh/uv/) installed, and a real-disk `CORO_TRANSCRIPT_SPILL_DIR` (not a tmpfs path) so host RAM stays flat on long audio. + +```bash +# GPU (NVIDIA): parakeet fp32 + NeMo diarization +./scripts/run-coro.sh gpu + +# CPU: parakeet int8 +./scripts/run-coro.sh cpu +``` + +Then point the API at it via the `TRANSCRIBE_BASE_URL` environment variable (see `.env`): + +``` +TRANSCRIBE_BASE_URL=http://localhost:8000/v1 +``` + +The API connects over HTTP/SSE and is hardware-agnostic — coro's device is independent of the API container's `TORCH_DEVICE`. + + ## Pipeline AymurAI’s backend utilizes a structured data processing pipeline to handle anonymized legal rulings and extract relevant information. This data is processed and made accessible via the API. For more details, please refer to the [pipeline documentation](docs/pipeline/README.md). diff --git a/scripts/run-coro.sh b/scripts/run-coro.sh new file mode 100755 index 00000000..db698529 --- /dev/null +++ b/scripts/run-coro.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +set -euo pipefail + +MODE="${1:-gpu}" +PORT="${CORO_PORT:-8000}" +SPILL_DIR="${CORO_TRANSCRIPT_SPILL_DIR:-./resources/cache/coro-spill}" +mkdir -p "$SPILL_DIR" + +export CORO_BACKEND_ASR=onnx-asr +export CORO_MODEL_ASR=nemo-parakeet-tdt-0.6b-v3 +export CORO_BACKEND_DIARIZATION=nemo +export CORO_PIPELINE=streaming +export CORO_TRANSCRIPT_SPILL_DIR="$SPILL_DIR" + +case "$MODE" in + gpu) + export CORO_ASR_DEVICE=cuda + export CORO_DIARIZATION_DEVICE=cuda + EXTRA="cuda" + ;; + cpu) + export CORO_ASR_DEVICE=cpu + export CORO_ASR_QUANTIZATION=int8 + export CORO_DIARIZATION_DEVICE=cpu + EXTRA="cpu" + ;; + *) + echo "usage: $0 [cpu|gpu]" >&2 + exit 2 + ;; +esac + +exec uvx --from "coro[$EXTRA] @ git+https://github.com/collectiveai-team/coro" \ + coro --port "$PORT" From 18581493838e73ed38d27478466ff99d1b656117 Mon Sep 17 00:00:00 2001 From: jedzill4 Date: Wed, 17 Jun 2026 01:27:11 -0400 Subject: [PATCH 13/22] test(asr): cover openai connection error -> RuntimeError mapping --- test/audio/test_asr_client.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/test/audio/test_asr_client.py b/test/audio/test_asr_client.py index 3f2145f3..71ea494f 100644 --- a/test/audio/test_asr_client.py +++ b/test/audio/test_asr_client.py @@ -2,7 +2,9 @@ import json from unittest.mock import AsyncMock, MagicMock, patch +import httpx import pytest +from openai import APIConnectionError from aymurai.api.meta.asr.coro import CoroSegment from aymurai.audio.asr_client import transcribe_audio_bytes @@ -71,3 +73,14 @@ def test_should_raise_when_base_url_not_configured(monkeypatch): monkeypatch.setattr(settings, "TRANSCRIBE_BASE_URL", None) with pytest.raises(RuntimeError, match="not configured"): asyncio.run(transcribe_audio_bytes(b"audio", "sample.wav", "audio/wav")) + + +def test_should_map_openai_connection_error_to_runtime_error(monkeypatch): + monkeypatch.setattr(settings, "TRANSCRIBE_BASE_URL", "http://coro.local/v1") + request = httpx.Request("POST", "http://coro.local/v1/audio/transcriptions") + create = AsyncMock(side_effect=APIConnectionError(request=request)) + client = MagicMock() + client.audio.transcriptions.create = create + with patch("aymurai.audio.asr_client.AsyncOpenAI", return_value=client): + with pytest.raises(RuntimeError, match="Transcription service error"): + asyncio.run(transcribe_audio_bytes(b"audio", "sample.wav", "audio/wav")) From b147c494f1a8a3a73f8446de6efd58c6e2894131 Mon Sep 17 00:00:00 2001 From: jedzill4 Date: Wed, 17 Jun 2026 01:39:24 -0400 Subject: [PATCH 14/22] test(asr): add coro SSE smoke test; pin uvx --python 3.12 in run-coro.sh --- README.md | 8 ++- scripts/run-coro.sh | 3 +- scripts/smoke_coro.py | 155 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 164 insertions(+), 2 deletions(-) create mode 100644 scripts/smoke_coro.py diff --git a/README.md b/README.md index 87c2b3a9..45a34431 100644 --- a/README.md +++ b/README.md @@ -80,7 +80,13 @@ Then point the API at it via the `TRANSCRIBE_BASE_URL` environment variable (see TRANSCRIBE_BASE_URL=http://localhost:8000/v1 ``` -The API connects over HTTP/SSE and is hardware-agnostic — coro's device is independent of the API container's `TORCH_DEVICE`. +The API connects over HTTP/SSE and is hardware-agnostic — coro's device is independent of the API container's `TORCH_DEVICE`. (`run-coro.sh` pins `uvx --python 3.12`, since NeMo/`kaldialign` lack wheels on newer Pythons.) + +To verify the whole path end-to-end (start coro, transcribe a sample over SSE via both the OpenAI SDK and aymurai's client): + +```bash +rtk uv run python scripts/smoke_coro.py cpu # or: gpu +``` ## Pipeline diff --git a/scripts/run-coro.sh b/scripts/run-coro.sh index db698529..5e4ac2ae 100755 --- a/scripts/run-coro.sh +++ b/scripts/run-coro.sh @@ -30,5 +30,6 @@ case "$MODE" in ;; esac -exec uvx --from "coro[$EXTRA] @ git+https://github.com/collectiveai-team/coro" \ +exec uvx --python 3.12 \ + --from "coro[$EXTRA] @ git+https://github.com/collectiveai-team/coro" \ coro --port "$PORT" diff --git a/scripts/smoke_coro.py b/scripts/smoke_coro.py new file mode 100644 index 00000000..1aecc6c2 --- /dev/null +++ b/scripts/smoke_coro.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python +"""Smoke test for the coro ASR integration. + +Starts the coro server (via ``scripts/run-coro.sh``), waits for it to become +healthy, then transcribes a sample audio file over the OpenAI-compatible SSE +endpoint two ways: + +1. directly with the ``openai`` SDK (``stream=True``) — shows the raw SSE events; +2. through aymurai's own client ``transcribe_audio_bytes`` — proves the wiring. + +Usage: + rtk uv run python scripts/smoke_coro.py [cpu|gpu] + +Env overrides: + CORO_PORT (default 8000) + SMOKE_AUDIO path to a local audio file (default: download sample) + SMOKE_AUDIO_URL sample to download if SMOKE_AUDIO is absent + SMOKE_HEALTH_TIMEOUT seconds to wait for /health (default 1200) +""" + +from __future__ import annotations + +import asyncio +import json +import os +import signal +import subprocess +import sys +import time +import urllib.request +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent +CORO_LOG = Path("/tmp/coro-smoke.log") +MODE = sys.argv[1] if len(sys.argv) > 1 else "cpu" +PORT = os.environ.get("CORO_PORT", "8000") +BASE_URL = f"http://localhost:{PORT}/v1" +HEALTH_URL = f"http://localhost:{PORT}/health" +HEALTH_TIMEOUT = int(os.environ.get("SMOKE_HEALTH_TIMEOUT", "1200")) +AUDIO_PATH = Path( + os.environ.get("SMOKE_AUDIO", str(REPO_ROOT / "resources/cache/smoke/sample.wav")) +) +AUDIO_URL = os.environ.get( + "SMOKE_AUDIO_URL", + "https://google-research.github.io/lingvo-lab/translatotron/fisher_src/775.wav", +) + + +def log(msg: str) -> None: + print(f"[smoke] {msg}", flush=True) + + +def ensure_audio() -> Path: + if AUDIO_PATH.exists(): + log(f"using existing audio: {AUDIO_PATH}") + return AUDIO_PATH + AUDIO_PATH.parent.mkdir(parents=True, exist_ok=True) + log(f"downloading sample audio: {AUDIO_URL}") + urllib.request.urlretrieve(AUDIO_URL, AUDIO_PATH) # noqa: S310 + log(f"saved sample audio: {AUDIO_PATH} ({AUDIO_PATH.stat().st_size} bytes)") + return AUDIO_PATH + + +def start_coro() -> subprocess.Popen: + log(f"starting coro ({MODE}) on port {PORT} ; logs -> {CORO_LOG}") + logfh = CORO_LOG.open("w") + return subprocess.Popen( + ["bash", str(REPO_ROOT / "scripts/run-coro.sh"), MODE], + cwd=REPO_ROOT, + stdout=logfh, + stderr=subprocess.STDOUT, + text=True, + start_new_session=True, + ) + + +def wait_healthy(proc: subprocess.Popen) -> None: + log(f"waiting for {HEALTH_URL} (timeout {HEALTH_TIMEOUT}s) ...") + deadline = time.time() + HEALTH_TIMEOUT + while time.time() < deadline: + if proc.poll() is not None: + raise RuntimeError("coro process exited before becoming healthy") + try: + with urllib.request.urlopen(HEALTH_URL, timeout=5) as resp: # noqa: S310 + if resp.status == 200: + log("coro is healthy") + return + except Exception: + pass + time.sleep(3) + raise TimeoutError("coro did not become healthy in time") + + +def transcribe_with_sdk(audio: Path) -> None: + from openai import OpenAI + + log("=== SSE transcription via openai SDK (stream=True) ===") + client = OpenAI(base_url=BASE_URL, api_key="not-needed") + deltas = 0 + with audio.open("rb") as fh: + stream = client.audio.transcriptions.create( + file=fh, model="whisper-1", language="es", stream=True + ) + for event in stream: + if event.type == "transcript.text.delta": + deltas += 1 + elif event.type == "transcript.text.done": + payload = json.loads(event.text) + segments = payload.get("segments", []) + log(f"received {deltas} delta events, {len(segments)} segments") + for seg in segments: + log( + f" [{seg['start']:.2f}-{seg['end']:.2f}] " + f"speaker {seg['speaker']}: {seg['text']}" + ) + + +def transcribe_with_aymurai_client(audio: Path) -> None: + os.environ["TRANSCRIBE_BASE_URL"] = BASE_URL + from aymurai.audio.asr_client import transcribe_audio_bytes + + log("=== transcription via aymurai transcribe_audio_bytes ===") + payload = audio.read_bytes() + segments = asyncio.run(transcribe_audio_bytes(payload, audio.name, "audio/wav")) + log(f"aymurai client returned {len(segments)} CoroSegment(s)") + for seg in segments: + log(f" speaker {seg.speaker} [{seg.start:.2f}-{seg.end:.2f}]: {seg.text}") + + +def main() -> int: + audio = ensure_audio() + proc = start_coro() + try: + wait_healthy(proc) + transcribe_with_sdk(audio) + transcribe_with_aymurai_client(audio) + log("SMOKE TEST PASSED") + return 0 + except Exception as exc: # noqa: BLE001 + log(f"SMOKE TEST FAILED: {exc}") + if CORO_LOG.exists(): + log(f"--- {CORO_LOG} tail ---") + print("\n".join(CORO_LOG.read_text().splitlines()[-40:]), flush=True) + return 1 + finally: + if proc.poll() is None: + os.killpg(os.getpgid(proc.pid), signal.SIGTERM) + try: + proc.wait(timeout=10) + except Exception: + os.killpg(os.getpgid(proc.pid), signal.SIGKILL) + + +if __name__ == "__main__": + raise SystemExit(main()) From 3302a59ad634cceb967945612f2d4d58893c0962 Mon Sep 17 00:00:00 2001 From: jansaldo Date: Fri, 19 Jun 2026 17:20:18 +0000 Subject: [PATCH 15/22] feat(asr): add Argentine Spanish ASR evaluation notebook with transcription and metrics --- .../01-es-ar-speech-evaluation.ipynb | 619 ++++++++++++++++++ 1 file changed, 619 insertions(+) create mode 100644 notebooks/experiments/speech-to-text/01-es-ar-speech-evaluation.ipynb diff --git a/notebooks/experiments/speech-to-text/01-es-ar-speech-evaluation.ipynb b/notebooks/experiments/speech-to-text/01-es-ar-speech-evaluation.ipynb new file mode 100644 index 00000000..2b867f8b --- /dev/null +++ b/notebooks/experiments/speech-to-text/01-es-ar-speech-evaluation.ipynb @@ -0,0 +1,619 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "7fb27b941602401d91542211134fc71a", + "metadata": {}, + "source": [ + "# Argentine Spanish ASR full evaluation\n", + "Resumable, parallel transcription plus raw and normalized WER/CER, breakdowns, and audio inspection. Set `SAMPLE_LIMIT = 20` for a smoke run.\n", + "\n", + "## Dataset\n", + "\n", + "This evaluation uses the [Crowdsourced High-Quality Argentine Spanish Speech Dataset](https://www.openslr.org/61/) from OpenSLR (SLR61). It contains nearly 6,000 short audio recordings from multiple speakers, paired with reference transcriptions in Argentine Spanish. The clips average approximately five seconds and include male, female, and weather-related subsets.\n", + "\n", + "The dataset and its accompanying resources can be downloaded from the [OpenSLR SLR61 page](https://www.openslr.org/61/)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "acae54e37e7d407bbb7b55eff062a284", + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "import os\n", + "import threading\n", + "import time\n", + "import unicodedata\n", + "from concurrent.futures import ThreadPoolExecutor, as_completed\n", + "from pathlib import Path\n", + "\n", + "import jiwer\n", + "import numpy as np\n", + "import pandas as pd\n", + "import requests\n", + "from IPython.display import Audio, display\n", + "from tqdm.auto import tqdm" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "90851e19", + "metadata": {}, + "outputs": [], + "source": [ + "DATA_DIR = Path(\"/resources/data/asr/es_ar_speech\")\n", + "API_BASE_URL = os.getenv(\"API_BASE_URL\", \"http://localhost:8999\")\n", + "API_TRANSCRIBE_ENDPOINT = f\"{API_BASE_URL}/asr/transcribe\"\n", + "MAX_WORKERS = 4\n", + "TIMEOUT = (10, 180)\n", + "MAX_RETRIES = 3\n", + "SAMPLE_LIMIT = None\n", + "CHECKPOINT = Path(\"results/es_ar_asr_transcriptions.csv\")\n", + "RESULT_COLUMNS = [\n", + " \"file\",\n", + " \"hypothesis\",\n", + " \"response_json\",\n", + " \"status\",\n", + " \"error\",\n", + " \"attempts\",\n", + " \"elapsed_seconds\",\n", + "]" + ] + }, + { + "cell_type": "markdown", + "id": "9a63283cbaf04dbcab1f6479b197f3a8", + "metadata": {}, + "source": [ + "## Load and validate annotations" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8dd0d8092fe74a7c96281538738b07e2", + "metadata": {}, + "outputs": [], + "source": [ + "parts = []\n", + "for path in sorted(DATA_DIR.glob(\"*.tsv\")):\n", + " part = pd.read_csv(path, sep=\"\\t\", header=None, names=[\"file\", \"transcription\"])\n", + " part[\"source_tsv\"] = path.name\n", + " parts.append(part)\n", + "df = pd.concat(parts, ignore_index=True)\n", + "df.info()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b37b31db", + "metadata": {}, + "outputs": [], + "source": [ + "df.sample(5)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "506b6dfc", + "metadata": {}, + "outputs": [], + "source": [ + "df.loc[df[\"file\"].duplicated(keep=False)].sort_values(\"file\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a386674f", + "metadata": {}, + "outputs": [], + "source": [ + "df.loc[df.duplicated(subset=[\"file\", \"transcription\"], keep=False)].sort_values(\"file\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a601b74e", + "metadata": {}, + "outputs": [], + "source": [ + "# Remove duplicates with same file and transcription, keeping the first occurrence\n", + "df.drop_duplicates(subset=[\"file\", \"transcription\"], inplace=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1709fb5f", + "metadata": {}, + "outputs": [], + "source": [ + "df.info()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4cc86bed", + "metadata": {}, + "outputs": [], + "source": [ + "# Verify that each file has a unique annotation and a corresponding WAV file\n", + "wavs = {p.stem: p for p in sorted(DATA_DIR.glob(\"*.wav\"))}\n", + "assert df[[\"file\", \"transcription\"]].notna().all().all()\n", + "assert df[\"file\"].is_unique and set(df[\"file\"]) == set(\n", + " wavs\n", + "), \"Annotations and WAV files differ\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2c871375", + "metadata": {}, + "outputs": [], + "source": [ + "# Add audio paths and file prefixes for later use\n", + "df[\"audio_path\"] = df[\"file\"].map(wavs)\n", + "df[\"file_prefix\"] = df[\"file\"].str.extract(r\"^([^_]+)\", expand=False)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f20ce36e", + "metadata": {}, + "outputs": [], + "source": [ + "# Summarize the number of samples per source TSV file\n", + "df.groupby(\"source_tsv\").size().rename(\"samples\").to_frame()" + ] + }, + { + "cell_type": "markdown", + "id": "72eea5119410473aa328ad9291626812", + "metadata": {}, + "source": [ + "## Parsing, normalization, and synthetic checks\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8edb47106e1a46a883d545849b8ab81b", + "metadata": {}, + "outputs": [], + "source": [ + "def parse_response(payload: dict) -> str:\n", + " \"\"\"\n", + " Parse the ASR API response to extract the transcribed text.\n", + "\n", + " Args:\n", + " payload (dict): The JSON response from the ASR API,\n", + " expected to contain a \"document\" key with a list of segments.\n", + "\n", + " Raises:\n", + " ValueError: If the response does not contain a valid document list or if any segment is not a dictionary.\n", + "\n", + " Returns:\n", + " str: The concatenated transcribed text from the document segments, with leading and trailing whitespace removed.\n", + " \"\"\"\n", + " if not isinstance(payload, dict) or not isinstance(payload.get(\"document\"), list):\n", + " raise ValueError(\"Response must contain a document list\")\n", + "\n", + " if any(not isinstance(x, dict) for x in payload[\"document\"]):\n", + " raise ValueError(\"Document segments must be objects\")\n", + "\n", + " return \" \".join(\n", + " x.get(\"text\", \"\").strip()\n", + " for x in payload[\"document\"]\n", + " if x.get(\"text\", \"\").strip()\n", + " )\n", + "\n", + "\n", + "def normalize(text: str) -> str:\n", + " \"\"\"\n", + " Normalize the input text by removing accents, converting to lowercase,\n", + " and replacing non-alphanumeric characters with spaces.\n", + "\n", + " Args:\n", + " text (str): The input text to normalize.\n", + "\n", + " Returns:\n", + " str: The normalized text.\n", + " \"\"\"\n", + " text = unicodedata.normalize(\"NFKD\", str(text)).lower()\n", + " text = \"\".join(c for c in text if not unicodedata.combining(c))\n", + " return \" \".join(\n", + " \"\".join(c if c.isalnum() or c.isspace() else \" \" for c in text).split()\n", + " )\n", + "\n", + "\n", + "assert (\n", + " parse_response({\"document\": [{\"text\": \" Hola \"}, {\"text\": \"mundo\"}]})\n", + " == \"Hola mundo\"\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "10185d26023b46108eb7d9f57d49d2b3", + "metadata": {}, + "source": [ + "## Parallel transcription with retries and atomic checkpoints\n", + "Successful rows are skipped when rerun; failed rows are attempted again.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8763a12b2bbd4a93a75aff182afb95dc", + "metadata": {}, + "outputs": [], + "source": [ + "_local = threading.local()\n", + "\n", + "\n", + "def session():\n", + " if not hasattr(_local, \"session\"):\n", + " _local.session = requests.Session()\n", + " return _local.session\n", + "\n", + "\n", + "def transcribe_one(fid: str, path: str) -> dict:\n", + " \"\"\"\n", + " Transcribe a single audio file using the ASR API, with retry logic and error handling.\n", + "\n", + " Args:\n", + " fid (str): The file identifier.\n", + " path (str): The path to the audio file.\n", + "\n", + " Returns:\n", + " dict: A dictionary containing the transcription results and metadata.\n", + " \"\"\"\n", + " started = time.monotonic()\n", + " error = \"\"\n", + "\n", + " for attempt in range(1, MAX_RETRIES + 2):\n", + " try:\n", + " with path.open(\"rb\") as stream:\n", + " response = session().post(\n", + " API_TRANSCRIBE_ENDPOINT,\n", + " files={\"file\": (path.name, stream, \"audio/wav\")},\n", + " data={\"response_format\": \"diarized_json\"},\n", + " timeout=TIMEOUT,\n", + " )\n", + " response.raise_for_status()\n", + " payload = response.json()\n", + " return dict(\n", + " file=fid,\n", + " hypothesis=parse_response(payload),\n", + " response_json=json.dumps(payload, ensure_ascii=False),\n", + " status=\"success\",\n", + " error=\"\",\n", + " attempts=attempt,\n", + " elapsed_seconds=time.monotonic() - started,\n", + " )\n", + " except (requests.RequestException, ValueError) as exc:\n", + " error = f\"{type(exc).__name__}: {exc}\"\n", + " if isinstance(exc, ValueError) or attempt > MAX_RETRIES:\n", + " break\n", + " time.sleep(2 ** (attempt - 1))\n", + "\n", + " return dict(\n", + " file=fid,\n", + " hypothesis=\"\",\n", + " response_json=\"\",\n", + " status=\"error\",\n", + " error=error,\n", + " attempts=attempt,\n", + " elapsed_seconds=time.monotonic() - started,\n", + " )\n", + "\n", + "\n", + "def load_checkpoint() -> pd.DataFrame:\n", + " \"\"\"\n", + " Load the transcription results from a checkpoint CSV file if it exists,\n", + "\n", + " Returns:\n", + " pd.DataFrame: A DataFrame containing the transcription results,\n", + " or an empty DataFrame with predefined columns if the checkpoint file does not exist.\n", + " \"\"\"\n", + " return (\n", + " pd.read_csv(CHECKPOINT, keep_default_na=False)\n", + " if CHECKPOINT.exists()\n", + " else pd.DataFrame(columns=RESULT_COLUMNS)\n", + " )\n", + "\n", + "\n", + "def save_checkpoint(records: dict) -> None:\n", + " \"\"\"\n", + " Save the transcription results to a checkpoint CSV file.\n", + "\n", + " Args:\n", + " records (dict): A dictionary of transcription results, where keys are file identifiers\n", + " and values are dictionaries containing the transcription data.\n", + " \"\"\"\n", + " CHECKPOINT.parent.mkdir(parents=True, exist_ok=True)\n", + " temp = CHECKPOINT.with_suffix(\".tmp\")\n", + " pd.DataFrame(sorted(records.values(), key=lambda x: x[\"file\"])).to_csv(\n", + " temp, index=False, columns=RESULT_COLUMNS\n", + " )\n", + " temp.replace(CHECKPOINT)\n", + "\n", + "\n", + "def transcribe_dataset(data: pd.DataFrame) -> pd.DataFrame:\n", + " \"\"\"\n", + " Transcribe a dataset of audio files using the ASR API,\n", + " with checkpointing to allow resuming.\n", + "\n", + " Args:\n", + " data (pd.DataFrame): A DataFrame containing the dataset to transcribe,\n", + " expected to have \"file\" and \"audio_path\" columns.\n", + "\n", + " Returns:\n", + " pd.DataFrame: A DataFrame containing the transcription results for the dataset.\n", + " \"\"\"\n", + " records = {r[\"file\"]: r for r in load_checkpoint().to_dict(\"records\")}\n", + " done = {k for k, v in records.items() if v[\"status\"] == \"success\"}\n", + " work = data.loc[~data.file.isin(done), [\"file\", \"audio_path\"]]\n", + " work = work if SAMPLE_LIMIT is None else work.head(SAMPLE_LIMIT)\n", + "\n", + " with ThreadPoolExecutor(max_workers=MAX_WORKERS) as pool:\n", + " futures = [\n", + " pool.submit(transcribe_one, r.file, r.audio_path)\n", + " for r in work.itertuples(index=False)\n", + " ]\n", + " for n, future in enumerate(tqdm(as_completed(futures), total=len(futures)), 1):\n", + " result = future.result()\n", + " records[result[\"file\"]] = result\n", + " if n % 25 == 0:\n", + " save_checkpoint(records)\n", + "\n", + " if len(work):\n", + " save_checkpoint(records)\n", + "\n", + " return load_checkpoint()\n", + "\n", + "\n", + "results = transcribe_dataset(df)" + ] + }, + { + "cell_type": "markdown", + "id": "7623eae2785240b9bd12b16a66d81610", + "metadata": {}, + "source": [ + "## Raw and normalized metrics\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7cdc8c89c7104fffa095e18ddfef8986", + "metadata": {}, + "outputs": [], + "source": [ + "df = df.merge(results, on=\"file\", how=\"left\")\n", + "df[\"status\"] = df.status.fillna(\"pending\")\n", + "df[\"hypothesis\"] = df.hypothesis.fillna(\"\")\n", + "df[\"reference_normalized\"] = df.transcription.map(normalize)\n", + "df[\"hypothesis_normalized\"] = df.hypothesis.map(normalize)\n", + "evaluation = df[df.status == \"success\"].copy()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "250ccd41", + "metadata": {}, + "outputs": [], + "source": [ + "def edit_counts(reference: str, hypothesis: str, unit: str) -> tuple[int, int, float]:\n", + " \"\"\"\n", + " Evaluate the edit counts and error rate between a reference and hypothesis transcription\n", + " using the jiwer library, based on the specified unit of measurement.\n", + "\n", + " Args:\n", + " reference (str): The reference transcription.\n", + " hypothesis (str): The hypothesis transcription.\n", + " unit (str): The unit of measurement, either \"word\" or \"character\".\n", + "\n", + " Returns:\n", + " tuple[int, int, float]: A tuple containing the number of errors, the total number of units, and the error rate.\n", + " \"\"\"\n", + " out = (jiwer.process_words if unit == \"word\" else jiwer.process_characters)(\n", + " reference, hypothesis\n", + " )\n", + " units = out.hits + out.substitutions + out.deletions\n", + " errors = out.substitutions + out.deletions + out.insertions\n", + " return errors, units, errors / units if units else np.nan\n", + "\n", + "\n", + "for label, ref, hyp in [\n", + " (\"raw\", \"transcription\", \"hypothesis\"),\n", + " (\"normalized\", \"reference_normalized\", \"hypothesis_normalized\"),\n", + "]:\n", + " for unit, metric in [(\"word\", \"wer\"), (\"character\", \"cer\")]:\n", + " values = [\n", + " edit_counts(r, h, unit) for r, h in zip(evaluation[ref], evaluation[hyp])\n", + " ]\n", + " for i, name in enumerate([\"errors\", \"units\", \"rate\"]):\n", + " evaluation[f\"{label}_{metric}_{name}\"] = [v[i] for v in values]\n", + "\n", + "\n", + "def aggregate(group: pd.DataFrame) -> pd.Series:\n", + " \"\"\"\n", + " Aggregate the edit counts and error rates for a group of samples,\n", + " calculating the total number of samples and the overall error rates for both raw and normalized transcriptions.\n", + "\n", + " Args:\n", + " group (pd.DataFrame): A DataFrame group containing the samples to aggregate.\n", + "\n", + " Returns:\n", + " pd.Series: A Series containing the total number of samples\n", + " and the overall error rates for raw and normalized transcriptions.\n", + " \"\"\"\n", + " result = {\"samples\": len(group)}\n", + " for label in [\"raw\", \"normalized\"]:\n", + " for metric in [\"wer\", \"cer\"]:\n", + " units = group[f\"{label}_{metric}_units\"].sum()\n", + " result[f\"{label}_{metric}\"] = (\n", + " group[f\"{label}_{metric}_errors\"].sum() / units if units else np.nan\n", + " )\n", + " return pd.Series(result)\n", + "\n", + "\n", + "assert edit_counts(\"hola mundo\", \"hola gente\", \"word\")[2] == 0.5\n", + "\n", + "display(\n", + " pd.Series(\n", + " {\n", + " \"total\": len(df),\n", + " \"success\": len(evaluation),\n", + " \"failed\": (df.status == \"error\").sum(),\n", + " \"pending\": (df.status == \"pending\").sum(),\n", + " \"empty_hypotheses\": evaluation.hypothesis.str.strip().eq(\"\").sum(),\n", + " }\n", + " ).to_frame(\"value\")\n", + ")\n", + "display(aggregate(evaluation).to_frame(\"value\"))" + ] + }, + { + "cell_type": "markdown", + "id": "b118ea5561624da68c537baed56e602f", + "metadata": {}, + "source": [ + "## Summaries, breakdowns, and audio playback\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "938c804e27f84196a10c8828c723f798", + "metadata": {}, + "outputs": [], + "source": [ + "metric_cols = [\n", + " \"raw_wer_rate\",\n", + " \"raw_cer_rate\",\n", + " \"normalized_wer_rate\",\n", + " \"normalized_cer_rate\",\n", + "]\n", + "\n", + "print(\"Error rate percentiles:\")\n", + "display(\n", + " evaluation[metric_cols + [\"elapsed_seconds\"]].describe(\n", + " percentiles=[0.5, 0.75, 0.9, 0.95, 0.99]\n", + " )\n", + ")\n", + "\n", + "print(\"Error rates by source TSV:\")\n", + "display(evaluation.groupby(\"source_tsv\").apply(aggregate, include_groups=False))\n", + "\n", + "print(\"Worst transcriptions by normalized WER:\")\n", + "worst = evaluation.sort_values(\"normalized_wer_rate\", ascending=False).head(20)\n", + "display(\n", + " worst[\n", + " [\"file\", \"transcription\", \"hypothesis\", \"raw_wer_rate\", \"normalized_wer_rate\"]\n", + " ]\n", + ")\n", + "if df[\"status\"].eq(\"error\").any():\n", + " print(\"Errors in transcriptions:\")\n", + " display(df[df[\"status\"] == \"error\"][[\"file\", \"attempts\", \"error\"]].head(50))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c8071c8d", + "metadata": {}, + "outputs": [], + "source": [ + "def display_sample(index: int) -> None:\n", + " \"\"\"\n", + " Display a sample from the evaluation dataframe.\n", + "\n", + " Args:\n", + " index (int): The index of the sample to display.\n", + " \"\"\"\n", + " row = evaluation.loc[index]\n", + " display(\n", + " pd.Series(\n", + " {\n", + " \"file\": row.file,\n", + " \"reference\": row.transcription,\n", + " \"hypothesis\": row.hypothesis,\n", + " \"raw WER\": row.raw_wer_rate,\n", + " \"normalized WER\": row.normalized_wer_rate,\n", + " }\n", + " ).to_frame(\"value\")\n", + " )\n", + " display(Audio(filename=str(row.audio_path)))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bed3a2ed", + "metadata": {}, + "outputs": [], + "source": [ + "# Display a random sample from the evaluation results\n", + "if not evaluation.empty:\n", + " display_sample(evaluation.sample(1).index[0])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ca3f681e", + "metadata": {}, + "outputs": [], + "source": [ + "# Assess the worst transcriptions by normalized WER\n", + "for index in worst.index:\n", + " display_sample(index)\n", + " print(\"-\" * 40)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e8ea97b9", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "aymurai (3.10.20.final.0)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.20" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From 1c219907d5715939058f92ea37b5ba10c48e27b6 Mon Sep 17 00:00:00 2001 From: jedzill4 Date: Sun, 21 Jun 2026 20:10:19 -0400 Subject: [PATCH 16/22] feat(asr): implement streaming transcription with progress estimation and SSE support --- .../api/endpoints/routers/asr/transcribe.py | 209 +++++++++++++++- aymurai/api/meta/asr/coro.py | 15 ++ aymurai/audio/asr_client.py | 67 ++++- aymurai/audio/duration.py | 59 +++++ scripts/run-coro.sh | 2 +- .../endpoints/routers/asr/test_transcribe.py | 232 +++++++++++++++++- test/audio/test_asr_client.py | 97 +++++++- test/audio/test_duration.py | 40 +++ 8 files changed, 703 insertions(+), 18 deletions(-) create mode 100644 aymurai/audio/duration.py create mode 100644 test/audio/test_duration.py diff --git a/aymurai/api/endpoints/routers/asr/transcribe.py b/aymurai/api/endpoints/routers/asr/transcribe.py index 3c0dce78..4298104c 100644 --- a/aymurai/api/endpoints/routers/asr/transcribe.py +++ b/aymurai/api/endpoints/routers/asr/transcribe.py @@ -1,4 +1,10 @@ +import json +import math +from collections.abc import AsyncIterator +from typing import Any + from fastapi import Body, Depends, UploadFile +from fastapi.responses import StreamingResponse from fastapi.routing import APIRouter from pydantic import UUID5 from sqlmodel import Session @@ -9,7 +15,16 @@ NotFoundError, UpstreamServiceError, ) -from aymurai.audio.asr_client import transcribe_audio_bytes +from aymurai.api.meta.asr.coro import ( + CoroSegment, + CoroStreamDelta, + CoroStreamSegments, +) +from aymurai.audio.asr_client import ( + stream_transcribe_audio_bytes, + transcribe_audio_bytes, +) +from aymurai.audio.duration import probe_audio_duration from aymurai.database.crud.audio_transcription import ( audio_transcription_create_or_update, audio_transcription_get, @@ -27,6 +42,73 @@ router = APIRouter() logger = get_logger(__name__) +# Progress estimation tuning (see docs/superpowers/specs/2026-06-19-asr-streaming-progress-design.md). +# CHARS_PER_SEC biases the estimate: higher values make the bar lag and snap to +# 100% on completion rather than peg early. K = ln(10) puts the displayed value +# at ~90% when the char estimate thinks transcription is done (raw == 1.0). +ASR_PROGRESS_CHARS_PER_SEC = 17.0 +ASR_PROGRESS_K = math.log(10) + + +def _estimate_progress(cumulative_chars: int, duration_s: float | None) -> float | None: + """ + Estimate transcription progress with an asymptotic easing curve. + + Computes ``1 - exp(-k * raw)`` where ``raw`` is the ratio of characters seen + so far to the expected total (``duration_s * ASR_PROGRESS_CHARS_PER_SEC``). + The curve never reaches 1.0, so it decelerates near the top instead of + pinning to a hard cap; completion is signalled separately by the done event. + + Args: + cumulative_chars (int): Characters received from deltas so far. + duration_s (float | None): Audio duration in seconds, or None if unknown. + + Returns: + float | None: Progress in [0.0, 1.0), or None if duration is unknown. + """ + if not duration_s or duration_s <= 0: + return None + raw = cumulative_chars / (duration_s * ASR_PROGRESS_CHARS_PER_SEC) + value = 1.0 - math.exp(-ASR_PROGRESS_K * raw) + # Guard the float-underflow corner so deltas stay strictly below 1.0 + # (1.0 is reserved for the done event). This only corrects the artifact at + # extreme raw values; it is not a visible cap. + return min(value, math.nextafter(1.0, 0.0)) + + +def _segments_to_paragraphs(segments: list[CoroSegment]) -> list[ASRParagraph]: + """ + Map coro segments to ASRParagraph objects. + + Args: + segments (list[CoroSegment]): The coro speaker-attributed segments. + + Returns: + list[ASRParagraph]: The transcribed paragraphs. + """ + return [ + ASRParagraph( + speaker_no=int(segment.speaker), + start=segment.start, + end=segment.end, + text=segment.text, + ) + for segment in segments + ] + + +def _build_sse_message(payload: dict[str, Any]) -> str: + """ + Format a payload as an SSE data message. + + Args: + payload (dict[str, Any]): Dictionary to serialize into the SSE data field. + + Returns: + str: Serialized SSE message string. + """ + return f"data: {json.dumps(payload)}\n\n" + def get_transcribe_base_url() -> str: """ @@ -73,15 +155,7 @@ async def _transcribe_audio_bytes_with_error_handling( detail="Unexpected error during transcription" ) from exc - return [ - ASRParagraph( - speaker_no=int(segment.speaker), - start=segment.start, - end=segment.end, - text=segment.text, - ) - for segment in segments - ] + return _segments_to_paragraphs(segments) @router.post("/transcribe", response_model=ASRDocument) @@ -111,7 +185,7 @@ async def transcribe( transcription_id=document_id, session=session ) if cached_record is not None: - logger.debug(f"Audio transcription DB hit for {file.filename}") + logger.debug("Audio transcription DB hit for %s", file.filename) cached_document = ASRDocument( document_id=document_id, document=cached_record.validation or cached_record.transcription, @@ -130,11 +204,122 @@ async def transcribe( transcription=document.document, session=session, ) - logger.debug(f"Audio transcription stored in DB for {file.filename}") + logger.debug("Audio transcription stored in DB for %s", file.filename) return document +@router.post("/transcribe/stream") +async def transcribe_stream( + file: UploadFile, + use_cache: bool = True, + base_url: str = Depends(get_transcribe_base_url), + session: Session = Depends(get_session), +) -> StreamingResponse: + """ + Transcribe an uploaded audio file and stream progress via Server-Sent Events. + + Emits ``data: {json}`` SSE messages with a ``type`` discriminator: + - meta: ``{document_id, duration}`` (duration is null if unknown) + - delta: ``{text, progress?}`` incremental text (progress omitted if no duration) + - segments: ``{document}`` the final paragraph list + - done: ``{progress: 1.0}`` terminal success + - error: ``{detail}`` upstream/internal failure + + On a cache hit it emits meta, segments, done without calling coro. + + Args: + file (UploadFile): The audio file to be transcribed. + use_cache (bool, optional): Whether to use cached results. Defaults to True. + base_url (str, optional): The coro base URL. Defaults to Depends(get_transcribe_base_url). + session (Session, optional): The database session. Defaults to Depends(get_session). + + Returns: + StreamingResponse: A text/event-stream of transcription progress events. + """ + data = await file.read() + document_id = data_to_uuid(data) + filename = file.filename or str(document_id) + content_type = file.content_type or "application/octet-stream" + + cached_paragraphs: list[ASRParagraph] | None = None + if use_cache: + cached_record = audio_transcription_get( + transcription_id=document_id, session=session + ) + if cached_record is not None: + cached_paragraphs = [ + ASRParagraph.model_validate(item) + for item in (cached_record.validation or cached_record.transcription) + ] + + duration = probe_audio_duration(data) + + async def event_stream() -> AsyncIterator[str]: + yield _build_sse_message( + { + "type": "meta", + "document_id": str(document_id), + "duration": duration, + } + ) + + if cached_paragraphs is not None: + yield _build_sse_message( + { + "type": "segments", + "document": [ + paragraph.model_dump(mode="json") + for paragraph in cached_paragraphs + ], + } + ) + yield _build_sse_message({"type": "done", "progress": 1.0}) + return + + cumulative_chars = 0 + try: + async for event in stream_transcribe_audio_bytes( + data, filename, content_type + ): + if isinstance(event, CoroStreamDelta): + cumulative_chars += len(event.text) + payload: dict[str, Any] = {"type": "delta", "text": event.text} + progress = _estimate_progress(cumulative_chars, duration) + if progress is not None: + payload["progress"] = progress + yield _build_sse_message(payload) + elif isinstance(event, CoroStreamSegments): + paragraphs = _segments_to_paragraphs(event.segments) + audio_transcription_create_or_update( + transcription_id=document_id, + name=filename, + transcription=paragraphs, + session=session, + ) + yield _build_sse_message( + { + "type": "segments", + "document": [ + paragraph.model_dump(mode="json") + for paragraph in paragraphs + ], + } + ) + except RuntimeError as exc: + logger.error("asr stream error: %s", exc) + yield _build_sse_message({"type": "error", "detail": str(exc)}) + return + + yield _build_sse_message({"type": "done", "progress": 1.0}) + + return StreamingResponse( + event_stream(), + media_type="text/event-stream", + headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"}, + ) + + @router.get("/validation/document/{document_id}") async def asr_read_document_validation( document_id: UUID5, diff --git a/aymurai/api/meta/asr/coro.py b/aymurai/api/meta/asr/coro.py index 31db6df3..f2326855 100644 --- a/aymurai/api/meta/asr/coro.py +++ b/aymurai/api/meta/asr/coro.py @@ -60,3 +60,18 @@ class CoroSegment(BaseModel): end: float text: str speaker: str + + +class CoroStreamDelta(BaseModel): + """An incremental transcription text fragment from a coro delta event.""" + + text: str + + +class CoroStreamSegments(BaseModel): + """The final speaker-attributed segments parsed from the coro done frame.""" + + segments: list[CoroSegment] + + +CoroStreamEvent = CoroStreamDelta | CoroStreamSegments diff --git a/aymurai/audio/asr_client.py b/aymurai/audio/asr_client.py index da682e2f..26e310aa 100644 --- a/aymurai/audio/asr_client.py +++ b/aymurai/audio/asr_client.py @@ -1,14 +1,21 @@ import json +from collections.abc import AsyncGenerator from openai import APIConnectionError, APIError, AsyncOpenAI -from aymurai.api.meta.asr.coro import CoroSegment +from aymurai.api.meta.asr.coro import ( + CoroSegment, + CoroStreamDelta, + CoroStreamEvent, + CoroStreamSegments, +) from aymurai.logger import get_logger from aymurai.settings import settings logger = get_logger(__name__) DONE_EVENT_TYPE = "transcript.text.done" +DELTA_EVENT_TYPE = "transcript.text.delta" async def transcribe_audio_bytes( @@ -60,3 +67,61 @@ async def transcribe_audio_bytes( raise RuntimeError("Transcription service error") from exc raise RuntimeError("coro stream ended without a done frame") + + +async def stream_transcribe_audio_bytes( + payload: bytes, + filename: str, + content_type: str, +) -> AsyncGenerator[CoroStreamEvent, None]: + """ + Stream a transcription from coro, yielding deltas then the final segments. + + Sends the audio to coro's OpenAI-compatible ``/v1/audio/transcriptions`` + endpoint with ``stream=True``. Yields one ``CoroStreamDelta`` per + ``transcript.text.delta`` event as text arrives, then a single + ``CoroStreamSegments`` parsed from the ``transcript.text.done`` frame. + + Args: + payload (bytes): The raw audio file bytes (any ffmpeg-decodable format). + filename (str): The uploaded file name (used for the multipart part). + content_type (str): The uploaded file MIME type. + + Raises: + RuntimeError: If the base URL is unset, the service errors, or no done + frame is received. + + Yields: + CoroStreamEvent: Incremental deltas followed by the final segments. + """ + base_url = settings.TRANSCRIBE_BASE_URL + if not base_url: + raise RuntimeError("TRANSCRIBE_BASE_URL is not configured") + + client = AsyncOpenAI(base_url=base_url, api_key=settings.TRANSCRIBE_API_KEY) + + logger.info("streaming audio to coro for transcription (sse)") + try: + stream = await client.audio.transcriptions.create( + file=(filename, payload, content_type), + model="whisper-1", + language="es", + stream=True, + ) + async for event in stream: + if event.type == DELTA_EVENT_TYPE: + yield CoroStreamDelta(text=event.delta or "") + elif event.type == DONE_EVENT_TYPE: + data = json.loads(event.text) + yield CoroStreamSegments( + segments=[ + CoroSegment.model_validate(segment) + for segment in data.get("segments", []) + ] + ) + return + except (APIError, APIConnectionError) as exc: + logger.error("coro transcription error: %s", exc) + raise RuntimeError("Transcription service error") from exc + + raise RuntimeError("coro stream ended without a done frame") diff --git a/aymurai/audio/duration.py b/aymurai/audio/duration.py new file mode 100644 index 00000000..4e8f04c0 --- /dev/null +++ b/aymurai/audio/duration.py @@ -0,0 +1,59 @@ +import subprocess +import tempfile + +from aymurai.logger import get_logger + +logger = get_logger(__name__) + +_FFPROBE_ARGS = [ + "ffprobe", + "-v", + "error", + "-show_entries", + "format=duration", + "-of", + "default=noprint_wrappers=1:nokey=1", +] + +_FFPROBE_TIMEOUT_SECONDS = 30 + + +def probe_audio_duration(payload: bytes) -> float | None: + """ + Probe the duration of an audio payload in seconds using ffprobe. + + The bytes are written to a temporary file because ffprobe cannot read the + duration of container formats (e.g. WAV) from a non-seekable stdin pipe. Any + failure (missing ffprobe, undecodable input, timeout, non-numeric output) is + treated as "unknown" and returns None so callers can degrade gracefully. + + Args: + payload (bytes): The raw audio file bytes. + + Returns: + float | None: The duration in seconds, or None if it cannot be determined. + """ + if not payload: + return None + + try: + with tempfile.NamedTemporaryFile(suffix=".audio") as handle: + handle.write(payload) + handle.flush() + result = subprocess.run( + [*_FFPROBE_ARGS, handle.name], + capture_output=True, + timeout=_FFPROBE_TIMEOUT_SECONDS, + check=True, + ) + except (subprocess.SubprocessError, OSError) as exc: + logger.warning("ffprobe duration probe failed: %s", exc) + return None + + raw = result.stdout.decode("utf-8", errors="ignore").strip() + try: + duration = float(raw) + except ValueError: + return None + + return duration if duration > 0 else None diff --git a/scripts/run-coro.sh b/scripts/run-coro.sh index 5e4ac2ae..539c3615 100755 --- a/scripts/run-coro.sh +++ b/scripts/run-coro.sh @@ -31,5 +31,5 @@ case "$MODE" in esac exec uvx --python 3.12 \ - --from "coro[$EXTRA] @ git+https://github.com/collectiveai-team/coro" \ + --from "coro-asr[$EXTRA] @ git+https://github.com/collectiveai-team/coro" \ coro --port "$PORT" diff --git a/test/api/endpoints/routers/asr/test_transcribe.py b/test/api/endpoints/routers/asr/test_transcribe.py index 024bc1e2..b4e17084 100644 --- a/test/api/endpoints/routers/asr/test_transcribe.py +++ b/test/api/endpoints/routers/asr/test_transcribe.py @@ -1,15 +1,35 @@ +import json from datetime import timedelta from typing import Any, cast from unittest.mock import AsyncMock, patch from sqlmodel import Session -from aymurai.api.meta.asr.coro import CoroSegment +from aymurai.api.endpoints.routers.asr.transcribe import _estimate_progress +from aymurai.api.meta.asr.coro import CoroSegment, CoroStreamDelta, CoroStreamSegments from aymurai.database.meta.audio_transcription import AudioTranscription from aymurai.database.utils import data_to_uuid from aymurai.meta.api_interfaces import ASRDocument, ASRParagraph +def _parse_sse(body: str) -> list[dict[str, Any]]: + events: list[dict[str, Any]] = [] + for block in body.split("\n\n"): + block = block.strip() + if not block.startswith("data:"): + continue + events.append(json.loads(block[len("data:") :].strip())) + return events + + +def _fake_stream(events): + async def _gen(*args, **kwargs): + for event in events: + yield event + + return _gen + + # MARK: POST Transcribe def test_should_transcribe_and_persist_document_when_service_returns_paragraphs( asr_test_client, @@ -297,3 +317,213 @@ def test_should_return_null_speaker_name_when_none_stored( assert response.status_code == 200 payload = ASRDocument.model_validate(response.json()) assert payload.document[0].speaker_name is None + + +# MARK: progress estimation +def test_progress_should_be_zero_for_no_characters(): + assert _estimate_progress(0, 10.0) == 0.0 + + +def test_progress_should_be_about_ninety_percent_at_estimated_completion(): + # raw == 1.0 when chars == duration * 17 + progress = _estimate_progress(170, 10.0) + assert progress is not None + assert abs(progress - 0.9) < 1e-6 + + +def test_progress_should_stay_below_one_even_when_estimate_exceeded(): + progress = _estimate_progress(100_000, 1.0) + assert progress is not None + assert progress < 1.0 + + +def test_progress_should_be_monotonic_non_decreasing(): + values = [_estimate_progress(chars, 10.0) for chars in range(0, 500, 25)] + assert all(a is not None and b is not None for a, b in zip(values, values[1:])) + assert all(a <= b for a, b in zip(values, values[1:])) # type: ignore[operator] + + +def test_progress_should_be_none_when_duration_unavailable(): + assert _estimate_progress(100, None) is None + + +def test_progress_should_be_none_when_duration_non_positive(): + assert _estimate_progress(100, 0.0) is None + + +# MARK: POST Transcribe stream +def test_stream_should_emit_meta_deltas_segments_done_and_persist( + asr_test_client, + make_wav_bytes, +): + client, engine = asr_test_client + audio_bytes = make_wav_bytes(duration_seconds=2.0) + document_id = data_to_uuid(audio_bytes) + + events = [ + CoroStreamDelta(text="Hola "), + CoroStreamDelta(text="mundo"), + CoroStreamSegments( + segments=[CoroSegment(start=0.0, end=1.0, text="Hola mundo", speaker="1")] + ), + ] + + with patch( + "aymurai.api.endpoints.routers.asr.transcribe.stream_transcribe_audio_bytes", + new=_fake_stream(events), + ): + response = client.post( + "/asr/transcribe/stream?use_cache=false", + files={"file": ("sample.wav", audio_bytes, "audio/wav")}, + ) + + assert response.status_code == 200 + assert response.headers["content-type"].startswith("text/event-stream") + parsed = _parse_sse(response.text) + + types = [event["type"] for event in parsed] + assert types == ["meta", "delta", "delta", "segments", "done"] + + meta = parsed[0] + assert meta["document_id"] == str(document_id) + assert meta["duration"] is not None and abs(meta["duration"] - 2.0) < 0.1 + + deltas = [event for event in parsed if event["type"] == "delta"] + assert deltas[0]["text"] == "Hola " + assert deltas[1]["text"] == "mundo" + progresses = [event["progress"] for event in deltas] + assert all(0.0 <= p < 1.0 for p in progresses) + assert progresses == sorted(progresses) + + segments_event = next(event for event in parsed if event["type"] == "segments") + assert segments_event["document"][0]["text"] == "Hola mundo" + + assert parsed[-1] == {"type": "done", "progress": 1.0} + + with Session(engine) as session: + record = session.get(AudioTranscription, document_id) + assert record is not None + first_item = cast(dict[str, Any], record.transcription[0]) + assert first_item["text"] == "Hola mundo" + + +def test_stream_should_emit_cached_document_without_calling_coro( + asr_test_client, + make_wav_bytes, +): + client, engine = asr_test_client + audio_bytes = make_wav_bytes(freq_hz=210) + document_id = data_to_uuid(audio_bytes) + + with Session(engine) as session: + session.add( + AudioTranscription( + id=document_id, + name="cached.wav", + transcription=cast( + Any, + [ + ASRParagraph( + speaker_no=1, + start=timedelta(seconds=0), + end=timedelta(seconds=1), + text="Texto cacheado", + ).model_dump(mode="json") + ], + ), + validation=[], + ) + ) + session.commit() + + def _fail(*args, **kwargs): + raise AssertionError("coro must not be called on cache hit") + + with patch( + "aymurai.api.endpoints.routers.asr.transcribe.stream_transcribe_audio_bytes", + new=_fail, + ): + response = client.post( + "/asr/transcribe/stream", + files={"file": ("cached.wav", audio_bytes, "audio/wav")}, + ) + + assert response.status_code == 200 + parsed = _parse_sse(response.text) + types = [event["type"] for event in parsed] + assert types == ["meta", "segments", "done"] + segments_event = next(event for event in parsed if event["type"] == "segments") + assert segments_event["document"][0]["text"] == "Texto cacheado" + + +def test_stream_should_emit_error_event_on_upstream_failure( + asr_test_client, + make_wav_bytes, +): + client, engine = asr_test_client + audio_bytes = make_wav_bytes(freq_hz=190) + document_id = data_to_uuid(audio_bytes) + + def _raising_stream(*args, **kwargs): + async def _gen(): + raise RuntimeError("Transcription service error") + yield # pragma: no cover + + return _gen() + + with patch( + "aymurai.api.endpoints.routers.asr.transcribe.stream_transcribe_audio_bytes", + new=_raising_stream, + ): + response = client.post( + "/asr/transcribe/stream?use_cache=false", + files={"file": ("err.wav", audio_bytes, "audio/wav")}, + ) + + assert response.status_code == 200 + parsed = _parse_sse(response.text) + types = [event["type"] for event in parsed] + assert "error" in types + assert "done" not in types + error_event = next(event for event in parsed if event["type"] == "error") + assert "Transcription service error" in error_event["detail"] + + with Session(engine) as session: + assert session.get(AudioTranscription, document_id) is None + + +def test_stream_should_omit_progress_when_duration_unavailable( + asr_test_client, + make_wav_bytes, +): + client, _ = asr_test_client + audio_bytes = make_wav_bytes() + + events = [ + CoroStreamDelta(text="Hola"), + CoroStreamSegments( + segments=[CoroSegment(start=0.0, end=1.0, text="Hola", speaker="1")] + ), + ] + + with ( + patch( + "aymurai.api.endpoints.routers.asr.transcribe.stream_transcribe_audio_bytes", + new=_fake_stream(events), + ), + patch( + "aymurai.api.endpoints.routers.asr.transcribe.probe_audio_duration", + return_value=None, + ), + ): + response = client.post( + "/asr/transcribe/stream?use_cache=false", + files={"file": ("sample.wav", audio_bytes, "audio/wav")}, + ) + + assert response.status_code == 200 + parsed = _parse_sse(response.text) + meta = parsed[0] + assert meta["duration"] is None + delta = next(event for event in parsed if event["type"] == "delta") + assert "progress" not in delta diff --git a/test/audio/test_asr_client.py b/test/audio/test_asr_client.py index 71ea494f..6d4b47ff 100644 --- a/test/audio/test_asr_client.py +++ b/test/audio/test_asr_client.py @@ -6,15 +6,27 @@ import pytest from openai import APIConnectionError -from aymurai.api.meta.asr.coro import CoroSegment -from aymurai.audio.asr_client import transcribe_audio_bytes +from aymurai.api.meta.asr.coro import ( + CoroSegment, + CoroStreamDelta, + CoroStreamSegments, +) +from aymurai.audio.asr_client import ( + stream_transcribe_audio_bytes, + transcribe_audio_bytes, +) from aymurai.settings import settings class _FakeEvent: - def __init__(self, type: str, text: str | None = None): + def __init__(self, type: str, text: str | None = None, delta: str | None = None): self.type = type self.text = text + self.delta = delta + + +async def _collect(agen): + return [item async for item in agen] class _FakeStream: @@ -84,3 +96,82 @@ def test_should_map_openai_connection_error_to_runtime_error(monkeypatch): with patch("aymurai.audio.asr_client.AsyncOpenAI", return_value=client): with pytest.raises(RuntimeError, match="Transcription service error"): asyncio.run(transcribe_audio_bytes(b"audio", "sample.wav", "audio/wav")) + + +# MARK: streaming generator +def test_stream_should_yield_deltas_then_segments(monkeypatch): + monkeypatch.setattr(settings, "TRANSCRIBE_BASE_URL", "http://coro.local/v1") + done_text = json.dumps( + { + "segments": [ + {"start": 0.0, "end": 1.0, "text": "Hola", "speaker": "1"}, + ] + } + ) + events = [ + _FakeEvent("transcript.text.delta", delta="Ho"), + _FakeEvent("transcript.text.delta", delta="la"), + _FakeEvent("transcript.text.done", text=done_text), + ] + with _patched_client(events): + result = asyncio.run( + _collect(stream_transcribe_audio_bytes(b"audio", "sample.wav", "audio/wav")) + ) + + assert result == [ + CoroStreamDelta(text="Ho"), + CoroStreamDelta(text="la"), + CoroStreamSegments( + segments=[CoroSegment(start=0.0, end=1.0, text="Hola", speaker="1")] + ), + ] + + +def test_stream_should_stop_after_done_frame(monkeypatch): + monkeypatch.setattr(settings, "TRANSCRIBE_BASE_URL", "http://coro.local/v1") + done_text = json.dumps({"segments": []}) + events = [ + _FakeEvent("transcript.text.done", text=done_text), + _FakeEvent("transcript.text.delta", delta="ignored"), + ] + with _patched_client(events): + result = asyncio.run( + _collect(stream_transcribe_audio_bytes(b"audio", "sample.wav", "audio/wav")) + ) + + assert result == [CoroStreamSegments(segments=[])] + + +def test_stream_should_raise_when_stream_has_no_done_frame(monkeypatch): + monkeypatch.setattr(settings, "TRANSCRIBE_BASE_URL", "http://coro.local/v1") + events = [_FakeEvent("transcript.text.delta", delta="Ho")] + with _patched_client(events): + with pytest.raises(RuntimeError, match="done frame"): + asyncio.run( + _collect( + stream_transcribe_audio_bytes(b"audio", "sample.wav", "audio/wav") + ) + ) + + +def test_stream_should_raise_when_base_url_not_configured(monkeypatch): + monkeypatch.setattr(settings, "TRANSCRIBE_BASE_URL", None) + with pytest.raises(RuntimeError, match="not configured"): + asyncio.run( + _collect(stream_transcribe_audio_bytes(b"audio", "sample.wav", "audio/wav")) + ) + + +def test_stream_should_map_connection_error_to_runtime_error(monkeypatch): + monkeypatch.setattr(settings, "TRANSCRIBE_BASE_URL", "http://coro.local/v1") + request = httpx.Request("POST", "http://coro.local/v1/audio/transcriptions") + create = AsyncMock(side_effect=APIConnectionError(request=request)) + client = MagicMock() + client.audio.transcriptions.create = create + with patch("aymurai.audio.asr_client.AsyncOpenAI", return_value=client): + with pytest.raises(RuntimeError, match="Transcription service error"): + asyncio.run( + _collect( + stream_transcribe_audio_bytes(b"audio", "sample.wav", "audio/wav") + ) + ) diff --git a/test/audio/test_duration.py b/test/audio/test_duration.py new file mode 100644 index 00000000..0ca75d1c --- /dev/null +++ b/test/audio/test_duration.py @@ -0,0 +1,40 @@ +import io +import math +import struct +import wave + +from aymurai.audio.duration import probe_audio_duration + + +def _make_wav_bytes(duration_seconds: float = 1.0, sample_rate: int = 16000) -> bytes: + frame_count = int(duration_seconds * sample_rate) + pcm = bytearray() + for i in range(frame_count): + sample = int(32767 * 0.2 * math.sin(2 * math.pi * 440.0 * (i / sample_rate))) + pcm.extend(struct.pack(" Date: Mon, 22 Jun 2026 17:53:57 -0400 Subject: [PATCH 17/22] feat(asr): enable coro speaker diarization via diarized_json - request response_format=diarized_json in both coro transcription calls so the server returns speaker-attributed segments - bump openai>=2.43.0 (typed diarized_json literal) and override marker-pdf's openai<2.0.0 cap via tool.uv.override-dependencies (marker's openai LLM services are unused by aymurai) - install ffmpeg in the API image for ffprobe-based audio duration probing - add scripts/smoke_coro_diarization.py to verify diarization wiring against a running coro server with a 30s+ multi-speaker clip --- aymurai/audio/asr_client.py | 2 + docker/api/Dockerfile | 6 +- pyproject.toml | 8 +- scripts/smoke_coro_diarization.py | 145 ++++++++++++++++++++++++++++++ uv.lock | 11 ++- 5 files changed, 165 insertions(+), 7 deletions(-) create mode 100644 scripts/smoke_coro_diarization.py diff --git a/aymurai/audio/asr_client.py b/aymurai/audio/asr_client.py index 26e310aa..53823134 100644 --- a/aymurai/audio/asr_client.py +++ b/aymurai/audio/asr_client.py @@ -54,6 +54,7 @@ async def transcribe_audio_bytes( model="whisper-1", language="es", stream=True, + response_format="diarized_json", ) async for event in stream: if event.type == DONE_EVENT_TYPE: @@ -107,6 +108,7 @@ async def stream_transcribe_audio_bytes( model="whisper-1", language="es", stream=True, + response_format="diarized_json", ) async for event in stream: if event.type == DELTA_EVENT_TYPE: diff --git a/docker/api/Dockerfile b/docker/api/Dockerfile index 2462d4ec..aaf0a3d2 100644 --- a/docker/api/Dockerfile +++ b/docker/api/Dockerfile @@ -55,13 +55,15 @@ RUN apt update && \ ENV LANG=en_US.UTF-8 \ LC_ALL=en_US.UTF-8 -# Install LibreOffice for document processing +# Install LibreOffice for document processing and ffmpeg for audio probing +# (ffmpeg provides ffprobe, used by aymurai.audio.duration.probe_audio_duration) RUN apt update && \ apt-get install -y --no-install-recommends \ libreoffice-writer \ libreoffice-common \ default-jre \ - libmagic1 && \ + libmagic1 \ + ffmpeg && \ apt-get autoremove -y && \ apt-get clean && \ rm -rf /tmp/* /var/tmp/* && \ diff --git a/pyproject.toml b/pyproject.toml index f43d3c00..84438d97 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -85,7 +85,7 @@ dependencies = [ "pypandoc>=1.15", "python-docx>=1.2.0", "langextract[openai]==1.1.0", - "openai>=1.65.2", + "openai>=2.43.0", "ollama==0.6.1", "tiktoken==0.12.0", "marker-pdf==1.10.1", @@ -115,6 +115,12 @@ dev = [ ] tests = ["pytest>=9.0.2"] +[tool.uv] +# marker-pdf pins openai<2.0.0, but aymurai never uses marker's optional LLM +# services (the only code path touching openai). Override the cap so we can use +# openai>=2.x, which adds the typed "diarized_json" response_format for coro ASR. +override-dependencies = ["openai>=2.43.0"] + [tool.setuptools.packages.find] include = ["aymurai"] diff --git a/scripts/smoke_coro_diarization.py b/scripts/smoke_coro_diarization.py new file mode 100644 index 00000000..6e57437e --- /dev/null +++ b/scripts/smoke_coro_diarization.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python +"""Diarization smoke test against an already-running coro ASR server. + +Unlike ``scripts/smoke_coro.py`` (which boots its own server), this script +assumes a coro server is already up and focuses on verifying that speaker +diarization is correctly wired end-to-end through aymurai's client. It feeds a +30s+ multi-speaker clip and asserts: + +1. the audio is at least ``SMOKE_MIN_DURATION`` seconds long; +2. the client returns speaker-attributed segments; +3. at least two distinct speakers are detected (diarization is active); +4. every segment is well-formed (start < end, non-empty speaker); +5. the segments cover a reasonable fraction of the audio. + +Usage: + rtk uv run python scripts/smoke_coro_diarization.py /path/to/audio.wav + +Env overrides: + TRANSCRIBE_BASE_URL coro base URL (default: http://localhost:${CORO_PORT}/v1) + CORO_PORT port if TRANSCRIBE_BASE_URL is unset (default 8000) + TRANSCRIBE_API_KEY forwarded to the client (default "not-needed") + SMOKE_AUDIO audio path if not passed as argv[1] + SMOKE_MIN_DURATION minimum required duration in seconds (default 30) + SMOKE_MIN_COVERAGE min fraction of audio covered by segments (default 0.5) +""" + +from __future__ import annotations + +import asyncio +import mimetypes +import os +import sys +from pathlib import Path + + +def log(msg: str) -> None: + print(f"[diar-smoke] {msg}", flush=True) + + +def resolve_base_url() -> str: + base_url = os.environ.get("TRANSCRIBE_BASE_URL") + if base_url: + return base_url + port = os.environ.get("CORO_PORT", "8000") + return f"http://localhost:{port}/v1" + + +def resolve_audio() -> Path: + raw = sys.argv[1] if len(sys.argv) > 1 else os.environ.get("SMOKE_AUDIO") + if not raw: + raise SystemExit( + "no audio provided: pass a path as the first argument or set SMOKE_AUDIO" + ) + path = Path(raw).expanduser() + if not path.exists(): + raise SystemExit(f"audio file not found: {path}") + return path + + +def content_type_for(path: Path) -> str: + guessed, _ = mimetypes.guess_type(path.name) + return guessed or "application/octet-stream" + + +def main() -> int: + min_duration = float(os.environ.get("SMOKE_MIN_DURATION", "30")) + min_coverage = float(os.environ.get("SMOKE_MIN_COVERAGE", "0.5")) + + audio = resolve_audio() + base_url = resolve_base_url() + os.environ["TRANSCRIBE_BASE_URL"] = base_url + os.environ.setdefault("TRANSCRIBE_API_KEY", "not-needed") + + # Imports happen after env is set so settings pick up the base URL. + from aymurai.audio.asr_client import transcribe_audio_bytes + from aymurai.audio.duration import probe_audio_duration + + payload = audio.read_bytes() + log(f"audio: {audio} ({len(payload)} bytes)") + log(f"coro base URL: {base_url}") + + duration = probe_audio_duration(payload) + if duration is None: + log("WARNING: could not probe duration (ffprobe missing?); skipping check") + else: + log(f"audio duration: {duration:.1f}s (minimum required: {min_duration:.0f}s)") + if duration < min_duration: + log(f"FAILED: audio is shorter than {min_duration:.0f}s") + return 1 + + log("transcribing via aymurai transcribe_audio_bytes ...") + segments = asyncio.run( + transcribe_audio_bytes(payload, audio.name, content_type_for(audio)) + ) + + if not segments: + log("FAILED: client returned no segments") + return 1 + + malformed = [s for s in segments if not s.speaker or s.end < s.start or s.start < 0] + if malformed: + log(f"FAILED: {len(malformed)} malformed segment(s); first: {malformed[0]!r}") + return 1 + + zero_length = [s for s in segments if s.end == s.start] + if zero_length: + log(f"note: {len(zero_length)} zero-length segment(s) (start == end), allowed") + + speakers = sorted({s.speaker for s in segments}) + spoken = sum(s.end - s.start for s in segments) + span_end = max(s.end for s in segments) + coverage = (span_end / duration) if duration else None + + log(f"received {len(segments)} segment(s) across {len(speakers)} speaker(s)") + for speaker in speakers: + spk_segs = [s for s in segments if s.speaker == speaker] + spk_time = sum(s.end - s.start for s in spk_segs) + log(f" speaker {speaker}: {len(spk_segs)} segment(s), {spk_time:.1f}s") + log(f"total spoken time: {spoken:.1f}s ; last segment ends at {span_end:.1f}s") + + log("--- transcript (first 12 segments) ---") + for seg in segments[:12]: + log(f" [{seg.start:6.2f}-{seg.end:6.2f}] spk {seg.speaker}: {seg.text}") + if len(segments) > 12: + log(f" ... ({len(segments) - 12} more)") + + if len(speakers) < 2: + log( + "FAILED: only one speaker detected — diarization does not appear wired " + "(expected >= 2 distinct speakers in a multi-speaker clip)" + ) + return 1 + + if coverage is not None: + log(f"segment coverage: {coverage:.0%} of audio (minimum {min_coverage:.0%})") + if coverage < min_coverage: + log("FAILED: segments cover too little of the audio") + return 1 + + log("DIARIZATION SMOKE TEST PASSED") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/uv.lock b/uv.lock index 0ba99b3b..073b870a 100644 --- a/uv.lock +++ b/uv.lock @@ -7,6 +7,9 @@ resolution-markers = [ "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')", ] +[manifest] +overrides = [{ name = "openai", specifier = ">=2.43.0" }] + [[package]] name = "absl-py" version = "2.4.0" @@ -370,7 +373,7 @@ requires-dist = [ { name = "numpy", specifier = "<2.0.0" }, { name = "odfpy", specifier = ">=1.4.1" }, { name = "ollama", specifier = "==0.6.1" }, - { name = "openai", specifier = ">=1.65.2" }, + { name = "openai", specifier = ">=2.43.0" }, { name = "psutil", specifier = "==6.1.0" }, { name = "pydantic", specifier = ">=2.10.4" }, { name = "pydantic-settings", specifier = ">=2.7.0" }, @@ -2875,7 +2878,7 @@ wheels = [ [[package]] name = "openai" -version = "1.109.1" +version = "2.43.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, @@ -2887,9 +2890,9 @@ dependencies = [ { name = "tqdm" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/c6/a1/a303104dc55fc546a3f6914c842d3da471c64eec92043aef8f652eb6c524/openai-1.109.1.tar.gz", hash = "sha256:d173ed8dbca665892a6db099b4a2dfac624f94d20a93f46eb0b56aae940ed869", size = 564133, upload-time = "2025-09-24T13:00:53.075Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f3/fa/88d0c58a0c58df7e6758e66b99c5d028d5e0bb49f8812d7203940cd9dbf1/openai-2.43.0.tar.gz", hash = "sha256:e74d238200a26868977002190fb6631613480a93dfe0c9c982e77021ed60a017", size = 785369, upload-time = "2026-06-17T17:06:56.06Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/1d/2a/7dd3d207ec669cacc1f186fd856a0f61dbc255d24f6fdc1a6715d6051b0f/openai-1.109.1-py3-none-any.whl", hash = "sha256:6bcaf57086cf59159b8e27447e4e7dd019db5d29a438072fbd49c290c7e65315", size = 948627, upload-time = "2025-09-24T13:00:50.754Z" }, + { url = "https://files.pythonhosted.org/packages/a3/d2/ba767f4bbb30776c03d40906a2d3afad716a165ffa1771fc23b8992f7920/openai-2.43.0-py3-none-any.whl", hash = "sha256:65a670b54fadf2268c9e1330133373c963eb779ee969e5cbad419ec2c21dce97", size = 1355077, upload-time = "2026-06-17T17:06:53.614Z" }, ] [[package]] From 3f4e015409700b8c3deae436a03e106522084a39 Mon Sep 17 00:00:00 2001 From: jansaldo Date: Mon, 29 Jun 2026 20:30:17 +0000 Subject: [PATCH 18/22] feat(asr): add speaker turn handling and transcript processing for diarization --- .../api/endpoints/routers/asr/transcribe.py | 41 +++- aymurai/audio/transcript.py | 208 ++++++++++++++++++ aymurai/meta/api_interfaces.py | 10 + 3 files changed, 253 insertions(+), 6 deletions(-) create mode 100644 aymurai/audio/transcript.py diff --git a/aymurai/api/endpoints/routers/asr/transcribe.py b/aymurai/api/endpoints/routers/asr/transcribe.py index 4298104c..56cf7c28 100644 --- a/aymurai/api/endpoints/routers/asr/transcribe.py +++ b/aymurai/api/endpoints/routers/asr/transcribe.py @@ -25,6 +25,7 @@ transcribe_audio_bytes, ) from aymurai.audio.duration import probe_audio_duration +from aymurai.audio.transcript import transcript_turns from aymurai.database.crud.audio_transcription import ( audio_transcription_create_or_update, audio_transcription_get, @@ -36,6 +37,7 @@ ASRDocument, ASRParagraph, ASRParagraphRequest, + ASRSpeakerTurn, ) from aymurai.settings import settings @@ -97,6 +99,25 @@ def _segments_to_paragraphs(segments: list[CoroSegment]) -> list[ASRParagraph]: ] +def _speaker_turns_for_paragraphs( + paragraphs: list[ASRParagraph], +) -> list[ASRSpeakerTurn]: + if not paragraphs: + return [] + return [ + ASRSpeakerTurn.model_validate(turn) + for turn in transcript_turns({"document": paragraphs}) + ] + + +def _asr_document(document_id: UUID5, paragraphs: list[ASRParagraph]) -> ASRDocument: + return ASRDocument( + document_id=document_id, + document=paragraphs, + speaker_turns=_speaker_turns_for_paragraphs(paragraphs), + ) + + def _build_sse_message(payload: dict[str, Any]) -> str: """ Format a payload as an SSE data message. @@ -186,18 +207,18 @@ async def transcribe( ) if cached_record is not None: logger.debug("Audio transcription DB hit for %s", file.filename) - cached_document = ASRDocument( - document_id=document_id, - document=cached_record.validation or cached_record.transcription, - ) - return cached_document + cached_paragraphs = [ + ASRParagraph.model_validate(item) + for item in (cached_record.validation or cached_record.transcription) + ] + return _asr_document(document_id, cached_paragraphs) transcription_items = await _transcribe_audio_bytes_with_error_handling( data, file.filename or str(document_id), file.content_type or "application/octet-stream", ) - document = ASRDocument(document_id=document_id, document=transcription_items) + document = _asr_document(document_id, transcription_items) audio_transcription_create_or_update( transcription_id=document_id, name=file.filename or str(document_id), @@ -272,6 +293,10 @@ async def event_stream() -> AsyncIterator[str]: paragraph.model_dump(mode="json") for paragraph in cached_paragraphs ], + "speaker_turns": [ + turn.model_dump(mode="json") + for turn in _speaker_turns_for_paragraphs(cached_paragraphs) + ], } ) yield _build_sse_message({"type": "done", "progress": 1.0}) @@ -304,6 +329,10 @@ async def event_stream() -> AsyncIterator[str]: paragraph.model_dump(mode="json") for paragraph in paragraphs ], + "speaker_turns": [ + turn.model_dump(mode="json") + for turn in _speaker_turns_for_paragraphs(paragraphs) + ], } ) except RuntimeError as exc: diff --git a/aymurai/audio/transcript.py b/aymurai/audio/transcript.py new file mode 100644 index 00000000..1db05bf0 --- /dev/null +++ b/aymurai/audio/transcript.py @@ -0,0 +1,208 @@ +import re +import warnings +from datetime import timedelta +from typing import Any + + +SENTENCE_END_RE = re.compile(r"(?:[.!?]+|[.!?]+[\"')\]]+)\s*$") + + +def parse_time_seconds(value: str | int | float | timedelta) -> float: + if isinstance(value, timedelta): + return value.total_seconds() + if isinstance(value, (int, float)): + return float(value) + + text = str(value).strip() + + iso_match = re.fullmatch( + r"PT(?:(\d+(?:\.\d+)?)H)?(?:(\d+(?:\.\d+)?)M)?(?:(\d+(?:\.\d+)?)S)?", + text, + ) + if iso_match: + hours = float(iso_match.group(1) or 0) + minutes = float(iso_match.group(2) or 0) + seconds = float(iso_match.group(3) or 0) + return hours * 3600 + minutes * 60 + seconds + + hhmmss = text.split(":") + if len(hhmmss) == 3: + hours, minutes, seconds = hhmmss + return int(hours) * 3600 + int(minutes) * 60 + float(seconds) + + return float(text) + + +def format_timestamp(seconds: float) -> str: + normalized = max(0.0, seconds) + whole_seconds = int(normalized) + millis = int(round((normalized - whole_seconds) * 1000)) + if millis == 1000: + whole_seconds += 1 + millis = 0 + hours, remainder = divmod(whole_seconds, 3600) + minutes, secs = divmod(remainder, 60) + return f"{hours:02d}:{minutes:02d}:{secs:02d}.{millis:03d}" + + +def _segment_value(segment: Any, field: str) -> Any: + if isinstance(segment, dict): + return segment.get(field) + return getattr(segment, field) + + +def ends_sentence(text: str) -> bool: + return bool(SENTENCE_END_RE.search(text.strip())) + + +def speaker_label(segment: dict[str, Any]) -> str: + name = str(segment.get("speaker_name") or "").strip() + if name: + return name + return f"Speaker {segment['speaker_no']}" + + +def diarized_segments(payload: dict[str, Any]) -> list[dict[str, Any]]: + """Read diarized ASR paragraphs from a payload with a document field.""" + document = payload.get("document") + if not isinstance(document, list) or not document: + raise ValueError("Diarized response is missing a non-empty 'document' list") + + segments: list[dict[str, Any]] = [] + for index, item in enumerate(document): + missing = [ + field + for field in ("speaker_no", "start", "end", "text") + if _segment_value(item, field) is None + ] + if missing: + raise ValueError( + f"document[{index}] is missing diarization field(s): {missing}" + ) + + speaker_no = _segment_value(item, "speaker_no") + if str(speaker_no).strip() == "": + raise ValueError(f"document[{index}] has an empty speaker label") + + text = str(_segment_value(item, "text") or "").strip() + if not text: + continue + + start = parse_time_seconds(_segment_value(item, "start")) + end = parse_time_seconds(_segment_value(item, "end")) + if start < 0 or end < start: + raise ValueError( + f"document[{index}] has invalid timestamps: start={start}, end={end}" + ) + + if isinstance(item, dict): + segment = dict(item) + else: + segment = item.model_dump(mode="json") + segment["_start_seconds"] = start + segment["_end_seconds"] = end + segment["_text"] = text + segments.append(segment) + + if not segments: + raise ValueError("Diarized response contains no non-empty speech segments") + + ordered = sorted( + segments, key=lambda item: (item["_start_seconds"], item["_end_seconds"]) + ) + if [id(item) for item in ordered] != [id(item) for item in segments]: + warnings.warn( + "Diarized segments were not ordered by timestamp; transcript will be timestamp-sorted", + stacklevel=2, + ) + + return ordered + + +def _new_turn(segment: dict[str, Any]) -> dict[str, Any]: + return { + "speaker": speaker_label(segment), + "speaker_no": segment["speaker_no"], + "start": segment["_start_seconds"], + "end": segment["_end_seconds"], + "texts": [segment["_text"]], + "segments": [segment], + } + + +def merge_speaker_turns_by_sentence( + segments: list[dict[str, Any]], +) -> list[dict[str, Any]]: + turns: list[dict[str, Any]] = [] + for segment in segments: + label = speaker_label(segment) + previous = turns[-1] if turns else None + starts_new_turn = ( + previous is None + or previous["speaker"] != label + or ends_sentence(" ".join(previous["texts"])) + ) + + if starts_new_turn: + turns.append(_new_turn(segment)) + continue + + previous["end"] = segment["_end_seconds"] + previous["texts"].append(segment["_text"]) + previous["segments"].append(segment) + + for turn in turns: + turn["text"] = " ".join(turn["texts"]).strip() + return turns + + +def transcript_turns(payload: dict[str, Any]) -> list[dict[str, Any]]: + turns = merge_speaker_turns_by_sentence(diarized_segments(payload)) + return serialize_speaker_turns(turns) + + +def serialize_speaker_turns(turns: list[dict[str, Any]]) -> list[dict[str, Any]]: + serialized: list[dict[str, Any]] = [] + for turn in turns: + serialized.append( + { + "speaker": turn["speaker"], + "speaker_no": turn["speaker_no"], + "start": format_timestamp(turn["start"]), + "end": format_timestamp(turn["end"]), + "text": turn.get("text") or " ".join(turn["texts"]).strip(), + "segments": [ + { + key: value + for key, value in segment.items() + if not key.startswith("_") + } + for segment in turn["segments"] + ], + } + ) + return serialized + + +def markdown_transcript(payload: dict[str, Any], title: str) -> str: + segments = diarized_segments(payload) + turns = merge_speaker_turns_by_sentence(segments) + + distinct_speakers = {turn["speaker"] for turn in turns} + if len(distinct_speakers) < 2: + warnings.warn( + f"Only one speaker label found for {title}; verify the source audio is expected to be single-speaker", + stacklevel=2, + ) + + lines = [f"# {title}", ""] + for turn in turns: + timestamp = ( + f"{format_timestamp(turn['start'])} - {format_timestamp(turn['end'])}" + ) + lines.append(f"## {turn['speaker']} ({timestamp})") + lines.append("") + lines.append(turn.get("text") or " ".join(turn["texts"]).strip()) + lines.append("") + + return "\n".join(lines).strip() + "\n" diff --git a/aymurai/meta/api_interfaces.py b/aymurai/meta/api_interfaces.py index fe32211b..f4e3954e 100644 --- a/aymurai/meta/api_interfaces.py +++ b/aymurai/meta/api_interfaces.py @@ -156,9 +156,19 @@ class ASRParagraphRequest(BaseModel): text: str +class ASRSpeakerTurn(BaseModel): + speaker: str + speaker_no: int + start: str + end: str + text: str + segments: list[ASRParagraph] + + class ASRDocument(BaseModel): document: list[ASRParagraph] document_id: UUID + speaker_turns: list[ASRSpeakerTurn] = Field(default_factory=list) def to_txt(self) -> str: return "\n\n".join([paragraph.to_txt() for paragraph in self.document]) From 2e0438c1d1ab65dee0c96562ff043f7e8a79af81 Mon Sep 17 00:00:00 2001 From: jansaldo Date: Mon, 29 Jun 2026 20:30:43 +0000 Subject: [PATCH 19/22] feat(asr): enhance tests for speaker turn handling and add new transcript tests --- .../endpoints/routers/asr/test_transcribe.py | 5 + test/audio/test_transcript.py | 163 ++++++++++++++++++ 2 files changed, 168 insertions(+) create mode 100644 test/audio/test_transcript.py diff --git a/test/api/endpoints/routers/asr/test_transcribe.py b/test/api/endpoints/routers/asr/test_transcribe.py index b4e17084..e0fe4e29 100644 --- a/test/api/endpoints/routers/asr/test_transcribe.py +++ b/test/api/endpoints/routers/asr/test_transcribe.py @@ -57,6 +57,8 @@ def test_should_transcribe_and_persist_document_when_service_returns_paragraphs( assert str(payload.document_id) == str(document_id) assert len(payload.document) == 1 assert payload.document[0].text == "Hola mundo" + assert response.json()["speaker_turns"][0]["text"] == "Hola mundo" + assert response.json()["speaker_turns"][0]["speaker"] == "Speaker 1" with Session(engine) as session: record = session.get(AudioTranscription, document_id) @@ -397,6 +399,8 @@ def test_stream_should_emit_meta_deltas_segments_done_and_persist( segments_event = next(event for event in parsed if event["type"] == "segments") assert segments_event["document"][0]["text"] == "Hola mundo" + assert segments_event["speaker_turns"][0]["text"] == "Hola mundo" + assert segments_event["speaker_turns"][0]["speaker"] == "Speaker 1" assert parsed[-1] == {"type": "done", "progress": 1.0} @@ -454,6 +458,7 @@ def _fail(*args, **kwargs): assert types == ["meta", "segments", "done"] segments_event = next(event for event in parsed if event["type"] == "segments") assert segments_event["document"][0]["text"] == "Texto cacheado" + assert segments_event["speaker_turns"][0]["text"] == "Texto cacheado" def test_stream_should_emit_error_event_on_upstream_failure( diff --git a/test/audio/test_transcript.py b/test/audio/test_transcript.py new file mode 100644 index 00000000..3e89fbdb --- /dev/null +++ b/test/audio/test_transcript.py @@ -0,0 +1,163 @@ +from datetime import timedelta + +import pytest + +from aymurai.audio.transcript import ( + diarized_segments, + ends_sentence, + markdown_transcript, + merge_speaker_turns_by_sentence, +) +from aymurai.meta.api_interfaces import ASRParagraph + + +def _paragraph( + speaker_no: int, + start: float, + end: float, + text: str, + speaker_name: str | None = None, +) -> ASRParagraph: + return ASRParagraph( + speaker_no=speaker_no, + speaker_name=speaker_name, + start=timedelta(seconds=start), + end=timedelta(seconds=end), + text=text, + ) + + +def test_ends_sentence_detects_sentence_closing_punctuation(): + assert ends_sentence("Hola.") + assert ends_sentence("Hola?") + assert ends_sentence("Hola!") + assert ends_sentence("¿Se entiende?") + assert ends_sentence("¡Perfecto!") + assert ends_sentence("Bueno...") + + +def test_ends_sentence_ignores_non_final_punctuation_and_open_text(): + assert not ends_sentence("entonces,") + assert not ends_sentence("lo que queria decir es") + assert not ends_sentence("pero bueno:") + assert not ends_sentence("") + + +def test_merge_combines_consecutive_same_speaker_without_sentence_close(): + segments = diarized_segments( + { + "document": [ + _paragraph(1, 0.0, 1.0, "Nos quedamos,"), + _paragraph(1, 1.0, 2.0, "esperando en el lobby"), + ] + } + ) + + turns = merge_speaker_turns_by_sentence(segments) + + assert len(turns) == 1 + assert turns[0]["speaker"] == "Speaker 1" + assert turns[0]["speaker_no"] == 1 + assert turns[0]["start"] == 0.0 + assert turns[0]["end"] == 2.0 + assert turns[0]["text"] == "Nos quedamos, esperando en el lobby" + assert len(turns[0]["segments"]) == 2 + + +def test_merge_splits_same_speaker_after_sentence_close(): + segments = diarized_segments( + { + "document": [ + _paragraph(1, 0.0, 1.0, "Hola."), + _paragraph(1, 1.0, 2.0, "Seguimos con otro tema"), + ] + } + ) + + turns = merge_speaker_turns_by_sentence(segments) + + assert [turn["text"] for turn in turns] == [ + "Hola.", + "Seguimos con otro tema", + ] + assert turns[0]["end"] == 1.0 + assert turns[1]["start"] == 1.0 + + +def test_merge_splits_when_speaker_changes_even_without_sentence_close(): + segments = diarized_segments( + { + "document": [ + _paragraph(1, 0.0, 1.0, "Entonces,"), + _paragraph(2, 1.0, 2.0, "contesto yo"), + ] + } + ) + + turns = merge_speaker_turns_by_sentence(segments) + + assert [turn["speaker"] for turn in turns] == ["Speaker 1", "Speaker 2"] + assert [turn["text"] for turn in turns] == ["Entonces,", "contesto yo"] + + +def test_merge_preserves_timestamps_across_merged_segments(): + segments = diarized_segments( + { + "document": [ + _paragraph(1, 5.76, 6.24, "Nos quedamos,"), + _paragraph(1, 6.24, 12.48, "esperando en el lobby."), + ] + } + ) + + turns = merge_speaker_turns_by_sentence(segments) + + assert turns[0]["start"] == 5.76 + assert turns[0]["end"] == 12.48 + + +def test_merge_uses_speaker_name_when_available(): + segments = diarized_segments( + { + "document": [ + _paragraph(1, 0.0, 1.0, "Tiene la palabra,", "Jueza"), + _paragraph(1, 1.0, 2.0, "doctora.", "Jueza"), + ] + } + ) + + turns = merge_speaker_turns_by_sentence(segments) + + assert turns[0]["speaker"] == "Jueza" + assert turns[0]["speaker_no"] == 1 + + +def test_diarized_segments_ignores_empty_text_segments(): + segments = diarized_segments( + { + "document": [ + _paragraph(1, 0.0, 1.0, " "), + _paragraph(1, 1.0, 2.0, "Texto util"), + ] + } + ) + + assert len(segments) == 1 + assert segments[0]["_text"] == "Texto util" + + +def test_markdown_transcript_uses_sentence_aware_turns(): + with pytest.warns(UserWarning, match="Only one speaker label"): + markdown = markdown_transcript( + { + "document": [ + _paragraph(1, 0.0, 1.0, "Hola."), + _paragraph(1, 1.0, 2.0, "Seguimos."), + ] + }, + title="Audiencia", + ) + + assert "## Speaker 1 (00:00:00.000 - 00:00:01.000)" in markdown + assert "## Speaker 1 (00:00:01.000 - 00:00:02.000)" in markdown + assert "Hola.\n\n## Speaker 1" in markdown From f856fab7cd4af0f13a34e08c1c209563bc58f34d Mon Sep 17 00:00:00 2001 From: jansaldo Date: Tue, 30 Jun 2026 18:04:41 +0000 Subject: [PATCH 20/22] feat(asr): refactor speaker turn handling and add tests for validated paragraphs --- .../api/endpoints/routers/asr/transcribe.py | 70 ++-- aymurai/audio/transcript.py | 31 ++ .../endpoints/routers/asr/test_transcribe.py | 304 ++++++++++++++++++ test/audio/test_transcript.py | 31 ++ 4 files changed, 413 insertions(+), 23 deletions(-) diff --git a/aymurai/api/endpoints/routers/asr/transcribe.py b/aymurai/api/endpoints/routers/asr/transcribe.py index 56cf7c28..a5f06305 100644 --- a/aymurai/api/endpoints/routers/asr/transcribe.py +++ b/aymurai/api/endpoints/routers/asr/transcribe.py @@ -25,7 +25,10 @@ transcribe_audio_bytes, ) from aymurai.audio.duration import probe_audio_duration -from aymurai.audio.transcript import transcript_turns +from aymurai.audio.transcript import ( + speaker_turns_from_validated_paragraphs, + transcript_turns, +) from aymurai.database.crud.audio_transcription import ( audio_transcription_create_or_update, audio_transcription_get, @@ -99,7 +102,7 @@ def _segments_to_paragraphs(segments: list[CoroSegment]) -> list[ASRParagraph]: ] -def _speaker_turns_for_paragraphs( +def _speaker_turns_for_transcription( paragraphs: list[ASRParagraph], ) -> list[ASRSpeakerTurn]: if not paragraphs: @@ -110,14 +113,43 @@ def _speaker_turns_for_paragraphs( ] -def _asr_document(document_id: UUID5, paragraphs: list[ASRParagraph]) -> ASRDocument: +def _speaker_turns_for_validation( + paragraphs: list[ASRParagraph], +) -> list[ASRSpeakerTurn]: + return [ + ASRSpeakerTurn.model_validate(turn) + for turn in speaker_turns_from_validated_paragraphs(paragraphs) + ] + + +def _asr_document_from_transcription( + document_id: UUID5, paragraphs: list[ASRParagraph] +) -> ASRDocument: return ASRDocument( document_id=document_id, document=paragraphs, - speaker_turns=_speaker_turns_for_paragraphs(paragraphs), + speaker_turns=_speaker_turns_for_transcription(paragraphs), ) +def _paragraphs_from_storage(items: list[Any]) -> list[ASRParagraph]: + return [ASRParagraph.model_validate(item) for item in items] + + +def _asr_document_from_cached_record(document_id: UUID5, record: Any) -> ASRDocument: + validation = record.validation or [] + if validation: + paragraphs = _paragraphs_from_storage(validation) + return ASRDocument( + document_id=document_id, + document=paragraphs, + speaker_turns=_speaker_turns_for_validation(paragraphs), + ) + + paragraphs = _paragraphs_from_storage(record.transcription) + return _asr_document_from_transcription(document_id, paragraphs) + + def _build_sse_message(payload: dict[str, Any]) -> str: """ Format a payload as an SSE data message. @@ -207,18 +239,14 @@ async def transcribe( ) if cached_record is not None: logger.debug("Audio transcription DB hit for %s", file.filename) - cached_paragraphs = [ - ASRParagraph.model_validate(item) - for item in (cached_record.validation or cached_record.transcription) - ] - return _asr_document(document_id, cached_paragraphs) + return _asr_document_from_cached_record(document_id, cached_record) transcription_items = await _transcribe_audio_bytes_with_error_handling( data, file.filename or str(document_id), file.content_type or "application/octet-stream", ) - document = _asr_document(document_id, transcription_items) + document = _asr_document_from_transcription(document_id, transcription_items) audio_transcription_create_or_update( transcription_id=document_id, name=file.filename or str(document_id), @@ -263,16 +291,15 @@ async def transcribe_stream( filename = file.filename or str(document_id) content_type = file.content_type or "application/octet-stream" - cached_paragraphs: list[ASRParagraph] | None = None + cached_document: ASRDocument | None = None if use_cache: cached_record = audio_transcription_get( transcription_id=document_id, session=session ) if cached_record is not None: - cached_paragraphs = [ - ASRParagraph.model_validate(item) - for item in (cached_record.validation or cached_record.transcription) - ] + cached_document = _asr_document_from_cached_record( + document_id, cached_record + ) duration = probe_audio_duration(data) @@ -285,17 +312,17 @@ async def event_stream() -> AsyncIterator[str]: } ) - if cached_paragraphs is not None: + if cached_document is not None: yield _build_sse_message( { "type": "segments", "document": [ paragraph.model_dump(mode="json") - for paragraph in cached_paragraphs + for paragraph in cached_document.document ], "speaker_turns": [ turn.model_dump(mode="json") - for turn in _speaker_turns_for_paragraphs(cached_paragraphs) + for turn in cached_document.speaker_turns ], } ) @@ -331,7 +358,7 @@ async def event_stream() -> AsyncIterator[str]: ], "speaker_turns": [ turn.model_dump(mode="json") - for turn in _speaker_turns_for_paragraphs(paragraphs) + for turn in _speaker_turns_for_transcription(paragraphs) ], } ) @@ -372,10 +399,7 @@ async def asr_read_document_validation( if not record: raise NotFoundError(detail=f"Document not found: {document_id}") - return ASRDocument( - document_id=document_id, - document=record.validation or record.transcription, - ) + return _asr_document_from_cached_record(document_id, record) @router.post("/validation/document/{document_id}") diff --git a/aymurai/audio/transcript.py b/aymurai/audio/transcript.py index 1db05bf0..5e13a7a5 100644 --- a/aymurai/audio/transcript.py +++ b/aymurai/audio/transcript.py @@ -184,6 +184,37 @@ def serialize_speaker_turns(turns: list[dict[str, Any]]) -> list[dict[str, Any]] return serialized +def speaker_turns_from_validated_paragraphs( + paragraphs: list[Any], +) -> list[dict[str, Any]]: + turns: list[dict[str, Any]] = [] + for paragraph in paragraphs: + segment = ( + paragraph + if isinstance(paragraph, dict) + else paragraph.model_dump(mode="json") + ) + text = str(segment.get("text") or "").strip() + if not text: + continue + + start = parse_time_seconds(segment["start"]) + end = parse_time_seconds(segment["end"]) + turn_segment = dict(segment) + turn_segment["text"] = text + turns.append( + { + "speaker": speaker_label(turn_segment), + "speaker_no": turn_segment["speaker_no"], + "start": format_timestamp(start), + "end": format_timestamp(end), + "text": text, + "segments": [turn_segment], + } + ) + return turns + + def markdown_transcript(payload: dict[str, Any], title: str) -> str: segments = diarized_segments(payload) turns = merge_speaker_turns_by_sentence(segments) diff --git a/test/api/endpoints/routers/asr/test_transcribe.py b/test/api/endpoints/routers/asr/test_transcribe.py index e0fe4e29..2a64fb39 100644 --- a/test/api/endpoints/routers/asr/test_transcribe.py +++ b/test/api/endpoints/routers/asr/test_transcribe.py @@ -68,6 +68,64 @@ def test_should_transcribe_and_persist_document_when_service_returns_paragraphs( assert first_item["text"] == "Hola mundo" +def test_transcribe_cache_should_merge_speaker_turns_from_transcription_when_no_validation( + asr_test_client, + make_wav_bytes, +): + client, engine = asr_test_client + audio_bytes = make_wav_bytes(freq_hz=224) + document_id = data_to_uuid(audio_bytes) + + with Session(engine) as session: + session.add( + AudioTranscription( + id=document_id, + name="cached-transcription.wav", + transcription=cast( + Any, + [ + ASRParagraph( + speaker_no=1, + start=timedelta(seconds=0), + end=timedelta(seconds=1), + text="Nos quedamos,", + ).model_dump(mode="json"), + ASRParagraph( + speaker_no=1, + start=timedelta(seconds=1), + end=timedelta(seconds=2), + text="esperando en el lobby.", + ).model_dump(mode="json"), + ], + ), + validation=[], + ) + ) + session.commit() + + def _fail(*args, **kwargs): + raise AssertionError("coro must not be called on transcription cache hit") + + with patch( + "aymurai.api.endpoints.routers.asr.transcribe.transcribe_audio_bytes", + new=_fail, + ): + response = client.post( + "/asr/transcribe", + files={"file": ("cached-transcription.wav", audio_bytes, "audio/wav")}, + ) + + assert response.status_code == 200 + body = response.json() + assert [item["text"] for item in body["document"]] == [ + "Nos quedamos,", + "esperando en el lobby.", + ] + assert [turn["text"] for turn in body["speaker_turns"]] == [ + "Nos quedamos, esperando en el lobby." + ] + + # MARK: GET Validation def test_should_return_validation_document_when_document_exists_in_database( asr_test_client, @@ -116,6 +174,137 @@ def test_should_return_validation_document_when_document_exists_in_database( assert payload.document[0].text == "Texto validado" +def test_transcribe_cache_should_preserve_validation_turns_without_merging( + asr_test_client, + make_wav_bytes, +): + client, engine = asr_test_client + audio_bytes = make_wav_bytes(freq_hz=225) + document_id = data_to_uuid(audio_bytes) + + with Session(engine) as session: + session.add( + AudioTranscription( + id=document_id, + name="validated-cache.wav", + transcription=cast( + Any, + [ + ASRParagraph( + speaker_no=1, + start=timedelta(seconds=0), + end=timedelta(seconds=3), + text="Texto original que no debe usarse", + ).model_dump(mode="json") + ], + ), + validation=cast( + Any, + [ + ASRParagraph( + speaker_no=1, + speaker_name="Docente", + start=timedelta(seconds=10), + end=timedelta(seconds=11), + text="Bloque manual uno sin cierre", + ).model_dump(mode="json"), + ASRParagraph( + speaker_no=1, + speaker_name="Docente", + start=timedelta(seconds=11), + end=timedelta(seconds=12.5), + text="Bloque manual dos editado", + ).model_dump(mode="json"), + ], + ), + ) + ) + session.commit() + + def _fail(*args, **kwargs): + raise AssertionError("coro must not be called on validation cache hit") + + with patch( + "aymurai.api.endpoints.routers.asr.transcribe.transcribe_audio_bytes", + new=_fail, + ): + response = client.post( + "/asr/transcribe", + files={"file": ("validated-cache.wav", audio_bytes, "audio/wav")}, + ) + + assert response.status_code == 200 + body = response.json() + assert [item["text"] for item in body["document"]] == [ + "Bloque manual uno sin cierre", + "Bloque manual dos editado", + ] + assert [turn["text"] for turn in body["speaker_turns"]] == [ + "Bloque manual uno sin cierre", + "Bloque manual dos editado", + ] + assert body["speaker_turns"][0]["speaker"] == "Docente" + assert body["speaker_turns"][1]["end"] == "00:00:12.500" + + +def test_get_validation_should_include_source_aware_speaker_turns( + asr_test_client, + make_wav_bytes, +): + client, engine = asr_test_client + audio_bytes = make_wav_bytes(freq_hz=226) + document_id = data_to_uuid(audio_bytes) + + with Session(engine) as session: + session.add( + AudioTranscription( + id=document_id, + name="read-validation.wav", + transcription=cast( + Any, + [ + ASRParagraph( + speaker_no=1, + start=timedelta(seconds=0), + end=timedelta(seconds=4), + text="Texto original", + ).model_dump(mode="json") + ], + ), + validation=cast( + Any, + [ + ASRParagraph( + speaker_no=2, + speaker_name="Fiscal", + start=timedelta(seconds=4), + end=timedelta(seconds=5), + text="Validado A", + ).model_dump(mode="json"), + ASRParagraph( + speaker_no=2, + speaker_name="Fiscal", + start=timedelta(seconds=5), + end=timedelta(seconds=6), + text="Validado B", + ).model_dump(mode="json"), + ], + ), + ) + ) + session.commit() + + response = client.get(f"/asr/validation/document/{document_id}") + + assert response.status_code == 200 + body = response.json() + assert [turn["text"] for turn in body["speaker_turns"]] == [ + "Validado A", + "Validado B", + ] + assert body["speaker_turns"][0]["speaker"] == "Fiscal" + + # MARK: POST Validation def test_should_persist_validation_annotations_when_posting_validation_for_existing_document( asr_test_client, @@ -321,6 +510,48 @@ def test_should_return_null_speaker_name_when_none_stored( assert payload.document[0].speaker_name is None +def test_get_validation_should_compute_speaker_turns_from_transcription_when_unvalidated( + asr_test_client, + make_wav_bytes, +): + client, engine = asr_test_client + audio_bytes = make_wav_bytes(freq_hz=881) + document_id = data_to_uuid(audio_bytes) + + with Session(engine) as session: + session.add( + AudioTranscription( + id=document_id, + name="unvalidated-read.wav", + transcription=cast( + Any, + [ + ASRParagraph( + speaker_no=1, + start=timedelta(seconds=0), + end=timedelta(seconds=1), + text="Hola,", + ).model_dump(mode="json"), + ASRParagraph( + speaker_no=1, + start=timedelta(seconds=1), + end=timedelta(seconds=2), + text="mundo.", + ).model_dump(mode="json"), + ], + ), + validation=[], + ) + ) + session.commit() + + response = client.get(f"/asr/validation/document/{document_id}") + + assert response.status_code == 200 + body = response.json() + assert [turn["text"] for turn in body["speaker_turns"]] == ["Hola, mundo."] + + # MARK: progress estimation def test_progress_should_be_zero_for_no_characters(): assert _estimate_progress(0, 10.0) == 0.0 @@ -461,6 +692,79 @@ def _fail(*args, **kwargs): assert segments_event["speaker_turns"][0]["text"] == "Texto cacheado" +def test_stream_cache_should_emit_validation_turns_without_merging( + asr_test_client, + make_wav_bytes, +): + client, engine = asr_test_client + audio_bytes = make_wav_bytes(freq_hz=211) + document_id = data_to_uuid(audio_bytes) + + with Session(engine) as session: + session.add( + AudioTranscription( + id=document_id, + name="cached-validation.wav", + transcription=cast( + Any, + [ + ASRParagraph( + speaker_no=1, + start=timedelta(seconds=0), + end=timedelta(seconds=2), + text="Texto original", + ).model_dump(mode="json") + ], + ), + validation=cast( + Any, + [ + ASRParagraph( + speaker_no=1, + speaker_name="Defensa", + start=timedelta(seconds=2), + end=timedelta(seconds=3), + text="Turno validado uno", + ).model_dump(mode="json"), + ASRParagraph( + speaker_no=1, + speaker_name="Defensa", + start=timedelta(seconds=3), + end=timedelta(seconds=4), + text="Turno validado dos", + ).model_dump(mode="json"), + ], + ), + ) + ) + session.commit() + + def _fail(*args, **kwargs): + raise AssertionError("coro must not be called on validation cache hit") + + with patch( + "aymurai.api.endpoints.routers.asr.transcribe.stream_transcribe_audio_bytes", + new=_fail, + ): + response = client.post( + "/asr/transcribe/stream", + files={"file": ("cached-validation.wav", audio_bytes, "audio/wav")}, + ) + + assert response.status_code == 200 + parsed = _parse_sse(response.text) + segments_event = next(event for event in parsed if event["type"] == "segments") + assert [item["text"] for item in segments_event["document"]] == [ + "Turno validado uno", + "Turno validado dos", + ] + assert [turn["text"] for turn in segments_event["speaker_turns"]] == [ + "Turno validado uno", + "Turno validado dos", + ] + assert segments_event["speaker_turns"][0]["speaker"] == "Defensa" + + def test_stream_should_emit_error_event_on_upstream_failure( asr_test_client, make_wav_bytes, diff --git a/test/audio/test_transcript.py b/test/audio/test_transcript.py index 3e89fbdb..646ea8b6 100644 --- a/test/audio/test_transcript.py +++ b/test/audio/test_transcript.py @@ -7,6 +7,7 @@ ends_sentence, markdown_transcript, merge_speaker_turns_by_sentence, + speaker_turns_from_validated_paragraphs, ) from aymurai.meta.api_interfaces import ASRParagraph @@ -161,3 +162,33 @@ def test_markdown_transcript_uses_sentence_aware_turns(): assert "## Speaker 1 (00:00:00.000 - 00:00:01.000)" in markdown assert "## Speaker 1 (00:00:01.000 - 00:00:02.000)" in markdown assert "Hola.\n\n## Speaker 1" in markdown + + +def test_validated_paragraphs_become_one_turn_each_without_merging(): + paragraphs = [ + _paragraph(1, 0.0, 1.0, "Texto editado sin cierre", "Docente"), + _paragraph(1, 1.0, 2.5, "Sigue siendo otro bloque", "Docente"), + ] + + turns = speaker_turns_from_validated_paragraphs(paragraphs) + + assert [turn["text"] for turn in turns] == [ + "Texto editado sin cierre", + "Sigue siendo otro bloque", + ] + assert [turn["speaker"] for turn in turns] == ["Docente", "Docente"] + assert turns[0]["start"] == "00:00:00.000" + assert turns[1]["end"] == "00:00:02.500" + assert turns[0]["segments"][0]["text"] == "Texto editado sin cierre" + + +def test_validated_paragraphs_ignore_empty_text(): + paragraphs = [ + _paragraph(1, 0.0, 1.0, " "), + _paragraph(1, 1.0, 2.0, "Texto validado"), + ] + + turns = speaker_turns_from_validated_paragraphs(paragraphs) + + assert len(turns) == 1 + assert turns[0]["text"] == "Texto validado" From 0921c66fff9fd546dfa6bcd43e41a63421fcacda Mon Sep 17 00:00:00 2001 From: jansaldo Date: Wed, 1 Jul 2026 19:22:46 +0000 Subject: [PATCH 21/22] feat(asr): add ASRValidationDocumentRequest model and update document handling with title support --- .../api/endpoints/routers/asr/transcribe.py | 25 +++-- aymurai/meta/api_interfaces.py | 17 ++++ .../endpoints/routers/asr/test_transcribe.py | 98 +++++++++++++++++++ 3 files changed, 134 insertions(+), 6 deletions(-) diff --git a/aymurai/api/endpoints/routers/asr/transcribe.py b/aymurai/api/endpoints/routers/asr/transcribe.py index a5f06305..af175ff0 100644 --- a/aymurai/api/endpoints/routers/asr/transcribe.py +++ b/aymurai/api/endpoints/routers/asr/transcribe.py @@ -41,6 +41,7 @@ ASRParagraph, ASRParagraphRequest, ASRSpeakerTurn, + ASRValidationDocumentRequest, ) from aymurai.settings import settings @@ -123,10 +124,11 @@ def _speaker_turns_for_validation( def _asr_document_from_transcription( - document_id: UUID5, paragraphs: list[ASRParagraph] + document_id: UUID5, paragraphs: list[ASRParagraph], title: str | None = None ) -> ASRDocument: return ASRDocument( document_id=document_id, + title=title, document=paragraphs, speaker_turns=_speaker_turns_for_transcription(paragraphs), ) @@ -142,12 +144,13 @@ def _asr_document_from_cached_record(document_id: UUID5, record: Any) -> ASRDocu paragraphs = _paragraphs_from_storage(validation) return ASRDocument( document_id=document_id, + title=record.name, document=paragraphs, speaker_turns=_speaker_turns_for_validation(paragraphs), ) paragraphs = _paragraphs_from_storage(record.transcription) - return _asr_document_from_transcription(document_id, paragraphs) + return _asr_document_from_transcription(document_id, paragraphs, title=record.name) def _build_sse_message(payload: dict[str, Any]) -> str: @@ -246,10 +249,13 @@ async def transcribe( file.filename or str(document_id), file.content_type or "application/octet-stream", ) - document = _asr_document_from_transcription(document_id, transcription_items) + title = file.filename or str(document_id) + document = _asr_document_from_transcription( + document_id, transcription_items, title=title + ) audio_transcription_create_or_update( transcription_id=document_id, - name=file.filename or str(document_id), + name=title, transcription=document.document, session=session, ) @@ -405,7 +411,7 @@ async def asr_read_document_validation( @router.post("/validation/document/{document_id}") async def asr_save_document_validation( document_id: UUID5, - annotations: list[ASRParagraphRequest] = Body(...), + payload: list[ASRParagraphRequest] | ASRValidationDocumentRequest = Body(...), session: Session = Depends(get_session), ) -> None: """ @@ -413,7 +419,7 @@ async def asr_save_document_validation( Args: document_id (UUID5): The ID of the document to validate. - annotations (list[ASRParagraphRequest]): The list of annotations for the document. + payload (list[ASRParagraphRequest] | ASRValidationDocumentRequest): Legacy annotation list or validation payload with title and document. session (Session, optional): The database session. Defaults to Depends(get_session). Raises: @@ -423,6 +429,13 @@ async def asr_save_document_validation( if not record: raise NotFoundError(detail=f"Document not found: {document_id}") + if isinstance(payload, ASRValidationDocumentRequest): + annotations = payload.document + if payload.title is not None: + record.name = payload.title + else: + annotations = payload + # NOTE: we are serializing the paragraphs to JSON for writing to the DB record.validation = [ # type: ignore ASRParagraph.model_validate(item).model_dump(mode="json") diff --git a/aymurai/meta/api_interfaces.py b/aymurai/meta/api_interfaces.py index f4e3954e..cec81034 100644 --- a/aymurai/meta/api_interfaces.py +++ b/aymurai/meta/api_interfaces.py @@ -156,6 +156,21 @@ class ASRParagraphRequest(BaseModel): text: str +class ASRValidationDocumentRequest(BaseModel): + title: str | None = None + document: list[ASRParagraphRequest] + + @field_validator("title") + @classmethod + def validate_title(cls, value: str | None) -> str | None: + if value is None: + return None + title = value.strip() + if not title: + raise ValueError("title cannot be empty") + return title + + class ASRSpeakerTurn(BaseModel): speaker: str speaker_no: int @@ -168,6 +183,7 @@ class ASRSpeakerTurn(BaseModel): class ASRDocument(BaseModel): document: list[ASRParagraph] document_id: UUID + title: str | None = None speaker_turns: list[ASRSpeakerTurn] = Field(default_factory=list) def to_txt(self) -> str: @@ -178,6 +194,7 @@ def from_transcription(cls, transcription: AudioTranscriptionRead) -> ASRDocumen return cls( document=transcription.validation or transcription.transcription, document_id=transcription.id, + title=transcription.name, ) diff --git a/test/api/endpoints/routers/asr/test_transcribe.py b/test/api/endpoints/routers/asr/test_transcribe.py index 2a64fb39..24a2ffa5 100644 --- a/test/api/endpoints/routers/asr/test_transcribe.py +++ b/test/api/endpoints/routers/asr/test_transcribe.py @@ -55,6 +55,7 @@ def test_should_transcribe_and_persist_document_when_service_returns_paragraphs( assert response.status_code == 200 payload = ASRDocument.model_validate(response.json()) assert str(payload.document_id) == str(document_id) + assert payload.title == "sample.wav" assert len(payload.document) == 1 assert payload.document[0].text == "Hola mundo" assert response.json()["speaker_turns"][0]["text"] == "Hola mundo" @@ -346,6 +347,103 @@ def test_should_persist_validation_annotations_when_posting_validation_for_exist assert first_item["text"] == "Linea validada" +def test_should_persist_validation_and_title_when_posting_validation_object( + asr_test_client, + make_wav_bytes, +): + client, engine = asr_test_client + audio_bytes = make_wav_bytes(freq_hz=331) + document_id = data_to_uuid(audio_bytes) + + with Session(engine) as session: + session.add( + AudioTranscription( + id=document_id, + name="old-title.wav", + transcription=cast( + Any, + [ + ASRParagraph( + speaker_no=1, + start=timedelta(seconds=0), + end=timedelta(seconds=1), + text="Base", + ).model_dump(mode="json") + ], + ), + validation=[], + ) + ) + session.commit() + + body = { + "title": " Audiencia editada ", + "document": [{"speaker_no": 1, "start": 0, "end": 1, "text": "Linea validada"}], + } + response = client.post(f"/asr/validation/document/{document_id}", json=body) + + assert response.status_code == 200 + with Session(engine) as session: + record = session.get(AudioTranscription, document_id) + assert record is not None + assert record.name == "Audiencia editada" + first_item = cast(dict[str, Any], record.validation[0]) + assert first_item["text"] == "Linea validada" + + get_response = client.get(f"/asr/validation/document/{document_id}") + assert get_response.status_code == 200 + payload = ASRDocument.model_validate(get_response.json()) + assert payload.title == "Audiencia editada" + + transcribe_response = client.post( + "/asr/transcribe", + files={"file": ("new-upload-name.wav", audio_bytes, "audio/wav")}, + ) + assert transcribe_response.status_code == 200 + transcribe_payload = ASRDocument.model_validate(transcribe_response.json()) + assert transcribe_payload.title == "Audiencia editada" + + +def test_should_reject_empty_title_when_posting_validation_object( + asr_test_client, + make_wav_bytes, +): + client, engine = asr_test_client + audio_bytes = make_wav_bytes(freq_hz=332) + document_id = data_to_uuid(audio_bytes) + + with Session(engine) as session: + session.add( + AudioTranscription( + id=document_id, + name="existing.wav", + transcription=cast( + Any, + [ + ASRParagraph( + speaker_no=1, + start=timedelta(seconds=0), + end=timedelta(seconds=1), + text="Base", + ).model_dump(mode="json") + ], + ), + validation=[], + ) + ) + session.commit() + + response = client.post( + f"/asr/validation/document/{document_id}", + json={ + "title": " ", + "document": [{"speaker_no": 1, "start": 0, "end": 1, "text": "Validada"}], + }, + ) + + assert response.status_code == 422 + + # MARK: POST Validation with speaker_name per paragraph def test_should_persist_speaker_name_on_paragraph_when_posting_validation( asr_test_client, From 5a52a99fdda1befd7c305e46c8eb83e637ce564a Mon Sep 17 00:00:00 2001 From: jansaldo Date: Wed, 1 Jul 2026 20:35:45 +0000 Subject: [PATCH 22/22] feat(asr): add title support to SSE messages in transcribe_stream --- aymurai/api/endpoints/routers/asr/transcribe.py | 5 +++++ test/api/endpoints/routers/asr/test_transcribe.py | 8 +++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/aymurai/api/endpoints/routers/asr/transcribe.py b/aymurai/api/endpoints/routers/asr/transcribe.py index af175ff0..97ce8b30 100644 --- a/aymurai/api/endpoints/routers/asr/transcribe.py +++ b/aymurai/api/endpoints/routers/asr/transcribe.py @@ -314,6 +314,9 @@ async def event_stream() -> AsyncIterator[str]: { "type": "meta", "document_id": str(document_id), + "title": cached_document.title + if cached_document is not None + else filename, "duration": duration, } ) @@ -322,6 +325,7 @@ async def event_stream() -> AsyncIterator[str]: yield _build_sse_message( { "type": "segments", + "title": cached_document.title, "document": [ paragraph.model_dump(mode="json") for paragraph in cached_document.document @@ -358,6 +362,7 @@ async def event_stream() -> AsyncIterator[str]: yield _build_sse_message( { "type": "segments", + "title": filename, "document": [ paragraph.model_dump(mode="json") for paragraph in paragraphs diff --git a/test/api/endpoints/routers/asr/test_transcribe.py b/test/api/endpoints/routers/asr/test_transcribe.py index 24a2ffa5..10d61b1c 100644 --- a/test/api/endpoints/routers/asr/test_transcribe.py +++ b/test/api/endpoints/routers/asr/test_transcribe.py @@ -785,7 +785,10 @@ def _fail(*args, **kwargs): parsed = _parse_sse(response.text) types = [event["type"] for event in parsed] assert types == ["meta", "segments", "done"] + meta_event = next(event for event in parsed if event["type"] == "meta") + assert meta_event["title"] == "cached.wav" segments_event = next(event for event in parsed if event["type"] == "segments") + assert segments_event["title"] == "cached.wav" assert segments_event["document"][0]["text"] == "Texto cacheado" assert segments_event["speaker_turns"][0]["text"] == "Texto cacheado" @@ -802,7 +805,7 @@ def test_stream_cache_should_emit_validation_turns_without_merging( session.add( AudioTranscription( id=document_id, - name="cached-validation.wav", + name="Audiencia editada", transcription=cast( Any, [ @@ -851,7 +854,10 @@ def _fail(*args, **kwargs): assert response.status_code == 200 parsed = _parse_sse(response.text) + meta_event = next(event for event in parsed if event["type"] == "meta") + assert meta_event["title"] == "Audiencia editada" segments_event = next(event for event in parsed if event["type"] == "segments") + assert segments_event["title"] == "Audiencia editada" assert [item["text"] for item in segments_event["document"]] == [ "Turno validado uno", "Turno validado dos",