diff --git a/README.md b/README.md index d140e5ad..45a34431 100644 --- a/README.md +++ b/README.md @@ -61,6 +61,34 @@ docker load -i aymurai-api.tar For more information on Docker deployment, refer to the [Docker documentation](https://docs.docker.com/). If you need further assistance, feel free to contact us at [aymurai@datagenero.org](mailto:aymurai@datagenero.org). +## Running the ASR server (coro) +Audio transcription is handled by [coro](https://github.com/collectiveai-team/coro), an OpenAI-compatible ASR + speaker-diarization server. It runs as an isolated host process via `uv tool` (no dependency conflicts with the API), and the API talks to it over HTTP/SSE. + +Prerequisites (both modes): `ffmpeg` on the host, [`uv`](https://docs.astral.sh/uv/) installed, and a real-disk `CORO_TRANSCRIPT_SPILL_DIR` (not a tmpfs path) so host RAM stays flat on long audio. + +```bash +# GPU (NVIDIA): parakeet fp32 + NeMo diarization +./scripts/run-coro.sh gpu + +# CPU: parakeet int8 +./scripts/run-coro.sh cpu +``` + +Then point the API at it via the `TRANSCRIBE_BASE_URL` environment variable (see `.env`): + +``` +TRANSCRIBE_BASE_URL=http://localhost:8000/v1 +``` + +The API connects over HTTP/SSE and is hardware-agnostic — coro's device is independent of the API container's `TORCH_DEVICE`. (`run-coro.sh` pins `uvx --python 3.12`, since NeMo/`kaldialign` lack wheels on newer Pythons.) + +To verify the whole path end-to-end (start coro, transcribe a sample over SSE via both the OpenAI SDK and aymurai's client): + +```bash +rtk uv run python scripts/smoke_coro.py cpu # or: gpu +``` + + ## Pipeline AymurAI’s backend utilizes a structured data processing pipeline to handle anonymized legal rulings and extract relevant information. This data is processed and made accessible via the API. For more details, please refer to the [pipeline documentation](docs/pipeline/README.md). diff --git a/aymurai/api/endpoints/routers/asr/transcribe.py b/aymurai/api/endpoints/routers/asr/transcribe.py index 53a29f78..97ce8b30 100644 --- a/aymurai/api/endpoints/routers/asr/transcribe.py +++ b/aymurai/api/endpoints/routers/asr/transcribe.py @@ -1,4 +1,10 @@ +import json +import math +from collections.abc import AsyncIterator +from typing import Any + from fastapi import Body, Depends, UploadFile +from fastapi.responses import StreamingResponse from fastapi.routing import APIRouter from pydantic import UUID5 from sqlmodel import Session @@ -9,7 +15,20 @@ NotFoundError, UpstreamServiceError, ) -from aymurai.audio.asr_client import transcribe_audio_bytes +from aymurai.api.meta.asr.coro import ( + CoroSegment, + CoroStreamDelta, + CoroStreamSegments, +) +from aymurai.audio.asr_client import ( + stream_transcribe_audio_bytes, + transcribe_audio_bytes, +) +from aymurai.audio.duration import probe_audio_duration +from aymurai.audio.transcript import ( + speaker_turns_from_validated_paragraphs, + transcript_turns, +) from aymurai.database.crud.audio_transcription import ( audio_transcription_create_or_update, audio_transcription_get, @@ -21,76 +40,185 @@ ASRDocument, ASRParagraph, ASRParagraphRequest, + ASRSpeakerTurn, + ASRValidationDocumentRequest, ) from aymurai.settings import settings router = APIRouter() logger = get_logger(__name__) +# Progress estimation tuning (see docs/superpowers/specs/2026-06-19-asr-streaming-progress-design.md). +# CHARS_PER_SEC biases the estimate: higher values make the bar lag and snap to +# 100% on completion rather than peg early. K = ln(10) puts the displayed value +# at ~90% when the char estimate thinks transcription is done (raw == 1.0). +ASR_PROGRESS_CHARS_PER_SEC = 17.0 +ASR_PROGRESS_K = math.log(10) + + +def _estimate_progress(cumulative_chars: int, duration_s: float | None) -> float | None: + """ + Estimate transcription progress with an asymptotic easing curve. + + Computes ``1 - exp(-k * raw)`` where ``raw`` is the ratio of characters seen + so far to the expected total (``duration_s * ASR_PROGRESS_CHARS_PER_SEC``). + The curve never reaches 1.0, so it decelerates near the top instead of + pinning to a hard cap; completion is signalled separately by the done event. + + Args: + cumulative_chars (int): Characters received from deltas so far. + duration_s (float | None): Audio duration in seconds, or None if unknown. + + Returns: + float | None: Progress in [0.0, 1.0), or None if duration is unknown. + """ + if not duration_s or duration_s <= 0: + return None + raw = cumulative_chars / (duration_s * ASR_PROGRESS_CHARS_PER_SEC) + value = 1.0 - math.exp(-ASR_PROGRESS_K * raw) + # Guard the float-underflow corner so deltas stay strictly below 1.0 + # (1.0 is reserved for the done event). This only corrects the artifact at + # extreme raw values; it is not a visible cap. + return min(value, math.nextafter(1.0, 0.0)) + + +def _segments_to_paragraphs(segments: list[CoroSegment]) -> list[ASRParagraph]: + """ + Map coro segments to ASRParagraph objects. + + Args: + segments (list[CoroSegment]): The coro speaker-attributed segments. + + Returns: + list[ASRParagraph]: The transcribed paragraphs. + """ + return [ + ASRParagraph( + speaker_no=int(segment.speaker), + start=segment.start, + end=segment.end, + text=segment.text, + ) + for segment in segments + ] + + +def _speaker_turns_for_transcription( + paragraphs: list[ASRParagraph], +) -> list[ASRSpeakerTurn]: + if not paragraphs: + return [] + return [ + ASRSpeakerTurn.model_validate(turn) + for turn in transcript_turns({"document": paragraphs}) + ] + + +def _speaker_turns_for_validation( + paragraphs: list[ASRParagraph], +) -> list[ASRSpeakerTurn]: + return [ + ASRSpeakerTurn.model_validate(turn) + for turn in speaker_turns_from_validated_paragraphs(paragraphs) + ] + + +def _asr_document_from_transcription( + document_id: UUID5, paragraphs: list[ASRParagraph], title: str | None = None +) -> ASRDocument: + return ASRDocument( + document_id=document_id, + title=title, + document=paragraphs, + speaker_turns=_speaker_turns_for_transcription(paragraphs), + ) + + +def _paragraphs_from_storage(items: list[Any]) -> list[ASRParagraph]: + return [ASRParagraph.model_validate(item) for item in items] + -def get_transcribe_ws_uri() -> str: +def _asr_document_from_cached_record(document_id: UUID5, record: Any) -> ASRDocument: + validation = record.validation or [] + if validation: + paragraphs = _paragraphs_from_storage(validation) + return ASRDocument( + document_id=document_id, + title=record.name, + document=paragraphs, + speaker_turns=_speaker_turns_for_validation(paragraphs), + ) + + paragraphs = _paragraphs_from_storage(record.transcription) + return _asr_document_from_transcription(document_id, paragraphs, title=record.name) + + +def _build_sse_message(payload: dict[str, Any]) -> str: """ - Get the WebSocket URI for the transcription service from settings. + Format a payload as an SSE data message. + + Args: + payload (dict[str, Any]): Dictionary to serialize into the SSE data field. + + Returns: + str: Serialized SSE message string. + """ + return f"data: {json.dumps(payload)}\n\n" + + +def get_transcribe_base_url() -> str: + """ + Get the coro base URL for the transcription service from settings. Raises: - ConfigurationError: If the WebSocket URI is not configured in settings. + ConfigurationError: If the base URL is not configured in settings. Returns: - str: The WebSocket URI for the transcription service. + str: The coro base URL (e.g. http://localhost:8000/v1). """ - ws_uri = settings.TRANSCRIBE_WS_URI - if not ws_uri: - raise ConfigurationError(detail="TRANSCRIBE_WS_URI is not configured") - return ws_uri + base_url = settings.TRANSCRIBE_BASE_URL + if not base_url: + raise ConfigurationError(detail="TRANSCRIBE_BASE_URL is not configured") + return base_url async def _transcribe_audio_bytes_with_error_handling( data: bytes, + filename: str, + content_type: str, ) -> list[ASRParagraph]: """ - Transcribes audio bytes into a list of ASRParagraph objects. + Transcribe audio bytes into a list of ASRParagraph objects via coro. Args: data (bytes): The audio data to be transcribed. + filename (str): The uploaded file name. + content_type (str): The uploaded file MIME type. Raises: - UpstreamServiceError: If there is an error with the upstream transcription service. + UpstreamServiceError: If the coro service errors or returns no result. AymuraiAPIException: If there is an unexpected error during transcription. Returns: - list[ASRParagraph]: A list of ASRParagraph objects representing the transcribed audio. + list[ASRParagraph]: The transcribed paragraphs. """ try: - status = await transcribe_audio_bytes(data) + segments = await transcribe_audio_bytes(data, filename, content_type) except RuntimeError as exc: - message = str(exc) - if "websocket" in message.lower(): - raise UpstreamServiceError(detail=message) from exc - raise AymuraiAPIException(detail=message) from exc + raise UpstreamServiceError(detail=str(exc)) from exc except Exception as exc: raise AymuraiAPIException( detail="Unexpected error during transcription" ) from exc - if not status: - raise AymuraiAPIException(detail="No transcription result received") - - return [ - ASRParagraph( - speaker_no=line.speaker, - start=line.start, - end=line.end, - text=line.text, - ) - for line in status.lines - ] + return _segments_to_paragraphs(segments) @router.post("/transcribe", response_model=ASRDocument) async def transcribe( file: UploadFile, use_cache: bool = True, - ws_uri: str = Depends(get_transcribe_ws_uri), + base_url: str = Depends(get_transcribe_base_url), session: Session = Depends(get_session), ) -> ASRDocument: """ @@ -99,7 +227,7 @@ async def transcribe( Args: file (UploadFile): The audio file to be transcribed. use_cache (bool, optional): Whether to use cached transcription results. Defaults to True. - ws_uri (str, optional): The WebSocket URI for the transcription service. Defaults to Depends(get_transcribe_ws_uri). + base_url (str, optional): The coro base URL for the transcription service. Defaults to Depends(get_transcribe_base_url). session (Session, optional): The database session. Defaults to Depends(get_session). Returns: @@ -113,26 +241,152 @@ async def transcribe( transcription_id=document_id, session=session ) if cached_record is not None: - logger.debug(f"Audio transcription DB hit for {file.filename}") - cached_document = ASRDocument( - document_id=document_id, - document=cached_record.validation or cached_record.transcription, - ) - return cached_document + logger.debug("Audio transcription DB hit for %s", file.filename) + return _asr_document_from_cached_record(document_id, cached_record) - transcription_items = await _transcribe_audio_bytes_with_error_handling(data) - document = ASRDocument(document_id=document_id, document=transcription_items) + transcription_items = await _transcribe_audio_bytes_with_error_handling( + data, + file.filename or str(document_id), + file.content_type or "application/octet-stream", + ) + title = file.filename or str(document_id) + document = _asr_document_from_transcription( + document_id, transcription_items, title=title + ) audio_transcription_create_or_update( transcription_id=document_id, - name=file.filename or str(document_id), + name=title, transcription=document.document, session=session, ) - logger.debug(f"Audio transcription stored in DB for {file.filename}") + logger.debug("Audio transcription stored in DB for %s", file.filename) return document +@router.post("/transcribe/stream") +async def transcribe_stream( + file: UploadFile, + use_cache: bool = True, + base_url: str = Depends(get_transcribe_base_url), + session: Session = Depends(get_session), +) -> StreamingResponse: + """ + Transcribe an uploaded audio file and stream progress via Server-Sent Events. + + Emits ``data: {json}`` SSE messages with a ``type`` discriminator: + - meta: ``{document_id, duration}`` (duration is null if unknown) + - delta: ``{text, progress?}`` incremental text (progress omitted if no duration) + - segments: ``{document}`` the final paragraph list + - done: ``{progress: 1.0}`` terminal success + - error: ``{detail}`` upstream/internal failure + + On a cache hit it emits meta, segments, done without calling coro. + + Args: + file (UploadFile): The audio file to be transcribed. + use_cache (bool, optional): Whether to use cached results. Defaults to True. + base_url (str, optional): The coro base URL. Defaults to Depends(get_transcribe_base_url). + session (Session, optional): The database session. Defaults to Depends(get_session). + + Returns: + StreamingResponse: A text/event-stream of transcription progress events. + """ + data = await file.read() + document_id = data_to_uuid(data) + filename = file.filename or str(document_id) + content_type = file.content_type or "application/octet-stream" + + cached_document: ASRDocument | None = None + if use_cache: + cached_record = audio_transcription_get( + transcription_id=document_id, session=session + ) + if cached_record is not None: + cached_document = _asr_document_from_cached_record( + document_id, cached_record + ) + + duration = probe_audio_duration(data) + + async def event_stream() -> AsyncIterator[str]: + yield _build_sse_message( + { + "type": "meta", + "document_id": str(document_id), + "title": cached_document.title + if cached_document is not None + else filename, + "duration": duration, + } + ) + + if cached_document is not None: + yield _build_sse_message( + { + "type": "segments", + "title": cached_document.title, + "document": [ + paragraph.model_dump(mode="json") + for paragraph in cached_document.document + ], + "speaker_turns": [ + turn.model_dump(mode="json") + for turn in cached_document.speaker_turns + ], + } + ) + yield _build_sse_message({"type": "done", "progress": 1.0}) + return + + cumulative_chars = 0 + try: + async for event in stream_transcribe_audio_bytes( + data, filename, content_type + ): + if isinstance(event, CoroStreamDelta): + cumulative_chars += len(event.text) + payload: dict[str, Any] = {"type": "delta", "text": event.text} + progress = _estimate_progress(cumulative_chars, duration) + if progress is not None: + payload["progress"] = progress + yield _build_sse_message(payload) + elif isinstance(event, CoroStreamSegments): + paragraphs = _segments_to_paragraphs(event.segments) + audio_transcription_create_or_update( + transcription_id=document_id, + name=filename, + transcription=paragraphs, + session=session, + ) + yield _build_sse_message( + { + "type": "segments", + "title": filename, + "document": [ + paragraph.model_dump(mode="json") + for paragraph in paragraphs + ], + "speaker_turns": [ + turn.model_dump(mode="json") + for turn in _speaker_turns_for_transcription(paragraphs) + ], + } + ) + except RuntimeError as exc: + logger.error("asr stream error: %s", exc) + yield _build_sse_message({"type": "error", "detail": str(exc)}) + return + + yield _build_sse_message({"type": "done", "progress": 1.0}) + + return StreamingResponse( + event_stream(), + media_type="text/event-stream", + headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"}, + ) + + @router.get("/validation/document/{document_id}") async def asr_read_document_validation( document_id: UUID5, @@ -156,16 +410,13 @@ async def asr_read_document_validation( if not record: raise NotFoundError(detail=f"Document not found: {document_id}") - return ASRDocument( - document_id=document_id, - document=record.validation or record.transcription, - ) + return _asr_document_from_cached_record(document_id, record) @router.post("/validation/document/{document_id}") async def asr_save_document_validation( document_id: UUID5, - annotations: list[ASRParagraphRequest] = Body(...), + payload: list[ASRParagraphRequest] | ASRValidationDocumentRequest = Body(...), session: Session = Depends(get_session), ) -> None: """ @@ -173,7 +424,7 @@ async def asr_save_document_validation( Args: document_id (UUID5): The ID of the document to validate. - annotations (list[ASRParagraphRequest]): The list of annotations for the document. + payload (list[ASRParagraphRequest] | ASRValidationDocumentRequest): Legacy annotation list or validation payload with title and document. session (Session, optional): The database session. Defaults to Depends(get_session). Raises: @@ -183,6 +434,13 @@ async def asr_save_document_validation( if not record: raise NotFoundError(detail=f"Document not found: {document_id}") + if isinstance(payload, ASRValidationDocumentRequest): + annotations = payload.document + if payload.title is not None: + record.name = payload.title + else: + annotations = payload + # NOTE: we are serializing the paragraphs to JSON for writing to the DB record.validation = [ # type: ignore ASRParagraph.model_validate(item).model_dump(mode="json") diff --git a/aymurai/api/meta/asr/coro.py b/aymurai/api/meta/asr/coro.py new file mode 100644 index 00000000..f2326855 --- /dev/null +++ b/aymurai/api/meta/asr/coro.py @@ -0,0 +1,77 @@ +import re +from datetime import timedelta + +from pydantic import BaseModel, ConfigDict + +ISO8601_DURATION_RE = re.compile( + r"^PT(?:(?P\d+(?:\.\d+)?)H)?" + r"(?:(?P\d+(?:\.\d+)?)M)?" + r"(?:(?P\d+(?:\.\d+)?)S)?$" +) + + +def _parse_hhmmss(value: str | int | float | timedelta) -> timedelta: + """ + Parse a time value in HH:MM:SS format, ISO 8601 duration (PT#H#M#S), or seconds. + + Args: + value (str | int | float | timedelta): The time value to parse. + + Raises: + ValueError: If the value is not a valid time format. + + Returns: + timedelta: The parsed time value. + """ + if isinstance(value, timedelta): + return value + + if isinstance(value, (int, float)): + return timedelta(seconds=float(value)) + + if not isinstance(value, str): + raise ValueError("Invalid time format") + + if value.startswith("PT"): + match = ISO8601_DURATION_RE.match(value) + if match is None: + raise ValueError("Expected ISO 8601 duration format (PT#H#M#S)") + hours = float(match.group("hours") or 0) + minutes = float(match.group("minutes") or 0) + seconds = float(match.group("seconds") or 0) + return timedelta(hours=hours, minutes=minutes, seconds=seconds) + + parts = value.split(":") + if len(parts) != 3: + raise ValueError("Expected HH:MM:SS format") + + hours_str, minutes_str, seconds_str = parts + return timedelta( + hours=int(hours_str), minutes=int(minutes_str), seconds=float(seconds_str) + ) + + +class CoroSegment(BaseModel): + """A speaker-attributed transcription segment from the coro done-frame.""" + + model_config = ConfigDict(extra="ignore") + + start: float + end: float + text: str + speaker: str + + +class CoroStreamDelta(BaseModel): + """An incremental transcription text fragment from a coro delta event.""" + + text: str + + +class CoroStreamSegments(BaseModel): + """The final speaker-attributed segments parsed from the coro done frame.""" + + segments: list[CoroSegment] + + +CoroStreamEvent = CoroStreamDelta | CoroStreamSegments diff --git a/aymurai/api/meta/asr/websocket.py b/aymurai/api/meta/asr/websocket.py deleted file mode 100644 index c1e4a46e..00000000 --- a/aymurai/api/meta/asr/websocket.py +++ /dev/null @@ -1,115 +0,0 @@ -import re -from datetime import timedelta -from typing import ClassVar, Literal - -from pydantic import BaseModel, ConfigDict, field_validator - -ISO8601_DURATION_RE = re.compile( - r"^PT(?:(?P\d+(?:\.\d+)?)H)?" - r"(?:(?P\d+(?:\.\d+)?)M)?" - r"(?:(?P\d+(?:\.\d+)?)S)?$" -) - - -def _parse_hhmmss(value: str | int | float | timedelta) -> timedelta: - """ - Parse a time value in HH:MM:SS format, ISO 8601 duration format (PT#H#M#S), or as a number of seconds. - - Args: - value (str | int | float | timedelta): The time value to parse. - - Raises: - ValueError: If the time value is not a valid format, or if the ISO 8601 duration format is invalid. - - Returns: - timedelta: The parsed time value as a timedelta object. - """ - if isinstance(value, timedelta): - return value - - if isinstance(value, (int, float)): - return timedelta(seconds=float(value)) - - if not isinstance(value, str): - raise ValueError("Invalid time format") - - if value.startswith("PT"): - match = ISO8601_DURATION_RE.match(value) - if match is None: - raise ValueError("Expected ISO 8601 duration format (PT#H#M#S)") - - hours = float(match.group("hours") or 0) - minutes = float(match.group("minutes") or 0) - seconds = float(match.group("seconds") or 0) - - return timedelta(hours=hours, minutes=minutes, seconds=seconds) - - parts = value.split(":") - if len(parts) != 3: - raise ValueError("Expected HH:MM:SS format") - - hours_str, minutes_str, seconds_str = parts - hours = int(hours_str) - minutes = int(minutes_str) - seconds = float(seconds_str) - return timedelta(hours=hours, minutes=minutes, seconds=seconds) - - -class WLKMessageConfig(BaseModel): - type: Literal["config"] - useAudioWorklet: bool - - -class WLKMessageTranscriptionLine(BaseModel): - speaker: int - text: str - start: timedelta - end: timedelta - detected_language: str | None = None - - @field_validator("text", mode="before") - @classmethod - def normalize_text(cls, value: str | None) -> str: - if value is None: - return "" - return value - - @field_validator("start", "end", mode="before") - @classmethod - def parse_hhmmss(cls, value: str | int | float | timedelta) -> timedelta: - return _parse_hhmmss(value) - - -class WLKMessageStatus(BaseModel): - status: str - lines: list[WLKMessageTranscriptionLine] - buffer_transcription: str - buffer_diarization: str - buffer_translation: str - remaining_time_transcription: float - remaining_time_diarization: float - speaker_ids: dict[str, str] | None = None - - -class WLKMessageReadyToStopMessage(BaseModel): - type: Literal["ready_to_stop"] - - -WLKMessageRawResponse = ( - WLKMessageConfig | WLKMessageStatus | WLKMessageReadyToStopMessage -) - - -class TranscriptionItem(BaseModel): - model_config: ClassVar[ConfigDict] = ConfigDict(from_attributes=True) - - speaker_no: int - speaker_name: str | None = None - start: timedelta - end: timedelta - text: str - - @field_validator("start", "end", mode="before") - @classmethod - def parse_hhmmss(cls, value: str | int | float | timedelta) -> timedelta: - return _parse_hhmmss(value) diff --git a/aymurai/audio/asr_client.py b/aymurai/audio/asr_client.py index e045fb14..53823134 100644 --- a/aymurai/audio/asr_client.py +++ b/aymurai/audio/asr_client.py @@ -1,186 +1,129 @@ -import asyncio -import contextlib -import io import json -from pathlib import Path +from collections.abc import AsyncGenerator -import librosa -import numpy as np -import websockets -from pydantic import TypeAdapter, ValidationError +from openai import APIConnectionError, APIError, AsyncOpenAI -from aymurai.api.meta.asr.websocket import ( - WLKMessageRawResponse, - WLKMessageReadyToStopMessage, - WLKMessageStatus, +from aymurai.api.meta.asr.coro import ( + CoroSegment, + CoroStreamDelta, + CoroStreamEvent, + CoroStreamSegments, ) from aymurai.logger import get_logger from aymurai.settings import settings logger = get_logger(__name__) -SAMPLE_RATE_HZ = 16000 -CHUNK_SECONDS = 1 -CHUNK_SAMPLES = SAMPLE_RATE_HZ * CHUNK_SECONDS -MAX_WS_LOG_CHARS = 2000 -ASR_RAW_RESPONSE_ADAPTER = TypeAdapter(WLKMessageRawResponse) +DONE_EVENT_TYPE = "transcript.text.done" +DELTA_EVENT_TYPE = "transcript.text.delta" -async def _stream_audio_bytes( +async def transcribe_audio_bytes( payload: bytes, - websocket: websockets.ClientConnection, -) -> int: + filename: str, + content_type: str, +) -> list[CoroSegment]: """ - Streams audio bytes to a WebSocket connection in chunks. + Transcribe audio by streaming it to the coro server over SSE. - Args: - payload (bytes): The audio data to be streamed. - websocket (websockets.ClientConnection): The WebSocket connection to stream the audio data to. - - Returns: - int: The total number of bytes sent to the WebSocket. - """ - audio, _ = librosa.load(io.BytesIO(payload), sr=SAMPLE_RATE_HZ, mono=True) - total_bytes = 0 - for i in range(0, len(audio), CHUNK_SAMPLES): - chunk = audio[i : i + CHUNK_SAMPLES] - if len(chunk) == 0: - continue - chunk_int16 = (chunk * 32768).astype(np.int16) - data = chunk_int16.tobytes() - total_bytes += len(data) - await websocket.send(data) - return total_bytes - - -def _parse_ws_message(message: str | bytes) -> WLKMessageRawResponse | None: - """ - Parses a WebSocket message into a WLKMessageRawResponse object. + Sends the audio to coro's OpenAI-compatible ``/v1/audio/transcriptions`` + endpoint with ``stream=True`` and parses the ``transcript.text.done`` frame. Args: - message (str | bytes): The WebSocket message to be parsed. + payload (bytes): The raw audio file bytes (any ffmpeg-decodable format). + filename (str): The uploaded file name (used for the multipart part). + content_type (str): The uploaded file MIME type. + + Raises: + RuntimeError: If the base URL is unset, the service errors, or no done + frame is received. Returns: - WLKMessageRawResponse | None: The parsed WLKMessageRawResponse object, or None if parsing fails. + list[CoroSegment]: The speaker-attributed transcription segments. """ - if isinstance(message, bytes): - message = message.decode("utf-8", errors="replace") - - payload_preview = message - if len(payload_preview) > MAX_WS_LOG_CHARS: - payload_preview = ( - f"{payload_preview[:MAX_WS_LOG_CHARS]}" - f"...[truncated {len(payload_preview) - MAX_WS_LOG_CHARS} chars]" - ) + base_url = settings.TRANSCRIBE_BASE_URL + if not base_url: + raise RuntimeError("TRANSCRIBE_BASE_URL is not configured") - try: - parsed = json.loads(message) - except json.JSONDecodeError: - logger.warning("received non-json websocket payload: %s", payload_preview) - return None + client = AsyncOpenAI(base_url=base_url, api_key=settings.TRANSCRIBE_API_KEY) + logger.info("streaming audio to coro for transcription") try: - return ASR_RAW_RESPONSE_ADAPTER.validate_python(parsed) - except ValidationError as exc: - logger.warning( - "received unrecognized websocket payload: %s; payload=%s", - exc, - payload_preview, + stream = await client.audio.transcriptions.create( + file=(filename, payload, content_type), + model="whisper-1", + language="es", + stream=True, + response_format="diarized_json", ) - return None - + async for event in stream: + if event.type == DONE_EVENT_TYPE: + data = json.loads(event.text) + return [ + CoroSegment.model_validate(segment) + for segment in data.get("segments", []) + ] + except (APIError, APIConnectionError) as exc: + logger.error("coro transcription error: %s", exc) + raise RuntimeError("Transcription service error") from exc -async def _receive_updates( - websocket: websockets.ClientConnection, -) -> WLKMessageStatus | None: - """ - Receives updates from the WebSocket connection and returns the last active transcription status. + raise RuntimeError("coro stream ended without a done frame") - Args: - websocket (websockets.ClientConnection): The WebSocket connection to receive updates from. - Returns: - WLKMessageStatus | None: The last active transcription status, - or None if no active transcription was received. - """ - last_active_transcription: WLKMessageStatus | None = None - while True: - try: - msg = await websocket.recv() - except websockets.exceptions.ConnectionClosedOK: - logger.info("connection closed normally") - break - except websockets.exceptions.WebSocketException as exc: - logger.error("websocket error while receiving updates: %s", exc) - break - - parsed = _parse_ws_message(msg) - match parsed: - case None: - continue - case WLKMessageStatus(status="active_transcription") as message: - last_active_transcription = message - case WLKMessageReadyToStopMessage(): - break - - return last_active_transcription - - -async def transcribe_audio_bytes(payload: bytes) -> WLKMessageStatus | None: +async def stream_transcribe_audio_bytes( + payload: bytes, + filename: str, + content_type: str, +) -> AsyncGenerator[CoroStreamEvent, None]: """ - Transcribes audio bytes by streaming them to a WebSocket transcription service and receiving updates. + Stream a transcription from coro, yielding deltas then the final segments. + + Sends the audio to coro's OpenAI-compatible ``/v1/audio/transcriptions`` + endpoint with ``stream=True``. Yields one ``CoroStreamDelta`` per + ``transcript.text.delta`` event as text arrives, then a single + ``CoroStreamSegments`` parsed from the ``transcript.text.done`` frame. Args: - payload (bytes): The audio data to be transcribed. + payload (bytes): The raw audio file bytes (any ffmpeg-decodable format). + filename (str): The uploaded file name (used for the multipart part). + content_type (str): The uploaded file MIME type. Raises: - RuntimeError: If there is an error with the transcription service. + RuntimeError: If the base URL is unset, the service errors, or no done + frame is received. - Returns: - WLKMessageStatus | None: The last active transcription status received from the transcription service, - or None if no active transcription was received. + Yields: + CoroStreamEvent: Incremental deltas followed by the final segments. """ - ws_uri = settings.TRANSCRIBE_WS_URI - - if not ws_uri: - raise RuntimeError("TRANSCRIBE_WS_URI is not configured") + base_url = settings.TRANSCRIBE_BASE_URL + if not base_url: + raise RuntimeError("TRANSCRIBE_BASE_URL is not configured") - logger.info("streaming audio for transcription") + client = AsyncOpenAI(base_url=base_url, api_key=settings.TRANSCRIBE_API_KEY) + logger.info("streaming audio to coro for transcription (sse)") try: - async with websockets.connect(ws_uri) as websocket: - receive_task = asyncio.create_task(_receive_updates(websocket)) - try: - total_bytes = await _stream_audio_bytes(payload, websocket) - await websocket.send(b"") - logger.info("sent %s bytes to transcription service", total_bytes) - last_active_transcription = await receive_task - except Exception: - if not receive_task.done(): - receive_task.cancel() - with contextlib.suppress(asyncio.CancelledError): - await receive_task - raise - except websockets.exceptions.WebSocketException as exc: - logger.error("websocket error during transcription: %s", exc) - raise RuntimeError("Transcription service websocket error") from exc - except Exception as exc: - logger.error("unexpected error during transcription: %s", exc) - raise RuntimeError("Unexpected error during transcription") from exc - - return last_active_transcription - - -def transcribe_audio_path(path: Path) -> WLKMessageStatus | None: - """ - Transcribes an audio file at the given path by reading its bytes and sending them to the transcription service. - - Args: - path (Path): The path to the audio file to be transcribed. - - Returns: - WLKMessageStatus | None: The last active transcription status received from the transcription service, - or None if no active transcription was received. - """ - payload = path.read_bytes() - return asyncio.run(transcribe_audio_bytes(payload)) + stream = await client.audio.transcriptions.create( + file=(filename, payload, content_type), + model="whisper-1", + language="es", + stream=True, + response_format="diarized_json", + ) + async for event in stream: + if event.type == DELTA_EVENT_TYPE: + yield CoroStreamDelta(text=event.delta or "") + elif event.type == DONE_EVENT_TYPE: + data = json.loads(event.text) + yield CoroStreamSegments( + segments=[ + CoroSegment.model_validate(segment) + for segment in data.get("segments", []) + ] + ) + return + except (APIError, APIConnectionError) as exc: + logger.error("coro transcription error: %s", exc) + raise RuntimeError("Transcription service error") from exc + + raise RuntimeError("coro stream ended without a done frame") diff --git a/aymurai/audio/duration.py b/aymurai/audio/duration.py new file mode 100644 index 00000000..4e8f04c0 --- /dev/null +++ b/aymurai/audio/duration.py @@ -0,0 +1,59 @@ +import subprocess +import tempfile + +from aymurai.logger import get_logger + +logger = get_logger(__name__) + +_FFPROBE_ARGS = [ + "ffprobe", + "-v", + "error", + "-show_entries", + "format=duration", + "-of", + "default=noprint_wrappers=1:nokey=1", +] + +_FFPROBE_TIMEOUT_SECONDS = 30 + + +def probe_audio_duration(payload: bytes) -> float | None: + """ + Probe the duration of an audio payload in seconds using ffprobe. + + The bytes are written to a temporary file because ffprobe cannot read the + duration of container formats (e.g. WAV) from a non-seekable stdin pipe. Any + failure (missing ffprobe, undecodable input, timeout, non-numeric output) is + treated as "unknown" and returns None so callers can degrade gracefully. + + Args: + payload (bytes): The raw audio file bytes. + + Returns: + float | None: The duration in seconds, or None if it cannot be determined. + """ + if not payload: + return None + + try: + with tempfile.NamedTemporaryFile(suffix=".audio") as handle: + handle.write(payload) + handle.flush() + result = subprocess.run( + [*_FFPROBE_ARGS, handle.name], + capture_output=True, + timeout=_FFPROBE_TIMEOUT_SECONDS, + check=True, + ) + except (subprocess.SubprocessError, OSError) as exc: + logger.warning("ffprobe duration probe failed: %s", exc) + return None + + raw = result.stdout.decode("utf-8", errors="ignore").strip() + try: + duration = float(raw) + except ValueError: + return None + + return duration if duration > 0 else None diff --git a/aymurai/audio/transcript.py b/aymurai/audio/transcript.py new file mode 100644 index 00000000..5e13a7a5 --- /dev/null +++ b/aymurai/audio/transcript.py @@ -0,0 +1,239 @@ +import re +import warnings +from datetime import timedelta +from typing import Any + + +SENTENCE_END_RE = re.compile(r"(?:[.!?]+|[.!?]+[\"')\]]+)\s*$") + + +def parse_time_seconds(value: str | int | float | timedelta) -> float: + if isinstance(value, timedelta): + return value.total_seconds() + if isinstance(value, (int, float)): + return float(value) + + text = str(value).strip() + + iso_match = re.fullmatch( + r"PT(?:(\d+(?:\.\d+)?)H)?(?:(\d+(?:\.\d+)?)M)?(?:(\d+(?:\.\d+)?)S)?", + text, + ) + if iso_match: + hours = float(iso_match.group(1) or 0) + minutes = float(iso_match.group(2) or 0) + seconds = float(iso_match.group(3) or 0) + return hours * 3600 + minutes * 60 + seconds + + hhmmss = text.split(":") + if len(hhmmss) == 3: + hours, minutes, seconds = hhmmss + return int(hours) * 3600 + int(minutes) * 60 + float(seconds) + + return float(text) + + +def format_timestamp(seconds: float) -> str: + normalized = max(0.0, seconds) + whole_seconds = int(normalized) + millis = int(round((normalized - whole_seconds) * 1000)) + if millis == 1000: + whole_seconds += 1 + millis = 0 + hours, remainder = divmod(whole_seconds, 3600) + minutes, secs = divmod(remainder, 60) + return f"{hours:02d}:{minutes:02d}:{secs:02d}.{millis:03d}" + + +def _segment_value(segment: Any, field: str) -> Any: + if isinstance(segment, dict): + return segment.get(field) + return getattr(segment, field) + + +def ends_sentence(text: str) -> bool: + return bool(SENTENCE_END_RE.search(text.strip())) + + +def speaker_label(segment: dict[str, Any]) -> str: + name = str(segment.get("speaker_name") or "").strip() + if name: + return name + return f"Speaker {segment['speaker_no']}" + + +def diarized_segments(payload: dict[str, Any]) -> list[dict[str, Any]]: + """Read diarized ASR paragraphs from a payload with a document field.""" + document = payload.get("document") + if not isinstance(document, list) or not document: + raise ValueError("Diarized response is missing a non-empty 'document' list") + + segments: list[dict[str, Any]] = [] + for index, item in enumerate(document): + missing = [ + field + for field in ("speaker_no", "start", "end", "text") + if _segment_value(item, field) is None + ] + if missing: + raise ValueError( + f"document[{index}] is missing diarization field(s): {missing}" + ) + + speaker_no = _segment_value(item, "speaker_no") + if str(speaker_no).strip() == "": + raise ValueError(f"document[{index}] has an empty speaker label") + + text = str(_segment_value(item, "text") or "").strip() + if not text: + continue + + start = parse_time_seconds(_segment_value(item, "start")) + end = parse_time_seconds(_segment_value(item, "end")) + if start < 0 or end < start: + raise ValueError( + f"document[{index}] has invalid timestamps: start={start}, end={end}" + ) + + if isinstance(item, dict): + segment = dict(item) + else: + segment = item.model_dump(mode="json") + segment["_start_seconds"] = start + segment["_end_seconds"] = end + segment["_text"] = text + segments.append(segment) + + if not segments: + raise ValueError("Diarized response contains no non-empty speech segments") + + ordered = sorted( + segments, key=lambda item: (item["_start_seconds"], item["_end_seconds"]) + ) + if [id(item) for item in ordered] != [id(item) for item in segments]: + warnings.warn( + "Diarized segments were not ordered by timestamp; transcript will be timestamp-sorted", + stacklevel=2, + ) + + return ordered + + +def _new_turn(segment: dict[str, Any]) -> dict[str, Any]: + return { + "speaker": speaker_label(segment), + "speaker_no": segment["speaker_no"], + "start": segment["_start_seconds"], + "end": segment["_end_seconds"], + "texts": [segment["_text"]], + "segments": [segment], + } + + +def merge_speaker_turns_by_sentence( + segments: list[dict[str, Any]], +) -> list[dict[str, Any]]: + turns: list[dict[str, Any]] = [] + for segment in segments: + label = speaker_label(segment) + previous = turns[-1] if turns else None + starts_new_turn = ( + previous is None + or previous["speaker"] != label + or ends_sentence(" ".join(previous["texts"])) + ) + + if starts_new_turn: + turns.append(_new_turn(segment)) + continue + + previous["end"] = segment["_end_seconds"] + previous["texts"].append(segment["_text"]) + previous["segments"].append(segment) + + for turn in turns: + turn["text"] = " ".join(turn["texts"]).strip() + return turns + + +def transcript_turns(payload: dict[str, Any]) -> list[dict[str, Any]]: + turns = merge_speaker_turns_by_sentence(diarized_segments(payload)) + return serialize_speaker_turns(turns) + + +def serialize_speaker_turns(turns: list[dict[str, Any]]) -> list[dict[str, Any]]: + serialized: list[dict[str, Any]] = [] + for turn in turns: + serialized.append( + { + "speaker": turn["speaker"], + "speaker_no": turn["speaker_no"], + "start": format_timestamp(turn["start"]), + "end": format_timestamp(turn["end"]), + "text": turn.get("text") or " ".join(turn["texts"]).strip(), + "segments": [ + { + key: value + for key, value in segment.items() + if not key.startswith("_") + } + for segment in turn["segments"] + ], + } + ) + return serialized + + +def speaker_turns_from_validated_paragraphs( + paragraphs: list[Any], +) -> list[dict[str, Any]]: + turns: list[dict[str, Any]] = [] + for paragraph in paragraphs: + segment = ( + paragraph + if isinstance(paragraph, dict) + else paragraph.model_dump(mode="json") + ) + text = str(segment.get("text") or "").strip() + if not text: + continue + + start = parse_time_seconds(segment["start"]) + end = parse_time_seconds(segment["end"]) + turn_segment = dict(segment) + turn_segment["text"] = text + turns.append( + { + "speaker": speaker_label(turn_segment), + "speaker_no": turn_segment["speaker_no"], + "start": format_timestamp(start), + "end": format_timestamp(end), + "text": text, + "segments": [turn_segment], + } + ) + return turns + + +def markdown_transcript(payload: dict[str, Any], title: str) -> str: + segments = diarized_segments(payload) + turns = merge_speaker_turns_by_sentence(segments) + + distinct_speakers = {turn["speaker"] for turn in turns} + if len(distinct_speakers) < 2: + warnings.warn( + f"Only one speaker label found for {title}; verify the source audio is expected to be single-speaker", + stacklevel=2, + ) + + lines = [f"# {title}", ""] + for turn in turns: + timestamp = ( + f"{format_timestamp(turn['start'])} - {format_timestamp(turn['end'])}" + ) + lines.append(f"## {turn['speaker']} ({timestamp})") + lines.append("") + lines.append(turn.get("text") or " ".join(turn["texts"]).strip()) + lines.append("") + + return "\n".join(lines).strip() + "\n" diff --git a/aymurai/meta/api_interfaces.py b/aymurai/meta/api_interfaces.py index a576df9a..cec81034 100644 --- a/aymurai/meta/api_interfaces.py +++ b/aymurai/meta/api_interfaces.py @@ -3,11 +3,20 @@ import uuid from datetime import timedelta from functools import cached_property -from typing import TYPE_CHECKING, Literal - -from pydantic import UUID4, UUID5, BaseModel, Field, RootModel, computed_field - -from aymurai.api.meta.asr.websocket import TranscriptionItem +from typing import TYPE_CHECKING, ClassVar, Literal + +from pydantic import ( + UUID4, + UUID5, + BaseModel, + ConfigDict, + Field, + RootModel, + computed_field, + field_validator, +) + +from aymurai.api.meta.asr.coro import _parse_hhmmss from aymurai.database.utils import text_to_uuid from aymurai.meta.entities import EntityAttributes @@ -90,7 +99,22 @@ class Document(BaseModel): footer: list[str] | None = None -class ASRParagraph(TranscriptionItem): +class ASRParagraph(BaseModel): + """A speaker-attributed transcription paragraph (persisted and anonymized).""" + + model_config: ClassVar[ConfigDict] = ConfigDict(from_attributes=True) + + speaker_no: int + speaker_name: str | None = None + start: timedelta + end: timedelta + text: str + + @field_validator("start", "end", mode="before") + @classmethod + def parse_hhmmss(cls, value: str | int | float | timedelta) -> timedelta: + return _parse_hhmmss(value) + @computed_field @property def paragraph_id(self) -> UUID: @@ -132,9 +156,35 @@ class ASRParagraphRequest(BaseModel): text: str +class ASRValidationDocumentRequest(BaseModel): + title: str | None = None + document: list[ASRParagraphRequest] + + @field_validator("title") + @classmethod + def validate_title(cls, value: str | None) -> str | None: + if value is None: + return None + title = value.strip() + if not title: + raise ValueError("title cannot be empty") + return title + + +class ASRSpeakerTurn(BaseModel): + speaker: str + speaker_no: int + start: str + end: str + text: str + segments: list[ASRParagraph] + + class ASRDocument(BaseModel): document: list[ASRParagraph] document_id: UUID + title: str | None = None + speaker_turns: list[ASRSpeakerTurn] = Field(default_factory=list) def to_txt(self) -> str: return "\n\n".join([paragraph.to_txt() for paragraph in self.document]) @@ -144,6 +194,7 @@ def from_transcription(cls, transcription: AudioTranscriptionRead) -> ASRDocumen return cls( document=transcription.validation or transcription.transcription, document_id=transcription.id, + title=transcription.name, ) diff --git a/aymurai/settings.py b/aymurai/settings.py index e141afdc..5a97cff6 100644 --- a/aymurai/settings.py +++ b/aymurai/settings.py @@ -65,9 +65,10 @@ def assemble_cors_origins(cls, v) -> list[str]: SQLALCHEMY_DATABASE_URI: str = "sqlite:////resources/cache/sqlite/database.db" ########################################################################## - # ASR Config + # ASR Config (coro — OpenAI-compatible server) ########################################################################## - TRANSCRIBE_WS_URI: str | None = None + TRANSCRIBE_BASE_URL: str | None = None + TRANSCRIBE_API_KEY: str = "not-needed" ########################################################################## # Disambiguation Config diff --git a/docker-compose.yml b/docker-compose.yml index eb8d8fe0..92a247cc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -48,7 +48,10 @@ services: [ "/bin/sh", "-c", - "pip install --no-cache-dir psycopg2-binary boto3 && mlflow server --host 0.0.0.0 --port 5000 --backend-store-uri postgresql://mlflow:mlflow@postgres:5432/mlflow --default-artifact-root ${MLFLOW_ARTIFACT_ROOT}", + "pip install --no-cache-dir psycopg2-binary boto3 && mlflow + server --host 0.0.0.0 --port 5000 --backend-store-uri + postgresql://mlflow:mlflow@postgres:5432/mlflow + --default-artifact-root ${MLFLOW_ARTIFACT_ROOT}", ] depends_on: postgres: @@ -61,7 +64,8 @@ services: "CMD", "python", "-c", - "import urllib.request; urllib.request.urlopen('http://localhost:5000')", + "import urllib.request; + urllib.request.urlopen('http://localhost:5000')", ] interval: 10s timeout: 5s @@ -79,7 +83,7 @@ services: volumes: - postgres-data:/var/lib/postgresql/data healthcheck: - test: ["CMD-SHELL", "pg_isready -U mlflow"] + test: [ "CMD-SHELL", "pg_isready -U mlflow" ] interval: 10s timeout: 5s retries: 6 @@ -93,7 +97,7 @@ services: environment: MINIO_ROOT_USER: ${AWS_ACCESS_KEY_ID} MINIO_ROOT_PASSWORD: ${AWS_SECRET_ACCESS_KEY} - command: ["server", "/data", "--console-address", ":9001"] + command: [ "server", "/data", "--console-address", ":9001" ] volumes: - minio-data:/data # Healthcheck removed; minio image lacks wget and minio-mc already waits for readiness. @@ -108,7 +112,10 @@ services: [ "/bin/sh", "-c", - "until (/usr/bin/mc alias set local http://minio:9000 ${AWS_ACCESS_KEY_ID} ${AWS_SECRET_ACCESS_KEY}) do sleep 1; done; /usr/bin/mc mb -p local/mlflow; /usr/bin/mc anonymous set none local/mlflow; exit 0;", + "until (/usr/bin/mc alias set local http://minio:9000 + ${AWS_ACCESS_KEY_ID} ${AWS_SECRET_ACCESS_KEY}) do sleep 1; + done; /usr/bin/mc mb -p local/mlflow; /usr/bin/mc anonymous + set none local/mlflow; exit 0;", ] # CPU-friendly API diff --git a/docker/api/Dockerfile b/docker/api/Dockerfile index 2462d4ec..aaf0a3d2 100644 --- a/docker/api/Dockerfile +++ b/docker/api/Dockerfile @@ -55,13 +55,15 @@ RUN apt update && \ ENV LANG=en_US.UTF-8 \ LC_ALL=en_US.UTF-8 -# Install LibreOffice for document processing +# Install LibreOffice for document processing and ffmpeg for audio probing +# (ffmpeg provides ffprobe, used by aymurai.audio.duration.probe_audio_duration) RUN apt update && \ apt-get install -y --no-install-recommends \ libreoffice-writer \ libreoffice-common \ default-jre \ - libmagic1 && \ + libmagic1 \ + ffmpeg && \ apt-get autoremove -y && \ apt-get clean && \ rm -rf /tmp/* /var/tmp/* && \ diff --git a/notebooks/experiments/speech-to-text/01-es-ar-speech-evaluation.ipynb b/notebooks/experiments/speech-to-text/01-es-ar-speech-evaluation.ipynb new file mode 100644 index 00000000..2b867f8b --- /dev/null +++ b/notebooks/experiments/speech-to-text/01-es-ar-speech-evaluation.ipynb @@ -0,0 +1,619 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "7fb27b941602401d91542211134fc71a", + "metadata": {}, + "source": [ + "# Argentine Spanish ASR full evaluation\n", + "Resumable, parallel transcription plus raw and normalized WER/CER, breakdowns, and audio inspection. Set `SAMPLE_LIMIT = 20` for a smoke run.\n", + "\n", + "## Dataset\n", + "\n", + "This evaluation uses the [Crowdsourced High-Quality Argentine Spanish Speech Dataset](https://www.openslr.org/61/) from OpenSLR (SLR61). It contains nearly 6,000 short audio recordings from multiple speakers, paired with reference transcriptions in Argentine Spanish. The clips average approximately five seconds and include male, female, and weather-related subsets.\n", + "\n", + "The dataset and its accompanying resources can be downloaded from the [OpenSLR SLR61 page](https://www.openslr.org/61/)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "acae54e37e7d407bbb7b55eff062a284", + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "import os\n", + "import threading\n", + "import time\n", + "import unicodedata\n", + "from concurrent.futures import ThreadPoolExecutor, as_completed\n", + "from pathlib import Path\n", + "\n", + "import jiwer\n", + "import numpy as np\n", + "import pandas as pd\n", + "import requests\n", + "from IPython.display import Audio, display\n", + "from tqdm.auto import tqdm" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "90851e19", + "metadata": {}, + "outputs": [], + "source": [ + "DATA_DIR = Path(\"/resources/data/asr/es_ar_speech\")\n", + "API_BASE_URL = os.getenv(\"API_BASE_URL\", \"http://localhost:8999\")\n", + "API_TRANSCRIBE_ENDPOINT = f\"{API_BASE_URL}/asr/transcribe\"\n", + "MAX_WORKERS = 4\n", + "TIMEOUT = (10, 180)\n", + "MAX_RETRIES = 3\n", + "SAMPLE_LIMIT = None\n", + "CHECKPOINT = Path(\"results/es_ar_asr_transcriptions.csv\")\n", + "RESULT_COLUMNS = [\n", + " \"file\",\n", + " \"hypothesis\",\n", + " \"response_json\",\n", + " \"status\",\n", + " \"error\",\n", + " \"attempts\",\n", + " \"elapsed_seconds\",\n", + "]" + ] + }, + { + "cell_type": "markdown", + "id": "9a63283cbaf04dbcab1f6479b197f3a8", + "metadata": {}, + "source": [ + "## Load and validate annotations" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8dd0d8092fe74a7c96281538738b07e2", + "metadata": {}, + "outputs": [], + "source": [ + "parts = []\n", + "for path in sorted(DATA_DIR.glob(\"*.tsv\")):\n", + " part = pd.read_csv(path, sep=\"\\t\", header=None, names=[\"file\", \"transcription\"])\n", + " part[\"source_tsv\"] = path.name\n", + " parts.append(part)\n", + "df = pd.concat(parts, ignore_index=True)\n", + "df.info()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b37b31db", + "metadata": {}, + "outputs": [], + "source": [ + "df.sample(5)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "506b6dfc", + "metadata": {}, + "outputs": [], + "source": [ + "df.loc[df[\"file\"].duplicated(keep=False)].sort_values(\"file\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a386674f", + "metadata": {}, + "outputs": [], + "source": [ + "df.loc[df.duplicated(subset=[\"file\", \"transcription\"], keep=False)].sort_values(\"file\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a601b74e", + "metadata": {}, + "outputs": [], + "source": [ + "# Remove duplicates with same file and transcription, keeping the first occurrence\n", + "df.drop_duplicates(subset=[\"file\", \"transcription\"], inplace=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1709fb5f", + "metadata": {}, + "outputs": [], + "source": [ + "df.info()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4cc86bed", + "metadata": {}, + "outputs": [], + "source": [ + "# Verify that each file has a unique annotation and a corresponding WAV file\n", + "wavs = {p.stem: p for p in sorted(DATA_DIR.glob(\"*.wav\"))}\n", + "assert df[[\"file\", \"transcription\"]].notna().all().all()\n", + "assert df[\"file\"].is_unique and set(df[\"file\"]) == set(\n", + " wavs\n", + "), \"Annotations and WAV files differ\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2c871375", + "metadata": {}, + "outputs": [], + "source": [ + "# Add audio paths and file prefixes for later use\n", + "df[\"audio_path\"] = df[\"file\"].map(wavs)\n", + "df[\"file_prefix\"] = df[\"file\"].str.extract(r\"^([^_]+)\", expand=False)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f20ce36e", + "metadata": {}, + "outputs": [], + "source": [ + "# Summarize the number of samples per source TSV file\n", + "df.groupby(\"source_tsv\").size().rename(\"samples\").to_frame()" + ] + }, + { + "cell_type": "markdown", + "id": "72eea5119410473aa328ad9291626812", + "metadata": {}, + "source": [ + "## Parsing, normalization, and synthetic checks\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8edb47106e1a46a883d545849b8ab81b", + "metadata": {}, + "outputs": [], + "source": [ + "def parse_response(payload: dict) -> str:\n", + " \"\"\"\n", + " Parse the ASR API response to extract the transcribed text.\n", + "\n", + " Args:\n", + " payload (dict): The JSON response from the ASR API,\n", + " expected to contain a \"document\" key with a list of segments.\n", + "\n", + " Raises:\n", + " ValueError: If the response does not contain a valid document list or if any segment is not a dictionary.\n", + "\n", + " Returns:\n", + " str: The concatenated transcribed text from the document segments, with leading and trailing whitespace removed.\n", + " \"\"\"\n", + " if not isinstance(payload, dict) or not isinstance(payload.get(\"document\"), list):\n", + " raise ValueError(\"Response must contain a document list\")\n", + "\n", + " if any(not isinstance(x, dict) for x in payload[\"document\"]):\n", + " raise ValueError(\"Document segments must be objects\")\n", + "\n", + " return \" \".join(\n", + " x.get(\"text\", \"\").strip()\n", + " for x in payload[\"document\"]\n", + " if x.get(\"text\", \"\").strip()\n", + " )\n", + "\n", + "\n", + "def normalize(text: str) -> str:\n", + " \"\"\"\n", + " Normalize the input text by removing accents, converting to lowercase,\n", + " and replacing non-alphanumeric characters with spaces.\n", + "\n", + " Args:\n", + " text (str): The input text to normalize.\n", + "\n", + " Returns:\n", + " str: The normalized text.\n", + " \"\"\"\n", + " text = unicodedata.normalize(\"NFKD\", str(text)).lower()\n", + " text = \"\".join(c for c in text if not unicodedata.combining(c))\n", + " return \" \".join(\n", + " \"\".join(c if c.isalnum() or c.isspace() else \" \" for c in text).split()\n", + " )\n", + "\n", + "\n", + "assert (\n", + " parse_response({\"document\": [{\"text\": \" Hola \"}, {\"text\": \"mundo\"}]})\n", + " == \"Hola mundo\"\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "10185d26023b46108eb7d9f57d49d2b3", + "metadata": {}, + "source": [ + "## Parallel transcription with retries and atomic checkpoints\n", + "Successful rows are skipped when rerun; failed rows are attempted again.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8763a12b2bbd4a93a75aff182afb95dc", + "metadata": {}, + "outputs": [], + "source": [ + "_local = threading.local()\n", + "\n", + "\n", + "def session():\n", + " if not hasattr(_local, \"session\"):\n", + " _local.session = requests.Session()\n", + " return _local.session\n", + "\n", + "\n", + "def transcribe_one(fid: str, path: str) -> dict:\n", + " \"\"\"\n", + " Transcribe a single audio file using the ASR API, with retry logic and error handling.\n", + "\n", + " Args:\n", + " fid (str): The file identifier.\n", + " path (str): The path to the audio file.\n", + "\n", + " Returns:\n", + " dict: A dictionary containing the transcription results and metadata.\n", + " \"\"\"\n", + " started = time.monotonic()\n", + " error = \"\"\n", + "\n", + " for attempt in range(1, MAX_RETRIES + 2):\n", + " try:\n", + " with path.open(\"rb\") as stream:\n", + " response = session().post(\n", + " API_TRANSCRIBE_ENDPOINT,\n", + " files={\"file\": (path.name, stream, \"audio/wav\")},\n", + " data={\"response_format\": \"diarized_json\"},\n", + " timeout=TIMEOUT,\n", + " )\n", + " response.raise_for_status()\n", + " payload = response.json()\n", + " return dict(\n", + " file=fid,\n", + " hypothesis=parse_response(payload),\n", + " response_json=json.dumps(payload, ensure_ascii=False),\n", + " status=\"success\",\n", + " error=\"\",\n", + " attempts=attempt,\n", + " elapsed_seconds=time.monotonic() - started,\n", + " )\n", + " except (requests.RequestException, ValueError) as exc:\n", + " error = f\"{type(exc).__name__}: {exc}\"\n", + " if isinstance(exc, ValueError) or attempt > MAX_RETRIES:\n", + " break\n", + " time.sleep(2 ** (attempt - 1))\n", + "\n", + " return dict(\n", + " file=fid,\n", + " hypothesis=\"\",\n", + " response_json=\"\",\n", + " status=\"error\",\n", + " error=error,\n", + " attempts=attempt,\n", + " elapsed_seconds=time.monotonic() - started,\n", + " )\n", + "\n", + "\n", + "def load_checkpoint() -> pd.DataFrame:\n", + " \"\"\"\n", + " Load the transcription results from a checkpoint CSV file if it exists,\n", + "\n", + " Returns:\n", + " pd.DataFrame: A DataFrame containing the transcription results,\n", + " or an empty DataFrame with predefined columns if the checkpoint file does not exist.\n", + " \"\"\"\n", + " return (\n", + " pd.read_csv(CHECKPOINT, keep_default_na=False)\n", + " if CHECKPOINT.exists()\n", + " else pd.DataFrame(columns=RESULT_COLUMNS)\n", + " )\n", + "\n", + "\n", + "def save_checkpoint(records: dict) -> None:\n", + " \"\"\"\n", + " Save the transcription results to a checkpoint CSV file.\n", + "\n", + " Args:\n", + " records (dict): A dictionary of transcription results, where keys are file identifiers\n", + " and values are dictionaries containing the transcription data.\n", + " \"\"\"\n", + " CHECKPOINT.parent.mkdir(parents=True, exist_ok=True)\n", + " temp = CHECKPOINT.with_suffix(\".tmp\")\n", + " pd.DataFrame(sorted(records.values(), key=lambda x: x[\"file\"])).to_csv(\n", + " temp, index=False, columns=RESULT_COLUMNS\n", + " )\n", + " temp.replace(CHECKPOINT)\n", + "\n", + "\n", + "def transcribe_dataset(data: pd.DataFrame) -> pd.DataFrame:\n", + " \"\"\"\n", + " Transcribe a dataset of audio files using the ASR API,\n", + " with checkpointing to allow resuming.\n", + "\n", + " Args:\n", + " data (pd.DataFrame): A DataFrame containing the dataset to transcribe,\n", + " expected to have \"file\" and \"audio_path\" columns.\n", + "\n", + " Returns:\n", + " pd.DataFrame: A DataFrame containing the transcription results for the dataset.\n", + " \"\"\"\n", + " records = {r[\"file\"]: r for r in load_checkpoint().to_dict(\"records\")}\n", + " done = {k for k, v in records.items() if v[\"status\"] == \"success\"}\n", + " work = data.loc[~data.file.isin(done), [\"file\", \"audio_path\"]]\n", + " work = work if SAMPLE_LIMIT is None else work.head(SAMPLE_LIMIT)\n", + "\n", + " with ThreadPoolExecutor(max_workers=MAX_WORKERS) as pool:\n", + " futures = [\n", + " pool.submit(transcribe_one, r.file, r.audio_path)\n", + " for r in work.itertuples(index=False)\n", + " ]\n", + " for n, future in enumerate(tqdm(as_completed(futures), total=len(futures)), 1):\n", + " result = future.result()\n", + " records[result[\"file\"]] = result\n", + " if n % 25 == 0:\n", + " save_checkpoint(records)\n", + "\n", + " if len(work):\n", + " save_checkpoint(records)\n", + "\n", + " return load_checkpoint()\n", + "\n", + "\n", + "results = transcribe_dataset(df)" + ] + }, + { + "cell_type": "markdown", + "id": "7623eae2785240b9bd12b16a66d81610", + "metadata": {}, + "source": [ + "## Raw and normalized metrics\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7cdc8c89c7104fffa095e18ddfef8986", + "metadata": {}, + "outputs": [], + "source": [ + "df = df.merge(results, on=\"file\", how=\"left\")\n", + "df[\"status\"] = df.status.fillna(\"pending\")\n", + "df[\"hypothesis\"] = df.hypothesis.fillna(\"\")\n", + "df[\"reference_normalized\"] = df.transcription.map(normalize)\n", + "df[\"hypothesis_normalized\"] = df.hypothesis.map(normalize)\n", + "evaluation = df[df.status == \"success\"].copy()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "250ccd41", + "metadata": {}, + "outputs": [], + "source": [ + "def edit_counts(reference: str, hypothesis: str, unit: str) -> tuple[int, int, float]:\n", + " \"\"\"\n", + " Evaluate the edit counts and error rate between a reference and hypothesis transcription\n", + " using the jiwer library, based on the specified unit of measurement.\n", + "\n", + " Args:\n", + " reference (str): The reference transcription.\n", + " hypothesis (str): The hypothesis transcription.\n", + " unit (str): The unit of measurement, either \"word\" or \"character\".\n", + "\n", + " Returns:\n", + " tuple[int, int, float]: A tuple containing the number of errors, the total number of units, and the error rate.\n", + " \"\"\"\n", + " out = (jiwer.process_words if unit == \"word\" else jiwer.process_characters)(\n", + " reference, hypothesis\n", + " )\n", + " units = out.hits + out.substitutions + out.deletions\n", + " errors = out.substitutions + out.deletions + out.insertions\n", + " return errors, units, errors / units if units else np.nan\n", + "\n", + "\n", + "for label, ref, hyp in [\n", + " (\"raw\", \"transcription\", \"hypothesis\"),\n", + " (\"normalized\", \"reference_normalized\", \"hypothesis_normalized\"),\n", + "]:\n", + " for unit, metric in [(\"word\", \"wer\"), (\"character\", \"cer\")]:\n", + " values = [\n", + " edit_counts(r, h, unit) for r, h in zip(evaluation[ref], evaluation[hyp])\n", + " ]\n", + " for i, name in enumerate([\"errors\", \"units\", \"rate\"]):\n", + " evaluation[f\"{label}_{metric}_{name}\"] = [v[i] for v in values]\n", + "\n", + "\n", + "def aggregate(group: pd.DataFrame) -> pd.Series:\n", + " \"\"\"\n", + " Aggregate the edit counts and error rates for a group of samples,\n", + " calculating the total number of samples and the overall error rates for both raw and normalized transcriptions.\n", + "\n", + " Args:\n", + " group (pd.DataFrame): A DataFrame group containing the samples to aggregate.\n", + "\n", + " Returns:\n", + " pd.Series: A Series containing the total number of samples\n", + " and the overall error rates for raw and normalized transcriptions.\n", + " \"\"\"\n", + " result = {\"samples\": len(group)}\n", + " for label in [\"raw\", \"normalized\"]:\n", + " for metric in [\"wer\", \"cer\"]:\n", + " units = group[f\"{label}_{metric}_units\"].sum()\n", + " result[f\"{label}_{metric}\"] = (\n", + " group[f\"{label}_{metric}_errors\"].sum() / units if units else np.nan\n", + " )\n", + " return pd.Series(result)\n", + "\n", + "\n", + "assert edit_counts(\"hola mundo\", \"hola gente\", \"word\")[2] == 0.5\n", + "\n", + "display(\n", + " pd.Series(\n", + " {\n", + " \"total\": len(df),\n", + " \"success\": len(evaluation),\n", + " \"failed\": (df.status == \"error\").sum(),\n", + " \"pending\": (df.status == \"pending\").sum(),\n", + " \"empty_hypotheses\": evaluation.hypothesis.str.strip().eq(\"\").sum(),\n", + " }\n", + " ).to_frame(\"value\")\n", + ")\n", + "display(aggregate(evaluation).to_frame(\"value\"))" + ] + }, + { + "cell_type": "markdown", + "id": "b118ea5561624da68c537baed56e602f", + "metadata": {}, + "source": [ + "## Summaries, breakdowns, and audio playback\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "938c804e27f84196a10c8828c723f798", + "metadata": {}, + "outputs": [], + "source": [ + "metric_cols = [\n", + " \"raw_wer_rate\",\n", + " \"raw_cer_rate\",\n", + " \"normalized_wer_rate\",\n", + " \"normalized_cer_rate\",\n", + "]\n", + "\n", + "print(\"Error rate percentiles:\")\n", + "display(\n", + " evaluation[metric_cols + [\"elapsed_seconds\"]].describe(\n", + " percentiles=[0.5, 0.75, 0.9, 0.95, 0.99]\n", + " )\n", + ")\n", + "\n", + "print(\"Error rates by source TSV:\")\n", + "display(evaluation.groupby(\"source_tsv\").apply(aggregate, include_groups=False))\n", + "\n", + "print(\"Worst transcriptions by normalized WER:\")\n", + "worst = evaluation.sort_values(\"normalized_wer_rate\", ascending=False).head(20)\n", + "display(\n", + " worst[\n", + " [\"file\", \"transcription\", \"hypothesis\", \"raw_wer_rate\", \"normalized_wer_rate\"]\n", + " ]\n", + ")\n", + "if df[\"status\"].eq(\"error\").any():\n", + " print(\"Errors in transcriptions:\")\n", + " display(df[df[\"status\"] == \"error\"][[\"file\", \"attempts\", \"error\"]].head(50))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c8071c8d", + "metadata": {}, + "outputs": [], + "source": [ + "def display_sample(index: int) -> None:\n", + " \"\"\"\n", + " Display a sample from the evaluation dataframe.\n", + "\n", + " Args:\n", + " index (int): The index of the sample to display.\n", + " \"\"\"\n", + " row = evaluation.loc[index]\n", + " display(\n", + " pd.Series(\n", + " {\n", + " \"file\": row.file,\n", + " \"reference\": row.transcription,\n", + " \"hypothesis\": row.hypothesis,\n", + " \"raw WER\": row.raw_wer_rate,\n", + " \"normalized WER\": row.normalized_wer_rate,\n", + " }\n", + " ).to_frame(\"value\")\n", + " )\n", + " display(Audio(filename=str(row.audio_path)))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bed3a2ed", + "metadata": {}, + "outputs": [], + "source": [ + "# Display a random sample from the evaluation results\n", + "if not evaluation.empty:\n", + " display_sample(evaluation.sample(1).index[0])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ca3f681e", + "metadata": {}, + "outputs": [], + "source": [ + "# Assess the worst transcriptions by normalized WER\n", + "for index in worst.index:\n", + " display_sample(index)\n", + " print(\"-\" * 40)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e8ea97b9", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "aymurai (3.10.20.final.0)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.20" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/pyproject.toml b/pyproject.toml index d0421eb6..84438d97 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,7 +53,6 @@ classifiers = [ dependencies = [ "requests>=2.32.3", "numpy<2.0.0", - "librosa>=0.10.2", "more-itertools>=10.5.0", "odfpy>=1.4.1", "joblib>=1.4.2", @@ -86,6 +85,7 @@ dependencies = [ "pypandoc>=1.15", "python-docx>=1.2.0", "langextract[openai]==1.1.0", + "openai>=2.43.0", "ollama==0.6.1", "tiktoken==0.12.0", "marker-pdf==1.10.1", @@ -115,6 +115,12 @@ dev = [ ] tests = ["pytest>=9.0.2"] +[tool.uv] +# marker-pdf pins openai<2.0.0, but aymurai never uses marker's optional LLM +# services (the only code path touching openai). Override the cap so we can use +# openai>=2.x, which adds the typed "diarized_json" response_format for coro ASR. +override-dependencies = ["openai>=2.43.0"] + [tool.setuptools.packages.find] include = ["aymurai"] diff --git a/scripts/run-coro.sh b/scripts/run-coro.sh new file mode 100755 index 00000000..539c3615 --- /dev/null +++ b/scripts/run-coro.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +set -euo pipefail + +MODE="${1:-gpu}" +PORT="${CORO_PORT:-8000}" +SPILL_DIR="${CORO_TRANSCRIPT_SPILL_DIR:-./resources/cache/coro-spill}" +mkdir -p "$SPILL_DIR" + +export CORO_BACKEND_ASR=onnx-asr +export CORO_MODEL_ASR=nemo-parakeet-tdt-0.6b-v3 +export CORO_BACKEND_DIARIZATION=nemo +export CORO_PIPELINE=streaming +export CORO_TRANSCRIPT_SPILL_DIR="$SPILL_DIR" + +case "$MODE" in + gpu) + export CORO_ASR_DEVICE=cuda + export CORO_DIARIZATION_DEVICE=cuda + EXTRA="cuda" + ;; + cpu) + export CORO_ASR_DEVICE=cpu + export CORO_ASR_QUANTIZATION=int8 + export CORO_DIARIZATION_DEVICE=cpu + EXTRA="cpu" + ;; + *) + echo "usage: $0 [cpu|gpu]" >&2 + exit 2 + ;; +esac + +exec uvx --python 3.12 \ + --from "coro-asr[$EXTRA] @ git+https://github.com/collectiveai-team/coro" \ + coro --port "$PORT" diff --git a/scripts/smoke_coro.py b/scripts/smoke_coro.py new file mode 100644 index 00000000..1aecc6c2 --- /dev/null +++ b/scripts/smoke_coro.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python +"""Smoke test for the coro ASR integration. + +Starts the coro server (via ``scripts/run-coro.sh``), waits for it to become +healthy, then transcribes a sample audio file over the OpenAI-compatible SSE +endpoint two ways: + +1. directly with the ``openai`` SDK (``stream=True``) — shows the raw SSE events; +2. through aymurai's own client ``transcribe_audio_bytes`` — proves the wiring. + +Usage: + rtk uv run python scripts/smoke_coro.py [cpu|gpu] + +Env overrides: + CORO_PORT (default 8000) + SMOKE_AUDIO path to a local audio file (default: download sample) + SMOKE_AUDIO_URL sample to download if SMOKE_AUDIO is absent + SMOKE_HEALTH_TIMEOUT seconds to wait for /health (default 1200) +""" + +from __future__ import annotations + +import asyncio +import json +import os +import signal +import subprocess +import sys +import time +import urllib.request +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent +CORO_LOG = Path("/tmp/coro-smoke.log") +MODE = sys.argv[1] if len(sys.argv) > 1 else "cpu" +PORT = os.environ.get("CORO_PORT", "8000") +BASE_URL = f"http://localhost:{PORT}/v1" +HEALTH_URL = f"http://localhost:{PORT}/health" +HEALTH_TIMEOUT = int(os.environ.get("SMOKE_HEALTH_TIMEOUT", "1200")) +AUDIO_PATH = Path( + os.environ.get("SMOKE_AUDIO", str(REPO_ROOT / "resources/cache/smoke/sample.wav")) +) +AUDIO_URL = os.environ.get( + "SMOKE_AUDIO_URL", + "https://google-research.github.io/lingvo-lab/translatotron/fisher_src/775.wav", +) + + +def log(msg: str) -> None: + print(f"[smoke] {msg}", flush=True) + + +def ensure_audio() -> Path: + if AUDIO_PATH.exists(): + log(f"using existing audio: {AUDIO_PATH}") + return AUDIO_PATH + AUDIO_PATH.parent.mkdir(parents=True, exist_ok=True) + log(f"downloading sample audio: {AUDIO_URL}") + urllib.request.urlretrieve(AUDIO_URL, AUDIO_PATH) # noqa: S310 + log(f"saved sample audio: {AUDIO_PATH} ({AUDIO_PATH.stat().st_size} bytes)") + return AUDIO_PATH + + +def start_coro() -> subprocess.Popen: + log(f"starting coro ({MODE}) on port {PORT} ; logs -> {CORO_LOG}") + logfh = CORO_LOG.open("w") + return subprocess.Popen( + ["bash", str(REPO_ROOT / "scripts/run-coro.sh"), MODE], + cwd=REPO_ROOT, + stdout=logfh, + stderr=subprocess.STDOUT, + text=True, + start_new_session=True, + ) + + +def wait_healthy(proc: subprocess.Popen) -> None: + log(f"waiting for {HEALTH_URL} (timeout {HEALTH_TIMEOUT}s) ...") + deadline = time.time() + HEALTH_TIMEOUT + while time.time() < deadline: + if proc.poll() is not None: + raise RuntimeError("coro process exited before becoming healthy") + try: + with urllib.request.urlopen(HEALTH_URL, timeout=5) as resp: # noqa: S310 + if resp.status == 200: + log("coro is healthy") + return + except Exception: + pass + time.sleep(3) + raise TimeoutError("coro did not become healthy in time") + + +def transcribe_with_sdk(audio: Path) -> None: + from openai import OpenAI + + log("=== SSE transcription via openai SDK (stream=True) ===") + client = OpenAI(base_url=BASE_URL, api_key="not-needed") + deltas = 0 + with audio.open("rb") as fh: + stream = client.audio.transcriptions.create( + file=fh, model="whisper-1", language="es", stream=True + ) + for event in stream: + if event.type == "transcript.text.delta": + deltas += 1 + elif event.type == "transcript.text.done": + payload = json.loads(event.text) + segments = payload.get("segments", []) + log(f"received {deltas} delta events, {len(segments)} segments") + for seg in segments: + log( + f" [{seg['start']:.2f}-{seg['end']:.2f}] " + f"speaker {seg['speaker']}: {seg['text']}" + ) + + +def transcribe_with_aymurai_client(audio: Path) -> None: + os.environ["TRANSCRIBE_BASE_URL"] = BASE_URL + from aymurai.audio.asr_client import transcribe_audio_bytes + + log("=== transcription via aymurai transcribe_audio_bytes ===") + payload = audio.read_bytes() + segments = asyncio.run(transcribe_audio_bytes(payload, audio.name, "audio/wav")) + log(f"aymurai client returned {len(segments)} CoroSegment(s)") + for seg in segments: + log(f" speaker {seg.speaker} [{seg.start:.2f}-{seg.end:.2f}]: {seg.text}") + + +def main() -> int: + audio = ensure_audio() + proc = start_coro() + try: + wait_healthy(proc) + transcribe_with_sdk(audio) + transcribe_with_aymurai_client(audio) + log("SMOKE TEST PASSED") + return 0 + except Exception as exc: # noqa: BLE001 + log(f"SMOKE TEST FAILED: {exc}") + if CORO_LOG.exists(): + log(f"--- {CORO_LOG} tail ---") + print("\n".join(CORO_LOG.read_text().splitlines()[-40:]), flush=True) + return 1 + finally: + if proc.poll() is None: + os.killpg(os.getpgid(proc.pid), signal.SIGTERM) + try: + proc.wait(timeout=10) + except Exception: + os.killpg(os.getpgid(proc.pid), signal.SIGKILL) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/smoke_coro_diarization.py b/scripts/smoke_coro_diarization.py new file mode 100644 index 00000000..6e57437e --- /dev/null +++ b/scripts/smoke_coro_diarization.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python +"""Diarization smoke test against an already-running coro ASR server. + +Unlike ``scripts/smoke_coro.py`` (which boots its own server), this script +assumes a coro server is already up and focuses on verifying that speaker +diarization is correctly wired end-to-end through aymurai's client. It feeds a +30s+ multi-speaker clip and asserts: + +1. the audio is at least ``SMOKE_MIN_DURATION`` seconds long; +2. the client returns speaker-attributed segments; +3. at least two distinct speakers are detected (diarization is active); +4. every segment is well-formed (start < end, non-empty speaker); +5. the segments cover a reasonable fraction of the audio. + +Usage: + rtk uv run python scripts/smoke_coro_diarization.py /path/to/audio.wav + +Env overrides: + TRANSCRIBE_BASE_URL coro base URL (default: http://localhost:${CORO_PORT}/v1) + CORO_PORT port if TRANSCRIBE_BASE_URL is unset (default 8000) + TRANSCRIBE_API_KEY forwarded to the client (default "not-needed") + SMOKE_AUDIO audio path if not passed as argv[1] + SMOKE_MIN_DURATION minimum required duration in seconds (default 30) + SMOKE_MIN_COVERAGE min fraction of audio covered by segments (default 0.5) +""" + +from __future__ import annotations + +import asyncio +import mimetypes +import os +import sys +from pathlib import Path + + +def log(msg: str) -> None: + print(f"[diar-smoke] {msg}", flush=True) + + +def resolve_base_url() -> str: + base_url = os.environ.get("TRANSCRIBE_BASE_URL") + if base_url: + return base_url + port = os.environ.get("CORO_PORT", "8000") + return f"http://localhost:{port}/v1" + + +def resolve_audio() -> Path: + raw = sys.argv[1] if len(sys.argv) > 1 else os.environ.get("SMOKE_AUDIO") + if not raw: + raise SystemExit( + "no audio provided: pass a path as the first argument or set SMOKE_AUDIO" + ) + path = Path(raw).expanduser() + if not path.exists(): + raise SystemExit(f"audio file not found: {path}") + return path + + +def content_type_for(path: Path) -> str: + guessed, _ = mimetypes.guess_type(path.name) + return guessed or "application/octet-stream" + + +def main() -> int: + min_duration = float(os.environ.get("SMOKE_MIN_DURATION", "30")) + min_coverage = float(os.environ.get("SMOKE_MIN_COVERAGE", "0.5")) + + audio = resolve_audio() + base_url = resolve_base_url() + os.environ["TRANSCRIBE_BASE_URL"] = base_url + os.environ.setdefault("TRANSCRIBE_API_KEY", "not-needed") + + # Imports happen after env is set so settings pick up the base URL. + from aymurai.audio.asr_client import transcribe_audio_bytes + from aymurai.audio.duration import probe_audio_duration + + payload = audio.read_bytes() + log(f"audio: {audio} ({len(payload)} bytes)") + log(f"coro base URL: {base_url}") + + duration = probe_audio_duration(payload) + if duration is None: + log("WARNING: could not probe duration (ffprobe missing?); skipping check") + else: + log(f"audio duration: {duration:.1f}s (minimum required: {min_duration:.0f}s)") + if duration < min_duration: + log(f"FAILED: audio is shorter than {min_duration:.0f}s") + return 1 + + log("transcribing via aymurai transcribe_audio_bytes ...") + segments = asyncio.run( + transcribe_audio_bytes(payload, audio.name, content_type_for(audio)) + ) + + if not segments: + log("FAILED: client returned no segments") + return 1 + + malformed = [s for s in segments if not s.speaker or s.end < s.start or s.start < 0] + if malformed: + log(f"FAILED: {len(malformed)} malformed segment(s); first: {malformed[0]!r}") + return 1 + + zero_length = [s for s in segments if s.end == s.start] + if zero_length: + log(f"note: {len(zero_length)} zero-length segment(s) (start == end), allowed") + + speakers = sorted({s.speaker for s in segments}) + spoken = sum(s.end - s.start for s in segments) + span_end = max(s.end for s in segments) + coverage = (span_end / duration) if duration else None + + log(f"received {len(segments)} segment(s) across {len(speakers)} speaker(s)") + for speaker in speakers: + spk_segs = [s for s in segments if s.speaker == speaker] + spk_time = sum(s.end - s.start for s in spk_segs) + log(f" speaker {speaker}: {len(spk_segs)} segment(s), {spk_time:.1f}s") + log(f"total spoken time: {spoken:.1f}s ; last segment ends at {span_end:.1f}s") + + log("--- transcript (first 12 segments) ---") + for seg in segments[:12]: + log(f" [{seg.start:6.2f}-{seg.end:6.2f}] spk {seg.speaker}: {seg.text}") + if len(segments) > 12: + log(f" ... ({len(segments) - 12} more)") + + if len(speakers) < 2: + log( + "FAILED: only one speaker detected — diarization does not appear wired " + "(expected >= 2 distinct speakers in a multi-speaker clip)" + ) + return 1 + + if coverage is not None: + log(f"segment coverage: {coverage:.0%} of audio (minimum {min_coverage:.0%})") + if coverage < min_coverage: + log("FAILED: segments cover too little of the audio") + return 1 + + log("DIARIZATION SMOKE TEST PASSED") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/test/api/endpoints/routers/asr/conftest.py b/test/api/endpoints/routers/asr/conftest.py index 270c0bdc..f61c3d4e 100644 --- a/test/api/endpoints/routers/asr/conftest.py +++ b/test/api/endpoints/routers/asr/conftest.py @@ -20,10 +20,10 @@ def _override_get_session(): yield session app.dependency_overrides[get_session] = _override_get_session - original_ws_uri = settings.TRANSCRIBE_WS_URI - settings.TRANSCRIBE_WS_URI = "ws://test-transcribe.local/ws" + original_base_url = settings.TRANSCRIBE_BASE_URL + settings.TRANSCRIBE_BASE_URL = "http://test-coro.local/v1" with TestClient(app) as client: yield client, sqlite_engine - settings.TRANSCRIBE_WS_URI = original_ws_uri + settings.TRANSCRIBE_BASE_URL = original_base_url diff --git a/test/api/endpoints/routers/asr/test_transcribe.py b/test/api/endpoints/routers/asr/test_transcribe.py index 6fd489ed..10d61b1c 100644 --- a/test/api/endpoints/routers/asr/test_transcribe.py +++ b/test/api/endpoints/routers/asr/test_transcribe.py @@ -1,18 +1,35 @@ +import json from datetime import timedelta from typing import Any, cast from unittest.mock import AsyncMock, patch from sqlmodel import Session -from aymurai.api.meta.asr.websocket import ( - WLKMessageStatus, - WLKMessageTranscriptionLine, -) +from aymurai.api.endpoints.routers.asr.transcribe import _estimate_progress +from aymurai.api.meta.asr.coro import CoroSegment, CoroStreamDelta, CoroStreamSegments from aymurai.database.meta.audio_transcription import AudioTranscription from aymurai.database.utils import data_to_uuid from aymurai.meta.api_interfaces import ASRDocument, ASRParagraph +def _parse_sse(body: str) -> list[dict[str, Any]]: + events: list[dict[str, Any]] = [] + for block in body.split("\n\n"): + block = block.strip() + if not block.startswith("data:"): + continue + events.append(json.loads(block[len("data:") :].strip())) + return events + + +def _fake_stream(events): + async def _gen(*args, **kwargs): + for event in events: + yield event + + return _gen + + # MARK: POST Transcribe def test_should_transcribe_and_persist_document_when_service_returns_paragraphs( asr_test_client, @@ -22,27 +39,13 @@ def test_should_transcribe_and_persist_document_when_service_returns_paragraphs( audio_bytes = make_wav_bytes() document_id = data_to_uuid(audio_bytes) - fake_status = WLKMessageStatus( - status="active_transcription", - lines=[ - WLKMessageTranscriptionLine( - speaker=1, - text="Hola mundo", - start=timedelta(seconds=0), - end=timedelta(seconds=1), - ) - ], - buffer_transcription="", - buffer_diarization="", - buffer_translation="", - remaining_time_transcription=0.0, - remaining_time_diarization=0.0, - speaker_ids={}, - ) + fake_segments = [ + CoroSegment(start=0.0, end=1.0, text="Hola mundo", speaker="1"), + ] with patch( "aymurai.api.endpoints.routers.asr.transcribe.transcribe_audio_bytes", - new=AsyncMock(return_value=fake_status), + new=AsyncMock(return_value=fake_segments), ): response = client.post( "/asr/transcribe?use_cache=false", @@ -52,8 +55,11 @@ def test_should_transcribe_and_persist_document_when_service_returns_paragraphs( assert response.status_code == 200 payload = ASRDocument.model_validate(response.json()) assert str(payload.document_id) == str(document_id) + assert payload.title == "sample.wav" assert len(payload.document) == 1 assert payload.document[0].text == "Hola mundo" + assert response.json()["speaker_turns"][0]["text"] == "Hola mundo" + assert response.json()["speaker_turns"][0]["speaker"] == "Speaker 1" with Session(engine) as session: record = session.get(AudioTranscription, document_id) @@ -63,6 +69,64 @@ def test_should_transcribe_and_persist_document_when_service_returns_paragraphs( assert first_item["text"] == "Hola mundo" +def test_transcribe_cache_should_merge_speaker_turns_from_transcription_when_no_validation( + asr_test_client, + make_wav_bytes, +): + client, engine = asr_test_client + audio_bytes = make_wav_bytes(freq_hz=224) + document_id = data_to_uuid(audio_bytes) + + with Session(engine) as session: + session.add( + AudioTranscription( + id=document_id, + name="cached-transcription.wav", + transcription=cast( + Any, + [ + ASRParagraph( + speaker_no=1, + start=timedelta(seconds=0), + end=timedelta(seconds=1), + text="Nos quedamos,", + ).model_dump(mode="json"), + ASRParagraph( + speaker_no=1, + start=timedelta(seconds=1), + end=timedelta(seconds=2), + text="esperando en el lobby.", + ).model_dump(mode="json"), + ], + ), + validation=[], + ) + ) + session.commit() + + def _fail(*args, **kwargs): + raise AssertionError("coro must not be called on transcription cache hit") + + with patch( + "aymurai.api.endpoints.routers.asr.transcribe.transcribe_audio_bytes", + new=_fail, + ): + response = client.post( + "/asr/transcribe", + files={"file": ("cached-transcription.wav", audio_bytes, "audio/wav")}, + ) + + assert response.status_code == 200 + body = response.json() + assert [item["text"] for item in body["document"]] == [ + "Nos quedamos,", + "esperando en el lobby.", + ] + assert [turn["text"] for turn in body["speaker_turns"]] == [ + "Nos quedamos, esperando en el lobby." + ] + + # MARK: GET Validation def test_should_return_validation_document_when_document_exists_in_database( asr_test_client, @@ -111,6 +175,137 @@ def test_should_return_validation_document_when_document_exists_in_database( assert payload.document[0].text == "Texto validado" +def test_transcribe_cache_should_preserve_validation_turns_without_merging( + asr_test_client, + make_wav_bytes, +): + client, engine = asr_test_client + audio_bytes = make_wav_bytes(freq_hz=225) + document_id = data_to_uuid(audio_bytes) + + with Session(engine) as session: + session.add( + AudioTranscription( + id=document_id, + name="validated-cache.wav", + transcription=cast( + Any, + [ + ASRParagraph( + speaker_no=1, + start=timedelta(seconds=0), + end=timedelta(seconds=3), + text="Texto original que no debe usarse", + ).model_dump(mode="json") + ], + ), + validation=cast( + Any, + [ + ASRParagraph( + speaker_no=1, + speaker_name="Docente", + start=timedelta(seconds=10), + end=timedelta(seconds=11), + text="Bloque manual uno sin cierre", + ).model_dump(mode="json"), + ASRParagraph( + speaker_no=1, + speaker_name="Docente", + start=timedelta(seconds=11), + end=timedelta(seconds=12.5), + text="Bloque manual dos editado", + ).model_dump(mode="json"), + ], + ), + ) + ) + session.commit() + + def _fail(*args, **kwargs): + raise AssertionError("coro must not be called on validation cache hit") + + with patch( + "aymurai.api.endpoints.routers.asr.transcribe.transcribe_audio_bytes", + new=_fail, + ): + response = client.post( + "/asr/transcribe", + files={"file": ("validated-cache.wav", audio_bytes, "audio/wav")}, + ) + + assert response.status_code == 200 + body = response.json() + assert [item["text"] for item in body["document"]] == [ + "Bloque manual uno sin cierre", + "Bloque manual dos editado", + ] + assert [turn["text"] for turn in body["speaker_turns"]] == [ + "Bloque manual uno sin cierre", + "Bloque manual dos editado", + ] + assert body["speaker_turns"][0]["speaker"] == "Docente" + assert body["speaker_turns"][1]["end"] == "00:00:12.500" + + +def test_get_validation_should_include_source_aware_speaker_turns( + asr_test_client, + make_wav_bytes, +): + client, engine = asr_test_client + audio_bytes = make_wav_bytes(freq_hz=226) + document_id = data_to_uuid(audio_bytes) + + with Session(engine) as session: + session.add( + AudioTranscription( + id=document_id, + name="read-validation.wav", + transcription=cast( + Any, + [ + ASRParagraph( + speaker_no=1, + start=timedelta(seconds=0), + end=timedelta(seconds=4), + text="Texto original", + ).model_dump(mode="json") + ], + ), + validation=cast( + Any, + [ + ASRParagraph( + speaker_no=2, + speaker_name="Fiscal", + start=timedelta(seconds=4), + end=timedelta(seconds=5), + text="Validado A", + ).model_dump(mode="json"), + ASRParagraph( + speaker_no=2, + speaker_name="Fiscal", + start=timedelta(seconds=5), + end=timedelta(seconds=6), + text="Validado B", + ).model_dump(mode="json"), + ], + ), + ) + ) + session.commit() + + response = client.get(f"/asr/validation/document/{document_id}") + + assert response.status_code == 200 + body = response.json() + assert [turn["text"] for turn in body["speaker_turns"]] == [ + "Validado A", + "Validado B", + ] + assert body["speaker_turns"][0]["speaker"] == "Fiscal" + + # MARK: POST Validation def test_should_persist_validation_annotations_when_posting_validation_for_existing_document( asr_test_client, @@ -152,6 +347,103 @@ def test_should_persist_validation_annotations_when_posting_validation_for_exist assert first_item["text"] == "Linea validada" +def test_should_persist_validation_and_title_when_posting_validation_object( + asr_test_client, + make_wav_bytes, +): + client, engine = asr_test_client + audio_bytes = make_wav_bytes(freq_hz=331) + document_id = data_to_uuid(audio_bytes) + + with Session(engine) as session: + session.add( + AudioTranscription( + id=document_id, + name="old-title.wav", + transcription=cast( + Any, + [ + ASRParagraph( + speaker_no=1, + start=timedelta(seconds=0), + end=timedelta(seconds=1), + text="Base", + ).model_dump(mode="json") + ], + ), + validation=[], + ) + ) + session.commit() + + body = { + "title": " Audiencia editada ", + "document": [{"speaker_no": 1, "start": 0, "end": 1, "text": "Linea validada"}], + } + response = client.post(f"/asr/validation/document/{document_id}", json=body) + + assert response.status_code == 200 + with Session(engine) as session: + record = session.get(AudioTranscription, document_id) + assert record is not None + assert record.name == "Audiencia editada" + first_item = cast(dict[str, Any], record.validation[0]) + assert first_item["text"] == "Linea validada" + + get_response = client.get(f"/asr/validation/document/{document_id}") + assert get_response.status_code == 200 + payload = ASRDocument.model_validate(get_response.json()) + assert payload.title == "Audiencia editada" + + transcribe_response = client.post( + "/asr/transcribe", + files={"file": ("new-upload-name.wav", audio_bytes, "audio/wav")}, + ) + assert transcribe_response.status_code == 200 + transcribe_payload = ASRDocument.model_validate(transcribe_response.json()) + assert transcribe_payload.title == "Audiencia editada" + + +def test_should_reject_empty_title_when_posting_validation_object( + asr_test_client, + make_wav_bytes, +): + client, engine = asr_test_client + audio_bytes = make_wav_bytes(freq_hz=332) + document_id = data_to_uuid(audio_bytes) + + with Session(engine) as session: + session.add( + AudioTranscription( + id=document_id, + name="existing.wav", + transcription=cast( + Any, + [ + ASRParagraph( + speaker_no=1, + start=timedelta(seconds=0), + end=timedelta(seconds=1), + text="Base", + ).model_dump(mode="json") + ], + ), + validation=[], + ) + ) + session.commit() + + response = client.post( + f"/asr/validation/document/{document_id}", + json={ + "title": " ", + "document": [{"speaker_no": 1, "start": 0, "end": 1, "text": "Validada"}], + }, + ) + + assert response.status_code == 422 + + # MARK: POST Validation with speaker_name per paragraph def test_should_persist_speaker_name_on_paragraph_when_posting_validation( asr_test_client, @@ -314,3 +606,337 @@ def test_should_return_null_speaker_name_when_none_stored( assert response.status_code == 200 payload = ASRDocument.model_validate(response.json()) assert payload.document[0].speaker_name is None + + +def test_get_validation_should_compute_speaker_turns_from_transcription_when_unvalidated( + asr_test_client, + make_wav_bytes, +): + client, engine = asr_test_client + audio_bytes = make_wav_bytes(freq_hz=881) + document_id = data_to_uuid(audio_bytes) + + with Session(engine) as session: + session.add( + AudioTranscription( + id=document_id, + name="unvalidated-read.wav", + transcription=cast( + Any, + [ + ASRParagraph( + speaker_no=1, + start=timedelta(seconds=0), + end=timedelta(seconds=1), + text="Hola,", + ).model_dump(mode="json"), + ASRParagraph( + speaker_no=1, + start=timedelta(seconds=1), + end=timedelta(seconds=2), + text="mundo.", + ).model_dump(mode="json"), + ], + ), + validation=[], + ) + ) + session.commit() + + response = client.get(f"/asr/validation/document/{document_id}") + + assert response.status_code == 200 + body = response.json() + assert [turn["text"] for turn in body["speaker_turns"]] == ["Hola, mundo."] + + +# MARK: progress estimation +def test_progress_should_be_zero_for_no_characters(): + assert _estimate_progress(0, 10.0) == 0.0 + + +def test_progress_should_be_about_ninety_percent_at_estimated_completion(): + # raw == 1.0 when chars == duration * 17 + progress = _estimate_progress(170, 10.0) + assert progress is not None + assert abs(progress - 0.9) < 1e-6 + + +def test_progress_should_stay_below_one_even_when_estimate_exceeded(): + progress = _estimate_progress(100_000, 1.0) + assert progress is not None + assert progress < 1.0 + + +def test_progress_should_be_monotonic_non_decreasing(): + values = [_estimate_progress(chars, 10.0) for chars in range(0, 500, 25)] + assert all(a is not None and b is not None for a, b in zip(values, values[1:])) + assert all(a <= b for a, b in zip(values, values[1:])) # type: ignore[operator] + + +def test_progress_should_be_none_when_duration_unavailable(): + assert _estimate_progress(100, None) is None + + +def test_progress_should_be_none_when_duration_non_positive(): + assert _estimate_progress(100, 0.0) is None + + +# MARK: POST Transcribe stream +def test_stream_should_emit_meta_deltas_segments_done_and_persist( + asr_test_client, + make_wav_bytes, +): + client, engine = asr_test_client + audio_bytes = make_wav_bytes(duration_seconds=2.0) + document_id = data_to_uuid(audio_bytes) + + events = [ + CoroStreamDelta(text="Hola "), + CoroStreamDelta(text="mundo"), + CoroStreamSegments( + segments=[CoroSegment(start=0.0, end=1.0, text="Hola mundo", speaker="1")] + ), + ] + + with patch( + "aymurai.api.endpoints.routers.asr.transcribe.stream_transcribe_audio_bytes", + new=_fake_stream(events), + ): + response = client.post( + "/asr/transcribe/stream?use_cache=false", + files={"file": ("sample.wav", audio_bytes, "audio/wav")}, + ) + + assert response.status_code == 200 + assert response.headers["content-type"].startswith("text/event-stream") + parsed = _parse_sse(response.text) + + types = [event["type"] for event in parsed] + assert types == ["meta", "delta", "delta", "segments", "done"] + + meta = parsed[0] + assert meta["document_id"] == str(document_id) + assert meta["duration"] is not None and abs(meta["duration"] - 2.0) < 0.1 + + deltas = [event for event in parsed if event["type"] == "delta"] + assert deltas[0]["text"] == "Hola " + assert deltas[1]["text"] == "mundo" + progresses = [event["progress"] for event in deltas] + assert all(0.0 <= p < 1.0 for p in progresses) + assert progresses == sorted(progresses) + + segments_event = next(event for event in parsed if event["type"] == "segments") + assert segments_event["document"][0]["text"] == "Hola mundo" + assert segments_event["speaker_turns"][0]["text"] == "Hola mundo" + assert segments_event["speaker_turns"][0]["speaker"] == "Speaker 1" + + assert parsed[-1] == {"type": "done", "progress": 1.0} + + with Session(engine) as session: + record = session.get(AudioTranscription, document_id) + assert record is not None + first_item = cast(dict[str, Any], record.transcription[0]) + assert first_item["text"] == "Hola mundo" + + +def test_stream_should_emit_cached_document_without_calling_coro( + asr_test_client, + make_wav_bytes, +): + client, engine = asr_test_client + audio_bytes = make_wav_bytes(freq_hz=210) + document_id = data_to_uuid(audio_bytes) + + with Session(engine) as session: + session.add( + AudioTranscription( + id=document_id, + name="cached.wav", + transcription=cast( + Any, + [ + ASRParagraph( + speaker_no=1, + start=timedelta(seconds=0), + end=timedelta(seconds=1), + text="Texto cacheado", + ).model_dump(mode="json") + ], + ), + validation=[], + ) + ) + session.commit() + + def _fail(*args, **kwargs): + raise AssertionError("coro must not be called on cache hit") + + with patch( + "aymurai.api.endpoints.routers.asr.transcribe.stream_transcribe_audio_bytes", + new=_fail, + ): + response = client.post( + "/asr/transcribe/stream", + files={"file": ("cached.wav", audio_bytes, "audio/wav")}, + ) + + assert response.status_code == 200 + parsed = _parse_sse(response.text) + types = [event["type"] for event in parsed] + assert types == ["meta", "segments", "done"] + meta_event = next(event for event in parsed if event["type"] == "meta") + assert meta_event["title"] == "cached.wav" + segments_event = next(event for event in parsed if event["type"] == "segments") + assert segments_event["title"] == "cached.wav" + assert segments_event["document"][0]["text"] == "Texto cacheado" + assert segments_event["speaker_turns"][0]["text"] == "Texto cacheado" + + +def test_stream_cache_should_emit_validation_turns_without_merging( + asr_test_client, + make_wav_bytes, +): + client, engine = asr_test_client + audio_bytes = make_wav_bytes(freq_hz=211) + document_id = data_to_uuid(audio_bytes) + + with Session(engine) as session: + session.add( + AudioTranscription( + id=document_id, + name="Audiencia editada", + transcription=cast( + Any, + [ + ASRParagraph( + speaker_no=1, + start=timedelta(seconds=0), + end=timedelta(seconds=2), + text="Texto original", + ).model_dump(mode="json") + ], + ), + validation=cast( + Any, + [ + ASRParagraph( + speaker_no=1, + speaker_name="Defensa", + start=timedelta(seconds=2), + end=timedelta(seconds=3), + text="Turno validado uno", + ).model_dump(mode="json"), + ASRParagraph( + speaker_no=1, + speaker_name="Defensa", + start=timedelta(seconds=3), + end=timedelta(seconds=4), + text="Turno validado dos", + ).model_dump(mode="json"), + ], + ), + ) + ) + session.commit() + + def _fail(*args, **kwargs): + raise AssertionError("coro must not be called on validation cache hit") + + with patch( + "aymurai.api.endpoints.routers.asr.transcribe.stream_transcribe_audio_bytes", + new=_fail, + ): + response = client.post( + "/asr/transcribe/stream", + files={"file": ("cached-validation.wav", audio_bytes, "audio/wav")}, + ) + + assert response.status_code == 200 + parsed = _parse_sse(response.text) + meta_event = next(event for event in parsed if event["type"] == "meta") + assert meta_event["title"] == "Audiencia editada" + segments_event = next(event for event in parsed if event["type"] == "segments") + assert segments_event["title"] == "Audiencia editada" + assert [item["text"] for item in segments_event["document"]] == [ + "Turno validado uno", + "Turno validado dos", + ] + assert [turn["text"] for turn in segments_event["speaker_turns"]] == [ + "Turno validado uno", + "Turno validado dos", + ] + assert segments_event["speaker_turns"][0]["speaker"] == "Defensa" + + +def test_stream_should_emit_error_event_on_upstream_failure( + asr_test_client, + make_wav_bytes, +): + client, engine = asr_test_client + audio_bytes = make_wav_bytes(freq_hz=190) + document_id = data_to_uuid(audio_bytes) + + def _raising_stream(*args, **kwargs): + async def _gen(): + raise RuntimeError("Transcription service error") + yield # pragma: no cover + + return _gen() + + with patch( + "aymurai.api.endpoints.routers.asr.transcribe.stream_transcribe_audio_bytes", + new=_raising_stream, + ): + response = client.post( + "/asr/transcribe/stream?use_cache=false", + files={"file": ("err.wav", audio_bytes, "audio/wav")}, + ) + + assert response.status_code == 200 + parsed = _parse_sse(response.text) + types = [event["type"] for event in parsed] + assert "error" in types + assert "done" not in types + error_event = next(event for event in parsed if event["type"] == "error") + assert "Transcription service error" in error_event["detail"] + + with Session(engine) as session: + assert session.get(AudioTranscription, document_id) is None + + +def test_stream_should_omit_progress_when_duration_unavailable( + asr_test_client, + make_wav_bytes, +): + client, _ = asr_test_client + audio_bytes = make_wav_bytes() + + events = [ + CoroStreamDelta(text="Hola"), + CoroStreamSegments( + segments=[CoroSegment(start=0.0, end=1.0, text="Hola", speaker="1")] + ), + ] + + with ( + patch( + "aymurai.api.endpoints.routers.asr.transcribe.stream_transcribe_audio_bytes", + new=_fake_stream(events), + ), + patch( + "aymurai.api.endpoints.routers.asr.transcribe.probe_audio_duration", + return_value=None, + ), + ): + response = client.post( + "/asr/transcribe/stream?use_cache=false", + files={"file": ("sample.wav", audio_bytes, "audio/wav")}, + ) + + assert response.status_code == 200 + parsed = _parse_sse(response.text) + meta = parsed[0] + assert meta["duration"] is None + delta = next(event for event in parsed if event["type"] == "delta") + assert "progress" not in delta diff --git a/test/audio/test_asr_client.py b/test/audio/test_asr_client.py new file mode 100644 index 00000000..6d4b47ff --- /dev/null +++ b/test/audio/test_asr_client.py @@ -0,0 +1,177 @@ +import asyncio +import json +from unittest.mock import AsyncMock, MagicMock, patch + +import httpx +import pytest +from openai import APIConnectionError + +from aymurai.api.meta.asr.coro import ( + CoroSegment, + CoroStreamDelta, + CoroStreamSegments, +) +from aymurai.audio.asr_client import ( + stream_transcribe_audio_bytes, + transcribe_audio_bytes, +) +from aymurai.settings import settings + + +class _FakeEvent: + def __init__(self, type: str, text: str | None = None, delta: str | None = None): + self.type = type + self.text = text + self.delta = delta + + +async def _collect(agen): + return [item async for item in agen] + + +class _FakeStream: + def __init__(self, events): + self._events = events + + def __aiter__(self): + async def _gen(): + for event in self._events: + yield event + + return _gen() + + +def _patched_client(events): + create = AsyncMock(return_value=_FakeStream(events)) + client = MagicMock() + client.audio.transcriptions.create = create + return patch("aymurai.audio.asr_client.AsyncOpenAI", return_value=client) + + +def test_should_return_segments_when_done_frame_received(monkeypatch): + monkeypatch.setattr(settings, "TRANSCRIBE_BASE_URL", "http://coro.local/v1") + done_text = json.dumps( + { + "segments": [ + {"start": 0.0, "end": 1.0, "text": "Hola", "speaker": "1"}, + {"start": 1.0, "end": 2.0, "text": "mundo", "speaker": "2"}, + ] + } + ) + events = [ + _FakeEvent("transcript.text.delta", None), + _FakeEvent("transcript.text.done", done_text), + ] + with _patched_client(events): + result = asyncio.run( + transcribe_audio_bytes(b"audio", "sample.wav", "audio/wav") + ) + + assert result == [ + CoroSegment(start=0.0, end=1.0, text="Hola", speaker="1"), + CoroSegment(start=1.0, end=2.0, text="mundo", speaker="2"), + ] + + +def test_should_raise_when_stream_has_no_done_frame(monkeypatch): + monkeypatch.setattr(settings, "TRANSCRIBE_BASE_URL", "http://coro.local/v1") + events = [_FakeEvent("transcript.text.delta", None)] + with _patched_client(events): + with pytest.raises(RuntimeError, match="done frame"): + asyncio.run(transcribe_audio_bytes(b"audio", "sample.wav", "audio/wav")) + + +def test_should_raise_when_base_url_not_configured(monkeypatch): + monkeypatch.setattr(settings, "TRANSCRIBE_BASE_URL", None) + with pytest.raises(RuntimeError, match="not configured"): + asyncio.run(transcribe_audio_bytes(b"audio", "sample.wav", "audio/wav")) + + +def test_should_map_openai_connection_error_to_runtime_error(monkeypatch): + monkeypatch.setattr(settings, "TRANSCRIBE_BASE_URL", "http://coro.local/v1") + request = httpx.Request("POST", "http://coro.local/v1/audio/transcriptions") + create = AsyncMock(side_effect=APIConnectionError(request=request)) + client = MagicMock() + client.audio.transcriptions.create = create + with patch("aymurai.audio.asr_client.AsyncOpenAI", return_value=client): + with pytest.raises(RuntimeError, match="Transcription service error"): + asyncio.run(transcribe_audio_bytes(b"audio", "sample.wav", "audio/wav")) + + +# MARK: streaming generator +def test_stream_should_yield_deltas_then_segments(monkeypatch): + monkeypatch.setattr(settings, "TRANSCRIBE_BASE_URL", "http://coro.local/v1") + done_text = json.dumps( + { + "segments": [ + {"start": 0.0, "end": 1.0, "text": "Hola", "speaker": "1"}, + ] + } + ) + events = [ + _FakeEvent("transcript.text.delta", delta="Ho"), + _FakeEvent("transcript.text.delta", delta="la"), + _FakeEvent("transcript.text.done", text=done_text), + ] + with _patched_client(events): + result = asyncio.run( + _collect(stream_transcribe_audio_bytes(b"audio", "sample.wav", "audio/wav")) + ) + + assert result == [ + CoroStreamDelta(text="Ho"), + CoroStreamDelta(text="la"), + CoroStreamSegments( + segments=[CoroSegment(start=0.0, end=1.0, text="Hola", speaker="1")] + ), + ] + + +def test_stream_should_stop_after_done_frame(monkeypatch): + monkeypatch.setattr(settings, "TRANSCRIBE_BASE_URL", "http://coro.local/v1") + done_text = json.dumps({"segments": []}) + events = [ + _FakeEvent("transcript.text.done", text=done_text), + _FakeEvent("transcript.text.delta", delta="ignored"), + ] + with _patched_client(events): + result = asyncio.run( + _collect(stream_transcribe_audio_bytes(b"audio", "sample.wav", "audio/wav")) + ) + + assert result == [CoroStreamSegments(segments=[])] + + +def test_stream_should_raise_when_stream_has_no_done_frame(monkeypatch): + monkeypatch.setattr(settings, "TRANSCRIBE_BASE_URL", "http://coro.local/v1") + events = [_FakeEvent("transcript.text.delta", delta="Ho")] + with _patched_client(events): + with pytest.raises(RuntimeError, match="done frame"): + asyncio.run( + _collect( + stream_transcribe_audio_bytes(b"audio", "sample.wav", "audio/wav") + ) + ) + + +def test_stream_should_raise_when_base_url_not_configured(monkeypatch): + monkeypatch.setattr(settings, "TRANSCRIBE_BASE_URL", None) + with pytest.raises(RuntimeError, match="not configured"): + asyncio.run( + _collect(stream_transcribe_audio_bytes(b"audio", "sample.wav", "audio/wav")) + ) + + +def test_stream_should_map_connection_error_to_runtime_error(monkeypatch): + monkeypatch.setattr(settings, "TRANSCRIBE_BASE_URL", "http://coro.local/v1") + request = httpx.Request("POST", "http://coro.local/v1/audio/transcriptions") + create = AsyncMock(side_effect=APIConnectionError(request=request)) + client = MagicMock() + client.audio.transcriptions.create = create + with patch("aymurai.audio.asr_client.AsyncOpenAI", return_value=client): + with pytest.raises(RuntimeError, match="Transcription service error"): + asyncio.run( + _collect( + stream_transcribe_audio_bytes(b"audio", "sample.wav", "audio/wav") + ) + ) diff --git a/test/audio/test_duration.py b/test/audio/test_duration.py new file mode 100644 index 00000000..0ca75d1c --- /dev/null +++ b/test/audio/test_duration.py @@ -0,0 +1,40 @@ +import io +import math +import struct +import wave + +from aymurai.audio.duration import probe_audio_duration + + +def _make_wav_bytes(duration_seconds: float = 1.0, sample_rate: int = 16000) -> bytes: + frame_count = int(duration_seconds * sample_rate) + pcm = bytearray() + for i in range(frame_count): + sample = int(32767 * 0.2 * math.sin(2 * math.pi * 440.0 * (i / sample_rate))) + pcm.extend(struct.pack(" ASRParagraph: + return ASRParagraph( + speaker_no=speaker_no, + speaker_name=speaker_name, + start=timedelta(seconds=start), + end=timedelta(seconds=end), + text=text, + ) + + +def test_ends_sentence_detects_sentence_closing_punctuation(): + assert ends_sentence("Hola.") + assert ends_sentence("Hola?") + assert ends_sentence("Hola!") + assert ends_sentence("¿Se entiende?") + assert ends_sentence("¡Perfecto!") + assert ends_sentence("Bueno...") + + +def test_ends_sentence_ignores_non_final_punctuation_and_open_text(): + assert not ends_sentence("entonces,") + assert not ends_sentence("lo que queria decir es") + assert not ends_sentence("pero bueno:") + assert not ends_sentence("") + + +def test_merge_combines_consecutive_same_speaker_without_sentence_close(): + segments = diarized_segments( + { + "document": [ + _paragraph(1, 0.0, 1.0, "Nos quedamos,"), + _paragraph(1, 1.0, 2.0, "esperando en el lobby"), + ] + } + ) + + turns = merge_speaker_turns_by_sentence(segments) + + assert len(turns) == 1 + assert turns[0]["speaker"] == "Speaker 1" + assert turns[0]["speaker_no"] == 1 + assert turns[0]["start"] == 0.0 + assert turns[0]["end"] == 2.0 + assert turns[0]["text"] == "Nos quedamos, esperando en el lobby" + assert len(turns[0]["segments"]) == 2 + + +def test_merge_splits_same_speaker_after_sentence_close(): + segments = diarized_segments( + { + "document": [ + _paragraph(1, 0.0, 1.0, "Hola."), + _paragraph(1, 1.0, 2.0, "Seguimos con otro tema"), + ] + } + ) + + turns = merge_speaker_turns_by_sentence(segments) + + assert [turn["text"] for turn in turns] == [ + "Hola.", + "Seguimos con otro tema", + ] + assert turns[0]["end"] == 1.0 + assert turns[1]["start"] == 1.0 + + +def test_merge_splits_when_speaker_changes_even_without_sentence_close(): + segments = diarized_segments( + { + "document": [ + _paragraph(1, 0.0, 1.0, "Entonces,"), + _paragraph(2, 1.0, 2.0, "contesto yo"), + ] + } + ) + + turns = merge_speaker_turns_by_sentence(segments) + + assert [turn["speaker"] for turn in turns] == ["Speaker 1", "Speaker 2"] + assert [turn["text"] for turn in turns] == ["Entonces,", "contesto yo"] + + +def test_merge_preserves_timestamps_across_merged_segments(): + segments = diarized_segments( + { + "document": [ + _paragraph(1, 5.76, 6.24, "Nos quedamos,"), + _paragraph(1, 6.24, 12.48, "esperando en el lobby."), + ] + } + ) + + turns = merge_speaker_turns_by_sentence(segments) + + assert turns[0]["start"] == 5.76 + assert turns[0]["end"] == 12.48 + + +def test_merge_uses_speaker_name_when_available(): + segments = diarized_segments( + { + "document": [ + _paragraph(1, 0.0, 1.0, "Tiene la palabra,", "Jueza"), + _paragraph(1, 1.0, 2.0, "doctora.", "Jueza"), + ] + } + ) + + turns = merge_speaker_turns_by_sentence(segments) + + assert turns[0]["speaker"] == "Jueza" + assert turns[0]["speaker_no"] == 1 + + +def test_diarized_segments_ignores_empty_text_segments(): + segments = diarized_segments( + { + "document": [ + _paragraph(1, 0.0, 1.0, " "), + _paragraph(1, 1.0, 2.0, "Texto util"), + ] + } + ) + + assert len(segments) == 1 + assert segments[0]["_text"] == "Texto util" + + +def test_markdown_transcript_uses_sentence_aware_turns(): + with pytest.warns(UserWarning, match="Only one speaker label"): + markdown = markdown_transcript( + { + "document": [ + _paragraph(1, 0.0, 1.0, "Hola."), + _paragraph(1, 1.0, 2.0, "Seguimos."), + ] + }, + title="Audiencia", + ) + + assert "## Speaker 1 (00:00:00.000 - 00:00:01.000)" in markdown + assert "## Speaker 1 (00:00:01.000 - 00:00:02.000)" in markdown + assert "Hola.\n\n## Speaker 1" in markdown + + +def test_validated_paragraphs_become_one_turn_each_without_merging(): + paragraphs = [ + _paragraph(1, 0.0, 1.0, "Texto editado sin cierre", "Docente"), + _paragraph(1, 1.0, 2.5, "Sigue siendo otro bloque", "Docente"), + ] + + turns = speaker_turns_from_validated_paragraphs(paragraphs) + + assert [turn["text"] for turn in turns] == [ + "Texto editado sin cierre", + "Sigue siendo otro bloque", + ] + assert [turn["speaker"] for turn in turns] == ["Docente", "Docente"] + assert turns[0]["start"] == "00:00:00.000" + assert turns[1]["end"] == "00:00:02.500" + assert turns[0]["segments"][0]["text"] == "Texto editado sin cierre" + + +def test_validated_paragraphs_ignore_empty_text(): + paragraphs = [ + _paragraph(1, 0.0, 1.0, " "), + _paragraph(1, 1.0, 2.0, "Texto validado"), + ] + + turns = speaker_turns_from_validated_paragraphs(paragraphs) + + assert len(turns) == 1 + assert turns[0]["text"] == "Texto validado" diff --git a/test/integration/test_asr_pipeline.py b/test/integration/test_asr_pipeline.py index 21ff6a34..7857ee88 100644 --- a/test/integration/test_asr_pipeline.py +++ b/test/integration/test_asr_pipeline.py @@ -19,8 +19,8 @@ def test_should_anonymize_audio_document_when_running_real_asr_and_anonymizer_en isolated_diskcache, integration_audio_bytes, ): - if not settings.TRANSCRIBE_WS_URI: - pytest.skip("TRANSCRIBE_WS_URI is required for real ASR integration test") + if not settings.TRANSCRIBE_BASE_URL: + pytest.skip("TRANSCRIBE_BASE_URL is required for real ASR integration test") if shutil.which(settings.LIBREOFFICE_BIN) is None: pytest.skip("LibreOffice binary is required for /anonymizer/anonymize-document") diff --git a/uv.lock b/uv.lock index b41e1967..073b870a 100644 --- a/uv.lock +++ b/uv.lock @@ -7,6 +7,9 @@ resolution-markers = [ "(platform_machine != 'aarch64' and sys_platform == 'linux') or (sys_platform != 'darwin' and sys_platform != 'linux')", ] +[manifest] +overrides = [{ name = "openai", specifier = ">=2.43.0" }] + [[package]] name = "absl-py" version = "2.4.0" @@ -283,15 +286,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3a/2a/7cc015f5b9f5db42b7d48157e23356022889fc354a2813c15934b7cb5c0e/attrs-25.4.0-py3-none-any.whl", hash = "sha256:adcf7e2a1fb3b36ac48d97835bb6d8ade15b8dcce26aba8bf1d14847b57a3373", size = 67615, upload-time = "2025-10-06T13:54:43.17Z" }, ] -[[package]] -name = "audioread" -version = "3.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a1/4a/874ecf9b472f998130c2b5e145dcdb9f6131e84786111489103b66772143/audioread-3.1.0.tar.gz", hash = "sha256:1c4ab2f2972764c896a8ac61ac53e261c8d29f0c6ccd652f84e18f08a4cab190", size = 20082, upload-time = "2025-10-26T19:44:13.484Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7e/16/fbe8e1e185a45042f7cd3a282def5bb8d95bb69ab9e9ef6a5368aa17e426/audioread-3.1.0-py3-none-any.whl", hash = "sha256:b30d1df6c5d3de5dcef0fb0e256f6ea17bdcf5f979408df0297d8a408e2971b4", size = 23143, upload-time = "2025-10-26T19:44:12.016Z" }, -] - [[package]] name = "aymurai" source = { editable = "." } @@ -308,12 +302,12 @@ dependencies = [ { name = "jiwer" }, { name = "joblib" }, { name = "langextract", extra = ["openai"] }, - { name = "librosa" }, { name = "marker-pdf" }, { name = "more-itertools" }, { name = "numpy" }, { name = "odfpy" }, { name = "ollama" }, + { name = "openai" }, { name = "psutil" }, { name = "pydantic" }, { name = "pydantic-settings" }, @@ -374,12 +368,12 @@ requires-dist = [ { name = "jiwer", specifier = "==3.0.5" }, { name = "joblib", specifier = ">=1.4.2" }, { name = "langextract", extras = ["openai"], specifier = "==1.1.0" }, - { name = "librosa", specifier = ">=0.10.2" }, { name = "marker-pdf", specifier = "==1.10.1" }, { name = "more-itertools", specifier = ">=10.5.0" }, { name = "numpy", specifier = "<2.0.0" }, { name = "odfpy", specifier = ">=1.4.1" }, { name = "ollama", specifier = "==0.6.1" }, + { name = "openai", specifier = ">=2.43.0" }, { name = "psutil", specifier = "==6.1.0" }, { name = "pydantic", specifier = ">=2.10.4" }, { name = "pydantic-settings", specifier = ">=2.7.0" }, @@ -2186,42 +2180,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/82/3d/14ce75ef66813643812f3093ab17e46d3a206942ce7376d31ec2d36229e7/lark-1.3.1-py3-none-any.whl", hash = "sha256:c629b661023a014c37da873b4ff58a817398d12635d3bbb2c5a03be7fe5d1e12", size = 113151, upload-time = "2025-10-27T18:25:54.882Z" }, ] -[[package]] -name = "lazy-loader" -version = "0.4" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "packaging" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/6f/6b/c875b30a1ba490860c93da4cabf479e03f584eba06fe5963f6f6644653d8/lazy_loader-0.4.tar.gz", hash = "sha256:47c75182589b91a4e1a85a136c074285a5ad4d9f39c63e0d7fb76391c4574cd1", size = 15431, upload-time = "2024-04-05T13:03:12.261Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/83/60/d497a310bde3f01cb805196ac61b7ad6dc5dcf8dce66634dc34364b20b4f/lazy_loader-0.4-py3-none-any.whl", hash = "sha256:342aa8e14d543a154047afb4ba8ef17f5563baad3fc610d7b15b213b0f119efc", size = 12097, upload-time = "2024-04-05T13:03:10.514Z" }, -] - -[[package]] -name = "librosa" -version = "0.11.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "audioread" }, - { name = "decorator" }, - { name = "joblib" }, - { name = "lazy-loader" }, - { name = "msgpack" }, - { name = "numba" }, - { name = "numpy" }, - { name = "pooch" }, - { name = "scikit-learn" }, - { name = "scipy" }, - { name = "soundfile" }, - { name = "soxr" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/64/36/360b5aafa0238e29758729e9486c6ed92a6f37fa403b7875e06c115cdf4a/librosa-0.11.0.tar.gz", hash = "sha256:f5ed951ca189b375bbe2e33b2abd7e040ceeee302b9bbaeeffdfddb8d0ace908", size = 327001, upload-time = "2025-03-11T15:09:54.884Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b5/ba/c63c5786dfee4c3417094c4b00966e61e4a63efecee22cb7b4c0387dda83/librosa-0.11.0-py3-none-any.whl", hash = "sha256:0b6415c4fd68bff4c29288abe67c6d80b587e0e1e2cfb0aad23e4559504a7fa1", size = 260749, upload-time = "2025-03-11T15:09:52.982Z" }, -] - [[package]] name = "lightning-utilities" version = "0.3.0" @@ -2257,18 +2215,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ea/53/aa31e4d057b3746b3c323ca993003d6cf15ef987e7fe7ceb53681695ae87/litellm-1.80.0-py3-none-any.whl", hash = "sha256:fd0009758f4772257048d74bf79bb64318859adb4ea49a8b66fdbc718cd80b6e", size = 10492975, upload-time = "2025-11-16T00:03:49.182Z" }, ] -[[package]] -name = "llvmlite" -version = "0.46.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/74/cd/08ae687ba099c7e3d21fe2ea536500563ef1943c5105bf6ab4ee3829f68e/llvmlite-0.46.0.tar.gz", hash = "sha256:227c9fd6d09dce2783c18b754b7cd9d9b3b3515210c46acc2d3c5badd9870ceb", size = 193456, upload-time = "2025-12-08T18:15:36.295Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3d/a4/3959e1c61c5ca9db7921e5fd115b344c29b9d57a5dadd87bef97963ca1a5/llvmlite-0.46.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4323177e936d61ae0f73e653e2e614284d97d14d5dd12579adc92b6c2b0597b0", size = 37232766, upload-time = "2025-12-08T18:14:34.765Z" }, - { url = "https://files.pythonhosted.org/packages/c2/a5/a4d916f1015106e1da876028606a8e87fd5d5c840f98c87bc2d5153b6a2f/llvmlite-0.46.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0a2d461cb89537b7c20feb04c46c32e12d5ad4f0896c9dfc0f60336219ff248e", size = 56275176, upload-time = "2025-12-08T18:14:37.944Z" }, - { url = "https://files.pythonhosted.org/packages/79/7f/a7f2028805dac8c1a6fae7bda4e739b7ebbcd45b29e15bf6d21556fcd3d5/llvmlite-0.46.0-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b1f6595a35b7b39c3518b85a28bf18f45e075264e4b2dce3f0c2a4f232b4a910", size = 55128629, upload-time = "2025-12-08T18:14:41.674Z" }, - { url = "https://files.pythonhosted.org/packages/b2/bc/4689e1ba0c073c196b594471eb21be0aa51d9e64b911728aa13cd85ef0ae/llvmlite-0.46.0-cp310-cp310-win_amd64.whl", hash = "sha256:e7a34d4aa6f9a97ee006b504be6d2b8cb7f755b80ab2f344dda1ef992f828559", size = 38138651, upload-time = "2025-12-08T18:14:45.845Z" }, -] - [[package]] name = "lxml" version = "6.0.2" @@ -2580,22 +2526,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/43/e3/7d92a15f894aa0c9c4b49b8ee9ac9850d6e63b03c9c32c0367a13ae62209/mpmath-1.3.0-py3-none-any.whl", hash = "sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c", size = 536198, upload-time = "2023-03-07T16:47:09.197Z" }, ] -[[package]] -name = "msgpack" -version = "1.1.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/4d/f2/bfb55a6236ed8725a96b0aa3acbd0ec17588e6a2c3b62a93eb513ed8783f/msgpack-1.1.2.tar.gz", hash = "sha256:3b60763c1373dd60f398488069bcdc703cd08a711477b5d480eecc9f9626f47e", size = 173581, upload-time = "2025-10-08T09:15:56.596Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f5/a2/3b68a9e769db68668b25c6108444a35f9bd163bb848c0650d516761a59c0/msgpack-1.1.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0051fffef5a37ca2cd16978ae4f0aef92f164df86823871b5162812bebecd8e2", size = 81318, upload-time = "2025-10-08T09:14:38.722Z" }, - { url = "https://files.pythonhosted.org/packages/5b/e1/2b720cc341325c00be44e1ed59e7cfeae2678329fbf5aa68f5bda57fe728/msgpack-1.1.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a605409040f2da88676e9c9e5853b3449ba8011973616189ea5ee55ddbc5bc87", size = 83786, upload-time = "2025-10-08T09:14:40.082Z" }, - { url = "https://files.pythonhosted.org/packages/71/e5/c2241de64bfceac456b140737812a2ab310b10538a7b34a1d393b748e095/msgpack-1.1.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8b696e83c9f1532b4af884045ba7f3aa741a63b2bc22617293a2c6a7c645f251", size = 398240, upload-time = "2025-10-08T09:14:41.151Z" }, - { url = "https://files.pythonhosted.org/packages/b7/09/2a06956383c0fdebaef5aa9246e2356776f12ea6f2a44bd1368abf0e46c4/msgpack-1.1.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:365c0bbe981a27d8932da71af63ef86acc59ed5c01ad929e09a0b88c6294e28a", size = 406070, upload-time = "2025-10-08T09:14:42.821Z" }, - { url = "https://files.pythonhosted.org/packages/0e/74/2957703f0e1ef20637d6aead4fbb314330c26f39aa046b348c7edcf6ca6b/msgpack-1.1.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:41d1a5d875680166d3ac5c38573896453bbbea7092936d2e107214daf43b1d4f", size = 393403, upload-time = "2025-10-08T09:14:44.38Z" }, - { url = "https://files.pythonhosted.org/packages/a5/09/3bfc12aa90f77b37322fc33e7a8a7c29ba7c8edeadfa27664451801b9860/msgpack-1.1.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:354e81bcdebaab427c3df4281187edc765d5d76bfb3a7c125af9da7a27e8458f", size = 398947, upload-time = "2025-10-08T09:14:45.56Z" }, - { url = "https://files.pythonhosted.org/packages/4b/4f/05fcebd3b4977cb3d840f7ef6b77c51f8582086de5e642f3fefee35c86fc/msgpack-1.1.2-cp310-cp310-win32.whl", hash = "sha256:e64c8d2f5e5d5fda7b842f55dec6133260ea8f53c4257d64494c534f306bf7a9", size = 64769, upload-time = "2025-10-08T09:14:47.334Z" }, - { url = "https://files.pythonhosted.org/packages/d0/3e/b4547e3a34210956382eed1c85935fff7e0f9b98be3106b3745d7dec9c5e/msgpack-1.1.2-cp310-cp310-win_amd64.whl", hash = "sha256:db6192777d943bdaaafb6ba66d44bf65aa0e9c5616fa1d2da9bb08828c6b39aa", size = 71293, upload-time = "2025-10-08T09:14:48.665Z" }, -] - [[package]] name = "multidict" version = "6.7.1" @@ -2774,22 +2704,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f9/33/bd5b9137445ea4b680023eb0469b2bb969d61303dedb2aac6560ff3d14a1/notebook_shim-0.2.4-py3-none-any.whl", hash = "sha256:411a5be4e9dc882a074ccbcae671eda64cceb068767e9a3419096986560e1cef", size = 13307, upload-time = "2024-02-14T23:35:16.286Z" }, ] -[[package]] -name = "numba" -version = "0.64.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "llvmlite" }, - { name = "numpy" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/23/c9/a0fb41787d01d621046138da30f6c2100d80857bf34b3390dd68040f27a3/numba-0.64.0.tar.gz", hash = "sha256:95e7300af648baa3308127b1955b52ce6d11889d16e8cfe637b4f85d2fca52b1", size = 2765679, upload-time = "2026-02-18T18:41:20.974Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/4c/5e/604fed821cd7e3426bb3bc99a7ed6ac0bcb489f4cd93052256437d082f95/numba-0.64.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:cc09b79440952e3098eeebea4bf6e8d2355fb7f12734fcd9fc5039f0dca90727", size = 2683250, upload-time = "2026-02-18T18:40:45.829Z" }, - { url = "https://files.pythonhosted.org/packages/4f/9f/9275a723d050b5f1a9b1c7fb7dbfce324fef301a8e50c5f88338569db06c/numba-0.64.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1afe3a80b8c2f376b211fb7a49e536ef9eafc92436afc95a2f41ea5392f8cc65", size = 3742168, upload-time = "2026-02-18T18:40:48.066Z" }, - { url = "https://files.pythonhosted.org/packages/e2/d1/97ca7dddaa36b16f4c46319bdb6b4913ba15d0245317d0d8ccde7b2d7d92/numba-0.64.0-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:23804194b93b8cd416c6444b5fbc4956082a45fed2d25436ef49c594666e7f7e", size = 3449103, upload-time = "2026-02-18T18:40:49.905Z" }, - { url = "https://files.pythonhosted.org/packages/52/0a/b9e137ad78415373e3353564500e8bf29dbce3c0d73633bb384d4e5d7537/numba-0.64.0-cp310-cp310-win_amd64.whl", hash = "sha256:e2a9fe998bb2cf848960b34db02c2c3b5e02cf82c07a26d9eef3494069740278", size = 2749950, upload-time = "2026-02-18T18:40:51.536Z" }, -] - [[package]] name = "numpy" version = "1.26.4" @@ -2964,7 +2878,7 @@ wheels = [ [[package]] name = "openai" -version = "1.109.1" +version = "2.43.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, @@ -2976,9 +2890,9 @@ dependencies = [ { name = "tqdm" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/c6/a1/a303104dc55fc546a3f6914c842d3da471c64eec92043aef8f652eb6c524/openai-1.109.1.tar.gz", hash = "sha256:d173ed8dbca665892a6db099b4a2dfac624f94d20a93f46eb0b56aae940ed869", size = 564133, upload-time = "2025-09-24T13:00:53.075Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f3/fa/88d0c58a0c58df7e6758e66b99c5d028d5e0bb49f8812d7203940cd9dbf1/openai-2.43.0.tar.gz", hash = "sha256:e74d238200a26868977002190fb6631613480a93dfe0c9c982e77021ed60a017", size = 785369, upload-time = "2026-06-17T17:06:56.06Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/1d/2a/7dd3d207ec669cacc1f186fd856a0f61dbc255d24f6fdc1a6715d6051b0f/openai-1.109.1-py3-none-any.whl", hash = "sha256:6bcaf57086cf59159b8e27447e4e7dd019db5d29a438072fbd49c290c7e65315", size = 948627, upload-time = "2025-09-24T13:00:50.754Z" }, + { url = "https://files.pythonhosted.org/packages/a3/d2/ba767f4bbb30776c03d40906a2d3afad716a165ffa1771fc23b8992f7920/openai-2.43.0-py3-none-any.whl", hash = "sha256:65a670b54fadf2268c9e1330133373c963eb779ee969e5cbad419ec2c21dce97", size = 1355077, upload-time = "2026-06-17T17:06:53.614Z" }, ] [[package]] @@ -3258,20 +3172,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, ] -[[package]] -name = "pooch" -version = "1.9.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "packaging" }, - { name = "platformdirs" }, - { name = "requests" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/83/43/85ef45e8b36c6a48546af7b266592dc32d7f67837a6514d111bced6d7d75/pooch-1.9.0.tar.gz", hash = "sha256:de46729579b9857ffd3e741987a2f6d5e0e03219892c167c6578c0091fb511ed", size = 61788, upload-time = "2026-01-30T19:15:09.649Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2a/2d/d4bf65e47cea8ff2c794a600c4fd1273a7902f268757c531e0ee9f18aa58/pooch-1.9.0-py3-none-any.whl", hash = "sha256:f265597baa9f760d25ceb29d0beb8186c243d6607b0f60b83ecf14078dbc703b", size = 67175, upload-time = "2026-01-30T19:15:08.36Z" }, -] - [[package]] name = "pptree" version = "3.1" @@ -4277,25 +4177,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0", size = 29575, upload-time = "2021-05-16T22:03:41.177Z" }, ] -[[package]] -name = "soundfile" -version = "0.13.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cffi" }, - { name = "numpy" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/e1/41/9b873a8c055582859b239be17902a85339bec6a30ad162f98c9b0288a2cc/soundfile-0.13.1.tar.gz", hash = "sha256:b2c68dab1e30297317080a5b43df57e302584c49e2942defdde0acccc53f0e5b", size = 46156, upload-time = "2025-01-25T09:17:04.831Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/64/28/e2a36573ccbcf3d57c00626a21fe51989380636e821b341d36ccca0c1c3a/soundfile-0.13.1-py2.py3-none-any.whl", hash = "sha256:a23c717560da2cf4c7b5ae1142514e0fd82d6bbd9dfc93a50423447142f2c445", size = 25751, upload-time = "2025-01-25T09:16:44.235Z" }, - { url = "https://files.pythonhosted.org/packages/ea/ab/73e97a5b3cc46bba7ff8650a1504348fa1863a6f9d57d7001c6b67c5f20e/soundfile-0.13.1-py2.py3-none-macosx_10_9_x86_64.whl", hash = "sha256:82dc664d19831933fe59adad199bf3945ad06d84bc111a5b4c0d3089a5b9ec33", size = 1142250, upload-time = "2025-01-25T09:16:47.583Z" }, - { url = "https://files.pythonhosted.org/packages/a0/e5/58fd1a8d7b26fc113af244f966ee3aecf03cb9293cb935daaddc1e455e18/soundfile-0.13.1-py2.py3-none-macosx_11_0_arm64.whl", hash = "sha256:743f12c12c4054921e15736c6be09ac26b3b3d603aef6fd69f9dde68748f2593", size = 1101406, upload-time = "2025-01-25T09:16:49.662Z" }, - { url = "https://files.pythonhosted.org/packages/58/ae/c0e4a53d77cf6e9a04179535766b3321b0b9ced5f70522e4caf9329f0046/soundfile-0.13.1-py2.py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:9c9e855f5a4d06ce4213f31918653ab7de0c5a8d8107cd2427e44b42df547deb", size = 1235729, upload-time = "2025-01-25T09:16:53.018Z" }, - { url = "https://files.pythonhosted.org/packages/57/5e/70bdd9579b35003a489fc850b5047beeda26328053ebadc1fb60f320f7db/soundfile-0.13.1-py2.py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:03267c4e493315294834a0870f31dbb3b28a95561b80b134f0bd3cf2d5f0e618", size = 1313646, upload-time = "2025-01-25T09:16:54.872Z" }, - { url = "https://files.pythonhosted.org/packages/fe/df/8c11dc4dfceda14e3003bb81a0d0edcaaf0796dd7b4f826ea3e532146bba/soundfile-0.13.1-py2.py3-none-win32.whl", hash = "sha256:c734564fab7c5ddf8e9be5bf70bab68042cd17e9c214c06e365e20d64f9a69d5", size = 899881, upload-time = "2025-01-25T09:16:56.663Z" }, - { url = "https://files.pythonhosted.org/packages/14/e9/6b761de83277f2f02ded7e7ea6f07828ec78e4b229b80e4ca55dd205b9dc/soundfile-0.13.1-py2.py3-none-win_amd64.whl", hash = "sha256:1e70a05a0626524a69e9f0f4dd2ec174b4e9567f4d8b6c11d38b5c289be36ee9", size = 1019162, upload-time = "2025-01-25T09:16:59.573Z" }, -] - [[package]] name = "soupsieve" version = "2.8.3" @@ -4305,27 +4186,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/46/2c/1462b1d0a634697ae9e55b3cecdcb64788e8b7d63f54d923fcd0bb140aed/soupsieve-2.8.3-py3-none-any.whl", hash = "sha256:ed64f2ba4eebeab06cc4962affce381647455978ffc1e36bb79a545b91f45a95", size = 37016, upload-time = "2026-01-20T04:27:01.012Z" }, ] -[[package]] -name = "soxr" -version = "1.0.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/42/7e/f4b461944662ad75036df65277d6130f9411002bfb79e9df7dff40a31db9/soxr-1.0.0.tar.gz", hash = "sha256:e07ee6c1d659bc6957034f4800c60cb8b98de798823e34d2a2bba1caa85a4509", size = 171415, upload-time = "2025-09-07T13:22:21.317Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1e/a7/11c36d71595b52fe84a220040ace679035953acf06b83bf2c7117c565d2c/soxr-1.0.0-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:b876a3156f67c76aef0cff1084eaf4088d9ca584bb569cb993f89a52ec5f399f", size = 206459, upload-time = "2025-09-07T13:21:46.904Z" }, - { url = "https://files.pythonhosted.org/packages/43/5e/8962f2aeea7777d2a6e65a24a2b83c6aea1a28badeda027fd328f7f03bb7/soxr-1.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4d3b957a7b0cc19ae6aa45d40b2181474e53a8dd00efd7bce6bcf4e60e020892", size = 164808, upload-time = "2025-09-07T13:21:48.83Z" }, - { url = "https://files.pythonhosted.org/packages/fc/91/00384166f110a3888ea8efd44523ba7168dd2dc39e3e43c931cc2d069fa9/soxr-1.0.0-cp310-cp310-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b89685faedebc45af71f08f9957b61cc6143bc94ba43fe38e97067f81e272969", size = 208586, upload-time = "2025-09-07T13:21:50.341Z" }, - { url = "https://files.pythonhosted.org/packages/75/34/e18f1003e242aabed44ed8902534814d3e64209e4d1d874f5b9b67d73cde/soxr-1.0.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d255741b2f0084fd02d4a2ddd77cd495be9e7e7b6f9dba1c9494f86afefac65b", size = 242310, upload-time = "2025-09-07T13:21:51.56Z" }, - { url = "https://files.pythonhosted.org/packages/61/9c/a1c5ed106b40cc1e2e12cd58831b7f1b61c5fbdb8eceeca4b3a0b0dbef6c/soxr-1.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:158a4a9055958c4b95ef91dbbe280cabb00946b5423b25a9b0ce31bd9e0a271e", size = 173561, upload-time = "2025-09-07T13:21:53.03Z" }, - { url = "https://files.pythonhosted.org/packages/c5/c7/f92b81f1a151c13afb114f57799b86da9330bec844ea5a0d3fe6a8732678/soxr-1.0.0-cp312-abi3-macosx_10_14_x86_64.whl", hash = "sha256:abecf4e39017f3fadb5e051637c272ae5778d838e5c3926a35db36a53e3a607f", size = 205508, upload-time = "2025-09-07T13:22:01.252Z" }, - { url = "https://files.pythonhosted.org/packages/ff/1d/c945fea9d83ea1f2be9d116b3674dbaef26ed090374a77c394b31e3b083b/soxr-1.0.0-cp312-abi3-macosx_11_0_arm64.whl", hash = "sha256:e973d487ee46aa8023ca00a139db6e09af053a37a032fe22f9ff0cc2e19c94b4", size = 163568, upload-time = "2025-09-07T13:22:03.558Z" }, - { url = "https://files.pythonhosted.org/packages/b5/80/10640970998a1d2199bef6c4d92205f36968cddaf3e4d0e9fe35ddd405bd/soxr-1.0.0-cp312-abi3-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e8ce273cca101aff3d8c387db5a5a41001ba76ef1837883438d3c652507a9ccc", size = 204707, upload-time = "2025-09-07T13:22:05.125Z" }, - { url = "https://files.pythonhosted.org/packages/b1/87/2726603c13c2126cb8ded9e57381b7377f4f0df6ba4408e1af5ddbfdc3dd/soxr-1.0.0-cp312-abi3-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e8f2a69686f2856d37823bbb7b78c3d44904f311fe70ba49b893af11d6b6047b", size = 238032, upload-time = "2025-09-07T13:22:06.428Z" }, - { url = "https://files.pythonhosted.org/packages/ce/04/530252227f4d0721a5524a936336485dfb429bb206a66baf8e470384f4a2/soxr-1.0.0-cp312-abi3-win_amd64.whl", hash = "sha256:2a3b77b115ae7c478eecdbd060ed4f61beda542dfb70639177ac263aceda42a2", size = 172070, upload-time = "2025-09-07T13:22:07.62Z" }, -] - [[package]] name = "sqlalchemy" version = "2.0.48"