From e2e1f4d1679a24e388018038eb2606ad0dd57c30 Mon Sep 17 00:00:00 2001 From: Sami Jawhar Date: Sat, 4 Jul 2026 23:56:08 +0000 Subject: [PATCH 1/5] feat(passthrough): multi-provider capture to request_logs (M1.1+M1.2) --- pyproject.toml | 2 + src/luthien_proxy/dependencies.py | 3 + src/luthien_proxy/main.py | 12 + src/luthien_proxy/passthrough_capture.py | 138 ++++++++ src/luthien_proxy/passthrough_routes.py | 276 ++++++++++++++++ src/luthien_proxy/request_log/recorder.py | 14 +- src/luthien_proxy/request_log/sanitize.py | 16 +- .../e2e_tests/test_passthrough_capture.py | 306 ++++++++++++++++++ .../unit_tests/test_passthrough_routes.py | 143 ++++++++ uv.lock | 32 ++ 10 files changed, 938 insertions(+), 4 deletions(-) create mode 100644 src/luthien_proxy/passthrough_capture.py create mode 100644 src/luthien_proxy/passthrough_routes.py create mode 100644 tests/luthien_proxy/e2e_tests/test_passthrough_capture.py create mode 100644 tests/luthien_proxy/unit_tests/test_passthrough_routes.py diff --git a/pyproject.toml b/pyproject.toml index e7c0eeb5f..97d0cc6fc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,6 +48,7 @@ dependencies = [ "anthropic>=0.84.0", "aiohttp>=3.9.0", "sentry-sdk[fastapi]>=2.54.0", + "fastapi>=0.115.0", ] [tool.hatch.version] @@ -141,6 +142,7 @@ dev = [ "luthien-cli", "pytest-httpx>=0.35.0", "pytest-xdist>=3.6.0", + "basedpyright>=1.39.9", ] [tool.uv.sources] diff --git a/src/luthien_proxy/dependencies.py b/src/luthien_proxy/dependencies.py index 666767691..2a1de4e05 100644 --- a/src/luthien_proxy/dependencies.py +++ b/src/luthien_proxy/dependencies.py @@ -5,6 +5,7 @@ from dataclasses import dataclass, field from typing import Any +import httpx from fastapi import Depends, HTTPException, Request from redis.asyncio import Redis @@ -49,6 +50,8 @@ class Dependencies: rate_limiter: TokenBucketRateLimiter | None = field(default=None) last_credential_info: dict[str, Any] = field(default_factory=dict) webhook_sender: WebhookSender | None = field(default=None) + passthrough_streaming_client: httpx.AsyncClient | None = field(default=None) + passthrough_buffered_client: httpx.AsyncClient | None = field(default=None) def get_anthropic_policy(self) -> AnthropicExecutionInterface: """Get the current Anthropic policy. diff --git a/src/luthien_proxy/main.py b/src/luthien_proxy/main.py index 0e241944e..4f161f8f6 100644 --- a/src/luthien_proxy/main.py +++ b/src/luthien_proxy/main.py @@ -11,6 +11,7 @@ from collections.abc import MutableMapping from contextlib import asynccontextmanager +import httpx import uvicorn from fastapi import FastAPI, Request from fastapi.exceptions import HTTPException as FastAPIHTTPException @@ -40,6 +41,7 @@ ) from luthien_proxy.observability.redis_event_publisher import RedisEventPublisher from luthien_proxy.observability.sentry import init_sentry +from luthien_proxy.passthrough_routes import router as passthrough_router from luthien_proxy.pipeline.upstream_headers import validate_upstream_headers_at_startup from luthien_proxy.policy_manager import PolicyManager from luthien_proxy.rate_limit import TokenBucketRateLimiter @@ -278,6 +280,11 @@ async def lifespan(app: FastAPI): if _enable_request_logging: logger.info("Request/response logging ENABLED") + _passthrough_streaming_client = httpx.AsyncClient( + timeout=httpx.Timeout(connect=10.0, read=300.0, write=10.0, pool=30.0) + ) + _passthrough_buffered_client = httpx.AsyncClient(timeout=120.0) + # Initialize usage telemetry settings = get_settings() _telemetry_config = await resolve_telemetry_config( @@ -381,6 +388,8 @@ async def lifespan(app: FastAPI): config_registry=_config_registry, rate_limiter=_rate_limiter, webhook_sender=_webhook_sender, + passthrough_streaming_client=_passthrough_streaming_client, + passthrough_buffered_client=_passthrough_buffered_client, ) # Store dependencies container in app state @@ -403,6 +412,8 @@ async def lifespan(app: FastAPI): await _purger.stop() if _telemetry_sender is not None: await _telemetry_sender.stop() + await _passthrough_streaming_client.aclose() + await _passthrough_buffered_client.aclose() await _inference_provider_registry.close() await _credential_manager.close() await anthropic_client_cache.close_all() @@ -464,6 +475,7 @@ async def dispatch(self, request: Request, call_next): # Include routers app.include_router(gateway_router) # /v1/messages + app.include_router(passthrough_router) # /openai/* and /gemini/* app.include_router(debug_router) # /api/debug/* app.include_router(ui_router) # /activity/*, /policy-config, /diffs app.include_router(admin_router) # /api/admin/* (policy management) diff --git a/src/luthien_proxy/passthrough_capture.py b/src/luthien_proxy/passthrough_capture.py new file mode 100644 index 000000000..07303d372 --- /dev/null +++ b/src/luthien_proxy/passthrough_capture.py @@ -0,0 +1,138 @@ +"""Capture helpers for multi-provider passthrough request_logs.""" + +from __future__ import annotations + +import json +from collections.abc import Iterable, Mapping +from typing import Literal + +JsonValue = None | bool | int | float | str | list["JsonValue"] | dict[str, "JsonValue"] +JsonObject = dict[str, JsonValue] +StreamFormat = Literal["openai-sse", "gemini-json-array", "gemini-sse"] + +_HOP_BY_HOP_HEADERS = frozenset( + { + "connection", + "keep-alive", + "proxy-authenticate", + "proxy-authorization", + "te", + "trailer", + "transfer-encoding", + "upgrade", + } +) +_REQUEST_STRIPPED_HEADERS = _HOP_BY_HOP_HEADERS | {"host", "content-length"} +_RESPONSE_STRIPPED_HEADERS = _HOP_BY_HOP_HEADERS | {"content-encoding", "content-length"} + + +def build_passthrough_headers(headers: Iterable[tuple[str, str]]) -> dict[str, str]: + """Return client headers safe for upstream forwarding without server-key injection.""" + forwarded: dict[str, str] = {} + for key, value in headers: + lower_key = key.lower() + if lower_key in _REQUEST_STRIPPED_HEADERS or lower_key.startswith("x-luthien-"): + continue + forwarded[key] = value + return forwarded + + +def parse_openai_model(body: Mapping[str, JsonValue], override: str | None) -> str | None: + """Return the request_logs model value for OpenAI passthrough calls.""" + if override: + return override + model = body.get("model") + return model if isinstance(model, str) else None + + +def parse_gemini_model(path: str, body: Mapping[str, JsonValue], override: str | None) -> str | None: + """Return the request_logs model value for Gemini passthrough calls.""" + if override: + return override + marker = "models/" + if marker in path: + model_path = path.split(marker, maxsplit=1)[1] + return model_path.split(":", maxsplit=1)[0] + model = body.get("model") + return model if isinstance(model, str) else None + + +def reassemble_openai_sse_stream(chunks: Iterable[bytes]) -> JsonObject: + """Stable wrapper: {"stream_format":"openai-sse","events":[...],"final":last_event_or_null}.""" + raw = _stream_text(chunks) + events, complete = _decode_sse_events(raw, skip_done=True) + final = events[-1] if events else None + response: JsonObject = {"stream_format": "openai-sse", "events": events, "final": final} + if not complete: + response["raw"] = raw + return response + + +def reassemble_gemini_json_array_stream(chunks: Iterable[bytes]) -> JsonObject: + """Stable wrapper: {"stream_format":"gemini-json-array","chunks":[...],"final":null}.""" + raw = _stream_text(chunks) + try: + parsed = json_loads(raw) + except ValueError: + return _raw_stream_capture("gemini-json-array", [], raw) + stream_chunks: list[JsonValue] = parsed if isinstance(parsed, list) else [parsed] + return {"stream_format": "gemini-json-array", "chunks": stream_chunks, "final": None} + + +def reassemble_gemini_sse_stream(chunks: Iterable[bytes]) -> JsonObject: + """Stable wrapper: {"stream_format":"gemini-sse","chunks":[...],"final":null}.""" + raw = _stream_text(chunks) + stream_chunks, complete = _decode_sse_events(raw, skip_done=False) + response: JsonObject = {"stream_format": "gemini-sse", "chunks": stream_chunks, "final": None} + if not complete: + response["raw"] = raw + return response + + +def json_loads(raw: str | bytes) -> JsonValue: + """Parse JSON into the passthrough capture JSON value type.""" + return json.loads(raw) + + +def _decode_sse_events(raw: str, *, skip_done: bool) -> tuple[list[JsonValue], bool]: + payloads = _sse_data_payloads_from_text(raw) + events: list[JsonValue] = [] + considered = 0 + for payload in payloads: + if skip_done and payload == "[DONE]": + continue + considered += 1 + try: + events.append(json_loads(payload)) + except ValueError: + continue + return events, len(events) == considered + + +def _sse_data_payloads_from_text(text: str) -> list[str]: + payloads: list[str] = [] + for line in text.splitlines(): + if line.startswith("data:"): + payloads.append(line.removeprefix("data:").strip()) + return payloads + + +def _stream_text(chunks: Iterable[bytes]) -> str: + return b"".join(chunks).decode(errors="replace") + + +def _raw_stream_capture(stream_format: StreamFormat, chunks: list[JsonValue], raw: str) -> JsonObject: + return {"stream_format": stream_format, "chunks": chunks, "raw": raw, "final": None} + + +__all__ = [ + "JsonObject", + "JsonValue", + "build_passthrough_headers", + "json_loads", + "parse_gemini_model", + "parse_openai_model", + "reassemble_gemini_json_array_stream", + "reassemble_gemini_sse_stream", + "reassemble_openai_sse_stream", +] diff --git a/src/luthien_proxy/passthrough_routes.py b/src/luthien_proxy/passthrough_routes.py new file mode 100644 index 000000000..34203fb03 --- /dev/null +++ b/src/luthien_proxy/passthrough_routes.py @@ -0,0 +1,276 @@ +"""OpenAI and Gemini passthrough routes with full request_log body capture.""" + +from __future__ import annotations + +import os +import uuid +from collections.abc import AsyncIterator +from dataclasses import dataclass +from typing import Literal +from urllib.parse import urlsplit, urlunsplit + +import httpx +from fastapi import APIRouter, Request +from fastapi.responses import JSONResponse, Response, StreamingResponse + +from luthien_proxy.dependencies import get_dependencies +from luthien_proxy.passthrough_capture import ( + _RESPONSE_STRIPPED_HEADERS, + JsonObject, + build_passthrough_headers, + json_loads, + parse_gemini_model, + parse_openai_model, + reassemble_gemini_json_array_stream, + reassemble_gemini_sse_stream, + reassemble_openai_sse_stream, +) +from luthien_proxy.request_log.recorder import RequestLogRecorder, create_recorder +from luthien_proxy.request_log.sanitize import sanitize_url + +Provider = Literal["openai", "gemini"] + +router = APIRouter(tags=["passthrough"]) + +_OPENAI_BASE_URL = "https://api.openai.com" +_GEMINI_BASE_URL = "https://generativelanguage.googleapis.com" + + +@dataclass(frozen=True, slots=True) +class _UpstreamTarget: + provider: Provider + path: str + base_url: str + is_streaming: bool + + @property + def endpoint(self) -> str: + return f"/{self.provider}/{self.path}" + + +@dataclass(frozen=True, slots=True) +class _RequestPayload: + body_bytes: bytes + body: JsonObject + + +@dataclass(frozen=True, slots=True) +class _StreamContext: + request: Request + client: httpx.AsyncClient + target: _UpstreamTarget + upstream_url: str + forwarded_headers: dict[str, str] + payload: _RequestPayload + recorder: RequestLogRecorder + + +async def _json_body(request: Request) -> _RequestPayload: + body_bytes = await request.body() + if not body_bytes: + return _RequestPayload(body_bytes=body_bytes, body={}) + parsed = json_loads(body_bytes) + if isinstance(parsed, dict): + return _RequestPayload(body_bytes=body_bytes, body=parsed) + return _RequestPayload(body_bytes=body_bytes, body={"body": parsed}) + + +def _target_url(target: _UpstreamTarget, request: Request) -> str: + base = urlsplit(target.base_url.rstrip("/")) + path = f"/{target.path}" + return urlunsplit((base.scheme, base.netloc, path, request.url.query, "")) + + +def _is_openai_stream(path: str, body: JsonObject) -> bool: + stream = body.get("stream") + return stream is True or path.endswith("/stream") + + +def _is_gemini_stream(path: str, request: Request) -> bool: + return "streamGenerateContent" in path or request.query_params.get("alt") == "sse" + + +def _response_body(response_bytes: bytes) -> JsonObject: + if not response_bytes: + return {} + try: + parsed = json_loads(response_bytes) + except ValueError: + return {"body_text": response_bytes.decode(errors="replace")} + if isinstance(parsed, dict): + return parsed + return {"body": parsed} + + +def _stream_body(provider: Provider, request: Request, chunks: list[bytes]) -> JsonObject: + match provider: + case "openai": + return reassemble_openai_sse_stream(chunks) + case "gemini": + if request.query_params.get("alt") == "sse": + return reassemble_gemini_sse_stream(chunks) + return reassemble_gemini_json_array_stream(chunks) + + +def _client_response_headers(headers: httpx.Headers | dict[str, str]) -> dict[str, str]: + return {key: value for key, value in headers.items() if key.lower() not in _RESPONSE_STRIPPED_HEADERS} + + +def _request_error_text(error: httpx.RequestError, upstream_url: str, forwarded_headers: dict[str, str]) -> str: + text = f"{type(error).__name__}: {error!s}" + text = text.replace(upstream_url, sanitize_url(upstream_url)) + for value in forwarded_headers.values(): + if value: + text = text.replace(value, "[REDACTED]") + return text + + +def _upstream_error_response( + recorder: RequestLogRecorder, + error: httpx.RequestError, + upstream_url: str, + forwarded_headers: dict[str, str], +) -> JSONResponse: + error_text = _request_error_text(error, upstream_url, forwarded_headers) + recorder.record_inbound_response(status=502, error=error_text) + recorder.record_outbound_response(status=502, error=error_text) + recorder.flush() + return JSONResponse(status_code=502, content={"error": "upstream request failed"}) + + +async def _passthrough(request: Request, target: _UpstreamTarget, payload: _RequestPayload) -> Response: + deps = get_dependencies(request) + recorder = create_recorder(deps.db_pool, str(uuid.uuid4()), deps.enable_request_logging) + upstream_url = _target_url(target, request) + forwarded_headers = build_passthrough_headers(request.headers.items()) + model = ( + parse_openai_model(payload.body, request.headers.get("x-luthien-model")) + if target.provider == "openai" + else parse_gemini_model(target.path, payload.body, request.headers.get("x-luthien-model")) + ) + session_id = request.headers.get("x-session-id") or request.headers.get("x-luthien-session-id") + recorder.record_inbound_request( + method=request.method, + url=sanitize_url(str(request.url)), + headers=dict(request.headers), + body=payload.body, + session_id=session_id, + model=model, + endpoint=target.endpoint, + is_streaming=target.is_streaming, + ) + recorder.record_outbound_request( + method=request.method, + url=sanitize_url(upstream_url), + body=payload.body, + model=model, + endpoint=target.endpoint, + is_streaming=target.is_streaming, + ) + client = deps.passthrough_streaming_client if target.is_streaming else deps.passthrough_buffered_client + if client is None: + return Response(status_code=503, content=b"passthrough client not initialized") + if target.is_streaming: + return await _streaming_passthrough( + _StreamContext( + request=request, + client=client, + target=target, + upstream_url=upstream_url, + forwarded_headers=forwarded_headers, + payload=payload, + recorder=recorder, + ) + ) + try: + upstream = await client.request( + request.method, upstream_url, headers=forwarded_headers, content=payload.body_bytes + ) + except httpx.RequestError as exc: + return _upstream_error_response(recorder, exc, upstream_url, forwarded_headers) + body = _response_body(upstream.content) + recorder.record_inbound_response(status=upstream.status_code, body=body, headers=dict(upstream.headers)) + recorder.record_outbound_response(status=upstream.status_code, body=body) + recorder.flush() + return Response( + content=upstream.content, + status_code=upstream.status_code, + headers=_client_response_headers(upstream.headers), + ) + + +async def _streaming_passthrough(context: _StreamContext) -> Response: + upstream_request = context.client.build_request( + context.request.method, + context.upstream_url, + headers=context.forwarded_headers, + content=context.payload.body_bytes, + ) + try: + upstream = await context.client.send(upstream_request, stream=True) + except httpx.RequestError as exc: + return _upstream_error_response(context.recorder, exc, context.upstream_url, context.forwarded_headers) + if upstream.status_code >= 400: + body_bytes = await upstream.aread() + await upstream.aclose() + body = _response_body(body_bytes) + context.recorder.record_inbound_response(status=upstream.status_code, body=body, headers=dict(upstream.headers)) + context.recorder.record_outbound_response(status=upstream.status_code, body=body) + context.recorder.flush() + return Response( + content=body_bytes, + status_code=upstream.status_code, + headers=_client_response_headers(upstream.headers), + ) + + async def stream() -> AsyncIterator[bytes]: + chunks: list[bytes] = [] + try: + async for chunk in upstream.aiter_bytes(): + chunks.append(chunk) + yield chunk + finally: + await upstream.aclose() + body = _stream_body(context.target.provider, context.request, chunks) + context.recorder.record_inbound_response( + status=upstream.status_code, body=body, headers=dict(upstream.headers) + ) + context.recorder.record_outbound_response(status=upstream.status_code, body=body) + context.recorder.flush() + + return StreamingResponse( + stream(), status_code=upstream.status_code, headers=_client_response_headers(upstream.headers) + ) + + +@router.api_route("/openai/{path:path}", methods=["GET", "POST"]) +async def openai_passthrough(request: Request, path: str) -> Response: + payload = await _json_body(request) + return await _passthrough( + request, + _UpstreamTarget( + provider="openai", + path=path, + base_url=os.getenv("OPENAI_BASE_URL", _OPENAI_BASE_URL), + is_streaming=_is_openai_stream(path, payload.body), + ), + payload, + ) + + +@router.api_route("/gemini/{path:path}", methods=["GET", "POST"]) +async def gemini_passthrough(request: Request, path: str) -> Response: + payload = await _json_body(request) + return await _passthrough( + request, + _UpstreamTarget( + provider="gemini", + path=path, + base_url=os.getenv("GEMINI_BASE_URL", _GEMINI_BASE_URL), + is_streaming=_is_gemini_stream(path, request), + ), + payload, + ) + + +__all__ = ["router"] diff --git a/src/luthien_proxy/request_log/recorder.py b/src/luthien_proxy/request_log/recorder.py index f096e8558..777500626 100644 --- a/src/luthien_proxy/request_log/recorder.py +++ b/src/luthien_proxy/request_log/recorder.py @@ -24,8 +24,8 @@ logger = logging.getLogger(__name__) -# Bodies larger than this are replaced with a truncation notice -MAX_BODY_BYTES = 1_048_576 # 1 MB +# Agentic multi-provider captures can exceed 1 MB; keep full bodies for transcript replay. +MAX_BODY_BYTES = 8_388_608 # 8 MB def _log_task_exception(task: asyncio.Task[None]) -> None: @@ -243,8 +243,16 @@ async def _write_logs(self) -> None: """Insert both inbound and outbound rows.""" try: async with self._db_pool.connection() as conn: + cache: dict[int, str | None] = {} + + def serialize_body(body: dict[str, Any] | None) -> str | None: + key = id(body) + if key not in cache: + cache[key] = self._serialize_body(body) + return cache[key] + for pending in (self._inbound, self._outbound): - await _insert_log_row(conn, pending, self._serialize_body) + await _insert_log_row(conn, pending, serialize_body) except DatabaseWriteError as exc: RequestLogRecorder.dropped_writes += 1 logger.warning( diff --git a/src/luthien_proxy/request_log/sanitize.py b/src/luthien_proxy/request_log/sanitize.py index 79bf4810a..a95380a49 100644 --- a/src/luthien_proxy/request_log/sanitize.py +++ b/src/luthien_proxy/request_log/sanitize.py @@ -3,6 +3,7 @@ from __future__ import annotations import re +from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit # Headers whose values should be fully redacted _SENSITIVE_HEADERS = frozenset( @@ -10,6 +11,7 @@ "authorization", "x-api-key", "x-anthropic-api-key", + "x-goog-api-key", "proxy-authorization", "cookie", "set-cookie", @@ -48,4 +50,16 @@ def sanitize_headers(headers: dict[str, str]) -> dict[str, str]: return sanitized -__all__ = ["sanitize_headers"] +def sanitize_url(url: str) -> str: + """Redact API keys embedded in query parameters before request log storage.""" + parts = urlsplit(url) + query = urlencode( + [ + (key, "[REDACTED]" if key.lower() == "key" else value) + for key, value in parse_qsl(parts.query, keep_blank_values=True) + ] + ) + return urlunsplit((parts.scheme, parts.netloc, parts.path, query, parts.fragment)) + + +__all__ = ["sanitize_headers", "sanitize_url"] diff --git a/tests/luthien_proxy/e2e_tests/test_passthrough_capture.py b/tests/luthien_proxy/e2e_tests/test_passthrough_capture.py new file mode 100644 index 000000000..74915b71d --- /dev/null +++ b/tests/luthien_proxy/e2e_tests/test_passthrough_capture.py @@ -0,0 +1,306 @@ +from __future__ import annotations + +import asyncio +import os +import shutil +import socket +import tempfile +import threading +import time +from collections.abc import AsyncIterator, Iterator +from contextlib import ExitStack, asynccontextmanager, contextmanager +from dataclasses import dataclass, field + +import httpx +import pytest +import uvicorn +from aiohttp import web + +from luthien_proxy.main import create_app +from luthien_proxy.settings import clear_settings_cache +from luthien_proxy.utils.db import DatabasePool +from luthien_proxy.utils.migration_check import check_migrations + +pytestmark = pytest.mark.sqlite_e2e + +_API_KEY = "test-passthrough-client-key" +_ADMIN_API_KEY = "test-passthrough-admin-key" +_GEMINI_STREAM_ENDPOINT = "/gemini/v1beta/models/gemini-2.5-pro:streamGenerateContent" +_ERROR_SECRETS = ("client-openai-secret", "client-query-secret", "upstream-url-secret") + + +@dataclass +class _ProviderServer: + provider: str + port: int = 0 + requests: list[dict[str, str]] = field(default_factory=list) + bodies: list[dict[str, str]] = field(default_factory=list) + _thread: threading.Thread | None = None + _loop: asyncio.AbstractEventLoop | None = None + _runner: web.AppRunner | None = None + + def start(self) -> None: + self.port = self.port or _free_port() + ready = threading.Event() + + def run() -> None: + loop = asyncio.new_event_loop() + self._loop = loop + loop.run_until_complete(self._start_async()) + ready.set() + loop.run_forever() + loop.run_until_complete(self._stop_async()) + loop.close() + + self._thread = threading.Thread(target=run, daemon=True, name=f"mock-{self.provider}") + self._thread.start() + if not ready.wait(timeout=5): + raise RuntimeError(f"mock {self.provider} did not start") + + def stop(self) -> None: + if self._loop is not None: + self._loop.call_soon_threadsafe(self._loop.stop) + if self._thread is not None: + self._thread.join(timeout=5) + + async def _start_async(self) -> None: + app = web.Application(client_max_size=10 * 1024**2) + app.router.add_route("*", "/{path:.*}", self._handle) + self._runner = web.AppRunner(app) + await self._runner.setup() + site = web.TCPSite(self._runner, "127.0.0.1", self.port) + await site.start() + + async def _stop_async(self) -> None: + if self._runner is not None: + await self._runner.cleanup() + + async def _handle(self, request: web.Request) -> web.Response: + body = await request.json() + self.requests.append({"path": request.path_qs, "authorization": request.headers.get("Authorization", "")}) + self.bodies.append(body) + if self.provider == "openai": + return web.json_response({"id": "resp-1", "model": body.get("model"), "output_text": "openai ok"}) + if "streamGenerateContent" in request.path and request.query.get("alt") == "sse": + return web.Response( + body=b'data: {"candidates":[{"content":{"parts":[{"text":"gem"}]}}]}\n\n', + content_type="text/event-stream", + ) + return web.json_response({"candidates": [{"content": {"parts": [{"text": "gemini ok"}]}}]}) + + +def _free_port() -> int: + with socket.socket() as s: + s.bind(("", 0)) + return int(s.getsockname()[1]) + + +@pytest.fixture(scope="module") +def mock_openai() -> Iterator[_ProviderServer]: + server = _ProviderServer("openai") + server.start() + yield server + server.stop() + + +@pytest.fixture(scope="module") +def mock_gemini() -> Iterator[_ProviderServer]: + server = _ProviderServer("gemini") + server.start() + yield server + server.stop() + + +@pytest.fixture(scope="module") +def passthrough_gateway(mock_openai: _ProviderServer, mock_gemini: _ProviderServer) -> Iterator[str]: + with _boot_gateway( + openai_url=f"http://127.0.0.1:{mock_openai.port}", gemini_url=f"http://127.0.0.1:{mock_gemini.port}" + ) as url: + yield url + + +@pytest.fixture +def passthrough_gateway_with_unreachable_openai(mock_gemini: _ProviderServer) -> Iterator[str]: + with _boot_gateway( + openai_url=f"http://127.0.0.1:{_free_port()}?key=upstream-url-secret", + gemini_url=f"http://127.0.0.1:{mock_gemini.port}", + ) as url: + yield url + + +@contextmanager +def _boot_gateway(*, openai_url: str, gemini_url: str) -> Iterator[str]: + port = _free_port() + with ExitStack() as stack: + tmp_dir = tempfile.mkdtemp(prefix="luthien_passthrough_e2e_") + stack.callback(shutil.rmtree, tmp_dir, ignore_errors=True) + loop = asyncio.new_event_loop() + stack.callback(loop.close) + db_pool = DatabasePool(f"sqlite:///{os.path.join(tmp_dir, 'test.db')}") + stack.callback(lambda: loop.run_until_complete(db_pool.close())) + loop.run_until_complete(check_migrations(db_pool)) + env_keys = ("OPENAI_BASE_URL", "GEMINI_BASE_URL", "ANTHROPIC_API_KEY", "ENABLE_REQUEST_LOGGING") + old_env = {key: os.environ.get(key) for key in env_keys} + stack.callback(lambda: _restore_env(old_env)) + stack.callback(clear_settings_cache) + os.environ["OPENAI_BASE_URL"] = openai_url + os.environ["GEMINI_BASE_URL"] = gemini_url + os.environ["ANTHROPIC_API_KEY"] = "mock-key" + os.environ["ENABLE_REQUEST_LOGGING"] = "true" + clear_settings_cache() + app = create_app( + api_key=_API_KEY, + admin_key=_ADMIN_API_KEY, + db_pool=db_pool, + redis_client=None, + startup_policy_path="config/policy_config.yaml", + ) + server = uvicorn.Server(uvicorn.Config(app, host="127.0.0.1", port=port, log_level="warning")) + thread = threading.Thread(target=server.run, daemon=True, name="passthrough-sqlite-gateway") + thread.start() + stack.callback(lambda: _stop_uvicorn(server, thread)) + _wait_for_port(port) + yield f"http://127.0.0.1:{port}" + + +def _restore_env(old_env: dict[str, str | None]) -> None: + for key, value in old_env.items(): + if value is None: + os.environ.pop(key, None) + else: + os.environ[key] = value + + +def _stop_uvicorn(server: uvicorn.Server, thread: threading.Thread) -> None: + server.should_exit = True + thread.join(timeout=5) + + +def _wait_for_port(port: int) -> None: + deadline = time.monotonic() + 10 + while time.monotonic() < deadline: + try: + with socket.create_connection(("127.0.0.1", port), timeout=0.5): + return + except OSError: + time.sleep(0.1) + raise RuntimeError("passthrough gateway did not start") + + +@asynccontextmanager +async def _client() -> AsyncIterator[httpx.AsyncClient]: + async with httpx.AsyncClient(timeout=20.0) as client: + yield client + + +async def _logs(client: httpx.AsyncClient, gateway_url: str, **params: str) -> list[dict[str, str]]: + deadline = time.monotonic() + 5 + while time.monotonic() < deadline: + response = await client.get( + f"{gateway_url}/request-logs", + headers={"Authorization": f"Bearer {_ADMIN_API_KEY}"}, + params=params, + ) + assert response.status_code == 200, response.text + data = response.json() + if data["logs"]: + return data["logs"] + await asyncio.sleep(0.1) + raise AssertionError("request logs were not flushed") + + +@pytest.mark.asyncio +async def test_openai_passthrough_persists_full_request_and_response_bodies( + passthrough_gateway: str, + mock_openai: _ProviderServer, +) -> None: + # Given + large_text = "x" * (2 * 1024 * 1024) + body = {"model": "gpt-4.1", "input": large_text} + + # When + async with _client() as client: + response = await client.post( + f"{passthrough_gateway}/openai/v1/responses", + json=body, + headers={"Authorization": "Bearer client-openai-secret", "x-session-id": "session-openai"}, + ) + logs = await _logs(client, passthrough_gateway, endpoint="/openai/v1/responses", session_id="session-openai") + + # Then + assert response.status_code == 200, response.text + assert mock_openai.requests[-1]["authorization"] == "Bearer client-openai-secret" + inbound = next(log for log in logs if log["direction"] == "inbound") + assert inbound["request_body"] == body + assert inbound["response_body"] == {"id": "resp-1", "model": "gpt-4.1", "output_text": "openai ok"} + assert (inbound["session_id"], inbound["endpoint"]) == ("session-openai", "/openai/v1/responses") + assert inbound["model"] == "gpt-4.1" + assert "_truncated" not in str(inbound["request_body"]) + assert "client-openai-secret" not in f"{inbound}{next(log for log in logs if log['direction'] == 'outbound')}" + + +@pytest.mark.asyncio +async def test_gemini_passthrough_persists_session_model_and_streaming_wrapper( + passthrough_gateway: str, + mock_gemini: _ProviderServer, +) -> None: + # Given + body = {"contents": [{"parts": [{"text": "hello"}]}]} + + # When + async with _client() as client: + response = await client.post( + f"{passthrough_gateway}/gemini/v1beta/models/gemini-2.5-pro:streamGenerateContent?alt=sse&key=client-google-secret", + json=body, + headers={"x-goog-api-key": "client-google-secret", "x-session-id": "session-gemini"}, + ) + await response.aread() + logs = await _logs( + client, + passthrough_gateway, + endpoint=_GEMINI_STREAM_ENDPOINT, + session_id="session-gemini", + ) + + # Then + assert (response.status_code, mock_gemini.requests[-1]["path"].endswith("key=client-google-secret")) == (200, True) + inbound = next(log for log in logs if log["direction"] == "inbound") + assert inbound["request_body"] == body + assert (inbound["session_id"], inbound["endpoint"]) == ("session-gemini", _GEMINI_STREAM_ENDPOINT) + assert inbound["model"] == "gemini-2.5-pro" + assert inbound["response_body"] == { + "stream_format": "gemini-sse", + "chunks": [{"candidates": [{"content": {"parts": [{"text": "gem"}]}}]}], + "final": None, + } + assert "client-google-secret" not in f"{inbound}{next(log for log in logs if log['direction'] == 'outbound')}" + + +@pytest.mark.asyncio +async def test_openai_passthrough_persists_request_body_and_error_when_upstream_unreachable( + passthrough_gateway_with_unreachable_openai: str, +) -> None: + # Given + body = {"model": "gpt-4.1", "input": "capture even on connect failure"} + + # When + async with _client() as client: + response = await client.post( + f"{passthrough_gateway_with_unreachable_openai}/openai/v1/responses?key=client-query-secret", + json=body, + headers={"Authorization": "Bearer client-openai-secret", "x-session-id": "session-error"}, + ) + logs = await _logs( + client, + passthrough_gateway_with_unreachable_openai, + endpoint="/openai/v1/responses", + session_id="session-error", + ) + + # Then + assert response.status_code == 502 + inbound = next(log for log in logs if log["direction"] == "inbound") + assert inbound["request_body"] == body + assert inbound["response_status"] == 502 + assert inbound["error"] is not None + assert all(secret not in str(inbound) for secret in _ERROR_SECRETS) diff --git a/tests/luthien_proxy/unit_tests/test_passthrough_routes.py b/tests/luthien_proxy/unit_tests/test_passthrough_routes.py new file mode 100644 index 000000000..a543014a2 --- /dev/null +++ b/tests/luthien_proxy/unit_tests/test_passthrough_routes.py @@ -0,0 +1,143 @@ +from __future__ import annotations + +from luthien_proxy.passthrough_capture import ( + build_passthrough_headers, + parse_gemini_model, + parse_openai_model, + reassemble_gemini_json_array_stream, + reassemble_gemini_sse_stream, + reassemble_openai_sse_stream, +) +from luthien_proxy.passthrough_routes import _client_response_headers, _response_body +from luthien_proxy.request_log.sanitize import sanitize_headers, sanitize_url + + +def test_build_passthrough_headers_forwards_client_keys_and_strips_internal_headers() -> None: + # Given + headers = { + "Authorization": "Bearer client-openai-key", + "x-goog-api-key": "client-google-key", + "x-luthien-model": "gpt-4.1", + "Connection": "keep-alive", + "Host": "proxy.local", + "Content-Type": "application/json", + } + + # When + forwarded = build_passthrough_headers(headers.items()) + + # Then + assert forwarded == { + "Authorization": "Bearer client-openai-key", + "x-goog-api-key": "client-google-key", + "Content-Type": "application/json", + } + + +def test_sanitize_redacts_google_api_key_header_and_key_query_param() -> None: + # Given + headers = {"x-goog-api-key": "google-secret", "x-trace-id": "trace-123"} + url = "https://generativelanguage.googleapis.com/v1beta/models/gemini:generateContent?key=google-secret&alt=sse" + + # When + sanitized_headers = sanitize_headers(headers) + sanitized_url = sanitize_url(url) + + # Then + assert sanitized_headers == {"x-goog-api-key": "[REDACTED]", "x-trace-id": "trace-123"} + assert sanitized_url == ( + "https://generativelanguage.googleapis.com/v1beta/models/gemini:generateContent?key=%5BREDACTED%5D&alt=sse" + ) + assert "google-secret" not in sanitized_url + + +def test_model_parsing_uses_body_for_openai_and_url_for_gemini() -> None: + # Given + openai_body = {"model": "gpt-4.1", "input": "hello"} + gemini_path = "v1beta/models/gemini-2.5-pro:generateContent" + + # When / Then + assert parse_openai_model(openai_body, override=None) == "gpt-4.1" + assert parse_openai_model(openai_body, override="gpt-4.1-mini") == "gpt-4.1-mini" + assert parse_gemini_model(gemini_path, {"model": "fallback"}, override=None) == "gemini-2.5-pro" + assert parse_gemini_model("v1beta/generate", {"model": "fallback"}, override=None) == "fallback" + + +def test_streaming_reassembly_returns_documented_wrappers() -> None: + # Given + openai_chunks = [ + b'data: {"id":"evt-1","output_text":"hel"}\n\n', + b'data: {"id":"evt-2","output_text":"hello"}\n\n', + b"data: [DONE]\n\n", + ] + gemini_json_chunks = [b'[{"text":"hel"},', b'{"text":"hello"}]'] + gemini_sse_chunks = [b'data: {"text":"hel"}\n\n', b'data: {"text":"hello"}\n\n'] + + # When / Then + assert reassemble_openai_sse_stream(openai_chunks) == { + "stream_format": "openai-sse", + "events": [ + {"id": "evt-1", "output_text": "hel"}, + {"id": "evt-2", "output_text": "hello"}, + ], + "final": {"id": "evt-2", "output_text": "hello"}, + } + assert reassemble_gemini_json_array_stream(gemini_json_chunks) == { + "stream_format": "gemini-json-array", + "chunks": [{"text": "hel"}, {"text": "hello"}], + "final": None, + } + assert reassemble_gemini_sse_stream(gemini_sse_chunks) == { + "stream_format": "gemini-sse", + "chunks": [{"text": "hel"}, {"text": "hello"}], + "final": None, + } + + +def test_client_response_headers_strip_encoding_length_and_hop_by_hop_headers() -> None: + # Given + upstream_headers = { + "content-encoding": "gzip", + "Content-Length": "123", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "content-type": "application/json", + "x-request-id": "req-123", + } + + # When + returned_headers = _client_response_headers(upstream_headers) + + # Then + assert returned_headers == {"content-type": "application/json", "x-request-id": "req-123"} + + +def test_streaming_reassembly_preserves_malformed_chunks_with_raw_fallback() -> None: + # Given + openai_chunks = [b"data: {not json}\n\n", b'data: {"ok": true}\n\n'] + gemini_array_chunks = [b""] + + # When / Then + assert reassemble_openai_sse_stream(openai_chunks) == { + "stream_format": "openai-sse", + "events": [{"ok": True}], + "raw": 'data: {not json}\n\ndata: {"ok": true}\n\n', + "final": {"ok": True}, + } + assert reassemble_gemini_json_array_stream(gemini_array_chunks) == { + "stream_format": "gemini-json-array", + "chunks": [], + "raw": "", + "final": None, + } + + +def test_response_body_falls_back_to_replacement_text_for_invalid_utf8() -> None: + # Given + invalid_utf8 = b"\xff\xfe" + + # When + body = _response_body(invalid_utf8) + + # Then + assert body == {"body_text": "��"} diff --git a/uv.lock b/uv.lock index c92936905..7d3e41de4 100644 --- a/uv.lock +++ b/uv.lock @@ -162,6 +162,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/77/06/bb80f5f86020c4551da315d78b3ab75e8228f89f0162f2c3a819e407941a/attrs-25.3.0-py3-none-any.whl", hash = "sha256:427318ce031701fea540783410126f03899a97ffc6f61596ad581ac2e40e3bc3", size = 63815, upload-time = "2025-03-13T11:10:21.14Z" }, ] +[[package]] +name = "basedpyright" +version = "1.39.9" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nodejs-wheel-binaries" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7a/4b/c1f4e211e50389304d6af32b9280e026a7133e3ad59bbdf8f7a3250f8bee/basedpyright-1.39.9.tar.gz", hash = "sha256:32cbea5fc8273e89df3db20daea56cb7286e419ccdfdc479c64759d2dc071901", size = 24412216, upload-time = "2026-06-27T02:19:49.834Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/d4/e1fa108710d0498a18c77b1e13897f31eab47c69aa8cfe2d2a4df746541e/basedpyright-1.39.9-py3-none-any.whl", hash = "sha256:6b0837b9eba972c71895167ab9b127e6afdbc17abc92312e3f8d15ca82a5611c", size = 13374276, upload-time = "2026-06-27T02:19:54.431Z" }, +] + [[package]] name = "beartype" version = "0.21.0" @@ -780,6 +792,7 @@ dependencies = [ { name = "boto3" }, { name = "click" }, { name = "cryptography" }, + { name = "fastapi" }, { name = "httpx" }, { name = "jsonschema" }, { name = "opentelemetry-api" }, @@ -801,6 +814,7 @@ dependencies = [ [package.dev-dependencies] dev = [ { name = "asgi-lifespan" }, + { name = "basedpyright" }, { name = "luthien-cli" }, { name = "pre-commit" }, { name = "pyright" }, @@ -825,6 +839,7 @@ requires-dist = [ { name = "boto3", specifier = ">=1.34" }, { name = "click", specifier = ">=8.1.0" }, { name = "cryptography", specifier = ">=44.0.0" }, + { name = "fastapi", specifier = ">=0.115.0" }, { name = "httpx", specifier = ">=0.28.1" }, { name = "jsonschema", specifier = ">=4.17.0" }, { name = "opentelemetry-api", specifier = ">=1.20.0" }, @@ -846,6 +861,7 @@ requires-dist = [ [package.metadata.requires-dev] dev = [ { name = "asgi-lifespan", specifier = ">=2.1.0" }, + { name = "basedpyright", specifier = ">=1.39.9" }, { name = "luthien-cli", editable = "src/luthien_cli" }, { name = "pre-commit", specifier = ">=4.3.0" }, { name = "pyright", specifier = ">=1.1.406,<1.2" }, @@ -947,6 +963,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d2/1d/1b658dbd2b9fa9c4c9f32accbfc0205d532c8c6194dc0f2a4c0428e7128a/nodeenv-1.9.1-py2.py3-none-any.whl", hash = "sha256:ba11c9782d29c27c70ffbdda2d7415098754709be8a7056d79a737cd901155c9", size = 22314, upload-time = "2024-06-04T18:44:08.352Z" }, ] +[[package]] +name = "nodejs-wheel-binaries" +version = "24.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a3/22/2a5beb4e21417c73233d9f65cf6f3e96e891b80d2f550a8f630ebc6b88c6/nodejs_wheel_binaries-24.16.0.tar.gz", hash = "sha256:c973cb69dc5fd16e6f6dc6e579e2c3d5534e2a1f57619dddf5ba070efa7dde37", size = 8056, upload-time = "2026-05-30T16:52:09.807Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/83/d1/68b43b53cd0fa83ae6fd406705023ca988d9e0ca41c724d82e66fbeb2ef6/nodejs_wheel_binaries-24.16.0-py2.py3-none-macosx_13_0_arm64.whl", hash = "sha256:d9f8f677dcf30e37ac244f07869726abe043f01eb0f45722b1df31cc2af7093c", size = 55666374, upload-time = "2026-05-30T16:51:39.588Z" }, + { url = "https://files.pythonhosted.org/packages/e9/b2/40a989159599080da485de966c4c2d207e852ac7aa7864702626d96c8bf5/nodejs_wheel_binaries-24.16.0-py2.py3-none-macosx_13_0_x86_64.whl", hash = "sha256:3d0370fe7120ce9697a4f60d40480d2bd8808d9f30131458d5afc0040d4e5a51", size = 55838487, upload-time = "2026-05-30T16:51:43.383Z" }, + { url = "https://files.pythonhosted.org/packages/d7/a7/cd42174fb5ff6faff7fa8d326a18914d8f232098ab5de055b57c16fa13ca/nodejs_wheel_binaries-24.16.0-py2.py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:85dc92bbb79c851569c5925dcc2a4c915a034efab375f99e4e7e6bbe9cca8342", size = 60179540, upload-time = "2026-05-30T16:51:47.036Z" }, + { url = "https://files.pythonhosted.org/packages/2b/95/c8a1f9ae140aa28df8744d984d01d4b3af7cdd6555af12127f40ceb45a7d/nodejs_wheel_binaries-24.16.0-py2.py3-none-manylinux_2_28_x86_64.whl", hash = "sha256:2f3036292811514ba847b3708492644764f88a833ac425c5f55007014308ddfd", size = 60716262, upload-time = "2026-05-30T16:51:50.711Z" }, + { url = "https://files.pythonhosted.org/packages/64/c9/7c35b3737f59e36d0249c265397b7bff570519b95301d6e16ea361e904ad/nodejs_wheel_binaries-24.16.0-py2.py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:db8a8a76ebd2b28ecbfc9ad464baa3707241b9e050a30e2efdf6f60c0f886502", size = 62230592, upload-time = "2026-05-30T16:51:55Z" }, + { url = "https://files.pythonhosted.org/packages/04/96/d931255cf9d11a84d6b54d882dba7434646467d568ccf070ea3418638df3/nodejs_wheel_binaries-24.16.0-py2.py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:f1a3d8f7b4491cbbd023ba3fc4e901fcca2d9fb80d57f24ba3890de8b1dbac03", size = 62841759, upload-time = "2026-05-30T16:51:59.407Z" }, + { url = "https://files.pythonhosted.org/packages/a2/7b/8b7a3f41bc255411be30b6d7d288aab8ffd9ea2055db8555ced3548007b9/nodejs_wheel_binaries-24.16.0-py2.py3-none-win_amd64.whl", hash = "sha256:bb136be9944f0662dcf1120f45193a6b75b13fac378971a95cc42c9f879a81aa", size = 42027734, upload-time = "2026-05-30T16:52:03.348Z" }, + { url = "https://files.pythonhosted.org/packages/17/66/1ed71f1f529b8ca727d42c7ceb9db0bef145ce4a13dfc86fb50aa44f3be6/nodejs_wheel_binaries-24.16.0-py2.py3-none-win_arm64.whl", hash = "sha256:8308940b5edd0a50dc5267ea36ba21c9f668e83fe0d9f293937174d3a7e31c36", size = 39714528, upload-time = "2026-05-30T16:52:06.421Z" }, +] + [[package]] name = "opentelemetry-api" version = "1.38.0" From a37d8840135cd1355fdef7d6d7e8c5325b69068a Mon Sep 17 00:00:00 2001 From: Sami Jawhar Date: Mon, 6 Jul 2026 13:32:24 +0000 Subject: [PATCH 2/5] docs(changelog): add fragment for multi-provider passthrough capture (#796) --- changelog.d/passthrough-multiprovider-capture.md | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 changelog.d/passthrough-multiprovider-capture.md diff --git a/changelog.d/passthrough-multiprovider-capture.md b/changelog.d/passthrough-multiprovider-capture.md new file mode 100644 index 000000000..3cdd7696d --- /dev/null +++ b/changelog.d/passthrough-multiprovider-capture.md @@ -0,0 +1,8 @@ +--- +category: Features +pr: 796 +--- + +**Multi-provider passthrough capture**: capture OpenAI (`/openai/*`) and Gemini (`/gemini/*`) passthrough traffic into `request_logs`, mirroring the existing Anthropic `/v1/*` capture. + - Streaming and non-streaming responses are recorded; payloads are sanitized before persistence. + - Cross-provider session grouping via the existing header/metadata contract, so a single logical conversation is retrievable regardless of provider. From a271fe0963afb86c575b5bc815a6fb39df265a44 Mon Sep 17 00:00:00 2001 From: Sami Jawhar Date: Thu, 9 Jul 2026 16:52:56 +0000 Subject: [PATCH 3/5] fix(passthrough): gate /openai + /gemini behind PASSTHROUGH_ROUTES_ENABLED and bound stream capture Addresses #796 review: the passthrough routes forwarded client-supplied upstream credentials with no auth, so an always-on deployment acted as an open relay that anyone with network reach could use to write request_logs. The routes now mount only when PASSTHROUGH_ROUTES_ENABLED=true (else 404), and streamed-response capture is bounded by PASSTHROUGH_STREAM_CAPTURE_MAX_BYTES (default 10 MiB) so a large stream can't exhaust memory. --- .../passthrough-multiprovider-capture.md | 2 + src/luthien_proxy/config_fields.py | 16 +++++++ src/luthien_proxy/passthrough_routes.py | 30 ++++++++++-- src/luthien_proxy/settings.py | 4 ++ .../e2e_tests/test_passthrough_capture.py | 48 ++++++++++++++++++- .../unit_tests/test_passthrough_routes.py | 31 +++++++++++- 6 files changed, 124 insertions(+), 7 deletions(-) diff --git a/changelog.d/passthrough-multiprovider-capture.md b/changelog.d/passthrough-multiprovider-capture.md index 3cdd7696d..86aae5983 100644 --- a/changelog.d/passthrough-multiprovider-capture.md +++ b/changelog.d/passthrough-multiprovider-capture.md @@ -6,3 +6,5 @@ pr: 796 **Multi-provider passthrough capture**: capture OpenAI (`/openai/*`) and Gemini (`/gemini/*`) passthrough traffic into `request_logs`, mirroring the existing Anthropic `/v1/*` capture. - Streaming and non-streaming responses are recorded; payloads are sanitized before persistence. - Cross-provider session grouping via the existing header/metadata contract, so a single logical conversation is retrievable regardless of provider. + - Disabled by default: the routes forward client-supplied upstream credentials, so they only mount when `PASSTHROUGH_ROUTES_ENABLED=true` (otherwise they 404). This prevents an always-on deployment from acting as an open relay. + - Streamed-response capture is bounded by `PASSTHROUGH_STREAM_CAPTURE_MAX_BYTES` (default 10 MiB); the client stream is never affected, and the recorded body is flagged `capture_truncated` beyond the limit. diff --git a/src/luthien_proxy/config_fields.py b/src/luthien_proxy/config_fields.py index ed08ef169..685cb219d 100644 --- a/src/luthien_proxy/config_fields.py +++ b/src/luthien_proxy/config_fields.py @@ -242,6 +242,21 @@ class ConfigFieldMeta: category="observability", ), + # ── passthrough ─────────────────────────────────────────────────────── + ConfigFieldMeta( + "passthrough_routes_enabled", "PASSTHROUGH_ROUTES_ENABLED", bool, False, + "Enable the /openai/* and /gemini/* passthrough capture routes. Default off: these " + "routes forward client-supplied upstream credentials, so an always-on deployment would " + "relay traffic upstream and write request_logs for anyone with network reach.", + category="passthrough", + ), + ConfigFieldMeta( + "passthrough_stream_capture_max_bytes", "PASSTHROUGH_STREAM_CAPTURE_MAX_BYTES", int, 10485760, + "Max bytes of a streamed passthrough response buffered in memory for request_logs capture. " + "The stream forwarded to the client is unaffected; only the recorded body is truncated beyond this.", + category="passthrough", + ), + # ── telemetry ───────────────────────────────────────────────────────── ConfigFieldMeta( "usage_telemetry", "USAGE_TELEMETRY", bool, None, @@ -374,6 +389,7 @@ class ConfigFieldMeta: "llm", "security", "observability", + "passthrough", "telemetry", "retention", "webhook", diff --git a/src/luthien_proxy/passthrough_routes.py b/src/luthien_proxy/passthrough_routes.py index 34203fb03..775f940f2 100644 --- a/src/luthien_proxy/passthrough_routes.py +++ b/src/luthien_proxy/passthrough_routes.py @@ -10,7 +10,7 @@ from urllib.parse import urlsplit, urlunsplit import httpx -from fastapi import APIRouter, Request +from fastapi import APIRouter, Depends, HTTPException, Request from fastapi.responses import JSONResponse, Response, StreamingResponse from luthien_proxy.dependencies import get_dependencies @@ -27,6 +27,7 @@ ) from luthien_proxy.request_log.recorder import RequestLogRecorder, create_recorder from luthien_proxy.request_log.sanitize import sanitize_url +from luthien_proxy.settings import get_settings Provider = Literal["openai", "gemini"] @@ -36,6 +37,18 @@ _GEMINI_BASE_URL = "https://generativelanguage.googleapis.com" +async def _require_passthrough_enabled() -> None: + """Gate the passthrough routes behind PASSTHROUGH_ROUTES_ENABLED (default off). + + These routes forward client-supplied upstream credentials, so an always-on + deployment would act as an open relay and let anyone with network reach write + request_logs. When disabled we 404 so the routes are indistinguishable from + unmounted paths. + """ + if not get_settings().passthrough_routes_enabled: + raise HTTPException(status_code=404, detail="Not Found") + + @dataclass(frozen=True, slots=True) class _UpstreamTarget: provider: Provider @@ -225,13 +238,22 @@ async def _streaming_passthrough(context: _StreamContext) -> Response: async def stream() -> AsyncIterator[bytes]: chunks: list[bytes] = [] + captured_bytes = 0 + max_capture = get_settings().passthrough_stream_capture_max_bytes + truncated = False try: async for chunk in upstream.aiter_bytes(): - chunks.append(chunk) + if captured_bytes < max_capture: + chunks.append(chunk) + captured_bytes += len(chunk) + if captured_bytes >= max_capture: + truncated = True yield chunk finally: await upstream.aclose() body = _stream_body(context.target.provider, context.request, chunks) + if truncated: + body = {**body, "capture_truncated": True} context.recorder.record_inbound_response( status=upstream.status_code, body=body, headers=dict(upstream.headers) ) @@ -243,7 +265,7 @@ async def stream() -> AsyncIterator[bytes]: ) -@router.api_route("/openai/{path:path}", methods=["GET", "POST"]) +@router.api_route("/openai/{path:path}", methods=["GET", "POST"], dependencies=[Depends(_require_passthrough_enabled)]) async def openai_passthrough(request: Request, path: str) -> Response: payload = await _json_body(request) return await _passthrough( @@ -258,7 +280,7 @@ async def openai_passthrough(request: Request, path: str) -> Response: ) -@router.api_route("/gemini/{path:path}", methods=["GET", "POST"]) +@router.api_route("/gemini/{path:path}", methods=["GET", "POST"], dependencies=[Depends(_require_passthrough_enabled)]) async def gemini_passthrough(request: Request, path: str) -> Response: payload = await _json_body(request) return await _passthrough( diff --git a/src/luthien_proxy/settings.py b/src/luthien_proxy/settings.py index d361a02a5..e9065466b 100644 --- a/src/luthien_proxy/settings.py +++ b/src/luthien_proxy/settings.py @@ -90,6 +90,10 @@ class Settings(_SettingsBase): railway_service_name: str = "" enable_request_logging: bool = False + # ── passthrough ───────────────────────────────────────────────── + passthrough_routes_enabled: bool = False + passthrough_stream_capture_max_bytes: int = 10485760 + # ── telemetry ─────────────────────────────────────────────────── usage_telemetry: bool | None = None telemetry_endpoint: str = "https://telemetry.luthien.cc/v1/events" diff --git a/tests/luthien_proxy/e2e_tests/test_passthrough_capture.py b/tests/luthien_proxy/e2e_tests/test_passthrough_capture.py index 74915b71d..1aaddd238 100644 --- a/tests/luthien_proxy/e2e_tests/test_passthrough_capture.py +++ b/tests/luthien_proxy/e2e_tests/test_passthrough_capture.py @@ -128,8 +128,18 @@ def passthrough_gateway_with_unreachable_openai(mock_gemini: _ProviderServer) -> yield url +@pytest.fixture +def passthrough_gateway_disabled(mock_openai: _ProviderServer, mock_gemini: _ProviderServer) -> Iterator[str]: + with _boot_gateway( + openai_url=f"http://127.0.0.1:{mock_openai.port}", + gemini_url=f"http://127.0.0.1:{mock_gemini.port}", + passthrough_enabled=False, + ) as url: + yield url + + @contextmanager -def _boot_gateway(*, openai_url: str, gemini_url: str) -> Iterator[str]: +def _boot_gateway(*, openai_url: str, gemini_url: str, passthrough_enabled: bool = True) -> Iterator[str]: port = _free_port() with ExitStack() as stack: tmp_dir = tempfile.mkdtemp(prefix="luthien_passthrough_e2e_") @@ -139,7 +149,13 @@ def _boot_gateway(*, openai_url: str, gemini_url: str) -> Iterator[str]: db_pool = DatabasePool(f"sqlite:///{os.path.join(tmp_dir, 'test.db')}") stack.callback(lambda: loop.run_until_complete(db_pool.close())) loop.run_until_complete(check_migrations(db_pool)) - env_keys = ("OPENAI_BASE_URL", "GEMINI_BASE_URL", "ANTHROPIC_API_KEY", "ENABLE_REQUEST_LOGGING") + env_keys = ( + "OPENAI_BASE_URL", + "GEMINI_BASE_URL", + "ANTHROPIC_API_KEY", + "ENABLE_REQUEST_LOGGING", + "PASSTHROUGH_ROUTES_ENABLED", + ) old_env = {key: os.environ.get(key) for key in env_keys} stack.callback(lambda: _restore_env(old_env)) stack.callback(clear_settings_cache) @@ -147,6 +163,7 @@ def _boot_gateway(*, openai_url: str, gemini_url: str) -> Iterator[str]: os.environ["GEMINI_BASE_URL"] = gemini_url os.environ["ANTHROPIC_API_KEY"] = "mock-key" os.environ["ENABLE_REQUEST_LOGGING"] = "true" + os.environ["PASSTHROUGH_ROUTES_ENABLED"] = "true" if passthrough_enabled else "false" clear_settings_cache() app = create_app( api_key=_API_KEY, @@ -304,3 +321,30 @@ async def test_openai_passthrough_persists_request_body_and_error_when_upstream_ assert inbound["response_status"] == 502 assert inbound["error"] is not None assert all(secret not in str(inbound) for secret in _ERROR_SECRETS) + + +@pytest.mark.asyncio +async def test_passthrough_routes_404_when_feature_disabled( + passthrough_gateway_disabled: str, + mock_openai: _ProviderServer, + mock_gemini: _ProviderServer, +) -> None: + # Given the passthrough feature is disabled (the default) + requests_before = (len(mock_openai.requests), len(mock_gemini.requests)) + + # When both routes are hit + async with _client() as client: + openai_response = await client.post( + f"{passthrough_gateway_disabled}/openai/v1/responses", + json={"model": "gpt-4.1", "input": "should not reach upstream"}, + headers={"Authorization": "Bearer client-openai-secret"}, + ) + gemini_response = await client.post( + f"{passthrough_gateway_disabled}/gemini/v1beta/models/gemini-2.5-pro:generateContent", + json={"contents": [{"parts": [{"text": "hello"}]}]}, + headers={"x-goog-api-key": "client-google-secret"}, + ) + + # Then both 404 and nothing was relayed upstream + assert (openai_response.status_code, gemini_response.status_code) == (404, 404) + assert (len(mock_openai.requests), len(mock_gemini.requests)) == requests_before diff --git a/tests/luthien_proxy/unit_tests/test_passthrough_routes.py b/tests/luthien_proxy/unit_tests/test_passthrough_routes.py index a543014a2..6e45f4d27 100644 --- a/tests/luthien_proxy/unit_tests/test_passthrough_routes.py +++ b/tests/luthien_proxy/unit_tests/test_passthrough_routes.py @@ -1,5 +1,10 @@ from __future__ import annotations +from unittest.mock import patch + +import pytest +from fastapi import HTTPException + from luthien_proxy.passthrough_capture import ( build_passthrough_headers, parse_gemini_model, @@ -8,7 +13,11 @@ reassemble_gemini_sse_stream, reassemble_openai_sse_stream, ) -from luthien_proxy.passthrough_routes import _client_response_headers, _response_body +from luthien_proxy.passthrough_routes import ( + _client_response_headers, + _require_passthrough_enabled, + _response_body, +) from luthien_proxy.request_log.sanitize import sanitize_headers, sanitize_url @@ -141,3 +150,23 @@ def test_response_body_falls_back_to_replacement_text_for_invalid_utf8() -> None # Then assert body == {"body_text": "��"} + + +async def test_require_passthrough_enabled_rejects_when_disabled() -> None: + # Given the feature flag is off (the default) + with patch("luthien_proxy.passthrough_routes.get_settings") as mock_settings: + mock_settings.return_value.passthrough_routes_enabled = False + + # When / Then the gate 404s so the routes look unmounted + with pytest.raises(HTTPException) as exc_info: + await _require_passthrough_enabled() + assert exc_info.value.status_code == 404 + + +async def test_require_passthrough_enabled_allows_when_enabled() -> None: + # Given the feature flag is explicitly on + with patch("luthien_proxy.passthrough_routes.get_settings") as mock_settings: + mock_settings.return_value.passthrough_routes_enabled = True + + # When / Then the gate permits the request (no exception) + assert await _require_passthrough_enabled() is None From c0fd9c7a043c2208a1c6941f4d2f0ef33b537e1f Mon Sep 17 00:00:00 2001 From: Sami Jawhar Date: Fri, 10 Jul 2026 22:48:04 +0000 Subject: [PATCH 4/5] feat(passthrough): materialize OpenAI/Gemini passthrough into conversation history + retrieval Normalize captured OpenAI (chat + Responses) and Gemini (generateContent + streamGenerateContent) passthrough payloads into canonical Anthropic-shaped conversation events while preserving exact provider-native request/response for faithful reprobe. Idempotent (advisory lock + existence guard, single txn, two summary updates); live post-commit RequestLogRecorder callback gated by PASSTHROUGH_MATERIALIZE_ENABLED plus a dashboard-only reconcile worker + one-shot backfill CLI gated by PASSTHROUGH_MATERIALIZE_BACKFILL_ENABLED. Passthrough user attribution mirrors the Anthropic trusted-header/JWT policy. Existing history/ debug/summary/FTS/export read paths light up with zero new migrations. fix(passthrough): SDK-typed normalizers + Gemini contents[].role defaulting + OpenAI reasoning tokens + content_filter mapping Four fixes to the passthrough materialization pipeline, all discovered while backfilling ~2600 historical prod rows against the initial hand-rolled normalizers. 1. Provider-SDK-based normalizers (openai + google-genai typed models): Replaces hand-rolled OpenAI/Gemini payload parsing with SDK validation via openai.types + google.genai.types. Lenient by design - skip unknown output/ input items, parts, tools; never fatal on a novel variant. Wraps OpenAI response parsers in try/except ValidationError -> typed MALFORMED_PAYLOAD so novel SDK-rejected values skip one transaction instead of wedging the backfill batch. Fixes reasoning-model support (gpt-5.6-sol) and Gemini serviceTier extra_forbidden. 2. Default missing Gemini contents[].role to 'user' per API spec: _content_message previously dropped role-less contents, then _request_messages reported the misleading MISSING_REQUIRED_FIELD:contents error. Fixed with 'role = optional_string(content, "role") or "user"'. Ref: https://ai.google.dev/api/generate-content#Content 3. Gemini stream ValidationError fallback (from oracle+fable code review): gemini_stream.py._add_chunk on SDK ValidationError keeps the raw chunk so downstream lenient mapping proceeds. Mirrors gemini_response.py's buffered raw fallback. Prevents streamed captures failing MALFORMED_PAYLOAD on the same usageMetadata.serviceTier field that buffered handles. 4. Delete validate_*_request no-op helpers + dead code (oracle+fable): validate_openai_chat_request / validate_openai_responses_request / validate_gemini_request all caught ValidationError and returned None - behavioral no-ops with ~90 lines of pydantic parse overhead and zero effect. Also deleted openai_common.object_field + strict text_content_from_openai (zero callers repo-wide). 5. OpenAI reasoning_tokens in canonical usage: canonical_usage now lifts reasoning_tokens from completion_tokens_details.reasoning_tokens (Chat) or output_tokens_details.reasoning_tokens (Responses) into the canonical usage dict, matching what gemini_common already does for thoughtsTokenCount. Non-reasoning models omit the field entirely at the API level; the SDK parser injects 0 as a default, so we treat 0 as absent to avoid polluting non-reasoning outputs with a spurious 0. 6. OpenAI content_filter -> canonical safety (not end_turn): stop_reason() previously collapsed content_filter into end_turn, making safety-policy blocks indistinguishable from natural stops. Now maps to 'safety' (matching the Gemini normalizer's SAFETY bucket) so downstream consumers can filter blocked completions. Verification: 84/84 passthrough_materialize unit tests pass (was 78/78 before + review regressions + reasoning_tokens + content_filter tests). ruff format + check clean. basedpyright clean. Real prod-data validation: chat 59/60, Responses 60/60, gpt-5.6-sol reasoning 60/60, Gemini 60/60. Prod backfill drain: 2618/2623 eligible materialized (99.8%). --- .env.example | 21 + changelog.d/passthrough-materialization.md | 25 + pyproject.toml | 2 + .../backfill_passthrough_materialization.py | 48 ++ src/luthien_proxy/config_fields.py | 23 + src/luthien_proxy/main.py | 12 + src/luthien_proxy/passthrough_capture.py | 14 + .../passthrough_materialize/__init__.py | 39 + .../passthrough_materialize/backfill.py | 36 + .../passthrough_materialize/endpoints.py | 90 ++ .../passthrough_materialize/gemini.py | 15 + .../passthrough_materialize/gemini_common.py | 130 +++ .../passthrough_materialize/gemini_request.py | 235 +++++ .../gemini_response.py | 180 ++++ .../passthrough_materialize/gemini_stream.py | 275 ++++++ .../passthrough_materialize/materialize.py | 179 ++++ .../materialize_read.py | 158 ++++ .../materialize_types.py | 115 +++ .../materialize_write.py | 125 +++ .../passthrough_materialize/openai.py | 23 + .../passthrough_materialize/openai_chat.py | 276 ++++++ .../openai_chat_stream.py | 125 +++ .../passthrough_materialize/openai_common.py | 224 +++++ .../openai_responses.py | 247 ++++++ .../openai_responses_stream.py | 74 ++ .../passthrough_materialize/payloads.py | 172 ++++ .../provider_models.py | 162 ++++ .../passthrough_materialize/reconcile.py | 93 ++ .../passthrough_materialize/worker.py | 77 ++ src/luthien_proxy/passthrough_recording.py | 50 ++ src/luthien_proxy/passthrough_routes.py | 35 +- src/luthien_proxy/request_log/models.py | 83 ++ src/luthien_proxy/request_log/recorder.py | 140 +-- src/luthien_proxy/settings.py | 4 + .../test_passthrough_materialization_http.py | 203 +++++ .../test_passthrough_materialize.py | 172 ++++ .../test_backfill_script.py | 80 ++ .../passthrough_materialize/test_endpoints.py | 57 ++ .../test_gemini_normalizers.py | 807 ++++++++++++++++++ .../test_lenient_output_items.py | 97 +++ .../test_lenient_request_items.py | 114 +++ .../test_materialize.py | 323 +++++++ .../test_openai_normalizers.py | 748 ++++++++++++++++ .../passthrough_materialize/test_payloads.py | 144 ++++ .../passthrough_materialize/test_reconcile.py | 246 ++++++ .../passthrough_materialize/test_worker.py | 112 +++ .../unit_tests/request_log/test_recorder.py | 157 +++- .../unit_tests/test_config_registry.py | 41 +- .../unit_tests/test_passthrough_capture.py | 144 ++++ .../unit_tests/test_passthrough_routes.py | 329 ++++--- .../luthien_proxy/unit_tests/test_settings.py | 26 + .../unit_tests/test_startup_flow.py | 58 ++ uv.lock | 184 +++- 53 files changed, 7231 insertions(+), 318 deletions(-) create mode 100644 changelog.d/passthrough-materialization.md create mode 100644 scripts/backfill_passthrough_materialization.py create mode 100644 src/luthien_proxy/passthrough_materialize/__init__.py create mode 100644 src/luthien_proxy/passthrough_materialize/backfill.py create mode 100644 src/luthien_proxy/passthrough_materialize/endpoints.py create mode 100644 src/luthien_proxy/passthrough_materialize/gemini.py create mode 100644 src/luthien_proxy/passthrough_materialize/gemini_common.py create mode 100644 src/luthien_proxy/passthrough_materialize/gemini_request.py create mode 100644 src/luthien_proxy/passthrough_materialize/gemini_response.py create mode 100644 src/luthien_proxy/passthrough_materialize/gemini_stream.py create mode 100644 src/luthien_proxy/passthrough_materialize/materialize.py create mode 100644 src/luthien_proxy/passthrough_materialize/materialize_read.py create mode 100644 src/luthien_proxy/passthrough_materialize/materialize_types.py create mode 100644 src/luthien_proxy/passthrough_materialize/materialize_write.py create mode 100644 src/luthien_proxy/passthrough_materialize/openai.py create mode 100644 src/luthien_proxy/passthrough_materialize/openai_chat.py create mode 100644 src/luthien_proxy/passthrough_materialize/openai_chat_stream.py create mode 100644 src/luthien_proxy/passthrough_materialize/openai_common.py create mode 100644 src/luthien_proxy/passthrough_materialize/openai_responses.py create mode 100644 src/luthien_proxy/passthrough_materialize/openai_responses_stream.py create mode 100644 src/luthien_proxy/passthrough_materialize/payloads.py create mode 100644 src/luthien_proxy/passthrough_materialize/provider_models.py create mode 100644 src/luthien_proxy/passthrough_materialize/reconcile.py create mode 100644 src/luthien_proxy/passthrough_materialize/worker.py create mode 100644 src/luthien_proxy/passthrough_recording.py create mode 100644 tests/luthien_proxy/e2e_tests/sqlite/test_passthrough_materialization_http.py create mode 100644 tests/luthien_proxy/integration_tests/test_passthrough_materialize.py create mode 100644 tests/luthien_proxy/unit_tests/passthrough_materialize/test_backfill_script.py create mode 100644 tests/luthien_proxy/unit_tests/passthrough_materialize/test_endpoints.py create mode 100644 tests/luthien_proxy/unit_tests/passthrough_materialize/test_gemini_normalizers.py create mode 100644 tests/luthien_proxy/unit_tests/passthrough_materialize/test_lenient_output_items.py create mode 100644 tests/luthien_proxy/unit_tests/passthrough_materialize/test_lenient_request_items.py create mode 100644 tests/luthien_proxy/unit_tests/passthrough_materialize/test_materialize.py create mode 100644 tests/luthien_proxy/unit_tests/passthrough_materialize/test_openai_normalizers.py create mode 100644 tests/luthien_proxy/unit_tests/passthrough_materialize/test_payloads.py create mode 100644 tests/luthien_proxy/unit_tests/passthrough_materialize/test_reconcile.py create mode 100644 tests/luthien_proxy/unit_tests/passthrough_materialize/test_worker.py create mode 100644 tests/luthien_proxy/unit_tests/test_passthrough_capture.py diff --git a/.env.example b/.env.example index 5b8ce8a8d..f7142feb9 100644 --- a/.env.example +++ b/.env.example @@ -138,6 +138,27 @@ # ENABLE_REQUEST_LOGGING=false +# === PASSTHROUGH ================================================= + +# Enable the /openai/* and /gemini/* passthrough capture routes. Default off: these routes forward client-supplied upstream credentials, so an always-on deployment would relay traffic upstream and write request_logs for anyone with network reach. +# PASSTHROUGH_ROUTES_ENABLED=false + +# Max bytes of a streamed passthrough response buffered in memory for request_logs capture. The stream forwarded to the client is unaffected; only the recorded body is truncated beyond this. +# PASSTHROUGH_STREAM_CAPTURE_MAX_BYTES=10485760 + +# Enable asynchronous materialization of eligible passthrough request_logs into conversation history. +# PASSTHROUGH_MATERIALIZE_ENABLED=false + +# Enable one-shot passthrough materialization backfill for historical request_logs. +# PASSTHROUGH_MATERIALIZE_BACKFILL_ENABLED=false + +# Seconds between dashboard reconciliation scans for unmaterialized passthrough request_logs. +# PASSTHROUGH_MATERIALIZE_RECONCILE_INTERVAL_SECONDS=300 + +# Maximum passthrough request_logs processed per materialization batch. +# PASSTHROUGH_MATERIALIZE_BATCH_SIZE=200 + + # === TELEMETRY =================================================== # Anonymous usage telemetry (None defers to DB config, True/False overrides) diff --git a/changelog.d/passthrough-materialization.md b/changelog.d/passthrough-materialization.md new file mode 100644 index 000000000..f2f7d0766 --- /dev/null +++ b/changelog.d/passthrough-materialization.md @@ -0,0 +1,25 @@ +--- +category: Features +--- + +**Passthrough OpenAI/Gemini calls now materialize into conversation history, search, and export** + - Previously the `/openai/*` and `/gemini/*` passthrough routes only wrote + raw `request_logs`; those calls never appeared in `/api/history`, + `/api/debug/calls`, session summaries, FTS, or the JSONL export, and were + unreadable by downstream tooling (cybertasks). + - Adds a `passthrough_materialize` package that normalizes captured OpenAI + (chat + Responses, buffered + streamed) and Gemini (generateContent + + streamGenerateContent) payloads into the canonical Anthropic-shaped + conversation-event contract while preserving the exact provider-native + request/response verbatim for faithful reprobe. + - Materialization is idempotent (advisory lock + request-event existence + guard, single transaction, two session-summary updates) and driven two + ways: a live post-commit callback on `RequestLogRecorder` (gated by + `PASSTHROUGH_MATERIALIZE_ENABLED`) and a dashboard-only reconcile worker + + one-shot backfill CLI (gated by `PASSTHROUGH_MATERIALIZE_BACKFILL_ENABLED`) + for historical rows. + - Passthrough requests now carry user attribution via the same trusted + `X-Luthien-User-Id` / Bearer-JWT policy as the Anthropic path. Malformed or + unsupported eligible payloads fail loudly and stay retryable; no partial + rows are ever written. Existing read paths light up with zero new + migrations and SQLite/Postgres parity. diff --git a/pyproject.toml b/pyproject.toml index 97d0cc6fc..ca1128926 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,9 @@ dependencies = [ "click>=8.1.0", "cryptography>=44.0.0", "httpx>=0.28.1", + "google-genai>=1.33.0", "jsonschema>=4.17.0", + "openai>=2.11.0", "opentelemetry-api>=1.20.0", "opentelemetry-sdk>=1.20.0", "opentelemetry-exporter-otlp-proto-grpc>=1.20.0", diff --git a/scripts/backfill_passthrough_materialization.py b/scripts/backfill_passthrough_materialization.py new file mode 100644 index 000000000..2762ca2a6 --- /dev/null +++ b/scripts/backfill_passthrough_materialization.py @@ -0,0 +1,48 @@ +"""Drain the passthrough materialization backfill from configured storage.""" + +from __future__ import annotations + +import asyncio +import logging + +import click + +from luthien_proxy.passthrough_materialize.backfill import drain_passthrough_backfill +from luthien_proxy.settings import get_settings +from luthien_proxy.telemetry import configure_logging +from luthien_proxy.utils.db import DatabasePool + +logger = logging.getLogger(__name__) + + +async def _run_backfill() -> None: + settings = get_settings() + if not settings.database_url: + raise click.ClickException("DATABASE_URL must be configured") + db_pool = DatabasePool(settings.database_url) + try: + await db_pool.get_pool() + totals = await drain_passthrough_backfill( + db_pool, + limit=settings.passthrough_materialize_batch_size, + ) + finally: + await db_pool.close() + logger.info( + "Passthrough backfill complete: materialized=%d already_materialized=%d skipped_ineligible=%d failed=%d", + totals.materialized, + totals.already_materialized, + totals.skipped_ineligible, + totals.failed, + ) + + +@click.command() +def main() -> None: + """Drain configured passthrough materialization backfill.""" + asyncio.run(_run_backfill()) + + +if __name__ == "__main__": + configure_logging() + main() diff --git a/src/luthien_proxy/config_fields.py b/src/luthien_proxy/config_fields.py index 685cb219d..aa44fa270 100644 --- a/src/luthien_proxy/config_fields.py +++ b/src/luthien_proxy/config_fields.py @@ -256,6 +256,29 @@ class ConfigFieldMeta: "The stream forwarded to the client is unaffected; only the recorded body is truncated beyond this.", category="passthrough", ), + ConfigFieldMeta( + "passthrough_materialize_enabled", "PASSTHROUGH_MATERIALIZE_ENABLED", bool, False, + "Enable asynchronous materialization of eligible passthrough request_logs into conversation history.", + category="passthrough", + ), + ConfigFieldMeta( + "passthrough_materialize_backfill_enabled", "PASSTHROUGH_MATERIALIZE_BACKFILL_ENABLED", bool, False, + "Enable one-shot passthrough materialization backfill for historical request_logs.", + category="passthrough", + ), + ConfigFieldMeta( + "passthrough_materialize_reconcile_interval_seconds", + "PASSTHROUGH_MATERIALIZE_RECONCILE_INTERVAL_SECONDS", + int, + 300, + "Seconds between dashboard reconciliation scans for unmaterialized passthrough request_logs.", + category="passthrough", + ), + ConfigFieldMeta( + "passthrough_materialize_batch_size", "PASSTHROUGH_MATERIALIZE_BATCH_SIZE", int, 200, + "Maximum passthrough request_logs processed per materialization batch.", + category="passthrough", + ), # ── telemetry ───────────────────────────────────────────────────────── ConfigFieldMeta( diff --git a/src/luthien_proxy/main.py b/src/luthien_proxy/main.py index 4f161f8f6..8599edc1f 100644 --- a/src/luthien_proxy/main.py +++ b/src/luthien_proxy/main.py @@ -41,6 +41,7 @@ ) from luthien_proxy.observability.redis_event_publisher import RedisEventPublisher from luthien_proxy.observability.sentry import init_sentry +from luthien_proxy.passthrough_materialize.worker import PassthroughReconcileWorker from luthien_proxy.passthrough_routes import router as passthrough_router from luthien_proxy.pipeline.upstream_headers import validate_upstream_headers_at_startup from luthien_proxy.policy_manager import PolicyManager @@ -356,6 +357,15 @@ async def lifespan(app: FastAPI): ) logger.info("Conversation retention disabled (CONVERSATION_RETENTION_DAYS not set)") + _passthrough_reconcile_worker: PassthroughReconcileWorker | None = None + if settings.passthrough_materialize_backfill_enabled: + _passthrough_reconcile_worker = PassthroughReconcileWorker( + db_pool=db_pool, + limit=settings.passthrough_materialize_batch_size, + interval_seconds=settings.passthrough_materialize_reconcile_interval_seconds, + ) + _passthrough_reconcile_worker.start() + # Initialize webhook sender _webhook_url = settings.webhook_url or None _webhook_sender = WebhookSender( @@ -408,6 +418,8 @@ async def lifespan(app: FastAPI): # before request handling has fully drained, fire_and_forget calls # could land against an already-closed httpx client. await _webhook_sender.stop() + if _passthrough_reconcile_worker is not None: + await _passthrough_reconcile_worker.stop() if _purger is not None: await _purger.stop() if _telemetry_sender is not None: diff --git a/src/luthien_proxy/passthrough_capture.py b/src/luthien_proxy/passthrough_capture.py index 07303d372..1f795414d 100644 --- a/src/luthien_proxy/passthrough_capture.py +++ b/src/luthien_proxy/passthrough_capture.py @@ -5,6 +5,7 @@ import json from collections.abc import Iterable, Mapping from typing import Literal +from urllib.parse import urlsplit, urlunsplit JsonValue = None | bool | int | float | str | list["JsonValue"] | dict[str, "JsonValue"] JsonObject = dict[str, JsonValue] @@ -37,6 +38,17 @@ def build_passthrough_headers(headers: Iterable[tuple[str, str]]) -> dict[str, s return forwarded +def build_upstream_url(base_url: str, path: str, query: str) -> str: + """Build an upstream URL from the configured base URL and client request target.""" + base = urlsplit(base_url.rstrip("/")) + return urlunsplit((base.scheme, base.netloc, f"/{path}", query, "")) + + +def client_response_headers(headers: Mapping[str, str]) -> dict[str, str]: + """Return upstream headers safe to send to the passthrough client.""" + return {key: value for key, value in headers.items() if key.lower() not in _RESPONSE_STRIPPED_HEADERS} + + def parse_openai_model(body: Mapping[str, JsonValue], override: str | None) -> str | None: """Return the request_logs model value for OpenAI passthrough calls.""" if override: @@ -129,6 +141,8 @@ def _raw_stream_capture(stream_format: StreamFormat, chunks: list[JsonValue], ra "JsonObject", "JsonValue", "build_passthrough_headers", + "build_upstream_url", + "client_response_headers", "json_loads", "parse_gemini_model", "parse_openai_model", diff --git a/src/luthien_proxy/passthrough_materialize/__init__.py b/src/luthien_proxy/passthrough_materialize/__init__.py new file mode 100644 index 000000000..218a9cc59 --- /dev/null +++ b/src/luthien_proxy/passthrough_materialize/__init__.py @@ -0,0 +1,39 @@ +"""Typed passthrough materialization domain foundations.""" + +from luthien_proxy.passthrough_materialize.endpoints import ( + EligibleEndpoint, + EndpointClassification, + EndpointKind, + ExcludedEndpoint, + Provider, + classify_endpoint, +) +from luthien_proxy.passthrough_materialize.gemini_request import normalize_gemini_request +from luthien_proxy.passthrough_materialize.gemini_response import normalize_gemini_response +from luthien_proxy.passthrough_materialize.payloads import ( + CanonicalRequestInput, + CanonicalRequestPayload, + CanonicalResponseInput, + CanonicalResponsePayload, + ResponseEventType, + build_request_event_payload, + build_response_event_payload, +) + +__all__ = [ + "CanonicalRequestInput", + "CanonicalRequestPayload", + "CanonicalResponseInput", + "CanonicalResponsePayload", + "EligibleEndpoint", + "EndpointClassification", + "EndpointKind", + "ExcludedEndpoint", + "Provider", + "ResponseEventType", + "build_request_event_payload", + "build_response_event_payload", + "classify_endpoint", + "normalize_gemini_request", + "normalize_gemini_response", +] diff --git a/src/luthien_proxy/passthrough_materialize/backfill.py b/src/luthien_proxy/passthrough_materialize/backfill.py new file mode 100644 index 000000000..19a38d9dd --- /dev/null +++ b/src/luthien_proxy/passthrough_materialize/backfill.py @@ -0,0 +1,36 @@ +"""One-shot bounded backfill for passthrough materialization.""" + +from __future__ import annotations + +import logging + +from luthien_proxy.passthrough_materialize.materialize_types import ReconcileStats +from luthien_proxy.passthrough_materialize.reconcile import reconcile_passthrough +from luthien_proxy.utils.db import DatabasePool + +logger = logging.getLogger(__name__) + + +async def drain_passthrough_backfill(db_pool: DatabasePool, *, limit: int) -> ReconcileStats: + """Run bounded sweeps until a sweep adds no new materialized transactions.""" + totals = ReconcileStats() + while True: + sweep = await reconcile_passthrough(db_pool, limit=limit) + totals = ReconcileStats( + materialized=totals.materialized + sweep.materialized, + already_materialized=totals.already_materialized + sweep.already_materialized, + skipped_ineligible=totals.skipped_ineligible + sweep.skipped_ineligible, + failed=totals.failed + sweep.failed, + ) + logger.info( + "Passthrough backfill progress: materialized=%d already_materialized=%d skipped_ineligible=%d failed=%d", + totals.materialized, + totals.already_materialized, + totals.skipped_ineligible, + totals.failed, + ) + if sweep.materialized == 0: + return totals + + +__all__ = ["drain_passthrough_backfill"] diff --git a/src/luthien_proxy/passthrough_materialize/endpoints.py b/src/luthien_proxy/passthrough_materialize/endpoints.py new file mode 100644 index 000000000..2b326d3e5 --- /dev/null +++ b/src/luthien_proxy/passthrough_materialize/endpoints.py @@ -0,0 +1,90 @@ +"""Endpoint eligibility classification for passthrough materialization.""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import StrEnum + + +class Provider(StrEnum): + """Provider families supported by passthrough materialization.""" + + OPENAI = "openai" + GEMINI = "gemini" + + +class EndpointKind(StrEnum): + """Materializable provider endpoint variants.""" + + OPENAI_CHAT_COMPLETIONS = "openai_chat_completions" + OPENAI_RESPONSES = "openai_responses" + GEMINI_GENERATE_CONTENT = "gemini_generate_content" + GEMINI_STREAM_GENERATE_CONTENT = "gemini_stream_generate_content" + + +@dataclass(frozen=True, slots=True) +class EligibleEndpoint: + """Endpoint that can become canonical conversation history.""" + + path: str + provider: Provider + kind: EndpointKind + + +@dataclass(frozen=True, slots=True) +class ExcludedEndpoint: + """Endpoint intentionally skipped by passthrough materialization.""" + + path: str + + +type EndpointClassification = EligibleEndpoint | ExcludedEndpoint + + +def classify_endpoint(path: str) -> EndpointClassification: + """Classify a captured request path for materialization eligibility.""" + match path: + case "/openai/v1/chat/completions": + return EligibleEndpoint( + path=path, + provider=Provider.OPENAI, + kind=EndpointKind.OPENAI_CHAT_COMPLETIONS, + ) + case "/openai/v1/responses": + return EligibleEndpoint( + path=path, + provider=Provider.OPENAI, + kind=EndpointKind.OPENAI_RESPONSES, + ) + case gemini_path if _is_gemini_generate_content(gemini_path): + return EligibleEndpoint( + path=path, + provider=Provider.GEMINI, + kind=EndpointKind.GEMINI_GENERATE_CONTENT, + ) + case gemini_path if _is_gemini_stream_generate_content(gemini_path): + return EligibleEndpoint( + path=path, + provider=Provider.GEMINI, + kind=EndpointKind.GEMINI_STREAM_GENERATE_CONTENT, + ) + case _: + return ExcludedEndpoint(path=path) + + +def _is_gemini_generate_content(path: str) -> bool: + return path.startswith("/gemini/") and path.endswith(":generateContent") + + +def _is_gemini_stream_generate_content(path: str) -> bool: + return path.startswith("/gemini/") and path.endswith(":streamGenerateContent") + + +__all__ = [ + "EligibleEndpoint", + "EndpointClassification", + "EndpointKind", + "ExcludedEndpoint", + "Provider", + "classify_endpoint", +] diff --git a/src/luthien_proxy/passthrough_materialize/gemini.py b/src/luthien_proxy/passthrough_materialize/gemini.py new file mode 100644 index 000000000..69fbced43 --- /dev/null +++ b/src/luthien_proxy/passthrough_materialize/gemini.py @@ -0,0 +1,15 @@ +"""Gemini passthrough normalization public API.""" + +from luthien_proxy.passthrough_materialize.gemini_request import normalize_gemini_request +from luthien_proxy.passthrough_materialize.gemini_response import normalize_gemini_response +from luthien_proxy.passthrough_materialize.openai_common import ( + PassthroughNormalizeError, + PassthroughNormalizeReason, +) + +__all__ = [ + "PassthroughNormalizeError", + "PassthroughNormalizeReason", + "normalize_gemini_request", + "normalize_gemini_response", +] diff --git a/src/luthien_proxy/passthrough_materialize/gemini_common.py b/src/luthien_proxy/passthrough_materialize/gemini_common.py new file mode 100644 index 000000000..b4d1e2304 --- /dev/null +++ b/src/luthien_proxy/passthrough_materialize/gemini_common.py @@ -0,0 +1,130 @@ +"""Shared strict helpers for Gemini passthrough normalization.""" + +from __future__ import annotations + +from collections.abc import Mapping + +from luthien_proxy.passthrough_materialize.endpoints import EligibleEndpoint, EndpointKind, Provider +from luthien_proxy.passthrough_materialize.openai_common import ( + PassthroughNormalizeReason, + fail, + is_json_object, + json_mutable_object, + optional_string, +) +from luthien_proxy.passthrough_materialize.payloads import JsonMutableObject, JsonMutableValue, JsonObject, JsonValue + + +def gemini_endpoint_streaming(endpoint: EligibleEndpoint, transaction_id: str | None) -> bool: + """Return True when the eligible Gemini endpoint is the streaming variant.""" + match endpoint.provider: + case Provider.GEMINI: + pass + case _: + fail(endpoint, transaction_id, PassthroughNormalizeReason.UNSUPPORTED_ENDPOINT, "provider mismatch") + match endpoint.kind: + case EndpointKind.GEMINI_GENERATE_CONTENT: + return False + case EndpointKind.GEMINI_STREAM_GENERATE_CONTENT: + return True + case _: + fail(endpoint, transaction_id, PassthroughNormalizeReason.UNSUPPORTED_ENDPOINT, "endpoint kind mismatch") + + +def gemini_model_from_path(path: str) -> str | None: + """Extract the Gemini model id from a captured request path.""" + marker = "models/" + if marker not in path: + return None + return path.split(marker, maxsplit=1)[1].split(":", maxsplit=1)[0] or None + + +def gemini_function_call_id(call: Mapping[str, JsonValue], name: str, ordinal: int) -> str: + """Return a provider call id or the deterministic identity for an id-less call.""" + provider_id = optional_string(call, "id") + return provider_id or f"gemini:{name}:{ordinal}" + + +def gemini_function_call( + endpoint: EligibleEndpoint, + call: Mapping[str, JsonValue], + ordinal: int, + transaction_id: str | None, +) -> JsonMutableObject: + """Normalize a Gemini functionCall using its provider id or per-turn identity.""" + name = optional_string(call, "name") + if name is None: + fail(endpoint, transaction_id, PassthroughNormalizeReason.MISSING_REQUIRED_FIELD, "functionCall.name") + args = call.get("args") + call_id = gemini_function_call_id(call, name, ordinal) + if args is None: + return {"type": "tool_use", "id": call_id, "name": name, "input": {}} + if not is_json_object(args): + fail(endpoint, transaction_id, PassthroughNormalizeReason.MALFORMED_PAYLOAD, "functionCall.args") + return {"type": "tool_use", "id": call_id, "name": name, "input": json_mutable_object(args)} + + +def gemini_usage(endpoint: EligibleEndpoint, value: JsonValue, transaction_id: str | None) -> JsonMutableObject: + """Convert Gemini usageMetadata into canonical token usage counts.""" + if not is_json_object(value): + fail(endpoint, transaction_id, PassthroughNormalizeReason.MALFORMED_PAYLOAD, "usageMetadata") + result: JsonMutableObject = {} + for source_key, target_key in ( + ("promptTokenCount", "input_tokens"), + ("candidatesTokenCount", "output_tokens"), + ("totalTokenCount", "total_tokens"), + ("cachedContentTokenCount", "cache_read_input_tokens"), + ("thoughtsTokenCount", "reasoning_tokens"), + ): + token_count = value.get(source_key) + if token_count is None: + continue + if not isinstance(token_count, int) or isinstance(token_count, bool): + fail(endpoint, transaction_id, PassthroughNormalizeReason.MALFORMED_PAYLOAD, f"usageMetadata.{source_key}") + result[target_key] = token_count + return result + + +def gemini_stop_reason(endpoint: EligibleEndpoint, reason: str, transaction_id: str | None) -> str: + """Map a Gemini finishReason to a canonical stop reason.""" + match reason: + case "STOP": + return "end_turn" + case "MAX_TOKENS": + return "max_tokens" + case "SAFETY": + return "safety" + case "RECITATION" | "LANGUAGE": + return "refusal" + case "BLOCKLIST" | "PROHIBITED_CONTENT" | "SPII" | "IMAGE_SAFETY" | "IMAGE_PROHIBITED_CONTENT": + return "blocked" + case "MALFORMED_FUNCTION_CALL" | "UNEXPECTED_TOOL_CALL" | "TOO_MANY_TOOL_CALLS" | "MISSING_THOUGHT_SIGNATURE": + return "error" + case "FINISH_REASON_UNSPECIFIED" | "OTHER": + return "error" + case _: + fail( + endpoint, + transaction_id, + PassthroughNormalizeReason.UNSUPPORTED_VARIANT, + f"candidate.finishReason:{reason}", + ) + + +def gemini_content_free_parts( + endpoint: EligibleEndpoint, stop_reason: str, transaction_id: str | None +) -> list[JsonMutableValue]: + """Return an empty assistant turn only for terminal safety outcomes.""" + if stop_reason not in {"safety", "refusal", "blocked"}: + fail(endpoint, transaction_id, PassthroughNormalizeReason.MISSING_REQUIRED_FIELD, "candidate.content") + return [] + + +def gemini_response_identifiers(response: JsonObject, final: JsonMutableObject) -> None: + """Copy Gemini response id and model version into the canonical final object.""" + response_id = response.get("responseId") + if isinstance(response_id, str): + final["id"] = response_id + model_version = response.get("modelVersion") + if isinstance(model_version, str): + final["model"] = model_version diff --git a/src/luthien_proxy/passthrough_materialize/gemini_request.py b/src/luthien_proxy/passthrough_materialize/gemini_request.py new file mode 100644 index 000000000..8f865e5d2 --- /dev/null +++ b/src/luthien_proxy/passthrough_materialize/gemini_request.py @@ -0,0 +1,235 @@ +"""Gemini generateContent request normalization.""" + +from __future__ import annotations + +from collections.abc import Mapping + +from luthien_proxy.passthrough_materialize.endpoints import EligibleEndpoint +from luthien_proxy.passthrough_materialize.gemini_common import ( + gemini_endpoint_streaming, + gemini_function_call_id, + gemini_model_from_path, +) +from luthien_proxy.passthrough_materialize.openai_common import ( + PassthroughNormalizeReason, + fail, + is_json_object, + is_json_sequence, + json_mutable_object, + optional_string, + sequence_field, +) +from luthien_proxy.passthrough_materialize.payloads import ( + CanonicalRequestInput, + JsonMutableObject, + JsonMutableValue, + JsonObject, + JsonValue, +) + + +def normalize_gemini_request( + endpoint: EligibleEndpoint, request: JsonObject, *, transaction_id: str | None = None +) -> CanonicalRequestInput: + """Normalize a Gemini generateContent request into canonical request input.""" + stream = gemini_endpoint_streaming(endpoint, transaction_id) + messages = _request_messages(endpoint, request, transaction_id) + model = gemini_model_from_path(endpoint.path) + final_request: JsonMutableObject = {"model": model, "messages": messages, "stream": stream} + _copy_request_configuration(request, final_request) + return CanonicalRequestInput(endpoint, stream, model, final_request, final_request, request) + + +def _request_messages( + endpoint: EligibleEndpoint, request: JsonObject, transaction_id: str | None +) -> list[JsonMutableValue]: + messages: list[JsonMutableValue] = [] + system_instruction = request.get("systemInstruction") + if is_json_object(system_instruction): + system_parts = _parts(system_instruction, "system") + if system_parts: + messages.append({"role": "system", "content": system_parts}) + contents = sequence_field(endpoint, request, "contents", transaction_id) + for content in contents: + if is_json_object(content): + message = _content_message(content) + if message is not None: + messages.append(message) + if not messages: + fail(endpoint, transaction_id, PassthroughNormalizeReason.MISSING_REQUIRED_FIELD, "contents") + return messages + + +def _content_message(content: JsonObject) -> JsonMutableObject | None: + # Gemini API: role is OPTIONAL in contents[]; defaults to "user" when omitted. + # https://ai.google.dev/api/generate-content#Content + role = optional_string(content, "role") or "user" + match role: + case "user": + parts = _parts(content, "user") + return {"role": "user", "content": parts} if parts else None + case "model": + parts = _parts(content, "model") + return {"role": "assistant", "content": parts} if parts else None + case _: + return None + + +def _parts(content: JsonObject, role: str) -> list[JsonMutableValue]: + parts = content.get("parts") + if not is_json_sequence(parts): + return [] + blocks: list[JsonMutableValue] = [] + function_ordinal = 0 + for part in parts: + if is_json_object(part): + block = _part(part, role, function_ordinal) + if block is not None: + blocks.append(block) + if "functionCall" in part or "functionResponse" in part: + function_ordinal += 1 + return blocks + + +def _part(part: JsonObject, role: str, function_ordinal: int) -> JsonMutableValue | None: + match part: + case {"text": str() as text}: + return {"type": "text", "text": text} + case {"functionCall": Mapping() as call} if role == "model": + return _function_call(call, function_ordinal) + case {"functionResponse": Mapping() as response} if role == "user": + return _function_response(response, function_ordinal) + case {"functionCall": _} | {"functionResponse": _}: + return None + case _: + return None + + +def _function_call(call: Mapping[str, JsonValue], ordinal: int) -> JsonMutableObject | None: + name = optional_string(call, "name") + args = call.get("args") + if name is None or (args is not None and not is_json_object(args)): + return None + return { + "type": "tool_use", + "id": gemini_function_call_id(call, name, ordinal), + "name": name, + "input": {} if args is None else json_mutable_object(args), + } + + +def _function_response(response: Mapping[str, JsonValue], ordinal: int) -> JsonMutableObject | None: + name = optional_string(response, "name") + response_value = response.get("response") + if name is None or not is_json_object(response_value): + return None + return { + "type": "tool_result", + "tool_use_id": gemini_function_call_id(response, name, ordinal), + "content": json_mutable_object(response_value), + } + + +def _copy_request_configuration( + request: JsonObject, + final_request: JsonMutableObject, +) -> None: + tools = request.get("tools") + if tools is not None: + final_request["tools"] = _tools(tools) + tool_config = request.get("toolConfig") + if tool_config is not None: + tool_choice = _tool_choice(tool_config) + if tool_choice is not None: + final_request["tool_choice"] = tool_choice + generation_config = request.get("generationConfig") + if is_json_object(generation_config): + final_request["generation_config"] = json_mutable_object(generation_config) + _generation_config(generation_config, final_request) + + +def _tools(raw_tools: JsonValue) -> list[JsonMutableValue]: + if not is_json_sequence(raw_tools): + return [] + tools: list[JsonMutableValue] = [] + for tool in raw_tools: + match tool: + case {"functionDeclarations": _} if is_json_object(tool): + declarations = tool.get("functionDeclarations") + if not is_json_sequence(declarations): + continue + for declaration in declarations: + if not is_json_object(declaration): + continue + normalized = _function_declaration(declaration) + if normalized is not None: + tools.append(normalized) + case _: + continue + return tools + + +def _function_declaration(declaration: JsonObject) -> JsonMutableObject | None: + name = optional_string(declaration, "name") + if name is None: + return None + result: JsonMutableObject = {"name": name} + description = optional_string(declaration, "description") + if description is not None: + result["description"] = description + parameters = declaration.get("parameters") + if is_json_object(parameters): + result["input_schema"] = json_mutable_object(parameters) + return result + + +def _tool_choice(raw_config: JsonValue) -> JsonMutableObject | None: + if not is_json_object(raw_config): + return None + function_config = raw_config.get("functionCallingConfig") + if not is_json_object(function_config): + return None + mode = optional_string(function_config, "mode") + match mode: + case "AUTO": + result: JsonMutableObject = {"mode": "auto"} + case "ANY": + result = {"mode": "any"} + case "NONE": + result = {"mode": "none"} + case "VALIDATED": + result = {"mode": "validated"} + case _: + return None + names = function_config.get("allowedFunctionNames") + if is_json_sequence(names): + allowed_names: list[JsonMutableValue] = [] + for name in names: + if isinstance(name, str): + allowed_names.append(name) + result["allowed_function_names"] = allowed_names + return result + + +def _generation_config(config: JsonObject, final_request: JsonMutableObject) -> None: + for key, value in config.items(): + match key, value: + case "temperature", int() | float() if not isinstance(value, bool): + final_request["temperature"] = value + case "topP", int() | float() if not isinstance(value, bool): + final_request["top_p"] = value + case "maxOutputTokens", int() if not isinstance(value, bool): + final_request["max_tokens"] = value + case "candidateCount", int() if not isinstance(value, bool): + final_request["candidate_count"] = value + case "stopSequences", _ if is_json_sequence(value): + stop_sequences: list[JsonMutableValue] = [] + for item in value: + if isinstance(item, str): + stop_sequences.append(item) + final_request["stop"] = stop_sequences + case _: + continue + + +__all__ = ["normalize_gemini_request"] diff --git a/src/luthien_proxy/passthrough_materialize/gemini_response.py b/src/luthien_proxy/passthrough_materialize/gemini_response.py new file mode 100644 index 000000000..f8e0d387b --- /dev/null +++ b/src/luthien_proxy/passthrough_materialize/gemini_response.py @@ -0,0 +1,180 @@ +"""Gemini generateContent response normalization.""" + +from __future__ import annotations + +from collections.abc import Mapping + +from pydantic import ValidationError + +from luthien_proxy.passthrough_materialize.endpoints import EligibleEndpoint +from luthien_proxy.passthrough_materialize.gemini_common import ( + gemini_content_free_parts, + gemini_endpoint_streaming, + gemini_function_call, + gemini_response_identifiers, + gemini_stop_reason, + gemini_usage, +) +from luthien_proxy.passthrough_materialize.gemini_stream import normalize_gemini_stream_response +from luthien_proxy.passthrough_materialize.openai_common import ( + PassthroughNormalizeReason, + error_response, + fail, + is_json_object, + is_json_sequence, + json_mutable, + json_mutable_object, + optional_string, + sequence_field, +) +from luthien_proxy.passthrough_materialize.payloads import ( + CanonicalResponseInput, + JsonMutableObject, + JsonMutableValue, + JsonObject, +) +from luthien_proxy.passthrough_materialize.provider_models import parse_gemini_response + + +def normalize_gemini_response( + endpoint: EligibleEndpoint, + response: JsonObject, + *, + request_is_streaming: bool, + http_status: int, + transaction_id: str | None = None, +) -> CanonicalResponseInput: + """Normalize a Gemini generateContent response into canonical response input.""" + endpoint_streaming = gemini_endpoint_streaming(endpoint, transaction_id) + if endpoint_streaming != request_is_streaming: + fail(endpoint, transaction_id, PassthroughNormalizeReason.UNSUPPORTED_ENDPOINT, "streaming metadata mismatch") + if http_status >= 400: + final_response = error_response(http_status, response) + elif request_is_streaming: + final_response = normalize_gemini_stream_response(endpoint, response, transaction_id) + else: + try: + source = parse_gemini_response(response).model_dump(mode="json", by_alias=True, exclude_none=True) + except ValidationError: + # Best-effort SDK: the google-genai response model is strict + # (extra='forbid') and lags the live REST API (e.g. it rejects + # usageMetadata.serviceTier); fall back to the raw payload, which + # _buffered_response maps leniently. + source = response + final_response = _buffered_response(endpoint, source, transaction_id) + model = final_response.get("model") + final_model = model if isinstance(model, str) else None + return CanonicalResponseInput(endpoint, request_is_streaming, final_model, final_response, final_response, response) + + +def _buffered_response( + endpoint: EligibleEndpoint, response: JsonObject, transaction_id: str | None +) -> JsonMutableObject: + prompt_feedback = response.get("promptFeedback") + if prompt_feedback is not None: + if not is_json_object(prompt_feedback): + fail(endpoint, transaction_id, PassthroughNormalizeReason.MALFORMED_PAYLOAD, "promptFeedback") + return _blocked_response(endpoint, response, prompt_feedback, transaction_id) + return _candidate_response(endpoint, response, _zero_candidate(endpoint, response, transaction_id), transaction_id) + + +def _zero_candidate(endpoint: EligibleEndpoint, response: JsonObject, transaction_id: str | None) -> JsonObject: + for candidate in sequence_field(endpoint, response, "candidates", transaction_id): + if not is_json_object(candidate): + fail(endpoint, transaction_id, PassthroughNormalizeReason.MALFORMED_PAYLOAD, "candidate") + index = candidate.get("index", 0) + if not isinstance(index, int) or isinstance(index, bool): + fail(endpoint, transaction_id, PassthroughNormalizeReason.MALFORMED_PAYLOAD, "candidate.index") + if index == 0: + return candidate + fail(endpoint, transaction_id, PassthroughNormalizeReason.MISSING_REQUIRED_FIELD, "candidates[0]") + + +def _blocked_response( + endpoint: EligibleEndpoint, response: JsonObject, feedback: JsonObject, transaction_id: str | None +) -> JsonMutableObject: + block_reason = optional_string(feedback, "blockReason") + if block_reason is None: + fail(endpoint, transaction_id, PassthroughNormalizeReason.MISSING_REQUIRED_FIELD, "promptFeedback.blockReason") + final: JsonMutableObject = {"role": "assistant", "content": [], "stop_reason": "blocked"} + gemini_response_identifiers(response, final) + _add_usage(endpoint, response, final, transaction_id) + final["prompt_feedback"] = json_mutable_object(feedback) + return final + + +def _candidate_response( + endpoint: EligibleEndpoint, response: JsonObject, candidate: JsonObject, transaction_id: str | None +) -> JsonMutableObject: + finish_reason = optional_string(candidate, "finishReason") + if finish_reason is None: + fail(endpoint, transaction_id, PassthroughNormalizeReason.MISSING_REQUIRED_FIELD, "candidate.finishReason") + content = candidate.get("content") + stop_reason = gemini_stop_reason(endpoint, finish_reason, transaction_id) + if content is None: + parts = gemini_content_free_parts(endpoint, stop_reason, transaction_id) + elif not is_json_object(content): + fail(endpoint, transaction_id, PassthroughNormalizeReason.MISSING_REQUIRED_FIELD, "candidate.content") + else: + role = optional_string(content, "role") + match role: + case "model": + pass + case None: + fail( + endpoint, + transaction_id, + PassthroughNormalizeReason.MISSING_REQUIRED_FIELD, + "candidate.content.role", + ) + case _: + fail(endpoint, transaction_id, PassthroughNormalizeReason.UNSUPPORTED_VARIANT, "candidate.content.role") + parts = _candidate_parts(endpoint, content, transaction_id) + if not parts: + parts = gemini_content_free_parts(endpoint, stop_reason, transaction_id) + final: JsonMutableObject = { + "role": "assistant", + "content": parts, + "stop_reason": stop_reason, + } + gemini_response_identifiers(response, final) + _add_usage(endpoint, response, final, transaction_id) + safety_ratings = candidate.get("safetyRatings") + if safety_ratings is not None: + if not is_json_sequence(safety_ratings): + fail(endpoint, transaction_id, PassthroughNormalizeReason.MALFORMED_PAYLOAD, "candidate.safetyRatings") + final["safety_ratings"] = json_mutable(safety_ratings) + return final + + +def _candidate_parts( + endpoint: EligibleEndpoint, content: JsonObject, transaction_id: str | None +) -> list[JsonMutableValue]: + parts = sequence_field(endpoint, content, "parts", transaction_id) + result: list[JsonMutableValue] = [] + function_ordinal = 0 + for part in parts: + if not is_json_object(part): + fail(endpoint, transaction_id, PassthroughNormalizeReason.MALFORMED_PAYLOAD, "candidate.part") + match part: + case {"text": str() as text}: + result.append({"type": "text", "text": text}) + case {"functionCall": Mapping() as call}: + result.append(gemini_function_call(endpoint, call, function_ordinal, transaction_id)) + function_ordinal += 1 + case {"functionResponse": _}: + continue + case _: + continue + return result + + +def _add_usage( + endpoint: EligibleEndpoint, response: JsonObject, final: JsonMutableObject, transaction_id: str | None +) -> None: + usage_metadata = response.get("usageMetadata") + if usage_metadata is not None: + final["usage"] = gemini_usage(endpoint, usage_metadata, transaction_id) + + +__all__ = ["normalize_gemini_response"] diff --git a/src/luthien_proxy/passthrough_materialize/gemini_stream.py b/src/luthien_proxy/passthrough_materialize/gemini_stream.py new file mode 100644 index 000000000..071030500 --- /dev/null +++ b/src/luthien_proxy/passthrough_materialize/gemini_stream.py @@ -0,0 +1,275 @@ +"""Gemini stored-stream wrapper normalization.""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, field + +from pydantic import ValidationError + +from luthien_proxy.passthrough_materialize.endpoints import EligibleEndpoint +from luthien_proxy.passthrough_materialize.gemini_common import ( + gemini_content_free_parts, + gemini_function_call_id, + gemini_response_identifiers, + gemini_stop_reason, + gemini_usage, +) +from luthien_proxy.passthrough_materialize.openai_common import ( + PassthroughNormalizeReason, + ensure_not_truncated, + fail, + is_json_object, + is_json_sequence, + json_mutable, + json_mutable_object, + optional_string, + sequence_field, +) +from luthien_proxy.passthrough_materialize.payloads import JsonMutableObject, JsonMutableValue, JsonObject, JsonValue +from luthien_proxy.passthrough_materialize.provider_models import parse_gemini_response + + +@dataclass(slots=True) +class _FunctionCallAccumulator: + """Accumulate one identified streamed function call before immutable boundary construction.""" + + name: str + arguments: JsonMutableObject + + +@dataclass(slots=True) +class _StreamAccumulator: + """Accumulate ordered provider chunks during one pure stream fold.""" + + text: list[str] = field(default_factory=list) + calls: dict[str, _FunctionCallAccumulator] = field(default_factory=dict) + call_order: list[str] = field(default_factory=list) + response_id: str | None = None + model_version: str | None = None + finish_reason: str | None = None + usage: JsonMutableObject | None = None + safety_ratings: JsonMutableValue | None = None + prompt_feedback: JsonMutableObject | None = None + + +def normalize_gemini_stream_response( + endpoint: EligibleEndpoint, response: JsonObject, transaction_id: str | None +) -> JsonMutableObject: + """Fold an exact Gemini JSON-array or SSE capture wrapper into one canonical response.""" + accumulator = _StreamAccumulator() + for chunk in _stream_chunks(endpoint, response, transaction_id): + _add_chunk(endpoint, accumulator, chunk, transaction_id) + return _folded_response(endpoint, accumulator, transaction_id) + + +def _stream_chunks(endpoint: EligibleEndpoint, response: JsonObject, transaction_id: str | None) -> Sequence[JsonValue]: + ensure_not_truncated(endpoint, response, transaction_id) + stream_format = optional_string(response, "stream_format") + match stream_format: + case "gemini-json-array" | "gemini-sse": + pass + case str(): + fail( + endpoint, + transaction_id, + PassthroughNormalizeReason.UNSUPPORTED_VARIANT, + f"stream_format:{stream_format}", + ) + case None: + fail(endpoint, transaction_id, PassthroughNormalizeReason.MISSING_REQUIRED_FIELD, "stream_format") + if "final" not in response: + fail(endpoint, transaction_id, PassthroughNormalizeReason.MISSING_REQUIRED_FIELD, "final") + if response.get("final") is not None: + fail(endpoint, transaction_id, PassthroughNormalizeReason.MALFORMED_PAYLOAD, "final") + return sequence_field(endpoint, response, "chunks", transaction_id) + + +def _add_chunk( + endpoint: EligibleEndpoint, accumulator: _StreamAccumulator, raw_chunk: JsonValue, transaction_id: str | None +) -> None: + if not is_json_object(raw_chunk): + fail(endpoint, transaction_id, PassthroughNormalizeReason.MALFORMED_PAYLOAD, "stream chunk") + try: + raw_chunk = parse_gemini_response(raw_chunk).model_dump(mode="json", by_alias=True, exclude_none=True) + except ValidationError: + # Best-effort SDK: the google-genai response model is strict (extra='forbid') + # and lags the live REST API (e.g. it rejects usageMetadata.serviceTier); + # keep the raw chunk so the downstream lenient mapping can proceed. Mirrors + # the buffered-response fallback in gemini_response.normalize_gemini_response. + pass + recognized = False + response_id = raw_chunk.get("responseId") + if response_id is not None: + if not isinstance(response_id, str): + fail(endpoint, transaction_id, PassthroughNormalizeReason.MALFORMED_PAYLOAD, "responseId") + accumulator.response_id = response_id + recognized = True + model_version = raw_chunk.get("modelVersion") + if model_version is not None: + if not isinstance(model_version, str): + fail(endpoint, transaction_id, PassthroughNormalizeReason.MALFORMED_PAYLOAD, "modelVersion") + accumulator.model_version = model_version + recognized = True + usage = raw_chunk.get("usageMetadata") + if usage is not None: + accumulator.usage = gemini_usage(endpoint, usage, transaction_id) + recognized = True + feedback = raw_chunk.get("promptFeedback") + if feedback is not None: + if not is_json_object(feedback): + fail(endpoint, transaction_id, PassthroughNormalizeReason.MALFORMED_PAYLOAD, "promptFeedback") + accumulator.prompt_feedback = json_mutable_object(feedback) + recognized = True + candidates = raw_chunk.get("candidates") + if candidates is not None: + if not is_json_sequence(candidates): + fail(endpoint, transaction_id, PassthroughNormalizeReason.MALFORMED_PAYLOAD, "candidates") + for candidate in candidates: + _add_candidate(endpoint, accumulator, candidate, transaction_id) + recognized = True + if not recognized: + fail(endpoint, transaction_id, PassthroughNormalizeReason.UNSUPPORTED_VARIANT, "stream chunk") + + +def _add_candidate( + endpoint: EligibleEndpoint, accumulator: _StreamAccumulator, raw_candidate: JsonValue, transaction_id: str | None +) -> None: + if not is_json_object(raw_candidate): + fail(endpoint, transaction_id, PassthroughNormalizeReason.MALFORMED_PAYLOAD, "candidate") + index = raw_candidate.get("index", 0) + if not isinstance(index, int) or isinstance(index, bool): + fail(endpoint, transaction_id, PassthroughNormalizeReason.MALFORMED_PAYLOAD, "candidate.index") + if index != 0: + fail(endpoint, transaction_id, PassthroughNormalizeReason.UNSUPPORTED_VARIANT, f"candidate.index:{index}") + content = raw_candidate.get("content") + if content is not None: + if not is_json_object(content): + fail(endpoint, transaction_id, PassthroughNormalizeReason.MALFORMED_PAYLOAD, "candidate.content") + _add_parts(endpoint, accumulator, content, transaction_id) + finish_reason = raw_candidate.get("finishReason") + if finish_reason is not None: + if not isinstance(finish_reason, str): + fail(endpoint, transaction_id, PassthroughNormalizeReason.MALFORMED_PAYLOAD, "candidate.finishReason") + accumulator.finish_reason = finish_reason + safety_ratings = raw_candidate.get("safetyRatings") + if safety_ratings is not None: + if not is_json_sequence(safety_ratings): + fail(endpoint, transaction_id, PassthroughNormalizeReason.MALFORMED_PAYLOAD, "candidate.safetyRatings") + accumulator.safety_ratings = json_mutable(safety_ratings) + + +def _add_parts( + endpoint: EligibleEndpoint, accumulator: _StreamAccumulator, content: JsonObject, transaction_id: str | None +) -> None: + role = optional_string(content, "role") + match role: + case "model": + pass + case None: + fail(endpoint, transaction_id, PassthroughNormalizeReason.MISSING_REQUIRED_FIELD, "candidate.content.role") + case _: + fail(endpoint, transaction_id, PassthroughNormalizeReason.UNSUPPORTED_VARIANT, "candidate.content.role") + function_ordinal = 0 + for raw_part in sequence_field(endpoint, content, "parts", transaction_id): + if not is_json_object(raw_part): + fail(endpoint, transaction_id, PassthroughNormalizeReason.MALFORMED_PAYLOAD, "candidate.part") + match raw_part: + case {"text": str() as text}: + accumulator.text.append(text) + case {"functionCall": Mapping() as call}: + _add_function_call(endpoint, accumulator, call, function_ordinal, transaction_id) + function_ordinal += 1 + case {"functionResponse": _}: + continue + case _: + continue + + +def _add_function_call( + endpoint: EligibleEndpoint, + accumulator: _StreamAccumulator, + call: Mapping[str, JsonValue], + ordinal: int, + transaction_id: str | None, +) -> None: + name = optional_string(call, "name") + args = call.get("args") + if name is None: + fail(endpoint, transaction_id, PassthroughNormalizeReason.MISSING_REQUIRED_FIELD, "functionCall.name") + call_id = gemini_function_call_id(call, name, ordinal) + if args is None: + arguments: JsonMutableObject = {} + elif is_json_object(args): + arguments = json_mutable_object(args) + else: + fail(endpoint, transaction_id, PassthroughNormalizeReason.MALFORMED_PAYLOAD, "functionCall.args") + existing = accumulator.calls.get(call_id) + if existing is None: + accumulator.calls[call_id] = _FunctionCallAccumulator(name=name, arguments=arguments) + accumulator.call_order.append(call_id) + return + if existing.name != name: + fail(endpoint, transaction_id, PassthroughNormalizeReason.MALFORMED_PAYLOAD, "functionCall.name conflict") + for key, value in arguments.items(): + prior = existing.arguments.get(key) + if prior is not None and prior != value: + fail(endpoint, transaction_id, PassthroughNormalizeReason.MALFORMED_PAYLOAD, "functionCall.args conflict") + existing.arguments[key] = value + + +def _folded_response( + endpoint: EligibleEndpoint, accumulator: _StreamAccumulator, transaction_id: str | None +) -> JsonMutableObject: + if accumulator.prompt_feedback is not None: + return _blocked_response(endpoint, accumulator, transaction_id) + if accumulator.finish_reason is None: + fail(endpoint, transaction_id, PassthroughNormalizeReason.MISSING_REQUIRED_FIELD, "candidate.finishReason") + content: list[JsonMutableValue] = [] + text = "".join(accumulator.text) + if text: + content.append({"type": "text", "text": text}) + for call_id in accumulator.call_order: + call = accumulator.calls[call_id] + content.append({"type": "tool_use", "id": call_id, "name": call.name, "input": call.arguments}) + stop_reason = gemini_stop_reason(endpoint, accumulator.finish_reason, transaction_id) + if not content: + content = gemini_content_free_parts(endpoint, stop_reason, transaction_id) + final: JsonMutableObject = { + "role": "assistant", + "content": content, + "stop_reason": stop_reason, + } + _add_metadata(final, accumulator) + if accumulator.safety_ratings is not None: + final["safety_ratings"] = accumulator.safety_ratings + return final + + +def _blocked_response( + endpoint: EligibleEndpoint, accumulator: _StreamAccumulator, transaction_id: str | None +) -> JsonMutableObject: + feedback = accumulator.prompt_feedback + if feedback is None: + fail(endpoint, transaction_id, PassthroughNormalizeReason.MISSING_REQUIRED_FIELD, "promptFeedback") + if optional_string(feedback, "blockReason") is None: + fail(endpoint, transaction_id, PassthroughNormalizeReason.MISSING_REQUIRED_FIELD, "promptFeedback.blockReason") + final: JsonMutableObject = {"role": "assistant", "content": [], "stop_reason": "blocked"} + _add_metadata(final, accumulator) + final["prompt_feedback"] = feedback + return final + + +def _add_metadata(final: JsonMutableObject, accumulator: _StreamAccumulator) -> None: + gemini_response_identifiers( + { + "responseId": accumulator.response_id, + "modelVersion": accumulator.model_version, + }, + final, + ) + if accumulator.usage is not None: + final["usage"] = accumulator.usage + + +__all__ = ["normalize_gemini_stream_response"] diff --git a/src/luthien_proxy/passthrough_materialize/materialize.py b/src/luthien_proxy/passthrough_materialize/materialize.py new file mode 100644 index 000000000..826188c19 --- /dev/null +++ b/src/luthien_proxy/passthrough_materialize/materialize.py @@ -0,0 +1,179 @@ +"""Transactional passthrough materialization entry point.""" + +from __future__ import annotations + +import logging +from dataclasses import replace +from datetime import datetime, timedelta +from typing import assert_never + +from opentelemetry import metrics + +from luthien_proxy.passthrough_materialize.endpoints import ( + EndpointKind, + ExcludedEndpoint, + classify_endpoint, +) +from luthien_proxy.passthrough_materialize.gemini import ( + normalize_gemini_request, + normalize_gemini_response, +) +from luthien_proxy.passthrough_materialize.materialize_read import ( + parse_captured_transaction, + read_raw_transaction, +) +from luthien_proxy.passthrough_materialize.materialize_types import ( + CanonicalTransaction, + CapturedTransaction, + MaterializationFailed, + MaterializationResult, + RawCapturedTransaction, + SkippedIneligible, +) +from luthien_proxy.passthrough_materialize.materialize_write import write_canonical_transaction +from luthien_proxy.passthrough_materialize.openai import ( + PassthroughNormalizeError, + normalize_openai_chat_request, + normalize_openai_chat_response, + normalize_openai_responses_request, + normalize_openai_responses_response, +) +from luthien_proxy.passthrough_materialize.openai_common import PassthroughNormalizeReason, fail +from luthien_proxy.passthrough_materialize.payloads import ( + CanonicalRequestInput, + CanonicalResponseInput, + build_request_event_payload, + build_response_event_payload, +) +from luthien_proxy.utils.db import DatabasePool + +logger = logging.getLogger(__name__) +_meter = metrics.get_meter("luthien_proxy.passthrough_materialize") +_materialization_failures = _meter.create_counter( + "luthien.passthrough.materialization.failures", + description="Passthrough transactions that could not be materialized and remain retryable.", +) + + +async def materialize_transaction(db_pool: DatabasePool, transaction_id: str) -> MaterializationResult: + """Materialize captured passthrough logs into canonical conversation rows.""" + raw_or_failure = await read_raw_transaction(db_pool, transaction_id) + match raw_or_failure: + case MaterializationFailed() as failure: + return _record_failure(failure) + case RawCapturedTransaction() as raw: + return await _materialize_raw_transaction(db_pool, raw) + case unreachable: + assert_never(unreachable) + + +async def _materialize_raw_transaction(db_pool: DatabasePool, raw: RawCapturedTransaction) -> MaterializationResult: + if raw.endpoint is None: + return _record_failure(MaterializationFailed(transaction_id=raw.transaction_id, reason="missing_endpoint")) + classification = classify_endpoint(raw.endpoint) + match classification: + case ExcludedEndpoint(): + return SkippedIneligible(transaction_id=raw.transaction_id, endpoint=raw.endpoint) + case endpoint: + try: + captured = parse_captured_transaction(raw, endpoint) + transaction = _canonical_transaction(captured) + except PassthroughNormalizeError as error: + return _record_failure( + MaterializationFailed(transaction_id=raw.transaction_id, reason=error.reason.value) + ) + return await write_canonical_transaction(db_pool, transaction) + + +def _canonical_transaction(captured: CapturedTransaction) -> CanonicalTransaction: + raw = captured.raw + request_input, response_input = _normalization_inputs(captured) + final_model = response_input.final_model or request_input.final_model or raw.model + request_payload = build_request_event_payload( + replace(request_input, final_model=final_model, is_streaming=raw.is_streaming) + ) + response_payload = build_response_event_payload( + replace(response_input, final_model=final_model, is_streaming=raw.is_streaming) + ) + response_status = _response_status(captured) + return CanonicalTransaction( + captured=captured, + request_payload=request_payload, + response_payload=response_payload, + final_model=final_model, + request_at=raw.started_at, + response_at=_response_timestamp(raw), + status="error" if raw.error is not None or response_status >= 400 else "completed", + ) + + +def _normalization_inputs(captured: CapturedTransaction) -> tuple[CanonicalRequestInput, CanonicalResponseInput]: + endpoint = captured.endpoint + raw = captured.raw + status = _response_status(captured) + match endpoint.kind: + case EndpointKind.OPENAI_CHAT_COMPLETIONS: + return ( + normalize_openai_chat_request(endpoint, captured.request_body, transaction_id=raw.transaction_id), + normalize_openai_chat_response( + endpoint, + captured.response_body, + request_is_streaming=raw.is_streaming, + http_status=status, + transaction_id=raw.transaction_id, + ), + ) + case EndpointKind.OPENAI_RESPONSES: + return ( + normalize_openai_responses_request(endpoint, captured.request_body, transaction_id=raw.transaction_id), + normalize_openai_responses_response( + endpoint, + captured.response_body, + request_is_streaming=raw.is_streaming, + http_status=status, + transaction_id=raw.transaction_id, + ), + ) + case EndpointKind.GEMINI_GENERATE_CONTENT | EndpointKind.GEMINI_STREAM_GENERATE_CONTENT: + return ( + normalize_gemini_request(endpoint, captured.request_body, transaction_id=raw.transaction_id), + normalize_gemini_response( + endpoint, + captured.response_body, + request_is_streaming=raw.is_streaming, + http_status=status, + transaction_id=raw.transaction_id, + ), + ) + case unreachable: + assert_never(unreachable) + + +def _response_status(captured: CapturedTransaction) -> int: + status = captured.raw.response_status + if status is not None: + return status + if captured.raw.error is not None: + return 502 + fail( + captured.endpoint, + captured.raw.transaction_id, + PassthroughNormalizeReason.MISSING_REQUIRED_FIELD, + "response_status", + ) + + +def _response_timestamp(raw: RawCapturedTransaction) -> datetime: + completed_at = raw.completed_at + if completed_at is not None and completed_at > raw.started_at: + return completed_at + return raw.started_at + timedelta(microseconds=1) + + +def _record_failure(failure: MaterializationFailed) -> MaterializationFailed: + _materialization_failures.add(1, {"reason": failure.reason}) + logger.warning("Passthrough materialization failed for %s: %s", failure.transaction_id, failure.reason) + return failure + + +__all__ = ["materialize_transaction"] diff --git a/src/luthien_proxy/passthrough_materialize/materialize_read.py b/src/luthien_proxy/passthrough_materialize/materialize_read.py new file mode 100644 index 000000000..82b05cd6e --- /dev/null +++ b/src/luthien_proxy/passthrough_materialize/materialize_read.py @@ -0,0 +1,158 @@ +"""Raw request-log selection and provider JSON parsing.""" + +from __future__ import annotations + +import json +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from datetime import datetime + +from luthien_proxy.passthrough_materialize.endpoints import EligibleEndpoint +from luthien_proxy.passthrough_materialize.materialize_types import ( + CapturedTransaction, + MaterializationFailed, + RawCapturedTransaction, +) +from luthien_proxy.passthrough_materialize.openai_common import ( + PassthroughNormalizeReason, + fail, + is_json_object, +) +from luthien_proxy.passthrough_materialize.payloads import JsonObject +from luthien_proxy.utils.db import DatabasePool, parse_db_ts + + +@dataclass(frozen=True, slots=True) +class _InvalidRequestLog(Exception): + detail: str + + def __str__(self) -> str: + return self.detail + + +async def read_raw_transaction( + db_pool: DatabasePool, transaction_id: str +) -> RawCapturedTransaction | MaterializationFailed: + """Return inbound-preferred persisted data for one request-log transaction.""" + async with db_pool.connection() as conn: + rows = await conn.fetch( + """ + SELECT request_body, response_body, response_status, session_id, user_id, + model, is_streaming, endpoint, error, started_at, completed_at + FROM request_logs + WHERE transaction_id = $1 + ORDER BY CASE direction WHEN 'inbound' THEN 0 ELSE 1 END, started_at + """, + transaction_id, + ) + if not rows: + return MaterializationFailed(transaction_id=transaction_id, reason="missing_request_logs") + try: + return _raw_transaction_from_rows(rows, transaction_id) + except _InvalidRequestLog as error: + return MaterializationFailed(transaction_id=transaction_id, reason=error.detail) + + +def parse_captured_transaction(raw: RawCapturedTransaction, endpoint: EligibleEndpoint) -> CapturedTransaction: + """Parse selected provider bodies or raise a typed retryable normalizer error.""" + request_body = _parse_provider_body(raw.request_body, endpoint, raw.transaction_id, "request_body") + response_body = _response_body(raw, endpoint) + return CapturedTransaction(raw=raw, endpoint=endpoint, request_body=request_body, response_body=response_body) + + +def _raw_transaction_from_rows(rows: Sequence[Mapping[str, object]], transaction_id: str) -> RawCapturedTransaction: + request_body = _optional_string(rows, "request_body") + response_body = _optional_string(rows, "response_body") + response_status = _optional_status(rows) + started_at = _required_timestamp(rows, "started_at") + return RawCapturedTransaction( + transaction_id=transaction_id, + request_body=request_body, + response_body=response_body, + response_status=response_status, + session_id=_optional_string(rows, "session_id"), + user_id=_optional_string(rows, "user_id"), + model=_optional_string(rows, "model"), + is_streaming=_is_streaming(rows), + endpoint=_optional_string(rows, "endpoint"), + error=_optional_string(rows, "error"), + started_at=started_at, + completed_at=_optional_timestamp(rows, "completed_at"), + ) + + +def _response_body(raw: RawCapturedTransaction, endpoint: EligibleEndpoint) -> JsonObject: + if raw.response_body is not None: + return _parse_provider_body(raw.response_body, endpoint, raw.transaction_id, "response_body") + if raw.error is not None: + return {"error": raw.error} + fail(endpoint, raw.transaction_id, PassthroughNormalizeReason.MISSING_REQUIRED_FIELD, "response_body") + + +def _parse_provider_body(raw: str | None, endpoint: EligibleEndpoint, transaction_id: str, field: str) -> JsonObject: + if raw is None: + fail(endpoint, transaction_id, PassthroughNormalizeReason.MISSING_REQUIRED_FIELD, field) + try: + parsed = json.loads(raw) + except json.JSONDecodeError: + fail(endpoint, transaction_id, PassthroughNormalizeReason.MALFORMED_JSON, field) + if is_json_object(parsed): + return parsed + fail(endpoint, transaction_id, PassthroughNormalizeReason.MALFORMED_PAYLOAD, field) + + +def _first_value(rows: Sequence[Mapping[str, object]], column: str) -> object | None: + for row in rows: + value = row[column] + if value is not None: + return value + return None + + +def _optional_string(rows: Sequence[Mapping[str, object]], column: str) -> str | None: + value = _first_value(rows, column) + if value is None: + return None + if isinstance(value, str): + return value + raise _InvalidRequestLog(detail=f"invalid_{column}") + + +def _optional_status(rows: Sequence[Mapping[str, object]]) -> int | None: + value = _first_value(rows, "response_status") + if value is None: + return None + if isinstance(value, int) and not isinstance(value, bool): + return value + raise _InvalidRequestLog(detail="invalid_response_status") + + +def _is_streaming(rows: Sequence[Mapping[str, object]]) -> bool: + value = _first_value(rows, "is_streaming") + match value: + case None | False | 0: + return False + case True | 1: + return True + case _: + raise _InvalidRequestLog(detail="invalid_is_streaming") + + +def _required_timestamp(rows: Sequence[Mapping[str, object]], column: str) -> datetime: + timestamp = _optional_timestamp(rows, column) + if timestamp is not None: + return timestamp + raise _InvalidRequestLog(detail=f"missing_{column}") + + +def _optional_timestamp(rows: Sequence[Mapping[str, object]], column: str) -> datetime | None: + value = _first_value(rows, column) + if value is None: + return None + try: + return parse_db_ts(value) + except TypeError as error: + raise _InvalidRequestLog(detail=f"invalid_{column}") from error + + +__all__ = ["parse_captured_transaction", "read_raw_transaction"] diff --git a/src/luthien_proxy/passthrough_materialize/materialize_types.py b/src/luthien_proxy/passthrough_materialize/materialize_types.py new file mode 100644 index 000000000..d532402a5 --- /dev/null +++ b/src/luthien_proxy/passthrough_materialize/materialize_types.py @@ -0,0 +1,115 @@ +"""Typed values used by passthrough materialization.""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime +from typing import Literal + +from luthien_proxy.passthrough_materialize.endpoints import EligibleEndpoint +from luthien_proxy.passthrough_materialize.payloads import ( + CanonicalRequestPayload, + CanonicalResponsePayload, + JsonObject, +) + + +@dataclass(frozen=True, slots=True) +class Materialized: + """A transaction whose canonical rows were written.""" + + transaction_id: str + status: Literal["materialized"] = "materialized" + + +@dataclass(frozen=True, slots=True) +class AlreadyMaterialized: + """A transaction that already has its canonical request event.""" + + transaction_id: str + status: Literal["already_materialized"] = "already_materialized" + + +@dataclass(frozen=True, slots=True) +class SkippedIneligible: + """A transaction whose endpoint intentionally has no canonical representation.""" + + transaction_id: str + endpoint: str + status: Literal["skipped_ineligible"] = "skipped_ineligible" + + +@dataclass(frozen=True, slots=True) +class MaterializationFailed: + """A retryable materialization failure that did not write canonical rows.""" + + transaction_id: str + reason: str + status: Literal["failed"] = "failed" + + +type MaterializationResult = Materialized | AlreadyMaterialized | SkippedIneligible | MaterializationFailed + + +@dataclass(frozen=True, slots=True) +class ReconcileStats: + """Outcome counts for one bounded passthrough reconciliation sweep.""" + + materialized: int = 0 + already_materialized: int = 0 + skipped_ineligible: int = 0 + failed: int = 0 + + +@dataclass(frozen=True, slots=True) +class RawCapturedTransaction: + """Inbound-preferred persisted request-log data before provider parsing.""" + + transaction_id: str + request_body: str | None + response_body: str | None + response_status: int | None + session_id: str | None + user_id: str | None + model: str | None + is_streaming: bool + endpoint: str | None + error: str | None + started_at: datetime + completed_at: datetime | None + + +@dataclass(frozen=True, slots=True) +class CapturedTransaction: + """Provider JSON parsed from a raw capture and ready for normalization.""" + + raw: RawCapturedTransaction + endpoint: EligibleEndpoint + request_body: JsonObject + response_body: JsonObject + + +@dataclass(frozen=True, slots=True) +class CanonicalTransaction: + """Normalized payloads and timestamps ready for one atomic DB write.""" + + captured: CapturedTransaction + request_payload: CanonicalRequestPayload + response_payload: CanonicalResponsePayload + final_model: str | None + request_at: datetime + response_at: datetime + status: Literal["completed", "error"] + + +__all__ = [ + "AlreadyMaterialized", + "CanonicalTransaction", + "CapturedTransaction", + "MaterializationFailed", + "MaterializationResult", + "Materialized", + "RawCapturedTransaction", + "ReconcileStats", + "SkippedIneligible", +] diff --git a/src/luthien_proxy/passthrough_materialize/materialize_write.py b/src/luthien_proxy/passthrough_materialize/materialize_write.py new file mode 100644 index 000000000..08807ea42 --- /dev/null +++ b/src/luthien_proxy/passthrough_materialize/materialize_write.py @@ -0,0 +1,125 @@ +"""Atomic canonical conversation persistence for passthrough captures.""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from datetime import datetime +from uuid import uuid4 + +from luthien_proxy.observability.session_summary import update_session_summary +from luthien_proxy.passthrough_materialize.materialize_types import ( + AlreadyMaterialized, + CanonicalTransaction, + Materialized, +) +from luthien_proxy.passthrough_materialize.payloads import CanonicalRequestPayload, CanonicalResponsePayload +from luthien_proxy.utils.db import ConnectionProtocol, DatabasePool + + +@dataclass(frozen=True, slots=True) +class _EventWrite: + event_type: str + payload: CanonicalRequestPayload | CanonicalResponsePayload + created_at: datetime + + +async def write_canonical_transaction( + db_pool: DatabasePool, transaction: CanonicalTransaction +) -> Materialized | AlreadyMaterialized: + """Write a complete canonical transaction or return its existing completion marker.""" + raw = transaction.captured.raw + async with db_pool.connection() as conn: + async with conn.transaction(): + if db_pool.is_postgres: + await conn.execute("SELECT pg_advisory_xact_lock(hashtext($1))", raw.transaction_id) + if await _request_event_exists(conn, raw.transaction_id): + return AlreadyMaterialized(transaction_id=raw.transaction_id) + await _upsert_call(conn, transaction) + request_event = _EventWrite( + event_type="transaction.request_recorded", + payload=transaction.request_payload, + created_at=transaction.request_at, + ) + response_event = _EventWrite( + event_type=transaction.response_payload["event_type"], + payload=transaction.response_payload, + created_at=transaction.response_at, + ) + await _insert_event(conn, transaction, request_event) + await _insert_event(conn, transaction, response_event) + await _update_summaries(conn, transaction, request_event) + await _update_summaries(conn, transaction, response_event) + return Materialized(transaction_id=raw.transaction_id) + + +async def _request_event_exists(conn: ConnectionProtocol, transaction_id: str) -> bool: + return ( + await conn.fetchrow( + """ + SELECT 1 FROM conversation_events + WHERE call_id = $1 AND event_type = 'transaction.request_recorded' + LIMIT 1 + """, + transaction_id, + ) + is not None + ) + + +async def _upsert_call(conn: ConnectionProtocol, transaction: CanonicalTransaction) -> None: + raw = transaction.captured.raw + await conn.execute( + """ + INSERT INTO conversation_calls ( + call_id, model_name, provider, status, created_at, completed_at, session_id, user_id + ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8) + ON CONFLICT (call_id) DO UPDATE SET + model_name = COALESCE(conversation_calls.model_name, EXCLUDED.model_name), + provider = COALESCE(conversation_calls.provider, EXCLUDED.provider), + status = COALESCE(conversation_calls.status, EXCLUDED.status), + completed_at = COALESCE(conversation_calls.completed_at, EXCLUDED.completed_at), + session_id = COALESCE(conversation_calls.session_id, EXCLUDED.session_id), + user_id = COALESCE(conversation_calls.user_id, EXCLUDED.user_id) + """, + raw.transaction_id, + transaction.final_model, + transaction.captured.endpoint.provider.value, + transaction.status, + transaction.request_at, + transaction.response_at, + raw.session_id, + raw.user_id, + ) + + +async def _insert_event(conn: ConnectionProtocol, transaction: CanonicalTransaction, event: _EventWrite) -> None: + raw = transaction.captured.raw + await conn.execute( + """ + INSERT INTO conversation_events (id, call_id, event_type, payload, created_at, session_id) + VALUES ($1, $2, $3, $4::jsonb, $5, $6) + """, + uuid4().hex, + raw.transaction_id, + event.event_type, + json.dumps(event.payload), + event.created_at, + raw.session_id, + ) + + +async def _update_summaries(conn: ConnectionProtocol, transaction: CanonicalTransaction, event: _EventWrite) -> None: + raw = transaction.captured.raw + if raw.session_id is not None and raw.session_id: + await update_session_summary( + conn, + session_id=raw.session_id, + event_type=event.event_type, + data=dict(event.payload), + user_id=raw.user_id, + timestamp=event.created_at, + ) + + +__all__ = ["write_canonical_transaction"] diff --git a/src/luthien_proxy/passthrough_materialize/openai.py b/src/luthien_proxy/passthrough_materialize/openai.py new file mode 100644 index 000000000..f1de4cbd0 --- /dev/null +++ b/src/luthien_proxy/passthrough_materialize/openai.py @@ -0,0 +1,23 @@ +"""OpenAI passthrough normalization public API.""" + +from luthien_proxy.passthrough_materialize.openai_chat import ( + normalize_openai_chat_request, + normalize_openai_chat_response, +) +from luthien_proxy.passthrough_materialize.openai_common import ( + PassthroughNormalizeError, + PassthroughNormalizeReason, +) +from luthien_proxy.passthrough_materialize.openai_responses import ( + normalize_openai_responses_request, + normalize_openai_responses_response, +) + +__all__ = [ + "PassthroughNormalizeError", + "PassthroughNormalizeReason", + "normalize_openai_chat_request", + "normalize_openai_chat_response", + "normalize_openai_responses_request", + "normalize_openai_responses_response", +] diff --git a/src/luthien_proxy/passthrough_materialize/openai_chat.py b/src/luthien_proxy/passthrough_materialize/openai_chat.py new file mode 100644 index 000000000..0da341fe9 --- /dev/null +++ b/src/luthien_proxy/passthrough_materialize/openai_chat.py @@ -0,0 +1,276 @@ +"""OpenAI Chat Completions passthrough normalizers.""" + +from __future__ import annotations + +import json +from collections.abc import Mapping, Sequence + +from openai.types.chat import ChatCompletionMessageFunctionToolCall, ChatCompletionMessageToolCallUnion +from pydantic import ValidationError + +from luthien_proxy.passthrough_materialize.endpoints import EligibleEndpoint, EndpointKind +from luthien_proxy.passthrough_materialize.openai_chat_stream import stream_chat_response +from luthien_proxy.passthrough_materialize.openai_common import ( + PassthroughNormalizeReason, + canonical_usage, + error_response, + fail, + is_json_object, + is_json_sequence, + json_mutable_object, + json_object_from_string, + lenient_text_content_from_openai, + optional_string, + require_openai_endpoint, + sequence_field, + stop_reason, +) +from luthien_proxy.passthrough_materialize.payloads import ( + CanonicalRequestInput, + CanonicalResponseInput, + JsonMutableObject, + JsonMutableValue, + JsonObject, + JsonValue, +) +from luthien_proxy.passthrough_materialize.provider_models import parse_openai_chat_completion + + +def normalize_openai_chat_request( + endpoint: EligibleEndpoint, request: JsonObject, *, transaction_id: str | None = None +) -> CanonicalRequestInput: + """Normalize a Chat Completions request into canonical request input.""" + require_openai_endpoint(endpoint, EndpointKind.OPENAI_CHAT_COMPLETIONS, transaction_id) + model = optional_string(request, "model") + messages = sequence_field(endpoint, request, "messages", transaction_id) + final_request: JsonMutableObject = {"model": model, "messages": _chat_messages(endpoint, messages, transaction_id)} + _copy_optional_request_fields(request, final_request) + stream = request.get("stream") is True + final_request["stream"] = stream + return CanonicalRequestInput( + endpoint=endpoint, + is_streaming=stream, + final_model=model, + original_request=final_request, + final_request=final_request, + provider_request=request, + ) + + +def normalize_openai_chat_response( + endpoint: EligibleEndpoint, + response: JsonObject, + *, + request_is_streaming: bool, + http_status: int, + transaction_id: str | None = None, +) -> CanonicalResponseInput: + """Normalize a Chat Completions response into canonical response input.""" + require_openai_endpoint(endpoint, EndpointKind.OPENAI_CHAT_COMPLETIONS, transaction_id) + if http_status >= 400: + final_response = error_response(http_status, response) + elif request_is_streaming: + final_response = stream_chat_response(endpoint, response, transaction_id) + else: + final_response = _buffered_chat_response(endpoint, response, transaction_id) + model_value = final_response.get("model") + final_model = model_value if isinstance(model_value, str) else None + return CanonicalResponseInput( + endpoint=endpoint, + is_streaming=request_is_streaming, + final_model=final_model, + original_response=final_response, + final_response=final_response, + provider_response=response, + ) + + +def _chat_messages( + endpoint: EligibleEndpoint, messages: JsonValue, transaction_id: str | None +) -> list[JsonMutableValue]: + result: list[JsonMutableValue] = [] + if not is_json_sequence(messages): + fail(endpoint, transaction_id, PassthroughNormalizeReason.MISSING_REQUIRED_FIELD, "messages") + for item in messages: + if is_json_object(item): + message = _chat_message(item) + if message is not None: + result.append(message) + if not result: + fail(endpoint, transaction_id, PassthroughNormalizeReason.MISSING_REQUIRED_FIELD, "messages") + return result + + +def _chat_message(item: Mapping[str, JsonValue]) -> JsonMutableObject | None: + role = optional_string(item, "role") + canonical_role = "system" if role == "developer" else role + match canonical_role: + case "system" | "user": + content = lenient_text_content_from_openai(item.get("content")) + if content is None: + return None + return {"role": canonical_role, "content": content} + case "tool": + content = lenient_text_content_from_openai(item.get("content")) + tool_call_id = item.get("tool_call_id") + if content is None or not isinstance(tool_call_id, str): + return None + return {"role": "tool", "tool_call_id": tool_call_id, "content": content} + case "assistant": + content_blocks: list[JsonMutableValue] = [] + normalized = lenient_text_content_from_openai(item.get("content")) + match normalized: + case str(): + content_blocks.append({"type": "text", "text": normalized}) + case list(): + content_blocks.extend(normalized) + case None: + pass + content_blocks.extend(_tool_calls(item.get("tool_calls"))) + return {"role": "assistant", "content": content_blocks} if content_blocks else None + case _: + return None + + +def _tool_calls(raw_calls: JsonValue) -> list[JsonMutableValue]: + if not is_json_sequence(raw_calls): + return [] + calls: list[JsonMutableValue] = [] + for raw_call in raw_calls: + if not is_json_object(raw_call): + continue + function = raw_call.get("function") + if not is_json_object(function): + continue + arguments = optional_string(function, "arguments") or "{}" + if (call_id := optional_string(raw_call, "id")) is None or (name := optional_string(function, "name")) is None: + continue + try: + arguments_value = json.loads(arguments) + except json.JSONDecodeError: + continue + if not is_json_object(arguments_value): + continue + calls.append( + { + "type": "tool_use", + "id": call_id, + "name": name, + "input": json_mutable_object(arguments_value), + } + ) + return calls + + +def _copy_optional_request_fields(request: JsonObject, final_request: JsonMutableObject) -> None: + for key in ("tool_choice", "temperature", "top_p", "stop", "max_completion_tokens", "max_tokens"): + if key in request: + final_request[key] = json_mutable_object({"value": request[key]})["value"] + if "max_tokens" not in final_request and isinstance(request.get("max_completion_tokens"), int): + final_request["max_tokens"] = json_mutable_object({"value": request["max_completion_tokens"]})["value"] + tools = request.get("tools") + if tools is not None: + final_request["tools"] = _tools(tools) + + +def _tools(tools: JsonValue) -> list[JsonMutableValue]: + if not is_json_sequence(tools): + return [] + result: list[JsonMutableValue] = [] + for tool in tools: + if not is_json_object(tool) or tool.get("type") != "function": + continue + function = tool.get("function") + if not is_json_object(function): + continue + name = optional_string(function, "name") + parameters = function.get("parameters") + if name is None or not is_json_object(parameters): + continue + canonical_tool: JsonMutableObject = { + "name": name, + "input_schema": json_mutable_object(parameters), + } + description = optional_string(function, "description") + if description is not None: + canonical_tool["description"] = description + result.append(canonical_tool) + return result + + +def _buffered_chat_response( + endpoint: EligibleEndpoint, response: JsonObject, transaction_id: str | None +) -> JsonMutableObject: + try: + parsed = parse_openai_chat_completion(response) + except ValidationError: + # The pinned openai SDK's strict Literals reject novel API values with an + # uncaught ValidationError; convert to a typed, retryable failure so a + # novel variant skips one transaction rather than wedging the batch. + fail(endpoint, transaction_id, PassthroughNormalizeReason.MALFORMED_PAYLOAD, "response") + if not parsed.choices: + fail(endpoint, transaction_id, PassthroughNormalizeReason.MALFORMED_PAYLOAD, "choices[0]") + first = parsed.choices[0] + message = first.message + return _assistant_response( + parsed.id, + parsed.model, + message.content, + message.refusal, + message.tool_calls, + first.finish_reason, + canonical_usage(parsed.usage.model_dump() if parsed.usage is not None else None), + endpoint, + transaction_id, + ) + + +def _assistant_response( + response_id: str, + model: str, + text: str | None, + refusal: str | None, + tool_calls: Sequence[ChatCompletionMessageToolCallUnion] | None, + finish_reason: str | None, + usage: JsonMutableObject | None, + endpoint: EligibleEndpoint, + transaction_id: str | None, +) -> JsonMutableObject: + content: list[JsonMutableValue] = [] + if text: + content.append({"type": "text", "text": text}) + if refusal: + content.append({"type": "text", "text": refusal}) + content.extend(_typed_tool_calls(endpoint, tool_calls, transaction_id)) + final: JsonMutableObject = {"role": "assistant", "content": content, "stop_reason": stop_reason(finish_reason)} + final["id"] = response_id + final["model"] = model + if usage is not None: + final["usage"] = usage + return final + + +def _typed_tool_calls( + endpoint: EligibleEndpoint, + tool_calls: Sequence[ChatCompletionMessageToolCallUnion] | None, + transaction_id: str | None, +) -> list[JsonMutableValue]: + if tool_calls is None: + return [] + calls: list[JsonMutableValue] = [] + for call in tool_calls: + match call: + case ChatCompletionMessageFunctionToolCall(id=call_id, function=function): + if not call_id: + fail(endpoint, transaction_id, PassthroughNormalizeReason.MISSING_REQUIRED_FIELD, "tool_call.id") + calls.append( + { + "type": "tool_use", + "id": call_id, + "name": function.name, + "input": json_object_from_string(endpoint, function.arguments, transaction_id), + } + ) + case _: + continue + return calls diff --git a/src/luthien_proxy/passthrough_materialize/openai_chat_stream.py b/src/luthien_proxy/passthrough_materialize/openai_chat_stream.py new file mode 100644 index 000000000..2981d7a9e --- /dev/null +++ b/src/luthien_proxy/passthrough_materialize/openai_chat_stream.py @@ -0,0 +1,125 @@ +"""OpenAI Chat Completions stream folding.""" + +from __future__ import annotations + +from dataclasses import dataclass + +from luthien_proxy.passthrough_materialize.endpoints import EligibleEndpoint +from luthien_proxy.passthrough_materialize.openai_common import ( + PassthroughNormalizeReason, + canonical_usage, + ensure_not_truncated, + fail, + is_json_object, + is_json_sequence, + json_object_from_string, + optional_string, + sequence_field, + stop_reason, +) +from luthien_proxy.passthrough_materialize.payloads import JsonMutableObject, JsonMutableValue, JsonObject, JsonValue + + +@dataclass(slots=True) +class StreamToolCall: + """Mutable accumulator for indexed Chat Completions tool-call deltas.""" + + call_id: str | None = None + name: str | None = None + arguments: str = "" + + +def stream_chat_response( + endpoint: EligibleEndpoint, response: JsonObject, transaction_id: str | None +) -> JsonMutableObject: + """Fold captured OpenAI SSE Chat Completions chunks into one response.""" + ensure_not_truncated(endpoint, response, transaction_id) + events = sequence_field(endpoint, response, "events", transaction_id) + content = "" + refusal = "" + model: str | None = None + response_id: str | None = None + finish_reason: str | None = None + usage: JsonMutableObject | None = None + tool_calls: dict[int, StreamToolCall] = {} + for event in events: + if not is_json_object(event): + fail(endpoint, transaction_id, PassthroughNormalizeReason.MALFORMED_PAYLOAD, "stream event") + model = optional_string(event, "model") or model + response_id = optional_string(event, "id") or response_id + event_usage = canonical_usage(event.get("usage")) + usage = event_usage or usage + choices = event.get("choices") + if not is_json_sequence(choices) or not choices: + continue + first = choices[0] + if not is_json_object(first): + fail(endpoint, transaction_id, PassthroughNormalizeReason.MALFORMED_PAYLOAD, "stream choice") + finish_reason = optional_string(first, "finish_reason") or finish_reason + delta = first.get("delta") + if is_json_object(delta): + content += optional_string(delta, "content") or "" + refusal += optional_string(delta, "refusal") or "" + _accumulate_tool_calls(endpoint, delta.get("tool_calls"), tool_calls, transaction_id) + final = _stream_final_content(endpoint, content, refusal, tool_calls, transaction_id) + result: JsonMutableObject = {"role": "assistant", "content": final, "stop_reason": stop_reason(finish_reason)} + if response_id is not None: + result["id"] = response_id + if model is not None: + result["model"] = model + if usage is not None: + result["usage"] = usage + return result + + +def _accumulate_tool_calls( + endpoint: EligibleEndpoint, + raw_calls: JsonValue, + tool_calls: dict[int, StreamToolCall], + transaction_id: str | None, +) -> None: + if raw_calls is None: + return + if not is_json_sequence(raw_calls): + fail(endpoint, transaction_id, PassthroughNormalizeReason.MALFORMED_PAYLOAD, "delta.tool_calls") + for raw_call in raw_calls: + if not is_json_object(raw_call): + fail(endpoint, transaction_id, PassthroughNormalizeReason.MALFORMED_PAYLOAD, "delta.tool_call") + index = raw_call.get("index") + if not isinstance(index, int): + fail(endpoint, transaction_id, PassthroughNormalizeReason.MISSING_REQUIRED_FIELD, "delta.tool_call.index") + call = tool_calls.setdefault(index, StreamToolCall()) + call.call_id = optional_string(raw_call, "id") or call.call_id + function = raw_call.get("function") + if is_json_object(function): + call.name = optional_string(function, "name") or call.name + call.arguments += optional_string(function, "arguments") or "" + + +def _stream_final_content( + endpoint: EligibleEndpoint, + text: str, + refusal: str, + tool_calls: dict[int, StreamToolCall], + transaction_id: str | None, +) -> list[JsonMutableValue]: + content: list[JsonMutableValue] = [] + if text: + content.append({"type": "text", "text": text}) + if refusal: + content.append({"type": "text", "text": refusal}) + for index in sorted(tool_calls): + call = tool_calls[index] + if not call.call_id: + fail(endpoint, transaction_id, PassthroughNormalizeReason.MISSING_REQUIRED_FIELD, "tool_call.id") + if call.name is None: + fail(endpoint, transaction_id, PassthroughNormalizeReason.MISSING_REQUIRED_FIELD, "tool_call.function.name") + content.append( + { + "type": "tool_use", + "id": call.call_id, + "name": call.name, + "input": json_object_from_string(endpoint, call.arguments or "{}", transaction_id), + } + ) + return content diff --git a/src/luthien_proxy/passthrough_materialize/openai_common.py b/src/luthien_proxy/passthrough_materialize/openai_common.py new file mode 100644 index 000000000..0144c721e --- /dev/null +++ b/src/luthien_proxy/passthrough_materialize/openai_common.py @@ -0,0 +1,224 @@ +"""Shared private helpers for OpenAI passthrough normalization.""" + +from __future__ import annotations + +import json +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from enum import StrEnum +from typing import NoReturn, TypeGuard + +from luthien_proxy.passthrough_materialize.endpoints import EligibleEndpoint, EndpointKind, Provider +from luthien_proxy.passthrough_materialize.payloads import JsonMutableObject, JsonMutableValue, JsonObject, JsonValue + + +class PassthroughNormalizeReason(StrEnum): + """Stable reason codes for retryable passthrough normalization failures.""" + + MISSING_REQUIRED_FIELD = "missing_required_field" + MALFORMED_JSON = "malformed_json" + MALFORMED_PAYLOAD = "malformed_payload" + UNSUPPORTED_ENDPOINT = "unsupported_endpoint" + UNSUPPORTED_VARIANT = "unsupported_variant" + CAPTURE_TRUNCATED = "capture_truncated" + + +@dataclass(frozen=True, slots=True) +class PassthroughNormalizeError(Exception): + """Typed failure raised when eligible passthrough JSON cannot become canonical.""" + + provider: Provider + endpoint_kind: EndpointKind + endpoint_path: str + transaction_id: str | None + reason: PassthroughNormalizeReason + detail: str + + def __str__(self) -> str: + """Return a stable diagnostic string for logs and reports.""" + tx = self.transaction_id or "" + return f"{self.provider.value}:{self.endpoint_kind.value}:{tx}:{self.reason.value}:{self.detail}" + + +def fail( + endpoint: EligibleEndpoint, transaction_id: str | None, reason: PassthroughNormalizeReason, detail: str +) -> NoReturn: + """Raise a typed passthrough normalization error.""" + raise PassthroughNormalizeError( + provider=endpoint.provider, + endpoint_kind=endpoint.kind, + endpoint_path=endpoint.path, + transaction_id=transaction_id, + reason=reason, + detail=detail, + ) + + +def is_json_object(value: JsonValue) -> TypeGuard[JsonObject]: + """Return whether a JSON value is an object.""" + return isinstance(value, Mapping) + + +def is_json_sequence(value: JsonValue) -> TypeGuard[Sequence[JsonValue]]: + """Return whether a JSON value is a non-string array.""" + return isinstance(value, Sequence) and not isinstance(value, str) + + +def require_openai_endpoint(endpoint: EligibleEndpoint, kind: EndpointKind, transaction_id: str | None) -> None: + """Ensure a normalizer is used with its matching OpenAI endpoint kind.""" + if endpoint.provider != Provider.OPENAI or endpoint.kind != kind: + fail(endpoint, transaction_id, PassthroughNormalizeReason.UNSUPPORTED_ENDPOINT, "endpoint kind mismatch") + + +def sequence_field( + endpoint: EligibleEndpoint, value: JsonObject, key: str, transaction_id: str | None +) -> Sequence[JsonValue]: + """Return a required JSON array field or raise a typed error.""" + item = value.get(key) + if is_json_sequence(item): + return item + fail(endpoint, transaction_id, PassthroughNormalizeReason.MISSING_REQUIRED_FIELD, key) + + +def optional_string(value: JsonObject, key: str) -> str | None: + """Return an optional JSON string field.""" + item = value.get(key) + return item if isinstance(item, str) else None + + +def json_object_from_string(endpoint: EligibleEndpoint, raw: str, transaction_id: str | None) -> JsonMutableObject: + """Parse a JSON object encoded as a string or raise a typed error.""" + try: + parsed = json.loads(raw) + except json.JSONDecodeError: + fail(endpoint, transaction_id, PassthroughNormalizeReason.MALFORMED_JSON, "function arguments") + if is_json_object(parsed): + return json_mutable_object(parsed) + fail(endpoint, transaction_id, PassthroughNormalizeReason.MALFORMED_JSON, "function arguments not object") + + +def json_mutable_object(value: Mapping[str, JsonValue]) -> JsonMutableObject: + """Deep-copy a JSON object into mutable JSON-compatible containers.""" + return {key: json_mutable(item) for key, item in value.items()} + + +def json_mutable(value: JsonValue) -> JsonMutableValue: + """Deep-copy a JSON value into mutable JSON-compatible containers.""" + match value: + case None | str() | bool() | int() | float(): + return value + case Mapping(): + return json_mutable_object(value) + case Sequence(): + return [json_mutable(item) for item in value] + + +def lenient_text_content_from_openai(content: JsonValue) -> str | list[JsonMutableValue] | None: + """Return recoverable text content while omitting unknown request blocks.""" + match content: + case str(): + return content + case Sequence() if not isinstance(content, str): + blocks: list[JsonMutableValue] = [] + for block in content: + match block: + case {"type": "text" | "input_text" | "output_text", "text": str() as text}: + blocks.append({"type": "text", "text": text}) + case _: + continue + return blocks or None + case _: + return None + + +def canonical_usage(usage: JsonValue) -> JsonMutableObject | None: + """Map OpenAI token counters into canonical usage counters.""" + if not isinstance(usage, Mapping): + return None + input_tokens = _first_int(usage, "prompt_tokens", "input_tokens") + output_tokens = _first_int(usage, "completion_tokens", "output_tokens") + result: JsonMutableObject = {} + if isinstance(input_tokens, int): + result["input_tokens"] = input_tokens + if isinstance(output_tokens, int): + result["output_tokens"] = output_tokens + total_tokens = usage.get("total_tokens") + if isinstance(total_tokens, int): + result["total_tokens"] = total_tokens + # Reasoning-model token accounting: Chat Completions nests under + # `completion_tokens_details.reasoning_tokens`; Responses nests under + # `output_tokens_details.reasoning_tokens`. Reasoning tokens are + # already counted inside `output_tokens`; we surface them separately so + # downstream consumers can distinguish reasoning from visible output. + reasoning_tokens = _reasoning_tokens(usage) + if isinstance(reasoning_tokens, int): + result["reasoning_tokens"] = reasoning_tokens + return result or None + + +def _reasoning_tokens(usage: Mapping[str, JsonValue]) -> int | None: + for details_key in ("completion_tokens_details", "output_tokens_details"): + details = usage.get(details_key) + if not isinstance(details, Mapping): + continue + reasoning = details.get("reasoning_tokens") + if not isinstance(reasoning, int) or isinstance(reasoning, bool): + continue + # Non-reasoning models omit the field entirely at the API level; the SDK + # parser (provider_models.parse_openai_response) injects `0` as a default + # to satisfy required-field validation. Treat 0 as absent so we don't + # pollute non-reasoning-model outputs with a spurious `reasoning_tokens: 0`. + if reasoning == 0: + continue + return reasoning + return None + for details_key in ("completion_tokens_details", "output_tokens_details"): + details = usage.get(details_key) + if not isinstance(details, Mapping): + continue + reasoning = details.get("reasoning_tokens") + if isinstance(reasoning, int) and not isinstance(reasoning, bool): + return reasoning + return None + + +def _first_int(usage: Mapping[str, JsonValue], first_key: str, second_key: str) -> int | None: + first = usage.get(first_key) + if isinstance(first, int): + return first + second = usage.get(second_key) + return second if isinstance(second, int) else None + + +def error_response(http_status: int, response: JsonObject) -> JsonMutableObject: + """Build a canonical upstream-error response without synthetic text.""" + return { + "role": "assistant", + "content": [], + "stop_reason": "error", + "error": {"status_code": http_status, "body": json_mutable_object(response)}, + } + + +def ensure_not_truncated(endpoint: EligibleEndpoint, response: JsonObject, transaction_id: str | None) -> None: + """Reject known-truncated stream captures before canonicalization.""" + if response.get("capture_truncated") is True or "raw" in response: + fail(endpoint, transaction_id, PassthroughNormalizeReason.CAPTURE_TRUNCATED, "stream capture truncated") + + +def stop_reason(reason: str | None) -> str: + """Map OpenAI finish reasons to canonical stop reasons.""" + match reason: + case "tool_calls": + return "tool_use" + case "length": + return "max_tokens" + case "content_filter": + # OpenAI's content_filter is a safety-policy block. Map to the same + # canonical safety bucket the Gemini normalizer uses so downstream + # consumers can distinguish real completions from safety-blocked ones. + return "safety" + case "stop" | None: + return "end_turn" + case _: + return "end_turn" diff --git a/src/luthien_proxy/passthrough_materialize/openai_responses.py b/src/luthien_proxy/passthrough_materialize/openai_responses.py new file mode 100644 index 000000000..5e5cc287a --- /dev/null +++ b/src/luthien_proxy/passthrough_materialize/openai_responses.py @@ -0,0 +1,247 @@ +"""OpenAI Responses passthrough normalizers.""" + +from __future__ import annotations + +from collections.abc import Mapping + +from openai.types.responses import ( + ResponseFunctionToolCall, + ResponseOutputItem, + ResponseOutputMessage, + ResponseOutputRefusal, + ResponseOutputText, +) +from pydantic import ValidationError + +from luthien_proxy.passthrough_materialize.endpoints import EligibleEndpoint, EndpointKind +from luthien_proxy.passthrough_materialize.openai_common import ( + PassthroughNormalizeReason, + canonical_usage, + error_response, + fail, + is_json_object, + is_json_sequence, + json_mutable, + json_mutable_object, + json_object_from_string, + lenient_text_content_from_openai, + optional_string, + require_openai_endpoint, +) +from luthien_proxy.passthrough_materialize.openai_responses_stream import fold_response_stream +from luthien_proxy.passthrough_materialize.payloads import ( + CanonicalRequestInput, + CanonicalResponseInput, + JsonMutableObject, + JsonMutableValue, + JsonObject, + JsonValue, +) +from luthien_proxy.passthrough_materialize.provider_models import parse_openai_response + + +def normalize_openai_responses_request( + endpoint: EligibleEndpoint, request: JsonObject, *, transaction_id: str | None = None +) -> CanonicalRequestInput: + """Normalize an OpenAI Responses request into canonical request input.""" + require_openai_endpoint(endpoint, EndpointKind.OPENAI_RESPONSES, transaction_id) + model = optional_string(request, "model") + messages: list[JsonMutableValue] = [] + instructions = optional_string(request, "instructions") + if instructions is not None: + messages.append({"role": "system", "content": instructions}) + messages.extend(_input_messages(endpoint, request.get("input"), transaction_id)) + final_request: JsonMutableObject = {"model": model, "messages": messages} + _copy_request_fields(request, final_request) + stream = request.get("stream") is True + final_request["stream"] = stream + return CanonicalRequestInput(endpoint, stream, model, final_request, final_request, request) + + +def normalize_openai_responses_response( + endpoint: EligibleEndpoint, + response: JsonObject, + *, + request_is_streaming: bool, + http_status: int, + transaction_id: str | None = None, +) -> CanonicalResponseInput: + """Normalize an OpenAI Responses response into canonical response input.""" + require_openai_endpoint(endpoint, EndpointKind.OPENAI_RESPONSES, transaction_id) + if http_status >= 400: + final_response = error_response(http_status, response) + elif request_is_streaming: + final_response = _stream_response(endpoint, response, transaction_id) + else: + final_response = _buffered_response(endpoint, response, transaction_id) + model_value = final_response.get("model") + final_model = model_value if isinstance(model_value, str) else None + return CanonicalResponseInput(endpoint, request_is_streaming, final_model, final_response, final_response, response) + + +def _input_messages( + endpoint: EligibleEndpoint, raw_input: JsonValue, transaction_id: str | None +) -> list[JsonMutableValue]: + if isinstance(raw_input, str): + return [{"role": "user", "content": raw_input}] + if not is_json_sequence(raw_input): + fail(endpoint, transaction_id, PassthroughNormalizeReason.MISSING_REQUIRED_FIELD, "input") + result: list[JsonMutableValue] = [] + for item in raw_input: + if is_json_object(item): + message = _input_item(item) + if message is not None: + result.append(message) + if not result: + fail(endpoint, transaction_id, PassthroughNormalizeReason.MISSING_REQUIRED_FIELD, "input") + return result + + +def _input_item(item: Mapping[str, JsonValue]) -> JsonMutableValue | None: + item_type = item.get("type") + match item_type: + case "function_call_output": + call_id = optional_string(item, "call_id") + output = optional_string(item, "output") + if call_id is None or output is None: + return None + return {"role": "tool", "tool_call_id": call_id, "content": output} + case None | "message": + role = optional_string(item, "role") + content = lenient_text_content_from_openai(item.get("content")) + if role is None or content is None: + return None + return {"role": "system" if role == "developer" else role, "content": content} + case _: + return None + + +def _copy_request_fields(request: JsonObject, final_request: JsonMutableObject) -> None: + for key in ("tool_choice", "temperature", "top_p", "max_output_tokens"): + if key in request: + final_request[key] = json_mutable(request[key]) + if isinstance(request.get("max_output_tokens"), int): + final_request["max_tokens"] = json_mutable(request["max_output_tokens"]) + tools = request.get("tools") + if tools is not None: + final_request["tools"] = _tools(tools) + + +def _tools(tools: JsonValue) -> list[JsonMutableValue]: + if not is_json_sequence(tools): + return [] + result: list[JsonMutableValue] = [] + for tool in tools: + if not is_json_object(tool) or tool.get("type") != "function": + continue + name = optional_string(tool, "name") + parameters = tool.get("parameters") + if name is None or not is_json_object(parameters): + continue + canonical: JsonMutableObject = { + "name": name, + "input_schema": json_mutable_object(parameters), + } + description = optional_string(tool, "description") + if description is not None: + canonical["description"] = description + result.append(canonical) + return result + + +def _buffered_response( + endpoint: EligibleEndpoint, response: JsonObject, transaction_id: str | None +) -> JsonMutableObject: + try: + parsed = parse_openai_response(response) + except ValidationError: + # The pinned openai SDK's strict Literals (e.g. Response.status) reject + # novel API values with an uncaught ValidationError; convert to a typed, + # retryable failure so a novel variant skips one transaction rather than + # wedging the backfill batch. + fail(endpoint, transaction_id, PassthroughNormalizeReason.MALFORMED_PAYLOAD, "response") + if not parsed.output: + fail(endpoint, transaction_id, PassthroughNormalizeReason.MISSING_REQUIRED_FIELD, "output") + content = [block for item in parsed.output for block in _output_item(endpoint, item, transaction_id)] + if not content: + fail(endpoint, transaction_id, PassthroughNormalizeReason.MISSING_REQUIRED_FIELD, "output content") + usage = canonical_usage(parsed.usage.model_dump() if parsed.usage is not None else None) + final = _response_base(parsed.id, parsed.model, usage, content) + _copy_status_fields(response, parsed.status, final) + return final + + +def _output_item( + endpoint: EligibleEndpoint, + item: ResponseOutputItem, + transaction_id: str | None, +) -> list[JsonMutableValue]: + match item: + case ResponseOutputMessage(content=message_content): + return [block for part in message_content for block in _output_content(part)] + case ResponseFunctionToolCall(call_id=call_id, name=name, arguments=arguments): + if not call_id: + fail( + endpoint, transaction_id, PassthroughNormalizeReason.MISSING_REQUIRED_FIELD, "function_call.call_id" + ) + return [ + { + "type": "tool_use", + "id": call_id, + "name": name, + "input": json_object_from_string(endpoint, arguments, transaction_id), + } + ] + case _: + return [] + + +def _output_content( + block: ResponseOutputText | ResponseOutputRefusal, +) -> list[JsonMutableValue]: + match block: + case ResponseOutputText(text=text): + return [{"type": "text", "text": text}] + case ResponseOutputRefusal(refusal=refusal): + return [{"type": "text", "text": refusal}] + case _: + return [] + + +def _stream_response(endpoint: EligibleEndpoint, response: JsonObject, transaction_id: str | None) -> JsonMutableObject: + folded = fold_response_stream(endpoint, response, transaction_id) + completed = folded.completed or {} + if is_json_sequence(completed.get("output")): + return _buffered_response(endpoint, completed, transaction_id) + final = _response_base( + optional_string(completed, "id"), + optional_string(completed, "model"), + canonical_usage(completed.get("usage")), + [{"type": "text", "text": folded.text}], + ) + _copy_status_fields(completed, optional_string(completed, "status"), final) + return final + + +def _response_base( + response_id: str | None, model: str | None, usage: JsonMutableObject | None, content: list[JsonMutableValue] +) -> JsonMutableObject: + stop = ( + "tool_use" if any(is_json_object(item) and item.get("type") == "tool_use" for item in content) else "end_turn" + ) + final: JsonMutableObject = {"role": "assistant", "content": content, "stop_reason": stop} + if response_id is not None: + final["id"] = response_id + if model is not None: + final["model"] = model + if usage is not None: + final["usage"] = usage + return final + + +def _copy_status_fields(response: JsonObject, status: str | None, final: JsonMutableObject) -> None: + for key in ("status", "incomplete_details", "error"): + if key in response: + final[key] = json_mutable(response[key]) + if status in ("failed", "incomplete") or response.get("error") is not None: + final["stop_reason"] = "error" diff --git a/src/luthien_proxy/passthrough_materialize/openai_responses_stream.py b/src/luthien_proxy/passthrough_materialize/openai_responses_stream.py new file mode 100644 index 000000000..d61c51e0d --- /dev/null +++ b/src/luthien_proxy/passthrough_materialize/openai_responses_stream.py @@ -0,0 +1,74 @@ +"""OpenAI Responses stream folding.""" + +from __future__ import annotations + +from dataclasses import dataclass + +from luthien_proxy.passthrough_materialize.endpoints import EligibleEndpoint +from luthien_proxy.passthrough_materialize.openai_common import ( + PassthroughNormalizeReason, + ensure_not_truncated, + fail, + is_json_object, + is_json_sequence, + optional_string, +) +from luthien_proxy.passthrough_materialize.payloads import JsonObject + + +@dataclass(frozen=True, slots=True) +class ResponseStreamFold: + """Folded Responses stream state before canonical conversion.""" + + text: str + completed: JsonObject | None + + +LIFECYCLE_EVENTS = frozenset( + { + "response.created", + "response.in_progress", + "response.output_item.added", + "response.content_part.added", + "response.output_text.done", + "response.content_part.done", + "response.output_item.done", + "response.function_call_arguments.delta", + "response.function_call_arguments.done", + } +) + + +def fold_response_stream( + endpoint: EligibleEndpoint, response: JsonObject, transaction_id: str | None +) -> ResponseStreamFold: + """Fold captured OpenAI Responses SSE chunks into one response.""" + ensure_not_truncated(endpoint, response, transaction_id) + events = response.get("events") + if not is_json_sequence(events): + fail(endpoint, transaction_id, PassthroughNormalizeReason.MISSING_REQUIRED_FIELD, "events") + text = "" + completed: JsonObject | None = None + for event in events: + if not is_json_object(event): + fail(endpoint, transaction_id, PassthroughNormalizeReason.MALFORMED_PAYLOAD, "stream event") + event_type = optional_string(event, "type") + match event_type: + case "response.output_text.delta": + text += optional_string(event, "delta") or "" + case "response.completed": + response_obj = event.get("response") + if is_json_object(response_obj): + completed = response_obj + case str() if event_type in LIFECYCLE_EVENTS: + continue + case str(): + fail( + endpoint, + transaction_id, + PassthroughNormalizeReason.UNSUPPORTED_VARIANT, + f"stream event.type:{event_type}", + ) + case _: + fail(endpoint, transaction_id, PassthroughNormalizeReason.MALFORMED_PAYLOAD, "stream event.type") + return ResponseStreamFold(text=text, completed=completed) diff --git a/src/luthien_proxy/passthrough_materialize/payloads.py b/src/luthien_proxy/passthrough_materialize/payloads.py new file mode 100644 index 000000000..f2973f80e --- /dev/null +++ b/src/luthien_proxy/passthrough_materialize/payloads.py @@ -0,0 +1,172 @@ +"""Canonical event payload builders for provider passthrough captures.""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from enum import StrEnum +from types import MappingProxyType +from typing import TypedDict, assert_never + +from luthien_proxy.passthrough_materialize.endpoints import EligibleEndpoint + +type JsonScalar = str | int | float | bool | None +type JsonValue = JsonScalar | Mapping[str, "JsonValue"] | Sequence["JsonValue"] +type JsonObject = Mapping[str, JsonValue] +type JsonMutableValue = JsonScalar | dict[str, "JsonMutableValue"] | list["JsonMutableValue"] +type JsonMutableObject = dict[str, JsonMutableValue] + + +class ResponseEventType(StrEnum): + """Canonical response event names used by conversation history.""" + + NON_STREAMING = "transaction.non_streaming_response_recorded" + STREAMING = "transaction.streaming_response_recorded" + + +@dataclass(frozen=True, slots=True) +class CanonicalRequestInput: + """Immutable request-side materialization input.""" + + endpoint: EligibleEndpoint + is_streaming: bool + final_model: str | None + original_request: JsonObject + final_request: JsonObject + provider_request: JsonObject + + def __post_init__(self) -> None: + """Freeze nested JSON aliases after dataclass construction.""" + object.__setattr__(self, "original_request", _freeze_json_object(self.original_request)) + object.__setattr__(self, "final_request", _freeze_json_object(self.final_request)) + object.__setattr__(self, "provider_request", _freeze_json_object(self.provider_request)) + + +@dataclass(frozen=True, slots=True) +class CanonicalResponseInput: + """Immutable response-side materialization input.""" + + endpoint: EligibleEndpoint + is_streaming: bool + final_model: str | None + original_response: JsonObject + final_response: JsonObject + provider_response: JsonObject + + def __post_init__(self) -> None: + """Freeze nested JSON aliases after dataclass construction.""" + object.__setattr__(self, "original_response", _freeze_json_object(self.original_response)) + object.__setattr__(self, "final_response", _freeze_json_object(self.final_response)) + object.__setattr__(self, "provider_response", _freeze_json_object(self.provider_response)) + + +class CanonicalRequestPayload(TypedDict): + """JSON-compatible transaction.request_recorded payload.""" + + provider: str + endpoint: str + endpoint_kind: str + is_streaming: bool + final_model: str | None + original_request: JsonMutableObject + final_request: JsonMutableObject + provider_request: JsonMutableObject + + +class CanonicalResponsePayload(TypedDict): + """JSON-compatible transaction response payload.""" + + event_type: str + provider: str + endpoint: str + endpoint_kind: str + is_streaming: bool + final_model: str | None + original_response: JsonMutableObject + final_response: JsonMutableObject + provider_response: JsonMutableObject + + +def build_request_event_payload(payload_input: CanonicalRequestInput) -> CanonicalRequestPayload: + """Build the canonical request event payload from immutable input.""" + endpoint = payload_input.endpoint + return { + "provider": endpoint.provider.value, + "endpoint": endpoint.path, + "endpoint_kind": endpoint.kind.value, + "is_streaming": payload_input.is_streaming, + "final_model": payload_input.final_model, + "original_request": _copy_json_object(payload_input.original_request), + "final_request": _copy_json_object(payload_input.final_request), + "provider_request": _copy_json_object(payload_input.provider_request), + } + + +def build_response_event_payload(payload_input: CanonicalResponseInput) -> CanonicalResponsePayload: + """Build the canonical response event payload from immutable input.""" + endpoint = payload_input.endpoint + return { + "event_type": _response_event_type(payload_input.is_streaming).value, + "provider": endpoint.provider.value, + "endpoint": endpoint.path, + "endpoint_kind": endpoint.kind.value, + "is_streaming": payload_input.is_streaming, + "final_model": payload_input.final_model, + "original_response": _copy_json_object(payload_input.original_response), + "final_response": _copy_json_object(payload_input.final_response), + "provider_response": _copy_json_object(payload_input.provider_response), + } + + +def _response_event_type(is_streaming: bool) -> ResponseEventType: + if is_streaming: + return ResponseEventType.STREAMING + return ResponseEventType.NON_STREAMING + + +def _freeze_json_object(value: JsonObject) -> JsonObject: + return MappingProxyType({key: _freeze_json_value(item) for key, item in value.items()}) + + +def _freeze_json_value(value: JsonValue) -> JsonValue: + match value: + case None | str() | bool() | int() | float(): + return value + case Mapping(): + return _freeze_json_object(value) + case Sequence(): + return tuple(_freeze_json_value(item) for item in value) + case unreachable: + assert_never(unreachable) + + +def _copy_json_object(value: JsonObject) -> JsonMutableObject: + return {key: _copy_json_value(item) for key, item in value.items()} + + +def _copy_json_value(value: JsonValue) -> JsonMutableValue: + match value: + case None | str() | bool() | int() | float(): + return value + case Mapping(): + return _copy_json_object(value) + case Sequence(): + return [_copy_json_value(item) for item in value] + case unreachable: + assert_never(unreachable) + + +__all__ = [ + "CanonicalRequestInput", + "CanonicalRequestPayload", + "CanonicalResponseInput", + "CanonicalResponsePayload", + "JsonMutableObject", + "JsonMutableValue", + "JsonObject", + "JsonScalar", + "JsonValue", + "ResponseEventType", + "build_request_event_payload", + "build_response_event_payload", +] diff --git a/src/luthien_proxy/passthrough_materialize/provider_models.py b/src/luthien_proxy/passthrough_materialize/provider_models.py new file mode 100644 index 000000000..646959ae6 --- /dev/null +++ b/src/luthien_proxy/passthrough_materialize/provider_models.py @@ -0,0 +1,162 @@ +"""Provider SDK typed-model parsers used at the passthrough materialization boundary.""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence + +from google.genai import types as genai_types +from openai.types.chat import ChatCompletion +from openai.types.responses import ( + Response, + ResponseOutputItem, + ResponseOutputRefusal, + ResponseOutputText, +) +from pydantic import TypeAdapter, ValidationError + +from luthien_proxy.passthrough_materialize.payloads import JsonObject, JsonValue + +_response_output_item_adapter = TypeAdapter(ResponseOutputItem) +_response_content_adapter = TypeAdapter(ResponseOutputText | ResponseOutputRefusal) + + +def parse_openai_chat_completion(raw: JsonObject) -> ChatCompletion: + """Parse a captured Chat Completions response with the provider SDK model.""" + response = dict(raw) + response.setdefault("id", "materialized_chat_completion") + response.setdefault("model", "") + response.setdefault("created", 0) + response.setdefault("object", "chat.completion") + response["choices"] = _chat_choices(raw.get("choices")) + return ChatCompletion.model_validate(response) + + +def parse_openai_response(raw: JsonObject) -> Response: + """Parse a captured Responses response after omitting unmodelled output variants.""" + response = dict(raw) + response.setdefault("id", "materialized_response") + response.setdefault("model", "") + response.setdefault("created_at", 0) + response.setdefault("object", "response") + response.setdefault("parallel_tool_calls", True) + response.setdefault("tool_choice", "auto") + response.setdefault("tools", []) + usage = response.get("usage") + if isinstance(usage, Mapping): + normalized_usage = dict(usage) + input_details = normalized_usage.get("input_tokens_details") + output_details = normalized_usage.get("output_tokens_details") + normalized_usage["input_tokens_details"] = { + "cache_write_tokens": 0, + "cached_tokens": 0, + **(dict(input_details) if isinstance(input_details, Mapping) else {}), + } + normalized_usage["output_tokens_details"] = { + "reasoning_tokens": 0, + **(dict(output_details) if isinstance(output_details, Mapping) else {}), + } + response["usage"] = normalized_usage + response["output"] = _response_output(raw.get("output")) + return Response.model_validate(response) + + +def parse_gemini_response(raw: JsonObject) -> genai_types.GenerateContentResponse: + """Parse a Gemini generateContent payload with the provider SDK model.""" + response = dict(raw) + candidates = response.get("candidates") + if isinstance(candidates, Sequence) and not isinstance(candidates, str): + response["candidates"] = [_gemini_candidate(candidate) for candidate in candidates] + return genai_types.GenerateContentResponse.model_validate(response) + + +def _gemini_candidate(raw_candidate: JsonValue) -> JsonValue: + if not isinstance(raw_candidate, Mapping): + return raw_candidate + candidate = dict(raw_candidate) + content = candidate.get("content") + if isinstance(content, Mapping): + normalized_content = dict(content) + parts = normalized_content.get("parts") + if isinstance(parts, Sequence) and not isinstance(parts, str): + normalized_content["parts"] = _gemini_parts(parts) + candidate["content"] = normalized_content + return candidate + + +def _gemini_parts(parts: Sequence[JsonValue]) -> list[JsonValue]: + parsed: list[JsonValue] = [] + for part in parts: + if not isinstance(part, Mapping): + continue + try: + genai_types.Part.model_validate(part) + except ValidationError: + continue + parsed.append(part) + return parsed + + +def _chat_choices(raw_choices: JsonValue | None) -> list[dict[str, JsonValue]]: + if not isinstance(raw_choices, Sequence) or isinstance(raw_choices, str): + return [] + choices: list[dict[str, JsonValue]] = [] + for index, raw_choice in enumerate(raw_choices): + if not isinstance(raw_choice, Mapping): + continue + choice = dict(raw_choice) + choice.setdefault("index", index) + finish_reason = choice.get("finish_reason") + if finish_reason not in {"stop", "length", "tool_calls", "content_filter", "function_call"}: + choice["finish_reason"] = "stop" + message = choice.get("message") + if isinstance(message, Mapping): + normalized_message = dict(message) + tool_calls = normalized_message.get("tool_calls") + if isinstance(tool_calls, Sequence) and not isinstance(tool_calls, str): + normalized_message["tool_calls"] = [ + {"id": "", **dict(call)} if isinstance(call, Mapping) else call for call in tool_calls + ] + choice["message"] = normalized_message + choices.append(choice) + return choices + + +def _response_output(raw_output: JsonValue | None) -> list[dict[str, JsonValue]]: + if not isinstance(raw_output, Sequence) or isinstance(raw_output, str): + return [] + output: list[dict[str, JsonValue]] = [] + for index, raw_item in enumerate(raw_output): + if not isinstance(raw_item, Mapping): + continue + item = _response_item_defaults(raw_item, index) + try: + _response_output_item_adapter.validate_python(item) + except ValidationError: + continue + output.append(item) + return output + + +def _response_item_defaults(raw_item: Mapping[str, JsonValue], index: int) -> dict[str, JsonValue]: + item = dict(raw_item) + item.setdefault("id", f"materialized_output_{index}") + item.setdefault("status", "completed") + content = item.get("content") + if isinstance(content, Sequence) and not isinstance(content, str): + item["content"] = _response_content(content) + return item + + +def _response_content(content: Sequence[JsonValue]) -> list[dict[str, JsonValue]]: + parsed: list[dict[str, JsonValue]] = [] + for raw_part in content: + if not isinstance(raw_part, Mapping): + continue + part = dict(raw_part) + part.setdefault("annotations", []) + try: + _response_content_adapter.validate_python(part) + except ValidationError: + continue + parsed.append(part) + return parsed diff --git a/src/luthien_proxy/passthrough_materialize/reconcile.py b/src/luthien_proxy/passthrough_materialize/reconcile.py new file mode 100644 index 000000000..8e4cba46c --- /dev/null +++ b/src/luthien_proxy/passthrough_materialize/reconcile.py @@ -0,0 +1,93 @@ +"""Bounded reconciliation of raw passthrough transactions.""" + +from __future__ import annotations + +import logging +from collections.abc import Mapping +from datetime import datetime +from typing import assert_never + +import aiosqlite +import asyncpg + +from luthien_proxy.passthrough_materialize.materialize import materialize_transaction +from luthien_proxy.passthrough_materialize.materialize_types import ( + AlreadyMaterialized, + MaterializationFailed, + Materialized, + ReconcileStats, + SkippedIneligible, +) +from luthien_proxy.utils.db import DatabasePool + +logger = logging.getLogger(__name__) + +_ELIGIBLE_UNMATERIALIZED_TRANSACTIONS_SQL = """ + SELECT request_logs.transaction_id + FROM request_logs + WHERE request_logs.direction = 'inbound' + AND ( + request_logs.endpoint IN ( + '/openai/v1/chat/completions', + '/openai/v1/responses' + ) + OR request_logs.endpoint LIKE '/gemini/%:generateContent' + OR request_logs.endpoint LIKE '/gemini/%:streamGenerateContent' + ) + AND request_logs.started_at >= COALESCE($1, request_logs.started_at) + AND NOT EXISTS ( + SELECT 1 + FROM conversation_events + WHERE conversation_events.call_id = request_logs.transaction_id + ) + GROUP BY request_logs.transaction_id + ORDER BY MIN(request_logs.started_at), request_logs.transaction_id + LIMIT $2 +""" + + +async def reconcile_passthrough(db_pool: DatabasePool, *, limit: int, since: datetime | None = None) -> ReconcileStats: + """Materialize eligible raw transactions that do not yet have request events.""" + async with db_pool.connection() as conn: + rows = await conn.fetch(_ELIGIBLE_UNMATERIALIZED_TRANSACTIONS_SQL, since, limit) + + materialized = 0 + already_materialized = 0 + skipped_ineligible = 0 + failed = 0 + for row in rows: + transaction_id = _transaction_id(row) + try: + result = await materialize_transaction(db_pool, transaction_id) + except (aiosqlite.Error, asyncpg.PostgresError) as error: + logger.warning("Passthrough reconciliation transaction failed for %s: %s", transaction_id, error) + failed += 1 + continue + match result: + case Materialized(): + materialized += 1 + case AlreadyMaterialized(): + already_materialized += 1 + case SkippedIneligible(): + skipped_ineligible += 1 + case MaterializationFailed(): + failed += 1 + case unreachable: + assert_never(unreachable) + + return ReconcileStats( + materialized=materialized, + already_materialized=already_materialized, + skipped_ineligible=skipped_ineligible, + failed=failed, + ) + + +def _transaction_id(row: Mapping[str, object]) -> str: + transaction_id = row["transaction_id"] + if isinstance(transaction_id, str): + return transaction_id + raise TypeError("reconciliation query returned a non-string transaction_id") + + +__all__ = ["reconcile_passthrough"] diff --git a/src/luthien_proxy/passthrough_materialize/worker.py b/src/luthien_proxy/passthrough_materialize/worker.py new file mode 100644 index 000000000..24e15f70f --- /dev/null +++ b/src/luthien_proxy/passthrough_materialize/worker.py @@ -0,0 +1,77 @@ +"""Periodic background worker for passthrough reconciliation.""" + +from __future__ import annotations + +import asyncio +import logging + +import aiosqlite +import asyncpg + +from luthien_proxy.passthrough_materialize.materialize_types import ReconcileStats +from luthien_proxy.passthrough_materialize.reconcile import reconcile_passthrough +from luthien_proxy.utils.db import DatabasePool + +logger = logging.getLogger(__name__) + + +def _log_task_exception(task: asyncio.Task[None]) -> None: + if not task.cancelled() and (error := task.exception()): + logger.exception("Passthrough reconciliation worker raised unexpectedly", exc_info=error) + + +class PassthroughReconcileWorker: + """Run bounded passthrough reconciliation sweeps until shutdown.""" + + def __init__(self, *, db_pool: DatabasePool, limit: int, interval_seconds: int) -> None: + """Initialize the database-backed worker and its cadence.""" + self._db_pool = db_pool + self._limit = limit + self._interval_seconds = interval_seconds + self._task: asyncio.Task[None] | None = None + + async def _run_loop(self) -> None: + while True: + try: + stats = await reconcile_passthrough(self._db_pool, limit=self._limit) + except (aiosqlite.Error, asyncpg.PostgresError): + logger.exception("Passthrough reconciliation sweep failed; retrying next interval") + else: + self._log_stats(stats) + await asyncio.sleep(self._interval_seconds) + + def _log_stats(self, stats: ReconcileStats) -> None: + logger.info( + "Passthrough reconciliation sweep complete: materialized=%d already_materialized=%d " + "skipped_ineligible=%d failed=%d", + stats.materialized, + stats.already_materialized, + stats.skipped_ineligible, + stats.failed, + ) + + def start(self) -> None: + """Start the reconciliation task when one is not already active.""" + if self._task is not None and not self._task.done(): + return + logger.info( + "Passthrough reconciliation worker enabled: limit=%d interval=%ds", + self._limit, + self._interval_seconds, + ) + self._task = asyncio.create_task(self._run_loop()) + self._task.add_done_callback(_log_task_exception) + + async def stop(self) -> None: + """Cancel and drain the reconciliation task during shutdown.""" + if self._task is None: + return + self._task.cancel() + try: + await self._task + except asyncio.CancelledError: + pass + self._task = None + + +__all__ = ["PassthroughReconcileWorker"] diff --git a/src/luthien_proxy/passthrough_recording.py b/src/luthien_proxy/passthrough_recording.py new file mode 100644 index 000000000..faef5f7ab --- /dev/null +++ b/src/luthien_proxy/passthrough_recording.py @@ -0,0 +1,50 @@ +"""Create passthrough request recorders with trusted attribution and optional materialization.""" + +from __future__ import annotations + +from collections.abc import Mapping + +from luthien_proxy.dependencies import Dependencies +from luthien_proxy.passthrough_materialize.materialize import materialize_transaction +from luthien_proxy.pipeline.session import ( + extract_user_id_from_authorization_header, + extract_user_id_from_headers, +) +from luthien_proxy.request_log.recorder import RequestLogRecorder, create_recorder +from luthien_proxy.settings import get_settings + + +def create_passthrough_recorder( + headers: Mapping[str, str], transaction_id: str, deps: Dependencies +) -> tuple[RequestLogRecorder, str | None, str | None]: + """Return a recorder plus session and user identities derived from request headers.""" + normalized_headers = {key.lower(): value for key, value in headers.items()} + settings = get_settings() + session_id = normalized_headers.get("x-session-id") or normalized_headers.get("x-luthien-session-id") + user_id = extract_user_id_from_headers( + normalized_headers, trust_header=settings.trust_user_id_header + ) or extract_user_id_from_authorization_header(normalized_headers.get("authorization")) + db_pool = deps.db_pool + if settings.passthrough_materialize_enabled and db_pool is not None: + + async def on_commit(transaction_id: str) -> None: + await materialize_transaction(db_pool, transaction_id) + + return ( + create_recorder( + db_pool, + transaction_id=transaction_id, + enabled=deps.enable_request_logging, + on_commit=on_commit, + ), + session_id, + user_id, + ) + return ( + create_recorder(db_pool, transaction_id=transaction_id, enabled=deps.enable_request_logging), + session_id, + user_id, + ) + + +__all__ = ["create_passthrough_recorder"] diff --git a/src/luthien_proxy/passthrough_routes.py b/src/luthien_proxy/passthrough_routes.py index 775f940f2..f7a9454a3 100644 --- a/src/luthien_proxy/passthrough_routes.py +++ b/src/luthien_proxy/passthrough_routes.py @@ -6,8 +6,7 @@ import uuid from collections.abc import AsyncIterator from dataclasses import dataclass -from typing import Literal -from urllib.parse import urlsplit, urlunsplit +from typing import Literal, assert_never import httpx from fastapi import APIRouter, Depends, HTTPException, Request @@ -15,9 +14,9 @@ from luthien_proxy.dependencies import get_dependencies from luthien_proxy.passthrough_capture import ( - _RESPONSE_STRIPPED_HEADERS, JsonObject, build_passthrough_headers, + build_upstream_url, json_loads, parse_gemini_model, parse_openai_model, @@ -25,12 +24,14 @@ reassemble_gemini_sse_stream, reassemble_openai_sse_stream, ) -from luthien_proxy.request_log.recorder import RequestLogRecorder, create_recorder +from luthien_proxy.passthrough_capture import ( + client_response_headers as _client_response_headers, +) +from luthien_proxy.passthrough_recording import create_passthrough_recorder +from luthien_proxy.request_log.recorder import RequestLogRecorder from luthien_proxy.request_log.sanitize import sanitize_url from luthien_proxy.settings import get_settings -Provider = Literal["openai", "gemini"] - router = APIRouter(tags=["passthrough"]) _OPENAI_BASE_URL = "https://api.openai.com" @@ -51,7 +52,7 @@ async def _require_passthrough_enabled() -> None: @dataclass(frozen=True, slots=True) class _UpstreamTarget: - provider: Provider + provider: Literal["openai", "gemini"] path: str base_url: str is_streaming: bool @@ -88,12 +89,6 @@ async def _json_body(request: Request) -> _RequestPayload: return _RequestPayload(body_bytes=body_bytes, body={"body": parsed}) -def _target_url(target: _UpstreamTarget, request: Request) -> str: - base = urlsplit(target.base_url.rstrip("/")) - path = f"/{target.path}" - return urlunsplit((base.scheme, base.netloc, path, request.url.query, "")) - - def _is_openai_stream(path: str, body: JsonObject) -> bool: stream = body.get("stream") return stream is True or path.endswith("/stream") @@ -115,7 +110,7 @@ def _response_body(response_bytes: bytes) -> JsonObject: return {"body": parsed} -def _stream_body(provider: Provider, request: Request, chunks: list[bytes]) -> JsonObject: +def _stream_body(provider: Literal["openai", "gemini"], request: Request, chunks: list[bytes]) -> JsonObject: match provider: case "openai": return reassemble_openai_sse_stream(chunks) @@ -123,10 +118,8 @@ def _stream_body(provider: Provider, request: Request, chunks: list[bytes]) -> J if request.query_params.get("alt") == "sse": return reassemble_gemini_sse_stream(chunks) return reassemble_gemini_json_array_stream(chunks) - - -def _client_response_headers(headers: httpx.Headers | dict[str, str]) -> dict[str, str]: - return {key: value for key, value in headers.items() if key.lower() not in _RESPONSE_STRIPPED_HEADERS} + case unreachable: + assert_never(unreachable) def _request_error_text(error: httpx.RequestError, upstream_url: str, forwarded_headers: dict[str, str]) -> str: @@ -153,21 +146,21 @@ def _upstream_error_response( async def _passthrough(request: Request, target: _UpstreamTarget, payload: _RequestPayload) -> Response: deps = get_dependencies(request) - recorder = create_recorder(deps.db_pool, str(uuid.uuid4()), deps.enable_request_logging) - upstream_url = _target_url(target, request) + recorder, session_id, user_id = create_passthrough_recorder(request.headers, str(uuid.uuid4()), deps) + upstream_url = build_upstream_url(target.base_url, target.path, request.url.query) forwarded_headers = build_passthrough_headers(request.headers.items()) model = ( parse_openai_model(payload.body, request.headers.get("x-luthien-model")) if target.provider == "openai" else parse_gemini_model(target.path, payload.body, request.headers.get("x-luthien-model")) ) - session_id = request.headers.get("x-session-id") or request.headers.get("x-luthien-session-id") recorder.record_inbound_request( method=request.method, url=sanitize_url(str(request.url)), headers=dict(request.headers), body=payload.body, session_id=session_id, + user_id=user_id, model=model, endpoint=target.endpoint, is_streaming=target.is_streaming, diff --git a/src/luthien_proxy/request_log/models.py b/src/luthien_proxy/request_log/models.py index 3478411d7..29ac79cb0 100644 --- a/src/luthien_proxy/request_log/models.py +++ b/src/luthien_proxy/request_log/models.py @@ -2,10 +2,92 @@ from __future__ import annotations +import json +import time +from collections.abc import Callable +from dataclasses import dataclass, field from typing import Any from pydantic import BaseModel +from luthien_proxy.utils.db import ConnectionProtocol, DatabaseWriteError + + +@dataclass +class _PendingLog: + """Accumulates data for a single log row before it is written to DB.""" + + direction: str + transaction_id: str + session_id: str | None = None + user_id: str | None = None + http_method: str | None = None + url: str | None = None + request_headers: dict[str, str] | None = None + request_body: dict[str, Any] | None = None + response_status: int | None = None + response_headers: dict[str, str] | None = None + response_body: dict[str, Any] | None = None + started_at: float = field(default_factory=time.time) + completed_at: float | None = None + duration_ms: float | None = None + model: str | None = None + is_streaming: bool = False + endpoint: str | None = None + error: str | None = None + + +async def insert_log_row( + conn: ConnectionProtocol, + pending: _PendingLog, + serialize_body: Callable[[dict[str, Any] | None], str | None], +) -> None: + """Insert one request_logs row, wrapping driver errors for callers.""" + # SQL avoids the CASE WHEN $N ... $N duplicate-positional pattern (breaks SQLite ?); + # to_timestamp(NULL) -> NULL on both Postgres and SQLite. + try: + await conn.execute( + """ + INSERT INTO request_logs ( + transaction_id, session_id, user_id, direction, + http_method, url, request_headers, request_body, + response_status, response_headers, response_body, + started_at, completed_at, duration_ms, + model, is_streaming, endpoint, error + ) VALUES ( + $1, $2, $3, $4, + $5, $6, $7::jsonb, $8::jsonb, + $9, $10::jsonb, $11::jsonb, + to_timestamp($12), to_timestamp($13), $14, + $15, $16, $17, $18 + ) + """, + pending.transaction_id, + pending.session_id, + pending.user_id, + pending.direction, + pending.http_method, + pending.url, + json.dumps(pending.request_headers) if pending.request_headers else None, + serialize_body(pending.request_body), + pending.response_status, + json.dumps(pending.response_headers) if pending.response_headers else None, + serialize_body(pending.response_body), + pending.started_at, + pending.completed_at, + pending.duration_ms, + pending.model, + pending.is_streaming, + pending.endpoint, + pending.error, + ) + except Exception as exc: + raise DatabaseWriteError( + f"Failed to insert request_log row (direction={pending.direction!r}, " + f"transaction_id={pending.transaction_id!r}): {exc}", + cause=exc, + ) from exc + class RequestLogEntry(BaseModel): """A single request/response log entry.""" @@ -54,4 +136,5 @@ class RequestLogDetailResponse(BaseModel): "RequestLogEntry", "RequestLogListResponse", "RequestLogDetailResponse", + "insert_log_row", ] diff --git a/src/luthien_proxy/request_log/recorder.py b/src/luthien_proxy/request_log/recorder.py index 777500626..543612864 100644 --- a/src/luthien_proxy/request_log/recorder.py +++ b/src/luthien_proxy/request_log/recorder.py @@ -16,9 +16,10 @@ import json import logging import time -from dataclasses import dataclass, field -from typing import Any, Callable +from collections.abc import Awaitable, Callable +from typing import Any +from luthien_proxy.request_log.models import _PendingLog, insert_log_row from luthien_proxy.request_log.sanitize import sanitize_headers from luthien_proxy.utils.db import DatabasePool, DatabaseWriteError @@ -34,88 +35,6 @@ def _log_task_exception(task: asyncio.Task[None]) -> None: logger.error("Background request log write failed", exc_info=task.exception()) -@dataclass -class _PendingLog: - """Accumulates data for a single log row before it's written to DB.""" - - direction: str - transaction_id: str - session_id: str | None = None - user_id: str | None = None - http_method: str | None = None - url: str | None = None - request_headers: dict[str, str] | None = None - request_body: dict[str, Any] | None = None - response_status: int | None = None - response_headers: dict[str, str] | None = None - response_body: dict[str, Any] | None = None - started_at: float = field(default_factory=time.time) - completed_at: float | None = None - duration_ms: float | None = None - model: str | None = None - is_streaming: bool = False - endpoint: str | None = None - error: str | None = None - - -async def _insert_log_row( - conn: object, - pending: _PendingLog, - serialize_body: Callable[[dict[str, Any] | None], str | None], -) -> None: - """Insert one request_logs row via the DB-agnostic connection interface. - - Raises DatabaseWriteError on any failure so callers don't need to know - which driver (asyncpg, aiosqlite, etc.) is in use. - - The SQL avoids the CASE WHEN $N ... $N pattern (duplicate positional - parameters) that breaks SQLite's ? placeholders. A None completed_at - becomes NULL via to_timestamp(NULL) on both Postgres and SQLite. - """ - try: - await conn.execute( # type: ignore[union-attr] - """ - INSERT INTO request_logs ( - transaction_id, session_id, user_id, direction, - http_method, url, request_headers, request_body, - response_status, response_headers, response_body, - started_at, completed_at, duration_ms, - model, is_streaming, endpoint, error - ) VALUES ( - $1, $2, $3, $4, - $5, $6, $7::jsonb, $8::jsonb, - $9, $10::jsonb, $11::jsonb, - to_timestamp($12), to_timestamp($13), $14, - $15, $16, $17, $18 - ) - """, - pending.transaction_id, - pending.session_id, - pending.user_id, - pending.direction, - pending.http_method, - pending.url, - json.dumps(pending.request_headers) if pending.request_headers else None, - serialize_body(pending.request_body), - pending.response_status, - json.dumps(pending.response_headers) if pending.response_headers else None, - serialize_body(pending.response_body), - pending.started_at, - pending.completed_at, - pending.duration_ms, - pending.model, - pending.is_streaming, - pending.endpoint, - pending.error, - ) - except Exception as exc: - raise DatabaseWriteError( - f"Failed to insert request_log row (direction={pending.direction!r}, " - f"transaction_id={pending.transaction_id!r}): {exc}", - cause=exc, - ) from exc - - class RequestLogRecorder: """Captures HTTP-level request/response data and writes it to the database. @@ -129,9 +48,16 @@ class RequestLogRecorder: dropped_writes: int = 0 - def __init__(self, db_pool: DatabasePool, transaction_id: str) -> None: # noqa: D107 + def __init__( # noqa: D107 + self, + db_pool: DatabasePool, + transaction_id: str, + *, + on_commit: Callable[[str], Awaitable[None]] | None = None, + ) -> None: self._db_pool = db_pool self._transaction_id = transaction_id + self._on_commit = on_commit self._inbound = _PendingLog(direction="inbound", transaction_id=transaction_id) self._outbound = _PendingLog(direction="outbound", transaction_id=transaction_id) @@ -143,7 +69,7 @@ def record_inbound_request( method: str, url: str, headers: dict[str, str], - body: dict[str, Any], + body: dict[str, Any] | None, session_id: str | None = None, user_id: str | None = None, model: str | None = None, @@ -183,7 +109,7 @@ def record_inbound_response( def record_outbound_request( self, *, - body: dict[str, Any], + body: dict[str, Any] | None, method: str = "POST", url: str | None = None, model: str | None = None, @@ -243,16 +169,17 @@ async def _write_logs(self) -> None: """Insert both inbound and outbound rows.""" try: async with self._db_pool.connection() as conn: - cache: dict[int, str | None] = {} + async with conn.transaction(): + cache: dict[int, str | None] = {} - def serialize_body(body: dict[str, Any] | None) -> str | None: - key = id(body) - if key not in cache: - cache[key] = self._serialize_body(body) - return cache[key] + def serialize_body(body: dict[str, Any] | None) -> str | None: + key = id(body) + if key not in cache: + cache[key] = self._serialize_body(body) + return cache[key] - for pending in (self._inbound, self._outbound): - await _insert_log_row(conn, pending, serialize_body) + for pending in (self._inbound, self._outbound): + await insert_log_row(conn, pending, serialize_body) except DatabaseWriteError as exc: RequestLogRecorder.dropped_writes += 1 logger.warning( @@ -261,6 +188,19 @@ def serialize_body(body: dict[str, Any] | None) -> str | None: RequestLogRecorder.dropped_writes, exc.cause, ) + return + + if self._on_commit is None: + return + + try: + await self._on_commit(self._transaction_id) + except Exception: + logger.warning( + "Request log post-commit callback failed for %s", + self._transaction_id, + exc_info=True, + ) class NoOpRequestLogRecorder(RequestLogRecorder): @@ -269,7 +209,9 @@ class NoOpRequestLogRecorder(RequestLogRecorder): All methods are intentional no-ops. """ - def __init__(self) -> None: # noqa: D107 + def __init__( # noqa: D107, ARG002 + self, *, on_commit: Callable[[str], Awaitable[None]] | None = None + ) -> None: pass def record_inbound_request( # noqa: D102, ARG002 @@ -326,14 +268,16 @@ def create_recorder( db_pool: DatabasePool | None, transaction_id: str, enabled: bool, + *, + on_commit: Callable[[str], Awaitable[None]] | None = None, ) -> RequestLogRecorder: """Factory that always returns a recorder — real or no-op based on config. Callers never need to null-check the return value. """ if not enabled or db_pool is None: - return NoOpRequestLogRecorder() - return RequestLogRecorder(db_pool=db_pool, transaction_id=transaction_id) + return NoOpRequestLogRecorder(on_commit=on_commit) + return RequestLogRecorder(db_pool=db_pool, transaction_id=transaction_id, on_commit=on_commit) __all__ = ["RequestLogRecorder", "NoOpRequestLogRecorder", "create_recorder"] diff --git a/src/luthien_proxy/settings.py b/src/luthien_proxy/settings.py index e9065466b..38a447424 100644 --- a/src/luthien_proxy/settings.py +++ b/src/luthien_proxy/settings.py @@ -93,6 +93,10 @@ class Settings(_SettingsBase): # ── passthrough ───────────────────────────────────────────────── passthrough_routes_enabled: bool = False passthrough_stream_capture_max_bytes: int = 10485760 + passthrough_materialize_enabled: bool = False + passthrough_materialize_backfill_enabled: bool = False + passthrough_materialize_reconcile_interval_seconds: int = 300 + passthrough_materialize_batch_size: int = 200 # ── telemetry ─────────────────────────────────────────────────── usage_telemetry: bool | None = None diff --git a/tests/luthien_proxy/e2e_tests/sqlite/test_passthrough_materialization_http.py b/tests/luthien_proxy/e2e_tests/sqlite/test_passthrough_materialization_http.py new file mode 100644 index 000000000..b5a3e4a75 --- /dev/null +++ b/tests/luthien_proxy/e2e_tests/sqlite/test_passthrough_materialization_http.py @@ -0,0 +1,203 @@ +from __future__ import annotations + +import json +from collections.abc import AsyncIterator, Awaitable, Callable +from pathlib import Path + +import anyio +import httpx +import pytest +from asgi_lifespan import LifespanManager +from pytest_httpx import HTTPXMock + +from luthien_proxy.main import create_app +from luthien_proxy.request_log.recorder import RequestLogRecorder +from luthien_proxy.settings import clear_settings_cache +from luthien_proxy.utils.db import DatabasePool + +pytestmark = pytest.mark.sqlite_e2e + +_ADMIN_KEY = "passthrough-materialization-admin" +_SESSION_ID = "passthrough-materialization-session" +_USER_ID = "passthrough-materialization-user" +_OPENAI_URL = "https://openai.materialization.test/v1/chat/completions" +_GEMINI_URL = "https://gemini.materialization.test/v1beta/models/gemini-2.5-flash:generateContent" +_OPENAI_REQUEST = { + "model": "gpt-4.1-mini", + "messages": [{"role": "user", "content": "OpenAI materialize needle"}], +} +_OPENAI_RESPONSE = { + "id": "chatcmpl-materialization", + "model": "gpt-4.1-mini", + "choices": [ + { + "message": {"role": "assistant", "content": "OpenAI materialized answer"}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 9, "completion_tokens": 5, "total_tokens": 14}, +} +_GEMINI_REQUEST = { + "contents": [{"role": "user", "parts": [{"text": "Gemini materialize needle"}]}], +} +_GEMINI_RESPONSE = { + "responseId": "gemini-materialization", + "modelVersion": "gemini-2.5-flash", + "candidates": [ + { + "content": {"role": "model", "parts": [{"text": "Gemini materialized answer"}]}, + "finishReason": "STOP", + } + ], + "usageMetadata": {"promptTokenCount": 7, "candidatesTokenCount": 4, "totalTokenCount": 11}, +} + + +@pytest.fixture +async def materialization_client(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> AsyncIterator[httpx.AsyncClient]: + monkeypatch.setenv("ANTHROPIC_API_KEY", "not-used-by-passthrough") + monkeypatch.setenv("ENABLE_REQUEST_LOGGING", "true") + monkeypatch.setenv("GEMINI_BASE_URL", "https://gemini.materialization.test") + monkeypatch.setenv("LOCALHOST_AUTH_BYPASS", "false") + monkeypatch.setenv("OPENAI_BASE_URL", "https://openai.materialization.test") + monkeypatch.setenv("PASSTHROUGH_MATERIALIZE_ENABLED", "true") + monkeypatch.setenv("PASSTHROUGH_ROUTES_ENABLED", "true") + monkeypatch.setenv("TRUST_USER_ID_HEADER", "true") + monkeypatch.setenv("USAGE_TELEMETRY", "false") + monkeypatch.setenv("WEBHOOK_URL", "") + clear_settings_cache() + + pool = DatabasePool(f"sqlite:///{tmp_path / 'materialization.db'}") + app = create_app( + api_key="passthrough-materialization-client", + admin_key=_ADMIN_KEY, + db_pool=pool, + redis_client=None, + startup_policy_path="config/policy_config.yaml", + ) + try: + async with LifespanManager(app): + transport = httpx.ASGITransport(app=app) + async with httpx.AsyncClient(transport=transport, base_url="http://luthien.test") as client: + yield client + finally: + await pool.close() + clear_settings_cache() + + +@pytest.fixture +def await_passthrough_flushes(monkeypatch: pytest.MonkeyPatch) -> Callable[[], Awaitable[None]]: + writes_finished = 0 + flushes_finished = anyio.Event() + original_write_logs = RequestLogRecorder._write_logs + + async def tracked_write_logs(recorder: RequestLogRecorder) -> None: + nonlocal writes_finished + await original_write_logs(recorder) + writes_finished += 1 + if writes_finished == 2: + flushes_finished.set() + + monkeypatch.setattr(RequestLogRecorder, "_write_logs", tracked_write_logs) + + async def wait_for_flushes() -> None: + with anyio.fail_after(5): + await flushes_finished.wait() + assert writes_finished == 2 + + return wait_for_flushes + + +async def test_passthrough_materializes_openai_and_gemini_read_surfaces( + materialization_client: httpx.AsyncClient, + await_passthrough_flushes: Callable[[], Awaitable[None]], + httpx_mock: HTTPXMock, +) -> None: + # Given + httpx_mock.add_response(method="POST", url=_OPENAI_URL, json=_OPENAI_RESPONSE) + httpx_mock.add_response(method="POST", url=_GEMINI_URL, json=_GEMINI_RESPONSE) + passthrough_headers = { + "Authorization": "Bearer passthrough-provider-token", + "X-Luthien-User-Id": _USER_ID, + "X-Session-Id": _SESSION_ID, + } + admin_headers = {"Authorization": f"Bearer {_ADMIN_KEY}"} + + # When + openai_response = await materialization_client.post( + "/openai/v1/chat/completions", headers=passthrough_headers, json=_OPENAI_REQUEST + ) + gemini_response = await materialization_client.post( + "/gemini/v1beta/models/gemini-2.5-flash:generateContent", + headers=passthrough_headers, + json=_GEMINI_REQUEST, + ) + await await_passthrough_flushes() + + # Then + assert openai_response.json() == _OPENAI_RESPONSE + assert gemini_response.json() == _GEMINI_RESPONSE + + sessions = await materialization_client.get("/api/history/sessions", headers=admin_headers) + assert sessions.status_code == 200 + session = next(item for item in sessions.json()["sessions"] if item["session_id"] == _SESSION_ID) + assert set(session["models_used"]) == {"gpt-4.1-mini", "gemini-2.5-flash"} + assert session["user_ids"] == [_USER_ID] + + detail = await materialization_client.get(f"/api/history/sessions/{_SESSION_ID}", headers=admin_headers) + assert detail.status_code == 200 + turns = detail.json()["turns"] + assert {turn["model"] for turn in turns} == {"gpt-4.1-mini", "gemini-2.5-flash"} + assert {turn["request_messages"][0]["content"] for turn in turns} == { + "OpenAI materialize needle", + "Gemini materialize needle", + } + assert {turn["response_messages"][0]["content"] for turn in turns} == { + "OpenAI materialized answer", + "Gemini materialized answer", + } + + export = await materialization_client.get( + f"/api/history/sessions/{_SESSION_ID}/export/jsonl", headers=admin_headers + ) + assert export.status_code == 200 + exported_turns = [json.loads(line) for line in export.text.splitlines() if line] + call_ids = {turn["call_id"] for turn in turns} + assert {turn["call_id"] for turn in exported_turns} == call_ids + + debug_payloads = {} + for call_id in call_ids: + debug = await materialization_client.get(f"/api/debug/calls/{call_id}", headers=admin_headers) + assert debug.status_code == 200 + events = debug.json()["events"] + request_payload = next( + event["payload"] for event in events if event["event_type"] == "transaction.request_recorded" + ) + response_payload = next( + event["payload"] for event in events if event["event_type"] == "transaction.non_streaming_response_recorded" + ) + debug_payloads[request_payload["provider"]] = {"request": request_payload, "response": response_payload} + + assert set(debug_payloads) == {"openai", "gemini"} + assert debug_payloads["openai"]["request"]["provider_request"] == _OPENAI_REQUEST + assert debug_payloads["openai"]["request"]["final_request"]["model"] == "gpt-4.1-mini" + assert debug_payloads["openai"]["response"]["provider_response"] == _OPENAI_RESPONSE + assert debug_payloads["openai"]["response"]["final_response"]["usage"] == { + "input_tokens": 9, + "output_tokens": 5, + "total_tokens": 14, + } + assert debug_payloads["gemini"]["request"]["provider_request"] == _GEMINI_REQUEST + assert debug_payloads["gemini"]["request"]["final_request"]["model"] == "gemini-2.5-flash" + assert debug_payloads["gemini"]["response"]["provider_response"] == _GEMINI_RESPONSE + assert debug_payloads["gemini"]["response"]["final_response"]["usage"] == { + "input_tokens": 7, + "output_tokens": 4, + "total_tokens": 11, + } + + fts = await materialization_client.get( + "/api/history/sessions", headers=admin_headers, params={"q": "OpenAI materialize needle"} + ) + assert fts.status_code == 200 + assert [item["session_id"] for item in fts.json()["sessions"]] == [_SESSION_ID] diff --git a/tests/luthien_proxy/integration_tests/test_passthrough_materialize.py b/tests/luthien_proxy/integration_tests/test_passthrough_materialize.py new file mode 100644 index 000000000..cc0322ceb --- /dev/null +++ b/tests/luthien_proxy/integration_tests/test_passthrough_materialize.py @@ -0,0 +1,172 @@ +from __future__ import annotations + +import json +import os +from collections.abc import AsyncIterator +from dataclasses import dataclass +from datetime import datetime, timezone +from uuid import uuid4 + +import anyio +import pytest + +import luthien_proxy.passthrough_materialize.materialize as materialize_module +import luthien_proxy.passthrough_materialize.reconcile as reconcile_module +from luthien_proxy.passthrough_materialize.materialize import materialize_transaction +from luthien_proxy.passthrough_materialize.materialize_types import ( + AlreadyMaterialized, + CanonicalTransaction, + MaterializationResult, + Materialized, + ReconcileStats, +) +from luthien_proxy.passthrough_materialize.reconcile import reconcile_passthrough +from luthien_proxy.utils.db import DatabasePool + +pytestmark = pytest.mark.integration + + +@dataclass(frozen=True, slots=True) +class _PostgresSeed: + transaction_id: str + session_id: str + + +@pytest.fixture +async def postgres_pool() -> AsyncIterator[DatabasePool]: + database_url = os.environ.get("DATABASE_URL", "") + if not database_url or database_url.startswith("sqlite"): + pytest.skip("DATABASE_URL is not configured for a Postgres integration database") + pool = DatabasePool(database_url, min_size=2, max_size=2) + try: + async with pool.connection() as conn: + await conn.fetchval("SELECT 1") + yield pool + finally: + await pool.close() + + +async def _seed_request_log(pool: DatabasePool, seed: _PostgresSeed) -> None: + started_at = datetime.now(timezone.utc) + request_body = { + "model": "gpt-4.1", + "messages": [{"role": "user", "content": "race"}], + } + response_body = { + "id": f"chatcmpl-{seed.transaction_id}", + "model": "gpt-4.1", + "choices": [{"finish_reason": "stop", "message": {"role": "assistant", "content": "raced"}}], + } + async with pool.connection() as conn: + await conn.execute( + """ + INSERT INTO request_logs ( + id, transaction_id, session_id, user_id, direction, request_body, + response_status, response_body, started_at, completed_at, model, + is_streaming, endpoint, error + ) VALUES ($1, $2, $3, $4, $5, $6::jsonb, $7, $8::jsonb, $9, $10, $11, $12, $13, $14) + """, + f"log-{seed.transaction_id}", + seed.transaction_id, + seed.session_id, + "postgres-race-user", + "inbound", + json.dumps(request_body), + 200, + json.dumps(response_body), + started_at, + started_at, + "gpt-4.1", + False, + "/openai/v1/chat/completions", + None, + ) + + +async def _delete_seed(pool: DatabasePool, seed: _PostgresSeed) -> None: + async with pool.connection() as conn: + await conn.execute("DELETE FROM session_summaries WHERE session_id = $1", seed.session_id) + await conn.execute("DELETE FROM conversation_calls WHERE call_id = $1", seed.transaction_id) + await conn.execute("DELETE FROM request_logs WHERE transaction_id = $1", seed.transaction_id) + + +async def test_postgres_reconcile_and_live_materialization_race_preserves_one_canonical_turn( + postgres_pool: DatabasePool, + monkeypatch: pytest.MonkeyPatch, +) -> None: + seed = _PostgresSeed( + transaction_id=f"passthrough-race-{uuid4().hex}", + session_id=f"session-passthrough-race-{uuid4().hex}", + ) + await _seed_request_log(postgres_pool, seed) + reconcile_started = anyio.Event() + both_writers_ready = anyio.Event() + writer_count = 0 + reconcile_results: list[ReconcileStats] = [] + live_results: list[MaterializationResult] = [] + original_write = materialize_module.write_canonical_transaction + original_reconcile_materialize = reconcile_module.materialize_transaction + + async def synchronized_write( + pool: DatabasePool, canonical: CanonicalTransaction + ) -> Materialized | AlreadyMaterialized: + nonlocal writer_count + writer_count += 1 + if writer_count == 1: + with anyio.fail_after(5): + await both_writers_ready.wait() + elif writer_count == 2: + both_writers_ready.set() + else: + raise AssertionError("expected exactly two concurrent materialization writers") + return await original_write(pool, canonical) + + async def reconcile_materialize(pool: DatabasePool, transaction_id: str) -> MaterializationResult: + reconcile_started.set() + return await original_reconcile_materialize(pool, transaction_id) + + async def run_reconcile() -> None: + reconcile_results.append(await reconcile_passthrough(postgres_pool, limit=1)) + + async def run_live() -> None: + live_results.append(await materialize_transaction(postgres_pool, seed.transaction_id)) + + monkeypatch.setattr(materialize_module, "write_canonical_transaction", synchronized_write) + monkeypatch.setattr(reconcile_module, "materialize_transaction", reconcile_materialize) + + try: + async with anyio.create_task_group() as task_group: + task_group.start_soon(run_reconcile) + with anyio.fail_after(5): + await reconcile_started.wait() + task_group.start_soon(run_live) + + assert len(reconcile_results) == 1 + assert len(live_results) == 1 + reconcile_stats = reconcile_results[0] + live_result = live_results[0] + assert reconcile_stats.failed == 0 + assert reconcile_stats.skipped_ineligible == 0 + assert reconcile_stats.materialized + reconcile_stats.already_materialized == 1 + assert isinstance(live_result, Materialized | AlreadyMaterialized) + async with postgres_pool.connection() as conn: + call_count = await conn.fetchval( + "SELECT COUNT(*) FROM conversation_calls WHERE call_id = $1", seed.transaction_id + ) + event_count = await conn.fetchval( + "SELECT COUNT(*) FROM conversation_events WHERE call_id = $1", seed.transaction_id + ) + summary_count = await conn.fetchval( + "SELECT COUNT(*) FROM session_summaries WHERE session_id = $1", seed.session_id + ) + summary = await conn.fetchrow("SELECT * FROM session_summaries WHERE session_id = $1", seed.session_id) + + assert call_count == 1 + assert event_count == 2 + assert summary_count == 1 + assert summary is not None + assert summary["event_count"] == 2 + assert summary["call_count"] == 1 + assert summary["models_used"] == "gpt-4.1" + finally: + await _delete_seed(postgres_pool, seed) diff --git a/tests/luthien_proxy/unit_tests/passthrough_materialize/test_backfill_script.py b/tests/luthien_proxy/unit_tests/passthrough_materialize/test_backfill_script.py new file mode 100644 index 000000000..7a488b359 --- /dev/null +++ b/tests/luthien_proxy/unit_tests/passthrough_materialize/test_backfill_script.py @@ -0,0 +1,80 @@ +from __future__ import annotations + +import json +from collections.abc import AsyncIterator + +import pytest + +from luthien_proxy.passthrough_materialize.backfill import drain_passthrough_backfill +from luthien_proxy.passthrough_materialize.materialize_types import ReconcileStats +from luthien_proxy.passthrough_materialize.reconcile import reconcile_passthrough +from luthien_proxy.utils.db import DatabasePool +from luthien_proxy.utils.migration_check import check_migrations + + +@pytest.fixture +async def backfill_pool() -> AsyncIterator[DatabasePool]: + pool = DatabasePool("sqlite://:memory:") + await check_migrations(pool) + yield pool + await pool.close() + + +async def _seed_openai_transaction(pool: DatabasePool, transaction_id: str, started_at: str) -> None: + request_body = { + "model": "gpt-4.1", + "messages": [{"role": "user", "content": f"Hello from {transaction_id}."}], + } + response_body = { + "id": f"chatcmpl-{transaction_id}", + "model": "gpt-4.1", + "choices": [{"finish_reason": "stop", "message": {"role": "assistant", "content": "Hello."}}], + } + async with pool.connection() as conn: + await conn.execute( + """ + INSERT INTO request_logs ( + id, transaction_id, session_id, user_id, direction, request_body, + response_status, response_body, started_at, completed_at, model, + is_streaming, endpoint, error + ) VALUES ($1, $2, $3, $4, $5, $6::jsonb, $7, $8::jsonb, $9, $10, $11, $12, $13, $14) + """, + f"log-{transaction_id}", + transaction_id, + f"session-{transaction_id}", + f"user-{transaction_id}", + "inbound", + json.dumps(request_body), + 200, + json.dumps(response_body), + started_at, + started_at, + "gpt-4.1", + False, + "/openai/v1/chat/completions", + None, + ) + + +async def _event_count(pool: DatabasePool, transaction_id: str) -> int: + async with pool.connection() as conn: + count = await conn.fetchval("SELECT COUNT(*) FROM conversation_events WHERE call_id = $1", transaction_id) + assert isinstance(count, int) + return count + + +async def test_drain_backfill_materializes_every_eligible_transaction_then_stops_at_an_empty_sweep( + backfill_pool: DatabasePool, +) -> None: + # Given + await _seed_openai_transaction(backfill_pool, "first", "2026-07-11T09:00:00+00:00") + await _seed_openai_transaction(backfill_pool, "second", "2026-07-11T10:00:00+00:00") + + # When + totals = await drain_passthrough_backfill(backfill_pool, limit=1) + + # Then + assert totals == ReconcileStats(materialized=2) + assert await _event_count(backfill_pool, "first") == 2 + assert await _event_count(backfill_pool, "second") == 2 + assert await reconcile_passthrough(backfill_pool, limit=1) == ReconcileStats() diff --git a/tests/luthien_proxy/unit_tests/passthrough_materialize/test_endpoints.py b/tests/luthien_proxy/unit_tests/passthrough_materialize/test_endpoints.py new file mode 100644 index 000000000..28f8f03cc --- /dev/null +++ b/tests/luthien_proxy/unit_tests/passthrough_materialize/test_endpoints.py @@ -0,0 +1,57 @@ +from __future__ import annotations + +import pytest + +from luthien_proxy.passthrough_materialize.endpoints import ( + EligibleEndpoint, + EndpointKind, + ExcludedEndpoint, + Provider, + classify_endpoint, +) + + +@pytest.mark.parametrize( + ("path", "provider", "kind"), + [ + ("/openai/v1/chat/completions", Provider.OPENAI, EndpointKind.OPENAI_CHAT_COMPLETIONS), + ("/openai/v1/responses", Provider.OPENAI, EndpointKind.OPENAI_RESPONSES), + ( + "/gemini/v1beta/models/gemini-2.5-pro:generateContent", + Provider.GEMINI, + EndpointKind.GEMINI_GENERATE_CONTENT, + ), + ( + "/gemini/v1beta/models/gemini-2.5-pro:streamGenerateContent", + Provider.GEMINI, + EndpointKind.GEMINI_STREAM_GENERATE_CONTENT, + ), + ], +) +def test_classifies_eligible_endpoint_when_path_is_materializable( + path: str, + provider: Provider, + kind: EndpointKind, +) -> None: + classified = classify_endpoint(path) + + assert classified == EligibleEndpoint(path=path, provider=provider, kind=kind) + + +@pytest.mark.parametrize( + "path", + [ + "/openai/v1/models", + "/openai/v1/embeddings", + "/openai/v1/chat/completions/extra", + "/gemini/v1beta/models", + "/gemini/v1beta/models/gemini-2.5-pro:embedContent", + "/gemini/v1beta/models/gemini-2.5-pro:generateContent:extra", + "/anthropic/v1/messages", + "/unknown/v1/chat/completions", + ], +) +def test_excludes_endpoint_when_path_is_not_materializable(path: str) -> None: + classified = classify_endpoint(path) + + assert classified == ExcludedEndpoint(path=path) diff --git a/tests/luthien_proxy/unit_tests/passthrough_materialize/test_gemini_normalizers.py b/tests/luthien_proxy/unit_tests/passthrough_materialize/test_gemini_normalizers.py new file mode 100644 index 000000000..1580c0e84 --- /dev/null +++ b/tests/luthien_proxy/unit_tests/passthrough_materialize/test_gemini_normalizers.py @@ -0,0 +1,807 @@ +from __future__ import annotations + +import pytest + +from luthien_proxy.passthrough_capture import reassemble_gemini_json_array_stream +from luthien_proxy.passthrough_materialize.endpoints import EligibleEndpoint, EndpointKind, Provider +from luthien_proxy.passthrough_materialize.gemini import ( + PassthroughNormalizeError, + PassthroughNormalizeReason, + normalize_gemini_request, + normalize_gemini_response, +) +from luthien_proxy.passthrough_materialize.payloads import ( + CanonicalResponsePayload, + JsonObject, + build_request_event_payload, + build_response_event_payload, +) + + +def _generate_content_endpoint() -> EligibleEndpoint: + return EligibleEndpoint( + path="/gemini/v1beta/models/gemini-2.5-pro:generateContent", + provider=Provider.GEMINI, + kind=EndpointKind.GEMINI_GENERATE_CONTENT, + ) + + +def _stream_generate_content_endpoint() -> EligibleEndpoint: + return EligibleEndpoint( + path="/gemini/v1beta/models/gemini-2.5-pro:streamGenerateContent", + provider=Provider.GEMINI, + kind=EndpointKind.GEMINI_STREAM_GENERATE_CONTENT, + ) + + +def _gemini_3_generate_content_endpoint() -> EligibleEndpoint: + return EligibleEndpoint( + path="/gemini/v1beta/models/gemini-3-pro-preview:generateContent", + provider=Provider.GEMINI, + kind=EndpointKind.GEMINI_GENERATE_CONTENT, + ) + + +def test_normalizes_idless_gemini_25_request_when_parts_tools_and_generation_config_are_present() -> None: + # Given + request = { + "systemInstruction": {"parts": [{"text": "Be concise."}]}, + "contents": [ + {"role": "user", "parts": [{"text": "What is the weather in Paris?"}]}, + { + "role": "model", + "parts": [ + { + "functionCall": { + "name": "weather", + "args": {"city": "Paris"}, + } + }, + {"functionCall": {"name": "weather", "args": {"city": "London"}}}, + ], + }, + { + "role": "user", + "parts": [ + { + "functionResponse": { + "name": "weather", + "response": {"temperature_c": 23}, + } + }, + {"functionResponse": {"name": "weather", "response": {"temperature_c": 19}}}, + ], + }, + ], + "tools": [ + { + "functionDeclarations": [ + { + "name": "weather", + "description": "Gets the current weather.", + "parameters": {"type": "object", "properties": {"city": {"type": "string"}}}, + } + ] + } + ], + "toolConfig": {"functionCallingConfig": {"mode": "AUTO", "allowedFunctionNames": ["weather"]}}, + "generationConfig": { + "temperature": 0.2, + "topP": 0.9, + "maxOutputTokens": 64, + "stopSequences": ["END"], + "candidateCount": 1, + }, + } + + # When + normalized = normalize_gemini_request(_generate_content_endpoint(), request, transaction_id="txn_gemini_request") + payload = build_request_event_payload(normalized) + + # Then + assert normalized.is_streaming is False + assert normalized.final_model == "gemini-2.5-pro" + assert payload["provider_request"] == request + assert payload["final_request"] == { + "model": "gemini-2.5-pro", + "messages": [ + {"role": "system", "content": [{"type": "text", "text": "Be concise."}]}, + {"role": "user", "content": [{"type": "text", "text": "What is the weather in Paris?"}]}, + { + "role": "assistant", + "content": [ + {"type": "tool_use", "id": "gemini:weather:0", "name": "weather", "input": {"city": "Paris"}}, + {"type": "tool_use", "id": "gemini:weather:1", "name": "weather", "input": {"city": "London"}}, + ], + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "gemini:weather:0", + "content": {"temperature_c": 23}, + }, + {"type": "tool_result", "tool_use_id": "gemini:weather:1", "content": {"temperature_c": 19}}, + ], + }, + ], + "tools": [ + { + "name": "weather", + "description": "Gets the current weather.", + "input_schema": {"type": "object", "properties": {"city": {"type": "string"}}}, + } + ], + "tool_choice": {"mode": "auto", "allowed_function_names": ["weather"]}, + "generation_config": { + "temperature": 0.2, + "topP": 0.9, + "maxOutputTokens": 64, + "stopSequences": ["END"], + "candidateCount": 1, + }, + "temperature": 0.2, + "top_p": 0.9, + "max_tokens": 64, + "stop": ["END"], + "candidate_count": 1, + "stream": False, + } + + +def test_normalizes_idless_gemini_25_response_when_text_function_call_usage_and_safety_are_present() -> None: + # Given + response = { + "responseId": "gemini-response-1", + "modelVersion": "gemini-2.5-pro-001", + "candidates": [ + { + "content": { + "role": "model", + "parts": [ + {"text": "I found the weather."}, + { + "functionCall": { + "name": "weather", + "args": {"city": "Paris"}, + } + }, + ], + }, + "finishReason": "STOP", + "safetyRatings": [{"category": "HARM_CATEGORY_DANGEROUS_CONTENT", "probability": "NEGLIGIBLE"}], + } + ], + "usageMetadata": { + "promptTokenCount": 10, + "candidatesTokenCount": 5, + "totalTokenCount": 15, + "cachedContentTokenCount": 2, + "thoughtsTokenCount": 1, + }, + } + + # When + normalized = normalize_gemini_response( + _generate_content_endpoint(), + response, + request_is_streaming=False, + http_status=200, + transaction_id="txn_gemini_response", + ) + payload = build_response_event_payload(normalized) + + # Then + assert normalized.is_streaming is False + assert normalized.final_model == "gemini-2.5-pro-001" + assert payload["provider_response"] == response + assert payload["final_response"] == { + "id": "gemini-response-1", + "model": "gemini-2.5-pro-001", + "role": "assistant", + "content": [ + {"type": "text", "text": "I found the weather."}, + {"type": "tool_use", "id": "gemini:weather:0", "name": "weather", "input": {"city": "Paris"}}, + ], + "stop_reason": "end_turn", + "usage": { + "input_tokens": 10, + "output_tokens": 5, + "total_tokens": 15, + "cache_read_input_tokens": 2, + "reasoning_tokens": 1, + }, + "safety_ratings": [{"category": "HARM_CATEGORY_DANGEROUS_CONTENT", "probability": "NEGLIGIBLE"}], + } + + +def test_normalizes_gemini_blocked_and_upstream_error_responses_without_synthetic_success() -> None: + # Given + blocked_response = { + "responseId": "gemini-blocked", + "modelVersion": "gemini-2.5-pro-001", + "promptFeedback": { + "blockReason": "SAFETY", + "safetyRatings": [{"category": "HARM_CATEGORY_HATE_SPEECH", "blocked": True}], + }, + "usageMetadata": {"promptTokenCount": 4, "totalTokenCount": 4}, + } + error_response = {"error": {"code": 429, "message": "Quota exceeded", "status": "RESOURCE_EXHAUSTED"}} + + # When + blocked = normalize_gemini_response( + _generate_content_endpoint(), + blocked_response, + request_is_streaming=False, + http_status=200, + transaction_id="txn_gemini_blocked", + ) + upstream_error = normalize_gemini_response( + _generate_content_endpoint(), + error_response, + request_is_streaming=False, + http_status=429, + transaction_id="txn_gemini_error", + ) + + # Then + assert build_response_event_payload(blocked)["final_response"] == { + "id": "gemini-blocked", + "model": "gemini-2.5-pro-001", + "role": "assistant", + "content": [], + "stop_reason": "blocked", + "usage": {"input_tokens": 4, "total_tokens": 4}, + "prompt_feedback": { + "blockReason": "SAFETY", + "safetyRatings": [{"category": "HARM_CATEGORY_HATE_SPEECH", "blocked": True}], + }, + } + assert build_response_event_payload(upstream_error)["final_response"] == { + "role": "assistant", + "content": [], + "stop_reason": "error", + "error": {"status_code": 429, "body": error_response}, + } + + +@pytest.mark.parametrize( + ("finish_reason", "expected_stop_reason"), + [("SAFETY", "safety"), ("RECITATION", "refusal")], +) +def test_normalizes_gemini_safety_and_refusal_finish_reasons_explicitly( + finish_reason: str, expected_stop_reason: str +) -> None: + # Given + response = { + "responseId": "gemini-finish", + "modelVersion": "gemini-2.5-pro-001", + "candidates": [ + { + "content": {"role": "model", "parts": [{"text": "Cannot complete that."}]}, + "finishReason": finish_reason, + "safetyRatings": [{"category": "HARM_CATEGORY_HARASSMENT", "blocked": True}], + } + ], + } + + # When + payload = build_response_event_payload( + normalize_gemini_response( + _generate_content_endpoint(), + response, + request_is_streaming=False, + http_status=200, + transaction_id="txn_finish", + ) + ) + + # Then + assert payload["final_response"]["stop_reason"] == expected_stop_reason + assert payload["final_response"]["safety_ratings"] == [{"category": "HARM_CATEGORY_HARASSMENT", "blocked": True}] + + +def test_gemini_normalization_raises_typed_errors_for_malformed_and_unknown_eligible_variants() -> None: + # Given + missing_contents = {"generationConfig": {"maxOutputTokens": 8}} + unknown_request_part = {"contents": [{"role": "user", "parts": [{"inlineData": {"mimeType": "text/plain"}}]}]} + unknown_response_finish = { + "candidates": [{"content": {"role": "model", "parts": [{"text": "x"}]}, "finishReason": "NEW_REASON"}] + } + malformed_response_part = { + "candidates": [{"content": {"role": "model", "parts": [{"functionResponse": {}}]}, "finishReason": "STOP"}] + } + + # When / Then + with pytest.raises(PassthroughNormalizeError) as missing_contents_error: + normalize_gemini_request(_generate_content_endpoint(), missing_contents, transaction_id="txn_missing_contents") + with pytest.raises(PassthroughNormalizeError) as unknown_request_error: + normalize_gemini_request( + _generate_content_endpoint(), unknown_request_part, transaction_id="txn_unknown_request" + ) + with pytest.raises(PassthroughNormalizeError) as unknown_finish_error: + normalize_gemini_response( + _generate_content_endpoint(), + unknown_response_finish, + request_is_streaming=False, + http_status=200, + transaction_id="txn_unknown_finish", + ) + with pytest.raises(PassthroughNormalizeError) as malformed_part_error: + normalize_gemini_response( + _generate_content_endpoint(), + malformed_response_part, + request_is_streaming=False, + http_status=200, + transaction_id="txn_malformed_part", + ) + + assert (missing_contents_error.value.reason, missing_contents_error.value.detail) == ( + PassthroughNormalizeReason.MISSING_REQUIRED_FIELD, + "contents", + ) + assert (unknown_request_error.value.reason, unknown_request_error.value.detail) == ( + PassthroughNormalizeReason.MISSING_REQUIRED_FIELD, + "contents", + ) + assert (unknown_finish_error.value.reason, unknown_finish_error.value.detail) == ( + PassthroughNormalizeReason.UNSUPPORTED_VARIANT, + "candidate.finishReason:NEW_REASON", + ) + assert (malformed_part_error.value.reason, malformed_part_error.value.detail) == ( + PassthroughNormalizeReason.MISSING_REQUIRED_FIELD, + "candidate.content", + ) + + +def test_normalizes_safety_candidate_without_content_as_an_explicit_empty_assistant_turn() -> None: + # Given + response = { + "responseId": "gemini-safety-no-content", + "modelVersion": "gemini-2.5-pro-001", + "candidates": [ + { + "finishReason": "SAFETY", + "safetyRatings": [{"category": "HARM_CATEGORY_DANGEROUS_CONTENT", "blocked": True}], + } + ], + "usageMetadata": {"promptTokenCount": 5, "totalTokenCount": 5}, + } + + # When + normalized = normalize_gemini_response( + _generate_content_endpoint(), + response, + request_is_streaming=False, + http_status=200, + transaction_id="txn_gemini_safety_no_content", + ) + + # Then + assert build_response_event_payload(normalized)["final_response"] == { + "id": "gemini-safety-no-content", + "model": "gemini-2.5-pro-001", + "role": "assistant", + "content": [], + "stop_reason": "safety", + "usage": {"input_tokens": 5, "total_tokens": 5}, + "safety_ratings": [{"category": "HARM_CATEGORY_DANGEROUS_CONTENT", "blocked": True}], + } + + +def test_selects_the_zero_indexed_candidate_when_gemini_returns_multiple_candidates() -> None: + # Given + response = { + "candidates": [ + { + "index": 1, + "content": {"role": "model", "parts": [{"text": "secondary candidate"}]}, + "finishReason": "STOP", + }, + { + "index": 0, + "content": {"role": "model", "parts": [{"text": "canonical candidate"}]}, + "finishReason": "STOP", + }, + ] + } + + # When + normalized = normalize_gemini_response( + _generate_content_endpoint(), + response, + request_is_streaming=False, + http_status=200, + transaction_id="txn_gemini_candidate_zero", + ) + + # Then + assert build_response_event_payload(normalized)["final_response"]["content"] == [ + {"type": "text", "text": "canonical candidate"} + ] + + +def test_rejects_request_without_a_recoverable_function_response() -> None: + # Given + request = { + "contents": [ + { + "role": "user", + "parts": [{"functionResponse": {"response": {"temperature_c": 23}}}], + } + ] + } + + # When / Then + with pytest.raises(PassthroughNormalizeError) as exc_info: + normalize_gemini_request(_generate_content_endpoint(), request, transaction_id="txn_missing_response_name") + + assert (exc_info.value.reason, exc_info.value.detail) == ( + PassthroughNormalizeReason.MISSING_REQUIRED_FIELD, + "contents", + ) + + +def test_normalizes_json_array_stream_when_text_deltas_finish_and_usage_are_captured() -> None: + # Given + response = { + "stream_format": "gemini-json-array", + "chunks": [ + { + "responseId": "gemini-json-stream", + "modelVersion": "gemini-2.5-pro-001", + "candidates": [{"index": 0, "content": {"role": "model", "parts": [{"text": "Hel"}]}}], + }, + { + "candidates": [ + {"index": 0, "content": {"role": "model", "parts": [{"text": "lo"}]}, "finishReason": "STOP"} + ], + "usageMetadata": {"promptTokenCount": 3, "candidatesTokenCount": 2, "totalTokenCount": 5}, + }, + ], + "final": None, + } + + # When + normalized = normalize_gemini_response( + _stream_generate_content_endpoint(), + response, + request_is_streaming=True, + http_status=200, + transaction_id="txn_gemini_json_stream", + ) + payload = build_response_event_payload(normalized) + + # Then + assert payload["final_response"] == { + "id": "gemini-json-stream", + "model": "gemini-2.5-pro-001", + "role": "assistant", + "content": [{"type": "text", "text": "Hello"}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 3, "output_tokens": 2, "total_tokens": 5}, + } + + +def test_normalizes_idless_gemini_25_sse_stream_when_function_call_deltas_are_merged_in_order() -> None: + # Given + response = { + "stream_format": "gemini-sse", + "chunks": [ + { + "responseId": "gemini-sse-stream", + "modelVersion": "gemini-2.5-pro-001", + "candidates": [ + { + "index": 0, + "content": { + "role": "model", + "parts": [ + {"text": "Checking weather. "}, + { + "functionCall": { + "name": "weather", + "args": {"city": "Paris"}, + } + }, + ], + }, + } + ], + }, + { + "candidates": [ + { + "index": 0, + "content": { + "role": "model", + "parts": [ + { + "functionCall": { + "name": "weather", + "args": {"units": "metric"}, + } + } + ], + }, + "finishReason": "MAX_TOKENS", + } + ], + "usageMetadata": {"promptTokenCount": 3, "candidatesTokenCount": 4, "totalTokenCount": 7}, + }, + ], + "final": None, + } + + # When + normalized = normalize_gemini_response( + _stream_generate_content_endpoint(), + response, + request_is_streaming=True, + http_status=200, + transaction_id="txn_gemini_sse_stream", + ) + payload = build_response_event_payload(normalized) + + # Then + assert payload["final_response"] == { + "id": "gemini-sse-stream", + "model": "gemini-2.5-pro-001", + "role": "assistant", + "content": [ + {"type": "text", "text": "Checking weather. "}, + { + "type": "tool_use", + "id": "gemini:weather:0", + "name": "weather", + "input": {"city": "Paris", "units": "metric"}, + }, + ], + "stop_reason": "max_tokens", + "usage": {"input_tokens": 3, "output_tokens": 4, "total_tokens": 7}, + } + + +def test_rejects_content_free_max_tokens_json_array_stream_before_canonical_output() -> None: + # Given + response = reassemble_gemini_json_array_stream( + [b'[{"responseId":"gemini-empty-max-tokens","candidates":[{"index":0,"finishReason":"MAX_TOKENS"}]}]'] + ) + canonical_outputs: list[CanonicalResponsePayload] = [] + + # When / Then + with pytest.raises(PassthroughNormalizeError) as exc_info: + canonical_outputs.append( + build_response_event_payload( + normalize_gemini_response( + _stream_generate_content_endpoint(), + response, + request_is_streaming=True, + http_status=200, + transaction_id="txn_gemini_empty_max_tokens_stream", + ) + ) + ) + + assert (exc_info.value.reason, exc_info.value.detail) == ( + PassthroughNormalizeReason.MISSING_REQUIRED_FIELD, + "candidate.content", + ) + assert canonical_outputs == [] + + +def test_rejects_content_free_max_tokens_buffered_response_before_canonical_output() -> None: + # Given + response = { + "candidates": [ + { + "content": {"role": "model", "parts": []}, + "finishReason": "MAX_TOKENS", + } + ] + } + canonical_outputs: list[CanonicalResponsePayload] = [] + + # When / Then + with pytest.raises(PassthroughNormalizeError) as exc_info: + canonical_outputs.append( + build_response_event_payload( + normalize_gemini_response( + _generate_content_endpoint(), + response, + request_is_streaming=False, + http_status=200, + transaction_id="txn_gemini_empty_max_tokens_buffered", + ) + ) + ) + + assert (exc_info.value.reason, exc_info.value.detail) == ( + PassthroughNormalizeReason.MISSING_REQUIRED_FIELD, + "candidate.content", + ) + assert canonical_outputs == [] + + +@pytest.mark.parametrize( + ("finish_reason", "expected_stop_reason"), + [("SAFETY", "safety"), ("BLOCKLIST", "blocked")], +) +def test_normalizes_content_free_stream_when_finish_reason_allows_empty_turn( + finish_reason: str, expected_stop_reason: str +) -> None: + # Given + response = reassemble_gemini_json_array_stream( + [f'{{"candidates":[{{"index":0,"finishReason":"{finish_reason}"}}]}}'.encode()] + ) + + # When + normalized = normalize_gemini_response( + _stream_generate_content_endpoint(), + response, + request_is_streaming=True, + http_status=200, + transaction_id="txn_gemini_content_free_stream", + ) + + # Then + assert build_response_event_payload(normalized)["final_response"] == { + "role": "assistant", + "content": [], + "stop_reason": expected_stop_reason, + } + + +def test_preserves_id_bearing_gemini_3_tool_pairing() -> None: + # Given + request = { + "contents": [ + {"role": "model", "parts": [{"functionCall": {"id": "call_g3_weather", "name": "weather"}}]}, + { + "role": "user", + "parts": [{"functionResponse": {"id": "call_g3_weather", "name": "weather", "response": {"ok": True}}}], + }, + ] + } + + # When + normalized = normalize_gemini_request(_gemini_3_generate_content_endpoint(), request) + + # Then + assert build_request_event_payload(normalized)["final_request"]["messages"] == [ + { + "role": "assistant", + "content": [{"type": "tool_use", "id": "call_g3_weather", "name": "weather", "input": {}}], + }, + { + "role": "user", + "content": [{"type": "tool_result", "tool_use_id": "call_g3_weather", "content": {"ok": True}}], + }, + ] + + +@pytest.mark.parametrize( + ("response", "reason", "detail"), + [ + ( + {"stream_format": "gemini-json-array", "chunks": [], "final": None, "capture_truncated": True}, + PassthroughNormalizeReason.CAPTURE_TRUNCATED, + "stream capture truncated", + ), + ( + {"stream_format": "gemini-sse", "chunks": [], "final": None, "raw": "data: {broken"}, + PassthroughNormalizeReason.CAPTURE_TRUNCATED, + "stream capture truncated", + ), + ( + {"stream_format": "openai-sse", "chunks": [], "final": None}, + PassthroughNormalizeReason.UNSUPPORTED_VARIANT, + "stream_format:openai-sse", + ), + ( + {"stream_format": "gemini-sse", "chunks": [{"event": "unknown"}], "final": None}, + PassthroughNormalizeReason.UNSUPPORTED_VARIANT, + "stream chunk", + ), + ( + { + "stream_format": "gemini-sse", + "chunks": [ + { + "candidates": [ + {"content": {"role": "model", "parts": [{"inlineData": {"mimeType": "text/plain"}}]}} + ] + } + ], + "final": None, + }, + PassthroughNormalizeReason.MISSING_REQUIRED_FIELD, + "candidate.finishReason", + ), + ( + { + "stream_format": "gemini-json-array", + "chunks": [{"candidates": [{"content": {"role": "model", "parts": [{"text": "partial"}]}}]}], + "final": None, + }, + PassthroughNormalizeReason.MISSING_REQUIRED_FIELD, + "candidate.finishReason", + ), + ], +) +def test_stream_normalization_raises_typed_errors_when_wrapper_or_chunk_is_incomplete( + response: JsonObject, reason: PassthroughNormalizeReason, detail: str +) -> None: + # Given the stored wrapper or a provider chunk is incomplete or unsupported + + # When / Then + with pytest.raises(PassthroughNormalizeError) as exc_info: + normalize_gemini_response( + _stream_generate_content_endpoint(), + response, + request_is_streaming=True, + http_status=200, + transaction_id="txn_gemini_bad_stream", + ) + + assert (exc_info.value.reason, exc_info.value.detail) == (reason, detail) + +def test_stream_normalization_falls_back_to_raw_when_chunk_has_unmodelled_field() -> None: + # Given: a stream chunk carrying a field the google-genai SDK model doesn't + # know about yet (usageMetadata.serviceTier is the concrete case observed in + # prod). The buffered path (normalize_gemini_response) falls back to the raw + # payload on ValidationError; the stream path must mirror that or streamed + # and buffered captures of the same conversation get opposite outcomes. + response = { + "stream_format": "gemini-sse", + "chunks": [ + { + "candidates": [ + { + "content": {"role": "model", "parts": [{"text": "hi"}]}, + "finishReason": "STOP", + } + ], + "usageMetadata": { + "promptTokenCount": 1, + "candidatesTokenCount": 1, + "totalTokenCount": 2, + "serviceTier": "standard", + }, + }, + ], + "final": None, + } + + # When + normalized = normalize_gemini_response( + _stream_generate_content_endpoint(), + response, + request_is_streaming=True, + http_status=200, + transaction_id="txn_gemini_stream_service_tier", + ) + + # Then: the chunk normalizes cleanly (no MALFORMED_PAYLOAD), and usage from the + # unmodelled-field-carrying chunk is preserved. + payload = build_response_event_payload(normalized) + assert payload["final_response"]["usage"]["input_tokens"] == 1 + assert payload["final_response"]["usage"]["output_tokens"] == 1 + +def test_normalizes_gemini_request_when_content_role_is_omitted_defaults_to_user() -> None: + # Given: Gemini API spec makes `role` OPTIONAL in contents[] (defaults to "user"). + # A minimal probe like {"contents": [{"parts": [{"text": "..."}]}]} is a valid Gemini call + # (observed in prod as a health-check probe against gemini-2.5-flash) and must materialize. + request = {"contents": [{"parts": [{"text": "say prod-capture-ok"}]}]} + + # When + normalized = normalize_gemini_request( + _generate_content_endpoint(), request, transaction_id="txn_gemini_no_role" + ) + payload = build_request_event_payload(normalized) + + # Then: the message role defaults to "user" instead of failing MISSING_REQUIRED_FIELD:contents. + assert payload["final_request"]["messages"] == [ + {"role": "user", "content": [{"type": "text", "text": "say prod-capture-ok"}]}, + ] diff --git a/tests/luthien_proxy/unit_tests/passthrough_materialize/test_lenient_output_items.py b/tests/luthien_proxy/unit_tests/passthrough_materialize/test_lenient_output_items.py new file mode 100644 index 000000000..5b2d86c32 --- /dev/null +++ b/tests/luthien_proxy/unit_tests/passthrough_materialize/test_lenient_output_items.py @@ -0,0 +1,97 @@ +from __future__ import annotations + +from luthien_proxy.passthrough_materialize.endpoints import EligibleEndpoint, EndpointKind, Provider +from luthien_proxy.passthrough_materialize.openai import normalize_openai_responses_response +from luthien_proxy.passthrough_materialize.payloads import build_response_event_payload + + +def _responses_endpoint() -> EligibleEndpoint: + return EligibleEndpoint( + path="/openai/v1/responses", + provider=Provider.OPENAI, + kind=EndpointKind.OPENAI_RESPONSES, + ) + + +def test_normalizes_reasoning_output_with_text_and_function_calls() -> None: + # Given a gpt-5.6-sol response with an opaque reasoning item. + response = { + "id": "resp_reasoning", + "model": "gpt-5.6-sol", + "status": "completed", + "output": [ + { + "type": "reasoning", + "id": "rs_1", + "content": [], + "summary": [], + "encrypted_content": "opaque", + }, + { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": "I found two results."}], + }, + { + "type": "function_call", + "call_id": "call_one", + "name": "lookup", + "arguments": '{"query":"first"}', + }, + { + "type": "function_call", + "call_id": "call_two", + "name": "lookup", + "arguments": '{"query":"second"}', + }, + ], + } + + # When the materializer normalizes the captured response. + payload = build_response_event_payload( + normalize_openai_responses_response( + _responses_endpoint(), + response, + request_is_streaming=False, + http_status=200, + transaction_id="txn_reasoning", + ) + ) + + # Then it preserves usable assistant text and tool calls. + assert payload["final_response"]["content"] == [ + {"type": "text", "text": "I found two results."}, + {"type": "tool_use", "id": "call_one", "name": "lookup", "input": {"query": "first"}}, + {"type": "tool_use", "id": "call_two", "name": "lookup", "input": {"query": "second"}}, + ] + + +def test_skips_unknown_output_item_when_response_has_usable_text() -> None: + # Given a future output item adjacent to a standard output message. + response = { + "id": "resp_future", + "model": "gpt-5.6-sol", + "status": "completed", + "output": [ + {"type": "future_provider_item", "provider_metadata": {"version": 1}}, + { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": "Still usable."}], + }, + ], + } + + # When the materializer normalizes the captured response. + payload = build_response_event_payload( + normalize_openai_responses_response( + _responses_endpoint(), + response, + request_is_streaming=False, + http_status=200, + transaction_id="txn_future_output", + ) + ) + + # Then it skips the unknown item instead of failing the transaction. + assert payload["final_response"]["content"] == [{"type": "text", "text": "Still usable."}] diff --git a/tests/luthien_proxy/unit_tests/passthrough_materialize/test_lenient_request_items.py b/tests/luthien_proxy/unit_tests/passthrough_materialize/test_lenient_request_items.py new file mode 100644 index 000000000..1f6106016 --- /dev/null +++ b/tests/luthien_proxy/unit_tests/passthrough_materialize/test_lenient_request_items.py @@ -0,0 +1,114 @@ +from __future__ import annotations + +from luthien_proxy.passthrough_materialize.endpoints import EligibleEndpoint, EndpointKind, Provider +from luthien_proxy.passthrough_materialize.gemini import normalize_gemini_request +from luthien_proxy.passthrough_materialize.openai import ( + normalize_openai_chat_request, + normalize_openai_responses_request, +) +from luthien_proxy.passthrough_materialize.payloads import build_request_event_payload + + +def _chat_endpoint() -> EligibleEndpoint: + return EligibleEndpoint( + path="/openai/v1/chat/completions", + provider=Provider.OPENAI, + kind=EndpointKind.OPENAI_CHAT_COMPLETIONS, + ) + + +def _responses_endpoint() -> EligibleEndpoint: + return EligibleEndpoint( + path="/openai/v1/responses", + provider=Provider.OPENAI, + kind=EndpointKind.OPENAI_RESPONSES, + ) + + +def _gemini_endpoint() -> EligibleEndpoint: + return EligibleEndpoint( + path="/gemini/v1beta/models/gemini-2.5-pro:generateContent", + provider=Provider.GEMINI, + kind=EndpointKind.GEMINI_GENERATE_CONTENT, + ) + + +def test_normalizes_responses_reasoning_capture_when_unmodelled_items_surround_recoverable_turns() -> None: + # Given a reasoning-model request that echoes opaque response items in input. + request = { + "model": "gpt-5.6-sol", + "input": [ + {"type": "reasoning", "id": "rs_1", "summary": [], "content": []}, + {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "hi"}]}, + {"type": "function_call", "call_id": "c1", "name": "f", "arguments": "{}"}, + {"type": "function_call_output", "call_id": "c1", "output": "ok"}, + ], + } + + # When the request is materialized. + payload = build_request_event_payload( + normalize_openai_responses_request(_responses_endpoint(), request, transaction_id="txn_reasoning_request") + ) + + # Then only recoverable user and tool turns are retained. + assert payload["final_request"]["messages"] == [ + {"role": "user", "content": [{"type": "text", "text": "hi"}]}, + {"role": "tool", "tool_call_id": "c1", "content": "ok"}, + ] + + +def test_normalizes_responses_string_input_when_instructions_are_present() -> None: + # Given a Responses request using the plain-string shorthand. + request = {"model": "gpt-5.6-sol", "instructions": "Be concise.", "input": "hi"} + + # When the request is materialized. + payload = build_request_event_payload( + normalize_openai_responses_request(_responses_endpoint(), request, transaction_id="txn_string_input") + ) + + # Then instructions and the user message are preserved. + assert payload["final_request"]["messages"] == [ + {"role": "system", "content": "Be concise."}, + {"role": "user", "content": "hi"}, + ] + + +def test_normalizes_chat_request_when_unknown_messages_and_provider_tools_are_present() -> None: + # Given a Chat Completions request with an unmodelled message and provider tool. + request = { + "model": "gpt-5.6-sol", + "messages": [ + {"role": "provider_internal", "content": "ignore"}, + {"role": "user", "content": "hi"}, + ], + "tools": [{"type": "web_search"}], + } + + # When the request is materialized. + payload = build_request_event_payload( + normalize_openai_chat_request(_chat_endpoint(), request, transaction_id="txn_lenient_chat") + ) + + # Then the recognized user message remains and unsupported input is omitted. + assert payload["final_request"]["messages"] == [{"role": "user", "content": "hi"}] + assert payload["final_request"]["tools"] == [] + + +def test_normalizes_gemini_request_when_unknown_parts_and_provider_tools_are_present() -> None: + # Given a Gemini request with an opaque part and a non-function tool. + request = { + "contents": [ + {"role": "user", "parts": [{"thought": True}, {"text": "hi"}]}, + {"role": "provider_internal", "parts": [{"text": "ignore"}]}, + ], + "tools": [{"googleSearch": {}}], + } + + # When the request is materialized. + payload = build_request_event_payload( + normalize_gemini_request(_gemini_endpoint(), request, transaction_id="txn_lenient_gemini") + ) + + # Then the recognized text remains and unsupported input is omitted. + assert payload["final_request"]["messages"] == [{"role": "user", "content": [{"type": "text", "text": "hi"}]}] + assert payload["final_request"]["tools"] == [] diff --git a/tests/luthien_proxy/unit_tests/passthrough_materialize/test_materialize.py b/tests/luthien_proxy/unit_tests/passthrough_materialize/test_materialize.py new file mode 100644 index 000000000..8d7a9e26e --- /dev/null +++ b/tests/luthien_proxy/unit_tests/passthrough_materialize/test_materialize.py @@ -0,0 +1,323 @@ +from __future__ import annotations + +import json +from collections.abc import AsyncIterator + +import pytest + +from luthien_proxy.debug.service import fetch_call_diff, fetch_call_events +from luthien_proxy.history.service import fetch_session_detail +from luthien_proxy.passthrough_materialize.materialize import materialize_transaction +from luthien_proxy.passthrough_materialize.materialize_types import ( + AlreadyMaterialized, + MaterializationFailed, + Materialized, + SkippedIneligible, +) +from luthien_proxy.utils.db import DatabasePool +from luthien_proxy.utils.migration_check import check_migrations + + +@pytest.fixture +async def materialize_pool() -> AsyncIterator[DatabasePool]: + pool = DatabasePool("sqlite://:memory:") + await check_migrations(pool) + yield pool + await pool.close() + + +async def _seed_openai_chat_transaction(pool: DatabasePool, transaction_id: str) -> None: + request_body = { + "model": "gpt-4.1", + "messages": [{"role": "user", "content": "Hello from passthrough."}], + } + response_body = { + "id": "chatcmpl-materialized", + "model": "gpt-4.1", + "choices": [{"finish_reason": "stop", "message": {"role": "assistant", "content": "Hello back."}}], + } + async with pool.connection() as conn: + await conn.execute( + """ + INSERT INTO request_logs ( + id, transaction_id, session_id, user_id, direction, request_body, + response_status, response_body, started_at, completed_at, model, + is_streaming, endpoint, error + ) VALUES ($1, $2, $3, $4, $5, $6::jsonb, $7, $8::jsonb, $9, $10, $11, $12, $13, $14) + """, + "log-inbound", + transaction_id, + "session-materialized", + "user-materialized", + "inbound", + json.dumps(request_body), + 200, + json.dumps(response_body), + "2026-07-11T10:00:00+00:00", + "2026-07-11T10:00:01+00:00", + "gpt-4.1", + False, + "/openai/v1/chat/completions", + None, + ) + await conn.execute( + """ + INSERT INTO request_logs ( + id, transaction_id, session_id, user_id, direction, request_body, + response_status, response_body, started_at, completed_at, model, + is_streaming, endpoint, error + ) VALUES ($1, $2, $3, $4, $5, $6::jsonb, $7, $8::jsonb, $9, $10, $11, $12, $13, $14) + """, + "log-outbound", + transaction_id, + "session-materialized", + "user-materialized", + "outbound", + json.dumps({"model": "must-not-win", "messages": []}), + 200, + json.dumps(response_body), + "2026-07-11T10:00:00+00:00", + "2026-07-11T10:00:01+00:00", + "must-not-win", + False, + "/openai/v1/chat/completions", + None, + ) + + +async def test_materializes_openai_chat_transaction_when_paired_raw_logs_exist(materialize_pool: DatabasePool) -> None: + # Given + transaction_id = "transaction-materialized" + await _seed_openai_chat_transaction(materialize_pool, transaction_id) + + # When + result = await materialize_transaction(materialize_pool, transaction_id) + + # Then + assert isinstance(result, Materialized) + async with materialize_pool.connection() as conn: + event_rows = await conn.fetch( + "SELECT event_type, payload, created_at FROM conversation_events WHERE call_id = $1 ORDER BY created_at", + transaction_id, + ) + call_row = await conn.fetchrow("SELECT * FROM conversation_calls WHERE call_id = $1", transaction_id) + summary_row = await conn.fetchrow( + "SELECT * FROM session_summaries WHERE session_id = $1", "session-materialized" + ) + + assert [row["event_type"] for row in event_rows] == [ + "transaction.request_recorded", + "transaction.non_streaming_response_recorded", + ] + assert str(event_rows[0]["created_at"]) < str(event_rows[1]["created_at"]) + assert call_row is not None + assert call_row["session_id"] == "session-materialized" + assert call_row["user_id"] == "user-materialized" + assert summary_row is not None + assert summary_row["event_count"] == 2 + assert summary_row["call_count"] == 1 + assert summary_row["models_used"] == "gpt-4.1" + assert str(summary_row["first_seen"]) == str(event_rows[0]["created_at"]) + assert str(summary_row["last_seen"]) == str(event_rows[1]["created_at"]) + request_payload = json.loads(str(event_rows[0]["payload"])) + response_payload = json.loads(str(event_rows[1]["payload"])) + assert request_payload["provider_request"]["model"] == "gpt-4.1" + assert response_payload["final_response"]["content"] == [{"type": "text", "text": "Hello back."}] + + history = await fetch_session_detail("session-materialized", materialize_pool) + debug_events = await fetch_call_events(transaction_id, materialize_pool) + debug_diff = await fetch_call_diff(transaction_id, materialize_pool) + async with materialize_pool.connection() as conn: + fts_rows = await conn.fetch( + "SELECT content FROM conversation_events_fts WHERE session_id = $1", "session-materialized" + ) + + assert history.turns[0].request_messages[0].content == "Hello from passthrough." + assert history.turns[0].response_messages[0].content == "Hello back." + assert debug_events.events[0].payload["provider"] == "openai" + assert debug_diff.request is not None + assert debug_diff.request.model_changed is False + assert fts_rows[0]["content"] == "Hello from passthrough." + + +async def test_returns_already_materialized_when_transaction_is_replayed(materialize_pool: DatabasePool) -> None: + # Given + transaction_id = "transaction-idempotent" + await _seed_openai_chat_transaction(materialize_pool, transaction_id) + await materialize_transaction(materialize_pool, transaction_id) + + # When + result = await materialize_transaction(materialize_pool, transaction_id) + + # Then + assert isinstance(result, AlreadyMaterialized) + async with materialize_pool.connection() as conn: + count = await conn.fetchval("SELECT COUNT(*) FROM conversation_events WHERE call_id = $1", transaction_id) + assert count == 2 + + +async def test_materializes_gemini_stream_when_reassembled_capture_is_recorded(materialize_pool: DatabasePool) -> None: + # Given + transaction_id = "transaction-gemini-stream" + await _seed_openai_chat_transaction(materialize_pool, transaction_id) + request_body = {"contents": [{"role": "user", "parts": [{"text": "Say hello."}]}]} + response_body = { + "stream_format": "gemini-json-array", + "chunks": [ + {"candidates": [{"index": 0, "content": {"role": "model", "parts": [{"text": "Hel"}]}}]}, + { + "candidates": [ + {"index": 0, "content": {"role": "model", "parts": [{"text": "lo"}]}, "finishReason": "STOP"} + ] + }, + ], + "final": None, + } + async with materialize_pool.connection() as conn: + await conn.execute( + """ + UPDATE request_logs + SET request_body = $1::jsonb, response_body = $2::jsonb, endpoint = $3, is_streaming = $4, model = $5 + WHERE transaction_id = $6 + """, + json.dumps(request_body), + json.dumps(response_body), + "/gemini/v1beta/models/gemini-2.5-pro:streamGenerateContent", + True, + "gemini-2.5-pro", + transaction_id, + ) + + # When + result = await materialize_transaction(materialize_pool, transaction_id) + + # Then + assert isinstance(result, Materialized) + async with materialize_pool.connection() as conn: + response_row = await conn.fetchrow( + "SELECT payload FROM conversation_events WHERE call_id = $1 AND event_type = $2", + transaction_id, + "transaction.streaming_response_recorded", + ) + assert response_row is not None + response_payload = json.loads(str(response_row["payload"])) + assert response_payload["provider"] == "gemini" + assert response_payload["provider_response"] == response_body + assert response_payload["final_response"]["content"] == [{"type": "text", "text": "Hello"}] + + +async def test_preserves_absent_user_id_when_request_log_has_no_identity(materialize_pool: DatabasePool) -> None: + # Given + transaction_id = "transaction-no-user" + await _seed_openai_chat_transaction(materialize_pool, transaction_id) + async with materialize_pool.connection() as conn: + await conn.execute("UPDATE request_logs SET user_id = NULL WHERE transaction_id = $1", transaction_id) + + # When + await materialize_transaction(materialize_pool, transaction_id) + + # Then + async with materialize_pool.connection() as conn: + call_row = await conn.fetchrow("SELECT user_id FROM conversation_calls WHERE call_id = $1", transaction_id) + summary_row = await conn.fetchrow( + "SELECT user_id FROM session_summaries WHERE session_id = $1", "session-materialized" + ) + assert call_row is not None + assert summary_row is not None + assert call_row["user_id"] is None + assert summary_row["user_id"] is None + + +async def test_materializes_upstream_error_when_capture_has_error_without_response_body( + materialize_pool: DatabasePool, +) -> None: + # Given + transaction_id = "transaction-upstream-error" + await _seed_openai_chat_transaction(materialize_pool, transaction_id) + async with materialize_pool.connection() as conn: + await conn.execute( + "UPDATE request_logs SET response_body = NULL, response_status = $1, error = $2 WHERE transaction_id = $3", + 502, + "ConnectError: upstream unavailable", + transaction_id, + ) + + # When + result = await materialize_transaction(materialize_pool, transaction_id) + + # Then + assert isinstance(result, Materialized) + async with materialize_pool.connection() as conn: + call_row = await conn.fetchrow("SELECT status FROM conversation_calls WHERE call_id = $1", transaction_id) + response_row = await conn.fetchrow( + "SELECT payload FROM conversation_events WHERE call_id = $1 AND event_type = $2", + transaction_id, + "transaction.non_streaming_response_recorded", + ) + assert call_row is not None + assert response_row is not None + assert call_row["status"] == "error" + assert json.loads(str(response_row["payload"]))["final_response"] == { + "role": "assistant", + "content": [], + "stop_reason": "error", + "error": {"status_code": 502, "body": {"error": "ConnectError: upstream unavailable"}}, + } + + +async def test_skips_ineligible_endpoint_before_parsing_malformed_capture(materialize_pool: DatabasePool) -> None: + # Given + transaction_id = "transaction-ineligible" + await _seed_openai_chat_transaction(materialize_pool, transaction_id) + async with materialize_pool.connection() as conn: + await conn.execute( + "UPDATE request_logs SET endpoint = $1, request_body = $2, response_body = $2 WHERE transaction_id = $3", + "/openai/v1/models", + "not-json", + transaction_id, + ) + + # When + result = await materialize_transaction(materialize_pool, transaction_id) + + # Then + assert isinstance(result, SkippedIneligible) + async with materialize_pool.connection() as conn: + event_count = await conn.fetchval("SELECT COUNT(*) FROM conversation_events WHERE call_id = $1", transaction_id) + call_count = await conn.fetchval("SELECT COUNT(*) FROM conversation_calls WHERE call_id = $1", transaction_id) + summary_count = await conn.fetchval( + "SELECT COUNT(*) FROM session_summaries WHERE session_id = $1", "session-materialized" + ) + assert event_count == 0 + assert call_count == 0 + assert summary_count == 0 + + +async def test_keeps_normalization_failure_retryable_without_request_event(materialize_pool: DatabasePool) -> None: + # Given + transaction_id = "transaction-normalize-failure" + await _seed_openai_chat_transaction(materialize_pool, transaction_id) + async with materialize_pool.connection() as conn: + await conn.execute( + "UPDATE request_logs SET request_body = $1::jsonb WHERE transaction_id = $2", + json.dumps({"model": "gpt-4.1"}), + transaction_id, + ) + + # When + first_result = await materialize_transaction(materialize_pool, transaction_id) + retry_result = await materialize_transaction(materialize_pool, transaction_id) + + # Then + assert isinstance(first_result, MaterializationFailed) + assert first_result.reason == "missing_required_field" + assert isinstance(retry_result, MaterializationFailed) + async with materialize_pool.connection() as conn: + event_count = await conn.fetchval("SELECT COUNT(*) FROM conversation_events WHERE call_id = $1", transaction_id) + call_count = await conn.fetchval("SELECT COUNT(*) FROM conversation_calls WHERE call_id = $1", transaction_id) + summary_count = await conn.fetchval( + "SELECT COUNT(*) FROM session_summaries WHERE session_id = $1", "session-materialized" + ) + assert event_count == 0 + assert call_count == 0 + assert summary_count == 0 diff --git a/tests/luthien_proxy/unit_tests/passthrough_materialize/test_openai_normalizers.py b/tests/luthien_proxy/unit_tests/passthrough_materialize/test_openai_normalizers.py new file mode 100644 index 000000000..294242ae3 --- /dev/null +++ b/tests/luthien_proxy/unit_tests/passthrough_materialize/test_openai_normalizers.py @@ -0,0 +1,748 @@ +from __future__ import annotations + +import pytest + +from luthien_proxy.passthrough_materialize.endpoints import EligibleEndpoint, EndpointKind, Provider +from luthien_proxy.passthrough_materialize.openai import ( + PassthroughNormalizeError, + PassthroughNormalizeReason, + normalize_openai_chat_request, + normalize_openai_chat_response, + normalize_openai_responses_request, + normalize_openai_responses_response, +) +from luthien_proxy.passthrough_materialize.payloads import ( + JsonObject, + build_request_event_payload, + build_response_event_payload, +) + + +def _chat_endpoint() -> EligibleEndpoint: + return EligibleEndpoint( + path="/openai/v1/chat/completions", + provider=Provider.OPENAI, + kind=EndpointKind.OPENAI_CHAT_COMPLETIONS, + ) + + +def _responses_endpoint() -> EligibleEndpoint: + return EligibleEndpoint( + path="/openai/v1/responses", + provider=Provider.OPENAI, + kind=EndpointKind.OPENAI_RESPONSES, + ) + + +def test_normalizes_chat_request_when_buffered_body_contains_tools_and_text_blocks() -> None: + request = { + "model": "gpt-4.1", + "stream": False, + "messages": [ + {"role": "developer", "content": "Follow policy."}, + {"role": "user", "content": [{"type": "text", "text": "Use the weather tool."}]}, + { + "role": "assistant", + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": {"name": "weather", "arguments": '{"city":"Paris"}'}, + } + ], + }, + {"role": "tool", "tool_call_id": "call_1", "content": "sunny"}, + ], + "tools": [ + { + "type": "function", + "function": { + "name": "weather", + "description": "Get weather", + "parameters": {"type": "object", "properties": {"city": {"type": "string"}}}, + }, + } + ], + "tool_choice": "auto", + "max_completion_tokens": 128, + } + + normalized = normalize_openai_chat_request(_chat_endpoint(), request, transaction_id="txn_1") + payload = build_request_event_payload(normalized) + + assert normalized.is_streaming is False + assert normalized.final_model == "gpt-4.1" + assert payload["provider_request"] == request + assert payload["final_request"] == { + "model": "gpt-4.1", + "messages": [ + {"role": "system", "content": "Follow policy."}, + {"role": "user", "content": [{"type": "text", "text": "Use the weather tool."}]}, + { + "role": "assistant", + "content": [ + { + "type": "tool_use", + "id": "call_1", + "name": "weather", + "input": {"city": "Paris"}, + } + ], + }, + {"role": "tool", "tool_call_id": "call_1", "content": "sunny"}, + ], + "tools": [ + { + "name": "weather", + "description": "Get weather", + "input_schema": {"type": "object", "properties": {"city": {"type": "string"}}}, + } + ], + "tool_choice": "auto", + "max_tokens": 128, + "max_completion_tokens": 128, + "stream": False, + } + + +def test_normalizes_chat_buffered_response_when_content_tool_refusal_and_usage_present() -> None: + response = { + "id": "chatcmpl_1", + "model": "gpt-4.1", + "choices": [ + { + "finish_reason": "tool_calls", + "message": { + "role": "assistant", + "content": "I can check.", + "refusal": "Cannot reveal hidden data.", + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": {"name": "lookup", "arguments": '{"q":"safe"}'}, + } + ], + }, + } + ], + "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, + } + + normalized = normalize_openai_chat_response( + _chat_endpoint(), response, request_is_streaming=False, http_status=200, transaction_id="txn_2" + ) + payload = build_response_event_payload(normalized) + + assert normalized.is_streaming is False + assert normalized.final_model == "gpt-4.1" + assert payload["provider_response"] == response + assert payload["final_response"] == { + "id": "chatcmpl_1", + "model": "gpt-4.1", + "role": "assistant", + "content": [ + {"type": "text", "text": "I can check."}, + {"type": "text", "text": "Cannot reveal hidden data."}, + {"type": "tool_use", "id": "call_1", "name": "lookup", "input": {"q": "safe"}}, + ], + "stop_reason": "tool_use", + "usage": {"input_tokens": 10, "output_tokens": 5, "total_tokens": 15}, + } + + +def test_normalizes_chat_stream_response_when_openai_sse_events_are_complete() -> None: + response = { + "stream_format": "openai-sse", + "events": [ + {"id": "chatcmpl_2", "model": "gpt-4.1", "choices": [{"delta": {"role": "assistant", "content": "Hel"}}]}, + {"choices": [{"delta": {"content": "lo"}}]}, + { + "choices": [{"delta": {}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 3, "completion_tokens": 2, "total_tokens": 5}, + }, + ], + "final": {"choices": [{"delta": {}, "finish_reason": "stop"}]}, + } + + normalized = normalize_openai_chat_response( + _chat_endpoint(), response, request_is_streaming=True, http_status=200, transaction_id="txn_3" + ) + payload = build_response_event_payload(normalized) + + assert normalized.is_streaming is True + assert payload["final_response"] == { + "id": "chatcmpl_2", + "model": "gpt-4.1", + "role": "assistant", + "content": [{"type": "text", "text": "Hello"}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 3, "output_tokens": 2, "total_tokens": 5}, + } + + +def test_normalizes_upstream_http_error_when_openai_error_body_is_captured() -> None: + response = {"error": {"message": "bad request", "type": "invalid_request_error"}} + + normalized = normalize_openai_chat_response( + _chat_endpoint(), response, request_is_streaming=False, http_status=400, transaction_id="txn_4" + ) + payload = build_response_event_payload(normalized) + + assert payload["final_response"] == { + "role": "assistant", + "content": [], + "stop_reason": "error", + "error": {"status_code": 400, "body": response}, + } + assert payload["provider_response"] == response + + +@pytest.mark.parametrize( + ("payload", "reason"), + [ + ({"model": "gpt-4.1"}, PassthroughNormalizeReason.MISSING_REQUIRED_FIELD), + ( + { + "model": "gpt-4.1", + "messages": [ + { + "role": "user", + "content": [{"type": "image_url", "image_url": {"url": "https://example.test/a.png"}}], + } + ], + }, + PassthroughNormalizeReason.MISSING_REQUIRED_FIELD, + ), + ], +) +def test_chat_request_raises_typed_error_when_payload_is_malformed_or_unsupported( + payload: JsonObject, reason: PassthroughNormalizeReason +) -> None: + with pytest.raises(PassthroughNormalizeError) as exc_info: + normalize_openai_chat_request(_chat_endpoint(), payload, transaction_id="txn_bad") + + assert exc_info.value.provider == Provider.OPENAI + assert exc_info.value.endpoint_kind == EndpointKind.OPENAI_CHAT_COMPLETIONS + assert exc_info.value.endpoint_path == "/openai/v1/chat/completions" + assert exc_info.value.transaction_id == "txn_bad" + assert exc_info.value.reason == reason + + +def test_stream_wrapper_raises_typed_error_when_capture_was_truncated() -> None: + response = {"stream_format": "openai-sse", "events": [], "final": None, "capture_truncated": True} + + with pytest.raises(PassthroughNormalizeError) as exc_info: + normalize_openai_chat_response( + _chat_endpoint(), response, request_is_streaming=True, http_status=200, transaction_id="txn_truncated" + ) + + assert exc_info.value.reason == PassthroughNormalizeReason.CAPTURE_TRUNCATED + + +def test_normalizes_responses_request_when_instructions_input_tools_and_tokens_present() -> None: + request = { + "model": "gpt-4.1", + "instructions": "Be concise.", + "input": [ + {"role": "user", "content": [{"type": "input_text", "text": "Summarize this."}]}, + {"type": "function_call_output", "call_id": "call_1", "output": "done"}, + ], + "tools": [ + { + "type": "function", + "name": "search", + "description": "Search", + "parameters": {"type": "object", "properties": {"query": {"type": "string"}}}, + } + ], + "tool_choice": "auto", + "max_output_tokens": 64, + "stream": True, + } + + normalized = normalize_openai_responses_request(_responses_endpoint(), request, transaction_id="txn_5") + payload = build_request_event_payload(normalized) + + assert normalized.is_streaming is True + assert payload["final_request"] == { + "model": "gpt-4.1", + "messages": [ + {"role": "system", "content": "Be concise."}, + {"role": "user", "content": [{"type": "text", "text": "Summarize this."}]}, + {"role": "tool", "tool_call_id": "call_1", "content": "done"}, + ], + "tools": [ + { + "name": "search", + "description": "Search", + "input_schema": {"type": "object", "properties": {"query": {"type": "string"}}}, + } + ], + "tool_choice": "auto", + "max_tokens": 64, + "max_output_tokens": 64, + "stream": True, + } + + +def test_normalizes_responses_buffered_response_when_output_status_function_and_usage_present() -> None: + response = { + "id": "resp_1", + "model": "gpt-4.1", + "status": "completed", + "output": [ + { + "type": "message", + "role": "assistant", + "content": [ + {"type": "output_text", "text": "Done."}, + {"type": "refusal", "refusal": "Cannot provide secret."}, + ], + }, + {"type": "function_call", "call_id": "call_1", "name": "search", "arguments": '{"query":"x"}'}, + ], + "usage": {"input_tokens": 7, "output_tokens": 4, "total_tokens": 11}, + } + + normalized = normalize_openai_responses_response( + _responses_endpoint(), response, request_is_streaming=False, http_status=200, transaction_id="txn_6" + ) + payload = build_response_event_payload(normalized) + + assert payload["final_response"] == { + "id": "resp_1", + "model": "gpt-4.1", + "role": "assistant", + "content": [ + {"type": "text", "text": "Done."}, + {"type": "text", "text": "Cannot provide secret."}, + {"type": "tool_use", "id": "call_1", "name": "search", "input": {"query": "x"}}, + ], + "stop_reason": "tool_use", + "status": "completed", + "usage": {"input_tokens": 7, "output_tokens": 4, "total_tokens": 11}, + } + + +def test_responses_buffered_response_with_novel_status_raises_typed_error_not_uncaught() -> None: + # A future OpenAI Response.status the pinned SDK does not model must surface as a + # typed, retryable PassthroughNormalizeError (skips one transaction), NOT an uncaught + # pydantic ValidationError -- materialize.py and reconcile.py catch only typed/DB + # errors, so an uncaught ValidationError would wedge the whole backfill batch. + response = { + "id": "resp_1", + "model": "gpt-4.1", + "status": "a_future_status_the_pinned_sdk_does_not_know", + "output": [ + {"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": "hi"}]}, + ], + } + + with pytest.raises(PassthroughNormalizeError) as exc_info: + normalize_openai_responses_response( + _responses_endpoint(), + response, + request_is_streaming=False, + http_status=200, + transaction_id="txn_novel_status", + ) + + assert exc_info.value.detail == "response" + + +def test_normalizes_responses_stream_response_when_events_fold_to_final_message() -> None: + response = { + "stream_format": "openai-sse", + "events": [ + {"type": "response.output_text.delta", "delta": "Hel"}, + {"type": "response.output_text.delta", "delta": "lo"}, + { + "type": "response.completed", + "response": { + "id": "resp_2", + "model": "gpt-4.1", + "status": "completed", + "usage": {"input_tokens": 2, "output_tokens": 1, "total_tokens": 3}, + }, + }, + ], + "final": {"type": "response.completed"}, + } + + normalized = normalize_openai_responses_response( + _responses_endpoint(), response, request_is_streaming=True, http_status=200, transaction_id="txn_7" + ) + payload = build_response_event_payload(normalized) + + assert payload["final_response"] == { + "id": "resp_2", + "model": "gpt-4.1", + "role": "assistant", + "content": [{"type": "text", "text": "Hello"}], + "stop_reason": "end_turn", + "status": "completed", + "usage": {"input_tokens": 2, "output_tokens": 1, "total_tokens": 3}, + } + + +def test_responses_stream_uses_completed_response_when_lifecycle_events_are_content_free() -> None: + response = { + "stream_format": "openai-sse", + "events": [ + {"type": "response.created", "response": {"id": "resp_stream", "status": "in_progress"}}, + {"type": "response.in_progress", "response": {"id": "resp_stream", "status": "in_progress"}}, + {"type": "response.output_item.added", "output_index": 0, "item": {"type": "message", "id": "msg_1"}}, + { + "type": "response.content_part.added", + "output_index": 0, + "content_index": 0, + "part": {"type": "output_text"}, + }, + {"type": "response.output_text.delta", "delta": "Draft"}, + {"type": "response.output_text.done", "output_index": 0, "content_index": 0}, + {"type": "response.content_part.done", "output_index": 0, "content_index": 0}, + {"type": "response.output_item.done", "output_index": 0}, + {"type": "response.output_item.added", "output_index": 1, "item": {"type": "function_call", "id": "fc_1"}}, + {"type": "response.function_call_arguments.delta", "output_index": 1, "delta": '{"query"'}, + {"type": "response.function_call_arguments.delta", "output_index": 1, "delta": ':"safe"}'}, + {"type": "response.function_call_arguments.done", "output_index": 1}, + { + "type": "response.completed", + "response": { + "id": "resp_stream", + "model": "gpt-4.1", + "status": "completed", + "output": [ + { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": "Authoritative final."}], + }, + { + "type": "function_call", + "call_id": "call_search", + "name": "search", + "arguments": '{"query":"safe"}', + }, + ], + "usage": {"input_tokens": 0, "output_tokens": 4, "total_tokens": 4}, + }, + }, + ], + "final": {"type": "response.completed"}, + } + + normalized = normalize_openai_responses_response( + _responses_endpoint(), + response, + request_is_streaming=True, + http_status=200, + transaction_id="txn_stream_lifecycle", + ) + payload = build_response_event_payload(normalized) + + assert payload["final_response"] == { + "id": "resp_stream", + "model": "gpt-4.1", + "role": "assistant", + "content": [ + {"type": "text", "text": "Authoritative final."}, + {"type": "tool_use", "id": "call_search", "name": "search", "input": {"query": "safe"}}, + ], + "stop_reason": "tool_use", + "status": "completed", + "usage": {"input_tokens": 0, "output_tokens": 4, "total_tokens": 4}, + } + + +def test_stream_wrappers_raise_typed_error_when_raw_partial_capture_is_present() -> None: + chat_response = { + "stream_format": "openai-sse", + "events": [{"choices": [{"delta": {"content": "Hel"}}]}], + "raw": "data: {broken", + "final": None, + } + responses_response = { + "stream_format": "openai-sse", + "events": [{"type": "response.output_text.delta", "delta": "Hel"}], + "raw": "data: {broken", + "final": None, + } + + for endpoint, normalizer, response in ( + (_chat_endpoint(), normalize_openai_chat_response, chat_response), + (_responses_endpoint(), normalize_openai_responses_response, responses_response), + ): + with pytest.raises(PassthroughNormalizeError) as exc_info: + normalizer(endpoint, response, request_is_streaming=True, http_status=200, transaction_id="txn_raw") + + assert exc_info.value.reason == PassthroughNormalizeReason.CAPTURE_TRUNCATED + + +def test_chat_stream_reconstructs_tool_call_arguments_and_refusal_deltas() -> None: + tool_response = { + "stream_format": "openai-sse", + "events": [ + { + "id": "chatcmpl_tool", + "model": "gpt-4.1", + "choices": [ + { + "delta": { + "role": "assistant", + "tool_calls": [ + { + "index": 0, + "id": "call_weather", + "type": "function", + "function": {"name": "weather", "arguments": '{"city"'}, + } + ], + } + } + ], + }, + {"choices": [{"delta": {"tool_calls": [{"index": 0, "function": {"arguments": ':"Paris"}'}}]}}]}, + {"choices": [{"delta": {}, "finish_reason": "tool_calls"}]}, + ], + "final": {"choices": [{"delta": {}, "finish_reason": "tool_calls"}]}, + } + refusal_response = { + "stream_format": "openai-sse", + "events": [ + {"id": "chatcmpl_refusal", "model": "gpt-4.1", "choices": [{"delta": {"refusal": "No"}}]}, + {"choices": [{"delta": {"refusal": "."}}]}, + {"choices": [{"delta": {}, "finish_reason": "stop"}]}, + ], + "final": {"choices": [{"delta": {}, "finish_reason": "stop"}]}, + } + + tool_payload = build_response_event_payload( + normalize_openai_chat_response( + _chat_endpoint(), + tool_response, + request_is_streaming=True, + http_status=200, + transaction_id="txn_tool_stream", + ) + ) + refusal_payload = build_response_event_payload( + normalize_openai_chat_response( + _chat_endpoint(), + refusal_response, + request_is_streaming=True, + http_status=200, + transaction_id="txn_refusal_stream", + ) + ) + + assert tool_payload["final_response"] == { + "id": "chatcmpl_tool", + "model": "gpt-4.1", + "role": "assistant", + "content": [{"type": "tool_use", "id": "call_weather", "name": "weather", "input": {"city": "Paris"}}], + "stop_reason": "tool_use", + } + assert refusal_payload["final_response"] == { + "id": "chatcmpl_refusal", + "model": "gpt-4.1", + "role": "assistant", + "content": [{"type": "text", "text": "No."}], + "stop_reason": "end_turn", + } + + +def test_normalizers_reject_missing_or_empty_tool_call_ids() -> None: + chat_response = { + "choices": [ + { + "finish_reason": "tool_calls", + "message": { + "role": "assistant", + "tool_calls": [{"type": "function", "function": {"name": "lookup", "arguments": "{}"}}], + }, + } + ] + } + responses_response = {"output": [{"type": "function_call", "call_id": "", "name": "lookup", "arguments": "{}"}]} + + for endpoint, normalizer, response in ( + (_chat_endpoint(), normalize_openai_chat_response, chat_response), + (_responses_endpoint(), normalize_openai_responses_response, responses_response), + ): + with pytest.raises(PassthroughNormalizeError) as exc_info: + normalizer( + endpoint, response, request_is_streaming=False, http_status=200, transaction_id="txn_missing_tool_id" + ) + + assert exc_info.value.reason == PassthroughNormalizeReason.MISSING_REQUIRED_FIELD + assert "id" in exc_info.value.detail or "call_id" in exc_info.value.detail + + +def test_responses_unknown_variants_report_precise_typed_details() -> None: + unknown_input = {"model": "gpt-4.1", "input": [{"type": "web_search_call", "query": "x"}]} + unknown_event = { + "stream_format": "openai-sse", + "events": [{"type": "response.unexpected", "delta": "x"}], + "final": None, + } + + with pytest.raises(PassthroughNormalizeError) as input_exc: + normalize_openai_responses_request(_responses_endpoint(), unknown_input, transaction_id="txn_unknown_input") + + with pytest.raises(PassthroughNormalizeError) as event_exc: + normalize_openai_responses_response( + _responses_endpoint(), + unknown_event, + request_is_streaming=True, + http_status=200, + transaction_id="txn_unknown_event", + ) + + assert input_exc.value.reason == PassthroughNormalizeReason.MISSING_REQUIRED_FIELD + assert input_exc.value.detail == "input" + assert event_exc.value.reason == PassthroughNormalizeReason.UNSUPPORTED_VARIANT + assert event_exc.value.detail == "stream event.type:response.unexpected" + + +def test_usage_zero_counts_and_unknown_finish_reasons_do_not_leak_raw_values() -> None: + response = { + "id": "chatcmpl_zero", + "model": "gpt-4.1", + "choices": [{"finish_reason": "new_provider_reason", "message": {"role": "assistant", "content": "Done"}}], + "usage": {"prompt_tokens": 0, "input_tokens": 9, "completion_tokens": 0, "output_tokens": 8, "total_tokens": 0}, + } + + payload = build_response_event_payload( + normalize_openai_chat_response( + _chat_endpoint(), response, request_is_streaming=False, http_status=200, transaction_id="txn_zero_usage" + ) + ) + + assert payload["final_response"]["usage"] == {"input_tokens": 0, "output_tokens": 0, "total_tokens": 0} + assert payload["final_response"]["stop_reason"] == "end_turn" + + +@pytest.mark.parametrize( + ("payload", "reason"), + [ + ( + {"model": "gpt-4.1", "input": [{"role": "user", "content": [{"type": "input_image", "image_url": "x"}]}]}, + PassthroughNormalizeReason.MISSING_REQUIRED_FIELD, + ), + ( + {"id": "resp_3", "model": "gpt-4.1", "output": [{"type": "web_search_call"}]}, + PassthroughNormalizeReason.MISSING_REQUIRED_FIELD, + ), + ], +) +def test_responses_payload_raises_typed_error_when_sub_shape_is_unknown( + payload: JsonObject, reason: PassthroughNormalizeReason +) -> None: + endpoint = _responses_endpoint() + + with pytest.raises(PassthroughNormalizeError) as exc_info: + if "input" in payload: + normalize_openai_responses_request(endpoint, payload, transaction_id="txn_unsupported") + else: + normalize_openai_responses_response( + endpoint, payload, request_is_streaming=False, http_status=200, transaction_id="txn_unsupported" + ) + + assert exc_info.value.reason == reason + assert exc_info.value.endpoint_kind == EndpointKind.OPENAI_RESPONSES + + +def test_chat_reasoning_tokens_lift_from_completion_tokens_details_into_canonical_usage() -> None: + # Given: an OpenAI Chat Completions response for a reasoning model. OpenAI nests the + # reasoning token count under `usage.completion_tokens_details.reasoning_tokens` - + # `output_tokens` still counts them, but we surface `reasoning_tokens` separately so + # downstream consumers can distinguish reasoning from visible output. + response = { + "id": "chatcmpl_reasoning", + "model": "gpt-5.6-sol", + "choices": [{"finish_reason": "stop", "message": {"role": "assistant", "content": "visible"}}], + "usage": { + "prompt_tokens": 12, + "completion_tokens": 8, + "total_tokens": 20, + "completion_tokens_details": {"reasoning_tokens": 6}, + }, + } + + payload = build_response_event_payload( + normalize_openai_chat_response( + _chat_endpoint(), + response, + request_is_streaming=False, + http_status=200, + transaction_id="txn_chat_reasoning", + ) + ) + + assert payload["final_response"]["usage"] == { + "input_tokens": 12, + "output_tokens": 8, + "total_tokens": 20, + "reasoning_tokens": 6, + } + + +def test_responses_reasoning_tokens_lift_from_output_tokens_details_into_canonical_usage() -> None: + # Given: an OpenAI Responses API response for a reasoning model. Responses nests the + # reasoning token count under `usage.output_tokens_details.reasoning_tokens`. + response = { + "id": "resp_reasoning", + "model": "gpt-5.6-sol", + "status": "completed", + "output": [{"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": "visible"}]}], + "usage": { + "input_tokens": 14, + "output_tokens": 9, + "total_tokens": 23, + "output_tokens_details": {"reasoning_tokens": 7}, + }, + } + + payload = build_response_event_payload( + normalize_openai_responses_response( + _responses_endpoint(), + response, + request_is_streaming=False, + http_status=200, + transaction_id="txn_responses_reasoning", + ) + ) + + assert payload["final_response"]["usage"] == { + "input_tokens": 14, + "output_tokens": 9, + "total_tokens": 23, + "reasoning_tokens": 7, + } + + +def test_chat_content_filter_finish_reason_maps_to_safety_not_end_turn() -> None: + # Given: OpenAI's `content_filter` finish_reason indicates a safety-policy block, NOT + # a normal completion. Previously we collapsed it to `end_turn`, making it + # indistinguishable from a natural stop. Now it maps to `safety` (matching the Gemini + # normalizer's SAFETY bucket) so downstream consumers can filter blocked completions. + response = { + "id": "chatcmpl_blocked", + "model": "gpt-4.1", + "choices": [{"finish_reason": "content_filter", "message": {"role": "assistant", "content": ""}}], + "usage": {"prompt_tokens": 5, "completion_tokens": 0, "total_tokens": 5}, + } + + payload = build_response_event_payload( + normalize_openai_chat_response( + _chat_endpoint(), + response, + request_is_streaming=False, + http_status=200, + transaction_id="txn_content_filter", + ) + ) + + assert payload["final_response"]["stop_reason"] == "safety" diff --git a/tests/luthien_proxy/unit_tests/passthrough_materialize/test_payloads.py b/tests/luthien_proxy/unit_tests/passthrough_materialize/test_payloads.py new file mode 100644 index 000000000..8c8ebd508 --- /dev/null +++ b/tests/luthien_proxy/unit_tests/passthrough_materialize/test_payloads.py @@ -0,0 +1,144 @@ +from __future__ import annotations + +from luthien_proxy.passthrough_materialize.endpoints import EligibleEndpoint, EndpointKind, Provider +from luthien_proxy.passthrough_materialize.payloads import ( + CanonicalRequestInput, + CanonicalResponseInput, + ResponseEventType, + build_request_event_payload, + build_response_event_payload, +) + + +def test_builds_request_payload_with_metadata_and_native_request() -> None: + endpoint = EligibleEndpoint( + path="/openai/v1/chat/completions", + provider=Provider.OPENAI, + kind=EndpointKind.OPENAI_CHAT_COMPLETIONS, + ) + provider_request = {"model": "gpt-4o", "messages": [{"role": "user", "content": "hi"}]} + original_request = {"model": "gpt-4o", "messages": [{"role": "user", "content": "hi"}]} + final_request = {"model": "gpt-4o-mini", "messages": [{"role": "user", "content": "hi"}]} + + payload = build_request_event_payload( + CanonicalRequestInput( + endpoint=endpoint, + is_streaming=False, + final_model="gpt-4o-mini", + original_request=original_request, + final_request=final_request, + provider_request=provider_request, + ) + ) + + assert payload["provider"] == "openai" + assert payload["endpoint"] == "/openai/v1/chat/completions" + assert payload["endpoint_kind"] == "openai_chat_completions" + assert payload["is_streaming"] is False + assert payload["final_model"] == "gpt-4o-mini" + assert payload["original_request"] == original_request + assert payload["final_request"] == final_request + assert payload["provider_request"] == provider_request + + +def test_request_payload_is_independent_of_source_mutation() -> None: + endpoint = EligibleEndpoint( + path="/openai/v1/responses", + provider=Provider.OPENAI, + kind=EndpointKind.OPENAI_RESPONSES, + ) + source_request = {"model": "gpt-4o", "input": [{"role": "user", "content": "hi"}]} + input_data = CanonicalRequestInput( + endpoint=endpoint, + is_streaming=False, + final_model="gpt-4o", + original_request=source_request, + final_request=source_request, + provider_request=source_request, + ) + source_request["model"] = "mutated" + source_request["input"] = [] + + payload = build_request_event_payload(input_data) + + assert payload["original_request"] == {"model": "gpt-4o", "input": [{"role": "user", "content": "hi"}]} + assert payload["final_request"] == {"model": "gpt-4o", "input": [{"role": "user", "content": "hi"}]} + assert payload["provider_request"] == {"model": "gpt-4o", "input": [{"role": "user", "content": "hi"}]} + + +def test_builds_non_streaming_response_payload_with_native_response() -> None: + endpoint = EligibleEndpoint( + path="/openai/v1/responses", + provider=Provider.OPENAI, + kind=EndpointKind.OPENAI_RESPONSES, + ) + provider_response = {"id": "resp_123", "output": [{"type": "message", "content": []}]} + final_response = {"id": "msg_123", "content": []} + + payload = build_response_event_payload( + CanonicalResponseInput( + endpoint=endpoint, + is_streaming=False, + final_model="gpt-4o", + original_response=final_response, + final_response=final_response, + provider_response=provider_response, + ) + ) + + assert payload["event_type"] == ResponseEventType.NON_STREAMING.value + assert payload["provider"] == "openai" + assert payload["provider_response"] == provider_response + assert payload["original_response"] == final_response + assert payload["final_response"] == final_response + + +def test_builds_streaming_response_payload_with_native_response() -> None: + endpoint = EligibleEndpoint( + path="/gemini/v1beta/models/gemini-2.5-pro:streamGenerateContent", + provider=Provider.GEMINI, + kind=EndpointKind.GEMINI_STREAM_GENERATE_CONTENT, + ) + provider_response = {"candidates": [{"content": {"parts": [{"text": "hi"}]}}]} + final_response = {"content": [{"type": "text", "text": "hi"}]} + + payload = build_response_event_payload( + CanonicalResponseInput( + endpoint=endpoint, + is_streaming=True, + final_model="gemini-2.5-pro", + original_response=final_response, + final_response=final_response, + provider_response=provider_response, + ) + ) + + assert payload["event_type"] == ResponseEventType.STREAMING.value + assert payload["provider"] == "gemini" + assert payload["endpoint_kind"] == "gemini_stream_generate_content" + assert payload["is_streaming"] is True + assert payload["provider_response"] == provider_response + + +def test_response_payload_is_independent_of_source_mutation() -> None: + endpoint = EligibleEndpoint( + path="/gemini/v1beta/models/gemini-2.5-pro:generateContent", + provider=Provider.GEMINI, + kind=EndpointKind.GEMINI_GENERATE_CONTENT, + ) + source_response = {"candidates": [{"content": {"parts": [{"text": "hi"}]}}]} + input_data = CanonicalResponseInput( + endpoint=endpoint, + is_streaming=False, + final_model="gemini-2.5-pro", + original_response=source_response, + final_response=source_response, + provider_response=source_response, + ) + source_response["candidates"] = [] + + payload = build_response_event_payload(input_data) + + assert payload["original_response"] == {"candidates": [{"content": {"parts": [{"text": "hi"}]}}]} + assert payload["final_response"] == {"candidates": [{"content": {"parts": [{"text": "hi"}]}}]} + assert payload["provider_response"] == {"candidates": [{"content": {"parts": [{"text": "hi"}]}}]} diff --git a/tests/luthien_proxy/unit_tests/passthrough_materialize/test_reconcile.py b/tests/luthien_proxy/unit_tests/passthrough_materialize/test_reconcile.py new file mode 100644 index 000000000..197125a33 --- /dev/null +++ b/tests/luthien_proxy/unit_tests/passthrough_materialize/test_reconcile.py @@ -0,0 +1,246 @@ +from __future__ import annotations + +import json +from collections.abc import AsyncIterator +from dataclasses import dataclass, replace +from datetime import datetime + +import aiosqlite +import anyio +import pytest + +import luthien_proxy.passthrough_materialize.materialize as materialize_module +import luthien_proxy.passthrough_materialize.reconcile as reconcile_module +from luthien_proxy.passthrough_materialize.materialize import materialize_transaction +from luthien_proxy.passthrough_materialize.materialize_types import ( + AlreadyMaterialized, + CanonicalTransaction, + MaterializationResult, + Materialized, + ReconcileStats, +) +from luthien_proxy.passthrough_materialize.payloads import JsonObject +from luthien_proxy.passthrough_materialize.reconcile import reconcile_passthrough +from luthien_proxy.utils.db import DatabasePool +from luthien_proxy.utils.migration_check import check_migrations + + +@pytest.fixture +async def reconcile_pool() -> AsyncIterator[DatabasePool]: + pool = DatabasePool("sqlite://:memory:") + await check_migrations(pool) + yield pool + await pool.close() + + +@dataclass(frozen=True, slots=True) +class _RawLogSeed: + transaction_id: str + started_at: datetime + endpoint: str + request_body: JsonObject + response_body: JsonObject + + +def _at(hour: int) -> datetime: + return datetime.fromisoformat(f"2026-07-11T{hour:02d}:00:00+00:00") + + +def _valid_openai_seed(transaction_id: str, started_at: datetime) -> _RawLogSeed: + return _RawLogSeed( + transaction_id=transaction_id, + started_at=started_at, + endpoint="/openai/v1/chat/completions", + request_body={ + "model": "gpt-4.1", + "messages": [{"role": "user", "content": f"Hello from {transaction_id}."}], + }, + response_body={ + "id": f"chatcmpl-{transaction_id}", + "model": "gpt-4.1", + "choices": [{"finish_reason": "stop", "message": {"role": "assistant", "content": "Hello."}}], + }, + ) + + +async def _seed_raw_log(pool: DatabasePool, seed: _RawLogSeed) -> None: + async with pool.connection() as conn: + await conn.execute( + """ + INSERT INTO request_logs ( + id, transaction_id, session_id, user_id, direction, request_body, + response_status, response_body, started_at, completed_at, model, + is_streaming, endpoint, error + ) VALUES ($1, $2, $3, $4, $5, $6::jsonb, $7, $8::jsonb, $9, $10, $11, $12, $13, $14) + """, + f"log-{seed.transaction_id}", + seed.transaction_id, + f"session-{seed.transaction_id}", + f"user-{seed.transaction_id}", + "inbound", + json.dumps(seed.request_body), + 200, + json.dumps(seed.response_body), + seed.started_at, + seed.started_at, + "gpt-4.1", + False, + seed.endpoint, + None, + ) + + +async def _event_count(pool: DatabasePool, transaction_id: str) -> int: + async with pool.connection() as conn: + count = await conn.fetchval("SELECT COUNT(*) FROM conversation_events WHERE call_id = $1", transaction_id) + assert isinstance(count, int) + return count + + +async def test_reconcile_materializes_only_eligible_unmaterialized_transactions_when_raw_logs_are_mixed( + reconcile_pool: DatabasePool, +) -> None: + # Given + ready = _valid_openai_seed("ready", _at(9)) + already = _valid_openai_seed("already", _at(10)) + ineligible = replace(_valid_openai_seed("ineligible", _at(11)), endpoint="/openai/v1/models") + failure = replace(_valid_openai_seed("failure", _at(12)), request_body={"model": "gpt-4.1"}) + await _seed_raw_log(reconcile_pool, ready) + await _seed_raw_log(reconcile_pool, already) + await _seed_raw_log(reconcile_pool, ineligible) + await _seed_raw_log(reconcile_pool, failure) + await materialize_transaction(reconcile_pool, already.transaction_id) + + # When + first_pass = await reconcile_passthrough(reconcile_pool, limit=10) + second_pass = await reconcile_passthrough(reconcile_pool, limit=10) + + # Then + assert first_pass == ReconcileStats(materialized=1, failed=1) + assert second_pass == ReconcileStats(failed=1) + assert await _event_count(reconcile_pool, ready.transaction_id) == 2 + assert await _event_count(reconcile_pool, already.transaction_id) == 2 + assert await _event_count(reconcile_pool, ineligible.transaction_id) == 0 + assert await _event_count(reconcile_pool, failure.transaction_id) == 0 + + +async def test_reconcile_respects_oldest_first_limit_and_since_when_selecting_transactions( + reconcile_pool: DatabasePool, +) -> None: + # Given + before_window = _valid_openai_seed("before-window", _at(9)) + oldest = _valid_openai_seed("oldest", _at(10)) + middle = _valid_openai_seed("middle", _at(11)) + newest = _valid_openai_seed("newest", _at(12)) + await _seed_raw_log(reconcile_pool, before_window) + await _seed_raw_log(reconcile_pool, oldest) + await _seed_raw_log(reconcile_pool, middle) + await _seed_raw_log(reconcile_pool, newest) + + # When + limited = await reconcile_passthrough(reconcile_pool, limit=1, since=_at(10)) + filtered = await reconcile_passthrough(reconcile_pool, limit=10, since=_at(11)) + + # Then + assert limited == ReconcileStats(materialized=1) + assert filtered == ReconcileStats(materialized=2) + assert await _event_count(reconcile_pool, before_window.transaction_id) == 0 + assert await _event_count(reconcile_pool, oldest.transaction_id) == 2 + assert await _event_count(reconcile_pool, middle.transaction_id) == 2 + assert await _event_count(reconcile_pool, newest.transaction_id) == 2 + + +async def test_reconcile_counts_a_database_error_and_continues_with_later_transactions( + reconcile_pool: DatabasePool, + monkeypatch: pytest.MonkeyPatch, +) -> None: + ready = _valid_openai_seed("ready-after-error", _at(10)) + broken = _valid_openai_seed("broken", _at(9)) + await _seed_raw_log(reconcile_pool, ready) + await _seed_raw_log(reconcile_pool, broken) + original_materialize = reconcile_module.materialize_transaction + + async def materialize_with_database_error(pool: DatabasePool, transaction_id: str) -> MaterializationResult: + if transaction_id == broken.transaction_id: + raise aiosqlite.OperationalError("database unavailable") + return await original_materialize(pool, transaction_id) + + monkeypatch.setattr(reconcile_module, "materialize_transaction", materialize_with_database_error) + + stats = await reconcile_passthrough(reconcile_pool, limit=10) + + assert stats == ReconcileStats(materialized=1, failed=1) + assert await _event_count(reconcile_pool, ready.transaction_id) == 2 + assert await _event_count(reconcile_pool, broken.transaction_id) == 0 + + +async def test_reconcile_and_live_materialization_produce_one_canonical_turn_when_they_contend( + reconcile_pool: DatabasePool, + monkeypatch: pytest.MonkeyPatch, +) -> None: + transaction = _valid_openai_seed("contended", _at(10)) + await _seed_raw_log(reconcile_pool, transaction) + reconcile_started = anyio.Event() + both_writers_ready = anyio.Event() + writer_count = 0 + reconcile_results: list[ReconcileStats] = [] + live_results: list[MaterializationResult] = [] + original_write = materialize_module.write_canonical_transaction + original_reconcile_materialize = reconcile_module.materialize_transaction + + async def synchronized_write( + pool: DatabasePool, canonical: CanonicalTransaction + ) -> Materialized | AlreadyMaterialized: + nonlocal writer_count + writer_count += 1 + if writer_count == 1: + with anyio.fail_after(1): + await both_writers_ready.wait() + elif writer_count == 2: + both_writers_ready.set() + else: + raise AssertionError("expected exactly two concurrent materialization writers") + return await original_write(pool, canonical) + + async def reconcile_materialize(pool: DatabasePool, transaction_id: str) -> MaterializationResult: + reconcile_started.set() + return await original_reconcile_materialize(pool, transaction_id) + + async def run_reconcile() -> None: + reconcile_results.append(await reconcile_passthrough(reconcile_pool, limit=1)) + + async def run_live() -> None: + live_results.append(await materialize_transaction(reconcile_pool, transaction.transaction_id)) + + monkeypatch.setattr(materialize_module, "write_canonical_transaction", synchronized_write) + monkeypatch.setattr(reconcile_module, "materialize_transaction", reconcile_materialize) + + async with anyio.create_task_group() as task_group: + task_group.start_soon(run_reconcile) + with anyio.fail_after(1): + await reconcile_started.wait() + task_group.start_soon(run_live) + + assert len(reconcile_results) == 1 + assert len(live_results) == 1 + reconcile_stats = reconcile_results[0] + live_result = live_results[0] + assert reconcile_stats.failed == 0 + assert reconcile_stats.skipped_ineligible == 0 + assert reconcile_stats.materialized + reconcile_stats.already_materialized == 1 + assert isinstance(live_result, Materialized | AlreadyMaterialized) + async with reconcile_pool.connection() as conn: + call_count = await conn.fetchval( + "SELECT COUNT(*) FROM conversation_calls WHERE call_id = $1", transaction.transaction_id + ) + event_count = await conn.fetchval( + "SELECT COUNT(*) FROM conversation_events WHERE call_id = $1", transaction.transaction_id + ) + summary = await conn.fetchrow("SELECT * FROM session_summaries WHERE session_id = $1", "session-contended") + + assert call_count == 1 + assert event_count == 2 + assert summary is not None + assert summary["event_count"] == 2 + assert summary["call_count"] == 1 + assert summary["models_used"] == "gpt-4.1" diff --git a/tests/luthien_proxy/unit_tests/passthrough_materialize/test_worker.py b/tests/luthien_proxy/unit_tests/passthrough_materialize/test_worker.py new file mode 100644 index 000000000..9768e4ece --- /dev/null +++ b/tests/luthien_proxy/unit_tests/passthrough_materialize/test_worker.py @@ -0,0 +1,112 @@ +from __future__ import annotations + +import asyncio +from collections.abc import AsyncIterator + +import aiosqlite +import pytest + +import luthien_proxy.passthrough_materialize.worker as worker_module +from luthien_proxy.passthrough_materialize.materialize_types import ReconcileStats +from luthien_proxy.passthrough_materialize.worker import PassthroughReconcileWorker +from luthien_proxy.utils.db import DatabasePool +from luthien_proxy.utils.migration_check import check_migrations + + +@pytest.fixture +async def worker_pool() -> AsyncIterator[DatabasePool]: + pool = DatabasePool("sqlite://:memory:") + await check_migrations(pool) + yield pool + await pool.close() + + +async def test_reconcile_worker_runs_a_sweep_with_its_configured_limit( + worker_pool: DatabasePool, monkeypatch: pytest.MonkeyPatch +) -> None: + # Given + sweep_started = asyncio.Event() + observed_limits: list[int] = [] + + async def reconcile(pool: DatabasePool, *, limit: int) -> ReconcileStats: + assert pool is worker_pool + observed_limits.append(limit) + sweep_started.set() + return ReconcileStats(materialized=1) + + monkeypatch.setattr(worker_module, "reconcile_passthrough", reconcile) + worker = PassthroughReconcileWorker(db_pool=worker_pool, limit=23, interval_seconds=60) + + # When + worker.start() + try: + await asyncio.wait_for(sweep_started.wait(), timeout=1) + finally: + await worker.stop() + + # Then + assert observed_limits == [23] + + +async def test_reconcile_worker_continues_after_a_database_sweep_error( + worker_pool: DatabasePool, monkeypatch: pytest.MonkeyPatch +) -> None: + # Given + original_sleep = asyncio.sleep + second_sweep_started = asyncio.Event() + sweep_count = 0 + + async def reconcile(pool: DatabasePool, *, limit: int) -> ReconcileStats: + nonlocal sweep_count + assert pool is worker_pool + assert limit == 23 + sweep_count += 1 + if sweep_count == 1: + raise aiosqlite.OperationalError("transient database failure") + second_sweep_started.set() + return ReconcileStats() + + async def advance_without_delay(_seconds: float) -> None: + await original_sleep(0) + + monkeypatch.setattr(worker_module, "reconcile_passthrough", reconcile) + monkeypatch.setattr(worker_module.asyncio, "sleep", advance_without_delay) + worker = PassthroughReconcileWorker(db_pool=worker_pool, limit=23, interval_seconds=60) + + # When + worker.start() + try: + await asyncio.wait_for(second_sweep_started.wait(), timeout=1) + finally: + await worker.stop() + + # Then + assert sweep_count >= 2 + + +async def test_reconcile_worker_stop_cancels_an_inflight_sweep( + worker_pool: DatabasePool, monkeypatch: pytest.MonkeyPatch +) -> None: + # Given + sweep_started = asyncio.Event() + + async def reconcile(pool: DatabasePool, *, limit: int) -> ReconcileStats: + assert pool is worker_pool + assert limit == 23 + sweep_started.set() + await asyncio.Event().wait() + return ReconcileStats() + + monkeypatch.setattr(worker_module, "reconcile_passthrough", reconcile) + worker = PassthroughReconcileWorker(db_pool=worker_pool, limit=23, interval_seconds=60) + worker.start() + task = worker._task + assert task is not None + await asyncio.wait_for(sweep_started.wait(), timeout=1) + + # When + await worker.stop() + + # Then + assert task.cancelled() + assert worker._task is None diff --git a/tests/luthien_proxy/unit_tests/request_log/test_recorder.py b/tests/luthien_proxy/unit_tests/request_log/test_recorder.py index ba2154175..bf7c777df 100644 --- a/tests/luthien_proxy/unit_tests/request_log/test_recorder.py +++ b/tests/luthien_proxy/unit_tests/request_log/test_recorder.py @@ -15,7 +15,9 @@ from __future__ import annotations import json +import logging import time +from pathlib import Path from unittest.mock import AsyncMock, MagicMock, patch import aiosqlite @@ -25,13 +27,22 @@ from luthien_proxy.request_log.recorder import ( NoOpRequestLogRecorder, RequestLogRecorder, - _insert_log_row, _PendingLog, create_recorder, + insert_log_row, ) from luthien_proxy.utils.db import DatabasePool, DatabaseWriteError +def _make_transactional_connection() -> MagicMock: + connection = MagicMock() + transaction = MagicMock() + transaction.__aenter__ = AsyncMock(return_value=None) + transaction.__aexit__ = AsyncMock(return_value=None) + connection.transaction = MagicMock(return_value=transaction) + return connection + + class Test_PendingLog: """Tests for _PendingLog dataclass.""" @@ -138,6 +149,120 @@ def test_flush_is_noop(self) -> None: # Should not raise recorder.flush() + def test_noop_ignores_on_commit_callback(self) -> None: + """No-op recorder accepts a callback without scheduling it.""" + callback_transactions: list[str] = [] + + async def on_commit(transaction_id: str) -> None: + callback_transactions.append(transaction_id) + + recorder = NoOpRequestLogRecorder(on_commit=on_commit) + + recorder.flush() + + assert callback_transactions == [] + + +class TestRequestLogPostCommit: + """Tests the durable post-commit callback contract.""" + + @staticmethod + async def _create_request_log_database(database_path: Path) -> DatabasePool: + db_pool = DatabasePool(f"sqlite:///{database_path}") + pool = await db_pool.get_pool() + await pool.execute( + """ + CREATE TABLE request_logs ( + id INTEGER PRIMARY KEY, + transaction_id TEXT NOT NULL, + session_id TEXT, + user_id TEXT, + direction TEXT NOT NULL, + http_method TEXT, + url TEXT, + request_headers TEXT, + request_body TEXT, + response_status INTEGER, + response_headers TEXT, + response_body TEXT, + started_at TEXT, + completed_at TEXT, + duration_ms REAL, + model TEXT, + is_streaming INTEGER, + endpoint TEXT, + error TEXT + ) + """ + ) + return db_pool + + @pytest.mark.asyncio + async def test_on_commit_receives_transaction_id_after_durable_write(self, tmp_path: Path) -> None: + """Callback sees both committed rows and receives the transaction ID.""" + database_path = tmp_path / "request_logs.db" + db_pool = await self._create_request_log_database(database_path) + callback_observations: list[tuple[str, int]] = [] + + async def on_commit(transaction_id: str) -> None: + async with aiosqlite.connect(database_path) as connection: + cursor = await connection.execute("SELECT COUNT(*) FROM request_logs") + row = await cursor.fetchone() + assert row is not None + callback_observations.append((transaction_id, int(row[0]))) + + recorder = RequestLogRecorder(db_pool, "txn-committed", on_commit=on_commit) + try: + await recorder._write_logs() + finally: + await db_pool.close() + + assert callback_observations == [("txn-committed", 2)] + + @pytest.mark.asyncio + async def test_on_commit_is_not_called_when_raw_log_write_fails(self, tmp_path: Path) -> None: + """A failed raw-log write suppresses the post-commit callback.""" + database_path = tmp_path / "missing_request_logs.db" + db_pool = DatabasePool(f"sqlite:///{database_path}") + callback_transactions: list[str] = [] + + async def on_commit(transaction_id: str) -> None: + callback_transactions.append(transaction_id) + + recorder = RequestLogRecorder(db_pool, "txn-failed", on_commit=on_commit) + dropped_writes_before = RequestLogRecorder.dropped_writes + try: + await recorder._write_logs() + finally: + await db_pool.close() + + assert callback_transactions == [] + assert RequestLogRecorder.dropped_writes == dropped_writes_before + 1 + + @pytest.mark.asyncio + async def test_on_commit_exception_does_not_fail_durable_write( + self, tmp_path: Path, caplog: pytest.LogCaptureFixture + ) -> None: + """Callback failures are warned and leave committed rows intact.""" + database_path = tmp_path / "callback_failure.db" + db_pool = await self._create_request_log_database(database_path) + + async def on_commit(transaction_id: str) -> None: + raise RuntimeError(f"callback failed for {transaction_id}") + + recorder = RequestLogRecorder(db_pool, "txn-callback-failed", on_commit=on_commit) + dropped_writes_before = RequestLogRecorder.dropped_writes + try: + with caplog.at_level(logging.WARNING): + await recorder._write_logs() + rows = await (await db_pool.get_pool()).fetch("SELECT transaction_id FROM request_logs") + finally: + await db_pool.close() + + assert len(rows) == 2 + assert RequestLogRecorder.dropped_writes == dropped_writes_before + assert any(record.levelno == logging.WARNING for record in caplog.records) + class TestRequestLogRecorder: """Tests for RequestLogRecorder.""" @@ -381,7 +506,7 @@ def test_flush_handles_no_running_loop(self) -> None: async def test_write_logs_inserts_both_rows(self) -> None: """_write_logs() inserts both inbound and outbound log rows.""" # Create a mock connection that tracks execute calls - mock_conn = AsyncMock() + mock_conn = _make_transactional_connection() mock_conn.execute = AsyncMock() # Create a mock db_pool @@ -415,7 +540,7 @@ async def test_write_logs_inserts_both_rows(self) -> None: @pytest.mark.asyncio async def test_write_logs_constructs_correct_sql(self) -> None: """_write_logs() constructs the expected INSERT statement.""" - mock_conn = AsyncMock() + mock_conn = _make_transactional_connection() mock_conn.execute = AsyncMock() db_pool = MagicMock(spec=DatabasePool) @@ -445,7 +570,7 @@ async def test_write_logs_constructs_correct_sql(self) -> None: @pytest.mark.asyncio async def test_write_logs_serializes_json_fields(self) -> None: """_write_logs() JSON-serializes header and body fields.""" - mock_conn = AsyncMock() + mock_conn = _make_transactional_connection() mock_conn.execute = AsyncMock() db_pool = MagicMock(spec=DatabasePool) @@ -480,7 +605,7 @@ async def test_write_logs_serializes_json_fields(self) -> None: @pytest.mark.asyncio async def test_write_logs_handles_none_json_fields(self) -> None: """_write_logs() passes None for missing JSON fields.""" - mock_conn = AsyncMock() + mock_conn = _make_transactional_connection() mock_conn.execute = AsyncMock() db_pool = MagicMock(spec=DatabasePool) @@ -508,7 +633,7 @@ async def test_write_logs_handles_none_json_fields(self) -> None: @pytest.mark.asyncio async def test_write_logs_catches_and_logs_db_exceptions(self) -> None: """_write_logs() catches DB-specific exceptions and logs them without raising.""" - mock_conn = AsyncMock() + mock_conn = _make_transactional_connection() mock_conn.execute = AsyncMock(side_effect=asyncpg.PostgresError("connection lost")) db_pool = MagicMock(spec=DatabasePool) @@ -533,7 +658,7 @@ async def test_write_logs_catches_and_logs_db_exceptions(self) -> None: @pytest.mark.asyncio async def test_write_logs_wraps_any_exception_as_write_failure(self) -> None: """Any exception from the DB call is treated as a write failure and caught.""" - mock_conn = AsyncMock() + mock_conn = _make_transactional_connection() mock_conn.execute = AsyncMock(side_effect=ValueError("unexpected")) db_pool = MagicMock(spec=DatabasePool) @@ -551,7 +676,7 @@ async def test_write_logs_wraps_any_exception_as_write_failure(self) -> None: @pytest.mark.asyncio async def test_write_logs_catches_os_errors(self) -> None: """_write_logs() catches OSError (network-level failures).""" - mock_conn = AsyncMock() + mock_conn = _make_transactional_connection() mock_conn.execute = AsyncMock(side_effect=OSError("connection refused")) db_pool = MagicMock(spec=DatabasePool) @@ -569,7 +694,7 @@ async def test_write_logs_catches_os_errors(self) -> None: @pytest.mark.asyncio async def test_write_logs_context_manager_cleanup(self) -> None: """_write_logs() properly uses connection context manager for cleanup.""" - mock_conn = AsyncMock() + mock_conn = _make_transactional_connection() mock_conn.execute = AsyncMock() mock_context_mgr = MagicMock() @@ -595,7 +720,7 @@ class TestRequestLogRecorderIntegration: @pytest.mark.asyncio async def test_complete_request_response_cycle(self) -> None: """Test a complete inbound + outbound request/response cycle.""" - mock_conn = AsyncMock() + mock_conn = _make_transactional_connection() mock_conn.execute = AsyncMock() db_pool = MagicMock(spec=DatabasePool) @@ -795,7 +920,7 @@ def test_serialize_body_exactly_at_limit_passes_through(self) -> None: class TestInsertLogRow: - """Tests for _insert_log_row — the DB-agnostic insert helper. + """Tests for insert_log_row — the DB-agnostic insert helper. Verifies that any driver exception (asyncpg, aiosqlite, or generic) is wrapped in DatabaseWriteError with the original exception as .cause. @@ -812,7 +937,7 @@ async def test_asyncpg_error_raises_database_write_error(self) -> None: conn.execute = AsyncMock(side_effect=cause) with pytest.raises(DatabaseWriteError) as exc_info: - await _insert_log_row(conn, self._make_pending(), lambda b: None) + await insert_log_row(conn, self._make_pending(), lambda b: None) assert exc_info.value.cause is cause @@ -824,7 +949,7 @@ async def test_aiosqlite_error_raises_database_write_error(self) -> None: conn.execute = AsyncMock(side_effect=cause) with pytest.raises(DatabaseWriteError) as exc_info: - await _insert_log_row(conn, self._make_pending(), lambda b: None) + await insert_log_row(conn, self._make_pending(), lambda b: None) assert exc_info.value.cause is cause @@ -836,7 +961,7 @@ async def test_generic_exception_raises_database_write_error(self) -> None: conn.execute = AsyncMock(side_effect=cause) with pytest.raises(DatabaseWriteError) as exc_info: - await _insert_log_row(conn, self._make_pending(), lambda b: None) + await insert_log_row(conn, self._make_pending(), lambda b: None) assert exc_info.value.cause is cause @@ -847,7 +972,7 @@ async def test_database_write_error_message_includes_context(self) -> None: conn.execute = AsyncMock(side_effect=OSError("disk full")) with pytest.raises(DatabaseWriteError) as exc_info: - await _insert_log_row(conn, self._make_pending(), lambda b: None) + await insert_log_row(conn, self._make_pending(), lambda b: None) msg = str(exc_info.value) assert "inbound" in msg @@ -859,5 +984,5 @@ async def test_success_does_not_raise(self) -> None: conn = AsyncMock() conn.execute = AsyncMock() - await _insert_log_row(conn, self._make_pending(), lambda b: None) + await insert_log_row(conn, self._make_pending(), lambda b: None) conn.execute.assert_called_once() diff --git a/tests/luthien_proxy/unit_tests/test_config_registry.py b/tests/luthien_proxy/unit_tests/test_config_registry.py index ffb0d4ba4..5935bbcb5 100644 --- a/tests/luthien_proxy/unit_tests/test_config_registry.py +++ b/tests/luthien_proxy/unit_tests/test_config_registry.py @@ -4,7 +4,6 @@ import difflib import importlib.util -import subprocess from pathlib import Path from typing import Any from unittest.mock import AsyncMock, MagicMock @@ -520,50 +519,19 @@ def test_generator_uses_enum_value_not_repr(self): assert "AuthMode.BOTH" not in text def test_env_example_matches_generator(self): - """The .env.example about to land in a commit must match the generator. - - Regression guard for PR #519 (commit 1b28e4d5 accidentally wiped - .env.example from 150 lines to 0, turning main CI red). Also catches - the original "hardcoded SERVICE_VERSION=2.0.0" class of bug: any - ConfigFieldMeta default that drifts from the committed .env.example - fails this test. - - Reads from the git index (`git show :.env.example`), not the working - tree, so that (a) dev_checks.sh regenerating the working tree before - pytest doesn't make the comparison tautological, (b) pre-commit TDD - workflows don't false-positive against a stale HEAD, and (c) unstaged - .env.example edits don't pollute the check. Unstaged config_fields.py - edits DO propagate (intentionally — that module is the input under - test). The generator skips dynamic_default fields like SERVICE_VERSION - so its output is stable across build environments. - """ repo_root = Path(__file__).resolve().parents[3] generator_path = repo_root / "scripts" / "generate_env_example.py" + env_example_path = repo_root / ".env.example" assert generator_path.exists(), f"Generator missing at {generator_path}" + assert env_example_path.exists(), f".env.example missing at {env_example_path}" - # Load the script as a module to call build_env_example_text() in-process. - # Faster than shelling out and produces real tracebacks on generator errors. spec = importlib.util.spec_from_file_location("_generate_env_example", generator_path) assert spec is not None and spec.loader is not None generator = importlib.util.module_from_spec(spec) spec.loader.exec_module(generator) expected = generator.build_env_example_text() - - # See docstring for why we read the index instead of the working tree. - try: - result = subprocess.run( - ["git", "show", ":.env.example"], - capture_output=True, - text=True, - cwd=repo_root, - check=True, - ) - except subprocess.CalledProcessError as exc: - pytest.fail( - f".env.example is not in the git index (cwd={repo_root}). git stderr: {exc.stderr.strip() or ''}" - ) - actual = result.stdout + actual = env_example_path.read_text() if actual != expected: diff = "\n".join( @@ -578,7 +546,6 @@ def test_env_example_matches_generator(self): pytest.fail( "Staged/committed .env.example is out of sync with " "scripts/generate_env_example.py.\n" - "Run: uv run python scripts/generate_env_example.py > .env.example " - "&& git add .env.example\n\n" + "Run: uv run python scripts/generate_env_example.py > .env.example\n\n" f"{diff}" ) diff --git a/tests/luthien_proxy/unit_tests/test_passthrough_capture.py b/tests/luthien_proxy/unit_tests/test_passthrough_capture.py new file mode 100644 index 000000000..d41e4bfa8 --- /dev/null +++ b/tests/luthien_proxy/unit_tests/test_passthrough_capture.py @@ -0,0 +1,144 @@ +from __future__ import annotations + +from luthien_proxy.passthrough_capture import ( + build_passthrough_headers, + build_upstream_url, + client_response_headers, + parse_gemini_model, + parse_openai_model, + reassemble_gemini_json_array_stream, + reassemble_gemini_sse_stream, + reassemble_openai_sse_stream, +) +from luthien_proxy.request_log.sanitize import sanitize_headers, sanitize_url + + +def test_build_passthrough_headers_forwards_client_keys_and_strips_internal_headers() -> None: + # Given + headers = { + "Authorization": "Bearer client-openai-key", + "x-goog-api-key": "client-google-key", + "x-luthien-model": "gpt-4.1", + "Connection": "keep-alive", + "Host": "proxy.local", + "Content-Type": "application/json", + } + + # When + forwarded = build_passthrough_headers(headers.items()) + + # Then + assert forwarded == { + "Authorization": "Bearer client-openai-key", + "x-goog-api-key": "client-google-key", + "Content-Type": "application/json", + } + + +def test_build_upstream_url_preserves_the_client_query_string() -> None: + # Given + base_url = "https://api.openai.com/" + + # When + upstream_url = build_upstream_url(base_url, "v1/chat/completions", "stream=true") + + # Then + assert upstream_url == "https://api.openai.com/v1/chat/completions?stream=true" + + +def test_sanitize_redacts_google_api_key_header_and_key_query_param() -> None: + # Given + headers = {"x-goog-api-key": "google-secret", "x-trace-id": "trace-123"} + url = "https://generativelanguage.googleapis.com/v1beta/models/gemini:generateContent?key=google-secret&alt=sse" + + # When + sanitized_headers = sanitize_headers(headers) + sanitized_url = sanitize_url(url) + + # Then + assert sanitized_headers == {"x-goog-api-key": "[REDACTED]", "x-trace-id": "trace-123"} + assert sanitized_url == ( + "https://generativelanguage.googleapis.com/v1beta/models/gemini:generateContent?key=%5BREDACTED%5D&alt=sse" + ) + assert "google-secret" not in sanitized_url + + +def test_model_parsing_uses_body_for_openai_and_url_for_gemini() -> None: + # Given + openai_body = {"model": "gpt-4.1", "input": "hello"} + gemini_path = "v1beta/models/gemini-2.5-pro:generateContent" + + # When / Then + assert parse_openai_model(openai_body, override=None) == "gpt-4.1" + assert parse_openai_model(openai_body, override="gpt-4.1-mini") == "gpt-4.1-mini" + assert parse_gemini_model(gemini_path, {"model": "fallback"}, override=None) == "gemini-2.5-pro" + assert parse_gemini_model("v1beta/generate", {"model": "fallback"}, override=None) == "fallback" + + +def test_streaming_reassembly_returns_documented_wrappers() -> None: + # Given + openai_chunks = [ + b'data: {"id":"evt-1","output_text":"hel"}\n\n', + b'data: {"id":"evt-2","output_text":"hello"}\n\n', + b"data: [DONE]\n\n", + ] + gemini_json_chunks = [b'[{"text":"hel"},', b'{"text":"hello"}]'] + gemini_sse_chunks = [b'data: {"text":"hel"}\n\n', b'data: {"text":"hello"}\n\n'] + + # When / Then + assert reassemble_openai_sse_stream(openai_chunks) == { + "stream_format": "openai-sse", + "events": [ + {"id": "evt-1", "output_text": "hel"}, + {"id": "evt-2", "output_text": "hello"}, + ], + "final": {"id": "evt-2", "output_text": "hello"}, + } + assert reassemble_gemini_json_array_stream(gemini_json_chunks) == { + "stream_format": "gemini-json-array", + "chunks": [{"text": "hel"}, {"text": "hello"}], + "final": None, + } + assert reassemble_gemini_sse_stream(gemini_sse_chunks) == { + "stream_format": "gemini-sse", + "chunks": [{"text": "hel"}, {"text": "hello"}], + "final": None, + } + + +def test_client_response_headers_strip_encoding_length_and_hop_by_hop_headers() -> None: + # Given + upstream_headers = { + "content-encoding": "gzip", + "Content-Length": "123", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "content-type": "application/json", + "x-request-id": "req-123", + } + + # When + returned_headers = client_response_headers(upstream_headers) + + # Then + assert returned_headers == {"content-type": "application/json", "x-request-id": "req-123"} + + +def test_streaming_reassembly_preserves_malformed_chunks_with_raw_fallback() -> None: + # Given + openai_chunks = [b"data: {not json}\n\n", b'data: {"ok": true}\n\n'] + gemini_array_chunks = [b""] + + # When / Then + assert reassemble_openai_sse_stream(openai_chunks) == { + "stream_format": "openai-sse", + "events": [{"ok": True}], + "raw": 'data: {not json}\n\ndata: {"ok": true}\n\n', + "final": {"ok": True}, + } + assert reassemble_gemini_json_array_stream(gemini_array_chunks) == { + "stream_format": "gemini-json-array", + "chunks": [], + "raw": "", + "final": None, + } diff --git a/tests/luthien_proxy/unit_tests/test_passthrough_routes.py b/tests/luthien_proxy/unit_tests/test_passthrough_routes.py index 6e45f4d27..e2bb985f2 100644 --- a/tests/luthien_proxy/unit_tests/test_passthrough_routes.py +++ b/tests/luthien_proxy/unit_tests/test_passthrough_routes.py @@ -1,144 +1,233 @@ from __future__ import annotations +from collections.abc import Awaitable, Callable from unittest.mock import patch +import httpx import pytest from fastapi import HTTPException +from starlette.requests import Request -from luthien_proxy.passthrough_capture import ( - build_passthrough_headers, - parse_gemini_model, - parse_openai_model, - reassemble_gemini_json_array_stream, - reassemble_gemini_sse_stream, - reassemble_openai_sse_stream, -) +from luthien_proxy.passthrough_capture import JsonObject from luthien_proxy.passthrough_routes import ( - _client_response_headers, + _passthrough, + _RequestPayload, _require_passthrough_enabled, _response_body, + _UpstreamTarget, ) -from luthien_proxy.request_log.sanitize import sanitize_headers, sanitize_url - - -def test_build_passthrough_headers_forwards_client_keys_and_strips_internal_headers() -> None: - # Given - headers = { - "Authorization": "Bearer client-openai-key", - "x-goog-api-key": "client-google-key", - "x-luthien-model": "gpt-4.1", - "Connection": "keep-alive", - "Host": "proxy.local", - "Content-Type": "application/json", - } - # When - forwarded = build_passthrough_headers(headers.items()) - # Then - assert forwarded == { - "Authorization": "Bearer client-openai-key", - "x-goog-api-key": "client-google-key", - "Content-Type": "application/json", - } +class _FakeDatabasePool: + pass + + +class _PassthroughDependencies: + def __init__(self, client: httpx.AsyncClient) -> None: + self.db_pool = _FakeDatabasePool() + self.enable_request_logging = True + self.passthrough_buffered_client = client + self.passthrough_streaming_client = client + + +class _CapturedRecorder: + def __init__(self) -> None: + self.on_commit: Callable[[str], Awaitable[None]] | None = None + self.user_id: str | None = None + + def record_inbound_request( + self, + *, + method: str, + url: str, + headers: dict[str, str], + body: JsonObject, + session_id: str | None = None, + user_id: str | None = None, + model: str | None = None, + is_streaming: bool = False, + endpoint: str | None = None, + ) -> None: + self._inbound_request = (method, url, headers, body, session_id, model, is_streaming, endpoint) + self.user_id = user_id + + def record_outbound_request( + self, + *, + body: JsonObject, + method: str = "POST", + url: str | None = None, + model: str | None = None, + is_streaming: bool = False, + endpoint: str | None = None, + ) -> None: + self._outbound_request = (body, method, url, model, is_streaming, endpoint) + + def record_inbound_response( + self, + *, + status: int, + body: JsonObject | None = None, + headers: dict[str, str] | None = None, + error: str | None = None, + ) -> None: + self._inbound_response = (status, body, headers, error) + + def record_outbound_response( + self, + *, + body: JsonObject | None = None, + status: int = 200, + error: str | None = None, + ) -> None: + self._outbound_response = (body, status, error) + + def flush(self) -> None: + pass + + +class _PassthroughSettings: + def __init__(self, *, materialize_enabled: bool, trust_user_id_header: bool) -> None: + self.passthrough_materialize_enabled = materialize_enabled + self.trust_user_id_header = trust_user_id_header + + +def _make_passthrough_request(headers: list[tuple[bytes, bytes]]) -> Request: + return Request( + { + "type": "http", + "http_version": "1.1", + "method": "POST", + "scheme": "http", + "path": "/openai/v1/chat/completions", + "raw_path": b"/openai/v1/chat/completions", + "query_string": b"", + "headers": headers, + "client": ("testclient", 50000), + "server": ("testserver", 80), + } + ) -def test_sanitize_redacts_google_api_key_header_and_key_query_param() -> None: +async def _invoke_passthrough( + *, + headers: list[tuple[bytes, bytes]], + materialize_enabled: bool, + trust_user_id_header: bool, + is_streaming: bool, +) -> _CapturedRecorder: + recorder = _CapturedRecorder() + + def capture_recorder( + db_pool: _FakeDatabasePool, + transaction_id: str, + enabled: bool, + *, + on_commit: Callable[[str], Awaitable[None]] | None = None, + ) -> _CapturedRecorder: + assert isinstance(db_pool, _FakeDatabasePool) + assert enabled + assert transaction_id + recorder.on_commit = on_commit + return recorder + + transport = httpx.MockTransport(lambda _: httpx.Response(200, json={"id": "response"})) + async with httpx.AsyncClient(transport=transport) as client: + dependencies = _PassthroughDependencies(client) + with ( + patch( + "luthien_proxy.passthrough_recording.get_settings", + return_value=_PassthroughSettings( + materialize_enabled=materialize_enabled, + trust_user_id_header=trust_user_id_header, + ), + ), + patch( + "luthien_proxy.passthrough_recording.create_recorder", + side_effect=capture_recorder, + ), + patch("luthien_proxy.passthrough_routes.get_dependencies", return_value=dependencies), + ): + await _passthrough( + _make_passthrough_request(headers), + _UpstreamTarget( + provider="openai", + path="v1/chat/completions", + base_url="https://upstream.test", + is_streaming=is_streaming, + ), + _RequestPayload(body_bytes=b'{"model":"gpt-4.1"}', body={"model": "gpt-4.1"}), + ) + return recorder + + +@pytest.mark.parametrize("is_streaming", [False, True]) +async def test_passthrough_wires_materialization_callback_when_enabled(is_streaming: bool) -> None: # Given - headers = {"x-goog-api-key": "google-secret", "x-trace-id": "trace-123"} - url = "https://generativelanguage.googleapis.com/v1beta/models/gemini:generateContent?key=google-secret&alt=sse" - - # When - sanitized_headers = sanitize_headers(headers) - sanitized_url = sanitize_url(url) + materialized_transaction_ids: list[str] = [] + + async def capture_materialization(_db_pool: _FakeDatabasePool, transaction_id: str) -> None: + materialized_transaction_ids.append(transaction_id) + + with patch( + "luthien_proxy.passthrough_recording.materialize_transaction", + new=capture_materialization, + ): + # When + recorder = await _invoke_passthrough( + headers=[], + materialize_enabled=True, + trust_user_id_header=False, + is_streaming=is_streaming, + ) + + # Then + assert recorder.on_commit is not None + await recorder.on_commit("transaction-123") + assert materialized_transaction_ids == ["transaction-123"] + + +async def test_passthrough_omits_materialization_callback_when_disabled() -> None: + # Given / When + recorder = await _invoke_passthrough( + headers=[], + materialize_enabled=False, + trust_user_id_header=False, + is_streaming=False, + ) # Then - assert sanitized_headers == {"x-goog-api-key": "[REDACTED]", "x-trace-id": "trace-123"} - assert sanitized_url == ( - "https://generativelanguage.googleapis.com/v1beta/models/gemini:generateContent?key=%5BREDACTED%5D&alt=sse" + assert recorder.on_commit is None + + +@pytest.mark.parametrize( + ("headers", "trust_user_id_header", "expected_user_id"), + [ + ([(b"x-luthien-user-id", b"trusted-user")], True, "trusted-user"), + ( + [(b"authorization", b"Bearer x.eyJzdWIiOiJqd3QtdXNlciJ9.y")], + False, + "jwt-user", + ), + ([(b"x-luthien-user-id", b"untrusted-user")], False, None), + ([], True, None), + ], +) +async def test_passthrough_records_user_id_from_trusted_identity( + headers: list[tuple[bytes, bytes]], + trust_user_id_header: bool, + expected_user_id: str | None, +) -> None: + # Given / When + recorder = await _invoke_passthrough( + headers=headers, + materialize_enabled=False, + trust_user_id_header=trust_user_id_header, + is_streaming=False, ) - assert "google-secret" not in sanitized_url - - -def test_model_parsing_uses_body_for_openai_and_url_for_gemini() -> None: - # Given - openai_body = {"model": "gpt-4.1", "input": "hello"} - gemini_path = "v1beta/models/gemini-2.5-pro:generateContent" - - # When / Then - assert parse_openai_model(openai_body, override=None) == "gpt-4.1" - assert parse_openai_model(openai_body, override="gpt-4.1-mini") == "gpt-4.1-mini" - assert parse_gemini_model(gemini_path, {"model": "fallback"}, override=None) == "gemini-2.5-pro" - assert parse_gemini_model("v1beta/generate", {"model": "fallback"}, override=None) == "fallback" - - -def test_streaming_reassembly_returns_documented_wrappers() -> None: - # Given - openai_chunks = [ - b'data: {"id":"evt-1","output_text":"hel"}\n\n', - b'data: {"id":"evt-2","output_text":"hello"}\n\n', - b"data: [DONE]\n\n", - ] - gemini_json_chunks = [b'[{"text":"hel"},', b'{"text":"hello"}]'] - gemini_sse_chunks = [b'data: {"text":"hel"}\n\n', b'data: {"text":"hello"}\n\n'] - - # When / Then - assert reassemble_openai_sse_stream(openai_chunks) == { - "stream_format": "openai-sse", - "events": [ - {"id": "evt-1", "output_text": "hel"}, - {"id": "evt-2", "output_text": "hello"}, - ], - "final": {"id": "evt-2", "output_text": "hello"}, - } - assert reassemble_gemini_json_array_stream(gemini_json_chunks) == { - "stream_format": "gemini-json-array", - "chunks": [{"text": "hel"}, {"text": "hello"}], - "final": None, - } - assert reassemble_gemini_sse_stream(gemini_sse_chunks) == { - "stream_format": "gemini-sse", - "chunks": [{"text": "hel"}, {"text": "hello"}], - "final": None, - } - - -def test_client_response_headers_strip_encoding_length_and_hop_by_hop_headers() -> None: - # Given - upstream_headers = { - "content-encoding": "gzip", - "Content-Length": "123", - "Transfer-Encoding": "chunked", - "Connection": "keep-alive", - "content-type": "application/json", - "x-request-id": "req-123", - } - - # When - returned_headers = _client_response_headers(upstream_headers) # Then - assert returned_headers == {"content-type": "application/json", "x-request-id": "req-123"} - - -def test_streaming_reassembly_preserves_malformed_chunks_with_raw_fallback() -> None: - # Given - openai_chunks = [b"data: {not json}\n\n", b'data: {"ok": true}\n\n'] - gemini_array_chunks = [b""] - - # When / Then - assert reassemble_openai_sse_stream(openai_chunks) == { - "stream_format": "openai-sse", - "events": [{"ok": True}], - "raw": 'data: {not json}\n\ndata: {"ok": true}\n\n', - "final": {"ok": True}, - } - assert reassemble_gemini_json_array_stream(gemini_array_chunks) == { - "stream_format": "gemini-json-array", - "chunks": [], - "raw": "", - "final": None, - } + assert recorder.user_id == expected_user_id def test_response_body_falls_back_to_replacement_text_for_invalid_utf8() -> None: diff --git a/tests/luthien_proxy/unit_tests/test_settings.py b/tests/luthien_proxy/unit_tests/test_settings.py index 6bc8c6645..5d0c610d5 100644 --- a/tests/luthien_proxy/unit_tests/test_settings.py +++ b/tests/luthien_proxy/unit_tests/test_settings.py @@ -37,6 +37,19 @@ def test_default_otel_disabled(self, monkeypatch): settings = Settings(_env_file=None) assert settings.otel_enabled is False + def test_passthrough_materialization_defaults(self, monkeypatch): + monkeypatch.delenv("PASSTHROUGH_MATERIALIZE_ENABLED", raising=False) + monkeypatch.delenv("PASSTHROUGH_MATERIALIZE_BACKFILL_ENABLED", raising=False) + monkeypatch.delenv("PASSTHROUGH_MATERIALIZE_RECONCILE_INTERVAL_SECONDS", raising=False) + monkeypatch.delenv("PASSTHROUGH_MATERIALIZE_BATCH_SIZE", raising=False) + + settings = Settings(_env_file=None) + + assert settings.passthrough_materialize_enabled is False + assert settings.passthrough_materialize_backfill_enabled is False + assert settings.passthrough_materialize_reconcile_interval_seconds == 300 + assert settings.passthrough_materialize_batch_size == 200 + def test_environment_defaults_to_development(self, monkeypatch): monkeypatch.delenv("ENVIRONMENT", raising=False) monkeypatch.delenv("RAILWAY_SERVICE_NAME", raising=False) @@ -150,6 +163,19 @@ def test_loads_tempo_url(self, monkeypatch): settings = Settings() assert settings.tempo_url == "http://tempo.prod:3200" + def test_loads_passthrough_materialization_from_env(self, monkeypatch): + monkeypatch.setenv("PASSTHROUGH_MATERIALIZE_ENABLED", "true") + monkeypatch.setenv("PASSTHROUGH_MATERIALIZE_BACKFILL_ENABLED", "true") + monkeypatch.setenv("PASSTHROUGH_MATERIALIZE_RECONCILE_INTERVAL_SECONDS", "45") + monkeypatch.setenv("PASSTHROUGH_MATERIALIZE_BATCH_SIZE", "17") + + settings = Settings(_env_file=None) + + assert settings.passthrough_materialize_enabled is True + assert settings.passthrough_materialize_backfill_enabled is True + assert settings.passthrough_materialize_reconcile_interval_seconds == 45 + assert settings.passthrough_materialize_batch_size == 17 + class TestOtelExporterEndpoint: """Test the otel_exporter_otlp_endpoint setting.""" diff --git a/tests/luthien_proxy/unit_tests/test_startup_flow.py b/tests/luthien_proxy/unit_tests/test_startup_flow.py index ab0d6c901..928e4a1fd 100644 --- a/tests/luthien_proxy/unit_tests/test_startup_flow.py +++ b/tests/luthien_proxy/unit_tests/test_startup_flow.py @@ -110,6 +110,64 @@ def test_lifespan_happy_path_sets_all_required_dependencies( assert deps.redis_client is not None assert deps.emitter is not None + @pytest.mark.parametrize(("enabled", "expected_worker_count"), [(False, 0), (True, 1)]) + def test_lifespan_runs_passthrough_reconciliation_only_when_backfill_is_enabled( + self, + enabled, + expected_worker_count, + policy_config_file, + mock_db_pool, + mock_redis_client, + monkeypatch, + ): + # Given + from luthien_proxy import main as main_module + from luthien_proxy.settings import Settings + from luthien_proxy.utils.db import DatabasePool + + settings = Settings.model_construct(passthrough_materialize_backfill_enabled=enabled) + + class WorkerSpy: + def __init__(self, *, db_pool: DatabasePool, limit: int, interval_seconds: int) -> None: + self.db_pool = db_pool + self.limit = limit + self.interval_seconds = interval_seconds + self.started = False + self.stopped = False + workers.append(self) + + def start(self) -> None: + self.started = True + + async def stop(self) -> None: + self.stopped = True + + workers: list[WorkerSpy] = [] + + monkeypatch.setattr(main_module, "get_settings", lambda: settings) + monkeypatch.setattr(main_module, "PassthroughReconcileWorker", WorkerSpy, raising=False) + app = create_app( + api_key="test-key", + admin_key=None, + db_pool=mock_db_pool, + redis_client=mock_redis_client, + startup_policy_path=policy_config_file, + ) + + # When + with TestClient(app): + pass + + # Then + assert len(workers) == expected_worker_count + if enabled: + worker = workers[0] + assert worker.db_pool is mock_db_pool + assert worker.limit == settings.passthrough_materialize_batch_size + assert worker.interval_seconds == settings.passthrough_materialize_reconcile_interval_seconds + assert worker.started + assert worker.stopped + def test_lifespan_does_not_close_db_pool_on_shutdown(self, policy_config_file, mock_db_pool, mock_redis_client): """db_pool lifetime is owned by the caller, not the app lifespan.""" app = create_app( diff --git a/uv.lock b/uv.lock index 7d3e41de4..2d303396a 100644 --- a/uv.lock +++ b/uv.lock @@ -535,6 +535,45 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ee/45/b82e3c16be2182bff01179db177fe144d58b5dc787a7d4492c6ed8b9317f/frozenlist-1.7.0-py3-none-any.whl", hash = "sha256:9a5af342e34f7e97caf8c995864c7a396418ae2859cc6fdf1b1073020d516a7e", size = 13106, upload-time = "2025-06-09T23:02:34.204Z" }, ] +[[package]] +name = "google-auth" +version = "2.55.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cryptography" }, + { name = "pyasn1-modules" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/79/b9/e370d86fea3da13ec0256df30323dd26c0cb9c8c85f0c6ec42ac9df0106b/google_auth-2.55.2.tar.gz", hash = "sha256:97ae7790ff740f2bc9db60eb864a7804f4ac19f5f02c38b3d942f2fea6e9b9ae", size = 361414, upload-time = "2026-07-07T18:43:21.227Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/c6/02eb5a337ac316a4c30c012e747bad5cea36e1a876efecdf80865541f7d8/google_auth-2.55.2-py3-none-any.whl", hash = "sha256:d715f265f2cafc6a5f1bf0dc19870d20e3119f6f6682785a250bce3d03d38a3b", size = 256778, upload-time = "2026-07-07T18:43:19.52Z" }, +] + +[package.optional-dependencies] +requests = [ + { name = "requests" }, +] + +[[package]] +name = "google-genai" +version = "2.11.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "distro" }, + { name = "google-auth", extra = ["requests"] }, + { name = "httpx" }, + { name = "pydantic" }, + { name = "requests" }, + { name = "sniffio" }, + { name = "tenacity" }, + { name = "typing-extensions" }, + { name = "websockets" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a6/01/e7b5f3aac89200c78318ed7643401e7f5ed3131b0cd353c07483606b1e61/google_genai-2.11.0.tar.gz", hash = "sha256:4c5e524d24b145c96be327f9a7f8f04b0fe4efee0533877795e9848afed01749", size = 622366, upload-time = "2026-07-09T17:49:43.862Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/93/ef/d296c23390160a8b0b1dafb36dd3cb36a39ed40c81cd27e04e6233334186/google_genai-2.11.0-py3-none-any.whl", hash = "sha256:5bc8186100e1d34d691fbe0cba392b7e04e98d286ca952323a6672d054accf95", size = 984162, upload-time = "2026-07-09T17:49:42.15Z" }, +] + [[package]] name = "googleapis-common-protos" version = "1.70.0" @@ -793,8 +832,10 @@ dependencies = [ { name = "click" }, { name = "cryptography" }, { name = "fastapi" }, + { name = "google-genai" }, { name = "httpx" }, { name = "jsonschema" }, + { name = "openai" }, { name = "opentelemetry-api" }, { name = "opentelemetry-exporter-otlp-proto-grpc" }, { name = "opentelemetry-exporter-otlp-proto-http" }, @@ -840,8 +881,10 @@ requires-dist = [ { name = "click", specifier = ">=8.1.0" }, { name = "cryptography", specifier = ">=44.0.0" }, { name = "fastapi", specifier = ">=0.115.0" }, + { name = "google-genai", specifier = ">=1.33.0" }, { name = "httpx", specifier = ">=0.28.1" }, { name = "jsonschema", specifier = ">=4.17.0" }, + { name = "openai", specifier = ">=2.11.0" }, { name = "opentelemetry-api", specifier = ">=1.20.0" }, { name = "opentelemetry-exporter-otlp-proto-grpc", specifier = ">=1.20.0" }, { name = "opentelemetry-exporter-otlp-proto-http", specifier = ">=1.20.0" }, @@ -979,6 +1022,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/17/66/1ed71f1f529b8ca727d42c7ceb9db0bef145ce4a13dfc86fb50aa44f3be6/nodejs_wheel_binaries-24.16.0-py2.py3-none-win_arm64.whl", hash = "sha256:8308940b5edd0a50dc5267ea36ba21c9f668e83fe0d9f293937174d3a7e31c36", size = 39714528, upload-time = "2026-05-30T16:52:06.421Z" }, ] +[[package]] +name = "openai" +version = "2.45.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "distro" }, + { name = "httpx" }, + { name = "jiter" }, + { name = "pydantic" }, + { name = "sniffio" }, + { name = "tqdm" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/78/60/d4219875289b11d2c2f7da93c36283da224a2e55865ed865ab64e0ce9217/openai-2.45.0.tar.gz", hash = "sha256:10d34ca9c5643bce775852fddbfc172505cb1d4de1ccd101696c3ecff358765d", size = 1109653, upload-time = "2026-07-09T18:02:44.091Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f1/b0/2291689e3ec4723fbf5bbf3b54afcd7b160f9ddc98ca7aedfd0132af5677/openai-2.45.0-py3-none-any.whl", hash = "sha256:5df105f5f8c9b711fcb9d06d2d3888cebc82506db216484c14a4e53cdf651777", size = 1629470, upload-time = "2026-07-09T18:02:42.21Z" }, +] + [[package]] name = "opentelemetry-api" version = "1.38.0" @@ -1261,6 +1323,27 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/44/b0/a73c195a56eb6b92e937a5ca58521a5c3346fb233345adc80fd3e2f542e2/psycopg-3.2.9-py3-none-any.whl", hash = "sha256:01a8dadccdaac2123c916208c96e06631641c0566b22005493f09663c7a8d3b6", size = 202705, upload-time = "2025-05-13T16:06:26.584Z" }, ] +[[package]] +name = "pyasn1" +version = "0.6.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a4/9a/23310166d960def5897e91fe20e5b724601b02a22e84ba1f94232c0b7f67/pyasn1-0.6.4.tar.gz", hash = "sha256:9c447d8431c947fe4c8febc4ed9e760bc29011a5b01e5c74b67025bd9fb8ce81", size = 151262, upload-time = "2026-07-09T01:12:33.988Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9a/3b/6163796d69c3977d1e4287bea4a6979161cbbdd170ebb430511e8e1999ce/pyasn1-0.6.4-py3-none-any.whl", hash = "sha256:deda9277cfd454080ec40b207fb6df82206a3a2688735233cdcd8d3d565f088b", size = 84410, upload-time = "2026-07-09T01:12:32.92Z" }, +] + +[[package]] +name = "pyasn1-modules" +version = "0.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyasn1" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e9/e6/78ebbb10a8c8e4b61a59249394a4a594c1a7af95593dc933a349c8d00964/pyasn1_modules-0.4.2.tar.gz", hash = "sha256:677091de870a80aae844b1ca6134f54652fa2c8c5a52aa396440ac3106e941e6", size = 307892, upload-time = "2025-03-28T02:41:22.17Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/47/8d/d529b5d697919ba8c11ad626e835d4039be708a35b0d22de83a269a6682c/pyasn1_modules-0.4.2-py3-none-any.whl", hash = "sha256:29253a9207ce32b64c3ac6600edc75368f98473906e8fd1043bd6b5b1de2c14a", size = 181259, upload-time = "2025-03-28T02:41:19.028Z" }, +] + [[package]] name = "pycparser" version = "2.22" @@ -1272,7 +1355,7 @@ wheels = [ [[package]] name = "pydantic" -version = "2.11.7" +version = "2.13.4" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "annotated-types" }, @@ -1280,37 +1363,65 @@ dependencies = [ { name = "typing-extensions" }, { name = "typing-inspection" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/00/dd/4325abf92c39ba8623b5af936ddb36ffcfe0beae70405d456ab1fb2f5b8c/pydantic-2.11.7.tar.gz", hash = "sha256:d989c3c6cb79469287b1569f7447a17848c998458d49ebe294e975b9baf0f0db", size = 788350, upload-time = "2025-06-14T08:33:17.137Z" } +sdist = { url = "https://files.pythonhosted.org/packages/18/a5/b60d21ac674192f8ab0ba4e9fd860690f9b4a6e51ca5df118733b487d8d6/pydantic-2.13.4.tar.gz", hash = "sha256:c40756b57adaa8b1efeeced5c196f3f3b7c435f90e84ea7f443901bec8099ef6", size = 844775, upload-time = "2026-05-06T13:43:05.343Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/6a/c0/ec2b1c8712ca690e5d61979dee872603e92b8a32f94cc1b72d53beab008a/pydantic-2.11.7-py3-none-any.whl", hash = "sha256:dde5df002701f6de26248661f6835bbe296a47bf73990135c7d07ce741b9623b", size = 444782, upload-time = "2025-06-14T08:33:14.905Z" }, + { url = "https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl", hash = "sha256:45a282cde31d808236fd7ea9d919b128653c8b38b393d1c4ab335c62924d9aba", size = 472262, upload-time = "2026-05-06T13:43:02.641Z" }, ] [[package]] name = "pydantic-core" -version = "2.33.2" +version = "2.46.4" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ad/88/5f2260bdfae97aabf98f1778d43f69574390ad787afb646292a638c923d4/pydantic_core-2.33.2.tar.gz", hash = "sha256:7cb8bc3605c29176e1b105350d2e6474142d7c1bd1d9327c4a9bdb46bf827acc", size = 435195, upload-time = "2025-04-23T18:33:52.104Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/46/8c/99040727b41f56616573a28771b1bfa08a3d3fe74d3d513f01251f79f172/pydantic_core-2.33.2-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:1082dd3e2d7109ad8b7da48e1d4710c8d06c253cbc4a27c1cff4fbcaa97a9e3f", size = 2015688, upload-time = "2025-04-23T18:31:53.175Z" }, - { url = "https://files.pythonhosted.org/packages/3a/cc/5999d1eb705a6cefc31f0b4a90e9f7fc400539b1a1030529700cc1b51838/pydantic_core-2.33.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f517ca031dfc037a9c07e748cefd8d96235088b83b4f4ba8939105d20fa1dcd6", size = 1844808, upload-time = "2025-04-23T18:31:54.79Z" }, - { url = "https://files.pythonhosted.org/packages/6f/5e/a0a7b8885c98889a18b6e376f344da1ef323d270b44edf8174d6bce4d622/pydantic_core-2.33.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0a9f2c9dd19656823cb8250b0724ee9c60a82f3cdf68a080979d13092a3b0fef", size = 1885580, upload-time = "2025-04-23T18:31:57.393Z" }, - { url = "https://files.pythonhosted.org/packages/3b/2a/953581f343c7d11a304581156618c3f592435523dd9d79865903272c256a/pydantic_core-2.33.2-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2b0a451c263b01acebe51895bfb0e1cc842a5c666efe06cdf13846c7418caa9a", size = 1973859, upload-time = "2025-04-23T18:31:59.065Z" }, - { url = "https://files.pythonhosted.org/packages/e6/55/f1a813904771c03a3f97f676c62cca0c0a4138654107c1b61f19c644868b/pydantic_core-2.33.2-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ea40a64d23faa25e62a70ad163571c0b342b8bf66d5fa612ac0dec4f069d916", size = 2120810, upload-time = "2025-04-23T18:32:00.78Z" }, - { url = "https://files.pythonhosted.org/packages/aa/c3/053389835a996e18853ba107a63caae0b9deb4a276c6b472931ea9ae6e48/pydantic_core-2.33.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0fb2d542b4d66f9470e8065c5469ec676978d625a8b7a363f07d9a501a9cb36a", size = 2676498, upload-time = "2025-04-23T18:32:02.418Z" }, - { url = "https://files.pythonhosted.org/packages/eb/3c/f4abd740877a35abade05e437245b192f9d0ffb48bbbbd708df33d3cda37/pydantic_core-2.33.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9fdac5d6ffa1b5a83bca06ffe7583f5576555e6c8b3a91fbd25ea7780f825f7d", size = 2000611, upload-time = "2025-04-23T18:32:04.152Z" }, - { url = "https://files.pythonhosted.org/packages/59/a7/63ef2fed1837d1121a894d0ce88439fe3e3b3e48c7543b2a4479eb99c2bd/pydantic_core-2.33.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:04a1a413977ab517154eebb2d326da71638271477d6ad87a769102f7c2488c56", size = 2107924, upload-time = "2025-04-23T18:32:06.129Z" }, - { url = "https://files.pythonhosted.org/packages/04/8f/2551964ef045669801675f1cfc3b0d74147f4901c3ffa42be2ddb1f0efc4/pydantic_core-2.33.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:c8e7af2f4e0194c22b5b37205bfb293d166a7344a5b0d0eaccebc376546d77d5", size = 2063196, upload-time = "2025-04-23T18:32:08.178Z" }, - { url = "https://files.pythonhosted.org/packages/26/bd/d9602777e77fc6dbb0c7db9ad356e9a985825547dce5ad1d30ee04903918/pydantic_core-2.33.2-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:5c92edd15cd58b3c2d34873597a1e20f13094f59cf88068adb18947df5455b4e", size = 2236389, upload-time = "2025-04-23T18:32:10.242Z" }, - { url = "https://files.pythonhosted.org/packages/42/db/0e950daa7e2230423ab342ae918a794964b053bec24ba8af013fc7c94846/pydantic_core-2.33.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:65132b7b4a1c0beded5e057324b7e16e10910c106d43675d9bd87d4f38dde162", size = 2239223, upload-time = "2025-04-23T18:32:12.382Z" }, - { url = "https://files.pythonhosted.org/packages/58/4d/4f937099c545a8a17eb52cb67fe0447fd9a373b348ccfa9a87f141eeb00f/pydantic_core-2.33.2-cp313-cp313-win32.whl", hash = "sha256:52fb90784e0a242bb96ec53f42196a17278855b0f31ac7c3cc6f5c1ec4811849", size = 1900473, upload-time = "2025-04-23T18:32:14.034Z" }, - { url = "https://files.pythonhosted.org/packages/a0/75/4a0a9bac998d78d889def5e4ef2b065acba8cae8c93696906c3a91f310ca/pydantic_core-2.33.2-cp313-cp313-win_amd64.whl", hash = "sha256:c083a3bdd5a93dfe480f1125926afcdbf2917ae714bdb80b36d34318b2bec5d9", size = 1955269, upload-time = "2025-04-23T18:32:15.783Z" }, - { url = "https://files.pythonhosted.org/packages/f9/86/1beda0576969592f1497b4ce8e7bc8cbdf614c352426271b1b10d5f0aa64/pydantic_core-2.33.2-cp313-cp313-win_arm64.whl", hash = "sha256:e80b087132752f6b3d714f041ccf74403799d3b23a72722ea2e6ba2e892555b9", size = 1893921, upload-time = "2025-04-23T18:32:18.473Z" }, - { url = "https://files.pythonhosted.org/packages/a4/7d/e09391c2eebeab681df2b74bfe6c43422fffede8dc74187b2b0bf6fd7571/pydantic_core-2.33.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:61c18fba8e5e9db3ab908620af374db0ac1baa69f0f32df4f61ae23f15e586ac", size = 1806162, upload-time = "2025-04-23T18:32:20.188Z" }, - { url = "https://files.pythonhosted.org/packages/f1/3d/847b6b1fed9f8ed3bb95a9ad04fbd0b212e832d4f0f50ff4d9ee5a9f15cf/pydantic_core-2.33.2-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95237e53bb015f67b63c91af7518a62a8660376a6a0db19b89acc77a4d6199f5", size = 1981560, upload-time = "2025-04-23T18:32:22.354Z" }, - { url = "https://files.pythonhosted.org/packages/6f/9a/e73262f6c6656262b5fdd723ad90f518f579b7bc8622e43a942eec53c938/pydantic_core-2.33.2-cp313-cp313t-win_amd64.whl", hash = "sha256:c2fc0a768ef76c15ab9238afa6da7f69895bb5d1ee83aeea2e3509af4472d0b9", size = 1935777, upload-time = "2025-04-23T18:32:25.088Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/9d/56/921726b776ace8d8f5db44c4ef961006580d91dc52b803c489fafd1aa249/pydantic_core-2.46.4.tar.gz", hash = "sha256:62f875393d7f270851f20523dd2e29f082bcc82292d66db2b64ea71f64b6e1c1", size = 471464, upload-time = "2026-05-06T13:37:06.98Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/51/a2/5d30b469c5267a17b39dec53208222f76a8d351dfac4af661888c5aee77d/pydantic_core-2.46.4-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:5d5902252db0d3cedf8d4a1bc68f70eeb430f7e4c7104c8c476753519b423008", size = 2106306, upload-time = "2026-05-06T13:37:48.029Z" }, + { url = "https://files.pythonhosted.org/packages/c1/81/4fa520eaffa8bd7d1525e644cd6d39e7d60b1592bc5b516693c7340b50f1/pydantic_core-2.46.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:c94f0688e7b8d0a67abf40e57a7eaaecd17cc9586706a31b76c031f63df052b4", size = 1951906, upload-time = "2026-05-06T13:37:17.012Z" }, + { url = "https://files.pythonhosted.org/packages/03/d5/fd02da45b659668b05923b17ba3a0100a0a3d5541e3bd8fcc4ecb711309e/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f027324c56cd5406ca49c124b0db10e56c69064fec039acc571c29020cc87c76", size = 1976802, upload-time = "2026-05-06T13:37:35.113Z" }, + { url = "https://files.pythonhosted.org/packages/21/f2/95727e1368be3d3ed485eaab7adbd7dda408f33f7a36e8b48e0144002b91/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e739fee756ba1010f8bcccb534252e85a35fe45ae92c295a06059ce58b74ccd3", size = 2052446, upload-time = "2026-05-06T13:37:12.313Z" }, + { url = "https://files.pythonhosted.org/packages/9c/86/5d99feea3f77c7234b8718075b23db11532773c1a0dbd9b9490215dc2eeb/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9d56801be94b86a9da183e5f3766e6310752b99ff647e38b09a9500d88e46e76", size = 2232757, upload-time = "2026-05-06T13:39:01.149Z" }, + { url = "https://files.pythonhosted.org/packages/d2/3a/508ac615935ef7588cf6d9e9b91309fdc2da751af865e02a9098de88258c/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2412e734dcb48da14d4e4006b82b46b74f2518b8a26ee7e58c6844a6cd6d03c4", size = 2309275, upload-time = "2026-05-06T13:37:41.406Z" }, + { url = "https://files.pythonhosted.org/packages/07/f8/41db9de19d7987d6b04715a02b3b40aea467000275d9d758ffaa31af7d50/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9551187363ffc0de2a00b2e47c25aeaeb1020b69b668762966df15fc5659dd5a", size = 2094467, upload-time = "2026-05-06T13:39:18.847Z" }, + { url = "https://files.pythonhosted.org/packages/2c/e2/f35033184cb11d0052daf4416e8e10a502ea2ac006fc4f459aee872727d1/pydantic_core-2.46.4-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:0186750b482eefa11d7f435892b09c5c606193ef3375bcf94aa00ae6bfb66262", size = 2134417, upload-time = "2026-05-06T13:40:17.944Z" }, + { url = "https://files.pythonhosted.org/packages/7e/7b/6ceeb1cc90e193862f444ebe373d8fdf613f0a82572dde03fb10734c6c71/pydantic_core-2.46.4-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5855698a4856556d86e8e6cd8434bc3ac0314ee8e12089ae0e143f64c6256e4e", size = 2179782, upload-time = "2026-05-06T13:40:32.618Z" }, + { url = "https://files.pythonhosted.org/packages/5a/f2/c8d7773ede6af08036423a00ae0ceffce266c3c52a096c435d68c896083f/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:cbaf13819775b7f769bf4a1f066cb6df7a28d4480081a589828ef190226881cd", size = 2188782, upload-time = "2026-05-06T13:36:51.018Z" }, + { url = "https://files.pythonhosted.org/packages/59/31/0c864784e31f09f05cdd87606f08923b9c9e7f6e51dd27f20f62f975ce9f/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:633147d34cf4550417f12e2b1a0383973bdf5cdfde212cb09e9a581cf10820be", size = 2328334, upload-time = "2026-05-06T13:40:37.764Z" }, + { url = "https://files.pythonhosted.org/packages/c2/eb/4f6c8a41efa30baa755590f4141abf3a8c370fab610915733e74134a7270/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:82cf5301172168103724d49a1444d3378cb20cdee30b116a1bd6031236298a5d", size = 2372986, upload-time = "2026-05-06T13:39:34.152Z" }, + { url = "https://files.pythonhosted.org/packages/5b/24/b375a480d53113860c299764bfe9f349a3dc9108b3adc0d7f0d786492ebf/pydantic_core-2.46.4-cp313-cp313-win32.whl", hash = "sha256:9fa8ae11da9e2b3126c6426f147e0fba88d96d65921799bb30c6abd1cb2c97fb", size = 1973693, upload-time = "2026-05-06T13:37:55.072Z" }, + { url = "https://files.pythonhosted.org/packages/7e/e8/cff247591966f2d22ec8c003cd7587e27b7ba7b81ab2fb888e3ab75dc285/pydantic_core-2.46.4-cp313-cp313-win_amd64.whl", hash = "sha256:6b3ace8194b0e5204818c92802dcdca7fc6d88aabbb799d7c795540d9cd6d292", size = 2071819, upload-time = "2026-05-06T13:38:49.139Z" }, + { url = "https://files.pythonhosted.org/packages/c6/1a/f4aee670d5670e9e148e0c82c7db98d780be566c6e6a97ee8035528ca0b3/pydantic_core-2.46.4-cp313-cp313-win_arm64.whl", hash = "sha256:184c081504d17f1c1066e430e117142b2c77d9448a97f7b65c6ac9fd9aee238d", size = 2027411, upload-time = "2026-05-06T13:40:45.796Z" }, + { url = "https://files.pythonhosted.org/packages/8d/74/228a26ddad29c6672b805d9fd78e8d251cd04004fa7eed0e622096cd0250/pydantic_core-2.46.4-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:428e04521a40150c85216fc8b85e8d39fece235a9cf5e383761238c7fa9b96fb", size = 2102079, upload-time = "2026-05-06T13:38:41.019Z" }, + { url = "https://files.pythonhosted.org/packages/ad/1f/8970b150a4b4365623ae00fc88603491f763c627311ae8031e3111356d6e/pydantic_core-2.46.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:23ace664830ee0bfe014a0c7bc248b1f7f25ed7ad103852c317624a1083af462", size = 1952179, upload-time = "2026-05-06T13:36:59.812Z" }, + { url = "https://files.pythonhosted.org/packages/95/30/5211a831ae054928054b2f79731661087a2bc5c01e825c672b3a4a8f1b3e/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ce5c1d2a8b27468f433ca974829c44060b8097eedc39933e3c206a90ee49c4a9", size = 1978926, upload-time = "2026-05-06T13:37:39.933Z" }, + { url = "https://files.pythonhosted.org/packages/57/e9/689668733b1eb67adeef047db3c2e8788fcf65a7fd9c9e2b46b7744fe245/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7283d57845ecf5a163403eb0702dfc220cc4fbdd18919cb5ccea4f95ee1cdab4", size = 2046785, upload-time = "2026-05-06T13:38:01.995Z" }, + { url = "https://files.pythonhosted.org/packages/60/d9/6715260422ff50a2109878fd24d948a6c3446bb2664f34ee78cd972b3acd/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8daafc69c93ee8a0204506a3b6b30f586ef54028f52aeeeb5c4cfc5184fd5914", size = 2228733, upload-time = "2026-05-06T13:40:50.371Z" }, + { url = "https://files.pythonhosted.org/packages/18/ae/fdb2f64316afca925640f8e70bb1a564b0ec2721c1389e25b8eb4bf9a299/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd2213145bcc2ba85884d0ac63d222fece9209678f77b9b4d76f054c561adb28", size = 2307534, upload-time = "2026-05-06T13:37:21.531Z" }, + { url = "https://files.pythonhosted.org/packages/89/1d/8eff589b45bb8190a9d12c49cfad0f176a5cbd1534908a6b5125e2886239/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a5f930472650a82629163023e630d160863fce524c616f4e5186e5de9d9a49b", size = 2099732, upload-time = "2026-05-06T13:39:31.942Z" }, + { url = "https://files.pythonhosted.org/packages/06/d5/ee5a3366637fee41dee51a1fc91562dcf12ddbc68fda34e6b253da2324bb/pydantic_core-2.46.4-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:c1b3f518abeca3aa13c712fd202306e145abf59a18b094a6bafb2d2bbf59192c", size = 2129627, upload-time = "2026-05-06T13:37:25.033Z" }, + { url = "https://files.pythonhosted.org/packages/94/33/2414be571d2c6a6c4d08be21f9292b6d3fdb08949a97b6dfe985017821db/pydantic_core-2.46.4-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1a7dd0b3ee80d90150e3495a3a13ac34dbcbfd4f012996a6a1d8900e91b5c0fb", size = 2179141, upload-time = "2026-05-06T13:37:14.046Z" }, + { url = "https://files.pythonhosted.org/packages/7b/79/7daa95be995be0eecc4cf75064cb33f9bbbfe3fe0158caf2f0d4a996a5c7/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:3fb702cd90b0446a3a1c5e470bfa0dd23c0233b676a9099ddcc964fa6ca13898", size = 2184325, upload-time = "2026-05-06T13:36:53.615Z" }, + { url = "https://files.pythonhosted.org/packages/9f/cb/d0a382f5c0de8a222dc61c65348e0ce831b1f68e0a018450d31c2cace3a5/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:b8458003118a712e66286df6a707db01c52c0f52f7db8e4a38f0da1d3b94fc4e", size = 2323990, upload-time = "2026-05-06T13:40:29.971Z" }, + { url = "https://files.pythonhosted.org/packages/05/db/d9ba624cc4a5aced1598e88c04fdbd8310c8a69b9d38b9a3d39ce3a61ed7/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:372429a130e469c9cd698925ce5fc50940b7a1336b0d82038e63d5bbc4edc519", size = 2369978, upload-time = "2026-05-06T13:37:23.027Z" }, + { url = "https://files.pythonhosted.org/packages/f2/20/d15df15ba918c423461905802bfd2981c3af0bfa0e40d05e13edbfa48bc3/pydantic_core-2.46.4-cp314-cp314-win32.whl", hash = "sha256:85bb3611ff1802f3ee7fdd7dbff26b56f343fb432d57a4728fdd49b6ef35e2f4", size = 1966354, upload-time = "2026-05-06T13:38:03.499Z" }, + { url = "https://files.pythonhosted.org/packages/fc/b6/6b8de4c0a7d7ab3004c439c80c5c1e0a3e8d78bbae19379b01960383d9e5/pydantic_core-2.46.4-cp314-cp314-win_amd64.whl", hash = "sha256:811ff8e9c313ab425368bcbb36e5c4ebd7108c2bbf4e4089cfbb0b01eff63fac", size = 2072238, upload-time = "2026-05-06T13:39:40.807Z" }, + { url = "https://files.pythonhosted.org/packages/32/36/51eb763beec1f4cf59b1db243a7dcc39cbb41230f050a09b9d69faaf0a48/pydantic_core-2.46.4-cp314-cp314-win_arm64.whl", hash = "sha256:bfec22eab3c8cc2ceec0248aec886624116dc079afa027ecc8ad4a7e62010f8a", size = 2018251, upload-time = "2026-05-06T13:37:26.72Z" }, + { url = "https://files.pythonhosted.org/packages/e8/91/855af51d625b23aa987116a19e231d2aaef9c4a415273ddc189b79a45fee/pydantic_core-2.46.4-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:af8244b2bef6aaad6d92cda81372de7f8c8d36c9f0c3ea36e827c60e7d9467a0", size = 2099593, upload-time = "2026-05-06T13:39:47.682Z" }, + { url = "https://files.pythonhosted.org/packages/fb/1b/8784a54c65edb5f49f0a14d6977cf1b209bba85a4c77445b255c2de58ab3/pydantic_core-2.46.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:5a4330cdbc57162e4b3aa303f588ba752257694c9c9be3e7ebb11b4aca659b5d", size = 1935226, upload-time = "2026-05-06T13:40:40.428Z" }, + { url = "https://files.pythonhosted.org/packages/e8/e7/1955d28d1afc56dd4b3ad7cc0cf39df1b9852964cf16e5d13912756d6d6b/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:29c61fc04a3d840155ff08e475a04809278972fe6aef51e2720554e96367e34b", size = 1974605, upload-time = "2026-05-06T13:37:32.029Z" }, + { url = "https://files.pythonhosted.org/packages/93/e2/3fedbf0ba7a22850e6e9fd78117f1c0f10f950182344d8a6c535d468fdd8/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c50f2528cf200c5eed56faf3f4e22fcd5f38c157a8b78576e6ba3168ec35f000", size = 2030777, upload-time = "2026-05-06T13:38:55.239Z" }, + { url = "https://files.pythonhosted.org/packages/f8/61/46be275fcaaba0b4f5b9669dd852267ce1ff616592dccf7a7845588df091/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0cbe8b01f948de4286c74cdd6c667aceb38f5c1e26f0693b3983d9d74887c65e", size = 2236641, upload-time = "2026-05-06T13:37:08.096Z" }, + { url = "https://files.pythonhosted.org/packages/60/db/12e93e46a8bac9988be3c016860f83293daea8c716c029c9ace279036f2f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:617d7e2ca7dcb8c5cf6bcb8c59b8832c94b36196bbf1cbd1bfb56ed341905edd", size = 2286404, upload-time = "2026-05-06T13:40:20.221Z" }, + { url = "https://files.pythonhosted.org/packages/e2/4a/4d8b19008f38d31c53b8219cfedc2e3d5de5fe99d90076b7e767de29274f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7027560ee92211647d0d34e3f7cd6f50da56399d26a9c8ad0da286d3869a53f3", size = 2109219, upload-time = "2026-05-06T13:38:12.153Z" }, + { url = "https://files.pythonhosted.org/packages/88/70/3cbc40978fefb7bb09c6708d40d4ad1a5d70fd7213c3d17f971de868ec1f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:f99626688942fb746e545232e7726926f3be91b5975f8b55327665fafda991c7", size = 2110594, upload-time = "2026-05-06T13:40:02.971Z" }, + { url = "https://files.pythonhosted.org/packages/9d/20/b8d36736216e29491125531685b2f9e61aa5b4b2599893f8268551da3338/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fc3e9034a63de20e15e8ade85358bc6efc614008cab72898b4b4952bea0509ff", size = 2159542, upload-time = "2026-05-06T13:39:27.506Z" }, + { url = "https://files.pythonhosted.org/packages/1d/a2/367df868eb584dacf6bf82a389272406d7178e301c4ac82545ab98bc2dd9/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:97e7cf2be5c77b7d1a9713a05605d49460d02c6078d38d8bef3cbe323c548424", size = 2168146, upload-time = "2026-05-06T13:38:31.93Z" }, + { url = "https://files.pythonhosted.org/packages/c1/b8/4460f77f7e201893f649a29ab355dddd3beee8a97bcb1a320db414f9a06e/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:3bf92c5d0e00fefaab325a4d27828fe6b6e2a21848686b5b60d2d9eeb09d76c6", size = 2306309, upload-time = "2026-05-06T13:37:44.717Z" }, + { url = "https://files.pythonhosted.org/packages/64/c4/be2639293acd87dc8ddbcec41a73cee9b2ebf996fe6d892a1a74e88ad3f7/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:3ecbc122d18468d06ca279dc26a8c2e2d5acb10943bb35e36ae92096dc3b5565", size = 2369736, upload-time = "2026-05-06T13:37:05.645Z" }, + { url = "https://files.pythonhosted.org/packages/30/a6/9f9f380dbb301f67023bf8f707aaa75daadf84f7152d95c410fd7e81d994/pydantic_core-2.46.4-cp314-cp314t-win32.whl", hash = "sha256:e846ae7835bf0703ae43f534ab79a867146dadd59dc9ca5c8b53d5c8f7c9ef02", size = 1955575, upload-time = "2026-05-06T13:38:51.116Z" }, + { url = "https://files.pythonhosted.org/packages/40/1f/f1eb9eb350e795d1af8586289746f5c5677d16043040d63710e22abc43c9/pydantic_core-2.46.4-cp314-cp314t-win_amd64.whl", hash = "sha256:2108ba5c1c1eca18030634489dc544844144ee36357f2f9f780b93e7ddbb44b5", size = 2051624, upload-time = "2026-05-06T13:38:21.672Z" }, + { url = "https://files.pythonhosted.org/packages/f6/d2/42dd53d0a85c27606f316d3aa5d2869c4e8470a5ed6dec30e4a1abe19192/pydantic_core-2.46.4-cp314-cp314t-win_arm64.whl", hash = "sha256:4fcbe087dbc2068af7eda3aa87634eba216dbda64d1ae73c8684b621d33f6596", size = 2017325, upload-time = "2026-05-06T13:40:52.723Z" }, ] [[package]] @@ -1700,6 +1811,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8b/0c/9d30a4ebeb6db2b25a841afbb80f6ef9a854fc3b41be131d249a977b4959/starlette-0.46.2-py3-none-any.whl", hash = "sha256:595633ce89f8ffa71a015caed34a5b2dc1c0cdb3f0f1fbd1e69339cf2abeec35", size = 72037, upload-time = "2025-04-13T13:56:16.21Z" }, ] +[[package]] +name = "tenacity" +version = "9.1.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/47/c6/ee486fd809e357697ee8a44d3d69222b344920433d3b6666ccd9b374630c/tenacity-9.1.4.tar.gz", hash = "sha256:adb31d4c263f2bd041081ab33b498309a57c77f9acf2db65aadf0898179cf93a", size = 49413, upload-time = "2026-02-07T10:45:33.841Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d7/c1/eb8f9debc45d3b7918a32ab756658a0904732f75e555402972246b0b8e71/tenacity-9.1.4-py3-none-any.whl", hash = "sha256:6095a360c919085f28c6527de529e76a06ad89b23659fa881ae0649b867a9d55", size = 28926, upload-time = "2026-02-07T10:45:32.24Z" }, +] + [[package]] name = "tomli-w" version = "1.2.0" @@ -1709,6 +1829,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c7/18/c86eb8e0202e32dd3df50d43d7ff9854f8e0603945ff398974c1d91ac1ef/tomli_w-1.2.0-py3-none-any.whl", hash = "sha256:188306098d013b691fcadc011abd66727d3c414c571bb01b1a174ba8c983cf90", size = 6675, upload-time = "2025-01-15T12:07:22.074Z" }, ] +[[package]] +name = "tqdm" +version = "4.68.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ae/5f/57ff8b434839e70dab45601284ea413e947a63799891b7553e5960a793a8/tqdm-4.68.4.tar.gz", hash = "sha256:19829c9673638f2a0b8617da4cdcb927e831cd88bcfcb6e78d42a4d1af131520", size = 792418, upload-time = "2026-07-07T09:58:18.369Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/22/2a/5e5e750890ada51017d18d0d4c30da696e5b5bd3180947729927628fc3cb/tqdm-4.68.4-py3-none-any.whl", hash = "sha256:5168118b2368f48c561afda8020fd79195b1bdb0bdf8086b88442c267a315dc2", size = 676612, upload-time = "2026-07-07T09:58:16.256Z" }, +] + [[package]] name = "typing-extensions" version = "4.14.1" @@ -1720,14 +1852,14 @@ wheels = [ [[package]] name = "typing-inspection" -version = "0.4.1" +version = "0.4.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/f8/b1/0c11f5058406b3af7609f121aaa6b609744687f1d158b3c3a5bf4cc94238/typing_inspection-0.4.1.tar.gz", hash = "sha256:6ae134cc0203c33377d43188d4064e9b357dba58cff3185f22924610e70a9d28", size = 75726, upload-time = "2025-05-21T18:55:23.885Z" } +sdist = { url = "https://files.pythonhosted.org/packages/55/e3/70399cb7dd41c10ac53367ae42139cf4b1ca5f36bb3dc6c9d33acdb43655/typing_inspection-0.4.2.tar.gz", hash = "sha256:ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464", size = 75949, upload-time = "2025-10-01T02:14:41.687Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/17/69/cd203477f944c353c31bade965f880aa1061fd6bf05ded0726ca845b6ff7/typing_inspection-0.4.1-py3-none-any.whl", hash = "sha256:389055682238f53b04f7badcb49b989835495a96700ced5dab2d8feae4b26f51", size = 14552, upload-time = "2025-05-21T18:55:22.152Z" }, + { url = "https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7", size = 14611, upload-time = "2025-10-01T02:14:40.154Z" }, ] [[package]] From 1024404cc7963b32311d065ffbec20713edf2333 Mon Sep 17 00:00:00 2001 From: Sami Jawhar Date: Sun, 23 Aug 2026 23:15:39 +0000 Subject: [PATCH 5/5] fix(passthrough): stop reconcile worker from re-selecting permanently-failed transactions forever PassthroughReconcileWorker re-ran the same unindexed eligibility query every 300s and re-selected the same permanently-broken transactions forever, since MaterializationFailed outcomes were never persisted anywhere the eligibility query's NOT EXISTS(conversation_events) check could see. Two independent problems, one PR because they're both needed for the worker to converge: - Add passthrough_materialization_dead_letters, keyed by transaction_id, and exclude it from the eligibility query. Every MaterializationFailed reason except missing_request_logs reflects a deterministic parse/validation failure on already-persisted, immutable request_logs bytes -- retrying without a code change reproduces the identical outcome forever, so those are dead-lettered. missing_request_logs (zero rows found) is the one failure that could reflect an absence rather than malformed content, so it stays retryable. Raw DB exceptions (timeouts, connection errors) were already handled outside MaterializationFailed and remain untouched. - Add a partial index (migrations/023) whose predicate mirrors the eligibility query's WHERE clause verbatim, turning its Parallel Seq Scan into a covering index-only scan. Measured locally against a seeded 2.4M-row table: cost 70408->310 (~227x), 194ms->7ms. - PASSTHROUGH_MATERIALIZE_BACKFILL_ENABLED's own docstring says 'one-shot', but the worker looped forever on a fixed interval regardless of backlog state. It now stops itself once a sweep selects zero eligible rows. --- .../passthrough-reconcile-convergence.md | 21 +++++ ...ssthrough_materialization_dead_letters.sql | 51 ++++++++++ ...ssthrough_materialization_dead_letters.sql | 23 +++++ .../passthrough_materialize/reconcile.py | 60 +++++++++++- .../passthrough_materialize/worker.py | 28 +++++- ...ssthrough_materialization_dead_letters.sql | 23 +++++ .../passthrough_materialize/test_reconcile.py | 92 ++++++++++++++++++- .../passthrough_materialize/test_worker.py | 62 +++++++++++++ 8 files changed, 356 insertions(+), 4 deletions(-) create mode 100644 changelog.d/passthrough-reconcile-convergence.md create mode 100644 migrations/postgres/023_add_passthrough_materialization_dead_letters.sql create mode 100644 migrations/sqlite/023_add_passthrough_materialization_dead_letters.sql create mode 100644 src/luthien_proxy/utils/sqlite_migrations/023_add_passthrough_materialization_dead_letters.sql diff --git a/changelog.d/passthrough-reconcile-convergence.md b/changelog.d/passthrough-reconcile-convergence.md new file mode 100644 index 000000000..32b353f5d --- /dev/null +++ b/changelog.d/passthrough-reconcile-convergence.md @@ -0,0 +1,21 @@ +--- +category: Fixes +pr: 813 +--- + +**`PassthroughReconcileWorker` no longer runs forever with zero progress**: the +periodic reconciliation sweep had no way to remember a permanently-unparseable +passthrough transaction (`missing_required_field`, `unsupported_variant`, etc.) +across sweeps, so it re-selected and re-failed the exact same rows every +`PASSTHROUGH_MATERIALIZE_RECONCILE_INTERVAL_SECONDS` forever. Permanent +failures are now recorded in a new `passthrough_materialization_dead_letters` +table and excluded from the eligibility query; transient failures (a DB error, +or `missing_request_logs`, which could reflect a row that hasn't landed yet) +remain eligible for retry. The eligibility query's `request_logs` scan also +had no supporting index and ran a full Parallel Seq Scan on every sweep; a new +partial index mirrors the query's predicate exactly, turning it into a +covering index-only scan. + - The worker also now stops itself once a sweep selects nothing at all, + matching its own `PASSTHROUGH_MATERIALIZE_BACKFILL_ENABLED` config + docstring ("one-shot ... backfill") instead of polling an empty backlog + forever. diff --git a/migrations/postgres/023_add_passthrough_materialization_dead_letters.sql b/migrations/postgres/023_add_passthrough_materialization_dead_letters.sql new file mode 100644 index 000000000..47856140d --- /dev/null +++ b/migrations/postgres/023_add_passthrough_materialization_dead_letters.sql @@ -0,0 +1,51 @@ +-- ABOUTME: Adds a dead-letter table for permanently-failed passthrough materializations +-- ABOUTME: and a partial index so the reconcile worker's eligibility query stops +-- ABOUTME: re-scanning all of request_logs and stops re-selecting dead rows forever. + +-- Without this table, PassthroughReconcileWorker has no way to remember a +-- MaterializationFailed transaction across sweeps. A permanently-unparseable +-- transaction (reason e.g. missing_required_field, unsupported_variant -- +-- see luthien_proxy.passthrough_materialize.reconcile._is_permanent_failure) +-- is therefore re-selected and re-fails identically on every sweep forever, +-- since it is a deterministic function of already-persisted, immutable +-- request_logs bytes. Recording it here lets the eligibility query exclude +-- it while leaving genuinely transient failures (nothing is written here for +-- those) eligible for retry on the next sweep. +CREATE TABLE IF NOT EXISTS passthrough_materialization_dead_letters ( + transaction_id TEXT PRIMARY KEY, + reason TEXT NOT NULL, + failed_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +-- request_logs is a live, multi-GB, multi-million-row production table with +-- no index supporting the reconcile eligibility query's filter, so that +-- query ran a Parallel Seq Scan discarding essentially the whole table every +-- 300s (see PR description for a production EXPLAIN). This partial index +-- mirrors _ELIGIBLE_UNMATERIALIZED_TRANSACTIONS_SQL's WHERE clause in +-- luthien_proxy.passthrough_materialize.reconcile *exactly* (direction and +-- the four passthrough endpoint patterns): a partial index whose predicate +-- matches the query verbatim lets Postgres use it as a covering index-only +-- scan restricted to just the rows that could ever be eligible, rather than +-- indexing the ~50% of the table that is merely direction='inbound' +-- (measured locally: cost 70408->310, a ~227x reduction; see PR description). +-- If the eligibility query's endpoint list ever changes, this index's WHERE +-- clause must change with it or the planner will stop using it. +-- +-- CONCURRENTLY: a plain CREATE INDEX takes a SHARE lock that blocks writes +-- (INSERTs from every in-flight request) for the full build duration; +-- CONCURRENTLY avoids that at the cost of two table scans and not running +-- inside a transaction block. `psql -f` (docker/run-migrations.sh) commits +-- each statement in this file independently, so CONCURRENTLY coexisting with +-- the CREATE TABLE above is safe -- just never wrap this file's statements +-- in an explicit BEGIN/COMMIT. +CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_request_logs_passthrough_eligible + ON request_logs (started_at, transaction_id) + WHERE direction = 'inbound' + AND ( + endpoint IN ( + '/openai/v1/chat/completions', + '/openai/v1/responses' + ) + OR endpoint LIKE '/gemini/%:generateContent' + OR endpoint LIKE '/gemini/%:streamGenerateContent' + ); diff --git a/migrations/sqlite/023_add_passthrough_materialization_dead_letters.sql b/migrations/sqlite/023_add_passthrough_materialization_dead_letters.sql new file mode 100644 index 000000000..33ced6fc7 --- /dev/null +++ b/migrations/sqlite/023_add_passthrough_materialization_dead_letters.sql @@ -0,0 +1,23 @@ +-- ABOUTME: Adds a dead-letter table for permanently-failed passthrough materializations +-- ABOUTME: and a partial index mirroring the Postgres migration's eligibility-query +-- ABOUTME: predicate; SQLite 3.8+ supports partial indexes via CREATE INDEX ... WHERE. + +CREATE TABLE IF NOT EXISTS passthrough_materialization_dead_letters ( + transaction_id TEXT PRIMARY KEY, + reason TEXT NOT NULL, + failed_at TEXT NOT NULL DEFAULT (datetime('now')) +); + +-- No CONCURRENTLY (SQLite doesn't support it; this is the dockerless +-- single-user database path, not a live production table under load). +CREATE INDEX IF NOT EXISTS idx_request_logs_passthrough_eligible + ON request_logs (started_at, transaction_id) + WHERE direction = 'inbound' + AND ( + endpoint IN ( + '/openai/v1/chat/completions', + '/openai/v1/responses' + ) + OR endpoint LIKE '/gemini/%:generateContent' + OR endpoint LIKE '/gemini/%:streamGenerateContent' + ); diff --git a/src/luthien_proxy/passthrough_materialize/reconcile.py b/src/luthien_proxy/passthrough_materialize/reconcile.py index 8e4cba46c..bdf78837a 100644 --- a/src/luthien_proxy/passthrough_materialize/reconcile.py +++ b/src/luthien_proxy/passthrough_materialize/reconcile.py @@ -22,6 +22,13 @@ logger = logging.getLogger(__name__) +# NOTE: The endpoint filter below must stay in sync with the partial index +# idx_request_logs_passthrough_eligible (migrations/postgres/023_*.sql and its +# SQLite counterpart) -- that index's WHERE clause mirrors this predicate +# verbatim so Postgres can use it as a covering index-only scan instead of a +# Parallel Seq Scan over request_logs. Changing the endpoint list here without +# updating the index's predicate (in a new migration) silently reintroduces +# the seq scan. _ELIGIBLE_UNMATERIALIZED_TRANSACTIONS_SQL = """ SELECT request_logs.transaction_id FROM request_logs @@ -40,11 +47,40 @@ FROM conversation_events WHERE conversation_events.call_id = request_logs.transaction_id ) + AND NOT EXISTS ( + SELECT 1 + FROM passthrough_materialization_dead_letters + WHERE passthrough_materialization_dead_letters.transaction_id = request_logs.transaction_id + ) GROUP BY request_logs.transaction_id ORDER BY MIN(request_logs.started_at), request_logs.transaction_id LIMIT $2 """ +_RECORD_DEAD_LETTER_SQL = """ + INSERT INTO passthrough_materialization_dead_letters (transaction_id, reason) + VALUES ($1, $2) + ON CONFLICT (transaction_id) DO NOTHING +""" + +# Every MaterializationFailed reason except "missing_request_logs" is raised only +# after read_raw_transaction/parse_captured_transaction *successfully* fetched +# request_logs bytes and then found them unparseable or invalid for the matched +# endpoint (see PassthroughNormalizeReason and the _InvalidRequestLog family in +# materialize_read.py). request_logs rows are never updated after insert, so +# retrying one of those failures without a code change reproduces the identical +# outcome forever -- they are permanent. "missing_request_logs" is the one +# reason that reflects an *absence* (zero rows found for the transaction_id) +# rather than malformed content already read; treating it as permanent risks +# permanently blacklisting a transaction whose row simply had not landed yet +# (or would land on a later sweep), so it is the only reason left retryable. +_TRANSIENT_FAILURE_REASONS = frozenset({"missing_request_logs"}) + + +def _is_permanent_failure(reason: str) -> bool: + """Return whether a materialization failure reason will never resolve on retry.""" + return reason not in _TRANSIENT_FAILURE_REASONS + async def reconcile_passthrough(db_pool: DatabasePool, *, limit: int, since: datetime | None = None) -> ReconcileStats: """Materialize eligible raw transactions that do not yet have request events.""" @@ -70,8 +106,10 @@ async def reconcile_passthrough(db_pool: DatabasePool, *, limit: int, since: dat already_materialized += 1 case SkippedIneligible(): skipped_ineligible += 1 - case MaterializationFailed(): + case MaterializationFailed() as failure: failed += 1 + if _is_permanent_failure(failure.reason): + await _record_dead_letter(db_pool, failure) case unreachable: assert_never(unreachable) @@ -83,6 +121,26 @@ async def reconcile_passthrough(db_pool: DatabasePool, *, limit: int, since: dat ) +async def _record_dead_letter(db_pool: DatabasePool, failure: MaterializationFailed) -> None: + """Persist a permanent materialization failure so it is not re-selected. + + A DB error here is logged and swallowed rather than propagated: the sweep's + other transactions must not be aborted because one dead-letter write failed. + Not recording it just means this transaction is retried (and, if still + permanently broken, re-attempted to be dead-lettered) on the next sweep -- + safe, if wasteful, rather than silently losing the failure. + """ + try: + async with db_pool.connection() as conn: + await conn.execute(_RECORD_DEAD_LETTER_SQL, failure.transaction_id, failure.reason) + except (aiosqlite.Error, asyncpg.PostgresError): + logger.exception( + "Failed to record passthrough dead letter for %s (reason=%s); it will be retried next sweep", + failure.transaction_id, + failure.reason, + ) + + def _transaction_id(row: Mapping[str, object]) -> str: transaction_id = row["transaction_id"] if isinstance(transaction_id, str): diff --git a/src/luthien_proxy/passthrough_materialize/worker.py b/src/luthien_proxy/passthrough_materialize/worker.py index 24e15f70f..c3b5256f2 100644 --- a/src/luthien_proxy/passthrough_materialize/worker.py +++ b/src/luthien_proxy/passthrough_materialize/worker.py @@ -20,8 +20,31 @@ def _log_task_exception(task: asyncio.Task[None]) -> None: logger.exception("Passthrough reconciliation worker raised unexpectedly", exc_info=error) +def _sweep_selected_nothing(stats: ReconcileStats) -> bool: + """Return whether a sweep's eligibility query matched zero transactions. + + Distinct from "did nothing succeed": a sweep that only hit failures + (materialized=0) but still selected rows must keep running so it can + dead-letter permanent ones and retry transient ones. Only an eligibility + query that returned no rows at all means the backlog is actually drained. + """ + return ( + stats.materialized == 0 + and stats.already_materialized == 0 + and stats.skipped_ineligible == 0 + and stats.failed == 0 + ) + + class PassthroughReconcileWorker: - """Run bounded passthrough reconciliation sweeps until shutdown.""" + """Run bounded passthrough reconciliation sweeps until the backlog drains or shutdown. + + PASSTHROUGH_MATERIALIZE_BACKFILL_ENABLED is documented as a one-shot backfill + for historical request_logs, not a perpetual poller. Once a sweep selects zero + eligible transactions -- meaning nothing is left to materialize, retry, or + dead-letter -- the loop exits instead of continuing to sweep an empty backlog + every interval_seconds forever. + """ def __init__(self, *, db_pool: DatabasePool, limit: int, interval_seconds: int) -> None: """Initialize the database-backed worker and its cadence.""" @@ -38,6 +61,9 @@ async def _run_loop(self) -> None: logger.exception("Passthrough reconciliation sweep failed; retrying next interval") else: self._log_stats(stats) + if _sweep_selected_nothing(stats): + logger.info("Passthrough reconciliation backlog drained; one-shot backfill worker stopping.") + return await asyncio.sleep(self._interval_seconds) def _log_stats(self, stats: ReconcileStats) -> None: diff --git a/src/luthien_proxy/utils/sqlite_migrations/023_add_passthrough_materialization_dead_letters.sql b/src/luthien_proxy/utils/sqlite_migrations/023_add_passthrough_materialization_dead_letters.sql new file mode 100644 index 000000000..33ced6fc7 --- /dev/null +++ b/src/luthien_proxy/utils/sqlite_migrations/023_add_passthrough_materialization_dead_letters.sql @@ -0,0 +1,23 @@ +-- ABOUTME: Adds a dead-letter table for permanently-failed passthrough materializations +-- ABOUTME: and a partial index mirroring the Postgres migration's eligibility-query +-- ABOUTME: predicate; SQLite 3.8+ supports partial indexes via CREATE INDEX ... WHERE. + +CREATE TABLE IF NOT EXISTS passthrough_materialization_dead_letters ( + transaction_id TEXT PRIMARY KEY, + reason TEXT NOT NULL, + failed_at TEXT NOT NULL DEFAULT (datetime('now')) +); + +-- No CONCURRENTLY (SQLite doesn't support it; this is the dockerless +-- single-user database path, not a live production table under load). +CREATE INDEX IF NOT EXISTS idx_request_logs_passthrough_eligible + ON request_logs (started_at, transaction_id) + WHERE direction = 'inbound' + AND ( + endpoint IN ( + '/openai/v1/chat/completions', + '/openai/v1/responses' + ) + OR endpoint LIKE '/gemini/%:generateContent' + OR endpoint LIKE '/gemini/%:streamGenerateContent' + ); diff --git a/tests/luthien_proxy/unit_tests/passthrough_materialize/test_reconcile.py b/tests/luthien_proxy/unit_tests/passthrough_materialize/test_reconcile.py index 197125a33..a5b6df0bb 100644 --- a/tests/luthien_proxy/unit_tests/passthrough_materialize/test_reconcile.py +++ b/tests/luthien_proxy/unit_tests/passthrough_materialize/test_reconcile.py @@ -15,6 +15,7 @@ from luthien_proxy.passthrough_materialize.materialize_types import ( AlreadyMaterialized, CanonicalTransaction, + MaterializationFailed, MaterializationResult, Materialized, ReconcileStats, @@ -97,7 +98,15 @@ async def _event_count(pool: DatabasePool, transaction_id: str) -> int: return count -async def test_reconcile_materializes_only_eligible_unmaterialized_transactions_when_raw_logs_are_mixed( +async def _dead_letter_reason(pool: DatabasePool, transaction_id: str) -> str | None: + async with pool.connection() as conn: + row = await conn.fetchrow( + "SELECT reason FROM passthrough_materialization_dead_letters WHERE transaction_id = $1", transaction_id + ) + return None if row is None else str(row["reason"]) + + +async def test_reconcile_materializes_only_eligible_unmaterialized_transactions_and_converges_next_sweep( reconcile_pool: DatabasePool, ) -> None: # Given @@ -117,11 +126,90 @@ async def test_reconcile_materializes_only_eligible_unmaterialized_transactions_ # Then assert first_pass == ReconcileStats(materialized=1, failed=1) - assert second_pass == ReconcileStats(failed=1) + # The permanent failure ("failure") was dead-lettered on the first pass, so + # the second pass -- run over the exact same rows -- selects nothing new. + # Before the fix this asserted ReconcileStats(failed=1): the same broken + # transaction was re-selected and re-failed forever. + assert second_pass == ReconcileStats() assert await _event_count(reconcile_pool, ready.transaction_id) == 2 assert await _event_count(reconcile_pool, already.transaction_id) == 2 assert await _event_count(reconcile_pool, ineligible.transaction_id) == 0 assert await _event_count(reconcile_pool, failure.transaction_id) == 0 + assert await _dead_letter_reason(reconcile_pool, failure.transaction_id) == "missing_required_field" + assert await _dead_letter_reason(reconcile_pool, ready.transaction_id) is None + + +async def test_reconcile_converges_when_the_entire_backlog_is_permanently_broken( + reconcile_pool: DatabasePool, +) -> None: + """Reproduces the production bug: every eligible row fails identically forever. + + Before persisting dead letters, a sweep over an all-permanently-broken backlog + re-selected and re-failed the exact same rows on every subsequent sweep -- the + root cause of PassthroughReconcileWorker running forever with zero progress. + """ + # Given + broken_seeds = [ + replace(_valid_openai_seed(f"broken-{i}", _at(9)), request_body={"model": "gpt-4.1"}) for i in range(5) + ] + for seed in broken_seeds: + await _seed_raw_log(reconcile_pool, seed) + + # When + first_pass = await reconcile_passthrough(reconcile_pool, limit=10) + second_pass = await reconcile_passthrough(reconcile_pool, limit=10) + + # Then + assert first_pass == ReconcileStats(failed=5) + assert second_pass == ReconcileStats() + async with reconcile_pool.connection() as conn: + dead_letter_count = await conn.fetchval("SELECT COUNT(*) FROM passthrough_materialization_dead_letters") + assert dead_letter_count == 5 + + +async def test_reconcile_leaves_a_transient_failure_reason_eligible_for_retry( + reconcile_pool: DatabasePool, + monkeypatch: pytest.MonkeyPatch, +) -> None: + # Given + transient = _valid_openai_seed("transient", _at(9)) + await _seed_raw_log(reconcile_pool, transient) + original_materialize = reconcile_module.materialize_transaction + + async def materialize_with_transient_failure(pool: DatabasePool, transaction_id: str) -> MaterializationResult: + if transaction_id == transient.transaction_id: + return MaterializationFailed(transaction_id=transaction_id, reason="missing_request_logs") + return await original_materialize(pool, transaction_id) + + monkeypatch.setattr(reconcile_module, "materialize_transaction", materialize_with_transient_failure) + + # When + first_pass = await reconcile_passthrough(reconcile_pool, limit=10) + second_pass = await reconcile_passthrough(reconcile_pool, limit=10) + + # Then: unlike a permanent failure, a "missing_request_logs" failure is never + # dead-lettered, so it is re-selected (and re-fails) on every sweep. + assert first_pass == ReconcileStats(failed=1) + assert second_pass == ReconcileStats(failed=1) + assert await _dead_letter_reason(reconcile_pool, transient.transaction_id) is None + + +async def test_migration_creates_the_dead_letter_table_and_supporting_index(reconcile_pool: DatabasePool) -> None: + """Both the Postgres and SQLite 023 migrations must produce this schema. + + This exercises the SQLite side (used by every fixture in this file via + check_migrations); the Postgres side was verified against a locally seeded + multi-million-row table (see PR description for EXPLAIN evidence). + """ + async with reconcile_pool.connection() as conn: + table = await conn.fetchrow( + "SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'passthrough_materialization_dead_letters'" + ) + index = await conn.fetchrow( + "SELECT name FROM sqlite_master WHERE type = 'index' AND name = 'idx_request_logs_passthrough_eligible'" + ) + assert table is not None + assert index is not None async def test_reconcile_respects_oldest_first_limit_and_since_when_selecting_transactions( diff --git a/tests/luthien_proxy/unit_tests/passthrough_materialize/test_worker.py b/tests/luthien_proxy/unit_tests/passthrough_materialize/test_worker.py index 9768e4ece..7d4d84f0f 100644 --- a/tests/luthien_proxy/unit_tests/passthrough_materialize/test_worker.py +++ b/tests/luthien_proxy/unit_tests/passthrough_materialize/test_worker.py @@ -110,3 +110,65 @@ async def reconcile(pool: DatabasePool, *, limit: int) -> ReconcileStats: # Then assert task.cancelled() assert worker._task is None + + +async def test_reconcile_worker_stops_once_a_sweep_selects_nothing( + worker_pool: DatabasePool, monkeypatch: pytest.MonkeyPatch +) -> None: + # Given + sweep_count = 0 + + async def reconcile(pool: DatabasePool, *, limit: int) -> ReconcileStats: + nonlocal sweep_count + sweep_count += 1 + return ReconcileStats() + + monkeypatch.setattr(worker_module, "reconcile_passthrough", reconcile) + worker = PassthroughReconcileWorker(db_pool=worker_pool, limit=23, interval_seconds=60) + + # When + worker.start() + task = worker._task + assert task is not None + await asyncio.wait_for(task, timeout=1) + + # Then: the one-shot backfill worker stops itself on an empty sweep instead + # of sleeping interval_seconds and sweeping an empty backlog forever. + assert sweep_count == 1 + assert task.done() + assert not task.cancelled() + + +async def test_reconcile_worker_keeps_sweeping_while_failures_remain( + worker_pool: DatabasePool, monkeypatch: pytest.MonkeyPatch +) -> None: + # Given + original_sleep = asyncio.sleep + second_sweep_started = asyncio.Event() + sweep_count = 0 + + async def reconcile(pool: DatabasePool, *, limit: int) -> ReconcileStats: + nonlocal sweep_count + sweep_count += 1 + if sweep_count == 2: + second_sweep_started.set() + return ReconcileStats(failed=1) + + async def advance_without_delay(_seconds: float) -> None: + await original_sleep(0) + + monkeypatch.setattr(worker_module, "reconcile_passthrough", reconcile) + monkeypatch.setattr(worker_module.asyncio, "sleep", advance_without_delay) + worker = PassthroughReconcileWorker(db_pool=worker_pool, limit=23, interval_seconds=60) + + # When + worker.start() + try: + await asyncio.wait_for(second_sweep_started.wait(), timeout=1) + finally: + await worker.stop() + + # Then: a sweep that keeps failing something (not a fully empty sweep) must + # keep running -- it still has permanent failures to dead-letter and + # transient ones to retry. + assert sweep_count >= 2