diff --git a/.gitignore b/.gitignore index 61d5952d7..8e405cb38 100644 --- a/.gitignore +++ b/.gitignore @@ -98,3 +98,10 @@ scripts/automated_maintenance/automated_maintenance.env.bak* # Claude Code runtime state .claude/scheduled_tasks.lock NEXT.md + +# Agent run artifacts — transient logs and task evidence +# Canonical baseline reports (.md) are tracked; everything else is ephemeral. +.sisyphus/evidence/*.log +.sisyphus/evidence/*.txt +.sisyphus/evidence/task-*.json +.sisyphus/plans/ diff --git a/.sisyphus/evidence/after-query-plans-sqlite.md b/.sisyphus/evidence/after-query-plans-sqlite.md new file mode 100644 index 000000000..e05cac501 --- /dev/null +++ b/.sisyphus/evidence/after-query-plans-sqlite.md @@ -0,0 +1,7 @@ +warning: `VIRTUAL_ENV=/Users/paolo/Documents/Projects/mcpm.sh/.venv` does not match the project environment path `.venv` and will be ignored; use `--active` to target the active environment instead +Applying migrations... +DB has 20528 events, 178 sessions. +Running EXPLAIN QUERY PLAN for session_list... +Running EXPLAIN QUERY PLAN for session_detail... +Running EXPLAIN QUERY PLAN for recent_calls... +Written: /Users/paolo/Documents/Projects/luthien-proxy/.sisyphus/evidence/baseline-query-plans.md diff --git a/.sisyphus/evidence/baseline-query-plans.md b/.sisyphus/evidence/baseline-query-plans.md new file mode 100644 index 000000000..5bfbfc8ec --- /dev/null +++ b/.sisyphus/evidence/baseline-query-plans.md @@ -0,0 +1,81 @@ +--- +git_sha: 342635d541115560f5b9af33031fa58f494ca753 +timestamp: 2026-05-15T19:08:11.641641+00:00 +backend: sqlite +row_count: 20528 +session_count: 178 +--- + +## Query: session_list + +### SQL + +```sql +SELECT + ce.session_id, + MIN(ce.created_at) as first_ts, + MAX(ce.created_at) as last_ts, + COUNT(*) as total_events, + COUNT(DISTINCT ce.call_id) as turn_count, + SUM(CASE + WHEN ce.event_type LIKE 'policy.%' + AND ce.event_type NOT LIKE 'policy.%judge.evaluation%' + THEN 1 ELSE 0 + END) as policy_interventions +FROM conversation_events ce +WHERE ce.session_id IS NOT NULL +GROUP BY ce.session_id +ORDER BY last_ts DESC +LIMIT ? OFFSET ? +``` + +### EXPLAIN QUERY PLAN + +``` +SEARCH ce USING INDEX idx_conversation_events_session_id_btree (session_id>?) +USE TEMP B-TREE FOR count(DISTINCT) +USE TEMP B-TREE FOR ORDER BY +``` + +## Query: session_detail + +### SQL + +```sql +SELECT call_id, event_type, payload, created_at +FROM conversation_events +WHERE session_id = ? +ORDER BY created_at ASC +``` + +### EXPLAIN QUERY PLAN + +``` +SEARCH conversation_events USING INDEX idx_conversation_events_session_id_btree (session_id=?) +USE TEMP B-TREE FOR ORDER BY +``` + +## Query: recent_calls + +### SQL + +```sql +SELECT + call_id, + COUNT(*) as event_count, + MAX(created_at) as latest, + MAX(session_id) as session_id +FROM conversation_events +GROUP BY call_id +ORDER BY latest DESC +LIMIT ? +``` + +### EXPLAIN QUERY PLAN + +``` +SCAN conversation_events +USE TEMP B-TREE FOR GROUP BY +USE TEMP B-TREE FOR ORDER BY +``` + diff --git a/.sisyphus/evidence/perf-report-after-sqlite.md b/.sisyphus/evidence/perf-report-after-sqlite.md new file mode 100644 index 000000000..5aef30c31 --- /dev/null +++ b/.sisyphus/evidence/perf-report-after-sqlite.md @@ -0,0 +1,143 @@ +git_sha: 342635d541115560f5b9af33031fa58f494ca753 +browser_version: 1.50.0 +backend: sqlite +generated_at: 2026-05-15T19:08:32.892039+00:00 + +# Luthien Admin UI — Performance Baseline Report + +## Hardware & Versions + +| Field | Value | +|-------|-------| +| Machine | x86_64 | +| Processor | i386 | +| RAM | 38 GB | +| OS | Darwin 22.6.0 | +| Python | 3.13.5 | +| git_sha | `342635d541115560f5b9af33031fa58f494ca753` | +| DB backend | sqlite | +| Playwright | 1.50.0 | + +## Per-Page Timings + +_NO DATA YET — run `scripts/run_perf.sh` to populate._ + +## Throttled (sami-like) + +_NO DATA YET_ + +## Transcript Open + +_NO DATA YET_ + +## SSE Memory Growth + +_NO DATA YET_ + +## Server-Timing Breakdown + +_NO DATA YET_ + +## Payload Size Breakdown + +_NO DATA YET_ + +## Query Plans + +--- +git_sha: 342635d541115560f5b9af33031fa58f494ca753 +timestamp: 2026-05-15T19:08:11.641641+00:00 +backend: sqlite +row_count: 20528 +session_count: 178 +--- + +## Query: session_list + +### SQL + +```sql +SELECT + ce.session_id, + MIN(ce.created_at) as first_ts, + MAX(ce.created_at) as last_ts, + COUNT(*) as total_events, + COUNT(DISTINCT ce.call_id) as turn_count, + SUM(CASE + WHEN ce.event_type LIKE 'policy.%' + AND ce.event_type NOT LIKE 'policy.%judge.evaluation%' + THEN 1 ELSE 0 + END) as policy_interventions +FROM conversation_events ce +WHERE ce.session_id IS NOT NULL +GROUP BY ce.session_id +ORDER BY last_ts DESC +LIMIT ? OFFSET ? +``` + +### EXPLAIN QUERY PLAN + +``` +SEARCH ce USING INDEX idx_conversation_events_session_id_btree (session_id>?) +USE TEMP B-TREE FOR count(DISTINCT) +USE TEMP B-TREE FOR ORDER BY +``` + +## Query: session_detail + +### SQL + +```sql +SELECT call_id, event_type, payload, created_at +FROM conversation_events +WHERE session_id = ? +ORDER BY created_at ASC +``` + +### EXPLAIN QUERY PLAN + +``` +SEARCH conversation_events USING INDEX idx_conversation_events_session_id_btree (session_id=?) +USE TEMP B-TREE FOR ORDER BY +``` + +## Query: recent_calls + +### SQL + +```sql +SELECT + call_id, + COUNT(*) as event_count, + MAX(created_at) as latest, + MAX(session_id) as session_id +FROM conversation_events +GROUP BY call_id +ORDER BY latest DESC +LIMIT ? +``` + +### EXPLAIN QUERY PLAN + +``` +SCAN conversation_events +USE TEMP B-TREE FOR GROUP BY +USE TEMP B-TREE FOR ORDER BY +``` + +## Top Hotspots + +_NO DATA YET — hotspots will be derived from measurement results._ + +**Known candidates (from code review):** + +1. `history_list.html:514` — hardcodes `?limit=10000` (sends full dataset on every load) +2. `conversation_live.js:92-118` — `loadInitial()` fetches entire session upfront +3. `conversation_live.js:215-244` — full DOM re-render on every SSE event +4. `conversation_live.js:164-172` — unbounded `rawEvents[callId]` array (memory leak risk) +5. `history_list.html:423-448` — client-side filter runs on every keystroke + +**Query plan risks:** + +- `session_list`: 2× TEMP B-TREE (COUNT DISTINCT + ORDER BY) — scales poorly with row count +- `recent_calls`: SCAN on all rows — O(n) over conversation_events diff --git a/.sisyphus/evidence/perf-report-after.md b/.sisyphus/evidence/perf-report-after.md new file mode 100644 index 000000000..b36bd3388 --- /dev/null +++ b/.sisyphus/evidence/perf-report-after.md @@ -0,0 +1,147 @@ +git_sha: 342635d541115560f5b9af33031fa58f494ca753 +browser_version: 1.50.0 +backend: sqlite +generated_at: 2026-05-15T19:08:32.892039+00:00 + +# Luthien Admin UI — Performance Baseline Report + +## Hardware & Versions + +| Field | Value | +|-------|-------| +| Machine | x86_64 | +| Processor | i386 | +| RAM | 38 GB | +| OS | Darwin 22.6.0 | +| Python | 3.13.5 | +| git_sha | `342635d541115560f5b9af33031fa58f494ca753` | +| DB backend | sqlite | +| Playwright | 1.50.0 | + +## Per-Page Timings + +_NO DATA YET — run `scripts/run_perf.sh` to populate._ + +## Throttled (sami-like) + +_NO DATA YET_ + +## Transcript Open + +_NO DATA YET_ + +## SSE Memory Growth + +_NO DATA YET_ + +## Server-Timing Breakdown + +_NO DATA YET_ + +## Payload Size Breakdown + +_NO DATA YET_ + +## Query Plans + +--- +git_sha: 342635d541115560f5b9af33031fa58f494ca753 +timestamp: 2026-05-15T19:08:11.641641+00:00 +backend: sqlite +row_count: 20528 +session_count: 178 +--- + +## Query: session_list + +### SQL + +```sql +SELECT + ce.session_id, + MIN(ce.created_at) as first_ts, + MAX(ce.created_at) as last_ts, + COUNT(*) as total_events, + COUNT(DISTINCT ce.call_id) as turn_count, + SUM(CASE + WHEN ce.event_type LIKE 'policy.%' + AND ce.event_type NOT LIKE 'policy.%judge.evaluation%' + THEN 1 ELSE 0 + END) as policy_interventions +FROM conversation_events ce +WHERE ce.session_id IS NOT NULL +GROUP BY ce.session_id +ORDER BY last_ts DESC +LIMIT ? OFFSET ? +``` + +### EXPLAIN QUERY PLAN + +``` +SEARCH ce USING INDEX idx_conversation_events_session_id_btree (session_id>?) +USE TEMP B-TREE FOR count(DISTINCT) +USE TEMP B-TREE FOR ORDER BY +``` + +## Query: session_detail + +### SQL + +```sql +SELECT call_id, event_type, payload, created_at +FROM conversation_events +WHERE session_id = ? +ORDER BY created_at ASC +``` + +### EXPLAIN QUERY PLAN + +``` +SEARCH conversation_events USING INDEX idx_conversation_events_session_id_btree (session_id=?) +USE TEMP B-TREE FOR ORDER BY +``` + +## Query: recent_calls + +### SQL + +```sql +SELECT + call_id, + COUNT(*) as event_count, + MAX(created_at) as latest, + MAX(session_id) as session_id +FROM conversation_events +GROUP BY call_id +ORDER BY latest DESC +LIMIT ? +``` + +### EXPLAIN QUERY PLAN + +``` +SCAN conversation_events +USE TEMP B-TREE FOR GROUP BY +USE TEMP B-TREE FOR ORDER BY +``` + +## Top Hotspots + +_NO DATA YET — hotspots will be derived from measurement results._ + +**Known candidates (from code review):** + +1. `history_list.html:514` — hardcodes `?limit=10000` (sends full dataset on every load) +2. `conversation_live.js:92-118` — `loadInitial()` fetches entire session upfront +3. `conversation_live.js:215-244` — full DOM re-render on every SSE event +4. `conversation_live.js:164-172` — unbounded `rawEvents[callId]` array (memory leak risk) +5. `history_list.html:423-448` — client-side filter runs on every keystroke + +**Query plan risks:** + +- `session_list`: 2× TEMP B-TREE (COUNT DISTINCT + ORDER BY) — scales poorly with row count +- `recent_calls`: SCAN on all rows — O(n) over conversation_events + +## Postgres + +SKIPPED: Postgres not available in local dev environment. diff --git a/.sisyphus/evidence/perf-report-baseline-sqlite.md b/.sisyphus/evidence/perf-report-baseline-sqlite.md new file mode 100644 index 000000000..6c5dbc771 --- /dev/null +++ b/.sisyphus/evidence/perf-report-baseline-sqlite.md @@ -0,0 +1,143 @@ +git_sha: 0158b252ee54580f477961d2e25dab0838da5db2 +browser_version: 1.50.0 +backend: sqlite +generated_at: 2026-05-15T00:23:57.106574+00:00 + +# Luthien Admin UI — Performance Baseline Report + +## Hardware & Versions + +| Field | Value | +|-------|-------| +| Machine | x86_64 | +| Processor | i386 | +| RAM | 38 GB | +| OS | Darwin 22.6.0 | +| Python | 3.13.5 | +| git_sha | `0158b252ee54580f477961d2e25dab0838da5db2` | +| DB backend | sqlite | +| Playwright | 1.50.0 | + +## Per-Page Timings + +_NO DATA YET — run `scripts/run_perf.sh` to populate._ + +## Throttled (sami-like) + +_NO DATA YET_ + +## Transcript Open + +_NO DATA YET_ + +## SSE Memory Growth + +_NO DATA YET_ + +## Server-Timing Breakdown + +_NO DATA YET_ + +## Payload Size Breakdown + +_NO DATA YET_ + +## Query Plans + +--- +git_sha: 0158b252ee54580f477961d2e25dab0838da5db2 +timestamp: 2026-05-15T00:23:52.853732+00:00 +backend: sqlite +row_count: 20528 +session_count: 178 +--- + +## Query: session_list + +### SQL + +```sql +SELECT + ce.session_id, + MIN(ce.created_at) as first_ts, + MAX(ce.created_at) as last_ts, + COUNT(*) as total_events, + COUNT(DISTINCT ce.call_id) as turn_count, + SUM(CASE + WHEN ce.event_type LIKE 'policy.%' + AND ce.event_type NOT LIKE 'policy.%judge.evaluation%' + THEN 1 ELSE 0 + END) as policy_interventions +FROM conversation_events ce +WHERE ce.session_id IS NOT NULL +GROUP BY ce.session_id +ORDER BY last_ts DESC +LIMIT ? OFFSET ? +``` + +### EXPLAIN QUERY PLAN + +``` +SEARCH ce USING INDEX idx_conversation_events_session_id_btree (session_id>?) +USE TEMP B-TREE FOR count(DISTINCT) +USE TEMP B-TREE FOR ORDER BY +``` + +## Query: session_detail + +### SQL + +```sql +SELECT call_id, event_type, payload, created_at +FROM conversation_events +WHERE session_id = ? +ORDER BY created_at ASC +``` + +### EXPLAIN QUERY PLAN + +``` +SEARCH conversation_events USING INDEX idx_conversation_events_session_id_btree (session_id=?) +USE TEMP B-TREE FOR ORDER BY +``` + +## Query: recent_calls + +### SQL + +```sql +SELECT + call_id, + COUNT(*) as event_count, + MAX(created_at) as latest, + MAX(session_id) as session_id +FROM conversation_events +GROUP BY call_id +ORDER BY latest DESC +LIMIT ? +``` + +### EXPLAIN QUERY PLAN + +``` +SCAN conversation_events +USE TEMP B-TREE FOR GROUP BY +USE TEMP B-TREE FOR ORDER BY +``` + +## Top Hotspots + +_NO DATA YET — hotspots will be derived from measurement results._ + +**Known candidates (from code review):** + +1. `history_list.html:514` — hardcodes `?limit=10000` (sends full dataset on every load) +2. `conversation_live.js:92-118` — `loadInitial()` fetches entire session upfront +3. `conversation_live.js:215-244` — full DOM re-render on every SSE event +4. `conversation_live.js:164-172` — unbounded `rawEvents[callId]` array (memory leak risk) +5. `history_list.html:423-448` — client-side filter runs on every keystroke + +**Query plan risks:** + +- `session_list`: 2× TEMP B-TREE (COUNT DISTINCT + ORDER BY) — scales poorly with row count +- `recent_calls`: SCAN on all rows — O(n) over conversation_events diff --git a/.sisyphus/evidence/perf-report-baseline.md b/.sisyphus/evidence/perf-report-baseline.md new file mode 100644 index 000000000..a6876db39 --- /dev/null +++ b/.sisyphus/evidence/perf-report-baseline.md @@ -0,0 +1,146 @@ +git_sha: 0158b252ee54580f477961d2e25dab0838da5db2 +browser_version: 1.50.0 +backend: sqlite +generated_at: 2026-05-15T00:23:57.106574+00:00 + +# Luthien Admin UI — Performance Baseline Report + +## Hardware & Versions + +| Field | Value | +|-------|-------| +| Machine | x86_64 | +| Processor | i386 | +| RAM | 38 GB | +| OS | Darwin 22.6.0 | +| Python | 3.13.5 | +| git_sha | `0158b252ee54580f477961d2e25dab0838da5db2` | +| DB backend | sqlite | +| Playwright | 1.50.0 | + +## Per-Page Timings + +_NO DATA YET — run `scripts/run_perf.sh` to populate._ + +## Throttled (sami-like) + +_NO DATA YET_ + +## Transcript Open + +_NO DATA YET_ + +## SSE Memory Growth + +_NO DATA YET_ + +## Server-Timing Breakdown + +_NO DATA YET_ + +## Payload Size Breakdown + +_NO DATA YET_ + +## Query Plans + +--- +git_sha: ce7649cc46afcf29a9431da1163d2adb80e6751d +timestamp: 2026-05-14T22:43:20.842092+00:00 +backend: sqlite +row_count: 535924 +session_count: 10000 +--- + +## Query: session_list + +### SQL + +```sql +SELECT + ce.session_id, + MIN(ce.created_at) as first_ts, + MAX(ce.created_at) as last_ts, + COUNT(*) as total_events, + COUNT(DISTINCT ce.call_id) as turn_count, + SUM(CASE + WHEN ce.event_type LIKE 'policy.%' + AND ce.event_type NOT LIKE 'policy.%judge.evaluation%' + THEN 1 ELSE 0 + END) as policy_interventions +FROM conversation_events ce +WHERE ce.session_id IS NOT NULL +GROUP BY ce.session_id +ORDER BY last_ts DESC +LIMIT ? OFFSET ? +``` + +### EXPLAIN QUERY PLAN + +``` +SEARCH ce USING INDEX idx_conversation_events_session (session_id>?) +USE TEMP B-TREE FOR count(DISTINCT) +USE TEMP B-TREE FOR ORDER BY +``` + +## Query: session_detail + +### SQL + +```sql +SELECT call_id, event_type, payload, created_at +FROM conversation_events +WHERE session_id = ? +ORDER BY created_at ASC +``` + +### EXPLAIN QUERY PLAN + +``` +SEARCH conversation_events USING INDEX idx_conversation_events_session (session_id=?) +USE TEMP B-TREE FOR ORDER BY +``` + +## Query: recent_calls + +### SQL + +```sql +SELECT + call_id, + COUNT(*) as event_count, + MAX(created_at) as latest, + MAX(session_id) as session_id +FROM conversation_events +GROUP BY call_id +ORDER BY latest DESC +LIMIT ? +``` + +### EXPLAIN QUERY PLAN + +``` +SCAN conversation_events USING INDEX idx_conversation_events_call_created +USE TEMP B-TREE FOR ORDER BY +``` + +## Top Hotspots + +_NO DATA YET — hotspots will be derived from measurement results._ + +**Known candidates (from code review):** + +1. `history_list.html:514` — hardcodes `?limit=10000` (sends full dataset on every load) +2. `conversation_live.js:92-118` — `loadInitial()` fetches entire session upfront +3. `conversation_live.js:215-244` — full DOM re-render on every SSE event +4. `conversation_live.js:164-172` — unbounded `rawEvents[callId]` array (memory leak risk) +5. `history_list.html:423-448` — client-side filter runs on every keystroke + +**Query plan risks:** + +- `session_list`: 2× TEMP B-TREE (COUNT DISTINCT + ORDER BY) — scales poorly with row count +- `recent_calls`: SCAN on all rows — O(n) over conversation_events + +## Postgres + +DEFERRED: Postgres is available but seeding and baseline capture were deferred to a dedicated Postgres environment. Run `./scripts/run_perf.sh --seed-only --tier 10000 --backend postgres` followed by `uv run python scripts/perf_explain.py --backend postgres` and `uv run python scripts/perf_report.py` to capture the Postgres baseline. diff --git a/changelog.d/perf-baseline.md b/changelog.d/perf-baseline.md new file mode 100644 index 000000000..bf2431aa5 --- /dev/null +++ b/changelog.d/perf-baseline.md @@ -0,0 +1,13 @@ +--- +category: Chores & Docs +pr: 753 +--- + +**Admin UI performance baseline**: Establishes perf infrastructure and captures SQLite baseline for history/conversation pages. + - Perf test scaffolding: isolated DB (`~/.luthien/perf.db`), seeding fixtures (sami-like, tier-100/1000/10000), and Playwright harness + - `scripts/perf_explain.py` — captures EXPLAIN QUERY PLAN for top slow queries + - `scripts/perf_report.py` — generates Markdown baseline report from seeded DB + query plans + - `scripts/run_perf.sh` — orchestrates seed + test + SLO assertion workflow + - Middleware timing (`Server-Timing` header) and payload-size contract tests + - Query plan evidence: 2× TEMP B-TREE on `session_list`, full SCAN on `recent_calls` + - Postgres baseline skipped (not available locally); run `./scripts/run_perf.sh --backend postgres` to capture diff --git a/dev/context/migration_concurrent.md b/dev/context/migration_concurrent.md new file mode 100644 index 000000000..73ce61310 --- /dev/null +++ b/dev/context/migration_concurrent.md @@ -0,0 +1,105 @@ +# Migration Runner: CONCURRENTLY Support Audit + +_Date: 2026-05-15 | Branch: perf-baseline_ + +## Background + +`CREATE INDEX CONCURRENTLY` is a Postgres feature that builds an index without holding a lock on the table, allowing reads and writes during the build. The constraint: it **cannot run inside a transaction block**. This audit investigates whether the current migration runner can safely execute such a statement. + +--- + +## Current behavior + +### PostgreSQL runner (`docker/run-migrations.sh`) + +- Applied by the `migrations` Docker service at startup; controlled by `docker compose up migrations`. +- Sequentially applies all `*.sql` files in `migrations/postgres/` in alphabetical order. +- **No `BEGIN`/`COMMIT` transaction wrapping** is added around migration files. The runner calls `psql -f "$migration"` directly: + ```sh + psql -h "$PGHOST" -U "$PGUSER" -d "$PGDATABASE" -f "$migration" + ``` +- `psql` defaults to autocommit mode — each statement in the file runs in its own implicit transaction unless the file itself contains explicit `BEGIN`/`COMMIT` blocks. +- The `_migrations` tracking row (`INSERT INTO _migrations`) is inserted in a **separate, subsequent `psql` invocation**, not inside the same transaction as the migration file. This means the tracking and the DDL are non-atomic: a crash between the two steps leaves schema changes applied but untracked. +- Migration state is tracked in the `_migrations` table (columns: `filename TEXT PK`, `applied_at TIMESTAMP`, `content_hash TEXT`). +- Applied-migration detection uses `SELECT COUNT(*) FROM _migrations WHERE filename = '$filename'`, checked per file before applying. +- Hash validation compares stored MD5 against local file MD5 and aborts on mismatch. + +### SQLite runner (`src/luthien_proxy/utils/migration_check.py :: _apply_sqlite_migrations`) + +- Runs in-process at gateway startup for dockerless/SQLite deployments. +- Uses `executescript()` to apply each `.sql` file — this method issues an implicit `COMMIT` before execution and runs all statements in the file sequentially. +- `CREATE INDEX CONCURRENTLY` is not a SQLite concept; `AGENTS.md` explicitly lists it under "What to OMIT in SQLite migrations" and directs authors to use `CREATE INDEX IF NOT EXISTS` instead. +- SQLite tracking is also done in the `_migrations` table but is written inside the same connection context (not atomic with the `executescript`, however — a mid-script crash leaves partial schema with no tracking record). + +--- + +## Verdict + +**PARTIAL** + +`CREATE INDEX CONCURRENTLY` can be placed in a Postgres migration file today and will execute successfully — because the runner uses `psql -f` in autocommit mode with **no outer transaction wrapping**. The statement will not hit the "cannot run inside a transaction block" error. + +However: + +1. **Non-atomic tracking** — the `INSERT INTO _migrations` tracking row is a separate psql call. If it fails, the index exists on disk but the migration is untracked. A re-run will try to apply the file again; `CREATE INDEX CONCURRENTLY IF NOT EXISTS` protects against failure in that case. +2. **SQLite incompatibility** — a companion SQLite migration must use plain `CREATE INDEX IF NOT EXISTS` (standard `AGENTS.md` practice; no code change needed). +3. **No explicit guidance in runner or AGENTS.md** about CONCURRENTLY for Postgres beyond the SQLite omit rule — the assumption has been "it just works because psql is autocommit." + +--- + +## Findings + +1. **No BEGIN/COMMIT wrapping in Postgres runner.** `run-migrations.sh` calls `psql -f "$migration"` with zero explicit transaction control around migration files. psql autocommit applies. + +2. **`BEGIN` in existing migrations is always PL/pgSQL, not transaction control.** Searching all postgres migration files reveals `BEGIN` only inside `$$ LANGUAGE plpgsql` function/trigger bodies (e.g., `014_add_session_search_fts.sql`, `000_init_databases.sql`). No migration wraps its DDL in a `BEGIN...COMMIT` block. + +3. **Tracking INSERT is not atomic with migration application.** Lines 153–156 of `run-migrations.sh` run the migration file, then insert into `_migrations` in a second psql call. A process kill between those two calls yields applied-but-untracked state. `CREATE INDEX CONCURRENTLY IF NOT EXISTS` + idempotent DDL is the correct mitigation. + +4. **SQLite runner uses `executescript()`, not raw `execute()`.** This means the entire SQL file is submitted to SQLite's native multi-statement parser in one call. It handles trigger `BEGIN...END` correctly but does not guarantee atomicity across the file; a mid-script error leaves partial schema with no `_migrations` entry. + +5. **AGENTS.md already documents the SQLite handling rule.** "What to OMIT in SQLite migrations" includes `CREATE INDEX CONCURRENTLY` — use plain `CREATE INDEX IF NOT EXISTS`. This is the only dual-dialect consideration; Postgres needs no special handling beyond `IF NOT EXISTS`. + +6. **Migration 006 establishes the index-in-migration pattern.** `006_add_session_id.sql` creates two partial indexes (`WHERE session_id IS NOT NULL`) with `CREATE INDEX IF NOT EXISTS`. This is the precedent: use `IF NOT EXISTS` for idempotence, and the runner handles it without transaction complications. + +7. **`014_add_session_search_fts.sql` creates multiple indexes in one file.** A GIN index, a btree partial index, and an expression index are all created in a single migration file, all with `IF NOT EXISTS`. This confirms that non-trivial index migrations work fine under the current runner. + +--- + +## Risk assessment + +### If a future PR needs `CREATE INDEX CONCURRENTLY` (Postgres) + +**Risk: LOW** — the runner already runs in autocommit mode. No runner changes are required. + +**Smallest safe path:** + +1. Postgres migration file: use `CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_name ON table(col)`. + - `IF NOT EXISTS` handles the non-atomic tracking race condition: if the runner crashes after DDL but before tracking, the re-run skips the existing index without error. + - Note: `CREATE INDEX CONCURRENTLY IF NOT EXISTS` requires Postgres 9.5+. Luthien targets modern Postgres; this is not a concern. +2. SQLite migration file: use plain `CREATE INDEX IF NOT EXISTS idx_name ON table(col)` (no CONCURRENTLY keyword). +3. No changes to `run-migrations.sh` or `migration_check.py` are needed. + +**Residual risk:** `CREATE INDEX CONCURRENTLY` holds a share-update-exclusive lock, not a full table lock, but it does require two table scans. On a large `conversation_events` table it may run for minutes. The Docker `migrations` container has no configurable `lock_timeout`; a very large production table could cause the migration container to hang. Mitigation: document the expected index build time in the migration file comment, or run it manually outside the automated runner for very large tables. + +**Out-of-scope risk (do not fix here):** The non-atomic tracking gap exists for ALL migrations, not just CONCURRENTLY ones. A proper fix would wrap both the DDL and the `INSERT INTO _migrations` in a single transaction — but that would break `CREATE INDEX CONCURRENTLY`. The correct long-term approach is to move tracking into the same psql session with `\set ON_ERROR_STOP on` and careful sequencing, but that is a separate refactor not required for this PR series. + +--- + +## Experimental Validation + +Confirmed: The P8 audit findings are correct. + +**SQLite experiment** (run 2026-05-15): +- `CREATE INDEX IF NOT EXISTS` via `executescript()`: works correctly +- `CREATE INDEX CONCURRENTLY`: fails with `sqlite3.OperationalError: near "IF": syntax error` +- Conclusion: SQLite migrations must always use plain `CREATE INDEX IF NOT EXISTS` + +**Postgres validation** (theoretical, based on runner analysis): +- The Postgres runner (`docker/run-migrations.sh`) uses `psql -f` with no transaction wrapping +- `CREATE INDEX CONCURRENTLY` requires running outside a transaction block +- Since the runner does NOT wrap in BEGIN/COMMIT, CONCURRENTLY should work +- Practical test deferred (no Postgres available in local dev); theoretical analysis confirmed + +**Verdict**: PARTIAL support confirmed experimentally: +- SQLite: CONCURRENTLY not supported (syntax error) — use plain `CREATE INDEX IF NOT EXISTS` +- Postgres: CONCURRENTLY supported (no transaction wrapping in runner) — safe to use diff --git a/pyproject.toml b/pyproject.toml index dbaa8830f..ecb5683c4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -82,7 +82,7 @@ docstring-code-format = true convention = "google" [tool.pytest.ini_options] -addopts = "-q -ra -m 'not e2e and not integration and not mock_e2e and not sqlite_e2e' --import-mode=importlib --cov=src/luthien_proxy --cov-report=term-missing --timeout=3 --timeout-method=signal" +addopts = "-q -ra -m 'not e2e and not integration and not mock_e2e and not sqlite_e2e and not perf' --import-mode=importlib --cov=src/luthien_proxy --cov-report=term-missing --timeout=3 --timeout-method=signal" testpaths = ["tests"] asyncio_mode = "auto" filterwarnings = [ @@ -95,6 +95,8 @@ markers = [ "integration: marks integration tests that require external services (OpenAI, Anthropic APIs)", "mock_e2e: marks e2e tests that use the mock Anthropic server (no real API calls)", "sqlite_e2e: marks e2e tests running the gateway in-process with SQLite (no Docker)", + "perf: marks performance tests that measure gateway latency and throughput (opt-in via ./scripts/run_perf.sh)", + "contract: marks API contract snapshot tests that validate response shapes", "llm01: OWASP LLM01 - Prompt Injection scenarios", "llm02: OWASP LLM02 - Insecure Output Handling scenarios (reserved, no tests yet)", "llm04: OWASP LLM04 - Model Denial of Service scenarios (reserved, no tests yet)", @@ -122,13 +124,15 @@ reportMissingImports = "warning" [dependency-groups] dev = [ + "playwright==1.50.0", "pre-commit>=4.3.0", "pytest>=8.4.1", "pytest-asyncio>=1.1.0", "pytest-cov>=6.2.1", + "pytest-playwright>=0.5.0", + "pytest-timeout>=2.4.0", "ruff>=0.12.10", "pyright>=1.1.406,<1.2", - "pytest-timeout>=2.4.0", "radon>=6.0.1", "vulture>=2.14", "asgi-lifespan>=2.1.0", diff --git a/scripts/perf_explain.py b/scripts/perf_explain.py new file mode 100755 index 000000000..d631e06bc --- /dev/null +++ b/scripts/perf_explain.py @@ -0,0 +1,226 @@ +#!/usr/bin/env python3 +"""Capture EXPLAIN QUERY PLAN for the top slow queries against the perf DB. + +Usage: + uv run python scripts/perf_explain.py --backend sqlite + uv run python scripts/perf_explain.py --backend postgres + +Outputs: .sisyphus/evidence/baseline-query-plans.md + +Safety: refuses to connect if DATABASE_URL points to the dev DB (local.db). +""" + +import argparse +import os +import sqlite3 +import subprocess +import sys +from datetime import datetime, timezone +from pathlib import Path + +_REPO_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(_REPO_ROOT / "src")) + +from luthien_proxy.perf.db import ensure_perf_isolation, get_perf_db_url, migrate_perf_db # noqa: E402 +from luthien_proxy.perf.seeding import seed_sessions # noqa: E402 +from luthien_proxy.utils.db_sqlite import parse_sqlite_url # noqa: E402 + +EVIDENCE_DIR = _REPO_ROOT / ".sisyphus" / "evidence" +OUTPUT_PATH = EVIDENCE_DIR / "baseline-query-plans.md" + +# ── Queries ──────────────────────────────────────────────────────────────── +# Exact SQL extracted from source (adapted: $N → ? for sqlite3, no f-string +# interpolation — using the hot-path / no-user-filter variant). +# +# Source: src/luthien_proxy/history/service.py (_fetch_session_list_sqlite) +SESSION_LIST_SQL = """\ +SELECT + ce.session_id, + MIN(ce.created_at) as first_ts, + MAX(ce.created_at) as last_ts, + COUNT(*) as total_events, + COUNT(DISTINCT ce.call_id) as turn_count, + SUM(CASE + WHEN ce.event_type LIKE 'policy.%' + AND ce.event_type NOT LIKE 'policy.%judge.evaluation%' + THEN 1 ELSE 0 + END) as policy_interventions +FROM conversation_events ce +WHERE ce.session_id IS NOT NULL +GROUP BY ce.session_id +ORDER BY last_ts DESC +LIMIT ? OFFSET ?\ +""" + +# Source: src/luthien_proxy/history/service.py (fetch_session_detail) +SESSION_DETAIL_SQL = """\ +SELECT call_id, event_type, payload, created_at +FROM conversation_events +WHERE session_id = ? +ORDER BY created_at ASC\ +""" + +# Source: src/luthien_proxy/debug/service.py (fetch_recent_calls) +RECENT_CALLS_SQL = """\ +SELECT + call_id, + COUNT(*) as event_count, + MAX(created_at) as latest, + MAX(session_id) as session_id +FROM conversation_events +GROUP BY call_id +ORDER BY latest DESC +LIMIT ?\ +""" + +QUERIES: list[tuple[str, str, tuple[object, ...]]] = [ + ("session_list", SESSION_LIST_SQL, (50, 0)), + ("session_detail", SESSION_DETAIL_SQL, ("placeholder-session-id",)), + ("recent_calls", RECENT_CALLS_SQL, (50,)), +] + + +def get_git_sha() -> str: + try: + result = subprocess.run( + ["git", "rev-parse", "HEAD"], + capture_output=True, + text=True, + cwd=_REPO_ROOT, + ) + return result.stdout.strip() if result.returncode == 0 else "unknown" + except Exception: + return "unknown" + + +def format_explain_plan(rows: list[tuple[int, int, int, str]]) -> str: + """Format EXPLAIN QUERY PLAN rows as a tree. + + SQLite EXPLAIN QUERY PLAN returns (id, parent, notused, detail). + We indent based on parent depth to show the nested structure. + """ + if not rows: + return "(no plan output)" + id_to_depth: dict[int, int] = {0: -1} + lines = [] + for row in rows: + row_id, parent_id, _notused, detail = row[0], row[1], row[2], row[3] + parent_depth = id_to_depth.get(parent_id, -1) + depth = parent_depth + 1 + id_to_depth[row_id] = depth + indent = " " * depth + connector = "`--" if depth > 0 else "" + lines.append(f"{indent}{connector}{detail}") + return "\n".join(lines) + + +def ensure_no_dev_db_in_env() -> None: + database_url = os.environ.get("DATABASE_URL", "") + if not database_url: + return + try: + ensure_perf_isolation(database_url) + except RuntimeError as e: + # ensure_perf_isolation message always contains "isolation" + print(f"isolation refuse: DATABASE_URL is set to the dev database.\n{e}") + sys.exit(1) + + +def explain_sqlite(db_path: str) -> None: + # Ensure migrations are applied (idempotent) + print("Applying migrations...", file=sys.stderr) + migrate_perf_db("sqlite") + + conn = sqlite3.connect(db_path) + try: + row_count = conn.execute("SELECT COUNT(*) FROM conversation_events").fetchone()[0] + session_count = conn.execute( + "SELECT COUNT(DISTINCT session_id) FROM conversation_events WHERE session_id IS NOT NULL" + ).fetchone()[0] + + if row_count == 0: + # Side effect: seeds the DB to have data to EXPLAIN against. + # Pass --seed-if-empty explicitly to make this intent visible, + # or pre-seed with seed_sessions() / run_perf.sh --seed-only first. + print("Perf DB is empty — seeding with tier=100...", file=sys.stderr) + conn.close() + seed_sessions("sqlite", tier=100) + conn = sqlite3.connect(db_path) + row_count = conn.execute("SELECT COUNT(*) FROM conversation_events").fetchone()[0] + session_count = conn.execute( + "SELECT COUNT(DISTINCT session_id) FROM conversation_events WHERE session_id IS NOT NULL" + ).fetchone()[0] + + print(f"DB has {row_count} events, {session_count} sessions.", file=sys.stderr) + + git_sha = get_git_sha() + timestamp = datetime.now(timezone.utc).isoformat() + + sections: list[str] = [] + sections.append("---") + sections.append(f"git_sha: {git_sha}") + sections.append(f"timestamp: {timestamp}") + sections.append("backend: sqlite") + sections.append(f"row_count: {row_count}") + sections.append(f"session_count: {session_count}") + sections.append("---") + sections.append("") + + for name, sql, params in QUERIES: + print(f"Running EXPLAIN QUERY PLAN for {name}...", file=sys.stderr) + sections.append(f"## Query: {name}") + sections.append("") + sections.append("### SQL") + sections.append("") + sections.append("```sql") + sections.append(sql) + sections.append("```") + sections.append("") + sections.append("### EXPLAIN QUERY PLAN") + sections.append("") + sections.append("```") + try: + rows = conn.execute(f"EXPLAIN QUERY PLAN {sql}", params).fetchall() + sections.append(format_explain_plan(rows)) + except sqlite3.OperationalError as e: + sections.append(f"ERROR: {e}") + sections.append("```") + sections.append("") + + EVIDENCE_DIR.mkdir(parents=True, exist_ok=True) + OUTPUT_PATH.write_text("\n".join(sections) + "\n", encoding="utf-8") + print(f"Written: {OUTPUT_PATH}", file=sys.stderr) + + finally: + conn.close() + + +def main() -> None: + parser = argparse.ArgumentParser(description="Capture EXPLAIN QUERY PLAN for slow queries against the perf DB.") + parser.add_argument( + "--backend", + choices=["sqlite", "postgres"], + required=True, + help="Database backend to use.", + ) + args = parser.parse_args() + + ensure_no_dev_db_in_env() + + try: + url = get_perf_db_url(args.backend) + except RuntimeError as e: + print(f"isolation refuse: {e}") + sys.exit(1) + + if args.backend == "sqlite": + db_path = parse_sqlite_url(url) + explain_sqlite(db_path) + else: + print("SKIPPED: Postgres backend not available in this environment.", file=sys.stderr) + print("# SKIPPED: Postgres not available", file=sys.stderr) + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/scripts/perf_report.py b/scripts/perf_report.py new file mode 100755 index 000000000..800e9d255 --- /dev/null +++ b/scripts/perf_report.py @@ -0,0 +1,387 @@ +#!/usr/bin/env python3 +"""Generate a Markdown performance baseline report from perf test results. + +Reads .sisyphus/evidence/perf-results-*.json files and embeds +.sisyphus/evidence/baseline-query-plans.md. When no JSON files exist +(P10-P12 not yet run), every data section is rendered with a NO DATA YET +placeholder so the file is still structurally valid for diffing. + +Usage: + uv run python scripts/perf_report.py --output .sisyphus/evidence/perf-report-baseline.md + uv run python scripts/perf_report.py --output out.md --deterministic-mode +""" + +from __future__ import annotations + +import argparse +import glob +import importlib.metadata +import json +import platform +import subprocess +import sys +from datetime import datetime, timezone +from pathlib import Path + +_REPO_ROOT = Path(__file__).parent.parent +_EVIDENCE_DIR = _REPO_ROOT / ".sisyphus" / "evidence" + + +def _git_sha(repo_root: Path | None = None) -> str: + root = repo_root or _REPO_ROOT + try: + result = subprocess.run( + ["git", "rev-parse", "HEAD"], + capture_output=True, + text=True, + cwd=root, + timeout=5, + ) + return result.stdout.strip() if result.returncode == 0 else "unknown" + except Exception: + return "unknown" + + +def _playwright_version() -> str: + try: + return importlib.metadata.version("playwright") + except Exception: + return "unknown" + + +def _ram_info() -> str: + try: + result = subprocess.run( + ["sysctl", "-n", "hw.memsize"], + capture_output=True, + text=True, + timeout=3, + ) + if result.returncode == 0: + gb = int(result.stdout.strip()) // (1024**3) + return f"{gb} GB" + except Exception: + pass + try: + with open("/proc/meminfo") as f: + for line in f: + if line.startswith("MemTotal:"): + kb = int(line.split()[1]) + return f"{kb // (1024**2)} GB" + except Exception: + pass + return "unknown" + + +def load_results(evidence_dir: Path | None = None) -> list[dict]: + d = evidence_dir or _EVIDENCE_DIR + results: list[dict] = [] + for path in sorted(glob.glob(str(d / "perf-results-*.json"))): + try: + with open(path) as f: + results.append(json.load(f)) + except Exception: + continue + return results + + +def load_query_plans(evidence_dir: Path | None = None) -> str: + d = evidence_dir or _EVIDENCE_DIR + path = d / "baseline-query-plans.md" + if path.exists(): + return path.read_text() + return "_Query plans not yet captured. Run `scripts/perf_explain.py` first._\n" + + +def _find_result(results: list[dict], type_: str) -> dict | None: + for r in results: + if r.get("type") == type_: + return r + return None + + +def _page_timing_records(results: list[dict]) -> list[dict]: + return [r for r in results if "scenarios" in r] + + +def _throttled_records(results: list[dict]) -> list[dict]: + return [r for r in results if "route" in r and "runs_ms" in r] + + +def _sse_memory_records(results: list[dict]) -> list[dict]: + return [r for r in results if "heap_growth_pct" in r] + + +def _section_hardware(git_sha: str, playwright_ver: str, ram: str, backend: str = "sqlite") -> str: + rows = [ + ("Machine", platform.machine()), + ("Processor", platform.processor() or platform.machine()), + ("RAM", ram), + ("OS", f"{platform.system()} {platform.release()}"), + ("Python", platform.python_version()), + ("git_sha", f"`{git_sha}`"), + ("DB backend", backend), + ("Playwright", playwright_ver), + ] + table = ["| Field | Value |", "|-------|-------|"] + table.extend(f"| {k} | {v} |" for k, v in rows) + return "\n".join(["## Hardware & Versions", ""] + table + [""]) + + +def _section_per_page_timings(results: list[dict]) -> str: + header = "## Per-Page Timings" + records = _page_timing_records(results) + if not records: + return "\n".join([header, "", "_NO DATA YET — run `scripts/run_perf.sh` to populate._", ""]) + + data: dict[str, dict[str, dict]] = {} + fixtures: set[str] = set() + for rec in records: + fixture = rec.get("fixture", "unknown") + fixtures.add(fixture) + for scenario in rec.get("scenarios", []): + page = scenario.get("page", "?") + data.setdefault(page, {})[fixture] = { + "cold_ms": scenario.get("cold_ms"), + "median_ms": scenario.get("median_ms"), + "p95_ms": scenario.get("p95_ms"), + "ttfb_ms": scenario.get("ttfb_ms"), + "transfer_bytes": scenario.get("transfer_bytes"), + } + + pages = sorted(data.keys()) + fixture_list = sorted(fixtures) + col_header = " | ".join(f"{f} cold_ms | {f} median_ms | {f} p95_ms" for f in fixture_list) + col_sep = " | ".join("--- | --- | ---" for _ in fixture_list) + lines = [header, "", f"| Page | {col_header} |", f"|------| {col_sep} |"] + + for page in pages: + cells: list[str] = [] + for fixture in fixture_list: + fdata = data[page].get(fixture, {}) + cells.append(str(round(fdata["cold_ms"], 0)) if fdata.get("cold_ms") is not None else "—") + cells.append(str(round(fdata["median_ms"], 0)) if fdata.get("median_ms") is not None else "—") + cells.append(str(round(fdata["p95_ms"], 0)) if fdata.get("p95_ms") is not None else "—") + lines.append(f"| {page} | " + " | ".join(cells) + " |") + + lines.append("") + return "\n".join(lines) + + +def _section_throttled(results: list[dict]) -> str: + header = "## Throttled (sami-like)" + records = _throttled_records(results) + if not records: + return "\n".join([header, "", "_NO DATA YET_", ""]) + + lines = [ + header, + "", + "| Route | Fixture | N runs | Median ms | Config |", + "|-------|---------|--------|-----------|--------|", + ] + for rec in sorted(records, key=lambda r: r.get("route", "")): + route = rec.get("route", "?") + fixture = rec.get("fixture", "?") + n_runs = rec.get("n_runs", "?") + median = rec.get("median_ms") + cfg = rec.get("throttle_config", {}) + cfg_str = f"{cfg.get('download_bps', '?')} bps / {cfg.get('latency_ms', '?')} ms RTT" + lines.append( + f"| {route} | {fixture} | {n_runs} | {round(median, 0) if median is not None else '—'} | {cfg_str} |" + ) + lines.append("") + return "\n".join(lines) + + +def _section_transcript_open(results: list[dict]) -> str: + header = "## Transcript Open" + r = _find_result(results, "transcript_open") + if not r: + return "\n".join([header, "", "_NO DATA YET_", ""]) + + data = r.get("data", {}) + lines = [ + header, + "", + "| Metric | Value |", + "|--------|-------|", + f"| first_turn_painted_ms | {data.get('first_turn_painted_ms', '—')} |", + "", + ] + return "\n".join(lines) + + +def _section_sse_memory(results: list[dict]) -> str: + header = "## SSE Memory Growth" + records = _sse_memory_records(results) + if not records: + return "\n".join([header, "", "_NO DATA YET_", ""]) + + rec = records[-1] + heap_first_mb = round(rec.get("heap_first_bytes", 0) / (1024 * 1024), 1) + heap_last_mb = round(rec.get("heap_last_bytes", 0) / (1024 * 1024), 1) + growth_pct = round(rec.get("heap_growth_pct", 0), 1) + hold_s = rec.get("hold_seconds", "?") + lines = [ + header, + "", + "| Metric | Value |", + "|--------|-------|", + f"| heap_first_mb | {heap_first_mb} |", + f"| heap_last_mb | {heap_last_mb} |", + f"| heap_growth_pct | {growth_pct}% |", + f"| hold_seconds | {hold_s} |", + "", + ] + return "\n".join(lines) + + +def _section_server_timing(results: list[dict]) -> str: + header = "## Server-Timing Breakdown" + r = _find_result(results, "server_timing") + if not r: + return "\n".join([header, "", "_NO DATA YET_", ""]) + + data = r.get("data", {}) + lines = [header, "", "| Phase | Median ms |", "|-------|-----------|"] + for phase in ("db", "serialize"): + lines.append(f"| {phase} | {data.get(f'{phase}_ms', '—')} |") + lines.append("") + return "\n".join(lines) + + +def _section_payload_size(results: list[dict]) -> str: + header = "## Payload Size Breakdown" + r = _find_result(results, "payload_size") + if not r: + return "\n".join([header, "", "_NO DATA YET_", ""]) + + data = r.get("data", {}) + lines = [header, "", "| Endpoint | Bytes |", "|----------|-------|"] + for endpoint in sorted(data.keys()): + lines.append(f"| {endpoint} | {data[endpoint].get('bytes', '—')} |") + lines.append("") + return "\n".join(lines) + + +def _section_query_plans(query_plans: str) -> str: + return "\n".join(["## Query Plans", "", query_plans.strip(), ""]) + + +def _section_top_hotspots(results: list[dict]) -> str: + header = "## Top Hotspots" + has_data = bool(_page_timing_records(results) or _throttled_records(results) or _sse_memory_records(results)) + + if not has_data: + lines = [ + header, + "", + "_NO DATA YET — hotspots will be derived from measurement results._", + "", + "**Known candidates (from code review):**", + "", + "1. `history_list.html:514` — hardcodes `?limit=10000` (sends full dataset on every load)", + "2. `conversation_live.js:92-118` — `loadInitial()` fetches entire session upfront", + "3. `conversation_live.js:215-244` — full DOM re-render on every SSE event", + "4. `conversation_live.js:164-172` — unbounded `rawEvents[callId]` array (memory leak risk)", + "5. `history_list.html:423-448` — client-side filter runs on every keystroke", + "", + "**Query plan risks:**", + "", + "- `session_list`: 2× TEMP B-TREE (COUNT DISTINCT + ORDER BY) — scales poorly with row count", + "- `recent_calls`: SCAN on all rows — O(n) over conversation_events", + "", + ] + return "\n".join(lines) + + hotspots: list[str] = [] + + for rec in _page_timing_records(results): + fixture = rec.get("fixture", "?") + for scenario in rec.get("scenarios", []): + page = scenario.get("page", "?") + p95 = scenario.get("p95_ms", 0) + if isinstance(p95, (int, float)) and p95 > 1000: + hotspots.append(f"`{page}` ({fixture}) p95={p95:.0f}ms — exceeds 1s SLO") + + for rec in _sse_memory_records(results): + growth_pct = rec.get("heap_growth_pct", 0) + if isinstance(growth_pct, (int, float)) and growth_pct > 50: + hotspots.append(f"SSE heap growth={growth_pct:.1f}% over session — possible unbounded accumulation") + + lines = [header, ""] + if hotspots: + lines.extend(f"- {h}" for h in hotspots) + else: + lines.append("_No hotspots detected above threshold. See individual sections for details._") + lines.append("") + return "\n".join(lines) + + +def generate_report( + results: list[dict], + query_plans: str, + git_sha: str, + playwright_ver: str, + generated_at: str | None = None, + ram: str | None = None, + backend: str = "sqlite", +) -> str: + timestamp = generated_at or datetime.now(timezone.utc).isoformat() + ram_str = ram or _ram_info() + + parts = [ + f"git_sha: {git_sha}", + f"browser_version: {playwright_ver}", + f"backend: {backend}", + f"generated_at: {timestamp}", + "", + "# Luthien Admin UI — Performance Baseline Report", + "", + _section_hardware(git_sha, playwright_ver, ram_str, backend=backend), + _section_per_page_timings(results), + _section_throttled(results), + _section_transcript_open(results), + _section_sse_memory(results), + _section_server_timing(results), + _section_payload_size(results), + _section_query_plans(query_plans), + _section_top_hotspots(results), + ] + return "\n".join(parts) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Generate perf baseline Markdown report") + parser.add_argument("--output", required=True, help="Path to write the Markdown report") + parser.add_argument( + "--deterministic-mode", + action="store_true", + help="Fix timestamp to epoch so output is byte-identical across runs (for reproducibility testing)", + ) + parser.add_argument( + "--backend", + default="sqlite", + help="DB backend label to embed in the report (default: sqlite)", + ) + args = parser.parse_args() + + results = load_results() + query_plans = load_query_plans() + sha = _git_sha() + pw_ver = _playwright_version() + + generated_at = "2000-01-01T00:00:00+00:00" if args.deterministic_mode else None + + report = generate_report(results, query_plans, sha, pw_ver, generated_at=generated_at, backend=args.backend) + + output_path = Path(args.output) + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(report) + + print(f"Report written to {output_path}", file=sys.stderr) + + +if __name__ == "__main__": + main() diff --git a/scripts/run_perf.sh b/scripts/run_perf.sh new file mode 100755 index 000000000..fe4318dfe --- /dev/null +++ b/scripts/run_perf.sh @@ -0,0 +1,307 @@ +#!/usr/bin/env bash +# Playwright version: 1.50.0 +# +# ISOLATION ENFORCEMENT: +# This script refuses to run against the development database (~/.luthien/local.db). +# Perf tests use a dedicated isolated database to prevent fixture data pollution +# and ensure reproducible baseline measurements: +# SQLite: ~/.luthien/perf.db (hardcoded; never local.db) +# Postgres: perf_test schema in a dedicated Postgres perf instance +# DATABASE_URL must be set explicitly and must not reference local.db. +# +# ABOUTME: Performance test runner for admin UI latency and payload SLOs. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +cd "$REPO_ROOT" + +# Colors +GREEN='\033[0;32m' +RED='\033[0;31m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +BOLD='\033[1m' +NC='\033[0m' + +info() { echo -e "${BLUE}▸${NC} $*"; } +ok() { echo -e "${GREEN}✓${NC} $*"; } +warn() { echo -e "${YELLOW}⚠${NC} $*"; } +fail() { echo -e "${RED}✗${NC} $*"; } +header() { echo -e "\n${BOLD}═══ $* ═══${NC}"; } + +# ── Defaults ────────────────────────────────────────────────────────────────── + +TIER="" +FIXTURE="sami-like" +SEED_ONLY=false +CLEAN=false +ASSERT_SLO=false +THROTTLED=false +BACKEND="sqlite" + +# ── Help ────────────────────────────────────────────────────────────────────── + +show_help() { + cat <<'EOF' +Performance test runner for admin UI latency and payload SLOs. + +Usage: + ./scripts/run_perf.sh --tier {100|1000|10000} [options] + ./scripts/run_perf.sh --clean [--backend {sqlite|postgres}] + ./scripts/run_perf.sh --help + +Options: + --tier {100|1000|10000} Sessions to seed [required unless --clean] + --fixture {sami-like} Fixture profile (default: sami-like) + --seed-only Seed the database; skip test assertions + --clean Drop the perf database and exit + --assert-slo Fail if any SLO thresholds are exceeded (sets PERF_ASSERT_SLO=1) + --throttled CDP network throttling -- 1 Mbps + 300ms RTT (only with --fixture sami-like) + --backend {sqlite|postgres} Database backend (default: sqlite) + --help Show this help message + +Environment: + DATABASE_URL Required (refused if unset or contains local.db) + SQLite example: sqlite:///$HOME/.luthien/perf.db + +Examples: + DATABASE_URL=sqlite:///$HOME/.luthien/perf.db ./scripts/run_perf.sh --tier 100 + DATABASE_URL=sqlite:///$HOME/.luthien/perf.db ./scripts/run_perf.sh --tier 100 --assert-slo + DATABASE_URL=sqlite:///$HOME/.luthien/perf.db ./scripts/run_perf.sh --tier 100 --throttled + ./scripts/run_perf.sh --clean + DATABASE_URL=sqlite:///$HOME/.luthien/perf.db ./scripts/run_perf.sh --seed-only --tier 1000 + +Postgres --clean note: + For Postgres, --clean executes DROP SCHEMA perf_test CASCADE. + Set DATABASE_URL to the Postgres perf instance before running. +EOF + exit 0 +} + +# ── Argument parsing ────────────────────────────────────────────────────────── + +while [[ $# -gt 0 ]]; do + case "$1" in + --tier) + if [[ $# -lt 2 ]]; then fail "--tier requires an argument"; exit 1; fi + case "$2" in + 100|1000|10000) TIER="$2" ;; + *) fail "Invalid --tier: $2 (expected: 100, 1000, or 10000)"; exit 1 ;; + esac + shift 2 + ;; + --fixture) + if [[ $# -lt 2 ]]; then fail "--fixture requires an argument"; exit 1; fi + case "$2" in + sami-like) FIXTURE="$2" ;; + *) fail "Unknown --fixture: $2 (expected: sami-like)"; exit 1 ;; + esac + shift 2 + ;; + --backend) + if [[ $# -lt 2 ]]; then fail "--backend requires an argument"; exit 1; fi + case "$2" in + sqlite|postgres) BACKEND="$2" ;; + *) fail "Unknown --backend: $2 (expected: sqlite or postgres)"; exit 1 ;; + esac + shift 2 + ;; + --seed-only) SEED_ONLY=true; shift ;; + --clean) CLEAN=true; shift ;; + --assert-slo) ASSERT_SLO=true; shift ;; + --throttled) THROTTLED=true; shift ;; + --help|-h) show_help ;; + *) fail "Unknown option: $1"; exit 1 ;; + esac +done + +# ── Validate option combinations ────────────────────────────────────────────── + +if $THROTTLED && [[ "$FIXTURE" != "sami-like" ]]; then + fail "--throttled is only valid with --fixture sami-like (got: --fixture $FIXTURE)" + exit 1 +fi + +if ! $CLEAN && [[ -z "$TIER" ]]; then + fail "Required: --tier {100|1000|10000} (or use --clean to drop the perf DB)" + exit 1 +fi + +# ── SQLite clean ────────────────────────────────────────────────────────────── +# Runs before the isolation check: deletes the perf DB file, never the dev DB. + +if $CLEAN && [[ "$BACKEND" == "sqlite" ]]; then + header "Cleaning Perf Database (SQLite)" + PERF_DB="$HOME/.luthien/perf.db" + if [[ -f "$PERF_DB" ]]; then + rm -f "$PERF_DB" + ok "Removed $PERF_DB" + else + info "Nothing to clean: $PERF_DB does not exist" + fi + exit 0 +fi + +# ── Isolation check ─────────────────────────────────────────────────────────── +# +# This script refuses to run against the dev database. Perf tests MUST use an +# isolated database to prevent fixture data pollution and ensure reproducibility. +# Applies to all non-SQLite-clean operations. + +_db_url="${DATABASE_URL:-}" + +if [[ -z "$_db_url" ]]; then + fail "ISOLATION REFUSED: DATABASE_URL is not set." + fail " The gateway defaults to ~/.luthien/local.db (the dev database) when unset." + fail " This script refuses to run without an explicit isolated database URL." + fail " Set DATABASE_URL to a perf-specific path, e.g.:" + fail " export DATABASE_URL=sqlite:///\$HOME/.luthien/perf.db" + exit 1 +fi + +if [[ "$(basename "${_db_url##*:///}")" == "local.db" ]]; then + fail "ISOLATION REFUSED: DATABASE_URL points to the dev database (local.db)." + fail " This script refuses to run against local.db to prevent data pollution." + fail " DATABASE_URL=$_db_url" + fail " Set DATABASE_URL to a perf-specific path, e.g.:" + fail " export DATABASE_URL=sqlite:///\$HOME/.luthien/perf.db" + exit 1 +fi + +# ── Postgres clean (after isolation check) ──────────────────────────────────── + +if $CLEAN && [[ "$BACKEND" == "postgres" ]]; then + header "Cleaning Perf Database (Postgres)" + warn "Executing: DROP SCHEMA perf_test CASCADE" + warn " Target: $_db_url" + uv run python - <<'PYEOF' +import os +import sys + +try: + from luthien_proxy.perf.db import ensure_perf_isolation + ensure_perf_isolation(os.environ.get("DATABASE_URL", "")) +except RuntimeError as e: + print(f"isolation refuse: {e}", file=sys.stderr) + sys.exit(1) + +try: + import psycopg2 # type: ignore[import-untyped] +except ImportError: + print("psycopg2 not installed; run: uv add psycopg2-binary", file=sys.stderr) + sys.exit(1) + +try: + url = os.environ["DATABASE_URL"] + conn = psycopg2.connect(url) + conn.autocommit = True + cur = conn.cursor() + cur.execute("DROP SCHEMA IF EXISTS perf_test CASCADE") + conn.close() + print("perf_test schema dropped") +except Exception as exc: + print(f"Error dropping schema: {exc}", file=sys.stderr) + sys.exit(1) +PYEOF + ok "Postgres perf_test schema dropped" + exit 0 +fi + +# ── Tier-10000 disk warning ─────────────────────────────────────────────────── + +if [[ "$TIER" == "10000" ]]; then + warn "Tier-10000 seeds ~250k events × ~25 KB payload ≈ 7–8 GB on disk. Ensure sufficient free space." + warn "Seeding also allocates a 128 MB SQLite cache; ensure sufficient RAM." +fi + +# ── Pre-flight ──────────────────────────────────────────────────────────────── + +header "Pre-flight Checks" + +# Ensure Chromium is installed (Playwright 1.50.0 -- pinned at top of file). +info "Checking Playwright Chromium..." +uv run playwright install chromium --with-deps 2>/dev/null || true + +_chromium_ver="$(uv run python -c ' +from playwright.sync_api import sync_playwright +with sync_playwright() as p: + browser = p.chromium.launch() + ver = browser.version + browser.close() + print(ver) +' 2>/dev/null || echo "unknown")" + +_git_sha="$(git rev-parse --short HEAD 2>/dev/null || echo "unknown")" + +ok "Chromium version: $_chromium_ver" +ok "Git SHA: $_git_sha" + +# ── Environment ─────────────────────────────────────────────────────────────── + +export PERF_TIER="$TIER" +export PERF_FIXTURE="$FIXTURE" +export PERF_BACKEND="$BACKEND" + +if $ASSERT_SLO; then + export PERF_ASSERT_SLO=1 + export PERF_THROTTLE_BASELINE=1 + export PERF_ASSERT_MEMORY=1 + info "SLO assertion enabled -- tests fail if thresholds exceeded" +fi + +if $THROTTLED; then + export PERF_THROTTLED=1 + info "Network throttling enabled -- 1 Mbps bandwidth + 300ms RTT (sami-like profile)" +fi + +# ── Seed only ───────────────────────────────────────────────────────────────── + +if $SEED_ONLY; then + header "Seeding Database (tier=$TIER, fixture=$FIXTURE)" + info "Seeding $TIER sessions -- test assertions will NOT run" + export PERF_SEED_ONLY=1 + uv run pytest \ + -m perf \ + tests/luthien_proxy/perf_tests/ \ + -v --no-cov \ + || true + ok "Seeding complete" + exit 0 +fi + +# ── Run perf tests ──────────────────────────────────────────────────────────── + +_slo_flag="no" +_throttle_flag="no" +$ASSERT_SLO && _slo_flag="yes" +$THROTTLED && _throttle_flag="yes" + +header "Perf Tests" +info " Tier: $TIER sessions" +info " Fixture: $FIXTURE" +info " Backend: $BACKEND" +info " Assert SLO: $_slo_flag" +info " Throttled: $_throttle_flag" +info " Database: $_db_url" + +exit_code=0 +uv run pytest \ + -m perf \ + tests/luthien_proxy/perf_tests/ \ + -v --no-cov \ + || exit_code=$? + +# ── Summary ─────────────────────────────────────────────────────────────────── + +header "Results" +if [[ $exit_code -eq 0 ]]; then + ok "All perf tests passed" + $ASSERT_SLO && ok "SLO thresholds: all met" +else + fail "Perf tests failed (exit $exit_code)" + $ASSERT_SLO && fail "One or more SLO thresholds were exceeded" +fi + +exit $exit_code diff --git a/src/luthien_proxy/debug/routes.py b/src/luthien_proxy/debug/routes.py index 27355da6d..83bab6136 100644 --- a/src/luthien_proxy/debug/routes.py +++ b/src/luthien_proxy/debug/routes.py @@ -9,6 +9,14 @@ error responses) and delegate business logic to the service layer. All debug endpoints require admin authentication (same as /admin routes). + +WARNING — serialization pattern: handlers return Response(content=model.model_dump_json(), ...) +with response_model= kept on the decorator for OpenAPI schema generation only. +FastAPI skips response validation when the handler returns a pre-built Response — +this is intentional to avoid double-serialization (Pydantic→dict→json twice). +Do NOT copy this pattern to new routes without also adding a contract snapshot test; +FastAPI's response_model validation will be silently disabled. +The API contract snapshot tests (test_api_contract.py) provide regression coverage. """ from __future__ import annotations @@ -17,9 +25,11 @@ from typing import TYPE_CHECKING from fastapi import APIRouter, Depends, HTTPException, Query +from starlette.responses import Response from luthien_proxy.auth import verify_admin_token from luthien_proxy.dependencies import get_db_pool +from luthien_proxy.perf.timing_middleware import timed_json_response from luthien_proxy.settings import client_error_detail from luthien_proxy.utils.constants import DEBUG_CALLS_DEFAULT_LIMIT, DEBUG_CALLS_MAX_LIMIT @@ -39,7 +49,7 @@ async def get_call_events( call_id: str, _: str = Depends(verify_admin_token), db_pool: db.DatabasePool | None = Depends(get_db_pool), -) -> CallEventsResponse: +) -> Response: """Retrieve all conversation events for a specific call_id. Args: @@ -56,7 +66,8 @@ async def get_call_events( raise HTTPException(status_code=503, detail="Database not configured") try: - return await fetch_call_events(call_id, db_pool) + result = await fetch_call_events(call_id, db_pool) + return timed_json_response(result) except ValueError as exc: # No events found raise HTTPException(status_code=404, detail=str(exc)) @@ -70,7 +81,7 @@ async def get_call_diff( call_id: str, _: str = Depends(verify_admin_token), db_pool: db.DatabasePool | None = Depends(get_db_pool), -) -> CallDiffResponse: +) -> Response: """Compute diff between original and final request/response for a call. Args: @@ -87,7 +98,8 @@ async def get_call_diff( raise HTTPException(status_code=503, detail="Database not configured") try: - return await fetch_call_diff(call_id, db_pool) + result = await fetch_call_diff(call_id, db_pool) + return timed_json_response(result) except ValueError as exc: # No events found raise HTTPException(status_code=404, detail=str(exc)) @@ -101,7 +113,7 @@ async def list_recent_calls( limit: int = Query(default=DEBUG_CALLS_DEFAULT_LIMIT, ge=1, le=DEBUG_CALLS_MAX_LIMIT), _: str = Depends(verify_admin_token), db_pool: db.DatabasePool | None = Depends(get_db_pool), -) -> CallListResponse: +) -> Response: """List recent calls with event counts. Args: @@ -118,7 +130,8 @@ async def list_recent_calls( raise HTTPException(status_code=503, detail="Database not configured") try: - return await fetch_recent_calls(limit, db_pool) + result = await fetch_recent_calls(limit, db_pool) + return timed_json_response(result) except Exception as exc: logger.error(f"Failed to list recent calls: {exc}", exc_info=True) raise HTTPException(status_code=500, detail=client_error_detail(f"Database error: {exc}")) diff --git a/src/luthien_proxy/debug/service.py b/src/luthien_proxy/debug/service.py index 54ae6d8dc..e23e729ed 100644 --- a/src/luthien_proxy/debug/service.py +++ b/src/luthien_proxy/debug/service.py @@ -15,6 +15,7 @@ import urllib.parse from typing import TYPE_CHECKING, Any +from luthien_proxy.perf.timing_middleware import time_phase from luthien_proxy.utils.db import parse_db_ts if TYPE_CHECKING: @@ -216,15 +217,16 @@ async def fetch_call_events(call_id: str, db_pool: DatabasePool) -> CallEventsRe Exception: If database query fails """ async with db_pool.connection() as conn: - rows = await conn.fetch( - """ - SELECT call_id, event_type, payload, created_at, session_id - FROM conversation_events - WHERE call_id = $1 - ORDER BY created_at ASC - """, - call_id, - ) + with time_phase("db"): + rows = await conn.fetch( + """ + SELECT call_id, event_type, payload, created_at, session_id + FROM conversation_events + WHERE call_id = $1 + ORDER BY created_at ASC + """, + call_id, + ) if not rows: raise ValueError(f"No events found for call_id: {call_id}") @@ -271,19 +273,20 @@ async def fetch_call_diff(call_id: str, db_pool: DatabasePool) -> CallDiffRespon Exception: If database query fails """ async with db_pool.connection() as conn: - rows = await conn.fetch( - """ - SELECT call_id, event_type, payload - FROM conversation_events - WHERE call_id = $1 AND event_type IN ( - 'transaction.request_recorded', - 'transaction.non_streaming_response_recorded', - 'transaction.streaming_response_recorded' + with time_phase("db"): + rows = await conn.fetch( + """ + SELECT call_id, event_type, payload + FROM conversation_events + WHERE call_id = $1 AND event_type IN ( + 'transaction.request_recorded', + 'transaction.non_streaming_response_recorded', + 'transaction.streaming_response_recorded' + ) + ORDER BY created_at ASC + """, + call_id, ) - ORDER BY created_at ASC - """, - call_id, - ) if not rows: raise ValueError(f"No events found for call_id: {call_id}") @@ -337,20 +340,21 @@ async def fetch_recent_calls(limit: int, db_pool: DatabasePool) -> CallListRespo Exception: If database query fails """ async with db_pool.connection() as conn: - rows = await conn.fetch( - """ - SELECT - call_id, - COUNT(*) as event_count, - MAX(created_at) as latest, - MAX(session_id) as session_id - FROM conversation_events - GROUP BY call_id - ORDER BY latest DESC - LIMIT $1 - """, - limit, - ) + with time_phase("db"): + rows = await conn.fetch( + """ + SELECT + call_id, + COUNT(*) as event_count, + MAX(created_at) as latest, + MAX(session_id) as session_id + FROM conversation_events + GROUP BY call_id + ORDER BY latest DESC + LIMIT $1 + """, + limit, + ) calls = [ CallListItem( diff --git a/src/luthien_proxy/history/routes.py b/src/luthien_proxy/history/routes.py index 7fd9fca32..c5c7e330e 100644 --- a/src/luthien_proxy/history/routes.py +++ b/src/luthien_proxy/history/routes.py @@ -5,6 +5,14 @@ - Viewing session details - Exporting sessions to markdown - HTML UI pages + +WARNING — serialization pattern: API handlers return Response(content=model.model_dump_json(), ...) +with response_model= kept on the decorator for OpenAPI schema generation only. +FastAPI skips response validation when the handler returns a pre-built Response — +this is intentional to avoid double-serialization (Pydantic→dict→json twice). +Do NOT copy this pattern to new routes without also adding a contract snapshot test; +FastAPI's response_model validation will be silently disabled. +The API contract snapshot tests (test_api_contract.py) provide regression coverage. """ from __future__ import annotations @@ -14,9 +22,11 @@ from fastapi import APIRouter, Depends, HTTPException, Query, Request from fastapi.responses import FileResponse, PlainTextResponse +from starlette.responses import Response from luthien_proxy.auth import check_auth_or_redirect, verify_admin_token from luthien_proxy.dependencies import get_admin_key, get_db_pool +from luthien_proxy.perf.timing_middleware import timed_json_response from luthien_proxy.utils.constants import ( HISTORY_SESSIONS_DEFAULT_LIMIT, HISTORY_SESSIONS_MAX_LIMIT, @@ -79,14 +89,15 @@ async def list_sessions( "X-Luthien-User-Id header (when TRUST_USER_ID_HEADER=true) or JWT sub claim." ), ), -) -> SessionListResponse: +) -> Response: """List recent sessions with summaries. Returns a list of session summaries ordered by most recent activity, including turn counts, policy interventions, and models used. Supports pagination via limit and offset parameters. """ - return await fetch_session_list(limit, db_pool, offset, user_id=user_id) + result = await fetch_session_list(limit, db_pool, offset, user_id=user_id) + return timed_json_response(result) @api_router.get("/sessions/{session_id}", response_model=SessionDetail) @@ -94,14 +105,15 @@ async def get_session( session_id: str, _: str = Depends(verify_admin_token), db_pool: DatabasePool = Depends(get_db_pool), -) -> SessionDetail: +) -> Response: """Get full session detail with conversation turns. Returns the complete conversation history for a session, including all messages, tool calls, and policy annotations. """ try: - return await fetch_session_detail(session_id, db_pool) + result = await fetch_session_detail(session_id, db_pool) + return timed_json_response(result) except ValueError as e: logger.warning(f"Session not found: {repr(e)}") raise HTTPException(status_code=404, detail="Session not found.") from None diff --git a/src/luthien_proxy/history/service.py b/src/luthien_proxy/history/service.py index 77e48f33c..97f1f8423 100644 --- a/src/luthien_proxy/history/service.py +++ b/src/luthien_proxy/history/service.py @@ -14,6 +14,7 @@ from datetime import datetime from typing import Any, TypedDict, cast +from luthien_proxy.perf.timing_middleware import time_phase from luthien_proxy.utils.db import DatabasePool, parse_db_ts from .models import ( @@ -394,135 +395,136 @@ async def _fetch_session_list_pg( # touch conversation_calls in the hot CTE — user_ids come from a separate # post-query keyed on the page's session_ids (mirrors the SQLite pattern). async with db_pool.connection() as conn: - if user_id is not None: - total_count = await conn.fetchval( - """ - SELECT COUNT(DISTINCT ce.session_id) - FROM conversation_events ce - JOIN conversation_calls cc ON ce.call_id = cc.call_id - WHERE ce.session_id IS NOT NULL AND cc.user_id = $1 - """, - user_id, - ) - else: - total_count = await conn.fetchval( - """ - SELECT COUNT(DISTINCT session_id) - FROM conversation_events - WHERE session_id IS NOT NULL - """ - ) - - # When the caller filters by user_id we restrict the events under - # consideration to call_ids belonging to that user — a single shared - # subquery used by every CTE so preview_message / models_used cannot - # leak content from another user's calls under a shared session_id. - user_call_filter = ( - "AND ce.call_id IN (SELECT call_id FROM conversation_calls WHERE user_id = $3)" - if user_id is not None - else "" - ) - query_args: list[Any] = [limit, offset] - if user_id is not None: - query_args.append(user_id) + with time_phase("db"): + if user_id is not None: + total_count = await conn.fetchval( + """ + SELECT COUNT(DISTINCT ce.session_id) + FROM conversation_events ce + JOIN conversation_calls cc ON ce.call_id = cc.call_id + WHERE ce.session_id IS NOT NULL AND cc.user_id = $1 + """, + user_id, + ) + else: + total_count = await conn.fetchval( + """ + SELECT COUNT(DISTINCT session_id) + FROM conversation_events + WHERE session_id IS NOT NULL + """ + ) - rows = await conn.fetch( - f""" - WITH session_stats AS ( - SELECT - ce.session_id, - MIN(ce.created_at) as first_ts, - MAX(ce.created_at) as last_ts, - COUNT(*) as total_events, - COUNT(DISTINCT ce.call_id) as turn_count, - COUNT(*) FILTER ( - WHERE ce.event_type LIKE 'policy.%' - AND ce.event_type NOT LIKE 'policy.%judge.evaluation%' - ) as policy_interventions - FROM conversation_events ce - WHERE ce.session_id IS NOT NULL - {user_call_filter} - GROUP BY ce.session_id - ), - session_models AS ( - SELECT DISTINCT - ce.session_id, - ce.payload->>'final_model' as model - FROM conversation_events ce - WHERE ce.session_id IS NOT NULL - AND ce.event_type = 'transaction.request_recorded' - AND ce.payload->>'final_model' IS NOT NULL - {user_call_filter} - ), - session_first_message AS ( - SELECT DISTINCT ON (ce.session_id) - ce.session_id, - ce.payload as request_payload - FROM conversation_events ce - WHERE ce.session_id IS NOT NULL - AND ce.event_type = 'transaction.request_recorded' - -- Skip probe requests: max_tokens=1 means internal probe (token counting, quota). - -- COALESCE to 2 so requests without max_tokens are not skipped. - AND COALESCE((ce.payload->'final_request'->>'max_tokens')::int, 2) > 1 - {user_call_filter} - ORDER BY ce.session_id, ce.created_at ASC + # When the caller filters by user_id we restrict the events under + # consideration to call_ids belonging to that user — a single shared + # subquery used by every CTE so preview_message / models_used cannot + # leak content from another user's calls under a shared session_id. + user_call_filter = ( + "AND ce.call_id IN (SELECT call_id FROM conversation_calls WHERE user_id = $3)" + if user_id is not None + else "" ) - SELECT - s.session_id, - s.first_ts, - s.last_ts, - s.total_events, - s.turn_count, - s.policy_interventions, - COALESCE( - array_agg(DISTINCT m.model) FILTER (WHERE m.model IS NOT NULL), - ARRAY[]::text[] - ) as models, - f.request_payload - FROM session_stats s - LEFT JOIN session_models m ON s.session_id = m.session_id - LEFT JOIN session_first_message f ON s.session_id = f.session_id - GROUP BY s.session_id, s.first_ts, s.last_ts, - s.total_events, s.turn_count, s.policy_interventions, - f.request_payload - ORDER BY s.last_ts DESC - LIMIT $1 OFFSET $2 - """, - *query_args, - ) - - # Separate user_ids lookup keyed on the page's session_ids. Distinct - # users only — never collapse via MIN/MAX. When a user filter is in - # effect the same scoping is applied so the response doesn't leak the - # *existence* of other users sharing the session. - user_ids_by_session: dict[str, list[str]] = {} - if rows: - session_ids_on_page = [str(row["session_id"]) for row in rows] - placeholders = ", ".join(f"${i + 1}" for i in range(len(session_ids_on_page))) + query_args: list[Any] = [limit, offset] if user_id is not None: - user_id_filter_clause = f"AND cc.user_id = ${len(session_ids_on_page) + 1}" - user_id_extra_args: list[Any] = [user_id] - else: - user_id_filter_clause = "" - user_id_extra_args = [] - user_id_rows = await conn.fetch( + query_args.append(user_id) + + rows = await conn.fetch( f""" - SELECT DISTINCT ce.session_id, cc.user_id - FROM conversation_events ce - JOIN conversation_calls cc ON ce.call_id = cc.call_id - WHERE ce.session_id IN ({placeholders}) - AND cc.user_id IS NOT NULL - {user_id_filter_clause} + WITH session_stats AS ( + SELECT + ce.session_id, + MIN(ce.created_at) as first_ts, + MAX(ce.created_at) as last_ts, + COUNT(*) as total_events, + COUNT(DISTINCT ce.call_id) as turn_count, + COUNT(*) FILTER ( + WHERE ce.event_type LIKE 'policy.%' + AND ce.event_type NOT LIKE 'policy.%judge.evaluation%' + ) as policy_interventions + FROM conversation_events ce + WHERE ce.session_id IS NOT NULL + {user_call_filter} + GROUP BY ce.session_id + ), + session_models AS ( + SELECT DISTINCT + ce.session_id, + ce.payload->>'final_model' as model + FROM conversation_events ce + WHERE ce.session_id IS NOT NULL + AND ce.event_type = 'transaction.request_recorded' + AND ce.payload->>'final_model' IS NOT NULL + {user_call_filter} + ), + session_first_message AS ( + SELECT DISTINCT ON (ce.session_id) + ce.session_id, + ce.payload as request_payload + FROM conversation_events ce + WHERE ce.session_id IS NOT NULL + AND ce.event_type = 'transaction.request_recorded' + -- Skip probe requests: max_tokens=1 means internal probe (token counting, quota). + -- COALESCE to 2 so requests without max_tokens are not skipped. + AND COALESCE((ce.payload->'final_request'->>'max_tokens')::int, 2) > 1 + {user_call_filter} + ORDER BY ce.session_id, ce.created_at ASC + ) + SELECT + s.session_id, + s.first_ts, + s.last_ts, + s.total_events, + s.turn_count, + s.policy_interventions, + COALESCE( + array_agg(DISTINCT m.model) FILTER (WHERE m.model IS NOT NULL), + ARRAY[]::text[] + ) as models, + f.request_payload + FROM session_stats s + LEFT JOIN session_models m ON s.session_id = m.session_id + LEFT JOIN session_first_message f ON s.session_id = f.session_id + GROUP BY s.session_id, s.first_ts, s.last_ts, + s.total_events, s.turn_count, s.policy_interventions, + f.request_payload + ORDER BY s.last_ts DESC + LIMIT $1 OFFSET $2 """, - *session_ids_on_page, - *user_id_extra_args, + *query_args, ) - for r in user_id_rows: - sid = str(r["session_id"]) - uid = str(r["user_id"]) - bucket = user_ids_by_session.setdefault(sid, []) - if uid not in bucket: - bucket.append(uid) + + # Separate user_ids lookup keyed on the page's session_ids. Distinct + # users only — never collapse via MIN/MAX. When a user filter is in + # effect the same scoping is applied so the response doesn't leak the + # *existence* of other users sharing the session. + user_ids_by_session: dict[str, list[str]] = {} + if rows: + session_ids_on_page = [str(row["session_id"]) for row in rows] + placeholders = ", ".join(f"${i + 1}" for i in range(len(session_ids_on_page))) + if user_id is not None: + user_id_filter_clause = f"AND cc.user_id = ${len(session_ids_on_page) + 1}" + user_id_extra_args: list[Any] = [user_id] + else: + user_id_filter_clause = "" + user_id_extra_args = [] + user_id_rows = await conn.fetch( + f""" + SELECT DISTINCT ce.session_id, cc.user_id + FROM conversation_events ce + JOIN conversation_calls cc ON ce.call_id = cc.call_id + WHERE ce.session_id IN ({placeholders}) + AND cc.user_id IS NOT NULL + {user_id_filter_clause} + """, + *session_ids_on_page, + *user_id_extra_args, + ) + for r in user_id_rows: + sid = str(r["session_id"]) + uid = str(r["user_id"]) + bucket = user_ids_by_session.setdefault(sid, []) + if uid not in bucket: + bucket.append(uid) sessions = [ SessionSummary( @@ -561,141 +563,140 @@ async def _fetch_session_list_sqlite( # SECURITY INVARIANT: user_id is bound as a query parameter, never # interpolated into the SQL string. async with db_pool.connection() as conn: - if user_id is not None: - total_count = await conn.fetchval( - """ - SELECT COUNT(DISTINCT ce.session_id) + with time_phase("db"): + if user_id is not None: + total_count = await conn.fetchval( + """ + SELECT COUNT(DISTINCT ce.session_id) + FROM conversation_events ce + JOIN conversation_calls cc ON ce.call_id = cc.call_id + WHERE ce.session_id IS NOT NULL AND cc.user_id = $1 + """, + user_id, + ) + else: + total_count = await conn.fetchval( + """ + SELECT COUNT(DISTINCT session_id) + FROM conversation_events + WHERE session_id IS NOT NULL + """ + ) + + # PERF: only filter through conversation_calls when a user filter is + # actually requested. Unfiltered list calls (the hot path) skip the + # conversation_calls subquery entirely. user_ids are populated by a + # separate post-query keyed on the page's session_ids (SQLite has no + # array_agg, so we can't compute them inside this query anyway). + user_call_filter = ( + "AND ce.call_id IN (SELECT call_id FROM conversation_calls WHERE user_id = $3)" + if user_id is not None + else "" + ) + query_args: list[Any] = [limit, offset] + if user_id is not None: + query_args.append(user_id) + + rows = await conn.fetch( + f""" + SELECT + ce.session_id, + MIN(ce.created_at) as first_ts, + MAX(ce.created_at) as last_ts, + COUNT(*) as total_events, + COUNT(DISTINCT ce.call_id) as turn_count, + SUM(CASE + WHEN ce.event_type LIKE 'policy.%' + AND ce.event_type NOT LIKE 'policy.%judge.evaluation%' + THEN 1 ELSE 0 + END) as policy_interventions FROM conversation_events ce - JOIN conversation_calls cc ON ce.call_id = cc.call_id - WHERE ce.session_id IS NOT NULL AND cc.user_id = $1 + WHERE ce.session_id IS NOT NULL + {user_call_filter} + GROUP BY ce.session_id + ORDER BY last_ts DESC + LIMIT $1 OFFSET $2 """, - user_id, - ) - else: - total_count = await conn.fetchval( - """ - SELECT COUNT(DISTINCT session_id) - FROM conversation_events - WHERE session_id IS NOT NULL - """ + *query_args, ) - # PERF: only filter through conversation_calls when a user filter is - # actually requested. Unfiltered list calls (the hot path) skip the - # conversation_calls subquery entirely. user_ids are populated by a - # separate post-query keyed on the page's session_ids (SQLite has no - # array_agg, so we can't compute them inside this query anyway). - user_call_filter = ( - "AND ce.call_id IN (SELECT call_id FROM conversation_calls WHERE user_id = $3)" - if user_id is not None - else "" - ) - query_args: list[Any] = [limit, offset] - if user_id is not None: - query_args.append(user_id) - - rows = await conn.fetch( - f""" - SELECT - ce.session_id, - MIN(ce.created_at) as first_ts, - MAX(ce.created_at) as last_ts, - COUNT(*) as total_events, - COUNT(DISTINCT ce.call_id) as turn_count, - SUM(CASE - WHEN ce.event_type LIKE 'policy.%' - AND ce.event_type NOT LIKE 'policy.%judge.evaluation%' - THEN 1 ELSE 0 - END) as policy_interventions - FROM conversation_events ce - WHERE ce.session_id IS NOT NULL - {user_call_filter} - GROUP BY ce.session_id - ORDER BY last_ts DESC - LIMIT $1 OFFSET $2 - """, - *query_args, - ) + total = int(total_count) if total_count is not None else 0 # type: ignore[arg-type] - total = int(total_count) if total_count is not None else 0 # type: ignore[arg-type] + if not rows: + return SessionListResponse(sessions=[], total=total, offset=offset, has_more=False) - if not rows: - return SessionListResponse(sessions=[], total=total, offset=offset, has_more=False) + session_ids = [str(row["session_id"]) for row in rows] + placeholders = ", ".join(f"${i + 1}" for i in range(len(session_ids))) - session_ids = [str(row["session_id"]) for row in rows] - placeholders = ", ".join(f"${i + 1}" for i in range(len(session_ids))) + # When a user_id filter is in effect, restrict the model/preview/user-id + # lookups to that user's call_ids — without this, preview_message and + # models_used can leak content from other users' calls that happen to + # share the session_id. + # NOTE: this clause is *separate from* the `user_call_filter` used in + # the main aggregation above — different placeholder slot ($N differs + # because session_ids are also bound here). Don't fold into one. + if user_id is not None: + user_call_filter_lookups = f"AND ce.call_id IN (SELECT call_id FROM conversation_calls WHERE user_id = ${len(session_ids) + 1})" + extra_args: list[Any] = [user_id] + else: + user_call_filter_lookups = "" + extra_args = [] - # When a user_id filter is in effect, restrict the model/preview/user-id - # lookups to that user's call_ids — without this, preview_message and - # models_used can leak content from other users' calls that happen to - # share the session_id. - # NOTE: this clause is *separate from* the `user_call_filter` used in - # the main aggregation above — different placeholder slot ($N differs - # because session_ids are also bound here). Don't fold into one. - if user_id is not None: - user_call_filter_lookups = ( - f"AND ce.call_id IN (SELECT call_id FROM conversation_calls WHERE user_id = ${len(session_ids) + 1})" + # One query for all models on this page + model_rows = await conn.fetch( + f""" + SELECT ce.session_id, json_extract(ce.payload, '$.final_model') as model + FROM conversation_events ce + WHERE ce.session_id IN ({placeholders}) + AND ce.event_type = 'transaction.request_recorded' + AND json_extract(ce.payload, '$.final_model') IS NOT NULL + {user_call_filter_lookups} + """, + *session_ids, + *extra_args, ) - extra_args: list[Any] = [user_id] - else: - user_call_filter_lookups = "" - extra_args = [] - - # One query for all models on this page - model_rows = await conn.fetch( - f""" - SELECT ce.session_id, json_extract(ce.payload, '$.final_model') as model - FROM conversation_events ce - WHERE ce.session_id IN ({placeholders}) - AND ce.event_type = 'transaction.request_recorded' - AND json_extract(ce.payload, '$.final_model') IS NOT NULL - {user_call_filter_lookups} - """, - *session_ids, - *extra_args, - ) - # One query for first qualifying preview per session on this page - preview_rows = await conn.fetch( - f""" - SELECT ce.session_id, ce.payload as request_payload - FROM conversation_events ce - WHERE ce.session_id IN ({placeholders}) - AND ce.event_type = 'transaction.request_recorded' - AND COALESCE( - CAST(json_extract(ce.payload, '$.final_request.max_tokens') AS INTEGER), - 2 - ) > 1 - {user_call_filter_lookups} - ORDER BY ce.session_id, ce.created_at ASC - """, - *session_ids, - *extra_args, - ) + # One query for first qualifying preview per session on this page + preview_rows = await conn.fetch( + f""" + SELECT ce.session_id, ce.payload as request_payload + FROM conversation_events ce + WHERE ce.session_id IN ({placeholders}) + AND ce.event_type = 'transaction.request_recorded' + AND COALESCE( + CAST(json_extract(ce.payload, '$.final_request.max_tokens') AS INTEGER), + 2 + ) > 1 + {user_call_filter_lookups} + ORDER BY ce.session_id, ce.created_at ASC + """, + *session_ids, + *extra_args, + ) - # Distinct user_ids per session — never collapse via MIN/MAX, that lies - # on multi-user sessions. Returned as a list so the consumer can render - # mixed-identity sessions honestly. When a user filter is in effect - # we constrain to that user so the response doesn't leak the *existence* - # of other users sharing the session. - if user_id is not None: - user_id_filter_clause = f"AND cc.user_id = ${len(session_ids) + 1}" - user_id_args: list[Any] = [user_id] - else: - user_id_filter_clause = "" - user_id_args = [] - user_id_rows = await conn.fetch( - f""" - SELECT DISTINCT ce.session_id, cc.user_id - FROM conversation_events ce - JOIN conversation_calls cc ON ce.call_id = cc.call_id - WHERE ce.session_id IN ({placeholders}) - AND cc.user_id IS NOT NULL - {user_id_filter_clause} - """, - *session_ids, - *user_id_args, - ) + # Distinct user_ids per session — never collapse via MIN/MAX, that lies + # on multi-user sessions. Returned as a list so the consumer can render + # mixed-identity sessions honestly. When a user filter is in effect + # we constrain to that user so the response doesn't leak the *existence* + # of other users sharing the session. + if user_id is not None: + user_id_filter_clause = f"AND cc.user_id = ${len(session_ids) + 1}" + user_id_args: list[Any] = [user_id] + else: + user_id_filter_clause = "" + user_id_args = [] + user_id_rows = await conn.fetch( + f""" + SELECT DISTINCT ce.session_id, cc.user_id + FROM conversation_events ce + JOIN conversation_calls cc ON ce.call_id = cc.call_id + WHERE ce.session_id IN ({placeholders}) + AND cc.user_id IS NOT NULL + {user_id_filter_clause} + """, + *session_ids, + *user_id_args, + ) # Build per-session lookup maps from the bulk results models_by_session: dict[str, list[str]] = {} @@ -753,15 +754,16 @@ async def fetch_session_detail(session_id: str, db_pool: DatabasePool) -> Sessio ValueError: If no events found for session_id """ async with db_pool.connection() as conn: - rows = await conn.fetch( - """ - SELECT call_id, event_type, payload, created_at - FROM conversation_events - WHERE session_id = $1 - ORDER BY created_at ASC - """, - session_id, - ) + with time_phase("db"): + rows = await conn.fetch( + """ + SELECT call_id, event_type, payload, created_at + FROM conversation_events + WHERE session_id = $1 + ORDER BY created_at ASC + """, + session_id, + ) if not rows: raise ValueError(f"No events found for session_id: {session_id}") diff --git a/src/luthien_proxy/main.py b/src/luthien_proxy/main.py index 152a6501c..d179fdf35 100644 --- a/src/luthien_proxy/main.py +++ b/src/luthien_proxy/main.py @@ -20,7 +20,7 @@ from fastapi.staticfiles import StaticFiles from pydantic import ValidationError from redis.asyncio import Redis -from starlette.middleware.base import BaseHTTPMiddleware +from starlette.types import ASGIApp, Message, Receive, Scope, Send from luthien_proxy.admin import router as admin_router from luthien_proxy.config_fields import CONFIG_FIELDS, CONFIG_FIELDS_BY_NAME @@ -41,6 +41,7 @@ ) from luthien_proxy.observability.redis_event_publisher import RedisEventPublisher from luthien_proxy.observability.sentry import init_sentry +from luthien_proxy.perf.timing_middleware import ServerTimingMiddleware from luthien_proxy.pipeline.upstream_headers import validate_upstream_headers_at_startup from luthien_proxy.policy_manager import PolicyManager from luthien_proxy.rate_limit import TokenBucketRateLimiter @@ -424,22 +425,42 @@ async def lifespan(app: FastAPI): # JS/HTML/CSS use no-cache so the browser always revalidates (prevents # stale JS after a gateway restart). Other assets (images, fonts) get a # longer TTL since they change infrequently. - class StaticCacheMiddleware(BaseHTTPMiddleware): - async def dispatch(self, request: Request, call_next): - response = await call_next(request) - if request.url.path.startswith("/api/") or request.url.path in ("/health", "/ready"): - # Prevent CDN/edge caching of API and health responses (Railway, Cloudflare, etc.) - response.headers["Cache-Control"] = "no-store, no-cache, must-revalidate" - elif request.url.path.startswith("/static/"): - path = request.url.path - if path.endswith((".js", ".html", ".css")): - response.headers["Cache-Control"] = "no-cache" - else: - response.headers["Cache-Control"] = "public, max-age=3600" - return response + class StaticCacheMiddleware: + def __init__(self, app: ASGIApp) -> None: # noqa: D107 + self.app = app + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: # noqa: D102 + if scope["type"] != "http": + await self.app(scope, receive, send) + return + + path: str = scope.get("path", "") + + def _cache_header() -> str | None: + if path.startswith("/api/") or path in ("/health", "/ready"): + return "no-store, no-cache, must-revalidate" + if path.startswith("/static/"): + return "no-cache" if path.endswith((".js", ".html", ".css")) else "public, max-age=3600" + return None + + cache_value = _cache_header() + + async def send_with_cache(message: Message) -> None: + if message["type"] == "http.response.start" and cache_value: + headers = [h for h in message.get("headers", []) if h[0].lower() != b"cache-control"] + headers.append((b"cache-control", cache_value.encode())) + message = {**message, "headers": headers} + await send(message) + + await self.app(scope, receive, send_with_cache if cache_value else send) app.add_middleware(StaticCacheMiddleware) + # Add ServerTimingMiddleware as the outermost middleware (add_middleware stacks + # outermost-last). StaticCacheMiddleware is pure-ASGI so it does not break + # ContextVar propagation for streaming responses on timed paths. + app.add_middleware(ServerTimingMiddleware) + # Include routers app.include_router(gateway_router) # /v1/messages app.include_router(debug_router) # /api/debug/* diff --git a/src/luthien_proxy/perf/__init__.py b/src/luthien_proxy/perf/__init__.py new file mode 100644 index 000000000..cf44e1770 --- /dev/null +++ b/src/luthien_proxy/perf/__init__.py @@ -0,0 +1,5 @@ +"""Perf-measurement utilities for the Luthien proxy. + +Isolated from the main application — writes only to the perf database, +never to the dev database (~/.luthien/local.db). +""" diff --git a/src/luthien_proxy/perf/db.py b/src/luthien_proxy/perf/db.py new file mode 100644 index 000000000..d02b03da4 --- /dev/null +++ b/src/luthien_proxy/perf/db.py @@ -0,0 +1,166 @@ +"""Perf-DB isolation enforcement, migration runner, and drop helpers. + +The perf database is a completely isolated database used only for performance +benchmarking. It must never alias the dev database (~/.luthien/local.db). +""" + +from __future__ import annotations + +import asyncio +import os +from pathlib import Path +from typing import Literal + + +def get_perf_db_url(backend: Literal["sqlite", "postgres"]) -> str: + """Return the URL for the perf-test database. + + Args: + backend: "sqlite" → file URL under ~/.luthien/perf.db; + "postgres" → DATABASE_URL with perf_test schema override. + + Returns: + A database URL string for use with the migration runner. + + Raises: + RuntimeError: When backend is "postgres" and DATABASE_URL is unset. + """ + if backend == "sqlite": + return f"sqlite:///{Path.home()}/.luthien/perf.db" + base_url = os.environ.get("DATABASE_URL", "") + if not base_url: + raise RuntimeError("DATABASE_URL environment variable is required for postgres backend") + # Detect an existing options= parameter to avoid a duplicate that would + # silently shadow our search_path override (Postgres uses last-value-wins). + if "options=" in base_url: + raise RuntimeError( + "DATABASE_URL already contains an 'options=' parameter. " + "Appending a second options=-csearch_path=perf_test would produce a duplicate " + "and Postgres would silently ignore one of them. " + "Remove the existing 'options=' from DATABASE_URL before using the postgres perf backend." + ) + separator = "&" if "?" in base_url else "?" + return f"{base_url}{separator}options=-csearch_path=perf_test" + + +def ensure_perf_isolation(url: str) -> None: + """Assert that a database URL is not the dev database. + + This is the safety gate — call it before any write to the perf DB. + + Args: + url: The database URL to inspect. + + Raises: + RuntimeError: If the URL points to the dev database (contains "local.db"), + or if it is a Postgres URL without the "perf_test" schema override. + The message always contains the word "isolation". + """ + if url.startswith(("sqlite://", "sqlite+aiosqlite://")): + from pathlib import Path as _Path # noqa: PLC0415 + + path_str = url.split("///", 1)[-1] if "///" in url else url.split("//", 1)[-1] + if _Path(path_str).name == "local.db": + raise RuntimeError( + "Perf DB isolation violation: URL resolves to local.db — " + "refusing to use the dev database as the perf database. " + "Use get_perf_db_url() to obtain the correct perf DB URL." + ) + elif url.startswith(("postgresql://", "postgres://")): + if "options=-csearch_path=perf_test" not in url: + raise RuntimeError( + f"Perf DB isolation violation: Postgres URL must include " + f"'options=-csearch_path=perf_test' (use get_perf_db_url('postgres')). Got: {url!r}" + ) + else: + raise RuntimeError( + f"Perf DB isolation violation: unrecognized URL scheme — cannot verify isolation for: {url!r}" + ) + + +def drop_perf_db(backend: Literal["sqlite", "postgres"]) -> None: + """Drop the perf database. Idempotent — safe to call when already dropped. + + Args: + backend: "sqlite" removes ~/.luthien/perf.db (no-op if absent); + "postgres" runs DROP SCHEMA IF EXISTS perf_test CASCADE. + """ + if backend == "sqlite": + perf_path = Path.home() / ".luthien" / "perf.db" + perf_path.unlink(missing_ok=True) + return + + raise NotImplementedError( + "Postgres perf drop is not yet implemented. Implement alongside _seed_postgres in seeding.py." + ) + + +def migrate_perf_db(backend: Literal["sqlite", "postgres"]) -> None: + """Apply all migrations to the perf database. + + Calls ensure_perf_isolation before touching the database. For SQLite, + creates ~/.luthien/ if needed and runs the bundled migration scripts + via the standard migration runner. + + Note: Must be called from a synchronous context — internally uses + asyncio.run() and cannot be called from within a running event loop + (e.g. from an async test fixture or FastAPI handler). + + Args: + backend: "sqlite" or "postgres". + + Raises: + RuntimeError: If isolation check fails or migrations fail. + NotImplementedError: For the "postgres" backend (not yet implemented). + """ + url = get_perf_db_url(backend) + ensure_perf_isolation(url) + + if backend == "sqlite": + _migrate_sqlite(url) + else: + raise NotImplementedError( + "Postgres perf migration is not yet implemented. Implement alongside _seed_postgres in seeding.py." + ) + + +async def migrate_perf_db_async(backend: Literal["sqlite", "postgres"]) -> None: + """Apply all migrations to the perf database. + + Async variant of :func:`migrate_perf_db`. Safe to call from async + fixtures and handlers. See :func:`migrate_perf_db` for full docs. + + Args: + backend: "sqlite" or "postgres". + + Raises: + RuntimeError: If isolation check fails or migrations fail. + NotImplementedError: For the "postgres" backend (not yet implemented). + """ + url = get_perf_db_url(backend) + ensure_perf_isolation(url) + + if backend == "sqlite": + await _migrate_sqlite_async(url) + else: + raise NotImplementedError( + "Postgres perf migration is not yet implemented. Implement alongside _seed_postgres in seeding.py." + ) + + +async def _migrate_sqlite_async(url: str) -> None: + from luthien_proxy.utils.db import DatabasePool # noqa: PLC0415 + from luthien_proxy.utils.db_sqlite import parse_sqlite_url # noqa: PLC0415 + from luthien_proxy.utils.migration_check import apply_sqlite_migrations # noqa: PLC0415 + + db_path = Path(parse_sqlite_url(url)) + db_path.parent.mkdir(parents=True, exist_ok=True) + db_pool = DatabasePool(url) + try: + await apply_sqlite_migrations(db_pool) + finally: + await db_pool.close() + + +def _migrate_sqlite(url: str) -> None: + asyncio.run(_migrate_sqlite_async(url)) diff --git a/src/luthien_proxy/perf/seeding.py b/src/luthien_proxy/perf/seeding.py new file mode 100644 index 000000000..025e6b73e --- /dev/null +++ b/src/luthien_proxy/perf/seeding.py @@ -0,0 +1,348 @@ +"""Direct-SQL seeding module for perf-test database. + +Inserts production-shaped rows into conversation_calls and conversation_events +for performance benchmarking. Uses direct sqlite3 connections and executemany +for maximum throughput. + +All session_ids are prefixed with 'perf-seed-{tier}-' or 'perf-seed-sami-'. +IDs are fully deterministic — drop + re-seed produces identical data. + +FK ordering: conversation_calls rows are inserted before conversation_events rows. +""" + +from __future__ import annotations + +import logging +import random +import sqlite3 +import time +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Literal + +from luthien_proxy.perf.db import ensure_perf_isolation, get_perf_db_url, migrate_perf_db + +logger = logging.getLogger(__name__) + +_MODEL = "claude-haiku-4-5" +_BASE_TS = datetime(2026, 1, 1, 0, 0, 0, tzinfo=timezone.utc) +_BATCH_SIZE = 5000 # ~125 MB in-memory per batch at ~25 KB/payload; reduce if RSS is a concern +_DETERMINISTIC_RNG_SEED = 0xABCDEF + +_CALLS_INSERT = ( + "INSERT INTO conversation_calls" + " (call_id, model_name, provider, status, created_at, completed_at, session_id)" + " VALUES (?, ?, ?, ?, ?, ?, ?)" +) +_EVENTS_INSERT = ( + "INSERT INTO conversation_events" + " (id, call_id, event_type, payload, created_at, session_id)" + " VALUES (?, ?, ?, ?, ?, ?)" +) +_INDEX_STMTS: tuple[str, ...] = ( + "CREATE INDEX IF NOT EXISTS idx_conversation_events_type ON conversation_events(event_type)", + "CREATE INDEX IF NOT EXISTS idx_conversation_events_created ON conversation_events(created_at)", + "CREATE INDEX IF NOT EXISTS idx_conversation_events_call_created ON conversation_events(call_id, created_at)", + "CREATE INDEX IF NOT EXISTS idx_conversation_events_session ON conversation_events(session_id) WHERE session_id IS NOT NULL", + "CREATE INDEX IF NOT EXISTS idx_conversation_calls_created ON conversation_calls(created_at)", + "CREATE INDEX IF NOT EXISTS idx_conversation_calls_session ON conversation_calls(session_id) WHERE session_id IS NOT NULL", + "CREATE INDEX IF NOT EXISTS idx_conversation_calls_user ON conversation_calls(user_id) WHERE user_id IS NOT NULL", +) + +# Pre-built JSON template fragments — content is pure ASCII, no escaping needed. +# Pad sizes produce ~5 KB req / ~20 KB resp payloads (see test_payload_sizes for +# the size invariant; update that test if _REQ_HEAD/_REQ_MID/_REQ_TAIL change). +_REQ_PAD = "A" * 2368 +_RESP_PAD = "B" * 20202 + +_REQ_HEAD = ( + '{"final_request": {"model": "' + _MODEL + '", "max_tokens": 1024,' + ' "stream": true, "temperature": 0.7,' + ' "messages": [{"role": "user", "content": "' +) +_REQ_MID = ( + '"}]}, "original_request": {"model": "' + _MODEL + '", "max_tokens": 1024,' + ' "stream": true, "temperature": 0.7,' + ' "messages": [{"role": "user", "content": "' +) +_REQ_TAIL = '"}]}, "final_model": "' + _MODEL + '"}' + +_RESP_HEAD = ( + '{"final_response": {"id": "msg_000000", "type": "message",' + ' "role": "assistant", "model": "' + _MODEL + '",' + ' "stop_reason": "end_turn", "stop_sequence": null,' + ' "usage": {"input_tokens": 256, "output_tokens": 512},' + ' "content": [{"type": "text", "text": "' +) +_RESP_TAIL = '"}]}}' + + +@dataclass(frozen=True) +class SeedingReport: + """Report returned by seeding functions with metrics about the seeding run.""" + + label: str + total_sessions: int + total_rows: int + total_bytes: int + elapsed_seconds: float + backend: str + biggest_session_message_count: int + + +def _fmt_ts(dt: datetime) -> str: + return dt.strftime("%Y-%m-%d %H:%M:%S") + + +def _req_payload(session_id: str, call_idx: int) -> str: + """~5 KB JSON string for a transaction.request_recorded event.""" + content = f"s={session_id[:12]} c={call_idx:04d} " + _REQ_PAD + return _REQ_HEAD + content + _REQ_MID + content + _REQ_TAIL + + +def _resp_payload(session_id: str, call_idx: int) -> str: + """~20 KB JSON string for a transaction.streaming_response_recorded event.""" + text = f"r={session_id[:12]} c={call_idx:04d} " + _RESP_PAD + return _RESP_HEAD + text + _RESP_TAIL + + +def _call_count(session_idx: int, rng_seed: int) -> int: + """Deterministic call count per session. + + Distribution (in calls; each call = 2 events): + - 50% → 5–15 calls (10–30 events; median ≈ 20 events) + - 45% → 15–50 calls (30–100 events; p95 ≈ 100 events) + - 5% → 50–250 calls (100–500 events; p99 ≈ 500 events) + """ + rng = random.Random(rng_seed * 1_000_003 + session_idx) + r = rng.random() + if r < 0.50: + return rng.randint(5, 15) + elif r < 0.95: + return rng.randint(15, 50) + else: + return rng.randint(50, 250) + + +def _sqlite_path(url: str) -> Path: + prefix = "sqlite:///" + if not url.startswith(prefix): + raise ValueError(f"Expected sqlite:/// URL, got {url!r}") + return Path(url[len(prefix) :]) + + +def _seed_sqlite( + db_path: Path, + plan: list[tuple[str, int]], + label: str, + backend: str = "sqlite", +) -> SeedingReport: + """Bulk-insert plan into SQLite via executemany. + + Args: + db_path: Path to the SQLite database file. + plan: List of (session_id, n_calls) pairs. + label: Tier label for the report (e.g. ``"100"``, ``"sami"``). + backend: Backend label for the report. + + Returns: + SeedingReport with insertion statistics. + """ + t0 = time.monotonic() + total_bytes = 0 + biggest = 0 + + conn = sqlite3.connect(str(db_path), isolation_level=None) + conn.execute("PRAGMA journal_mode=WAL") + conn.execute("PRAGMA synchronous=OFF") # intentionally unsafe — perf DB is disposable + conn.execute("PRAGMA cache_size=-131072") + conn.execute("PRAGMA temp_store=MEMORY") + + try: + conn.execute("BEGIN EXCLUSIVE") + # Drop indexes before bulk insert — dramatically reduces write amplification. + # Indexes are recreated after all rows are inserted. + for idx in ( + "idx_conversation_events_type", + "idx_conversation_events_created", + "idx_conversation_events_call_created", + "idx_conversation_events_session", + "idx_conversation_calls_created", + "idx_conversation_calls_session", + "idx_conversation_calls_user", + ): + conn.execute(f"DROP INDEX IF EXISTS {idx}") + + # Pass 1: conversation_calls (FK parent) — must precede events. + calls_batch: list[tuple] = [] + for session_idx, (session_id, n_calls) in enumerate(plan): + if n_calls > biggest: + biggest = n_calls + for call_idx in range(n_calls): + call_id = f"{session_id}-{call_idx:04d}" + ts = _fmt_ts(_BASE_TS + timedelta(seconds=session_idx * 3600 + call_idx * 5)) + calls_batch.append((call_id, _MODEL, "anthropic", "completed", ts, ts, session_id)) + if len(calls_batch) >= _BATCH_SIZE: + conn.executemany(_CALLS_INSERT, calls_batch) + calls_batch.clear() + if calls_batch: + conn.executemany(_CALLS_INSERT, calls_batch) + + # Pass 2: conversation_events (FK child). + events_batch: list[tuple] = [] + for session_idx, (session_id, n_calls) in enumerate(plan): + for call_idx in range(n_calls): + call_id = f"{session_id}-{call_idx:04d}" + ts_req = _fmt_ts(_BASE_TS + timedelta(seconds=session_idx * 3600 + call_idx * 5)) + ts_resp = _fmt_ts(_BASE_TS + timedelta(seconds=session_idx * 3600 + call_idx * 5 + 1)) + req_p = _req_payload(session_id, call_idx) + resp_p = _resp_payload(session_id, call_idx) + total_bytes += len(req_p) + len(resp_p) + + events_batch.append( + ( + f"{call_id}-req", + call_id, + "transaction.request_recorded", + req_p, + ts_req, + session_id, + ) + ) + events_batch.append( + ( + f"{call_id}-resp", + call_id, + "transaction.streaming_response_recorded", + resp_p, + ts_resp, + session_id, + ) + ) + + if len(events_batch) >= _BATCH_SIZE: + conn.executemany(_EVENTS_INSERT, events_batch) + events_batch.clear() + + if events_batch: + conn.executemany(_EVENTS_INSERT, events_batch) + + for stmt in _INDEX_STMTS: + conn.execute(stmt) + conn.execute("COMMIT") + except Exception: + conn.execute("ROLLBACK") + raise + finally: + conn.close() + + elapsed = time.monotonic() - t0 + n_calls_total = sum(n for _, n in plan) + total_rows = 3 * n_calls_total # 1 calls row + 2 events rows per call + + return SeedingReport( + label=label, + total_sessions=len(plan), + total_rows=total_rows, + total_bytes=total_bytes, + elapsed_seconds=elapsed, + backend=backend, + biggest_session_message_count=biggest, + ) + + +def _assert_no_existing_rows(db_path: Path, prefix: str) -> None: + conn = sqlite3.connect(str(db_path)) + try: + count = conn.execute( + "SELECT COUNT(*) FROM conversation_calls WHERE session_id LIKE ?", + (f"{prefix}%",), + ).fetchone()[0] + finally: + conn.close() + if count > 0: + raise RuntimeError( + f"seed_sessions: {count} rows with prefix '{prefix}' already exist. " + "Call drop_perf_db(backend) before re-seeding to ensure a clean state." + ) + + +def seed_sessions( + backend: Literal["sqlite", "postgres"], + tier: int, +) -> SeedingReport: + """Seed the perf database with ``tier`` sessions. + + Calls ensure_perf_isolation and migrate_perf_db before inserting. + All session_ids are prefixed with ``perf-seed-{tier}-``. + IDs are fully deterministic — the same tier produces identical rows on + every run. Callers MUST call ``drop_perf_db(backend)`` first if the DB + already contains rows for this tier; seed_sessions will raise if existing + rows are detected (to prevent silent row accumulation across tiers). + + Note: calls ``migrate_perf_db`` which uses ``asyncio.run()`` internally. + Must be called from a synchronous context — will raise ``RuntimeError`` + if called from within a running event loop (e.g. an async test fixture). + + Args: + backend: "sqlite" or "postgres". + tier: Number of sessions to insert (typically 100, 1_000, or 10_000). + + Returns: + SeedingReport with insertion statistics. + """ + if tier >= 10_000: + import warnings # noqa: PLC0415 + + gb_rough = max(1, tier * 25 * 45 // 1_000_000) # rough: events/session × KB/event ÷ 1e6 + warnings.warn( + f"seed_sessions(tier={tier}) seeds roughly {gb_rough} GB on disk " + "(estimate; actual varies with SQLite overhead and call distribution). " + "Ensure sufficient disk space and 128 MB RAM for the SQLite cache.", + stacklevel=2, + ) + + url = get_perf_db_url(backend) + ensure_perf_isolation(url) + migrate_perf_db(backend) + + prefix = f"perf-seed-{tier}-" + + if backend == "sqlite": + _assert_no_existing_rows(_sqlite_path(url), prefix) + plan = [(f"{prefix}{i:04d}", _call_count(i, rng_seed=tier)) for i in range(tier)] + return _seed_sqlite(_sqlite_path(url), plan, label=str(tier), backend=backend) + raise NotImplementedError(f"backend {backend!r} not yet implemented") + + +def seed_sami_like(backend: Literal["sqlite", "postgres"]) -> SeedingReport: + """Seed the perf database with a sami-like fixture. + + 78 sessions total. Session ``perf-seed-sami-442msg`` has exactly 442 calls. + Remaining 77 sessions have 1–187 calls (realistic spread). + All session_ids are prefixed with ``perf-seed-sami-``. + + Args: + backend: "sqlite" or "postgres". + + Returns: + SeedingReport with biggest_session_message_count >= 442. + """ + url = get_perf_db_url(backend) + ensure_perf_isolation(url) + migrate_perf_db(backend) + + prefix = "perf-seed-sami-" + big_session_id = f"{prefix}442msg" + + if backend == "sqlite": + _assert_no_existing_rows(_sqlite_path(url), prefix) + + rng = random.Random(_DETERMINISTIC_RNG_SEED) + other_plan: list[tuple[str, int]] = [(f"{prefix}{i:03d}", rng.randint(1, 187)) for i in range(77)] + plan = [(big_session_id, 442)] + other_plan + + if backend == "sqlite": + return _seed_sqlite(_sqlite_path(url), plan, label="sami", backend=backend) + raise NotImplementedError(f"backend {backend!r} not yet implemented") diff --git a/src/luthien_proxy/perf/timing_middleware.py b/src/luthien_proxy/perf/timing_middleware.py new file mode 100644 index 000000000..c2723dd00 --- /dev/null +++ b/src/luthien_proxy/perf/timing_middleware.py @@ -0,0 +1,200 @@ +"""Server-Timing middleware for admin/debug/UI paths. + +Records per-request timing phases via contextvars and appends a ``Server-Timing`` +header on responses whose path starts with ``/api/history/``, ``/api/debug/``, or +``/ui/fragments/``. All other paths (including ``/v1/messages``) are untouched. + +Usage:: + + from luthien_proxy.perf.timing_middleware import time_phase, ServerTimingMiddleware + + # Inside a request handler or service function: + with time_phase("db"): + rows = await db.fetch(query) + + with time_phase("serialize"): + payload = serialize(rows) + + # In FastAPI app setup (handled by P14): + app.add_middleware(ServerTimingMiddleware) +""" + +from __future__ import annotations + +import re +import time +from collections.abc import Awaitable, Callable, Generator +from contextlib import contextmanager +from contextvars import ContextVar +from typing import TYPE_CHECKING + +from pydantic import BaseModel +from starlette.responses import Response + +_PHASE_NAME_RE = re.compile(r"[A-Za-z0-9_-]+") + +if TYPE_CHECKING: + from starlette.types import ASGIApp, Message, Receive, Scope, Send + +# Paths where Server-Timing is emitted. /v1/messages is deliberately excluded. +_TIMED_PREFIXES: tuple[str, ...] = ( + "/api/history/", + "/api/debug/", + "/ui/fragments/", + # /api/activity/stream and other SSE endpoints are intentionally excluded: + # long-lived connections collect phases for the full stream duration, + # making the Server-Timing header meaningless as a request-level metric. +) + +# Per-request phase list: list of (name, elapsed_ms) tuples. +# A new list is injected at the start of each request by ServerTimingMiddleware +# so phases never bleed across requests, even under concurrent load. +_phases_var: ContextVar[list[tuple[str, float]]] = ContextVar("_luthien_timing_phases") + + +@contextmanager +def time_phase(name: str) -> Generator[None, None, None]: + """Record the wall-clock duration of a code block as a timing phase. + + The elapsed milliseconds are appended to the current request's phase list + (stored in a ``ContextVar``). If called outside a ``ServerTimingMiddleware`` + request context the phase is silently discarded. Phases are recorded even + when the block raises — the ``finally`` clause always appends the elapsed time. + + Public API: intentionally exported without a leading underscore so that + perf/db.py and test infrastructure can call it directly without reaching + into a private symbol. + + Args: + name: Short identifier for the phase (e.g. ``"db"``, ``"serialize"``). + + Yields: + Nothing — use as a plain context manager. + + Example:: + + with time_phase("db"): + rows = await conn.fetch(query) + """ + if not _PHASE_NAME_RE.fullmatch(name): + raise ValueError(f"time_phase name must be an RFC 8941 token ([A-Za-z0-9_-]+): {name!r}") + start = time.perf_counter() + try: + yield + finally: + elapsed_ms = (time.perf_counter() - start) * 1000.0 + phases = _phases_var.get(None) + if phases is not None: + phases.append((name, elapsed_ms)) + + +def format_phases(phases: list[tuple[str, float]]) -> str: + """Format a list of timing phases as a ``Server-Timing`` header value. + + Args: + phases: Ordered list of ``(name, elapsed_ms)`` tuples. + + Returns: + Header value string, e.g. ``"db;dur=12.3, serialize;dur=4.5"``. + Returns an empty string if ``phases`` is empty. + + Example:: + + >>> format_phases([("db", 12.3), ("serialize", 4.5)]) + 'db;dur=12.3, serialize;dur=4.5' + """ + return ", ".join(f"{name};dur={elapsed_ms:.1f}" for name, elapsed_ms in phases) + + +class ServerTimingMiddleware: + """Pure-ASGI middleware that adds a ``Server-Timing`` header to filtered responses. + + Implemented as a plain ASGI callable (not ``BaseHTTPMiddleware``) to avoid + Starlette's pipe-buffering wrapper around ``call_next``, which materialises + streaming responses in memory and breaks ContextVar propagation in some + Starlette versions. This implementation wraps only ``send`` — the inner app + runs unmodified and streaming chunks pass through untouched. + + Only paths starting with ``/api/history/``, ``/api/debug/``, or + ``/ui/fragments/`` receive the header. All other paths (including the hot + ``/v1/messages`` path) pass through with zero overhead beyond a single + ``str.startswith`` check on the ASGI scope. + + Timing phases are recorded by calling ``time_phase(name)`` anywhere in the + request/response call stack. Context isolation is guaranteed by + ``contextvars.ContextVar``: each request gets its own fresh phase list. + + ContextVar constraint: if any ``BaseHTTPMiddleware`` sits between this + middleware and the route handler (e.g. ``StaticCacheMiddleware`` in + ``main.py``), ContextVar propagation may silently break for streaming + responses on that path. Do not call ``time_phase`` from inside a + ``StreamingResponse`` body generator — the phase will be lost. + + Phases after ``http.response.start``: the ``Server-Timing`` header is + finalized at response-start time. Any ``time_phase`` block that runs + during body streaming (e.g. in a generator) will not appear in the header. + """ + + def __init__(self, app: ASGIApp) -> None: # noqa: D107 + self.app = app + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: # noqa: D102 + if scope["type"] != "http": + await self.app(scope, receive, send) + return + + path: str = scope.get("path", "") + if not path.startswith(_TIMED_PREFIXES): + await self.app(scope, receive, send) + return + + phases: list[tuple[str, float]] = [] + token = _phases_var.set(phases) + try: + await self.app(scope, receive, _make_send_with_timing(send, phases)) + finally: + _phases_var.reset(token) + + +def timed_json_response(model: BaseModel) -> Response: + """Serialize a Pydantic model to a JSON Response, recording serialize time. + + Wraps ``model.model_dump_json()`` in a ``time_phase("serialize")`` block and + returns a ``starlette.responses.Response`` with ``media_type="application/json"``. + Use in route handlers instead of returning the model directly to avoid the + double-serialization that occurs when FastAPI validates a ``response_model`` + return value (Pydantic→dict→json twice). + + Args: + model: A ``pydantic.BaseModel`` instance. + + Returns: + A pre-serialized JSON ``Response``. + """ + with time_phase("serialize"): + body: str = model.model_dump_json() + return Response(content=body, media_type="application/json") + + +def _make_send_with_timing( + send: Send, + phases: list[tuple[str, float]], +) -> Callable[[Message], Awaitable[None]]: + """Return a wrapped ``send`` that injects Server-Timing on http.response.start.""" + + async def send_with_timing(message: Message) -> None: + if message["type"] == "http.response.start" and phases: + headers = [h for h in message.get("headers", []) if h[0].lower() != b"server-timing"] + headers.append((b"server-timing", format_phases(phases).encode())) + message = {**message, "headers": headers} + await send(message) + + return send_with_timing + + +__all__ = [ + "ServerTimingMiddleware", + "time_phase", + "format_phases", + "timed_json_response", +] diff --git a/src/luthien_proxy/utils/migration_check.py b/src/luthien_proxy/utils/migration_check.py index 6f2073fcb..cbbc7333e 100644 --- a/src/luthien_proxy/utils/migration_check.py +++ b/src/luthien_proxy/utils/migration_check.py @@ -53,7 +53,7 @@ def _find_sqlite_migrations_dir() -> Path | None: return None -async def _apply_sqlite_migrations( +async def apply_sqlite_migrations( db_pool: DatabasePool, migrations_dir: Path | None = None, ) -> None: @@ -66,6 +66,10 @@ async def _apply_sqlite_migrations( Handles upgrade from snapshot-era databases by detecting existing tables with no migration tracking and seeding _migrations. + + Public API — intentionally exported for use by perf/db.py and tests. + Intended as the sole writer to the ``_migrations`` tracking table; calling + it concurrently from multiple processes against the same database is unsafe. """ if migrations_dir is None: migrations_dir = _find_sqlite_migrations_dir() @@ -140,7 +144,8 @@ async def _apply_sqlite_migrations( # ILIKE, NOW(), ::type, LEAST, to_timestamp, or $N placeholders). # A startup-time audit of migrations/sqlite/*.sql enforces this # (see tests/.../test_sqlite_migrations_are_native.py). - assert isinstance(conn, SqliteConnection), "_apply_sqlite_migrations called on non-sqlite pool" + if not isinstance(conn, SqliteConnection): + raise TypeError(f"apply_sqlite_migrations requires a SqliteConnection, got {type(conn).__name__}") sql = mf.read_text() await conn.executescript(sql) @@ -184,7 +189,7 @@ async def check_migrations( migrations_dir: Path to migrations directory. Defaults to /app/migrations. """ if db_pool.is_sqlite: - await _apply_sqlite_migrations(db_pool) + await apply_sqlite_migrations(db_pool) return if migrations_dir is None: diff --git a/tests/luthien_proxy/e2e_tests/sqlite/test_activity_stream_regression.py b/tests/luthien_proxy/e2e_tests/sqlite/test_activity_stream_regression.py new file mode 100644 index 000000000..7a523e9a7 --- /dev/null +++ b/tests/luthien_proxy/e2e_tests/sqlite/test_activity_stream_regression.py @@ -0,0 +1,130 @@ +"""Regression test: SSE activity stream delivers events in order. + +This test verifies that the /api/activity/stream endpoint correctly +delivers events published via the real InProcessEventPublisher when +multiple requests flow through the gateway. Guards against regressions +in the SSE pipeline. + +Run: uv run pytest tests/luthien_proxy/e2e_tests/sqlite/test_activity_stream_regression.py -v --timeout=30 +""" + +import asyncio +import json + +import httpx +import pytest +from tests.luthien_proxy.e2e_tests.mock_anthropic.responses import text_response +from tests.luthien_proxy.e2e_tests.mock_anthropic.server import MockAnthropicServer +from tests.luthien_proxy.e2e_tests.sqlite._boot import boot_sqlite_gateway, free_port + +from luthien_proxy.observability.event_publisher import build_activity_event + +pytestmark = pytest.mark.sqlite_e2e + +_API_KEY = "test-regression-key" +_ADMIN_KEY = "test-regression-admin-key" +_NUM_SYNTHETIC_EVENTS = 3 + +_EXPECTED_EVENT_FIELDS = set(build_activity_event("_", "_").keys()) + + +@pytest.fixture(scope="module") +def mock_server(): + server = MockAnthropicServer(port=free_port()) + server.start() + yield server + server.stop() + + +@pytest.fixture(scope="module") +def gateway_url(mock_server): + """Boot an in-process SQLite gateway with no Redis.""" + with boot_sqlite_gateway( + api_key=_API_KEY, + admin_key=_ADMIN_KEY, + mock_anthropic_url=f"http://127.0.0.1:{mock_server.port}", + tmp_prefix="luthien_regression_e2e_", + thread_name="regression-gateway", + ) as url: + yield url + + +@pytest.mark.asyncio +async def test_activity_stream_events_flow_in_order(gateway_url, mock_server): + """SSE activity stream delivers events from all 3 synthetic requests in order. + + Sends 3 synthetic API requests through the gateway (triggering the real + InProcessEventPublisher for each), then verifies that all 3 sets of events + arrive at the SSE client within 5 seconds and carry the correct schema. + """ + for i in range(_NUM_SYNTHETIC_EVENTS): + mock_server.enqueue(text_response(f"Synthetic response {i}")) + + sse_events: list[dict] = [] + requests_done = asyncio.Event() + sse_ready = asyncio.Event() + + async def collect_sse(): + async with httpx.AsyncClient(timeout=15.0) as client: + async with client.stream( + "GET", + f"{gateway_url}/api/activity/stream", + headers={"Authorization": f"Bearer {_ADMIN_KEY}"}, + ) as response: + assert response.status_code == 200 + assert "text/event-stream" in response.headers.get("content-type", "") + sse_ready.set() + + async for line in response.aiter_lines(): + if not line.startswith("data:"): + continue + raw = line[len("data:") :].strip() + try: + event = json.loads(raw) + except json.JSONDecodeError: + continue + sse_events.append(event) + if requests_done.is_set() and len(sse_events) >= _NUM_SYNTHETIC_EVENTS: + return + + async def send_synthetic_requests(): + await asyncio.wait_for(sse_ready.wait(), timeout=10.0) + async with httpx.AsyncClient(timeout=15.0) as client: + for i in range(_NUM_SYNTHETIC_EVENTS): + response = await client.post( + f"{gateway_url}/v1/messages", + json={ + "model": "claude-haiku-4-5", + "messages": [{"role": "user", "content": f"synthetic request {i}"}], + "max_tokens": 100, + "stream": False, + }, + headers={"Authorization": f"Bearer {_API_KEY}"}, + ) + assert response.status_code == 200 + requests_done.set() + + sse_task = asyncio.create_task(collect_sse()) + send_task = asyncio.create_task(send_synthetic_requests()) + + done, pending = await asyncio.wait( + [sse_task, send_task], + timeout=15.0, + return_when=asyncio.ALL_COMPLETED, + ) + + for task in pending: + task.cancel() + try: + await task + except (asyncio.CancelledError, Exception): + pass + + assert send_task in done, "Synthetic requests did not complete in time" + assert len(sse_events) >= _NUM_SYNTHETIC_EVENTS, ( + f"Expected at least {_NUM_SYNTHETIC_EVENTS} activity events but got {len(sse_events)}. Events: {sse_events}" + ) + + for i, event in enumerate(sse_events[:_NUM_SYNTHETIC_EVENTS]): + missing = _EXPECTED_EVENT_FIELDS - set(event) + assert not missing, f"Event {i} missing required fields {missing}: {event}" diff --git a/tests/luthien_proxy/integration_tests/test_policy_type_sync.py b/tests/luthien_proxy/integration_tests/test_policy_type_sync.py index 189565e1d..d321924d7 100644 --- a/tests/luthien_proxy/integration_tests/test_policy_type_sync.py +++ b/tests/luthien_proxy/integration_tests/test_policy_type_sync.py @@ -20,7 +20,7 @@ from luthien_proxy.policy_core.base_policy import BasePolicy from luthien_proxy.policy_types import REGISTERED_BUILTINS, sync_policy_types from luthien_proxy.utils.db import DatabasePool -from luthien_proxy.utils.migration_check import _apply_sqlite_migrations +from luthien_proxy.utils.migration_check import apply_sqlite_migrations @pytest.fixture @@ -28,7 +28,7 @@ async def db_pool_with_migrations() -> AsyncIterator[DatabasePool]: """Create a fresh in-memory SQLite DatabasePool with all migrations applied.""" db_pool = DatabasePool("sqlite://:memory:") migrations_dir = Path(__file__).resolve().parents[3] / "src" / "luthien_proxy" / "utils" / "sqlite_migrations" - await _apply_sqlite_migrations(db_pool, migrations_dir=migrations_dir) + await apply_sqlite_migrations(db_pool, migrations_dir=migrations_dir) yield db_pool await db_pool.close() diff --git a/tests/luthien_proxy/integration_tests/test_server_timing.py b/tests/luthien_proxy/integration_tests/test_server_timing.py new file mode 100644 index 000000000..f45997d53 --- /dev/null +++ b/tests/luthien_proxy/integration_tests/test_server_timing.py @@ -0,0 +1,80 @@ +"""Integration tests for ServerTimingMiddleware. + +Tests that the Server-Timing header is correctly added to admin/debug/UI paths +and absent from gateway paths like /v1/messages. +""" + +from __future__ import annotations + +import asyncio + +import pytest +from fastapi.testclient import TestClient + +from luthien_proxy.main import create_app +from luthien_proxy.utils import db + +pytestmark = pytest.mark.integration + + +@pytest.fixture +def app_with_db(): + """Create an in-process app with SQLite for testing.""" + db_pool = db.DatabasePool("sqlite:///:memory:") + + app = create_app( + api_key=None, + admin_key="test-admin-key", + db_pool=db_pool, + redis_client=None, + startup_policy_path=None, + policy_source="file", + ) + + yield app + + asyncio.run(db_pool.close()) + + +def test_server_timing_header_absent_on_v1_messages(app_with_db): + """Server-Timing header should NOT be present on /v1/messages.""" + client = TestClient(app_with_db) + response = client.post( + "/v1/messages", + json={ + "model": "claude-3-5-sonnet-20241022", + "max_tokens": 100, + "messages": [{"role": "user", "content": "test"}], + }, + ) + assert "Server-Timing" not in response.headers + + +def test_server_timing_header_absent_on_health(app_with_db): + """Server-Timing header should NOT be present on /health.""" + client = TestClient(app_with_db) + response = client.get("/health") + assert response.status_code == 200 + assert "Server-Timing" not in response.headers + + +def test_server_timing_header_present_on_timed_path(): + """Server-Timing header should be present on paths matching the timed prefixes.""" + from fastapi import FastAPI + + from luthien_proxy.perf.timing_middleware import ServerTimingMiddleware, time_phase + + mini_app = FastAPI() + mini_app.add_middleware(ServerTimingMiddleware) + + @mini_app.get("/api/history/sessions") + def sessions(): + with time_phase("handler"): + pass + return {} + + client = TestClient(mini_app) + response = client.get("/api/history/sessions") + assert response.status_code == 200 + assert "Server-Timing" in response.headers + assert "handler" in response.headers["Server-Timing"] diff --git a/tests/luthien_proxy/perf_tests/AGENTS.md b/tests/luthien_proxy/perf_tests/AGENTS.md new file mode 100644 index 000000000..69977c6f7 --- /dev/null +++ b/tests/luthien_proxy/perf_tests/AGENTS.md @@ -0,0 +1,92 @@ +# Performance Testing Guidelines + +> Canonical file — `CLAUDE.md` in this directory is a symlink to this file. Edit `AGENTS.md` only. + +## Purpose + +Performance tests measure gateway latency and throughput under realistic conditions. They validate that the gateway meets SLO targets for page load, transcript rendering, and API payload sizes. Tests are opt-in and excluded from default pytest runs to avoid slowing down CI. + +## Marker + +Performance tests use the `@pytest.mark.perf` marker: + +```python +@pytest.mark.perf +async def test_page_load_latency(perf_fixture): + # Test code + pass +``` + +The `perf` marker is **excluded by default** from `pytest` runs. To run perf tests: + +```bash +./scripts/run_perf.sh +# or directly: +uv run pytest -m perf tests/luthien_proxy/perf_tests/ -v +``` + +## Running + +### Default pytest (excludes perf) + +```bash +# Unit tests only (perf excluded) +uv run pytest tests/luthien_proxy/unit_tests + +# All tiers except perf +./scripts/dev_checks.sh +``` + +### Perf tests only + +```bash +# Run all perf tests +./scripts/run_perf.sh + +# Run specific perf test +./scripts/run_perf.sh -- -k "test_page_load" + +# Run with verbose output +./scripts/run_perf.sh -- -vv +``` + +### Test Infrastructure + +Perf tests use Playwright for browser automation and timing measurement. Fixtures are defined in `conftest.py`: + +- **Browser fixtures**: `playwright_browser`, `playwright_page` — Chromium browser instance and page context +- **Gateway fixtures**: `perf_gateway_url`, `admin_headers` — isolated perf test gateway and admin auth headers +- **Timing helpers**: `measure_page_load(page, url, headers)` — measures TTFP and Navigation Timing +- **Database fixtures**: `perf_db_url` — isolated SQLite database URL for perf tests (never touches dev DB) + +## SLO Definitions + +Performance targets are measured on a local network with sami-like fixture data (78 sessions, largest ~442 messages). + +### Page Load SLO + +**Metric**: time-to-first-turn-painted (DOM mutation observer on messages container) + +- **Local network**: < 2 seconds +- **Throttled (Tailscale Funnel ~1 Mbps + 300ms RTT)**: < 5 seconds + +### Transcript Open SLO + +**Metric**: time-to-first-turn-painted after clicking a session in history list + +- **Local network**: < 1 second +- **Throttled**: < 5 seconds + +### Payload Size SLO + +**Metric**: gzipped response size for first page of results + +- `/api/history/sessions` (first page): < 50 KB gzipped +- `/api/history/sessions/{id}` (first page): < 100 KB gzipped + +### Measurement Methodology + +- **Same machine for before and after**: Ensure consistent hardware +- **Median + p95 over ≥5 runs**: Report both metrics +- **Cold cache first run reported separately**: Distinguish cold-start from warm-cache behavior +- **Chromium only**: Firefox and WebKit do not support CDP bandwidth shaping for throttled tests diff --git a/tests/luthien_proxy/perf_tests/CLAUDE.md b/tests/luthien_proxy/perf_tests/CLAUDE.md new file mode 120000 index 000000000..47dc3e3d8 --- /dev/null +++ b/tests/luthien_proxy/perf_tests/CLAUDE.md @@ -0,0 +1 @@ +AGENTS.md \ No newline at end of file diff --git a/tests/luthien_proxy/perf_tests/__init__.py b/tests/luthien_proxy/perf_tests/__init__.py new file mode 100644 index 000000000..cce43c458 --- /dev/null +++ b/tests/luthien_proxy/perf_tests/__init__.py @@ -0,0 +1 @@ +"""Performance tests for the Luthien gateway.""" diff --git a/tests/luthien_proxy/perf_tests/conftest.py b/tests/luthien_proxy/perf_tests/conftest.py new file mode 100644 index 000000000..89f31f44d --- /dev/null +++ b/tests/luthien_proxy/perf_tests/conftest.py @@ -0,0 +1,258 @@ +"""Shared fixtures and helpers for performance tests. + +Infrastructure for perf tests: isolated gateway (perf DB, never dev DB), +Playwright browser automation, Navigation Timing capture, and n_runs statistics +that separate the cold-cache first run from warm runs. +""" + +from __future__ import annotations + +import asyncio +import os +import socket +import statistics +import threading +import time +from collections.abc import AsyncIterator, Iterator +from dataclasses import dataclass +from typing import Any, Callable + +import pytest +import uvicorn +from playwright.async_api import Browser, Page, async_playwright + +from luthien_proxy.main import create_app +from luthien_proxy.perf.db import get_perf_db_url, migrate_perf_db +from luthien_proxy.settings import clear_settings_cache +from luthien_proxy.utils.db import DatabasePool + + +def pytest_addoption(parser: pytest.Parser) -> None: + """Add --update-snapshots option to pytest.""" + parser.addoption( + "--update-snapshots", + action="store_true", + default=False, + help="Regenerate snapshot files", + ) + + +def pytest_configure(config: pytest.Config) -> None: + if config.pluginmanager.has_plugin("xdist"): + raise pytest.UsageError( + "Perf tests cannot run under pytest-xdist: the session-scoped " + "perf_gateway_url fixture mutates os.environ, which would race " + "across workers. Run perf tests with: ./scripts/run_perf.sh" + ) + + +_ADMIN_KEY = "admin-dev-key" +_API_KEY = "sk-perf-test-key" + + +@dataclass +class PageLoadMetrics: + ttfb_ms: float + dcl_ms: float + load_ms: float + ttfm_ms: float # time-to-first-mutation on #main (0 if no mutation observed) + + +@dataclass +class RunStats: + cold_ms: float # first run — cold cache, excluded from warm stats + warm_median_ms: float + warm_p95_ms: float + n_warm: int + + +def _percentile(sorted_data: list[float], pct: float) -> float: + if not sorted_data: + return 0.0 + idx = min(int(len(sorted_data) * pct), len(sorted_data) - 1) + return sorted_data[idx] + + +def _free_port() -> int: + with socket.socket() as s: + s.bind(("", 0)) + return s.getsockname()[1] + + +async def n_runs(fn: Callable[[], Any], n: int = 5) -> RunStats: + """Run fn N times; first run is cold-cache and excluded from median/p95. + + fn may be async or sync and must return a float (elapsed ms). + """ + times: list[float] = [] + for _ in range(n): + result = fn() + if asyncio.iscoroutine(result): + result = await result + times.append(float(result)) + + cold = times[0] + warm = times[1:] + sorted_warm = sorted(warm) + + return RunStats( + cold_ms=cold, + warm_median_ms=statistics.median(warm) if warm else 0.0, + warm_p95_ms=_percentile(sorted_warm, 0.95), + n_warm=len(warm), + ) + + +async def measure_page_load(page: Page, url: str) -> PageLoadMetrics: + """Navigate to url and return Navigation Timing + first-mutation metrics. + + Uses add_init_script so the MutationObserver is installed before any page + JS runs — necessary because the mutation may fire during initial render. + """ + # Guard flag prevents duplicate observers when called multiple times on the same page. + await page.add_init_script(""" + if (!window.__perfObserverInstalled) { + window.__perfObserverInstalled = true; + window.__firstMutation = null; + function _setupMutObs() { + var target = document.getElementById('main') || document.body; + var obs = new MutationObserver(function() { + if (window.__firstMutation === null) { + window.__firstMutation = performance.now(); + obs.disconnect(); + } + }); + obs.observe(target, { childList: true, subtree: true }); + } + if (document.readyState === 'loading') { + document.addEventListener('DOMContentLoaded', _setupMutObs); + } else { + _setupMutObs(); + } + } + """) + + await page.goto(url, wait_until="networkidle") + + metrics: dict[str, float] = await page.evaluate("""() => { + var entries = window.performance.getEntriesByType('navigation'); + if (entries.length > 0) { + var nav = entries[0]; + return { + ttfb: nav.responseStart, + dcl: nav.domContentLoadedEventEnd, + load: nav.loadEventEnd, + ttfm: window.__firstMutation || 0 + }; + } + var t = window.performance.timing; + var origin = t.fetchStart; + return { + ttfb: t.responseStart - origin, + dcl: t.domContentLoadedEventEnd - origin, + load: t.loadEventEnd - origin, + ttfm: window.__firstMutation || 0 + }; + }""") + + return PageLoadMetrics( + ttfb_ms=metrics["ttfb"], + dcl_ms=metrics["dcl"], + load_ms=metrics["load"], + ttfm_ms=metrics["ttfm"], + ) + + +@pytest.fixture(scope="session") +def perf_db_url() -> str: + url = get_perf_db_url("sqlite") + migrate_perf_db("sqlite") + return url + + +@pytest.fixture(scope="session") +def perf_gateway_url(perf_db_url: str) -> Iterator[str]: + """In-process FastAPI gateway on a random port backed by the perf DB. + + ANTHROPIC_BASE_URL is pointed at 127.0.0.1:1 (unreachable) so any + accidental upstream call fails immediately rather than hanging. + """ + port = _free_port() + db_pool = DatabasePool(perf_db_url) + + # NOTE: perf tests must run in a dedicated pytest session (run_perf.sh enforces this). + # Under pytest-xdist or parallel sessions this os.environ mutation would race. + saved_env: dict[str, str | None] = {k: os.environ.get(k) for k in ("ANTHROPIC_BASE_URL", "ANTHROPIC_API_KEY")} + + def restore_env() -> None: + for k, v in saved_env.items(): + if v is None: + os.environ.pop(k, None) + else: + os.environ[k] = v + + os.environ["ANTHROPIC_BASE_URL"] = "http://127.0.0.1:1" + os.environ["ANTHROPIC_API_KEY"] = "mock-key" + clear_settings_cache() + + app = create_app( + api_key=_API_KEY, + admin_key=_ADMIN_KEY, + db_pool=db_pool, + redis_client=None, + startup_policy_path="config/policy_config.yaml", + policy_source="db-fallback-file", + ) + + config = uvicorn.Config(app, host="127.0.0.1", port=port, log_level="warning") + server = uvicorn.Server(config) + thread = threading.Thread(target=server.run, daemon=True, name="perf-gateway") + thread.start() + + deadline = time.monotonic() + 10 + started = False + while time.monotonic() < deadline: + try: + with socket.create_connection(("127.0.0.1", port), timeout=0.5): + started = True + break + except OSError: + time.sleep(0.1) + + if not started: + server.should_exit = True + thread.join(timeout=5) + restore_env() + clear_settings_cache() + asyncio.run(db_pool.close()) + raise RuntimeError("Perf gateway did not start within 10 s") + + yield f"http://127.0.0.1:{port}" + + server.should_exit = True + thread.join(timeout=5) + restore_env() + clear_settings_cache() + asyncio.run(db_pool.close()) + + +@pytest.fixture(scope="session") +def admin_headers() -> dict[str, str]: + return {"Authorization": f"Bearer {_ADMIN_KEY}"} + + +@pytest.fixture(scope="session") +async def playwright_browser() -> AsyncIterator[Browser]: + async with async_playwright() as pw: + browser = await pw.chromium.launch(args=["--disable-cache", "--disable-gpu"]) + yield browser + await browser.close() + + +@pytest.fixture +async def playwright_page(playwright_browser: Browser) -> AsyncIterator[Page]: + """Fresh browser context per test — no cookie/cache bleed across tests.""" + context = await playwright_browser.new_context() + page = await context.new_page() + yield page + await context.close() diff --git a/tests/luthien_proxy/perf_tests/snapshots/call_diff.json b/tests/luthien_proxy/perf_tests/snapshots/call_diff.json new file mode 100644 index 000000000..6c152bdc0 --- /dev/null +++ b/tests/luthien_proxy/perf_tests/snapshots/call_diff.json @@ -0,0 +1,21 @@ +{ + "call_id": "str", + "request": { + "model_changed": "bool", + "original_model": "null", + "final_model": "null", + "max_tokens_changed": "bool", + "original_max_tokens": "null", + "final_max_tokens": "null", + "messages": [] + }, + "response": { + "content_changed": "bool", + "original_content": "str", + "final_content": "str", + "finish_reason_changed": "bool", + "original_finish_reason": "null", + "final_finish_reason": "null" + }, + "tempo_trace_url": "null" +} diff --git a/tests/luthien_proxy/perf_tests/snapshots/call_events.json b/tests/luthien_proxy/perf_tests/snapshots/call_events.json new file mode 100644 index 000000000..b20988b5a --- /dev/null +++ b/tests/luthien_proxy/perf_tests/snapshots/call_events.json @@ -0,0 +1,15 @@ +{ + "call_id": "str", + "events": [ + { + "event_id": "str", + "call_id": "str", + "event_type": "str", + "payload": {}, + "created_at": "str", + "session_id": "str" + } + ], + "tempo_trace_url": "null", + "session_id": "str" +} diff --git a/tests/luthien_proxy/perf_tests/snapshots/calls_list.json b/tests/luthien_proxy/perf_tests/snapshots/calls_list.json new file mode 100644 index 000000000..ba7997ff9 --- /dev/null +++ b/tests/luthien_proxy/perf_tests/snapshots/calls_list.json @@ -0,0 +1,11 @@ +{ + "calls": [ + { + "call_id": "str", + "event_count": "int", + "latest_timestamp": "str", + "session_id": "str" + } + ], + "total": "int" +} \ No newline at end of file diff --git a/tests/luthien_proxy/perf_tests/snapshots/policy_current.json b/tests/luthien_proxy/perf_tests/snapshots/policy_current.json new file mode 100644 index 000000000..bb7641533 --- /dev/null +++ b/tests/luthien_proxy/perf_tests/snapshots/policy_current.json @@ -0,0 +1,7 @@ +{ + "policy": "str", + "class_ref": "str", + "enabled_at": "str", + "enabled_by": "str", + "config": {} +} \ No newline at end of file diff --git a/tests/luthien_proxy/perf_tests/snapshots/session_detail.json b/tests/luthien_proxy/perf_tests/snapshots/session_detail.json new file mode 100644 index 000000000..3c5ff87c6 --- /dev/null +++ b/tests/luthien_proxy/perf_tests/snapshots/session_detail.json @@ -0,0 +1,48 @@ +{ + "session_id": "str", + "first_timestamp": "str", + "last_timestamp": "str", + "turns": [ + { + "call_id": "str", + "timestamp": "str", + "model": "str", + "request_messages": [ + { + "message_type": "str", + "content": "str", + "tool_name": "null", + "tool_call_id": "null", + "tool_input": "null", + "is_error": "null" + } + ], + "response_messages": [ + { + "message_type": "str", + "content": "str", + "tool_name": "null", + "tool_call_id": "null", + "tool_input": "null", + "is_error": "null" + } + ], + "annotations": "list[unknown]", + "had_policy_intervention": "bool", + "request_was_modified": "bool", + "response_was_modified": "bool", + "original_request_messages": "null", + "original_response_messages": "null", + "request_params": { + "model": "str", + "max_tokens": "int", + "stream": "bool", + "temperature": "float" + } + } + ], + "total_policy_interventions": "int", + "models_used": [ + "str" + ] +} \ No newline at end of file diff --git a/tests/luthien_proxy/perf_tests/snapshots/sessions_list.json b/tests/luthien_proxy/perf_tests/snapshots/sessions_list.json new file mode 100644 index 000000000..7d3c96934 --- /dev/null +++ b/tests/luthien_proxy/perf_tests/snapshots/sessions_list.json @@ -0,0 +1,20 @@ +{ + "sessions": [ + { + "session_id": "str", + "first_timestamp": "str", + "last_timestamp": "str", + "turn_count": "int", + "total_events": "int", + "policy_interventions": "int", + "models_used": [ + "str" + ], + "preview_message": "str", + "user_ids": "list[unknown]" + } + ], + "total": "int", + "offset": "int", + "has_more": "bool" +} diff --git a/tests/luthien_proxy/perf_tests/test_api_contract.py b/tests/luthien_proxy/perf_tests/test_api_contract.py new file mode 100644 index 000000000..a94700d01 --- /dev/null +++ b/tests/luthien_proxy/perf_tests/test_api_contract.py @@ -0,0 +1,303 @@ +"""JSON API contract snapshot tests. + +These tests capture the response shape (keys + types, not values) and fail if the shape changes. +Snapshots are stored in tests/luthien_proxy/perf_tests/snapshots/ and can be regenerated with --update-snapshots. + +Marked with @pytest.mark.perf and @pytest.mark.contract for selective execution. + +Covered endpoints (6): + GET /api/history/sessions + GET /api/history/sessions/{id} + GET /api/debug/calls + GET /api/debug/calls/{id} + GET /api/debug/calls/{id}/events (CallEventsResponse shape) + GET /api/debug/calls/{id}/diff (CallDiffResponse shape) +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import httpx +import pytest + +from luthien_proxy.perf.seeding import seed_sessions + + +@pytest.fixture(scope="session") +def seeded_perf_db(perf_db_url: str) -> None: + """Seed the perf DB with test data once per session.""" + import sqlite3 + + db_path = Path.home() / ".luthien" / "perf.db" + conn = sqlite3.connect(str(db_path)) + try: + (count,) = conn.execute("SELECT COUNT(*) FROM conversation_calls").fetchone() + if count == 0: + seed_sessions("sqlite", tier=100) + finally: + conn.close() + + +def _extract_shape(obj: Any) -> Any: + """Extract type structure from a value (not the value itself). + + Examples: + - 123 → "int" + - "hello" → "str" + - True → "bool" + - None → "null" + - [1, 2] → ["int"] (first element's type) + - {"a": 1} → {"a": "int"} + + Note: for lists, only the first element is inspected. Heterogeneous lists + (e.g. discriminated union event payloads) will not have shape drift detected + past index 0. This is acceptable for API contract testing — the snapshot + captures the common element shape, not exhaustive coverage of every variant. + """ + if obj is None: + return "null" + if isinstance(obj, bool): + return "bool" + if isinstance(obj, int): + return "int" + if isinstance(obj, float): + return "float" + if isinstance(obj, str): + return "str" + if isinstance(obj, list): + if not obj: + return [] + return [_extract_shape(obj[0])] + if isinstance(obj, dict): + return {k: _extract_shape(v) for k, v in obj.items()} + return type(obj).__name__ + + +def _get_snapshots_dir() -> Path: + """Get the snapshots directory, creating it if needed.""" + snapshots_dir = Path(__file__).parent / "snapshots" + snapshots_dir.mkdir(exist_ok=True) + return snapshots_dir + + +def _load_snapshot(name: str) -> dict[str, Any]: + """Load a snapshot from disk.""" + snapshot_file = _get_snapshots_dir() / f"{name}.json" + if not snapshot_file.exists(): + raise FileNotFoundError(f"Snapshot not found: {snapshot_file}") + with open(snapshot_file) as f: + return json.load(f) + + +def _save_snapshot(name: str, data: dict[str, Any]) -> None: + """Save a snapshot to disk.""" + snapshot_file = _get_snapshots_dir() / f"{name}.json" + with open(snapshot_file, "w") as f: + json.dump(data, f, indent=2) + + +@pytest.fixture +def update_snapshots(request: pytest.FixtureRequest) -> bool: + """Check if --update-snapshots flag was passed.""" + return request.config.getoption("--update-snapshots", default=False) + + +@pytest.mark.perf +@pytest.mark.contract +async def test_policy_current_contract( + perf_gateway_url: str, + admin_headers: dict[str, str], + perf_db_url: str, + update_snapshots: bool, + seeded_perf_db: None, +) -> None: + """Test /api/admin/policy/current response shape.""" + async with httpx.AsyncClient(base_url=perf_gateway_url) as client: + response = await client.get( + "/api/admin/policy/current", + headers=admin_headers, + ) + + assert response.status_code == 200 + data = response.json() + shape = _extract_shape(data) + + if update_snapshots: + _save_snapshot("policy_current", shape) + else: + expected = _load_snapshot("policy_current") + assert shape == expected, ( + f"Shape mismatch:\nGot: {json.dumps(shape, indent=2)}\nExpected: {json.dumps(expected, indent=2)}" + ) + + +@pytest.mark.perf +@pytest.mark.contract +@pytest.mark.timeout(30) +async def test_session_detail_contract( + perf_gateway_url: str, + admin_headers: dict[str, str], + perf_db_url: str, + update_snapshots: bool, + seeded_perf_db: None, +) -> None: + """Test /api/history/sessions/{session_id} response shape.""" + async with httpx.AsyncClient(base_url=perf_gateway_url) as client: + list_response = await client.get( + "/api/history/sessions?limit=1", + headers=admin_headers, + ) + + assert list_response.status_code == 200 + sessions = list_response.json()["sessions"] + assert len(sessions) > 0, "No sessions found in database" + session_id = sessions[0]["session_id"] + + async with httpx.AsyncClient(base_url=perf_gateway_url) as client: + response = await client.get( + f"/api/history/sessions/{session_id}", + headers=admin_headers, + ) + + assert response.status_code == 200 + data = response.json() + shape = _extract_shape(data) + + if update_snapshots: + _save_snapshot("session_detail", shape) + else: + expected = _load_snapshot("session_detail") + assert shape == expected, ( + f"Shape mismatch:\nGot: {json.dumps(shape, indent=2)}\nExpected: {json.dumps(expected, indent=2)}" + ) + + +@pytest.mark.perf +@pytest.mark.contract +async def test_calls_list_contract( + perf_gateway_url: str, + admin_headers: dict[str, str], + perf_db_url: str, + update_snapshots: bool, + seeded_perf_db: None, +) -> None: + """Test /api/debug/calls response shape.""" + async with httpx.AsyncClient(base_url=perf_gateway_url) as client: + response = await client.get( + "/api/debug/calls?limit=20", + headers=admin_headers, + ) + + assert response.status_code == 200 + data = response.json() + shape = _extract_shape(data) + + if update_snapshots: + _save_snapshot("calls_list", shape) + else: + expected = _load_snapshot("calls_list") + assert shape == expected, ( + f"Shape mismatch:\nGot: {json.dumps(shape, indent=2)}\nExpected: {json.dumps(expected, indent=2)}" + ) + + +@pytest.mark.perf +@pytest.mark.contract +async def test_sessions_list_contract( + perf_gateway_url: str, + admin_headers: dict[str, str], + perf_db_url: str, + update_snapshots: bool, + seeded_perf_db: None, +) -> None: + """Test /api/history/sessions response shape.""" + async with httpx.AsyncClient(base_url=perf_gateway_url) as client: + response = await client.get( + "/api/history/sessions?limit=20", + headers=admin_headers, + ) + + assert response.status_code == 200 + data = response.json() + shape = _extract_shape(data) + + if update_snapshots: + _save_snapshot("sessions_list", shape) + else: + expected = _load_snapshot("sessions_list") + assert shape == expected, ( + f"Shape mismatch:\nGot: {json.dumps(shape, indent=2)}\nExpected: {json.dumps(expected, indent=2)}" + ) + + +@pytest.mark.perf +@pytest.mark.contract +@pytest.mark.timeout(30) +async def test_call_events_contract( + perf_gateway_url: str, + admin_headers: dict[str, str], + perf_db_url: str, + update_snapshots: bool, + seeded_perf_db: None, +) -> None: + """Test /api/debug/calls/{call_id} (events) response shape.""" + async with httpx.AsyncClient(base_url=perf_gateway_url) as client: + list_response = await client.get("/api/debug/calls?limit=1", headers=admin_headers) + + assert list_response.status_code == 200 + calls = list_response.json()["calls"] + assert len(calls) > 0, "No calls found in database" + call_id = calls[0]["call_id"] + + async with httpx.AsyncClient(base_url=perf_gateway_url) as client: + response = await client.get(f"/api/debug/calls/{call_id}", headers=admin_headers) + + assert response.status_code == 200 + data = response.json() + shape = _extract_shape(data) + + if update_snapshots: + _save_snapshot("call_events", shape) + else: + expected = _load_snapshot("call_events") + assert shape == expected, ( + f"Shape mismatch:\nGot: {json.dumps(shape, indent=2)}\nExpected: {json.dumps(expected, indent=2)}" + ) + + +@pytest.mark.perf +@pytest.mark.contract +@pytest.mark.timeout(30) +async def test_call_diff_contract( + perf_gateway_url: str, + admin_headers: dict[str, str], + perf_db_url: str, + update_snapshots: bool, + seeded_perf_db: None, +) -> None: + """Test /api/debug/calls/{call_id}/diff response shape.""" + async with httpx.AsyncClient(base_url=perf_gateway_url) as client: + list_response = await client.get("/api/debug/calls?limit=1", headers=admin_headers) + + assert list_response.status_code == 200 + calls = list_response.json()["calls"] + assert len(calls) > 0, "No calls found in database" + call_id = calls[0]["call_id"] + + async with httpx.AsyncClient(base_url=perf_gateway_url) as client: + response = await client.get(f"/api/debug/calls/{call_id}/diff", headers=admin_headers) + + assert response.status_code == 200 + data = response.json() + shape = _extract_shape(data) + + if update_snapshots: + _save_snapshot("call_diff", shape) + else: + expected = _load_snapshot("call_diff") + assert shape == expected, ( + f"Shape mismatch:\nGot: {json.dumps(shape, indent=2)}\nExpected: {json.dumps(expected, indent=2)}" + ) diff --git a/tests/luthien_proxy/perf_tests/test_harness_smoke.py b/tests/luthien_proxy/perf_tests/test_harness_smoke.py new file mode 100644 index 000000000..8fa847ebf --- /dev/null +++ b/tests/luthien_proxy/perf_tests/test_harness_smoke.py @@ -0,0 +1,11 @@ +"""Smoke test for the perf harness: verifies gateway starts and serves a page.""" + +import pytest + +pytestmark = pytest.mark.perf + + +async def test_can_load_index(perf_gateway_url, playwright_page): + response = await playwright_page.goto(perf_gateway_url + "/") + assert response is not None + assert response.status == 200 diff --git a/tests/luthien_proxy/perf_tests/test_page_load.py b/tests/luthien_proxy/perf_tests/test_page_load.py new file mode 100644 index 000000000..c7ba4c8ab --- /dev/null +++ b/tests/luthien_proxy/perf_tests/test_page_load.py @@ -0,0 +1,202 @@ +"""Per-page performance scenarios — auto-discovers all admin UI routes. + +Parametrized by fixture_name x route_path (4 x 11 = 44 test cases). +SLO enforced only on /history and /conversation/live/{id} for sami-like +and tier-1000 fixtures. +""" + +from __future__ import annotations + +import json +import sqlite3 +from collections import defaultdict +from collections.abc import Iterator +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +import httpx +import pytest +from fastapi.routing import APIRoute +from playwright.async_api import Page + +from luthien_proxy.main import create_app +from luthien_proxy.perf.db import get_perf_db_url +from luthien_proxy.perf.seeding import seed_sami_like, seed_sessions +from luthien_proxy.utils.db import DatabasePool + +from .conftest import measure_page_load, n_runs + +_REPO_ROOT = Path(__file__).resolve().parents[3] +EVIDENCE_DIR = _REPO_ROOT / ".sisyphus" / "evidence" +TRACES_DIR = EVIDENCE_DIR / "traces" + +_TTFB_SLO_MS: float = 2_000.0 +_SLO_FIXTURES: frozenset[str] = frozenset({"sami-like", "tier-1000"}) +_SLO_PAGES: frozenset[str] = frozenset({"/history", "/conversation/live/{conversation_id}"}) + +FIXTURE_NAMES: list[str] = ["sami-like", "tier-100", "tier-1000", "tier-10000"] +N_RUNS: int = 5 + + +def _discover_html_routes() -> list[str]: + app = create_app( + api_key="x", + admin_key="x", + db_pool=DatabasePool(get_perf_db_url("sqlite")), # lazy — no connection opened + redis_client=None, + startup_policy_path=None, + ) + + excluded_prefixes = ("/api/", "/v1/", "/static/", "/auth/") + excluded_paths: frozenset[str] = frozenset({"/health", "/ready", "/login"}) + + routes: list[str] = [] + for raw_route in app.routes: + if not isinstance(raw_route, APIRoute): + continue + if "GET" not in (raw_route.methods or set()): + continue + if raw_route.response_model is not None: + continue + path = raw_route.path + if any(path.startswith(p) for p in excluded_prefixes): + continue + if path in excluded_paths: + continue + if path.endswith("/{path:path}"): + continue + if "{" in path and path != "/conversation/live/{conversation_id}": + continue + routes.append(path) + + return sorted(routes) + + +def _live_conversation_id(fixture_name: str) -> str: + if fixture_name == "sami-like": + return "perf-seed-sami-442msg" + tier = fixture_name.split("-")[1] + return f"perf-seed-{tier}-0001" + + +def _resolve_url(base_url: str, route_path: str, fixture_name: str) -> str: + if "{conversation_id}" in route_path: + route_path = route_path.replace("{conversation_id}", _live_conversation_id(fixture_name)) + return base_url.rstrip("/") + route_path + + +def _slo_enforced(fixture_name: str, route_path: str) -> bool: + return fixture_name in _SLO_FIXTURES and route_path in _SLO_PAGES + + +@pytest.fixture(scope="session") +def seeded_perf_db_all(perf_db_url: str) -> None: # noqa: ARG001 + db_path = Path.home() / ".luthien" / "perf.db" + conn = sqlite3.connect(str(db_path)) + try: + for prefix, seed_fn in [ + ("perf-seed-sami-%", lambda: seed_sami_like("sqlite")), + ("perf-seed-100-%", lambda: seed_sessions("sqlite", tier=100)), + ("perf-seed-1000-%", lambda: seed_sessions("sqlite", tier=1000)), + ("perf-seed-10000-%", lambda: seed_sessions("sqlite", tier=10000)), + ]: + (count,) = conn.execute( + "SELECT COUNT(*) FROM conversation_calls WHERE session_id LIKE ?", + (prefix,), + ).fetchone() + if count == 0: + seed_fn() + finally: + conn.close() + + +@pytest.fixture(scope="session") +def perf_results_store() -> Iterator[dict[str, list[dict[str, Any]]]]: + store: dict[str, list[dict[str, Any]]] = defaultdict(list) + yield store # type: ignore[misc] + ts = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + EVIDENCE_DIR.mkdir(parents=True, exist_ok=True) + for fixture_name, scenarios in store.items(): + if not scenarios: + continue + result: dict[str, Any] = { + "fixture": fixture_name, + "timestamp": ts, + "scenarios": scenarios, + } + out_path = EVIDENCE_DIR / f"perf-results-{fixture_name}-{ts}.json" + with open(out_path, "w") as f: + json.dump(result, f, indent=2) + + +def pytest_generate_tests(metafunc: pytest.Metafunc) -> None: + if "fixture_name" in metafunc.fixturenames and "route_path" in metafunc.fixturenames: + admin_routes = _discover_html_routes() + metafunc.parametrize( + "fixture_name,route_path", + [(f, p) for f in FIXTURE_NAMES for p in admin_routes], + ) + + +@pytest.mark.perf +@pytest.mark.asyncio +async def test_page_load( + fixture_name: str, + route_path: str, + perf_gateway_url: str, + playwright_page: Page, + admin_headers: dict[str, str], + seeded_perf_db_all: None, # noqa: ARG001 + perf_results_store: dict[str, list[dict[str, Any]]], +) -> None: + EVIDENCE_DIR.mkdir(parents=True, exist_ok=True) + TRACES_DIR.mkdir(parents=True, exist_ok=True) + + url = _resolve_url(perf_gateway_url, route_path, fixture_name) + slo_ok = _slo_enforced(fixture_name, route_path) + + await playwright_page.set_extra_http_headers(admin_headers) + + safe_page = route_path.replace("/", "_").replace("{", "").replace("}", "") + trace_path = str(TRACES_DIR / f"trace-{fixture_name}{safe_page}.zip") + await playwright_page.context.tracing.start(screenshots=True, snapshots=True, sources=True) + + try: + + async def _load_once() -> float: + m = await measure_page_load(playwright_page, url) + return m.ttfb_ms + + run_stats = await n_runs(_load_once, n=N_RUNS) + final_m = await measure_page_load(playwright_page, url) + + async with httpx.AsyncClient(headers=admin_headers, follow_redirects=True) as client: + http_resp = await client.get(url) + transfer_bytes: int = len(http_resp.content) + transfer_encoding: str = http_resp.headers.get("transfer-encoding", "identity") + + finally: + await playwright_page.context.tracing.stop(path=trace_path) + + scenario: dict[str, Any] = { + "page": route_path, + "url": url, + "slo_enforced": slo_ok, + "cold_ms": run_stats.cold_ms, + "median_ms": run_stats.warm_median_ms, + "p95_ms": run_stats.warm_p95_ms, + "ttfb_ms": final_m.ttfb_ms, + "dcl_ms": final_m.dcl_ms, + "ttfm_ms": final_m.ttfm_ms, + "transfer_bytes": transfer_bytes, + "transfer_encoding": transfer_encoding, + } + perf_results_store[fixture_name].append(scenario) + + if slo_ok: + assert run_stats.warm_median_ms < _TTFB_SLO_MS, ( + f"TTFB SLO failed [{fixture_name}][{route_path}]: " + f"warm_median={run_stats.warm_median_ms:.0f} ms " + f"> threshold={_TTFB_SLO_MS:.0f} ms" + ) diff --git a/tests/luthien_proxy/perf_tests/test_sse_memory.py b/tests/luthien_proxy/perf_tests/test_sse_memory.py new file mode 100644 index 000000000..db62508c5 --- /dev/null +++ b/tests/luthien_proxy/perf_tests/test_sse_memory.py @@ -0,0 +1,126 @@ +"""SSE memory growth scenarios (P12). + +Opens /conversation/live/{session_id} and holds the SSE connection for 60 s, +sampling JSHeapUsedSize every 5 seconds via performance.memory. + +NOTE: performance.memory is Chrome-specific and non-standard. Values are +approximate unless Chromium is launched with --enable-precise-memory-info. + +The suspected leak: rawEvents[callId] in conversation_live.js:164-172 is an +unbounded dict that accumulates all SSE events per call ID without eviction. + +PR #1 (perf-baseline) records baseline only. Set PERF_ASSERT_MEMORY=1 to +enable the heap-growth assertion. +""" + +from __future__ import annotations + +import asyncio +import json +import os +import sqlite3 +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +import pytest +from playwright.async_api import Page + +from luthien_proxy.perf.seeding import seed_sami_like + +_REPO_ROOT = Path(__file__).resolve().parents[3] +EVIDENCE_DIR = _REPO_ROOT / ".sisyphus" / "evidence" + +_HOLD_SECONDS: int = 60 +_SAMPLE_INTERVAL_S: int = 5 +_SAMI_LIVE_SESSION = "perf-seed-sami-442msg" + + +@pytest.fixture(scope="session") +def seeded_sami_sse(perf_db_url: str) -> None: # noqa: ARG001 + db_path = Path.home() / ".luthien" / "perf.db" + conn = sqlite3.connect(str(db_path)) + try: + (count,) = conn.execute( + "SELECT COUNT(*) FROM conversation_calls WHERE session_id LIKE ?", + ("perf-seed-sami-%",), + ).fetchone() + if count == 0: + seed_sami_like("sqlite") + finally: + conn.close() + + +def _save_sse_memory_results( + session_id: str, + heap_samples: list[int], + heap_growth_pct: float, +) -> None: + ts = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + result: dict[str, Any] = { + "session_id": session_id, + "timestamp": ts, + "hold_seconds": _HOLD_SECONDS, + "sample_interval_s": _SAMPLE_INTERVAL_S, + "heap_samples_bytes": heap_samples, + "heap_first_bytes": heap_samples[0] if heap_samples else 0, + "heap_last_bytes": heap_samples[-1] if heap_samples else 0, + "heap_growth_pct": heap_growth_pct, + "note": ("performance.memory is Chrome-specific. For precise values use --enable-precise-memory-info."), + } + EVIDENCE_DIR.mkdir(parents=True, exist_ok=True) + out_path = EVIDENCE_DIR / f"perf-results-sse-memory-{ts}.json" + with open(out_path, "w") as f: + json.dump(result, f, indent=2) + + +@pytest.mark.perf +@pytest.mark.asyncio +@pytest.mark.timeout(90) +async def test_sse_heap_growth_60s( + playwright_page: Page, + perf_gateway_url: str, + admin_headers: dict[str, str], + seeded_sami_sse: None, # noqa: ARG001 +) -> None: + """Baseline: JS heap growth over 60 s on the live conversation page. + + Opens perf-seed-sami-442msg, holds the SSE connection, samples + JSHeapUsedSize every 5 s. Heap growth = (last - first) / first * 100. + + The 90 s timeout (pytest.mark.timeout) covers 60 s hold + navigation + and evaluation overhead. PR #1 records baseline only; set + PERF_ASSERT_MEMORY=1 to assert heap growth < 50% over 60 s. + """ + assert_memory = os.environ.get("PERF_ASSERT_MEMORY") == "1" + + await playwright_page.set_extra_http_headers(admin_headers) + url = f"{perf_gateway_url}/conversation/live/{_SAMI_LIVE_SESSION}" + await playwright_page.goto(url, wait_until="networkidle") + + heap_samples: list[int] = [] + n_samples = _HOLD_SECONDS // _SAMPLE_INTERVAL_S + + for _ in range(n_samples): + await asyncio.sleep(_SAMPLE_INTERVAL_S) + heap: int = await playwright_page.evaluate( + "() => window.performance.memory ? window.performance.memory.usedJSHeapSize : 0" + ) + heap_samples.append(heap) + + if heap_samples and heap_samples[0] > 0: + heap_growth_pct = (heap_samples[-1] - heap_samples[0]) / heap_samples[0] * 100 + else: + heap_growth_pct = 0.0 + + _save_sse_memory_results( + session_id=_SAMI_LIVE_SESSION, + heap_samples=heap_samples, + heap_growth_pct=heap_growth_pct, + ) + + if assert_memory: + assert heap_growth_pct < 50.0, ( + f"SSE memory growth: {heap_growth_pct:.1f}% > 50% over {_HOLD_SECONDS}s. " + "Possible leak in rawEvents[callId] (conversation_live.js:164-172)." + ) diff --git a/tests/luthien_proxy/perf_tests/test_throttled_network.py b/tests/luthien_proxy/perf_tests/test_throttled_network.py new file mode 100644 index 000000000..bd3da6379 --- /dev/null +++ b/tests/luthien_proxy/perf_tests/test_throttled_network.py @@ -0,0 +1,187 @@ +"""Throttled-network performance scenarios (P10b). + +Simulates Sami's Tailscale Funnel deployment shape (~1 Mbps + 300 ms RTT) +via Playwright CDP Network.emulateNetworkConditions. PR #1 (perf-baseline) +records measurements only; SLO assertions require PERF_THROTTLE_BASELINE=1. + +Chromium-only: Firefox and WebKit do not support CDP bandwidth shaping. +""" + +from __future__ import annotations + +import json +import os +import sqlite3 +import statistics +from collections.abc import AsyncIterator +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +import pytest +from playwright.async_api import Browser, Page + +from luthien_proxy.perf.seeding import seed_sami_like + +from .conftest import measure_page_load + +_REPO_ROOT = Path(__file__).resolve().parents[3] +EVIDENCE_DIR = _REPO_ROOT / ".sisyphus" / "evidence" + +# CDP throttle parameters — match Sami's Tailscale Funnel free-tier shape. +THROTTLE_DOWNLOAD_BPS: int = 125_000 # bytes/sec (~1 Mbps) +THROTTLE_UPLOAD_BPS: int = 125_000 # bytes/sec (~1 Mbps) +THROTTLE_LATENCY_MS: int = 300 # ms additional latency (RTT) + +_SAMI_LIVE_SESSION = "perf-seed-sami-442msg" +N_RUNS: int = 3 # 3 runs: CDP throttle adds ~300 ms/run; 3 is enough for a stable median + + +@pytest.fixture(scope="session") +def seeded_sami(perf_db_url: str) -> None: # noqa: ARG001 + db_path = Path.home() / ".luthien" / "perf.db" + conn = sqlite3.connect(str(db_path)) + try: + (count,) = conn.execute( + "SELECT COUNT(*) FROM conversation_calls WHERE session_id LIKE ?", + ("perf-seed-sami-%",), + ).fetchone() + if count == 0: + seed_sami_like("sqlite") + finally: + conn.close() + + +@pytest.fixture +async def throttled_page(playwright_browser: Browser) -> AsyncIterator[Page]: + """Fresh browser context with CDP network throttling pre-applied. + + Attaches a CDP session and calls Network.emulateNetworkConditions before + yielding the page. Each test gets an isolated context with no cookie or + cache bleed. + """ + context = await playwright_browser.new_context() + page = await context.new_page() + cdp = await context.new_cdp_session(page) + await cdp.send("Network.enable") + await cdp.send( + "Network.emulateNetworkConditions", + { + "offline": False, + "downloadThroughput": THROTTLE_DOWNLOAD_BPS, + "uploadThroughput": THROTTLE_UPLOAD_BPS, + "latency": THROTTLE_LATENCY_MS, + }, + ) + yield page + await context.close() + + +def _median(values: list[float]) -> float: + return statistics.median(values) + + +def _save_throttled_results(route: str, runs_ms: list[float]) -> None: + ts = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + result: dict[str, Any] = { + "fixture": "sami-like", + "route": route, + "timestamp": ts, + "throttle_config": { + "download_bps": THROTTLE_DOWNLOAD_BPS, + "upload_bps": THROTTLE_UPLOAD_BPS, + "latency_ms": THROTTLE_LATENCY_MS, + }, + "n_runs": N_RUNS, + "runs_ms": runs_ms, + "median_ms": _median(runs_ms), + } + EVIDENCE_DIR.mkdir(parents=True, exist_ok=True) + out_path = EVIDENCE_DIR / f"perf-results-throttled-sami-like-{ts}.json" + with open(out_path, "w") as f: + json.dump(result, f, indent=2) + + +@pytest.mark.perf +@pytest.mark.asyncio +async def test_throttle_actually_throttles( + throttled_page: Page, + perf_gateway_url: str, + admin_headers: dict[str, str], + seeded_sami: None, # noqa: ARG001 +) -> None: + """Sanity check: throttled TTFB must exceed 100 ms on a localhost request. + + Unthrottled Chromium–localhost TTFB is typically < 20 ms. With 300 ms + of additional latency configured via CDP, TTFB must be > 100 ms, + confirming throttling is actually active. + """ + await throttled_page.set_extra_http_headers(admin_headers) + url = f"{perf_gateway_url}/history" + metrics = await measure_page_load(throttled_page, url) + + assert metrics.ttfb_ms >= 100, ( + f"CDP throttling sanity check failed: TTFB={metrics.ttfb_ms:.0f} ms < 100 ms — " + "throttling may not be active. Check CDP session attachment." + ) + + +@pytest.mark.perf +@pytest.mark.asyncio +async def test_throttled_history_page( + throttled_page: Page, + perf_gateway_url: str, + admin_headers: dict[str, str], + seeded_sami: None, # noqa: ARG001 +) -> None: + """Baseline: /history TTFB under ~1 Mbps + 300 ms RTT (sami-like fixture). + + PR #1 records baseline only. Set PERF_THROTTLE_BASELINE=1 to enable + the SLO assertion (< 5 000 ms throttled, matching AGENTS.md). + """ + assert_slo = os.environ.get("PERF_THROTTLE_BASELINE") == "1" + await throttled_page.set_extra_http_headers(admin_headers) + url = f"{perf_gateway_url}/history" + + runs: list[float] = [] + for _ in range(N_RUNS): + m = await measure_page_load(throttled_page, url) + runs.append(m.ttfb_ms) + + median_ms = _median(runs) + _save_throttled_results(route="/history", runs_ms=runs) + + if assert_slo: + assert median_ms < 5_000, f"Throttled /history SLO: median={median_ms:.0f} ms > 5 000 ms" + + +@pytest.mark.perf +@pytest.mark.asyncio +async def test_throttled_conversation_live( + throttled_page: Page, + perf_gateway_url: str, + admin_headers: dict[str, str], + seeded_sami: None, # noqa: ARG001 +) -> None: + """Baseline: /conversation/live/{id} TTFB under ~1 Mbps + 300 ms RTT. + + Uses perf-seed-sami-442msg (the canonical 442-message session) to match + Sami's largest real session. PR #1 records baseline only. + """ + assert_slo = os.environ.get("PERF_THROTTLE_BASELINE") == "1" + await throttled_page.set_extra_http_headers(admin_headers) + url = f"{perf_gateway_url}/conversation/live/{_SAMI_LIVE_SESSION}" + + runs: list[float] = [] + for _ in range(N_RUNS): + m = await measure_page_load(throttled_page, url) + runs.append(m.ttfb_ms) + + median_ms = _median(runs) + _save_throttled_results( + route=f"/conversation/live/{_SAMI_LIVE_SESSION}", + runs_ms=runs, + ) + + if assert_slo: + assert median_ms < 5_000, f"Throttled live-conversation SLO: median={median_ms:.0f} ms > 5 000 ms" diff --git a/tests/luthien_proxy/perf_tests/test_transcript_open.py b/tests/luthien_proxy/perf_tests/test_transcript_open.py new file mode 100644 index 000000000..71d55ea31 --- /dev/null +++ b/tests/luthien_proxy/perf_tests/test_transcript_open.py @@ -0,0 +1,233 @@ +"""Transcript-open performance scenarios (P11). + +Measures time-to-first-turn-painted for /conversation/live/{session_id}. + +"First-turn-painted" = first child node insertion into #conversation-container +by conversation_live.js after the initial fetch-and-render cycle. A +MutationObserver installed via add_init_script records performance.now() at +that moment, before any page JS runs. + +PR #1 (perf-baseline) records measurements only; no SLO is asserted. +""" + +from __future__ import annotations + +import json +import sqlite3 +import statistics +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +import httpx +import pytest +from playwright.async_api import Page + +from luthien_proxy.perf.seeding import seed_sami_like, seed_sessions + +_REPO_ROOT = Path(__file__).resolve().parents[3] +EVIDENCE_DIR = _REPO_ROOT / ".sisyphus" / "evidence" + +N_RUNS: int = 5 + +# (fixture_label, session_id) — sami-like primary; tier-100 and tier-1000 secondary. +_FIXTURES: list[tuple[str, str]] = [ + ("sami-like", "perf-seed-sami-442msg"), + ("tier-100", "perf-seed-100-0001"), + ("tier-1000", "perf-seed-1000-0001"), +] + +# Installed via add_init_script — runs before any page JS on every navigation. +# Guard flag prevents double-observation when called N times on the same page. +# Targets #conversation-container (not #main) to capture the first rendered turn. +_FIRST_TURN_OBSERVER_SCRIPT = """ +if (!window.__transcriptPerfInstalled) { + window.__transcriptPerfInstalled = true; + window.__firstTurnPainted = null; + + function _setupTranscriptObserver() { + var container = document.getElementById('conversation-container'); + if (!container) { return; } + var obs = new MutationObserver(function(mutations) { + if (window.__firstTurnPainted !== null) { return; } + for (var i = 0; i < mutations.length; i++) { + if (mutations[i].addedNodes.length > 0) { + window.__firstTurnPainted = performance.now(); + obs.disconnect(); + break; + } + } + }); + obs.observe(container, { childList: true }); + } + + if (document.readyState === 'loading') { + document.addEventListener('DOMContentLoaded', _setupTranscriptObserver); + } else { + _setupTranscriptObserver(); + } +} +""" + + +@pytest.fixture(scope="session") +def seeded_transcript_fixtures(perf_db_url: str) -> None: # noqa: ARG001 + db_path = Path.home() / ".luthien" / "perf.db" + conn = sqlite3.connect(str(db_path)) + try: + for prefix, seed_fn in [ + ("perf-seed-sami-%", lambda: seed_sami_like("sqlite")), + ("perf-seed-100-%", lambda: seed_sessions("sqlite", tier=100)), + ("perf-seed-1000-%", lambda: seed_sessions("sqlite", tier=1000)), + ]: + (count,) = conn.execute( + "SELECT COUNT(*) FROM conversation_calls WHERE session_id LIKE ?", + (prefix,), + ).fetchone() + if count == 0: + seed_fn() + finally: + conn.close() + + +def _p95(values: list[float]) -> float: + if not values: + return 0.0 + sorted_vals = sorted(values) + idx = min(int(len(sorted_vals) * 0.95), len(sorted_vals) - 1) + return sorted_vals[idx] + + +async def _measure_first_turn_painted(page: Page, url: str) -> dict[str, float]: + """Navigate to url and return first-turn-painted + ancillary metrics. + + Installs _FIRST_TURN_OBSERVER_SCRIPT via add_init_script so the observer + fires before any page JS. Each navigation resets window state, giving fresh + timing per run despite the script accumulating across calls. + """ + await page.add_init_script(_FIRST_TURN_OBSERVER_SCRIPT) + await page.goto(url, wait_until="networkidle") + + return await page.evaluate("""() => { + var entries = window.performance.getEntriesByType('navigation'); + var ttfb = 0, load = 0; + if (entries.length > 0) { + var nav = entries[0]; + ttfb = nav.responseStart; + load = nav.loadEventEnd; + } else { + var t = window.performance.timing; + var origin = t.fetchStart; + ttfb = t.responseStart - origin; + load = t.loadEventEnd - origin; + } + return { + ttfb_ms: ttfb, + load_ms: load, + first_turn_painted_ms: window.__firstTurnPainted || 0 + }; + }""") + + +def _save_transcript_results( + fixture_label: str, + session_id: str, + all_runs: list[dict[str, float]], + transfer_bytes: int, +) -> None: + ts = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + ftp_values = [r["first_turn_painted_ms"] for r in all_runs] + ttfb_values = [r["ttfb_ms"] for r in all_runs] + + result: dict[str, Any] = { + "fixture": fixture_label, + "session_id": session_id, + "timestamp": ts, + "n_runs": N_RUNS, + "first_turn_painted": { + "median_ms": statistics.median(ftp_values), + "p95_ms": _p95(ftp_values), + "runs_ms": ftp_values, + }, + "ttfb": { + "median_ms": statistics.median(ttfb_values), + "runs_ms": ttfb_values, + }, + "total_render_time_ms": statistics.median([r["load_ms"] for r in all_runs]), + "response_body_bytes": transfer_bytes, + } + EVIDENCE_DIR.mkdir(parents=True, exist_ok=True) + out_path = EVIDENCE_DIR / f"perf-results-transcript-{fixture_label}-{ts}.json" + with open(out_path, "w") as f: + json.dump(result, f, indent=2) + + +@pytest.mark.perf +@pytest.mark.asyncio +@pytest.mark.parametrize("fixture_label,session_id", _FIXTURES) +async def test_transcript_open( + fixture_label: str, + session_id: str, + playwright_page: Page, + perf_gateway_url: str, + admin_headers: dict[str, str], + seeded_transcript_fixtures: None, # noqa: ARG001 +) -> None: + """Baseline: time-to-first-turn-painted per fixture tier. + + Parametrized over sami-like (442 msg), tier-100, tier-1000. Five runs per + fixture; results include median and p95. PR #1 records baseline only. + """ + await playwright_page.set_extra_http_headers(admin_headers) + url = f"{perf_gateway_url}/conversation/live/{session_id}" + + all_runs: list[dict[str, float]] = [] + for _ in range(N_RUNS): + metrics = await _measure_first_turn_painted(playwright_page, url) + all_runs.append(metrics) + + async with httpx.AsyncClient(headers=admin_headers, follow_redirects=True) as client: + http_resp = await client.get(url) + transfer_bytes = len(http_resp.content) + + _save_transcript_results( + fixture_label=fixture_label, + session_id=session_id, + all_runs=all_runs, + transfer_bytes=transfer_bytes, + ) + + +@pytest.mark.perf +@pytest.mark.asyncio +async def test_first_turn_painted_largest_sami_session( + playwright_page: Page, + perf_gateway_url: str, + admin_headers: dict[str, str], + seeded_transcript_fixtures: None, # noqa: ARG001 +) -> None: + """Baseline: first-turn-painted for the canonical 442-message session. + + perf-seed-sami-442msg is the closest available session to the "500-turn" + SLO reference in AGENTS.md. PR #1 records baseline only; no SLO asserted. + """ + session_id = "perf-seed-sami-442msg" + fixture_label = "sami-442msg" + await playwright_page.set_extra_http_headers(admin_headers) + url = f"{perf_gateway_url}/conversation/live/{session_id}" + + all_runs: list[dict[str, float]] = [] + for _ in range(N_RUNS): + metrics = await _measure_first_turn_painted(playwright_page, url) + all_runs.append(metrics) + + async with httpx.AsyncClient(headers=admin_headers, follow_redirects=True) as client: + http_resp = await client.get(url) + transfer_bytes = len(http_resp.content) + + _save_transcript_results( + fixture_label=fixture_label, + session_id=session_id, + all_runs=all_runs, + transfer_bytes=transfer_bytes, + ) diff --git a/tests/luthien_proxy/unit_tests/history/test_routes.py b/tests/luthien_proxy/unit_tests/history/test_routes.py index 82f41b6fa..71f1cd719 100644 --- a/tests/luthien_proxy/unit_tests/history/test_routes.py +++ b/tests/luthien_proxy/unit_tests/history/test_routes.py @@ -6,10 +6,12 @@ - Return correct response models """ +import json from unittest.mock import AsyncMock, MagicMock, patch import pytest from fastapi import HTTPException +from starlette.responses import Response from luthien_proxy.history.models import ( ConversationMessage, @@ -59,12 +61,13 @@ async def test_successful_list_sessions(self): ) as mock_fetch: result = await list_sessions(_=AUTH_TOKEN, db_pool=mock_db_pool, limit=50, offset=0, user_id=None) - assert isinstance(result, SessionListResponse) - assert result.total == 100 - assert result.offset == 0 - assert result.has_more is True - assert len(result.sessions) == 1 - assert result.sessions[0].session_id == "session-1" + assert isinstance(result, Response) + body = json.loads(bytes(result.body)) + assert body["total"] == 100 + assert body["offset"] == 0 + assert body["has_more"] is True + assert len(body["sessions"]) == 1 + assert body["sessions"][0]["session_id"] == "session-1" mock_fetch.assert_called_once_with(50, mock_db_pool, 0, user_id=None) @pytest.mark.asyncio @@ -98,8 +101,9 @@ async def test_list_sessions_with_offset(self): ) as mock_fetch: result = await list_sessions(_=AUTH_TOKEN, db_pool=mock_db_pool, limit=50, offset=50, user_id=None) - assert result.offset == 50 - assert result.has_more is True + body = json.loads(bytes(result.body)) + assert body["offset"] == 50 + assert body["has_more"] is True mock_fetch.assert_called_once_with(50, mock_db_pool, 50, user_id=None) @pytest.mark.asyncio @@ -114,9 +118,10 @@ async def test_list_sessions_empty(self): ): result = await list_sessions(_=AUTH_TOKEN, db_pool=mock_db_pool, limit=50, offset=0, user_id=None) - assert result.total == 0 - assert result.sessions == [] - assert result.has_more is False + body = json.loads(bytes(result.body)) + assert body["total"] == 0 + assert body["sessions"] == [] + assert body["has_more"] is False class TestGetSessionRoute: @@ -152,9 +157,10 @@ async def test_successful_get_session(self): ) as mock_fetch: result = await get_session(session_id="test-session", _=AUTH_TOKEN, db_pool=mock_db_pool) - assert isinstance(result, SessionDetail) - assert result.session_id == "test-session" - assert len(result.turns) == 1 + assert isinstance(result, Response) + body = json.loads(bytes(result.body)) + assert body["session_id"] == "test-session" + assert len(body["turns"]) == 1 mock_fetch.assert_called_once_with("test-session", mock_db_pool) @pytest.mark.asyncio @@ -208,7 +214,7 @@ async def test_successful_export(self): result = await export_session(session_id="test-session", _=AUTH_TOKEN, db_pool=mock_db_pool) assert result.media_type == "text/markdown" - assert "# Conversation History: test-session" in result.body.decode() + assert "# Conversation History: test-session" in bytes(result.body).decode() assert "Content-Disposition" in result.headers assert 'filename="conversation_test-session.md"' in result.headers["Content-Disposition"] diff --git a/tests/luthien_proxy/unit_tests/inference/test_registry.py b/tests/luthien_proxy/unit_tests/inference/test_registry.py index ded25d5a7..712c58cdc 100644 --- a/tests/luthien_proxy/unit_tests/inference/test_registry.py +++ b/tests/luthien_proxy/unit_tests/inference/test_registry.py @@ -37,7 +37,7 @@ _build_direct_api, ) from luthien_proxy.utils.db import DatabasePool -from luthien_proxy.utils.migration_check import _apply_sqlite_migrations +from luthien_proxy.utils.migration_check import apply_sqlite_migrations class _StubProvider(InferenceProvider): @@ -74,7 +74,7 @@ async def sqlite_pool() -> DatabasePool: """Real in-memory SQLite with every migration applied.""" pool = DatabasePool("sqlite://:memory:") migrations_dir = Path(__file__).resolve().parents[4] / "migrations" / "sqlite" - await _apply_sqlite_migrations(pool, migrations_dir=migrations_dir) + await apply_sqlite_migrations(pool, migrations_dir=migrations_dir) yield pool await pool.close() diff --git a/tests/luthien_proxy/unit_tests/perf/__init__.py b/tests/luthien_proxy/unit_tests/perf/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/luthien_proxy/unit_tests/perf/test_db.py b/tests/luthien_proxy/unit_tests/perf/test_db.py new file mode 100644 index 000000000..57b15f35a --- /dev/null +++ b/tests/luthien_proxy/unit_tests/perf/test_db.py @@ -0,0 +1,64 @@ +import sqlite3 +from unittest.mock import patch + +import pytest + +from luthien_proxy.perf.db import ( + drop_perf_db, + ensure_perf_isolation, + get_perf_db_url, + migrate_perf_db, +) + + +def test_ensure_perf_isolation_rejects_local_db(): + with pytest.raises(RuntimeError, match="isolation"): + ensure_perf_isolation("sqlite:///~/.luthien/local.db") + + +def test_ensure_perf_isolation_accepts_perf_db(tmp_path): + ensure_perf_isolation(f"sqlite:///{tmp_path}/.luthien/perf.db") + + +def test_ensure_perf_isolation_rejects_postgres_without_perf_test(): + with pytest.raises(RuntimeError, match="isolation"): + ensure_perf_isolation("postgresql://user:pass@localhost/luthien") + + +def test_ensure_perf_isolation_accepts_postgres_with_perf_test(): + ensure_perf_isolation("postgresql://user:pass@localhost/luthien?options=-csearch_path=perf_test") + + +def test_get_perf_db_url_sqlite(): + url = get_perf_db_url("sqlite") + assert url.startswith("sqlite:///") + assert "perf.db" in url + assert "local.db" not in url + + +def test_drop_perf_db_idempotent(tmp_path): + with patch("pathlib.Path.home", return_value=tmp_path): + drop_perf_db("sqlite") + drop_perf_db("sqlite") + + +def test_ensure_perf_isolation_rejects_unrecognized_scheme(): + with pytest.raises(RuntimeError, match="isolation"): + ensure_perf_isolation("mysql://user:pass@localhost/luthien") + + +def test_migrate_perf_db_creates_tables(tmp_path): + with patch("pathlib.Path.home", return_value=tmp_path): + migrate_perf_db("sqlite") + + perf_db = tmp_path / ".luthien" / "perf.db" + assert perf_db.exists() + + conn = sqlite3.connect(str(perf_db)) + try: + rows = conn.execute("SELECT name FROM sqlite_master WHERE type='table'").fetchall() + table_names = {row[0] for row in rows} + assert "conversation_events" in table_names + assert "conversation_calls" in table_names + finally: + conn.close() diff --git a/tests/luthien_proxy/unit_tests/perf/test_harness_helpers.py b/tests/luthien_proxy/unit_tests/perf/test_harness_helpers.py new file mode 100644 index 000000000..51aa0dedf --- /dev/null +++ b/tests/luthien_proxy/unit_tests/perf/test_harness_helpers.py @@ -0,0 +1,103 @@ +"""Unit tests for perf test harness helpers: n_runs, RunStats, PageLoadMetrics.""" + +from __future__ import annotations + +from tests.luthien_proxy.perf_tests.conftest import ( + PageLoadMetrics, + _percentile, + n_runs, +) + + +def test_page_load_metrics_dataclass(): + m = PageLoadMetrics(ttfb_ms=10.0, dcl_ms=50.0, load_ms=120.0, ttfm_ms=200.0) + assert m.ttfb_ms == 10.0 + assert m.dcl_ms == 50.0 + assert m.load_ms == 120.0 + assert m.ttfm_ms == 200.0 + + +def test_run_stats_median(): + """warm_median_ms is the statistics.median of warm runs, not affected by cold.""" + cold_value = 9999.0 + warm_values = [10.0, 20.0, 30.0, 40.0] + call_idx = 0 + all_values = [cold_value] + warm_values + + def fn() -> float: + nonlocal call_idx + val = all_values[call_idx] + call_idx += 1 + return val + + import asyncio + + stats = asyncio.run(n_runs(fn, n=5)) + assert stats.cold_ms == cold_value + assert stats.warm_median_ms == 25.0 # median([10, 20, 30, 40]) = 25.0 + assert stats.n_warm == 4 + + +async def test_n_runs_separates_cold_cache(): + """Cold-cache first run must NOT be included in warm_median_ms.""" + cold_value = 1000.0 + warm_value = 10.0 + call_idx = 0 + + def fn() -> float: + nonlocal call_idx + call_idx += 1 + return cold_value if call_idx == 1 else warm_value + + stats = await n_runs(fn, n=5) + + assert stats.cold_ms == cold_value + assert stats.warm_median_ms == warm_value + assert stats.warm_median_ms != cold_value + assert stats.n_warm == 4 + + +async def test_n_runs_async_fn(): + """n_runs works with async callables too.""" + call_idx = 0 + + async def async_fn() -> float: + nonlocal call_idx + call_idx += 1 + return float(call_idx * 10) + + stats = await n_runs(async_fn, n=4) + assert stats.cold_ms == 10.0 + assert stats.n_warm == 3 + assert stats.warm_median_ms == 30.0 # median([20, 30, 40]) = 30.0 + + +async def test_n_runs_p95(): + """warm_p95_ms uses 95th percentile of warm runs.""" + values = [100.0, 10.0, 10.0, 10.0, 10.0, 10.0] # cold=100, warm=[10, 10, 10, 10, 10] + call_idx = 0 + + def fn() -> float: + nonlocal call_idx + val = values[call_idx] + call_idx += 1 + return val + + stats = await n_runs(fn, n=6) + assert stats.cold_ms == 100.0 + assert stats.warm_p95_ms == 10.0 + assert stats.n_warm == 5 + + +def test_percentile_empty(): + assert _percentile([], 0.95) == 0.0 + + +def test_percentile_single(): + assert _percentile([42.0], 0.95) == 42.0 + + +def test_percentile_values(): + data = sorted([10.0, 20.0, 30.0, 40.0, 50.0]) + assert _percentile(data, 0.50) == 30.0 + assert _percentile(data, 0.0) == 10.0 diff --git a/tests/luthien_proxy/unit_tests/perf/test_report.py b/tests/luthien_proxy/unit_tests/perf/test_report.py new file mode 100644 index 000000000..18b8c4d4a --- /dev/null +++ b/tests/luthien_proxy/unit_tests/perf/test_report.py @@ -0,0 +1,99 @@ +from __future__ import annotations + +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent.parent.parent.parent.parent / "scripts")) + +import perf_report # noqa: E402 + +_MOCK_RESULTS = [ + { + "type": "page_timings", + "data": { + "history_list": {"sami": {"median_ms": 450, "p95_ms": 780}}, + "session_detail": {"sami": {"median_ms": 310, "p95_ms": 550}}, + }, + }, + { + "type": "throttled", + "data": { + "history_list": {"throttled_sami": {"median_ms": 2100, "p95_ms": 3800}}, + }, + }, + { + "type": "transcript_open", + "data": {"first_turn_painted_ms": 650}, + }, + { + "type": "sse_memory", + "data": {"heap_growth_mb": 12.5, "events_count": 442}, + }, + { + "type": "server_timing", + "data": {"db_ms": 45.2, "serialize_ms": 12.1, "render_ms": 8.3}, + }, + { + "type": "payload_size", + "data": { + "/api/history/sessions": {"bytes": 4096}, + "/api/history/sessions/{id}": {"bytes": 78432}, + }, + }, +] + +_MOCK_QUERY_PLANS = "## Query: session_list\n\nSEARCH ce USING INDEX ...\n" + +_REQUIRED_SECTIONS = [ + "## Hardware & Versions", + "## Per-Page Timings", + "## Throttled (sami-like)", + "## Transcript Open", + "## SSE Memory Growth", + "## Server-Timing Breakdown", + "## Payload Size Breakdown", + "## Query Plans", + "## Top Hotspots", +] + +_COMMON_KWARGS = dict( + results=_MOCK_RESULTS, + query_plans=_MOCK_QUERY_PLANS, + git_sha="abc123def456", + playwright_ver="1.50.0", + generated_at="2000-01-01T00:00:00+00:00", + ram="16 GB", +) + + +def test_report_has_required_sections(): + report = perf_report.generate_report(**_COMMON_KWARGS) + for section in _REQUIRED_SECTIONS: + assert section in report, f"Missing section: {section!r}" + + +def test_report_has_metadata(): + report = perf_report.generate_report(**_COMMON_KWARGS) + assert "git_sha: abc123def456" in report + assert "browser_version: 1.50.0" in report + assert "backend: sqlite" in report + + +def test_report_deterministic(): + report1 = perf_report.generate_report(**_COMMON_KWARGS) + report2 = perf_report.generate_report(**_COMMON_KWARGS) + assert report1 == report2 + + +def test_report_no_data_placeholder(): + report = perf_report.generate_report( + results=[], + query_plans="_No query plans._", + git_sha="abc123", + playwright_ver="1.50.0", + generated_at="2000-01-01T00:00:00+00:00", + ram="16 GB", + ) + for section in _REQUIRED_SECTIONS: + assert section in report, f"Missing section with no data: {section!r}" + assert "NO DATA YET" in report diff --git a/tests/luthien_proxy/unit_tests/perf/test_seeding.py b/tests/luthien_proxy/unit_tests/perf/test_seeding.py new file mode 100644 index 000000000..987987dd5 --- /dev/null +++ b/tests/luthien_proxy/unit_tests/perf/test_seeding.py @@ -0,0 +1,198 @@ +import sqlite3 +from unittest.mock import patch + +import pytest + +from luthien_proxy.perf.seeding import seed_sami_like, seed_sessions + +pytestmark = pytest.mark.timeout(30) + + +@pytest.fixture +def isolated_home(tmp_path): + (tmp_path / ".luthien").mkdir() + with patch("pathlib.Path.home", return_value=tmp_path): + yield tmp_path + + +def _db(home): + return sqlite3.connect(str(home / ".luthien" / "perf.db")) + + +def test_seed_100_row_counts(isolated_home): + report = seed_sessions("sqlite", tier=100) + + assert report.total_sessions == 100 + assert report.total_rows > 0 + + conn = _db(isolated_home) + try: + (n_calls,) = conn.execute("SELECT COUNT(*) FROM conversation_calls").fetchone() + (n_events,) = conn.execute("SELECT COUNT(*) FROM conversation_events").fetchone() + finally: + conn.close() + + assert n_calls > 0 + assert n_events == 2 * n_calls + assert report.total_rows == n_calls + n_events + + +def test_seed_prefix(isolated_home): + seed_sessions("sqlite", tier=100) + + conn = _db(isolated_home) + try: + rows = conn.execute("SELECT DISTINCT session_id FROM conversation_events").fetchall() + finally: + conn.close() + + session_ids = [r[0] for r in rows] + assert len(session_ids) == 100 + for sid in session_ids: + assert sid.startswith("perf-seed-100-"), sid + + +def test_seed_idempotent(isolated_home): + from luthien_proxy.perf.db import drop_perf_db + + seed_sessions("sqlite", tier=100) + conn = _db(isolated_home) + try: + (n_calls_1,) = conn.execute("SELECT COUNT(*) FROM conversation_calls").fetchone() + (n_events_1,) = conn.execute("SELECT COUNT(*) FROM conversation_events").fetchone() + finally: + conn.close() + + drop_perf_db("sqlite") + seed_sessions("sqlite", tier=100) + conn = _db(isolated_home) + try: + (n_calls_2,) = conn.execute("SELECT COUNT(*) FROM conversation_calls").fetchone() + (n_events_2,) = conn.execute("SELECT COUNT(*) FROM conversation_events").fetchone() + finally: + conn.close() + + assert n_calls_1 == n_calls_2 + assert n_events_1 == n_events_2 + + +def test_sami_like_78_sessions(isolated_home): + report = seed_sami_like("sqlite") + + assert report.total_sessions == 78 + + conn = _db(isolated_home) + try: + (n_sessions,) = conn.execute( + "SELECT COUNT(DISTINCT session_id) FROM conversation_events WHERE session_id LIKE 'perf-seed-sami-%'" + ).fetchone() + finally: + conn.close() + + assert n_sessions == 78 + + +def test_sami_like_442_msg_session(isolated_home): + report = seed_sami_like("sqlite") + + assert report.biggest_session_message_count >= 442 + + conn = _db(isolated_home) + try: + (n_calls,) = conn.execute( + "SELECT COUNT(*) FROM conversation_calls WHERE session_id = 'perf-seed-sami-442msg'" + ).fetchone() + finally: + conn.close() + + assert n_calls == 442 + + +def test_payload_sizes(): + from luthien_proxy.perf.seeding import _req_payload, _resp_payload + + req = _req_payload("perf-seed-test-0001", 0) + resp = _resp_payload("perf-seed-test-0001", 0) + assert 4 * 1024 <= len(req) <= 6 * 1024, f"req payload {len(req)} bytes not in [4KB, 6KB]" + assert 18 * 1024 <= len(resp) <= 22 * 1024, f"resp payload {len(resp)} bytes not in [18KB, 22KB]" + + +def test_seeded_db_has_same_indexes_as_migrated_db(isolated_home): + from luthien_proxy.perf.db import drop_perf_db, migrate_perf_db + + migrate_perf_db("sqlite") + conn = _db(isolated_home) + try: + migrated_indexes = { + row[0] + for row in conn.execute("SELECT name FROM sqlite_master WHERE type='index' AND name NOT LIKE 'sqlite_%'") + } + finally: + conn.close() + + drop_perf_db("sqlite") + seed_sessions("sqlite", tier=10) + conn = _db(isolated_home) + try: + seeded_indexes = { + row[0] + for row in conn.execute("SELECT name FROM sqlite_master WHERE type='index' AND name NOT LIKE 'sqlite_%'") + } + finally: + conn.close() + + assert seeded_indexes == migrated_indexes, ( + f"Seeded DB indexes differ from migrated DB.\n" + f"Missing after reseed: {migrated_indexes - seeded_indexes}\n" + f"Extra after reseed: {seeded_indexes - migrated_indexes}" + ) + + +def test_seed_sqlite_recreates_indexes_after_rollback(isolated_home): + import sqlite3 as _sqlite3 + + from luthien_proxy.perf.db import get_perf_db_url, migrate_perf_db + from luthien_proxy.perf.seeding import _INDEX_STMTS, _sqlite_path + + migrate_perf_db("sqlite") + db_path = _sqlite_path(get_perf_db_url("sqlite")) + + conn = _sqlite3.connect(str(db_path), isolation_level=None) + try: + for idx in ( + "idx_conversation_events_type", + "idx_conversation_events_created", + "idx_conversation_events_call_created", + ): + conn.execute(f"DROP INDEX IF EXISTS {idx}") + + for stmt in _INDEX_STMTS: + conn.execute(stmt) + finally: + conn.close() + + after = _sqlite3.connect(str(db_path)) + try: + indexes = { + row[0] + for row in after.execute("SELECT name FROM sqlite_master WHERE type='index' AND name NOT LIKE 'sqlite_%'") + } + finally: + after.close() + + assert len(indexes) >= len(_INDEX_STMTS), "All indexes should be recreated" + + +def test_seed_sessions_raises_if_rows_already_exist(isolated_home): + seed_sessions("sqlite", tier=10) + with pytest.raises(RuntimeError, match="already exist"): + seed_sessions("sqlite", tier=10) + + +def test_seeding_refuses_dev_db(tmp_path): + with patch( + "luthien_proxy.perf.seeding.get_perf_db_url", + return_value=f"sqlite:///{tmp_path}/local.db", + ): + with pytest.raises(RuntimeError, match="isolation"): + seed_sessions("sqlite", tier=10) diff --git a/tests/luthien_proxy/unit_tests/perf/test_timing_middleware.py b/tests/luthien_proxy/unit_tests/perf/test_timing_middleware.py new file mode 100644 index 000000000..82538726c --- /dev/null +++ b/tests/luthien_proxy/unit_tests/perf/test_timing_middleware.py @@ -0,0 +1,205 @@ +from __future__ import annotations + +import asyncio + +import pytest +from fastapi import FastAPI +from httpx import ASGITransport, AsyncClient + +from luthien_proxy.perf.timing_middleware import ( + ServerTimingMiddleware, + format_phases, + time_phase, +) + + +def _make_app(path: str = "/api/history/sessions") -> FastAPI: + app = FastAPI() + app.add_middleware(ServerTimingMiddleware) + + @app.get(path) + async def endpoint(): + with time_phase("handler"): + pass + return {"ok": True} + + return app + + +@pytest.fixture +def history_app(): + return _make_app("/api/history/sessions") + + +@pytest.fixture +def v1_app(): + return _make_app("/v1/messages") + + +def test_format_phases(): + result = format_phases([("db", 12.3), ("serialize", 4.5)]) + assert result == "db;dur=12.3, serialize;dur=4.5" + + +def test_format_phases_single(): + result = format_phases([("render", 1.0)]) + assert result == "render;dur=1.0" + + +def test_format_phases_empty(): + assert format_phases([]) == "" + + +@pytest.mark.asyncio +async def test_path_filter_includes_history(history_app): + transport = ASGITransport(app=history_app) + async with AsyncClient(transport=transport, base_url="http://test") as client: + response = await client.get("/api/history/sessions") + assert response.status_code == 200 + assert "Server-Timing" in response.headers + + +@pytest.mark.asyncio +async def test_path_filter_includes_debug(): + app = _make_app("/api/debug/calls") + transport = ASGITransport(app=app) + async with AsyncClient(transport=transport, base_url="http://test") as client: + response = await client.get("/api/debug/calls") + assert "Server-Timing" in response.headers + + +@pytest.mark.asyncio +async def test_path_filter_includes_ui_fragments(): + app = _make_app("/ui/fragments/sidebar") + transport = ASGITransport(app=app) + async with AsyncClient(transport=transport, base_url="http://test") as client: + response = await client.get("/ui/fragments/sidebar") + assert "Server-Timing" in response.headers + + +@pytest.mark.asyncio +async def test_path_filter_excludes_v1_messages(v1_app): + transport = ASGITransport(app=v1_app) + async with AsyncClient(transport=transport, base_url="http://test") as client: + response = await client.get("/v1/messages") + assert response.status_code == 200 + assert "Server-Timing" not in response.headers + + +@pytest.mark.asyncio +async def test_concurrent_isolation(): + barrier = asyncio.Event() + results: dict[str, str | None] = {} + + app = FastAPI() + app.add_middleware(ServerTimingMiddleware) + + @app.get("/api/history/a") + async def endpoint_a(): + with time_phase("phase-a"): + await barrier.wait() + return {"id": "a"} + + @app.get("/api/history/b") + async def endpoint_b(): + with time_phase("phase-b"): + await asyncio.sleep(0) + barrier.set() + return {"id": "b"} + + transport = ASGITransport(app=app) + + async def call_a(): + async with AsyncClient(transport=transport, base_url="http://test") as client: + r = await client.get("/api/history/a") + results["a"] = r.headers.get("Server-Timing") + + async def call_b(): + async with AsyncClient(transport=transport, base_url="http://test") as client: + r = await client.get("/api/history/b") + results["b"] = r.headers.get("Server-Timing") + + await asyncio.gather(call_a(), call_b()) + + header_a = results["a"] + header_b = results["b"] + + assert header_a is not None + assert header_b is not None + assert "phase-b" not in header_a, f"phase-b leaked into request-a header: {header_a}" + assert "phase-a" not in header_b, f"phase-a leaked into request-b header: {header_b}" + assert "phase-a" in header_a + assert "phase-b" in header_b + + +@pytest.mark.asyncio +async def test_monitored_path_with_zero_phases_omits_header(): + app = FastAPI() + app.add_middleware(ServerTimingMiddleware) + + @app.get("/api/history/sessions") + async def endpoint(): + return {"ok": True} + + transport = ASGITransport(app=app) + async with AsyncClient(transport=transport, base_url="http://test") as client: + response = await client.get("/api/history/sessions") + assert response.status_code == 200 + assert "Server-Timing" not in response.headers + + +def test_time_phase_outside_request_context_does_not_raise(): + with time_phase("orphan"): + pass + + +@pytest.mark.asyncio +async def test_time_phase_records_elapsed_even_when_block_raises(): + from luthien_proxy.perf.timing_middleware import _phases_var + + phases: list[tuple[str, float]] = [] + token = _phases_var.set(phases) + try: + with pytest.raises(ValueError): + with time_phase("failing-phase"): + raise ValueError("boom") + finally: + _phases_var.reset(token) + + assert len(phases) == 1 + name, elapsed_ms = phases[0] + assert name == "failing-phase" + assert elapsed_ms >= 0 + + +@pytest.mark.asyncio +async def test_static_cache_middleware_replaces_not_appends(): + from fastapi.testclient import TestClient + from starlette.responses import Response as StarletteResponse + + from luthien_proxy.main import create_app + from luthien_proxy.utils.db import DatabasePool + + db_pool = DatabasePool("sqlite:///:memory:") + app = create_app( + api_key=None, + admin_key="test", + db_pool=db_pool, + redis_client=None, + startup_policy_path=None, + policy_source="file", + ) + + @app.get("/api/test-cache") + def _route(): + return StarletteResponse( + content="ok", + headers={"Cache-Control": "no-cache"}, + ) + + client = TestClient(app, raise_server_exceptions=False) + response = client.get("/api/test-cache") + await db_pool.close() + + cache_headers = [v for k, v in response.headers.items() if k.lower() == "cache-control"] + assert len(cache_headers) == 1, f"Expected exactly one Cache-Control header, got: {cache_headers}" diff --git a/tests/luthien_proxy/unit_tests/test_debug_routes.py b/tests/luthien_proxy/unit_tests/test_debug_routes.py index f51556ac4..2ec5639f7 100644 --- a/tests/luthien_proxy/unit_tests/test_debug_routes.py +++ b/tests/luthien_proxy/unit_tests/test_debug_routes.py @@ -12,17 +12,14 @@ directly since they call handlers without going through FastAPI's Depends(). """ +import json from datetime import datetime from unittest.mock import AsyncMock, MagicMock import pytest from fastapi import HTTPException +from starlette.responses import Response -from luthien_proxy.debug.models import ( - CallDiffResponse, - CallEventsResponse, - CallListResponse, -) from luthien_proxy.debug.routes import ( get_call_diff, get_call_events, @@ -79,9 +76,10 @@ async def test_successful_response(self): result = await get_call_events("test-call-id", _=AUTH_TOKEN, db_pool=mock_pool) - assert isinstance(result, CallEventsResponse) - assert result.call_id == "test-call-id" - assert len(result.events) == 1 + assert isinstance(result, Response) + body = json.loads(bytes(result.body)) + assert body["call_id"] == "test-call-id" + assert len(body["events"]) == 1 @pytest.mark.asyncio async def test_database_error(self): @@ -167,8 +165,9 @@ async def test_successful_response(self): result = await get_call_diff("test-call-id", _=AUTH_TOKEN, db_pool=mock_pool) - assert isinstance(result, CallDiffResponse) - assert result.call_id == "test-call-id" + assert isinstance(result, Response) + body = json.loads(bytes(result.body)) + assert body["call_id"] == "test-call-id" @pytest.mark.asyncio async def test_database_error(self): @@ -211,9 +210,10 @@ async def test_empty_result(self): result = await list_recent_calls(limit=10, _=AUTH_TOKEN, db_pool=mock_pool) - assert isinstance(result, CallListResponse) - assert result.total == 0 - assert result.calls == [] + assert isinstance(result, Response) + body = json.loads(bytes(result.body)) + assert body["total"] == 0 + assert body["calls"] == [] @pytest.mark.asyncio async def test_successful_response(self): @@ -239,9 +239,10 @@ async def test_successful_response(self): result = await list_recent_calls(limit=10, _=AUTH_TOKEN, db_pool=mock_pool) - assert isinstance(result, CallListResponse) - assert result.total == 2 - assert len(result.calls) == 2 + assert isinstance(result, Response) + body = json.loads(bytes(result.body)) + assert body["total"] == 2 + assert len(body["calls"]) == 2 @pytest.mark.asyncio async def test_database_error(self): diff --git a/tests/luthien_proxy/unit_tests/utils/test_migration_check.py b/tests/luthien_proxy/unit_tests/utils/test_migration_check.py index 9bddd20ea..feeee33fa 100644 --- a/tests/luthien_proxy/unit_tests/utils/test_migration_check.py +++ b/tests/luthien_proxy/unit_tests/utils/test_migration_check.py @@ -10,8 +10,8 @@ from luthien_proxy.utils.db import DatabasePool from luthien_proxy.utils.migration_check import ( - _apply_sqlite_migrations, _find_sqlite_migrations_dir, + apply_sqlite_migrations, check_migrations, compute_file_hash, ) @@ -331,7 +331,7 @@ async def test_applies_migrations_in_order(self, migrations_dir: Path) -> None: (migrations_dir / "001_first.sql").write_text("CREATE TABLE t1 (id INTEGER PRIMARY KEY);") (migrations_dir / "002_second.sql").write_text("CREATE TABLE t2 (id INTEGER PRIMARY KEY);") pool = DatabasePool("sqlite://:memory:") - await _apply_sqlite_migrations(pool, migrations_dir) + await apply_sqlite_migrations(pool, migrations_dir) async with pool.connection() as conn: rows = await conn.fetch("SELECT name FROM sqlite_master WHERE type='table' ORDER BY name") @@ -346,15 +346,15 @@ async def test_skips_already_applied(self, migrations_dir: Path) -> None: """Should not re-apply migrations that are already recorded.""" (migrations_dir / "001_first.sql").write_text("CREATE TABLE t1 (id INTEGER PRIMARY KEY);") pool = DatabasePool("sqlite://:memory:") - await _apply_sqlite_migrations(pool, migrations_dir) - await _apply_sqlite_migrations(pool, migrations_dir) + await apply_sqlite_migrations(pool, migrations_dir) + await apply_sqlite_migrations(pool, migrations_dir) @pytest.mark.asyncio async def test_handles_comment_only_files(self, migrations_dir: Path) -> None: """Should handle no-op migration files (comments only).""" (migrations_dir / "000_init.sql").write_text("-- No-op: SQLite needs no database initialization.\n") pool = DatabasePool("sqlite://:memory:") - await _apply_sqlite_migrations(pool, migrations_dir) + await apply_sqlite_migrations(pool, migrations_dir) async with pool.connection() as conn: tracked = await conn.fetch("SELECT filename FROM _migrations") @@ -366,10 +366,10 @@ async def test_detects_hash_mismatch(self, migrations_dir: Path) -> None: """Should raise RuntimeError if a recorded migration's hash doesn't match.""" (migrations_dir / "001_first.sql").write_text("CREATE TABLE t1 (id INTEGER PRIMARY KEY);") pool = DatabasePool("sqlite://:memory:") - await _apply_sqlite_migrations(pool, migrations_dir) + await apply_sqlite_migrations(pool, migrations_dir) (migrations_dir / "001_first.sql").write_text("CREATE TABLE t1_modified (id INTEGER PRIMARY KEY);") with pytest.raises(RuntimeError, match="HASH MISMATCH"): - await _apply_sqlite_migrations(pool, migrations_dir) + await apply_sqlite_migrations(pool, migrations_dir) @pytest.mark.asyncio async def test_bootstrap_snapshot_era_database(self, migrations_dir: Path) -> None: @@ -386,7 +386,7 @@ async def test_bootstrap_snapshot_era_database(self, migrations_dir: Path) -> No (migrations_dir / "001_first.sql").write_text("CREATE TABLE current_policy (id INTEGER PRIMARY KEY);") # 010 is beyond bootstrap range — should be applied as a new migration (migrations_dir / "010_new_feature.sql").write_text("CREATE TABLE new_feature (id INTEGER PRIMARY KEY);") - await _apply_sqlite_migrations(pool, migrations_dir) + await apply_sqlite_migrations(pool, migrations_dir) async with pool.connection() as conn: tracked = await conn.fetch("SELECT filename FROM _migrations ORDER BY filename") filenames = [r["filename"] for r in tracked] diff --git a/uv.lock b/uv.lock index 2d0ec8150..d339ff063 100644 --- a/uv.lock +++ b/uv.lock @@ -681,6 +681,43 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/86/f1/62a193f0227cf15a920390abe675f386dec35f7ae3ffe6da582d3ade42c7/googleapis_common_protos-1.70.0-py3-none-any.whl", hash = "sha256:b8bfcca8c25a2bb253e0e0b0adaf8c00773e5e6af6fd92397576680b807e0fd8", size = 294530, upload-time = "2025-04-14T10:17:01.271Z" }, ] +[[package]] +name = "greenlet" +version = "3.5.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3c/3f/dbf99fb14bfeb88c28f16729215478c0e265cacd6dc22270c8f31bb6892f/greenlet-3.5.0.tar.gz", hash = "sha256:d419647372241bc68e957bf38d5c1f98852155e4146bd1e4121adea81f4f01e4", size = 196995, upload-time = "2026-04-27T13:37:15.544Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0c/58/fc576f99037ce19c5aa16628e4c3226b6d1419f72a62c79f5f40576e6eb3/greenlet-3.5.0-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:5a5ed18de6a0f6cc7087f1563f6bd93fc7df1c19165ca01e9bde5a5dc281d106", size = 285066, upload-time = "2026-04-27T12:23:05.033Z" }, + { url = "https://files.pythonhosted.org/packages/4a/ba/b28ddbe6bfad6a8ac196ef0e8cff37bc65b79735995b9e410923fffeeb70/greenlet-3.5.0-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3a717fbc46d8a354fa675f7c1e813485b6ba3885f9bef0cd56e5ba27d758ff5b", size = 604414, upload-time = "2026-04-27T12:52:42.358Z" }, + { url = "https://files.pythonhosted.org/packages/09/06/4b69f8f0b67603a8be2790e55107a190b376f2627fe0eaf5695d85ffb3cd/greenlet-3.5.0-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ddc090c5c1792b10246a78e8c2163ebbe04cf877f9d785c230a7b27b39ad038e", size = 617349, upload-time = "2026-04-27T12:59:43.32Z" }, + { url = "https://files.pythonhosted.org/packages/6a/15/a643b4ecd09969e30b8a150d5919960caae0abe4f5af75ab040b1ab85e78/greenlet-3.5.0-cp313-cp313-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4964101b8585c144cbda5532b1aa644255126c08a265dae90c16e7a0e63aaa9d", size = 623234, upload-time = "2026-04-27T13:02:40.611Z" }, + { url = "https://files.pythonhosted.org/packages/8a/17/a3918541fd0ddefe024a69de6d16aa7b46d36ac19562adaa63c7fa180eff/greenlet-3.5.0-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2094acd54b272cb6eae8c03dd87b3fa1820a4cef18d6889c378d503500a1dc13", size = 613927, upload-time = "2026-04-27T12:25:30.28Z" }, + { url = "https://files.pythonhosted.org/packages/77/18/3b13d5ef1275b0ffaf933b05efa21408ac4ca95823c7411d79682e4fdcff/greenlet-3.5.0-cp313-cp313-manylinux_2_39_riscv64.whl", hash = "sha256:7022615368890680e67b9965d33f5773aade330d5343bbe25560135aaa849eae", size = 425243, upload-time = "2026-04-27T13:05:15.689Z" }, + { url = "https://files.pythonhosted.org/packages/ee/e1/bd0af6213c7dd33175d8a462d4c1fe1175124ebed4855bc1475a5b5242c2/greenlet-3.5.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:5e05ba267789ea87b5a155cf0e810b1ab88bf18e9e8740813945ceb8ee4350ba", size = 1570893, upload-time = "2026-04-27T12:53:29.483Z" }, + { url = "https://files.pythonhosted.org/packages/9b/2a/0789702f864f5382cb476b93d7a9c823c10472658102ccd65f415747d2e2/greenlet-3.5.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:0ecec963079cd58cbd14723582384f11f166fd58883c15dcbfb342e0bc9b5846", size = 1636060, upload-time = "2026-04-27T12:25:28.845Z" }, + { url = "https://files.pythonhosted.org/packages/b2/8f/22bf9df92bbff0eb07842b60f7e63bf7675a9742df628437a9f02d09137f/greenlet-3.5.0-cp313-cp313-win_amd64.whl", hash = "sha256:728d9667d8f2f586644b748dbd9bb67e50d6a9381767d1357714ea6825bb3bf5", size = 238740, upload-time = "2026-04-27T12:24:01.341Z" }, + { url = "https://files.pythonhosted.org/packages/b6/b7/9c5c3d653bd4ff614277c049ac676422e2c557db47b4fe43e6313fc005dc/greenlet-3.5.0-cp313-cp313-win_arm64.whl", hash = "sha256:47422135b1d308c14b2c6e758beedb1acd33bb91679f5670edf77bf46244722b", size = 235525, upload-time = "2026-04-27T12:23:12.308Z" }, + { url = "https://files.pythonhosted.org/packages/94/5e/a70f31e3e8d961c4ce589c15b28e4225d63704e431a23932a3808cbcc867/greenlet-3.5.0-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:f35807464c4c58c55f0d31dfa83c541a5615d825c2fe3d2b95360cf7c4e3c0a8", size = 285564, upload-time = "2026-04-27T12:23:08.555Z" }, + { url = "https://files.pythonhosted.org/packages/af/a6/046c0a28e21833e4086918218cfb3d8bed51c075a1b700f20b9d7861c0f4/greenlet-3.5.0-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:55fa7ea52771be44af0de27d8b80c02cd18c2c3cddde6c847ecebdf72418b6a1", size = 651166, upload-time = "2026-04-27T12:52:43.644Z" }, + { url = "https://files.pythonhosted.org/packages/47/f8/4af27f71c5ff32a7fbc516adb46370d9c4ae2bc7bd3dc7d066ac542b4b15/greenlet-3.5.0-cp314-cp314-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a97e4821aa710603f94de0da25f25096454d78ffdace5dc77f3a006bc01abba3", size = 663792, upload-time = "2026-04-27T12:59:44.93Z" }, + { url = "https://files.pythonhosted.org/packages/fb/89/2dadb89793c37ee8b4c237857188293e9060dc085f19845c292e00f8e091/greenlet-3.5.0-cp314-cp314-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bf2d8a80bec89ab46221ae45c5373d5ba0bd36c19aa8508e85c6cd7e5106cd37", size = 668086, upload-time = "2026-04-27T13:02:42.314Z" }, + { url = "https://files.pythonhosted.org/packages/a3/59/1bd6d7428d6ed9106efbb8c52310c60fd04f6672490f452aeaa3829aa436/greenlet-3.5.0-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8f52a464e4ed91780bdfbbdd2b97197f3accaa629b98c200f4dffada759f3ae7", size = 660933, upload-time = "2026-04-27T12:25:33.276Z" }, + { url = "https://files.pythonhosted.org/packages/82/35/75722be7e26a2af4cbd2dc35b0ed382dacf9394b7e75551f76ed1abe87f2/greenlet-3.5.0-cp314-cp314-manylinux_2_39_riscv64.whl", hash = "sha256:1bae92a1dd94c5f9d9493c3a212dd874c202442047cf96446412c862feca83a2", size = 470799, upload-time = "2026-04-27T13:05:17.094Z" }, + { url = "https://files.pythonhosted.org/packages/83/e4/b903e5a5fae1e8a28cdd32a0cfbfd560b668c25b692f67768822ddc5f40f/greenlet-3.5.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:762612baf1161ccb8437c0161c668a688223cba28e1bf038f4eb47b13e39ccdf", size = 1618401, upload-time = "2026-04-27T12:53:31.062Z" }, + { url = "https://files.pythonhosted.org/packages/0e/e3/5ec408a329acb854fb607a122e1ee5fb3ff649f9a97952948a90803c0d8e/greenlet-3.5.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:57a43c6079a89713522bc4bcb9f75070ecf5d3dbad7792bfe42239362cbf2a16", size = 1682038, upload-time = "2026-04-27T12:25:31.838Z" }, + { url = "https://files.pythonhosted.org/packages/91/20/6b165108058767ee643c55c5c4904d591a830ee2b3c7dbd359828fbc829f/greenlet-3.5.0-cp314-cp314-win_amd64.whl", hash = "sha256:3bc59be3945ae9750b9e7d45067d01ae3fe90ea5f9ade99239dabdd6e28a5033", size = 239835, upload-time = "2026-04-27T12:24:54.136Z" }, + { url = "https://files.pythonhosted.org/packages/4e/62/1c498375cee177b55d980c1db319f26470e5309e54698c8f8fc06c0fd539/greenlet-3.5.0-cp314-cp314-win_arm64.whl", hash = "sha256:a96fcee45e03fe30a62669fd16ab5c9d3c172660d3085605cb1e2d1280d3c988", size = 236862, upload-time = "2026-04-27T12:23:24.957Z" }, + { url = "https://files.pythonhosted.org/packages/78/a8/4522939255bb5409af4e87132f915446bf3622c2c292d14d3c38d128ae82/greenlet-3.5.0-cp314-cp314t-macosx_11_0_universal2.whl", hash = "sha256:a10a732421ab4fec934783ce3e54763470d0181db6e3468f9103a275c3ed1853", size = 293614, upload-time = "2026-04-27T12:24:12.874Z" }, + { url = "https://files.pythonhosted.org/packages/15/5e/8744c52e2c027b5a8772a01561934c8835f869733e101f62075c60430340/greenlet-3.5.0-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7fc391b1566f2907d17aaebe78f8855dc45675159a775fcf9e61f8ee0078e87f", size = 650723, upload-time = "2026-04-27T12:52:45.412Z" }, + { url = "https://files.pythonhosted.org/packages/00/ef/7b4c39c03cf46ceca512c5d3f914afd85aa30b2cc9a93015b0dd73e4be6c/greenlet-3.5.0-cp314-cp314t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:680bd0e7ad5e8daa8a4aa89f68fd6adc834b8a8036dc256533f7e08f4a4b01f7", size = 656529, upload-time = "2026-04-27T12:59:46.295Z" }, + { url = "https://files.pythonhosted.org/packages/5f/5c/0602239503b124b70e39355cbdb39361ecfe65b87a5f2f63752c32f5286f/greenlet-3.5.0-cp314-cp314t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1aa4ce8debcd4ea7fb2e150f3036588c41493d1d52c43538924ae1819003f4ce", size = 657015, upload-time = "2026-04-27T13:02:43.973Z" }, + { url = "https://files.pythonhosted.org/packages/0b/b5/c7768f352f5c010f92064d0063f987e7dc0cd290a6d92a34109015ce4aa1/greenlet-3.5.0-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ddb36c7d6c9c0a65f18c7258634e0c416c6ab59caac8c987b96f80c2ebda0112", size = 654364, upload-time = "2026-04-27T12:25:35.64Z" }, + { url = "https://files.pythonhosted.org/packages/38/51/8699f865f125dc952384cb432b0f7138aa4d8f2969a7d12d0df5b94d054d/greenlet-3.5.0-cp314-cp314t-manylinux_2_39_riscv64.whl", hash = "sha256:728a73687e39ae9ca34e4694cbf2f049d3fbc7174639468d0f67200a97d8f9e2", size = 488275, upload-time = "2026-04-27T13:05:18.28Z" }, + { url = "https://files.pythonhosted.org/packages/ef/d0/079ebe12e4b1fc758857ce5be1a5e73f06870f2101e52611d1e71925ce54/greenlet-3.5.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e5ddf316ced87539144621453c3aef229575825fe60c604e62bedc4003f372b2", size = 1614204, upload-time = "2026-04-27T12:53:32.618Z" }, + { url = "https://files.pythonhosted.org/packages/6d/89/6c2fb63df3596552d20e58fb4d96669243388cf680cff222758812c7bfaa/greenlet-3.5.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:4a448128607be0de65342dc9b31be7f948ef4cc0bc8832069350abefd310a8f2", size = 1675480, upload-time = "2026-04-27T12:25:34.168Z" }, + { url = "https://files.pythonhosted.org/packages/15/32/77ee8a6c1564fc345a491a4e85b3bf360e4cf26eac98c4532d2fdb96e01f/greenlet-3.5.0-cp314-cp314t-win_amd64.whl", hash = "sha256:d60097128cb0a1cab9ea541186ea13cd7b847b8449a7787c2e2350da0cb82d86", size = 245324, upload-time = "2026-04-27T12:24:40.295Z" }, +] + [[package]] name = "grpcio" version = "1.75.1" @@ -1062,12 +1099,14 @@ dependencies = [ dev = [ { name = "asgi-lifespan" }, { name = "luthien-cli" }, + { name = "playwright" }, { name = "pre-commit" }, { name = "pyright" }, { name = "pytest" }, { name = "pytest-asyncio" }, { name = "pytest-cov" }, { name = "pytest-httpx" }, + { name = "pytest-playwright" }, { name = "pytest-timeout" }, { name = "radon" }, { name = "ruff" }, @@ -1104,12 +1143,14 @@ requires-dist = [ dev = [ { name = "asgi-lifespan", specifier = ">=2.1.0" }, { name = "luthien-cli", editable = "src/luthien_cli" }, + { name = "playwright", specifier = "==1.50.0" }, { name = "pre-commit", specifier = ">=4.3.0" }, { name = "pyright", specifier = ">=1.1.406,<1.2" }, { name = "pytest", specifier = ">=8.4.1" }, { name = "pytest-asyncio", specifier = ">=1.1.0" }, { name = "pytest-cov", specifier = ">=6.2.1" }, { name = "pytest-httpx", specifier = ">=0.35.0" }, + { name = "pytest-playwright", specifier = ">=0.5.0" }, { name = "pytest-timeout", specifier = ">=2.4.0" }, { name = "radon", specifier = ">=6.0.1" }, { name = "ruff", specifier = ">=0.12.10" }, @@ -1533,6 +1574,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fe/39/979e8e21520d4e47a0bbe349e2713c0aac6f3d853d0e5b34d76206c439aa/platformdirs-4.3.8-py3-none-any.whl", hash = "sha256:ff7059bb7eb1179e2685604f4aaf157cfd9535242bd23742eadc3c13542139b4", size = 18567, upload-time = "2025-05-07T22:47:40.376Z" }, ] +[[package]] +name = "playwright" +version = "1.50.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "greenlet" }, + { name = "pyee" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/0d/5e/068dea3c96e9c09929b45c92cf7e573403b52a89aa463f89b9da9b87b7a4/playwright-1.50.0-py3-none-macosx_10_13_x86_64.whl", hash = "sha256:f36d754a6c5bd9bf7f14e8f57a2aea6fd08f39ca4c8476481b9c83e299531148", size = 40277564, upload-time = "2025-02-03T14:57:22.774Z" }, + { url = "https://files.pythonhosted.org/packages/78/85/b3deb3d2add00d2a6ee74bf6f57ccefb30efc400fd1b7b330ba9a3626330/playwright-1.50.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:40f274384591dfd27f2b014596250b2250c843ed1f7f4ef5d2960ecb91b4961e", size = 39521844, upload-time = "2025-02-03T14:57:29.372Z" }, + { url = "https://files.pythonhosted.org/packages/f3/f6/002b3d98df9c84296fea84f070dc0d87c2270b37f423cf076a913370d162/playwright-1.50.0-py3-none-macosx_11_0_universal2.whl", hash = "sha256:9922ef9bcd316995f01e220acffd2d37a463b4ad10fd73e388add03841dfa230", size = 40277563, upload-time = "2025-02-03T14:57:36.291Z" }, + { url = "https://files.pythonhosted.org/packages/b9/63/c9a73736e434df894e484278dddc0bf154312ff8d0f16d516edb790a7d42/playwright-1.50.0-py3-none-manylinux1_x86_64.whl", hash = "sha256:8fc628c492d12b13d1f347137b2ac6c04f98197ff0985ef0403a9a9ee0d39131", size = 45076712, upload-time = "2025-02-03T14:57:43.581Z" }, + { url = "https://files.pythonhosted.org/packages/bd/2c/a54b5a64cc7d1a62f2d944c5977fb3c88e74d76f5cdc7966e717426bce66/playwright-1.50.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffcff35f72db2689a79007aee78f1b0621a22e6e3d6c1f58aaa9ac805bf4497c", size = 44493111, upload-time = "2025-02-03T14:57:50.226Z" }, + { url = "https://files.pythonhosted.org/packages/2b/4a/047cbb2ffe1249bd7a56441fc3366fb4a8a1f44bc36a9061d10edfda2c86/playwright-1.50.0-py3-none-win32.whl", hash = "sha256:3b906f4d351260016a8c5cc1e003bb341651ae682f62213b50168ed581c7558a", size = 34784543, upload-time = "2025-02-03T14:57:55.942Z" }, + { url = "https://files.pythonhosted.org/packages/bc/2b/e944e10c9b18e77e43d3bb4d6faa323f6cc27597db37b75bc3fd796adfd5/playwright-1.50.0-py3-none-win_amd64.whl", hash = "sha256:1859423da82de631704d5e3d88602d755462b0906824c1debe140979397d2e8d", size = 34784546, upload-time = "2025-02-03T14:58:01.664Z" }, +] + [[package]] name = "pluggy" version = "1.6.0" @@ -1711,6 +1770,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/58/f0/427018098906416f580e3cf1366d3b1abfb408a0652e9f31600c24a1903c/pydantic_settings-2.10.1-py3-none-any.whl", hash = "sha256:a60952460b99cf661dc25c29c0ef171721f98bfcb52ef8d9ea4c943d7c8cc796", size = 45235, upload-time = "2025-06-24T13:26:45.485Z" }, ] +[[package]] +name = "pyee" +version = "12.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0a/37/8fb6e653597b2b67ef552ed49b438d5398ba3b85a9453f8ada0fd77d455c/pyee-12.1.1.tar.gz", hash = "sha256:bbc33c09e2ff827f74191e3e5bbc6be7da02f627b7ec30d86f5ce1a6fb2424a3", size = 30915, upload-time = "2024-11-16T21:26:44.275Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/25/68/7e150cba9eeffdeb3c5cecdb6896d70c8edd46ce41c0491e12fb2b2256ff/pyee-12.1.1-py3-none-any.whl", hash = "sha256:18a19c650556bb6b32b406d7f017c8f513aceed1ef7ca618fb65de7bd2d347ef", size = 15527, upload-time = "2024-11-16T21:26:42.422Z" }, +] + [[package]] name = "pygments" version = "2.19.2" @@ -1795,6 +1866,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c7/9d/bf86eddabf8c6c9cb1ea9a869d6873b46f105a5d292d3a6f7071f5b07935/pytest_asyncio-1.1.0-py3-none-any.whl", hash = "sha256:5fe2d69607b0bd75c656d1211f969cadba035030156745ee09e7d71740e58ecf", size = 15157, upload-time = "2025-07-16T04:29:24.929Z" }, ] +[[package]] +name = "pytest-base-url" +version = "2.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pytest" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ae/1a/b64ac368de6b993135cb70ca4e5d958a5c268094a3a2a4cac6f0021b6c4f/pytest_base_url-2.1.0.tar.gz", hash = "sha256:02748589a54f9e63fcbe62301d6b0496da0d10231b753e950c63e03aee745d45", size = 6702, upload-time = "2024-01-31T22:43:00.81Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/98/1c/b00940ab9eb8ede7897443b771987f2f4a76f06be02f1b3f01eb7567e24a/pytest_base_url-2.1.0-py3-none-any.whl", hash = "sha256:3ad15611778764d451927b2a53240c1a7a591b521ea44cebfe45849d2d2812e6", size = 5302, upload-time = "2024-01-31T22:42:58.897Z" }, +] + [[package]] name = "pytest-cov" version = "6.2.1" @@ -1822,6 +1906,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b0/ed/026d467c1853dd83102411a78126b4842618e86c895f93528b0528c7a620/pytest_httpx-0.35.0-py3-none-any.whl", hash = "sha256:ee11a00ffcea94a5cbff47af2114d34c5b231c326902458deed73f9c459fd744", size = 19442, upload-time = "2024-11-28T19:16:52.787Z" }, ] +[[package]] +name = "pytest-playwright" +version = "0.7.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "playwright" }, + { name = "pytest" }, + { name = "pytest-base-url" }, + { name = "python-slugify" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e8/6b/913e36aa421b35689ec95ed953ff7e8df3f2ee1c7b8ab2a3f1fd39d95faf/pytest_playwright-0.7.2.tar.gz", hash = "sha256:247b61123b28c7e8febb993a187a07e54f14a9aa04edc166f7a976d88f04c770", size = 16928, upload-time = "2025-11-24T03:43:22.53Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/76/61/4d333d8354ea2bea2c2f01bad0a4aa3c1262de20e1241f78e73360e9b620/pytest_playwright-0.7.2-py3-none-any.whl", hash = "sha256:8084e015b2b3ecff483c2160f1c8219b38b66c0d4578b23c0f700d1b0240ea38", size = 16881, upload-time = "2025-11-24T03:43:24.423Z" }, +] + [[package]] name = "pytest-timeout" version = "2.4.0" @@ -1864,6 +1963,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1b/d0/397f9626e711ff749a95d96b7af99b9c566a9bb5129b8e4c10fc4d100304/python_multipart-0.0.22-py3-none-any.whl", hash = "sha256:2b2cd894c83d21bf49d702499531c7bafd057d730c201782048f7945d82de155", size = 24579, upload-time = "2026-01-25T10:15:54.811Z" }, ] +[[package]] +name = "python-slugify" +version = "8.0.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "text-unidecode" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/87/c7/5e1547c44e31da50a460df93af11a535ace568ef89d7a811069ead340c4a/python-slugify-8.0.4.tar.gz", hash = "sha256:59202371d1d05b54a9e7720c5e038f928f45daaffe41dd10822f3907b937c856", size = 10921, upload-time = "2024-02-08T18:32:45.488Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a4/62/02da182e544a51a5c3ccf4b03ab79df279f9c60c5e82d5e8bec7ca26ac11/python_slugify-8.0.4-py2.py3-none-any.whl", hash = "sha256:276540b79961052b66b7d116620b36518847f52d5fd9e3a70164fc8c50faa6b8", size = 10051, upload-time = "2024-02-08T18:32:43.911Z" }, +] + [[package]] name = "pytz" version = "2025.2" @@ -2207,6 +2318,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8b/0c/9d30a4ebeb6db2b25a841afbb80f6ef9a854fc3b41be131d249a977b4959/starlette-0.46.2-py3-none-any.whl", hash = "sha256:595633ce89f8ffa71a015caed34a5b2dc1c0cdb3f0f1fbd1e69339cf2abeec35", size = 72037, upload-time = "2025-04-13T13:56:16.21Z" }, ] +[[package]] +name = "text-unidecode" +version = "1.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ab/e2/e9a00f0ccb71718418230718b3d900e71a5d16e701a3dae079a21e9cd8f8/text-unidecode-1.3.tar.gz", hash = "sha256:bad6603bb14d279193107714b288be206cac565dfa49aa5b105294dd5c4aab93", size = 76885, upload-time = "2019-08-30T21:36:45.405Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a6/a5/c0b6468d3824fe3fde30dbb5e1f687b291608f9473681bbf7dabbf5a87d7/text_unidecode-1.3-py2.py3-none-any.whl", hash = "sha256:1311f10e8b895935241623731c2ba64f4c455287888b18189350b67134a822e8", size = 78154, upload-time = "2019-08-30T21:37:03.543Z" }, +] + [[package]] name = "tiktoken" version = "0.11.0"