From 1261464a232f4b46c95789cf91a80c1d9ad0de11 Mon Sep 17 00:00:00 2001 From: Lukin Date: Mon, 6 Jul 2026 20:52:04 +0800 Subject: [PATCH] fix(admin): speed up request model search Co-Authored-By: Codex --- implementation-notes.md | 8 ++ package.json | 2 +- .../core/src/store/postgres/migrate.test.ts | 73 +++++++++++++++- packages/core/src/store/postgres/migrate.ts | 25 ++++++ packages/core/src/store/postgres/telemetry.ts | 10 +-- .../store/sqlite/memory-dedup-migrate.test.ts | 3 +- .../src/store/sqlite/memory-schema.test.ts | 3 +- packages/core/src/store/sqlite/migrate.ts | 27 ++++++ .../store/sqlite/oauth-quota-migrate.test.ts | 3 +- .../store/sqlite/oauth-usage-migrate.test.ts | 3 +- packages/core/src/store/sqlite/schema.test.ts | 86 +++++++++++++++++-- packages/core/src/store/sqlite/telemetry.ts | 10 +-- 12 files changed, 229 insertions(+), 24 deletions(-) diff --git a/implementation-notes.md b/implementation-notes.md index 9900ab83..66b388b3 100644 --- a/implementation-notes.md +++ b/implementation-notes.md @@ -15,6 +15,14 @@ - **观测决策**：触发时写入 passthrough mutation `body_shims_applied: ["anthropic_billing_cch_stabilized"]`，并清掉 raw_body 走重序列化，避免 telemetry 显示仍是旧的客户端 raw body。后续线上可按该 mutation 与 `cached_tokens/cache_creation_tokens` 验证成本改善。 - **风险边界**：如果 Anthropic 将来开始强校验官方 `cch` 与整请求字节一致，这个 shim 可能引发上游拒绝；届时可通过 native passthrough flag 或后续 runtime setting 回滚。当前生产证据显示 `cch` 更像缓存/归因指纹而非认证字段。 +## 2026-07-06 · Admin 请求列表模型关键词搜索改走预计算列（Admin requests performance，docs/07/11，原则 1/7） + +- **背景（Lukin）**：线上 `/admin/api/requests?...&model=fable&key_id=...` 首字节约 5.26s。生产 SQLite 虽能用 `(api_key_id, created_at)` 索引把范围缩到当天该 key 的 7922 行，但 `model` 筛选仍要对每行 `decision_json` 做 `requested_model` / `final.model_alias` JSON 提取和 LIKE，`COUNT` 单独就约 2.36s。 +- **查询决策**：保持 `model=` 的既有语义（substring 匹配 requested model、served alias、selected lane；大小写不敏感），新增生成列 `model_search`：把三段文本 lower 后拼成一个小搜索面。SQLite 用 VIRTUAL generated column，Postgres 用 STORED generated column。 +- **索引决策**：新增 `idx_telemetry_admin_model_window(created_at, model_search)` 与 `idx_telemetry_admin_key_model_window(api_key_id, created_at, model_search)`，让 admin 列表和计数扫描小索引值，不再为候选行反复解析 JSON。 +- **保持不变**：不改 API 参数、不改 UI、不改 payload/telemetry retention；`payload_retention_days` 继续保持 3 天。 +- **验证计划**：覆盖 SQLite/Postgres 迁移、SQLite telemetry 查询、跨 adapter store contract；发布后用同一线上 URL 比较 TTFB/total，并确认 `EXPLAIN QUERY PLAN` 使用新索引。 + ## 2026-07-06 · Admin 请求详情 payload 改为分段懒加载（Admin requests performance，docs/07/11，原则 1/7） - **背景（Lukin）**：线上 `/admin/requests/:traceId` 详情页会在首屏同时拉完整 request/response/upstream payload；部分记录超过 1MB，经公网和未压缩 JSON 传输后容易出现长时间白屏/卡顿。 diff --git a/package.json b/package.json index c051b4ea..435927df 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "helm-api", - "version": "0.25.11", + "version": "0.25.12", "private": true, "type": "module", "engines": { diff --git a/packages/core/src/store/postgres/migrate.test.ts b/packages/core/src/store/postgres/migrate.test.ts index b09c3748..65b584b9 100644 --- a/packages/core/src/store/postgres/migrate.test.ts +++ b/packages/core/src/store/postgres/migrate.test.ts @@ -140,6 +140,71 @@ describe("runPgMigrations — per-migration atomicity", () => { await db.$close(); }); + it("v36 adds model_search for admin request keyword filtering", async () => { + const client = new PGlite(); + const db = Object.assign(drizzlePglite(client), { $close: () => client.close() }); + await db.execute( + sql.raw("CREATE TABLE _migrations (version INTEGER PRIMARY KEY, applied_at BIGINT NOT NULL)"), + ); + await db.execute( + sql.raw(` + CREATE TABLE telemetry ( + id TEXT PRIMARY KEY, + request_id TEXT NOT NULL UNIQUE, + api_key_id TEXT NOT NULL, + decision_json JSONB NOT NULL, + final_status TEXT, + cost_usd DOUBLE PRECISION, + prompt_tokens INTEGER, + completion_tokens INTEGER, + cached_tokens INTEGER, + cache_creation_tokens INTEGER, + served_model TEXT, + generation_ms INTEGER, + latency_total_ms INTEGER, + created_at BIGINT NOT NULL + ) + `), + ); + for (let version = 1; version <= 35; version++) { + await db.execute( + sql.raw(`INSERT INTO _migrations (version, applied_at) VALUES (${version}, 1000)`), + ); + } + await db.execute( + sql.raw(` + INSERT INTO telemetry ( + id, request_id, api_key_id, decision_json, final_status, created_at + ) VALUES ( + 't1', + 'req_1', + 'k1', + '{"requested_model":"Claude-Fable-5","final":{"model_alias":"anthropic/claude-fable-5"},"lane":{"selected_lane":"premium"}}'::jsonb, + 'ok', + 1000 + ) + `), + ); + + await expect(runPgMigrations(db)).resolves.toBeUndefined(); + + const row = (await db.execute( + sql.raw("SELECT model_search FROM telemetry WHERE request_id = 'req_1'"), + )) as { rows: Array<{ model_search: string }> }; + expect(row.rows[0]?.model_search).toContain("claude-fable-5"); + expect(row.rows[0]?.model_search).toContain("premium"); + const indexes = (await db.execute( + sql.raw("SELECT indexname FROM pg_indexes WHERE tablename = 'telemetry'"), + )) as { rows: Array<{ indexname: string }> }; + expect(indexes.rows.map((r) => r.indexname)).toEqual( + expect.arrayContaining([ + "idx_telemetry_admin_model_window", + "idx_telemetry_admin_key_model_window", + ]), + ); + await db.$close(); + }); + it("creates memory job admin-stats indexes", async () => { const db = await createPgliteDb(); const indexes = (await db.execute( @@ -232,10 +297,10 @@ describe("runPgMigrations — per-migration atomicity", () => { // oauth_quota → pre-mark applied (out of scope). // v27 (pgvector + tsvector over memory_facts): this fixture never creates // memory_facts (+ no pgvector here) → pre-mark applied, out of scope. - // v29/v31 alter telemetry (absent here) → pre-mark applied; v30 alters api_keys + // v29/v31/v36 alter telemetry (absent here) → pre-mark applied; v30 alters api_keys // (absent here). v28 (payload_blobs CREATE) is pre-marked too (out of scope). // biome-ignore format: keep the version ledger on one readable line - for (const version of [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 27, 28, 29, 30, 31]) { + for (const version of [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 27, 28, 29, 30, 31, 36]) { await db.execute( sql.raw(`INSERT INTO _migrations (version, applied_at) VALUES (${version}, 1000)`), ); @@ -333,11 +398,11 @@ describe("runPgMigrations — per-migration atomicity", () => { // v26 (idx_memory_jobs_claim): no memory_jobs table in this messages fixture → pre-mark. // v27 (pgvector + tsvector over memory_facts): this messages fixture never creates // memory_facts (+ no pgvector here) → pre-mark applied, out of scope. - // v28 (payload_blobs), v29/v31 (telemetry generated/aggregate columns), and + // v28 (payload_blobs), v29/v31/v36 (telemetry generated/aggregate columns), and // v30 (api_keys allow_fast_mode) are out of scope here too. for (const version of [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 21, 22, 23, 24, 25, 26, 27, - 28, 29, 30, 31, + 28, 29, 30, 31, 36, ]) { await db.execute( sql.raw(`INSERT INTO _migrations (version, applied_at) VALUES (${version}, 1000)`), diff --git a/packages/core/src/store/postgres/migrate.ts b/packages/core/src/store/postgres/migrate.ts index ac24f518..f2622e62 100644 --- a/packages/core/src/store/postgres/migrate.ts +++ b/packages/core/src/store/postgres/migrate.ts @@ -753,6 +753,31 @@ const MIGRATIONS: readonly Migration[] = [ } }, }, + { + // Admin /requests keyword search — pg mirror of sqlite v37. Keep the existing + // requested_model OR final.model_alias OR lane semantics, but compute them once + // into a lowercase STORED column so model= scans a narrow indexed value instead + // of extracting three jsonb paths for every candidate row. + version: 36, + sql: ` + ALTER TABLE telemetry ADD COLUMN IF NOT EXISTS model_search TEXT + GENERATED ALWAYS AS ( + lower( + coalesce(decision_json ->> 'requested_model', '') || + chr(31) || + coalesce(decision_json -> 'final' ->> 'model_alias', '') || + chr(31) || + coalesce(decision_json -> 'lane' ->> 'selected_lane', '') + ) + ) STORED; + + CREATE INDEX IF NOT EXISTS idx_telemetry_admin_model_window + ON telemetry (created_at, model_search); + + CREATE INDEX IF NOT EXISTS idx_telemetry_admin_key_model_window + ON telemetry (api_key_id, created_at, model_search); + `, + }, ]; function resultRows(result: unknown): T[] { diff --git a/packages/core/src/store/postgres/telemetry.ts b/packages/core/src/store/postgres/telemetry.ts index ce934aaa..db71b46b 100644 --- a/packages/core/src/store/postgres/telemetry.ts +++ b/packages/core/src/store/postgres/telemetry.ts @@ -140,12 +140,12 @@ export class PgTelemetryStore implements TelemetryStore { if (query.decidedBy !== undefined) conds.push(sql`decided_by = ${query.decidedBy}`); if (query.lane !== undefined) conds.push(sql`lane = ${query.lane}`); // Broad operator search: requested model, served alias, or selected - // lane/channel. This backs the admin "Requested model / lane" box. + // lane/channel. `model_search` is a generated lowercase concat indexed by the + // admin migrations, so wide windows scan a small index value instead of + // parsing decision_json for every candidate row. if (query.model !== undefined) { - const pat = likeContains(query.model); - conds.push( - sql`(${telemetry.decisionJson} ->> 'requested_model' ILIKE ${pat} ESCAPE '\\' OR ${telemetry.decisionJson} -> 'final' ->> 'model_alias' ILIKE ${pat} ESCAPE '\\' OR lane ILIKE ${pat} ESCAPE '\\')`, - ); + const pat = likeContains(query.model.toLowerCase()); + conds.push(sql`model_search LIKE ${pat} ESCAPE '\\'`); } return and(...conds); } diff --git a/packages/core/src/store/sqlite/memory-dedup-migrate.test.ts b/packages/core/src/store/sqlite/memory-dedup-migrate.test.ts index f497e0e5..5425e925 100644 --- a/packages/core/src/store/sqlite/memory-dedup-migrate.test.ts +++ b/packages/core/src/store/sqlite/memory-dedup-migrate.test.ts @@ -47,12 +47,13 @@ function seedPreV21(): string { // v28 alters memory_facts (+ FTS); this memory_messages-only fixture never creates // memory_facts → pre-mark applied (out of scope for the v21 dedup test). ins.run(28); - // v30/v32 alter telemetry and v31 alters api_keys; both are absent here → pre-mark. + // v30/v32/v37 alter telemetry and v31 alters api_keys; both are absent here → pre-mark. // v29 (payload_blobs) is out of scope too. ins.run(29); ins.run(30); ins.run(31); ins.run(32); + ins.run(37); raw.exec(` CREATE TABLE memory_messages ( id TEXT PRIMARY KEY, diff --git a/packages/core/src/store/sqlite/memory-schema.test.ts b/packages/core/src/store/sqlite/memory-schema.test.ts index b808bf02..a3b255eb 100644 --- a/packages/core/src/store/sqlite/memory-schema.test.ts +++ b/packages/core/src/store/sqlite/memory-schema.test.ts @@ -374,12 +374,13 @@ describe("sqlite v18 forgetting schema deltas", () => { rec.run(26, Date.now()); // v27 indexes memory_jobs; this memory-only fixture never creates it → pre-mark. rec.run(27, Date.now()); - // v30/v32 alter telemetry and v31 alters api_keys; both are absent here → pre-mark. + // v30/v32/v37 alter telemetry and v31 alters api_keys; both are absent here → pre-mark. // v29 (payload_blobs) is out of scope too. rec.run(29, Date.now()); rec.run(30, Date.now()); rec.run(31, Date.now()); rec.run(32, Date.now()); + rec.run(37, Date.now()); seed.close(); expect(() => runMigrations(path)).not.toThrow(); diff --git a/packages/core/src/store/sqlite/migrate.ts b/packages/core/src/store/sqlite/migrate.ts index 9bdcfbeb..e68bbbfb 100644 --- a/packages/core/src/store/sqlite/migrate.ts +++ b/packages/core/src/store/sqlite/migrate.ts @@ -851,6 +851,33 @@ const MIGRATIONS: readonly Migration[] = [ } }, }, + { + // Admin /requests keyword search: model= used to parse decision_json for every + // candidate row (requested_model OR final.model_alias OR lane), which made + // day+key searches slow on large SQLite files. Promote that concatenated + // search surface to a VIRTUAL generated column and cover the admin date/key + // windows with small indexes. The search remains substring + case-insensitive + // for ASCII model/lane ids by storing lowercase text and lowercasing the query. + version: 37, + sql: ` + ALTER TABLE telemetry ADD COLUMN model_search TEXT + GENERATED ALWAYS AS ( + lower( + coalesce(json_extract(decision_json, '$.requested_model'), '') || + char(31) || + coalesce(json_extract(decision_json, '$.final.model_alias'), '') || + char(31) || + coalesce(json_extract(decision_json, '$.lane.selected_lane'), '') + ) + ) VIRTUAL; + + CREATE INDEX IF NOT EXISTS idx_telemetry_admin_model_window + ON telemetry (created_at, model_search); + + CREATE INDEX IF NOT EXISTS idx_telemetry_admin_key_model_window + ON telemetry (api_key_id, created_at, model_search); + `, + }, ]; function sqliteTableHasColumns( diff --git a/packages/core/src/store/sqlite/oauth-quota-migrate.test.ts b/packages/core/src/store/sqlite/oauth-quota-migrate.test.ts index c650335a..dfad5cb8 100644 --- a/packages/core/src/store/sqlite/oauth-quota-migrate.test.ts +++ b/packages/core/src/store/sqlite/oauth-quota-migrate.test.ts @@ -32,12 +32,13 @@ function seedPreV26(): string { ins.run(27); // v28 alters memory_facts (+ FTS), absent from this oauth_quota-only fixture → pre-mark. ins.run(28); - // v30/v32 alter telemetry and v31 alters api_keys; both are absent here → pre-mark. + // v30/v32/v37 alter telemetry and v31 alters api_keys; both are absent here → pre-mark. // v29 (payload_blobs) is out of scope too. ins.run(29); ins.run(30); ins.run(31); ins.run(32); + ins.run(37); raw.exec(` CREATE TABLE oauth_quota ( provider_id TEXT NOT NULL, diff --git a/packages/core/src/store/sqlite/oauth-usage-migrate.test.ts b/packages/core/src/store/sqlite/oauth-usage-migrate.test.ts index 2fc28012..fe403b4d 100644 --- a/packages/core/src/store/sqlite/oauth-usage-migrate.test.ts +++ b/packages/core/src/store/sqlite/oauth-usage-migrate.test.ts @@ -43,12 +43,13 @@ function seedPreV23(): string { ins.run(27); // v28 alters memory_facts (+ FTS), absent from this oauth_usage-only fixture → pre-mark. ins.run(28); - // v30/v32 alter telemetry and v31 alters api_keys; both are absent here → pre-mark. + // v30/v32/v37 alter telemetry and v31 alters api_keys; both are absent here → pre-mark. // v29 (payload_blobs) is out of scope too. ins.run(29); ins.run(30); ins.run(31); ins.run(32); + ins.run(37); raw.exec(` CREATE TABLE oauth_usage ( provider_id TEXT NOT NULL, diff --git a/packages/core/src/store/sqlite/schema.test.ts b/packages/core/src/store/sqlite/schema.test.ts index ab59dce2..e7b62467 100644 --- a/packages/core/src/store/sqlite/schema.test.ts +++ b/packages/core/src/store/sqlite/schema.test.ts @@ -66,11 +66,11 @@ describe("sqlite schema + migrations", () => { // pre-mark applied, out of scope. // v28 alters memory_facts (+ FTS); this memory_jobs-only fixture never creates // memory_facts → pre-mark applied (out of scope for this v14–v16 test). - // v30/v32 alter telemetry and v31 alters api_keys; both tables are absent from + // v30/v32/v37 alter telemetry and v31 alters api_keys; both tables are absent from // this memory_jobs-only fixture → pre-mark applied. v29 (payload_blobs CREATE) // has no dependency but is pre-marked too. // biome-ignore format: keep the version ledger on one readable line - for (const v of [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 28, 29, 30, 31, 32]) + for (const v of [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 28, 29, 30, 31, 32, 37]) rec.run(v, Date.now()); const insert = seed.prepare( "INSERT INTO memory_jobs (id, type, scope_id, status, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?)", @@ -130,7 +130,7 @@ describe("sqlite schema + migrations", () => { .all() .map((c) => (c as { name: string }).name); const telCols = raw - .prepare("PRAGMA table_info(telemetry)") + .prepare("PRAGMA table_xinfo(telemetry)") .all() .map((c) => (c as { name: string }).name); @@ -166,6 +166,7 @@ describe("sqlite schema + migrations", () => { "cache_creation_tokens", "served_model", "generation_ms", + "model_search", "created_at", ]) { expect(telCols).toContain(c); @@ -176,6 +177,8 @@ describe("sqlite schema + migrations", () => { .map((idx) => (idx as { name: string }).name); expect(telemetryIndexes).toContain("idx_telemetry_admin_window_cover"); expect(telemetryIndexes).toContain("idx_telemetry_admin_key_window_cover"); + expect(telemetryIndexes).toContain("idx_telemetry_admin_model_window"); + expect(telemetryIndexes).toContain("idx_telemetry_admin_key_model_window"); raw.close(); }); @@ -250,6 +253,79 @@ describe("sqlite schema + migrations", () => { } }); + it("v37 adds model_search for admin request keyword filtering", () => { + const dir = mkdtempSync(join(tmpdir(), "helm-sqlite-v37-")); + const path = join(dir, "helm.db"); + try { + const seed = new Database(path); + seed.exec( + "CREATE TABLE _migrations (version INTEGER PRIMARY KEY, applied_at INTEGER NOT NULL);", + ); + seed.exec(` + CREATE TABLE telemetry ( + id TEXT PRIMARY KEY, + request_id TEXT NOT NULL UNIQUE, + api_key_id TEXT NOT NULL, + decision_json TEXT NOT NULL, + final_status TEXT, + cost_usd REAL, + prompt_tokens INTEGER, + completion_tokens INTEGER, + cached_tokens INTEGER, + cache_creation_tokens INTEGER, + served_model TEXT, + generation_ms INTEGER, + latency_total_ms INTEGER, + created_at INTEGER NOT NULL + ); + `); + const rec = seed.prepare("INSERT INTO _migrations (version, applied_at) VALUES (?, ?)"); + for (let v = 1; v <= 36; v++) rec.run(v, Date.now()); + seed + .prepare( + `INSERT INTO telemetry ( + id, request_id, api_key_id, decision_json, final_status, created_at + ) VALUES (?, ?, ?, ?, ?, ?)`, + ) + .run( + "t1", + "req_1", + "k1", + JSON.stringify({ + requested_model: "Claude-Fable-5", + final: { model_alias: "anthropic/claude-fable-5" }, + lane: { selected_lane: "premium" }, + }), + "ok", + 1000, + ); + seed.close(); + + runMigrations(path); + + const after = new Database(path); + const cols = after + .prepare("PRAGMA table_xinfo(telemetry)") + .all() + .map((c) => (c as { name: string }).name); + expect(cols).toContain("model_search"); + const row = after + .prepare("SELECT model_search FROM telemetry WHERE request_id = ?") + .get("req_1") as { model_search: string }; + expect(row.model_search).toContain("claude-fable-5"); + expect(row.model_search).toContain("premium"); + const telemetryIndexes = after + .prepare("PRAGMA index_list(telemetry)") + .all() + .map((idx) => (idx as { name: string }).name); + expect(telemetryIndexes).toContain("idx_telemetry_admin_model_window"); + expect(telemetryIndexes).toContain("idx_telemetry_admin_key_model_window"); + after.close(); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + it("v6 rebuild relaxes cost_usd to REAL and preserves existing rows", () => { // Simulate an OLD database already migrated through v5 (INTEGER cost_usd) // carrying a telemetry row, then let runMigrations apply only v6. @@ -364,8 +440,8 @@ describe("sqlite schema + migrations", () => { // api_keys-only fixture) → pre-mark applied. // v25 (telemetry.generation_ms): no telemetry table here → pre-mark applied. // v28 alters memory_facts (absent from this api_keys-only fixture) → pre-mark. - // v30/v32 alter telemetry (absent here) → pre-mark; v29 (payload_blobs) pre-marked too. - for (const v of [1, 2, 3, 4, 5, 6, 7, 8, 9, 18, 20, 21, 22, 24, 25, 28, 29, 30, 32]) + // v30/v32/v37 alter telemetry (absent here) → pre-mark; v29 (payload_blobs) pre-marked too. + for (const v of [1, 2, 3, 4, 5, 6, 7, 8, 9, 18, 20, 21, 22, 24, 25, 28, 29, 30, 32, 37]) rec.run(v, Date.now()); seed .prepare( diff --git a/packages/core/src/store/sqlite/telemetry.ts b/packages/core/src/store/sqlite/telemetry.ts index b4865d2a..6aedef49 100644 --- a/packages/core/src/store/sqlite/telemetry.ts +++ b/packages/core/src/store/sqlite/telemetry.ts @@ -127,12 +127,12 @@ export class SqliteTelemetryStore implements TelemetryStore { if (query.decidedBy !== undefined) conds.push(sql`decided_by = ${query.decidedBy}`); if (query.lane !== undefined) conds.push(sql`lane = ${query.lane}`); // Broad operator search: requested model, served alias, or selected - // lane/channel. This backs the admin "Requested model / lane" box. + // lane/channel. `model_search` is a generated lowercase concat indexed by the + // admin migrations, so wide windows scan a small index value instead of + // parsing decision_json for every candidate row. if (query.model !== undefined) { - const pat = likeContains(query.model); - conds.push( - sql`(json_extract(${telemetry.decisionJson}, '$.requested_model') LIKE ${pat} ESCAPE '\\' OR json_extract(${telemetry.decisionJson}, '$.final.model_alias') LIKE ${pat} ESCAPE '\\' OR lane LIKE ${pat} ESCAPE '\\')`, - ); + const pat = likeContains(query.model.toLowerCase()); + conds.push(sql`model_search LIKE ${pat} ESCAPE '\\'`); } return and(...conds); }