diff --git a/CHANGELOG.md b/CHANGELOG.md index 0da428d..49d6f7f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,12 +7,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Fixed +- **Queries were attributed to the wrong request when a connection pool was full.** pg-pool, mysql2 and mysql call a queued request's callback from whichever request released a connection, so its queries landed in that other request's tree, or in none. Pool callbacks, and every callback a tracked driver call takes, now keep the caller's context. With a pool of 1 and 4 concurrent requests, 3 of the 4 showed no queries before (found while adding pool-wait tracking). + ### Fixed (found by the new evaluation set) - `run` reported each of several `Promise.all` calls as "dominant" (each spans most of the request). Parallel calls are no longer dominant, and a dominant call must take at least 50 ms (`dominantMinMs`). - Blocking code was reported three times: as blocking, as its own route queueing, and as the process being busy. Only the bystanders that waited behind it are listed now. - `traceperf/auto` wrote each request tree with its own `appendFileSync`, and under load that showed up as the app's "CPU cost" (`writeFileUtf8`). Writes are batched every 20 ms. ### Added +- **Missing indexes:** `traceperf run` EXPLAINs (plans, never runs) the app's Postgres `SELECT`s after the load and reports full scans of tables with 10,000+ rows that an index would avoid, with the `CREATE INDEX` to add. `--no-explain` turns it off. +- **Pool exhaustion:** `pg` and `mysql2` pools track the wait for a free connection when every connection is in use (`pg pool wait`, `mysql pool wait`); `run`/`explain` report it with the setting to change. - **GitHub Action** (`uses: thelastbackspace/traceperf@`): runs the baseline check, fails the job on more calls per request, and posts one pull-request comment it updates on every push. `traceperf run --markdown file.md` writes the same summary. - `traceperf run` reads `traceperf.config.json` (or `--config file`); command-line options win. A `login` step logs in before the traffic and sends the returned token or cookies with every request; `${NAME}` in the file is read from the environment. - `npm run eval` scores `traceperf run` on 20 apps in `examples/slow-apps`: 12 with a planted problem, 8 clean ones where the right answer is "no issues". CI fails on any missed problem or false alarm. Before these fixes: 17/17 found, 15 false alarms; after: 17/17, 0. diff --git a/README.md b/README.md index 6f0a73b..4bade7d 100644 --- a/README.md +++ b/README.md @@ -93,6 +93,14 @@ The database was fast; `buildReport` was blocking it. That only shows up when re fix: Mongoose is building full documents: add .lean() to read-only queries … and .select() only the fields you send ``` - **Timings you can trust:** after the warm-up, `run` sends a few requests one at a time (`--probe`) and takes the per-request findings (a call taking most of a request, sequential awaits, N+1) from those. Under concurrent load every `await` also waits for the busy event loop, which makes fast queries look slow. +- **Missing indexes (Postgres):** after the load, the app runs plain `EXPLAIN` (the query is planned, not executed) on the `SELECT`s it made and reports full scans of big tables that an index would avoid: + + ``` + 🔍 SELECT * FROM orders WHERE customer_id = $1 (run 1219 times) reads all ~60,000 rows of orders to return ~12 (Seq Scan, Filter: (customer_id = 0)) + fix: add an index: CREATE INDEX ON orders (customer_id) + ``` + Postgres scans small tables on purpose, so a test database with little data rarely shows one: point the run at realistic data. Tables under 10,000 rows are ignored. `--no-explain` turns it off. +- **Pool exhaustion:** with `pg` and `mysql2`, the time a request waits for a free connection from a full pool is tracked (`pg pool wait`) and reported, with the pool setting to change: "GET /report waited 65 ms for a free pg connection (79% of the request): every connection in the pool was in use". - **Wrong setup is said first:** "2117 of 2117 responses failed (most: 401 Unauthorized)…" instead of timings of error pages. Checked on real apps, not only on the planted ones: Prisma's REST example, the RealWorld API (Express + Prisma), an Express + Mongoose boilerplate and a NestJS + TypeORM app. diff --git a/examples/slow-apps/cases.json b/examples/slow-apps/cases.json index 102faf1..3225c3b 100644 --- a/examples/slow-apps/cases.json +++ b/examples/slow-apps/cases.json @@ -214,5 +214,43 @@ "/flags" ], "expect": [] + }, + "pool-exhaustion": { + "paths": [ + "/report" + ], + "requires": "pg", + "expect": [ + { + "kind": "pool", + "includes": "for a free pg connection" + } + ] + }, + "missing-index": { + "paths": [ + "/orders" + ], + "requires": "pg", + "expect": [ + { + "kind": "index", + "includes": "traceperf_eval_orders" + } + ] + }, + "clean-indexed": { + "paths": [ + "/orders" + ], + "requires": "pg", + "expect": [] + }, + "clean-pool": { + "paths": [ + "/report" + ], + "requires": "pg", + "expect": [] } } diff --git a/examples/slow-apps/clean-indexed/server.js b/examples/slow-apps/clean-indexed/server.js new file mode 100644 index 0000000..a0bc9c5 --- /dev/null +++ b/examples/slow-apps/clean-indexed/server.js @@ -0,0 +1,10 @@ +// Clean: the fixed version of missing-index (customer_id is indexed). Needs TRACEPERF_PG_URL. +const serve = require('../serve'); +const setup = require('../missing-index/setup'); + +setup('traceperf_eval_orders_indexed', true).then(pool => { + let n = 0; + serve({ + '/orders': async () => (await pool.query('SELECT * FROM traceperf_eval_orders_indexed WHERE customer_id = $1', [n++ % 5000])).rows, + }); +}); diff --git a/examples/slow-apps/clean-pool/server.js b/examples/slow-apps/clean-pool/server.js new file mode 100644 index 0000000..13ad9a1 --- /dev/null +++ b/examples/slow-apps/clean-pool/server.js @@ -0,0 +1,9 @@ +// Clean: the fixed version of pool-exhaustion (pool of 20 for 10 concurrent requests). Needs TRACEPERF_PG_URL. +const pg = require('pg'); +const serve = require('../serve'); + +const pool = new pg.Pool({ connectionString: process.env.TRACEPERF_PG_URL, max: 20 }); + +serve({ + '/report': async () => (await pool.query('SELECT pg_sleep(0.015), now() AS at')).rows, +}); diff --git a/examples/slow-apps/missing-index/server.js b/examples/slow-apps/missing-index/server.js new file mode 100644 index 0000000..f54d7cb --- /dev/null +++ b/examples/slow-apps/missing-index/server.js @@ -0,0 +1,11 @@ +// Planted cause: orders are looked up by customer_id, which has no index: +// every request makes Postgres read all 60,000 rows. Needs TRACEPERF_PG_URL. +const serve = require('../serve'); +const setup = require('./setup'); + +setup('traceperf_eval_orders', false).then(pool => { + let n = 0; + serve({ + '/orders': async () => (await pool.query('SELECT * FROM traceperf_eval_orders WHERE customer_id = $1', [n++ % 5000])).rows, + }); +}); diff --git a/examples/slow-apps/missing-index/setup.js b/examples/slow-apps/missing-index/setup.js new file mode 100644 index 0000000..10ba4f1 --- /dev/null +++ b/examples/slow-apps/missing-index/setup.js @@ -0,0 +1,14 @@ +// A 60,000-row orders table (created once), with or without an index on customer_id +const pg = require('pg'); + +module.exports = async function setup(table, indexed) { + const pool = new pg.Pool({ connectionString: process.env.TRACEPERF_PG_URL, max: 10 }); + await pool.query(`CREATE TABLE IF NOT EXISTS ${table} (id int PRIMARY KEY, customer_id int, total numeric, created_at timestamptz)`); + const { rows } = await pool.query(`SELECT count(*)::int AS n FROM ${table}`); + if (rows[0].n === 0) { + await pool.query(`INSERT INTO ${table} SELECT g, g % 5000, g * 1.5, now() - g * interval '1 minute' FROM generate_series(1, 60000) g`); + } + if (indexed) await pool.query(`CREATE INDEX IF NOT EXISTS ${table}_customer ON ${table} (customer_id)`); + await pool.query(`ANALYZE ${table}`); + return pool; +}; diff --git a/examples/slow-apps/pool-exhaustion/server.js b/examples/slow-apps/pool-exhaustion/server.js new file mode 100644 index 0000000..d8ba81f --- /dev/null +++ b/examples/slow-apps/pool-exhaustion/server.js @@ -0,0 +1,11 @@ +// Planted cause: a pg pool of 2 connections serving 10 concurrent requests, +// each holding its connection for a ~15 ms query: most of every request is +// waiting for a free connection, not querying. Needs TRACEPERF_PG_URL. +const pg = require('pg'); +const serve = require('../serve'); + +const pool = new pg.Pool({ connectionString: process.env.TRACEPERF_PG_URL, max: 2 }); + +serve({ + '/report': async () => (await pool.query('SELECT pg_sleep(0.015), now() AS at')).rows, +}); diff --git a/src/__tests__/drivers.test.ts b/src/__tests__/drivers.test.ts index f297fd7..e48f9df 100644 --- a/src/__tests__/drivers.test.ts +++ b/src/__tests__/drivers.test.ts @@ -25,6 +25,7 @@ import { instrumentRedis, sqlLabel, } from '../index'; +import { observeSql, ObservedSql } from '../drivers'; function setup() { const roots: ExecutionRecord[] = []; @@ -477,3 +478,81 @@ describe('instrumentBetterSqlite3 (database shaped like better-sqlite3: Statemen expect(labels(roots[0])).toEqual([]); }); }); + +describe('pg pools without a server (fake pg)', () => { + // Shaped like pg: Client.query(text, values, cb) and pg-pool's counters and callback queue + function fakePg() { + class Client { + query(text: string, values: unknown, cb?: (e: unknown, r: unknown) => void) { + const done = typeof values === 'function' ? values as typeof cb : cb; + const result = { rows: [{ 'QUERY PLAN': [{ Plan: { 'Node Type': 'Seq Scan' } }] }] }; + if (done) { setTimeout(() => done(null, result), 2); return undefined; } + return new Promise(r => setTimeout(() => r(result), 2)); + } + } + class Pool { + options = { max: 1 }; + totalCount = 0; + idleCount = 0; + private queue: Array<(e: unknown, c: Client, release: () => void) => void> = []; + private client = new Client(); + connect(cb: (e: unknown, c: Client, release: () => void) => void) { + const release = () => { + const next = this.queue.shift(); + // The next waiter is called from inside this release: another request's context + if (next) next(null, this.client, release); + else this.idleCount = 1; + }; + if (this.totalCount < this.options.max) { this.totalCount++; setTimeout(() => cb(null, this.client, release), 1); return; } + if (this.idleCount > 0) { this.idleCount = 0; setTimeout(() => cb(null, this.client, release), 1); return; } + this.queue.push(cb); + } + query(text: string, values?: unknown[]) { + return new Promise((resolve, reject) => this.connect((e, c, release) => { + if (e) return reject(e); + (c.query(text, values) as Promise).then(r => { release(); resolve(r); }, reject); + })); + } + } + return { Client, Pool }; + } + + test('a full pool: each request keeps its own queries, and the wait is tracked', async () => { + const pg = fakePg(); + const { tp, roots } = setup(); + const uninstall = instrumentPg(tp, pg); + const pool = new pg.Pool(); + try { + await Promise.all([1, 2, 3].map(i => tp.track(async () => { + await pool.query(`SELECT ${i}`); + }, { label: `req${i}` }))); + } finally { + uninstall(); + } + const requests = roots.filter(r => r.label.startsWith('req')); + expect(requests).toHaveLength(3); + for (const r of requests) expect(labels(r).filter(l => l !== 'pg pool wait')).toEqual(['pg SELECT']); + expect(requests.filter(r => labels(r).includes('pg pool wait')).length).toBeGreaterThanOrEqual(1); + }); + + test('observeSql sees statements and runs its own on the same pool, untracked', async () => { + const pg = fakePg(); + const { tp, roots } = setup(); + const uninstall = instrumentPg(tp, pg); + const seen: ObservedSql[] = []; + observeSql(q => seen.push(q)); + const pool = new pg.Pool(); + try { + await tp.track(() => pool.query('SELECT * FROM users WHERE id = $1', [7]), { label: 'job' }); + expect(seen.map(q => [q.text, q.values])).toEqual([['SELECT * FROM users WHERE id = $1', [7]]]); + const rows = await seen[0].run('EXPLAIN (FORMAT JSON) SELECT 1'); + expect(rows[0]['QUERY PLAN']).toBeDefined(); + // The EXPLAIN was neither observed again nor tracked as the app's query + expect(seen).toHaveLength(1); + } finally { + observeSql(undefined); + uninstall(); + } + expect(roots.map(r => r.label)).toEqual(['job']); + }); +}); diff --git a/src/__tests__/run.test.ts b/src/__tests__/run.test.ts index 42ec458..14f4834 100644 --- a/src/__tests__/run.test.ts +++ b/src/__tests__/run.test.ts @@ -137,7 +137,9 @@ const cases: Record { - it.each(Object.keys(cases))('%s', async name => { + // Cases that need a real database run where CI provides one (TRACEPERF_PG_URL) + const runnable = Object.keys(cases).filter(name => !(cases[name] as { requires?: string }).requires || !!process.env.TRACEPERF_PG_URL); + it.each(runnable)('%s', async name => { const c = cases[name]; const out: string[] = []; const argv = [path.join(APPS, name, 'server.js'), ...c.paths.flatMap(p => ['--path', p]), '-d', '1s', '--json']; @@ -430,3 +432,24 @@ describe('markdownReport', () => { expect(markdownReport({ trees: 5, findings: [] })).toMatch(/^### traceperf: no issues found/); }); }); + +describe('indexFindings', () => { + const { indexFindings } = jest.requireActual('../run'); + const scan = (over: object) => ({ table: 'orders', filter: '(customer_id = 5)', rows: 12, columns: ['customer_id'], tableRows: 60000, ...over }); + it('reports selective full scans of big tables, once per index', () => { + const [f, ...rest] = indexFindings([ + { driver: 'pg', text: 'SELECT * FROM orders WHERE customer_id = $1', count: 500, scans: [scan({})] }, + { driver: 'pg', text: 'SELECT id FROM orders WHERE customer_id = $1', count: 20, scans: [scan({})] }, + ]); + expect(rest).toEqual([]); + expect(f.kind).toBe('index'); + expect(f.message).toMatch(/\(run 500 times\) reads all ~60,000 rows of orders to return ~12/); + expect(f.fix).toMatch(/^add an index: CREATE INDEX ON orders \(customer_id\)/); + }); + it('small tables, unselective filters and filters without usable columns are fine', () => { + expect(indexFindings([{ driver: 'pg', text: 'q', count: 9, scans: [scan({ tableRows: 500 })] }])).toEqual([]); + expect(indexFindings([{ driver: 'pg', text: 'q', count: 9, scans: [scan({ rows: 30000 })] }])).toEqual([]); + expect(indexFindings([{ driver: 'pg', text: 'q', count: 9, scans: [scan({ columns: [] })] }])).toEqual([]); + expect(indexFindings([{ driver: 'pg', text: 'q', count: 9, scans: [scan({ tableRows: 500 })] }], 100)).toHaveLength(1); + }); +}); diff --git a/src/__tests__/sql-plan.test.ts b/src/__tests__/sql-plan.test.ts new file mode 100644 index 0000000..9b8de13 --- /dev/null +++ b/src/__tests__/sql-plan.test.ts @@ -0,0 +1,35 @@ +import { filterColumns, indexStatement, seqScans } from '../core/sql-plan'; + +describe('sql plans (real Postgres 16 EXPLAIN output)', () => { + test('filter columns: equality first, then ranges; <> and LIKE left out', () => { + expect(filterColumns('(customer_id = 5)')).toEqual(['customer_id']); + expect(filterColumns('(customer_id = $1)')).toEqual(['customer_id']); + expect(filterColumns("((customer_id <> 3) AND (status = 'paid'::text) AND (created_at > (now() - '1 day'::interval)))")).toEqual(['status', 'created_at']); + expect(filterColumns("((status)::text = 'paid'::text)")).toEqual(['status']); + expect(filterColumns('("userId" = $1)')).toEqual(['userId']); + expect(filterColumns("(name ~~ '%x%'::text)")).toEqual([]); + expect(filterColumns('(id = ANY ($1))')).toEqual(['id']); + }); + + test('every filtered seq scan in the plan tree', () => { + const plan = [{ Plan: { + 'Node Type': 'Nested Loop', 'Plan Rows': 12, + Plans: [ + { 'Node Type': 'Seq Scan', 'Relation Name': 'tp_plan_demo', Alias: 'p', 'Plan Rows': 1, Filter: '(id = 7)' }, + { 'Node Type': 'Seq Scan', 'Relation Name': 'tp_plan_demo', Alias: 'o', 'Plan Rows': 12, Filter: '(customer_id = 7)' }, + { 'Node Type': 'Seq Scan', 'Relation Name': 'everything' }, + { 'Node Type': 'Index Scan', 'Relation Name': 'users', Filter: '(x = 1)' }, + ], + } }]; + expect(seqScans(plan)).toEqual([ + { table: 'tp_plan_demo', filter: '(id = 7)', rows: 1, columns: ['id'] }, + { table: 'tp_plan_demo', filter: '(customer_id = 7)', rows: 12, columns: ['customer_id'] }, + ]); + expect(seqScans(undefined)).toEqual([]); + }); + + test('index statement quotes what needs quoting', () => { + expect(indexStatement('orders', ['customer_id', 'created_at'])).toBe('CREATE INDEX ON orders (customer_id, created_at)'); + expect(indexStatement('public.Post', ['authorId'])).toBe('CREATE INDEX ON public."Post" ("authorId")'); + }); +}); diff --git a/src/core/explain.ts b/src/core/explain.ts index 454b465..9a86ac6 100644 --- a/src/core/explain.ts +++ b/src/core/explain.ts @@ -24,7 +24,7 @@ export interface ExplainNode { } /** 'busy' (the process was saturated) comes from traceperf run, which measures the whole process */ -export type FindingKind = 'blocked' | 'blocking' | 'sequential' | 'repeated' | 'dominant' | 'error' | 'busy'; +export type FindingKind = 'blocked' | 'blocking' | 'sequential' | 'repeated' | 'dominant' | 'error' | 'busy' | 'pool' | 'index'; export interface Finding { kind: FindingKind; @@ -223,7 +223,8 @@ export function explain(roots: ExplainNode[], options: ExplainOptions = {}): Fin } walk(root, node => { - const kids = node.children; + // A wait for a pooled connection is reported as such (5b), not as a call to reorder or batch + const kids = node.children.filter(k => !POOL_WAIT.test(k.label)); if (kids.length === 0) return; // What a library does inside one call (Prisma's include = two queries) is not the app's to reorder if (LIBRARY_CALL.test(node.label)) return; @@ -288,8 +289,26 @@ export function explain(roots: ExplainNode[], options: ExplainOptions = {}): Fin }); }); + // 5b. Waited for a free pooled connection (the drivers track it only when the pool was full) + const waits = new Map(); + walk(root, n => { + const m = POOL_WAIT.exec(n.label); + if (m) waits.set(m[1], (waits.get(m[1]) || 0) + n.duration); + }); + waits.forEach((ms, driver) => { + if (ms < minMs || ms < 0.2 * root.duration) return; + findings.push({ + kind: 'pool', + root: rootLabel, + ms, + group: `pool ${driver}`, + message: `${rootLabel} waited ${fmt(ms)} for a free ${driver} connection (${pct(ms, root.duration)} of the request): every connection in the pool was in use`, + }); + }); + // 5. One child dominates the top-level call for (const k of root.children) { + if (POOL_WAIT.test(k.label)) continue; // A request that makes one call is always "mostly" that call: only say so when it is slow if (root.children.length === 1 && k.duration < 100) continue; // A call under dominantMinMs is not worth chasing, whatever its share @@ -395,7 +414,7 @@ export function explain(roots: ExplainNode[], options: ExplainOptions = {}): Fin if (g.ms < minMs || rare) grouped.delete(key); else if (g.kind !== 'blocking' && g.roots.size === 1 && total > g.times) share.set(g, `in ${g.times} of ${total} ${g.root}`); } - const order: Record = { error: 0, busy: 1, blocked: 1, blocking: 2, repeated: 3, sequential: 4, dominant: 5 }; + const order: Record = { error: 0, busy: 1, blocked: 1, pool: 1, blocking: 2, index: 3, repeated: 3, sequential: 4, dominant: 5 }; return [...grouped.values()] // Failures first: they matter more than timing; then by total time involved .sort((a, b) => Number(b.kind === 'error') - Number(a.kind === 'error') || b.ms * b.times - a.ms * a.times || order[a.kind] - order[b.kind]) @@ -430,6 +449,7 @@ const SYNC_FIX: Array<[RegExp, string]> = [ [/garbage collection/, 'it allocates a lot: reuse objects/buffers and avoid building large temporary arrays or strings'], ]; /** Labels traceperf gives driver / ORM calls */ +const POOL_WAIT = /^(\w+) pool wait$/; const LIBRARY_CALL = /^(prisma\.|(pg|mysql|sqlite|mongo|redis|aws) )/; const SQL = /\b(select|insert|update|delete|pg|mysql|postgres|sql|query|prisma|knex|sequelize|typeorm|drizzle)\b/i; @@ -461,6 +481,11 @@ function suggestFix(f: Finding, hotspots?: Record, me if (!names.every(n => /^[\w.$]+$/.test(n))) return `if they don't depend on each other, start them together with Promise.all instead of awaiting each in turn`; return `if they don't depend on each other, start them together: await Promise.all([${names.map(n => `${n}()`).join(', ')}${labels.length > 3 ? ', ...' : ''}])`; } + case 'pool': { + const driver = /free (\w+) connection/.exec(f.message)?.[1]; + const setting = driver === 'mysql' ? 'connectionLimit (mysql2 default 10)' : driver === 'pg' ? 'max (pg default 10)' : 'the pool size'; + return `requests queued for a connection: raise ${setting} if the database can take more connections, or hold each one for less time (no other awaits inside a transaction; release clients you check out yourself). Compare with --concurrency below the pool size`; + } case 'repeated': { const label = labels[0] || ''; if (/^prisma\./.test(label)) return `load them in one call: include the relation in the parent query (include: { ... }), or findMany({ where: { id: { in: ids } } }) and match results in memory`; @@ -487,7 +512,7 @@ function suggestFix(f: Finding, hotspots?: Record, me } const ICON: Record = { - error: '✖', blocked: '⏳', blocking: '🔥', repeated: '🔁', sequential: '⇉', dominant: '●', busy: '🔥', + error: '✖', blocked: '⏳', blocking: '🔥', repeated: '🔁', sequential: '⇉', dominant: '●', busy: '🔥', pool: '⏳', index: '🔍', }; /** Render findings as text, one per line (and the fix, if any, below it) */ diff --git a/src/core/sql-plan.ts b/src/core/sql-plan.ts new file mode 100644 index 0000000..dfe8908 --- /dev/null +++ b/src/core/sql-plan.ts @@ -0,0 +1,67 @@ +/** + * Reading Postgres query plans (EXPLAIN (FORMAT JSON), not ANALYZE: the + * query is planned, never run) for scans an index would avoid. + */ + +export interface PlanNode { + 'Node Type'?: string; + 'Relation Name'?: string; + 'Alias'?: string; + 'Filter'?: string; + 'Plan Rows'?: number; + 'Total Cost'?: number; + Plans?: PlanNode[]; + [key: string]: unknown; +} + +export interface SeqScan { + table: string; + filter: string; + /** Rows the planner expects the scan to return (after the filter) */ + rows: number; + /** Columns compared in the filter, in order: the index to suggest */ + columns: string[]; +} + +/** Every sequential scan with a filter in a plan (the root of EXPLAIN (FORMAT JSON) output, or its "Plan") */ +export function seqScans(plan: unknown): SeqScan[] { + const out: SeqScan[] = []; + const top: PlanNode | undefined = Array.isArray(plan) ? (plan[0] && (plan[0] as any).Plan) : (plan as any)?.Plan ?? plan; + const visit = (node: PlanNode | undefined) => { + if (!node || typeof node !== 'object') return; + if (node['Node Type'] === 'Seq Scan' && node['Relation Name'] && typeof node.Filter === 'string') { + out.push({ table: node['Relation Name'], filter: node.Filter, rows: Number(node['Plan Rows']) || 0, columns: filterColumns(node.Filter) }); + } + for (const child of node.Plans || []) visit(child); + }; + visit(top); + return out; +} + +/** + * Columns compared with a value in a Postgres filter expression: + * "(customer_id = $1)", "((status)::text = 'paid'::text)", "((a = 1) AND (b > $2))". + * Equality columns come first (they lead a composite index), then ranges. + */ +export function filterColumns(filter: string): string[] { + const eq: string[] = []; + const range: string[] = []; + const re = /\(*("?[a-z_][\w$]*"?)\)?(?:::[\w ]+)?\s*(=|<>|!=|<=|>=|<|>|~~\*?|= ANY)\s*(\(?\$\d+|'|\(?-?\d|ANY|\(?'|true|false|NULL|\(?[a-z_]+\()/gi; + let m: RegExpExecArray | null; + while ((m = re.exec(filter))) { + const column = m[1].replace(/"/g, ''); + if (/^(and|or|not|true|false|null)$/i.test(column)) continue; + // "<>" and LIKE '%x' can't use a btree index: leave them out + if (m[2] === '<>' || m[2] === '!=') continue; + if (/^~~/.test(m[2])) continue; + const list = m[2] === '=' || m[2] === '= ANY' ? eq : range; + if (!eq.includes(column) && !range.includes(column)) list.push(column); + } + return [...eq, ...range]; +} + +/** "CREATE INDEX ON orders (customer_id, created_at)" */ +export function indexStatement(table: string, columns: string[]): string { + const quote = (name: string) => (/^[a-z_][a-z0-9_]*$/.test(name) ? name : `"${name.replace(/"/g, '""')}"`); + return `CREATE INDEX ON ${table.split('.').map(quote).join('.')} (${columns.map(quote).join(', ')})`; +} diff --git a/src/drivers.ts b/src/drivers.ts index 08c8e42..9f6380b 100644 --- a/src/drivers.ts +++ b/src/drivers.ts @@ -8,6 +8,7 @@ import type { OptimizedPerformanceMonitor, Span } from './core/optimized-performance-monitor'; import type { TrackOptions } from './types/options'; import type { Uninstall } from './instrument'; +import { loadNodeBuiltin } from './core/env'; const PATCHED = Symbol.for('traceperf.instrumented'); @@ -44,10 +45,12 @@ function trackCall( const cb = callbackIndex >= 0 ? args[callbackIndex] : undefined; if (typeof cb === 'function') { const wrappedArgs = args.slice(); - wrappedArgs[callbackIndex] = function (this: unknown, error: unknown, ...rest: unknown[]) { + // Bound: drivers call it from whatever request's I/O completed it (queued + // queries on one connection, pools), and work done in it belongs to the caller + wrappedArgs[callbackIndex] = bindContext(function (this: unknown, error: unknown, ...rest: unknown[]) { end(error || undefined); return (cb as Function).call(this, error, ...rest); - }; + }); try { return span.run(() => original.apply(thisArg, wrappedArgs)); } catch (error) { @@ -85,6 +88,66 @@ function trackCall( return result; } +/** A SQL statement as the app sent it, with a way to run another statement on the same database */ +export interface ObservedSql { + driver: 'pg'; + text: string; + values?: unknown[]; + /** Runs a statement on the pool the app used (or the client, when it has no pool) */ + run(text: string, values?: unknown[]): Promise; +} + +const OWN_SQL = '/* traceperf */ '; +let sqlObserver: ((q: ObservedSql) => void) | undefined; +let lastPgPool: any; + +/** @internal traceperf run's index check: sees each SQL statement (one observer at a time) */ +export function observeSql(observer: ((q: ObservedSql) => void) | undefined): void { + sqlObserver = observer; +} + +const AsyncResource: any = (loadNodeBuiltin('async_hooks') || {}).AsyncResource; + +/** + * Pools keep a queue of callbacks and call them when another request releases + * a connection, i.e. in that other request's async context: without binding, + * the queries made in the callback were attributed to the wrong request (or to + * none) whenever the pool was full. + */ +function bindContext(fn: F): F { + return typeof fn === 'function' && AsyncResource && typeof AsyncResource.bind === 'function' ? AsyncResource.bind(fn) : fn; +} + +/** Bind a trailing callback argument to the caller's async context */ +function bindLastCallback(args: unknown[]): unknown[] { + const last = args.length - 1; + if (last < 0 || typeof args[last] !== 'function') return args; + const copy = args.slice(); + copy[last] = bindContext(copy[last]); + return copy; +} + +/** + * Pool.connect / getConnection: callbacks keep the caller's context, and when + * every connection is in use the wait is tracked as " pool wait" + * (traceperf run reports requests that queued for a connection). + */ +function patchPoolAcquire( + tp: OptimizedPerformanceMonitor, proto: any, method: string, label: string, isFull: (pool: any) => boolean, options: DriverOptions, restores: Uninstall[] +): void { + patchMethod(proto, method, original => function (this: any, ...args: unknown[]) { + const bound = bindLastCallback(args); + let full = false; + try { + full = isFull(this); + } catch { + full = false; + } + if (!full) return original.apply(this, bound); + return trackCall(tp, label, options, this, original, bound, typeof bound[bound.length - 1] === 'function' ? bound.length - 1 : -1); + }, restores); +} + function patchMethod(target: any, name: string, makePatched: (original: Function) => Function, restores: Uninstall[]): void { if (!target) return; const original = target[name]; @@ -124,9 +187,36 @@ export function sqlLabel(sql: unknown): string { */ export function instrumentPg(tp: OptimizedPerformanceMonitor, pg: any, options: DriverOptions = {}): Uninstall { const restores: Uninstall[] = []; - patchMethod(pg && pg.Client && pg.Client.prototype, 'query', original => function (this: unknown, ...args: unknown[]) { + patchMethod(pg && pg.Client && pg.Client.prototype, 'query', original => function (this: any, ...args: unknown[]) { + // traceperf's own statements (the index check) are not the app's + if (typeof args[0] === 'string' && args[0].startsWith(OWN_SQL)) return original.apply(this, args); + if (sqlObserver) { + const first = args[0] as any; + const text = typeof first === 'string' ? first : first && typeof first.text === 'string' ? first.text : undefined; + if (text !== undefined) { + const values = Array.isArray(args[1]) ? args[1] : first && Array.isArray(first.values) ? first.values : undefined; + const pool = lastPgPool; + try { + sqlObserver({ + driver: 'pg', text, values, + run: (sql, vals) => (pool && !pool.ending && !pool.ended ? pool.query(OWN_SQL + sql, vals) : this.query(OWN_SQL + sql, vals)).then((r: any) => r.rows), + }); + } catch { + // observing must never break the query + } + } + } return trackCall(tp, `pg ${sqlLabel(args[0])}`, options, this, original, args); }, restores); + patchMethod(pg && pg.Pool && pg.Pool.prototype, 'query', original => function (this: unknown, ...args: unknown[]) { + // The index check runs its EXPLAINs on the app's own pool: remember the last one used + // eslint-disable-next-line @typescript-eslint/no-this-alias + lastPgPool = this; + return original.apply(this, args); + }, restores); + // pg-pool: totalCount / idleCount / options.max + patchPoolAcquire(tp, pg && pg.Pool && pg.Pool.prototype, 'connect', 'pg pool wait', + pool => pool.totalCount >= pool.options.max && pool.idleCount === 0, options, restores); return () => restores.forEach(r => r()); } @@ -143,6 +233,9 @@ export function instrumentMysql2(tp: OptimizedPerformanceMonitor, mysql2: any, o return trackCall(tp, `mysql ${sqlLabel(args[0])}`, options, this, original, args); }, restores); } + // pool.query and pool.execute get a connection through getConnection(cb) + patchPoolAcquire(tp, mysql2 && mysql2.Pool && mysql2.Pool.prototype, 'getConnection', 'mysql pool wait', + pool => pool._allConnections.length >= pool.config.connectionLimit && pool._freeConnections.length === 0, options, restores); return () => restores.forEach(r => r()); } @@ -180,6 +273,10 @@ export function instrumentMysql(tp: OptimizedPerformanceMonitor, mysql: any, opt if (result && typeof result === 'object') result[POOLED] = true; return result; }, restores); + // pool.getConnection(cb): the callback runs when another request releases a connection + patchMethod(poolProto, 'getConnection', original => function (this: unknown, ...args: unknown[]) { + return original.apply(this, bindLastCallback(args)); + }, restores); return () => restores.forEach(r => r()); } diff --git a/src/register.ts b/src/register.ts index a5e0c9f..d2140d4 100644 --- a/src/register.ts +++ b/src/register.ts @@ -30,6 +30,8 @@ import { blockingRanges, explain, formatFindings } from './core/explain'; import { BlockingProfileEntry, blockingHotspots, profileSlowCalls, SlowCallProfiler } from './core/cpu-profile'; import { jsonLinesExporter } from './core/exporters'; import { loadNodeBuiltin } from './core/env'; +import { observeSql, ObservedSql } from './drivers'; +import { seqScans } from './core/sql-plan'; import type { ExecutionRecord } from './core/records'; const env: Record = (typeof process !== 'undefined' && process.env) || {}; @@ -97,6 +99,55 @@ if (flag(env.TRACEPERF_ENABLED, true) && !g[AUTO_KEY]) { // reporting must never break the app } }; + // traceperf run's index check: remember each distinct SELECT; when run asks + // (a request file, after the load), EXPLAIN them (planned, never executed) + const explainOutput = env.TRACEPERF_EXPLAIN_OUTPUT; + if (explainOutput && fs) { + const seen = new Map(); + let explaining = false; + observeSql(q => { + if (explaining || !/^\s*select\b/i.test(q.text)) return; + const entry = seen.get(q.text); + if (entry) entry.count++; + else if (seen.size < 500) seen.set(q.text, { count: 1, sample: q }); + }); + const timer = setInterval(() => { + if (!fs.existsSync(`${explainOutput}.request`)) return; + clearInterval(timer); + explaining = true; + const top = [...seen.entries()].sort((a, b) => b[1].count - a[1].count).slice(0, 30); + const lines: string[] = []; + const sizes = new Map(); + (async () => { + for (const [text, { count, sample }] of top) { + try { + const rows = await sample.run(`EXPLAIN (FORMAT JSON) ${text}`, sample.values); + const plan = rows[0] && (rows[0]['QUERY PLAN'] ?? Object.values(rows[0])[0]); + const scans = seqScans(typeof plan === 'string' ? JSON.parse(plan) : plan); + for (const scan of scans) { + if (!sizes.has(scan.table)) { + const r = await sample.run('SELECT reltuples::bigint AS n FROM pg_class WHERE oid = to_regclass($1)', [scan.table]).catch(() => []); + sizes.set(scan.table, Number(r[0] && r[0].n) || 0); + } + } + if (scans.length) lines.push(JSON.stringify({ driver: sample.driver, text, count, scans: scans.map(x => ({ ...x, tableRows: sizes.get(x.table) || 0 })) })); + } catch { + // a statement EXPLAIN can't take (parameters it can't infer, permissions): skip it + } + } + })().finally(() => { + try { + fs.writeFileSync(explainOutput, lines.join('\n') + (lines.length ? '\n' : '')); + fs.writeFileSync(`${explainOutput}.done`, '1'); + } catch { + // reporting must never break the app + } + explaining = false; + }); + }, 100); + if (typeof timer.unref === 'function') timer.unref(); + } + const perf = (globalThis as any).performance; if (loadOutput && perf && typeof perf.eventLoopUtilization === 'function' && typeof process.cpuUsage === 'function') { let lastElu = perf.eventLoopUtilization(); diff --git a/src/run.ts b/src/run.ts index f22ef47..1133c72 100644 --- a/src/run.ts +++ b/src/run.ts @@ -9,6 +9,7 @@ * Node-only; builtins are passed in so this module stays free of require(). */ import { explain, formatFindings, Finding } from './core/explain'; +import { indexStatement, SeqScan } from './core/sql-plan'; import { blockingHotspots, BlockingProfileEntry } from './core/cpu-profile'; import { normalizePath } from './instrument'; @@ -47,6 +48,10 @@ export interface RunArgs { profile: boolean; /** Ms to wait for the app to listen before giving up */ startTimeoutMs: number; + /** EXPLAIN the app's SELECTs after the load and report full scans an index would avoid (Postgres; default on) */ + explain: boolean; + /** Tables smaller than this are fine to scan (default 10000 rows) */ + indexMinRows: number; /** From the config file: log in first and send the token with every request */ login?: RunLogin; } @@ -75,6 +80,8 @@ export const RUN_USAGE = ` traceperf run [app args] [options] --config f.json read these options from a file (default: traceperf.config.json if present; command-line options win). It can also log in first: "login": {"path": "/api/login", "body": {...}, "token": "user.token"} + --no-explain don't EXPLAIN the app's Postgres SELECTs after the load + (it plans them, never runs them, to find missing indexes) --no-profile don't sample the CPU (blocking code is then named only by its tracked call)`; @@ -194,7 +201,7 @@ export function parseRunArgs(argv: string[], readConfig?: (file?: string) => str function parseArgv(argv: string[], partial = false, start?: RunArgs): RunArgs | string { const args: RunArgs = start ? { ...start } : { command: [], paths: [], concurrency: 10, durationMs: 5000, host: '127.0.0.1', - method: 'GET', warmup: 3, probe: 10, json: false, profile: true, max: 10, startTimeoutMs: 15000, + method: 'GET', warmup: 3, probe: 10, json: false, profile: true, max: 10, startTimeoutMs: 15000, explain: true, indexMinRows: 10000, }; const num = (name: string, v: string | undefined): number | string => { const n = Number(v); @@ -259,6 +266,7 @@ function parseArgv(argv: string[], partial = false, start?: RunArgs): RunArgs | case '--save-baseline': if (!(args.saveBaseline = argv[++i])) return '--save-baseline expects a file'; break; case '--baseline': if (!(args.baseline = argv[++i])) return '--baseline expects a file'; break; case '--no-profile': args.profile = false; break; + case '--no-explain': args.explain = false; break; case '--probe': { const n = Number(argv[++i]); if (!(n >= 0)) return '--probe expects a number'; @@ -631,6 +639,7 @@ export async function runApp(args: RunArgs, deps: RunDeps): Promise { const readyFile = path.join(dir, 'port'); const profileFile = path.join(dir, 'profiles.jsonl'); const loadFile = path.join(dir, 'load.jsonl'); + const explainFile = path.join(dir, 'explain.jsonl'); let loadWindow: { start: number; end: number } | undefined; if (args.output) fs.writeFileSync(output, ''); const env: Record = { @@ -641,6 +650,7 @@ export async function runApp(args: RunArgs, deps: RunDeps): Promise { TRACEPERF_READY_FILE: readyFile, ...(args.profile ? { TRACEPERF_PROFILE_BLOCKING: '1', TRACEPERF_PROFILE_OUTPUT: profileFile } : {}), TRACEPERF_LOAD_OUTPUT: loadFile, + ...(args.explain ? { TRACEPERF_EXPLAIN_OUTPUT: explainFile } : {}), } as Record; const [cmd, ...cmdArgs] = args.command; const quoted = JSON.stringify(deps.registerPath); @@ -754,8 +764,13 @@ export async function runApp(args: RunArgs, deps: RunDeps): Promise { const loadStart = Date.now(); load = await sendLoad(deps.http, args, port); loadWindow = { start: loadStart, end: Date.now() }; - // One more load sample covers the last second + // One more load sample covers the last second; meanwhile the app EXPLAINs its queries + if (args.explain) fs.writeFileSync(`${explainFile}.request`, '1'); await new Promise(r => setTimeout(r, 1100)); + if (args.explain) { + const until = Date.now() + 5000; + while (!fs.existsSync(`${explainFile}.done`) && Date.now() < until && !exited) await new Promise(r => setTimeout(r, 100)); + } await stopAndWait(); } else { deps.err('traceperf: app running; use it, then press Ctrl-C for the report'); @@ -839,6 +854,15 @@ export async function runApp(args: RunArgs, deps: RunDeps): Promise { } queued = []; } + if (args.explain) { + let plans: ExplainedQuery[] = []; + try { + plans = fs.readFileSync(explainFile, 'utf8').split('\n').filter((l: string) => l.trim()).map((l: string) => JSON.parse(l)); + } catch { + plans = []; + } + summary.findings.push(...indexFindings(plans, args.indexMinRows)); + } // After blocking findings, which explain them const after = summary.findings.filter(f => f.kind === 'error' || f.kind === 'blocking').length; summary.findings.splice(after, 0, ...queued); @@ -894,6 +918,42 @@ export async function runApp(args: RunArgs, deps: RunDeps): Promise { return !summary.trees && exitCode ? 1 : 0; } +/** One EXPLAINed statement from the app (TRACEPERF_EXPLAIN_OUTPUT) */ +export interface ExplainedQuery { + driver: string; + text: string; + count: number; + scans: Array; +} + +/** + * Scans of whole tables that an index would avoid: a sequential scan of a + * table with at least `minRows` rows (planner estimate) whose filter keeps + * under 10% of them. Small tables are scanned on purpose, so test databases + * with little data rarely show one: run against realistic data. + */ +export function indexFindings(plans: ExplainedQuery[], minRows = 10000): Finding[] { + const out: Finding[] = []; + const said = new Set(); + for (const p of plans.slice().sort((a, b) => b.count - a.count)) { + for (const scan of p.scans) { + if (scan.tableRows < minRows || !scan.columns.length || scan.rows > 0.1 * scan.tableRows) continue; + const statement = indexStatement(scan.table, scan.columns); + if (said.has(statement)) continue; + said.add(statement); + const sql = p.text.replace(/\s+/g, ' ').trim(); + out.push({ + kind: 'index', + root: p.text, + ms: 0, + message: `${sql.length > 120 ? `${sql.slice(0, 117)}...` : sql} (run ${p.count} time${p.count === 1 ? '' : 's'}) reads all ~${scan.tableRows.toLocaleString('en-US')} rows of ${scan.table} to return ~${scan.rows} (Seq Scan, Filter: ${scan.filter})`, + fix: `add an index: ${statement} (from EXPLAIN on this database: check it on production-sized data)`, + }); + } + } + return out; +} + /** A short Markdown summary for a pull-request comment or a CI job summary */ export function markdownReport(r: { load?: LoadResult; findings: Finding[]; trees: number; diff --git a/test/compat/compat.js b/test/compat/compat.js index 3e1037e..10e8222 100644 --- a/test/compat/compat.js +++ b/test/compat/compat.js @@ -40,6 +40,29 @@ async function trace(instrument, body) { const cb = fn => new Promise((resolve, reject) => fn((err, res) => (err ? reject(err) : resolve(res)))); const db = () => `traceperf_compat_${process.pid}`; + +/** Pool of 1, 4 requests at once: each request's query must stay in its own tree, with the wait tracked */ +async function contention(instrument, query, waitLabel) { + const roots = []; + const tp = createTracePerf({ silent: true, trackMemory: false, onComplete: r => roots.push(r) }); + const uninstall = instrument(tp); + try { + await Promise.all([1, 2, 3, 4].map(i => tp.track(async () => { + await query(i); + await query(i); + }, { label: `req${i}` }))); + } finally { + uninstall(); + } + const requests = roots.filter(r => r.label.startsWith('req')); + assert.strictEqual(requests.length, 4, `4 request trees, got ${roots.map(r => r.label).join(', ')}`); + for (const r of requests) { + const queries = r.children.filter(c => c.label !== waitLabel); + assert.strictEqual(queries.length, 2, `${r.label} has its 2 queries: ${r.children.map(c => c.label).join(', ')}`); + } + assert.ok(requests.some(r => r.children.some(c => c.label === waitLabel)), `some request waited (${waitLabel})`); +} + const pgCheck = pkg => async () => { const pg = require(pkg); const client = new pg.Client({ connectionString: env.pg }); @@ -56,6 +79,12 @@ const pgCheck = pkg => async () => { }); assert.deepStrictEqual(labels, ['pg CREATE users', 'pg INSERT users', 'pg SELECT users', 'pg SELECT pg_class', 'pg SELECT missing_table']); assert.ok(job.children[4].error, 'failed query is marked as an error'); + const one = new pg.Pool({ connectionString: env.pg, max: 1 }); + try { + await contention(tp => instrumentPg(tp, pg), () => one.query('SELECT pg_sleep(0.01)'), 'pg pool wait'); + } finally { + await one.end(); + } } finally { await client.end(); await pool.end(); @@ -77,6 +106,14 @@ const mysql2Check = pkg => async () => { }); assert.deepStrictEqual(labels, ['mysql CREATE users', 'mysql INSERT users', 'mysql SELECT users', 'mysql SELECT DUAL', 'mysql SELECT missing_table']); assert.ok(job.children[4].error, 'failed query is marked as an error'); + const one = mysql.createPool({ uri: env.mysql, connectionLimit: 1 }); + try { + await contention(tp => instrumentMysql2(tp, mysql), () => one.promise().query('SELECT SLEEP(0.01)'), 'mysql pool wait'); + // callback API too + await contention(tp => instrumentMysql2(tp, mysql), () => cb(done => one.query('SELECT SLEEP(0.01)', done)), 'mysql pool wait'); + } finally { + await one.promise().end(); + } } finally { conn.end(); await pool.promise().end();