diff --git a/bench/README.md b/bench/README.md index 189b4081..4b2ffb88 100644 --- a/bench/README.md +++ b/bench/README.md @@ -63,7 +63,8 @@ node capture.mjs --app actual --cases C12,C18,C20 --out results/pilot/capture ``` These commands validate local integration. Whole-run diagnostics include setup input, -recorded separately from scenario interactions. This is author-run integration evidence. +recorded separately from scenario interactions. The first overhead measurement is in +[actual-overhead-2026-11-10.md](../docs/benchmarks/actual-overhead-2026-11-10.md). This is author-run integration evidence. ## Environment and Outputs diff --git a/bench/bench.mjs b/bench/bench.mjs index 17982735..6ef2b09d 100644 --- a/bench/bench.mjs +++ b/bench/bench.mjs @@ -312,6 +312,56 @@ export async function drainObservers(page) { }); } +/** + * The same measures as the whole-run ones, from the scenario steps alone, so the setup clicks that + * build the demo budget do not count. Fewer than 50 interactions, so the INP estimate is the worst. + */ +export function scenarioMetrics(interactions, loaf, longTasks) { + return { + interactionCount: interactions.length, + inp: interactions.length ? Math.max(...interactions.map((i) => i.duration)) : null, + totalInteractionMs: interactions.reduce((s, i) => s + i.duration, 0), + loafCount: loaf.length, + loafBlockingMs: loaf.reduce((s, f) => s + (f.blockingDuration || 0), 0), + longTaskCount: longTasks.length, + longTaskBlockingMs: longTasks.reduce((s, t) => s + Math.max(0, t.duration - 50), 0), + }; +} + +/** + * The scripts this run received, matched to the build's files, with the gzip size of each. The local + * server sends them uncompressed, so the gzip figure is what a compressing server would deliver for + * this route. A script that matches no build file is listed and left out of the totals. + */ +export function deliveredJs(app, config, delivered) { + const root = path.join(app.dir, app.dists[config]); + const seen = new Map(); + const unmatched = []; + for (const d of delivered) { + let file = null; + try { + const u = new URL(d.url); + if (u.protocol.startsWith('http')) file = path.join(root, decodeURIComponent(u.pathname)); + } catch { + /* not a URL */ + } + if (!file || !file.startsWith(root + path.sep) || !fs.existsSync(file)) { + unmatched.push(d.url.slice(0, 200)); + continue; + } + if (seen.has(file)) continue; + const bytes = fs.readFileSync(file); + seen.set(file, { path: path.relative(root, file), wireBytes: d.bytes, bytes: bytes.length, gzipBytes: zlib.gzipSync(bytes, { level: 9 }).length }); + } + const files = [...seen.values()]; + return { + files, + unmatched, + totalBytes: files.reduce((s, f) => s + f.bytes, 0), + totalGzipBytes: files.reduce((s, f) => s + f.gzipBytes, 0), + }; +} + // caseId, when given, runs only that case's step: the accuracy capture takes one case per fresh run. export async function runOnce({ browser, app, config, url, throttle, runIndex, warmup, schedule = null, caseId = null }) { // The app's own deterministic reset (database back to its seed) before every run, so a run never @@ -352,6 +402,33 @@ export async function runOnce({ browser, app, config, url, throttle, runIndex, w const cdp = await context.newCDPSession(page); await cdp.send('Emulation.setCPUThrottlingRate', { rate: throttle }); + // The page throttle does not reach dedicated workers. Ask each worker too and keep the browser's + // answer, so a throttled run says whether its workers were slowed rather than assuming it. + const workerThrottle = []; + if (throttle !== 1) { + let id = 0; + const pending = new Map(); + cdp.on('Target.receivedMessageFromTarget', (e) => { + const m = JSON.parse(e.message); + const entry = m.id && pending.get(m.id); + if (!entry) return; + pending.delete(m.id); + entry.applied = !m.error; + entry.error = m.error ? m.error.message : null; + }); + cdp.on('Target.attachedToTarget', (e) => { + if (!e.targetInfo.type.includes('worker')) return; + const entry = { type: e.targetInfo.type, url: e.targetInfo.url.slice(0, 200), applied: null, error: null }; + workerThrottle.push(entry); + pending.set(++id, entry); + const message = JSON.stringify({ id, method: 'Emulation.setCPUThrottlingRate', params: { rate: throttle } }); + cdp.send('Target.sendMessageToTarget', { sessionId: e.sessionId, message }).catch((err) => { + entry.applied = false; + entry.error = String(err && err.message); + }); + }); + await cdp.send('Target.setAutoAttach', { autoAttach: true, waitForDebuggerOnStart: false, flatten: false }); + } const t0 = Date.now(); await page.goto(url, { waitUntil: 'load', timeout: 120000 }); @@ -388,6 +465,7 @@ export async function runOnce({ browser, app, config, url, throttle, runIndex, w // Keep the original whole-run diagnostics, and identify setup input explicitly. const scenarioInteractions = data.interactions.filter(inScenario); const setupInteractions = data.interactions.filter((event) => !inScenario(event)); + const scenario = scenarioMetrics(scenarioInteractions, data.loafEntries.filter(inScenario), data.longTaskEntries.filter(inScenario)); await cdp.send('Emulation.setCPUThrottlingRate', { rate: 1 }); if (config !== 'A' && !data.lib) { @@ -409,6 +487,9 @@ export async function runOnce({ browser, app, config, url, throttle, runIndex, w measurementStatus: app.measurementStatus ?? null, scenarioInteractions, setupInteractions, + scenario, + workerThrottle, + deliveredJs: deliveredJs(app, config, delivered), readyMs, readiness: readiness ?? null, steps: stepResults, diff --git a/bench/measure.test.mjs b/bench/measure.test.mjs new file mode 100644 index 00000000..7bd7c16a --- /dev/null +++ b/bench/measure.test.mjs @@ -0,0 +1,39 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import zlib from 'node:zlib'; +import { scenarioMetrics, deliveredJs } from './bench.mjs'; + +test('scenario metrics count only what they are given, and the INP estimate is the worst', () => { + const m = scenarioMetrics( + [{ duration: 40 }, { duration: 120 }, { duration: 64 }], + [{ blockingDuration: 30 }, { blockingDuration: 0 }], + [{ duration: 80 }, { duration: 40 }], + ); + assert.deepEqual(m, { interactionCount: 3, inp: 120, totalInteractionMs: 224, loafCount: 2, loafBlockingMs: 30, longTaskCount: 2, longTaskBlockingMs: 30 }); + assert.equal(scenarioMetrics([], [], []).inp, null); +}); + +test('delivered JS matches build files once each and leaves out what is not in the build', () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'bench-js-')); + try { + fs.mkdirSync(path.join(dir, 'dist-a', 'static'), { recursive: true }); + const body = 'console.log("x");'.repeat(50); + fs.writeFileSync(path.join(dir, 'dist-a', 'static', 'a.js'), body); + const app = { dir, dists: { A: 'dist-a' } }; + const out = deliveredJs(app, 'A', [ + { url: 'http://127.0.0.1:5310/static/a.js', bytes: body.length }, + { url: 'http://127.0.0.1:5310/static/a.js?again', bytes: body.length }, + { url: 'blob:http://127.0.0.1:5310/abc', bytes: 10 }, + { url: 'http://127.0.0.1:5310/../outside.js', bytes: 10 }, + ]); + assert.equal(out.files.length, 1); + assert.equal(out.totalBytes, body.length); + assert.equal(out.totalGzipBytes, zlib.gzipSync(Buffer.from(body), { level: 9 }).length); + assert.deepEqual(out.unmatched, ['blob:http://127.0.0.1:5310/abc', 'http://127.0.0.1:5310/../outside.js']); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); diff --git a/bench/package.json b/bench/package.json index d66485c9..db068366 100644 --- a/bench/package.json +++ b/bench/package.json @@ -5,7 +5,7 @@ "scripts": { "bench": "node bench.mjs", "report": "node report.mjs", - "test": "node --test configs.test.mjs report.test.mjs schedule.test.mjs cases.test.mjs stats.test.mjs actual.test.mjs", + "test": "node --test configs.test.mjs report.test.mjs schedule.test.mjs cases.test.mjs stats.test.mjs actual.test.mjs measure.test.mjs", "test:drain": "node --test drain.test.mjs" }, "devDependencies": { diff --git a/bench/report.mjs b/bench/report.mjs index 395c506b..75989f2d 100644 --- a/bench/report.mjs +++ b/bench/report.mjs @@ -105,6 +105,12 @@ const METRICS = [ { key: 'loafBlocking', label: 'LoAF blocking (ms)', get: (r) => r.loafBlockingMs }, { key: 'loafCount', label: 'LoAF count', get: (r) => r.loafCount }, { key: 'ready', label: 'Load to ready (ms)', get: (r) => r.readyMs }, + { key: 'scenarioInp', label: 'Scenario INP estimate (ms)', get: (r) => r.scenario?.inp ?? null }, + { key: 'scenarioTotal', label: 'Scenario interaction time (ms)', get: (r) => r.scenario?.totalInteractionMs ?? null }, + { key: 'scenarioLoafBlocking', label: 'Scenario LoAF blocking (ms)', get: (r) => r.scenario?.loafBlockingMs ?? null }, + { key: 'scenarioLongTaskBlocking', label: 'Scenario long-task blocking (ms)', get: (r) => r.scenario?.longTaskBlockingMs ?? null }, + { key: 'longTaskBlocking', label: 'Long-task blocking (ms)', get: (r) => r.longTaskBlockingMs ?? null }, + { key: 'jsGzipKB', label: 'Delivered JS, gzip (KB)', get: (r) => (r.deliveredJs ? r.deliveredJs.totalGzipBytes / 1024 : null) }, { key: 'heap', label: 'JS heap used (MB)', @@ -318,7 +324,7 @@ function main() { if (!vals.A.length) continue; const dB = bootstrapMedianDelta(paired(runs, app, throttle, m, 'A', 'B')); const dC = bootstrapMedianDelta(paired(runs, app, throttle, m, 'A', 'C')); - const unit = m.key === 'heap' ? 'MB' : m.key.startsWith('loafCount') || m.key === 'captured' ? '' : 'ms'; + const unit = m.key === 'heap' ? 'MB' : m.key === 'jsGzipKB' ? 'KB' : m.key.startsWith('loafCount') || m.key === 'captured' ? '' : 'ms'; p( `| ${m.label} | ${fmt(median(vals.A))} | ${fmt(quantile(vals.A, 0.9))} | ${fmt(median(vals.B))} | ${fmt(quantile(vals.B, 0.9))} | ${fmt(median(vals.C))} | ${fmt(quantile(vals.C, 0.9))} | ${deltaCell(dB, unit)} | ${deltaCell(dC, unit)} |`, ); diff --git a/docs/benchmarks/README.md b/docs/benchmarks/README.md index 2ab24e78..11b3aafb 100644 --- a/docs/benchmarks/README.md +++ b/docs/benchmarks/README.md @@ -1,9 +1,11 @@ # Benchmarks -The current benchmark cohort is [five everyday apps](real-apps.md). +The current benchmark cohort is [five everyday apps](real-apps.md). The first overhead +measurement is for [Actual](actual-overhead-2026-11-10.md). Results are added here as each app's runs finish. - [Current apps and integration status](real-apps.md) +- [Actual overhead, 2026-11-10](actual-overhead-2026-11-10.md) - [Harness setup and dry runs](../../bench/README.md) - [Accuracy and overhead protocol, 2026-10-14](protocol-2026-10-14.md) - [Selection-method amendment, 2026-10-15](amendment-2026-10-15.md) diff --git a/docs/benchmarks/actual-overhead-2026-11-10.md b/docs/benchmarks/actual-overhead-2026-11-10.md new file mode 100644 index 00000000..b036ac5a --- /dev/null +++ b/docs/benchmarks/actual-overhead-2026-11-10.md @@ -0,0 +1,82 @@ +# Actual overhead, 2026-11-10 + +One author-run measurement of react-inp-blame 1.0.0 in [Actual](https://github.com/actualbudget/actual), +following the overhead section of the [protocol](protocol-2026-10-14.md). Actual's maintainers +had no part in it. + +## Setup + +- Actual at `6fa81d2f49b7562ec813dca72f2ebdb5ed3dc91e`. The served A/B/C builds are the functional + pilot's builds file for file. Build manifest SHA-256: + A `c6a1ea7200e9b26c30a9f857073115852e2128ec50342de7504dfe7ed879e22d`, + B `d21c8133312b3a3a68eace787872d9601bd7ec0d803a9b385f8847cb766cf401`, + C `56d7874031329d65cf19d583d1e52baba8be08eea18c09fe33598115bd12758d`. +- react-inp-blame 1.0.0, integrity + `sha512-wEQcaqyjZaZ6rnIC7vqkK3eSX4Zl9Y1wBlIFjg1LlfwcsLmsU5li0gBtzxlXhEwiSFnRSU/arBduorTMltkgJg==`. The installed copy matched the npm + tarball when those builds were made. +- Harness at `40cbd17`. Command: `node bench.mjs --app actual --blocks 5`. +- Node 24.22.2, Playwright 1.59.1, Chromium 156.0.8078.4, Ubuntu 26.04, AMD EPYC 9354P with 8 + logical CPUs and 31 GiB. +- Five balanced blocks of all six A/B/C orders at 1x and 4x CPU, one discarded warmup per + configuration, a fresh browser context and demo budget reset for every run. Every completed run asserted + all 20 [cases](../../bench/adapters/actual.cases.json) changed the intended state. +- 180 runs planned, 178 completed, no page or observer errors. Raw results SHA-256 + `60e1a5532e6ecabd44256d4b249870dbeab7dfa4be3dc45c44d3c0a498297aae`. +- The machine was shared. The 1-minute load average was 3.3 to 16.1 (median 5.6) during the 1x + pass and 1.4 to 10.4 (median 3.8) during the 4x pass. Pairing by block cancels most of that + drift, but the absolute medians move with it. + +## Results + +Each cell is the median paired difference against A with its 95% bootstrap interval (seed 359, +10,000 resamples). An interval that contains zero is inconclusive. +Scenario rows count only the 20 cases, not the setup clicks that build the budget. Times are in +ms. + +### 1x CPU, 30 pairs + +| Metric | A median | B − A | C − A | +| --- | ---: | --- | --- | +| Scenario INP estimate | 164.0 | +4.0 [-8.0, 16.0] | 0.0 [-8.0, 16.0] | +| Scenario interaction time | 2256.0 | +4.0 [-96.0, 52.0] | **+56.0 [36.0, 144.0]** | +| Scenario long-task blocking | 322.5 | -2.5 [-40.5, 39.5] | +25.5 [-13.5, 40.5] | +| Scenario LoAF blocking | 336.2 | -6.2 [-65.5, 35.7] | +6.0 [-50.8, 49.2] | +| Load to ready | 4910.5 | +9.0 [-455.0, 564.0] | -17.0 [-471.5, 397.5] | +| Whole-run INP | 136.0 | -8.0 [-12.0, 4.0] | 0.0 [-8.0, 8.0] | +| Whole-run interaction time | 3304.0 | +20.0 [-84.0, 96.0] | **+92.0 [20.0, 156.0]** | +| Delivered JS, gzip | 2090.1 KB | **+89.1 KB** | **+97.0 KB** | + +### 4x CPU, 29 pairs for B and 28 for C + +| Metric | A median | B − A | C − A | +| --- | ---: | --- | --- | +| Scenario INP estimate | 792.0 | +24.0 [-32.0, 64.0] | **+48.0 [4.0, 72.0]** | +| Scenario interaction time | 10864.0 | **+184.0 [32.0, 672.0]** | +500.0 [-240.0, 708.0] | +| Scenario long-task blocking | 8583.0 | **+232.0 [57.0, 622.0]** | +456.0 [-344.0, 687.0] | +| Scenario LoAF blocking | 8685.1 | **+208.3 [64.3, 628.2]** | +378.0 [-456.0, 667.3] | +| Load to ready | 8150.0 | +18.0 [-273.0, 707.0] | **+673.5 [324.0, 887.0]** | +| Whole-run INP | 704.0 | **+24.0 [8.0, 48.0]** | +32.0 [-4.0, 48.0] | +| Whole-run interaction time | 14592.0 | **+488.0 [224.0, 816.0]** | **+812.0 [152.0, 1120.0]** | +| Delivered JS, gzip | 2090.1 KB | **+89.1 KB** | **+97.0 KB** | + +At 1x, B has no measurable timing cost in any row. C adds 56 ms of scenario interaction time over +the 20 cases, about 2.5%. At 4x, B adds 184 ms of scenario interaction time (1.7%) and C adds 48 +ms to the scenario INP estimate (6%) and 674 ms to load to ready (8%). The library adds 89 KB of +gzipped JS, 97 KB with the overlay, about 4.5% of what Actual already sends. + +The library's own `stats()` put its tree walks at a median of 299 ms per run for B at 1x and +1215 ms at 4x, counted over the whole run including setup. + +## Run notes + +- 4x slows the page's main thread only. For each of Actual's three workers Chromium 156 answers + "Operation is only supported for pages, not workers", and every 4x run records it. Actual's + backend runs in a worker, so it ran at full speed at 4x in all three configurations. The library + runs on the main thread. +- Two 4x runs failed because the observer drain did not settle within 2000 ms: A in block 1 (BAC) + and C in block 2 (CAB). They are kept as failures and were not rerun, so 4x is below the + protocol's 30 pairs. +- The library bytes are checked through the builds. The installed copy had been removed from the + Actual checkout before this run, so the run's own library check reports it missing. +- JS heap is one unforced sample at the end of a run and is not a memory result. +- One app and one build per configuration. diff --git a/docs/benchmarks/real-apps.md b/docs/benchmarks/real-apps.md index 4c3b94bd..70f9c219 100644 --- a/docs/benchmarks/real-apps.md +++ b/docs/benchmarks/real-apps.md @@ -1,7 +1,8 @@ # Real-app benchmarks The current cohort contains exactly five everyday open-source apps. The machine-readable -registry is [bench/apps.mjs](../../bench/apps.mjs). +registry is [bench/apps.mjs](../../bench/apps.mjs). Actual has an +[overhead measurement](actual-overhead-2026-11-10.md). Results are added here as each app's runs finish. | ID | App | Purpose | Integration work to verify | @@ -12,7 +13,7 @@ Results are added here as each app's runs finish. | `manage-my-damn-life-nextjs` | [Manage My Damn Life](https://github.com/intri-in/manage-my-damn-life-nextjs) | Productivity and finance | Next.js; verify database setup through Sequelize and deterministic reset | | `reactive-resume` | [Reactive Resume](https://github.com/amruthpillai/reactive-resume) | Resume building | Verify current React/framework versions, production build and PostgreSQL requirements from upstream source | -Actual has a verified author-run functional integration. The table lists the setup work for each app. +Actual has a verified author-run functional integration and an overhead measurement. The table lists the setup work for each app. Selection came from a category shortlist, not a random sample. The departure from the original seeded selection is recorded in the [selection amendment](amendment-2026-10-15.md).