Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion bench/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,8 @@ node capture.mjs --app actual --cases C12,C18,C20 --out results/pilot/capture
```

These commands validate local integration. Whole-run diagnostics include setup input,
recorded separately from scenario interactions. This is author-run integration evidence.
recorded separately from scenario interactions. The first overhead measurement is in
[actual-overhead-2026-11-10.md](../docs/benchmarks/actual-overhead-2026-11-10.md). This is author-run integration evidence.

## Environment and Outputs

Expand Down
81 changes: 81 additions & 0 deletions bench/bench.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -312,6 +312,56 @@ export async function drainObservers(page) {
});
}

/**
* The same measures as the whole-run ones, from the scenario steps alone, so the setup clicks that
* build the demo budget do not count. Fewer than 50 interactions, so the INP estimate is the worst.
*/
export function scenarioMetrics(interactions, loaf, longTasks) {
return {
interactionCount: interactions.length,
inp: interactions.length ? Math.max(...interactions.map((i) => i.duration)) : null,
totalInteractionMs: interactions.reduce((s, i) => s + i.duration, 0),
loafCount: loaf.length,
loafBlockingMs: loaf.reduce((s, f) => s + (f.blockingDuration || 0), 0),
longTaskCount: longTasks.length,
longTaskBlockingMs: longTasks.reduce((s, t) => s + Math.max(0, t.duration - 50), 0),
};
}

/**
* The scripts this run received, matched to the build's files, with the gzip size of each. The local
* server sends them uncompressed, so the gzip figure is what a compressing server would deliver for
* this route. A script that matches no build file is listed and left out of the totals.
*/
export function deliveredJs(app, config, delivered) {
const root = path.join(app.dir, app.dists[config]);
const seen = new Map();
const unmatched = [];
for (const d of delivered) {
let file = null;
try {
const u = new URL(d.url);
if (u.protocol.startsWith('http')) file = path.join(root, decodeURIComponent(u.pathname));
} catch {
/* not a URL */
}
if (!file || !file.startsWith(root + path.sep) || !fs.existsSync(file)) {
unmatched.push(d.url.slice(0, 200));
continue;
}
if (seen.has(file)) continue;
const bytes = fs.readFileSync(file);
seen.set(file, { path: path.relative(root, file), wireBytes: d.bytes, bytes: bytes.length, gzipBytes: zlib.gzipSync(bytes, { level: 9 }).length });
}
const files = [...seen.values()];
return {
files,
unmatched,
totalBytes: files.reduce((s, f) => s + f.bytes, 0),
totalGzipBytes: files.reduce((s, f) => s + f.gzipBytes, 0),
};
}

// caseId, when given, runs only that case's step: the accuracy capture takes one case per fresh run.
export async function runOnce({ browser, app, config, url, throttle, runIndex, warmup, schedule = null, caseId = null }) {
// The app's own deterministic reset (database back to its seed) before every run, so a run never
Expand Down Expand Up @@ -352,6 +402,33 @@ export async function runOnce({ browser, app, config, url, throttle, runIndex, w

const cdp = await context.newCDPSession(page);
await cdp.send('Emulation.setCPUThrottlingRate', { rate: throttle });
// The page throttle does not reach dedicated workers. Ask each worker too and keep the browser's
// answer, so a throttled run says whether its workers were slowed rather than assuming it.
const workerThrottle = [];
if (throttle !== 1) {
let id = 0;
const pending = new Map();
cdp.on('Target.receivedMessageFromTarget', (e) => {
const m = JSON.parse(e.message);
const entry = m.id && pending.get(m.id);
if (!entry) return;
pending.delete(m.id);
entry.applied = !m.error;
entry.error = m.error ? m.error.message : null;
});
cdp.on('Target.attachedToTarget', (e) => {
if (!e.targetInfo.type.includes('worker')) return;
const entry = { type: e.targetInfo.type, url: e.targetInfo.url.slice(0, 200), applied: null, error: null };
workerThrottle.push(entry);
pending.set(++id, entry);
const message = JSON.stringify({ id, method: 'Emulation.setCPUThrottlingRate', params: { rate: throttle } });
cdp.send('Target.sendMessageToTarget', { sessionId: e.sessionId, message }).catch((err) => {
entry.applied = false;
entry.error = String(err && err.message);
});
});
await cdp.send('Target.setAutoAttach', { autoAttach: true, waitForDebuggerOnStart: false, flatten: false });
}

const t0 = Date.now();
await page.goto(url, { waitUntil: 'load', timeout: 120000 });
Expand Down Expand Up @@ -388,6 +465,7 @@ export async function runOnce({ browser, app, config, url, throttle, runIndex, w
// Keep the original whole-run diagnostics, and identify setup input explicitly.
const scenarioInteractions = data.interactions.filter(inScenario);
const setupInteractions = data.interactions.filter((event) => !inScenario(event));
const scenario = scenarioMetrics(scenarioInteractions, data.loafEntries.filter(inScenario), data.longTaskEntries.filter(inScenario));
await cdp.send('Emulation.setCPUThrottlingRate', { rate: 1 });

if (config !== 'A' && !data.lib) {
Expand All @@ -409,6 +487,9 @@ export async function runOnce({ browser, app, config, url, throttle, runIndex, w
measurementStatus: app.measurementStatus ?? null,
scenarioInteractions,
setupInteractions,
scenario,
workerThrottle,
deliveredJs: deliveredJs(app, config, delivered),
readyMs,
readiness: readiness ?? null,
steps: stepResults,
Expand Down
39 changes: 39 additions & 0 deletions bench/measure.test.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
import test from 'node:test';
import assert from 'node:assert/strict';
import fs from 'node:fs';
import os from 'node:os';
import path from 'node:path';
import zlib from 'node:zlib';
import { scenarioMetrics, deliveredJs } from './bench.mjs';

test('scenario metrics count only what they are given, and the INP estimate is the worst', () => {
const m = scenarioMetrics(
[{ duration: 40 }, { duration: 120 }, { duration: 64 }],
[{ blockingDuration: 30 }, { blockingDuration: 0 }],
[{ duration: 80 }, { duration: 40 }],
);
assert.deepEqual(m, { interactionCount: 3, inp: 120, totalInteractionMs: 224, loafCount: 2, loafBlockingMs: 30, longTaskCount: 2, longTaskBlockingMs: 30 });
assert.equal(scenarioMetrics([], [], []).inp, null);
});

test('delivered JS matches build files once each and leaves out what is not in the build', () => {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'bench-js-'));
try {
fs.mkdirSync(path.join(dir, 'dist-a', 'static'), { recursive: true });
const body = 'console.log("x");'.repeat(50);
fs.writeFileSync(path.join(dir, 'dist-a', 'static', 'a.js'), body);
const app = { dir, dists: { A: 'dist-a' } };
const out = deliveredJs(app, 'A', [
{ url: 'http://127.0.0.1:5310/static/a.js', bytes: body.length },
{ url: 'http://127.0.0.1:5310/static/a.js?again', bytes: body.length },
{ url: 'blob:http://127.0.0.1:5310/abc', bytes: 10 },
{ url: 'http://127.0.0.1:5310/../outside.js', bytes: 10 },
]);
assert.equal(out.files.length, 1);
assert.equal(out.totalBytes, body.length);
assert.equal(out.totalGzipBytes, zlib.gzipSync(Buffer.from(body), { level: 9 }).length);
assert.deepEqual(out.unmatched, ['blob:http://127.0.0.1:5310/abc', 'http://127.0.0.1:5310/../outside.js']);
} finally {
fs.rmSync(dir, { recursive: true, force: true });
}
});
2 changes: 1 addition & 1 deletion bench/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
"scripts": {
"bench": "node bench.mjs",
"report": "node report.mjs",
"test": "node --test configs.test.mjs report.test.mjs schedule.test.mjs cases.test.mjs stats.test.mjs actual.test.mjs",
"test": "node --test configs.test.mjs report.test.mjs schedule.test.mjs cases.test.mjs stats.test.mjs actual.test.mjs measure.test.mjs",
"test:drain": "node --test drain.test.mjs"
},
"devDependencies": {
Expand Down
8 changes: 7 additions & 1 deletion bench/report.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,12 @@ const METRICS = [
{ key: 'loafBlocking', label: 'LoAF blocking (ms)', get: (r) => r.loafBlockingMs },
{ key: 'loafCount', label: 'LoAF count', get: (r) => r.loafCount },
{ key: 'ready', label: 'Load to ready (ms)', get: (r) => r.readyMs },
{ key: 'scenarioInp', label: 'Scenario INP estimate (ms)', get: (r) => r.scenario?.inp ?? null },
{ key: 'scenarioTotal', label: 'Scenario interaction time (ms)', get: (r) => r.scenario?.totalInteractionMs ?? null },
{ key: 'scenarioLoafBlocking', label: 'Scenario LoAF blocking (ms)', get: (r) => r.scenario?.loafBlockingMs ?? null },
{ key: 'scenarioLongTaskBlocking', label: 'Scenario long-task blocking (ms)', get: (r) => r.scenario?.longTaskBlockingMs ?? null },
{ key: 'longTaskBlocking', label: 'Long-task blocking (ms)', get: (r) => r.longTaskBlockingMs ?? null },
{ key: 'jsGzipKB', label: 'Delivered JS, gzip (KB)', get: (r) => (r.deliveredJs ? r.deliveredJs.totalGzipBytes / 1024 : null) },
{
key: 'heap',
label: 'JS heap used (MB)',
Expand Down Expand Up @@ -318,7 +324,7 @@ function main() {
if (!vals.A.length) continue;
const dB = bootstrapMedianDelta(paired(runs, app, throttle, m, 'A', 'B'));
const dC = bootstrapMedianDelta(paired(runs, app, throttle, m, 'A', 'C'));
const unit = m.key === 'heap' ? 'MB' : m.key.startsWith('loafCount') || m.key === 'captured' ? '' : 'ms';
const unit = m.key === 'heap' ? 'MB' : m.key === 'jsGzipKB' ? 'KB' : m.key.startsWith('loafCount') || m.key === 'captured' ? '' : 'ms';
p(
`| ${m.label} | ${fmt(median(vals.A))} | ${fmt(quantile(vals.A, 0.9))} | ${fmt(median(vals.B))} | ${fmt(quantile(vals.B, 0.9))} | ${fmt(median(vals.C))} | ${fmt(quantile(vals.C, 0.9))} | ${deltaCell(dB, unit)} | ${deltaCell(dC, unit)} |`,
);
Expand Down
4 changes: 3 additions & 1 deletion docs/benchmarks/README.md
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
# Benchmarks

The current benchmark cohort is [five everyday apps](real-apps.md).
The current benchmark cohort is [five everyday apps](real-apps.md). The first overhead
measurement is for [Actual](actual-overhead-2026-11-10.md).
Results are added here as each app's runs finish.

- [Current apps and integration status](real-apps.md)
- [Actual overhead, 2026-11-10](actual-overhead-2026-11-10.md)
- [Harness setup and dry runs](../../bench/README.md)
- [Accuracy and overhead protocol, 2026-10-14](protocol-2026-10-14.md)
- [Selection-method amendment, 2026-10-15](amendment-2026-10-15.md)
Expand Down
82 changes: 82 additions & 0 deletions docs/benchmarks/actual-overhead-2026-11-10.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
# Actual overhead, 2026-11-10

One author-run measurement of react-inp-blame 1.0.0 in [Actual](https://github.com/actualbudget/actual),
following the overhead section of the [protocol](protocol-2026-10-14.md). Actual's maintainers
had no part in it.

## Setup

- Actual at `6fa81d2f49b7562ec813dca72f2ebdb5ed3dc91e`. The served A/B/C builds are the functional
pilot's builds file for file. Build manifest SHA-256:
A `c6a1ea7200e9b26c30a9f857073115852e2128ec50342de7504dfe7ed879e22d`,
B `d21c8133312b3a3a68eace787872d9601bd7ec0d803a9b385f8847cb766cf401`,
C `56d7874031329d65cf19d583d1e52baba8be08eea18c09fe33598115bd12758d`.
- react-inp-blame 1.0.0, integrity
`sha512-wEQcaqyjZaZ6rnIC7vqkK3eSX4Zl9Y1wBlIFjg1LlfwcsLmsU5li0gBtzxlXhEwiSFnRSU/arBduorTMltkgJg==`. The installed copy matched the npm
tarball when those builds were made.
- Harness at `40cbd17`. Command: `node bench.mjs --app actual --blocks 5`.
- Node 24.22.2, Playwright 1.59.1, Chromium 156.0.8078.4, Ubuntu 26.04, AMD EPYC 9354P with 8
logical CPUs and 31 GiB.
- Five balanced blocks of all six A/B/C orders at 1x and 4x CPU, one discarded warmup per
configuration, a fresh browser context and demo budget reset for every run. Every completed run asserted
all 20 [cases](../../bench/adapters/actual.cases.json) changed the intended state.
- 180 runs planned, 178 completed, no page or observer errors. Raw results SHA-256
`60e1a5532e6ecabd44256d4b249870dbeab7dfa4be3dc45c44d3c0a498297aae`.
- The machine was shared. The 1-minute load average was 3.3 to 16.1 (median 5.6) during the 1x
pass and 1.4 to 10.4 (median 3.8) during the 4x pass. Pairing by block cancels most of that
drift, but the absolute medians move with it.

## Results

Each cell is the median paired difference against A with its 95% bootstrap interval (seed 359,
10,000 resamples). An interval that contains zero is inconclusive.
Scenario rows count only the 20 cases, not the setup clicks that build the budget. Times are in
ms.

### 1x CPU, 30 pairs

| Metric | A median | B − A | C − A |
| --- | ---: | --- | --- |
| Scenario INP estimate | 164.0 | +4.0 [-8.0, 16.0] | 0.0 [-8.0, 16.0] |
| Scenario interaction time | 2256.0 | +4.0 [-96.0, 52.0] | **+56.0 [36.0, 144.0]** |
| Scenario long-task blocking | 322.5 | -2.5 [-40.5, 39.5] | +25.5 [-13.5, 40.5] |
| Scenario LoAF blocking | 336.2 | -6.2 [-65.5, 35.7] | +6.0 [-50.8, 49.2] |
| Load to ready | 4910.5 | +9.0 [-455.0, 564.0] | -17.0 [-471.5, 397.5] |
| Whole-run INP | 136.0 | -8.0 [-12.0, 4.0] | 0.0 [-8.0, 8.0] |
| Whole-run interaction time | 3304.0 | +20.0 [-84.0, 96.0] | **+92.0 [20.0, 156.0]** |
| Delivered JS, gzip | 2090.1 KB | **+89.1 KB** | **+97.0 KB** |

### 4x CPU, 29 pairs for B and 28 for C

| Metric | A median | B − A | C − A |
| --- | ---: | --- | --- |
| Scenario INP estimate | 792.0 | +24.0 [-32.0, 64.0] | **+48.0 [4.0, 72.0]** |
| Scenario interaction time | 10864.0 | **+184.0 [32.0, 672.0]** | +500.0 [-240.0, 708.0] |
| Scenario long-task blocking | 8583.0 | **+232.0 [57.0, 622.0]** | +456.0 [-344.0, 687.0] |
| Scenario LoAF blocking | 8685.1 | **+208.3 [64.3, 628.2]** | +378.0 [-456.0, 667.3] |
| Load to ready | 8150.0 | +18.0 [-273.0, 707.0] | **+673.5 [324.0, 887.0]** |
| Whole-run INP | 704.0 | **+24.0 [8.0, 48.0]** | +32.0 [-4.0, 48.0] |
| Whole-run interaction time | 14592.0 | **+488.0 [224.0, 816.0]** | **+812.0 [152.0, 1120.0]** |
| Delivered JS, gzip | 2090.1 KB | **+89.1 KB** | **+97.0 KB** |

At 1x, B has no measurable timing cost in any row. C adds 56 ms of scenario interaction time over
the 20 cases, about 2.5%. At 4x, B adds 184 ms of scenario interaction time (1.7%) and C adds 48
ms to the scenario INP estimate (6%) and 674 ms to load to ready (8%). The library adds 89 KB of
gzipped JS, 97 KB with the overlay, about 4.5% of what Actual already sends.

The library's own `stats()` put its tree walks at a median of 299 ms per run for B at 1x and
1215 ms at 4x, counted over the whole run including setup.

## Run notes

- 4x slows the page's main thread only. For each of Actual's three workers Chromium 156 answers
"Operation is only supported for pages, not workers", and every 4x run records it. Actual's
backend runs in a worker, so it ran at full speed at 4x in all three configurations. The library
runs on the main thread.
- Two 4x runs failed because the observer drain did not settle within 2000 ms: A in block 1 (BAC)
and C in block 2 (CAB). They are kept as failures and were not rerun, so 4x is below the
protocol's 30 pairs.
- The library bytes are checked through the builds. The installed copy had been removed from the
Actual checkout before this run, so the run's own library check reports it missing.
- JS heap is one unforced sample at the end of a run and is not a memory result.
- One app and one build per configuration.
5 changes: 3 additions & 2 deletions docs/benchmarks/real-apps.md
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
# Real-app benchmarks

The current cohort contains exactly five everyday open-source apps. The machine-readable
registry is [bench/apps.mjs](../../bench/apps.mjs).
registry is [bench/apps.mjs](../../bench/apps.mjs). Actual has an
[overhead measurement](actual-overhead-2026-11-10.md).
Results are added here as each app's runs finish.

| ID | App | Purpose | Integration work to verify |
Expand All @@ -12,7 +13,7 @@ Results are added here as each app's runs finish.
| `manage-my-damn-life-nextjs` | [Manage My Damn Life](https://github.com/intri-in/manage-my-damn-life-nextjs) | Productivity and finance | Next.js; verify database setup through Sequelize and deterministic reset |
| `reactive-resume` | [Reactive Resume](https://github.com/amruthpillai/reactive-resume) | Resume building | Verify current React/framework versions, production build and PostgreSQL requirements from upstream source |

Actual has a verified author-run functional integration. The table lists the setup work for each app.
Actual has a verified author-run functional integration and an overhead measurement. The table lists the setup work for each app.

Selection came from a category shortlist, not a random sample. The departure from the original
seeded selection is recorded in the [selection amendment](amendment-2026-10-15.md).
Expand Down
Loading