From 02c85223178609d7de8c3cab281931f91e94aa08 Mon Sep 17 00:00:00 2001 From: aamir Date: Mon, 31 Aug 2026 21:05:05 -0500 Subject: [PATCH] fix(evals): keep large benchmark viewers bounded --- src/trace-foundry-viewer.ts | 42 ++++++++++++++++++++++++++++++++++-- src/trace-foundry.ts | 16 +++++++++----- tests/trace-foundry.test.mjs | 21 ++++++++++++++++++ 3 files changed, 72 insertions(+), 7 deletions(-) diff --git a/src/trace-foundry-viewer.ts b/src/trace-foundry-viewer.ts index 09209708..8f4907bc 100644 --- a/src/trace-foundry-viewer.ts +++ b/src/trace-foundry-viewer.ts @@ -1,10 +1,48 @@ type Obj = Record; +const MAX_INLINE_CAPTURE_BYTES = 16 * 1024 * 1024; +const MAX_INLINE_CAPTURE_COUNT = 250; + +function asObject(value: unknown): Obj { + return value !== null && typeof value === "object" && !Array.isArray(value) ? value as Obj : {}; +} + +function prepareViewerPayload(payload: Obj): { inline: boolean; payload: Obj } { + const captures = asObject(payload.captures); + const references: Obj = {}; + let captureCount = 0; + let captureBytes = 0; + let sizesKnown = true; + for (const key in captures) { + if (!Object.hasOwn(captures, key)) continue; + captureCount += 1; + const capture = asObject(captures[key]); + const bytes = capture.bytes; + if (typeof bytes !== "number" || !Number.isSafeInteger(bytes) || bytes < 0 || !Number.isSafeInteger(captureBytes + bytes)) { + sizesKnown = false; + } else { + captureBytes += bytes; + } + const { data: _data, ...reference } = capture; + references[key] = reference; + } + + const inline = sizesKnown && captureCount <= MAX_INLINE_CAPTURE_COUNT && captureBytes <= MAX_INLINE_CAPTURE_BYTES; + return { + inline, + payload: { ...payload, captures: inline ? captures : references }, + }; +} + export function traceFoundryViewer(payload: Obj): string { - const data = JSON.stringify(payload).replaceAll("Large benchmark: capture bodies load through the local server. Run understudy traces serve --benchmark <benchmark-directory>.'; return String.raw` Understudy · benchmark orchard -
benchmark orchard
Source DAG

+${serveNotice}
benchmark orchard
Source DAG

`; } diff --git a/src/trace-foundry.ts b/src/trace-foundry.ts index d01ae2c5..36364c86 100644 --- a/src/trace-foundry.ts +++ b/src/trace-foundry.ts @@ -695,7 +695,12 @@ function lineageAnalysis(rows: Obj[], requireProvableLineage: boolean): { summar const viewerHtml = (payload: Obj) => `Understudy · benchmark orchard
benchmark orchard

`; -function writeJson(path: string, value: unknown): void { mkdirSync(resolve(path, ".."), { recursive: true }); writeFileSync(path, `${JSON.stringify(value, null, 2)}\n`, { mode: 0o600 }); } +function writeJson(path: string, value: unknown): number { + const contents = `${JSON.stringify(value, null, 2)}\n`; + mkdirSync(resolve(path, ".."), { recursive: true }); + writeFileSync(path, contents, { mode: 0o600 }); + return Buffer.byteLength(contents); +} function finalizePrimeVerifierV021Package(pkg: string): void { const tasksetPath = join(pkg, "taskset.py"); @@ -2478,13 +2483,14 @@ function writeFoundryArtifacts(ctx: { source: string; output: string; files: str // RECOMPUTES this name from the pointer, so the two must never fork. const fileId = captureFileId({ capture_id: String(row.capture_id), sha256: String(row.source.sha256) }); const path = join(capturesDir, `${fileId}.json`); - writeJson(path, row); - // Inline the private normalized capture as well as writing the split data - // file. This keeps the viewer genuinely self-contained when opened via - // file://, where browsers block fetch() of sibling files. + const bytes = writeJson(path, row); + // Keep the normalized capture available to the viewer builder. Small + // benchmarks inline it for file:// use; large ones retain only this + // sidecar pointer and load through the existing local server. captureIndex[row.capture_key] = { path: `data/captures/${fileId}.json`, source: row.source, + bytes, data: row, }; } diff --git a/tests/trace-foundry.test.mjs b/tests/trace-foundry.test.mjs index bff7c554..95cb19f9 100644 --- a/tests/trace-foundry.test.mjs +++ b/tests/trace-foundry.test.mjs @@ -7,6 +7,7 @@ import { spawnSync } from "node:child_process"; import test from "node:test"; import { once } from "node:events"; import { compileTraceFoundry, createTraceReplayPlan, extractJsonPayload, importTraceReviews, requestSystemPrompt, runTraceReplays } from "../dist/trace-foundry.js"; +import { traceFoundryViewer } from "../dist/trace-foundry-viewer.js"; import { serveTraceFoundry } from "../dist/trace-foundry-server.js"; const capture = (id, ts, messages, response) => ({ @@ -15,6 +16,26 @@ const capture = (id, ts, messages, response) => ({ response_body: JSON.stringify(response), status_code: 200, }); +test("large benchmark viewers keep capture bodies out of the HTML payload", () => { + const overByteLimit = { + capture: { + path: "data/captures/capture.json", + bytes: Number.MAX_SAFE_INTEGER, + data: { private_marker: "must-not-be-inlined" }, + }, + }; + const overCountLimit = Object.fromEntries(Array.from({ length: 251 }, (_, index) => [ + `capture-${index}`, + { path: `data/captures/${index}.json`, bytes: 0, data: { private_marker: "must-not-be-inlined" } }, + ])); + + for (const captures of [overByteLimit, overCountLimit]) { + const viewer = traceFoundryViewer({ tasks: [], nodes: [], issues: [], captures, benchmark: {} }); + assert.match(viewer, /understudy traces serve --benchmark/); + assert.doesNotMatch(viewer, /must-not-be-inlined/); + } +}); + test("builds a fresh generic DAG, self-contained viewer, and raw/parsed inspector", () => { const root = mkdtempSync(join(tmpdir(), "understudy-foundry-")); const source = join(root, ".understudy", "captures"), output = join(root, ".understudy", "benchmarks", "latest");