Skip to content

fix(evals): keep large benchmark viewers bounded #478

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
aamir merged 1 commit into from
Sep 1, 2026
Merged

fix(evals): keep large benchmark viewers bounded #478

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 40 additions & 2 deletions src/trace-foundry-viewer.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,48 @@
type Obj = Record<string, any>;

const MAX_INLINE_CAPTURE_BYTES = 16 * 1024 * 1024;
const MAX_INLINE_CAPTURE_COUNT = 250;

function asObject(value: unknown): Obj {
return value !== null && typeof value === "object" && !Array.isArray(value) ? value as Obj : {};
}

function prepareViewerPayload(payload: Obj): { inline: boolean; payload: Obj } {
const captures = asObject(payload.captures);
const references: Obj = {};
let captureCount = 0;
let captureBytes = 0;
let sizesKnown = true;
for (const key in captures) {
if (!Object.hasOwn(captures, key)) continue;
captureCount += 1;
const capture = asObject(captures[key]);
const bytes = capture.bytes;
if (typeof bytes !== "number" || !Number.isSafeInteger(bytes) || bytes < 0 || !Number.isSafeInteger(captureBytes + bytes)) {
sizesKnown = false;
} else {
captureBytes += bytes;
}
const { data: _data, ...reference } = capture;
references[key] = reference;
}

const inline = sizesKnown && captureCount <= MAX_INLINE_CAPTURE_COUNT && captureBytes <= MAX_INLINE_CAPTURE_BYTES;
return {
inline,
payload: { ...payload, captures: inline ? captures : references },
};
}

export function traceFoundryViewer(payload: Obj): string {
const data = JSON.stringify(payload).replaceAll("</", "<\\/");
const prepared = prepareViewerPayload(payload);
const data = JSON.stringify(prepared.payload).replaceAll("</", "<\\/");
const serveNotice = prepared.inline
? ""
: '<aside style="position:fixed;z-index:10;right:16px;bottom:16px;max-width:520px;padding:10px 12px;border:1px solid #e8b86d;background:#111;color:#ecebe5;font:11px IBM Plex Mono">Large benchmark: capture bodies load through the local server. Run <code>understudy traces serve --benchmark &lt;benchmark-directory&gt;</code>.</aside>';
return String.raw`<!doctype html>
<html><head><meta charset="utf-8"><meta name="viewport" content="width=device-width"><title>Understudy · benchmark orchard</title>
<style>@import url('https://fonts.googleapis.com/css2?family=IBM+Plex+Mono:wght@400;500&family=IBM+Plex+Sans:wght@400;500&display=swap');:root{--paper:#ecebe5;--ink:#111;--dim:#777;--line:#292929;--mint:#9edbd3;--amber:#e8b86d;--violet:#a78bfa;--cyan:#67e8f9}*{box-sizing:border-box}body{margin:0;background:#050505;color:var(--paper);font:13px 'IBM Plex Sans';overflow:hidden}header{height:48px;display:flex;align-items:center;padding:0 16px;border-bottom:1px solid var(--line);gap:14px}.brand{font:500 11px 'IBM Plex Mono';letter-spacing:.14em;text-transform:uppercase}.brand:before{content:'●';color:var(--mint);margin-right:9px}.meta{flex:1;color:var(--dim)}button{font:inherit;color:inherit;background:none;border:1px solid var(--line);border-radius:5px;padding:7px 9px;cursor:pointer}.layout{height:calc(100vh - 48px);display:grid;grid-template-columns:250px 340px 1fr}.pane{min-width:0;border-right:1px solid var(--line);display:flex;flex-direction:column}.head{height:44px;padding:0 13px;border-bottom:1px solid var(--line);display:flex;align-items:center;justify-content:space-between;color:var(--dim);font:10px 'IBM Plex Mono';text-transform:uppercase;letter-spacing:.1em}.scroll{overflow:auto;min-height:0}.item{display:block;width:100%;text-align:left;border:0;border-bottom:1px solid var(--line);border-radius:0;padding:11px 13px}.item:hover,.item.on{background:#171717}.sub{color:var(--dim);font:10px 'IBM Plex Mono'}.node{margin:10px 12px;padding-left:18px;border-left:1px solid var(--violet)}.node button{width:100%;text-align:left}.node.on button{border-color:var(--cyan)}.edge{color:var(--violet);font:10px 'IBM Plex Mono'}.error{color:var(--amber)}.inspect{padding:16px 18px 90px}.tabs{display:flex;gap:7px;padding:10px 16px;border-bottom:1px solid var(--line)}.tabs button.on,.mode button.on{background:var(--paper);color:var(--ink)}h1{font:400 19px 'IBM Plex Mono';margin:0;padding:15px 18px;border-bottom:1px solid var(--line)}.facts{display:grid;grid-template-columns:repeat(4,1fr);gap:1px;background:var(--line);margin-bottom:16px}.fact{background:#090909;padding:10px}.fact b{display:block;margin-top:4px;font:12px 'IBM Plex Mono'}details{border-top:1px solid var(--line);padding-left:10px}summary{padding:8px 0;cursor:pointer;font:11px 'IBM Plex Mono';color:var(--cyan)}.leaf{padding:4px 0 8px 13px;font:11px/1.5 'IBM Plex Mono';white-space:pre-wrap;word-break:break-word}.mode{display:flex;gap:6px;justify-content:flex-end;margin-bottom:8px}pre{background:#0b0b0b;border:1px solid var(--line);padding:12px;white-space:pre-wrap;word-break:break-word;max-height:56vh;overflow:auto}.review{position:absolute;right:0;bottom:0;width:calc(100% - 590px);padding:10px 16px;background:#090909ee;border-top:1px solid var(--line);display:flex;justify-content:flex-end;gap:6px}.accept{color:var(--mint)}@media(max-width:900px){.layout{grid-template-columns:210px 280px 1fr}.review{width:calc(100% - 490px)}}</style></head>
<body><header><span class="brand">benchmark orchard</span><span class="meta" id="meta"></span><button onclick="exportReviews()">Export reviews</button></header><main class="layout"><aside class="pane"><div class="head">Tasks <span id="taskCount"></span></div><div id="tasks" class="scroll"></div></aside><section class="pane"><div class="head">Source DAG <span id="roundCount"></span></div><div id="dag" class="scroll"></div></section><section class="pane"><h1 id="title"></h1><div id="tabs" class="tabs"></div><div id="body" class="inspect scroll"></div></section></main><div class="review"><button class="accept" onclick="judge('accept')">Accept</button><button onclick="judge('restrict')">Restrict</button><button onclick="judge('needs_more')">Needs more</button><button onclick="judge('reject')">Reject</button></div>
<body>${serveNotice}<header><span class="brand">benchmark orchard</span><span class="meta" id="meta"></span><button onclick="exportReviews()">Export reviews</button></header><main class="layout"><aside class="pane"><div class="head">Tasks <span id="taskCount"></span></div><div id="tasks" class="scroll"></div></aside><section class="pane"><div class="head">Source DAG <span id="roundCount"></span></div><div id="dag" class="scroll"></div></section><section class="pane"><h1 id="title"></h1><div id="tabs" class="tabs"></div><div id="body" class="inspect scroll"></div></section></main><div class="review"><button class="accept" onclick="judge('accept')">Accept</button><button onclick="judge('restrict')">Restrict</button><button onclick="judge('needs_more')">Needs more</button><button onclick="judge('reject')">Reject</button></div>
<script>const D=${data};let task=D.tasks[0],node=task?.candidate_boundary,tab='task',mode='parsed';const cache={},reviews=JSON.parse(localStorage.getItem('understudy-reviews')||'{}');const esc=x=>String(x??'').replace(/[&<>\"]/g,c=>({'&':'&amp;','<':'&lt;','>':'&gt;','\"':'&quot;'}[c]));function tree(x,k='root'){if(x===null||typeof x!=='object')return '<div class="leaf">'+esc(JSON.stringify(x))+'</div>';const entries=Array.isArray(x)?x.map((v,i)=>[i,v]):Object.entries(x);return '<details open><summary>'+esc(k)+' · '+entries.length+'</summary>'+entries.map(([a,b])=>tree(b,String(a))).join('')+'</details>'}async function load(){if(!node||cache[node])return;const capture=D.captures[node];if(capture?.data){cache[node]=capture.data;return}cache[node]=await fetch(capture.path).then(r=>{if(!r.ok)throw new Error('capture '+node+' failed to load: '+r.status);return r.json()})}function render(){if(!task)return;document.querySelector('#meta').textContent=D.tasks.length+' tasks · '+D.nodes.length+' captures · '+(D.issues?.length||0)+' DAG issues · contamination '+(D.benchmark?.splits?.contamination||'unknown')+' · '+(D.benchmark?.promotion_blockers?.length||0)+' blockers';document.querySelector('#taskCount').textContent=D.tasks.length;document.querySelector('#tasks').innerHTML=D.tasks.map((t,i)=>'<button class="item '+(t.task_id===task.task_id?'on':'')+'" onclick="pickTask(\''+t.task_id+'\')"><span class="sub">'+String(i+1).padStart(2,'0')+' · '+esc(t.capability_fit?.classification)+'</span><br>'+esc(t.title)+'<br><span class="sub">'+esc(reviews[t.task_id]?.decision||t.status)+' · '+esc(t.task_hash?.slice(0,10))+'</span></button>').join('');const ids=new Set(task.source.node_ids),ns=D.nodes.filter(n=>ids.has(n.id)).sort((a,b)=>a.captured_at.localeCompare(b.captured_at));document.querySelector('#roundCount').textContent=ns.length;document.querySelector('#dag').innerHTML=ns.map((n,i)=>{const edge=task.source.edges.find(e=>e.to===n.id);return '<div class="node '+(n.id===node?'on':'')+'"><button onclick="pickNode(\''+n.id+'\')"><span class="sub">round '+(i+1)+' · '+esc(n.capture_id)+' · '+esc(n.id.slice(0,8))+'</span><br>'+n.message_count+' messages '+(n.has_error?'<span class="error">error</span>':'')+'<br><span class="edge">'+esc(edge?.type||'root')+(edge?.evidence?.ambiguous_parent?' · ambiguous':'')+'</span></button></div>'}).join('');document.querySelector('#title').textContent=task.title;document.querySelector('#tabs').innerHTML=['task','request','upstream_request','response','contract','source'].map(x=>'<button class="'+(tab===x?'on':'')+'" onclick="setTab(\''+x+'\')">'+x+'</button>').join('');const c=cache[node]||{};let value;if(tab==='task')value={capability_fit:task.capability_fit,claims:task.claims,sentinels:task.sentinels,review:reviews[task.task_id]||task.review,promotion:D.benchmark};else if(tab==='contract')value={world_model:task.world_model,outcome_contract:task.outcome_contract};else if(tab==='source')value={task_hash:task.task_hash,source:task.source,dag_issues:D.issues};else value=c[tab]??{};const rawKey=tab==='request'?'customer_request':tab;const raw=c.raw?.[rawKey];document.querySelector('#body').innerHTML='<div class="facts"><div class="fact">Confidence<b>'+esc(task.machine_confidence)+'</b></div><div class="fact">Split<b>'+esc(task.split)+'</b></div><div class="fact">Fit<b>'+esc(task.capability_fit?.classification)+'</b></div><div class="fact">Tools<b>'+task.tool_surface.length+'</b></div></div>'+((['request','upstream_request','response'].includes(tab))?'<div class="mode"><button class="'+(mode==='parsed'?'on':'')+'" onclick="setMode(\'parsed\')">Parsed JSON</button><button class="'+(mode==='raw'?'on':'')+'" onclick="setMode(\'raw\')">Raw</button></div>':'')+(mode==='raw'&&['request','upstream_request','response'].includes(tab)?'<pre>'+esc(typeof raw==='string'?raw:JSON.stringify(raw,null,2))+'</pre>':tree(value,tab))}async function pickTask(id){task=D.tasks.find(t=>t.task_id===id);node=task.candidate_boundary;tab='task';mode='parsed';await load();render()}async function pickNode(id){node=id;tab='request';mode='parsed';await load();render()}function setTab(x){tab=x;mode='parsed';render()}function setMode(x){mode=x;render()}function judge(decision){const restrictions=decision==='restrict'?[prompt('Restriction to apply')||''].filter(Boolean):[];reviews[task.task_id]={decision,restrictions,task_hash:task.task_hash,reviewed_at:new Date().toISOString()};localStorage.setItem('understudy-reviews',JSON.stringify(reviews));render()}function exportReviews(){const body=Object.entries(reviews).map(([task_id,r])=>JSON.stringify({task_id,...r})).join('\n')+'\n';const a=document.createElement('a');a.href=URL.createObjectURL(new Blob([body],{type:'application/x-ndjson'}));a.download='benchmark-reviews.jsonl';a.click()}load().then(render).catch(error=>{document.querySelector('#body').innerHTML='<pre class="error">'+esc(error.message)+'</pre>';render()})</script></body></html>`;
}
Loading