diff --git a/metainfer/tasks/evolve_kernel/static/ok-detail.js b/metainfer/tasks/evolve_kernel/static/ok-detail.js index 0770a0a3..b6c3b015 100644 --- a/metainfer/tasks/evolve_kernel/static/ok-detail.js +++ b/metainfer/tasks/evolve_kernel/static/ok-detail.js @@ -9,9 +9,10 @@ import { html } from "htm/preact"; import { useCallback, useEffect, useState } from "preact/hooks"; import { StateGraph } from "app/ok-state-graph"; -import { KernelLibrary } from "app/ok-kernel-library"; +import { KernelLibrary, formatMs } from "app/ok-kernel-library"; import { AgentsPanel } from "app/agents-panel"; import { Timeline } from "app/timeline"; +import { fmtDur } from "app/utils"; import { getIterations, getCharts, getStateGraph, getKernelLibrary, getCorrectnessHarness, getPerfHarness, @@ -128,6 +129,76 @@ function MetricCards({ library, iterations }) { `; } +// ---- Iteration progress ---- + +// One row per optimization iteration. The metric cards only surface the *last* +// iteration's speedup and the library's current best, so a run that is stuck — +// or that regressed after a good iteration — looks identical to one that is +// climbing. This table is the per-iteration trail: what each iteration set out +// to do (`goal`), how it ended, and what it measured. Rows that set a new best +// exec time are marked, which is the "is it actually improving?" signal. +// `status` is written by IterationRecord as one of running|success|failed. +const ITER_STATUS_CLASS = { + success: "green", + failed: "red", + running: "yellow", // in flight — no result yet, but nothing broke either +}; + +function IterationProgress({ iterations }) { + const rows = iterations || []; + if (rows.length === 0) { + return html`

No iterations recorded yet.

`; + } + + // Best-so-far marking. Iterations that failed or are still running carry no + // `perf.exec_time_ms`, so they simply never take the mark. + const bestIterations = new Set(); + let best = Infinity; + for (const it of rows) { + const ms = it.perf && it.perf.exec_time_ms; + if (typeof ms === "number" && ms > 0 && ms < best) { + best = ms; + bestIterations.add(it.iteration); + } + } + + return html`
+ + + + + + + + + ${rows.map((it) => { + const perf = it.perf || {}; + const improved = bestIterations.has(it.iteration); + // These numbers come out of an LLM-generated benchmark harness, so + // coerce rather than trusting the type (a string "1.02" is fine). + const speedup = perf.speedup == null ? null : Number(perf.speedup); + return html` + + + + + + + + + `; + })} + +
#StatusExec TimeSpeedupLibraryDurationGoal
${it.iteration} + ${it.status || "—"} + ${perf.exec_time_ms != null ? formatMs(perf.exec_time_ms) : "—"} + ${improved ? html`▲` : null} + ${Number.isFinite(speedup) ? speedup.toFixed(3) + "×" : "—"}${it.kernel_library_size ?? "—"}${it.duration_s ? fmtDur(it.duration_s) : "—"} + ${it.goal || (it.failure_reason ? "failed: " + it.failure_reason : "—")} +
+
`; +} + // ---- Harness status badges ---- function HarnessStatus({ harness, label }) { @@ -262,6 +333,15 @@ export default function OptKernelDetailView({ <${MetricCards} library=${rt.library} iterations=${rt.iterations} /> +
+

Iteration Progress + + (▲ = new best exec time) + +

+ <${IterationProgress} iterations=${rt.iterations} /> +
+

State Machine

diff --git a/metainfer/tasks/evolve_kernel/static/ok-kernel-library.js b/metainfer/tasks/evolve_kernel/static/ok-kernel-library.js index 6bfc58ac..31dc6576 100644 --- a/metainfer/tasks/evolve_kernel/static/ok-kernel-library.js +++ b/metainfer/tasks/evolve_kernel/static/ok-kernel-library.js @@ -10,7 +10,7 @@ function scoreClass(val) { return "ok-score-bad"; } -function formatMs(ms) { +export function formatMs(ms) { if (ms == null || ms === 0) return "—"; if (ms < 1) return (ms * 1000).toFixed(1) + " μs"; if (ms < 1000) return ms.toFixed(3) + " ms"; diff --git a/metainfer/tasks/evolve_kernel/static/ok.css b/metainfer/tasks/evolve_kernel/static/ok.css index 2ce837b0..47a17ae7 100644 --- a/metainfer/tasks/evolve_kernel/static/ok.css +++ b/metainfer/tasks/evolve_kernel/static/ok.css @@ -115,6 +115,33 @@ color: #d29922; } +.ok-badge.red { + background: rgba(248, 81, 73, 0.15); + color: #f85149; +} + +/* ---- Iteration progress table ---- */ + +.ok-best-tag { + color: #3fb950; + margin-left: 0.25rem; +} + +/* Goals are free-form sentences; keep rows scannable and let the full text + show on hover (the cell carries it as a title). */ +.ok-goal { + max-width: 26rem; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +/* A long run logs dozens of iterations — keep the panel bounded. */ +.ok-iter-scroll { + max-height: 22rem; + overflow: auto; +} + .ok-flow-indicator { display: flex; align-items: center; diff --git a/metainfer/tasks/evolve_kernel/tests/test_iteration_progress.py b/metainfer/tasks/evolve_kernel/tests/test_iteration_progress.py new file mode 100644 index 00000000..fe6df0a3 --- /dev/null +++ b/metainfer/tasks/evolve_kernel/tests/test_iteration_progress.py @@ -0,0 +1,63 @@ +"""Contract for the fields the WebUI "Iteration Progress" table renders. + +``static/ok-detail.js`` builds one row per iteration straight from +``read_iterations``: ``status``, ``goal``, ``duration_s``, +``kernel_library_size`` and ``perf.{exec_time_ms,speedup}``. Iterations that +never got as far as measuring (failed / still running) must still come back as +rows — the table falls back to "—" for the metrics it doesn't have — so the +reader must not require a complete record, and an iteration that was only +partially written must not take the whole list down. +""" + +from __future__ import annotations + +import json + +from metainfer.tasks.evolve_kernel.server import _state_readers as sr + + +def _write_iteration(state_dir, n, **fields): + # The writer dumps `IterationRecord.to_dict()` (== asdict), so every field is + # always on disk — an unmeasured iteration carries `perf: {}`, not a missing + # key. Fixtures pass the fields explicitly to mirror that shape. + iters = state_dir / "iterations" + iters.mkdir(parents=True, exist_ok=True) + (iters / f"{n:03d}.json").write_text( + json.dumps({"iteration": n, **fields}), encoding="utf-8") + + +def test_read_iterations_keeps_the_fields_the_table_renders(tmp_path): + _write_iteration(tmp_path, 2, status="success", goal="wider tiles + double buffer", + duration_s=41.5, kernel_library_size=3, + perf={"exec_time_ms": 0.121, "speedup": 1.02, + "combined_score": 58.1}) + _write_iteration(tmp_path, 1, status="failed", duration_s=2.0, + failure_reason="harness crashed", perf={}) + + rows = sr.read_iterations(tmp_path) + assert [r["iteration"] for r in rows] == [1, 2] + + failed, measured = rows + # A failed iteration is still a row; its empty `perf` leaves the metric + # columns at "—" in the table. + assert failed["status"] == "failed" + assert failed["failure_reason"] == "harness crashed" + assert failed["perf"] == {} + + assert measured["perf"]["exec_time_ms"] == 0.121 + assert measured["perf"]["speedup"] == 1.02 + assert measured["goal"] == "wider tiles + double buffer" + assert measured["duration_s"] == 41.5 + assert measured["kernel_library_size"] == 3 + + +def test_read_iterations_skips_an_unreadable_record(tmp_path): + _write_iteration(tmp_path, 1, status="success", perf={"exec_time_ms": 0.2}) + (tmp_path / "iterations" / "002.json").write_text("{not json", encoding="utf-8") + + rows = sr.read_iterations(tmp_path) + assert [r["iteration"] for r in rows] == [1] + + +def test_read_iterations_empty_when_no_directory(tmp_path): + assert sr.read_iterations(tmp_path) == []