Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
82 changes: 81 additions & 1 deletion metainfer/tasks/evolve_kernel/static/ok-detail.js
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,10 @@
import { html } from "htm/preact";
import { useCallback, useEffect, useState } from "preact/hooks";
import { StateGraph } from "app/ok-state-graph";
import { KernelLibrary } from "app/ok-kernel-library";
import { KernelLibrary, formatMs } from "app/ok-kernel-library";
import { AgentsPanel } from "app/agents-panel";
import { Timeline } from "app/timeline";
import { fmtDur } from "app/utils";
import {
getIterations, getCharts, getStateGraph,
getKernelLibrary, getCorrectnessHarness, getPerfHarness,
Expand Down Expand Up @@ -128,6 +129,76 @@ function MetricCards({ library, iterations }) {
</div>`;
}

// ---- Iteration progress ----

// One row per optimization iteration. The metric cards only surface the *last*
// iteration's speedup and the library's current best, so a run that is stuck —
// or that regressed after a good iteration — looks identical to one that is
// climbing. This table is the per-iteration trail: what each iteration set out
// to do (`goal`), how it ended, and what it measured. Rows that set a new best
// exec time are marked, which is the "is it actually improving?" signal.
// `status` is written by IterationRecord as one of running|success|failed.
const ITER_STATUS_CLASS = {
success: "green",
failed: "red",
running: "yellow", // in flight — no result yet, but nothing broke either
};

function IterationProgress({ iterations }) {
const rows = iterations || [];
if (rows.length === 0) {
return html`<p class="muted">No iterations recorded yet.</p>`;
}

// Best-so-far marking. Iterations that failed or are still running carry no
// `perf.exec_time_ms`, so they simply never take the mark.
const bestIterations = new Set();
let best = Infinity;
for (const it of rows) {
const ms = it.perf && it.perf.exec_time_ms;
if (typeof ms === "number" && ms > 0 && ms < best) {
best = ms;
bestIterations.add(it.iteration);
}
}

return html`<div class="ok-iter-scroll">
<table class="ok-kernel-table">
<thead>
<tr>
<th>#</th><th>Status</th><th>Exec Time</th><th>Speedup</th>
<th>Library</th><th>Duration</th><th>Goal</th>
</tr>
</thead>
<tbody>
${rows.map((it) => {
const perf = it.perf || {};
const improved = bestIterations.has(it.iteration);
// These numbers come out of an LLM-generated benchmark harness, so
// coerce rather than trusting the type (a string "1.02" is fine).
const speedup = perf.speedup == null ? null : Number(perf.speedup);
return html`
<tr key=${it.iteration}>
<td><strong>${it.iteration}</strong></td>
<td><span class=${"ok-badge " + (ITER_STATUS_CLASS[it.status] || "")}>
${it.status || "—"}</span></td>
<td class=${improved ? "ok-score-good" : ""}>
${perf.exec_time_ms != null ? formatMs(perf.exec_time_ms) : "—"}
${improved ? html`<span class="ok-best-tag" title="new best exec time">▲</span>` : null}
</td>
<td>${Number.isFinite(speedup) ? speedup.toFixed(3) + "×" : "—"}</td>
<td class="muted">${it.kernel_library_size ?? "—"}</td>
<td class="muted">${it.duration_s ? fmtDur(it.duration_s) : "—"}</td>
<td class="ok-goal" title=${it.goal || ""}>
${it.goal || (it.failure_reason ? "failed: " + it.failure_reason : "—")}
</td>
</tr>`;
})}
</tbody>
</table>
</div>`;
}

// ---- Harness status badges ----

function HarnessStatus({ harness, label }) {
Expand Down Expand Up @@ -262,6 +333,15 @@ export default function OptKernelDetailView({

<${MetricCards} library=${rt.library} iterations=${rt.iterations} />

<section class="panel ok-panel-full">
<h2>Iteration Progress
<span class="muted" style="font-size:0.75rem;margin-left:0.5rem;">
(▲ = new best exec time)
</span>
</h2>
<${IterationProgress} iterations=${rt.iterations} />
</section>

<div class="ok-grid">
<section class="panel">
<h2>State Machine</h2>
Expand Down
2 changes: 1 addition & 1 deletion metainfer/tasks/evolve_kernel/static/ok-kernel-library.js
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ function scoreClass(val) {
return "ok-score-bad";
}

function formatMs(ms) {
export function formatMs(ms) {
if (ms == null || ms === 0) return "—";
if (ms < 1) return (ms * 1000).toFixed(1) + " μs";
if (ms < 1000) return ms.toFixed(3) + " ms";
Expand Down
27 changes: 27 additions & 0 deletions metainfer/tasks/evolve_kernel/static/ok.css
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,33 @@
color: #d29922;
}

.ok-badge.red {
background: rgba(248, 81, 73, 0.15);
color: #f85149;
}

/* ---- Iteration progress table ---- */

.ok-best-tag {
color: #3fb950;
margin-left: 0.25rem;
}

/* Goals are free-form sentences; keep rows scannable and let the full text
show on hover (the cell carries it as a title). */
.ok-goal {
max-width: 26rem;
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}

/* A long run logs dozens of iterations — keep the panel bounded. */
.ok-iter-scroll {
max-height: 22rem;
overflow: auto;
}

.ok-flow-indicator {
display: flex;
align-items: center;
Expand Down
63 changes: 63 additions & 0 deletions metainfer/tasks/evolve_kernel/tests/test_iteration_progress.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
"""Contract for the fields the WebUI "Iteration Progress" table renders.

``static/ok-detail.js`` builds one row per iteration straight from
``read_iterations``: ``status``, ``goal``, ``duration_s``,
``kernel_library_size`` and ``perf.{exec_time_ms,speedup}``. Iterations that
never got as far as measuring (failed / still running) must still come back as
rows — the table falls back to "—" for the metrics it doesn't have — so the
reader must not require a complete record, and an iteration that was only
partially written must not take the whole list down.
"""

from __future__ import annotations

import json

from metainfer.tasks.evolve_kernel.server import _state_readers as sr


def _write_iteration(state_dir, n, **fields):
# The writer dumps `IterationRecord.to_dict()` (== asdict), so every field is
# always on disk — an unmeasured iteration carries `perf: {}`, not a missing
# key. Fixtures pass the fields explicitly to mirror that shape.
iters = state_dir / "iterations"
iters.mkdir(parents=True, exist_ok=True)
(iters / f"{n:03d}.json").write_text(
json.dumps({"iteration": n, **fields}), encoding="utf-8")


def test_read_iterations_keeps_the_fields_the_table_renders(tmp_path):
_write_iteration(tmp_path, 2, status="success", goal="wider tiles + double buffer",
duration_s=41.5, kernel_library_size=3,
perf={"exec_time_ms": 0.121, "speedup": 1.02,
"combined_score": 58.1})
_write_iteration(tmp_path, 1, status="failed", duration_s=2.0,
failure_reason="harness crashed", perf={})

rows = sr.read_iterations(tmp_path)
assert [r["iteration"] for r in rows] == [1, 2]

failed, measured = rows
# A failed iteration is still a row; its empty `perf` leaves the metric
# columns at "—" in the table.
assert failed["status"] == "failed"
assert failed["failure_reason"] == "harness crashed"
assert failed["perf"] == {}

assert measured["perf"]["exec_time_ms"] == 0.121
assert measured["perf"]["speedup"] == 1.02
assert measured["goal"] == "wider tiles + double buffer"
assert measured["duration_s"] == 41.5
assert measured["kernel_library_size"] == 3


def test_read_iterations_skips_an_unreadable_record(tmp_path):
_write_iteration(tmp_path, 1, status="success", perf={"exec_time_ms": 0.2})
(tmp_path / "iterations" / "002.json").write_text("{not json", encoding="utf-8")

rows = sr.read_iterations(tmp_path)
assert [r["iteration"] for r in rows] == [1]


def test_read_iterations_empty_when_no_directory(tmp_path):
assert sr.read_iterations(tmp_path) == []
Loading