render-bench-table.py

#!/usr/bin/env python3
"""Regenerate the "Models tested on AINode" table in README.md from bench/results/*.json.

Those numbers appear in prose in two repos (this README and the tested-models guide on
docs.ainode.dev), so the table is generated rather than typed. Add a result file, run
this, commit both:

    python3 scripts/render-bench-table.py            # rewrite README.md in place
    python3 scripts/render-bench-table.py --print    # emit the table, touch nothing
    python3 scripts/render-bench-table.py --check    # exit 1 if README.md is stale

Any file carrying the two markers can be the target, which is how the same table
lands in the tested-models guide on docs.ainode.dev (MDX needs JSX comments, not
HTML ones):

    python3 scripts/render-bench-table.py \
        --readme ../ainode-docs/guides/tested-models.mdx \
        --begin '{/* BEGIN bench-table */}' --end '{/* END bench-table */}'

Nothing is computed or extrapolated. A measurement that is missing from the JSON
renders as "not measured" and a run with no rubric renders as "not run"; neither
ever becomes a zero or a guess. See bench/SCHEMA.md for the record format.

A second table, "Coding harness runs", renders the records that carry a `harness`
block instead of throughput. It sits between its own markers,
`--harness-begin` / `--harness-end`, right after the speed table. A third,
"Agentic rubric runs", does the same for records carrying an `agentic` block,
between `--agentic-begin` / `--agentic-end`. A fourth, "Decision runs", does it for
records carrying a `decide` block, between `--decide-begin` / `--decide-end`. That
table holds both decision measurements: a record with a `decide.jevals` block was
scored by the Jevals recipe over the public question sets and fills the Decision
Score, ECE and hand-off columns; a legacy 110-item record fills Brier and the
wrong-at-gate count instead, and each says "not measured" in the other's columns
rather than borrowing a number computed under a different definition. A
fifth, "Embedding runs", does it for records carrying an `embed` block, between
`--embed-begin` / `--embed-end`. A sixth, "Speech runs", does it for records carrying
a `speech` block, between `--speech-begin` / `--speech-end`. `--check` covers all six
tables.

# ponytail: stdlib only, so it runs in CI and in the slim image with nothing installed.
"""
import argparse
import json
import pathlib
import sys

BEGIN = "<!-- BEGIN bench-table (generated by scripts/render-bench-table.py) -->"
END = "<!-- END bench-table -->"
HARNESS_BEGIN = "<!-- BEGIN harness-bench-table (generated by scripts/render-bench-table.py) -->"
HARNESS_END = "<!-- END harness-bench-table -->"
AGENTIC_BEGIN = "<!-- BEGIN agentic-bench-table (generated by scripts/render-bench-table.py) -->"
AGENTIC_END = "<!-- END agentic-bench-table -->"
DECIDE_BEGIN = "<!-- BEGIN decide-bench-table (generated by scripts/render-bench-table.py) -->"
DECIDE_END = "<!-- END decide-bench-table -->"
EMBED_BEGIN = "<!-- BEGIN embed-bench-table (generated by scripts/render-bench-table.py) -->"
EMBED_END = "<!-- END embed-bench-table -->"
SPEECH_BEGIN = "<!-- BEGIN speech-bench-table (generated by scripts/render-bench-table.py) -->"
SPEECH_END = "<!-- END speech-bench-table -->"
BASE_URL = "https://github.com/getainode/ainode/blob/main/bench/results"
NOT_MEASURED = "not measured"

HEADERS = [
    "Model",
    "Params / active",
    "Quant",
    "Placement",
    "Single-stream tok/s",
    "16 streams tok/s",
    "Rubric",
    "Date",
    "Run",
]

HARNESS_HEADERS = [
    "Model",
    "Harness",
    "pass@1",
    "pass@2",
    "Mean s",
    "Placement",
    "Date",
    "Run",
]

AGENTIC_HEADERS = [
    "Model",
    "Placement",
    "Score",
    "Tools (B+G)",
    "Coding (C)",
    "Reasoning (D)",
    "Needle",
    "Date",
    "Run",
]

DECIDE_HEADERS = [
    "Backend/Model",
    "Placement",
    "Items",
    "Accuracy",
    "Decision Score",
    "Brier",
    "ECE",
    "Hand-off at 95%",
    "Wrong at 0.9",
    "p50 ms",
    "Cost",
    "Date",
    "Run",
]

EMBED_HEADERS = [
    "Model",
    "Placement",
    "Dims",
    "p50 ms",
    "Texts/s at 64",
    "Tokens/s",
    "Pairs ordered",
    "Date",
    "Run",
]

SPEECH_HEADERS = [
    "Model",
    "Placement",
    "Clips",
    "WER",
    "Exact",
    "p50 ms",
    "RTF",
    "Date",
    "Run",
]

#: The batch size the embedding table reports throughput at. The record carries
#: every size the sweep ran; one column is enough in a README, and 64 is the one an
#: indexer uses and the engine's own --max-num-seqs in the proven recipe.
EMBED_BATCH = 64

#: Which groups make up the "Tools" column: B is one call at a time, G is the
#: agentic loop. They are the same skill measured shallow and deep, so the table
#: adds them and the record keeps them apart.
TOOL_GROUPS = ("B", "G")

#: The confidence gate the decision table reports errors at. The record carries 0.8
#: as well; one column is enough in a README, and 0.9 is the one people automate at.
DECIDE_THRESHOLD = "0.9"


def load_runs(results_dir):
    """Every result JSON, newest run first."""
    runs = []
    for path in sorted(results_dir.glob("*.json")):
        with path.open() as fh:
            run = json.load(fh)
        run["_file"] = path.name
        runs.append(run)
    runs.sort(key=lambda r: r.get("stamp", ""), reverse=True)
    return runs


def is_harness_run(run):
    """A harness-bench record: it measured a coding agent, not throughput.

    Those runs carry a `harness` block and no `single_stream`, so every column of
    this table would be "not measured" for them. They belong in their own table
    (see bench/harness/README.md), not as a row of blanks in this one.
    """
    return bool(run.get("harness")) and not (run.get("results") or {}).get("single_stream")


def is_agentic_run(run):
    """An agentic-rubric record: an `agentic` block and no throughput.

    Same reasoning as `is_harness_run`: the run scored a capability rubric and took
    no tok/s, so every column of the speed table would read "not measured".
    """
    return bool(run.get("agentic")) and not (run.get("results") or {}).get("single_stream")


def is_decide_run(run):
    """A decision-bench record: a `decide` block and no throughput.

    Same reasoning as the two above: the run measured typed decisions and took no
    tok/s, so every column of the speed table would read "not measured".
    """
    return bool(run.get("decide")) and not (run.get("results") or {}).get("single_stream")


def is_embed_run(run):
    """An embedding-bench record: an `embed` block and no throughput.

    Same reasoning as the three above: the run measured an embedding model, which
    generates no tokens at all, so every column of the speed table would read "not
    measured" and the tok/s ones would be meaningless even if it did not.
    """
    return bool(run.get("embed")) and not (run.get("results") or {}).get("single_stream")


def is_speech_run(run):
    """A speech-bench record: a `speech` block and no throughput.

    Same reasoning as the four above: the run measured a transcription model, which
    is scored against known words rather than timed writing tokens, so every column
    of the speed table would read "not measured".
    """
    return bool(run.get("speech")) and not (run.get("results") or {}).get("single_stream")


def throughput_runs(runs):
    return [run for run in runs
            if not is_harness_run(run) and not is_agentic_run(run)
            and not is_decide_run(run) and not is_embed_run(run)
            and not is_speech_run(run)]


def fmt_params(model):
    params = model.get("params_b")
    active = model.get("active_b")
    if params is None:
        return NOT_MEASURED
    if model.get("arch") == "moe" and (active is None or active == params):
        # MoE with no active count in the record is still not dense.
        return f"{params:g}B MoE"
    if model.get("arch") == "dense" or active is None or active == params:
        return f"{params:g}B dense"
    return f"{params:g}B / {active:g}B active"


def fmt_placement(placement):
    bits = []
    node = placement.get("node")
    if node:
        bits.append(node)
    gpus = placement.get("gpus")
    gpu = placement.get("gpu", "GPU")
    if gpus:
        bits.append(f"{gpus}× {gpu.replace('NVIDIA ', '')}")
    tp = placement.get("tp")
    if tp:
        bits.append(f"TP={tp}")
    line = ", ".join(bits) if bits else NOT_MEASURED
    if placement.get("stacked_with"):
        # Stacked runs share the node's memory with another engine, which caps
        # gpu_memory_utilization and therefore the KV cache. Say so in the row.
        line += " (stacked)"
    return line


def fmt_tok_s(value):
    return NOT_MEASURED if value is None else f"{value:.1f}"


def fmt_single_stream(results):
    return fmt_tok_s((results.get("single_stream") or {}).get("decode_tok_s"))


def fmt_aggregate(results, streams=16):
    for point in results.get("concurrency") or []:
        if point.get("streams") == streams:
            return fmt_tok_s(point.get("aggregate_tok_s"))
    return NOT_MEASURED


def fmt_rubric(run):
    rubric = run.get("rubric") or {}
    passed, total = rubric.get("pass"), rubric.get("total")
    if passed is None or total is None:
        return "not run"
    return f"{passed}/{total}"


def fmt_date(stamp):
    if not stamp or len(stamp) < 8:
        return "unknown"
    return f"{stamp[0:4]}-{stamp[4:6]}-{stamp[6:8]}"


def row_for(run, base_url):
    model = run.get("model") or {}
    label = run.get("label") or "run"
    return [
        model.get("name") or model.get("id") or "unknown",
        fmt_params(model),
        model.get("quant") or NOT_MEASURED,
        fmt_placement(run.get("placement") or {}),
        fmt_single_stream(run.get("results") or {}),
        fmt_aggregate(run.get("results") or {}),
        fmt_rubric(run),
        fmt_date(run.get("stamp")),
        f"[{label}]({base_url}/{run['_file']})",
    ]


def render_table(runs, base_url=BASE_URL):
    runs = throughput_runs(runs)
    # Unpadded cells and a `|---|` rule, matching the other tables in README.md.
    lines = [
        "| " + " | ".join(HEADERS) + " |",
        "|" + "---|" * len(HEADERS),
    ]
    for run in runs:
        lines.append("| " + " | ".join(row_for(run, base_url)) + " |")
    return "\n".join(lines)


def note_marks_not_measured(run, hr):
    """A record's notes can flag one harness's rows as "not measured".

    The notes are free text and name the offending harness before they say so,
    either with its version string ("opencode 1.18.31") or its bare name
    ("claude"), so a note counts only when it carries both the name and the
    words "not measured".
    """
    version = hr.get("version")
    name = hr.get("harness") or ""
    candidates = []
    if version:
        candidates.append(version)
        if name:
            candidates.append(f"{name} {version}")
    if name:
        candidates.append(name)
    if not candidates:
        return False
    wanted = [c.lower() for c in candidates]
    for note in run.get("notes") or []:
        lowered = note.lower()
        if "not measured" in lowered and any(w in lowered for w in wanted):
            return True
    return False


def fmt_harness(hr):
    """The harness column: its name, and its version when the run knew one."""
    name = hr.get("harness") or NOT_MEASURED
    version = hr.get("version")
    if version:
        return f"{name} {version}"
    return name


def fmt_pass(hr, key):
    """pass@1 / pass@2 rendered as a count over the tasks the run attempted."""
    scores = hr.get("scores") or {}
    passed = scores.get(key)
    tasks = scores.get("tasks")
    if passed is None or tasks is None:
        return NOT_MEASURED
    return f"{passed}/{tasks}"


def fmt_mean(hr):
    value = (hr.get("scores") or {}).get("mean_wall_s")
    return NOT_MEASURED if value is None else f"{value:.1f}"


def fmt_placement_tp(placement):
    """Placement narrowed to node and tensor-parallel size for the harness table."""
    placement = placement or {}
    bits = []
    node = placement.get("node")
    if node:
        bits.append(node)
    tp = placement.get("tp")
    if tp:
        bits.append(f"TP={tp}")
    return ", ".join(bits) if bits else NOT_MEASURED


def row_for_harness(run, hr, base_url):
    model = run.get("model") or {}
    label = run.get("label") or "run"
    if note_marks_not_measured(run, hr):
        pass1 = pass2 = mean = NOT_MEASURED
    else:
        pass1 = fmt_pass(hr, "passed_at_1")
        pass2 = fmt_pass(hr, "passed_at_2")
        mean = fmt_mean(hr)
    return [
        model.get("name") or model.get("id") or "unknown",
        fmt_harness(hr),
        pass1,
        pass2,
        mean,
        fmt_placement_tp(run.get("placement") or {}),
        fmt_date(run.get("stamp")),
        f"[{label}]({base_url}/{run['_file']})",
    ]


def dedup_harness_runs(runs):
    """One (run, harness-run) per (model, harness); the latest stamp wins.

    `runs` arrive newest-first, so the first record seen for a pair is its
    winner and every earlier-dated record for the same pair is skipped.
    """
    chosen = {}
    for run in runs:
        model = run.get("model") or {}
        label = model.get("name") or model.get("id") or "unknown"
        for hr in (run.get("harness") or {}).get("runs") or []:
            key = (label, hr.get("harness") or "")
            if key not in chosen:
                chosen[key] = (run, hr)
    return list(chosen.values())


def render_harness_table(runs, base_url=BASE_URL):
    """One row per (model, harness), newest record winning, sorted by model then harness.

    A run whose notes flag a harness as "not measured" renders that phrase in the
    score cells instead of a number.
    """
    chosen = dedup_harness_runs(runs)

    def sort_key(pair):
        run, hr = pair
        model = run.get("model") or {}
        return (model.get("name") or model.get("id") or "", hr.get("harness") or "")

    chosen.sort(key=sort_key)

    lines = [
        "| " + " | ".join(HARNESS_HEADERS) + " |",
        "|" + "---|" * len(HARNESS_HEADERS),
    ]
    for run, hr in chosen:
        lines.append("| " + " | ".join(row_for_harness(run, hr, base_url)) + " |")
    return "\n".join(lines)


def fmt_group(agentic, groups):
    """`pass/total` summed over the named probe groups, or "not measured".

    Summed rather than averaged: the cell is a count of probes, and a run that
    skipped a group (`--groups`, `--quick`) contributes nothing to it instead of a
    zero it never measured.
    """
    blocks = agentic.get("groups") or {}
    present = [blocks[g] for g in groups if isinstance(blocks.get(g), dict)]
    if not present:
        return NOT_MEASURED
    passed = sum(b.get("pass") or 0 for b in present)
    total = sum(b.get("total") or 0 for b in present)
    if not total:
        return NOT_MEASURED
    return f"{passed}/{total}"


def fmt_score(agentic):
    score = agentic.get("score") or {}
    passed, total = score.get("pass"), score.get("total")
    if passed is None or total is None:
        return NOT_MEASURED
    return f"{passed}/{total}"


def fmt_needle(agentic):
    """The largest prompt size whose needle probe passed.

    A run where every size failed says so rather than naming the smallest one, and
    a run that skipped group E reads "not measured".
    """
    needle = agentic.get("needle") or {}
    sizes = []
    for size, ok in needle.items():
        try:
            sizes.append((int(size), bool(ok)))
        except (TypeError, ValueError):
            continue
    if not sizes:
        return NOT_MEASURED
    passing = [size for size, ok in sizes if ok]
    if not passing:
        return "none"
    return f"{max(passing) // 1000}k"


def row_for_agentic(run, base_url):
    model = run.get("model") or {}
    agentic = run.get("agentic") or {}
    label = run.get("label") or "run"
    return [
        model.get("name") or model.get("id") or "unknown",
        fmt_placement_tp(run.get("placement") or {}),
        fmt_score(agentic),
        fmt_group(agentic, TOOL_GROUPS),
        fmt_group(agentic, ("C",)),
        fmt_group(agentic, ("D",)),
        fmt_needle(agentic),
        fmt_date(run.get("stamp")),
        f"[{label}]({base_url}/{run['_file']})",
    ]


def dedup_agentic_runs(runs):
    """One record per model; the latest stamp wins.

    `runs` arrive newest-first, so the first record seen for a model is its winner.
    The same rule the harness table uses, keyed on the model alone because an
    agentic run has one score rather than one per harness.
    """
    chosen = {}
    for run in runs:
        if not is_agentic_run(run) and not run.get("agentic"):
            continue
        model = run.get("model") or {}
        key = model.get("name") or model.get("id") or "unknown"
        if key not in chosen:
            chosen[key] = run
    return list(chosen.values())


def render_agentic_table(runs, base_url=BASE_URL):
    """One row per model, newest record winning, sorted by model name."""
    chosen = dedup_agentic_runs(runs)

    def sort_key(run):
        model = run.get("model") or {}
        return model.get("name") or model.get("id") or ""

    chosen.sort(key=sort_key)

    lines = [
        "| " + " | ".join(AGENTIC_HEADERS) + " |",
        "|" + "---|" * len(AGENTIC_HEADERS),
    ]
    for run in chosen:
        lines.append("| " + " | ".join(row_for_agentic(run, base_url)) + " |")
    return "\n".join(lines)


def fmt_backend_model(run):
    """The first column: which backend answered, and as what model.

    Both, because the same model id behind two backends is two different
    measurements: `/v1/decide` and the lettered-options chat fallback ask a served
    model in different shapes, and averaging their rows together would hide that.
    """
    decide = run.get("decide") or {}
    model = run.get("model") or {}
    name = model.get("name") or model.get("id") or "unknown"
    backend = decide.get("backend")
    return f"{backend} / {name}" if backend else name


def fmt_ratio(value, places=3):
    return NOT_MEASURED if value is None else f"{value:.{places}f}"


def fmt_items(decide):
    overall = decide.get("overall") or {}
    count = overall.get("n")
    if count is None:
        return NOT_MEASURED
    errors = overall.get("errors") or 0
    return f"{count}" if not errors else f"{count} ({errors} failed)"


def fmt_wrong_at(decide, threshold=DECIDE_THRESHOLD):
    """`wrong of kept` at the gate: the failure mode, over the answers it let through.

    Two numbers rather than a rate, because "2 of 106" is the sentence a reader
    needs: at 0.9 the backend answered 106 of the items and got 2 of those wrong.
    """
    block = ((decide.get("overall") or {}).get("thresholds") or {}).get(threshold)
    if not isinstance(block, dict):
        return NOT_MEASURED
    wrong, kept = block.get("wrong"), block.get("kept")
    if wrong is None or kept is None:
        return NOT_MEASURED
    return f"{wrong} of {kept}"


def decide_set_blocks(decide):
    """The per-set metrics blocks of a Jevals-recipe record, or ``[]`` for a legacy one.

    A record carrying a `decide.jevals` block was scored by the recipe recorded in
    `bench/decide/JEVALS.md` over the public question sets; one with no such block is a
    legacy 110-item run. The two are not the same measurement, so the three columns
    below read from the first and say "not measured" for the second rather than
    borrowing a number from a different definition.
    """
    return list(((decide.get("jevals") or {}).get("sets") or {}).values())


def fmt_decision_score(decide):
    """The mean Decision Score over the sets this run scored, or "not measured".

    100 is perfect, 0 is answering with the label base rates and negative is worse than
    that, so this column is the one to read first on a Jevals-recipe row. The mean over
    several sets is only shown when every set has a score, which is the recipe's rule.
    """
    overall = (decide.get("jevals") or {}).get("overall") or {}
    value = overall.get("mean_decision_score")
    if value is None:
        blocks = decide_set_blocks(decide)
        scores = [b.get("decision_score") for b in blocks]
        if not scores or any(s is None for s in scores):
            return NOT_MEASURED
        value = sum(scores) / len(scores)
    return f"{value:.1f}"


def fmt_decide_ece(decide):
    """The calibration gap, in the units of whichever recipe produced it.

    A Jevals-recipe row shows the ten-bin top-label ECE in POINTS with the unit in the
    cell (`5.8 pt`, which is 0.058); a legacy row shows its own five-bin ratio
    (`0.059`). The unit travels with the number because two definitions in one column
    with no unit is how a comparison becomes a lie.
    """
    blocks = decide_set_blocks(decide)
    if blocks:
        points = [b.get("ece_points") for b in blocks]
        points = [p for p in points if p is not None]
        if not points:
            return NOT_MEASURED
        return f"{sum(points) / len(points):.1f} pt"
    value = (decide.get("overall") or {}).get("ece")
    return NOT_MEASURED if value is None else f"{value:.3f}"


def fmt_handoff_share(decide):
    """`share @ threshold`: the largest share it can take alone while staying 95% right.

    Averaged over the sets when there are several, and "not measured" as soon as one set
    never reached 95 percent at any threshold, because a mean over the sets that did
    would read as a hand-off the model does not have.
    """
    blocks = decide_set_blocks(decide)
    if not blocks:
        return NOT_MEASURED
    hands = [b.get("handoff_95") for b in blocks]
    if not hands or any(not h for h in hands):
        return NOT_MEASURED
    shares = [h.get("share") for h in hands]
    thresholds = [h.get("threshold") for h in hands]
    if any(s is None for s in shares) or any(t is None for t in thresholds):
        return NOT_MEASURED
    return (f"{sum(shares) / len(shares):.2f} @ "
            f"{sum(thresholds) / len(thresholds):.2f}")


def fmt_latency_ms(decide):
    value = (decide.get("overall") or {}).get("p50_ms")
    return NOT_MEASURED if value is None else f"{value:.0f}"


def fmt_usd(decide):
    """The run's bill. `$0` for a backend nobody charges per token for."""
    value = (decide.get("overall") or {}).get("cost_usd")
    if value is None:
        return NOT_MEASURED
    if not value:
        return "$0"
    return f"${value:.4f}" if value >= 0.0001 else f"${value:.6f}"


def fmt_placement_decide(run):
    """Placement for a decision row, or the hosted service's own name."""
    placement = run.get("placement") or {}
    if placement.get("node") and not placement.get("gpus") and not placement.get("tp"):
        # A hosted backend has one honest placement string and no GPU of ours.
        return placement["node"]
    return fmt_placement_tp(placement)


def row_for_decide(run, base_url):
    decide = run.get("decide") or {}
    overall = decide.get("overall") or {}
    label = run.get("label") or "run"
    return [
        fmt_backend_model(run),
        fmt_placement_decide(run),
        fmt_items(decide),
        fmt_ratio(overall.get("accuracy")),
        fmt_decision_score(decide),
        fmt_ratio(overall.get("brier")),
        fmt_decide_ece(decide),
        fmt_handoff_share(decide),
        fmt_wrong_at(decide),
        fmt_latency_ms(decide),
        fmt_usd(decide),
        fmt_date(run.get("stamp")),
        f"[{label}]({base_url}/{run['_file']})",
    ]


def dedup_decide_runs(runs):
    """One record per (backend, model); the latest stamp wins.

    Keyed on the pair because a model measured through `/v1/decide` and through the
    chat fallback are two backends' numbers, not one model's, and both belong in the
    table.
    """
    chosen = {}
    for run in runs:
        if not run.get("decide"):
            continue
        model = run.get("model") or {}
        key = ((run["decide"].get("backend") or ""),
               model.get("name") or model.get("id") or "unknown")
        if key not in chosen:
            chosen[key] = run
    return list(chosen.values())


def render_decide_table(runs, base_url=BASE_URL):
    """One row per (backend, model), newest record winning, sorted by that pair."""
    chosen = dedup_decide_runs(runs)

    def sort_key(run):
        model = run.get("model") or {}
        return ((run.get("decide") or {}).get("backend") or "",
                model.get("name") or model.get("id") or "")

    chosen.sort(key=sort_key)

    lines = [
        "| " + " | ".join(DECIDE_HEADERS) + " |",
        "|" + "---|" * len(DECIDE_HEADERS),
    ]
    for run in chosen:
        lines.append("| " + " | ".join(row_for_decide(run, base_url)) + " |")
    return "\n".join(lines)


def fmt_dims(embed):
    value = embed.get("dimensions")
    return NOT_MEASURED if not value else str(value)


def fmt_embed_latency(embed):
    """p50 of a single-text request, end to end from wherever the bench ran.

    The record also carries the measured transport floor; the column is the number a
    caller waits, and the floor is in the record's notes where the sentence that
    explains it fits.
    """
    value = (embed.get("latency") or {}).get("p50_ms")
    return NOT_MEASURED if value is None else f"{value:.0f}"


def embed_row_at(embed, batch=EMBED_BATCH):
    for row in embed.get("throughput") or []:
        if row.get("batch") == batch:
            return row
    return None


def fmt_texts_per_s(embed, batch=EMBED_BATCH):
    row = embed_row_at(embed, batch)
    if not row or row.get("texts_per_s") is None:
        return NOT_MEASURED
    return f"{row['texts_per_s']:.0f}"


def fmt_tokens_per_s(embed, batch=EMBED_BATCH):
    """Tokens per second at the same batch as the texts column, not the best of all.

    Two rates from two different batch sizes side by side would read as one
    measurement and be neither.
    """
    row = embed_row_at(embed, batch)
    if not row or row.get("tokens_per_s") is None:
        return NOT_MEASURED
    return f"{row['tokens_per_s']:.0f}"


def fmt_pairs_ordered(embed):
    """The quality verdict: every related pair above every unrelated one.

    A run that could not score all six pairs reads "not measured" rather than "no",
    because nothing failed: nothing was measured.
    """
    quality = embed.get("quality") or {}
    ordered = quality.get("ordered")
    if ordered is None:
        return NOT_MEASURED
    margin = quality.get("margin")
    if margin is None:
        return "yes" if ordered else "NO"
    return f"{'yes' if ordered else 'NO'} ({margin:+.2f})"


def row_for_embed(run, base_url):
    model = run.get("model") or {}
    embed = run.get("embed") or {}
    label = run.get("label") or "run"
    return [
        model.get("name") or model.get("id") or "unknown",
        fmt_placement_tp(run.get("placement") or {}),
        fmt_dims(embed),
        fmt_embed_latency(embed),
        fmt_texts_per_s(embed),
        fmt_tokens_per_s(embed),
        fmt_pairs_ordered(embed),
        fmt_date(run.get("stamp")),
        f"[{label}]({base_url}/{run['_file']})",
    ]


def dedup_embed_runs(runs):
    """One record per (model, endpoint); the latest stamp wins.

    Keyed on the pair because the same model measured through the fleet router and
    straight at its engine port are two different measurements of the same instance,
    and the difference between them is the routing hop this table exists to make
    visible.
    """
    chosen = {}
    for run in runs:
        if not run.get("embed"):
            continue
        model = run.get("model") or {}
        key = (model.get("name") or model.get("id") or "unknown",
               (run["embed"].get("endpoint") or ""))
        if key not in chosen:
            chosen[key] = run
    return list(chosen.values())


def render_embed_table(runs, base_url=BASE_URL):
    """One row per (model, endpoint), newest record winning, sorted by that pair."""
    chosen = dedup_embed_runs(runs)

    def sort_key(run):
        model = run.get("model") or {}
        return (model.get("name") or model.get("id") or "",
                (run.get("embed") or {}).get("endpoint") or "")

    chosen.sort(key=sort_key)

    lines = [
        "| " + " | ".join(EMBED_HEADERS) + " |",
        "|" + "---|" * len(EMBED_HEADERS),
    ]
    for run in chosen:
        lines.append("| " + " | ".join(row_for_embed(run, base_url)) + " |")
    return "\n".join(lines)


def fmt_speech_clips(speech):
    """Clips scored out of clips sent, so a run that lost one does not read as ten."""
    accuracy = speech.get("accuracy") or {}
    sent = accuracy.get("clips")
    scored = accuracy.get("scored")
    if not sent:
        return NOT_MEASURED
    return str(sent) if scored == sent else f"{scored} of {sent}"


def fmt_wer(speech):
    """The pooled word error rate as a percentage, with the orthographic one beside it.

    Both, because the difference between them is spelling rather than hearing and one
    number alone invites the reader to pick which it is. A run that scored nothing
    reads "not measured" rather than a rate nobody took.
    """
    accuracy = speech.get("accuracy") or {}
    wer = accuracy.get("wer")
    if wer is None:
        return NOT_MEASURED
    strict = accuracy.get("wer_orthographic")
    if strict is None:
        return f"{wer * 100:.1f}%"
    return f"{wer * 100:.1f}% ({strict * 100:.1f}% raw)"


def fmt_exact(speech):
    """Clips transcribed word for word, which is the number a user feels."""
    accuracy = speech.get("accuracy") or {}
    exact = accuracy.get("clips_exact")
    scored = accuracy.get("scored")
    if exact is None or not scored:
        return NOT_MEASURED
    return f"{exact} of {scored}"


def fmt_speech_latency(speech):
    """p50 of one clip uploaded on its own, end to end from wherever the bench ran.

    The record also carries the measured transport floor; the column is the number a
    caller waits, and the floor is in the record's notes where the sentence that
    explains it fits.
    """
    value = (speech.get("latency") or {}).get("p50_ms")
    return NOT_MEASURED if value is None else f"{value:.0f}"


def fmt_rtf(speech):
    """Pooled real-time factor: wall seconds per second of audio.

    Below 1 is faster than the clip plays, which is what decides whether a live
    stream can be kept up with. The wire is in it, so the record's floor is part of
    reading it.
    """
    value = (speech.get("rtf") or {}).get("pooled")
    return NOT_MEASURED if value is None else f"{value:.3f}"


def row_for_speech(run, base_url):
    model = run.get("model") or {}
    speech = run.get("speech") or {}
    label = run.get("label") or "run"
    return [
        model.get("name") or model.get("id") or "unknown",
        fmt_placement_tp(run.get("placement") or {}),
        fmt_speech_clips(speech),
        fmt_wer(speech),
        fmt_exact(speech),
        fmt_speech_latency(speech),
        fmt_rtf(speech),
        fmt_date(run.get("stamp")),
        f"[{label}]({base_url}/{run['_file']})",
    ]


def dedup_speech_runs(runs):
    """One record per (model, endpoint, path); the latest stamp wins.

    Keyed on all three because the same model measured through the fleet router and
    straight at its engine port are two different measurements of the same instance,
    and because transcription and translation are different tasks that must never
    share a row.
    """
    chosen = {}
    for run in runs:
        if not run.get("speech"):
            continue
        model = run.get("model") or {}
        key = (model.get("name") or model.get("id") or "unknown",
               (run["speech"].get("endpoint") or ""),
               (run["speech"].get("path") or ""))
        if key not in chosen:
            chosen[key] = run
    return list(chosen.values())


def render_speech_table(runs, base_url=BASE_URL):
    """One row per (model, endpoint, path), newest record winning, sorted by that."""
    chosen = dedup_speech_runs(runs)

    def sort_key(run):
        model = run.get("model") or {}
        speech = run.get("speech") or {}
        return (model.get("name") or model.get("id") or "",
                speech.get("endpoint") or "", speech.get("path") or "")

    chosen.sort(key=sort_key)

    lines = [
        "| " + " | ".join(SPEECH_HEADERS) + " |",
        "|" + "---|" * len(SPEECH_HEADERS),
    ]
    for run in chosen:
        lines.append("| " + " | ".join(row_for_speech(run, base_url)) + " |")
    return "\n".join(lines)


def splice(text, table, begin=BEGIN, end=END):
    """Replace whatever sits between the markers. Both markers must already exist."""
    start = text.find(begin)
    stop = text.find(end)
    if start == -1 or stop == -1:
        raise SystemExit(f"markers not found in target file; expected\n  {begin}\n  {end}")
    if stop < start:
        raise SystemExit("end marker appears before begin marker")
    head = text[: start + len(begin)]
    tail = text[stop:]
    return f"{head}\n\n{table}\n\n{tail}"


def main():
    repo = pathlib.Path(__file__).resolve().parent.parent
    ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
    ap.add_argument("--results", type=pathlib.Path, default=repo / "bench" / "results")
    ap.add_argument("--readme", type=pathlib.Path, default=repo / "README.md",
                    help="file to rewrite; any file carrying the two markers works, which "
                         "is how the docs-site guide gets the same table")
    ap.add_argument("--begin", default=BEGIN, help="opening marker (MDX needs {/* ... */})")
    ap.add_argument("--end", default=END, help="closing marker")
    ap.add_argument("--harness-begin", default=HARNESS_BEGIN,
                    help="opening marker for the Coding harness runs table")
    ap.add_argument("--harness-end", default=HARNESS_END,
                    help="closing marker for the Coding harness runs table")
    ap.add_argument("--agentic-begin", default=AGENTIC_BEGIN,
                    help="opening marker for the Agentic rubric runs table")
    ap.add_argument("--agentic-end", default=AGENTIC_END,
                    help="closing marker for the Agentic rubric runs table")
    ap.add_argument("--decide-begin", default=DECIDE_BEGIN,
                    help="opening marker for the Decision runs table")
    ap.add_argument("--decide-end", default=DECIDE_END,
                    help="closing marker for the Decision runs table")
    ap.add_argument("--embed-begin", default=EMBED_BEGIN,
                    help="opening marker for the Embedding runs table")
    ap.add_argument("--embed-end", default=EMBED_END,
                    help="closing marker for the Embedding runs table")
    ap.add_argument("--speech-begin", default=SPEECH_BEGIN,
                    help="opening marker for the Speech runs table")
    ap.add_argument("--speech-end", default=SPEECH_END,
                    help="closing marker for the Speech runs table")
    ap.add_argument("--base-url", default=BASE_URL, help="where the JSON files are browsable")
    ap.add_argument("--print", dest="print_only", action="store_true",
                    help="write the throughput table to stdout and leave README.md alone")
    ap.add_argument("--check", action="store_true",
                    help="exit 1 if README.md does not already match the JSON for any "
                         "of the six tables")
    args = ap.parse_args()

    runs = load_runs(args.results)
    if not runs:
        raise SystemExit(f"no result files in {args.results}")
    shown = len(throughput_runs(runs))
    if not shown:
        raise SystemExit(f"{args.results} holds no throughput runs; this table is "
                         "the throughput one and would render empty")
    shown_harness = len(dedup_harness_runs(runs))
    shown_agentic = len(dedup_agentic_runs(runs))
    shown_decide = len(dedup_decide_runs(runs))
    shown_embed = len(dedup_embed_runs(runs))
    shown_speech = len(dedup_speech_runs(runs))
    speed_table = render_table(runs, args.base_url)
    harness_table = render_harness_table(runs, args.base_url)
    agentic_table = render_agentic_table(runs, args.base_url)
    decide_table = render_decide_table(runs, args.base_url)
    embed_table = render_embed_table(runs, args.base_url)
    speech_table = render_speech_table(runs, args.base_url)

    if args.print_only:
        print(speed_table)
        return 0

    current = args.readme.read_text()
    updated = splice(current, speed_table, args.begin, args.end)
    # The harness table lives in its own markers, so any other file that does not
    # carry them (the docs-site guide, say) keeps only the throughput table.
    if args.harness_begin in current and args.harness_end in current:
        updated = splice(updated, harness_table, args.harness_begin, args.harness_end)
    if args.agentic_begin in current and args.agentic_end in current:
        updated = splice(updated, agentic_table, args.agentic_begin, args.agentic_end)
    if args.decide_begin in current and args.decide_end in current:
        updated = splice(updated, decide_table, args.decide_begin, args.decide_end)
    if args.embed_begin in current and args.embed_end in current:
        updated = splice(updated, embed_table, args.embed_begin, args.embed_end)
    if args.speech_begin in current and args.speech_end in current:
        updated = splice(updated, speech_table, args.speech_begin, args.speech_end)
    counted = (f"{shown} speed rows, {shown_harness} harness rows, "
               f"{shown_agentic} agentic rows, {shown_decide} decide rows, "
               f"{shown_embed} embed rows, {shown_speech} speech rows")
    if args.check:
        if current != updated:
            print(f"{args.readme} is stale; run python3 scripts/render-bench-table.py",
                  file=sys.stderr)
            return 1
        print(f"{args.readme} is up to date ({counted})")
        return 0
    if current == updated:
        print(f"{args.readme} already up to date ({counted})")
        return 0
    args.readme.write_text(updated)
    print(f"{args.readme} updated ({counted})")
    return 0


if __name__ == "__main__":
    sys.exit(main())