From 2fbd8ecc7ffaed252a74bd56a0ff211621932e2b Mon Sep 17 00:00:00 2001 From: Andrew Anderson Date: Tue, 27 Oct 2026 11:39:25 -0400 Subject: [PATCH] feat(evidence): reproducible mediated-vs-unmediated outcome benchmark with README outcomes block Adds pkg/outcomebench (fold evidence bundles per PR, split by whether the review swarm touched any head, revert/change-failure/time-to-merge/CI/ changes-requested/cost with Wilson intervals, n<10 not reported, generated SVG charts), hivectl evidence benchmark, src/scripts/outcome-benchmark.sh, the first report from hive's own spoke (2026-10) and the README Outcomes block it regenerates; a test fails when README, report or charts drift from the committed JSON. Signed-off-by: Andrew Anderson --- README.md | 20 + changelog.d/added-outcome-benchmark.md | 1 + src/docs/benchmarks/README.md | 44 ++ .../assets/outcomes-2026-10-bars.svg | 36 ++ .../assets/outcomes-2026-10-trend.svg | 43 ++ src/docs/benchmarks/outcomes-2026-10.json | 425 ++++++++++++++++++ src/docs/benchmarks/outcomes-2026-10.md | 63 +++ src/docs/documentation-map.md | 1 + src/docs/hivectl.md | 19 + src/pkg/hivectl/commands/evidence.go | 3 +- .../hivectl/commands/evidence_benchmark.go | 147 ++++++ .../commands/evidence_benchmark_test.go | 88 ++++ src/pkg/outcomebench/bench.go | 416 +++++++++++++++++ src/pkg/outcomebench/drift_test.go | 60 +++ src/pkg/outcomebench/outcomebench_test.go | 331 ++++++++++++++ src/pkg/outcomebench/readme.go | 79 ++++ src/pkg/outcomebench/render.go | 206 +++++++++ src/pkg/outcomebench/sort.go | 5 + src/pkg/outcomebench/svg.go | 222 +++++++++ src/pkg/outcomebench/wilson.go | 64 +++ src/scripts/outcome-benchmark.sh | 37 ++ 21 files changed, 2309 insertions(+), 1 deletion(-) create mode 100644 changelog.d/added-outcome-benchmark.md create mode 100644 src/docs/benchmarks/README.md create mode 100644 src/docs/benchmarks/assets/outcomes-2026-10-bars.svg create mode 100644 src/docs/benchmarks/assets/outcomes-2026-10-trend.svg create mode 100644 src/docs/benchmarks/outcomes-2026-10.json create mode 100644 src/docs/benchmarks/outcomes-2026-10.md create mode 100644 src/pkg/hivectl/commands/evidence_benchmark.go create mode 100644 src/pkg/hivectl/commands/evidence_benchmark_test.go create mode 100644 src/pkg/outcomebench/bench.go create mode 100644 src/pkg/outcomebench/drift_test.go create mode 100644 src/pkg/outcomebench/outcomebench_test.go create mode 100644 src/pkg/outcomebench/readme.go create mode 100644 src/pkg/outcomebench/render.go create mode 100644 src/pkg/outcomebench/sort.go create mode 100644 src/pkg/outcomebench/svg.go create mode 100644 src/pkg/outcomebench/wilson.go create mode 100755 src/scripts/outcome-benchmark.sh diff --git a/README.md b/README.md index 9d179536e3..d796a31864 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,26 @@ A terminal agent is a power tool: you hold it, aim it, and it stops when you sto Around that, Hive picks work from your real queue, sets limits in code with six trust levels from the **AI Codebase Maturity Model (ACMM)** — a framework for how much autonomy a codebase is ready to give agents ([levels](#acmm-levels) · [paper](https://arxiv.org/abs/2604.09388)), checks before and after with deterministic filters and gates, writes every action to an audit log owned by a named human, and runs around the clock. Same agents you use today — governed, scheduled, scoped, and accountable. +## Outcomes + +Measured from the review evidence Hive already records, not from a demo. Every figure carries its n and a 95% Wilson interval; anything with n < 10 is not reported. Regenerate with `src/scripts/outcome-benchmark.sh --readme`; a test fails if this block drifts from the committed report. + + + +- **34% of reviewed PRs were flagged for changes before they merged** — a `changes_requested` verdict on an earlier head of 490 of 1455 mediated merges [31.3–36.1], 2026-10-27. +- **Revert rate 0.0% mediated vs 0.1% unmediated** — 0/43 vs 1/1513 merges matured 14 days, Wilson [0.0–8.2] vs [0.0–0.4], 2026-10-27. +- **Median time-to-merge 27 min mediated vs 12 min unmediated** — review costs about 15 minutes on the median PR (n=1455 vs 2224), 2026-10-27. +- **≈$2.13 estimated agent spend per merged PR** — `/api/cost` estimate over 2869 hive-attributed merges, 2026-08-08 → 2026-10-27. +- **11571 pull requests across 10 repositories, 15054 review verdicts recorded** — the evidence this is measured from, 2026-10-27. + +How measured: per-PR evidence bundles the spoke already writes, folded per PR; 95% Wilson intervals; n < 10 is never reported. Full tables, trend and methodology in [src/docs/benchmarks/outcomes-2026-10.md](src/docs/benchmarks/outcomes-2026-10.md). + +![mediated vs unmediated](src/docs/benchmarks/assets/outcomes-2026-10-bars.svg) +![trend by month](src/docs/benchmarks/assets/outcomes-2026-10-trend.svg) + + +**One real example.** On [hivecommons/hive#13523](https://github.com/hivecommons/hive/pull/13523) the swarm's [posted review](https://github.com/hivecommons/hive/pull/13523#pullrequestreview-5613922737) flagged a changelog fragment that started with prose (the release compile would have rejected it) and a duplicate fragment; the author fixed both four minutes later and the next head was approved. Methodology and the full tables: [src/docs/benchmarks/](src/docs/benchmarks/README.md). + ## Hub and spoke - **Spoke** — one running Hive, serving your repo(s). It is where agents actually work. Runs on your Kubernetes cluster, Podman, or Docker Compose. diff --git a/changelog.d/added-outcome-benchmark.md b/changelog.d/added-outcome-benchmark.md new file mode 100644 index 0000000000..1e0f2201b9 --- /dev/null +++ b/changelog.d/added-outcome-benchmark.md @@ -0,0 +1 @@ +- `hivectl evidence benchmark` and `src/scripts/outcome-benchmark.sh` compute a reproducible mediated-vs-unmediated outcome report (revert rate, change-failure, median time-to-merge, CI at merged head, changes-requested-before-merge, cost per merged PR; Wilson intervals, n < 10 not reported) with generated SVG charts under `src/docs/benchmarks/` and a `--readme` flag that rewrites the README's `` block, guarded by a drift test diff --git a/src/docs/benchmarks/README.md b/src/docs/benchmarks/README.md new file mode 100644 index 0000000000..3e651f8e33 --- /dev/null +++ b/src/docs/benchmarks/README.md @@ -0,0 +1,44 @@ +# Outcome benchmarks + +Reproducible comparison of pull requests that went through Hive's review swarm (**mediated**) with pull requests merged without it (**unmediated**), computed from the per-PR evidence bundles a spoke already writes under `/data/evidence`. No new telemetry; every rate carries its n and a 95% Wilson interval, and n < 10 is printed as `not reported`. + +## Reports + +| Month | Report | Charts | +|---|---|---| +| 2026-10 | [outcomes-2026-10.md](outcomes-2026-10.md) | [bars](assets/outcomes-2026-10-bars.svg) · [trend](assets/outcomes-2026-10-trend.svg) | + +Each report ships with `outcomes-YYYY-MM.json`, the computed `Report` the markdown, charts and README block are rendered from. `pkg/outcomebench` `TestCommittedReportMatchesReadme` re-renders the newest JSON and fails when the committed markdown, SVGs or the repository README's `…` block differ, so none of them can drift. + +## Regenerate + +```sh +# on a spoke, or anywhere with a copy of its evidence tree +src/scripts/outcome-benchmark.sh --root /data/evidence --month 2026-10 \ + --spend cost.json # saved GET /api/cost, for cost per merged PR + --hub hub.json # saved GET /api/hub/evidence (admin session), optional + --source "hive-oke spoke snapshot 2026-10-27" \ + --readme # also rewrite README.md's outcomes block +``` + +The script wraps `hivectl evidence benchmark`; see [hivectl.md](../hivectl.md#evidence--benchmark) for every flag. To take a read-only copy of a spoke's evidence: `kubectl exec deploy/hive -c hive -- tar czf - -C /data evidence | tar xzf -`. + +## Metrics + +| Metric | Definition | +|---|---| +| Revert rate | merged PRs the evidence linker tagged `reverted_by` ÷ merged PRs observed ≥ 14 days after merge | +| Change-failure rate | `reverted_by` or `followed_up_by` ÷ the same matured denominator | +| Median time-to-merge | lower median of merge − open time over merged PRs whose bundle recorded an open time | +| CI failure at merged head | captured `failure` ÷ captured success/failure on merged heads; not comparable when a cohort has a captured status on < 50 % of its merges | +| Changes requested pre-merge | mediated merged PRs with ≥ 1 `changes_requested` verdict on any head before the merge | +| Cost per merged PR | `/api/cost` estimated spend ÷ its hive-attributed merged-PR count, same window | + +A PR is mediated when any of its heads carries a Hive verdict or a posted Hive review. Backfilled history is unmediated by construction, so the window must start no earlier than the first mediated merge or the comparison is confounded by time; the report states that date. + +## Caveats + +- Revert and follow-up links depend on the evidence linker having seen the later PR; a spoke that was down misses them. +- CI status is captured only for heads Hive reviewed, so the unmediated CI cell is a small sample until the sweeper records it for every merged head. +- Cost is an estimate: model prices are approximate and some models are unpriced. +- The hub's `GET /api/hub/evidence` needs an admin session; when it is not supplied the report says so and the numbers are one spoke's. diff --git a/src/docs/benchmarks/assets/outcomes-2026-10-bars.svg b/src/docs/benchmarks/assets/outcomes-2026-10-bars.svg new file mode 100644 index 0000000000..1c6e386026 --- /dev/null +++ b/src/docs/benchmarks/assets/outcomes-2026-10-bars.svg @@ -0,0 +1,36 @@ + +Mediated vs unmediated — 2026-10 (95% Wilson intervals) +Mediated vs unmediated — 2026-10 (95% Wilson intervals) +mediated +unmediated +0% +10% +20% +30% +40% + + +0.0% (n=43) + + +0.1% (n=1513) +Revert rate + + +2.3% (n=43) + + +1.1% (n=1513) +Change-failure rate + + +6.9% (n=798) +n=62 +CI failure at head + + +33.7% (n=1455) +Changes requested pre-merge + +Hatched = n < 10 or CI not captured for most of the cohort: not reported. Change-failure = revert or follow-up fix within 14 days. + diff --git a/src/docs/benchmarks/assets/outcomes-2026-10-trend.svg b/src/docs/benchmarks/assets/outcomes-2026-10-trend.svg new file mode 100644 index 0000000000..c811f256ea --- /dev/null +++ b/src/docs/benchmarks/assets/outcomes-2026-10-trend.svg @@ -0,0 +1,43 @@ + +Median time-to-merge by month (hours) — revert rate annotated +Median time-to-merge by month (hours) — revert rate annotated +mediated +unmediated +0h +0.1h +0.2h +0.3h +0.4h +0.5h +2026-04 +2026-05 +2026-06 +2026-07 +2026-08 +2026-09 +2026-10 + +0.01h · rev 0.0% + +0.01h · rev 0.0% + + +0.01h · rev 0.1% + + +0.09h · rev 0.0% + + +0.31h · rev 0.1% + + +0.34h · rev 0.1% + + +0.19h · rev 0.1% + + +0.44h · rev 0.0% + +Months with n < 10 are gaps. Mediated series starts at the first mediated merge. + diff --git a/src/docs/benchmarks/outcomes-2026-10.json b/src/docs/benchmarks/outcomes-2026-10.json new file mode 100644 index 0000000000..2e74e841b2 --- /dev/null +++ b/src/docs/benchmarks/outcomes-2026-10.json @@ -0,0 +1,425 @@ +{ + "GeneratedAt": "2026-10-27T00:00:00Z", + "Since": "2026-10-01T00:00:00Z", + "Until": "2026-11-01T00:00:00Z", + "Now": "2026-10-27T00:00:00Z", + "Source": "hive's own spoke (hive-oke) evidence snapshot taken 2026-10-27", + "Bundles": 13060, + "Unreadable": 10, + "PRs": 11571, + "Repos": [ + { + "Repo": "hivecommons/hive", + "Merged": 2451 + }, + { + "Repo": "hivecommons/docs", + "Merged": 223 + }, + { + "Repo": "hivecommons/dibs", + "Merged": 179 + }, + { + "Repo": "hivecommons/rationguard", + "Merged": 172 + }, + { + "Repo": "hivecommons/pluk", + "Merged": 155 + }, + { + "Repo": "hivecommons/promptargs", + "Merged": 147 + }, + { + "Repo": "hivecommons/hivecommons.github.io", + "Merged": 145 + }, + { + "Repo": "hivecommons/hotshot", + "Merged": 122 + }, + { + "Repo": "hivecommons/homebrew-hive", + "Merged": 78 + }, + { + "Repo": "hivecommons/crux", + "Merged": 7 + } + ], + "Verdicts": 15054, + "Mediated": { + "Name": "mediated", + "Merged": 1455, + "Matured": 43, + "RevertRate": { + "K": 0, + "N": 43, + "Rate": 0, + "Lo": 0, + "Hi": 0.08201257002322722, + "Reported": true + }, + "ChangeFailure": { + "K": 1, + "N": 43, + "Rate": 0.023255813953488372, + "Lo": 0.004116981177419918, + "Hi": 0.12059267861216882, + "Reported": true + }, + "TimeToMerge": { + "Value": 0.4418315681741667, + "N": 1455, + "Reported": true + }, + "CIFailureAtHead": { + "K": 55, + "N": 798, + "Rate": 0.06892230576441102, + "Lo": 0.05333222787551566, + "Hi": 0.08864294527588978, + "Reported": true + }, + "CICoverage": { + "K": 798, + "N": 1455, + "Rate": 0.5484536082474227, + "Lo": 0.5227885036932317, + "Hi": 0.5738635248751888, + "Reported": true + }, + "ChangesRequested": { + "K": 490, + "N": 1455, + "Rate": 0.33676975945017185, + "Lo": 0.312943612754814, + "Hi": 0.36145558178419573, + "Reported": true + }, + "ByAuthorKind": { + "agent": 857, + "bot": 70, + "human": 528 + } + }, + "Unmediated": { + "Name": "unmediated", + "Merged": 2224, + "Matured": 1513, + "RevertRate": { + "K": 1, + "N": 1513, + "Rate": 0.0006609385327164573, + "Lo": 0.00011667840798450777, + "Hi": 0.0037344817922631644, + "Reported": true + }, + "ChangeFailure": { + "K": 16, + "N": 1513, + "Rate": 0.010575016523463317, + "Lo": 0.006519662289637756, + "Hi": 0.01710943649035351, + "Reported": true + }, + "TimeToMerge": { + "Value": 0.19166666666666668, + "N": 2224, + "Reported": true + }, + "CIFailureAtHead": { + "K": 17, + "N": 62, + "Rate": 0.27419354838709675, + "Lo": 0.17880918599092133, + "Hi": 0.39592775843327255, + "Reported": true + }, + "CICoverage": { + "K": 62, + "N": 2224, + "Rate": 0.02787769784172662, + "Lo": 0.021807488449304663, + "Hi": 0.035576124461056645, + "Reported": true + }, + "ChangesRequested": { + "K": 0, + "N": 2224, + "Rate": 0, + "Lo": 0, + "Hi": 0.0017243595774493123, + "Reported": true + }, + "ByAuthorKind": { + "agent": 1186, + "bot": 103, + "human": 935 + } + }, + "FirstMediatedMerge": "2026-10-09T18:04:32Z", + "Months": [ + "2026-04", + "2026-05", + "2026-06", + "2026-07", + "2026-08", + "2026-09", + "2026-10" + ], + "Trend": { + "mediated": [ + { + "Month": "2026-04", + "Merged": 0, + "RevertRate": { + "K": 0, + "N": 0, + "Rate": 0, + "Lo": 0, + "Hi": 0, + "Reported": false + }, + "TimeToMerge": { + "Value": 0, + "N": 0, + "Reported": false + } + }, + { + "Month": "2026-05", + "Merged": 0, + "RevertRate": { + "K": 0, + "N": 0, + "Rate": 0, + "Lo": 0, + "Hi": 0, + "Reported": false + }, + "TimeToMerge": { + "Value": 0, + "N": 0, + "Reported": false + } + }, + { + "Month": "2026-06", + "Merged": 0, + "RevertRate": { + "K": 0, + "N": 0, + "Rate": 0, + "Lo": 0, + "Hi": 0, + "Reported": false + }, + "TimeToMerge": { + "Value": 0, + "N": 0, + "Reported": false + } + }, + { + "Month": "2026-07", + "Merged": 0, + "RevertRate": { + "K": 0, + "N": 0, + "Rate": 0, + "Lo": 0, + "Hi": 0, + "Reported": false + }, + "TimeToMerge": { + "Value": 0, + "N": 0, + "Reported": false + } + }, + { + "Month": "2026-08", + "Merged": 0, + "RevertRate": { + "K": 0, + "N": 0, + "Rate": 0, + "Lo": 0, + "Hi": 0, + "Reported": false + }, + "TimeToMerge": { + "Value": 0, + "N": 0, + "Reported": false + } + }, + { + "Month": "2026-09", + "Merged": 0, + "RevertRate": { + "K": 0, + "N": 0, + "Rate": 0, + "Lo": 0, + "Hi": 0, + "Reported": false + }, + "TimeToMerge": { + "Value": 0, + "N": 0, + "Reported": false + } + }, + { + "Month": "2026-10", + "Merged": 1455, + "RevertRate": { + "K": 0, + "N": 43, + "Rate": 0, + "Lo": 0, + "Hi": 0.08201257002322722, + "Reported": true + }, + "TimeToMerge": { + "Value": 0.4418315681741667, + "N": 1455, + "Reported": true + } + } + ], + "unmediated": [ + { + "Month": "2026-04", + "Merged": 184, + "RevertRate": { + "K": 0, + "N": 184, + "Rate": 0, + "Lo": 0, + "Hi": 0.02045127383923476, + "Reported": true + }, + "TimeToMerge": { + "Value": 0.005555555555555556, + "N": 184, + "Reported": true + } + }, + { + "Month": "2026-05", + "Merged": 432, + "RevertRate": { + "K": 0, + "N": 432, + "Rate": 0, + "Lo": 0, + "Hi": 0.008814211401573416, + "Reported": true + }, + "TimeToMerge": { + "Value": 0.013888888888888888, + "N": 432, + "Reported": true + } + }, + { + "Month": "2026-06", + "Merged": 943, + "RevertRate": { + "K": 1, + "N": 943, + "Rate": 0.0010604453870625664, + "Lo": 0.00018721436692565457, + "Hi": 0.005982349792486016, + "Reported": true + }, + "TimeToMerge": { + "Value": 0.013333333333333334, + "N": 943, + "Reported": true + } + }, + { + "Month": "2026-07", + "Merged": 611, + "RevertRate": { + "K": 0, + "N": 611, + "Rate": 0, + "Lo": 0, + "Hi": 0.006248113335206986, + "Reported": true + }, + "TimeToMerge": { + "Value": 0.08694444444444445, + "N": 611, + "Reported": true + } + }, + { + "Month": "2026-08", + "Merged": 1881, + "RevertRate": { + "K": 2, + "N": 1881, + "Rate": 0.001063264221158958, + "Lo": 0.00029162773813870926, + "Hi": 0.0038687218636527625, + "Reported": true + }, + "TimeToMerge": { + "Value": 0.30833333333333335, + "N": 1881, + "Reported": true + } + }, + { + "Month": "2026-09", + "Merged": 3088, + "RevertRate": { + "K": 4, + "N": 3088, + "Rate": 0.0012953367875647669, + "Lo": 0.0005038347363301571, + "Hi": 0.0033261156725135554, + "Reported": true + }, + "TimeToMerge": { + "Value": 0.33694444444444444, + "N": 3088, + "Reported": true + } + }, + { + "Month": "2026-10", + "Merged": 2224, + "RevertRate": { + "K": 1, + "N": 1513, + "Rate": 0.0006609385327164573, + "Lo": 0.00011667840798450777, + "Hi": 0.0037344817922631644, + "Reported": true + }, + "TimeToMerge": { + "Value": 0.19166666666666668, + "N": 2224, + "Reported": true + } + } + ] + }, + "Spend": { + "usd": 6124.75144225, + "merged_prs": 2869, + "window_start": "2026-08-08T00:00:00Z", + "window_end": "2026-10-27T14:50:41.723Z", + "basis": "hive-attributed" + }, + "Hub": null +} diff --git a/src/docs/benchmarks/outcomes-2026-10.md b/src/docs/benchmarks/outcomes-2026-10.md new file mode 100644 index 0000000000..ffee1b9301 --- /dev/null +++ b/src/docs/benchmarks/outcomes-2026-10.md @@ -0,0 +1,63 @@ +# Outcome benchmark — 2026-10 + +Generated 2026-10-27 from hive's own spoke (hive-oke) evidence snapshot taken 2026-10-27. Window: merged 2026-10-01 → 2026-11-01; maturation 14 days (observed as of 2026-10-27). + +Pull requests that went through Hive's review swarm (**mediated**: at least one recorded verdict or posted review) versus pull requests merged without it (**unmediated**). Every rate carries its n and a 95% Wilson interval; anything with n < 10 is `not reported`. + +## Headline + +| Metric | Mediated | Unmediated | +|---|---|---| +| Merged PRs in window | 1455 | 2224 | +| Revert rate (matured) | 0.0% [0.0–8.2] (n=43) | 0.1% [0.0–0.4] (n=1513) | +| Change-failure rate (revert or follow-up fix, matured) | 2.3% [0.4–12.1] (n=43) | 1.1% [0.7–1.7] (n=1513) | +| Median time-to-merge | 0.44 h (n=1455) | 0.19 h (n=2224) | +| CI failure at merged head | 6.9% [5.3–8.9] (n=798) | 27.4% [17.9–39.6] (n=62) — captured on only 3% of merges, not comparable | +| Had a `changes_requested` verdict before merge | 33.7% [31.3–36.1] (n=1455) | — (no verdicts by definition) | +| Cost per merged PR | ≈$2.13 est. (n=2869 merged, 2026-08-08 → 2026-10-27) | — | + +![mediated vs unmediated](assets/outcomes-2026-10-bars.svg) + +### Reading the numbers + +- Revert rates are indistinguishable at this n: 0.0% [0.0–8.2] (n=43) mediated vs 0.1% [0.0–0.4] (n=1513) unmediated (overlapping intervals). +- Median time-to-merge is 0.44 h (n=1455) for mediated vs 0.19 h (n=2224) for unmediated: review adds about 15 minutes to the median PR. +- 33.7% [31.3–36.1] (n=1455) of mediated merged PRs were flagged `changes_requested` on an earlier head before merging — findings an unmediated PR would have carried to main unseen. +- CI at merged head is only captured for reviewed heads (mediated 6.9% [5.3–8.9] (n=798)); the unmediated cell is not comparable. +- Estimated agent spend is ≈$2.13 per hive-attributed merged PR (n=2869). + +## Trend by month + +Whole evidence history, not just the window; months with n < 10 are gaps. + +![trend](assets/outcomes-2026-10-trend.svg) + +| Month | Mediated merged | Mediated revert | Mediated TTM | Unmediated merged | Unmediated revert | Unmediated TTM | +|---|---|---|---|---|---|---| +| 2026-04 | 0 | not reported (n=0) | not reported (n=0) | 184 | 0.0% [0.0–2.0] (n=184) | 0.01 h (n=184) | +| 2026-05 | 0 | not reported (n=0) | not reported (n=0) | 432 | 0.0% [0.0–0.9] (n=432) | 0.01 h (n=432) | +| 2026-06 | 0 | not reported (n=0) | not reported (n=0) | 943 | 0.1% [0.0–0.6] (n=943) | 0.01 h (n=943) | +| 2026-07 | 0 | not reported (n=0) | not reported (n=0) | 611 | 0.0% [0.0–0.6] (n=611) | 0.09 h (n=611) | +| 2026-08 | 0 | not reported (n=0) | not reported (n=0) | 1881 | 0.1% [0.0–0.4] (n=1881) | 0.31 h (n=1881) | +| 2026-09 | 0 | not reported (n=0) | not reported (n=0) | 3088 | 0.1% [0.1–0.3] (n=3088) | 0.34 h (n=3088) | +| 2026-10 | 1455 | 0.0% [0.0–8.2] (n=43) | 0.44 h (n=1455) | 2224 | 0.1% [0.0–0.4] (n=1513) | 0.19 h (n=2224) | + +## Scale + +- 13060 evidence bundles (10 unreadable) folding to 11571 pull requests across 10 repositories; 15054 verdicts recorded. +- Merged in window by repository: hivecommons/hive 2451, hivecommons/docs 223, hivecommons/dibs 179, hivecommons/rationguard 172, hivecommons/pluk 155, hivecommons/promptargs 147, hivecommons/hivecommons.github.io 145, hivecommons/hotshot 122, hivecommons/homebrew-hive 78, hivecommons/crux 7. +- Merged in window by author kind — mediated: agent 857, bot 70, human 528; unmediated: agent 1186, bot 103, human 935. +- First mediated merge in the evidence: 2026-10-09. +- Hub `GET /api/hub/evidence` was not consulted (it requires an admin session); numbers are this spoke's evidence only. + +## Methodology + +- **Source.** Only the per-PR evidence bundles the spoke already writes under `/data/evidence` (schema v1), folded per PR with `pkg/evidence.FoldPRs`. No new telemetry. +- **Cohorts.** A PR is *mediated* if any head carries a Hive verdict or a posted Hive review; otherwise *unmediated*. Backfilled history is unmediated by construction, so the window must start no earlier than the first mediated merge or the comparison is confounded by time. +- **Revert / change-failure.** Denominator is merged PRs observed at least 14 days after merge. A revert is a later PR the evidence linker tagged `reverted_by`; change-failure adds `followed_up_by` fixes. +- **Time-to-merge.** Lower median of merge time minus open time, over merged PRs whose bundle recorded an open time. +- **CI failure at head.** Share of merged heads with a captured `failure` status among heads with a captured success/failure. The unmediated cohort rarely has a captured status (Hive only records CI for heads it reviews), so its cell reports coverage and is not comparable when coverage is low. +- **Changes requested.** Share of mediated merged PRs with at least one `changes_requested` verdict on any head before the merge — findings the author had a chance to address pre-merge. Whether the later head fixed the finding, or a human overrode it, is not verified here. +- **Cost.** `/api/cost` estimated spend divided by its hive-attributed merged-PR count over the same window; model prices are estimates and some models are unpriced, so treat it as an upper-bound order of magnitude. +- **Intervals.** 95% Wilson score; n < 10 is never reported. +- **Regenerate.** `src/scripts/outcome-benchmark.sh --month 2026-10` (see `docs/benchmarks/README.md`). diff --git a/src/docs/documentation-map.md b/src/docs/documentation-map.md index f730072c79..aa4ab1d170 100644 --- a/src/docs/documentation-map.md +++ b/src/docs/documentation-map.md @@ -21,6 +21,7 @@ Documentation for the current Hive line (branch `v5`; the code and docs live und - [Work-source terminology](work-source-terminology.md) — glossary and audit notes for keeping user-facing copy neutral across GitHub, GitLab, Gitea, Linear, Jira, and future work sources. - [Release channels](release-channels.md) — `stable`/`candidate`/`edge` moving image tags, per-line channel ownership (`v5` → `candidate`/`latest`, `v6` → `edge`, `v4` → none), switching a hive to a channel, the version pill, and the timeline for when a hive follows `stable` after the tag moves. - [Digest-verifiable rollback](release-rollback.md) - the operator runbook for pinning a hive back to a prior immutable short-SHA build for all three images (`hive`, `hive-contributor`, `hive-hub`), stopping the hub automation that would undo it, and verifying by digest rather than by tag that the pin landed on the running spoke. +- [Outcome benchmarks](benchmarks/README.md) — reproducible mediated-vs-unmediated comparison (revert rate, change-failure, time-to-merge, CI at merged head, changes requested pre-merge, cost per merged PR) from stored review evidence, with the monthly reports, generated charts and the README outcomes block it keeps in sync. - [v5 GA readiness bar](v5-ga.md) — measurable release-train, migration, safety, and governance criteria that must be evidenced before v5 can be promoted beyond the active-development `edge` channel. - [v5 GA candidate week sequencing](v5-candidate-week.md) — dependency-ordered plan for the remaining human-gated GA-bar steps: candidate-SHA designation, candidate-pinned measurements, the two bound exercises, and the v4 freeze evaluation. - [Stable soak and promotion policy (v5 line)](stable-soak-policy.md) — the CI-enforced gate between `candidate` and `stable`: soak conditions, immutable short-SHA rollback tags, the emergency exception path (what it waives, and the post-hoc evidence that closes its follow-up issue), and the ledger of exceptions taken with their per-image rollback digests. diff --git a/src/docs/hivectl.md b/src/docs/hivectl.md index 2fd4f38380..a26f4f3b26 100644 --- a/src/docs/hivectl.md +++ b/src/docs/hivectl.md @@ -288,6 +288,25 @@ Columns: `repo`, `pr`, `head_sha`, `bundle_id`, `author_login`, `author_kind`, ` Reporting only: nothing here reads into or alters merge eligibility. See [Review evidence bundle](review-evidence.md#exporting). +### evidence — benchmark + +```bash +hivectl evidence benchmark --month 2026-10 --out-dir src/docs/benchmarks --readme README.md +hivectl evidence benchmark --root ./evidence --month 2026-10 --spend cost.json --hub hub.json --source "hive-oke snapshot" +``` + +`evidence benchmark` folds the schema v1 bundles under `--root` per PR and compares the pull requests Hive's review +swarm touched (**mediated**: at least one verdict or posted review on any head) with those merged without it +(**unmediated**): revert rate and change-failure rate over merges observed ≥ 14 days, median time-to-merge, CI failure +at merged head, the share of mediated merges that carried a `changes_requested` verdict before merging, and — when +`--spend` names a saved `GET /api/cost` response — cost per merged PR. Every rate carries its n and a 95 % Wilson +interval; n < 10 is `not reported`. `--month YYYY-MM` bounds the merge time (default all), `--now` fixes the +observation time for maturation, `--hub` adds a saved `GET /api/hub/evidence` response as fleet context, `--source` +names the data in the methodology section. It writes `outcomes-YYYY-MM.{md,json}` and +`assets/outcomes-YYYY-MM-{bars,trend}.svg` under `--out-dir`, and `--readme FILE` rewrites that file's +`…` block from the same report. Local filesystem only; no hive session, no +telemetry. `src/scripts/outcome-benchmark.sh` wraps it; see [benchmarks/](benchmarks/README.md). + ### evidence — backfill bundles from GitHub history ```bash diff --git a/src/pkg/hivectl/commands/evidence.go b/src/pkg/hivectl/commands/evidence.go index 97366657c8..a0ca13d830 100644 --- a/src/pkg/hivectl/commands/evidence.go +++ b/src/pkg/hivectl/commands/evidence.go @@ -102,11 +102,12 @@ func newEvidenceCommand(env *commandEnv) *cobra.Command { Use: "evidence", Short: "Per-PR review evidence: export stored bundles or backfill them from forge history", Long: "Work with the review evidence bundles Hive stores under the evidence root (default /data/evidence).\nexport reads the local filesystem only; backfill talks to the GitHub API. Neither contacts a hive and nothing here\ninfluences a merge.", - Example: " hivectl evidence export --since 168h --format csv --out evidence.csv\n hivectl evidence export --since 2026-10-01T00:00:00Z --repo hivecommons/hive\n hivectl evidence backfill --repo ublue-os/bluefin --since 2025-01-01 --dry-run", + Example: " hivectl evidence export --since 168h --format csv --out evidence.csv\n hivectl evidence export --since 2026-10-01T00:00:00Z --repo hivecommons/hive\n hivectl evidence backfill --repo ublue-os/bluefin --since 2025-01-01 --dry-run\n hivectl evidence benchmark --month 2026-10 --out-dir src/docs/benchmarks --readme README.md", } cmd.AddCommand(newEvidenceExportCommand(env)) cmd.AddCommand(newEvidenceBackfillCommand(env)) cmd.AddCommand(newEvidenceBackfillIssuesCommand(env)) + cmd.AddCommand(newEvidenceBenchmarkCommand(env)) return cmd } diff --git a/src/pkg/hivectl/commands/evidence_benchmark.go b/src/pkg/hivectl/commands/evidence_benchmark.go new file mode 100644 index 0000000000..0b26647396 --- /dev/null +++ b/src/pkg/hivectl/commands/evidence_benchmark.go @@ -0,0 +1,147 @@ +package commands + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "github.com/spf13/cobra" + + ghpkg "github.com/hivecommons/hive/pkg/github" + "github.com/hivecommons/hive/pkg/outcomebench" +) + +// evidenceBenchmarkNow is overridable in tests. +var evidenceBenchmarkNow = func() time.Time { return time.Now().UTC() } + +// benchmarkReadmeDocDir is the benchmark directory as the repository README +// links to it. +const benchmarkReadmeDocDir = "src/docs/benchmarks" + +func newEvidenceBenchmarkCommand(_ *commandEnv) *cobra.Command { + var root, month, now, spendPath, hubPath, outDir, readme, source string + cmd := &cobra.Command{ + Use: "benchmark", + Short: "Compare mediated and unmediated merged PRs from stored evidence and write the outcomes report", + Long: "Fold the schema v1 evidence bundles under --root per PR and compare the pull requests Hive's review swarm\n" + + "touched (at least one verdict or posted review) with those merged without it: revert rate, change-failure rate,\n" + + "median time-to-merge, CI failure at merged head, changes-requested-before-merge, and cost per merged PR when a\n" + + "saved /api/cost response is given with --spend. Every rate carries its n and a 95% Wilson interval; n < 10 is\n" + + "printed as \"not reported\". No telemetry is added and nothing contacts a hive.\n\n" + + "Writes /outcomes-YYYY-MM.md, outcomes-YYYY-MM.json and assets/outcomes-YYYY-MM-{bars,trend}.svg.\n" + + "--readme rewrites the … block of that README from the same\n" + + "report, so the README never drifts from the committed report (a test asserts they match).", + Example: " hivectl evidence benchmark --root ./evidence --month 2026-10 --spend cost.json --out-dir src/docs/benchmarks --readme README.md\n" + + " hivectl evidence benchmark --root /data/evidence --month 2026-10 --out-dir /tmp/bench", + Args: argsNone(), + RunE: func(cmd *cobra.Command, _ []string) error { + var opts outcomebench.Options + var err error + if strings.TrimSpace(month) != "" { + if opts.Since, opts.Until, err = outcomebench.ParseWindow(month); err != nil { + return &usageError{message: err.Error()} + } + } + opts.Now = evidenceBenchmarkNow() + if strings.TrimSpace(now) != "" { + if opts.Now, err = time.Parse(time.RFC3339, strings.TrimSpace(now)); err != nil { + return &usageError{message: fmt.Sprintf("invalid --now %q; expected RFC3339", now)} + } + } + if spendPath != "" { + if opts.Spend, err = outcomebench.LoadSpend(spendPath); err != nil { + return err + } + } + if hubPath != "" { + if opts.Hub, err = outcomebench.LoadHub(hubPath); err != nil { + return err + } + } + opts.Source = source + if opts.Source == "" { + opts.Source = "evidence bundles under " + root + } + bundles, unreadable, err := outcomebench.Load(root) + if err != nil { + return err + } + if unreadable > 0 { + fmt.Fprintf(cmd.ErrOrStderr(), "skipped %d unreadable or non-v1 bundle(s) under %s\n", unreadable, root) + } + rep := outcomebench.Compute(bundles, unreadable, opts) + paths, err := writeBenchmark(rep, outDir) + if err != nil { + return err + } + for _, p := range paths { + fmt.Fprintf(cmd.ErrOrStderr(), "wrote %s\n", p) + } + if readme != "" { + data, err := os.ReadFile(readme) + if err != nil { + return fmt.Errorf("reading %s: %w", readme, err) + } + updated, err := outcomebench.UpdateReadme(string(data), rep.ReadmeBlock(benchmarkReadmeDocDir)) + if err != nil { + return fmt.Errorf("%s: %w", readme, err) + } + if err := os.WriteFile(readme, []byte(updated), 0o644); err != nil { + return fmt.Errorf("writing %s: %w", readme, err) + } + fmt.Fprintf(cmd.ErrOrStderr(), "updated outcomes block in %s\n", readme) + } + fmt.Fprint(cmd.OutOrStdout(), benchmarkSummary(rep)) + return nil + }, + } + cmd.Flags().StringVar(&root, "root", ghpkg.ReviewEvidenceRoot, "evidence root holding ///.json bundles") + cmd.Flags().StringVar(&month, "month", "", "only PRs merged in this YYYY-MM month (default: all)") + cmd.Flags().StringVar(&now, "now", "", "observation time (RFC3339) for the maturation window; default now") + cmd.Flags().StringVar(&spendPath, "spend", "", "saved GET /api/cost JSON for cost per merged PR") + cmd.Flags().StringVar(&hubPath, "hub", "", "saved GET /api/hub/evidence JSON for fleet context") + cmd.Flags().StringVar(&outDir, "out-dir", "docs/benchmarks", "directory for the report, JSON and assets/") + cmd.Flags().StringVar(&readme, "readme", "", "README whose outcomes block to rewrite from this report") + cmd.Flags().StringVar(&source, "source", "", "how the report names its source (default: the --root path)") + return cmd +} + +// writeBenchmark writes the markdown, JSON and SVG outputs and returns their +// paths. +func writeBenchmark(rep outcomebench.Report, outDir string) ([]string, error) { + if err := os.MkdirAll(filepath.Join(outDir, "assets"), 0o755); err != nil { + return nil, err + } + js, err := json.MarshalIndent(rep, "", " ") + if err != nil { + return nil, err + } + base := filepath.Join(outDir, "outcomes-"+rep.Month()) + files := map[string][]byte{ + base + ".md": []byte(rep.Markdown()), + base + ".json": append(js, '\n'), + filepath.Join(outDir, "assets", rep.AssetBase()+"-bars.svg"): []byte(rep.BarChartSVG()), + filepath.Join(outDir, "assets", rep.AssetBase()+"-trend.svg"): []byte(rep.TrendSVG()), + } + paths := make([]string, 0, len(files)) + for _, p := range []string{base + ".md", base + ".json", filepath.Join(outDir, "assets", rep.AssetBase()+"-bars.svg"), filepath.Join(outDir, "assets", rep.AssetBase()+"-trend.svg")} { + if err := os.WriteFile(p, files[p], 0o644); err != nil { + return nil, fmt.Errorf("writing %s: %w", p, err) + } + paths = append(paths, p) + } + return paths, nil +} + +func benchmarkSummary(rep outcomebench.Report) string { + var b strings.Builder + fmt.Fprintf(&b, "outcomes %s: mediated %d merged, unmediated %d merged, %d PRs / %d repos / %d verdicts\n", + rep.Month(), rep.Mediated.Merged, rep.Unmediated.Merged, rep.PRs, len(rep.Repos), rep.Verdicts) + for _, f := range rep.Findings() { + fmt.Fprintf(&b, "- %s\n", f) + } + return b.String() +} diff --git a/src/pkg/hivectl/commands/evidence_benchmark_test.go b/src/pkg/hivectl/commands/evidence_benchmark_test.go new file mode 100644 index 0000000000..9dabea1b63 --- /dev/null +++ b/src/pkg/hivectl/commands/evidence_benchmark_test.go @@ -0,0 +1,88 @@ +package commands + +import ( + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/outcomebench" +) + +func TestEvidenceBenchmarkWritesReportAndReadme(t *testing.T) { + root := t.TempDir() + writeExportFixtures(t, root) + out := filepath.Join(t.TempDir(), "bench") + readme := filepath.Join(t.TempDir(), "README.md") + if err := os.WriteFile(readme, []byte("# x\n\n"+outcomebench.ReadmeStart+"\nold\n"+outcomebench.ReadmeEnd+"\n\nrest\n"), 0o644); err != nil { + t.Fatal(err) + } + spend := filepath.Join(t.TempDir(), "cost.json") + if err := os.WriteFile(spend, []byte(`{"estimated":{"total_usd":30},"merged_prs":15}`), 0o644); err != nil { + t.Fatal(err) + } + hub := filepath.Join(t.TempDir(), "hub.json") + if err := os.WriteFile(hub, []byte(`{"spokes":3,"reporting":2,"window":{"from":"a","to":"b"}}`), 0o644); err != nil { + t.Fatal(err) + } + evidenceBenchmarkNow = func() time.Time { return time.Date(2026, 10, 27, 0, 0, 0, 0, time.UTC) } + t.Cleanup(func() { evidenceBenchmarkNow = func() time.Time { return time.Now().UTC() } }) + server := httptest.NewServer(nil) + defer server.Close() + + stdout, stderr, err := execute(t, server, "", "evidence", "benchmark", "--root", root, "--month", "2026-10", + "--spend", spend, "--hub", hub, "--out-dir", out, "--readme", readme, "--source", "fixtures") + if err != nil { + t.Fatal(err) + } + if !strings.Contains(stderr, "skipped 2 unreadable") || !strings.Contains(stderr, "updated outcomes block") { + t.Fatalf("stderr = %q", stderr) + } + if !strings.HasPrefix(stdout, "outcomes 2026-10:") || !strings.Contains(stdout, "mediated 1 merged, unmediated 0 merged") { + t.Fatalf("stdout = %q", stdout) + } + for _, f := range []string{"outcomes-2026-10.md", "outcomes-2026-10.json", "assets/outcomes-2026-10-bars.svg", "assets/outcomes-2026-10-trend.svg"} { + if _, err := os.Stat(filepath.Join(out, f)); err != nil { + t.Errorf("missing %s: %v", f, err) + } + } + md, _ := os.ReadFile(filepath.Join(out, "outcomes-2026-10.md")) + if !strings.Contains(string(md), "from fixtures") || !strings.Contains(string(md), "3 spokes registered") || !strings.Contains(string(md), "≈$2.00") { + t.Fatalf("report:\n%s", md) + } + got, _ := os.ReadFile(readme) + if strings.Contains(string(got), "\nold\n") || !strings.HasPrefix(string(got), "# x\n\n"+outcomebench.ReadmeStart) || !strings.HasSuffix(string(got), "\n\nrest\n") { + t.Fatalf("readme:\n%s", got) + } +} + +func TestEvidenceBenchmarkRejectsBadInput(t *testing.T) { + server := httptest.NewServer(nil) + defer server.Close() + root := t.TempDir() + cases := [][]string{ + {"--month", "October"}, + {"--now", "yesterday"}, + {"--spend", filepath.Join(root, "missing.json")}, + {"--hub", filepath.Join(root, "missing.json")}, + {"--readme", filepath.Join(root, "missing.md")}, + } + for _, extra := range cases { + args := append([]string{"evidence", "benchmark", "--root", root, "--out-dir", filepath.Join(root, "out")}, extra...) + if _, _, err := execute(t, server, "", args...); err == nil { + t.Errorf("%v: expected error", extra) + } + } + noMarkers := filepath.Join(root, "plain.md") + if err := os.WriteFile(noMarkers, []byte("# no block\n"), 0o644); err != nil { + t.Fatal(err) + } + if _, _, err := execute(t, server, "", "evidence", "benchmark", "--root", root, "--out-dir", filepath.Join(root, "out"), "--readme", noMarkers); err == nil || !strings.Contains(err.Error(), "outcomes:start") { + t.Fatalf("expected marker error, got %v", err) + } + if _, err := writeBenchmark(outcomebench.Report{}, filepath.Join(noMarkers, "x")); err == nil { + t.Fatal("unwritable out-dir must fail") + } +} diff --git a/src/pkg/outcomebench/bench.go b/src/pkg/outcomebench/bench.go new file mode 100644 index 0000000000..1155a1a734 --- /dev/null +++ b/src/pkg/outcomebench/bench.go @@ -0,0 +1,416 @@ +// Package outcomebench compares pull requests that went through Hive's +// review swarm with pull requests merged without it, from the evidence +// bundles a spoke already writes. It adds no telemetry: every number is a +// fold over /data/evidence, and every rate carries its n and a Wilson +// interval or is not reported at all. +package outcomebench + +import ( + "encoding/json" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "sort" + "strings" + "time" + + "github.com/hivecommons/hive/pkg/evidence" +) + +// Maturation is how long after merge a PR must have been observed before it +// counts in a revert or change-failure denominator: the evidence linker +// captures reverts and follow-ups within this window, so a younger merge has +// not had its chance to fail yet. +const Maturation = 14 * 24 * time.Hour + +// Cohort names. +const ( + CohortMediated = "mediated" + CohortUnmediated = "unmediated" +) + +// Options selects the PRs and supplies what the bundles do not carry. +type Options struct { + // Since/Until bound the merge time, [Since, Until). Zero means open. + Since, Until time.Time + // Now is the observation time for maturation; zero means time.Now(). + Now time.Time + // Spend is the spoke's estimated agent spend and the hive-attributed + // merged-PR count over the same window, as /api/cost reports them. Nil + // means cost per merged PR is not reported. + Spend *Spend + // Hub is a saved GET /api/hub/evidence response for fleet context; nil + // means the section is omitted. + Hub *HubContext + // Source names where the bundles came from, for the methodology section. + Source string +} + +// Spend is what /api/cost already publishes. +type Spend struct { + USD float64 `json:"usd"` + MergedPRs int `json:"merged_prs"` + WindowStart time.Time `json:"window_start"` + WindowEnd time.Time `json:"window_end"` + Basis string `json:"basis"` +} + +// PerMergedPR is USD/MergedPRs, or 0 when there is nothing to divide. +func (s *Spend) PerMergedPR() (float64, bool) { + if s == nil || s.MergedPRs < MinN || s.USD <= 0 { + return 0, false + } + return s.USD / float64(s.MergedPRs), true +} + +// HubContext is the fleet-level subset of the hub's evidence view that the +// report quotes verbatim: how many spokes report, over what window. +type HubContext struct { + Spokes int `json:"spokes"` + Reporting int `json:"reporting"` + Window struct { + From string `json:"from"` + To string `json:"to"` + } `json:"window"` + RevertRateByAuthorKind map[string]struct { + N int `json:"n"` + Rate *float64 `json:"rate"` + Lo *float64 `json:"lo"` + Hi *float64 `json:"hi"` + } `json:"revert_rate_by_author_kind"` +} + +// Cohort is one side of the comparison. +type Cohort struct { + Name string + Merged int + // Matured is the revert/change-failure denominator: merged at least + // Maturation before Now. + Matured int + RevertRate Rate + ChangeFailure Rate + // TimeToMerge is the median hours from open to merge over the merged PRs + // whose bundle recorded an open time. + TimeToMerge Median + // CIFailureAtHead is the share of merged heads whose captured CI status + // was failure, over heads with a captured success/failure; CICoverage is + // how many merged PRs had one at all. A cohort whose coverage is low is + // a sample, not a population, and the renderer says so. + CIFailureAtHead Rate + CICoverage Rate + // ChangesRequested is the share of merged PRs that carried at least one + // changes_requested verdict on some head before merging; by definition + // zero and unreported for the unmediated cohort. + ChangesRequested Rate + // ByAuthorKind splits Merged by evidence author kind. + ByAuthorKind map[string]int +} + +// MonthPoint is one calendar month of one cohort for the trend chart. +type MonthPoint struct { + Month string + Merged int + RevertRate Rate + TimeToMerge Median +} + +// Report is the computed benchmark. +type Report struct { + GeneratedAt time.Time + Since, Until time.Time + Now time.Time + Source string + Bundles int + Unreadable int + PRs int + Repos []RepoCount + Verdicts int + Mediated Cohort + Unmediated Cohort + // FirstMediatedMerge is when the mediated cohort begins; before it every + // PR is unmediated by construction and the comparison is confounded by + // time, which the methodology section states. + FirstMediatedMerge time.Time + // Months and Trend cover the whole evidence history, not just the window. + Months []string + Trend map[string][]MonthPoint + Spend *Spend + Hub *HubContext +} + +// RepoCount is merged PRs per repository in the window. +type RepoCount struct { + Repo string + Merged int +} + +// Load reads every v1 bundle under root and returns them with the count of +// files that were not readable bundles. A missing root is empty, not an error. +func Load(root string) ([]*evidence.Bundle, int, error) { + var bundles []*evidence.Bundle + unreadable := 0 + err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return nil + } + return err + } + if d.IsDir() || filepath.Ext(path) != ".json" { + return nil + } + data, rerr := os.ReadFile(path) + if rerr != nil { + unreadable++ + return nil + } + var b evidence.Bundle + if json.Unmarshal(data, &b) != nil || b.SchemaVersion != evidence.SchemaVersion || b.Repo == "" || b.Number <= 0 { + unreadable++ + return nil + } + bundles = append(bundles, &b) + return nil + }) + if err != nil { + return nil, unreadable, fmt.Errorf("reading evidence under %s: %w", root, err) + } + return bundles, unreadable, nil +} + +// review is what the swarm recorded on a PR across every head, which the +// folded PR (newest bundle only) does not carry. +type review struct { + verdicts int + posted int + changesRequested bool +} + +// Mediated reports whether Hive's review swarm touched the PR: at least one +// recorded verdict or one posted review on any head. +func (r review) Mediated() bool { return r.verdicts > 0 || r.posted > 0 } + +// reviewsByPR folds verdicts and posted reviews over all heads of each PR. +func reviewsByPR(bundles []*evidence.Bundle) map[string]review { + out := map[string]review{} + for _, b := range bundles { + if b == nil { + continue + } + key := evidence.PRKey(b.Repo, b.Number) + r := out[key] + r.verdicts += len(b.Verdicts) + r.posted += len(b.PostedReviews) + for _, v := range b.Verdicts { + if v.Verdict == "changes_requested" { + r.changesRequested = true + } + } + out[key] = r + } + return out +} + +type cohortAcc struct { + merged, matured, reverted, failed int + ttm []float64 + ciFail, ciKnown int + changesRequested int + byKind map[string]int +} + +func (a *cohortAcc) add(p *evidence.PR, rv review, now time.Time) { + if a.byKind == nil { + a.byKind = map[string]int{} + } + a.merged++ + a.byKind[kindOrUnknown(p.Author.Kind)]++ + if !p.Merge.At.Add(Maturation).After(now) { + a.matured++ + reverted := len(p.RevertedBy) > 0 + if reverted { + a.reverted++ + } + if reverted || len(p.FollowedUpBy) > 0 { + a.failed++ + } + } + opened := p.OpenedAt + if opened.IsZero() { + opened = p.Merge.OpenedAt + } + if !opened.IsZero() && p.Merge.At.After(opened) { + a.ttm = append(a.ttm, p.Merge.At.Sub(opened).Hours()) + } + switch p.CI.Status { + case "failure": + a.ciFail++ + a.ciKnown++ + case "success": + a.ciKnown++ + } + if rv.changesRequested { + a.changesRequested++ + } +} + +func (a *cohortAcc) cohort(name string) Cohort { + return Cohort{ + Name: name, + Merged: a.merged, + Matured: a.matured, + RevertRate: NewRate(a.reverted, a.matured), + ChangeFailure: NewRate(a.failed, a.matured), + TimeToMerge: NewMedian(a.ttm), + CIFailureAtHead: NewRate(a.ciFail, a.ciKnown), + CICoverage: NewRate(a.ciKnown, a.merged), + ChangesRequested: NewRate(a.changesRequested, a.merged), + ByAuthorKind: a.byKind, + } +} + +func kindOrUnknown(k string) string { + if k == "" { + return "unknown" + } + return k +} + +// Compute folds the bundles per PR and builds the report over the merged +// PRs in the window. +func Compute(bundles []*evidence.Bundle, unreadable int, opts Options) Report { + now := opts.Now + if now.IsZero() { + now = time.Now().UTC() + } + prs := evidence.FoldPRs(bundles, "", "") + reviews := reviewsByPR(bundles) + rep := Report{ + GeneratedAt: now, Since: opts.Since, Until: opts.Until, Now: now, Source: opts.Source, + Bundles: len(bundles), Unreadable: unreadable, PRs: len(prs), + Spend: opts.Spend, Hub: opts.Hub, Trend: map[string][]MonthPoint{}, + } + var med, unmed cohortAcc + repos := map[string]int{} + months := map[string]map[string]*cohortAcc{} + for key, p := range prs { + rv := reviews[key] + rep.Verdicts += rv.verdicts + if p.Merge == nil { + continue + } + at := p.Merge.At + isMed := rv.Mediated() + if isMed && (rep.FirstMediatedMerge.IsZero() || at.Before(rep.FirstMediatedMerge)) { + rep.FirstMediatedMerge = at + } + name := CohortUnmediated + acc := &unmed + if isMed { + name, acc = CohortMediated, &med + } + // The trend spans the whole evidence history; only the headline + // cohorts are bounded by the window. + m := at.UTC().Format("2006-01") + if months[m] == nil { + months[m] = map[string]*cohortAcc{CohortMediated: {}, CohortUnmediated: {}} + } + months[m][name].add(p, rv, now) + if (!opts.Since.IsZero() && at.Before(opts.Since)) || (!opts.Until.IsZero() && !at.Before(opts.Until)) { + continue + } + repos[p.Repo]++ + acc.add(p, rv, now) + } + rep.Mediated = med.cohort(CohortMediated) + rep.Unmediated = unmed.cohort(CohortUnmediated) + for r, n := range repos { + rep.Repos = append(rep.Repos, RepoCount{Repo: r, Merged: n}) + } + sort.Slice(rep.Repos, func(i, j int) bool { + if rep.Repos[i].Merged != rep.Repos[j].Merged { + return rep.Repos[i].Merged > rep.Repos[j].Merged + } + return rep.Repos[i].Repo < rep.Repos[j].Repo + }) + for m := range months { + rep.Months = append(rep.Months, m) + } + sort.Strings(rep.Months) + for _, m := range rep.Months { + for _, name := range []string{CohortMediated, CohortUnmediated} { + c := months[m][name].cohort(name) + rep.Trend[name] = append(rep.Trend[name], MonthPoint{Month: m, Merged: c.Merged, RevertRate: c.RevertRate, TimeToMerge: c.TimeToMerge}) + } + } + return rep +} + +// LoadSpend reads a saved /api/cost response (or a minimal {usd,merged_prs} +// document) into a Spend. +func LoadSpend(path string) (*Spend, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, err + } + var raw struct { + Spend + Estimated struct { + TotalUSD float64 `json:"total_usd"` + } `json:"estimated"` + MergedPRs int `json:"merged_prs"` + CountWindowStart int64 `json:"count_window_start"` + CountWindowEnd int64 `json:"count_window_end"` + CountBasis string `json:"count_basis"` + } + if err := json.Unmarshal(data, &raw); err != nil { + return nil, fmt.Errorf("parsing %s: %w", path, err) + } + s := raw.Spend + if s.USD == 0 { + s.USD = raw.Estimated.TotalUSD + } + if s.MergedPRs == 0 { + s.MergedPRs = raw.MergedPRs + } + if s.WindowStart.IsZero() && raw.CountWindowStart > 0 { + s.WindowStart = time.UnixMilli(raw.CountWindowStart).UTC() + } + if s.WindowEnd.IsZero() && raw.CountWindowEnd > 0 { + s.WindowEnd = time.UnixMilli(raw.CountWindowEnd).UTC() + } + if s.Basis == "" { + s.Basis = raw.CountBasis + } + if s.USD <= 0 || s.MergedPRs <= 0 { + return nil, fmt.Errorf("%s: no spend or merged-PR count found", path) + } + return &s, nil +} + +// LoadHub reads a saved GET /api/hub/evidence response. +func LoadHub(path string) (*HubContext, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, err + } + var h HubContext + if err := json.Unmarshal(data, &h); err != nil { + return nil, fmt.Errorf("parsing %s: %w", path, err) + } + if h.Spokes == 0 && h.Reporting == 0 { + return nil, fmt.Errorf("%s: not a hub evidence view", path) + } + return &h, nil +} + +// ParseWindow turns a YYYY-MM month into its [since, until) bounds. +func ParseWindow(month string) (since, until time.Time, err error) { + since, err = time.Parse("2006-01", strings.TrimSpace(month)) + if err != nil { + return time.Time{}, time.Time{}, fmt.Errorf("invalid month %q; expected YYYY-MM", month) + } + return since, since.AddDate(0, 1, 0), nil +} diff --git a/src/pkg/outcomebench/drift_test.go b/src/pkg/outcomebench/drift_test.go new file mode 100644 index 0000000000..bfaea10263 --- /dev/null +++ b/src/pkg/outcomebench/drift_test.go @@ -0,0 +1,60 @@ +package outcomebench + +import ( + "encoding/json" + "os" + "path/filepath" + "sort" + "strings" + "testing" +) + +// TestCommittedReportMatchesReadme regenerates the newest committed report +// from its JSON and asserts the README outcomes block, the markdown and the +// SVG assets are exactly what the generator emits, so none of them can +// drift from the data. Fix with: src/scripts/outcome-benchmark.sh --readme +// (or hivectl evidence benchmark … --readme README.md). +func TestCommittedReportMatchesReadme(t *testing.T) { + docDir := filepath.Join("..", "..", "docs", "benchmarks") + reports, err := filepath.Glob(filepath.Join(docDir, "outcomes-*.json")) + if err != nil || len(reports) == 0 { + t.Skipf("no committed report under %s (%v)", docDir, err) + } + sort.Strings(reports) + latest := reports[len(reports)-1] + data, err := os.ReadFile(latest) + if err != nil { + t.Fatal(err) + } + var rep Report + if err := json.Unmarshal(data, &rep); err != nil { + t.Fatalf("%s: %v", latest, err) + } + want := rep.ReadmeBlock("src/docs/benchmarks") + + readme, err := os.ReadFile(filepath.Join("..", "..", "..", "README.md")) + if err != nil { + t.Fatal(err) + } + got, err := ExtractReadmeBlock(string(readme)) + if err != nil { + t.Fatal(err) + } + if got != want { + t.Errorf("README outcomes block drifted from %s; regenerate with src/scripts/outcome-benchmark.sh --readme\n--- README\n%s\n--- regenerated\n%s", latest, got, want) + } + base := strings.TrimSuffix(latest, ".json") + check := func(path, want string) { + t.Helper() + b, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if string(b) != want { + t.Errorf("%s drifted from %s; regenerate with src/scripts/outcome-benchmark.sh", path, latest) + } + } + check(base+".md", rep.Markdown()) + check(filepath.Join(docDir, "assets", rep.AssetBase()+"-bars.svg"), rep.BarChartSVG()) + check(filepath.Join(docDir, "assets", rep.AssetBase()+"-trend.svg"), rep.TrendSVG()) +} diff --git a/src/pkg/outcomebench/outcomebench_test.go b/src/pkg/outcomebench/outcomebench_test.go new file mode 100644 index 0000000000..1ef4ead385 --- /dev/null +++ b/src/pkg/outcomebench/outcomebench_test.go @@ -0,0 +1,331 @@ +package outcomebench + +import ( + "encoding/json" + "math" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/evidence" +) + +var now = time.Date(2026, 10, 27, 0, 0, 0, 0, time.UTC) + +func TestWilsonMatchesKnownValues(t *testing.T) { + lo, hi := Wilson(0, 41) + if lo != 0 || math.Abs(hi-0.0859) > 0.001 { + t.Fatalf("Wilson(0,41) = [%f, %f]", lo, hi) + } + lo, hi = Wilson(5, 10) + if math.Abs(lo-0.2366) > 0.001 || math.Abs(hi-0.7634) > 0.001 { + t.Fatalf("Wilson(5,10) = [%f, %f]", lo, hi) + } + if lo, hi := Wilson(0, 0); lo != 0 || hi != 0 { + t.Fatalf("Wilson(0,0) = [%f, %f]", lo, hi) + } +} + +func TestRateAndMedianRefuseSmallN(t *testing.T) { + if r := NewRate(3, MinN-1); r.Reported || r.Rate != 0 { + t.Fatalf("n= 1 { + t.Fatalf("n=MinN must be reported with interval: %+v", r) + } + if m := NewMedian([]float64{1, 2, 3}); m.Reported { + t.Fatalf("median below MinN reported: %+v", m) + } + xs := []float64{9, 1, 8, 2, 7, 3, 6, 4, 5, 10} + if m := NewMedian(xs); !m.Reported || m.Value != 6 || xs[0] != 9 { + t.Fatalf("median = %+v (input mutated: %v)", m, xs) + } + if !strings.Contains(pct(NewRate(1, 2)), NotReported) || !strings.Contains(hours(NewMedian(nil)), NotReported) { + t.Fatal("renderers must print not reported below MinN") + } +} + +// bundle builds a merged v1 bundle; verdict "" means unmediated. +func bundle(repo string, n int, merged time.Time, ttm time.Duration, verdict string, kind string) *evidence.Bundle { + b := &evidence.Bundle{SchemaVersion: evidence.SchemaVersion, ID: repo + "-" + time.Duration(n).String(), Repo: repo, Number: n, HeadSHA: "h", CreatedAt: merged} + b.Author.Kind = kind + b.Merge = &evidence.MergeEvent{At: merged, OpenedAt: merged.Add(-ttm), SHA: "m"} + if verdict != "" { + b.Verdicts = []evidence.Verdict{{Perspective: "p", Verdict: verdict}} + b.CI.Status = "success" + } + return b +} + +func fixture() []*evidence.Bundle { + var bs []*evidence.Bundle + oct := time.Date(2026, 10, 2, 0, 0, 0, 0, time.UTC) + for i := 1; i <= 12; i++ { + v := "approve" + if i%3 == 0 { + v = "changes_requested" + } + bs = append(bs, bundle("o/a", i, oct.Add(time.Duration(i)*time.Hour), 30*time.Minute, v, "agent")) + } + for i := 100; i < 112; i++ { + b := bundle("o/b", i, oct.Add(time.Duration(i)*time.Hour), 10*time.Minute, "", "human") + if i == 100 { + b.RevertedBy = []string{"o/b#200"} + } + if i == 101 { + b.FollowedUpBy = []string{"o/b#201"} + } + bs = append(bs, b) + } + // An earlier head with a changes_requested verdict on PR 1 and a merge + // too young to have matured; both must be folded correctly. + early := bundle("o/a", 1, oct.Add(time.Hour), 30*time.Minute, "changes_requested", "agent") + early.Merge, early.CreatedAt, early.HeadSHA = nil, oct, "h0" + bs = append(bs, early) + bs = append(bs, bundle("o/a", 50, now.Add(-time.Hour), time.Hour, "approve", "agent")) + // September history for the trend and a PR outside the window. + for i := 300; i < 311; i++ { + bs = append(bs, bundle("o/a", i, time.Date(2026, 9, 5, 0, 0, 0, 0, time.UTC).Add(time.Duration(i)*time.Minute), time.Hour, "", "bot")) + } + return bs +} + +func TestComputeCohorts(t *testing.T) { + since, until, err := ParseWindow("2026-10") + if err != nil { + t.Fatal(err) + } + rep := Compute(fixture(), 2, Options{Since: since, Until: until, Now: now, Source: "fixture", + Spend: &Spend{USD: 26, MergedPRs: 13}}) + m, u := rep.Mediated, rep.Unmediated + if m.Merged != 13 || u.Merged != 12 || rep.PRs != 36 || len(rep.Repos) != 2 || rep.Repos[0].Repo != "o/a" { + t.Fatalf("cohorts: med %d unmed %d prs %d repos %+v", m.Merged, u.Merged, rep.PRs, rep.Repos) + } + if m.Matured != 12 || m.RevertRate.K != 0 || !m.RevertRate.Reported { + t.Fatalf("mediated maturation: %+v", m) + } + if u.RevertRate.K != 1 || u.ChangeFailure.K != 2 || u.ChangeFailure.N != 12 { + t.Fatalf("unmediated links: revert %+v change-failure %+v", u.RevertRate, u.ChangeFailure) + } + if m.TimeToMerge.Value < 0.49 || m.TimeToMerge.Value > 1.01 || math.Abs(u.TimeToMerge.Value-1.0/6) > 1e-9 { + t.Fatalf("ttm: %+v / %+v", m.TimeToMerge, u.TimeToMerge) + } + // PRs 3,6,9,12 plus PR 1 via its earlier head. + if m.ChangesRequested.K != 5 || m.ChangesRequested.N != 13 { + t.Fatalf("changes requested: %+v", m.ChangesRequested) + } + if !m.CIFailureAtHead.Reported || m.CIFailureAtHead.K != 0 || u.CIFailureAtHead.Reported || u.CICoverage.K != 0 { + t.Fatalf("ci: %+v / %+v", m.CIFailureAtHead, u.CIFailureAtHead) + } + if m.ByAuthorKind["agent"] != 13 || u.ByAuthorKind["human"] != 12 || rep.Verdicts != 14 { + t.Fatalf("kinds %v %v verdicts %d", m.ByAuthorKind, u.ByAuthorKind, rep.Verdicts) + } + if rep.FirstMediatedMerge != oct(1) { + t.Fatalf("first mediated merge %s", rep.FirstMediatedMerge) + } + if strings.Join(rep.Months, ",") != "2026-09,2026-10" || rep.Trend[CohortUnmediated][0].Merged != 11 || rep.Trend[CohortMediated][1].Merged != 13 { + t.Fatalf("trend: %v %+v", rep.Months, rep.Trend) + } + if !rep.SplitSupported() || rep.Month() != "2026-10" || rep.AssetBase() != "outcomes-2026-10" { + t.Fatalf("split/month: %v %s", rep.SplitSupported(), rep.Month()) + } + if per, ok := rep.Spend.PerMergedPR(); !ok || per != 2 { + t.Fatalf("per merged PR %f %v", per, ok) + } + md := rep.Markdown() + for _, want := range []string{"# Outcome benchmark — 2026-10", "| Merged PRs in window | 13 | 12 |", "## Methodology", "≈$2.00 est.", "o/a 13, o/b 12", "not consulted", "assets/outcomes-2026-10-bars.svg", "flagged `changes_requested`"} { + if !strings.Contains(md, want) { + t.Errorf("markdown missing %q", want) + } + } + if strings.Contains(md, "does not support a clean") { + t.Error("split must be reported as supported") + } + for _, want := range []string{"Revert rates are indistinguishable", "Median time-to-merge", "not comparable", "per hive-attributed merged PR"} { + if !strings.Contains(strings.Join(rep.Findings(), "\n"), want) { + t.Errorf("findings missing %q: %v", want, rep.Findings()) + } + } +} + +func oct(h int) time.Time { return time.Date(2026, 10, 2, h, 0, 0, 0, time.UTC) } + +func TestComputeRefusesUnsupportedSplit(t *testing.T) { + bs := fixture()[:3] + rep := Compute(bs, 0, Options{Now: now}) + if rep.SplitSupported() { + t.Fatal("3 PRs cannot support a split") + } + md := rep.Markdown() + if !strings.Contains(md, "does not support a clean mediated-vs-unmediated split") || !strings.Contains(md, NotReported) { + t.Fatalf("markdown must say so:\n%s", md) + } + if !strings.Contains(strings.Join(rep.Findings(), ""), "nothing is claimed") { + t.Fatalf("findings: %v", rep.Findings()) + } + block := rep.ReadmeBlock("src/docs/benchmarks") + if !strings.Contains(block, "does not support a mediated-vs-unmediated comparison") || strings.Contains(block, "Revert rate") { + t.Fatalf("readme block must not claim a comparison:\n%s", block) + } + for _, r := range []Rate{rep.Mediated.RevertRate, rep.Unmediated.TimeToMergeRate()} { + if r.Reported { + t.Fatalf("reported below MinN: %+v", r) + } + } +} + +// TimeToMergeRate adapts the median to the Rate check above. +func (c Cohort) TimeToMergeRate() Rate { + return Rate{N: c.TimeToMerge.N, Reported: c.TimeToMerge.Reported} +} + +func TestSVGsAreWellFormedAndHonest(t *testing.T) { + since, until, _ := ParseWindow("2026-10") + rep := Compute(fixture(), 0, Options{Since: since, Until: until, Now: now}) + bars, trend := rep.BarChartSVG(), rep.TrendSVG() + for _, svg := range []string{bars, trend} { + if !strings.HasPrefix(svg, "\n") || strings.Count(svg, "2026-09<") || !strings.Contains(trend, ">2026-10<") || !strings.Contains(trend, "\n") { + t.Fatal("empty trend must still be a document") + } +} + +func TestReadmeBlockRoundTrip(t *testing.T) { + since, until, _ := ParseWindow("2026-10") + rep := Compute(fixture(), 0, Options{Since: since, Until: until, Now: now, Spend: &Spend{USD: 26, MergedPRs: 13}}) + block := rep.ReadmeBlock("src/docs/benchmarks") + for _, want := range []string{ReadmeStart, ReadmeEnd, "**Revert rate", "**Median time-to-merge", "flagged for changes", "≈$2.00", "36 pull requests across 2 repositories", "src/docs/benchmarks/outcomes-2026-10.md", "assets/outcomes-2026-10-bars.svg", "assets/outcomes-2026-10-trend.svg"} { + if !strings.Contains(block, want) { + t.Errorf("block missing %q:\n%s", want, block) + } + } + if n := strings.Count(block, "\n"); n > 14 { + t.Fatalf("block is %d lines; README additions must stay small", n+1) + } + readme := "# Hive\n\nintro\n\n" + ReadmeStart + "\nstale\n" + ReadmeEnd + "\n\n## Next\n" + updated, err := UpdateReadme(readme, block) + if err != nil { + t.Fatal(err) + } + if strings.Contains(updated, "stale") || !strings.HasPrefix(updated, "# Hive\n\nintro\n\n"+ReadmeStart) || !strings.HasSuffix(updated, ReadmeEnd+"\n\n## Next\n") { + t.Fatalf("update:\n%s", updated) + } + got, err := ExtractReadmeBlock(updated) + if err != nil || got != block { + t.Fatalf("extract mismatch (%v):\n%s", err, got) + } + if _, err := UpdateReadme("no markers", block); err != ErrNoMarkers { + t.Fatalf("expected ErrNoMarkers, got %v", err) + } + if _, err := ExtractReadmeBlock(ReadmeEnd + ReadmeStart); err != ErrNoMarkers { + t.Fatalf("reversed markers: %v", err) + } +} + +func TestLoadAndLoaders(t *testing.T) { + dir := t.TempDir() + write := func(rel string, v any) { + p := filepath.Join(dir, rel) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + data, _ := json.Marshal(v) + if err := os.WriteFile(p, data, 0o644); err != nil { + t.Fatal(err) + } + } + write("o/a/1/h.json", bundle("o/a", 1, now, time.Hour, "approve", "agent")) + write("o/a/2/h.json", map[string]any{"schema_version": "v0", "repo": "o/a", "number": 2}) + if err := os.WriteFile(filepath.Join(dir, "o/a/junk.json"), []byte("{"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "o/a/notes.txt"), []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + bs, unreadable, err := Load(dir) + if err != nil || len(bs) != 1 || unreadable != 2 { + t.Fatalf("Load = %d bundles, %d unreadable, %v", len(bs), unreadable, err) + } + if bs, _, err := Load(filepath.Join(dir, "missing")); err != nil || len(bs) != 0 { + t.Fatalf("missing root must be empty: %v", err) + } + + write("cost.json", map[string]any{"estimated": map[string]any{"total_usd": 6124.75}, "merged_prs": 2869, "count_window_start": 1786147200000, "count_window_end": 1793112641723, "count_basis": "hive-attributed"}) + sp, err := LoadSpend(filepath.Join(dir, "cost.json")) + if err != nil || sp.MergedPRs != 2869 || sp.Basis != "hive-attributed" || sp.WindowStart.Format("2006-01-02") != "2026-08-08" { + t.Fatalf("LoadSpend = %+v, %v", sp, err) + } + if per, ok := sp.PerMergedPR(); !ok || math.Abs(per-2.135) > 0.001 { + t.Fatalf("per merged PR %f", per) + } + write("minimal.json", map[string]any{"usd": 10, "merged_prs": 5}) + small, err := LoadSpend(filepath.Join(dir, "minimal.json")) + if err != nil || small.USD != 10 { + t.Fatalf("minimal spend: %+v %v", small, err) + } + if _, ok := small.PerMergedPR(); ok { + t.Fatal("5 merged PRs is below MinN") + } + write("empty.json", map[string]any{}) + if _, err := LoadSpend(filepath.Join(dir, "empty.json")); err == nil { + t.Fatal("empty spend must fail") + } + if _, err := LoadSpend(filepath.Join(dir, "o/a/junk.json")); err == nil { + t.Fatal("junk spend must fail") + } + if _, err := LoadSpend(filepath.Join(dir, "nope.json")); err == nil { + t.Fatal("missing spend must fail") + } + + write("hub.json", map[string]any{"spokes": 7, "reporting": 5, "window": map[string]string{"from": "2026-07-29", "to": "2026-10-27"}}) + hub, err := LoadHub(filepath.Join(dir, "hub.json")) + if err != nil || hub.Spokes != 7 || hub.Window.From != "2026-07-29" { + t.Fatalf("LoadHub = %+v, %v", hub, err) + } + rep := Compute(nil, 0, Options{Now: now, Hub: hub}) + if !strings.Contains(rep.Markdown(), "7 spokes registered, 5 reporting") { + t.Fatal("hub context must be quoted") + } + if _, err := LoadHub(filepath.Join(dir, "empty.json")); err == nil { + t.Fatal("empty hub must fail") + } + if _, err := LoadHub(filepath.Join(dir, "o/a/junk.json")); err == nil { + t.Fatal("junk hub must fail") + } + if _, err := LoadHub(filepath.Join(dir, "nope.json")); err == nil { + t.Fatal("missing hub must fail") + } + if _, _, err := ParseWindow("Oct 2026"); err == nil { + t.Fatal("bad month must fail") + } +} + +func TestOpenWindowUsesGenerationMonth(t *testing.T) { + rep := Compute(nil, 0, Options{Now: now}) + if rep.Month() != "2026-10" || rep.GeneratedAt != now { + t.Fatalf("month %s generated %s", rep.Month(), rep.GeneratedAt) + } + if !strings.Contains(rep.Markdown(), "Window: merged open → open") { + t.Fatal("open window must be stated") + } +} diff --git a/src/pkg/outcomebench/readme.go b/src/pkg/outcomebench/readme.go new file mode 100644 index 0000000000..d9e523e43c --- /dev/null +++ b/src/pkg/outcomebench/readme.go @@ -0,0 +1,79 @@ +package outcomebench + +import ( + "errors" + "fmt" + "strings" +) + +// README markers; everything between them is owned by the generator. +const ( + ReadmeStart = "" + ReadmeEnd = "" +) + +// ErrNoMarkers is returned when a README has no outcomes block to rewrite. +var ErrNoMarkers = errors.New("README has no " + ReadmeStart + " … " + ReadmeEnd + " block") + +// ReadmeBlock renders the README "Outcomes" section: 3–4 bold headline +// numbers, each with n and date and a one-line "how measured", plus the two +// generated charts. Only reported figures appear; docDir is the path from +// the README to the benchmark directory. +func (r Report) ReadmeBlock(docDir string) string { + report := fmt.Sprintf("%s/outcomes-%s.md", docDir, r.Month()) + var lines []string + add := func(s string) { lines = append(lines, s) } + add(ReadmeStart) + add(fmt.Sprintf("", report)) + m, u := r.Mediated, r.Unmediated + asOf := day(r.GeneratedAt) + if r.SplitSupported() { + if m.ChangesRequested.Reported { + add(fmt.Sprintf("- **%.0f%% of reviewed PRs were flagged for changes before they merged** — a `changes_requested` verdict on an earlier head of %d of %d mediated merges [%.1f–%.1f], %s.", + 100*m.ChangesRequested.Rate, m.ChangesRequested.K, m.ChangesRequested.N, 100*m.ChangesRequested.Lo, 100*m.ChangesRequested.Hi, asOf)) + } + if m.RevertRate.Reported && u.RevertRate.Reported { + add(fmt.Sprintf("- **Revert rate %.1f%% mediated vs %.1f%% unmediated** — %d/%d vs %d/%d merges matured %d days, Wilson [%.1f–%.1f] vs [%.1f–%.1f], %s.", + 100*m.RevertRate.Rate, 100*u.RevertRate.Rate, m.RevertRate.K, m.RevertRate.N, u.RevertRate.K, u.RevertRate.N, int(Maturation.Hours()/24), + 100*m.RevertRate.Lo, 100*m.RevertRate.Hi, 100*u.RevertRate.Lo, 100*u.RevertRate.Hi, asOf)) + } + if m.TimeToMerge.Reported && u.TimeToMerge.Reported { + add(fmt.Sprintf("- **Median time-to-merge %.0f min mediated vs %.0f min unmediated** — review costs about %.0f minutes on the median PR (n=%d vs %d), %s.", + 60*m.TimeToMerge.Value, 60*u.TimeToMerge.Value, 60*(m.TimeToMerge.Value-u.TimeToMerge.Value), m.TimeToMerge.N, u.TimeToMerge.N, asOf)) + } + } else { + add(fmt.Sprintf("- The current evidence does not support a mediated-vs-unmediated comparison (mediated n=%d, unmediated n=%d); only scale facts are shown.", m.Merged, u.Merged)) + } + if per, ok := r.Spend.PerMergedPR(); ok { + add(fmt.Sprintf("- **≈$%.2f estimated agent spend per merged PR** — `/api/cost` estimate over %d hive-attributed merges, %s → %s.", per, r.Spend.MergedPRs, day(r.Spend.WindowStart), day(r.Spend.WindowEnd))) + } + add(fmt.Sprintf("- **%d pull requests across %d repositories, %d review verdicts recorded** — the evidence this is measured from, %s.", r.PRs, len(r.Repos), r.Verdicts, asOf)) + add("") + add(fmt.Sprintf("How measured: per-PR evidence bundles the spoke already writes, folded per PR; 95%% Wilson intervals; n < %d is never reported. Full tables, trend and methodology in [%s](%s).", MinN, report, report)) + add("") + add(fmt.Sprintf("![mediated vs unmediated](%s/assets/%s-bars.svg)", docDir, r.AssetBase())) + add(fmt.Sprintf("![trend by month](%s/assets/%s-trend.svg)", docDir, r.AssetBase())) + add(ReadmeEnd) + return strings.Join(lines, "\n") +} + +// UpdateReadme replaces the marker block in readme with block. The block is +// expected to start with ReadmeStart and end with ReadmeEnd. +func UpdateReadme(readme, block string) (string, error) { + start := strings.Index(readme, ReadmeStart) + end := strings.Index(readme, ReadmeEnd) + if start < 0 || end < start { + return "", ErrNoMarkers + } + return readme[:start] + block + readme[end+len(ReadmeEnd):], nil +} + +// ExtractReadmeBlock returns the current marker block, markers included. +func ExtractReadmeBlock(readme string) (string, error) { + start := strings.Index(readme, ReadmeStart) + end := strings.Index(readme, ReadmeEnd) + if start < 0 || end < start { + return "", ErrNoMarkers + } + return readme[start : end+len(ReadmeEnd)], nil +} diff --git a/src/pkg/outcomebench/render.go b/src/pkg/outcomebench/render.go new file mode 100644 index 0000000000..a4fb62f604 --- /dev/null +++ b/src/pkg/outcomebench/render.go @@ -0,0 +1,206 @@ +package outcomebench + +import ( + "fmt" + "sort" + "strings" + "time" +) + +// NotReported is printed wherever n < MinN. +const NotReported = "not reported" + +func pct(r Rate) string { + if !r.Reported { + return fmt.Sprintf("%s (n=%d)", NotReported, r.N) + } + return fmt.Sprintf("%.1f%% [%.1f–%.1f] (n=%d)", 100*r.Rate, 100*r.Lo, 100*r.Hi, r.N) +} + +func hours(m Median) string { + if !m.Reported { + return fmt.Sprintf("%s (n=%d)", NotReported, m.N) + } + return fmt.Sprintf("%.2f h (n=%d)", m.Value, m.N) +} + +func day(t time.Time) string { + if t.IsZero() { + return "open" + } + return t.UTC().Format("2006-01-02") +} + +// Month is the YYYY-MM the report is filed under (its Since month, or the +// generation month when the window is open). +func (r Report) Month() string { + if !r.Since.IsZero() { + return r.Since.UTC().Format("2006-01") + } + return r.GeneratedAt.UTC().Format("2006-01") +} + +// AssetBase is the file stem the charts are written under. +func (r Report) AssetBase() string { return "outcomes-" + r.Month() } + +// SplitSupported reports whether both cohorts have at least MinN merged PRs, +// i.e. whether a mediated-vs-unmediated comparison is defensible at all. +func (r Report) SplitSupported() bool { + return r.Mediated.Merged >= MinN && r.Unmediated.Merged >= MinN +} + +// Markdown renders the full report, methodology included. +func (r Report) Markdown() string { + var b strings.Builder + w := func(f string, a ...any) { fmt.Fprintf(&b, f+"\n", a...) } + w("# Outcome benchmark — %s", r.Month()) + w("") + w("Generated %s from %s. Window: merged %s → %s; maturation %d days (observed as of %s).", + r.GeneratedAt.UTC().Format("2006-01-02"), r.Source, day(r.Since), day(r.Until), int(Maturation.Hours()/24), day(r.Now)) + w("") + w("Pull requests that went through Hive's review swarm (**mediated**: at least one recorded verdict or posted review) versus pull requests merged without it (**unmediated**). Every rate carries its n and a 95%% Wilson interval; anything with n < %d is `%s`.", MinN, NotReported) + w("") + if !r.SplitSupported() { + w("> **The data does not support a clean mediated-vs-unmediated split** (mediated n=%d, unmediated n=%d). Only the scale facts below are defensible.", r.Mediated.Merged, r.Unmediated.Merged) + w("") + } + w("## Headline") + w("") + w("| Metric | Mediated | Unmediated |") + w("|---|---|---|") + w("| Merged PRs in window | %d | %d |", r.Mediated.Merged, r.Unmediated.Merged) + w("| Revert rate (matured) | %s | %s |", pct(r.Mediated.RevertRate), pct(r.Unmediated.RevertRate)) + w("| Change-failure rate (revert or follow-up fix, matured) | %s | %s |", pct(r.Mediated.ChangeFailure), pct(r.Unmediated.ChangeFailure)) + w("| Median time-to-merge | %s | %s |", hours(r.Mediated.TimeToMerge), hours(r.Unmediated.TimeToMerge)) + w("| CI failure at merged head | %s | %s |", ciCell(r.Mediated), ciCell(r.Unmediated)) + w("| Had a `changes_requested` verdict before merge | %s | — (no verdicts by definition) |", pct(r.Mediated.ChangesRequested)) + w("| Cost per merged PR | %s | — |", r.costCell()) + w("") + w("![mediated vs unmediated](assets/%s-bars.svg)", r.AssetBase()) + w("") + w("### Reading the numbers") + w("") + for _, f := range r.Findings() { + w("- %s", f) + } + w("") + w("## Trend by month") + w("") + w("Whole evidence history, not just the window; months with n < %d are gaps.", MinN) + w("") + w("![trend](assets/%s-trend.svg)", r.AssetBase()) + w("") + w("| Month | Mediated merged | Mediated revert | Mediated TTM | Unmediated merged | Unmediated revert | Unmediated TTM |") + w("|---|---|---|---|---|---|---|") + for i, m := range r.Months { + a, u := r.Trend[CohortMediated][i], r.Trend[CohortUnmediated][i] + w("| %s | %d | %s | %s | %d | %s | %s |", m, a.Merged, pct(a.RevertRate), hours(a.TimeToMerge), u.Merged, pct(u.RevertRate), hours(u.TimeToMerge)) + } + w("") + w("## Scale") + w("") + w("- %d evidence bundles (%d unreadable) folding to %d pull requests across %d repositories; %d verdicts recorded.", r.Bundles, r.Unreadable, r.PRs, len(r.Repos), r.Verdicts) + w("- Merged in window by repository: %s.", r.repoList()) + w("- Merged in window by author kind — mediated: %s; unmediated: %s.", kinds(r.Mediated.ByAuthorKind), kinds(r.Unmediated.ByAuthorKind)) + if !r.FirstMediatedMerge.IsZero() { + w("- First mediated merge in the evidence: %s.", day(r.FirstMediatedMerge)) + } + if r.Hub != nil { + w("- Hub fleet view: %d spokes registered, %d reporting, window %s → %s.", r.Hub.Spokes, r.Hub.Reporting, r.Hub.Window.From, r.Hub.Window.To) + } else { + w("- Hub `GET /api/hub/evidence` was not consulted (it requires an admin session); numbers are this spoke's evidence only.") + } + w("") + w("## Methodology") + w("") + w("- **Source.** Only the per-PR evidence bundles the spoke already writes under `/data/evidence` (schema v1), folded per PR with `pkg/evidence.FoldPRs`. No new telemetry.") + w("- **Cohorts.** A PR is *mediated* if any head carries a Hive verdict or a posted Hive review; otherwise *unmediated*. Backfilled history is unmediated by construction, so the window must start no earlier than the first mediated merge or the comparison is confounded by time.") + w("- **Revert / change-failure.** Denominator is merged PRs observed at least %d days after merge. A revert is a later PR the evidence linker tagged `reverted_by`; change-failure adds `followed_up_by` fixes.", int(Maturation.Hours()/24)) + w("- **Time-to-merge.** Lower median of merge time minus open time, over merged PRs whose bundle recorded an open time.") + w("- **CI failure at head.** Share of merged heads with a captured `failure` status among heads with a captured success/failure. The unmediated cohort rarely has a captured status (Hive only records CI for heads it reviews), so its cell reports coverage and is not comparable when coverage is low.") + w("- **Changes requested.** Share of mediated merged PRs with at least one `changes_requested` verdict on any head before the merge — findings the author had a chance to address pre-merge. Whether the later head fixed the finding, or a human overrode it, is not verified here.") + w("- **Cost.** `/api/cost` estimated spend divided by its hive-attributed merged-PR count over the same window; model prices are estimates and some models are unpriced, so treat it as an upper-bound order of magnitude.") + w("- **Intervals.** 95%% Wilson score; n < %d is never reported.", MinN) + w("- **Regenerate.** `src/scripts/outcome-benchmark.sh --month %s` (see `docs/benchmarks/README.md`).", r.Month()) + return b.String() +} + +func ciCell(c Cohort) string { + if !c.CIFailureAtHead.Reported { + return fmt.Sprintf("%s (captured on %d of %d)", NotReported, c.CICoverage.K, c.CICoverage.N) + } + cov := "" + if c.CICoverage.Reported && c.CICoverage.Rate < 0.5 { + cov = fmt.Sprintf(" — captured on only %.0f%% of merges, not comparable", 100*c.CICoverage.Rate) + } + return pct(c.CIFailureAtHead) + cov +} + +func (r Report) costCell() string { + per, ok := r.Spend.PerMergedPR() + if !ok { + return NotReported + " (no spend data)" + } + return fmt.Sprintf("≈$%.2f est. (n=%d merged, %s → %s)", per, r.Spend.MergedPRs, day(r.Spend.WindowStart), day(r.Spend.WindowEnd)) +} + +func (r Report) repoList() string { + parts := make([]string, 0, len(r.Repos)) + for _, rc := range r.Repos { + parts = append(parts, fmt.Sprintf("%s %d", rc.Repo, rc.Merged)) + } + if len(parts) == 0 { + return "none" + } + return strings.Join(parts, ", ") +} + +func kinds(m map[string]int) string { + if len(m) == 0 { + return "none" + } + ks := make([]string, 0, len(m)) + for k := range m { + ks = append(ks, k) + } + sort.Strings(ks) + parts := make([]string, 0, len(ks)) + for _, k := range ks { + parts = append(parts, fmt.Sprintf("%s %d", k, m[k])) + } + return strings.Join(parts, ", ") +} + +// Findings are the plain-language sentences a reader should take away, +// derived only from reported figures. +func (r Report) Findings() []string { + var out []string + m, u := r.Mediated, r.Unmediated + if m.RevertRate.Reported && u.RevertRate.Reported { + switch { + case m.RevertRate.Hi < u.RevertRate.Lo: + out = append(out, fmt.Sprintf("Mediated PRs are reverted less often (%s vs %s); the intervals do not overlap.", pct(m.RevertRate), pct(u.RevertRate))) + case u.RevertRate.Hi < m.RevertRate.Lo: + out = append(out, fmt.Sprintf("Mediated PRs are reverted more often (%s vs %s); the intervals do not overlap.", pct(m.RevertRate), pct(u.RevertRate))) + default: + out = append(out, fmt.Sprintf("Revert rates are indistinguishable at this n: %s mediated vs %s unmediated (overlapping intervals).", pct(m.RevertRate), pct(u.RevertRate))) + } + } + if m.TimeToMerge.Reported && u.TimeToMerge.Reported { + d := m.TimeToMerge.Value - u.TimeToMerge.Value + out = append(out, fmt.Sprintf("Median time-to-merge is %s for mediated vs %s for unmediated: review adds about %.0f minutes to the median PR.", hours(m.TimeToMerge), hours(u.TimeToMerge), 60*d)) + } + if m.ChangesRequested.Reported { + out = append(out, fmt.Sprintf("%s of mediated merged PRs were flagged `changes_requested` on an earlier head before merging — findings an unmediated PR would have carried to main unseen.", pct(m.ChangesRequested))) + } + if m.CIFailureAtHead.Reported && !(u.CIFailureAtHead.Reported && u.CICoverage.Rate >= 0.5) { + out = append(out, fmt.Sprintf("CI at merged head is only captured for reviewed heads (mediated %s); the unmediated cell is not comparable.", pct(m.CIFailureAtHead))) + } + if per, ok := r.Spend.PerMergedPR(); ok { + out = append(out, fmt.Sprintf("Estimated agent spend is ≈$%.2f per hive-attributed merged PR (n=%d).", per, r.Spend.MergedPRs)) + } + if len(out) == 0 { + out = append(out, "No metric reached n="+fmt.Sprint(MinN)+" in both cohorts; nothing is claimed.") + } + return out +} diff --git a/src/pkg/outcomebench/sort.go b/src/pkg/outcomebench/sort.go new file mode 100644 index 0000000000..551fe47bd8 --- /dev/null +++ b/src/pkg/outcomebench/sort.go @@ -0,0 +1,5 @@ +package outcomebench + +import "sort" + +func sortFloats(s []float64) { sort.Float64s(s) } diff --git a/src/pkg/outcomebench/svg.go b/src/pkg/outcomebench/svg.go new file mode 100644 index 0000000000..db1c54c407 --- /dev/null +++ b/src/pkg/outcomebench/svg.go @@ -0,0 +1,222 @@ +package outcomebench + +import ( + "fmt" + "html" + "math" + "strings" +) + +// Chart palette: Hive's brand amber for the mediated series and a neutral +// slate for the unmediated one, both legible on GitHub's light and dark +// backgrounds. Text uses currentColor-free mid greys for the same reason. +const ( + colourMediated = "#f4c75f" + colourUnmediated = "#8a97a8" + colourAxis = "#7d8a9b" + colourText = "#8a97a8" + colourGrid = "#7d8a9b" +) + +const ( + svgWidth = 760 + svgHeight = 300 + svgPadL = 56 + svgPadR = 24 + svgPadT = 36 + svgPadB = 56 + svgFont = "ui-sans-serif, system-ui, sans-serif" + svgFontPx = 12 + barGap = 10 + groupGap = 36 + markerSize = 3.5 +) + +type barMetric struct { + label string + med, u Rate +} + +// minCIComparableCoverage is the share of merged PRs that must have a +// captured CI status before a cohort's CI-failure rate is drawn as a bar +// rather than a placeholder: below it the figure is a sample of reviewed +// heads, not the cohort. +const minCIComparableCoverage = 0.5 + +func (r Report) barMetrics() []barMetric { + ci := func(c Cohort) Rate { + if !c.CICoverage.Reported || c.CICoverage.Rate < minCIComparableCoverage { + return Rate{N: c.CIFailureAtHead.N} + } + return c.CIFailureAtHead + } + return []barMetric{ + {"Revert rate", r.Mediated.RevertRate, r.Unmediated.RevertRate}, + {"Change-failure rate", r.Mediated.ChangeFailure, r.Unmediated.ChangeFailure}, + {"CI failure at head", ci(r.Mediated), ci(r.Unmediated)}, + {"Changes requested pre-merge", r.Mediated.ChangesRequested, Rate{}}, + } +} + +// niceHours is niceMax for an hours axis. +func niceHours(v float64) float64 { + for _, m := range []float64{0.5, 1, 2, 4, 8, 24, 48, 168} { + if v <= m { + return m + } + } + return math.Ceil(v/168) * 168 +} + +// niceMax picks the smallest axis maximum from a short list of round values +// so the quarter gridlines land on readable percentages. +func niceMax(v float64) float64 { + for _, m := range []float64{0.05, 0.1, 0.2, 0.4, 0.6, 0.8, 1} { + if v <= m { + return m + } + } + return 1 +} + +func svgHeader(title string) string { + return fmt.Sprintf(` +%s +%s +`, svgWidth, svgHeight, svgWidth, svgHeight, html.EscapeString(title), svgFont, svgFontPx, html.EscapeString(title), svgPadL, colourText, html.EscapeString(title)) +} + +func legend(b *strings.Builder, x, y int) { + fmt.Fprintf(b, `mediated`+"\n", x, y, colourMediated, x+16, y+10, colourText) + fmt.Fprintf(b, `unmediated`+"\n", x+90, y, colourUnmediated, x+106, y+10, colourText) +} + +// BarChartSVG draws mediated vs unmediated per rate metric with Wilson +// whiskers. Unreported bars are drawn as a hatched placeholder labelled n. +func (r Report) BarChartSVG() string { + metrics := r.barMetrics() + var b strings.Builder + b.WriteString(svgHeader("Mediated vs unmediated — " + r.Month() + " (95% Wilson intervals)")) + legend(&b, svgWidth-svgPadR-200, 8) + plotW := svgWidth - svgPadL - svgPadR + plotH := svgHeight - svgPadT - svgPadB + maxV := 0.05 + for _, m := range metrics { + for _, rt := range []Rate{m.med, m.u} { + if rt.Reported && rt.Hi > maxV { + maxV = rt.Hi + } + } + } + maxV = niceMax(maxV) + y := func(v float64) float64 { return float64(svgPadT+plotH) - v/maxV*float64(plotH) } + for i := 0; i <= 4; i++ { + v := maxV * float64(i) / 4 + fmt.Fprintf(&b, `%.0f%%`+"\n", + svgPadL, y(v), svgWidth-svgPadR, y(v), colourGrid, svgPadL-6, y(v)+4, colourText, 100*v) + } + groupW := (plotW - groupGap*(len(metrics)-1)) / len(metrics) + barW := (groupW - barGap) / 2 + for gi, m := range metrics { + gx := svgPadL + gi*(groupW+groupGap) + drawBar := func(x int, rt Rate, fill string, skip bool) { + if skip { + return + } + if !rt.Reported { + fmt.Fprintf(&b, `n=%d`+"\n", + x, y(0)-18, barW, 18, fill, x+barW/2, y(0)-6, colourText, rt.N) + return + } + top := y(rt.Rate) + fmt.Fprintf(&b, ``+"\n", x, top, barW, y(0)-top, fill) + cx := x + barW/2 + fmt.Fprintf(&b, ``+"\n", + cx, y(rt.Lo), cx, y(rt.Hi), colourAxis, cx-5, y(rt.Lo), cx+5, y(rt.Lo), colourAxis, cx-5, y(rt.Hi), cx+5, y(rt.Hi), colourAxis) + fmt.Fprintf(&b, `%.1f%% (n=%d)`+"\n", cx, y(rt.Hi)-4, colourText, 100*rt.Rate, rt.N) + } + drawBar(gx, m.med, colourMediated, false) + drawBar(gx+barW+barGap, m.u, colourUnmediated, m.label == "Changes requested pre-merge") + fmt.Fprintf(&b, `%s`+"\n", gx+groupW/2, svgPadT+plotH+18, colourText, html.EscapeString(m.label)) + } + fmt.Fprintf(&b, ``+"\n", svgPadL, y(0), svgWidth-svgPadR, y(0), colourAxis) + fmt.Fprintf(&b, `Hatched = n < %d or CI not captured for most of the cohort: not reported. Change-failure = revert or follow-up fix within %d days.`+"\n", svgPadL, svgHeight-10, colourText, MinN, int(Maturation.Hours()/24)) + b.WriteString("\n") + return b.String() +} + +// TrendSVG draws median time-to-merge per month per cohort as lines, with +// the monthly revert rate annotated where reported. Months below MinN are +// gaps, not zeros. +func (r Report) TrendSVG() string { + var b strings.Builder + b.WriteString(svgHeader("Median time-to-merge by month (hours) — revert rate annotated")) + legend(&b, svgWidth-svgPadR-200, 8) + plotW := svgWidth - svgPadL - svgPadR + plotH := svgHeight - svgPadT - svgPadB + maxV := 0.5 + for _, pts := range r.Trend { + for _, p := range pts { + if p.TimeToMerge.Reported && p.TimeToMerge.Value > maxV { + maxV = p.TimeToMerge.Value + } + } + } + maxV = niceHours(maxV) + n := len(r.Months) + x := func(i int) float64 { + if n <= 1 { + return float64(svgPadL + plotW/2) + } + return float64(svgPadL) + float64(i)*float64(plotW)/float64(n-1) + } + y := func(v float64) float64 { return float64(svgPadT+plotH) - v/maxV*float64(plotH) } + for i := 0; i <= 5; i++ { + v := maxV * float64(i) / 5 + fmt.Fprintf(&b, `%.2gh`+"\n", + svgPadL, y(v), svgWidth-svgPadR, y(v), colourGrid, svgPadL-6, y(v)+4, colourText, v) + } + for i, m := range r.Months { + fmt.Fprintf(&b, `%s`+"\n", x(i), svgPadT+plotH+18, colourText, m) + } + for _, name := range []string{CohortUnmediated, CohortMediated} { + colour := colourMediated + if name == CohortUnmediated { + colour = colourUnmediated + } + var path []string + for i, p := range r.Trend[name] { + if !p.TimeToMerge.Reported { + path = nil + continue + } + cmd := "L" + if len(path) == 0 { + cmd = "M" + } + path = append(path, fmt.Sprintf("%s%.1f %.1f", cmd, x(i), y(p.TimeToMerge.Value))) + fmt.Fprintf(&b, ``+"\n", x(i), y(p.TimeToMerge.Value), markerSize, colour) + label := fmt.Sprintf("%.2fh", p.TimeToMerge.Value) + if p.RevertRate.Reported { + label += fmt.Sprintf(" · rev %.1f%%", 100*p.RevertRate.Rate) + } + anchor := "middle" + switch { + case i == 0 && n > 1: + anchor = "start" + case i == n-1 && n > 1: + anchor = "end" + } + fmt.Fprintf(&b, `%s`+"\n", x(i), y(p.TimeToMerge.Value)-8, colourText, anchor, label) + if len(path) > 1 { + fmt.Fprintf(&b, ``+"\n", strings.Join(path, " "), colour) + path = path[len(path)-1:] + path[0] = "M" + path[0][1:] + } + } + } + fmt.Fprintf(&b, ``+"\n", svgPadL, y(0), svgWidth-svgPadR, y(0), colourAxis) + fmt.Fprintf(&b, `Months with n < %d are gaps. Mediated series starts at the first mediated merge.`+"\n", svgPadL, svgHeight-10, colourText, MinN) + b.WriteString("\n") + return b.String() +} diff --git a/src/pkg/outcomebench/wilson.go b/src/pkg/outcomebench/wilson.go new file mode 100644 index 0000000000..1902235ce6 --- /dev/null +++ b/src/pkg/outcomebench/wilson.go @@ -0,0 +1,64 @@ +package outcomebench + +import "math" + +// MinN is the smallest sample a rate or median is reported for; below it the +// figure is "not reported", never a number. It matches the hub's evidence +// panel (pkg/hub minEvidenceN). +const MinN = 10 + +// Wilson returns the 95% Wilson score interval for k successes in n trials. +// It is the same formula the hub's evidence panel uses (pkg/hub/evidence.go +// wilson); it lives here too so the benchmark does not import the hub. +func Wilson(k, n int) (lo, hi float64) { + if n <= 0 { + return 0, 0 + } + const z = 1.96 + nf, p := float64(n), float64(k)/float64(n) + d := 1 + z*z/nf + centre := (p + z*z/(2*nf)) / d + half := z * math.Sqrt(p*(1-p)/nf+z*z/(4*nf*nf)) / d + return math.Max(0, centre-half), math.Min(1, centre+half) +} + +// Rate is k/n with its Wilson interval. Reported is false when n < MinN; a +// renderer must then print "not reported" and nothing numeric. +type Rate struct { + K, N int + Rate float64 + Lo, Hi float64 + Reported bool +} + +// NewRate builds a Rate, leaving it unreported below MinN. +func NewRate(k, n int) Rate { + r := Rate{K: k, N: n} + if n < MinN { + return r + } + r.Rate = float64(k) / float64(n) + r.Lo, r.Hi = Wilson(k, n) + r.Reported = true + return r +} + +// Median is a median over n samples, unreported below MinN. +type Median struct { + Value float64 + N int + Reported bool +} + +// NewMedian sorts a copy of xs and takes the lower median. +func NewMedian(xs []float64) Median { + m := Median{N: len(xs)} + if len(xs) < MinN { + return m + } + s := append([]float64(nil), xs...) + sortFloats(s) + m.Value = s[len(s)/2] + m.Reported = true + return m +} diff --git a/src/scripts/outcome-benchmark.sh b/src/scripts/outcome-benchmark.sh new file mode 100755 index 0000000000..7931141c4e --- /dev/null +++ b/src/scripts/outcome-benchmark.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# Regenerate the mediated-vs-unmediated outcome benchmark from stored review +# evidence and, with --readme, the README's block. +# +# src/scripts/outcome-benchmark.sh --root ./evidence --month 2026-10 --spend cost.json --readme +# +# Flags are passed through to `hivectl evidence benchmark`; --readme (no +# argument) rewrites the repository README. Defaults: --root /data/evidence, +# --month = current UTC month, --out-dir src/docs/benchmarks. A saved +# GET /api/cost response (--spend) adds cost per merged PR; a saved +# GET /api/hub/evidence response (--hub) adds fleet context. Nothing here +# contacts a hive or adds telemetry; a test (pkg/outcomebench +# TestCommittedReportMatchesReadme) fails when the README block, report or +# charts no longer match the committed JSON. +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +cd "$repo_root/src" + +args=() +readme="" +month="" +out_dir="docs/benchmarks" +while [ $# -gt 0 ]; do + case "$1" in + --readme) readme="$repo_root/README.md"; shift ;; + --month) month="$2"; shift 2 ;; + --out-dir) out_dir="$2"; shift 2 ;; + -h|--help) sed -n '2,15p' "$0"; exit 0 ;; + *) args+=("$1"); shift ;; + esac +done +[ -n "$month" ] || month="$(date -u +%Y-%m)" + +cmd=(go run ./cmd/hivectl evidence benchmark --month "$month" --out-dir "$out_dir") +[ -n "$readme" ] && cmd+=(--readme "$readme") +exec "${cmd[@]}" "${args[@]}"