From 079f7fbb86d4fe1385e4fc64d39f0ac11dbab5ff Mon Sep 17 00:00:00 2001
From: Andrew Anderson
Date: Tue, 27 Oct 2026 12:51:38 -0400
Subject: [PATCH 1/6] docs: rewrite README around plain-language evidence cards
Generate dark/light SVG cards from the committed outcome snapshot (sent
back before merge, two real pull requests, prompt injection, cost, an
also strip and a how-it-works ladder); the README block lists Accuracy,
Security, Cost and Also with no statistical vocabulary and a minimum
base of 200 for production counts. Detailed charts and intervals stay in
src/docs/benchmarks; the former overview moves to src/docs/what-is-hive.md.
Adds prose-budget, vocabulary and card tests.
Fixes #14602
Signed-off-by: Andrew Anderson
---
CONTRIBUTING.md | 2 +-
README.md | 192 ++-----
bin/README.md | 2 +-
changelog.d/changed-readme-evidence-cards.md | 1 +
examples/kubestellar/README.md | 2 +-
security-insights.yml | 2 +-
src/docs/benchmarks/README.md | 20 +-
.../outcomes-2026-10-card-also-dark.svg | 29 +
.../outcomes-2026-10-card-also-light.svg | 29 +
.../outcomes-2026-10-card-cost-dark.svg | 11 +
.../outcomes-2026-10-card-cost-light.svg | 11 +
...utcomes-2026-10-card-how-it-works-dark.svg | 36 ++
...tcomes-2026-10-card-how-it-works-light.svg | 36 ++
...mes-2026-10-card-prompt-injection-dark.svg | 11 +
...es-2026-10-card-prompt-injection-light.svg | 11 +
.../outcomes-2026-10-card-sent-back-dark.svg | 10 +
.../outcomes-2026-10-card-sent-back-light.svg | 10 +
.../outcomes-2026-10-card-stories-dark.svg | 30 +
.../outcomes-2026-10-card-stories-light.svg | 30 +
src/docs/benchmarks/cost-2026-10.json | 1 +
src/docs/benchmarks/outcomes-2026-10.json | 33 +-
src/docs/benchmarks/outcomes-2026-10.md | 21 +
src/docs/benchmarks/red-team-2026-10.json | 8 +
src/docs/benchmarks/stories-2026-10.json | 22 +
src/docs/documentation-map.md | 3 +-
src/docs/hivectl.md | 12 +-
src/docs/what-is-hive.md | 213 +++++++
.../hivectl/commands/evidence_benchmark.go | 39 +-
src/pkg/outcomebench/bench.go | 11 +
src/pkg/outcomebench/cards.go | 518 ++++++++++++++++++
src/pkg/outcomebench/cards_test.go | 114 ++++
src/pkg/outcomebench/drift_test.go | 14 +
src/pkg/outcomebench/outcomebench_test.go | 18 +-
src/pkg/outcomebench/readme.go | 102 +++-
src/pkg/outcomebench/readme_budget_test.go | 125 +++++
src/pkg/outcomebench/render.go | 56 ++
36 files changed, 1583 insertions(+), 202 deletions(-)
create mode 100644 changelog.d/changed-readme-evidence-cards.md
create mode 100644 src/docs/benchmarks/assets/outcomes-2026-10-card-also-dark.svg
create mode 100644 src/docs/benchmarks/assets/outcomes-2026-10-card-also-light.svg
create mode 100644 src/docs/benchmarks/assets/outcomes-2026-10-card-cost-dark.svg
create mode 100644 src/docs/benchmarks/assets/outcomes-2026-10-card-cost-light.svg
create mode 100644 src/docs/benchmarks/assets/outcomes-2026-10-card-how-it-works-dark.svg
create mode 100644 src/docs/benchmarks/assets/outcomes-2026-10-card-how-it-works-light.svg
create mode 100644 src/docs/benchmarks/assets/outcomes-2026-10-card-prompt-injection-dark.svg
create mode 100644 src/docs/benchmarks/assets/outcomes-2026-10-card-prompt-injection-light.svg
create mode 100644 src/docs/benchmarks/assets/outcomes-2026-10-card-sent-back-dark.svg
create mode 100644 src/docs/benchmarks/assets/outcomes-2026-10-card-sent-back-light.svg
create mode 100644 src/docs/benchmarks/assets/outcomes-2026-10-card-stories-dark.svg
create mode 100644 src/docs/benchmarks/assets/outcomes-2026-10-card-stories-light.svg
create mode 100644 src/docs/benchmarks/cost-2026-10.json
create mode 100644 src/docs/benchmarks/red-team-2026-10.json
create mode 100644 src/docs/benchmarks/stories-2026-10.json
create mode 100644 src/docs/what-is-hive.md
create mode 100644 src/pkg/outcomebench/cards.go
create mode 100644 src/pkg/outcomebench/cards_test.go
create mode 100644 src/pkg/outcomebench/readme_budget_test.go
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index e04c8ecea3..f6e656fa4d 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -1,6 +1,6 @@
# Contributing to Hive
-Thank you for helping improve Hive. This guide is for contributing code and documentation to this repository. If you want to donate compute to a running hive, see [Contribute to a Hive](README.md#contribute-to-a-hive) instead.
+Thank you for helping improve Hive. This guide is for contributing code and documentation to this repository. If you want to donate compute to a running hive, see [Contribute to a Hive](src/docs/what-is-hive.md#contribute-to-a-hive) instead.
**New here?** Start with the [getting-started guide for first-time contributors](docs/getting-started-contributing.md) — it walks the end-to-end journey (finding an issue, local setup, testing without a cluster, key concepts, and what the review/CI process looks like) and links back into this guide for the mechanics. For live community help, [Join our Discord](https://hivecommons.dev/discord).
diff --git a/README.md b/README.md
index d796a31864..b34a91ee32 100644
--- a/README.md
+++ b/README.md
@@ -7,184 +7,76 @@
[](https://www.bestpractices.dev/projects/14261)
[](LICENSE)
-Hive is a co-maintainer for your GitHub project that never sleeps. Point it at a repo, set the limits, and let short-lived AI agents handle the maintenance chores while humans keep the decisions that matter.
-
-## What Hive is
-
-Hive watches the work that piles up in a repository — issues, pull requests, failing CI, CVEs, and dependabot alerts — and sends AI coding agents (Claude, Copilot, Codex, Antigravity, Goose…) to deal with it: triage, fixes, reviews, releases. You decide how much it is allowed to do. Humans keep every decision that matters; agents do the chores.
-
-## "I already use an agent. How is this different?"
-
-A terminal agent is a power tool: you hold it, aim it, and it stops when you stop. Hive is the workshop that runs the power tools for you — on three rules the industry is only now discovering:
-
-1. **Short-lived agents.** Every task starts a fresh agent with a clean context and ends it. No "long-running" agent drifting for hours.
-2. **External context.** What the agent needs to know lives outside the model — in your repo, your knowledge base, your audit trail — and is re-injected each time, so nothing is forgotten and nothing is hallucinated from stale memory.
-3. **Prompts driven by deterministic facts.** The instructions an agent gets are templates filled from real data (the issue, the CI log, the diff, the trust level), not hand-tuned prose.
-
-Around that, Hive picks work from your real queue, sets limits in code with six trust levels from the **AI Codebase Maturity Model (ACMM)** — a framework for how much autonomy a codebase is ready to give agents ([levels](#acmm-levels) · [paper](https://arxiv.org/abs/2604.09388)), checks before and after with deterministic filters and gates, writes every action to an audit log owned by a named human, and runs around the clock. Same agents you use today — governed, scheduled, scoped, and accountable.
-
-## Outcomes
-
-Measured from the review evidence Hive already records, not from a demo. Every figure carries its n and a 95% Wilson interval; anything with n < 10 is not reported. Regenerate with `src/scripts/outcome-benchmark.sh --readme`; a test fails if this block drifts from the committed report.
+Hive points short-lived coding agents at a repository's issues, pull requests and failing checks, reviews what they produce, and leaves the decisions to humans.
-- **34% of reviewed PRs were flagged for changes before they merged** — a `changes_requested` verdict on an earlier head of 490 of 1455 mediated merges [31.3–36.1], 2026-10-27.
-- **Revert rate 0.0% mediated vs 0.1% unmediated** — 0/43 vs 1/1513 merges matured 14 days, Wilson [0.0–8.2] vs [0.0–0.4], 2026-10-27.
-- **Median time-to-merge 27 min mediated vs 12 min unmediated** — review costs about 15 minutes on the median PR (n=1455 vs 2224), 2026-10-27.
-- **≈$2.13 estimated agent spend per merged PR** — `/api/cost` estimate over 2869 hive-attributed merges, 2026-08-08 → 2026-10-27.
-- **11571 pull requests across 10 repositories, 15054 review verdicts recorded** — the evidence this is measured from, 2026-10-27.
+Measured on 11,571 pull requests across 10 repositories and 15,054 review verdicts, as of 2026-10-27.
-How measured: per-PR evidence bundles the spoke already writes, folded per PR; 95% Wilson intervals; n < 10 is never reported. Full tables, trend and methodology in [src/docs/benchmarks/outcomes-2026-10.md](src/docs/benchmarks/outcomes-2026-10.md).
+## Accuracy
-
-
-
-
-**One real example.** On [hivecommons/hive#13523](https://github.com/hivecommons/hive/pull/13523) the swarm's [posted review](https://github.com/hivecommons/hive/pull/13523#pullrequestreview-5613922737) flagged a changelog fragment that started with prose (the release compile would have rejected it) and a duplicate fragment; the author fixed both four minutes later and the next head was approved. Methodology and the full tables: [src/docs/benchmarks/](src/docs/benchmarks/README.md).
+A separate reviewer reads every pull request an agent opens and can send it back before it merges.
-## Hub and spoke
+
+
+
-- **Spoke** — one running Hive, serving your repo(s). It is where agents actually work. Runs on your Kubernetes cluster, Podman, or Docker Compose.
-- **Hub** — the control plane for many spokes: registers, provisions, upgrades and watches them; fleet view, leaderboards, routes contributor compute to the hives that need it.
+
+
+
-A spoke works alone. A hub is how you see and manage a fleet of them.
+Pull requests merged without review come from other repositories and periods, not a controlled comparison, so differences may not be caused by review. Revert rates are not distinguishable so far.
-## Contributor relay: a swarm working on your behalf
+## Security
-Your own agents are only the start. The **contributor relay** (ClankeR) lets anyone who wants to help your project **donate their compute and model tokens** without you handing out anything:
+Agents work with short-lived, task-scoped access; pull requests from untrusted authors are held for a human; every merge is signed off and credits the issue author.
-- **They bring their own agent.** A contributor runs one command on their laptop, cluster, or Compose box; it connects outbound to your hive, takes one curated task at a time, and runs *their* CLI with *their* model and *their* subscription.
-- **Their secrets never reach you.** API keys and CLI logins stay on the contributor's machine. The hive never sees them.
-- **Your forge settings never change.** Nobody is added as a collaborator. For each task the hive mints a short-lived, task-scoped GitHub token so the PR can be opened, then it expires. Revoke a contributor and their access is gone.
-- **Your prompts, your limits.** Contributors work the queue you curated, under the same trust level, filters and gates as your resident agents.
+
+
+
-The result is an **elastic swarm**: a three-person project can have a dozen contributors' agents grinding its backlog overnight, each accountable to a named human, while the maintainers hold every merge decision. See [src/docs/contributor-relay.md](src/docs/contributor-relay.md).
+## Cost
-## Hive the project vs the hosted service
+Each agent card in the dashboard shows what the current task has cost so far. The figure below is an estimate from provider pricing; some models are unpriced.
-- **Hive the project**: Apache-2.0 code in this repo. Run it yourself.
-- **Hive the hosted service**: [hive.hivecommons.dev](https://hive.hivecommons.dev) — a hub we operate so you can onboard a spoke without a cluster. Same software, one deployment of it.
+
+
+
-## Does Hive fit my project?
+## Also
-**Do I need a paid agent subscription?** No. Hive drives the agent CLIs you already have (Claude, Copilot, Codex, Antigravity, Goose) *or* any OpenAI-compatible model through its Model Gateway (OpenRouter key, no CLI login). Contributors can bring their own.
+
+
+
-**Who tells it what to work on?** Your backlog does. The Governor reads your issues/PRs/CI, filters them deterministically (labels, areas, complexity), and only hands agents work you have allowed. You choose the areas and the trust level. Nothing is invented.
+Production data, not a lab run. Every count above is regenerated from live records by `src/scripts/outcome-benchmark.sh`; bases, dates, exclusions and the full tables are in [how we count →](src/docs/benchmarks/outcomes-2026-10.md).
+
-**We require design discussion before non-trivial PRs.** That is a trust level, not a workaround: *Measured* mode = agents may only file findings and proposals as issues — no code, no PRs — so the discussion happens first; *Holdgated* = PRs are opened with a `hold` label and a human must review and release; *Full* (L6) = merge on green. Start at Measured. You never have to move up.
+## How it works
-**Can it open PRs to our upstream repo?** Yes, if you install it there. Or run it on a fork (`upstream_watch` keeps the fork current) and push reviewed PRs upstream yourself.
+
+
+
-More answers: https://docs.hivecommons.dev/docs/community/what-is-hive-commons#does-hive-fit-my-project
+The trust level ([ACMM](src/docs/acmm-policy-matrix.md)) sets how far the loop runs on its own, from filing findings only to merging on green.
## Run it
-| Path | Start here |
-| --- | --- |
-| Docker Compose | [Quick Start (Docker Compose)](src/docs/quickstart-docker-compose.md) |
-| Podman | [Quick Start (Podman)](src/docs/quickstart-podman.md) |
-| Kubernetes | [Kubernetes deployment](src/docs/kubernetes-deployment.md) |
-| Hosted hub | [https://hive.hivecommons.dev](https://hive.hivecommons.dev) and the [hosted hub guide](src/docs/hosted-hub.md) |
-| Getting started | [Zero to Automation: Getting Started with Hive](src/docs/getting-started.md) |
-
-Configuration lives in `hive.yaml`; see the [operator reference](src/docs/operator-reference.md), [environment variable reference](src/docs/env-vars.md), and [agent configuration](src/docs/agent-configuration.md).
-
-## ACMM Levels
-
-Hive uses an **AI Codebase Maturity Model** (ACMM) with six levels that control what agents are allowed to do:
-
-| Level | Name | What agents can do |
-|-------|------|-------------------|
-| L1 | Inception (Assisted) | Interactive advisor and project inception. Advisory beads only. |
-| L2 | Advisory (Instructed) | Observe and report findings as dashboard beads. No GitHub interaction. |
-| L3 | Quality-Gated (Measured) | Quality agent opens issues and PRs gated by the literal `hold` level-gate label. Others remain advisory. |
-| L4 | Security-Aware (Adaptive) | All agents file issues. Quality, sec-check, and CI open PRs gated by `hold`. |
-| L5 | Semi-Autonomous (Semi-Automated) | All agents open PRs gated by `hold`. Humans batch-review and approve. |
-| L6 | Autonomous | Agents open PRs and auto-merge when checks pass. Non-outreach PRs have no level hold; outreach PRs remain held. |
-
-See [src/docs/acmm-policy-matrix.md](src/docs/acmm-policy-matrix.md) for the full matrix and [src/docs/labels-and-control-signals.md](src/docs/labels-and-control-signals.md) for label behavior.
-
-## Architecture
-
-Hive runs as a single container with a Go control plane, an authenticated Node.js gateway, and ttyd for web terminal access. The Governor evaluates queue depth, deterministic filters decide what an agent may see, scoped tokens and a MITM proxy enforce what it may do, and gates decide what may land.
-
-See [src/docs/architecture.md](src/docs/architecture.md) for process model, governor loop, guardrails, hub/spoke, and walkthrough diagrams.
-
-## Terminal dashboard
-
-`hivectl tui` is a full-screen terminal view of the same dashboard API the web UI uses: agents, governor, token spend, live activity, pause/resume, kick, ACMM level, and hive-profile actions.
-
-See [`hivectl tui` in the command reference](src/docs/hivectl.md#tui--live-terminal-dashboard).
-
-## Contribute to a Hive
-
-Community members can contribute compute to any hive through **ClankeR**, the contributor relay (see [above](#contributor-relay-a-swarm-working-on-your-behalf)): it hands tasks from a hive's backlog to the CLI agent running on your own machine, under the hive's prompts and limits, with no repo permissions and no shared secrets.
-
-Start with [src/docs/contributor-relay.md](src/docs/contributor-relay.md) or the [Hive Hub contribute page](https://hive.hivecommons.dev).
-
-Homebrew is an additional install option:
-
```bash
-brew install hivecommons/hive/contribute
+git clone https://github.com/hivecommons/hive.git && cd hive
+cp src/hive.yaml.example src/hive.yaml # set project.org, project.repos and dashboard.port 3002
+docker compose -f src/docker-compose.yaml up -d
```
-Follow the [Homebrew setup instructions](src/docs/contributor-relay.md#homebrew)
-to register without a checkout before running `hive-contribute`. The
-[`just` workflow](src/docs/contributor-relay.md#from-source) remains the
-from-source path.
-
-## Moved sections
-
-## Quick Start
-
-Moved to [Quick Start (Docker Compose)](src/docs/quickstart-docker-compose.md), [Quick Start (Podman)](src/docs/quickstart-podman.md), and [Kubernetes deployment](src/docs/kubernetes-deployment.md).
-
-## Quick Start (Docker Compose)
-
-Moved to [src/docs/quickstart-docker-compose.md](src/docs/quickstart-docker-compose.md).
-
-## Quick Start (Podman)
-
-Moved to [src/docs/quickstart-podman.md](src/docs/quickstart-podman.md).
-
-## Kubernetes Deployment
+Tokens go in `src/.env`; the [Docker Compose quick start](src/docs/quickstart-docker-compose.md) lists them. Podman, Kubernetes and the hosted hub: [Run it](src/docs/what-is-hive.md#run-it).
-Moved to [src/docs/kubernetes-deployment.md](src/docs/kubernetes-deployment.md).
+## Learn more
-## Works with
-
-Hive drives Claude Code, GitHub Copilot CLI, Codex, Goose, Antigravity and OpenAI-compatible gateways; reads work from GitHub Issues, GitHub Projects, Linear and Jira; and runs on GitHub today with GitLab and Gitea/Forgejo planned. See the [integrations page](src/docs/integrations.md) for the full logo grid with status.
-
-## Related projects
-
-- [Goose](https://github.com/aaif-goose/goose), an agent backend whose upstream moved to the [Agentic AI Foundation (AAIF)](https://aaif.io/). Hive integrates Goose as an independent CNCF project; Hive does not join AAIF, seek AAIF membership, or rely on AAIF to host or govern Hive. See [unattended Goose with Hive](docs/goose-at-scale.md) and [Goose upstream tracking](src/docs/landscape.md#agentic-ai-foundation-aaif-and-goose).
-
-## Repositories
-
-Hive is one of several repositories in the hivecommons organization. See [docs/repositories.md](docs/repositories.md) for the full list, what each one is for, and which ones ship inside a Hive release.
-
-## Adopters
-
-See [ADOPTERS.md](ADOPTERS.md) for the organizations running Hive and how far along each one is. If you use Hive, you can add your organization to that file with a pull request.
-
-## Contributing
-
-See the [Hive Hub](https://hive.hivecommons.dev) to browse registered hives, view leaderboards, and find hives accepting contributions.
-
-To contribute to Hive itself, see [CONTRIBUTING.md](CONTRIBUTING.md) and open issues or PRs on this repository. Contributors are expected to follow the [CNCF Code of Conduct](https://github.com/cncf/foundation/blob/main/code-of-conduct.md), adopted in [CODE_OF_CONDUCT.md](CODE_OF_CONDUCT.md).
-
-Recent user-visible changes are recorded in [CHANGELOG.md](CHANGELOG.md).
-
-## Security
-
-For a code-grounded inventory of agent safety controls, enforcing versus advisory modes, known gaps, and running-hive audit steps, see the [Security Model](src/docs/security-model.md).
-
-Please see [SECURITY.md](SECURITY.md) for the vulnerability disclosure process. Data collection and telemetry paths are documented in [src/docs/telemetry.md](src/docs/telemetry.md). Do not report security vulnerabilities through public issues or pull requests.
+- [What Hive is](src/docs/what-is-hive.md) — the longer overview: how it differs from a terminal agent, hub and spoke, the contributor relay, ACMM levels, "does Hive fit my project".
+- [Architecture](src/docs/architecture.md) · [Security model](src/docs/security-model.md) · [Terminal dashboard](src/docs/hivectl.md#tui--live-terminal-dashboard) · [Integrations](src/docs/integrations.md)
+- [Contribute compute to a hive](src/docs/contributor-relay.md) · [Contribute to Hive itself](CONTRIBUTING.md) · [Adopters](ADOPTERS.md) · [Repositories](docs/repositories.md) · [Changelog](CHANGELOG.md)
+- [Report a vulnerability](SECURITY.md) · [Telemetry](src/docs/telemetry.md) · [Hosted hub](https://hive.hivecommons.dev)
---
-Apache 2.0
-
-Hive is a [Cloud Native Computing Foundation](https://www.cncf.io/) sandbox project candidate; onboarding status is tracked in [src/docs/cncf-sandbox-onboarding.md](src/docs/cncf-sandbox-onboarding.md). This does not imply CNCF acceptance or endorsement.
-Copyright The Linux Foundation and Hive contributors. The Linux Foundation has registered trademarks and uses trademarks. For a list of trademarks of The Linux Foundation, please see the [Linux Foundation Trademark Usage page](https://www.linuxfoundation.org/legal/trademark-usage). Hive project names and logos are used in accordance with that policy once ownership is transferred to the Linux Foundation/CNCF.
+Apache-2.0. Hive is a [Cloud Native Computing Foundation](https://www.cncf.io/) sandbox project candidate ([onboarding status](src/docs/cncf-sandbox-onboarding.md)); this does not imply CNCF acceptance or endorsement. Copyright The Linux Foundation and Hive contributors; trademarks are used per the [Linux Foundation Trademark Usage page](https://www.linuxfoundation.org/legal/trademark-usage).
diff --git a/bin/README.md b/bin/README.md
index 5b40f4e591..a67c42788a 100644
--- a/bin/README.md
+++ b/bin/README.md
@@ -57,7 +57,7 @@ Most production scripts are installed under `/usr/local/bin` by `bin/hive-deploy
| Script | Stage | Purpose |
|---|---|---|
| `hive.sh` | CLI | Operator command wrapper for starting the supervisor, checking status, attaching, kicking agents, reading logs, and stopping agents. |
-| `hive-setup.sh` | Bootstrap | All-in-one Ubuntu 24.04 LXC setup for a Docker-based Hive v2 instance, including generated `hive.yaml` and env files. **Docker-only**: it installs Docker Engine and drives `docker compose`, and predates `HIVE_DEPLOY_RUNTIME`, so that variable does not redirect it. For a Podman install follow the [README's Quick Start (Podman)](../README.md#quick-start-podman); see [Deployment helper scripts](../src/docs/deployment-scripts.md#all-in-one-lxc-setup). |
+| `hive-setup.sh` | Bootstrap | All-in-one Ubuntu 24.04 LXC setup for a Docker-based Hive v2 instance, including generated `hive.yaml` and env files. **Docker-only**: it installs Docker Engine and drives `docker compose`, and predates `HIVE_DEPLOY_RUNTIME`, so that variable does not redirect it. For a Podman install follow the [Quick Start (Podman)](../src/docs/quickstart-podman.md); see [Deployment helper scripts](../src/docs/deployment-scripts.md#all-in-one-lxc-setup). |
| `hive-prereq-check.sh` | Bootstrap | Validates host prerequisites for a Docker-based Hive v2 install and can attempt fixes with `--fix`. |
| `hive-deploy.sh` | Deploy | Pulls the latest Hive repository and syncs scripts to `/usr/local/bin`; also restarts Discord bot components when their files change. |
| `hive-checkout-guard.sh` | Deploy | `ExecStartPre` guard for systemd units that execute code out of a checkout (e.g. `hive-discord.service`, `hive-snapshot.service`): refuses to let the unit start when the checkout directory or its entrypoint files could have been planted by an unprivileged local user (#5435). Its exit status is the assertion — every unsafe path exits non-zero, so a violation actually stops `ExecStart`. |
diff --git a/changelog.d/changed-readme-evidence-cards.md b/changelog.d/changed-readme-evidence-cards.md
new file mode 100644
index 0000000000..d4fffe41bd
--- /dev/null
+++ b/changelog.d/changed-readme-evidence-cards.md
@@ -0,0 +1 @@
+- README rewritten around plain-language evidence cards: `hivectl evidence benchmark` now also renders dark/light SVG cards (sent back before merge, real examples, prompt injection, cost, "also" strip, how-it-works ladder) gated by `--readme-min-base` (default 200), takes `--stories` and `--red-team` inputs, and the generated README block lists Accuracy / Security / Cost / Also with no statistical vocabulary; the detailed charts and intervals stay in `src/docs/benchmarks/`, and the former README overview moved to `src/docs/what-is-hive.md`.
diff --git a/examples/kubestellar/README.md b/examples/kubestellar/README.md
index 3edffcdb24..c08f06be22 100644
--- a/examples/kubestellar/README.md
+++ b/examples/kubestellar/README.md
@@ -6,7 +6,7 @@
> of the v1 approach and as context for the agent definitions in
> [`agents/`](agents/), which remain current-style. Do not follow the
> installation steps below for a new deployment — use the
-> [Quick Start](../../README.md#quick-start) instead, and see
+> [Quick Start](../../src/docs/what-is-hive.md#run-it) instead, and see
> [`docs/LEGACY-v1-architecture.md`](../../docs/LEGACY-v1-architecture.md)
> for the retired v1 architecture this example ran on.
diff --git a/security-insights.yml b/security-insights.yml
index eff5c7c8bc..ec0bf0a2ed 100644
--- a/security-insights.yml
+++ b/security-insights.yml
@@ -18,7 +18,7 @@ project:
social: https://github.com/clubanderson
primary: true
documentation:
- quickstart-guide: https://github.com/hivecommons/hive/blob/v5/README.md#quick-start
+ quickstart-guide: https://github.com/hivecommons/hive/blob/v5/src/docs/what-is-hive.md#run-it
detailed-guide: https://docs.hivecommons.dev
code-of-conduct: https://github.com/hivecommons/.github/blob/main/CODE_OF_CONDUCT.md
repositories:
diff --git a/src/docs/benchmarks/README.md b/src/docs/benchmarks/README.md
index 3e651f8e33..c5245e3650 100644
--- a/src/docs/benchmarks/README.md
+++ b/src/docs/benchmarks/README.md
@@ -6,16 +6,30 @@ Reproducible comparison of pull requests that went through Hive's review swarm (
| Month | Report | Charts |
|---|---|---|
-| 2026-10 | [outcomes-2026-10.md](outcomes-2026-10.md) | [bars](assets/outcomes-2026-10-bars.svg) · [trend](assets/outcomes-2026-10-trend.svg) |
+| 2026-10 | [outcomes-2026-10.md](outcomes-2026-10.md) | [bars](assets/outcomes-2026-10-bars.svg) · [trend](assets/outcomes-2026-10-trend.svg) · README cards `assets/outcomes-2026-10-card-*.svg` |
-Each report ships with `outcomes-YYYY-MM.json`, the computed `Report` the markdown, charts and README block are rendered from. `pkg/outcomebench` `TestCommittedReportMatchesReadme` re-renders the newest JSON and fails when the committed markdown, SVGs or the repository README's `…` block differ, so none of them can drift.
+Each report ships with `outcomes-YYYY-MM.json`, the computed `Report` the markdown, charts, README cards and README block are rendered from, plus the inputs that are not derivable from the evidence tree: `cost-YYYY-MM.json` (saved `GET /api/cost`), `stories-YYYY-MM.json` (curated caught → fixed → merged examples, timestamps taken from the evidence bundles) and `red-team-YYYY-MM.json` (prompt-injection evaluation counts from [ioscan-red-team.md](../ioscan-red-team.md)). `pkg/outcomebench` `TestCommittedReportMatchesReadme` re-renders the newest JSON and fails when the committed markdown, SVGs, cards or the repository README's `…` block differ, so none of them can drift.
+
+## README cards
+
+The repository README shows this data as plain-language cards (one number, one sentence, a footer naming the base and month), with the detailed charts and intervals kept in the monthly report. Rules the generator enforces for the README:
+
+- A production-count card is drawn only when its base is at least `--readme-min-base` (default 200), separate from the n ≥ 10 report threshold. The 43-merge revert bucket, for example, stays in the report.
+- The checks-passing card is drawn only when a check result is recorded on at least 80 % of reviewed merges; below that the pass rate would describe a biased subset.
+- No statistical vocabulary (intervals, medians, cohorts, n=) appears in the README or on a card; `TestReadmeVocabulary` enforces the list and `TestReadmeProseBudget` keeps README prose under 600 words and 10 sections.
+- Every card is a real-text SVG in dark and light (`` + `prefers-color-scheme`) using the dashboard's own palette.
+- The prompt-injection card comes from Hive's own evaluation set, says so on the card, and is not subject to the minimum base because it is not a production count.
+
+Each report's "What the README shows and leaves out" section records, per card, whether it was drawn and on what base.
## Regenerate
```sh
# on a spoke, or anywhere with a copy of its evidence tree
src/scripts/outcome-benchmark.sh --root /data/evidence --month 2026-10 \
- --spend cost.json # saved GET /api/cost, for cost per merged PR
+ --spend cost-2026-10.json # saved GET /api/cost, for cost per merged PR
+ --stories stories-2026-10.json # curated caught → fixed → merged examples
+ --red-team red-team-2026-10.json # prompt-injection evaluation counts
--hub hub.json # saved GET /api/hub/evidence (admin session), optional
--source "hive-oke spoke snapshot 2026-10-27" \
--readme # also rewrite README.md's outcomes block
diff --git a/src/docs/benchmarks/assets/outcomes-2026-10-card-also-dark.svg b/src/docs/benchmarks/assets/outcomes-2026-10-card-also-dark.svg
new file mode 100644
index 0000000000..fb42f3f08a
--- /dev/null
+++ b/src/docs/benchmarks/assets/outcomes-2026-10-card-also-dark.svg
@@ -0,0 +1,29 @@
+
diff --git a/src/docs/benchmarks/assets/outcomes-2026-10-card-also-light.svg b/src/docs/benchmarks/assets/outcomes-2026-10-card-also-light.svg
new file mode 100644
index 0000000000..e39375615c
--- /dev/null
+++ b/src/docs/benchmarks/assets/outcomes-2026-10-card-also-light.svg
@@ -0,0 +1,29 @@
+
diff --git a/src/docs/benchmarks/assets/outcomes-2026-10-card-cost-dark.svg b/src/docs/benchmarks/assets/outcomes-2026-10-card-cost-dark.svg
new file mode 100644
index 0000000000..835ee868d5
--- /dev/null
+++ b/src/docs/benchmarks/assets/outcomes-2026-10-card-cost-dark.svg
@@ -0,0 +1,11 @@
+
diff --git a/src/docs/benchmarks/assets/outcomes-2026-10-card-cost-light.svg b/src/docs/benchmarks/assets/outcomes-2026-10-card-cost-light.svg
new file mode 100644
index 0000000000..71d0737807
--- /dev/null
+++ b/src/docs/benchmarks/assets/outcomes-2026-10-card-cost-light.svg
@@ -0,0 +1,11 @@
+
diff --git a/src/docs/benchmarks/assets/outcomes-2026-10-card-how-it-works-dark.svg b/src/docs/benchmarks/assets/outcomes-2026-10-card-how-it-works-dark.svg
new file mode 100644
index 0000000000..6561d20b81
--- /dev/null
+++ b/src/docs/benchmarks/assets/outcomes-2026-10-card-how-it-works-dark.svg
@@ -0,0 +1,36 @@
+
diff --git a/src/docs/benchmarks/assets/outcomes-2026-10-card-how-it-works-light.svg b/src/docs/benchmarks/assets/outcomes-2026-10-card-how-it-works-light.svg
new file mode 100644
index 0000000000..1c60209a40
--- /dev/null
+++ b/src/docs/benchmarks/assets/outcomes-2026-10-card-how-it-works-light.svg
@@ -0,0 +1,36 @@
+
diff --git a/src/docs/benchmarks/assets/outcomes-2026-10-card-prompt-injection-dark.svg b/src/docs/benchmarks/assets/outcomes-2026-10-card-prompt-injection-dark.svg
new file mode 100644
index 0000000000..7b5170109d
--- /dev/null
+++ b/src/docs/benchmarks/assets/outcomes-2026-10-card-prompt-injection-dark.svg
@@ -0,0 +1,11 @@
+
diff --git a/src/docs/benchmarks/assets/outcomes-2026-10-card-prompt-injection-light.svg b/src/docs/benchmarks/assets/outcomes-2026-10-card-prompt-injection-light.svg
new file mode 100644
index 0000000000..e4da9ffe7c
--- /dev/null
+++ b/src/docs/benchmarks/assets/outcomes-2026-10-card-prompt-injection-light.svg
@@ -0,0 +1,11 @@
+
diff --git a/src/docs/benchmarks/assets/outcomes-2026-10-card-sent-back-dark.svg b/src/docs/benchmarks/assets/outcomes-2026-10-card-sent-back-dark.svg
new file mode 100644
index 0000000000..671dfc76ea
--- /dev/null
+++ b/src/docs/benchmarks/assets/outcomes-2026-10-card-sent-back-dark.svg
@@ -0,0 +1,10 @@
+
diff --git a/src/docs/benchmarks/assets/outcomes-2026-10-card-sent-back-light.svg b/src/docs/benchmarks/assets/outcomes-2026-10-card-sent-back-light.svg
new file mode 100644
index 0000000000..be5e7e305b
--- /dev/null
+++ b/src/docs/benchmarks/assets/outcomes-2026-10-card-sent-back-light.svg
@@ -0,0 +1,10 @@
+
diff --git a/src/docs/benchmarks/assets/outcomes-2026-10-card-stories-dark.svg b/src/docs/benchmarks/assets/outcomes-2026-10-card-stories-dark.svg
new file mode 100644
index 0000000000..9ee1f62a3d
--- /dev/null
+++ b/src/docs/benchmarks/assets/outcomes-2026-10-card-stories-dark.svg
@@ -0,0 +1,30 @@
+
diff --git a/src/docs/benchmarks/assets/outcomes-2026-10-card-stories-light.svg b/src/docs/benchmarks/assets/outcomes-2026-10-card-stories-light.svg
new file mode 100644
index 0000000000..d7f5358475
--- /dev/null
+++ b/src/docs/benchmarks/assets/outcomes-2026-10-card-stories-light.svg
@@ -0,0 +1,30 @@
+
diff --git a/src/docs/benchmarks/cost-2026-10.json b/src/docs/benchmarks/cost-2026-10.json
new file mode 100644
index 0000000000..90656a0190
--- /dev/null
+++ b/src/docs/benchmarks/cost-2026-10.json
@@ -0,0 +1 @@
+{"estimated":{"total_usd":6124.75144225,"coin_label":"BC","by_model":[{"name":"claude-fable-5","usd":13.0314995,"source":"estimated","input":85072,"output":50517,"cache_read":4028842,"cache_create":450087},{"name":"claude-opus-5.5","usd":26.785830600000004,"source":"estimated","input":309228,"output":306964,"cache_read":44891893,"cache_create":2086252},{"name":"gpt-4o-mini","usd":1.67606425,"source":"unpriced","input":6501753,"output":25313,"cache_read":0,"cache_create":0},{"name":"claude-fable-5.1","usd":6025.2392685,"source":"estimated","input":72482882,"output":34513979,"cache_read":3808347594,"cache_create":209809968},{"name":"claude-sonnet-5.5","usd":57.8325393,"source":"estimated","input":7538768,"output":476734,"cache_read":61926183,"cache_create":12718018},{"name":"claude-sonnet-5","usd":0.18624010000000002,"source":"estimated","input":5581,"output":1805,"cache_read":152178,"cache_create":50637}],"by_agent":[{"name":"reviewer","usd":1250.6676095,"source":"unpriced","input":17698267,"output":6702463,"cache_read":640275179,"cache_create":46295010},{"name":"ci-maintainer","usd":541.1905579999999,"source":"unpriced","input":7438603,"output":3015737,"cache_read":270980094,"cache_create":20443398},{"name":"strategist","usd":4.957559799999999,"source":"unpriced","input":1328852,"output":25917,"cache_read":3798713,"cache_create":998862},{"name":"reviewer-2","usd":1150.4665407500001,"source":"estimated","input":9472363,"output":6604680,"cache_read":627023043,"cache_create":45500252},{"name":"adjudicator","usd":721.9997615,"source":"estimated","input":6213386,"output":4563777,"cache_read":424278006,"cache_create":26048604},{"name":"scanner","usd":1989.6345952500005,"source":"unpriced","input":27282120,"output":11684366,"cache_read":1557645144,"cache_create":60263080},{"name":"telemetry","usd":12.9796011,"source":"unpriced","input":856612,"output":137382,"cache_read":15200936,"cache_create":3353515},{"name":"sec-check","usd":113.21250425000001,"source":"unpriced","input":3084882,"output":532319,"cache_read":82827459,"cache_create":3518954},{"name":"brainstorm","usd":0.18624010000000002,"source":"estimated","input":5581,"output":1805,"cache_read":152178,"cache_create":50637},{"name":"outreach","usd":26.785830600000004,"source":"estimated","input":309228,"output":306964,"cache_read":44891893,"cache_create":2086252},{"name":"quality","usd":261.4214055,"source":"unpriced","input":6296148,"output":1434192,"cache_read":205365453,"cache_create":7883331},{"name":"guide","usd":16.92538265,"source":"unpriced","input":3798671,"output":108524,"cache_read":17275109,"cache_create":3123565},{"name":"dedupe","usd":10.966232999999999,"source":"estimated","input":84960,"output":45835,"cache_read":3982058,"cache_create":307426},{"name":"architect","usd":9.20738255,"source":"unpriced","input":2202175,"output":54409,"cache_read":6171093,"cache_create":1670111},{"name":"operations","usd":14.1502377,"source":"estimated","input":851436,"output":156942,"cache_read":19480332,"cache_create":3571965}],"unpriced_models":["gpt-4o-mini"],"window_start":1787785874000,"window_end":1793111300000,"by_session":[{"session_id":"cdb83d43-ff4","agent":"scanner","model":"claude-fable-5","usd":0.030182,"source":"estimated","input":2,"output":90,"cache_read":1462,"messages":2,"started":1793109799225,"last_active":1793109803911},{"session_id":"a45fe164-252","agent":"scanner","model":"claude-fable-5","usd":0.028057,"source":"estimated","input":2,"output":57,"cache_read":1462,"messages":2,"started":1793109568765,"last_active":1793109573041},{"session_id":"b6628e24-712","agent":"ci-maintainer","model":"claude-fable-5","usd":0.046682499999999995,"source":"estimated","input":2,"output":84,"cache_read":0,"messages":2,"started":1793109554873,"last_active":1793109559117},{"session_id":"c615171c-95c","agent":"scanner","model":"claude-fable-5","usd":0.0452575,"source":"estimated","input":2,"output":59,"cache_read":0,"messages":2,"started":1793108791934,"last_active":1793108795763},{"session_id":"65a8d34c-c32","agent":"scanner","model":"claude-fable-5","usd":0.028432000000000002,"source":"estimated","input":2,"output":63,"cache_read":1462,"messages":2,"started":1793108129883,"last_active":1793108133948},{"session_id":"0cea34a6-7d2","agent":"scanner","model":"claude-fable-5","usd":0.045895,"source":"estimated","input":2,"output":74,"cache_read":0,"messages":2,"started":1793108116603,"last_active":1793108120159},{"session_id":"dce8797e-3b1","agent":"scanner","model":"claude-fable-5","usd":0.032407,"source":"estimated","input":2,"output":136,"cache_read":1462,"messages":2,"started":1793106342677,"last_active":1793106347535},{"session_id":"c20402c1-66f","agent":"scanner","model":"claude-fable-5","usd":0.0556575,"source":"estimated","input":2,"output":264,"cache_read":0,"messages":2,"started":1793106326117,"last_active":1793106333175},{"session_id":"fa2425c0-cb6","agent":"scanner","model":"claude-fable-5","usd":0.039556999999999995,"source":"estimated","input":2,"output":278,"cache_read":1462,"messages":2,"started":1793104309901,"last_active":1793104316731},{"session_id":"a050381b-e92","agent":"scanner","model":"claude-fable-5","usd":0.045845,"source":"estimated","input":2,"output":69,"cache_read":0,"messages":2,"started":1793104296475,"last_active":1793104300258},{"session_id":"4af6bfdc-8f4","agent":"scanner","model":"claude-fable-5","usd":0.0465075,"source":"estimated","input":2,"output":79,"cache_read":0,"messages":2,"started":1793103867423,"last_active":1793103871442},{"session_id":"58a0a724-594","agent":"ci-maintainer","model":"claude-fable-5","usd":0.0509575,"source":"estimated","input":2,"output":170,"cache_read":0,"messages":2,"started":1793102769336,"last_active":1793102775203},{"session_id":"6d5a30db-ada","agent":"scanner","model":"claude-fable-5","usd":0.0461075,"source":"estimated","input":2,"output":73,"cache_read":0,"messages":2,"started":1793102430951,"last_active":1793102435702},{"session_id":"083858c1-c4f","agent":"scanner","model":"claude-fable-5","usd":0.0282695,"source":"estimated","input":2,"output":58,"cache_read":1462,"messages":2,"started":1793101504547,"last_active":1793101508341},{"session_id":"762cf0c1-2e8","agent":"scanner","model":"claude-fable-5","usd":0.0293945,"source":"estimated","input":2,"output":76,"cache_read":1462,"messages":2,"started":1793101490107,"last_active":1793101494582},{"session_id":"5f15da2d-1cc","agent":"scanner","model":"claude-fable-5","usd":0.045445,"source":"estimated","input":2,"output":60,"cache_read":0,"messages":2,"started":1793101475625,"last_active":1793101479870},{"session_id":"00ae7383-983","agent":"scanner","model":"claude-fable-5","usd":0.0471825,"source":"estimated","input":2,"output":96,"cache_read":0,"messages":2,"started":1793101093871,"last_active":1793101097970},{"session_id":"912fdf76-ac5","agent":"scanner","model":"claude-fable-5","usd":0.046057499999999994,"source":"estimated","input":2,"output":74,"cache_read":0,"messages":2,"started":1793100657634,"last_active":1793100662401},{"session_id":"470cb199-3e4","agent":"scanner","model":"claude-fable-5","usd":0.045795,"source":"estimated","input":2,"output":69,"cache_read":0,"messages":2,"started":1793098940961,"last_active":1793098945718},{"session_id":"08cad316-ff6","agent":"scanner","model":"claude-fable-5","usd":0.047232500000000004,"source":"estimated","input":2,"output":97,"cache_read":0,"messages":2,"started":1793096150094,"last_active":1793096154129},{"session_id":"61865831-9a3","agent":"scanner","model":"claude-fable-5","usd":0.0459825,"source":"estimated","input":2,"output":73,"cache_read":0,"messages":2,"started":1793095756031,"last_active":1793095761268},{"session_id":"1b03a3de-4a3","agent":"scanner","model":"claude-fable-5","usd":0.045544999999999995,"source":"estimated","input":2,"output":67,"cache_read":0,"messages":2,"started":1793093947154,"last_active":1793093950832},{"session_id":"9270dc6d-dbb","agent":"scanner","model":"claude-fable-5","usd":0.0306195,"source":"estimated","input":2,"output":104,"cache_read":1462,"messages":2,"started":1793092133641,"last_active":1793092138644},{"session_id":"cc620766-185","agent":"scanner","model":"claude-fable-5","usd":0.029082,"source":"estimated","input":2,"output":69,"cache_read":1462,"messages":2,"started":1793092119529,"last_active":1793092124013},{"session_id":"c5fd0152-b98","agent":"scanner","model":"claude-fable-5","usd":0.0459825,"source":"estimated","input":2,"output":73,"cache_read":0,"messages":2,"started":1793092105798,"last_active":1793092109872},{"session_id":"550ebc99-4b5","agent":"scanner","model":"claude-fable-5","usd":0.0284945,"source":"estimated","input":2,"output":64,"cache_read":1462,"messages":2,"started":1793081814295,"last_active":1793081818694},{"session_id":"cc5d94c4-659","agent":"scanner","model":"claude-fable-5","usd":0.028469500000000002,"source":"estimated","input":2,"output":65,"cache_read":1462,"messages":2,"started":1793081800029,"last_active":1793081804149},{"session_id":"eae331fc-b19","agent":"scanner","model":"claude-fable-5","usd":0.0278195,"source":"estimated","input":2,"output":52,"cache_read":1462,"messages":2,"started":1793081786119,"last_active":1793081790381},{"session_id":"247413d0-c99","agent":"scanner","model":"claude-fable-5","usd":0.0286695,"source":"estimated","input":2,"output":69,"cache_read":1462,"messages":2,"started":1793081770732,"last_active":1793081776114},{"session_id":"3313184d-fb5","agent":"scanner","model":"claude-fable-5","usd":0.0278195,"source":"estimated","input":2,"output":51,"cache_read":1462,"messages":2,"started":1793081757537,"last_active":1793081761153},{"session_id":"2263f109-f95","agent":"scanner","model":"claude-fable-5","usd":0.04712,"source":"estimated","input":2,"output":90,"cache_read":0,"messages":2,"started":1793081744204,"last_active":1793081747881},{"session_id":"36a79fac-ad5","agent":"sec-check","model":"claude-fable-5","usd":0.028032,"source":"estimated","input":2,"output":53,"cache_read":1462,"messages":2,"started":1793080093665,"last_active":1793080098406},{"session_id":"ee65a857-2e2","agent":"scanner","model":"claude-fable-5","usd":0.027957,"source":"estimated","input":2,"output":52,"cache_read":1462,"messages":2,"started":1793080079776,"last_active":1793080084015},{"session_id":"321151e9-c84","agent":"sec-check","model":"claude-fable-5","usd":0.028332,"source":"estimated","input":2,"output":62,"cache_read":1462,"messages":2,"started":1793080065698,"last_active":1793080070202},{"session_id":"b9c4fe22-c30","agent":"scanner","model":"claude-fable-5","usd":0.027906999999999998,"source":"estimated","input":2,"output":54,"cache_read":1462,"messages":2,"started":1793080052216,"last_active":1793080056119},{"session_id":"02bf151d-da7","agent":"scanner","model":"claude-fable-5","usd":0.0278195,"source":"estimated","input":2,"output":52,"cache_read":1462,"messages":2,"started":1793080038774,"last_active":1793080042323},{"session_id":"2cbd5c83-34b","agent":"scanner","model":"claude-fable-5","usd":0.046707500000000006,"source":"estimated","input":2,"output":83,"cache_read":0,"messages":2,"started":1793080024059,"last_active":1793080029196},{"session_id":"43e114ee-0a8","agent":"scanner","model":"claude-fable-5","usd":0.045395,"source":"estimated","input":2,"output":59,"cache_read":0,"messages":2,"started":1793073967980,"last_active":1793073971363},{"session_id":"b0b40ecf-0a4","agent":"scanner","model":"claude-fable-5","usd":0.028807,"source":"estimated","input":2,"output":64,"cache_read":1462,"messages":2,"started":1793073217066,"last_active":1793073220723},{"session_id":"240cd640-5a8","agent":"scanner","model":"claude-fable-5","usd":0.030581999999999998,"source":"estimated","input":2,"output":98,"cache_read":1462,"messages":2,"started":1793073202620,"last_active":1793073206543},{"session_id":"9023a652-6e7","agent":"scanner","model":"claude-fable-5","usd":0.045720000000000004,"source":"estimated","input":2,"output":66,"cache_read":0,"messages":2,"started":1793073189402,"last_active":1793073193134},{"session_id":"1a3c1aab-9f6","agent":"scanner","model":"claude-fable-5","usd":0.04782,"source":"estimated","input":2,"output":110,"cache_read":0,"messages":2,"started":1793072602573,"last_active":1793072607333},{"session_id":"519d338c-db2","agent":"scanner","model":"claude-fable-5","usd":0.028607,"source":"estimated","input":2,"output":59,"cache_read":1462,"messages":2,"started":1793071722297,"last_active":1793071726804},{"session_id":"aaf41614-042","agent":"scanner","model":"claude-fable-5","usd":0.0461325,"source":"estimated","input":2,"output":77,"cache_read":0,"messages":2,"started":1793071708276,"last_active":1793071712811},{"session_id":"532f1b32-dd2","agent":"scanner","model":"claude-fable-5","usd":0.0352945,"source":"estimated","input":2,"output":194,"cache_read":1462,"messages":2,"started":1793070797728,"last_active":1793070804533},{"session_id":"5e8c7d99-ac9","agent":"scanner","model":"claude-fable-5","usd":0.032482,"source":"estimated","input":2,"output":139,"cache_read":1462,"messages":2,"started":1793070783450,"last_active":1793070788219},{"session_id":"f7fa1140-5c1","agent":"scanner","model":"claude-fable-5","usd":0.029319500000000002,"source":"estimated","input":2,"output":73,"cache_read":1462,"messages":2,"started":1793070769824,"last_active":1793070773883},{"session_id":"910f0a9d-7f4","agent":"scanner","model":"claude-fable-5","usd":0.047245,"source":"estimated","input":2,"output":99,"cache_read":0,"messages":2,"started":1793070755029,"last_active":1793070760044},{"session_id":"dc4fe15b-f63","agent":"scanner","model":"claude-fable-5","usd":0.027632,"source":"estimated","input":2,"output":48,"cache_read":1462,"messages":2,"started":1793069856043,"last_active":1793069859695},{"session_id":"22e68c24-79c","agent":"scanner","model":"claude-fable-5","usd":0.027932,"source":"estimated","input":2,"output":48,"cache_read":1462,"messages":2,"started":1793069842980,"last_active":1793069846261},{"session_id":"3ced6296-f70","agent":"scanner","model":"claude-fable-5","usd":0.028631999999999998,"source":"estimated","input":2,"output":60,"cache_read":1462,"messages":2,"started":1793069829706,"last_active":1793069833345},{"session_id":"a307aab5-3bb","agent":"scanner","model":"claude-fable-5","usd":0.0288195,"source":"estimated","input":2,"output":72,"cache_read":1462,"messages":2,"started":1793069816322,"last_active":1793069820131},{"session_id":"da57358b-427","agent":"scanner","model":"claude-fable-5","usd":0.0287195,"source":"estimated","input":2,"output":64,"cache_read":1462,"messages":2,"started":1793069802800,"last_active":1793069806670},{"session_id":"aa0e6825-955","agent":"ci-maintainer","model":"claude-fable-5","usd":0.0291195,"source":"estimated","input":2,"output":69,"cache_read":1462,"messages":2,"started":1793069789183,"last_active":1793069793102},{"session_id":"4c0d2874-f6c","agent":"scanner","model":"claude-fable-5","usd":0.0277445,"source":"estimated","input":2,"output":50,"cache_read":1462,"messages":2,"started":1793069775937,"last_active":1793069779696},{"session_id":"c9499190-269","agent":"scanner","model":"claude-fable-5","usd":0.045982499999999996,"source":"estimated","input":2,"output":74,"cache_read":0,"messages":2,"started":1793069761768,"last_active":1793069766258},{"session_id":"copilot-live-reviewer-2","agent":"reviewer-2","model":"claude-fable-5.1","usd":1150.4665407500001,"source":"estimated","input":9472363,"output":6604680,"cache_read":627023043,"messages":1835,"started":1792435510000,"last_active":1793110888000},{"session_id":"copilot-live-dedupe","agent":"dedupe","model":"claude-fable-5","usd":10.966232999999999,"source":"estimated","input":84960,"output":45835,"cache_read":3982058,"messages":100,"started":1791905263000,"last_active":1793100949000},{"session_id":"copilot-live-brainstorm","agent":"brainstorm","model":"claude-sonnet-5","usd":0.18624010000000002,"source":"estimated","input":5581,"output":1805,"cache_read":152178,"messages":10,"started":1791035255000,"last_active":1792805650000},{"session_id":"copilot-live-adjudicator","agent":"adjudicator","model":"claude-fable-5.1","usd":721.9997615,"source":"estimated","input":6213386,"output":4563777,"cache_read":424278006,"messages":1845,"started":1790702123000,"last_active":1793109400000},{"session_id":"copilot-live-telemetry","agent":"telemetry","model":"claude-sonnet-5.5","usd":12.9778951,"source":"estimated","input":850612,"output":137279,"cache_read":15200936,"messages":964,"started":1790693751000,"last_active":1793110699000},{"session_id":"copilot-live-operations","agent":"operations","model":"claude-sonnet-5.5","usd":14.1502377,"source":"estimated","input":851436,"output":156942,"cache_read":19480332,"messages":1063,"started":1790693655000,"last_active":1793109655000},{"session_id":"copilot-live-architect","agent":"architect","model":"claude-sonnet-5.5","usd":9.1303328,"source":"estimated","input":1901848,"output":53425,"cache_read":6171093,"messages":500,"started":1790532277000,"last_active":1793104017000},{"session_id":"copilot-live-outreach","agent":"outreach","model":"claude-opus-5.5","usd":26.785830600000004,"source":"estimated","input":309228,"output":306964,"cache_read":44891893,"messages":852,"started":1790515852000,"last_active":1793094205000},{"session_id":"copilot-live-scanner","agent":"scanner","model":"claude-fable-5.1","usd":1987.4898815,"source":"estimated","input":26259407,"output":11676663,"cache_read":1557602746,"messages":1783,"started":1790022219000,"last_active":1793111300000},{"session_id":"copilot-live-ci-maintainer","agent":"ci-maintainer","model":"claude-fable-5.1","usd":540.8748055,"source":"estimated","input":6709241,"output":3012087,"cache_read":270978632,"messages":1646,"started":1790021526000,"last_active":1793109320000},{"session_id":"copilot-live-strategist","agent":"strategist","model":"claude-sonnet-5.5","usd":4.8366682999999995,"source":"estimated","input":858486,"output":24267,"cache_read":3798713,"messages":317,"started":1790021225000,"last_active":1793099318000},{"session_id":"copilot-live-sec-check","agent":"sec-check","model":"claude-fable-5.1","usd":112.92331875,"source":"estimated","input":2188736,"output":527811,"cache_read":82824535,"messages":1713,"started":1790020926000,"last_active":1793110853000},{"session_id":"copilot-live-quality","agent":"quality","model":"claude-fable-5.1","usd":260.82237075,"source":"estimated","input":3961257,"output":1426536,"cache_read":205365453,"messages":1667,"started":1790014615000,"last_active":1793110207000},{"session_id":"copilot-live-reviewer","agent":"reviewer","model":"claude-fable-5.1","usd":1250.66258975,"source":"estimated","input":17678492,"output":6702425,"cache_read":640275179,"messages":1711,"started":1790013431000,"last_active":1793110846000},{"session_id":"copilot-live-guide","agent":"guide","model":"claude-sonnet-5.5","usd":16.7374054,"source":"estimated","input":3076386,"output":104821,"cache_read":17275109,"messages":904,"started":1790012806000,"last_active":1793109697000},{"session_id":"inference-reviewer","agent":"reviewer","model":"gpt-4o-mini","usd":0.00501975,"source":"unpriced","input":19775,"output":38,"cache_read":0,"messages":2,"started":1790007399000,"last_active":1790011537000},{"session_id":"inference-architect","agent":"architect","model":"gpt-4o-mini","usd":0.07704975,"source":"unpriced","input":300327,"output":984,"cache_read":0,"messages":2,"started":1788274071000,"last_active":1789689082000},{"session_id":"inference-strategist","agent":"strategist","model":"gpt-4o-mini","usd":0.1208915,"source":"unpriced","input":470366,"output":1650,"cache_read":0,"messages":2,"started":1788274010000,"last_active":1790006588000},{"session_id":"inference-telemetry","agent":"telemetry","model":"gpt-4o-mini","usd":0.001706,"source":"unpriced","input":6000,"output":103,"cache_read":0,"messages":2,"started":1788273918000,"last_active":1788406985000},{"session_id":"inference-sec-check","agent":"sec-check","model":"gpt-4o-mini","usd":0.2328215,"source":"unpriced","input":896142,"output":4393,"cache_read":0,"messages":2,"started":1787791127000,"last_active":1790006289000},{"session_id":"inference-quality","agent":"quality","model":"gpt-4o-mini","usd":0.59903475,"source":"unpriced","input":2334891,"output":7656,"cache_read":0,"messages":2,"started":1787787382000,"last_active":1790007322000},{"session_id":"inference-ci-maintainer","agent":"ci-maintainer","model":"gpt-4o-mini","usd":0.188993,"source":"unpriced","input":729356,"output":3327,"cache_read":0,"messages":2,"started":1787785987000,"last_active":1790006771000},{"session_id":"inference-guide","agent":"guide","model":"gpt-4o-mini","usd":0.18797724999999998,"source":"unpriced","input":722285,"output":3703,"cache_read":0,"messages":2,"started":1787785930000,"last_active":1789998066000},{"session_id":"inference-scanner","agent":"scanner","model":"gpt-4o-mini","usd":0.26257074999999996,"source":"unpriced","input":1022611,"output":3459,"cache_read":0,"messages":2,"started":1787785874000,"last_active":1790007460000}]},"coin_label":"BC","gateways":[],"price_table_date":"2026-10-26","disclaimer":"Estimated from token counts × published list prices. Subscription plans (Claude, Copilot), self-hosted inference (vLLM), and negotiated rates differ from list price. Not a bill.","merged_prs":2869,"closed_issues":2955,"count_author":"hivecommons-hive[bot]","count_basis":"hive-attributed","count_window_start":1786147200000,"count_window_end":1793112641723,"count_updated_at":"2026-10-27T14:46:14Z","count_status":"fresh","count_source":"forgestate","count_available":true,"daily_rollup":[{"day":"2026-08-08","cost_usd":38.9304759500003,"input_tokens":8022926,"output_tokens":46227,"samples":45},{"day":"2026-08-09","cost_usd":147.90401645000043,"input_tokens":34762434,"output_tokens":189742,"samples":28},{"day":"2026-08-10","cost_usd":393.2616113000016,"input_tokens":98897210,"output_tokens":612397,"samples":24},{"day":"2026-08-11","cost_usd":107.01969177500564,"input_tokens":24505378,"output_tokens":146958,"samples":150},{"day":"2026-08-12","cost_usd":199.3599360000046,"input_tokens":28159051,"output_tokens":103221,"samples":106},{"day":"2026-08-13","cost_usd":11.501422250001042,"input_tokens":1893592,"output_tokens":12959,"samples":22},{"day":"2026-08-14","cost_usd":108.00359445000004,"input_tokens":26369316,"output_tokens":161198,"samples":2},{"day":"2026-08-20","cost_usd":502.4785596500101,"input_tokens":129787446,"output_tokens":620829,"samples":96},{"day":"2026-08-21","cost_usd":125.13099125001008,"input_tokens":10431025,"output_tokens":83344,"samples":92},{"day":"2026-08-22","cost_usd":15.45825675001106,"input_tokens":1289186,"output_tokens":12151,"samples":96},{"day":"2026-08-23","cost_usd":0.0010430000011183438,"input_tokens":4068,"output_tokens":13,"samples":6},{"day":"2026-08-27","cost_usd":1258.4085199999995,"input_tokens":104959153,"output_tokens":761861,"samples":1},{"day":"2026-08-28","cost_usd":310.3188079000215,"input_tokens":31858702,"output_tokens":221094,"samples":128},{"day":"2026-08-29","cost_usd":377.7144686000456,"input_tokens":33767300,"output_tokens":277015,"samples":262},{"day":"2026-08-30","cost_usd":207.84388425006,"input_tokens":17536941,"output_tokens":126331,"samples":267},{"day":"2026-08-31","cost_usd":175.56794415004515,"input_tokens":15607690,"output_tokens":131399,"samples":262},{"day":"2026-09-01","cost_usd":145.48462740005198,"input_tokens":14698584,"output_tokens":116718,"samples":267},{"day":"2026-09-02","cost_usd":221.42737625002883,"input_tokens":18878239,"output_tokens":154051,"samples":167},{"day":"2026-09-03","cost_usd":524.5506297500115,"input_tokens":44301243,"output_tokens":334555,"samples":72},{"day":"2026-09-04","cost_usd":0,"input_tokens":2330325,"output_tokens":18561,"samples":2},{"day":"2026-09-16","cost_usd":8129.188672975001,"input_tokens":828650561,"output_tokens":6585590,"samples":1},{"day":"2026-09-18","cost_usd":710.6469106999975,"input_tokens":63516717,"output_tokens":515134,"samples":3},{"day":"2026-09-23","cost_usd":0.00661000000036438,"input_tokens":1295871,"output_tokens":3053,"samples":33},{"day":"2026-09-24","cost_usd":616.2903384500005,"input_tokens":50559545,"output_tokens":418835,"samples":93},{"day":"2026-09-25","cost_usd":0.07554875000732864,"input_tokens":390335,"output_tokens":2867,"samples":160},{"day":"2026-09-26","cost_usd":0.07717900000307054,"input_tokens":2258641,"output_tokens":13608,"samples":261},{"day":"2026-09-27","cost_usd":705.231650850008,"input_tokens":79113628,"output_tokens":578094,"samples":266},{"day":"2026-09-28","cost_usd":1493.5609004500138,"input_tokens":161741378,"output_tokens":1299781,"samples":261},{"day":"2026-09-29","cost_usd":457.8319045000253,"input_tokens":39116320,"output_tokens":351079,"samples":222},{"day":"2026-09-30","cost_usd":0.3272460000207502,"input_tokens":1343716,"output_tokens":2790,"samples":256},{"day":"2026-10-01","cost_usd":0.2584472500175252,"input_tokens":1042475,"output_tokens":2433,"samples":259},{"day":"2026-10-02","cost_usd":0.2553210000041872,"input_tokens":1010403,"output_tokens":2347,"samples":260},{"day":"2026-10-03","cost_usd":0.38515425000196046,"input_tokens":1545379,"output_tokens":3166,"samples":264},{"day":"2026-10-04","cost_usd":0.6834777500012024,"input_tokens":2708512,"output_tokens":4655,"samples":259},{"day":"2026-10-05","cost_usd":0.2212410000033742,"input_tokens":890815,"output_tokens":1666,"samples":205},{"day":"2026-10-06","cost_usd":0.5549840000000774,"input_tokens":2214570,"output_tokens":4415,"samples":261},{"day":"2026-10-07","cost_usd":0.7155697499999993,"input_tokens":2798039,"output_tokens":8030,"samples":259},{"day":"2026-10-08","cost_usd":0.7053845000000001,"input_tokens":2748938,"output_tokens":9075,"samples":256},{"day":"2026-10-09","cost_usd":0.5321210000000001,"input_tokens":2065052,"output_tokens":7929,"samples":245},{"day":"2026-10-10","cost_usd":0.4674862500000021,"input_tokens":1825393,"output_tokens":5658,"samples":229},{"day":"2026-10-11","cost_usd":0.5054077499999998,"input_tokens":1977199,"output_tokens":5554,"samples":246},{"day":"2026-10-12","cost_usd":0.4168147499999968,"input_tokens":1626979,"output_tokens":5065,"samples":235},{"day":"2026-10-13","cost_usd":0.38747249999999944,"input_tokens":1506018,"output_tokens":5484,"samples":194},{"day":"2026-10-14","cost_usd":0.4589437500000013,"input_tokens":1783359,"output_tokens":6553,"samples":175},{"day":"2026-10-15","cost_usd":0.5309149999999985,"input_tokens":2058372,"output_tokens":8161,"samples":219},{"day":"2026-10-16","cost_usd":0.50005275,"input_tokens":1934123,"output_tokens":8261,"samples":201},{"day":"2026-10-17","cost_usd":0.6885615000000005,"input_tokens":2689606,"output_tokens":8080,"samples":250},{"day":"2026-10-18","cost_usd":0.8397672499999995,"input_tokens":3281053,"output_tokens":9752,"samples":256},{"day":"2026-10-19","cost_usd":0.7491307500000008,"input_tokens":2923755,"output_tokens":9096,"samples":250},{"day":"2026-10-20","cost_usd":0.9947679999999988,"input_tokens":3903208,"output_tokens":9483,"samples":247},{"day":"2026-10-21","cost_usd":1.193594250000002,"input_tokens":4686721,"output_tokens":10957,"samples":245},{"day":"2026-10-22","cost_usd":2008.6326094249998,"input_tokens":54180941,"output_tokens":7113321,"samples":272},{"day":"2026-10-23","cost_usd":1324.9088885250003,"input_tokens":7312989,"output_tokens":8208435,"samples":254},{"day":"2026-10-24","cost_usd":1153.5635407999994,"input_tokens":5243978,"output_tokens":7159967,"samples":211},{"day":"2026-10-25","cost_usd":901.603876300002,"input_tokens":4239986,"output_tokens":5682283,"samples":214},{"day":"2026-10-26","cost_usd":720.8247865000021,"input_tokens":4185691,"output_tokens":4316300,"samples":218}],"recording":{"agent_models":["claude-fable-5","claude-fable-5.1","claude-opus-5.5","claude-sonnet-5.5"],"unrecorded_models":[],"recent_models":["claude-fable-5","claude-fable-5.1","claude-opus-5.5","claude-sonnet-5.5"],"last_priced_growth_at":1793111849060,"last_agent_model_growth_at":1793111849060,"stale":false,"stale_after_ms":86400000}}
diff --git a/src/docs/benchmarks/outcomes-2026-10.json b/src/docs/benchmarks/outcomes-2026-10.json
index 2e74e841b2..fa240cb57e 100644
--- a/src/docs/benchmarks/outcomes-2026-10.json
+++ b/src/docs/benchmarks/outcomes-2026-10.json
@@ -421,5 +421,36 @@
"window_end": "2026-10-27T14:50:41.723Z",
"basis": "hive-attributed"
},
- "Hub": null
+ "Hub": null,
+ "Stories": [
+ {
+ "repo": "hivecommons/hive",
+ "number": 13523,
+ "review_url": "https://github.com/hivecommons/hive/pull/13523#pullrequestreview-5613922737",
+ "caught": "The changelog note was written as a sentence, not a list entry, so the release build would have rejected it; a second, duplicate note was also added.",
+ "opened_at": "2026-10-22T12:07:50Z",
+ "caught_at": "2026-10-22T12:12:08Z",
+ "fix_approved_at": "2026-10-22T12:16:15Z",
+ "merged_at": "2026-10-22T12:23:31Z"
+ },
+ {
+ "repo": "hivecommons/pluk",
+ "number": 328,
+ "review_url": "https://github.com/hivecommons/pluk/pull/328#pullrequestreview-5607075007",
+ "caught": "An apostrophe inside a quoted message ended the string early, so the project would not compile and no tests could run.",
+ "opened_at": "2026-10-21T22:30:54Z",
+ "caught_at": "2026-10-21T22:36:56Z",
+ "fix_approved_at": "2026-10-21T22:46:52Z",
+ "merged_at": "2026-10-21T22:54:09Z"
+ }
+ ],
+ "RedTeam": {
+ "attacks": 63,
+ "caught": 61,
+ "benign": 12,
+ "false_alarms": 1,
+ "source": "src/docs/ioscan-red-team.md (Results table; 61 blocked of 63 attack payloads, 1 of 12 benign controls wrongly redacted)",
+ "run_at": "2026-10-23T00:00:00Z"
+ },
+ "ReadmeMinBase": 200
}
diff --git a/src/docs/benchmarks/outcomes-2026-10.md b/src/docs/benchmarks/outcomes-2026-10.md
index ffee1b9301..8efe7df1a3 100644
--- a/src/docs/benchmarks/outcomes-2026-10.md
+++ b/src/docs/benchmarks/outcomes-2026-10.md
@@ -42,6 +42,11 @@ Whole evidence history, not just the window; months with n < 10 are gaps.
| 2026-09 | 0 | not reported (n=0) | not reported (n=0) | 3088 | 0.1% [0.1–0.3] (n=3088) | 0.34 h (n=3088) |
| 2026-10 | 1455 | 0.0% [0.0–8.2] (n=43) | 0.44 h (n=1455) | 2224 | 0.1% [0.0–0.4] (n=1513) | 0.19 h (n=2224) |
+## Real examples
+
+- [hivecommons/hive#13523](https://github.com/hivecommons/hive/pull/13523): the [posted review](https://github.com/hivecommons/hive/pull/13523#pullrequestreview-5613922737) caught: The changelog note was written as a sentence, not a list entry, so the release build would have rejected it; a second, duplicate note was also added. The fixed head was approved 4 minutes later; the PR merged 16 minutes after opening (opened 12:07, caught 12:12, approved 12:16, merged 2026-10-22 12:23 UTC).
+- [hivecommons/pluk#328](https://github.com/hivecommons/pluk/pull/328): the [posted review](https://github.com/hivecommons/pluk/pull/328#pullrequestreview-5607075007) caught: An apostrophe inside a quoted message ended the string early, so the project would not compile and no tests could run. The fixed head was approved 10 minutes later; the PR merged 23 minutes after opening (opened 22:30, caught 22:36, approved 22:46, merged 2026-10-21 22:54 UTC).
+
## Scale
- 13060 evidence bundles (10 unreadable) folding to 11571 pull requests across 10 repositories; 15054 verdicts recorded.
@@ -50,8 +55,24 @@ Whole evidence history, not just the window; months with n < 10 are gaps.
- First mediated merge in the evidence: 2026-10-09.
- Hub `GET /api/hub/evidence` was not consulted (it requires an admin session); numbers are this spoke's evidence only.
+## What the README shows and leaves out
+
+The README carries only plain-language cards; this section says which numbers made it and why the rest did not.
+
+- **Sent back before merge** is shown: 490 of 1455 reviewed merged PRs (full base, no exclusions beyond the window).
+- **Checks passing at merge** is not shown: a check result is recorded on 798 of 1455 reviewed merges (55%), under the 80% floor, so the pass rate would describe a biased subset.
+- **Revert rate** is not shown: the reviewed base is 43 merges observed 14+ days after merge, below the README minimum of 200, and the two groups are not distinguishable on this data.
+- **Cost per merged PR** is shown as an estimate: $2.13 = $6124.75 estimated spend / 2869 hive-attributed merges, 2026-08-08 → 2026-10-27.
+- **Time to merge** is shown in the Also strip as 27 minutes (typical, i.e. median) for 1455 reviewed merges, against 12 minutes for 2224 merges without review; the extra minutes are stated as the cost of review.
+- **Not a controlled comparison.** Pull requests merged without a Hive review come from other repositories and periods; differences between the two groups may not be caused by review.
+- **Pull requests sent back.** Counts a `changes_requested` verdict on any head before merge. Whether the next head fixed the finding, or a human overrode it, is not verified here.
+- **Prompt injection.** The catch rate is from Hive's own evaluation set, not production traffic, and is not subject to the README minimum base because it is a self-run test and labelled as such on the card.
+- **Post-merge CI pass rate** is drawn only when at least 80% of reviewed merges carry a recorded check result; a roadmap item is to record check status for every merged head so it can be reported.
+
## Methodology
+How measured: per-PR evidence bundles the spoke already writes, folded per PR; 95% Wilson intervals; n < 10 is never reported. The README shows only cards whose base is at least 200 (`--readme-min-base`), in plain words and without intervals; everything below that threshold, and every interval, is in this report.
+
- **Source.** Only the per-PR evidence bundles the spoke already writes under `/data/evidence` (schema v1), folded per PR with `pkg/evidence.FoldPRs`. No new telemetry.
- **Cohorts.** A PR is *mediated* if any head carries a Hive verdict or a posted Hive review; otherwise *unmediated*. Backfilled history is unmediated by construction, so the window must start no earlier than the first mediated merge or the comparison is confounded by time.
- **Revert / change-failure.** Denominator is merged PRs observed at least 14 days after merge. A revert is a later PR the evidence linker tagged `reverted_by`; change-failure adds `followed_up_by` fixes.
diff --git a/src/docs/benchmarks/red-team-2026-10.json b/src/docs/benchmarks/red-team-2026-10.json
new file mode 100644
index 0000000000..c301fd9d48
--- /dev/null
+++ b/src/docs/benchmarks/red-team-2026-10.json
@@ -0,0 +1,8 @@
+{
+ "attacks": 63,
+ "caught": 61,
+ "benign": 12,
+ "false_alarms": 1,
+ "source": "src/docs/ioscan-red-team.md (Results table; 61 blocked of 63 attack payloads, 1 of 12 benign controls wrongly redacted)",
+ "run_at": "2026-10-23T00:00:00Z"
+}
diff --git a/src/docs/benchmarks/stories-2026-10.json b/src/docs/benchmarks/stories-2026-10.json
new file mode 100644
index 0000000000..561b4e6cfa
--- /dev/null
+++ b/src/docs/benchmarks/stories-2026-10.json
@@ -0,0 +1,22 @@
+[
+ {
+ "repo": "hivecommons/hive",
+ "number": 13523,
+ "review_url": "https://github.com/hivecommons/hive/pull/13523#pullrequestreview-5613922737",
+ "caught": "The changelog note was written as a sentence, not a list entry, so the release build would have rejected it; a second, duplicate note was also added.",
+ "opened_at": "2026-10-22T12:07:50Z",
+ "caught_at": "2026-10-22T12:12:08Z",
+ "fix_approved_at": "2026-10-22T12:16:15Z",
+ "merged_at": "2026-10-22T12:23:31Z"
+ },
+ {
+ "repo": "hivecommons/pluk",
+ "number": 328,
+ "review_url": "https://github.com/hivecommons/pluk/pull/328#pullrequestreview-5607075007",
+ "caught": "An apostrophe inside a quoted message ended the string early, so the project would not compile and no tests could run.",
+ "opened_at": "2026-10-21T22:30:54Z",
+ "caught_at": "2026-10-21T22:36:56Z",
+ "fix_approved_at": "2026-10-21T22:46:52Z",
+ "merged_at": "2026-10-21T22:54:09Z"
+ }
+]
diff --git a/src/docs/documentation-map.md b/src/docs/documentation-map.md
index aa4ab1d170..d12defba59 100644
--- a/src/docs/documentation-map.md
+++ b/src/docs/documentation-map.md
@@ -21,7 +21,7 @@ Documentation for the current Hive line (branch `v5`; the code and docs live und
- [Work-source terminology](work-source-terminology.md) — glossary and audit notes for keeping user-facing copy neutral across GitHub, GitLab, Gitea, Linear, Jira, and future work sources.
- [Release channels](release-channels.md) — `stable`/`candidate`/`edge` moving image tags, per-line channel ownership (`v5` → `candidate`/`latest`, `v6` → `edge`, `v4` → none), switching a hive to a channel, the version pill, and the timeline for when a hive follows `stable` after the tag moves.
- [Digest-verifiable rollback](release-rollback.md) - the operator runbook for pinning a hive back to a prior immutable short-SHA build for all three images (`hive`, `hive-contributor`, `hive-hub`), stopping the hub automation that would undo it, and verifying by digest rather than by tag that the pin landed on the running spoke.
-- [Outcome benchmarks](benchmarks/README.md) — reproducible mediated-vs-unmediated comparison (revert rate, change-failure, time-to-merge, CI at merged head, changes requested pre-merge, cost per merged PR) from stored review evidence, with the monthly reports, generated charts and the README outcomes block it keeps in sync.
+- [Outcome benchmarks](benchmarks/README.md) — reproducible mediated-vs-unmediated comparison (revert rate, change-failure, time-to-merge, CI at merged head, changes requested pre-merge, cost per merged PR) from stored review evidence, with the monthly reports, generated charts, the plain-language README cards (minimum-base and vocabulary rules) and the README outcomes block it keeps in sync.
- [v5 GA readiness bar](v5-ga.md) — measurable release-train, migration, safety, and governance criteria that must be evidenced before v5 can be promoted beyond the active-development `edge` channel.
- [v5 GA candidate week sequencing](v5-candidate-week.md) — dependency-ordered plan for the remaining human-gated GA-bar steps: candidate-SHA designation, candidate-pinned measurements, the two bound exercises, and the v4 freeze evaluation.
- [Stable soak and promotion policy (v5 line)](stable-soak-policy.md) — the CI-enforced gate between `candidate` and `stable`: soak conditions, immutable short-SHA rollback tags, the emergency exception path (what it waives, and the post-hoc evidence that closes its follow-up issue), and the ledger of exceptions taken with their per-image rollback digests.
@@ -185,6 +185,7 @@ Documentation for the current Hive line (branch `v5`; the code and docs live und
## Architecture and design
+- [What Hive is](what-is-hive.md) — the project overview that used to open the README: how Hive differs from a terminal agent, hub and spoke, the contributor relay, project vs hosted service, "does Hive fit my project", run-it paths, the ACMM level table, works-with, related projects, and where adopters/contributing/security live.
- [Architecture](architecture.md) — process model, governor loop, guardrails, hub/spoke, and walkthrough.
- [Automatic repo ACMM onboarding reconciler](design/repo-acmm-onboarding.md) — proposed design for per-repo ACMM targets, gap-to-work reconciliation, human hand-off, waivers, and anti-gaming controls (RFC #6235).
- [Hive federation design](../../docs/federation-design.md) — the multi-hive registry: live `/api/hives` endpoints, project onboarding, contributor flow across hubs, and what remains future design work.
diff --git a/src/docs/hivectl.md b/src/docs/hivectl.md
index a26f4f3b26..4479879f79 100644
--- a/src/docs/hivectl.md
+++ b/src/docs/hivectl.md
@@ -302,10 +302,14 @@ at merged head, the share of mediated merges that carried a `changes_requested`
`--spend` names a saved `GET /api/cost` response — cost per merged PR. Every rate carries its n and a 95 % Wilson
interval; n < 10 is `not reported`. `--month YYYY-MM` bounds the merge time (default all), `--now` fixes the
observation time for maturation, `--hub` adds a saved `GET /api/hub/evidence` response as fleet context, `--source`
-names the data in the methodology section. It writes `outcomes-YYYY-MM.{md,json}` and
-`assets/outcomes-YYYY-MM-{bars,trend}.svg` under `--out-dir`, and `--readme FILE` rewrites that file's
-`…` block from the same report. Local filesystem only; no hive session, no
-telemetry. `src/scripts/outcome-benchmark.sh` wraps it; see [benchmarks/](benchmarks/README.md).
+names the data in the methodology section. It writes `outcomes-YYYY-MM.{md,json}`,
+`assets/outcomes-YYYY-MM-{bars,trend}.svg` and the plain-language README cards
+`assets/outcomes-YYYY-MM-card--{dark,light}.svg` under `--out-dir`, and `--readme FILE` rewrites that file's
+`…` block from the same report. The README block carries only the cards
+and flat sentences, no intervals: `--readme-min-base N` (default 200) is the smallest base a production-count card
+may rest on, separate from the n ≥ 10 report threshold; `--stories FILE` supplies curated caught → fixed → merged
+examples and `--red-team FILE` the prompt-injection evaluation counts (see [benchmarks/](benchmarks/README.md) for
+both formats). Local filesystem only; no hive session, no telemetry. `src/scripts/outcome-benchmark.sh` wraps it; see [benchmarks/](benchmarks/README.md).
### evidence — backfill bundles from GitHub history
diff --git a/src/docs/what-is-hive.md b/src/docs/what-is-hive.md
new file mode 100644
index 0000000000..6fb2cea56c
--- /dev/null
+++ b/src/docs/what-is-hive.md
@@ -0,0 +1,213 @@
+# What Hive is
+
+This page holds the project overview that used to live in the repository
+README: what Hive does, how it differs from a terminal agent, the hub/spoke
+model, the contributor relay, the ACMM trust levels, and the questions people
+ask before adopting it. The README itself is kept short and carries the
+measured outcomes; everything else is here or linked from here.
+
+## Overview
+
+Hive watches the work that piles up in a repository — issues, pull requests,
+failing CI, CVEs, and dependabot alerts — and sends AI coding agents (Claude,
+Copilot, Codex, Antigravity, Goose…) to deal with it: triage, fixes, reviews,
+releases. You decide how much it is allowed to do. Humans keep every decision
+that matters; agents do the chores.
+
+## "I already use an agent. How is this different?"
+
+A terminal agent is a tool you hold, aim, and stop. Hive runs those tools for
+you, on three rules:
+
+1. **Short-lived agents.** Every task starts a fresh agent with a clean context
+ and ends it. No long-running agent drifting for hours.
+2. **External context.** What the agent needs to know lives outside the model —
+ in your repo, your knowledge base, your audit trail — and is re-injected
+ each time, so nothing is forgotten and nothing is reconstructed from stale
+ memory.
+3. **Prompts driven by deterministic facts.** The instructions an agent gets
+ are templates filled from real data (the issue, the CI log, the diff, the
+ trust level), not hand-tuned prose.
+
+Around that, Hive picks work from your real queue, sets limits in code with six
+trust levels from the **AI Codebase Maturity Model (ACMM)** — a framework for
+how much autonomy a codebase is ready to give agents ([levels](#acmm-levels) ·
+[paper](https://arxiv.org/abs/2604.09388)), checks before and after with
+deterministic filters and gates, writes every action to an audit log owned by
+a named human, and runs around the clock. Same agents you use today —
+governed, scheduled, scoped, and accountable.
+
+## Hub and spoke
+
+- **Spoke** — one running Hive, serving your repo(s). It is where agents
+ actually work. Runs on your Kubernetes cluster, Podman, or Docker Compose.
+- **Hub** — the control plane for many spokes: registers, provisions, upgrades
+ and watches them; fleet view, leaderboards, routes contributor compute to the
+ hives that need it.
+
+A spoke works alone. A hub is how you see and manage a fleet of them.
+
+## Contributor relay: a swarm working on your behalf
+
+Your own agents are only the start. The **contributor relay** (ClankeR) lets
+anyone who wants to help your project donate their compute and model tokens
+without you handing out anything:
+
+- **They bring their own agent.** A contributor runs one command on their
+ laptop, cluster, or Compose box; it connects outbound to your hive, takes
+ one curated task at a time, and runs *their* CLI with *their* model and
+ *their* subscription.
+- **Their secrets never reach you.** API keys and CLI logins stay on the
+ contributor's machine. The hive never sees them.
+- **Your forge settings never change.** Nobody is added as a collaborator. For
+ each task the hive mints a short-lived, task-scoped GitHub token so the PR
+ can be opened, then it expires. Revoke a contributor and their access is
+ gone.
+- **Your prompts, your limits.** Contributors work the queue you curated,
+ under the same trust level, filters and gates as your resident agents.
+
+The result is an elastic swarm: a three-person project can have a dozen
+contributors' agents working its backlog overnight, each accountable to a
+named human, while the maintainers hold every merge decision. See
+[contributor-relay.md](contributor-relay.md).
+
+### Contribute to a Hive
+
+
+
+Community members can contribute compute to any hive through ClankeR: it hands
+tasks from a hive's backlog to the CLI agent running on your own machine, under
+the hive's prompts and limits, with no repo permissions and no shared secrets.
+
+Start with [contributor-relay.md](contributor-relay.md) or the
+[Hive Hub contribute page](https://hive.hivecommons.dev).
+
+Homebrew is an additional install option:
+
+```bash
+brew install hivecommons/hive/contribute
+```
+
+Follow the [Homebrew setup instructions](contributor-relay.md#homebrew) to
+register without a checkout before running `hive-contribute`. The
+[`just` workflow](contributor-relay.md#from-source) remains the from-source
+path.
+
+## Hive the project vs the hosted service
+
+- **Hive the project**: Apache-2.0 code in this repo. Run it yourself.
+- **Hive the hosted service**: [hive.hivecommons.dev](https://hive.hivecommons.dev)
+ — a hub we operate so you can onboard a spoke without a cluster. Same
+ software, one deployment of it. See [hosted-hub.md](hosted-hub.md).
+
+## Does Hive fit my project?
+
+**Do I need a paid agent subscription?** No. Hive drives the agent CLIs you
+already have (Claude, Copilot, Codex, Antigravity, Goose) *or* any
+OpenAI-compatible model through its Model Gateway (OpenRouter key, no CLI
+login). Contributors can bring their own.
+
+**Who tells it what to work on?** Your backlog does. The Governor reads your
+issues/PRs/CI, filters them deterministically (labels, areas, complexity), and
+only hands agents work you have allowed. You choose the areas and the trust
+level. Nothing is invented.
+
+**We require design discussion before non-trivial PRs.** That is a trust
+level, not a workaround: *Measured* mode = agents may only file findings and
+proposals as issues — no code, no PRs — so the discussion happens first;
+*Holdgated* = PRs are opened with a `hold` label and a human must review and
+release; *Full* (L6) = merge on green. Start at Measured. You never have to
+move up.
+
+**Can it open PRs to our upstream repo?** Yes, if you install it there. Or run
+it on a fork (`upstream_watch` keeps the fork current) and push reviewed PRs
+upstream yourself.
+
+More answers:
+
+
+## Run it
+
+
+
+| Path | Start here |
+| --- | --- |
+| Docker Compose | [Quick Start (Docker Compose)](quickstart-docker-compose.md) |
+| Podman | [Quick Start (Podman)](quickstart-podman.md) |
+| Kubernetes | [Kubernetes deployment](kubernetes-deployment.md) |
+| Hosted hub | [https://hive.hivecommons.dev](https://hive.hivecommons.dev) and the [hosted hub guide](hosted-hub.md) |
+| Getting started | [Zero to Automation: Getting Started with Hive](getting-started.md) |
+
+Configuration lives in `hive.yaml`; see the
+[operator reference](operator-reference.md),
+[environment variable reference](env-vars.md), and
+[agent configuration](agent-configuration.md).
+
+## ACMM Levels
+
+Hive uses an **AI Codebase Maturity Model** (ACMM) with six levels that control
+what agents are allowed to do:
+
+| Level | Name | What agents can do |
+|-------|------|-------------------|
+| L1 | Inception (Assisted) | Interactive advisor and project inception. Advisory beads only. |
+| L2 | Advisory (Instructed) | Observe and report findings as dashboard beads. No GitHub interaction. |
+| L3 | Quality-Gated (Measured) | Quality agent opens issues and PRs gated by the literal `hold` level-gate label. Others remain advisory. |
+| L4 | Security-Aware (Adaptive) | All agents file issues. Quality, sec-check, and CI open PRs gated by `hold`. |
+| L5 | Semi-Autonomous (Semi-Automated) | All agents open PRs gated by `hold`. Humans batch-review and approve. |
+| L6 | Autonomous | Agents open PRs and auto-merge when checks pass. Non-outreach PRs have no level hold; outreach PRs remain held. |
+
+See [acmm-policy-matrix.md](acmm-policy-matrix.md) for the full matrix and
+[labels-and-control-signals.md](labels-and-control-signals.md) for label
+behavior.
+
+## Architecture
+
+Hive runs as a single container with a Go control plane, an authenticated
+Node.js gateway, and ttyd for web terminal access. The Governor evaluates
+queue depth, deterministic filters decide what an agent may see, scoped tokens
+and a MITM proxy enforce what it may do, and gates decide what may land.
+
+See [architecture.md](architecture.md) for process model, governor loop,
+guardrails, hub/spoke, and walkthrough diagrams.
+
+## Terminal dashboard
+
+`hivectl tui` is a full-screen terminal view of the same dashboard API the web
+UI uses: agents, governor, token spend, live activity, pause/resume, kick,
+ACMM level, and hive-profile actions. See
+[`hivectl tui` in the command reference](hivectl.md#tui--live-terminal-dashboard).
+
+## Works with
+
+Hive drives Claude Code, GitHub Copilot CLI, Codex, Goose, Antigravity and
+OpenAI-compatible gateways; reads work from GitHub Issues, GitHub Projects,
+Linear and Jira; and runs on GitHub today with GitLab and Gitea/Forgejo
+planned. See the [integrations page](integrations.md) for the full logo grid
+with status.
+
+## Related projects
+
+- [Goose](https://github.com/aaif-goose/goose), an agent backend whose
+ upstream moved to the [Agentic AI Foundation (AAIF)](https://aaif.io/). Hive
+ integrates Goose as an independent CNCF project; Hive does not join AAIF,
+ seek AAIF membership, or rely on AAIF to host or govern Hive. See
+ [unattended Goose with Hive](../../docs/goose-at-scale.md) and
+ [Goose upstream tracking](landscape.md#agentic-ai-foundation-aaif-and-goose).
+
+## Repositories, adopters, contributing, security
+
+- [Repositories](../../docs/repositories.md) — the other repositories in the
+ hivecommons organization, what each is for, and which ship inside a release.
+- [ADOPTERS.md](../../ADOPTERS.md) — organizations running Hive. Add yours
+ with a pull request.
+- [CONTRIBUTING.md](../../CONTRIBUTING.md) — contributing to Hive itself.
+ Contributors follow the
+ [CNCF Code of Conduct](https://github.com/cncf/foundation/blob/main/code-of-conduct.md),
+ adopted in [CODE_OF_CONDUCT.md](../../CODE_OF_CONDUCT.md). Recent changes
+ are in [CHANGELOG.md](../../CHANGELOG.md).
+- [Security model](security-model.md) — code-grounded inventory of agent
+ safety controls, enforcing versus advisory modes, known gaps, and audit
+ steps. Vulnerability disclosure: [SECURITY.md](../../SECURITY.md); telemetry
+ and data collection: [telemetry.md](telemetry.md). Do not report security
+ vulnerabilities through public issues or pull requests.
diff --git a/src/pkg/hivectl/commands/evidence_benchmark.go b/src/pkg/hivectl/commands/evidence_benchmark.go
index 0b26647396..6339771f44 100644
--- a/src/pkg/hivectl/commands/evidence_benchmark.go
+++ b/src/pkg/hivectl/commands/evidence_benchmark.go
@@ -22,7 +22,8 @@ var evidenceBenchmarkNow = func() time.Time { return time.Now().UTC() }
const benchmarkReadmeDocDir = "src/docs/benchmarks"
func newEvidenceBenchmarkCommand(_ *commandEnv) *cobra.Command {
- var root, month, now, spendPath, hubPath, outDir, readme, source string
+ var root, month, now, spendPath, hubPath, storiesPath, redTeamPath, outDir, readme, source string
+ var readmeMinBase int
cmd := &cobra.Command{
Use: "benchmark",
Short: "Compare mediated and unmediated merged PRs from stored evidence and write the outcomes report",
@@ -31,9 +32,11 @@ func newEvidenceBenchmarkCommand(_ *commandEnv) *cobra.Command {
"median time-to-merge, CI failure at merged head, changes-requested-before-merge, and cost per merged PR when a\n" +
"saved /api/cost response is given with --spend. Every rate carries its n and a 95% Wilson interval; n < 10 is\n" +
"printed as \"not reported\". No telemetry is added and nothing contacts a hive.\n\n" +
- "Writes /outcomes-YYYY-MM.md, outcomes-YYYY-MM.json and assets/outcomes-YYYY-MM-{bars,trend}.svg.\n" +
- "--readme rewrites the … block of that README from the same\n" +
- "report, so the README never drifts from the committed report (a test asserts they match).",
+ "Writes /outcomes-YYYY-MM.md, outcomes-YYYY-MM.json, assets/outcomes-YYYY-MM-{bars,trend}.svg and the\n" +
+ "plain-language README cards assets/outcomes-YYYY-MM-card--{dark,light}.svg (one number per card, no\n" +
+ "statistics; cards whose base is under --readme-min-base are not written). --stories supplies curated real PRs\n" +
+ "for the story card; --red-team the prompt-injection eval summary. --readme rewrites the … block of that README\n" +
+ "from the same report, so the README never drifts from the committed report (a test asserts they match).",
Example: " hivectl evidence benchmark --root ./evidence --month 2026-10 --spend cost.json --out-dir src/docs/benchmarks --readme README.md\n" +
" hivectl evidence benchmark --root /data/evidence --month 2026-10 --out-dir /tmp/bench",
Args: argsNone(),
@@ -61,6 +64,17 @@ func newEvidenceBenchmarkCommand(_ *commandEnv) *cobra.Command {
return err
}
}
+ if storiesPath != "" {
+ if opts.Stories, err = outcomebench.LoadStories(storiesPath); err != nil {
+ return err
+ }
+ }
+ if redTeamPath != "" {
+ if opts.RedTeam, err = outcomebench.LoadRedTeam(redTeamPath); err != nil {
+ return err
+ }
+ }
+ opts.ReadmeMinBase = readmeMinBase
opts.Source = source
if opts.Source == "" {
opts.Source = "evidence bundles under " + root
@@ -103,6 +117,9 @@ func newEvidenceBenchmarkCommand(_ *commandEnv) *cobra.Command {
cmd.Flags().StringVar(&now, "now", "", "observation time (RFC3339) for the maturation window; default now")
cmd.Flags().StringVar(&spendPath, "spend", "", "saved GET /api/cost JSON for cost per merged PR")
cmd.Flags().StringVar(&hubPath, "hub", "", "saved GET /api/hub/evidence JSON for fleet context")
+ cmd.Flags().StringVar(&storiesPath, "stories", "", "saved JSON list of real PRs for the README story card")
+ cmd.Flags().StringVar(&redTeamPath, "red-team", "", "saved JSON summary of the prompt-injection eval (docs/ioscan-red-team.md) for the security card")
+ cmd.Flags().IntVar(&readmeMinBase, "readme-min-base", outcomebench.DefaultReadmeMinBase, "smallest count a README card may be built on")
cmd.Flags().StringVar(&outDir, "out-dir", "docs/benchmarks", "directory for the report, JSON and assets/")
cmd.Flags().StringVar(&readme, "readme", "", "README whose outcomes block to rewrite from this report")
cmd.Flags().StringVar(&source, "source", "", "how the report names its source (default: the --root path)")
@@ -126,8 +143,20 @@ func writeBenchmark(rep outcomebench.Report, outDir string) ([]string, error) {
filepath.Join(outDir, "assets", rep.AssetBase()+"-bars.svg"): []byte(rep.BarChartSVG()),
filepath.Join(outDir, "assets", rep.AssetBase()+"-trend.svg"): []byte(rep.TrendSVG()),
}
+ order := []string{base + ".md", base + ".json", filepath.Join(outDir, "assets", rep.AssetBase()+"-bars.svg"), filepath.Join(outDir, "assets", rep.AssetBase()+"-trend.svg")}
+ cards := rep.Cards()
+ for _, name := range outcomebench.CardNames() {
+ for _, theme := range []string{"dark", "light"} {
+ file := rep.CardFile(name, theme)
+ if svg, ok := cards[file]; ok {
+ p := filepath.Join(outDir, "assets", file)
+ files[p] = []byte(svg)
+ order = append(order, p)
+ }
+ }
+ }
paths := make([]string, 0, len(files))
- for _, p := range []string{base + ".md", base + ".json", filepath.Join(outDir, "assets", rep.AssetBase()+"-bars.svg"), filepath.Join(outDir, "assets", rep.AssetBase()+"-trend.svg")} {
+ for _, p := range order {
if err := os.WriteFile(p, files[p], 0o644); err != nil {
return nil, fmt.Errorf("writing %s: %w", p, err)
}
diff --git a/src/pkg/outcomebench/bench.go b/src/pkg/outcomebench/bench.go
index 1155a1a734..7f787a23ce 100644
--- a/src/pkg/outcomebench/bench.go
+++ b/src/pkg/outcomebench/bench.go
@@ -46,6 +46,12 @@ type Options struct {
Hub *HubContext
// Source names where the bundles came from, for the methodology section.
Source string
+ // Stories are curated real PRs for the README story card; nil omits it.
+ Stories []Story
+ // RedTeam is the saved prompt-injection eval summary; nil omits the card.
+ RedTeam *RedTeam
+ // ReadmeMinBase overrides DefaultReadmeMinBase (0 keeps the default).
+ ReadmeMinBase int
}
// Spend is what /api/cost already publishes.
@@ -138,6 +144,10 @@ type Report struct {
Trend map[string][]MonthPoint
Spend *Spend
Hub *HubContext
+ // Stories and ReadmeMinBase feed the README cards; see cards.go.
+ Stories []Story `json:",omitempty"`
+ RedTeam *RedTeam `json:",omitempty"`
+ ReadmeMinBase int `json:",omitempty"`
}
// RepoCount is merged PRs per repository in the window.
@@ -291,6 +301,7 @@ func Compute(bundles []*evidence.Bundle, unreadable int, opts Options) Report {
GeneratedAt: now, Since: opts.Since, Until: opts.Until, Now: now, Source: opts.Source,
Bundles: len(bundles), Unreadable: unreadable, PRs: len(prs),
Spend: opts.Spend, Hub: opts.Hub, Trend: map[string][]MonthPoint{},
+ Stories: opts.Stories, RedTeam: opts.RedTeam, ReadmeMinBase: opts.ReadmeMinBase,
}
var med, unmed cohortAcc
repos := map[string]int{}
diff --git a/src/pkg/outcomebench/cards.go b/src/pkg/outcomebench/cards.go
new file mode 100644
index 0000000000..7b1ce3f7e6
--- /dev/null
+++ b/src/pkg/outcomebench/cards.go
@@ -0,0 +1,518 @@
+package outcomebench
+
+import (
+ "encoding/json"
+ "fmt"
+ "html"
+ "math"
+ "os"
+ "strings"
+ "time"
+)
+
+// The README cards are the plain-language face of the benchmark: one idea
+// per card, one number in big type, one flat sentence, and a footer that
+// says in words what was counted. Intervals, n and cohort names stay in the
+// generated report; the cards never print them, and a card whose base is
+// below MinBase is not written at all.
+
+// Story is one real pull request where the review swarm caught something
+// before merge and the author fixed it. Stories are curated input (a saved
+// JSON list), not computed, because choosing a readable, non-sensitive
+// example is a human call; every field is checkable against the PR.
+type Story struct {
+ Repo string `json:"repo"`
+ Number int `json:"number"`
+ // ReviewURL is the posted review that carried the changes_requested
+ // verdict.
+ ReviewURL string `json:"review_url"`
+ // Caught is the plain-English description of what the reviewer found.
+ Caught string `json:"caught"`
+ // OpenedAt, CaughtAt, FixApprovedAt and MergedAt come from the evidence
+ // bundles: PR opened, first changes_requested verdict, first approve on
+ // the following head, merge.
+ OpenedAt time.Time `json:"opened_at"`
+ CaughtAt time.Time `json:"caught_at"`
+ FixApprovedAt time.Time `json:"fix_approved_at"`
+ MergedAt time.Time `json:"merged_at"`
+}
+
+// URL is the pull request page.
+func (s Story) URL() string { return fmt.Sprintf("https://github.com/%s/pull/%d", s.Repo, s.Number) }
+
+// FixMinutes is how long after the catch the fixed head was approved.
+func (s Story) FixMinutes() int { return int(math.Round(s.FixApprovedAt.Sub(s.CaughtAt).Minutes())) }
+
+// MergeMinutes is open-to-merge for the whole PR.
+func (s Story) MergeMinutes() int { return int(math.Round(s.MergedAt.Sub(s.OpenedAt).Minutes())) }
+
+// LoadStories reads a saved JSON list of stories and rejects any whose
+// timeline is not in order, so a typo cannot produce a negative minute.
+func LoadStories(path string) ([]Story, error) {
+ data, err := os.ReadFile(path)
+ if err != nil {
+ return nil, err
+ }
+ var out []Story
+ if err := json.Unmarshal(data, &out); err != nil {
+ return nil, fmt.Errorf("parsing %s: %w", path, err)
+ }
+ for i, s := range out {
+ if s.Repo == "" || s.Number <= 0 || strings.TrimSpace(s.Caught) == "" {
+ return nil, fmt.Errorf("%s: story %d needs repo, number and caught", path, i)
+ }
+ if !(s.OpenedAt.Before(s.CaughtAt) && s.CaughtAt.Before(s.FixApprovedAt) && s.FixApprovedAt.Before(s.MergedAt)) {
+ return nil, fmt.Errorf("%s: story %s#%d timeline is out of order", path, s.Repo, s.Number)
+ }
+ }
+ return out, nil
+}
+
+// RedTeam is the result of Hive's own prompt-injection evaluation
+// (docs/ioscan-red-team.md): how many attack payloads the scanner caught
+// and how many harmless requests it wrongly flagged. It is a lab set, not
+// production traffic, and the card says so; MinBase does not apply because
+// the base is the size of the eval set, which the footer states.
+type RedTeam struct {
+ Attacks int `json:"attacks"`
+ Caught int `json:"caught"`
+ Benign int `json:"benign"`
+ FalseAlarms int `json:"false_alarms"`
+ Source string `json:"source"`
+ // RunAt is when the eval set was last run, for the card footer.
+ RunAt time.Time `json:"run_at"`
+}
+
+// LoadRedTeam reads a saved red-team summary.
+func LoadRedTeam(path string) (*RedTeam, error) {
+ data, err := os.ReadFile(path)
+ if err != nil {
+ return nil, err
+ }
+ var rt RedTeam
+ if err := json.Unmarshal(data, &rt); err != nil {
+ return nil, fmt.Errorf("parsing %s: %w", path, err)
+ }
+ if rt.Attacks <= 0 || rt.Caught < 0 || rt.Caught > rt.Attacks || rt.Benign < 0 || rt.FalseAlarms > rt.Benign {
+ return nil, fmt.Errorf("%s: counts are inconsistent", path)
+ }
+ return &rt, nil
+}
+
+// DefaultReadmeMinBase is the smallest production count a README card may
+// be built on. It is deliberately far above MinN: the report may discuss a
+// 43-merge bucket with its interval, but a card with one big number and no
+// interval must rest on a base a reader would trust without one.
+const DefaultReadmeMinBase = 200
+
+// MinBase is the README card threshold in force for this report.
+func (r Report) MinBase() int {
+ if r.ReadmeMinBase > 0 {
+ return r.ReadmeMinBase
+ }
+ return DefaultReadmeMinBase
+}
+
+// minCICardCoverage is the share of reviewed merges that must carry a
+// recorded check result before the CI card is drawn; below it the figure
+// describes a sample the footer cannot explain in one line.
+const minCICardCoverage = 0.8
+
+// cardTheme mirrors the dashboard's design tokens (src/pkg/dashboard/static/
+// tokens.css): surface-2 panels, amber brand, muted slate text. One SVG is
+// written per theme and the README picks one with .
+type cardTheme struct {
+ Name, BG, Border, Text, Muted, Brand, Neutral, OK string
+}
+
+var cardThemes = []cardTheme{
+ {Name: "dark", BG: "#121922", Border: "#263545", Text: "#f6f8fb", Muted: "#a8b3c2", Brand: "#f4c75f", Neutral: "#56657a", OK: "#74df9a"},
+ {Name: "light", BG: "#ffffff", Border: "#d9dee5", Text: "#1f2937", Muted: "#4b5563", Brand: "#9a5f0b", Neutral: "#9aa5b4", OK: "#16a34a"},
+}
+
+const (
+ cardFont = "Inter, ui-sans-serif, system-ui, -apple-system, 'Segoe UI', sans-serif"
+ cardW = 280
+ cardH = 210
+ cardWideW = 880
+ cardPad = 20
+ cardRadius = 12
+ eyebrowPx = 10
+ claimPx = 14
+ footerPx = 10
+ bigPx = 52
+ claimLineH = 19
+ claimChars = 32 // characters per claim line at claimPx in a 280px card
+ footChars = 46 // characters per footer line at footerPx in a 280px card
+ wideChars = 58 // characters per line in a two-column wide card
+ tileChars = 28 // characters per line in a four-tile strip
+ ladderChars = 70
+)
+
+// Card names; the asset file is -card--.svg.
+const (
+ CardSentBack = "sent-back"
+ CardChecksGreen = "checks-green"
+ CardStories = "stories"
+ CardPromptInjection = "prompt-injection"
+ CardCost = "cost"
+ CardAlso = "also"
+ CardHowItWorks = "how-it-works"
+)
+
+// CardNames lists every card the generator knows, in README order.
+func CardNames() []string {
+ return []string{CardSentBack, CardChecksGreen, CardStories, CardPromptInjection, CardCost, CardAlso, CardHowItWorks}
+}
+
+// CardFile is the asset filename for a card in a theme.
+func (r Report) CardFile(name, theme string) string {
+ return fmt.Sprintf("%s-card-%s-%s.svg", r.AssetBase(), name, theme)
+}
+
+// Cards renders every card in every theme, keyed by filename. Cards whose
+// data is missing or below MinBase are omitted, never drawn as placeholders.
+func (r Report) Cards() map[string]string {
+ out := map[string]string{}
+ for _, th := range cardThemes {
+ for _, name := range CardNames() {
+ if svg, ok := r.Card(name, th); ok {
+ out[r.CardFile(name, th.Name)] = svg
+ }
+ }
+ }
+ return out
+}
+
+// Card renders one named card; ok is false when its data is not shown.
+func (r Report) Card(name string, th cardTheme) (string, bool) {
+ switch name {
+ case CardSentBack:
+ return r.sentBackCard(th)
+ case CardChecksGreen:
+ return r.checksGreenCard(th)
+ case CardStories:
+ return r.storiesCard(th)
+ case CardPromptInjection:
+ return r.promptInjectionCard(th)
+ case CardCost:
+ return r.costCard(th)
+ case CardAlso:
+ return r.alsoCard(th)
+ case CardHowItWorks:
+ return howItWorksCard(th), true
+ }
+ return "", false
+}
+
+// HasCard reports whether a card has data to show.
+func (r Report) HasCard(name string) bool {
+ _, ok := r.Card(name, cardThemes[0])
+ return ok
+}
+
+// windowLabel is "Oct 2026" for a monthly report, or the generation month.
+func (r Report) windowLabel() string {
+ t := r.Since
+ if t.IsZero() {
+ t = r.GeneratedAt
+ }
+ return monthLabel(t)
+}
+
+// monthLabel is "Aug 2026".
+func monthLabel(t time.Time) string { return t.UTC().Format("Jan 2006") }
+
+// commas formats 11571 as 11,571.
+func commas(n int) string {
+ if n < 0 {
+ return "-" + commas(-n)
+ }
+ s := fmt.Sprint(n)
+ for i := len(s) - 3; i > 0; i -= 3 {
+ s = s[:i] + "," + s[i:]
+ }
+ return s
+}
+
+// oneIn turns a rate into "1 in N" (N rounded), the way people say it.
+func oneIn(rate float64) string {
+ if rate <= 0 {
+ return "0"
+ }
+ return fmt.Sprintf("1 in %d", int(math.Round(1/rate)))
+}
+
+// minutes rounds hours to whole minutes.
+func minutes(hours float64) int { return int(math.Round(60 * hours)) }
+
+// wrap splits s into lines of at most width characters on word boundaries.
+func wrap(s string, width int) []string {
+ var lines []string
+ line := ""
+ for _, w := range strings.Fields(s) {
+ switch {
+ case line == "":
+ line = w
+ case len(line)+1+len(w) > width:
+ lines = append(lines, line)
+ line = w
+ default:
+ line += " " + w
+ }
+ }
+ if line != "" {
+ lines = append(lines, line)
+ }
+ return lines
+}
+
+func esc(s string) string { return html.EscapeString(s) }
+
+func cardOpen(b *strings.Builder, th cardTheme, w, h int, title string) {
+ fmt.Fprintf(b, `\n")
+ return b.String()
+}
+
+func (r Report) sentBackCard(th cardTheme) (string, bool) {
+ m := r.Mediated.ChangesRequested
+ if !r.SplitSupported() || !m.Reported || m.Rate <= 0 || m.N < r.MinBase() {
+ return "", false
+ }
+ return numberCard(th,
+ fmt.Sprintf("%s reviewed pull requests was sent back for fixes before it merged", oneIn(m.Rate)),
+ "Accuracy · sent back before merge",
+ oneIn(m.Rate),
+ "reviewed pull requests was sent back for fixes before it merged.",
+ fmt.Sprintf("from %s merged pull requests Hive reviewed, %s", commas(m.N), r.windowLabel()),
+ ), true
+}
+
+func (r Report) checksGreenCard(th cardTheme) (string, bool) {
+ c := r.Mediated
+ if !r.SplitSupported() || !c.CIFailureAtHead.Reported || c.CIFailureAtHead.N < r.MinBase() || !c.CICoverage.Reported || c.CICoverage.Rate < minCICardCoverage {
+ return "", false
+ }
+ pass := int(math.Round(100 * (1 - c.CIFailureAtHead.Rate)))
+ return numberCard(th,
+ fmt.Sprintf("%d in 100 reviewed pull requests merged with every check passing", pass),
+ "Accuracy · checks at merge",
+ fmt.Sprintf("%d in 100", pass),
+ "reviewed pull requests merged with every check passing.",
+ fmt.Sprintf("from %s reviewed merges with a recorded check result, %s", commas(c.CIFailureAtHead.N), r.windowLabel()),
+ ), true
+}
+
+func (r Report) promptInjectionCard(th cardTheme) (string, bool) {
+ rt := r.RedTeam
+ if rt == nil || rt.Attacks <= 0 {
+ return "", false
+ }
+ pct := int(math.Round(100 * float64(rt.Caught) / float64(rt.Attacks)))
+ return numberCard(th,
+ fmt.Sprintf("%d%% of prompt-injection attempts were caught before reaching an agent, with %d false alarm in %d harmless requests", pct, rt.FalseAlarms, rt.Benign),
+ "Security · prompt injection",
+ fmt.Sprintf("%d%%", pct),
+ fmt.Sprintf("of prompt-injection attempts were caught; %d false alarm in %d harmless requests.", rt.FalseAlarms, rt.Benign),
+ fmt.Sprintf("Hive's own test set: %d attacks, %d harmless texts, %s. Not production traffic.", rt.Attacks, rt.Benign, monthLabel(rt.RunAt)),
+ ), true
+}
+
+func (r Report) costCard(th cardTheme) (string, bool) {
+ per, ok := r.Spend.PerMergedPR()
+ if !ok || r.Spend.MergedPRs < r.MinBase() {
+ return "", false
+ }
+ big := fmt.Sprintf("about $%.0f", per)
+ if per < 1 {
+ big = fmt.Sprintf("about %d¢", int(math.Round(per*100)))
+ }
+ return numberCard(th,
+ big+" of agent spend per merged pull request",
+ "Cost · per merged pull request",
+ big,
+ "of agent spend per merged pull request, estimated from provider pricing.",
+ fmt.Sprintf("from %s merged pull requests, %s to %s", commas(r.Spend.MergedPRs), monthLabel(r.Spend.WindowStart), monthLabel(r.Spend.WindowEnd)),
+ ), true
+}
+
+// storiesCard shows up to two real PRs side by side: what was caught, how
+// long the fix took, when it merged.
+func (r Report) storiesCard(th cardTheme) (string, bool) {
+ if len(r.Stories) == 0 {
+ return "", false
+ }
+ stories := r.Stories
+ if len(stories) > 2 {
+ stories = stories[:2]
+ }
+ colW := (cardWideW - 3*cardPad) / 2
+ // Lay out each column first so the card height follows the longest one.
+ type col struct {
+ s Story
+ lines []string
+ }
+ cols := make([]col, 0, len(stories))
+ maxLines := 0
+ for _, s := range stories {
+ l := wrap(s.Caught, wideChars)
+ if len(l) > maxLines {
+ maxLines = len(l)
+ }
+ cols = append(cols, col{s, l})
+ }
+ const stepGap = 30
+ h := 84 + 16 + maxLines*(claimLineH-2) + 2*stepGap + 10 + cardPad + 28
+ var b strings.Builder
+ cardOpen(&b, th, cardWideW, h, "Two real pull requests where the reviewer caught a problem before merge")
+ eyebrow(&b, th, cardPad, 32, "Accuracy · two real pull requests")
+ for i, c := range cols {
+ s := c.s
+ x := cardPad + i*(colW+cardPad)
+ fmt.Fprintf(&b, `%s #%d`+"\n", esc(s.URL()), x, 62, th.Brand, claimPx+1, esc(s.Repo), s.Number)
+ y := 84
+ step := func(label string, lines []string, fill string) {
+ fmt.Fprintf(&b, ``+"\n", x+4, y-4, fill)
+ fmt.Fprintf(&b, `%s`+"\n", x+16, y, th.Muted, footerPx, esc(strings.ToUpper(label)))
+ y = textLines(&b, th, x+16, y+16, claimPx-1, claimLineH-2, th.Text, lines) + 10
+ }
+ step("Caught", c.lines, th.Brand)
+ y = 84 + 16 + maxLines*(claimLineH-2) + 10
+ step("Fixed", []string{fmt.Sprintf("The author fixed it; the new version was approved %d minutes later.", s.FixMinutes())}, th.OK)
+ step("Merged", []string{fmt.Sprintf("Merged %d minutes after the pull request was opened.", s.MergeMinutes())}, th.OK)
+ }
+ footer(&b, th, cardPad, h-cardPad+4, 120, "Times from the review records Hive keeps. Click a title to open the pull request.")
+ b.WriteString("\n")
+ return b.String(), true
+}
+
+// tile is one cell of the "also" strip.
+type tile struct{ Head, Big, Text string }
+
+// alsoTiles are the four secondary properties. Tiles with a number take it
+// from the report; the others are plain statements of how Hive works.
+func (r Report) alsoTiles() []tile {
+ tiles := []tile{
+ {"Consistency", commas(r.Verdicts), "review verdicts, each against the same checklist; one house style file for every task."},
+ {"Readability", "", "Every verdict is written as what this is, the problem, the fix, and what happens if it is skipped."},
+ {"Efficiency", "", "Agents run only when there is work; capacity is shared fairly across repositories."},
+ }
+ m, u := r.Mediated.TimeToMerge, r.Unmediated.TimeToMerge
+ if r.SplitSupported() && m.Reported && u.Reported && m.N >= r.MinBase() && u.N >= r.MinBase() {
+ d := minutes(m.Value) - minutes(u.Value)
+ text := fmt.Sprintf("minutes from open to merge for a typical reviewed pull request; review adds about %d.", d)
+ if d <= 0 {
+ text = "minutes from open to merge for a typical reviewed pull request."
+ }
+ tiles = append(tiles, tile{"Speed", fmt.Sprint(minutes(m.Value)), text})
+ } else {
+ tiles = append(tiles, tile{"Speed", "", "Time to merge is reported once enough reviewed pull requests are on record."})
+ }
+ return tiles
+}
+
+func (r Report) alsoCard(th cardTheme) (string, bool) {
+ if r.Verdicts < r.MinBase() {
+ return "", false
+ }
+ const h = 190
+ tiles := r.alsoTiles()
+ var b strings.Builder
+ cardOpen(&b, th, cardWideW, h, "Also: consistency, readability, efficiency, speed")
+ eyebrow(&b, th, cardPad, 32, "Also · "+r.windowLabel())
+ tileW := (cardWideW - 2*cardPad) / len(tiles)
+ for i, t := range tiles {
+ x := cardPad + i*tileW
+ if i > 0 {
+ fmt.Fprintf(&b, ``+"\n", x-10, x-10, h-cardPad-8, th.Border)
+ }
+ fmt.Fprintf(&b, `%s`+"\n", x, 62, th.Text, claimPx, esc(t.Head))
+ y := 84
+ if t.Big != "" {
+ fmt.Fprintf(&b, `%s`+"\n", x, 94, th.Brand, esc(t.Big))
+ y = 114
+ }
+ textLines(&b, th, x, y, footerPx+2, 15, th.Muted, wrap(t.Text, tileChars))
+ }
+ b.WriteString("\n")
+ return b.String(), true
+}
+
+// rung is one step of the Hive loop for the how-it-works card.
+type rung struct{ Step, What, Outcome string }
+
+// hiveLoop is the loop every task goes through, in order.
+var hiveLoop = []rung{
+ {"Scan", "Hive reads your open issues, pull requests and failing checks.", "Work comes from your backlog"},
+ {"Claim", "One agent claims one task, so nothing is done twice.", "No duplicate work"},
+ {"Work", "A fresh, short-lived agent does the job and then stops.", "Nothing runs for hours unattended"},
+ {"Verify", "CI runs, a separate reviewer checks the change, and planted instructions are screened out.", "Problems are found before merge"},
+ {"Merge", "Trust level decides: hold for a human, or merge once every check passes.", "Humans keep the decisions"},
+ {"Credit", "The person who opened the issue is credited on the merge.", "The asker is credited"},
+}
+
+func howItWorksCard(th cardTheme) string {
+ const baseRowH, lineH = 44, 14
+ rows := make([][]string, len(hiveLoop))
+ h := 72
+ for i, r := range hiveLoop {
+ rows[i] = wrap(r.What, ladderChars)
+ h += baseRowH + (len(rows[i])-1)*lineH
+ }
+ var b strings.Builder
+ cardOpen(&b, th, cardWideW, h, "How Hive works: scan, claim, work, verify, merge, credit")
+ eyebrow(&b, th, cardPad, 32, "How it works · every task")
+ y := 64
+ for i, r := range hiveLoop {
+ rowH := baseRowH + (len(rows[i])-1)*lineH
+ fmt.Fprintf(&b, `%d`+"\n", cardPad+12, y-4, th.Brand, cardPad+12, y, th.BG, i+1)
+ fmt.Fprintf(&b, `%s`+"\n", cardPad+36, y, th.Text, claimPx+1, esc(r.Step))
+ textLines(&b, th, cardPad+36, y+17, footerPx+2, lineH, th.Muted, rows[i])
+ fmt.Fprintf(&b, `%s`+"\n", cardWideW-cardPad, y, th.OK, footerPx+2, esc(r.Outcome))
+ if i < len(hiveLoop)-1 {
+ fmt.Fprintf(&b, ``+"\n", cardPad+12, y+10, cardPad+12, y+rowH-18, th.Border)
+ }
+ y += rowH
+ }
+ b.WriteString("\n")
+ return b.String()
+}
diff --git a/src/pkg/outcomebench/cards_test.go b/src/pkg/outcomebench/cards_test.go
new file mode 100644
index 0000000000..f141d7c2d4
--- /dev/null
+++ b/src/pkg/outcomebench/cards_test.go
@@ -0,0 +1,114 @@
+package outcomebench
+
+import (
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+)
+
+func TestCardsRespectMinimumBase(t *testing.T) {
+ since, until, _ := ParseWindow("2026-10")
+ opts := Options{Since: since, Until: until, Now: now, Spend: &Spend{USD: 26, MergedPRs: 13}}
+ // The fixture has 13 reviewed merges: below the default README base, so
+ // no production-count card is drawn, but the ladder always is.
+ rep := Compute(fixture(), 0, opts)
+ if rep.MinBase() != DefaultReadmeMinBase {
+ t.Fatalf("MinBase = %d", rep.MinBase())
+ }
+ for _, c := range []string{CardSentBack, CardChecksGreen, CardCost, CardAlso, CardStories, CardPromptInjection} {
+ if rep.HasCard(c) {
+ t.Errorf("%s drawn with base below %d", c, DefaultReadmeMinBase)
+ }
+ }
+ if !rep.HasCard(CardHowItWorks) {
+ t.Error("how-it-works card is data-free and must always render")
+ }
+ // Lowering the threshold draws the count cards; cost is never below base
+ // because MergedPRs is its own base.
+ opts.ReadmeMinBase = 10
+ rep = Compute(fixture(), 0, opts)
+ for _, c := range []string{CardSentBack, CardCost, CardAlso} {
+ if !rep.HasCard(c) {
+ t.Errorf("%s not drawn at min base 10", c)
+ }
+ }
+ // The fixture records a check for every reviewed merge, so checks-green
+ // is drawn; dropping coverage below the floor must suppress it.
+ if !rep.HasCard(CardChecksGreen) {
+ t.Error("checks-green not drawn with full CI coverage")
+ }
+ low := rep
+ low.Mediated.CICoverage.Rate = minCICardCoverage - 0.01
+ if low.HasCard(CardChecksGreen) {
+ t.Error("checks-green drawn with CI coverage below the floor")
+ }
+ svg, _ := rep.Card(CardSentBack, cardThemes[0])
+ for _, want := range []string{"", ` 15 && !strings.Contains(l, " ") == false {
+ t.Errorf("line too long: %q", l)
+ }
+ }
+ if strings.Join(lines, " ") != "the quick brown fox jumps over the lazy dog" {
+ t.Errorf("wrap lost words: %v", lines)
+ }
+}
+
+func TestLoadStoriesAndRedTeam(t *testing.T) {
+ dir := t.TempDir()
+ write := func(name, body string) string {
+ p := filepath.Join(dir, name)
+ if err := os.WriteFile(p, []byte(body), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ return p
+ }
+ good := write("stories.json", `[{"repo":"hivecommons/hive","number":1,"caught":"x","opened_at":"2026-10-01T00:00:00Z","caught_at":"2026-10-01T00:05:00Z","fix_approved_at":"2026-10-01T00:09:00Z","merged_at":"2026-10-01T00:16:00Z"}]`)
+ stories, err := LoadStories(good)
+ if err != nil || len(stories) != 1 || stories[0].FixMinutes() != 4 || stories[0].MergeMinutes() != 16 {
+ t.Fatalf("LoadStories: %v %+v", err, stories)
+ }
+ if _, err := LoadStories(write("bad-order.json", `[{"repo":"a/b","number":1,"caught":"x","opened_at":"2026-10-01T00:10:00Z","caught_at":"2026-10-01T00:05:00Z","fix_approved_at":"2026-10-01T00:09:00Z","merged_at":"2026-10-01T00:16:00Z"}]`)); err == nil {
+ t.Error("out-of-order timeline accepted")
+ }
+ if _, err := LoadStories(write("bad-fields.json", `[{"repo":"a/b"}]`)); err == nil {
+ t.Error("story without number/caught accepted")
+ }
+ rt, err := LoadRedTeam(write("rt.json", `{"attacks":63,"caught":61,"benign":12,"false_alarms":1,"run_at":"2026-10-23T00:00:00Z"}`))
+ if err != nil || rt.Attacks != 63 || rt.RunAt.IsZero() {
+ t.Fatalf("LoadRedTeam: %v %+v", err, rt)
+ }
+ if _, err := LoadRedTeam(write("rt-bad.json", `{"attacks":10,"caught":11,"benign":12,"false_alarms":1}`)); err == nil {
+ t.Error("caught > attacks accepted")
+ }
+ rep := Report{RedTeam: rt, GeneratedAt: now}
+ svg, ok := rep.Card(CardPromptInjection, cardThemes[1])
+ if !ok || !strings.Contains(svg, ">97%<") || !strings.Contains(svg, "63 attacks, 12 harmless") || !strings.Contains(svg, "texts, Oct 2026") || !strings.Contains(svg, "Not production traffic") {
+ t.Fatalf("prompt-injection card:\n%s", svg)
+ }
+}
diff --git a/src/pkg/outcomebench/drift_test.go b/src/pkg/outcomebench/drift_test.go
index bfaea10263..ed90d45ceb 100644
--- a/src/pkg/outcomebench/drift_test.go
+++ b/src/pkg/outcomebench/drift_test.go
@@ -43,6 +43,10 @@ func TestCommittedReportMatchesReadme(t *testing.T) {
if got != want {
t.Errorf("README outcomes block drifted from %s; regenerate with src/scripts/outcome-benchmark.sh --readme\n--- README\n%s\n--- regenerated\n%s", latest, got, want)
}
+ // The ladder card sits outside the generated block but is still generated.
+ if !strings.Contains(string(readme), rep.HowItWorksPicture("src/docs/benchmarks")) {
+ t.Errorf("README does not embed the how-it-works card: expected\n%s", rep.HowItWorksPicture("src/docs/benchmarks"))
+ }
base := strings.TrimSuffix(latest, ".json")
check := func(path, want string) {
t.Helper()
@@ -57,4 +61,14 @@ func TestCommittedReportMatchesReadme(t *testing.T) {
check(base+".md", rep.Markdown())
check(filepath.Join(docDir, "assets", rep.AssetBase()+"-bars.svg"), rep.BarChartSVG())
check(filepath.Join(docDir, "assets", rep.AssetBase()+"-trend.svg"), rep.TrendSVG())
+ for file, svg := range rep.Cards() {
+ check(filepath.Join(docDir, "assets", file), svg)
+ }
+ // A card the generator no longer emits must not linger as a stale asset.
+ stale, _ := filepath.Glob(filepath.Join(docDir, "assets", rep.AssetBase()+"-card-*.svg"))
+ for _, p := range stale {
+ if _, ok := rep.Cards()[filepath.Base(p)]; !ok {
+ t.Errorf("%s is not emitted by the generator any more; delete it", p)
+ }
+ }
}
diff --git a/src/pkg/outcomebench/outcomebench_test.go b/src/pkg/outcomebench/outcomebench_test.go
index 1ef4ead385..3b517e9e08 100644
--- a/src/pkg/outcomebench/outcomebench_test.go
+++ b/src/pkg/outcomebench/outcomebench_test.go
@@ -166,8 +166,8 @@ func TestComputeRefusesUnsupportedSplit(t *testing.T) {
t.Fatalf("findings: %v", rep.Findings())
}
block := rep.ReadmeBlock("src/docs/benchmarks")
- if !strings.Contains(block, "does not support a mediated-vs-unmediated comparison") || strings.Contains(block, "Revert rate") {
- t.Fatalf("readme block must not claim a comparison:\n%s", block)
+ if !strings.Contains(block, "Not enough reviewed pull requests are on record yet") || strings.Contains(block, "card-sent-back") || strings.Contains(block, "card-also") {
+ t.Fatalf("readme block must not claim a comparison or draw a card below the minimum base:\n%s", block)
}
for _, r := range []Rate{rep.Mediated.RevertRate, rep.Unmediated.TimeToMergeRate()} {
if r.Reported {
@@ -212,14 +212,22 @@ func TestSVGsAreWellFormedAndHonest(t *testing.T) {
func TestReadmeBlockRoundTrip(t *testing.T) {
since, until, _ := ParseWindow("2026-10")
- rep := Compute(fixture(), 0, Options{Since: since, Until: until, Now: now, Spend: &Spend{USD: 26, MergedPRs: 13}})
+ rep := Compute(fixture(), 0, Options{Since: since, Until: until, Now: now, Spend: &Spend{USD: 26, MergedPRs: 13}, ReadmeMinBase: 1,
+ Stories: []Story{{Repo: "hivecommons/hive", Number: 1, Caught: "A test was skipped.", OpenedAt: now.Add(-20 * time.Minute), CaughtAt: now.Add(-15 * time.Minute), FixApprovedAt: now.Add(-10 * time.Minute), MergedAt: now}},
+ RedTeam: &RedTeam{Attacks: 10, Caught: 9, Benign: 5, FalseAlarms: 1, RunAt: now}})
block := rep.ReadmeBlock("src/docs/benchmarks")
- for _, want := range []string{ReadmeStart, ReadmeEnd, "**Revert rate", "**Median time-to-merge", "flagged for changes", "≈$2.00", "36 pull requests across 2 repositories", "src/docs/benchmarks/outcomes-2026-10.md", "assets/outcomes-2026-10-bars.svg", "assets/outcomes-2026-10-trend.svg"} {
+ for _, want := range []string{ReadmeStart, ReadmeEnd, "## Accuracy", "## Security", "## Cost", "## Also", "36 pull requests across 2 repositories", "src/docs/benchmarks/outcomes-2026-10.md",
+ "card-sent-back-dark.svg", "card-sent-back-light.svg", "card-stories-dark.svg", "card-prompt-injection-light.svg", "card-cost-dark.svg", "card-also-light.svg", "prefers-color-scheme: dark", "how we count"} {
if !strings.Contains(block, want) {
t.Errorf("block missing %q:\n%s", want, block)
}
}
- if n := strings.Count(block, "\n"); n > 14 {
+ for _, banned := range []string{"Wilson", "interval", "(n=", " n=", "median", "mediated", "95%", "≈"} {
+ if strings.Contains(block, banned) {
+ t.Errorf("block contains %q; statistical vocabulary belongs in the report", banned)
+ }
+ }
+ if n := strings.Count(block, "\n"); n > 60 {
t.Fatalf("block is %d lines; README additions must stay small", n+1)
}
readme := "# Hive\n\nintro\n\n" + ReadmeStart + "\nstale\n" + ReadmeEnd + "\n\n## Next\n"
diff --git a/src/pkg/outcomebench/readme.go b/src/pkg/outcomebench/readme.go
index d9e523e43c..4f47cd987d 100644
--- a/src/pkg/outcomebench/readme.go
+++ b/src/pkg/outcomebench/readme.go
@@ -15,48 +15,92 @@ const (
// ErrNoMarkers is returned when a README has no outcomes block to rewrite.
var ErrNoMarkers = errors.New("README has no " + ReadmeStart + " … " + ReadmeEnd + " block")
-// ReadmeBlock renders the README "Outcomes" section: 3–4 bold headline
-// numbers, each with n and date and a one-line "how measured", plus the two
-// generated charts. Only reported figures appear; docDir is the path from
-// the README to the benchmark directory.
+// ReadmeBlock renders the README evidence section for a general reader:
+// one line of scale, then Accuracy / Security / Cost / Also, each a card
+// (a with a dark and a light SVG) and at most two flat sentences,
+// one of which states the limit of the number. No statistical vocabulary
+// appears here; it lives in the report. docDir is the path from the README
+// to the benchmark directory.
func (r Report) ReadmeBlock(docDir string) string {
report := fmt.Sprintf("%s/outcomes-%s.md", docDir, r.Month())
var lines []string
- add := func(s string) { lines = append(lines, s) }
- add(ReadmeStart)
- add(fmt.Sprintf("", report))
- m, u := r.Mediated, r.Unmediated
- asOf := day(r.GeneratedAt)
- if r.SplitSupported() {
- if m.ChangesRequested.Reported {
- add(fmt.Sprintf("- **%.0f%% of reviewed PRs were flagged for changes before they merged** — a `changes_requested` verdict on an earlier head of %d of %d mediated merges [%.1f–%.1f], %s.",
- 100*m.ChangesRequested.Rate, m.ChangesRequested.K, m.ChangesRequested.N, 100*m.ChangesRequested.Lo, 100*m.ChangesRequested.Hi, asOf))
+ add := func(s ...string) { lines = append(lines, s...) }
+ cards := func(width int, names ...string) {
+ var shown []string
+ for _, n := range names {
+ if r.HasCard(n) {
+ shown = append(shown, r.picture(docDir, n, width))
+ }
}
- if m.RevertRate.Reported && u.RevertRate.Reported {
- add(fmt.Sprintf("- **Revert rate %.1f%% mediated vs %.1f%% unmediated** — %d/%d vs %d/%d merges matured %d days, Wilson [%.1f–%.1f] vs [%.1f–%.1f], %s.",
- 100*m.RevertRate.Rate, 100*u.RevertRate.Rate, m.RevertRate.K, m.RevertRate.N, u.RevertRate.K, u.RevertRate.N, int(Maturation.Hours()/24),
- 100*m.RevertRate.Lo, 100*m.RevertRate.Hi, 100*u.RevertRate.Lo, 100*u.RevertRate.Hi, asOf))
+ if len(shown) > 0 {
+ add(`
`)
+ add(shown...)
+ add("
", "")
}
- if m.TimeToMerge.Reported && u.TimeToMerge.Reported {
- add(fmt.Sprintf("- **Median time-to-merge %.0f min mediated vs %.0f min unmediated** — review costs about %.0f minutes on the median PR (n=%d vs %d), %s.",
- 60*m.TimeToMerge.Value, 60*u.TimeToMerge.Value, 60*(m.TimeToMerge.Value-u.TimeToMerge.Value), m.TimeToMerge.N, u.TimeToMerge.N, asOf))
+ }
+ add(ReadmeStart)
+ add(fmt.Sprintf("", report))
+ add(r.HeroLine(), "")
+
+ add("## Accuracy", "")
+ if r.HasCard(CardSentBack) || r.HasCard(CardStories) {
+ add("A separate reviewer reads every pull request an agent opens and can send it back before it merges.", "")
+ cards(cardW, CardSentBack, CardChecksGreen)
+ cards(cardWideW, CardStories)
+ if r.Mediated.RevertRate.Reported && r.Unmediated.RevertRate.Reported {
+ add("Pull requests merged without review come from other repositories and periods, not a controlled comparison, so differences may not be caused by review. Revert rates are not distinguishable so far.", "")
}
} else {
- add(fmt.Sprintf("- The current evidence does not support a mediated-vs-unmediated comparison (mediated n=%d, unmediated n=%d); only scale facts are shown.", m.Merged, u.Merged))
+ add("Not enough reviewed pull requests are on record yet to report accuracy figures.", "")
}
- if per, ok := r.Spend.PerMergedPR(); ok {
- add(fmt.Sprintf("- **≈$%.2f estimated agent spend per merged PR** — `/api/cost` estimate over %d hive-attributed merges, %s → %s.", per, r.Spend.MergedPRs, day(r.Spend.WindowStart), day(r.Spend.WindowEnd)))
+
+ add("## Security", "")
+ add("Agents work with short-lived, task-scoped access; pull requests from untrusted authors are held for a human; every merge is signed off and credits the issue author.", "")
+ cards(cardW, CardPromptInjection)
+
+ add("## Cost", "")
+ if r.HasCard(CardCost) {
+ add("Each agent card in the dashboard shows what the current task has cost so far. The figure below is an estimate from provider pricing; some models are unpriced.", "")
+ cards(cardW, CardCost)
+ } else {
+ add("Cost per merged pull request is reported once enough hive-attributed merges are on record.", "")
}
- add(fmt.Sprintf("- **%d pull requests across %d repositories, %d review verdicts recorded** — the evidence this is measured from, %s.", r.PRs, len(r.Repos), r.Verdicts, asOf))
- add("")
- add(fmt.Sprintf("How measured: per-PR evidence bundles the spoke already writes, folded per PR; 95%% Wilson intervals; n < %d is never reported. Full tables, trend and methodology in [%s](%s).", MinN, report, report))
- add("")
- add(fmt.Sprintf("", docDir, r.AssetBase()))
- add(fmt.Sprintf("", docDir, r.AssetBase()))
+
+ add("## Also", "")
+ cards(cardWideW, CardAlso)
+
+ add(fmt.Sprintf("Production data, not a lab run. Every count above is regenerated from live records by `src/scripts/outcome-benchmark.sh`; bases, dates, exclusions and the full tables are in [how we count →](%s).", report))
add(ReadmeEnd)
return strings.Join(lines, "\n")
}
+// HeroLine is the one sentence of scale at the top of the README.
+func (r Report) HeroLine() string {
+ s := fmt.Sprintf("Measured on %s pull requests across %d repositories", commas(r.PRs), len(r.Repos))
+ if r.Verdicts > 0 {
+ s += fmt.Sprintf(" and %s review verdicts", commas(r.Verdicts))
+ }
+ return s + fmt.Sprintf(", as of %s.", day(r.GeneratedAt))
+}
+
+// picture is the GitHub-flavoured that swaps the card SVG with
+// the reader's colour scheme; the alt text is the card's title sentence.
+func (r Report) picture(docDir, card string, width int) string {
+ alt := card
+ if svg, ok := r.Card(card, cardThemes[0]); ok {
+ if i, j := strings.Index(svg, ""), strings.Index(svg, ""); i >= 0 && j > i {
+ alt = svg[i+len("") : j]
+ }
+ }
+ return fmt.Sprintf(``,
+ docDir, r.CardFile(card, "dark"), alt, width, docDir, r.CardFile(card, "light"))
+}
+
+// HowItWorksPicture is the ladder card for the README's own section.
+func (r Report) HowItWorksPicture(docDir string) string {
+ return r.picture(docDir, CardHowItWorks, cardWideW)
+}
+
// UpdateReadme replaces the marker block in readme with block. The block is
// expected to start with ReadmeStart and end with ReadmeEnd.
func UpdateReadme(readme, block string) (string, error) {
diff --git a/src/pkg/outcomebench/readme_budget_test.go b/src/pkg/outcomebench/readme_budget_test.go
new file mode 100644
index 0000000000..ae821fd8b6
--- /dev/null
+++ b/src/pkg/outcomebench/readme_budget_test.go
@@ -0,0 +1,125 @@
+package outcomebench
+
+import (
+ "os"
+ "path/filepath"
+ "regexp"
+ "strings"
+ "testing"
+ "unicode"
+)
+
+// The README is deliberately short: the cards carry the evidence and the
+// prose is one or two sentences per section. These ratchets fail when prose
+// grows past the budget, a section is added, or statistical or marketing
+// vocabulary creeps back in. Lower the baselines when the README shrinks;
+// do not raise them without the maintainers' agreement.
+const (
+ readmeMaxProseWords = 600
+ readmeMaxSections = 10
+)
+
+var (
+ reHTMLComment = regexp.MustCompile(`(?s)`)
+ reFenced = regexp.MustCompile("(?s)```.*?```")
+ reHTMLTag = regexp.MustCompile(`(?s)<[^>]+>`)
+ reImage = regexp.MustCompile(`!\[[^\]]*\]\([^)]*\)`)
+ reLinkTarget = regexp.MustCompile(`\]\([^)]*\)`)
+ reAutoLink = regexp.MustCompile(`?`)
+ reInlineCode = regexp.MustCompile("`[^`]*`")
+)
+
+// readmeProse strips everything that is not read as prose: HTML (the
+// cards), comments, fenced code, badges and images, link URLs and
+// inline code. Link text stays, because the reader reads it.
+func readmeProse(md string) string {
+ md = reHTMLComment.ReplaceAllString(md, " ")
+ md = reFenced.ReplaceAllString(md, " ")
+ md = reImage.ReplaceAllString(md, " ")
+ md = reHTMLTag.ReplaceAllString(md, " ")
+ md = reLinkTarget.ReplaceAllString(md, "]")
+ md = reAutoLink.ReplaceAllString(md, " ")
+ md = reInlineCode.ReplaceAllString(md, " ")
+ return md
+}
+
+// countWords counts whitespace-separated tokens that contain a letter or
+// digit, so punctuation-only tokens such as "·" or "---" are not words.
+func countWords(s string) int {
+ n := 0
+ for _, tok := range strings.Fields(s) {
+ if strings.IndexFunc(tok, func(r rune) bool { return unicode.IsLetter(r) || unicode.IsDigit(r) }) >= 0 {
+ n++
+ }
+ }
+ return n
+}
+
+func readRepoReadme(t *testing.T) string {
+ t.Helper()
+ b, err := os.ReadFile(filepath.Join("..", "..", "..", "README.md"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ return string(b)
+}
+
+func TestReadmeProseBudget(t *testing.T) {
+ readme := readRepoReadme(t)
+ words := countWords(readmeProse(readme))
+ t.Logf("README prose: %d words (budget %d)", words, readmeMaxProseWords)
+ if words > readmeMaxProseWords {
+ t.Errorf("README prose is %d words; budget is %d. Move detail into src/docs/ (what-is-hive.md or the benchmarks doc) instead of the README.", words, readmeMaxProseWords)
+ }
+ sections := 0
+ for _, line := range strings.Split(readme, "\n") {
+ if strings.HasPrefix(line, "## ") {
+ sections++
+ }
+ }
+ if sections > readmeMaxSections {
+ t.Errorf("README has %d H2 sections; budget is %d", sections, readmeMaxSections)
+ }
+}
+
+// Vocabulary that belongs in the benchmarks report, not the README, plus the
+// persuasive words the README copy avoids. Matched case-insensitively on
+// word boundaries against the prose and against the generated card text.
+var readmeForbiddenWords = []string{
+ // statistical
+ "wilson", "interval", "intervals", "n=", "median", "cohort", "cohorts",
+ "mediated", "unmediated", "hatched", "95%", "confidence", "p-value", "whisker",
+ // marketing
+ "powerful", "seamless", "seamlessly", "blazing", "revolutionary", "best-in-class",
+ "effortless", "effortlessly", "supercharge", "game-changing", "cutting-edge",
+ "world-class", "unleash", "amazing", "incredible", "never sleeps",
+}
+
+func TestReadmeVocabulary(t *testing.T) {
+ readme := readRepoReadme(t)
+ texts := map[string]string{"README prose": strings.ToLower(readmeProse(readme))}
+ docDir := filepath.Join("..", "..", "docs", "benchmarks")
+ svgs, _ := filepath.Glob(filepath.Join(docDir, "assets", "outcomes-*-card-*.svg"))
+ for _, p := range svgs {
+ b, err := os.ReadFile(p)
+ if err != nil {
+ t.Fatal(err)
+ }
+ texts[filepath.Base(p)] = strings.ToLower(reHTMLTag.ReplaceAllString(string(b), " "))
+ }
+ for name, text := range texts {
+ for _, w := range readmeForbiddenWords {
+ re := regexp.MustCompile(`(^|[^a-z0-9-])` + regexp.QuoteMeta(w) + `($|[^a-z0-9-])`)
+ if re.MatchString(text) {
+ t.Errorf("%s contains %q; keep statistical and marketing vocabulary out of the README and its cards", name, w)
+ }
+ }
+ if strings.Contains(text, "!") && name == "README prose" {
+ for _, line := range strings.Split(text, "\n") {
+ if strings.Contains(line, "!") {
+ t.Errorf("README prose has an exclamation mark: %q", strings.TrimSpace(line))
+ }
+ }
+ }
+ }
+}
diff --git a/src/pkg/outcomebench/render.go b/src/pkg/outcomebench/render.go
index a4fb62f604..a1eda62b03 100644
--- a/src/pkg/outcomebench/render.go
+++ b/src/pkg/outcomebench/render.go
@@ -97,6 +97,17 @@ func (r Report) Markdown() string {
w("| %s | %d | %s | %s | %d | %s | %s |", m, a.Merged, pct(a.RevertRate), hours(a.TimeToMerge), u.Merged, pct(u.RevertRate), hours(u.TimeToMerge))
}
w("")
+ w("## Real examples")
+ w("")
+ if len(r.Stories) == 0 {
+ w("No curated examples were supplied (`--stories`).")
+ }
+ for _, s := range r.Stories {
+ w("- [%s#%d](%s): the [posted review](%s) caught: %s The fixed head was approved %d minutes later; the PR merged %d minutes after opening (opened %s, caught %s, approved %s, merged %s UTC).",
+ s.Repo, s.Number, s.URL(), s.ReviewURL, strings.TrimSpace(s.Caught), s.FixMinutes(), s.MergeMinutes(),
+ s.OpenedAt.UTC().Format("15:04"), s.CaughtAt.UTC().Format("15:04"), s.FixApprovedAt.UTC().Format("15:04"), s.MergedAt.UTC().Format("2006-01-02 15:04"))
+ }
+ w("")
w("## Scale")
w("")
w("- %d evidence bundles (%d unreadable) folding to %d pull requests across %d repositories; %d verdicts recorded.", r.Bundles, r.Unreadable, r.PRs, len(r.Repos), r.Verdicts)
@@ -111,8 +122,22 @@ func (r Report) Markdown() string {
w("- Hub `GET /api/hub/evidence` was not consulted (it requires an admin session); numbers are this spoke's evidence only.")
}
w("")
+ w("## What the README shows and leaves out")
+ w("")
+ w("The README carries only plain-language cards; this section says which numbers made it and why the rest did not.")
+ w("")
+ for _, line := range r.readmeDecisions() {
+ w("- %s", line)
+ }
+ w("- **Not a controlled comparison.** Pull requests merged without a Hive review come from other repositories and periods; differences between the two groups may not be caused by review.")
+ w("- **Pull requests sent back.** Counts a `changes_requested` verdict on any head before merge. Whether the next head fixed the finding, or a human overrode it, is not verified here.")
+ w("- **Prompt injection.** The catch rate is from Hive's own evaluation set, not production traffic, and is not subject to the README minimum base because it is a self-run test and labelled as such on the card.")
+ w("- **Post-merge CI pass rate** is drawn only when at least %d%% of reviewed merges carry a recorded check result; a roadmap item is to record check status for every merged head so it can be reported.", int(100*minCICardCoverage))
+ w("")
w("## Methodology")
w("")
+ w("How measured: per-PR evidence bundles the spoke already writes, folded per PR; 95%% Wilson intervals; n < %d is never reported. The README shows only cards whose base is at least %d (`--readme-min-base`), in plain words and without intervals; everything below that threshold, and every interval, is in this report.", MinN, r.MinBase())
+ w("")
w("- **Source.** Only the per-PR evidence bundles the spoke already writes under `/data/evidence` (schema v1), folded per PR with `pkg/evidence.FoldPRs`. No new telemetry.")
w("- **Cohorts.** A PR is *mediated* if any head carries a Hive verdict or a posted Hive review; otherwise *unmediated*. Backfilled history is unmediated by construction, so the window must start no earlier than the first mediated merge or the comparison is confounded by time.")
w("- **Revert / change-failure.** Denominator is merged PRs observed at least %d days after merge. A revert is a later PR the evidence linker tagged `reverted_by`; change-failure adds `followed_up_by` fixes.", int(Maturation.Hours()/24))
@@ -204,3 +229,34 @@ func (r Report) Findings() []string {
}
return out
}
+
+// readmeDecisions lists, per README card, whether it was drawn and the base
+// it rests on, so the report states every exclusion the README is silent on.
+func (r Report) readmeDecisions() []string {
+ min := r.MinBase()
+ var out []string
+ cr := r.Mediated.ChangesRequested
+ if r.HasCard(CardSentBack) {
+ out = append(out, fmt.Sprintf("**Sent back before merge** is shown: %d of %d reviewed merged PRs (full base, no exclusions beyond the window).", cr.K, cr.N))
+ } else {
+ out = append(out, fmt.Sprintf("**Sent back before merge** is not shown: base %d is below the README minimum of %d, or the split is unsupported.", cr.N, min))
+ }
+ ci := r.Mediated.CIFailureAtHead
+ if r.HasCard(CardChecksGreen) {
+ out = append(out, fmt.Sprintf("**Checks passing at merge** is shown: %d of %d reviewed merges failed a recorded check.", ci.K, ci.N))
+ } else {
+ out = append(out, fmt.Sprintf("**Checks passing at merge** is not shown: a check result is recorded on %d of %d reviewed merges (%.0f%%), under the %d%% floor, so the pass rate would describe a biased subset.", r.Mediated.CICoverage.K, r.Mediated.CICoverage.N, 100*r.Mediated.CICoverage.Rate, int(100*minCICardCoverage)))
+ }
+ rv := r.Mediated.RevertRate
+ out = append(out, fmt.Sprintf("**Revert rate** is not shown: the reviewed base is %d merges observed %d+ days after merge, below the README minimum of %d, and the two groups are not distinguishable on this data.", rv.N, int(Maturation.Hours()/24), min))
+ if per, ok := r.Spend.PerMergedPR(); ok {
+ out = append(out, fmt.Sprintf("**Cost per merged PR** is shown as an estimate: $%.2f = $%.2f estimated spend / %d hive-attributed merges, %s → %s.", per, r.Spend.USD, r.Spend.MergedPRs, day(r.Spend.WindowStart), day(r.Spend.WindowEnd)))
+ } else {
+ out = append(out, "**Cost per merged PR** is not shown: no spend sample was supplied or its merged-PR count is zero.")
+ }
+ m, u := r.Mediated.TimeToMerge, r.Unmediated.TimeToMerge
+ if m.Reported && u.Reported {
+ out = append(out, fmt.Sprintf("**Time to merge** is shown in the Also strip as %d minutes (typical, i.e. median) for %d reviewed merges, against %d minutes for %d merges without review; the extra minutes are stated as the cost of review.", minutes(m.Value), m.N, minutes(u.Value), u.N))
+ }
+ return out
+}
From 9c42d41a890665b6881c1c833dc8d925181e4831 Mon Sep 17 00:00:00 2001
From: Andrew Anderson
Date: Tue, 27 Oct 2026 12:57:55 -0400
Subject: [PATCH 2/6] docs: README quick start writes src/.env with the
required tokens
The Compose quick start contract requires every install recipe to write
src/.env and set HIVE_DASHBOARD_TOKEN.
Signed-off-by: Andrew Anderson
---
README.md | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/README.md b/README.md
index b34a91ee32..7552d3b452 100644
--- a/README.md
+++ b/README.md
@@ -65,10 +65,11 @@ The trust level ([ACMM](src/docs/acmm-policy-matrix.md)) sets how far the loop r
```bash
git clone https://github.com/hivecommons/hive.git && cd hive
cp src/hive.yaml.example src/hive.yaml # set project.org, project.repos and dashboard.port 3002
+(umask 077; printf 'HIVE_GITHUB_TOKEN=ghp_...\nHIVE_DASHBOARD_TOKEN=%s\n' "$(openssl rand -hex 32)" > src/.env)
docker compose -f src/docker-compose.yaml up -d
```
-Tokens go in `src/.env`; the [Docker Compose quick start](src/docs/quickstart-docker-compose.md) lists them. Podman, Kubernetes and the hosted hub: [Run it](src/docs/what-is-hive.md#run-it).
+The [Docker Compose quick start](src/docs/quickstart-docker-compose.md) explains each line. Podman, Kubernetes and the hosted hub: [Run it](src/docs/what-is-hive.md#run-it).
## Learn more
From 43beb1876dde3340ff8cc1e45c08f59f21578190 Mon Sep 17 00:00:00 2001
From: Andrew Anderson
Date: Tue, 27 Oct 2026 12:58:26 -0400
Subject: [PATCH 3/6] docs: README quick start writes src/.env with the
required tokens
The Compose quick start contract requires every install recipe to write
src/.env and set HIVE_DASHBOARD_TOKEN.
Signed-off-by: Andrew Anderson
---
README.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/README.md b/README.md
index 7552d3b452..4a3c216b77 100644
--- a/README.md
+++ b/README.md
@@ -65,7 +65,7 @@ The trust level ([ACMM](src/docs/acmm-policy-matrix.md)) sets how far the loop r
```bash
git clone https://github.com/hivecommons/hive.git && cd hive
cp src/hive.yaml.example src/hive.yaml # set project.org, project.repos and dashboard.port 3002
-(umask 077; printf 'HIVE_GITHUB_TOKEN=ghp_...\nHIVE_DASHBOARD_TOKEN=%s\n' "$(openssl rand -hex 32)" > src/.env)
+printf 'HIVE_GITHUB_TOKEN=ghp_...\nHIVE_DASHBOARD_TOKEN=%s\n' "$(openssl rand -hex 32)" > src/.env && chmod 600 src/.env
docker compose -f src/docker-compose.yaml up -d
```
From 8a7a01d2f81e58bee15220dd3111e395f1ab2824 Mon Sep 17 00:00:00 2001
From: Andrew Anderson
Date: Tue, 27 Oct 2026 13:44:31 -0400
Subject: [PATCH 4/6] docs: rework README Accuracy card around what happened
after review
Replace the sent-back-before-merge card, which counted reviewer
objections rather than outcomes, with two follow-up measures computed
per head from the same evidence bundles: the share of sent-back pull
requests that came back fixed and were approved (369 of 563), and the
typical time from change request to approval of the fixed version
(24 minutes, 369 PRs). The report states why reverts, post-fix CI and
reopened issues are not drawn yet, and the methodology defines the new
measures.
Signed-off-by: Andrew Anderson
---
README.md | 7 +-
changelog.d/changed-readme-evidence-cards.md | 2 +-
...omes-2026-10-card-came-back-fixed-dark.svg | 11 ++
...mes-2026-10-card-came-back-fixed-light.svg | 11 ++
.../outcomes-2026-10-card-sent-back-dark.svg | 10 --
.../outcomes-2026-10-card-sent-back-light.svg | 10 --
...mes-2026-10-card-time-to-approval-dark.svg | 11 ++
...es-2026-10-card-time-to-approval-light.svg | 11 ++
src/docs/benchmarks/outcomes-2026-10.json | 30 +++++
src/docs/benchmarks/outcomes-2026-10.md | 9 +-
src/pkg/outcomebench/bench.go | 114 ++++++++++++++++++
src/pkg/outcomebench/cards.go | 59 +++++++--
src/pkg/outcomebench/cards_test.go | 18 ++-
src/pkg/outcomebench/outcomebench_test.go | 23 +++-
src/pkg/outcomebench/readme.go | 28 ++++-
src/pkg/outcomebench/render.go | 26 +++-
16 files changed, 321 insertions(+), 59 deletions(-)
create mode 100644 src/docs/benchmarks/assets/outcomes-2026-10-card-came-back-fixed-dark.svg
create mode 100644 src/docs/benchmarks/assets/outcomes-2026-10-card-came-back-fixed-light.svg
delete mode 100644 src/docs/benchmarks/assets/outcomes-2026-10-card-sent-back-dark.svg
delete mode 100644 src/docs/benchmarks/assets/outcomes-2026-10-card-sent-back-light.svg
create mode 100644 src/docs/benchmarks/assets/outcomes-2026-10-card-time-to-approval-dark.svg
create mode 100644 src/docs/benchmarks/assets/outcomes-2026-10-card-time-to-approval-light.svg
diff --git a/README.md b/README.md
index 4a3c216b77..54ab18a436 100644
--- a/README.md
+++ b/README.md
@@ -15,17 +15,18 @@ Measured on 11,571 pull requests across 10 repositories and 15,054 review verdic
## Accuracy
-A separate reviewer reads every pull request an agent opens and can send it back before it merges.
+A separate reviewer reads every pull request an agent opens and can send it back before it merges. The question is whether its objections hold up.
-
+
+
-Pull requests merged without review come from other repositories and periods, not a controlled comparison, so differences may not be caused by review. Revert rates are not distinguishable so far.
+140 of the sent-back pull requests merged unchanged, 133 of them after another review pass approved the same version; 19 did not merge. Whether reviewed code later had to be reverted needs merges at least 14 days old; reviewed merges only began Oct 2026, so those figures are in the report, not here.
## Security
diff --git a/changelog.d/changed-readme-evidence-cards.md b/changelog.d/changed-readme-evidence-cards.md
index d4fffe41bd..06bcea6f09 100644
--- a/changelog.d/changed-readme-evidence-cards.md
+++ b/changelog.d/changed-readme-evidence-cards.md
@@ -1 +1 @@
-- README rewritten around plain-language evidence cards: `hivectl evidence benchmark` now also renders dark/light SVG cards (sent back before merge, real examples, prompt injection, cost, "also" strip, how-it-works ladder) gated by `--readme-min-base` (default 200), takes `--stories` and `--red-team` inputs, and the generated README block lists Accuracy / Security / Cost / Also with no statistical vocabulary; the detailed charts and intervals stay in `src/docs/benchmarks/`, and the former README overview moved to `src/docs/what-is-hive.md`.
+- README rewritten around plain-language evidence cards: `hivectl evidence benchmark` now also renders dark/light SVG cards (sent back then fixed, time to fix, real examples, prompt injection, cost, "also" strip, how-it-works ladder) gated by `--readme-min-base` (default 200), takes `--stories` and `--red-team` inputs, and the generated README block lists Accuracy / Security / Cost / Also with no statistical vocabulary; the detailed charts and intervals stay in `src/docs/benchmarks/`, and the former README overview moved to `src/docs/what-is-hive.md`.
diff --git a/src/docs/benchmarks/assets/outcomes-2026-10-card-came-back-fixed-dark.svg b/src/docs/benchmarks/assets/outcomes-2026-10-card-came-back-fixed-dark.svg
new file mode 100644
index 0000000000..38add3ca2b
--- /dev/null
+++ b/src/docs/benchmarks/assets/outcomes-2026-10-card-came-back-fixed-dark.svg
@@ -0,0 +1,11 @@
+
diff --git a/src/docs/benchmarks/assets/outcomes-2026-10-card-came-back-fixed-light.svg b/src/docs/benchmarks/assets/outcomes-2026-10-card-came-back-fixed-light.svg
new file mode 100644
index 0000000000..bb47744c92
--- /dev/null
+++ b/src/docs/benchmarks/assets/outcomes-2026-10-card-came-back-fixed-light.svg
@@ -0,0 +1,11 @@
+
diff --git a/src/docs/benchmarks/assets/outcomes-2026-10-card-sent-back-dark.svg b/src/docs/benchmarks/assets/outcomes-2026-10-card-sent-back-dark.svg
deleted file mode 100644
index 671dfc76ea..0000000000
--- a/src/docs/benchmarks/assets/outcomes-2026-10-card-sent-back-dark.svg
+++ /dev/null
@@ -1,10 +0,0 @@
-
diff --git a/src/docs/benchmarks/assets/outcomes-2026-10-card-sent-back-light.svg b/src/docs/benchmarks/assets/outcomes-2026-10-card-sent-back-light.svg
deleted file mode 100644
index be5e7e305b..0000000000
--- a/src/docs/benchmarks/assets/outcomes-2026-10-card-sent-back-light.svg
+++ /dev/null
@@ -1,10 +0,0 @@
-
diff --git a/src/docs/benchmarks/assets/outcomes-2026-10-card-time-to-approval-dark.svg b/src/docs/benchmarks/assets/outcomes-2026-10-card-time-to-approval-dark.svg
new file mode 100644
index 0000000000..302e4cd59f
--- /dev/null
+++ b/src/docs/benchmarks/assets/outcomes-2026-10-card-time-to-approval-dark.svg
@@ -0,0 +1,11 @@
+
diff --git a/src/docs/benchmarks/assets/outcomes-2026-10-card-time-to-approval-light.svg b/src/docs/benchmarks/assets/outcomes-2026-10-card-time-to-approval-light.svg
new file mode 100644
index 0000000000..613a0a648a
--- /dev/null
+++ b/src/docs/benchmarks/assets/outcomes-2026-10-card-time-to-approval-light.svg
@@ -0,0 +1,11 @@
+
diff --git a/src/docs/benchmarks/outcomes-2026-10.json b/src/docs/benchmarks/outcomes-2026-10.json
index fa240cb57e..c118b5ed6d 100644
--- a/src/docs/benchmarks/outcomes-2026-10.json
+++ b/src/docs/benchmarks/outcomes-2026-10.json
@@ -422,6 +422,36 @@
"basis": "hive-attributed"
},
"Hub": null,
+ "Followup": {
+ "SentBack": 563,
+ "NewHead": 404,
+ "FixedAndApproved": {
+ "K": 369,
+ "N": 563,
+ "Rate": 0.655417406749556,
+ "Lo": 0.615227050920689,
+ "Hi": 0.6935011722725277,
+ "Reported": true
+ },
+ "MergedSameHead": 140,
+ "MergedSameHeadApproved": 133,
+ "Unresolved": 19,
+ "TimeToApproval": {
+ "Value": 0.4026072397602778,
+ "N": 369,
+ "Reported": true
+ },
+ "CIPairs": 140,
+ "CIFixed": 48,
+ "CICoverage": {
+ "K": 140,
+ "N": 404,
+ "Rate": 0.3465346534653465,
+ "Lo": 0.30177321979967897,
+ "Hi": 0.3941871726664157,
+ "Reported": true
+ }
+ },
"Stories": [
{
"repo": "hivecommons/hive",
diff --git a/src/docs/benchmarks/outcomes-2026-10.md b/src/docs/benchmarks/outcomes-2026-10.md
index 8efe7df1a3..564de06796 100644
--- a/src/docs/benchmarks/outcomes-2026-10.md
+++ b/src/docs/benchmarks/outcomes-2026-10.md
@@ -59,7 +59,11 @@ Whole evidence history, not just the window; months with n < 10 are gaps.
The README carries only plain-language cards; this section says which numbers made it and why the rest did not.
-- **Sent back before merge** is shown: 490 of 1455 reviewed merged PRs (full base, no exclusions beyond the window).
+- **Sent back, then fixed** is shown: 369 of 563 PRs that received a changes_requested verdict in the window pushed a new head that was then approved (404 pushed a new head at all; 140 merged on the same head, 133 of those because another perspective approved it; 19 were unresolved at observation time).
+- **Time to fix** is shown: 24 minutes is the median from the first changes_requested verdict to the first approve verdict on a later head, over 369 PRs.
+- **Sent back before merge** (490 of 1455 reviewed merges) is in the tables but not the README: it counts reviewer objections, not whether the shipped code was right.
+- **Checks fixed after a change request** is not shown: a check result is captured on both the sent-back head and the next head for 140 of 404 PRs (35%), under the 80% floor; of those, 48 went from failing to passing.
+- **Issues that stayed closed** is not shown: issue reopen records exist only for the newest part of the history and no closure via a reviewed merge has been observed for 14 days yet.
- **Checks passing at merge** is not shown: a check result is recorded on 798 of 1455 reviewed merges (55%), under the 80% floor, so the pass rate would describe a biased subset.
- **Revert rate** is not shown: the reviewed base is 43 merges observed 14+ days after merge, below the README minimum of 200, and the two groups are not distinguishable on this data.
- **Cost per merged PR** is shown as an estimate: $2.13 = $6124.75 estimated spend / 2869 hive-attributed merges, 2026-08-08 → 2026-10-27.
@@ -79,6 +83,9 @@ How measured: per-PR evidence bundles the spoke already writes, folded per PR; 9
- **Time-to-merge.** Lower median of merge time minus open time, over merged PRs whose bundle recorded an open time.
- **CI failure at head.** Share of merged heads with a captured `failure` status among heads with a captured success/failure. The unmediated cohort rarely has a captured status (Hive only records CI for heads it reviews), so its cell reports coverage and is not comparable when coverage is low.
- **Changes requested.** Share of mediated merged PRs with at least one `changes_requested` verdict on any head before the merge — findings the author had a chance to address pre-merge. Whether the later head fixed the finding, or a human overrode it, is not verified here.
+- **Sent back, then fixed.** Over PRs whose first `changes_requested` verdict falls in the window: the share that pushed a later head which then received an `approve` verdict. Heads are ordered by bundle creation time; a PR counted as *merged on the same head* merged with no later head, and the report says how many of those heads also carried an `approve` from another perspective, which is how the gate let them through. This measures whether the author acted on the reviewer and the fix passed re-review, not whether the shipped code was correct.
+- **Time to fix.** Lower median of first `approve` on a later head minus the first `changes_requested`, over the PRs counted as fixed above.
+- **Outcome measures after review** (reverts, follow-up fixes, reopened issues) need merges observed at least 14 days; the reviewed cohort is too young for a base of 200, so they stay in the tables and are not drawn in the README.
- **Cost.** `/api/cost` estimated spend divided by its hive-attributed merged-PR count over the same window; model prices are estimates and some models are unpriced, so treat it as an upper-bound order of magnitude.
- **Intervals.** 95% Wilson score; n < 10 is never reported.
- **Regenerate.** `src/scripts/outcome-benchmark.sh --month 2026-10` (see `docs/benchmarks/README.md`).
diff --git a/src/pkg/outcomebench/bench.go b/src/pkg/outcomebench/bench.go
index 7f787a23ce..e50bfde119 100644
--- a/src/pkg/outcomebench/bench.go
+++ b/src/pkg/outcomebench/bench.go
@@ -144,6 +144,8 @@ type Report struct {
Trend map[string][]MonthPoint
Spend *Spend
Hub *HubContext
+ // Followup is what happened after the reviewer sent PRs back in the window.
+ Followup Followup
// Stories and ReadmeMinBase feed the README cards; see cards.go.
Stories []Story `json:",omitempty"`
RedTeam *RedTeam `json:",omitempty"`
@@ -337,6 +339,7 @@ func Compute(bundles []*evidence.Bundle, unreadable int, opts Options) Report {
}
rep.Mediated = med.cohort(CohortMediated)
rep.Unmediated = unmed.cohort(CohortUnmediated)
+ rep.Followup = followups(bundles, prs, opts)
for r, n := range repos {
rep.Repos = append(rep.Repos, RepoCount{Repo: r, Merged: n})
}
@@ -425,3 +428,114 @@ func ParseWindow(month string) (since, until time.Time, err error) {
}
return since, since.AddDate(0, 1, 0), nil
}
+
+// Followup is what happened after the reviewer sent a pull request back:
+// whether the author pushed a fixed version, whether that version was
+// approved, how long that took, and how often the same version merged
+// anyway. It is the closest thing to "was the reviewer right" the evidence
+// records; whether the shipped code then failed is the matured
+// revert/follow-up rate, which needs older merges.
+type Followup struct {
+ // SentBack is the number of pull requests whose first
+ // changes_requested verdict was recorded in the window.
+ SentBack int
+ // NewHead is how many of them pushed a new head after that verdict.
+ NewHead int
+ // FixedAndApproved is K = pull requests whose later head received an
+ // approve verdict, N = SentBack.
+ FixedAndApproved Rate
+ // MergedSameHead merged without a new head; MergedSameHeadApproved is
+ // the subset where another perspective approved that same head, which
+ // is how the gate let it through.
+ MergedSameHead int
+ MergedSameHeadApproved int
+ // Unresolved pushed no new head and had not merged when observed.
+ Unresolved int
+ // TimeToApproval is the median hours from the first changes_requested
+ // verdict to the first approve verdict on a later head.
+ TimeToApproval Median
+ // CIPairs counts pull requests with a captured success/failure on both
+ // the sent-back head and the next head; CIFixed is how many went from
+ // failure to success; CICoverage is CIPairs over NewHead.
+ CIPairs, CIFixed int
+ CICoverage Rate
+}
+
+// followups folds every head of every PR, oldest first, and classifies the
+// PRs the reviewer sent back in the window.
+func followups(bundles []*evidence.Bundle, prs map[string]*evidence.PR, opts Options) Followup {
+ byPR := map[string][]*evidence.Bundle{}
+ for _, b := range bundles {
+ if b != nil {
+ k := evidence.PRKey(b.Repo, b.Number)
+ byPR[k] = append(byPR[k], b)
+ }
+ }
+ var f Followup
+ var fixed, ciKnown int
+ var hours []float64
+ for key, heads := range byPR {
+ sort.SliceStable(heads, func(i, j int) bool { return heads[i].CreatedAt.Before(heads[j].CreatedAt) })
+ var sentBack *evidence.Bundle
+ var at time.Time
+ for _, b := range heads {
+ for _, v := range b.Verdicts {
+ if v.Verdict == "changes_requested" && (at.IsZero() || v.RecordedAt.Before(at)) {
+ sentBack, at = b, v.RecordedAt
+ }
+ }
+ if sentBack != nil {
+ break
+ }
+ }
+ if sentBack == nil || (!opts.Since.IsZero() && at.Before(opts.Since)) || (!opts.Until.IsZero() && !at.Before(opts.Until)) {
+ continue
+ }
+ f.SentBack++
+ var later []*evidence.Bundle
+ for _, b := range heads {
+ if b.CreatedAt.After(sentBack.CreatedAt) && b.HeadSHA != sentBack.HeadSHA {
+ later = append(later, b)
+ }
+ }
+ merged := prs[key] != nil && prs[key].Merge != nil
+ if len(later) == 0 {
+ if merged {
+ f.MergedSameHead++
+ for _, v := range sentBack.Verdicts {
+ if v.Verdict == "approve" {
+ f.MergedSameHeadApproved++
+ break
+ }
+ }
+ } else {
+ f.Unresolved++
+ }
+ continue
+ }
+ f.NewHead++
+ var approved time.Time
+ for _, b := range later {
+ for _, v := range b.Verdicts {
+ if v.Verdict == "approve" && v.RecordedAt.After(at) && (approved.IsZero() || v.RecordedAt.Before(approved)) {
+ approved = v.RecordedAt
+ }
+ }
+ }
+ if !approved.IsZero() {
+ fixed++
+ hours = append(hours, approved.Sub(at).Hours())
+ }
+ if a, b := sentBack.CI.Status, later[0].CI.Status; (a == "success" || a == "failure") && (b == "success" || b == "failure") {
+ ciKnown++
+ if a == "failure" && b == "success" {
+ f.CIFixed++
+ }
+ }
+ }
+ f.FixedAndApproved = NewRate(fixed, f.SentBack)
+ f.TimeToApproval = NewMedian(hours)
+ f.CIPairs = ciKnown
+ f.CICoverage = NewRate(ciKnown, f.NewHead)
+ return f
+}
diff --git a/src/pkg/outcomebench/cards.go b/src/pkg/outcomebench/cards.go
index 7b1ce3f7e6..0038ee007b 100644
--- a/src/pkg/outcomebench/cards.go
+++ b/src/pkg/outcomebench/cards.go
@@ -151,7 +151,8 @@ const (
// Card names; the asset file is -card--.svg.
const (
- CardSentBack = "sent-back"
+ CardCameBack = "came-back-fixed"
+ CardTimeToApproval = "time-to-approval"
CardChecksGreen = "checks-green"
CardStories = "stories"
CardPromptInjection = "prompt-injection"
@@ -162,7 +163,7 @@ const (
// CardNames lists every card the generator knows, in README order.
func CardNames() []string {
- return []string{CardSentBack, CardChecksGreen, CardStories, CardPromptInjection, CardCost, CardAlso, CardHowItWorks}
+ return []string{CardCameBack, CardTimeToApproval, CardChecksGreen, CardStories, CardPromptInjection, CardCost, CardAlso, CardHowItWorks}
}
// CardFile is the asset filename for a card in a theme.
@@ -187,8 +188,10 @@ func (r Report) Cards() map[string]string {
// Card renders one named card; ok is false when its data is not shown.
func (r Report) Card(name string, th cardTheme) (string, bool) {
switch name {
- case CardSentBack:
- return r.sentBackCard(th)
+ case CardCameBack:
+ return r.cameBackCard(th)
+ case CardTimeToApproval:
+ return r.timeToApprovalCard(th)
case CardChecksGreen:
return r.checksGreenCard(th)
case CardStories:
@@ -243,6 +246,22 @@ func oneIn(rate float64) string {
return fmt.Sprintf("1 in %d", int(math.Round(1/rate)))
}
+// share renders a rate above one half as "k in d" with the smallest
+// denominator within a few points of the true value, and otherwise as
+// oneIn.
+func share(rate float64) string {
+ if rate < 0.5 {
+ return oneIn(rate)
+ }
+ for d := 2; d <= 5; d++ {
+ k := int(math.Round(rate * float64(d)))
+ if math.Abs(float64(k)/float64(d)-rate) < 0.04 && k < d {
+ return fmt.Sprintf("%d in %d", k, d)
+ }
+ }
+ return fmt.Sprintf("%d in 10", int(math.Round(rate*10)))
+}
+
// minutes rounds hours to whole minutes.
func minutes(hours float64) int { return int(math.Round(60 * hours)) }
@@ -313,17 +332,33 @@ func numberCard(th cardTheme, title, eye, big, claim, base string) string {
return b.String()
}
-func (r Report) sentBackCard(th cardTheme) (string, bool) {
- m := r.Mediated.ChangesRequested
- if !r.SplitSupported() || !m.Reported || m.Rate <= 0 || m.N < r.MinBase() {
+func (r Report) cameBackCard(th cardTheme) (string, bool) {
+ f := r.Followup
+ fa := f.FixedAndApproved
+ if !fa.Reported || fa.K <= 0 || fa.N < r.MinBase() {
+ return "", false
+ }
+ return numberCard(th,
+ fmt.Sprintf("%s pull requests the reviewer sent back came back fixed and were approved", share(fa.Rate)),
+ "Accuracy · sent back, then fixed",
+ share(fa.Rate),
+ "pull requests the reviewer sent back came back fixed and were approved.",
+ fmt.Sprintf("from %s pull requests sent back for fixes, %s", commas(fa.N), r.windowLabel()),
+ ), true
+}
+
+func (r Report) timeToApprovalCard(th cardTheme) (string, bool) {
+ m := r.Followup.TimeToApproval
+ if !m.Reported || m.N < r.MinBase() {
return "", false
}
+ mins := minutes(m.Value)
return numberCard(th,
- fmt.Sprintf("%s reviewed pull requests was sent back for fixes before it merged", oneIn(m.Rate)),
- "Accuracy · sent back before merge",
- oneIn(m.Rate),
- "reviewed pull requests was sent back for fixes before it merged.",
- fmt.Sprintf("from %s merged pull requests Hive reviewed, %s", commas(m.N), r.windowLabel()),
+ fmt.Sprintf("%d minutes, typically, from the reviewer's change request to approval of the fixed version", mins),
+ "Accuracy · time to fix",
+ fmt.Sprintf("%d min", mins),
+ "typically, from the reviewer's change request to approval of the fixed version.",
+ fmt.Sprintf("from %s pull requests that were sent back and then approved, %s", commas(m.N), r.windowLabel()),
), true
}
diff --git a/src/pkg/outcomebench/cards_test.go b/src/pkg/outcomebench/cards_test.go
index f141d7c2d4..14ffa60bd7 100644
--- a/src/pkg/outcomebench/cards_test.go
+++ b/src/pkg/outcomebench/cards_test.go
@@ -16,7 +16,7 @@ func TestCardsRespectMinimumBase(t *testing.T) {
if rep.MinBase() != DefaultReadmeMinBase {
t.Fatalf("MinBase = %d", rep.MinBase())
}
- for _, c := range []string{CardSentBack, CardChecksGreen, CardCost, CardAlso, CardStories, CardPromptInjection} {
+ for _, c := range []string{CardCameBack, CardTimeToApproval, CardChecksGreen, CardCost, CardAlso, CardStories, CardPromptInjection} {
if rep.HasCard(c) {
t.Errorf("%s drawn with base below %d", c, DefaultReadmeMinBase)
}
@@ -28,7 +28,7 @@ func TestCardsRespectMinimumBase(t *testing.T) {
// because MergedPRs is its own base.
opts.ReadmeMinBase = 10
rep = Compute(fixture(), 0, opts)
- for _, c := range []string{CardSentBack, CardCost, CardAlso} {
+ for _, c := range []string{CardCameBack, CardTimeToApproval, CardCost, CardAlso} {
if !rep.HasCard(c) {
t.Errorf("%s not drawn at min base 10", c)
}
@@ -43,13 +43,19 @@ func TestCardsRespectMinimumBase(t *testing.T) {
if low.HasCard(CardChecksGreen) {
t.Error("checks-green drawn with CI coverage below the floor")
}
- svg, _ := rep.Card(CardSentBack, cardThemes[0])
- for _, want := range []string{"", `", ` 0 {
+ parts = append(parts, fmt.Sprintf("%s of the sent-back pull requests merged unchanged, %s of them after another review pass approved the same version", commas(f.MergedSameHead), commas(f.MergedSameHeadApproved)))
+ }
+ if f.Unresolved > 0 {
+ parts = append(parts, fmt.Sprintf("%s did not merge", commas(f.Unresolved)))
+ }
+ s := ""
+ if len(parts) > 0 {
+ s = strings.Join(parts, "; ") + ". "
+ }
+ return s + fmt.Sprintf("Whether reviewed code later had to be reverted needs merges at least %d days old; reviewed merges only began %s, so those figures are in the report, not here.", int(Maturation.Hours()/24), monthLabel(r.FirstMediatedMerge))
+}
+
// HeroLine is the one sentence of scale at the top of the README.
func (r Report) HeroLine() string {
s := fmt.Sprintf("Measured on %s pull requests across %d repositories", commas(r.PRs), len(r.Repos))
diff --git a/src/pkg/outcomebench/render.go b/src/pkg/outcomebench/render.go
index a1eda62b03..b19b074694 100644
--- a/src/pkg/outcomebench/render.go
+++ b/src/pkg/outcomebench/render.go
@@ -144,6 +144,9 @@ func (r Report) Markdown() string {
w("- **Time-to-merge.** Lower median of merge time minus open time, over merged PRs whose bundle recorded an open time.")
w("- **CI failure at head.** Share of merged heads with a captured `failure` status among heads with a captured success/failure. The unmediated cohort rarely has a captured status (Hive only records CI for heads it reviews), so its cell reports coverage and is not comparable when coverage is low.")
w("- **Changes requested.** Share of mediated merged PRs with at least one `changes_requested` verdict on any head before the merge — findings the author had a chance to address pre-merge. Whether the later head fixed the finding, or a human overrode it, is not verified here.")
+ w("- **Sent back, then fixed.** Over PRs whose first `changes_requested` verdict falls in the window: the share that pushed a later head which then received an `approve` verdict. Heads are ordered by bundle creation time; a PR counted as *merged on the same head* merged with no later head, and the report says how many of those heads also carried an `approve` from another perspective, which is how the gate let them through. This measures whether the author acted on the reviewer and the fix passed re-review, not whether the shipped code was correct.")
+ w("- **Time to fix.** Lower median of first `approve` on a later head minus the first `changes_requested`, over the PRs counted as fixed above.")
+ w("- **Outcome measures after review** (reverts, follow-up fixes, reopened issues) need merges observed at least %d days; the reviewed cohort is too young for a base of %d, so they stay in the tables and are not drawn in the README.", int(Maturation.Hours()/24), DefaultReadmeMinBase)
w("- **Cost.** `/api/cost` estimated spend divided by its hive-attributed merged-PR count over the same window; model prices are estimates and some models are unpriced, so treat it as an upper-bound order of magnitude.")
w("- **Intervals.** 95%% Wilson score; n < %d is never reported.", MinN)
w("- **Regenerate.** `src/scripts/outcome-benchmark.sh --month %s` (see `docs/benchmarks/README.md`).", r.Month())
@@ -235,12 +238,27 @@ func (r Report) Findings() []string {
func (r Report) readmeDecisions() []string {
min := r.MinBase()
var out []string
- cr := r.Mediated.ChangesRequested
- if r.HasCard(CardSentBack) {
- out = append(out, fmt.Sprintf("**Sent back before merge** is shown: %d of %d reviewed merged PRs (full base, no exclusions beyond the window).", cr.K, cr.N))
+ f := r.Followup
+ fa := f.FixedAndApproved
+ if r.HasCard(CardCameBack) {
+ out = append(out, fmt.Sprintf("**Sent back, then fixed** is shown: %d of %d PRs that received a changes_requested verdict in the window pushed a new head that was then approved (%d pushed a new head at all; %d merged on the same head, %d of those because another perspective approved it; %d were unresolved at observation time).", fa.K, fa.N, f.NewHead, f.MergedSameHead, f.MergedSameHeadApproved, f.Unresolved))
+ } else {
+ out = append(out, fmt.Sprintf("**Sent back, then fixed** is not shown: base %d is below the README minimum of %d.", fa.N, min))
+ }
+ ta := f.TimeToApproval
+ if r.HasCard(CardTimeToApproval) {
+ out = append(out, fmt.Sprintf("**Time to fix** is shown: %d minutes is the median from the first changes_requested verdict to the first approve verdict on a later head, over %d PRs.", minutes(ta.Value), ta.N))
} else {
- out = append(out, fmt.Sprintf("**Sent back before merge** is not shown: base %d is below the README minimum of %d, or the split is unsupported.", cr.N, min))
+ out = append(out, fmt.Sprintf("**Time to fix** is not shown: base %d is below the README minimum of %d.", ta.N, min))
+ }
+ cr := r.Mediated.ChangesRequested
+ if cr.Reported {
+ out = append(out, fmt.Sprintf("**Sent back before merge** (%d of %d reviewed merges) is in the tables but not the README: it counts reviewer objections, not whether the shipped code was right.", cr.K, cr.N))
+ }
+ if f.NewHead > 0 {
+ out = append(out, fmt.Sprintf("**Checks fixed after a change request** is not shown: a check result is captured on both the sent-back head and the next head for %d of %d PRs (%.0f%%), under the %d%% floor; of those, %d went from failing to passing.", f.CIPairs, f.NewHead, 100*f.CICoverage.Rate, int(100*minCICardCoverage), f.CIFixed))
}
+ out = append(out, fmt.Sprintf("**Issues that stayed closed** is not shown: issue reopen records exist only for the newest part of the history and no closure via a reviewed merge has been observed for %d days yet.", int(Maturation.Hours()/24)))
ci := r.Mediated.CIFailureAtHead
if r.HasCard(CardChecksGreen) {
out = append(out, fmt.Sprintf("**Checks passing at merge** is shown: %d of %d reviewed merges failed a recorded check.", ci.K, ci.N))
From 48a877761c8835acd4cc958d9971924c5046757d Mon Sep 17 00:00:00 2001
From: Andrew Anderson
Date: Tue, 27 Oct 2026 14:21:00 -0400
Subject: [PATCH 5/6] docs: restyle README cards as full-width warm dark
monospace cards
- One card builder: eyebrow, story headline, amber numerals with glow,
footnote; 880px content width declared at 2x (1760) for crisp rendering
- Warm near-black surface and off-white text; light theme via
- Accuracy tiles merged into one card with two stats; two-bar time
comparison in the Also strip; numbered how-it-works ladder
- Drift check and tests updated; old came-back-fixed/time-to-approval
assets removed
Signed-off-by: Andrew Anderson
---
README.md | 19 +-
changelog.d/changed-readme-evidence-cards.md | 2 +-
.../outcomes-2026-10-card-accuracy-dark.svg | 22 +
.../outcomes-2026-10-card-accuracy-light.svg | 20 +
.../outcomes-2026-10-card-also-dark.svg | 74 +--
.../outcomes-2026-10-card-also-light.svg | 71 +--
...omes-2026-10-card-came-back-fixed-dark.svg | 11 -
...mes-2026-10-card-came-back-fixed-light.svg | 11 -
.../outcomes-2026-10-card-cost-dark.svg | 29 +-
.../outcomes-2026-10-card-cost-light.svg | 27 +-
...utcomes-2026-10-card-how-it-works-dark.svg | 85 ++--
...tcomes-2026-10-card-how-it-works-light.svg | 79 +--
...mes-2026-10-card-prompt-injection-dark.svg | 28 +-
...es-2026-10-card-prompt-injection-light.svg | 26 +-
.../outcomes-2026-10-card-stories-dark.svg | 69 +--
.../outcomes-2026-10-card-stories-light.svg | 69 +--
...mes-2026-10-card-time-to-approval-dark.svg | 11 -
...es-2026-10-card-time-to-approval-light.svg | 11 -
src/pkg/outcomebench/cards.go | 468 +++++++++++-------
src/pkg/outcomebench/cards_test.go | 21 +-
src/pkg/outcomebench/outcomebench_test.go | 4 +-
src/pkg/outcomebench/readme.go | 13 +-
src/pkg/outcomebench/render.go | 4 +-
23 files changed, 698 insertions(+), 476 deletions(-)
create mode 100644 src/docs/benchmarks/assets/outcomes-2026-10-card-accuracy-dark.svg
create mode 100644 src/docs/benchmarks/assets/outcomes-2026-10-card-accuracy-light.svg
delete mode 100644 src/docs/benchmarks/assets/outcomes-2026-10-card-came-back-fixed-dark.svg
delete mode 100644 src/docs/benchmarks/assets/outcomes-2026-10-card-came-back-fixed-light.svg
delete mode 100644 src/docs/benchmarks/assets/outcomes-2026-10-card-time-to-approval-dark.svg
delete mode 100644 src/docs/benchmarks/assets/outcomes-2026-10-card-time-to-approval-light.svg
diff --git a/README.md b/README.md
index 54ab18a436..0da8c3e566 100644
--- a/README.md
+++ b/README.md
@@ -17,14 +17,9 @@ Measured on 11,571 pull requests across 10 repositories and 15,054 review verdic
A separate reviewer reads every pull request an agent opens and can send it back before it merges. The question is whether its objections hold up.
-
-
-
-
+
-
-
140 of the sent-back pull requests merged unchanged, 133 of them after another review pass approved the same version; 19 did not merge. Whether reviewed code later had to be reverted needs merges at least 14 days old; reviewed merges only began Oct 2026, so those figures are in the report, not here.
@@ -32,23 +27,17 @@ A separate reviewer reads every pull request an agent opens and can send it back
Agents work with short-lived, task-scoped access; pull requests from untrusted authors are held for a human; every merge is signed off and credits the issue author.
-
-
-
+
## Cost
Each agent card in the dashboard shows what the current task has cost so far. The figure below is an estimate from provider pricing; some models are unpriced.
-
-
-
+
## Also
-
-
-
+
Production data, not a lab run. Every count above is regenerated from live records by `src/scripts/outcome-benchmark.sh`; bases, dates, exclusions and the full tables are in [how we count →](src/docs/benchmarks/outcomes-2026-10.md).
diff --git a/changelog.d/changed-readme-evidence-cards.md b/changelog.d/changed-readme-evidence-cards.md
index 06bcea6f09..ed3b60e7a2 100644
--- a/changelog.d/changed-readme-evidence-cards.md
+++ b/changelog.d/changed-readme-evidence-cards.md
@@ -1 +1 @@
-- README rewritten around plain-language evidence cards: `hivectl evidence benchmark` now also renders dark/light SVG cards (sent back then fixed, time to fix, real examples, prompt injection, cost, "also" strip, how-it-works ladder) gated by `--readme-min-base` (default 200), takes `--stories` and `--red-team` inputs, and the generated README block lists Accuracy / Security / Cost / Also with no statistical vocabulary; the detailed charts and intervals stay in `src/docs/benchmarks/`, and the former README overview moved to `src/docs/what-is-hive.md`.
+- README rewritten around plain-language evidence cards: `hivectl evidence benchmark` now also renders dark/light full-width SVG cards (accuracy, real examples, prompt injection, cost, "also" strip, how-it-works ladder) gated by `--readme-min-base` (default 200), takes `--stories` and `--red-team` inputs, and the generated README block lists Accuracy / Security / Cost / Also with no statistical vocabulary; the detailed charts and intervals stay in `src/docs/benchmarks/`, and the former README overview moved to `src/docs/what-is-hive.md`.
diff --git a/src/docs/benchmarks/assets/outcomes-2026-10-card-accuracy-dark.svg b/src/docs/benchmarks/assets/outcomes-2026-10-card-accuracy-dark.svg
new file mode 100644
index 0000000000..694105dd9f
--- /dev/null
+++ b/src/docs/benchmarks/assets/outcomes-2026-10-card-accuracy-dark.svg
@@ -0,0 +1,22 @@
+
diff --git a/src/docs/benchmarks/assets/outcomes-2026-10-card-accuracy-light.svg b/src/docs/benchmarks/assets/outcomes-2026-10-card-accuracy-light.svg
new file mode 100644
index 0000000000..598415db0a
--- /dev/null
+++ b/src/docs/benchmarks/assets/outcomes-2026-10-card-accuracy-light.svg
@@ -0,0 +1,20 @@
+
diff --git a/src/docs/benchmarks/assets/outcomes-2026-10-card-also-dark.svg b/src/docs/benchmarks/assets/outcomes-2026-10-card-also-dark.svg
index fb42f3f08a..96a0054db5 100644
--- a/src/docs/benchmarks/assets/outcomes-2026-10-card-also-dark.svg
+++ b/src/docs/benchmarks/assets/outcomes-2026-10-card-also-dark.svg
@@ -1,29 +1,47 @@
-
-The trust level ([ACMM](src/docs/acmm-policy-matrix.md)) sets how far the loop runs on its own, from filing findings only to merging on green.
-
## Run it
```bash
@@ -59,14 +57,13 @@ printf 'HIVE_GITHUB_TOKEN=ghp_...\nHIVE_DASHBOARD_TOKEN=%s\n' "$(openssl rand -h
docker compose -f src/docker-compose.yaml up -d
```
-The [Docker Compose quick start](src/docs/quickstart-docker-compose.md) explains each line. Podman, Kubernetes and the hosted hub: [Run it](src/docs/what-is-hive.md#run-it).
+Each line is explained in the [quick start](src/docs/quickstart-docker-compose.md); [Podman and Kubernetes](src/docs/what-is-hive.md#run-it).
## Learn more
-- [What Hive is](src/docs/what-is-hive.md) — the longer overview: how it differs from a terminal agent, hub and spoke, the contributor relay, ACMM levels, "does Hive fit my project".
-- [Architecture](src/docs/architecture.md) · [Security model](src/docs/security-model.md) · [Terminal dashboard](src/docs/hivectl.md#tui--live-terminal-dashboard) · [Integrations](src/docs/integrations.md)
-- [Contribute compute to a hive](src/docs/contributor-relay.md) · [Contribute to Hive itself](CONTRIBUTING.md) · [Adopters](ADOPTERS.md) · [Repositories](docs/repositories.md) · [Changelog](CHANGELOG.md)
-- [Report a vulnerability](SECURITY.md) · [Telemetry](src/docs/telemetry.md) · [Hosted hub](https://hive.hivecommons.dev)
+[Overview](src/docs/what-is-hive.md) · [Architecture](src/docs/architecture.md) · [Security model](src/docs/security-model.md) · [Terminal UI](src/docs/hivectl.md#tui--live-terminal-dashboard) · [Integrations](src/docs/integrations.md) · [Contribute compute](src/docs/contributor-relay.md) · [Contributing](CONTRIBUTING.md) · [Adopters](ADOPTERS.md) · [Repositories](docs/repositories.md) · [Changelog](CHANGELOG.md) · [Vulnerabilities](SECURITY.md) · [Telemetry](src/docs/telemetry.md) · [Hosted service](https://hive.hivecommons.dev)
+
+That maintainer now reads every repo. Only run images from `ghcr.io/hivecommons/hive`.
---
diff --git a/changelog.d/changed-readme-evidence-cards.md b/changelog.d/changed-readme-evidence-cards.md
index ed3b60e7a2..6f9ad90900 100644
--- a/changelog.d/changed-readme-evidence-cards.md
+++ b/changelog.d/changed-readme-evidence-cards.md
@@ -1 +1 @@
-- README rewritten around plain-language evidence cards: `hivectl evidence benchmark` now also renders dark/light full-width SVG cards (accuracy, real examples, prompt injection, cost, "also" strip, how-it-works ladder) gated by `--readme-min-base` (default 200), takes `--stories` and `--red-team` inputs, and the generated README block lists Accuracy / Security / Cost / Also with no statistical vocabulary; the detailed charts and intervals stay in `src/docs/benchmarks/`, and the former README overview moved to `src/docs/what-is-hive.md`.
+- README rewritten around plain-language evidence cards: `hivectl evidence benchmark` now also renders dark/light full-width SVG cards (accuracy, a before/after scenario from one real pull request, prompt injection, cost, "also" strip, how-it-works ladder) gated by `--readme-min-base` (default 200), takes `--stories` and `--red-team` inputs, and the generated README block lists Accuracy / Security / Cost / Also with no statistical vocabulary; the detailed charts and intervals stay in `src/docs/benchmarks/`, and the former README overview moved to `src/docs/what-is-hive.md`.
diff --git a/src/docs/benchmarks/assets/outcomes-2026-10-card-accuracy-dark.svg b/src/docs/benchmarks/assets/outcomes-2026-10-card-accuracy-dark.svg
index 694105dd9f..2086db4830 100644
--- a/src/docs/benchmarks/assets/outcomes-2026-10-card-accuracy-dark.svg
+++ b/src/docs/benchmarks/assets/outcomes-2026-10-card-accuracy-dark.svg
@@ -1,22 +1,20 @@
-
-2 in 3 pull requests the reviewer sent back came back fixed and were approved; 24 minutes typical from change request to approval of the fix
+
+2 in 3 pull requests the review agent sent back came back fixed and were approved; 24 minutes typical from sent back to approved
-
-ACCURACY · 563 PULL REQUESTS SENT BACK · OCT 2026
-The reviewer sent 563 back. Two in three
-came back fixed, typically in 24 minutes.
-2 in 3
-2 in 3
-came back fixed and were approved, of 563
-sent back.
-24 min
-24 min
-typical, from the change request to
-approval of the fix. Of 369 fixed.
-
-140 merged unchanged after another review pass approved the same version; 19 did not
-merge. Times and verdicts from the review records Hive keeps.
+
+ACCURACY · 563 PULL REQUESTS SENT BACK · (oct '26)
+The review agent sent 563 pull requests
+back. Two in three came back fixed,
+typically in 24 minutes.
+2 in 3
+2 in 3
+came back fixed
+24 min
+24 min
+typical time to fix
+
+563 sent back (oct '26); 140 merged as-is after a second review.
diff --git a/src/docs/benchmarks/assets/outcomes-2026-10-card-accuracy-light.svg b/src/docs/benchmarks/assets/outcomes-2026-10-card-accuracy-light.svg
index 598415db0a..664cc66e3a 100644
--- a/src/docs/benchmarks/assets/outcomes-2026-10-card-accuracy-light.svg
+++ b/src/docs/benchmarks/assets/outcomes-2026-10-card-accuracy-light.svg
@@ -1,20 +1,18 @@
-
-2 in 3 pull requests the reviewer sent back came back fixed and were approved; 24 minutes typical from change request to approval of the fix
+
+2 in 3 pull requests the review agent sent back came back fixed and were approved; 24 minutes typical from sent back to approved
-
-ACCURACY · 563 PULL REQUESTS SENT BACK · OCT 2026
-The reviewer sent 563 back. Two in three
-came back fixed, typically in 24 minutes.
-2 in 3
-came back fixed and were approved, of 563
-sent back.
-24 min
-typical, from the change request to
-approval of the fix. Of 369 fixed.
-
-140 merged unchanged after another review pass approved the same version; 19 did not
-merge. Times and verdicts from the review records Hive keeps.
+
+ACCURACY · 563 PULL REQUESTS SENT BACK · (oct '26)
+The review agent sent 563 pull requests
+back. Two in three came back fixed,
+typically in 24 minutes.
+2 in 3
+came back fixed
+24 min
+typical time to fix
+
+563 sent back (oct '26); 140 merged as-is after a second review.
diff --git a/src/docs/benchmarks/assets/outcomes-2026-10-card-also-dark.svg b/src/docs/benchmarks/assets/outcomes-2026-10-card-also-dark.svg
index 96a0054db5..1f8ddbd16f 100644
--- a/src/docs/benchmarks/assets/outcomes-2026-10-card-also-dark.svg
+++ b/src/docs/benchmarks/assets/outcomes-2026-10-card-also-dark.svg
@@ -1,38 +1,30 @@
-
+Also: consistent, readable, efficient, fast
-
-ALSO · OCT 2026
-Every pull request gets the same
-reviewer. It costs about 15 minutes.
+
+ALSO · (oct '26)
+Every pull request gets the same AI
+review. It adds about 15 minutes.CONSISTENT15,05415,054
-review verdicts,
-each against the
-same checklist and
-one house style
-file.
+reviews, one
+checklist, one
+shared style guideREADABLE4 parts4 parts
-to every verdict:
-what this is, the
-problem, the fix,
-what happens if
-skipped.
+to every review:
+what, problem, fix,
+if skippedEFFICIENTOn demandOn demand
-agents start when
-there is work and
-stop when it is
-done; capacity is
-shared across
-repositories.
+jobs start on work
+and stop when doneFASTmerged without review
@@ -40,8 +32,6 @@
reviewed by Hive27 min
-typical minutes from
-open to merge;
-review adds about
-15.
+review adds about 15
+minutes
diff --git a/src/docs/benchmarks/assets/outcomes-2026-10-card-also-light.svg b/src/docs/benchmarks/assets/outcomes-2026-10-card-also-light.svg
index 81f69ebef4..2bdfcc7654 100644
--- a/src/docs/benchmarks/assets/outcomes-2026-10-card-also-light.svg
+++ b/src/docs/benchmarks/assets/outcomes-2026-10-card-also-light.svg
@@ -1,35 +1,27 @@
-
+Also: consistent, readable, efficient, fast
-
-ALSO · OCT 2026
-Every pull request gets the same
-reviewer. It costs about 15 minutes.
+
+ALSO · (oct '26)
+Every pull request gets the same AI
+review. It adds about 15 minutes.CONSISTENT15,054
-review verdicts,
-each against the
-same checklist and
-one house style
-file.
+reviews, one
+checklist, one
+shared style guideREADABLE4 parts
-to every verdict:
-what this is, the
-problem, the fix,
-what happens if
-skipped.
+to every review:
+what, problem, fix,
+if skippedEFFICIENTOn demand
-agents start when
-there is work and
-stop when it is
-done; capacity is
-shared across
-repositories.
+jobs start on work
+and stop when doneFASTmerged without review
@@ -37,8 +29,6 @@
reviewed by Hive27 min
-typical minutes from
-open to merge;
-review adds about
-15.
+review adds about 15
+minutes
diff --git a/src/docs/benchmarks/assets/outcomes-2026-10-card-cost-dark.svg b/src/docs/benchmarks/assets/outcomes-2026-10-card-cost-dark.svg
index a18d6372c4..fb718cce0b 100644
--- a/src/docs/benchmarks/assets/outcomes-2026-10-card-cost-dark.svg
+++ b/src/docs/benchmarks/assets/outcomes-2026-10-card-cost-dark.svg
@@ -1,22 +1,20 @@
-
+about $2 of agent spend per merged pull request
-
-COST · 2,869 MERGED PULL REQUESTS · AUG 2026 TO OCT 2026
+
+COST · 2,869 MERGED PULL REQUESTS · (aug–oct '26)Agents spent about $6,125 to merge 2,869
-pull requests. That is about $2 each.
-$2
-$2
-about $2 per merged pull request, Aug
-2026 to Oct 2026.
-$6,125
-$6,125
-estimated agent spend across 2,869 merged
-pull requests.
-
-Estimate from provider list prices; some models are unpriced. Each agent card in the
-dashboard shows what its task has cost so far.
+pull requests, about $2 each. It is
+tracked like CI minutes.
+$2
+$2
+per merged pull request
+$6,125
+$6,125
+total agent spend
+
+Estimate from provider list prices (aug–oct '26); some models unpriced.
diff --git a/src/docs/benchmarks/assets/outcomes-2026-10-card-cost-light.svg b/src/docs/benchmarks/assets/outcomes-2026-10-card-cost-light.svg
index a9e1d9d4dc..e06263bc04 100644
--- a/src/docs/benchmarks/assets/outcomes-2026-10-card-cost-light.svg
+++ b/src/docs/benchmarks/assets/outcomes-2026-10-card-cost-light.svg
@@ -1,20 +1,18 @@
-
+about $2 of agent spend per merged pull request
-
-COST · 2,869 MERGED PULL REQUESTS · AUG 2026 TO OCT 2026
+
+COST · 2,869 MERGED PULL REQUESTS · (aug–oct '26)Agents spent about $6,125 to merge 2,869
-pull requests. That is about $2 each.
-$2
-about $2 per merged pull request, Aug
-2026 to Oct 2026.
-$6,125
-estimated agent spend across 2,869 merged
-pull requests.
-
-Estimate from provider list prices; some models are unpriced. Each agent card in the
-dashboard shows what its task has cost so far.
+pull requests, about $2 each. It is
+tracked like CI minutes.
+$2
+per merged pull request
+$6,125
+total agent spend
+
+Estimate from provider list prices (aug–oct '26); some models unpriced.
diff --git a/src/docs/benchmarks/assets/outcomes-2026-10-card-how-it-works-dark.svg b/src/docs/benchmarks/assets/outcomes-2026-10-card-how-it-works-dark.svg
index 93e2f039d7..266edf7b40 100644
--- a/src/docs/benchmarks/assets/outcomes-2026-10-card-how-it-works-dark.svg
+++ b/src/docs/benchmarks/assets/outcomes-2026-10-card-how-it-works-dark.svg
@@ -1,53 +1,47 @@
-
+How Hive works: scan, claim, work, verify, merge, credit
-
+HOW IT WORKS · EVERY TASK
-An agent does the work. A reviewer checks
-it before it merges.
+A coding agent does the work. A review
+agent and your CI gate the merge.0101Scan
-Hive reads your open issues, pull requests
-and failing checks.
-Work comes from your backlog
+Reads open issues, pull requests, failing
+checks.
+like a webhook trigger0202Claim
-One agent claims one task, so nothing is
-done twice.
-No duplicate work
-
-03
-03
-Work
-A fresh, short-lived agent does the job
-and then stops.
-Nothing runs for hours unattended
-
-04
-04
-Verify
-CI runs, a separate reviewer checks the
-change, and planted instructions are
-screened out.
-Problems are found before merge
-
-05
-05
-Merge
-Trust level decides: hold for a human, or
-merge once every check passes.
-Humans keep the decisions
-
-06
-06
-Credit
-The person who opened the issue is
-credited on the merge.
-The asker is credited
+One agent takes one task.
+one job per task
+
+03
+03
+Work
+A short-lived agent does it, then stops.
+ephemeral runner
+
+04
+04
+Verify
+AI review plus your CI must pass.
+both are required checks
+
+05
+05
+Merge
+Held for a human, or merged on green.
+PRs only → merge on green
+
+06
+06
+Credit
+The issue opener is named on the merge.
+co-author trailer
diff --git a/src/docs/benchmarks/assets/outcomes-2026-10-card-how-it-works-light.svg b/src/docs/benchmarks/assets/outcomes-2026-10-card-how-it-works-light.svg
index 826552b976..55fce31690 100644
--- a/src/docs/benchmarks/assets/outcomes-2026-10-card-how-it-works-light.svg
+++ b/src/docs/benchmarks/assets/outcomes-2026-10-card-how-it-works-light.svg
@@ -1,47 +1,41 @@
-
+How Hive works: scan, claim, work, verify, merge, credit
-
+HOW IT WORKS · EVERY TASK
-An agent does the work. A reviewer checks
-it before it merges.
+A coding agent does the work. A review
+agent and your CI gate the merge.01Scan
-Hive reads your open issues, pull requests
-and failing checks.
-Work comes from your backlog
+Reads open issues, pull requests, failing
+checks.
+like a webhook trigger02Claim
-One agent claims one task, so nothing is
-done twice.
-No duplicate work
-
-03
-Work
-A fresh, short-lived agent does the job
-and then stops.
-Nothing runs for hours unattended
-
-04
-Verify
-CI runs, a separate reviewer checks the
-change, and planted instructions are
-screened out.
-Problems are found before merge
-
-05
-Merge
-Trust level decides: hold for a human, or
-merge once every check passes.
-Humans keep the decisions
-
-06
-Credit
-The person who opened the issue is
-credited on the merge.
-The asker is credited
+One agent takes one task.
+one job per task
+
+03
+Work
+A short-lived agent does it, then stops.
+ephemeral runner
+
+04
+Verify
+AI review plus your CI must pass.
+both are required checks
+
+05
+Merge
+Held for a human, or merged on green.
+PRs only → merge on green
+
+06
+Credit
+The issue opener is named on the merge.
+co-author trailer
diff --git a/src/docs/benchmarks/assets/outcomes-2026-10-card-prompt-injection-dark.svg b/src/docs/benchmarks/assets/outcomes-2026-10-card-prompt-injection-dark.svg
index 4d11401a5e..7717c38fa8 100644
--- a/src/docs/benchmarks/assets/outcomes-2026-10-card-prompt-injection-dark.svg
+++ b/src/docs/benchmarks/assets/outcomes-2026-10-card-prompt-injection-dark.svg
@@ -1,21 +1,21 @@
-
-97% of prompt-injection attempts were caught before reaching an agent, with 1 false alarm in 12 harmless requests
+
+97% of planted text that tries to redirect an agent (prompt injection) was screened out before an agent read it, with 1 false alarm in 12 harmless texts
-
-SECURITY · HIVE'S OWN TEST SET · OCT 2026
-We planted 63 prompt-injection attacks in
-our own test set. The scanner caught 61.
-97%
-97%
-of the planted attacks were caught before
-an agent read them. Of 63.
-1 in 12
-1 in 12
-harmless texts wrongly flagged. Of 12.
-
-Hive's own test set, not production traffic. A planted attack is text in an issue or
-file that tries to redirect the agent.
+
+SECURITY · HIVE'S OWN TEST SET · (oct '26)
+We hid 63 texts that try to redirect an
+agent (prompt injection) in issues and
+files. Hive screened out 61 before an
+agent read them.
+97%
+97%
+screened out before an agent read it
+1 in 12
+1 in 12
+harmless texts wrongly flagged
+
+Hive's own test set of 75 texts (oct '26); not production traffic.
diff --git a/src/docs/benchmarks/assets/outcomes-2026-10-card-prompt-injection-light.svg b/src/docs/benchmarks/assets/outcomes-2026-10-card-prompt-injection-light.svg
index 4535ed426d..1c574d1ede 100644
--- a/src/docs/benchmarks/assets/outcomes-2026-10-card-prompt-injection-light.svg
+++ b/src/docs/benchmarks/assets/outcomes-2026-10-card-prompt-injection-light.svg
@@ -1,19 +1,19 @@
-
-97% of prompt-injection attempts were caught before reaching an agent, with 1 false alarm in 12 harmless requests
+
+97% of planted text that tries to redirect an agent (prompt injection) was screened out before an agent read it, with 1 false alarm in 12 harmless texts
-
-SECURITY · HIVE'S OWN TEST SET · OCT 2026
-We planted 63 prompt-injection attacks in
-our own test set. The scanner caught 61.
-97%
-of the planted attacks were caught before
-an agent read them. Of 63.
-1 in 12
-harmless texts wrongly flagged. Of 12.
-
-Hive's own test set, not production traffic. A planted attack is text in an issue or
-file that tries to redirect the agent.
+
+SECURITY · HIVE'S OWN TEST SET · (oct '26)
+We hid 63 texts that try to redirect an
+agent (prompt injection) in issues and
+files. Hive screened out 61 before an
+agent read them.
+97%
+screened out before an agent read it
+1 in 12
+harmless texts wrongly flagged
+
+Hive's own test set of 75 texts (oct '26); not production traffic.
diff --git a/src/docs/benchmarks/assets/outcomes-2026-10-card-stories-dark.svg b/src/docs/benchmarks/assets/outcomes-2026-10-card-stories-dark.svg
index 3dc917c4d7..fc8a4f4ddd 100644
--- a/src/docs/benchmarks/assets/outcomes-2026-10-card-stories-dark.svg
+++ b/src/docs/benchmarks/assets/outcomes-2026-10-card-stories-dark.svg
@@ -1,43 +1,49 @@
-
-Two real pull requests where the reviewer caught a problem before merge
+
+Before and after: hivecommons/hive #13523 was sent back once and merged 16 minutes after opening
-
-ACCURACY · TWO REAL PULL REQUESTS · OCT 2026
-The reviewer caught it. The author fixed
-it in minutes.
-hivecommons/hive #13523
-
-CAUGHT
-The changelog note was written as a
-sentence, not a list entry, so the
-release build would have rejected it; a
-second, duplicate note was also added.
-
-FIXED
-The author fixed it; the new version was
-approved 4 minutes later.
-
-MERGED
-Merged 16 minutes after the pull request
-was opened.
-hivecommons/pluk #328
-
-CAUGHT
-An apostrophe inside a quoted message
-ended the string early, so the project
-would not compile and no tests could
-run.
-
-FIXED
-The author fixed it; the new version was
-approved 10 minutes later.
-
-MERGED
-Merged 23 minutes after the pull request
-was opened.
-
-Times from the review records Hive keeps. Click a title to open the pull request.
+
+BEFORE / AFTER · ONE REAL PULL REQUEST · (oct '26)
+A coding agent opened a pull request. The
+review agent sent it back once. It
+merged, green, 16 minutes after opening.
+Without Hive
+
+ISSUE
+A failing check or an open issue waits
+for someone to pick it up.
+
+PULL REQUEST
+Someone writes the fix when they have
+time.
+
+REVIEW
+Someone else reads it when they have
+time.
+
+MERGE
+Thursday, maybe.
+With Hive: hivecommons/hive #13523
+
+PULL REQUEST
+A coding agent opened it.
+
+SENT BACK AT 4 MIN
+The changelog note was written as a
+sentence, not a list entry, so the
+release build would have rejected it; a
+second, duplicate note was also added.
+
+FIXED AT 8 MIN
+The coding agent pushed a fix; the
+review agent approved it.
+
+MERGED AT 16 MIN
+CI green, a human merged it.
+
+
+Same shape: hivecommons/pluk #328, sent back at 6 min, fixed at 16, merged at 23. Times
+from Hive's review records.
diff --git a/src/docs/benchmarks/assets/outcomes-2026-10-card-stories-light.svg b/src/docs/benchmarks/assets/outcomes-2026-10-card-stories-light.svg
index 5b38bf06d7..6304fc0559 100644
--- a/src/docs/benchmarks/assets/outcomes-2026-10-card-stories-light.svg
+++ b/src/docs/benchmarks/assets/outcomes-2026-10-card-stories-light.svg
@@ -1,43 +1,49 @@
-
-Two real pull requests where the reviewer caught a problem before merge
+
+Before and after: hivecommons/hive #13523 was sent back once and merged 16 minutes after opening
-
-ACCURACY · TWO REAL PULL REQUESTS · OCT 2026
-The reviewer caught it. The author fixed
-it in minutes.
-hivecommons/hive #13523
-
-CAUGHT
-The changelog note was written as a
-sentence, not a list entry, so the
-release build would have rejected it; a
-second, duplicate note was also added.
-
-FIXED
-The author fixed it; the new version was
-approved 4 minutes later.
-
-MERGED
-Merged 16 minutes after the pull request
-was opened.
-hivecommons/pluk #328
-
-CAUGHT
-An apostrophe inside a quoted message
-ended the string early, so the project
-would not compile and no tests could
-run.
-
-FIXED
-The author fixed it; the new version was
-approved 10 minutes later.
-
-MERGED
-Merged 23 minutes after the pull request
-was opened.
-
-Times from the review records Hive keeps. Click a title to open the pull request.
+
+BEFORE / AFTER · ONE REAL PULL REQUEST · (oct '26)
+A coding agent opened a pull request. The
+review agent sent it back once. It
+merged, green, 16 minutes after opening.
+Without Hive
+
+ISSUE
+A failing check or an open issue waits
+for someone to pick it up.
+
+PULL REQUEST
+Someone writes the fix when they have
+time.
+
+REVIEW
+Someone else reads it when they have
+time.
+
+MERGE
+Thursday, maybe.
+With Hive: hivecommons/hive #13523
+
+PULL REQUEST
+A coding agent opened it.
+
+SENT BACK AT 4 MIN
+The changelog note was written as a
+sentence, not a list entry, so the
+release build would have rejected it; a
+second, duplicate note was also added.
+
+FIXED AT 8 MIN
+The coding agent pushed a fix; the
+review agent approved it.
+
+MERGED AT 16 MIN
+CI green, a human merged it.
+
+
+Same shape: hivecommons/pluk #328, sent back at 6 min, fixed at 16, merged at 23. Times
+from Hive's review records.
diff --git a/src/pkg/outcomebench/cards.go b/src/pkg/outcomebench/cards.go
index 1b1584cfec..9b3b0ac6fd 100644
--- a/src/pkg/outcomebench/cards.go
+++ b/src/pkg/outcomebench/cards.go
@@ -6,6 +6,7 @@ import (
"html"
"math"
"os"
+ "regexp"
"strings"
"time"
)
@@ -26,7 +27,7 @@ type Story struct {
// ReviewURL is the posted review that carried the changes_requested
// verdict.
ReviewURL string `json:"review_url"`
- // Caught is the plain-English description of what the reviewer found.
+ // Caught is the plain-English description of what the review agent found.
Caught string `json:"caught"`
// OpenedAt, CaughtAt, FixApprovedAt and MergedAt come from the evidence
// bundles: PR opened, first changes_requested verdict, first approve on
@@ -41,6 +42,9 @@ type Story struct {
func (s Story) URL() string { return fmt.Sprintf("https://github.com/%s/pull/%d", s.Repo, s.Number) }
// FixMinutes is how long after the catch the fixed head was approved.
+// CaughtMinutes is how long after opening the review agent sent it back.
+func (s Story) CaughtMinutes() int { return int(math.Round(s.CaughtAt.Sub(s.OpenedAt).Minutes())) }
+
func (s Story) FixMinutes() int { return int(math.Round(s.FixApprovedAt.Sub(s.CaughtAt).Minutes())) }
// MergeMinutes is open-to-merge for the whole PR.
@@ -232,8 +236,22 @@ func (r Report) windowLabel() string {
return monthLabel(t)
}
-// monthLabel is "Aug 2026".
-func monthLabel(t time.Time) string { return t.UTC().Format("Jan 2006") }
+// monthLabel is "(oct '26)": lowercase month, apostrophe year, in
+// parentheses. No ISO dates appear in the README or its cards.
+func monthLabel(t time.Time) string { return "(" + shortMonth(t) + " '" + t.UTC().Format("06") + ")" }
+
+// rangeLabel is "(aug–oct '26)", or monthLabel when both fall in one month.
+func rangeLabel(a, b time.Time) string {
+ if a.UTC().Year() == b.UTC().Year() && a.UTC().Month() == b.UTC().Month() {
+ return monthLabel(b)
+ }
+ if a.UTC().Year() == b.UTC().Year() {
+ return "(" + shortMonth(a) + "–" + shortMonth(b) + " '" + b.UTC().Format("06") + ")"
+ }
+ return "(" + shortMonth(a) + " '" + a.UTC().Format("06") + "–" + shortMonth(b) + " '" + b.UTC().Format("06") + ")"
+}
+
+func shortMonth(t time.Time) string { return strings.ToLower(t.UTC().Format("Jan")) }
// commas formats 11571 as 11,571.
func commas(n int) string {
@@ -329,7 +347,15 @@ func (c *card) lines(x, y, px, lineH int, fill string, lines []string) int {
// eyebrow is the tracked uppercase line at the top: section, base, window.
func (c *card) eyebrow(y int, s string) {
- c.text(cardPad, y, eyebrowPx, c.th.Muted, ` font-weight="500" letter-spacing="2.2"`, strings.ToUpper(s))
+ c.text(cardPad, y, eyebrowPx, c.th.Muted, ` font-weight="500" letter-spacing="2.2"`, upperKeepDates(s))
+}
+
+var reDateLabel = regexp.MustCompile(`\([A-Z]{3}(?: '\d\d)?(?:–[A-Z]{3})? '\d\d\)`)
+
+// upperKeepDates uppercases an eyebrow but leaves the "(oct '26)" date
+// labels lowercase, as the README style requires.
+func upperKeepDates(s string) string {
+ return reDateLabel.ReplaceAllStringFunc(strings.ToUpper(s), strings.ToLower)
}
// headline is the finding in one plain sentence; the biggest text after
@@ -413,26 +439,22 @@ func (r Report) accuracyCard(th cardTheme) (string, bool) {
if !fa.Reported || fa.K <= 0 || fa.N < r.MinBase() {
return "", false
}
- stats := []stat{{share(fa.Rate), fmt.Sprintf("came back fixed and were approved, of %s sent back.", commas(fa.N))}}
- title := fmt.Sprintf("%s pull requests the reviewer sent back came back fixed and were approved", share(fa.Rate))
- head := fmt.Sprintf("The reviewer sent %s pull requests back. %s came back fixed.", commas(fa.N), strings.ToUpper(shareWords(fa.Rate)[:1])+shareWords(fa.Rate)[1:])
+ words := strings.ToUpper(shareWords(fa.Rate)[:1]) + shareWords(fa.Rate)[1:]
+ stats := []stat{{share(fa.Rate), "came back fixed"}}
+ title := fmt.Sprintf("%s pull requests the review agent sent back came back fixed and were approved", share(fa.Rate))
+ head := fmt.Sprintf("The review agent sent %s pull requests back. %s came back fixed.", commas(fa.N), words)
if ta.Reported && ta.N >= r.MinBase() {
- stats = append(stats, stat{fmt.Sprintf("%d min", minutes(ta.Value)), fmt.Sprintf("typical, from the change request to approval of the fix. Of %s fixed.", commas(ta.N))})
- title += fmt.Sprintf("; %d minutes typical from change request to approval of the fix", minutes(ta.Value))
- head = fmt.Sprintf("The reviewer sent %s back. %s came back fixed, typically in %d minutes.", commas(fa.N), strings.ToUpper(shareWords(fa.Rate)[:1])+shareWords(fa.Rate)[1:], minutes(ta.Value))
+ stats = append(stats, stat{fmt.Sprintf("%d min", minutes(ta.Value)), "typical time to fix"})
+ title += fmt.Sprintf("; %d minutes typical from sent back to approved", minutes(ta.Value))
+ head = fmt.Sprintf("The review agent sent %s pull requests back. %s came back fixed, typically in %d minutes.", commas(fa.N), words, minutes(ta.Value))
}
- var foot []string
+ foot := fmt.Sprintf("%s sent back %s", commas(fa.N), r.windowLabel())
if f.MergedSameHead > 0 {
- foot = append(foot, fmt.Sprintf("%s merged unchanged after another review pass approved the same version", commas(f.MergedSameHead)))
- }
- if f.Unresolved > 0 {
- foot = append(foot, fmt.Sprintf("%s did not merge", commas(f.Unresolved)))
+ foot += fmt.Sprintf("; %s merged as-is after a second review", commas(f.MergedSameHead))
}
return statCard(th, title,
fmt.Sprintf("Accuracy · %s pull requests sent back · %s", commas(fa.N), r.windowLabel()),
- head,
- stats,
- strings.Join(foot, "; ")+". Times and verdicts from the review records Hive keeps.",
+ head, stats, foot+".",
), true
}
@@ -446,8 +468,8 @@ func (r Report) checksGreenCard(th cardTheme) (string, bool) {
fmt.Sprintf("%d in 100 reviewed pull requests merged with every check passing", pass),
fmt.Sprintf("Accuracy · %s reviewed merges · %s", commas(c.CIFailureAtHead.N), r.windowLabel()),
fmt.Sprintf("%d in 100 reviewed pull requests merged with every check green.", pass),
- []stat{{fmt.Sprintf("%d in 100", pass), fmt.Sprintf("merged with every check passing, of %s with a recorded check result.", commas(c.CIFailureAtHead.N))}},
- "Check results as recorded on the merged version.",
+ []stat{{fmt.Sprintf("%d in 100", pass), "merged with every check green"}},
+ fmt.Sprintf("%s reviewed merges with a recorded check result %s.", commas(c.CIFailureAtHead.N), r.windowLabel()),
), true
}
@@ -462,14 +484,14 @@ func (r Report) promptInjectionCard(th cardTheme) (string, bool) {
fa = "0"
}
return statCard(th,
- fmt.Sprintf("%d%% of prompt-injection attempts were caught before reaching an agent, with %d false alarm in %d harmless requests", pct, rt.FalseAlarms, rt.Benign),
+ fmt.Sprintf("%d%% of planted text that tries to redirect an agent (prompt injection) was screened out before an agent read it, with %d false alarm in %d harmless texts", pct, rt.FalseAlarms, rt.Benign),
fmt.Sprintf("Security · Hive's own test set · %s", monthLabel(rt.RunAt)),
- fmt.Sprintf("We planted %d prompt-injection attacks in our own test set. The scanner caught %d.", rt.Attacks, rt.Caught),
+ fmt.Sprintf("We hid %d texts that try to redirect an agent (prompt injection) in issues and files. Hive screened out %d before an agent read them.", rt.Attacks, rt.Caught),
[]stat{
- {fmt.Sprintf("%d%%", pct), fmt.Sprintf("of the planted attacks were caught before an agent read them. Of %d.", rt.Attacks)},
- {fa, fmt.Sprintf("harmless texts wrongly flagged. Of %d.", rt.Benign)},
+ {fmt.Sprintf("%d%%", pct), "screened out before an agent read it"},
+ {fa, "harmless texts wrongly flagged"},
},
- "Hive's own test set, not production traffic. A planted attack is text in an issue or file that tries to redirect the agent.",
+ fmt.Sprintf("Hive's own test set of %d texts %s; not production traffic.", rt.Attacks+rt.Benign, monthLabel(rt.RunAt)),
), true
}
@@ -482,66 +504,69 @@ func (r Report) costCard(th cardTheme) (string, bool) {
if per < 1 {
big = fmt.Sprintf("about %d¢", int(math.Round(per*100)))
}
- window := fmt.Sprintf("%s to %s", monthLabel(r.Spend.WindowStart), monthLabel(r.Spend.WindowEnd))
+ window := rangeLabel(r.Spend.WindowStart, r.Spend.WindowEnd)
return statCard(th,
big+" of agent spend per merged pull request",
fmt.Sprintf("Cost · %s merged pull requests · %s", commas(r.Spend.MergedPRs), window),
- fmt.Sprintf("Agents spent about $%s to merge %s pull requests. That is %s each.", commas(int(math.Round(r.Spend.USD))), commas(r.Spend.MergedPRs), big),
+ fmt.Sprintf("Agents spent about $%s to merge %s pull requests, %s each. It is tracked like CI minutes.", commas(int(math.Round(r.Spend.USD))), commas(r.Spend.MergedPRs), big),
[]stat{
- {strings.TrimPrefix(big, "about "), fmt.Sprintf("%s per merged pull request, %s.", big, window)},
- {"$" + commas(int(math.Round(r.Spend.USD))), fmt.Sprintf("estimated agent spend across %s merged pull requests.", commas(r.Spend.MergedPRs))},
+ {strings.TrimPrefix(big, "about "), "per merged pull request"},
+ {"$" + commas(int(math.Round(r.Spend.USD))), "total agent spend"},
},
- "Estimate from provider list prices; some models are unpriced. Each agent card in the dashboard shows what its task has cost so far.",
+ fmt.Sprintf("Estimate from provider list prices %s; some models unpriced.", window),
), true
}
-// storiesCard shows up to two real PRs side by side: what was caught, how
-// long the fix took, when it merged.
+// storiesCard is the before/after scenario: the left column is the familiar
+// path without Hive, in plain words and without invented numbers; the right
+// column is one real pull request with its recorded timings. A second real
+// pull request, if present, is summarised in the footnote.
func (r Report) storiesCard(th cardTheme) (string, bool) {
if len(r.Stories) == 0 {
return "", false
}
- stories := r.Stories
- if len(stories) > 2 {
- stories = stories[:2]
- }
- c := &card{th: th, title: "Two real pull requests where the reviewer caught a problem before merge"}
+ s := r.Stories[0]
+ c := &card{th: th, title: fmt.Sprintf("Before and after: %s #%d was sent back once and merged %d minutes after opening", s.Repo, s.Number, s.MergeMinutes())}
y := cardPad + eyebrowPx
- c.eyebrow(y, fmt.Sprintf("Accuracy · two real pull requests · %s", r.windowLabel()))
- y = c.headline(y+eyebrowGap, "The reviewer caught it. The author fixed it in minutes.") + 4
+ c.eyebrow(y, fmt.Sprintf("Before / after · one real pull request · %s", r.windowLabel()))
+ y = c.headline(y+eyebrowGap, fmt.Sprintf("A coding agent opened a pull request. The review agent sent it back once. It merged, green, %d minutes after opening.", s.MergeMinutes())) + 4
colW := (colX - cardPad) / 2
- chars := halfChars
- if len(stories) == 1 {
- colW, chars = colX, wideChars
- }
- caught := make([][]string, len(stories))
- maxLines := 0
- for i, s := range stories {
- caught[i] = wrap(s.Caught, chars)
- if len(caught[i]) > maxLines {
- maxLines = len(caught[i])
- }
- }
top := y + 8
- bottom := top
- for i, s := range stories {
+ col := func(i int, label, labelFill string, steps [][2]string, dot []string) int {
x := cardPad + i*(colW+cardPad)
- fmt.Fprintf(&c.b, `%s #%d`+"\n", esc(s.URL()), x, top, th.Brand, bodyPx+2, esc(s.Repo), s.Number)
+ c.text(x, top, bodyPx+2, labelFill, ` font-weight="700"`, label)
yy := top + 36
- step := func(label string, lines []string, fill string) {
- fmt.Fprintf(&c.b, ``+"\n", x+4, yy-4, fill)
- c.text(x+18, yy, eyebrowPx, th.Muted, ` font-weight="500" letter-spacing="2.2"`, strings.ToUpper(label))
- yy = c.lines(x+18, yy+bodyLineH, bodyPx, bodyLineH, th.Text, lines) + 12
+ for j, st := range steps {
+ fmt.Fprintf(&c.b, ``+"\n", x+4, yy-4, dot[j])
+ c.text(x+18, yy, eyebrowPx, th.Muted, ` font-weight="500" letter-spacing="2.2"`, strings.ToUpper(st[0]))
+ yy = c.lines(x+18, yy+bodyLineH, bodyPx, bodyLineH, th.Text, wrap(st[1], halfChars)) + 12
}
- step("Caught", caught[i], th.Brand)
- yy = top + 36 + bodyLineH + maxLines*bodyLineH + 12
- step("Fixed", wrap(fmt.Sprintf("The author fixed it; the new version was approved %d minutes later.", s.FixMinutes()), chars), th.OK)
- step("Merged", wrap(fmt.Sprintf("Merged %d minutes after the pull request was opened.", s.MergeMinutes()), chars), th.OK)
- if yy > bottom {
- bottom = yy
- }
- }
- y = c.foot(bottom, "Times from the review records Hive keeps. Click a title to open the pull request.") - footLineH
+ return yy
+ }
+ left := col(0, "Without Hive", th.Muted, [][2]string{
+ {"Issue", "A failing check or an open issue waits for someone to pick it up."},
+ {"Pull request", "Someone writes the fix when they have time."},
+ {"Review", "Someone else reads it when they have time."},
+ {"Merge", "Thursday, maybe."},
+ }, []string{th.Neutral, th.Neutral, th.Neutral, th.Neutral})
+ right := col(1, fmt.Sprintf("With Hive: %s #%d", s.Repo, s.Number), th.Brand, [][2]string{
+ {"Pull request", "A coding agent opened it."},
+ {fmt.Sprintf("Sent back at %d min", s.CaughtMinutes()), s.Caught},
+ {fmt.Sprintf("Fixed at %d min", s.CaughtMinutes()+s.FixMinutes()), "The coding agent pushed a fix; the review agent approved it."},
+ {fmt.Sprintf("Merged at %d min", s.MergeMinutes()), "CI green, a human merged it."},
+ }, []string{th.Brand, th.Brand, th.OK, th.OK})
+ bottom := left
+ if right > bottom {
+ bottom = right
+ }
+ x := cardPad + colW + cardPad
+ fmt.Fprintf(&c.b, ``+"\n", esc(s.URL()), x, top-20, colW, right-top+20)
+ foot := "Times from Hive's review records; the right column links to the pull request."
+ if len(r.Stories) > 1 {
+ t := r.Stories[1]
+ foot = fmt.Sprintf("Same shape: %s #%d, sent back at %d min, fixed at %d, merged at %d. Times from Hive's review records.", t.Repo, t.Number, t.CaughtMinutes(), t.CaughtMinutes()+t.FixMinutes(), t.MergeMinutes())
+ }
+ y = c.foot(bottom, foot) - footLineH
return c.String(y + cardPad - 4), true
}
@@ -552,9 +577,9 @@ type tile struct{ Head, Big, Text string }
// from the report; the others are plain statements of how Hive works.
func (r Report) alsoTiles() []tile {
return []tile{
- {"Consistent", commas(r.Verdicts), "review verdicts, each against the same checklist and one house style file."},
- {"Readable", "4 parts", "to every verdict: what this is, the problem, the fix, what happens if skipped."},
- {"Efficient", "On demand", "agents start when there is work and stop when it is done; capacity is shared across repositories."},
+ {"Consistent", commas(r.Verdicts), "reviews, one checklist, one shared style guide"},
+ {"Readable", "4 parts", "to every review: what, problem, fix, if skipped"},
+ {"Efficient", "On demand", "jobs start on work and stop when done"},
}
}
@@ -576,9 +601,9 @@ func (r Report) alsoCard(th cardTheme) (string, bool) {
c := &card{th: th, title: "Also: consistent, readable, efficient, fast"}
y := cardPad + eyebrowPx
c.eyebrow(y, "Also · "+r.windowLabel())
- head := "Every pull request gets the same reviewer, and the verdict is written for people."
+ head := "Every pull request gets the same AI review, written for people."
if with, without, ok := r.speedBars(); ok && with > without {
- head = fmt.Sprintf("Every pull request gets the same reviewer. It costs about %d minutes.", with-without)
+ head = fmt.Sprintf("Every pull request gets the same AI review. It adds about %d minutes.", with-without)
}
y = c.headline(y+eyebrowGap, head) + 8
tiles := r.alsoTiles()
@@ -613,16 +638,16 @@ func (r Report) alsoCard(th cardTheme) (string, bool) {
bar(top+28, "merged without review", without, th.Neutral)
bar(top+68, "reviewed by Hive", with, "url(#amber)")
d := with - without
- note := "typical minutes from open to merge."
+ note := "typical minutes, open to merge"
if d > 0 {
- note = fmt.Sprintf("typical minutes from open to merge; review adds about %d.", d)
+ note = fmt.Sprintf("review adds about %d minutes", d)
}
yy := c.lines(x, top+112, bodyPx-1, bodyLineH-2, th.Text, wrap(note, tileChars))
if yy > bottom {
bottom = yy
}
} else {
- yy := c.lines(x, top+28, bodyPx-1, bodyLineH-2, th.Text, wrap("Time to merge is reported once enough reviewed pull requests are on record.", tileChars))
+ yy := c.lines(x, top+28, bodyPx-1, bodyLineH-2, th.Text, wrap("Time to merge is reported once enough pull requests are on record.", tileChars))
if yy > bottom {
bottom = yy
}
@@ -635,19 +660,19 @@ type rung struct{ Step, What, Outcome string }
// hiveLoop is the loop every task goes through, in order.
var hiveLoop = []rung{
- {"Scan", "Hive reads your open issues, pull requests and failing checks.", "Work comes from your backlog"},
- {"Claim", "One agent claims one task, so nothing is done twice.", "No duplicate work"},
- {"Work", "A fresh, short-lived agent does the job and then stops.", "Nothing runs for hours unattended"},
- {"Verify", "CI runs, a separate reviewer checks the change, and planted instructions are screened out.", "Problems are found before merge"},
- {"Merge", "Trust level decides: hold for a human, or merge once every check passes.", "Humans keep the decisions"},
- {"Credit", "The person who opened the issue is credited on the merge.", "The asker is credited"},
+ {"Scan", "Reads open issues, pull requests, failing checks.", "like a webhook trigger"},
+ {"Claim", "One agent takes one task.", "one job per task"},
+ {"Work", "A short-lived agent does it, then stops.", "ephemeral runner"},
+ {"Verify", "AI review plus your CI must pass.", "both are required checks"},
+ {"Merge", "Held for a human, or merged on green.", "PRs only → merge on green"},
+ {"Credit", "The issue opener is named on the merge.", "co-author trailer"},
}
func howItWorksCard(th cardTheme) string {
c := &card{th: th, title: "How Hive works: scan, claim, work, verify, merge, credit"}
y := cardPad + eyebrowPx
c.eyebrow(y, "How it works · every task")
- y = c.headline(y+eyebrowGap, "An agent does the work. A reviewer checks it before it merges.")
+ y = c.headline(y+eyebrowGap, "A coding agent does the work. A review agent and your CI gate the merge.")
const stepX, textX = cardPad + 48, cardPad + 48 + 96
y += 6
for i, r := range hiveLoop {
diff --git a/src/pkg/outcomebench/cards_test.go b/src/pkg/outcomebench/cards_test.go
index 66999c35e7..806de4c45d 100644
--- a/src/pkg/outcomebench/cards_test.go
+++ b/src/pkg/outcomebench/cards_test.go
@@ -44,7 +44,7 @@ func TestCardsRespectMinimumBase(t *testing.T) {
t.Error("checks-green drawn with CI coverage below the floor")
}
svg, _ := rep.Card(CardAccuracy, cardThemes[0])
- for _, want := range []string{"", `3 in 4<", ">20 min<", "of 15", "Of 11 fixed", "4 merged unchanged", `filter="url(#glow)"`, `width="1760"`, `viewBox="0 0 880 `} {
+ for _, want := range []string{"", `3 in 4<", ">20 min<", "15 sent back (oct '26); 4 merged as-is", `filter="url(#glow)"`, `width="1760"`, `viewBox="0 0 880 `} {
if !strings.Contains(svg, want) {
t.Errorf("accuracy card missing %q:\n%s", want, svg)
}
@@ -111,7 +111,7 @@ func TestLoadStoriesAndRedTeam(t *testing.T) {
}
rep := Report{RedTeam: rt, GeneratedAt: now}
svg, ok := rep.Card(CardPromptInjection, cardThemes[1])
- if !ok || !strings.Contains(svg, ">97%<") || !strings.Contains(svg, ">1 in 12<") || !strings.Contains(svg, "We planted 63 prompt-injection") || !strings.Contains(svg, "TEST SET · OCT 2026") || !strings.Contains(svg, "not production traffic") {
+ if !ok || !strings.Contains(svg, ">97%<") || !strings.Contains(svg, ">1 in 12<") || !strings.Contains(svg, "We hid 63 texts that try to redirect an") || !strings.Contains(svg, "TEST SET · (oct '26)") || !strings.Contains(svg, "75 texts (oct '26)") || !strings.Contains(svg, "not production traffic") {
t.Fatalf("prompt-injection card:\n%s", svg)
}
}
diff --git a/src/pkg/outcomebench/outcomebench_test.go b/src/pkg/outcomebench/outcomebench_test.go
index 2ba78b2970..f21e0ce0d8 100644
--- a/src/pkg/outcomebench/outcomebench_test.go
+++ b/src/pkg/outcomebench/outcomebench_test.go
@@ -227,8 +227,8 @@ func TestReadmeBlockRoundTrip(t *testing.T) {
Stories: []Story{{Repo: "hivecommons/hive", Number: 1, Caught: "A test was skipped.", OpenedAt: now.Add(-20 * time.Minute), CaughtAt: now.Add(-15 * time.Minute), FixApprovedAt: now.Add(-10 * time.Minute), MergedAt: now}},
RedTeam: &RedTeam{Attacks: 10, Caught: 9, Benign: 5, FalseAlarms: 1, RunAt: now}})
block := rep.ReadmeBlock("src/docs/benchmarks")
- for _, want := range []string{ReadmeStart, ReadmeEnd, "## Accuracy", "## Security", "## Cost", "## Also", "46 pull requests across 2 repositories", "src/docs/benchmarks/outcomes-2026-10.md",
- "card-accuracy-dark.svg", "card-accuracy-light.svg", "card-stories-dark.svg", "card-prompt-injection-light.svg", "card-cost-dark.svg", "card-also-light.svg", "prefers-color-scheme: dark", "how we count"} {
+ for _, want := range []string{ReadmeStart, ReadmeEnd, "## Accuracy", "## Security", "## Cost", "## Also", "46 pull requests, 2 repositories", "(oct '26)", "Humans still merge", "open PRs only", "src/docs/benchmarks/outcomes-2026-10.md",
+ "card-accuracy-dark.svg", "card-accuracy-light.svg", "card-stories-dark.svg", "card-prompt-injection-light.svg", "card-cost-dark.svg", "card-also-light.svg", "prefers-color-scheme: dark", "How we count"} {
if !strings.Contains(block, want) {
t.Errorf("block missing %q:\n%s", want, block)
}
diff --git a/src/pkg/outcomebench/readme.go b/src/pkg/outcomebench/readme.go
index 2f557f8493..0b2e27776d 100644
--- a/src/pkg/outcomebench/readme.go
+++ b/src/pkg/outcomebench/readme.go
@@ -43,59 +43,46 @@ func (r Report) ReadmeBlock(docDir string) string {
add("## Accuracy", "")
if r.HasCard(CardAccuracy) || r.HasCard(CardStories) {
- add("A separate reviewer reads every pull request an agent opens and can send it back before it merges. The question is whether its objections hold up.", "")
cards(cardW, CardAccuracy, CardChecksGreen)
+ add(fmt.Sprintf("Humans still merge; what reaches them has passed review and CI. Start at \"open PRs only\"; once you trust it, turn the [setting](%s/acmm-policy-matrix.md) up to merge on green, like a merge queue.", strings.TrimSuffix(docDir, "/benchmarks")), "")
cards(cardW, CardStories)
- add(r.followupCaveat(), "")
} else {
add("Not enough reviewed pull requests are on record yet to report accuracy figures.", "")
}
add("## Security", "")
- add("Agents work with short-lived, task-scoped access; pull requests from untrusted authors are held for a human; every merge is signed off and credits the issue author.", "")
+ add("Hidden instructions in issues or files are screened out before an agent reads them; agents run as a GitHub App, not your PAT.", "")
cards(cardW, CardPromptInjection)
add("## Cost", "")
if r.HasCard(CardCost) {
- add("Each agent card in the dashboard shows what the current task has cost so far. The figure below is an estimate from provider pricing; some models are unpriced.", "")
+ add("Tracked like CI minutes; each job's cost is on the dashboard.", "")
cards(cardW, CardCost)
} else {
- add("Cost per merged pull request is reported once enough hive-attributed merges are on record.", "")
+ add("Cost per merged pull request is reported once enough Hive-attributed merges are on record.", "")
}
add("## Also", "")
cards(cardW, CardAlso)
- add(fmt.Sprintf("Production data, not a lab run. Every count above is regenerated from live records by `src/scripts/outcome-benchmark.sh`; bases, dates, exclusions and the full tables are in [how we count →](%s).", report))
+ add(fmt.Sprintf("Production data, not a lab run; counts regenerate from live records. %s [How we count →](%s)", r.followupCaveat(), report))
add(ReadmeEnd)
return strings.Join(lines, "\n")
}
-// followupCaveat is the one honest sentence under the Accuracy cards: what
-// happened to the rest, and why shipped-code outcomes are not shown yet.
+// followupCaveat is the one honest clause under the cards: shipped-code
+// outcomes (reverts) are not shown because reviewed merges are too new.
func (r Report) followupCaveat() string {
- f := r.Followup
- var parts []string
- if f.MergedSameHead > 0 {
- parts = append(parts, fmt.Sprintf("%s of the sent-back pull requests merged unchanged, %s of them after another review pass approved the same version", commas(f.MergedSameHead), commas(f.MergedSameHeadApproved)))
- }
- if f.Unresolved > 0 {
- parts = append(parts, fmt.Sprintf("%s did not merge", commas(f.Unresolved)))
- }
- s := ""
- if len(parts) > 0 {
- s = strings.Join(parts, "; ") + ". "
- }
- return s + fmt.Sprintf("Whether reviewed code later had to be reverted needs merges at least %d days old; reviewed merges only began %s, so those figures are in the report, not here.", int(Maturation.Hours()/24), monthLabel(r.FirstMediatedMerge))
+ return "Reverts are too new to report."
}
// HeroLine is the one sentence of scale at the top of the README.
func (r Report) HeroLine() string {
- s := fmt.Sprintf("Measured on %s pull requests across %d repositories", commas(r.PRs), len(r.Repos))
+ s := fmt.Sprintf("Numbers below: %s pull requests, %d repositories", commas(r.PRs), len(r.Repos))
if r.Verdicts > 0 {
- s += fmt.Sprintf(" and %s review verdicts", commas(r.Verdicts))
+ s += fmt.Sprintf(", %s AI reviews", commas(r.Verdicts))
}
- return s + fmt.Sprintf(", as of %s.", day(r.GeneratedAt))
+ return s + " " + monthLabel(r.GeneratedAt) + "."
}
// picture is the GitHub-flavoured that swaps the card SVG with
diff --git a/src/pkg/outcomebench/readme_budget_test.go b/src/pkg/outcomebench/readme_budget_test.go
index ae821fd8b6..6c007dc42c 100644
--- a/src/pkg/outcomebench/readme_budget_test.go
+++ b/src/pkg/outcomebench/readme_budget_test.go
@@ -15,7 +15,7 @@ import (
// vocabulary creeps back in. Lower the baselines when the README shrinks;
// do not raise them without the maintainers' agreement.
const (
- readmeMaxProseWords = 600
+ readmeMaxProseWords = 300
readmeMaxSections = 10
)
@@ -93,6 +93,19 @@ var readmeForbiddenWords = []string{
"powerful", "seamless", "seamlessly", "blazing", "revolutionary", "best-in-class",
"effortless", "effortlessly", "supercharge", "game-changing", "cutting-edge",
"world-class", "unleash", "amazing", "incredible", "never sleeps",
+ // insider nouns: a first-time reader does not know them. The two actor
+ // names are "the coding agent" and "the review agent".
+ "reviewer", "reviewers", "scanner", "author", "authors", "verdict", "verdicts",
+ "change request", "change requests", "changes requested", "house style", "agent card",
+ "acmm", "trust level", "trust tier", "hub and spoke", "hosted hub", "spoke", "spokes",
+ "contributor relay", "kick", "kicks", "ioscan", "instruction-override",
+}
+
+// Dates in the README and cards are "(oct '26)" or "(aug–oct '26)"; never
+// ISO dates or bare four-digit years.
+var readmeForbiddenDates = []*regexp.Regexp{
+ regexp.MustCompile(`\d{4}-\d{2}-\d{2}`),
+ regexp.MustCompile(`\b20\d\d\b`),
}
func TestReadmeVocabulary(t *testing.T) {
@@ -114,6 +127,11 @@ func TestReadmeVocabulary(t *testing.T) {
t.Errorf("%s contains %q; keep statistical and marketing vocabulary out of the README and its cards", name, w)
}
}
+ for _, re := range readmeForbiddenDates {
+ if m := re.FindString(text); m != "" {
+ t.Errorf("%s contains date %q; write dates as (oct '26)", name, m)
+ }
+ }
if strings.Contains(text, "!") && name == "README prose" {
for _, line := range strings.Split(text, "\n") {
if strings.Contains(line, "!") {