diff --git a/changelog.d/added-17694-benchmarks.md b/changelog.d/added-17694-benchmarks.md new file mode 100644 index 0000000000..8dad632bf3 --- /dev/null +++ b/changelog.d/added-17694-benchmarks.md @@ -0,0 +1 @@ +- Benchmarks: every hive now contributes **anonymized** per-PR benchmark records to the public leaderboard at (data repo `hivecommons/benchmarks`), built from the review evidence bundles it already seals. A record carries a salted hash of the hive id, the agent, harness, model, sandbox runtime, task kind, ACMM level, review verdicts and merge/CI/revert outcome — never the hive id, PR numbers, SHAs, prompts, diffs, logins or costs. Publication is **on by default**; `benchmarks.publish: false` opts a hive out, and `benchmarks.repo_consent` (default on) names repositories only when GitHub reports them public, so no new information is disclosed. Cadence is hub-driven and hourly: the spoke pushes new records to the hub's `POST /api/benchmarks/ingest` with its heartbeat bearer, and the hub commits month files to the data repo through its App identity; `GET /api/benchmarks/export?since=` and the `benchmarks` block in `/api/status` expose the same data and last-push state on the spoke, `GET /api/hub/benchmarks` on the hub, and Settings → Features → Benchmarks shows the toggles and the hive's hash. New bundles are stamped with the agent's sandbox runtime and task kind. See `docs/benchmarks.md` (#17694). diff --git a/dashboard/openapi.json b/dashboard/openapi.json index 1cc70f84a0..ec82036abe 100644 --- a/dashboard/openapi.json +++ b/dashboard/openapi.json @@ -25993,6 +25993,67 @@ } } }, + "/api/benchmarks/export": { + "get": { + "tags": [ + "System" + ], + "summary": "Export this hive's anonymized benchmark records", + "description": "Owner session. Folds the review evidence bundles under /data/evidence into hivecommons/benchmarks schema records (one per PR and agent role): model, harness, sandbox runtime, outcome, time-to-merge, ACMM level, keyed by this hive's salted hash. Repository and org names appear only for public repositories unless `benchmarks.repo_consent: false`. Returns `records: []` when `benchmarks.publish: false`. The same records are pushed hourly to the hub (hivecommons/hive#17694).", + "parameters": [ + { + "name": "since", + "in": "query", + "required": false, + "schema": { + "type": "string" + }, + "description": "RFC3339 or YYYY-MM-DD; PRs whose newest bundle is older are skipped. Default: 90 days ago." + } + ], + "responses": { + "200": { + "description": "Benchmark records", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "hive_hash": { + "type": "string" + }, + "generated_at": { + "type": "string", + "format": "date-time" + }, + "publish": { + "type": "boolean" + }, + "since": { + "type": "string", + "format": "date-time" + }, + "count": { + "type": "integer" + }, + "records": { + "type": "array", + "items": { + "type": "object", + "description": "hivecommons/benchmarks schema/result.schema.json record" + } + } + } + } + } + } + }, + "400": { + "description": "Bad since" + } + } + } + }, "/api/evidence/summary": { "get": { "tags": [ diff --git a/src/cmd/hive/main_sweeps.go b/src/cmd/hive/main_sweeps.go index e3023e49e5..ab9064d0ab 100644 --- a/src/cmd/hive/main_sweeps.go +++ b/src/cmd/hive/main_sweeps.go @@ -19,6 +19,7 @@ import ( "github.com/hivecommons/hive/pkg/config" "github.com/hivecommons/hive/pkg/dashboard" "github.com/hivecommons/hive/pkg/escalation" + "github.com/hivecommons/hive/pkg/evidence/benchmarks" "github.com/hivecommons/hive/pkg/forge" "github.com/hivecommons/hive/pkg/github" "github.com/hivecommons/hive/pkg/github/automerge" @@ -2015,6 +2016,11 @@ func installReviewRelaySettings(client *github.Client, cfg *config.Config, logge client.SetReviewEvidence(func(repo string) github.ReviewEvidenceSettings { return reviewEvidenceSettings(cfg, repo) }) + // hivecommons/hive#17694: stamp the sandbox runtime and benchmark task + // kind on agent-authored bundles so the exporter need not guess. + github.EvidenceAgentContext = func(agent string) (string, string) { + return cfg.BenchmarkSandboxRuntime(agent), benchmarks.TaskKindFor(agent) + } // hivecommons/hive#14581: issue-author credit on every merge path, read // live so a reload of auto_merge.credit_issue_authors applies. client.SetCreditIssueAuthors(func(string) bool { return cfg.AutoMerge.CreditIssueAuthorsEnabled() }) diff --git a/src/docs/api-reference/agents-and-controls.md b/src/docs/api-reference/agents-and-controls.md index 086d85bb17..bf60205f3f 100644 --- a/src/docs/api-reference/agents-and-controls.md +++ b/src/docs/api-reference/agents-and-controls.md @@ -32,6 +32,7 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/review/evidence/list` | Merger or owner | Every retained evidence bundle id and head for one PR (`repo`, `number`), oldest first, with the newest marked latest; `400` for a bad `repo`/`number` | `pkg/dashboard/api.go:528` | | `GET` | `/api/evidence/write-counters` | Merger or owner | Durable attempted vs blocked agent GitHub write counters ([#12069](https://github.com/hivecommons/hive/issues/12069)): `totals` by intent tier (ACMM mode) with `blocked_by_reason`, and `weeks` keyed by the UTC Monday of each ISO week with per-tier and per-reason counts. Persisted across restarts; observation only, never gates a write or merge. Empty `totals`/`weeks` before anything is recorded | `pkg/dashboard/api.go:530` | | `GET` | `/api/evidence/summary` | Dashboard auth/session | Read-only aggregate over `/data/evidence` bundles (schema v1) plus reviewer accuracy. Query `hours` (default 24, capped like `/api/pr-throughput`; `0` = all time) or a custom `from`/`to` range (RFC3339 or `YYYY-MM-DD`, date-only `to` inclusive, `to` defaults to now; `400` when `to` precedes `from`, the span exceeds 3 × 366 days, or `to` is given without `from`; the response echoes `from`/`to`), `repo` (`owner/name`) and `author_kind=human\|agent\|bot`; `400` for a bad value. A PR falls in the window by its open time (earliest bundle `created_at` or `merge.opened_at`), never by when a bundle was written; all bundles of a PR are folded so the newest supplies verdicts and the merge / `reverted_by` / backfilled close come from whichever record knows them. Returns `available_windows` (`{"24","168","720","all"}` → PR count under the same filters), `outcomes_by_author_kind` (merged/closed/open; `closed` comes from backfilled records, `reverted_14d` is null until bundles record it), `reviewer_calibration` buckets per perspective, `reviewer_accuracy_by_model` split by author kind (blocks split into `enforced_blocks` / `advisory_blocks` by `policy.require_approval`; advisory blocks that preceded a merge are `dissent_before_merge`, not false blocks), `sentinel_hits_per_week`, `shadow_gate` (`n`, `would_have_held`, `would_have_blocked`, and per gate `review_gate` / `intent` the evaluated `n` with `held` / `passed` outcomes `{n, merged, closed, open, reverted}`, plus `review_gate_by_lane` `{merge_eligible, self_merge}`), `write_counters` (per intent tier `{attempted, blocked, blocked_by_reason}` from the proxy write counter file; omitted when absent or unreadable), `write_activity` (blocked ÷ every agent write in the window: `total_writes`, `blocked`, `direct` `{recorded, since, partial_coverage, attempted, blocked, by_tier, blocked_by_reason}` from the proxy's weekly counter buckets under `/data/hive-metrics/write-counters.json`, `mediated` `{n, refused, by_action}` from the audit log), `claim_vs_verdict` (per ISO week of agent-authored PRs: `n`, `claimed`, `ci_captured`, `ci_not_green`, `verdict_not_approve`, `both`; records with empty CI are excluded from the CI column and disclosed under `missing.records_empty_ci`), `ci_recovery` (`policy` `{self_authored_automerge, required_checks}`, `n`, `at_claim` `{green, red, pending, none}`, `red` `{n, recovered_merged, green_open, abandoned, pending, outcome_unknown, enriched, not_enriched, median_pushes_to_green, median_hours_to_green, same_head_recovered}`, `gate` `{merged_n, with_checks, green, green_from_bundles, non_required_red, required_unknown, breaches, breach_prs[], suspected, suspected_prs[], pending, unknown, green_to_merge_n, median_green_to_merge_hours}` over every merged PR by GitHub-confirmed state of the *required* checks on the merged head, as the merge gate judges (`breach_prs[]` carry `red`, `merged_by`; a red non-required check is disclosed, not a breach; a red review-time snapshot, a required check with no run, or a red read without a resolvable required set is only suspected), and `convergence` `{n, passes {"1","2","3","4+","never"}, settled, in_flight, unknown, median_passes, from_github, from_bundles, by_week[]}` over agent PRs by CI passes to first green, from the durable `/data/hive-metrics/ci-recovery.json` GitHub check-run trail rollup), `work_balance` (per ISO week: `n`, `opened` by actor kind `{human, agent, bot}`, `closed`, `prs_opened`, `prs_merged`, `prs_closed`, `queue_issues`, `queue_prs`, `issue_source` `github|audit`, `throttled`) with `work_balance_sources` (`issues_github_since`, `issues_fetched_at`, `issues_known`, `audit_only_weeks`, `issue_series_weeks`, `queue_samples`, `queue_retention_days`; issues from the durable GitHub rollup `/data/hive-metrics/issue-activity.json` for weeks at or after `issues_github_since` and, for earlier weeks, from the backfilled issue series `///issues/.json` written by the issue backfill ([#12527](https://github.com/hivecommons/hive/pull/12527)) when its coverage spans the requested portion of the week (`issue_series_weeks` counts the repo-weeks it supplied), else the audit log (`audit_only_weeks`), PRs from folded bundles, queue depth from governor snapshots; `missing.issues_human_opens_unobserved` is true while no GitHub rollup exists unless the issue series covers every row, and whenever any row fell back to the audit log, `missing.throttle_windows_unavailable` discloses that no governor throttle windows are reported), `intent_counters` (per intent tier `{attempted, blocked}` from `/data/hive-metrics/intent-counters.json` — evaluations and the subset the intent gate would have blocked with enforcement on, [#12362](https://github.com/hivecommons/hive/issues/12362); omitted when absent or unreadable), `complexity_by_repo` (per managed repo the newest 12 monthly complexity snapshots, oldest first, each `{month, ref, captured_at, files, functions, mean_cyclomatic, median_cyclomatic, p90_cyclomatic, max_cyclomatic, functions_over_threshold, threshold, static_warnings, static_warnings_by_tool}`; omitted when no repo has a snapshot), the `/api/reviewer/accuracy` summary, and `n` plus `missing` counts (unknown-model verdicts, empty CI, unreadable records). Never consulted by merge decisions. | `pkg/dashboard/api.go:533` | +| `GET` | `/api/benchmarks/export` | Owner only | Anonymized per-PR [benchmark records](../benchmarks.md) (schema `hivecommons/benchmarks` `result.schema.json`) built from the evidence bundles updated since `since` (RFC3339 or `YYYY-MM-DD`; default 90 days; `400` when malformed) plus `hive_hash`, `publish`, `generated_at` and `count`. `records` is empty when `benchmarks.publish: false`; repo/org names appear only for GitHub-public repos when `benchmarks.repo_consent` is on | `pkg/dashboard/api.go:534` | | `GET` | `/api/agents` | Dashboard auth/session | Agents List | `pkg/dashboard/api.go:320` | | `POST` | `/api/agents` | Owner only | Agent Create | `pkg/dashboard/api.go:321` | | `POST` | `/api/agents/import` | Owner only | Agent Import | `pkg/dashboard/api.go:322` | diff --git a/src/docs/api-reference/auth.md b/src/docs/api-reference/auth.md index e4114ce752..36be77b0ee 100644 --- a/src/docs/api-reference/auth.md +++ b/src/docs/api-reference/auth.md @@ -24,6 +24,6 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/openrouter/models` | Dashboard auth/session | Open Router Models | `pkg/dashboard/openrouter.go:47` | | `GET` | `/api/openrouter/credit` | Dashboard auth/session | Open Router Credit | `pkg/dashboard/openrouter.go:48` | | `GET` | `/openrouter/callback` | Public | Open Router Callback | `pkg/dashboard/openrouter.go:49` | -| `POST` | `/api/github-app/recheck` | Dashboard auth/session | GitHub App Recheck | `pkg/dashboard/server.go:1675` | -| `POST` | `/api/github-app/install-clicked` | Dashboard auth/session | GitHub App Install Clicked | `pkg/dashboard/server.go:1676` | -| `GET` | `/gh-setup` | Public | GitHub App Setup Callback | `pkg/dashboard/server.go:1677` | +| `POST` | `/api/github-app/recheck` | Dashboard auth/session | GitHub App Recheck | `pkg/dashboard/server.go:1681` | +| `POST` | `/api/github-app/install-clicked` | Dashboard auth/session | GitHub App Install Clicked | `pkg/dashboard/server.go:1682` | +| `GET` | `/gh-setup` | Public | GitHub App Setup Callback | `pkg/dashboard/server.go:1683` | diff --git a/src/docs/api-reference/configuration.md b/src/docs/api-reference/configuration.md index fb22892d16..00f31ea1e5 100644 --- a/src/docs/api-reference/configuration.md +++ b/src/docs/api-reference/configuration.md @@ -85,7 +85,7 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/compliance/export` | Owner only | Evidence download for `?since=`/`?until=`: `kind=controls` (`json`/`md`), `posture` (`json`/`csv`), `audit` (`json`), `config` (redacted effective config + sha256, `json`), `attestations` (`json`/`csv`) or `bundle` (`json`). Every format includes source, period, generator, selected profile versions, attesters, retention and a per-section SHA-256 manifest; audited as `compliance_export` with the SHA-256 of the delivered bytes ([#11081](https://github.com/hivecommons/hive/issues/11081), [#13582](https://github.com/hivecommons/hive/issues/13582)) | `pkg/dashboard/api.go:369` | | `GET` | `/api/compliance/attestations` | Owner only | Owner attestations, newest first (`?framework=` filters) ([#11081](https://github.com/hivecommons/hive/issues/11081)) | `pkg/dashboard/api.go:370` | | `POST` | `/api/compliance/attestations` | Owner only | Record `{"framework", "reviewed_on", "note"}` for the signed-in owner; audited as `compliance_attestation`; `201` with the attestation, `400` for an unknown framework, a bad or future date, or a note over 2000 characters ([#11081](https://github.com/hivecommons/hive/issues/11081)) | `pkg/dashboard/api.go:371` | -| `PUT` | `/api/config/governor/compliance` | Owner only | Settings → Compliance framework picker: `{"frameworks": [...]}` sets `compliance.frameworks` (empty list deselects all, absent key leaves it unchanged; unknown/duplicate ids are a `400` before any change); audited as `config_compliance`; returns the re-evaluated `compliance.Report` ([#11080](https://github.com/hivecommons/hive/issues/11080)) | `pkg/dashboard/api.go:540` | +| `PUT` | `/api/config/governor/compliance` | Owner only | Settings → Compliance framework picker: `{"frameworks": [...]}` sets `compliance.frameworks` (empty list deselects all, absent key leaves it unchanged; unknown/duplicate ids are a `400` before any change); audited as `config_compliance`; returns the re-evaluated `compliance.Report` ([#11080](https://github.com/hivecommons/hive/issues/11080)) | `pkg/dashboard/api.go:541` | | `GET` | `/api/config/governor/threshold-scaling` | Owner only | Governor Threshold Scaling Get | `pkg/dashboard/api.go:220` | | `PUT` | `/api/config/governor/threshold-scaling` | Owner only | Governor Threshold Scaling Set | `pkg/dashboard/api.go:221` | @@ -94,8 +94,8 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `PUT` | `/api/config/governor/watchdog` | Owner only | Governor Watchdog | `pkg/dashboard/api.go:230` | | `GET` | `/api/config/escalation` | Owner only | Escalation Config Get | `pkg/dashboard/api.go:233` | | `PUT` | `/api/config/escalation` | Owner only | Escalation Config Set | `pkg/dashboard/api.go:234` | -| `GET` | `/api/config/write-surface` | Owner only | Lane write allowlist (`write_surface.allowlist`, [#9587](https://github.com/hivecommons/hive/issues/9587)): `allowlist`, `ops`, `warnings` | `pkg/dashboard/api.go:536` | -| `PUT` | `/api/config/write-surface` | Owner only | Replace the lane write allowlist; `{"allowlist": {}}` clears it; unknown operation or bad lane name is a 400 with nothing changed | `pkg/dashboard/api.go:537` | +| `GET` | `/api/config/write-surface` | Owner only | Lane write allowlist (`write_surface.allowlist`, [#9587](https://github.com/hivecommons/hive/issues/9587)): `allowlist`, `ops`, `warnings` | `pkg/dashboard/api.go:537` | +| `PUT` | `/api/config/write-surface` | Owner only | Replace the lane write allowlist; `{"allowlist": {}}` clears it; unknown operation or bad lane name is a 400 with nothing changed | `pkg/dashboard/api.go:538` | | `GET` | `/api/config/review` | Dashboard auth/session | Review Config Get | `pkg/dashboard/api.go:237` | | `PUT` | `/api/config/review` | Owner only | Review Config Set | `pkg/dashboard/api.go:240` | | `PUT` | `/api/config/governor/features` | Owner only | Governor Features | `pkg/dashboard/api.go:246` | diff --git a/src/docs/api-reference/dashboard-misc.md b/src/docs/api-reference/dashboard-misc.md index 404a030c9b..1bdc09aed9 100644 --- a/src/docs/api-reference/dashboard-misc.md +++ b/src/docs/api-reference/dashboard-misc.md @@ -51,7 +51,7 @@ The monitor renders CI and Review concurrently on one shared PR row; see | `GET` | `/api/upstream-watch` | Owner only | Upstream-watch divergence: per watched fork the upstream followed, the watermark and last-run time, surfaced/ported/dismissed/skipped counts, and the recent upstream refs with their upstream URL, diff URL, fork issue number and state. Read-only — it reads the config and the watch state file, never GitHub. Also carries `interval`, `project_repos` and each repo's configured entry (`config`) for the editor. | `pkg/dashboard/api.go:306` | | `PUT` | `/api/config/upstream-watch` | Owner only | Upstream Watch Config Update: edit `upstream_watch` from the Features tab. Body `{enabled?, interval?, repos?: {: {upstream?, sources?, pr_labels?, max_issues_per_run?, label?} \| null}}` — absent keys are unchanged, `null` removes a repo, keys are stored as the bare repo name. Validated with the loader's rules before the config changes (400 on failure), then persisted and audited as `config_upstream_watch`. | `pkg/dashboard/api.go:307` | -| `GET` | `/api/hotspots` | Dashboard auth/session | Merge-hotspot advisor report (`?repo=owner/name` optional): files repeatedly touched by concurrent PRs over the last `governor.hotspot_window_days`, each with `prs` (distinct PRs), `pr_numbers`, `overlaps` (PR pairs open at the same time), `conflicts` (rebase conflicts blamed on the file), `lines`, `kind` (`code`/`doc`/`lock`/`generated`/`data`), `score`, `symbol_groups`, `recommendation` (`split-code`/`split-doc`/`split-source`/`report-only`/`none`), `reason`, `generated_from`, `source_files` (files under `generated_from`, when known), `issue_number`/`issue_url` of the modularize issue filed, `last_updated`; top-level `count`/`actionable`/`filed`. Derived from the merge train's own PR file lists and the rebase follow-through's conflict records — never lists PRs or files from GitHub. See [operator-reference.md § Merge-hotspot advisor](../operator-reference.md#merge-hotspot-advisor-governorhotspot_). | `pkg/dashboard/api.go:547` | +| `GET` | `/api/hotspots` | Dashboard auth/session | Merge-hotspot advisor report (`?repo=owner/name` optional): files repeatedly touched by concurrent PRs over the last `governor.hotspot_window_days`, each with `prs` (distinct PRs), `pr_numbers`, `overlaps` (PR pairs open at the same time), `conflicts` (rebase conflicts blamed on the file), `lines`, `kind` (`code`/`doc`/`lock`/`generated`/`data`), `score`, `symbol_groups`, `recommendation` (`split-code`/`split-doc`/`split-source`/`report-only`/`none`), `reason`, `generated_from`, `source_files` (files under `generated_from`, when known), `issue_number`/`issue_url` of the modularize issue filed, `last_updated`; top-level `count`/`actionable`/`filed`. Derived from the merge train's own PR file lists and the rebase follow-through's conflict records — never lists PRs or files from GitHub. See [operator-reference.md § Merge-hotspot advisor](../operator-reference.md#merge-hotspot-advisor-governorhotspot_). | `pkg/dashboard/api.go:548` | | `GET` | `/api/widget` | Dashboard auth/session | Widget | `pkg/dashboard/api.go:118` | | `GET` | `/api/pane/{agent}` | Dashboard auth/session | Pane | `pkg/dashboard/api.go:123` | | `GET` | `/api/role` | Dashboard auth/session | Role | `pkg/dashboard/api.go:137` | @@ -84,7 +84,7 @@ The monitor renders CI and Review concurrently on one shared PR row; see | `POST` | `/api/hives/{id}/heartbeat` | Dashboard auth/session | Hives Heartbeat | `pkg/dashboard/api_contribute.go:377` | | `DELETE` | `/api/hives/{id}` | Owner only | Hives Delete | `pkg/dashboard/api_contribute.go:378` | | `POST` | `/api/hives/onboard` | Dashboard auth/session | Hives Onboard | `pkg/dashboard/api_contribute.go:379` | -| `GET` | `/sso` | Public | SSO | `pkg/dashboard/server.go:1682` | +| `GET` | `/sso` | Public | SSO | `pkg/dashboard/server.go:1688` | ### Activity observation provenance The CI snapshot retains real check/node IDs, head, App ID and transition times. diff --git a/src/docs/api-reference/health-status-events.md b/src/docs/api-reference/health-status-events.md index e992c4eff0..c59b00248b 100644 --- a/src/docs/api-reference/health-status-events.md +++ b/src/docs/api-reference/health-status-events.md @@ -24,17 +24,17 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/version` | Dashboard auth/session | Build/version metadata; includes `upgradeMarker` (`target`, `current`, `attempts`, `maxAttempts`, `failed`, `requestedAt`, `lastError`) while a self-upgrade is in flight or has failed ([#6765](https://github.com/hivecommons/hive/issues/6765)), the hub-delivered `upgradePolicy` (`hub_managed`, `spoke_managed`, `schedule`, `schedule_hour`, `schedule_timezone`, `schedule_weekday`, `paused`, `branch`, `channel`, `target_sha`, `target_resolved`, `armed_target`, and `next_update_at` — the hub's RFC3339 ETA for the next promotion into the `stable` channel, omitted when unknown ([#10256](https://github.com/hivecommons/hive/issues/10256))) when available, and an `autoUpdate` object (`enabled`, `state` — one of `disabled`/`up_to_date`/`behind`/`retrying`/`failed`/`unknown` — `healthy`, `period`, `targetBranch`, `targetCommit`, `currentCommit`, `commitsBehind`, `lastAttemptAt`, `lastError`, `detail`, and `nextUpdateAt` — the hub's RFC3339 ETA for the next update, omitted when unknown, disabled or paused ([#10257](https://github.com/hivecommons/hive/issues/10257))) that never reports a failed or unknown update as healthy ([#6962](https://github.com/hivecommons/hive/issues/6962), [#6963](https://github.com/hivecommons/hive/issues/6963)). Also `startedAt` (RFC3339 process start — changes on every pod roll even when the build does not, so the dashboard can say "Restarted" vs "Upgraded") and, until the first status is published, `startupPhase`/`startupPhaseLabel` (the boot phase a fresh pod is in) ([#15788](https://github.com/hivecommons/hive/issues/15788)) | `pkg/dashboard/api.go:45` | | `GET` | `/api/version/release-notes` | Dashboard auth/session | Changelog gap between two revisions: `?from=&to=` returns `{from:{sha,version?}, to:{sha,ref,version?}, sections:[{version,date,categories:{added,changed,fixed,security,deprecated}}], unreleased?, truncated, source, error?}` — the `CHANGELOG.md` release sections present at `to` and absent at `from`, newest first, plus (when `to` is not a release tag) the `changelog.d/` fragments new at `to` under an `Unreleased (in this build)` section. Fetch failures return `source: "unavailable"` with `error` rather than an error status; results are cached per (from, to) for an hour ([#11065](https://github.com/hivecommons/hive/issues/11065)) | `pkg/dashboard/api.go:46` | | `POST` | `/api/release-channel` | Owner only | Hosted spoke self-service release-channel selector; relays `stable`/`candidate`/`edge` to the hub's existing switch-branch endpoint with the spoke dashboard-token proof, and reports the requested channel as pending until the Deployment image lands | `pkg/dashboard/api.go:70` | -| `GET` | `/api/health` | Public | Basic health probe | `pkg/dashboard/server.go:1655` | -| `GET` | `/api/health/deep` | Public | Deep health probe | `pkg/dashboard/server.go:1656` | -| `GET` | `/api/livez` | Public | Kubernetes liveness probe | `pkg/dashboard/server.go:1657` | -| `GET` | `/metrics` | Registered only when `HIVE_METRICS_ENABLED`; requires `Authorization: ****** (403 if the token is unset) | Prometheus metrics | `pkg/dashboard/server.go:1663` | -| `GET` | `/api/status` | Dashboard auth/session | Dashboard aggregate status; `labelPresets[]` (`{key,label,description,labels[],states[]}`) lists the built-in label-filter presets for the Projects cards, led by `needs-human` whose `labels[]` are the effective hard-suppress labels plus `needs-signal`, the hold labels, `stage/071-needs-author` and the reporter-trust awaiting label (#12172); accepts `?fields=a,b` for top-level selection and `?omit=repos` to drop heavy sections, and honors `Accept-Encoding: gzip`; `agents[]` includes busy-pane visibility fields (`kicksUndeliverable`, `busySince`, `lastTranscriptActivity`, `condition`, `conditionMessage`) when a Working pane cannot accept kicks; a working agent also carries `agents[].run` — the live per-run line (#14576): `{since, elapsedS, costUSD, tokens, priced, tests, filesChanged}`, the estimated spend/tokens since the kick was delivered (`lastKickAt`, falling back to `busySince`) (delta of the token collector's per-agent cumulative figure; `priced:false` when the agent has none yet), the last visible test-runner result (`pass`/`fail`/empty) and the last `N files changed` git summary (`-1` when none) parsed from the captured pane; top-level `fleetRun` `{running, hourUSD, hourPriced, dayUSD, weekUSD, monthUSD, windowsPriced}` is the governor strip's rotating `running N · $X this hour / today / this week / this month` figure (#15614) — the hour is the rise since the top of the clock hour, the wider windows are the daily cost ledger summed from 00:00 UTC today, Monday 00:00 UTC (ISO week) and the 1st 00:00 UTC plus the rise since the ledger's last snapshot, `windowsPriced:false` when there is no ledger yet; `sla` (fleet) and `repos[].sla` (per repo) carry the Delivery SLA block derived from the lifecycle timeline with no extra GitHub calls: `windows.{7d,30d}.{open_to_triage,triage_to_start,start_to_pr,pr_to_merge,open_to_merge}` as `{p50_h,p95_h,n}` (hours, nearest-rank), `history[]` of up to 30 daily buckets `{day,start_to_pr_p50,pr_to_merge_p50,open_to_merge_p50,n_started,n_pr,n_merged}` persisted under `/data/sla-history.json`, `stale_in_progress[]` (`{number,repo,started_at,age_h,agent}` — issues carrying `hive/in-progress` longer than the agent claim TTL with no linked PR), `stale_count`, `merged_7d`, `claim_ttl_s` and, on the fleet block, the effective `targets` (`start_to_pr_h`, `pr_to_merge_h`, `open_to_merge_h`); `staleSince` and `repos[].countsStale` / `countsStaleSince` mark totals served from last-good figures — the previous enumeration (#13382) or the ForgeState cache's last-good counts (#13376); `ci_failure_signals` `{count, last, kicks, unmatched}` is the webhook CI-failure signal store — distinct red (repo, head, check) signals retained, the most recent one `{repo, number, head_sha, check, conclusion, details_url, agent, at}`, webhook-triggered kicks/nudges since boot, and failure deliveries whose head matched no open PR; `workflow_approvals` `{approved, denied, permission_errors, last}` counts the hive's approvals of GitHub-gated workflow runs since boot (#16603) — runs approved, runs refused by the trust policy (once per head), `403`/`404` refusals for want of `actions: write`, and the most recent decision `{repo, number, head, run_id, outcome, at}`; `hotspots` `{count, actionable, filed}` is the merge-hotspot advisor's one-line summary (files at/above the thresholds, those with a split recommendation, those with a modularize issue filed), absent when the advisor is not wired — details on `GET /api/hotspots`; `writeback.dedupe` `{enabled, dedupe_same_item, dedupe_burst, rate_capped, hourly_cap, burst_items, burst_window_s, same_item_ttl_s}` is the write-back gate's content layer — the effective `writeback.dedupe.*` settings and the last-hour counts of writes held as `duplicate`, `burst` and `rate-capped` (each also listed under `GET /api/writeback/suppressed`); `config.gate_regressions[]` `{gate, previous, effective, source}` is the boot-time safety-gate regression list (#16438) — gates a previous runtime overlay on the PVC carried that the running config lacks or loosens (`review.require_labels`, `project.issue_filter.require_labels`, `project.paused_repos[]`, `writeback.per_repo[].mode`), empty on a healthy boot | `pkg/dashboard/server.go:1668` | -| `GET` | `/api/status/summary` | Dashboard auth/session | Compact agent/governor/budget status summary for pollers; honors `Accept-Encoding: gzip` | `pkg/dashboard/server.go:1669` | -| `GET` | `/api/overview/issues.csv` | Dashboard auth/session | Cached Overview issue-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1670` | -| `GET` | `/api/overview/issues.json` | Dashboard auth/session | Cached Overview issue-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1671` | -| `GET` | `/api/overview/prs.csv` | Dashboard auth/session | Cached Overview PR-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1672` | -| `GET` | `/api/overview/prs.json` | Dashboard auth/session | Cached Overview PR-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1673` | -| `GET` | `/api/events` | Dashboard auth/session | Server-sent event stream | `pkg/dashboard/server.go:1674` | +| `GET` | `/api/health` | Public | Basic health probe | `pkg/dashboard/server.go:1661` | +| `GET` | `/api/health/deep` | Public | Deep health probe | `pkg/dashboard/server.go:1662` | +| `GET` | `/api/livez` | Public | Kubernetes liveness probe | `pkg/dashboard/server.go:1663` | +| `GET` | `/metrics` | Registered only when `HIVE_METRICS_ENABLED`; requires `Authorization: ****** (403 if the token is unset) | Prometheus metrics | `pkg/dashboard/server.go:1669` | +| `GET` | `/api/status` | Dashboard auth/session | Dashboard aggregate status; `labelPresets[]` (`{key,label,description,labels[],states[]}`) lists the built-in label-filter presets for the Projects cards, led by `needs-human` whose `labels[]` are the effective hard-suppress labels plus `needs-signal`, the hold labels, `stage/071-needs-author` and the reporter-trust awaiting label (#12172); accepts `?fields=a,b` for top-level selection and `?omit=repos` to drop heavy sections, and honors `Accept-Encoding: gzip`; `agents[]` includes busy-pane visibility fields (`kicksUndeliverable`, `busySince`, `lastTranscriptActivity`, `condition`, `conditionMessage`) when a Working pane cannot accept kicks; a working agent also carries `agents[].run` — the live per-run line (#14576): `{since, elapsedS, costUSD, tokens, priced, tests, filesChanged}`, the estimated spend/tokens since the kick was delivered (`lastKickAt`, falling back to `busySince`) (delta of the token collector's per-agent cumulative figure; `priced:false` when the agent has none yet), the last visible test-runner result (`pass`/`fail`/empty) and the last `N files changed` git summary (`-1` when none) parsed from the captured pane; top-level `fleetRun` `{running, hourUSD, hourPriced, dayUSD, weekUSD, monthUSD, windowsPriced}` is the governor strip's rotating `running N · $X this hour / today / this week / this month` figure (#15614) — the hour is the rise since the top of the clock hour, the wider windows are the daily cost ledger summed from 00:00 UTC today, Monday 00:00 UTC (ISO week) and the 1st 00:00 UTC plus the rise since the ledger's last snapshot, `windowsPriced:false` when there is no ledger yet; `sla` (fleet) and `repos[].sla` (per repo) carry the Delivery SLA block derived from the lifecycle timeline with no extra GitHub calls: `windows.{7d,30d}.{open_to_triage,triage_to_start,start_to_pr,pr_to_merge,open_to_merge}` as `{p50_h,p95_h,n}` (hours, nearest-rank), `history[]` of up to 30 daily buckets `{day,start_to_pr_p50,pr_to_merge_p50,open_to_merge_p50,n_started,n_pr,n_merged}` persisted under `/data/sla-history.json`, `stale_in_progress[]` (`{number,repo,started_at,age_h,agent}` — issues carrying `hive/in-progress` longer than the agent claim TTL with no linked PR), `stale_count`, `merged_7d`, `claim_ttl_s` and, on the fleet block, the effective `targets` (`start_to_pr_h`, `pr_to_merge_h`, `open_to_merge_h`); `staleSince` and `repos[].countsStale` / `countsStaleSince` mark totals served from last-good figures — the previous enumeration (#13382) or the ForgeState cache's last-good counts (#13376); `ci_failure_signals` `{count, last, kicks, unmatched}` is the webhook CI-failure signal store — distinct red (repo, head, check) signals retained, the most recent one `{repo, number, head_sha, check, conclusion, details_url, agent, at}`, webhook-triggered kicks/nudges since boot, and failure deliveries whose head matched no open PR; `workflow_approvals` `{approved, denied, permission_errors, last}` counts the hive's approvals of GitHub-gated workflow runs since boot (#16603) — runs approved, runs refused by the trust policy (once per head), `403`/`404` refusals for want of `actions: write`, and the most recent decision `{repo, number, head, run_id, outcome, at}`; `hotspots` `{count, actionable, filed}` is the merge-hotspot advisor's one-line summary (files at/above the thresholds, those with a split recommendation, those with a modularize issue filed), absent when the advisor is not wired — details on `GET /api/hotspots`; `writeback.dedupe` `{enabled, dedupe_same_item, dedupe_burst, rate_capped, hourly_cap, burst_items, burst_window_s, same_item_ttl_s}` is the write-back gate's content layer — the effective `writeback.dedupe.*` settings and the last-hour counts of writes held as `duplicate`, `burst` and `rate-capped` (each also listed under `GET /api/writeback/suppressed`); `config.gate_regressions[]` `{gate, previous, effective, source}` is the boot-time safety-gate regression list (#16438) — gates a previous runtime overlay on the PVC carried that the running config lacks or loosens (`review.require_labels`, `project.issue_filter.require_labels`, `project.paused_repos[]`, `writeback.per_repo[].mode`), empty on a healthy boot | `pkg/dashboard/server.go:1674` | +| `GET` | `/api/status/summary` | Dashboard auth/session | Compact agent/governor/budget status summary for pollers; honors `Accept-Encoding: gzip` | `pkg/dashboard/server.go:1675` | +| `GET` | `/api/overview/issues.csv` | Dashboard auth/session | Cached Overview issue-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1676` | +| `GET` | `/api/overview/issues.json` | Dashboard auth/session | Cached Overview issue-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1677` | +| `GET` | `/api/overview/prs.csv` | Dashboard auth/session | Cached Overview PR-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1678` | +| `GET` | `/api/overview/prs.json` | Dashboard auth/session | Cached Overview PR-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1679` | +| `GET` | `/api/events` | Dashboard auth/session | Server-sent event stream | `pkg/dashboard/server.go:1680` | | `GET` | `/api/swarm` | Dashboard auth/session | Current swarm status: display name, duration, active repo, expiry, persisted prep metrics when available, and idle-unlock fields. | `pkg/dashboard/swarm.go:310` | | `POST` | `/api/swarm` | Owner only | Start a 24h repo swarm for a configured repo, returning 409 when another swarm is active and 423 when a previous swarm keeps the next swarm locked until enough agents are idle unless body `{"repo":"owner/name","force":true}` is used; saves the active record before repo prep, persists prep metrics, and announces the start to Discord when configured. | `pkg/dashboard/swarm.go:311` | | `DELETE` | `/api/swarm` | Owner only | End the active swarm, score issues closed and PRs merged in the window, append it to history, and announce the result to Discord when configured. | `pkg/dashboard/swarm.go:312` | diff --git a/src/docs/api-reference/hub-server.md b/src/docs/api-reference/hub-server.md index 91265f4c73..3112db2c7d 100644 --- a/src/docs/api-reference/hub-server.md +++ b/src/docs/api-reference/hub-server.md @@ -32,9 +32,11 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/access-denied` | Hub handler-specific | Access Denied | `pkg/hub/saas.go:370` | | `GET` | `/api/hub/clusters` | Hub auth | List Clusters | `pkg/hub/saas.go:531` | | `GET` | `/api/hub/image-pulls` | Hub auth | Per-Release Image Pull Series | `pkg/hub/saas.go:377` | +| `POST` | `/api/benchmarks/ingest` | Per-hive heartbeat bearer | Spoke push target for anonymized [benchmark records](../benchmarks.md): body `{hive_id, records[]}`; the hub recomputes the hive hash from the authenticated id, rejects records whose `source.hive_id` differs (`401` bad bearer, `403` unregistered hive, `400` malformed), merges accepted records by id into `/data/benchmarks//.json` and returns `202 {accepted, rejected, files}` | `pkg/hub/benchmarks.go` | +| `GET` | `/api/hub/benchmarks` | Public | Benchmark publisher status: `last_ingest_at`, `last_publish_at`, `last_publish_files`, `last_error`, `pending_files`, `hive_count`, `record_count` and per-hive-hash `hives{}` counts; the hub commits pending month files hourly to `hivecommons/benchmarks` (`HIVE_BENCHMARKS_REPO`) via its App identity | `pkg/hub/benchmarks.go` | | `GET` | `/api/reach` | Hub admin | PR Reach Report (?pr=NNN or ?recent=K) | `pkg/hub/saas.go:522` | -| `GET` | `/fleet` | Hub handler-specific | My-Hives Fleet Page (static; data via `/api/saas/my-hives`) | `pkg/hub/server.go:1780` | -| `GET` | `/my-hives` | Hub handler-specific | 301 redirect to `/fleet` (query preserved) | `pkg/hub/server.go:1781` | +| `GET` | `/fleet` | Hub handler-specific | My-Hives Fleet Page (static; data via `/api/saas/my-hives`) | `pkg/hub/server.go:1783` | +| `GET` | `/my-hives` | Hub handler-specific | 301 redirect to `/fleet` (query preserved) | `pkg/hub/server.go:1784` | | `GET` | `/activity` | Signed-in user | Fleet activity page ([#16063](https://github.com/hivecommons/hive/issues/16063)): read-only, data-free shell that polls `/api/hub/activity` every 15 s with `If-None-Match`. No session → `307` to `/login`; blocked user → `403`. Own Content-Security-Policy (scripts and styles from `'self'` only, `script-src-attr`/`style-src-attr` `'none'`), `Cache-Control: no-cache` + content `ETag`. See [Hub Fleet activity page](../hub-fleet-activity.md) | `pkg/hub/activity_page.go` | | `GET` | `/activity/public` | Public | Public-safe Fleet activity page ([#15966](https://github.com/hivecommons/hive/issues/15966)): the same data-free shell as `/activity`, served without a session; on this path the page polls `/api/hub/activity/public` with `credentials: 'omit'`. Source of the `twitch.tv/hivecommonsfleet` broadcast. Same CSP and cache headers as `/activity`. See [Hub Fleet activity page](../hub-fleet-activity.md#public-safe-view) | `pkg/hub/activity_public.go` | | `GET` | `/activity/assets/{name}` | Public | The shared tracker assets the Fleet activity page loads, byte-identical: the row asset (`pkg/webassets/trackerrow`, [#16062](https://github.com/hivecommons/hive/issues/16062)) and the `am.*` URL-view and display-playlist helpers (`pkg/webassets/trackerview`, [#16064](https://github.com/hivecommons/hive/issues/16064)); only names in `trackerrow.Files` or `trackerview.Files`, else `404`. `Cache-Control: no-cache` + content `ETag` | `pkg/hub/activity_page.go` | @@ -43,29 +45,29 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `DELETE` | `/api/shared-agent-pool/leases` | Hub handler-specific | Release a shared CLI session lease | `pkg/hub/server.go:1717` | | `POST` | `/api/heartbeat` | Hub handler-specific | Heartbeat. A request carrying `X-Hive-Activity-Update` is an activity-only update ([#15963](https://github.com/hivecommons/hive/issues/15963)): body `{"activity_update":{"hive_id","activity"}}`, the same per-hive bearer, accepted only for a hive already in the registry (`404` otherwise); it replaces the in-memory activity digest and nothing else, and the signed response carries `ok`, `activity_etag` and `activity_events_ack` (the highest activity event sequence number the hub holds for the hive, [#15964](https://github.com/hivecommons/hive/issues/15964); full heartbeat responses carry it too) | `pkg/hub/server.go:1718` | | `POST` | `/api/task-status` | Hub handler-specific | Task Status | `pkg/hub/server.go:1719` | -| `GET` | `/api/registry` | Hub handler-specific | Registry | `pkg/hub/server.go:1727` | -| `GET` | `/api/hub/leaderboard` | Hub handler-specific | Leaderboard | `pkg/hub/server.go:1728` | -| `GET` | `/api/hub/stats` | Hub handler-specific | Stats | `pkg/hub/server.go:1729` | -| `GET` | `/api/fleet-stats` | Hub handler-specific | Fleet Stats | `pkg/hub/server.go:1730` | +| `GET` | `/api/registry` | Hub handler-specific | Registry | `pkg/hub/server.go:1730` | +| `GET` | `/api/hub/leaderboard` | Hub handler-specific | Leaderboard | `pkg/hub/server.go:1731` | +| `GET` | `/api/hub/stats` | Hub handler-specific | Stats | `pkg/hub/server.go:1732` | +| `GET` | `/api/fleet-stats` | Hub handler-specific | Fleet Stats | `pkg/hub/server.go:1733` | | `GET` | `/api/hub/evidence` | Hub admin | Read-only cross-hive evidence drill-down from each spoke's stored heartbeat `evidence` summary ([#12116](https://github.com/hivecommons/hive/issues/12116)); non-admins get `403`. Returns `spokes`, `reporting`, pooled `calibration` (bucket 0–5 → `{n, bad, rate, lo, hi}`; the hub view folds the six buckets into four reviewer-confidence ranges), `findings[]` (the plain-language *What this says* sentences, generated from the same fields; [#12456](https://github.com/hivecommons/hive/issues/12456)), `by_acmm_level` (`level`, `hives`, `merged_90d`, `revert_rate`, and `time_to_merge` — `median_of_hive_medians_s`, `hives`, `n` — the median of the level's per-hive median time-to-merge, omitted when no hive at the level reports one; [#12464](https://github.com/hivecommons/hive/issues/12464)), `family_false_approve_rate` (`same`/`cross`/`mixed`, hives classed by their reviewer vs author family sets), fleet `record_coverage` (merged PRs with an evidence record over `prs_merged_90d`), fleet panels ([#12370](https://github.com/hivecommons/hive/issues/12370)) `outcomes_by_author_kind[]` (`kind`, `hives`, `n`, `merged`, `reverted`, `merge_rate`, `revert_rate`), `write_counters[]` (`tier`, `hives`, `attempted`, `blocked`, `block_rate`) and `shadow_gate` (`review_gate` / `intent` → `held` / `passed` outcomes with `revert_rate`), each omitted while no spoke reports it and repeated per `by_acmm_level[]` row, and per hive `hives[]` with `acmm_level`, `prs_merged_90d`, `record_coverage`, `family_mix`, false-approve / false-block / revert rates, families, a calibration series the raw `counts` behind them (including `by_author_kind`) so the drill-down can re-pool a hive / ACMM-level selection client-side, and `criteria[]`: the per-axis criteria table derived hub-side from the quadrant criteria (`axis`, `label`, `scored`, `criteria[]` with `name`, `label`, `status` `met`/`unmet`/`not_reported`, `display`/`score` when measured, `reason` otherwise; offline, evidence-less or unscored hives are `not_reported`, never `unmet`; [#12463](https://github.com/hivecommons/hive/issues/12463)). Every rate is `{n, rate, lo, hi}` with a 95% Wilson interval; `rate`/`lo`/`hi` are null when `n` < 10. Window selection ([RFC #12062](https://github.com/hivecommons/hive/issues/12062)): `?hours=24|168|720` pools each hive's matching heartbeat `windows` block instead of its 90-day block (any other or absent `hours` → 90 days); `?from=YYYY-MM-DD|RFC3339[&to=…]` (date-only `to` is inclusive, `to` defaults to now, `to` ≥ `from`, span ≤ 366 days, else `400`) is served from the smallest reported window that reaches back to `from` and flagged `window.approximate`. The response carries `window` (`hours`, `key`, `label`, `from`, `to`, `approximate`, `coverage_available` — `record_coverage` is only computed for the 90-day window because `prs_merged_90d` is a 90-day count), `window_unreported` and per-hive `window_unreported` for hives whose spoke predates per-window reporting. `?hive=[,…]` (repeatable) and `?level=` restrict the registry server-side to those hives / that ACMM level before pooling. Hub admins open it from the Fleet quadrant card or its **🔬 Evidence** button ([#12117](https://github.com/hivecommons/hive/issues/12117)), pick a window (24h / 7d / 30d / 90d / custom), filter it by hive and ACMM level and read each section's ⓘ explainer; the per-hive author-kind revert counts behind the *Revert rate by author kind* panel are `hives[].counts.by_author_kind` ([#12326](https://github.com/hivecommons/hive/issues/12326)). Never consulted by merge decisions | `pkg/hub/evidence.go` | | `GET` | `/api/hub/evidence/report` | Hub admin | Plain-language report of the hub evidence view ([#12456](https://github.com/hivecommons/hive/issues/12456)), rendered server-side from the same `EvidenceView` as `/api/hub/evidence` so every figure reconciles with the modal; non-admins get `403`. Same `hours=` / `from=` / `to=` window and `hive=` / `level=` filter parameters. `format=md` (default, `text/markdown`) or `format=html` (standalone page, inline CSS, no scripts or external assets, hive names escaped); any other value → `400`. `download=1` sets `Content-Disposition: attachment; filename="hive-evidence---.md|html"`, otherwise inline. Sections: Summary (the generated findings), What we measured and why, The numbers, How to read these, What is not reported yet and why, Grade per section, footer link to `review-evidence.md`. `Cache-Control: no-store`. Read-only; never consulted by merge decisions | `pkg/hub/evidence_report.go` | | `GET` | `/api/hub/activity` | Signed-in user | Read-only hub-wide activity view ([#15951](https://github.com/hivecommons/hive/issues/15951), RFC [#15861](https://github.com/hivecommons/hive/issues/15861)) merged from each spoke's heartbeat `activity` digest, held in hub memory only. Authorization is per user and per hive before aggregation: hub admins see every registered hive; everyone else only hives they own or hold an unexpired grant for; anonymous → `401`. Returns `hives[]` (`hive_id`, `name`, `status` `ok`/`stale`/`not_reporting`/`missing`, `reporting`, `stale` once the block is older than two heartbeats, `last_heartbeat`, `received_at`, `generated_at`, `totals` — `items`, `by_stage`, `by_modifier`, counting private and truncated items too — `truncated` and `events_dropped`), `items[]` (`hive_id`, `repo`, `n`, `kind`, `title`, `author_kind`, `stage`, `stage_n`, `since`, `mods`, `private`) ordered needs-human/needs-decision, stalled, furthest stage, oldest, a response-level `truncated` (cap 2000), and `events[]` ([#15964](https://github.com/hivecommons/hive/issues/15964): `hive_id`, `received_at`, `seq`, `type` `opened`/`stage`/`merged`, `repo`, `n`, `kind`, `title`, `from`, `stage`, `at`, `private`), oldest first, from a fleet-wide in-memory ring of at most 1,000 events, none held longer than 24 hours (ageing out is not counted in `events_dropped`; when full, the oldest event of the hive holding the most events is dropped, counted in that hive's `events_dropped`, so one hive cannot evict the others' events). Events are filtered per user and per hive exactly like items, before aggregation, so an event for a hive the viewer cannot see is never delivered; `mod`/`author_kind` filters keep only events whose item is still on the hive's board. A spoke block carries at most 200 events; drops are disclosed per hive as `events_dropped`. New events change the `ETag`. Each digest carries at most 100 items by default and never more than the 250-item hub ceiling; private-repository items appear only when the spoke opts in. Filters: `hive`, `repo`, `stage`, `mod`, `author_kind`, `q` (title substring or `#`), each ≤ 200 characters (else `400`). `ETag` + `If-None-Match` → `304`; `Cache-Control: private, no-cache`, `Vary: Cookie, Authorization`. No actions | `pkg/hub/activity.go` | | `GET` | `/api/hub/activity/public` | Public | Public-safe activity view ([#15966](https://github.com/hivecommons/hive/issues/15966)): the `/api/hub/activity` shape for anyone, never reading a cookie, token or `Authorization` header (credentials do not change the response). Only hives the registry lists as public on github.com, and only items and events not marked `private`; `totals`, per-hive `truncated` and `events_dropped`, hive owners and per-user grants are omitted. Same filters (`400` on an invalid one). The body is built at most once per 2 s per parsed filter and shared by every anonymous caller, so rows, `generated_at` and the `ETag` change at most every 2 s; query strings that parse to the default view share one entry. Toggling a hive's visibility drops the cache at once. `ETag` + `If-None-Match` → `304`; `Cache-Control: no-cache` | `pkg/hub/activity_public.go` | -| `GET` | `/api/hub/version` | Hub handler-specific | Hub Version | `pkg/hub/server.go:1738` | -| `DELETE` | `/api/hub/registry/{id}` | Hub handler-specific | Registry Delete | `pkg/hub/server.go:1748` | -| `POST` | `/api/contribute/register` | Hub handler-specific | Contribute Proxy | `pkg/hub/server.go:1749` | -| `GET` | `/api/contribute/status` | Hub handler-specific | Contribute Status | `pkg/hub/server.go:1750` | -| `GET` | `/api/contribute/ws` | Hub handler-specific | Contribute WSProxy | `pkg/hub/server.go:1751` | -| `POST` | `/api/github/webhook` | Hub handler-specific | GitHub Webhook | `pkg/hub/server.go:1752` | -| `GET` | `/gh-setup` | Hub handler-specific | GitHub App Setup Router | `pkg/hub/server.go:1753` | -| `GET` | `/learn` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1754` | -| `GET` | `/get-started` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1755` | -| `GET` | `/api/docs` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1758` | -| `GET` | `/api/reading-list` | Hub handler-specific | Reading List | `pkg/hub/server.go:1761` | -| `GET` | `/reading` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1762` | -| `GET` | `/cncf-reference-architecture` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1769` | -| `GET` | `/cncf-reference-architecture/console` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1770` | -| `GET` | `/cncf-reference-architecture/bluefin` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1771` | -| `GET` | `/{$}` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1788` | -| `GET` | `/og-card.png` | Hub handler-specific | OGCard | `pkg/hub/server.go:1793` | -| `GET` | `/` | Public | Static asset fallback (`http.FileServerFS` over the embedded `static/` tree) for any path no other route claims | `pkg/hub/server.go:1795` | +| `GET` | `/api/hub/version` | Hub handler-specific | Hub Version | `pkg/hub/server.go:1741` | +| `DELETE` | `/api/hub/registry/{id}` | Hub handler-specific | Registry Delete | `pkg/hub/server.go:1751` | +| `POST` | `/api/contribute/register` | Hub handler-specific | Contribute Proxy | `pkg/hub/server.go:1752` | +| `GET` | `/api/contribute/status` | Hub handler-specific | Contribute Status | `pkg/hub/server.go:1753` | +| `GET` | `/api/contribute/ws` | Hub handler-specific | Contribute WSProxy | `pkg/hub/server.go:1754` | +| `POST` | `/api/github/webhook` | Hub handler-specific | GitHub Webhook | `pkg/hub/server.go:1755` | +| `GET` | `/gh-setup` | Hub handler-specific | GitHub App Setup Router | `pkg/hub/server.go:1756` | +| `GET` | `/learn` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1757` | +| `GET` | `/get-started` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1758` | +| `GET` | `/api/docs` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1761` | +| `GET` | `/api/reading-list` | Hub handler-specific | Reading List | `pkg/hub/server.go:1764` | +| `GET` | `/reading` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1765` | +| `GET` | `/cncf-reference-architecture` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1772` | +| `GET` | `/cncf-reference-architecture/console` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1773` | +| `GET` | `/cncf-reference-architecture/bluefin` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1774` | +| `GET` | `/{$}` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1791` | +| `GET` | `/og-card.png` | Hub handler-specific | OGCard | `pkg/hub/server.go:1796` | +| `GET` | `/` | Public | Static asset fallback (`http.FileServerFS` over the embedded `static/` tree) for any path no other route claims | `pkg/hub/server.go:1798` | diff --git a/src/docs/api-reference/packs-and-acmm.md b/src/docs/api-reference/packs-and-acmm.md index 0c15c8d508..d0067822c6 100644 --- a/src/docs/api-reference/packs-and-acmm.md +++ b/src/docs/api-reference/packs-and-acmm.md @@ -25,7 +25,7 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `POST` | `/api/repos/{owner}/{repo}/items/{number}/decide-delegated/plan` | Same as `…/decide-delegated` | Preview for *Let hive decide*: runs the same owner-delegate policy with **no side effects** (nothing written, no job) and answers `{would, rule, acting, title, summary, why, consequences[], ladder, mergeable_as_is, alternatives[], writes: false, prior, notes}` — `summary` is one verb-first sentence with no "Hive would:" prefix (*Send #15332 back to its author with one instruction: make CI green, nothing else.*), `why` one short line (*CI is red.*), `consequences` at most two items, `ladder` the rung that decided with its `ruled_out` list, and `mergeable_as_is` whether an as-is merge is honest (the queue-merge and merge-now alternatives are dropped when it is not). The dialog shows **Cancel** / **Let hive do that**, plus **Approve and let hive merge** (or **Merge now as maintainer** when the hive would approve anyway) only when `mergeable_as_is`; only then does the SPA post `…/decide-delegated` with `chosen` naming the button. A transient GitHub failure (5xx, rate limit, timeout) is retried twice with backoff before the plan gives up. A failure answers `{ok: false, error, reason, retryable, attempts, elapsed_ms, hive, hint}` under **409** when the item is no longer open (`reason: not-open`) or **424** when a read it depends on failed (`github`), is not configured (`config`) or the plan itself failed (`plan`) — never 502/503, which the hosted Ingress intercepts and replaces with its generic *hive unavailable* page (hivecommons/hive#16776). See [dashboard-sections.md](../dashboard-sections.md#let-hive-decide). | `pkg/dashboard/api.go:356`, `pkg/dashboard/api_repo_item_decide_delegated_plan.go` | | `POST` | `/api/repos/{owner}/{repo}/items/{number}/blocker/recheck` | Same as `…/hold` (owner or repo write), signed-in GitHub user | *Re-check permissions* on an issue parked on a capability blocker (#15099): probes the GitHub App installation's permissions; when the missing permission is granted, clears every ledger entry blocked on it and un-parks the item (`{result: {outcome: "released", …}}`), otherwise `{result: {outcome: "no-effect", reason: "blocker", options: [...]}}`; 404 when the item has no blocker, 409 when the blocker is not a capability gap | `pkg/dashboard/api.go:357` | | `POST` | `/api/repos/{owner}/{repo}/items/{number}/blocker/close-partial` | Same as `…/hold` (owner or repo write), signed-in GitHub user | Option (c) on a capability blocker (#15099): labels the issue `hive/partially-addressed`, posts the operator's `{"comment"}` naming what remains and closes it; the ledger entry is cleared | `pkg/dashboard/api.go:358` | -| `POST` | `/api/repos/{owner}/{repo}/items/{number}/accept-triage` | Owner or repo write/maintain/admin | Accept triage on an issue from the activity monitor's row menu — what a maintainer accepting triage on GitHub does by hand: adds the reporter-trust triage label (`triage/accepted`, or the first of `issue_filter.reporter_trust.untrusted_require_labels`; the label is created in the repo if missing) and removes the awaiting label (`needs-triage` / `reporter_trust.awaiting_label`) and a Hive-written `stage/015-triage-needed`. Empty body. Returns `{ok, repo, number, acceptLabel, added, removed, alreadyAccepted, minStatusSeq}`; an issue already carrying the accept label with nothing left to remove is a no-op (`alreadyAccepted: true`, no write, no audit). 404 for a pull request or an unknown item. The status snapshot's labels are patched so the next reconcile reads 010 Actionable before the engine's own 015 → 010 derivation. Every write is audited as `repo_item_accept_triage` under the dashboard user. | `pkg/dashboard/api.go:554`, `pkg/dashboard/api_repo_item_accept_triage.go` | +| `POST` | `/api/repos/{owner}/{repo}/items/{number}/accept-triage` | Owner or repo write/maintain/admin | Accept triage on an issue from the activity monitor's row menu — what a maintainer accepting triage on GitHub does by hand: adds the reporter-trust triage label (`triage/accepted`, or the first of `issue_filter.reporter_trust.untrusted_require_labels`; the label is created in the repo if missing) and removes the awaiting label (`needs-triage` / `reporter_trust.awaiting_label`) and a Hive-written `stage/015-triage-needed`. Empty body. Returns `{ok, repo, number, acceptLabel, added, removed, alreadyAccepted, minStatusSeq}`; an issue already carrying the accept label with nothing left to remove is a no-op (`alreadyAccepted: true`, no write, no audit). 404 for a pull request or an unknown item. The status snapshot's labels are patched so the next reconcile reads 010 Actionable before the engine's own 015 → 010 derivation. Every write is audited as `repo_item_accept_triage` under the dashboard user. | `pkg/dashboard/api.go:555`, `pkg/dashboard/api_repo_item_accept_triage.go` | | `POST` | `/api/repos/{owner}/{repo}/items/{number}/sentinel-confirm` | Owner or repo write/maintain/admin | **Confirm safe** on the activity monitor's human-gate pill ([#17470](https://github.com/hivecommons/hive/issues/17470)): a person confirms a sentinel finding is intended. Body `{comment?, type?}` (`type` is `pr` or `issue`, comment ≤ 4000 characters). Posts the confirmation as a comment first — ``, `@user confirmed the sentinel finding safe from the dashboard.`, then the comment text — and removes every label matching the configured sentinel label (`sentinel.label`, default `sentinel-alert`, case-insensitive). Returns `{ok, repo, number, label, removed, alreadyClear, result}`; with no sentinel label on the item the comment is still posted and `alreadyClear` is true. 403 for a repository the hive does not watch. Audited as `repo_item_sentinel_confirm` under the dashboard user; the operator-action ledger records it as `confirm-sentinel`. | `pkg/dashboard/api.go:348`, `pkg/dashboard/api_repo_item_sentinel_confirm.go` | | `GET` | `/api/acmm/evaluation` | Dashboard auth/session | ACMMEvaluation — combined codebase + operational result, cached server-side for 1 hour (`acmmEvalTTL`). `?refresh=1` (the dashboard's "🔄 Re-evaluate" button, #5877) bypasses the hourly TTL but is debounced server-side: requests within 1 minute of the last evaluation (`acmmRefreshDebounce`) still serve the cache, since a full refresh costs up to ~29 GitHub GetContents calls per repo. The response's `last_evaluated_at` timestamp reports when the cached evaluation was computed | `pkg/dashboard/api.go:360`, `pkg/dashboard/api_acmm_eval.go` | | `POST` | `/api/acmm/issue` | Owner only | ACMMCreate Issue — files on GitHub or, with `governor.acmm.issue_tracker: work_source` / body `tracker: "work_source"` on a Linear-sourced hive, on Linear; response `tracker` says which. See [ACMM policy matrix](../acmm-policy-matrix.md#where-acmm-gap-issues-are-filed) | `pkg/dashboard/api.go:361` | diff --git a/src/docs/api-reference/snapshots-and-style.md b/src/docs/api-reference/snapshots-and-style.md index 49513410a0..8dae7348f7 100644 --- a/src/docs/api-reference/snapshots-and-style.md +++ b/src/docs/api-reference/snapshots-and-style.md @@ -6,7 +6,7 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | Method | Path | Auth | Purpose | Source | |---|---|---|---|---| | `GET` | `/api/style` | Public | Sanitized custom dashboard CSS | `pkg/dashboard/api.go:47` | -| `GET` | `/branding/custom.css` | Dashboard auth/session | Operator branding stylesheet override, read per request (see [branding](../branding.md)) | `pkg/dashboard/server.go:1745` | +| `GET` | `/branding/custom.css` | Dashboard auth/session | Operator branding stylesheet override, read per request (see [branding](../branding.md)) | `pkg/dashboard/server.go:1751` | | `GET` | `/api/snapshot/frame-ancestors` | Public | Snapshot framing allowlist | `pkg/dashboard/api.go:83` | | `GET` | `/api/snapshot` | Public | Snapshot data | `pkg/dashboard/api.go:84` | | `GET` | `/snapshot` | Public | Public read-only snapshot page | `pkg/dashboard/api.go:85` | diff --git a/src/docs/benchmarks.md b/src/docs/benchmarks.md new file mode 100644 index 0000000000..be9cf68661 --- /dev/null +++ b/src/docs/benchmarks.md @@ -0,0 +1,101 @@ +# Benchmarks: publishing per-PR agent/model/runtime outcomes + +Every hive already seals one [review evidence bundle](review-evidence.md) per +PR head. Those bundles record which agent and model opened the PR, which +backend (harness) ran it, how the review swarm scored it, whether CI was green +at merge, when it was opened and merged, and whether it was reverted. The +`benchmarks` feature turns that evidence into **anonymized per-PR benchmark +records** and publishes them to the public leaderboard at + (data repo +[`hivecommons/benchmarks`](https://github.com/hivecommons/benchmarks)), so the +fleet can compare models, harnesses and sandbox runtimes on real work rather +than on vendor evals. Design discussion: hivecommons/hive#17694. + +## What is published + +One record per PR per role, conforming to +[`schema/result.schema.json`](https://github.com/hivecommons/benchmarks/blob/main/schema/result.schema.json) +(vendored in `pkg/evidence/benchmarks/testdata/`): + +| Field | Source | +|---|---| +| `source.hive_id` | **Salted hash** of the hive id: `sha256("hivecommons-benchmarks/v1/" + hive_id)[:16]`. The hive id itself never leaves the spoke. | +| `source.hive_version`, `source.cluster` | Build version and `HIVE_CLUSTER_ID`. | +| `agent.name`, `agent.role` (`author` / `reviewer`), `agent.harness` | Bundle `author.agent`, `author.backend`; reviewer records come from the dominant `(backend, model)` among the swarm verdicts. | +| `model.provider`, `model.name` | Bundle `author.model`; the provider is inferred from the model name prefix. | +| `sandbox_runtime` | Stamped on new bundles from the agent's effective crux sandbox (`podman`, `kubernetes-job`, …) or `none` when agents run in-process. Old bundles fall back to the current config. | +| `task.kind` | Stamped from the agent's lane (`docs`, `security-fix`, `ci-repair`, `release`, `dependency-update`, `triage`, `refactor`), else `unknown`. Reviewer records are `pr-review`. | +| `task.acmm_level` | `policy.acmm_level` at review time. | +| `outcome.*` | `pr_opened`, `pr_merged`, `merged_at`, `time_to_merge_s`, `ci_status_at_merge`, `ci_green_first_try`, `review_verdict`, `reverted`, `closed_unmerged`, `human_interventions` (human approvals / label changes). | +| `quality.*` | Swarm verdict counts and confidence. | + +**What never leaves the spoke:** the hive id, repository and organization +names (unless the repository is public — see below), PR numbers, commit SHAs, +agent prompts, diff content, reviewer comments, human logins, and cost/token +figures (not present in bundles today). + +## Privacy model: default on, anonymized + +| `hive.yaml` key | Default | Effect | +|---|---|---| +| `benchmarks.publish` | `true` | Publish anonymized records (hashed hive id, no repo/org). Set `false` to **opt out** entirely: `/api/benchmarks/export` returns zero records and the hourly push to the hub stops. | +| `benchmarks.repo_consent` | `true` | Attach `task.repo` / `source.org` **only for repositories GitHub reports as public** (visibility is checked through the App client and cached for 24h; unknown ⇒ treated as private). Public-repo activity is already public, so this adds no new disclosure. Set `false` to anonymize even public repos. Private repos are never named regardless of this setting. | + +Both toggles are in **Settings → Features → Benchmarks** in the dashboard, +which also shows this hive's hash so an operator can find their row in the +leaderboard's per-hive table. + +## Cadence: hub-driven, hourly + +``` +spoke (/data/evidence) --hourly POST /api/benchmarks/ingest--> hub (/data/benchmarks//.json) + | + +--hourly Contents-API commit--> hivecommons/benchmarks data/fleet//.json + | + +--GitHub Pages rebuild on push--> hivecommons.dev/benchmarks +``` + +1. **Spoke → hub (push, hourly).** Three minutes after start and then every + hour, the spoke converts bundles updated since its persisted watermark + (`/data/benchmarks-publish.json`), caps a batch at 2000 records and POSTs + them to `/api/benchmarks/ingest` with the same per-hive heartbeat + bearer used for NPS and task-status pushes. The hub recomputes the hash + from the authenticated hive id and rejects any record claiming a different + `source.hive_id`, so one spoke cannot write into another's rows. Push was + chosen over hub-pull because the hub holds no credential for spokes and + most spoke dashboards are cluster-private. +2. **Hub → GitHub (hourly).** The hub merges records by id into month files + under `/data/benchmarks/`, marks changed files pending, and once an hour + (ten minutes after start, at most 60 files per run) writes them to + `hivecommons/benchmarks` at `data/fleet//.json` through the + Contents API using the hub's GitHub App identity (the installation needs + `contents: write` on the data repo). Identical content is skipped. +3. **Pages** rebuilds `data/index.json` on every push, so the site lags the + spoke by at most ~2 hours. + +Until the hub publisher is rolled out, the data repo's +`scripts/backfill-from-cluster.sh` pulls `/data/evidence` straight from spoke +pods and a nightly workflow rebuilds the index. + +## Endpoints + +| Endpoint | Where | Auth | Purpose | +|---|---|---|---| +| `GET /api/benchmarks/export?since=` | spoke | owner | Schema-valid records for bundles updated since `since` (default 90 days). Empty when `benchmarks.publish: false`. | +| `GET /api/status` → `benchmarks{}` | spoke | owner | `hive_hash`, `publish`, `repo_consent`, `last_push_at`, `last_push_count`, `total_pushed`, `last_error`, `watermark_at`, `site_url`. | +| `POST /api/benchmarks/ingest` | hub | per-hive heartbeat bearer | Spoke push target. | +| `GET /api/hub/benchmarks` | hub | public status | Last ingest/publish time, pending files, per-hive-hash record counts. | + +## Hub environment + +| Variable | Default | Purpose | +|---|---|---| +| `HIVE_BENCHMARKS_REPO` | `hivecommons/benchmarks` | Target data repo (`owner/name`). | +| `HIVE_BENCHMARKS_PUBLISH_DISABLED` | unset | Any value stops the GitHub publisher; ingest still stores records locally. | + +## Code map + +- `pkg/evidence/benchmarks` — record types, `HiveHash`, `Convert` (bundles → records), schema golden test. +- `pkg/dashboard/api_benchmarks.go` — export endpoint, status block, hourly pusher, Settings panel data. +- `pkg/hub/benchmarks.go` — ingest, month-file store, hourly GitHub publisher, status endpoint. +- `pkg/evidence` `Author.runtime` / `Author.task_kind` — stamped at bundle write via `github.EvidenceAgentContext`. diff --git a/src/docs/documentation-map.md b/src/docs/documentation-map.md index 590617f66f..64a28aa6f7 100644 --- a/src/docs/documentation-map.md +++ b/src/docs/documentation-map.md @@ -195,6 +195,7 @@ Documentation for the current Hive line (branch `v5`; the code and docs live und - [Review swarm](review-swarm.md) — the five review perspectives, the verdict collector and its report contract, and the opt-in merge-gate integration and bounded auto-fix cycle for review findings. - [Swarm mode](swarm.md) — focusing the hive on one repository for a bounded window: Discord announcements, theming (`swarm:` config, `HIVE_SWARM_EVENT_NAME` / `HIVE_SWARM_CALL_TO_ARMS`), the `/api/leaderboard/swarm` endpoint and scoring. - [Review evidence bundle](review-evidence.md) — the per-PR evidence bundle schema (v1), canonical hash, Ed25519 signing, offline verification, and retention design. +- [Benchmarks](benchmarks.md) — default-on, anonymized per-PR agent/model/runtime records (opt-out `benchmarks.publish`, public-repo attribution via `benchmarks.repo_consent`), the hourly spoke→hub→hivecommons/benchmarks cadence, and the `/api/benchmarks/export`, `/api/benchmarks/ingest` and `/api/hub/benchmarks` endpoints. - [Evidence reporting](evidence.md) — the guarded control-arm (`control_window: on|off`) evidence report: per-repository outcome comparison, the minimum-denominator rule, and why it is reporting groundwork rather than an enabled experiment. - [SOC 2 control mapping (not a compliance claim)](soc2-control-mapping.md) — maps common auditor change-management questions to bundle fields, with sample policy language and the gaps Hive does not cover. - [Duplicate PR sweep](duplicate-sweep.md) — **Opt-in; recommendations only:** identify overlapping PRs and suggest which to keep. Never closes them automatically. diff --git a/src/docs/operator-reference/README.md b/src/docs/operator-reference/README.md index 4e2ec10d20..35267a3a32 100644 --- a/src/docs/operator-reference/README.md +++ b/src/docs/operator-reference/README.md @@ -266,6 +266,8 @@ Go callers keep reading `cfg.Agents`, `cfg.Governor.Gateways` and so on through | `evidence.enabled` | On by default. | The review relay writes one [review evidence bundle](../review-evidence.md) per PR head to `/data/evidence////.json` when it records a verdict or posts a review; the sentinel sweep adds its findings to the flagged head's bundle; and every merge path adds the head's CI check-run summary, the human approvals and label changes, and the merge event. Owners and mergers read bundles through `GET /api/review/evidence` and `GET /api/review/evidence/list` (see the [API reference](../api-reference.md)). Set `false` to stop writing new bundles; existing ones are left in place. | | `evidence.signing_key_file` | Unset (bundles unsigned). | Path to an Ed25519 private key (32-byte seed or 64-byte key, hex or base64). When set and readable, bundles are signed. Dashboard/API readers require a valid signature under its public half and fail closed when the configured key is unavailable; unsigned mode verifies only the bundle content hash. Writers still record `"signed": false` when the key is empty, absent or unusable and log a warning for an unusable key, so an evidence write failure never blocks review or merge work. Keep the key off shared hosts. | | `evidence.complexity.enabled` | On by default (when `evidence.enabled` is on). | Once a month, for each managed repo, the hive downloads the default-branch tarball through the GitHub App and writes one Go cyclomatic-complexity snapshot to `/data/evidence///complexity/.json` (see [Monthly complexity snapshot](../review-evidence.md#monthly-complexity-snapshot-report-only)). Report-only: it never affects merge eligibility. Cheap by design — one archive download and an AST pass per repo per month, no module downloads and no compile. An existing month is never rewritten; the hive re-checks hourly and retries a failed repo on the next check. The last 24 months are kept per repo. Set `false` to stop writing snapshots; existing ones stay. | +| `benchmarks.publish` | **On by default.** | Hourly, the spoke converts review evidence bundles into anonymized per-PR benchmark records (salted hive hash, agent/harness/model/sandbox runtime/task kind/outcome/review quality — never the hive id, PR numbers, SHAs, prompts, diffs, logins or costs) and pushes them to the hub, which commits them hourly to [hivecommons/benchmarks](https://github.com/hivecommons/benchmarks) for . `GET /api/benchmarks/export?since=` serves the same records to the owner. Set `false` to opt out: export returns nothing and the push stops. See [Benchmarks](../benchmarks.md). | +| `benchmarks.repo_consent` | On by default. | Adds `task.repo` / `source.org` to records **only for repositories GitHub reports as public** (visibility checked through the App client, cached 24h, unknown ⇒ private). Private repos are never named. Set `false` to anonymize public repos too. | | `evidence.complexity.static_analysis` | `false`. | Also run `go vet` and (when installed) `staticcheck` on the snapshot checkout and record finding counts per tool. Off by default because it compiles each repo and downloads its modules, which is not cheap on a spoke with many or large repos. | | `review.all_authors` | Deprecated no-op; accepted and logged once. | Every open PR is reviewed regardless of author — hive agent, human, clanker, dependabot or any other bot — whenever review is on (`require_approval` and `fan_out`); there is no author-class opt-out below that switch ([#12974](https://github.com/hivecommons/hive/issues/12974)). The knob used to be required for human PRs. It never lets an agent push to those PRs (see the next row). Remove it from `hive.yaml`. | | `review.fix_human_prs` | Retained for compatibility; treated as push-disabled. | No longer lets review-fix kicks push commits onto PRs whose login author is not a hive agent/App, even when the PR is hive-attributed ([#13124](https://github.com/hivecommons/hive/issues/13124)); the one exception is a PR the stage reconciler adopted in place (`stage/074-adopted-by-hive`). A `changes_requested` verdict on such a PR stops at the published review, with the proposed fix as a suggestion or patch block in the comment, and the refusal is audited as `review_fix_withheld` with the setting's actual value. When the setting is on, the review-thread reconciler still answers review-bot threads on those PRs, by suggestion only, never by checkout or push ([#12897](https://github.com/hivecommons/hive/issues/12897)). Existing configs may still store `true` for UI/audit continuity, but DCO squash attribution means Hive cannot add a compliant follow-up commit without forging the PR author's sign-off. See [review-swarm.md](../review-swarm.md#who-may-be-pushed-to-all_authors-and-the-retired-fix_human_prs-push). | diff --git a/src/pkg/config/benchmarks.go b/src/pkg/config/benchmarks.go new file mode 100644 index 0000000000..ec82f5552e --- /dev/null +++ b/src/pkg/config/benchmarks.go @@ -0,0 +1,60 @@ +package config + +import ( + "strings" + + cruxconfig "github.com/hivecommons/hive/crux/config" +) + +// BenchmarksConfig (top-level `benchmarks:`) governs what this hive +// contributes to the fleet leaderboard at https://hivecommons.dev/benchmarks/ +// (RFC hivecommons/hive#17694, docs/benchmarks.md). +// +// Default ON, anonymized: every hive publishes per-PR records keyed by a +// salted hash of its hive id — model, harness, sandbox runtime, outcome, +// time-to-merge and ACMM level; never a login, prompt, finding or code. +// Repository and org names ride along only for PUBLIC repositories, whose +// activity is already public. Both knobs are opt-outs. +type BenchmarksConfig struct { + // Publish false stops the spoke exporting records to the hub (and empties + // GET /api/benchmarks/export). nil means on. + Publish *bool `yaml:"publish,omitempty" json:"publish,omitempty"` + // RepoConsent false anonymizes even public repositories: task.repo and + // source.org are dropped from every record. nil means public repos are + // named. + RepoConsent *bool `yaml:"repo_consent,omitempty" json:"repo_consent,omitempty"` +} + +// PublishEnabled reports whether records leave this hive. +func (b BenchmarksConfig) PublishEnabled() bool { + return b.Publish == nil || *b.Publish +} + +// RepoConsentEnabled reports whether public repositories may be named. +func (b BenchmarksConfig) RepoConsentEnabled() bool { + return b.RepoConsent == nil || *b.RepoConsent +} + +// BenchmarkSandboxRuntimeNone is the runtime published for in-process agents. +const BenchmarkSandboxRuntimeNone = "none" + +// BenchmarkSandboxRuntime is the sandbox runtime an agent's kicks run under +// as the benchmark schema names it: the crux runtime when sandboxing is on +// for that agent, else "none". +func (c *Config) BenchmarkSandboxRuntime(agent string) string { + if c == nil { + return BenchmarkSandboxRuntimeNone + } + var ac *cruxconfig.AgentConfig + if a, ok := c.Agents[agent]; ok { + ac = &a + } + enabled := c.AgentSandbox.Enabled + if ac != nil && ac.Sandbox != nil && ac.Sandbox.Enabled != nil { + enabled = *ac.Sandbox.Enabled + } + if !enabled { + return BenchmarkSandboxRuntimeNone + } + return strings.TrimSpace(ac.SandboxRuntime(c.AgentSandbox)) +} diff --git a/src/pkg/config/benchmarks_test.go b/src/pkg/config/benchmarks_test.go new file mode 100644 index 0000000000..e660c6ae6d --- /dev/null +++ b/src/pkg/config/benchmarks_test.go @@ -0,0 +1,52 @@ +package config + +import ( + "testing" + + cruxconfig "github.com/hivecommons/hive/crux/config" +) + +func TestBenchmarksConfigDefaultsOn(t *testing.T) { + var b BenchmarksConfig + if !b.PublishEnabled() || !b.RepoConsentEnabled() { + t.Fatal("unset benchmarks block must publish and name public repos") + } + off := false + b = BenchmarksConfig{Publish: &off, RepoConsent: &off} + if b.PublishEnabled() || b.RepoConsentEnabled() { + t.Fatal("explicit false must opt out") + } +} + +func TestBenchmarkSandboxRuntime(t *testing.T) { + var nilCfg *Config + if got := nilCfg.BenchmarkSandboxRuntime("a"); got != BenchmarkSandboxRuntimeNone { + t.Fatalf("nil config = %q", got) + } + on, off := true, false + cfg := &Config{} + cfg.AgentSandbox = cruxconfig.AgentSandboxConfig{Enabled: false, Runtime: "kubernetes-job"} + cfg.Agents = map[string]cruxconfig.AgentConfig{ + "opted-in": {Sandbox: &cruxconfig.AgentSandboxOverride{Enabled: &on, Runtime: " openshell "}}, + "inherits": {Sandbox: &cruxconfig.AgentSandboxOverride{Enabled: &on}}, + "opted-out": {Sandbox: &cruxconfig.AgentSandboxOverride{Enabled: &off}}, + } + for agent, want := range map[string]string{ + "opted-in": "openshell", + "inherits": "kubernetes-job", + "opted-out": BenchmarkSandboxRuntimeNone, + "unknown": BenchmarkSandboxRuntimeNone, // global sandbox off + } { + if got := cfg.BenchmarkSandboxRuntime(agent); got != want { + t.Errorf("%s: got %q want %q", agent, got, want) + } + } + cfg.AgentSandbox.Enabled = true + if got := cfg.BenchmarkSandboxRuntime("unknown"); got != "kubernetes-job" { + t.Errorf("global on, unlisted agent: got %q", got) + } + cfg.AgentSandbox.Runtime = "" + if got := cfg.BenchmarkSandboxRuntime("unknown"); got != cruxconfig.SandboxRuntimePodman { + t.Errorf("default runtime: got %q", got) + } +} diff --git a/src/pkg/config/config.go b/src/pkg/config/config.go index 0e18cf591c..810c5256dc 100644 --- a/src/pkg/config/config.go +++ b/src/pkg/config/config.go @@ -159,6 +159,9 @@ type Config struct { // Evidence controls the review evidence bundles the review relay writes // (hivecommons/hive#11060). Default on, unsigned; see EvidenceConfig. Evidence EvidenceConfig `yaml:"evidence,omitempty" json:"evidence,omitempty"` + // Benchmarks governs the anonymized per-PR records this hive contributes + // to hivecommons/benchmarks (hivecommons/hive#17694); see BenchmarksConfig. + Benchmarks BenchmarksConfig `yaml:"benchmarks,omitempty" json:"benchmarks,omitempty"` // Compliance selects framework profiles; see ComplianceConfig. Compliance ComplianceConfig `yaml:"compliance,omitempty" json:"compliance,omitempty"` Escalation EscalationConfig `yaml:"escalation,omitempty" json:"escalation,omitempty"` diff --git a/src/pkg/config/testdata/agent_runtime_golden.json b/src/pkg/config/testdata/agent_runtime_golden.json index b5c0896436..891512459a 100644 --- a/src/pkg/config/testdata/agent_runtime_golden.json +++ b/src/pkg/config/testdata/agent_runtime_golden.json @@ -710,6 +710,7 @@ "evidence": { "complexity": {} }, + "benchmarks": {}, "compliance": { "posture_checks": {} }, diff --git a/src/pkg/dashboard/api.go b/src/pkg/dashboard/api.go index 3f540af75c..52c63d7fa0 100644 --- a/src/pkg/dashboard/api.go +++ b/src/pkg/dashboard/api.go @@ -531,6 +531,7 @@ func (s *Server) RegisterAPI(deps *Dependencies) { // Evidence summary (#12063): read-only aggregate over the bundles; never // consulted by any merge decision. s.mux.HandleFunc("GET /api/evidence/summary", s.handleEvidenceSummary) + s.mux.HandleFunc("GET /api/benchmarks/export", s.handleBenchmarksExport) // write_surface is a top-level Config field; the lane write allowlist // editor lives on the governor Security tab (#9587, api_config_write_surface.go). s.mux.HandleFunc("GET /api/config/write-surface", s.handleWriteSurfaceGet) diff --git a/src/pkg/dashboard/api_benchmarks.go b/src/pkg/dashboard/api_benchmarks.go new file mode 100644 index 0000000000..ae05bddc3c --- /dev/null +++ b/src/pkg/dashboard/api_benchmarks.go @@ -0,0 +1,377 @@ +package dashboard + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "log/slog" + "net/http" + "os" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/evidence/benchmarks" + ghpkg "github.com/hivecommons/hive/pkg/github" + "github.com/hivecommons/hive/pkg/hub/spoke" +) + +// Benchmark publication (hivecommons/hive#17694). The spoke converts its +// review evidence bundles into anonymized leaderboard records +// (pkg/benchmarks) and, hourly, pushes the ones newer than the hub's last +// acknowledgement to POST /api/benchmarks/ingest on the hub — the same +// spoke→hub bearer pattern as NPS and task-status, chosen over a hub pull +// because the hub holds no bearer for spokes and most spoke URLs are +// cluster-private. The hub batches and commits to hivecommons/benchmarks. + +const ( + // benchmarksPushInterval is the spoke→hub cadence; the hub publishes + // hourly too, so the public site lags the fleet by ≤ ~2 h. + benchmarksPushInterval = time.Hour + // benchmarksPushInitialDelay lets the evidence walk and the heartbeat + // registration settle before the first push. + benchmarksPushInitialDelay = 3 * time.Minute + // benchmarksPushTimeout bounds one hub POST. + benchmarksPushTimeout = 60 * time.Second + // benchmarksPushMaxRecords caps one batch; the rest go next hour. + benchmarksPushMaxRecords = 2000 + // benchmarksHubIngestPath is the hub route. + benchmarksHubIngestPath = "/api/benchmarks/ingest" + // benchmarksRepoVisibilityTTL bounds how long a repo's public/private + // answer is reused. + benchmarksRepoVisibilityTTL = 24 * time.Hour + // benchmarksExportMaxSinceDays bounds ?since= so one request cannot + // re-emit years of history on a hot path. + benchmarksExportDefaultWindow = 90 * 24 * time.Hour +) + +// benchmarksPublishStatePath persists the last hub acknowledgement so a +// restart does not resend history. A var for tests. +var benchmarksPublishStatePath = filepath.Join(ghpkg.ReviewLinksDir, "benchmarks-publish.json") + +// BenchmarksStatus is the /api/status block and the Settings → Benchmarks +// panel source. +type BenchmarksStatus struct { + HiveHash string `json:"hive_hash"` + Publish bool `json:"publish"` + RepoConsent bool `json:"repo_consent"` + LastPushAt string `json:"last_push_at,omitempty"` + LastPushCount int `json:"last_push_count"` + TotalPushed int `json:"total_pushed"` + LastError string `json:"last_error,omitempty"` + WatermarkAt string `json:"watermark_at,omitempty"` + SiteURL string `json:"site_url"` +} + +// benchmarksSiteURL is where the records end up. +const benchmarksSiteURL = "https://hivecommons.dev/benchmarks/" + +type benchmarksPublishState struct { + // Watermark is the newest bundle UpdatedAt the hub has acknowledged. + Watermark time.Time `json:"watermark"` + LastPushAt time.Time `json:"last_push_at"` + LastCount int `json:"last_count"` + TotalPushed int `json:"total_pushed"` + LastError string `json:"last_error,omitempty"` +} + +type benchmarksPublisher struct { + mu sync.Mutex + state benchmarksPublishState + vis map[string]benchmarksVisibility +} + +type benchmarksVisibility struct { + public bool + at time.Time +} + +func (s *Server) benchmarks() *benchmarksPublisher { + s.benchmarksOnce.Do(func() { + p := &benchmarksPublisher{vis: map[string]benchmarksVisibility{}} + if data, err := os.ReadFile(benchmarksPublishStatePath); err == nil { + _ = json.Unmarshal(data, &p.state) + } + s.benchmarksPub = p + }) + return s.benchmarksPub +} + +func (p *benchmarksPublisher) save() { + data, err := json.Marshal(p.state) + if err != nil { + return + } + _ = os.MkdirAll(filepath.Dir(benchmarksPublishStatePath), 0o755) + _ = os.WriteFile(benchmarksPublishStatePath, data, 0o644) +} + +// repoPublic answers through the hive's GitHub client with a daily cache. +// Unknown (API error, no client) is private: the record stays anonymous. +func (s *Server) benchmarksRepoPublic(ctx context.Context, repo string) bool { + p := s.benchmarks() + now := time.Now() + p.mu.Lock() + if v, ok := p.vis[repo]; ok && now.Sub(v.at) < benchmarksRepoVisibilityTTL { + p.mu.Unlock() + return v.public + } + p.mu.Unlock() + public := false + if s.deps != nil && s.deps.GHClient != nil { + if owner, name, ok := strings.Cut(repo, "/"); ok { + rctx, cancel := context.WithTimeout(ctx, 10*time.Second) + if r, _, err := s.deps.GHClient.GetRepo(rctx, owner, name); err == nil && r != nil { + public = !r.GetPrivate() + } + cancel() + } + } + p.mu.Lock() + p.vis[repo] = benchmarksVisibility{public: public, at: now} + p.mu.Unlock() + return public +} + +// benchmarksOptions builds the converter options from live config. +func (s *Server) benchmarksOptions(ctx context.Context, since time.Time) benchmarks.Options { + cfg := s.deps.Config + release, _ := fleetReportBuildInfo() + return benchmarks.Options{ + HiveID: cfg.HiveID, + HiveVersion: release, + Since: since, + RepoConsent: cfg.Benchmarks.RepoConsentEnabled(), + RepoPublic: func(repo string) bool { return s.benchmarksRepoPublic(ctx, repo) }, + SandboxRuntime: cfg.BenchmarkSandboxRuntime, + } +} + +// benchmarksRecords converts every bundle updated at/after since. +func (s *Server) benchmarksRecords(ctx context.Context, since time.Time) ([]benchmarks.Record, time.Time, error) { + verifier, err := s.reviewEvidenceVerifier() + if err != nil { + return nil, time.Time{}, err + } + bundles, _, err := loadEvidenceBundles(ghpkg.ReviewEvidenceRoot, verifier) + if err != nil { + return nil, time.Time{}, err + } + var newest time.Time + for _, b := range bundles { + if b != nil && b.UpdatedAt.After(newest) { + newest = b.UpdatedAt + } + } + return benchmarks.Convert(bundles, s.benchmarksOptions(ctx, since)), newest, nil +} + +// handleBenchmarksExport serves GET /api/benchmarks/export?since=. +// Owner-authenticated like every /api route. Returns no records when +// benchmarks.publish is false so the endpoint and the hub agree. +func (s *Server) handleBenchmarksExport(w http.ResponseWriter, r *http.Request) { + if s.deps == nil || s.deps.Config == nil { + jsonError(w, "config not loaded", http.StatusServiceUnavailable) + return + } + cfg := s.deps.Config + since := time.Now().Add(-benchmarksExportDefaultWindow) + if raw := strings.TrimSpace(r.URL.Query().Get("since")); raw != "" { + t, err := time.Parse(time.RFC3339, raw) + if err != nil { + t, err = time.Parse("2006-01-02", raw) + } + if err != nil { + jsonError(w, "since must be RFC3339 or YYYY-MM-DD", http.StatusBadRequest) + return + } + since = t + } + resp := map[string]any{ + "hive_hash": benchmarks.HiveHash(cfg.HiveID), + "generated_at": time.Now().UTC().Format(time.RFC3339), + "publish": cfg.Benchmarks.PublishEnabled(), + "since": since.UTC().Format(time.RFC3339), + "records": []benchmarks.Record{}, + "count": 0, + } + if cfg.Benchmarks.PublishEnabled() { + recs, _, err := s.benchmarksRecords(r.Context(), since) + if err != nil && !errors.Is(err, os.ErrNotExist) { + jsonError(w, "load evidence: "+err.Error(), http.StatusInternalServerError) + return + } + if recs == nil { + recs = []benchmarks.Record{} + } + resp["records"] = recs + resp["count"] = len(recs) + } + jsonResponse(w, resp) +} + +// BenchmarksStatusBlock is the /api/status view. +func (s *Server) BenchmarksStatusBlock() *BenchmarksStatus { + if s.deps == nil || s.deps.Config == nil { + return nil + } + cfg := s.deps.Config + p := s.benchmarks() + p.mu.Lock() + st := p.state + p.mu.Unlock() + out := &BenchmarksStatus{ + HiveHash: benchmarks.HiveHash(cfg.HiveID), + Publish: cfg.Benchmarks.PublishEnabled(), + RepoConsent: cfg.Benchmarks.RepoConsentEnabled(), + LastPushCount: st.LastCount, + TotalPushed: st.TotalPushed, + LastError: st.LastError, + SiteURL: benchmarksSiteURL, + } + if !st.LastPushAt.IsZero() { + out.LastPushAt = st.LastPushAt.UTC().Format(time.RFC3339) + } + if !st.Watermark.IsZero() { + out.WatermarkAt = st.Watermark.UTC().Format(time.RFC3339) + } + return out +} + +// StartBenchmarksPublisher runs the hourly spoke→hub push until ctx ends. +// It returns at once when there is no hub, no hive id or no hub credential. +func (s *Server) StartBenchmarksPublisher(ctx context.Context) { + if s.deps == nil || s.deps.Config == nil { + return + } + cfg := s.deps.Config + hubURL := strings.TrimRight(strings.TrimSpace(cfg.Hub.URL), "/") + if hubURL == "" || strings.TrimSpace(cfg.HiveID) == "" { + return + } + logger := s.logger + if logger == nil { + logger = slog.Default() + } + go func() { + select { + case <-ctx.Done(): + return + case <-time.After(benchmarksPushInitialDelay): + } + ticker := time.NewTicker(benchmarksPushInterval) + defer ticker.Stop() + for { + s.pushBenchmarks(ctx, hubURL, logger) + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + } + }() +} + +// benchmarksIngestPayload is the body of POST /api/benchmarks/ingest. +type benchmarksIngestPayload struct { + HiveID string `json:"hive_id"` + HiveHash string `json:"hive_hash"` + Watermark string `json:"watermark"` + Records []benchmarks.Record `json:"records"` +} + +// pushBenchmarks sends one batch and advances the watermark on 2xx. +func (s *Server) pushBenchmarks(ctx context.Context, hubURL string, logger *slog.Logger) { + cfg := s.deps.Config + p := s.benchmarks() + if !cfg.Benchmarks.PublishEnabled() { + return + } + bearer := spoke.SpokeHeartbeatKey() + if bearer == "" { + return + } + p.mu.Lock() + since := p.state.Watermark + p.mu.Unlock() + recs, newest, err := s.benchmarksRecords(ctx, since) + if err != nil { + if !errors.Is(err, os.ErrNotExist) { + s.benchmarksFail(p, "load evidence: "+err.Error()) + } + return + } + if len(recs) == 0 { + return + } + if len(recs) > benchmarksPushMaxRecords { + recs = recs[:benchmarksPushMaxRecords] + newest = since // partial batch: keep the watermark so the rest follows + } + body, err := json.Marshal(benchmarksIngestPayload{ + HiveID: cfg.HiveID, + HiveHash: benchmarks.HiveHash(cfg.HiveID), + Watermark: newest.UTC().Format(time.RFC3339), + Records: recs, + }) + if err != nil { + return + } + reqCtx, cancel := context.WithTimeout(ctx, benchmarksPushTimeout) + defer cancel() + req, err := http.NewRequestWithContext(reqCtx, http.MethodPost, hubURL+benchmarksHubIngestPath, bytes.NewReader(body)) + if err != nil { + return + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+bearer) + resp, err := http.DefaultClient.Do(req) + if err != nil { + s.benchmarksFail(p, "post: "+err.Error()) + return + } + defer closeHTTPBody(resp.Body) + if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { + s.benchmarksFail(p, "hub answered "+resp.Status) + return + } + p.mu.Lock() + if newest.After(p.state.Watermark) { + p.state.Watermark = newest + } + p.state.LastPushAt = time.Now() + p.state.LastCount = len(recs) + p.state.TotalPushed += len(recs) + p.state.LastError = "" + p.save() + p.mu.Unlock() + logger.Info("benchmarks: pushed records to hub", "count", len(recs), "watermark", newest.UTC().Format(time.RFC3339)) +} + +func (s *Server) benchmarksFail(p *benchmarksPublisher, msg string) { + p.mu.Lock() + p.state.LastError = msg + p.save() + p.mu.Unlock() + if s.logger != nil { + s.logger.Warn("benchmarks: push failed", "error", msg) + } +} + +// featuresBenchmarksResponse is the Settings → Features → Benchmarks panel +// source (config_features.js). +func (s *Server) featuresBenchmarksResponse(cfg *config.Config) map[string]interface{} { + out := map[string]interface{}{ + "benchmarksPublish": cfg.Benchmarks.PublishEnabled(), + "benchmarksRepoConsent": cfg.Benchmarks.RepoConsentEnabled(), + "benchmarksHiveHash": benchmarks.HiveHash(cfg.HiveID), + } + if st := s.BenchmarksStatusBlock(); st != nil { + out["benchmarksLastPushAt"] = st.LastPushAt + out["benchmarksLastPushCount"] = st.LastPushCount + } + return out +} diff --git a/src/pkg/dashboard/api_benchmarks_test.go b/src/pkg/dashboard/api_benchmarks_test.go new file mode 100644 index 0000000000..c5f8589a8d --- /dev/null +++ b/src/pkg/dashboard/api_benchmarks_test.go @@ -0,0 +1,167 @@ +package dashboard + +import ( + "encoding/json" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/evidence" + "github.com/hivecommons/hive/pkg/evidence/benchmarks" + ghpkg "github.com/hivecommons/hive/pkg/github" + "github.com/hivecommons/hive/pkg/hub/spoke" +) + +func writeBenchmarkFixtureBundle(t *testing.T, root string, b *evidence.Bundle) { + t.Helper() + if err := evidence.Seal(b, nil); err != nil { + t.Fatal(err) + } + dir := filepath.Join(root, filepath.FromSlash(b.Repo), "1") + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + data, _ := json.Marshal(b) + if err := os.WriteFile(filepath.Join(dir, b.HeadSHA+".json"), data, 0o644); err != nil { + t.Fatal(err) + } +} + +func benchmarksTestServer(t *testing.T, publish bool) *Server { + t.Helper() + root := t.TempDir() + prev := ghpkg.ReviewEvidenceRoot + ghpkg.ReviewEvidenceRoot = root + t.Cleanup(func() { ghpkg.ReviewEvidenceRoot = prev }) + now := time.Now().UTC().Add(-time.Hour) + writeBenchmarkFixtureBundle(t, root, &evidence.Bundle{ + SchemaVersion: evidence.SchemaVersion, Repo: "o/r", Number: 1, HeadSHA: "abc1234", + Author: evidence.Author{Login: "bot", Kind: evidence.AuthorAgent, Model: "claude-fable-5.1", Backend: "copilot", Runtime: "podman", TaskKind: "docs"}, + CreatedAt: now, UpdatedAt: now, + Merge: &evidence.MergeEvent{Actor: "hive", Method: "squash", At: now.Add(30 * time.Minute), OpenedAt: now}, + }) + s := newTestServer() + s.deps = &Dependencies{Config: &config.Config{HiveID: "h1", Benchmarks: config.BenchmarksConfig{Publish: &publish}}} + return s +} + +func TestBenchmarksExportEmitsSchemaRecordsAndHonoursSince(t *testing.T) { + s := benchmarksTestServer(t, true) + rec := httptest.NewRecorder() + s.handleBenchmarksExport(rec, httptest.NewRequest(http.MethodGet, "/api/benchmarks/export", nil)) + if rec.Code != http.StatusOK { + t.Fatalf("code=%d body=%s", rec.Code, rec.Body.String()) + } + var resp struct { + HiveHash string `json:"hive_hash"` + Count int `json:"count"` + Records []benchmarks.Record `json:"records"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil { + t.Fatal(err) + } + if resp.HiveHash != benchmarks.HiveHash("h1") || resp.Count != 1 { + t.Fatalf("resp=%+v", resp) + } + r := resp.Records[0] + if r.Source.HiveID != resp.HiveHash || r.Model.Name != "claude-fable-5.1" || r.SandboxRuntime != "podman" || r.Task.Kind != "docs" || !r.Outcome.PRMerged { + t.Errorf("record=%+v", r) + } + if r.Task.Repo != "" { + t.Errorf("repo of unknown visibility must be anonymized, got %q", r.Task.Repo) + } + + rec = httptest.NewRecorder() + s.handleBenchmarksExport(rec, httptest.NewRequest(http.MethodGet, "/api/benchmarks/export?since="+time.Now().UTC().Format(time.RFC3339), nil)) + if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil || resp.Count != 0 { + t.Errorf("since in the future should yield 0 records: count=%d err=%v", resp.Count, err) + } + rec = httptest.NewRecorder() + s.handleBenchmarksExport(rec, httptest.NewRequest(http.MethodGet, "/api/benchmarks/export?since=yesterday", nil)) + if rec.Code != http.StatusBadRequest { + t.Errorf("bad since: code=%d", rec.Code) + } +} + +func TestBenchmarksExportRespectsPublishOptOut(t *testing.T) { + s := benchmarksTestServer(t, false) + rec := httptest.NewRecorder() + s.handleBenchmarksExport(rec, httptest.NewRequest(http.MethodGet, "/api/benchmarks/export", nil)) + var resp struct { + Publish bool `json:"publish"` + Count int `json:"count"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil || rec.Code != http.StatusOK { + t.Fatalf("code=%d err=%v", rec.Code, err) + } + if resp.Publish || resp.Count != 0 { + t.Errorf("opt-out must export nothing: %+v", resp) + } + if st := s.BenchmarksStatusBlock(); st == nil || st.Publish || st.HiveHash != benchmarks.HiveHash("h1") { + t.Errorf("status block = %+v", st) + } +} + +func TestPushBenchmarksAdvancesWatermarkAndRecordsFailures(t *testing.T) { + s := benchmarksTestServer(t, true) + prevState := benchmarksPublishStatePath + benchmarksPublishStatePath = filepath.Join(t.TempDir(), "benchmarks-publish.json") + t.Cleanup(func() { benchmarksPublishStatePath = prevState }) + t.Setenv(spoke.EnvHeartbeatKey, "spoke-key") + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + + var got struct { + HiveID string `json:"hive_id"` + Records []benchmarks.Record `json:"records"` + } + var bearer string + status := http.StatusAccepted + hub := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + bearer = r.Header.Get("Authorization") + _ = json.NewDecoder(r.Body).Decode(&got) + w.WriteHeader(status) + })) + defer hub.Close() + + s.pushBenchmarks(t.Context(), hub.URL, logger) + if bearer != "Bearer spoke-key" || got.HiveID != "h1" || len(got.Records) != 1 { + t.Fatalf("push: bearer=%q hive=%q n=%d", bearer, got.HiveID, len(got.Records)) + } + st := s.BenchmarksStatusBlock() + if st.TotalPushed != 1 || st.LastPushAt == "" || st.WatermarkAt == "" || st.LastError != "" { + t.Fatalf("status after push = %+v", st) + } + if _, err := os.Stat(benchmarksPublishStatePath); err != nil { + t.Fatalf("watermark not persisted: %v", err) + } + + // Nothing new since the watermark: no second request. + got.Records = nil + s.pushBenchmarks(t.Context(), hub.URL, logger) + if got.Records != nil { + t.Fatal("pushed again with nothing newer than the watermark") + } + + // A hub error is surfaced and keeps the watermark where it was. + s.benchmarks().state.Watermark = time.Time{} + status = http.StatusInternalServerError + s.pushBenchmarks(t.Context(), hub.URL, logger) + if st := s.BenchmarksStatusBlock(); st.LastError == "" || st.TotalPushed != 1 { + t.Fatalf("failure not recorded: %+v", st) + } + + // Opt-out short-circuits before any request. + off := false + s.deps.Config.Benchmarks.Publish = &off + bearer = "" + s.pushBenchmarks(t.Context(), hub.URL, logger) + if bearer != "" { + t.Fatal("publish:false must not push") + } +} diff --git a/src/pkg/dashboard/api_governor_features.go b/src/pkg/dashboard/api_governor_features.go index 9b664a4c90..3dfece4a37 100644 --- a/src/pkg/dashboard/api_governor_features.go +++ b/src/pkg/dashboard/api_governor_features.go @@ -497,6 +497,9 @@ func (s *Server) featuresLinkedView() map[string]interface{} { // which is what the governor config GET serves. func (s *Server) featuresSectionWithLinked(cfg *config.Config) map[string]interface{} { out := featuresSectionResponse(cfg) + for k, v := range s.featuresBenchmarksResponse(cfg) { + out[k] = v + } for k, v := range s.featuresLinkedView() { out[k] = v } diff --git a/src/pkg/dashboard/api_governor_features_platform.go b/src/pkg/dashboard/api_governor_features_platform.go index 5fe2a5b1c8..1d8c865d73 100644 --- a/src/pkg/dashboard/api_governor_features_platform.go +++ b/src/pkg/dashboard/api_governor_features_platform.go @@ -36,6 +36,9 @@ type featuresPlatformBody struct { EvidenceEnabled *bool `json:"evidenceEnabled"` + BenchmarksPublish *bool `json:"benchmarksPublish"` + BenchmarksRepoConsent *bool `json:"benchmarksRepoConsent"` + GiteaURL *string `json:"giteaUrl"` GiteaTokenEnv *string `json:"giteaTokenEnv"` GitLabURL *string `json:"gitlabUrl"` @@ -174,6 +177,14 @@ func (b featuresPlatformBody) prepare() (func(*config.Config), error) { v := *b.EvidenceEnabled steps = append(steps, func(c *config.Config) { c.Evidence.Enabled = &v }) } + if b.BenchmarksPublish != nil { + v := *b.BenchmarksPublish + steps = append(steps, func(c *config.Config) { c.Benchmarks.Publish = &v }) + } + if b.BenchmarksRepoConsent != nil { + v := *b.BenchmarksRepoConsent + steps = append(steps, func(c *config.Config) { c.Benchmarks.RepoConsent = &v }) + } for _, f := range []struct { raw *string diff --git a/src/pkg/dashboard/server.go b/src/pkg/dashboard/server.go index 0787453285..233bebed05 100644 --- a/src/pkg/dashboard/server.go +++ b/src/pkg/dashboard/server.go @@ -139,6 +139,12 @@ type Server struct { logger *slog.Logger mux *http.ServeMux deps *Dependencies + + // benchmarksPub is the hourly hivecommons/benchmarks publisher state + // (api_benchmarks.go), built lazily by benchmarks(). + benchmarksOnce sync.Once + benchmarksPub *benchmarksPublisher + // stageStore is the stage-label memory /api/stage-tracker reads; nil // means the process-wide stagelabels.Shared(). Tests inject one. stageStore *stagelabels.Store @@ -1782,6 +1788,8 @@ func (s *Server) Start() error { followCtx, cancelFollow := context.WithCancel(context.Background()) defer cancelFollow() s.StartRebaseMergeFollowThrough(followCtx) + // Hourly anonymized benchmark push to the hub (hivecommons/hive#17694). + s.StartBenchmarksPublisher(followCtx) return srv.Serve(ln) } @@ -3763,6 +3771,7 @@ func (s *Server) handleStatus(w http.ResponseWriter, r *http.Request) { payload.UpgradeProgress = s.upgradeProgress(status, now) payload.Writeback = s.WritebackStatus() payload.Hotspots = s.HotspotStatusBlock() + payload.Benchmarks = s.BenchmarksStatusBlock() payload.Config = s.configHealthStatus() if s.deps != nil { payload.WorkflowApprovals = s.deps.GHClient.WorkflowApprovalsStatus() @@ -3800,6 +3809,9 @@ type statusWithWebhooks struct { // WorkflowApprovals counts the hive's approvals of GitHub-gated // workflow runs (hivecommons/hive#16603). WorkflowApprovals github.WorkflowApprovalsStatus `json:"workflow_approvals"` + // Benchmarks is this hive's leaderboard hash and last publish to + // hivecommons/benchmarks (hivecommons/hive#17694). + Benchmarks *BenchmarksStatus `json:"benchmarks,omitempty"` *EvalLoopStatus // MatrixEnabled reports whether the Matrix block is active (enabled with // at least one repo, see config.MatrixConfig.Active) so the header shows diff --git a/src/pkg/dashboard/static-src/js/config-features.js b/src/pkg/dashboard/static-src/js/config-features.js index f294e2d203..4677f288de 100644 --- a/src/pkg/dashboard/static-src/js/config-features.js +++ b/src/pkg/dashboard/static-src/js/config-features.js @@ -633,6 +633,20 @@
Write evidence bundles ${yamlHint('evidence.enabled')} ${settingHelpMark('settings', 'features', 'Write evidence bundles', `On by default. Off stops writing review evidence bundles. Applies to the next review.`)} +
Benchmarks ${settingHelpMark('settings', 'features', 'Benchmarks', `This hive contributes anonymized per-PR records (model, harness, sandbox runtime, outcome, time-to-merge, ACMM level) to the fleet leaderboard at hivecommons.dev/benchmarks. No logins, prompts, findings or code ever leave the hive; the hive is identified only by the hash below. Repository names are included only for public repositories.`)}
+
+
+ Publish anonymized benchmark records ${yamlHint('benchmarks.publish')} ${settingHelpMark('settings', 'features', 'Publish benchmark records', `On by default. Off stops the hourly push to the hub and empties GET /api/benchmarks/export.`)} +
+
+
+ Name public repositories ${yamlHint('benchmarks.repo_consent')} ${settingHelpMark('settings', 'features', 'Name public repositories', `On by default: records for PUBLIC repositories carry the repository and org name, since that activity is already public. Off anonymizes every record. Private repositories are never named.`)} +
+
Look this hash up in the "Contributing hives" table at hivecommons.dev/benchmarks to verify your records arrived${f.benchmarksLastPushAt ? ` · last pushed ${esc(f.benchmarksLastPushAt)} (${esc(String(f.benchmarksLastPushCount || 0))} records)` : ' · nothing pushed yet'}.
+ + +
+
Review evidence signing
Read-only here: signing keys are provisioned on disk and set in hive.yaml.
`)} diff --git a/src/pkg/dashboard/static-src/js/config-state.js b/src/pkg/dashboard/static-src/js/config-state.js index 95017b5b64..6bb1f4d866 100644 --- a/src/pkg/dashboard/static-src/js/config-state.js +++ b/src/pkg/dashboard/static-src/js/config-state.js @@ -109,6 +109,7 @@ { tab: 'Features', section: 'Integrations & observability', label: 'Standalone deployment upgrades', key: 'deployment.podman_mode', text: 'deployment podman quadlet rootless rootful docker compose upgrade helper request dir deployment.podman_mode deployment.upgrade_helper deployment.upgrade_request_dir' }, { tab: 'Features', section: 'Review gate', label: 'Review gate: comments, dispatch and out-of-scope findings', key: 'review.post_comments', text: 'review swarm post comments fan out parallel perspectives combined all authors fix human prs acknowledge no findings confidence score priority labels fixer agent human decision label self authored decision policy reviews per head revise verdicts webhook event driven debounce pending ttl redispatch out of scope backlog issues recommendations review.post_comments review.self_authored_decision_policy review.event_driven review.event_debounce_s review.pending_ttl review.max_redispatch review.head_max_retries review.max_reviews_per_head review.revise_verdicts_before review.out_of_scope_backlog_disabled review.max_out_of_scope_backlog_issues review.recommendations.enabled review.recommendations.min_ready_to_open' }, { tab: 'Features', section: 'Review gate', label: 'Review evidence bundles', key: 'evidence.enabled', text: 'review evidence bundles audit signing key ed25519 evidence.enabled evidence.signing_key_file' }, + { tab: 'Features', section: 'Review gate', label: 'Benchmarks', key: 'benchmarks.publish', text: 'benchmarks leaderboard publish anonymized hive hash repo consent benchmarks.publish benchmarks.repo_consent hivecommons.dev/benchmarks' }, { tab: 'Features', section: 'Integrations & observability', label: 'GitLab and Gitea forges', key: 'gitlab.gitlab_url', text: 'gitlab gitea forgejo codeberg forge instance url token env gitlab.gitlab_url gitlab.token_env gitea.gitea_url gitea.token_env' }, { tab: 'Features', section: 'Agents & tooling', label: 'Token mint key and TTL', key: 'mint.max_ttl_seconds', text: 'token mint signing key path max ttl lifetime jwt mint.key_path mint.max_ttl_seconds' }, { tab: 'Features', section: 'Autonomy & ACMM', label: 'Design-first planning', key: 'planning.design_approved_label', text: 'planning design approved label architect revisions concurrent designs plan from label planning.design_approved_label planning.max_concurrent_designs planning.max_design_revisions planning.plan_from_label' }, diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index 68a4810cc5..e280e84772 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -54686,6 +54686,7 @@

Configure LiteLLM

{ tab: 'Features', section: 'Integrations & observability', label: 'Standalone deployment upgrades', key: 'deployment.podman_mode', text: 'deployment podman quadlet rootless rootful docker compose upgrade helper request dir deployment.podman_mode deployment.upgrade_helper deployment.upgrade_request_dir' }, { tab: 'Features', section: 'Review gate', label: 'Review gate: comments, dispatch and out-of-scope findings', key: 'review.post_comments', text: 'review swarm post comments fan out parallel perspectives combined all authors fix human prs acknowledge no findings confidence score priority labels fixer agent human decision label self authored decision policy reviews per head revise verdicts webhook event driven debounce pending ttl redispatch out of scope backlog issues recommendations review.post_comments review.self_authored_decision_policy review.event_driven review.event_debounce_s review.pending_ttl review.max_redispatch review.head_max_retries review.max_reviews_per_head review.revise_verdicts_before review.out_of_scope_backlog_disabled review.max_out_of_scope_backlog_issues review.recommendations.enabled review.recommendations.min_ready_to_open' }, { tab: 'Features', section: 'Review gate', label: 'Review evidence bundles', key: 'evidence.enabled', text: 'review evidence bundles audit signing key ed25519 evidence.enabled evidence.signing_key_file' }, + { tab: 'Features', section: 'Review gate', label: 'Benchmarks', key: 'benchmarks.publish', text: 'benchmarks leaderboard publish anonymized hive hash repo consent benchmarks.publish benchmarks.repo_consent hivecommons.dev/benchmarks' }, { tab: 'Features', section: 'Integrations & observability', label: 'GitLab and Gitea forges', key: 'gitlab.gitlab_url', text: 'gitlab gitea forgejo codeberg forge instance url token env gitlab.gitlab_url gitlab.token_env gitea.gitea_url gitea.token_env' }, { tab: 'Features', section: 'Agents & tooling', label: 'Token mint key and TTL', key: 'mint.max_ttl_seconds', text: 'token mint signing key path max ttl lifetime jwt mint.key_path mint.max_ttl_seconds' }, { tab: 'Features', section: 'Autonomy & ACMM', label: 'Design-first planning', key: 'planning.design_approved_label', text: 'planning design approved label architect revisions concurrent designs plan from label planning.design_approved_label planning.max_concurrent_designs planning.max_design_revisions planning.plan_from_label' }, @@ -65494,6 +65495,20 @@

${esc(s.title)}

Write evidence bundles ${yamlHint('evidence.enabled')} ${settingHelpMark('settings', 'features', 'Write evidence bundles', `On by default. Off stops writing review evidence bundles. Applies to the next review.`)} +
Benchmarks ${settingHelpMark('settings', 'features', 'Benchmarks', `This hive contributes anonymized per-PR records (model, harness, sandbox runtime, outcome, time-to-merge, ACMM level) to the fleet leaderboard at hivecommons.dev/benchmarks. No logins, prompts, findings or code ever leave the hive; the hive is identified only by the hash below. Repository names are included only for public repositories.`)}
+
+
+ Publish anonymized benchmark records ${yamlHint('benchmarks.publish')} ${settingHelpMark('settings', 'features', 'Publish benchmark records', `On by default. Off stops the hourly push to the hub and empties GET /api/benchmarks/export.`)} +
+
+
+ Name public repositories ${yamlHint('benchmarks.repo_consent')} ${settingHelpMark('settings', 'features', 'Name public repositories', `On by default: records for PUBLIC repositories carry the repository and org name, since that activity is already public. Off anonymizes every record. Private repositories are never named.`)} +
+
Look this hash up in the "Contributing hives" table at hivecommons.dev/benchmarks to verify your records arrived${f.benchmarksLastPushAt ? ` · last pushed ${esc(f.benchmarksLastPushAt)} (${esc(String(f.benchmarksLastPushCount || 0))} records)` : ' · nothing pushed yet'}.
+ + +
+
Review evidence signing
Read-only here: signing keys are provisioned on disk and set in hive.yaml.
`)} diff --git a/src/pkg/evidence/benchmarks/convert.go b/src/pkg/evidence/benchmarks/convert.go new file mode 100644 index 0000000000..0f9f6c5966 --- /dev/null +++ b/src/pkg/evidence/benchmarks/convert.go @@ -0,0 +1,357 @@ +package benchmarks + +import ( + "sort" + "strconv" + "strings" + "time" + + "github.com/hivecommons/hive/pkg/evidence" +) + +// Options parameterise one export. +type Options struct { + HiveID string + HiveVersion string + Cluster string + // Since drops PRs whose newest record is not strictly newer than it (the + // pusher passes its last watermark). Zero exports all. + Since time.Time + // Now stamps published_at; zero means time.Now(). + Now time.Time + // RepoPublic reports whether owner/name is a public repository. nil means + // unknown, which anonymizes every repo. + RepoPublic func(repo string) bool + // RepoConsent is benchmarks.repo_consent: false anonymizes even public repos. + RepoConsent bool + // SandboxRuntime resolves the runtime an agent's kicks ran under + // ("none" when sandboxing is off). nil means "none" for every agent. + SandboxRuntime func(agent string) string +} + +// Roles. +const ( + RoleAuthor = "author" + RoleReviewer = "reviewer" +) + +// SandboxRuntimeNone is the published runtime when agents run in-process. +const SandboxRuntimeNone = "none" + +const ( + kindPRReview = "pr-review" + kindUnknown = "unknown" + verdictNone = "none" + providerUnknown = "unknown" + harnessOther = "other" + ciSuccess = "success" + ciFailure = "failure" + ciPending = "pending" + ciUnknown = "unknown" + reviewSwarmAgent = "review-swarm" +) + +var knownHarnesses = map[string]bool{"agy": true, "aider": true, "bob": true, "claude": true, "cline": true, "codex": true, + "copilot": true, "gemini": true, "goose": true, "kilo": true, "muse": true, "omnigent": true, "omp": true, + "opencode": true, "openhands": true, "pi": true, "clanker": true} + +var verdictRank = map[string]int{"reject": 4, "changes_requested": 3, "requires_human": 2, "comment": 1, "approve": 0} + +var kindHints = []struct{ hint, kind string }{ + {"doc", "docs"}, {"sec", "security-fix"}, {"ci", "ci-repair"}, {"release", "release"}, + {"dep", "dependency-update"}, {"renovate", "dependency-update"}, {"triage", "triage"}, {"refactor", "refactor"}, +} + +var providerPrefixes = []struct{ prefix, provider string }{ + {"claude", "anthropic"}, {"gpt", "openai"}, {"o1", "openai"}, {"o3", "openai"}, {"o4", "openai"}, + {"gemini", "google"}, {"granite", "ibm"}, {"llama", "meta"}, {"mistral", "mistral"}, + {"qwen", "alibaba"}, {"deepseek", "deepseek"}, {"grok", "xai"}, {"kimi", "moonshot"}, +} + +// Convert folds bundles per PR and emits an author record for each PR a hive +// agent opened and a reviewer record for each PR the swarm scored. Output is +// sorted by id so exports are byte-stable. +func Convert(bundles []*evidence.Bundle, opts Options) []Record { + now := opts.Now + if now.IsZero() { + now = time.Now() + } + now = now.UTC() + hash := HiveHash(opts.HiveID) + version := strings.TrimSpace(opts.HiveVersion) + if version == "" { + version = kindUnknown + } + byPR := map[string][]*evidence.Bundle{} + for _, b := range bundles { + if b == nil || b.Repo == "" || b.Number == 0 { + continue + } + k := evidence.PRKey(b.Repo, b.Number) + byPR[k] = append(byPR[k], b) + } + folded := evidence.FoldPRs(bundles, "", "") + var out []Record + for key, group := range byPR { + pr := folded[key] + if pr == nil { + continue + } + latest := pr.Bundle + if !opts.Since.IsZero() && !latest.UpdatedAt.After(opts.Since) { + continue + } + public := opts.RepoConsent && opts.RepoPublic != nil && opts.RepoPublic(latest.Repo) + base := Record{ + SchemaVersion: SchemaVersion, + PublishedAt: now.Format(time.RFC3339), + Source: Source{HiveID: hash, HiveVersion: version, Cluster: opts.Cluster, Exporter: Exporter}, + Task: Task{RepoConsent: public}, + Outcome: outcomeOf(pr, group), + Quality: qualityOf(latest, pr), + } + if public { + base.Task.Repo = latest.Repo + base.Task.RepoVisibility = "public" + if owner, _, ok := strings.Cut(latest.Repo, "/"); ok { + base.Source.Org = owner + } + } + if lvl := latest.Policy.ACMMLevel; lvl >= 0 && lvl <= maxACMM { + l := lvl + base.Task.ACMMLevel = &l + } + a := latest.Author + if a.Kind == evidence.AuthorAgent && strings.TrimSpace(a.Model) != "" { + rec := base + rec.ID = recordID(hash, latest.Repo, latest.Number, RoleAuthor) + name := strings.TrimSpace(a.Agent) + if name == "" { + name = "agent" + } + rec.Agent = Agent{Name: name, Role: RoleAuthor, Harness: harnessOf(a.Backend)} + rec.Model = Model{Provider: providerOf(a.Model), Name: strings.TrimSpace(a.Model)} + rec.SandboxRuntime = runtimeOf(opts, a, name) + rec.Task.Kind = kindOf(a.TaskKind, name) + out = append(out, rec) + } + if backend, model, ok := dominantReviewer(latest.Verdicts); ok { + rec := base + rec.ID = recordID(hash, latest.Repo, latest.Number, RoleReviewer) + rec.Agent = Agent{Name: reviewSwarmAgent, Role: RoleReviewer, Harness: harnessOf(backend)} + rec.Model = Model{Provider: providerOf(model), Name: model} + rec.SandboxRuntime = SandboxRuntimeNone + rec.Task.Kind = kindPRReview + out = append(out, rec) + } + } + sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID }) + return out +} + +// maxACMM is the highest ACMM level the schema accepts. +const maxACMM = 6 + +func runtimeOf(opts Options, a evidence.Author, agent string) string { + if rt := strings.TrimSpace(a.Runtime); rt != "" { + return rt + } + if opts.SandboxRuntime != nil { + if rt := strings.TrimSpace(opts.SandboxRuntime(agent)); rt != "" { + return rt + } + } + return SandboxRuntimeNone +} + +func outcomeOf(pr *evidence.PR, group []*evidence.Bundle) Outcome { + latest := pr.Bundle + first := group[0] + heads := map[string]bool{} + for _, b := range group { + if b.CreatedAt.Before(first.CreatedAt) { + first = b + } + if b.HeadSHA != "" { + heads[b.HeadSHA] = true + } + } + merged := pr.Merge != nil || (pr.Outcome != nil && pr.Outcome.State == evidence.BackfillStateMerged) || + (latest.Backfill != nil && latest.Backfill.State == evidence.BackfillStateMerged) + closed := !merged && pr.Closed + o := Outcome{ + PROpened: true, + PRMerged: merged, + PRClosedUnmerged: closed, + CIStatusAtMerge: ciStatus(latest.CI.Status), + ReviewVerdict: worstVerdict(latest.Verdicts), + HumanInterventions: len(latest.HumanActions), + Retries: max(len(heads)-1, 0), + } + switch first.CI.Status { + case ciSuccess: + t := true + o.CIGreenFirstTry = &t + case ciFailure: + f := false + o.CIGreenFirstTry = &f + } + opened := pr.OpenedAt + var end time.Time + if merged { + if pr.Merge != nil { + end = pr.Merge.At + switch pr.Merge.Method { + case "merge", "squash", "rebase": + o.MergeMethod = pr.Merge.Method + default: + o.MergeMethod = evidence.MergeUnrecorded + } + } else { + o.MergeMethod = evidence.MergeUnrecorded + if pr.Outcome != nil { + end = pr.Outcome.At + } + } + if !end.IsZero() { + o.MergedAt = end.UTC().Format(time.RFC3339) + } + } else if closed && pr.Outcome != nil { + end = pr.Outcome.At + } + if !opened.IsZero() { + o.OpenedAt = opened.UTC().Format(time.RFC3339) + if !end.IsZero() && !end.Before(opened) { + secs := int64(end.Sub(opened).Seconds()) + o.WallClockS = &secs + } + } + if len(latest.Verdicts) > 0 { + minConf, seen := 1.0, false + models := map[string]bool{} + for _, v := range latest.Verdicts { + if v.Confidence < minConf { + minConf = v.Confidence + } + seen = true + if m := strings.TrimSpace(v.Model); m != "" { + models[m] = true + } + } + if seen { + c := min(max(minConf, 0), 1) + o.ReviewConfidence = &c + } + for m := range models { + o.ReviewerModels = append(o.ReviewerModels, m) + } + sort.Strings(o.ReviewerModels) + } + return o +} + +func qualityOf(latest *evidence.Bundle, pr *evidence.PR) *Quality { + q := &Quality{} + any := false + if pr.Merge != nil || (pr.Outcome != nil && pr.Outcome.State == evidence.BackfillStateMerged) { + rv, fu, ro := len(pr.RevertedBy) > 0, len(pr.FollowedUpBy) > 0, len(pr.IssueReopens) > 0 + q.RevertedWithin14d, q.FollowedUpWithin14d, q.IssueReopened = &rv, &fu, &ro + any = true + } + if lvl := latest.Policy.ACMMLevel; lvl >= 0 && lvl <= maxACMM { + l := lvl + q.ACMMLevel = &l + any = true + } + if n := len(latest.Sentinel); n > 0 { + q.SentinelFindings = &n + any = true + } + if !any { + return nil + } + return q +} + +func ciStatus(s string) string { + switch s { + case ciSuccess, ciFailure, ciPending: + return s + } + return ciUnknown +} + +func worstVerdict(vs []evidence.Verdict) string { + best, bestRank := verdictNone, -1 + for _, v := range vs { + if r, ok := verdictRank[v.Verdict]; ok && r > bestRank { + best, bestRank = v.Verdict, r + } + } + return best +} + +// dominantReviewer is the (backend, model) pair that cast most verdicts. +func dominantReviewer(vs []evidence.Verdict) (backend, model string, ok bool) { + type pair struct{ b, m string } + counts := map[pair]int{} + for _, v := range vs { + m := strings.TrimSpace(v.Model) + if m == "" { + continue + } + counts[pair{strings.TrimSpace(v.Backend), m}]++ + } + best := pair{} + for p, n := range counts { + if n > counts[best] || (n == counts[best] && (best.m == "" || p.m < best.m)) { + best = p + } + } + return best.b, best.m, best.m != "" +} + +func harnessOf(backend string) string { + b := strings.ToLower(strings.TrimSpace(backend)) + if knownHarnesses[b] { + return b + } + return harnessOther +} + +func providerOf(model string) string { + m := strings.ToLower(strings.TrimSpace(model)) + if p, _, ok := strings.Cut(m, "/"); ok && p != "" { + return p + } + for _, pp := range providerPrefixes { + if strings.HasPrefix(m, pp.prefix) { + return pp.provider + } + } + return providerUnknown +} + +// kindOf prefers the task kind stamped on the bundle and otherwise guesses +// from the agent's name. +func kindOf(stamped, agent string) string { + if k := strings.TrimSpace(stamped); k != "" { + return k + } + return TaskKindFor(agent) +} + +// TaskKindFor guesses the benchmark task kind from an agent's name +// ("docs-writer" → docs); "unknown" when nothing matches. It is what gets +// stamped on new bundles until agents declare a kind. +func TaskKindFor(agent string) string { + n := strings.ToLower(agent) + for _, h := range kindHints { + if strings.Contains(n, h.hint) { + return h.kind + } + } + return kindUnknown +} + +func itoa(n int) string { return strconv.Itoa(n) } diff --git a/src/pkg/evidence/benchmarks/convert_test.go b/src/pkg/evidence/benchmarks/convert_test.go new file mode 100644 index 0000000000..b70a87d042 --- /dev/null +++ b/src/pkg/evidence/benchmarks/convert_test.go @@ -0,0 +1,272 @@ +package benchmarks + +import ( + "encoding/json" + "os" + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/evidence" +) + +func t0(d time.Duration) time.Time { return time.Date(2026, 10, 3, 12, 0, 0, 0, time.UTC).Add(d) } + +func fixtureBundles() []*evidence.Bundle { + first := &evidence.Bundle{ + SchemaVersion: evidence.SchemaVersion, ID: "a", Repo: "hivecommons/hive", Number: 42, + Author: evidence.Author{Login: "hive[bot]", Kind: evidence.AuthorAgent, Agent: "docs-writer", Backend: "copilot", Model: "claude-fable-5.1"}, + HeadSHA: "h1", Policy: evidence.Policy{ACMMLevel: 3}, + CI: evidence.CI{Status: "failure"}, + CreatedAt: t0(0), UpdatedAt: t0(time.Hour), + } + latest := &evidence.Bundle{ + SchemaVersion: evidence.SchemaVersion, ID: "b", Repo: "hivecommons/hive", Number: 42, + Author: first.Author, HeadSHA: "h2", Policy: evidence.Policy{ACMMLevel: 3}, + CI: evidence.CI{Status: "success"}, + Verdicts: []evidence.Verdict{ + {Perspective: "security", Model: "claude-fable-5.1", Backend: "copilot", Verdict: "approve", Confidence: 0.9}, + {Perspective: "style", Model: "claude-fable-5.1", Backend: "copilot", Verdict: "comment", Confidence: 0.7}, + }, + HumanActions: []evidence.Action{{Actor: "andy", Kind: "label", At: t0(2 * time.Hour)}}, + Merge: &evidence.MergeEvent{Actor: "hive", Method: "squash", At: t0(3 * time.Hour), OpenedAt: t0(-time.Hour)}, + CreatedAt: t0(2 * time.Hour), UpdatedAt: t0(3 * time.Hour), + } + private := &evidence.Bundle{ + SchemaVersion: evidence.SchemaVersion, ID: "c", Repo: "acme/secret", Number: 7, + Author: evidence.Author{Login: "x", Kind: evidence.AuthorAgent, Agent: "fixer", Backend: "pi", Model: "gpt-5.3-codex", Runtime: "podman", TaskKind: "issue-fix"}, + HeadSHA: "p1", Policy: evidence.Policy{ACMMLevel: 2}, + Outcome: &evidence.Outcome{Source: "ledger", State: evidence.BackfillStateClosed, At: t0(5 * time.Hour)}, + CreatedAt: t0(0), UpdatedAt: t0(5 * time.Hour), + } + human := &evidence.Bundle{ + SchemaVersion: evidence.SchemaVersion, ID: "d", Repo: "hivecommons/hive", Number: 43, + Author: evidence.Author{Login: "someone", Kind: evidence.AuthorHuman}, + HeadSHA: "q1", CreatedAt: t0(0), UpdatedAt: t0(time.Hour), + } + return []*evidence.Bundle{first, latest, private, human} +} + +func testOptions() Options { + return Options{ + HiveID: "hosted-test", HiveVersion: "v5.1.0", Now: t0(6 * time.Hour), + RepoConsent: true, + RepoPublic: func(repo string) bool { return strings.HasPrefix(repo, "hivecommons/") }, + SandboxRuntime: func(agent string) string { return "none" }, + } +} + +func byRole(recs []Record, repo string, role string) *Record { + for i := range recs { + if recs[i].Agent.Role == role && (recs[i].Task.Repo == repo || (repo == "" && recs[i].Task.Repo == "")) { + return &recs[i] + } + } + return nil +} + +func TestConvertFoldsPRAndRedacts(t *testing.T) { + recs := Convert(fixtureBundles(), testOptions()) + if len(recs) != 3 { + t.Fatalf("want 3 records (author+reviewer for #42, author for private #7), got %d", len(recs)) + } + a := byRole(recs, "hivecommons/hive", RoleAuthor) + if a == nil { + t.Fatal("missing author record for hivecommons/hive#42") + } + if a.Source.HiveID != HiveHash("hosted-test") || a.Source.Org != "hivecommons" || !a.Task.RepoConsent { + t.Errorf("source/task attribution wrong: %+v %+v", a.Source, a.Task) + } + if a.Task.Kind != "docs" || a.Agent.Harness != "copilot" || a.Model.Provider != "anthropic" { + t.Errorf("classification wrong: kind=%s harness=%s provider=%s", a.Task.Kind, a.Agent.Harness, a.Model.Provider) + } + o := a.Outcome + if !o.PRMerged || o.MergeMethod != "squash" || o.Retries != 1 || o.HumanInterventions != 1 { + t.Errorf("outcome wrong: %+v", o) + } + if o.CIGreenFirstTry == nil || *o.CIGreenFirstTry { + t.Errorf("first head failed CI, want ci_green_first_try=false, got %v", o.CIGreenFirstTry) + } + if o.WallClockS == nil || *o.WallClockS != int64(4*time.Hour/time.Second) { + t.Errorf("wall clock: opened T-1h, merged T+3h, want 14400 got %v", o.WallClockS) + } + if o.ReviewVerdict != "comment" || o.ReviewConfidence == nil || *o.ReviewConfidence != 0.7 { + t.Errorf("review summary wrong: %+v", o) + } + if a.Quality == nil || a.Quality.ACMMLevel == nil || *a.Quality.ACMMLevel != 3 || a.Quality.RevertedWithin14d == nil || *a.Quality.RevertedWithin14d { + t.Errorf("quality wrong: %+v", a.Quality) + } + + r := byRole(recs, "hivecommons/hive", RoleReviewer) + if r == nil || r.Task.Kind != "pr-review" || r.Agent.Name != reviewSwarmAgent || r.Model.Name != "claude-fable-5.1" { + t.Errorf("reviewer record wrong: %+v", r) + } + + p := byRole(recs, "", RoleAuthor) + if p == nil { + t.Fatal("missing anonymized record for private repo") + } + if p.Task.Repo != "" || p.Source.Org != "" || p.Task.RepoConsent || p.Task.RepoVisibility != "" { + t.Errorf("private repo leaked: %+v %+v", p.Task, p.Source) + } + if p.SandboxRuntime != "podman" || p.Task.Kind != "issue-fix" { + t.Errorf("stamped runtime/kind not honoured: %s %s", p.SandboxRuntime, p.Task.Kind) + } + if !p.Outcome.PRClosedUnmerged || p.Outcome.PRMerged { + t.Errorf("ledger-closed PR should be closed unmerged: %+v", p.Outcome) + } + for _, rec := range recs { + if strings.Contains(rec.Agent.Name, "andy") || strings.Contains(rec.Agent.Name, "[bot]") { + t.Errorf("login leaked into %+v", rec.Agent) + } + } +} + +func TestConvertRepoConsentOffAnonymizesPublic(t *testing.T) { + opts := testOptions() + opts.RepoConsent = false + for _, r := range Convert(fixtureBundles(), opts) { + if r.Task.Repo != "" || r.Source.Org != "" || r.Task.RepoConsent { + t.Fatalf("repo_consent=false must drop names: %+v", r.Task) + } + } +} + +func TestConvertSinceAndStableIDs(t *testing.T) { + opts := testOptions() + all := Convert(fixtureBundles(), opts) + opts.Since = t0(4 * time.Hour) + recent := Convert(fixtureBundles(), opts) + if len(recent) != 1 || recent[0].Task.Repo != "" { + t.Fatalf("since=T+4h should keep only the private PR updated at T+5h, got %d", len(recent)) + } + again := Convert(fixtureBundles(), testOptions()) + for i := range all { + if all[i].ID != again[i].ID { + t.Fatalf("ids must be stable across runs") + } + } + if HiveHash("hosted-test") == HiveHash("hosted-test2") || len(HiveHash("x")) != hiveHashLen { + t.Fatal("hive hash shape") + } +} + +// TestConvertOutputMatchesSchema checks every emitted record against the +// vendored hivecommons/benchmarks schema: required keys, enum values and +// no unknown keys at each object the schema closes. +func TestConvertOutputMatchesSchema(t *testing.T) { + raw, err := os.ReadFile("testdata/result.schema.json") + if err != nil { + t.Fatal(err) + } + var schema map[string]any + if err := json.Unmarshal(raw, &schema); err != nil { + t.Fatal(err) + } + for _, rec := range Convert(fixtureBundles(), testOptions()) { + data, _ := json.Marshal(rec) + var obj map[string]any + _ = json.Unmarshal(data, &obj) + checkSchema(t, "", schema, obj) + } +} + +func checkSchema(t *testing.T, path string, schema map[string]any, v any) { + t.Helper() + if enum, ok := schema["enum"].([]any); ok { + found := false + for _, e := range enum { + if e == v { + found = true + } + } + if !found { + t.Errorf("%s: %v not in enum %v", path, v, enum) + } + } + props, _ := schema["properties"].(map[string]any) + obj, isObj := v.(map[string]any) + if !isObj { + return + } + if req, ok := schema["required"].([]any); ok { + for _, r := range req { + if _, present := obj[r.(string)]; !present { + t.Errorf("%s: missing required %s", path, r) + } + } + } + for k, val := range obj { + sub, ok := props[k].(map[string]any) + if !ok { + if ap, closed := schema["additionalProperties"].(bool); closed && !ap { + t.Errorf("%s: unexpected key %s", path, k) + } + continue + } + if val == nil { + continue + } + checkSchema(t, path+"."+k, sub, val) + } +} + +func TestConvertEdgeCases(t *testing.T) { + // Backfilled merge (no Merge event), first bundle green, sentinel + // findings, nameless agent on an unknown backend/model, and a bundle + // that cannot be keyed — all with zero-value Options. + ledgerMerged := &evidence.Bundle{ + SchemaVersion: evidence.SchemaVersion, ID: "e", Repo: "o/r", Number: 9, HeadSHA: "s1", + Author: evidence.Author{Login: "bot", Kind: evidence.AuthorAgent, Backend: "mystery", Model: "zeta-9000"}, + CI: evidence.CI{Status: "success"}, + Sentinel: []evidence.SentinelFinding{{}}, + Outcome: &evidence.Outcome{Source: "ledger", State: evidence.BackfillStateMerged, At: t0(4 * time.Hour)}, + Verdicts: []evidence.Verdict{{Perspective: "security", Verdict: "approve"}}, // no model → no reviewer record + CreatedAt: t0(2 * time.Hour), UpdatedAt: t0(4 * time.Hour), + } + earlier := &evidence.Bundle{ + SchemaVersion: evidence.SchemaVersion, ID: "f", Repo: "o/r", Number: 9, HeadSHA: "s0", + Author: ledgerMerged.Author, CI: evidence.CI{Status: "success"}, + CreatedAt: t0(0), UpdatedAt: t0(time.Hour), + } + unkeyed := &evidence.Bundle{SchemaVersion: evidence.SchemaVersion, ID: "g"} + recs := Convert([]*evidence.Bundle{ledgerMerged, nil, unkeyed, earlier}, Options{HiveID: "x"}) + if len(recs) != 1 { + t.Fatalf("want 1 author record, got %d: %+v", len(recs), recs) + } + r := recs[0] + if r.Agent.Name != "agent" || r.Agent.Harness != "other" || r.Model.Provider != "unknown" || r.Task.Kind != "unknown" { + t.Errorf("defaults: %+v %+v %+v", r.Agent, r.Model, r.Task) + } + if r.SandboxRuntime != SandboxRuntimeNone || r.Source.HiveVersion == "" || r.PublishedAt == "" { + t.Errorf("fallbacks: runtime=%q version=%q published=%q", r.SandboxRuntime, r.Source.HiveVersion, r.PublishedAt) + } + if !r.Outcome.PRMerged || r.Outcome.MergeMethod != evidence.MergeUnrecorded || r.Outcome.MergedAt == "" || r.Outcome.Retries != 1 { + t.Errorf("ledger-merged outcome: %+v", r.Outcome) + } + if r.Outcome.CIGreenFirstTry == nil || !*r.Outcome.CIGreenFirstTry { + t.Errorf("first bundle was green: %+v", r.Outcome.CIGreenFirstTry) + } + if r.Quality == nil || r.Quality.SentinelFindings == nil || *r.Quality.SentinelFindings != 1 { + t.Errorf("quality: %+v", r.Quality) + } + + // A merge event with an unknown method is recorded as unrecorded. + odd := &evidence.Bundle{ + SchemaVersion: evidence.SchemaVersion, ID: "h", Repo: "o/r", Number: 10, HeadSHA: "t1", + Author: evidence.Author{Kind: evidence.AuthorAgent, Model: "openai/gpt-5", Agent: "ci-fixer"}, + Policy: evidence.Policy{ACMMLevel: 99}, + Merge: &evidence.MergeEvent{Actor: "x", Method: "teleport", At: t0(time.Hour)}, + CreatedAt: t0(0), UpdatedAt: t0(time.Hour), + } + recs = Convert([]*evidence.Bundle{odd}, Options{HiveID: "x", Now: t0(2 * time.Hour)}) + if len(recs) != 1 || recs[0].Outcome.MergeMethod != evidence.MergeUnrecorded || recs[0].Model.Provider != "openai" || recs[0].Task.Kind != "ci-repair" { + t.Errorf("odd merge: %+v", recs) + } + if recs[0].Quality != nil && recs[0].Quality.ACMMLevel != nil { + t.Errorf("out-of-range ACMM level must be dropped: %+v", recs[0].Quality) + } + if got := qualityOf(&evidence.Bundle{Policy: evidence.Policy{ACMMLevel: 99}}, &evidence.PR{}); got != nil { + t.Errorf("qualityOf with nothing to report = %+v, want nil", got) + } +} diff --git a/src/pkg/evidence/benchmarks/record.go b/src/pkg/evidence/benchmarks/record.go new file mode 100644 index 0000000000..50b2aaf0f9 --- /dev/null +++ b/src/pkg/evidence/benchmarks/record.go @@ -0,0 +1,117 @@ +// Package benchmarks turns review evidence bundles into the records published +// at https://hivecommons.dev/benchmarks/ (schema: hivecommons/benchmarks +// schema/result.schema.json; RFC hivecommons/hive#17694). +// +// Privacy contract: a record never carries a login, a prompt, finding text or +// code. The hive is identified only by HiveHash. Repository and org names are +// present only when Options.RepoPublic reports the repo public AND the owner +// has not set benchmarks.repo_consent: false; otherwise task.repo is absent. +package benchmarks + +import ( + "crypto/sha256" + "encoding/hex" + "strings" +) + +// SchemaVersion is the published record schema version. +const SchemaVersion = "1" + +// hashDomain salts the hive id so the published hash cannot be joined with +// any other sha256(hive_id) in the wild. +const hashDomain = "hivecommons-benchmarks/v1/" + +// hiveHashLen is the published hex prefix length of the hive hash. +const hiveHashLen = 16 + +// recordIDLen is the hex length of a record id. +const recordIDLen = 24 + +// Exporter identifies the producer in source.exporter. +const Exporter = "hive" + +// HiveHash is the anonymous, stable identifier of a hive on the leaderboard. +// Owners see theirs in Settings → Benchmarks and in /api/status. +func HiveHash(hiveID string) string { + sum := sha256.Sum256([]byte(hashDomain + strings.TrimSpace(hiveID))) + return hex.EncodeToString(sum[:])[:hiveHashLen] +} + +// Record is one benchmark result: one agent role on one PR. +type Record struct { + SchemaVersion string `json:"schema_version"` + ID string `json:"id"` + PublishedAt string `json:"published_at"` + Source Source `json:"source"` + Agent Agent `json:"agent"` + Model Model `json:"model"` + SandboxRuntime string `json:"sandbox_runtime"` + Task Task `json:"task"` + Outcome Outcome `json:"outcome"` + Quality *Quality `json:"quality,omitempty"` +} + +// Source says which hive emitted the record. +type Source struct { + HiveID string `json:"hive_id"` + Org string `json:"org,omitempty"` + Cluster string `json:"cluster,omitempty"` + HiveVersion string `json:"hive_version"` + Exporter string `json:"exporter,omitempty"` +} + +// Agent is the hive agent (or the review swarm) the record scores. +type Agent struct { + Name string `json:"name"` + Role string `json:"role,omitempty"` + Harness string `json:"harness"` +} + +// Model is the model the agent ran on. +type Model struct { + Provider string `json:"provider"` + Name string `json:"name"` +} + +// Task describes the unit of work. +type Task struct { + Kind string `json:"kind"` + Repo string `json:"repo,omitempty"` + RepoConsent bool `json:"repo_consent"` + RepoVisibility string `json:"repo_visibility,omitempty"` + ACMMLevel *int `json:"acmm_level,omitempty"` +} + +// Outcome is what happened to the PR. +type Outcome struct { + PROpened bool `json:"pr_opened"` + PRMerged bool `json:"pr_merged"` + PRClosedUnmerged bool `json:"pr_closed_unmerged"` + MergeMethod string `json:"merge_method,omitempty"` + CIGreenFirstTry *bool `json:"ci_green_first_try"` + CIStatusAtMerge string `json:"ci_status_at_merge"` + ReviewVerdict string `json:"review_verdict"` + ReviewConfidence *float64 `json:"review_confidence,omitempty"` + ReviewerModels []string `json:"reviewer_models,omitempty"` + HumanInterventions int `json:"human_interventions"` + Retries int `json:"retries"` + WallClockS *int64 `json:"wall_clock_s,omitempty"` + OpenedAt string `json:"opened_at,omitempty"` + MergedAt string `json:"merged_at,omitempty"` +} + +// Quality carries post-merge signals and the governing ACMM level. +type Quality struct { + RevertedWithin14d *bool `json:"reverted_within_14d,omitempty"` + FollowedUpWithin14d *bool `json:"followed_up_within_14d,omitempty"` + IssueReopened *bool `json:"issue_reopened,omitempty"` + ACMMLevel *int `json:"acmm_level,omitempty"` + SentinelFindings *int `json:"sentinel_findings,omitempty"` +} + +// recordID derives the stable id of one (hive, PR, role) triple so re-exports +// overwrite rather than duplicate. +func recordID(hiveHash, repo string, number int, role string) string { + sum := sha256.Sum256([]byte(hiveHash + "|" + repo + "|" + itoa(number) + "|" + role)) + return hex.EncodeToString(sum[:])[:recordIDLen] +} diff --git a/src/pkg/evidence/benchmarks/testdata/result.schema.json b/src/pkg/evidence/benchmarks/testdata/result.schema.json new file mode 100644 index 0000000000..c4ecd39ec6 --- /dev/null +++ b/src/pkg/evidence/benchmarks/testdata/result.schema.json @@ -0,0 +1,449 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://hivecommons.dev/benchmarks/schema/result.schema.json", + "title": "Hive Commons benchmark result", + "description": "One benchmark record: a single unit of agent work (usually one pull request) with its outcome, cost and quality signals. Field names follow the hive evidence bundle (pkg/evidence.Bundle) and GET /api/cost so a spoke can emit records directly.", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "id", + "published_at", + "source", + "agent", + "model", + "sandbox_runtime", + "task", + "outcome" + ], + "properties": { + "schema_version": { + "const": "1" + }, + "id": { + "type": "string", + "description": "Stable unique id. Convention: sha256 of (hive_id, repo, pr number, head sha) hex-encoded, or any URN.", + "minLength": 8 + }, + "published_at": { + "type": "string", + "format": "date-time" + }, + "synthetic": { + "type": "boolean", + "default": false, + "description": "True for hand-written sample records. The site excludes these from every leaderboard." + }, + "source": { + "type": "object", + "additionalProperties": false, + "required": [ + "hive_id", + "hive_version" + ], + "properties": { + "hive_id": { + "type": "string", + "description": "Stable anonymous id: sha256('hivecommons-benchmarks/v1/' + hive id)[:16]. Shown to the owner in their dashboard (Settings \u2192 Benchmarks)." + }, + "hive_name": { + "type": "string", + "description": "Only present when the owner opted in (benchmarks.publish_name)." + }, + "org": { + "type": "string", + "description": "GitHub org of the repository. Present only when task.repo_consent is true (public repo)." + }, + "cluster": { + "type": "string", + "description": "Free-form cluster / environment label (e.g. kind, openshift, bare-metal)." + }, + "hive_version": { + "type": "string" + }, + "exporter": { + "type": "string", + "description": "What produced the record, e.g. hive/api/benchmarks/export, hub/aggregate, manual.", + "examples": [ + "hive/api/benchmarks/export", + "hub/publisher", + "backfill-from-cluster", + "manual" + ] + } + } + }, + "agent": { + "type": "object", + "additionalProperties": false, + "required": [ + "name", + "harness" + ], + "properties": { + "name": { + "type": "string", + "description": "Agent name as configured in hive.yaml (evidence author.agent)." + }, + "role": { + "type": "string", + "description": "author = the agent opened the PR; reviewer = the agent reviewed someone else's PR. Other values are free-form lane names.", + "examples": [ + "author", + "reviewer", + "fix", + "review", + "triage", + "ci-repair" + ] + }, + "harness": { + "type": "string", + "description": "CLI harness / backend (evidence author.backend, crux IsCLIBackend list).", + "enum": [ + "agy", + "aider", + "bob", + "claude", + "cline", + "codex", + "copilot", + "gemini", + "goose", + "kilo", + "muse", + "omnigent", + "omp", + "opencode", + "openhands", + "pi", + "clanker", + "other" + ] + }, + "harness_version": { + "type": "string" + } + } + }, + "model": { + "type": "object", + "additionalProperties": false, + "required": [ + "provider", + "name" + ], + "properties": { + "provider": { + "type": "string", + "description": "Vendor family (pkg/modelfamily), e.g. anthropic, openai, google, ibm, local." + }, + "name": { + "type": "string", + "description": "Exact model id as reported by the harness (evidence author.model)." + }, + "version": { + "type": "string" + }, + "effort": { + "type": "string", + "description": "Reasoning effort / thinking budget label if the harness exposes one." + } + } + }, + "sandbox_runtime": { + "type": "string", + "description": "Where the agent process ran (crux/config SandboxRuntime*). Open enum.", + "anyOf": [ + { + "enum": [ + "podman", + "kubernetes-job", + "openshell", + "substrate", + "none" + ] + }, + { + "type": "string", + "pattern": "^x-[a-z0-9-]+$" + } + ] + }, + "task": { + "type": "object", + "additionalProperties": false, + "required": [ + "kind" + ], + "properties": { + "kind": { + "type": "string", + "enum": [ + "issue-fix", + "pr-review", + "ci-repair", + "triage", + "release", + "docs", + "security-fix", + "dependency-update", + "refactor", + "experiment", + "other", + "unknown" + ] + }, + "repo": { + "type": "string", + "description": "owner/name. MUST be omitted unless repo_consent is true.", + "pattern": "^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$" + }, + "repo_consent": { + "type": "boolean", + "description": "Repository owner consented to being named in published results." + }, + "repo_visibility": { + "type": "string", + "enum": [ + "public", + "private" + ] + }, + "language": { + "type": "string", + "description": "Primary language of the touched files." + }, + "size": { + "type": "object", + "additionalProperties": false, + "properties": { + "files_changed": { + "type": "integer", + "minimum": 0 + }, + "additions": { + "type": "integer", + "minimum": 0 + }, + "deletions": { + "type": "integer", + "minimum": 0 + }, + "bucket": { + "type": "string", + "enum": [ + "xs", + "s", + "m", + "l", + "xl" + ] + } + } + }, + "acmm_level": { + "type": "integer", + "minimum": 0, + "maximum": 6, + "description": "Policy ACMM level the hive ran at (evidence policy.acmm_level)." + } + }, + "if": { + "properties": { + "repo_consent": { + "const": false + } + } + }, + "then": { + "not": { + "required": [ + "repo" + ] + } + } + }, + "outcome": { + "type": "object", + "additionalProperties": false, + "required": [ + "pr_opened" + ], + "properties": { + "pr_opened": { + "type": "boolean" + }, + "pr_merged": { + "type": "boolean" + }, + "pr_closed_unmerged": { + "type": "boolean" + }, + "merge_method": { + "type": "string", + "enum": [ + "merge", + "squash", + "rebase", + "unrecorded" + ] + }, + "ci_green_first_try": { + "type": [ + "boolean", + "null" + ], + "description": "CI status was success on the first head (evidence ci.status on the bundle with no previous_bundle_id). null = no CI captured." + }, + "ci_status_at_merge": { + "type": "string", + "enum": [ + "success", + "failure", + "pending", + "unknown" + ] + }, + "review_verdict": { + "type": "string", + "description": "Worst swarm verdict on the final head (evidence verdicts[].verdict); reject > changes_requested > requires_human > comment > approve.", + "enum": [ + "approve", + "comment", + "requires_human", + "changes_requested", + "reject", + "none" + ] + }, + "review_confidence": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "reviewer_models": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Exact model ids of the reviewing perspectives." + }, + "human_interventions": { + "type": "integer", + "minimum": 0, + "description": "len(evidence human_actions)." + }, + "wall_clock_s": { + "type": "number", + "minimum": 0, + "description": "merge.at - merge.opened_at, or close time for unmerged." + }, + "retries": { + "type": "integer", + "minimum": 0, + "description": "Number of additional heads pushed (bundles chained by previous_bundle_id)." + }, + "opened_at": { + "type": "string", + "format": "date-time" + }, + "merged_at": { + "type": "string", + "format": "date-time" + } + } + }, + "cost": { + "type": "object", + "additionalProperties": false, + "properties": { + "input_tokens": { + "type": "integer", + "minimum": 0 + }, + "output_tokens": { + "type": "integer", + "minimum": 0 + }, + "cache_read_tokens": { + "type": "integer", + "minimum": 0 + }, + "cache_write_tokens": { + "type": "integer", + "minimum": 0 + }, + "usd": { + "type": "number", + "minimum": 0 + }, + "basis": { + "type": "string", + "description": "How usd was computed (GET /api/cost: estimated from public price table, or apportioned from per-agent totals).", + "enum": [ + "estimated", + "apportioned", + "billed", + "unpriced" + ] + }, + "price_table_date": { + "type": "string", + "format": "date" + } + } + }, + "quality": { + "type": "object", + "additionalProperties": false, + "properties": { + "reverted_within_14d": { + "type": [ + "boolean", + "null" + ] + }, + "followed_up_within_14d": { + "type": [ + "boolean", + "null" + ] + }, + "issue_reopened": { + "type": [ + "boolean", + "null" + ] + }, + "acmm_level": { + "type": "integer", + "minimum": 0, + "maximum": 6, + "description": "Repo overall ACMM level at the time (GET /api/acmm/evaluation overall_level); context, not an agent score." + }, + "acmm_scores": { + "type": "object", + "description": "Per-level detected_score 0..1 keyed by level number, from GET /api/acmm/evaluation levels[].", + "patternProperties": { + "^[0-6]$": { + "type": "number", + "minimum": 0, + "maximum": 1 + } + }, + "additionalProperties": false + }, + "sentinel_findings": { + "type": "integer", + "minimum": 0 + } + } + }, + "notes": { + "type": "string", + "maxLength": 2000, + "description": "Free text. MUST NOT contain prompts, logins, or code excerpts." + } + } +} diff --git a/src/pkg/evidence/evidence.go b/src/pkg/evidence/evidence.go index 8818e39c44..a3c27e11f1 100644 --- a/src/pkg/evidence/evidence.go +++ b/src/pkg/evidence/evidence.go @@ -91,13 +91,19 @@ type Bundle struct { // Author identifies the PR author. Kind is one of human, agent or bot. For // an agent-authored PR, Agent, Backend and Model are the launch metadata the -// hive recorded when the agent opened it; empty when unknown. +// hive recorded when the agent opened it; empty when unknown. Runtime is the +// sandbox runtime the agent's kicks ran under (podman, job, openshell, or +// "none" in-process) and TaskKind the benchmark task class +// (hivecommons/hive#17694); both empty on records written before they were +// stamped. type Author struct { - Login string `json:"login"` - Kind string `json:"kind"` - Agent string `json:"agent,omitempty"` - Backend string `json:"backend,omitempty"` - Model string `json:"model,omitempty"` + Login string `json:"login"` + Kind string `json:"kind"` + Agent string `json:"agent,omitempty"` + Backend string `json:"backend,omitempty"` + Model string `json:"model,omitempty"` + Runtime string `json:"runtime,omitempty"` + TaskKind string `json:"task_kind,omitempty"` } // Policy snapshots the review policy in force when the bundle was written. diff --git a/src/pkg/github/review_evidence.go b/src/pkg/github/review_evidence.go index 5b4574be5c..b0c5f6b327 100644 --- a/src/pkg/github/review_evidence.go +++ b/src/pkg/github/review_evidence.go @@ -803,6 +803,13 @@ func (c *Client) reviewEvidencePolicy(p evidence.Policy) evidence.Policy { return p } +// EvidenceAgentContext resolves the sandbox runtime and benchmark task kind +// stamped on agent-authored bundles (Author.Runtime / Author.TaskKind, +// hivecommons/hive#17694). The composition root installs the config-backed +// resolver; the default leaves both empty, which the exporter reads as +// "none" / "unknown". +var EvidenceAgentContext = func(agent string) (runtime, taskKind string) { return "", "" } + // evidenceAuthor classifies the PR author. A PR the audit log records as // opened by one of this hive's agents is "agent" even though GitHub shows the // App bot as its author, and carries the agent, backend and model that audit @@ -810,7 +817,8 @@ func (c *Client) reviewEvidencePolicy(p evidence.Policy) evidence.Policy { func evidenceAuthor(repo string, number int, user *gh.User) evidence.Author { login := strings.TrimSpace(user.GetLogin()) if m := auditAuthorMetaForPR(repo, number, reviewVerdictAuditPath, time.Now().UTC().Add(-reviewVerdictAuditLookback)); m.Agent != "" { - return evidence.Author{Login: login, Kind: evidence.AuthorAgent, Agent: m.Agent, Backend: m.Backend, Model: m.Model} + runtime, kind := EvidenceAgentContext(m.Agent) + return evidence.Author{Login: login, Kind: evidence.AuthorAgent, Agent: m.Agent, Backend: m.Backend, Model: m.Model, Runtime: runtime, TaskKind: kind} } switch { case strings.EqualFold(user.GetType(), "Bot") || strings.HasSuffix(strings.ToLower(login), "[bot]"): diff --git a/src/pkg/hub/background_pollers.go b/src/pkg/hub/background_pollers.go index 8f9021dabd..4bcd0f8043 100644 --- a/src/pkg/hub/background_pollers.go +++ b/src/pkg/hub/background_pollers.go @@ -48,6 +48,9 @@ func (s *HubServer) StartBackgroundPollers(ctx context.Context) <-chan struct{} start(s.startAlertStatePersistence) // Live-vs-approved org reconciliation for assigned hives (#11847). start(s.startPlaceholderOrgSweep) + // Hourly commit of spoke benchmark records to hivecommons/benchmarks + // (hivecommons/hive#17694). + start(s.StartBenchmarksPublisher) s.githubActivityMu.Lock() s.githubActivityCtx = ctx feed := s.githubActivityFeed diff --git a/src/pkg/hub/benchmarks.go b/src/pkg/hub/benchmarks.go new file mode 100644 index 0000000000..89f7b4d564 --- /dev/null +++ b/src/pkg/hub/benchmarks.go @@ -0,0 +1,457 @@ +package hub + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "net/http" + "os" + "path/filepath" + "regexp" + "sort" + "strings" + "sync" + "time" + + "github.com/hivecommons/hive/pkg/evidence/benchmarks" +) + +// Fleet benchmark publication (hivecommons/hive#17694). Spokes POST their +// anonymized per-PR records here hourly (pkg/dashboard/api_benchmarks.go); +// the hub merges them by record id into /data/benchmarks//.json +// and, hourly, commits every changed month file to the data repo through the +// GitHub Contents API as the hub's own identity. Direct pushes to main: the +// target is a data-only repo whose Pages workflow rebuilds on push. + +const ( + benchmarksIngestPath = "/api/benchmarks/ingest" + benchmarksStatusPath = "/api/hub/benchmarks" + // benchmarksIngestMaxBodyBytes caps one spoke batch (2000 records ≈ 2 MB). + benchmarksIngestMaxBodyBytes = 8 << 20 + // benchmarksPublishInterval is the hub→GitHub cadence. + benchmarksPublishInterval = time.Hour + // benchmarksPublishInitialDelay lets the first spoke batches land. + benchmarksPublishInitialDelay = 10 * time.Minute + // benchmarksCommitTimeout bounds one Contents API call. + benchmarksCommitTimeout = 45 * time.Second + // benchmarksRepoEnv overrides the target repo (owner/name). + benchmarksRepoEnv = "HIVE_BENCHMARKS_REPO" + // benchmarksRepoDefault is the public leaderboard data repo. + benchmarksRepoDefault = "hivecommons/benchmarks" + // benchmarksDataPrefix is the path inside the repo. + benchmarksDataPrefix = "data/fleet" + // benchmarksDisableEnv turns the GitHub publisher off (ingest still stores). + benchmarksDisableEnv = "HIVE_BENCHMARKS_PUBLISH_DISABLED" + // benchmarksMaxFilesPerRun bounds commits per hour so a cold start does + // not burn the installation's write budget in one go. + benchmarksMaxFilesPerRun = 60 +) + +var ( + // benchmarksStoreDir is the on-PVC mirror of the repo's data/fleet tree. + benchmarksStoreDir = "/data/benchmarks" + // benchmarksStatePath remembers the last publish for the status page. + benchmarksStatePath = "/data/benchmarks-publish.json" + // benchmarksGitHubAPI is the Contents API base; a var for tests. + benchmarksGitHubAPI = "https://api.github.com" + hiveHashRe = regexp.MustCompile(`^[0-9a-f]{16}$`) + monthRe = regexp.MustCompile(`^\d{4}-\d{2}$`) +) + +// BenchmarksPublishState is the hub status block. +type BenchmarksPublishState struct { + LastIngestAt string `json:"last_ingest_at,omitempty"` + LastPublishAt string `json:"last_publish_at,omitempty"` + LastPublishFiles int `json:"last_publish_files"` + LastError string `json:"last_error,omitempty"` + Repo string `json:"repo"` + PendingFiles int `json:"pending_files"` + HiveCount int `json:"hive_count"` + RecordCount int `json:"record_count"` + Hives map[string]int `json:"hives,omitempty"` + Enabled bool `json:"enabled"` +} + +type benchmarksHub struct { + mu sync.Mutex + pending map[string]bool // relative file paths (hash/month.json) changed since last publish + state BenchmarksPublishState + loaded bool +} + +var hubBenchmarks = &benchmarksHub{pending: map[string]bool{}} + +func (b *benchmarksHub) load() { + if b.loaded { + return + } + b.loaded = true + data, err := os.ReadFile(benchmarksStatePath) + if err != nil { + return + } + var st struct { + BenchmarksPublishState + Pending []string `json:"pending"` + } + if json.Unmarshal(data, &st) == nil { + b.state = st.BenchmarksPublishState + for _, p := range st.Pending { + b.pending[p] = true + } + } +} + +func (b *benchmarksHub) save() { + var pending []string + for p := range b.pending { + pending = append(pending, p) + } + sort.Strings(pending) + data, err := json.Marshal(struct { + BenchmarksPublishState + Pending []string `json:"pending"` + }{b.state, pending}) + if err != nil { + return + } + _ = os.MkdirAll(filepath.Dir(benchmarksStatePath), 0o755) + _ = os.WriteFile(benchmarksStatePath, data, 0o644) +} + +type benchmarksIngestBody struct { + HiveID string `json:"hive_id"` + HiveHash string `json:"hive_hash"` + Watermark string `json:"watermark"` + Records []benchmarks.Record `json:"records"` +} + +// handleBenchmarksIngest accepts one spoke batch. Auth is the per-hive +// heartbeat bearer bound to hive_id, exactly as NPS ingest. The hive hash is +// recomputed from hive_id, never trusted from the body. +func (s *HubServer) handleBenchmarksIngest(w http.ResponseWriter, r *http.Request) { + if s.hubSecret == "" { + npsJSONError(w, "benchmarks ingest requires a configured hub secret", http.StatusServiceUnavailable) + return + } + auth := r.Header.Get("Authorization") + if !strings.HasPrefix(auth, "Bearer ") { + npsJSONError(w, "unauthorized", http.StatusUnauthorized) + return + } + body, err := io.ReadAll(io.LimitReader(r.Body, benchmarksIngestMaxBodyBytes+1)) + if err != nil { + npsJSONError(w, "read error", http.StatusBadRequest) + return + } + if len(body) > benchmarksIngestMaxBodyBytes { + npsJSONError(w, "payload too large", http.StatusRequestEntityTooLarge) + return + } + var p benchmarksIngestBody + if err := json.Unmarshal(body, &p); err != nil { + npsJSONError(w, "invalid payload", http.StatusBadRequest) + return + } + hiveID := sanitizeHeartbeatField(p.HiveID) + if hiveID == "" || hiveID != p.HiveID || !isValidName(hiveID) { + npsJSONError(w, "invalid hive_id", http.StatusBadRequest) + return + } + if !s.verifyHeartbeatBearer(strings.TrimPrefix(auth, "Bearer "), hiveID) { + npsJSONError(w, "unauthorized", http.StatusUnauthorized) + return + } + if !s.npsHiveRegistered(hiveID) { + npsJSONError(w, "unknown hive - heartbeat first", http.StatusForbidden) + return + } + hash := benchmarks.HiveHash(hiveID) + stored, rejected, err := storeBenchmarkRecords(hash, p.Records) + if err != nil { + if s.logger != nil { + s.logger.Error("benchmarks: store failed", "hive_hash", hash, "error", err) + } + npsJSONError(w, "store failed", http.StatusInternalServerError) + return + } + hubBenchmarks.mu.Lock() + hubBenchmarks.load() + hubBenchmarks.state.LastIngestAt = time.Now().UTC().Format(time.RFC3339) + for _, f := range stored { + hubBenchmarks.pending[f] = true + } + hubBenchmarks.save() + hubBenchmarks.mu.Unlock() + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusAccepted) + _ = json.NewEncoder(w).Encode(map[string]any{"ok": true, "accepted": len(p.Records) - rejected, "rejected": rejected, "files": len(stored)}) +} + +// storeBenchmarkRecords merges records by id into /.json files +// and returns the relative paths it changed. A record whose source.hive_id +// is not the authenticated hash is rejected: a spoke publishes only as itself. +func storeBenchmarkRecords(hash string, recs []benchmarks.Record) (changed []string, rejected int, err error) { + byMonth := map[string][]benchmarks.Record{} + for _, r := range recs { + if r.Source.HiveID != hash || r.ID == "" { + rejected++ + continue + } + r.Source.HiveID = hash + byMonth[benchmarkMonth(r)] = append(byMonth[benchmarkMonth(r)], r) + } + for month, add := range byMonth { + rel := filepath.Join(hash, month+".json") + path := filepath.Join(benchmarksStoreDir, rel) + existing := map[string]benchmarks.Record{} + if data, rerr := os.ReadFile(path); rerr == nil { + var cur []benchmarks.Record + if json.Unmarshal(data, &cur) == nil { + for _, r := range cur { + existing[r.ID] = r + } + } + } + for _, r := range add { + existing[r.ID] = r + } + merged := make([]benchmarks.Record, 0, len(existing)) + for _, r := range existing { + merged = append(merged, r) + } + sort.Slice(merged, func(i, j int) bool { return merged[i].ID < merged[j].ID }) + data, merr := json.Marshal(merged) + if merr != nil { + return changed, rejected, merr + } + if werr := os.MkdirAll(filepath.Dir(path), 0o755); werr != nil { + return changed, rejected, werr + } + if werr := os.WriteFile(path, append(data, '\n'), 0o644); werr != nil { + return changed, rejected, werr + } + changed = append(changed, filepath.ToSlash(rel)) + } + sort.Strings(changed) + return changed, rejected, nil +} + +// benchmarkMonth files a record under the month it merged, else opened, +// else was published. +func benchmarkMonth(r benchmarks.Record) string { + for _, ts := range []string{r.Outcome.MergedAt, r.Outcome.OpenedAt, r.PublishedAt} { + if t, err := time.Parse(time.RFC3339, ts); err == nil { + return t.UTC().Format("2006-01") + } + } + return time.Now().UTC().Format("2006-01") +} + +// benchmarksTargetRepo is owner/name of the data repo. +func benchmarksTargetRepo() string { + if v := strings.TrimSpace(os.Getenv(benchmarksRepoEnv)); v != "" { + return v + } + return benchmarksRepoDefault +} + +func benchmarksPublishEnabled() bool { + return strings.TrimSpace(os.Getenv(benchmarksDisableEnv)) == "" +} + +// StartBenchmarksPublisher commits pending month files to the data repo +// every hour. Registered from StartBackgroundPollers. +func (s *HubServer) StartBenchmarksPublisher(ctx context.Context) { + select { + case <-ctx.Done(): + return + case <-time.After(benchmarksPublishInitialDelay): + } + ticker := time.NewTicker(benchmarksPublishInterval) + defer ticker.Stop() + for { + s.publishBenchmarks(ctx) + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + } +} + +// publishBenchmarks pushes every pending file; a file that fails stays +// pending for the next run. +func (s *HubServer) publishBenchmarks(ctx context.Context) { + logger := s.logger + if logger == nil { + logger = slog.Default() + } + if !benchmarksPublishEnabled() { + return + } + hubBenchmarks.mu.Lock() + hubBenchmarks.load() + var files []string + for f := range hubBenchmarks.pending { + files = append(files, f) + } + hubBenchmarks.mu.Unlock() + if len(files) == 0 { + return + } + sort.Strings(files) + if len(files) > benchmarksMaxFilesPerRun { + files = files[:benchmarksMaxFilesPerRun] + } + token := hubGitHubAuthTokenForRequest(ctx) + if token == "" { + s.benchmarksSetError("no hub GitHub identity (App or token) configured") + return + } + client := hubGitHubHTTPClient() + repo := benchmarksTargetRepo() + done := 0 + var lastErr error + for _, rel := range files { + if err := ctx.Err(); err != nil { + return + } + if err := commitBenchmarkFile(ctx, client, token, repo, rel); err != nil { + lastErr = err + logger.Warn("benchmarks: publish failed", "file", rel, "error", err) + continue + } + done++ + hubBenchmarks.mu.Lock() + delete(hubBenchmarks.pending, rel) + hubBenchmarks.mu.Unlock() + } + hubBenchmarks.mu.Lock() + hubBenchmarks.state.LastPublishAt = time.Now().UTC().Format(time.RFC3339) + hubBenchmarks.state.LastPublishFiles = done + hubBenchmarks.state.LastError = "" + if lastErr != nil { + hubBenchmarks.state.LastError = lastErr.Error() + } + hubBenchmarks.save() + hubBenchmarks.mu.Unlock() + logger.Info("benchmarks: published to data repo", "repo", repo, "files", done, "failed", len(files)-done) +} + +func (s *HubServer) benchmarksSetError(msg string) { + hubBenchmarks.mu.Lock() + hubBenchmarks.load() + hubBenchmarks.state.LastError = msg + hubBenchmarks.save() + hubBenchmarks.mu.Unlock() +} + +// commitBenchmarkFile PUTs one file through the Contents API, reading the +// current blob sha first so the write is an update, not a conflict. +func commitBenchmarkFile(ctx context.Context, client *http.Client, token, repo, rel string) error { + if !validBenchmarkRel(rel) { + return fmt.Errorf("refusing to publish path %q", rel) + } + data, err := os.ReadFile(filepath.Join(benchmarksStoreDir, filepath.FromSlash(rel))) + if err != nil { + return err + } + url := fmt.Sprintf("%s/repos/%s/contents/%s/%s", benchmarksGitHubAPI, repo, benchmarksDataPrefix, rel) + cctx, cancel := context.WithTimeout(hubGitHubCallerContext(ctx, "benchmarks"), benchmarksCommitTimeout) + defer cancel() + sha := "" + req, _ := http.NewRequestWithContext(cctx, http.MethodGet, url, nil) + req.Header.Set("Authorization", "Bearer "+token) + req.Header.Set("Accept", "application/vnd.github+json") + if resp, gerr := client.Do(req); gerr == nil { + var cur struct { + SHA string `json:"sha"` + Content string `json:"content"` + } + _ = json.NewDecoder(io.LimitReader(resp.Body, 16<<20)).Decode(&cur) + _ = resp.Body.Close() + if resp.StatusCode == http.StatusOK { + sha = cur.SHA + if existing, derr := base64.StdEncoding.DecodeString(strings.ReplaceAll(cur.Content, "\n", "")); derr == nil && bytes.Equal(existing, data) { + return nil // already published + } + } + } + hash := strings.SplitN(rel, "/", 2)[0] + payload := map[string]any{ + "message": fmt.Sprintf("🌱 fleet: hourly benchmark records for %s (%s)", hash, strings.TrimSuffix(filepath.Base(rel), ".json")), + "content": base64.StdEncoding.EncodeToString(data), + } + if sha != "" { + payload["sha"] = sha + } + body, _ := json.Marshal(payload) + req, err = http.NewRequestWithContext(cctx, http.MethodPut, url, bytes.NewReader(body)) + if err != nil { + return err + } + req.Header.Set("Authorization", "Bearer "+token) + req.Header.Set("Accept", "application/vnd.github+json") + req.Header.Set("Content-Type", "application/json") + resp, err := client.Do(req) + if err != nil { + return err + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated { + msg, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) + return fmt.Errorf("contents API %s: %s", resp.Status, strings.TrimSpace(string(msg))) + } + return nil +} + +func validBenchmarkRel(rel string) bool { + hash, file, ok := strings.Cut(rel, "/") + return ok && hiveHashRe.MatchString(hash) && strings.HasSuffix(file, ".json") && monthRe.MatchString(strings.TrimSuffix(file, ".json")) +} + +// handleBenchmarksStatus serves GET /api/hub/benchmarks: last ingest and +// publish times plus per-hive record counts from the on-disk mirror. +func (s *HubServer) handleBenchmarksStatus(w http.ResponseWriter, _ *http.Request) { + hubBenchmarks.mu.Lock() + hubBenchmarks.load() + st := hubBenchmarks.state + st.PendingFiles = len(hubBenchmarks.pending) + hubBenchmarks.mu.Unlock() + st.Repo = benchmarksTargetRepo() + st.Enabled = benchmarksPublishEnabled() + st.Hives = map[string]int{} + entries, err := os.ReadDir(benchmarksStoreDir) + if err != nil && !errors.Is(err, os.ErrNotExist) { + npsJSONError(w, "read store", http.StatusInternalServerError) + return + } + for _, e := range entries { + if !e.IsDir() || !hiveHashRe.MatchString(e.Name()) { + continue + } + n := 0 + months, _ := os.ReadDir(filepath.Join(benchmarksStoreDir, e.Name())) + for _, m := range months { + data, rerr := os.ReadFile(filepath.Join(benchmarksStoreDir, e.Name(), m.Name())) + if rerr != nil { + continue + } + var recs []json.RawMessage + if json.Unmarshal(data, &recs) == nil { + n += len(recs) + } + } + st.Hives[e.Name()] = n + st.RecordCount += n + } + st.HiveCount = len(st.Hives) + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(st) +} diff --git a/src/pkg/hub/benchmarks_test.go b/src/pkg/hub/benchmarks_test.go new file mode 100644 index 0000000000..73f5a50c55 --- /dev/null +++ b/src/pkg/hub/benchmarks_test.go @@ -0,0 +1,253 @@ +package hub + +import ( + "context" + "encoding/base64" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/evidence/benchmarks" +) + +func useTempBenchmarksStore(t *testing.T) string { + t.Helper() + dir := t.TempDir() + oldDir, oldState := benchmarksStoreDir, benchmarksStatePath + benchmarksStoreDir = filepath.Join(dir, "benchmarks") + benchmarksStatePath = filepath.Join(dir, "state.json") + hubBenchmarks.mu.Lock() + hubBenchmarks.pending = map[string]bool{} + hubBenchmarks.state = BenchmarksPublishState{} + hubBenchmarks.loaded = true + hubBenchmarks.mu.Unlock() + t.Cleanup(func() { + benchmarksStoreDir, benchmarksStatePath = oldDir, oldState + hubBenchmarks.mu.Lock() + hubBenchmarks.pending = map[string]bool{} + hubBenchmarks.state = BenchmarksPublishState{} + hubBenchmarks.loaded = false + hubBenchmarks.mu.Unlock() + }) + return dir +} + +func benchRecord(hash, id, mergedAt string) benchmarks.Record { + return benchmarks.Record{ + SchemaVersion: benchmarks.SchemaVersion, ID: id, PublishedAt: "2026-11-11T00:00:00Z", + Source: benchmarks.Source{HiveID: hash, HiveVersion: "v5"}, + Agent: benchmarks.Agent{Name: "a", Role: "author", Harness: "copilot"}, + Model: benchmarks.Model{Provider: "anthropic", Name: "claude-fable-5.1"}, + SandboxRuntime: "none", Task: benchmarks.Task{Kind: "docs"}, + Outcome: benchmarks.Outcome{PROpened: true, PRMerged: mergedAt != "", MergedAt: mergedAt, CIStatusAtMerge: "unknown", ReviewVerdict: "none"}, + } +} + +func benchIngest(s *HubServer, body string, bearer string) *httptest.ResponseRecorder { + req := httptest.NewRequest(http.MethodPost, benchmarksIngestPath, strings.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + if bearer != "" { + req.Header.Set("Authorization", "Bearer "+bearer) + } + rec := httptest.NewRecorder() + s.handleBenchmarksIngest(rec, req) + return rec +} + +func TestBenchmarksIngestStoresByMonthAndRejectsForeignHash(t *testing.T) { + useTempBenchmarksStore(t) + s := npsTestHub("h1") + hash := benchmarks.HiveHash("h1") + payload := benchmarksIngestBody{HiveID: "h1", Records: []benchmarks.Record{ + benchRecord(hash, "aaa", "2026-10-02T00:00:00Z"), + benchRecord(hash, "bbb", "2026-11-05T00:00:00Z"), + benchRecord(benchmarks.HiveHash("h2"), "ccc", "2026-11-05T00:00:00Z"), + }} + body, _ := json.Marshal(payload) + rec := benchIngest(s, string(body), s.heartbeatKeyFor("h1")) + if rec.Code != http.StatusAccepted { + t.Fatalf("code=%d body=%s", rec.Code, rec.Body.String()) + } + if !strings.Contains(rec.Body.String(), `"accepted":2`) || !strings.Contains(rec.Body.String(), `"rejected":1`) { + t.Errorf("response: %s", rec.Body.String()) + } + for _, f := range []string{"2026-10.json", "2026-11.json"} { + if _, err := os.Stat(filepath.Join(benchmarksStoreDir, hash, f)); err != nil { + t.Errorf("missing %s: %v", f, err) + } + } + if _, err := os.Stat(filepath.Join(benchmarksStoreDir, benchmarks.HiveHash("h2"))); err == nil { + t.Error("a record claiming another hive's hash was stored") + } + // Re-ingesting the same id merges rather than duplicates. + benchIngest(s, string(body), s.heartbeatKeyFor("h1")) + data, _ := os.ReadFile(filepath.Join(benchmarksStoreDir, hash, "2026-11.json")) + var recs []benchmarks.Record + _ = json.Unmarshal(data, &recs) + if len(recs) != 1 { + t.Errorf("want 1 record after re-ingest, got %d", len(recs)) + } + hubBenchmarks.mu.Lock() + pending := len(hubBenchmarks.pending) + hubBenchmarks.mu.Unlock() + if pending != 2 { + t.Errorf("pending files = %d, want 2", pending) + } + + // Status reflects the store. + st := httptest.NewRecorder() + s.handleBenchmarksStatus(st, httptest.NewRequest(http.MethodGet, benchmarksStatusPath, nil)) + var status BenchmarksPublishState + _ = json.Unmarshal(st.Body.Bytes(), &status) + if status.HiveCount != 1 || status.RecordCount != 2 || status.PendingFiles != 2 || status.Hives[hash] != 2 { + t.Errorf("status = %+v", status) + } +} + +func TestBenchmarksIngestAuth(t *testing.T) { + useTempBenchmarksStore(t) + s := npsTestHub("h1", "h2") + body := `{"hive_id":"h1","records":[]}` + for name, bearer := range map[string]string{"none": "", "other hive": s.heartbeatKeyFor("h2"), "fleet key": s.heartbeatKey()} { + if rec := benchIngest(s, body, bearer); rec.Code != http.StatusUnauthorized { + t.Errorf("%s: code=%d want 401", name, rec.Code) + } + } + if rec := benchIngest(s, `{"hive_id":"h3","records":[]}`, s.heartbeatKeyFor("h3")); rec.Code != http.StatusForbidden { + t.Errorf("unregistered hive: code=%d want 403", rec.Code) + } +} + +func TestPublishBenchmarksCommitsPendingFiles(t *testing.T) { + useTempBenchmarksStore(t) + hash := benchmarks.HiveHash("h1") + changed, _, err := storeBenchmarkRecords(hash, []benchmarks.Record{benchRecord(hash, "aaa", "2026-11-05T00:00:00Z")}) + if err != nil || len(changed) != 1 { + t.Fatalf("store: %v %v", changed, err) + } + hubBenchmarks.mu.Lock() + hubBenchmarks.pending[changed[0]] = true + hubBenchmarks.mu.Unlock() + + var puts atomic.Int32 + var gotPath, gotSHA string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !strings.HasPrefix(r.Header.Get("Authorization"), "Bearer tok") { + w.WriteHeader(http.StatusUnauthorized) + return + } + switch r.Method { + case http.MethodGet: + _ = json.NewEncoder(w).Encode(map[string]string{"sha": "oldsha", "content": base64.StdEncoding.EncodeToString([]byte("[]"))}) + case http.MethodPut: + puts.Add(1) + var p map[string]any + _ = json.NewDecoder(r.Body).Decode(&p) + gotPath = r.URL.Path + gotSHA, _ = p["sha"].(string) + w.WriteHeader(http.StatusOK) + } + })) + defer srv.Close() + oldAPI, oldTok := benchmarksGitHubAPI, hubGitHubAuthTokenForRequest + benchmarksGitHubAPI = srv.URL + hubGitHubAuthTokenForRequest = func(ctx context.Context) string { return "tok" } + t.Cleanup(func() { benchmarksGitHubAPI, hubGitHubAuthTokenForRequest = oldAPI, oldTok }) + t.Setenv(benchmarksRepoEnv, "example/data") + + s := newTestHubServer("secret") + s.publishBenchmarks(t.Context()) + if puts.Load() != 1 { + t.Fatalf("puts = %d, want 1", puts.Load()) + } + if want := "/repos/example/data/contents/data/fleet/" + hash + "/2026-11.json"; gotPath != want { + t.Errorf("path = %s, want %s", gotPath, want) + } + if gotSHA != "oldsha" { + t.Errorf("update must carry the existing blob sha, got %q", gotSHA) + } + hubBenchmarks.mu.Lock() + defer hubBenchmarks.mu.Unlock() + if len(hubBenchmarks.pending) != 0 || hubBenchmarks.state.LastPublishFiles != 1 || hubBenchmarks.state.LastPublishAt == "" { + t.Errorf("state after publish: pending=%v state=%+v", hubBenchmarks.pending, hubBenchmarks.state) + } +} + +func TestValidBenchmarkRel(t *testing.T) { + for rel, ok := range map[string]bool{"0123456789abcdef/2026-11.json": true, "../x/2026-11.json": false, "0123456789abcdef/evil.json": false, "0123456789abcdef/2026-11.json/..": false} { + if validBenchmarkRel(rel) != ok { + t.Errorf("%q: want %v", rel, ok) + } + } +} + +func TestBenchmarksIngestRejectsMalformedBodies(t *testing.T) { + useTempBenchmarksStore(t) + s := npsTestHub("h1") + for name, tc := range map[string]struct { + body string + want int + }{ + "not json": {"{", http.StatusBadRequest}, + "bad hive id": {`{"hive_id":"../x","records":[]}`, http.StatusBadRequest}, + "too large": {`{"hive_id":"h1","records":[` + strings.Repeat(`{},`, benchmarksIngestMaxBodyBytes/3) + `{}]}`, http.StatusRequestEntityTooLarge}, + "publish is off": {`{"hive_id":"h1","records":[]}`, http.StatusAccepted}, + } { + if rec := benchIngest(s, tc.body, s.heartbeatKeyFor("h1")); rec.Code != tc.want { + t.Errorf("%s: code=%d want %d (%s)", name, rec.Code, tc.want, rec.Body.String()) + } + } + nosecret := newTestHubServer("") + if rec := benchIngest(nosecret, `{"hive_id":"h1","records":[]}`, "x"); rec.Code != http.StatusServiceUnavailable { + t.Errorf("no hub secret: code=%d", rec.Code) + } +} + +func TestBenchmarksStatePersistsAcrossLoad(t *testing.T) { + useTempBenchmarksStore(t) + hash := benchmarks.HiveHash("h1") + changed, _, err := storeBenchmarkRecords(hash, []benchmarks.Record{benchRecord(hash, "aaa", "2026-11-05T00:00:00Z")}) + if err != nil { + t.Fatal(err) + } + hubBenchmarks.mu.Lock() + hubBenchmarks.pending[changed[0]] = true + hubBenchmarks.state.LastIngestAt = "2026-11-11T00:00:00Z" + hubBenchmarks.save() + hubBenchmarks.pending = map[string]bool{} + hubBenchmarks.state = BenchmarksPublishState{} + hubBenchmarks.loaded = false + hubBenchmarks.load() + defer hubBenchmarks.mu.Unlock() + if !hubBenchmarks.pending[changed[0]] || hubBenchmarks.state.LastIngestAt != "2026-11-11T00:00:00Z" { + t.Fatalf("state did not round-trip: pending=%v state=%+v", hubBenchmarks.pending, hubBenchmarks.state) + } +} + +func TestPublishBenchmarksRecordsMissingIdentity(t *testing.T) { + useTempBenchmarksStore(t) + hash := benchmarks.HiveHash("h1") + changed, _, _ := storeBenchmarkRecords(hash, []benchmarks.Record{benchRecord(hash, "aaa", "")}) + hubBenchmarks.mu.Lock() + hubBenchmarks.pending[changed[0]] = true + hubBenchmarks.mu.Unlock() + oldTok := hubGitHubAuthTokenForRequest + hubGitHubAuthTokenForRequest = func(ctx context.Context) string { return "" } + t.Cleanup(func() { hubGitHubAuthTokenForRequest = oldTok }) + s := newTestHubServer("secret") + s.publishBenchmarks(t.Context()) + hubBenchmarks.mu.Lock() + defer hubBenchmarks.mu.Unlock() + if hubBenchmarks.state.LastError == "" || len(hubBenchmarks.pending) != 1 { + t.Fatalf("missing identity must be surfaced and keep files pending: %+v pending=%v", hubBenchmarks.state, hubBenchmarks.pending) + } + if benchmarkMonth(benchmarks.Record{}) != time.Now().UTC().Format("2006-01") { + t.Error("record without timestamps must land in the current month") + } +} diff --git a/src/pkg/hub/server.go b/src/pkg/hub/server.go index d107147380..724c4c15e1 100644 --- a/src/pkg/hub/server.go +++ b/src/pkg/hub/server.go @@ -1721,6 +1721,9 @@ func NewHubServer(port int, logger *slog.Logger, gitHash, gitBranch string) *Hub // same credential as /api/task-status. The only read path is the admin-gated // GET registered in registerSaaSRoutes. s.mux.HandleFunc("POST "+npsIngestPath, s.handleNPSIngest) + // Fleet benchmark records from spokes (hivecommons/hive#17694). + s.mux.HandleFunc("POST "+benchmarksIngestPath, s.handleBenchmarksIngest) + s.mux.HandleFunc("GET "+benchmarksStatusPath, s.handleBenchmarksStatus) s.mux.HandleFunc("POST "+feedbackIngestPath, s.handleFeedbackIngest) s.mux.HandleFunc("POST "+selfReportIngestPath, s.handleSelfReportIngest) s.mux.HandleFunc("GET "+feedbackIssuesPath, s.handleFeedbackIssues)