Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/added-18717-pr-models-hive-cost.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- **Governor PRs by model: what each model's PRs cost the hive to review (#18717).** Each model bucket in `GET /api/governor/pr-models` now carries `hive_cost` (`usd`, `launches`, `prs_with_cost`, `usd_per_merged_pr`): the hive's own estimated spend on reviewing, adjudicating and fixing that model's PRs, summed from the PR refs' `share_usd` in the cached launch-cost snapshot (#12128). `usd` is `null`, not `0`, when none of the model's PRs had attributed usage, and `hive_cost` is omitted while the snapshot is not ready. A top-level `hive_cost_coverage` reports the launch-cost window, the unattributed and backend-unsupported spend, and the estimate disclaimer. The card shows the figure in each row's tooltip with a coverage note under the header. It reads only the cached snapshot: no new token scanning or GitHub calls, and existing fields are unchanged.
89 changes: 89 additions & 0 deletions dashboard/test/governor-pr-models-hive-cost-ui.test.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
'use strict';

// Governor PRs by model: hive review cost per model (#18717).

const { test } = require('node:test');
const assert = require('node:assert/strict');
const fs = require('node:fs');
const path = require('node:path');
const vm = require('node:vm');

const indexHtml = fs.readFileSync(path.join(__dirname, '..', '..', 'src', 'pkg', 'dashboard', 'static', 'index.html'), 'utf8');

function extractFunction(name) {
const start = indexHtml.indexOf('function ' + name + '(');
assert.notEqual(start, -1, `${name} function exists`);
const brace = indexHtml.indexOf('{', start);
let depth = 0;
for (let i = brace; i < indexHtml.length; i++) {
const ch = indexHtml[i];
if (ch === '{') depth++;
if (ch === '}') depth--;
if (depth === 0) return indexHtml.slice(start, i + 1);
}
throw new Error(`unterminated ${name}`);
}

function load() {
const context = { exports: {} };
vm.runInNewContext([
"function fmtUSD(n) { return n == null ? '—' : '$' + Number(n).toFixed(2); }",
"function escapeHtml(s) { return String(s).replace(/&/g, '&amp;').replace(/</g, '&lt;'); }",
extractFunction('governorPRModelHiveCostTip'),
extractFunction('governorPRModelsHiveCostNote'),
'exports.tip = governorPRModelHiveCostTip;',
'exports.note = governorPRModelsHiveCostNote;',
].join('\n'), context);
return context.exports;
}

const disclaimer = 'Estimated from token counts × published list prices. Not a bill.';
const data = {
total: 4,
hive_cost_coverage: { ready: true, window_hours: 336, unattributed_usd: 7.25, backend_unsupported_usd: null, disclaimer },
};

test('row tooltip shows the hive review cost and the disclaimer', () => {
const { tip } = load();
const out = tip({ hive_cost: { usd: 2, launches: 3, prs_with_cost: 2, usd_per_merged_pr: 1 } }, data);
assert.match(out, /Hive review cost: \$2\.00 est\. over 2 PRs with launch records, 3 launches · \$1\.00 per merged PR/);
assert.match(out, /launch-cost window: last 14d/);
assert.ok(out.includes(disclaimer), 'disclaimer travels with the figure');
assert.ok(!out.includes('not $0'), 'no null note for a measured figure');
});

test('null usd renders as a dash, never $0', () => {
const { tip } = load();
const out = tip({ hive_cost: { usd: null, launches: 1, prs_with_cost: 1, usd_per_merged_pr: null } }, data);
assert.match(out, /Hive review cost: — est\./);
assert.ok(out.includes('no attributed usage, not $0'));
assert.ok(!out.includes('per merged PR'));
assert.ok(!out.includes('$0.00'));
});

test('nothing renders when the snapshot is not ready or hive_cost is absent', () => {
const { tip, note } = load();
assert.equal(tip({}, data), '');
assert.equal(tip({ hive_cost: { usd: 1 } }, { total: 1, hive_cost_coverage: { ready: false } }), '');
assert.equal(note({ total: 1, hive_cost_coverage: { ready: false, disclaimer } }), '');
assert.equal(note({ total: 1 }), '');
});

test('coverage note names the window, the unattributed spend and the disclaimer', () => {
const { note } = load();
const out = note(data);
assert.match(out, /estimate over the last 14d of launch records/);
assert.match(out, /not tied to any PR: \$7\.25 unattributed/);
assert.ok(!out.includes('backends without a usage timeline'), 'null bucket is not shown as $0');
assert.ok(out.includes(disclaimer));
});

test('the tile wires the hive cost into the row tooltip and the coverage notes', () => {
for (const expected of [
'${governorPRModelHiveCostTip(b, data)}',
'const hiveCostNote = governorPRModelsHiveCostNote(data);',
'${hiveCostNote}',
]) {
assert.ok(indexHtml.includes(expected), `index.html missing ${expected}`);
}
});
2 changes: 1 addition & 1 deletion src/docs/api-reference/agents-and-controls.md
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f
| `POST` | `/api/unpin/{agent}/{dimension}` | Owner only | Unpin | `pkg/dashboard/api.go:135` |
| `POST` | `/api/restart/{agent}` | Owner only | Restart | `pkg/dashboard/api.go:139` |
| `GET` | `/api/model-advisor` | Dashboard auth/session | Model Advisor | `pkg/dashboard/api.go:156` |
| `GET` | `/api/governor/pr-models` | Dashboard auth/session | Agent-authored PR distribution by normalized attribution model/backend for `window=7d`, `30d`, or `all`, including per-model rework stats, Wilson 95% `*_ci` intervals on the rates (omitted when the denominator is 0), a `coverage` block (closed-PR lookback and run-log partiality, as on `/api/contribute/effective-models`), `earliest_pr_created_at`, and the top 10 most-reworked PRs. Optional `model=<id>` (matched after `modelfamily.Canonical`, so raw spellings such as `anthropic/claude-opus-5-5` work) adds `prs[]`: every PR in the window in that model's bucket, with the `most_reworked` fields plus `outcome` (`merged` / `closed_unmerged` / `open`), `agent`, `created_at`, `merged_at` and `first_pass`, sorted by review rounds + fix attempts then newest, capped at 200 with `prs_truncated: true` past that. An unknown model returns `prs: []`; without `model` the response is unchanged | `pkg/dashboard/api.go:157` |
| `GET` | `/api/governor/pr-models` | Dashboard auth/session | Agent-authored PR distribution by normalized attribution model/backend for `window=7d`, `30d`, or `all`, including per-model rework stats, Wilson 95% `*_ci` intervals on the rates (omitted when the denominator is 0), a `coverage` block (closed-PR lookback and run-log partiality, as on `/api/contribute/effective-models`), `earliest_pr_created_at`, the top 10 most-reworked PRs, and per-bucket `hive_cost` (`usd`, `launches`, `prs_with_cost`, `usd_per_merged_pr`: the hive's estimated review/fix spend on that model's PRs from the cached `/api/launch-cost` snapshot; `usd` is `null`, not `0`, when nothing was attributed; omitted while the snapshot is not ready) with a top-level `hive_cost_coverage` (`ready`, `window_hours`, `unattributed_usd`, `backend_unsupported_usd`, `disclaimer`). Optional `model=<id>` (matched after `modelfamily.Canonical`, so raw spellings such as `anthropic/claude-opus-5-5` work) adds `prs[]`: every PR in the window in that model's bucket, with the `most_reworked` fields plus `outcome` (`merged` / `closed_unmerged` / `open`), `agent`, `created_at`, `merged_at` and `first_pass`, sorted by review rounds + fix attempts then newest, capped at 200 with `prs_truncated: true` past that. An unknown model returns `prs: []`; without `model` the response is unchanged | `pkg/dashboard/api.go:157` |
| `GET` | `/api/reviewer/accuracy` | Dashboard auth/session | Reviewer verdict calibration for the configured window: false-approve/false-block rates and confidence buckets per perspective and reviewer model | `pkg/dashboard/api.go:220` |
| `GET` | `/api/review/queue` | Dashboard auth/session | PR review queue: every open PR (agent and contributor, actionable and held) ranked by triage class, confidence band, CI state and age, with per-PR `reasons`; unreviewed items carry `pending_since` and `redispatch_count` from the review dispatch state; each entry carries `stage` `{stage, entered_at, last_activity_at, quiet_for, stalled, waiting_on_hive, past_dwell, history[]}` from the stage-label store (`stalled`/`past_dwell` follow `governor.labels.stage.dwell` and `stalled.require_quiet`; `history[]` is `{from, to, at, reason, regressed}`, newest last, capped at 8) and the top-level `hive_held` counts PRs past their stage dwell while waiting on the hive (`waiting_on_hive` and `past_dwell` both true); paged with `limit` (1-200, default 50) and `offset` | `pkg/dashboard/api.go:502` |
| `POST` | `/api/review/queue/{owner}/{repo}/{number}/superseded/close` | Owner only | Close a superseded PR now (grace window or not): posts the close comment, adds `hive/superseded`, closes it | `pkg/dashboard/api.go:505` |
Expand Down
6 changes: 6 additions & 0 deletions src/docs/dashboard.md
Original file line number Diff line number Diff line change
Expand Up @@ -179,6 +179,12 @@ selected window (outcome, review rounds, fix attempts, first pass, author agent
and backend, each linked to GitHub). The list is fetched on demand with
`GET /api/governor/pr-models?window=…&model=<model>` and cached per window and
model.
Each model row's tooltip also shows **Hive review cost**: what the hive's own
agents (reviewers, adjudicator, fixers) are estimated to have spent on that
model's PRs, joined from the cached launch-cost snapshot (see
[Launch attribution](token-tracking.md#launch-attribution)). A note under the
header names the launch-cost window and the spend that could not be tied to any
PR. "—" means no attributed usage, not $0.

The Governor **actionable issues** and **actionable PRs** tiles use the shared
server-side actionable partition described in [Repository card legend, issue
Expand Down
2 changes: 1 addition & 1 deletion src/docs/token-tracking.md
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ The kick path records a `LaunchRecord` (`pkg/tokens/launch.go`) each time the go
- USD is `null` when nothing was attributed — never `$0.00`.
- `?ref=owner/repo#N` narrows `by_ref` and `by_launch` to one work item; totals stay whole-window.

The dashboard Cost panel renders the top rows as "cost by work item". Manual `send-kick` and startup prompts do not record launches, so their spend stays unattributed. This is visibility only: nothing here gates or budgets a kick, and the governor window budget remains the single enforced budget.
The dashboard Cost panel renders the top rows as "cost by work item". The Governor **PRs by model** card (`GET /api/governor/pr-models`) sums the PR refs' `share_usd` per authoring model as `hive_cost` (#18717): refs with `kind: "pr"` only, joined on the canonical `owner/repo#N` (a short configured repo name joins on the repo name), never `launch_usd`, and `null` when no matched ref carried usage. `hive_cost_coverage` copies the snapshot's `window_hours`, `unattributed` and `backend_unsupported` USD and the disclaimer; on backends without a usage timeline most models show `null`. Manual `send-kick` and startup prompts do not record launches, so their spend stays unattributed. This is visibility only: nothing here gates or budgets a kick, and the governor window budget remains the single enforced budget.

## Metering diagnostics and zero-token health

Expand Down
17 changes: 10 additions & 7 deletions src/pkg/dashboard/api_governor_pr_models.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,10 @@ type governorPRModelsResponse struct {
// created_at among the PRs counted, omitted when there are none.
Coverage contributeEffectiveModelsCoverage `json:"coverage"`
EarliestPRCreatedAt string `json:"earliest_pr_created_at,omitempty"`
// HiveCostCoverage describes the launch-cost snapshot behind each
// bucket's hive_cost (#18717). Always present; Ready=false means no
// bucket carries hive_cost.
HiveCostCoverage governorPRModelsHiveCostCoverage `json:"hive_cost_coverage"`
}

// governorPRModelBucket is one canonical model (modelfamily.Canonical folds
Expand All @@ -57,6 +61,9 @@ type governorPRModelBucket struct {
// Efforts splits the bucket by the trailer's effort= value ("unknown"
// when absent), sorted by PRs descending (hivecommons/hive#18715).
Efforts []governorPRModelEffort `json:"efforts,omitempty"`
// HiveCost is what this model's PRs cost the hive to review and repair
// (#18717); omitted when the launch-cost snapshot is not ready.
HiveCost *governorPRModelHiveCost `json:"hive_cost,omitempty"`

// ClosedUnmergedReasons splits ClosedUnmerged by why the PR was closed
// (#18719); the three counts always sum to ClosedUnmerged.
Expand Down Expand Up @@ -265,6 +272,8 @@ func (s *Server) handleGovernorPRModels(w http.ResponseWriter, r *http.Request)
resp.Coverage = effectiveModelsCoverage(governorPRModelsWindowDuration(window), ghpkg.AttributedClosedPRLookback(), runs, rotated, now)
effective := aggregateEffectiveModelsFromPRAggregation(resp, aggregateEffectiveRunStats(runs, effectiveModelsFilterAll), effectiveModelsFilterAll, minMergedPRs)
applyGovernorPRModelEffectiveness(&resp, effective, minMergedPRs)
launchCost, launchCostReady := s.launchCostSnapshotForPRModels()
applyGovernorPRModelHiveCost(&resp, actionable.PRs.Attributed, launchCost, launchCostReady, now)
w.Header().Set("Content-Type", "application/json")
if model, ok := governorPRModelsQueryModel(r); ok {
_ = json.NewEncoder(w).Encode(withGovernorPRModelPRs(resp, actionable.PRs.Attributed, model, now))
Expand Down Expand Up @@ -296,13 +305,7 @@ func normalizeGovernorPRModelsWindow(window string) string {

func aggregateGovernorPRModels(prs []ghpkg.PullRequest, window string, now time.Time) governorPRModelsResponse {
window = normalizeGovernorPRModelsWindow(window)
cutoff := time.Time{}
switch window {
case governorPRModelsWindow7d:
cutoff = now.AddDate(0, 0, -governorPRModelsDays7)
case governorPRModelsWindow30d:
cutoff = now.AddDate(0, 0, -governorPRModelsDays30)
}
cutoff := governorPRModelsCutoff(window, now)

type bucketState struct {
governorPRModelBucket
Expand Down
Loading
Loading