From 4e83e31490eaf3598222b80342e495a059ba11cf Mon Sep 17 00:00:00 2001 From: Danathar <6772335+Danathar@users.noreply.github.com> Date: Tue, 17 Nov 2026 11:58:31 -0500 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20Contributor=20run=20stats=20by=20mo?= =?UTF-8?q?del:=20why=20runs=20failed?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each per-model row of /api/contribute/run-stats/models (and the models array of /api/contribute/run-stats) gains abandoned, scenarios (same keys as backends[].scenarios, summing to runs) and failure_kinds (environment/task/unspecified, summing to the failed runs). failure_rate and existing fields are unchanged. The contributor wall's "failed" model-evidence figure shows the split as a tooltip. Refs #18721 Signed-off-by: Danathar <6772335+Danathar@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WcWm7qFmFJp26r9LPdmi3t --- ...added-18721-run-stats-failure-scenarios.md | 1 + dashboard/openapi.json | 2 +- src/docs/api-reference/contribute.md | 2 +- src/docs/backend-smoke.md | 10 ++ src/pkg/dashboard/contribute_landing.go | 19 +++- src/pkg/dashboard/contribute_wall.go | 46 ++++++++++ .../contribute_wall_evidence.test.js | 61 +++++++++++++ src/pkg/dashboard/contribute_wall_test.go | 91 +++++++++++++++++++ src/pkg/dashboard/task_run_log_test.go | 61 +++++++++++++ 9 files changed, 290 insertions(+), 3 deletions(-) create mode 100644 changelog.d/added-18721-run-stats-failure-scenarios.md create mode 100644 src/pkg/dashboard/contribute_wall_evidence.test.js diff --git a/changelog.d/added-18721-run-stats-failure-scenarios.md b/changelog.d/added-18721-run-stats-failure-scenarios.md new file mode 100644 index 0000000000..fd9bb5752d --- /dev/null +++ b/changelog.d/added-18721-run-stats-failure-scenarios.md @@ -0,0 +1 @@ +- **Contributor run stats by model now say why runs failed.** Each model row of `GET /api/contribute/run-stats/models` (and the `models` array of `GET /api/contribute/run-stats`) gains `abandoned`, `scenarios` (the same scenario counts run-stats already gives per backend, summing to `runs`) and `failure_kinds` (`environment`, `task`, `unspecified`, summing to the failed runs). A high failure rate can now be read as "mostly the contributors' environments" or "mostly the work". `failure_rate` and the other existing fields are unchanged. On the contributor wall, hovering a post's "failed" model-evidence figure shows the split, with the labels the run history uses. (#18721) diff --git a/dashboard/openapi.json b/dashboard/openapi.json index 14373a6618..9e1cf22101 100644 --- a/dashboard/openapi.json +++ b/dashboard/openapi.json @@ -37667,7 +37667,7 @@ "Contribute" ], "summary": "Contributor run stats by model", - "description": "Returns per-model run counts, verified-PR share, and failure rate for contributor-wall evidence.", + "description": "Returns per-model run counts, verified-PR share, and failure rate for contributor-wall evidence, plus why runs ended (#18721): `abandoned` (count), `scenarios` (scenario -> count, the same keys as /api/contribute/run-stats backends[].scenarios, summing to runs) and `failure_kinds` ({environment, task, unspecified}, summing to the failed runs behind failure_rate).", "responses": { "200": { "description": "OK" diff --git a/src/docs/api-reference/contribute.md b/src/docs/api-reference/contribute.md index 05fca9515e..2de912ceeb 100644 --- a/src/docs/api-reference/contribute.md +++ b/src/docs/api-reference/contribute.md @@ -21,7 +21,7 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/contribute/limits` | Public | Contribute Limits | `pkg/dashboard/api_contribute.go:275` | | `GET` | `/api/contribute/metrics` | Public | Contribute Metrics | `pkg/dashboard/api_contribute.go:281` | | `GET` | `/api/contribute/me` | Public path; caller identity resolved server-side (401 anonymous; a signed-in caller with no profile on this hive gets 200 with `has_profile:false` and no counters, [#12111](https://github.com/hivecommons/hive/issues/12111)) | The caller's OWN contribution stats — issues worked (24h and all-time), completions that produced a verified PR (24h and all-time, [#7894](https://github.com/hivecommons/hive/issues/7894)), and failures. No username parameter, so it only ever answers for its caller ([#6543](https://github.com/hivecommons/hive/issues/6543)) | `pkg/dashboard/api_contribute.go:291` | -| `GET` | `/api/contribute/run-stats/models` | Public | Per-model run evidence for the contributor wall: runs, verified-PR share, and failure rate over the selected window. | `pkg/dashboard/api_contribute.go` | +| `GET` | `/api/contribute/run-stats/models` | Public | Per-model run evidence for the contributor wall: runs, verified-PR share, and failure rate over the selected window, plus why runs ended ([#18721](https://github.com/hivecommons/hive/issues/18721)): `abandoned`, `scenarios` (same keys as `/api/contribute/run-stats` `backends[].scenarios`, summing to `runs`) and `failure_kinds` (`environment`/`task`/`unspecified`, summing to the failed runs). `/api/contribute/run-stats` returns the same rows as `models`. | `pkg/dashboard/api_contribute.go` | | `GET` | `/api/contribute/wall` | Public | Contributor wall posts when the hive-wide wall switch is enabled; anonymous readers see visible posts, authors can still see their own hidden posts. | `pkg/dashboard/api_contribute.go` | | `POST` | `/api/contribute/wall` | Signed-in contributor | Create a 500-character plain-text wall post or one-level reply, with optional `model`, `backend`, and `repo` tags. Returns 401 anonymous, 403 when disabled/revoked/muted, and 429 when rate-limited. | `pkg/dashboard/api_contribute.go` | | `DELETE` | `/api/contribute/wall/{id}` | Post author | Author deletes their own wall post; action is audit logged. | `pkg/dashboard/api_contribute.go` | diff --git a/src/docs/backend-smoke.md b/src/docs/backend-smoke.md index 8de48550a8..29c421de55 100644 --- a/src/docs/backend-smoke.md +++ b/src/docs/backend-smoke.md @@ -137,6 +137,16 @@ counts, the chrome-idle share, and duration p50/p95. The intended use is a ratchet: watch `idle_complete` share and `env_failure` counts per backend trend toward zero, and treat a jump as a regression to root-cause. +The same response's `models` array (also served alone by +`GET /api/contribute/run-stats/models`) gives the per-model view: `runs`, +`verified_pr_share` and `failure_rate` (failed ÷ runs; abandonments are not +failures), plus, since #18721, `abandoned`, `scenarios` (the keys above plus +`abandoned_handback`, `abandoned_disconnect` and `abandoned_other`, summing to +`runs`) and `failure_kinds` (`environment`, `task`, `unspecified`, summing to +the failed runs). That is the split that tells "this model's contributors have +broken environments" from "this model gets the work wrong". The contributor +wall shows it as the tooltip on a post's "failed" model-evidence figure. + This is DECLARE-only telemetry, per the boundary `src/pkg/dashboard/contribute_protocol.go` documents: nothing routes, cooldowns, or gates on any of it, and a telemetry write failure never fails a diff --git a/src/pkg/dashboard/contribute_landing.go b/src/pkg/dashboard/contribute_landing.go index 035b9511b4..e1dfa99933 100644 --- a/src/pkg/dashboard/contribute_landing.go +++ b/src/pkg/dashboard/contribute_landing.go @@ -5804,6 +5804,22 @@ function ccScenarioLabel(sc){ abandoned_handback:'handed back (relay asked for new work)',abandoned_disconnect:'handed back (connection lost)',abandoned_other:'handed back'}; return map[sc]||(sc||'').replace(/_/g,' '); } +// ccModelEvidenceFailTip (#18721) is the tooltip on a wall post's "failed" +// model-evidence figure: the model's runs split by scenario (labelled as in the +// run history above), then its failed runs by kind, so a high failure figure +// reads as "the contributors' environments" or "the work". Plain text for a +// title attribute; empty when the server sent no split (an older hub). +function ccModelEvidenceFailTip(me){ + var sc=(me&&me.scenarios)||{},lines=[]; + var order=['verdict_complete','idle_complete','headless_complete','env_failure','task_failure','unspecified_failure','abandoned_handback','abandoned_disconnect','abandoned_other']; + Object.keys(sc).filter(function(k){return order.indexOf(k)<0;}).sort().forEach(function(k){order.push(k);}); + order.forEach(function(k){if(sc[k])lines.push(ccScenarioLabel(k)+': '+sc[k]);}); + var fk=me&&me.failure_kinds; + if(fk&&((fk.environment||0)+(fk.task||0)+(fk.unspecified||0))>0){ + lines.push('Failed runs by kind: environment '+(fk.environment||0)+' \u00b7 task '+(fk.task||0)+' \u00b7 unspecified '+(fk.unspecified||0)); + } + return lines.join('\n'); +} function ccRenderRuns(user,data){ var el=document.getElementById('runs-list'),cnt=document.getElementById('runs-count'); if(!el)return; @@ -6062,7 +6078,8 @@ function ccWallPostHTML(p){ var tags=p.tags||{}; var evidence=''; if(p.model_evidence){ - evidence='
model evidence: '+(p.model_evidence.runs||0)+' runs · '+Math.round((p.model_evidence.verified_pr_share||0)*100)+'%% verified PR · '+Math.round((p.model_evidence.failure_rate||0)*100)+'%% failed
'; + var failTip=ccModelEvidenceFailTip(p.model_evidence); + evidence='
model evidence: '+(p.model_evidence.runs||0)+' runs · '+Math.round((p.model_evidence.verified_pr_share||0)*100)+'%% verified PR · '+Math.round((p.model_evidence.failure_rate||0)*100)+'%% failed
'; } var controls=''; if(ccMeUsername&&p.author&&p.author.toLowerCase()===ccMeUsername.toLowerCase())controls+=''; diff --git a/src/pkg/dashboard/contribute_wall.go b/src/pkg/dashboard/contribute_wall.go index 452f0066a6..1cd0faf810 100644 --- a/src/pkg/dashboard/contribute_wall.go +++ b/src/pkg/dashboard/contribute_wall.go @@ -75,6 +75,51 @@ type WallModelStats struct { Runs int `json:"runs"` VerifiedPRShare float64 `json:"verified_pr_share"` FailureRate float64 `json:"failure_rate"` + // Abandoned, Scenarios and FailureKinds (#18721) say WHY a model's runs + // ended as they did, from data the run log already holds. Scenarios uses + // deriveScenario's closed vocabulary — the same keys as run-stats' + // backends[].scenarios — and sums to Runs. FailureKinds splits the runs + // behind FailureRate (outcome "failed" only) and sums to that count; + // abandonments stay out of it exactly as they stay out of FailureRate. + Abandoned int `json:"abandoned"` + Scenarios map[string]int `json:"scenarios"` + FailureKinds WallModelFailureKinds `json:"failure_kinds"` +} + +// WallModelFailureKinds counts a model's failed runs by the failure kind the +// relay reported: the contributor's runtime (environment), the work itself +// (task), or no usable kind (unspecified, an older relay). +type WallModelFailureKinds struct { + Environment int `json:"environment"` + Task int `json:"task"` + Unspecified int `json:"unspecified"` +} + +// addRunCause books one run's scenario (and, for a failed run, its failure +// kind) onto the model's split (#18721). A record written before the scenario +// field existed is classified the same way appendTaskRun would have. +func (m *WallModelStats) addRunCause(rec TaskRunRecord) { + sc := rec.Scenario + if sc == "" { + sc = deriveScenario(rec.Outcome, rec.CompletionSignal, rec.FailureKind, rec.AbandonCause) + } + if m.Scenarios == nil { + m.Scenarios = map[string]int{} + } + m.Scenarios[sc]++ + switch rec.Outcome { + case outcomeAbandoned: + m.Abandoned++ + case outcomeFailed: + switch sc { + case scenarioEnvFailure: + m.FailureKinds.Environment++ + case scenarioTaskFailure: + m.FailureKinds.Task++ + default: + m.FailureKinds.Unspecified++ + } + } } type wallContext struct { @@ -620,6 +665,7 @@ func readTaskRunModelStats(path string, window time.Duration) (map[string]WallMo by[key] = a } a.Runs++ + a.addRunCause(rec) if rec.PRVerified { a.verified++ } diff --git a/src/pkg/dashboard/contribute_wall_evidence.test.js b/src/pkg/dashboard/contribute_wall_evidence.test.js new file mode 100644 index 0000000000..1e15cf2cb1 --- /dev/null +++ b/src/pkg/dashboard/contribute_wall_evidence.test.js @@ -0,0 +1,61 @@ +'use strict'; + +// #18721: a wall post's model evidence explains its "failed" figure with the +// model's scenario split, labelled as in the per-run history. + +const { test } = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const vm = require('node:vm'); + +const source = fs.readFileSync(path.join(__dirname, 'contribute_landing.go'), 'utf8'); + +function extractFunction(name) { + const start = source.indexOf('function ' + name + '('); + assert.ok(start >= 0, name + ' not found'); + const end = source.indexOf('\n}\n', start); + assert.ok(end > start, name + ' has no end'); + return source.slice(start, end + 3); +} + +const context = {}; +vm.runInNewContext(extractFunction('ccScenarioLabel') + extractFunction('ccModelEvidenceFailTip'), context); + +test('tooltip lists the scenario split in run-history labels, then failed runs by kind', () => { + const tip = context.ccModelEvidenceFailTip({ + runs: 7, + failure_rate: 4 / 7, + abandoned: 2, + scenarios: { + verdict_complete: 1, env_failure: 2, task_failure: 1, unspecified_failure: 1, + abandoned_handback: 1, abandoned_disconnect: 1, + }, + failure_kinds: { environment: 2, task: 1, unspecified: 1 }, + }); + assert.equal(tip, [ + 'completed on its own verdict: 1', + 'environment failure: 2', + 'failed on the work: 1', + 'failed (unspecified): 1', + 'handed back (relay asked for new work): 1', + 'handed back (connection lost): 1', + 'Failed runs by kind: environment 2 · task 1 · unspecified 1', + ].join('\n')); +}); + +test('an unknown scenario still shows, after the known ones', () => { + const tip = context.ccModelEvidenceFailTip({ scenarios: { zz_new: 1, idle_complete: 2 } }); + assert.equal(tip, 'completed via idle fallback: 2\nzz new: 1'); +}); + +test('evidence from an older hub (no split) yields no tooltip', () => { + assert.equal(context.ccModelEvidenceFailTip({ runs: 3, failure_rate: 0.5 }), ''); + assert.equal(context.ccModelEvidenceFailTip(null), ''); +}); + +test('the wall post puts the tooltip on the failed figure and escapes it', () => { + const body = extractFunction('ccWallPostHTML'); + assert.match(body, /var failTip=ccModelEvidenceFailTip\(p\.model_evidence\);/); + assert.match(body, /'\+Math\.round\(\(p\.model_evidence\.failure_rate\|\|0\)\*100\)\+'%% failed<\/span>/); +}); diff --git a/src/pkg/dashboard/contribute_wall_test.go b/src/pkg/dashboard/contribute_wall_test.go index affb4ba421..882d965fec 100644 --- a/src/pkg/dashboard/contribute_wall_test.go +++ b/src/pkg/dashboard/contribute_wall_test.go @@ -262,6 +262,97 @@ func TestContributeRunStatsByModel(t *testing.T) { } } +// #18721: per-model stats say WHY runs ended — the scenario split, the +// abandonment count and the failed runs by kind — without moving failure_rate. +func TestContributeRunStatsByModelScenarioSplit(t *testing.T) { + s, dir := newWallTestServer(t, true) + now := time.Now().UTC().Format(time.RFC3339) + mk := func(outcome, signal, kind, cause string) TaskRunRecord { + return TaskRunRecord{TS: now, Model: "m1", Outcome: outcome, CompletionSignal: signal, FailureKind: kind, AbandonCause: cause, + Scenario: deriveScenario(outcome, signal, kind, cause)} + } + recs := []TaskRunRecord{ + mk(outcomeFailed, "", TaskFailureKindEnvironment, ""), + mk(outcomeFailed, "", TaskFailureKindTask, ""), + mk(outcomeFailed, "", "", ""), + mk(outcomeAbandoned, "", "", abandonCauseHandback), + mk(outcomeAbandoned, "", "", abandonCauseDisconnect), + mk(outcomeCompleted, completionSignalVerdict, "", ""), + // A record written before the scenario field existed is classified + // the way appendTaskRun would have classified it. + {TS: now, Model: "M1", Outcome: outcomeFailed, FailureKind: TaskFailureKindEnvironment}, + } + var buf bytes.Buffer + for _, r := range recs { + b, _ := json.Marshal(r) + buf.Write(b) + buf.WriteByte('\n') + } + if err := os.WriteFile(filepath.Join(dir, taskRunLogFileName), buf.Bytes(), 0o600); err != nil { + t.Fatal(err) + } + rr := httptest.NewRecorder() + s.handleContributeRunStatsByModel(rr, httptest.NewRequest("GET", "/api/contribute/run-stats/models?days=1", nil)) + var resp struct { + Models []WallModelStats `json:"models"` + } + if err := json.Unmarshal(rr.Body.Bytes(), &resp); err != nil { + t.Fatalf("json %s: %v", rr.Body.String(), err) + } + if len(resp.Models) != 1 { + t.Fatalf("models %+v", resp.Models) + } + m := resp.Models[0] + wantScenarios := map[string]int{ + scenarioEnvFailure: 2, + scenarioTaskFailure: 1, + scenarioUnspecifiedFailure: 1, + scenarioAbandonedHandback: 1, + scenarioAbandonedDisconnect: 1, + scenarioVerdictComplete: 1, + } + if len(m.Scenarios) != len(wantScenarios) { + t.Fatalf("scenarios %+v, want %+v", m.Scenarios, wantScenarios) + } + sum := 0 + for k, v := range m.Scenarios { + if wantScenarios[k] != v { + t.Errorf("scenarios[%q] = %d, want %d", k, v, wantScenarios[k]) + } + sum += v + } + if m.Runs != 7 || sum != m.Runs { + t.Errorf("runs = %d, scenario sum = %d; want both 7", m.Runs, sum) + } + if m.Abandoned != 2 { + t.Errorf("abandoned = %d, want 2", m.Abandoned) + } + if m.FailureKinds != (WallModelFailureKinds{Environment: 2, Task: 1, Unspecified: 1}) { + t.Errorf("failure_kinds = %+v", m.FailureKinds) + } + failed := m.FailureKinds.Environment + m.FailureKinds.Task + m.FailureKinds.Unspecified + if m.FailureRate != float64(failed)/float64(m.Runs) || m.FailureRate != 4.0/7.0 { + t.Errorf("failure_rate = %v, want 4/7 (abandonments still excluded)", m.FailureRate) + } + if m.VerifiedPRShare != 0 { + t.Errorf("verified_pr_share = %v, want 0", m.VerifiedPRShare) + } + // The wire keys are the documented ones. + var raw struct { + Models []map[string]any `json:"models"` + } + if err := json.Unmarshal(rr.Body.Bytes(), &raw); err != nil { + t.Fatal(err) + } + fk, ok := raw.Models[0]["failure_kinds"].(map[string]any) + if !ok || fk["environment"] != 2.0 || fk["task"] != 1.0 || fk["unspecified"] != 1.0 { + t.Errorf("failure_kinds on the wire = %#v", raw.Models[0]["failure_kinds"]) + } + if raw.Models[0]["abandoned"] != 2.0 { + t.Errorf("abandoned on the wire = %#v", raw.Models[0]["abandoned"]) + } +} + func TestContributeWallSSEPayload(t *testing.T) { s, dir := newWallTestServer(t, true) writeWallProfile(t, dir, "alice", "contributor", 0) diff --git a/src/pkg/dashboard/task_run_log_test.go b/src/pkg/dashboard/task_run_log_test.go index 5fc6a52dea..ebd756c043 100644 --- a/src/pkg/dashboard/task_run_log_test.go +++ b/src/pkg/dashboard/task_run_log_test.go @@ -736,3 +736,64 @@ func TestTaskRunLog_RecordedOnHandback(t *testing.T) { t.Fatalf("run-stats wrong: %+v", stats.Backends) } } + +// #18721: the models array of /api/contribute/run-stats carries the same +// per-model split as /api/contribute/run-stats/models, with the scenario keys +// the backends array uses. +func TestHandleContributeRunStats_ModelsCarryScenarioSplit(t *testing.T) { + dir := t.TempDir() + now := time.Now().UTC().Format(time.RFC3339) + var lines []string + for _, r := range []TaskRunRecord{ + {TS: now, Backend: "claude", Model: "m1", Outcome: outcomeFailed, FailureKind: TaskFailureKindEnvironment, + Scenario: deriveScenario(outcomeFailed, "", TaskFailureKindEnvironment, "")}, + {TS: now, Backend: "claude", Model: "m1", Outcome: outcomeAbandoned, AbandonCause: abandonCauseHandback, + Scenario: deriveScenario(outcomeAbandoned, "", "", abandonCauseHandback)}, + {TS: now, Backend: "claude", Model: "m1", Outcome: outcomeCompleted, CompletionSignal: completionSignalVerdict, + Scenario: deriveScenario(outcomeCompleted, completionSignalVerdict, "", "")}, + } { + b, err := json.Marshal(r) + if err != nil { + t.Fatal(err) + } + lines = append(lines, string(b)) + } + if err := os.WriteFile(filepath.Join(dir, taskRunLogFileName), []byte(strings.Join(lines, "\n")+"\n"), 0o600); err != nil { + t.Fatal(err) + } + s := NewServer(0, nil) + s.SetContributorsDir(dir) + w := httptest.NewRecorder() + s.handleContributeRunStats(w, httptest.NewRequest(http.MethodGet, "/api/contribute/run-stats?days=1", nil)) + if w.Code != http.StatusOK { + t.Fatalf("status = %d", w.Code) + } + var resp struct { + Backends []taskRunBackendStats `json:"backends"` + Models []map[string]any `json:"models"` + } + if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil { + t.Fatalf("response: %v", err) + } + if len(resp.Models) != 1 || len(resp.Backends) != 1 { + t.Fatalf("models %+v backends %+v", resp.Models, resp.Backends) + } + m := resp.Models[0] + for _, key := range []string{"model", "runs", "verified_pr_share", "failure_rate", "abandoned", "scenarios", "failure_kinds"} { + if _, ok := m[key]; !ok { + t.Errorf("models[0] missing %q: %#v", key, m) + } + } + sc, _ := m["scenarios"].(map[string]any) + if len(sc) != len(resp.Backends[0].Scenarios) { + t.Fatalf("model scenarios %#v, backend scenarios %#v", sc, resp.Backends[0].Scenarios) + } + for k, v := range resp.Backends[0].Scenarios { + if sc[k] != float64(v) { + t.Errorf("scenarios[%q] = %#v on the model, %d on the backend", k, sc[k], v) + } + } + if m["abandoned"] != 1.0 || m["failure_rate"] != 1.0/3.0 { + t.Errorf("abandoned/failure_rate = %#v/%#v, want 1 and 1/3", m["abandoned"], m["failure_rate"]) + } +}