.
+// Owner-authenticated like every /api route. Returns no records when
+// benchmarks.publish is false so the endpoint and the hub agree.
+func (s *Server) handleBenchmarksExport(w http.ResponseWriter, r *http.Request) {
+ if s.deps == nil || s.deps.Config == nil {
+ jsonError(w, "config not loaded", http.StatusServiceUnavailable)
+ return
+ }
+ cfg := s.deps.Config
+ since := time.Now().Add(-benchmarksExportDefaultWindow)
+ if raw := strings.TrimSpace(r.URL.Query().Get("since")); raw != "" {
+ t, err := time.Parse(time.RFC3339, raw)
+ if err != nil {
+ t, err = time.Parse("2006-01-02", raw)
+ }
+ if err != nil {
+ jsonError(w, "since must be RFC3339 or YYYY-MM-DD", http.StatusBadRequest)
+ return
+ }
+ since = t
+ }
+ resp := map[string]any{
+ "hive_hash": benchmarks.HiveHash(cfg.HiveID),
+ "generated_at": time.Now().UTC().Format(time.RFC3339),
+ "publish": cfg.Benchmarks.PublishEnabled(),
+ "since": since.UTC().Format(time.RFC3339),
+ "records": []benchmarks.Record{},
+ "count": 0,
+ }
+ if cfg.Benchmarks.PublishEnabled() {
+ recs, _, err := s.benchmarksRecords(r.Context(), since)
+ if err != nil && !errors.Is(err, os.ErrNotExist) {
+ jsonError(w, "load evidence: "+err.Error(), http.StatusInternalServerError)
+ return
+ }
+ if recs == nil {
+ recs = []benchmarks.Record{}
+ }
+ resp["records"] = recs
+ resp["count"] = len(recs)
+ }
+ jsonResponse(w, resp)
+}
+
+// BenchmarksStatusBlock is the /api/status view.
+func (s *Server) BenchmarksStatusBlock() *BenchmarksStatus {
+ if s.deps == nil || s.deps.Config == nil {
+ return nil
+ }
+ cfg := s.deps.Config
+ p := s.benchmarks()
+ p.mu.Lock()
+ st := p.state
+ p.mu.Unlock()
+ out := &BenchmarksStatus{
+ HiveHash: benchmarks.HiveHash(cfg.HiveID),
+ Publish: cfg.Benchmarks.PublishEnabled(),
+ RepoConsent: cfg.Benchmarks.RepoConsentEnabled(),
+ LastPushCount: st.LastCount,
+ TotalPushed: st.TotalPushed,
+ LastError: st.LastError,
+ SiteURL: benchmarksSiteURL,
+ }
+ if !st.LastPushAt.IsZero() {
+ out.LastPushAt = st.LastPushAt.UTC().Format(time.RFC3339)
+ }
+ if !st.Watermark.IsZero() {
+ out.WatermarkAt = st.Watermark.UTC().Format(time.RFC3339)
+ }
+ return out
+}
+
+// StartBenchmarksPublisher runs the hourly spoke→hub push until ctx ends.
+// It returns at once when there is no hub, no hive id or no hub credential.
+func (s *Server) StartBenchmarksPublisher(ctx context.Context) {
+ if s.deps == nil || s.deps.Config == nil {
+ return
+ }
+ cfg := s.deps.Config
+ hubURL := strings.TrimRight(strings.TrimSpace(cfg.Hub.URL), "/")
+ if hubURL == "" || strings.TrimSpace(cfg.HiveID) == "" {
+ return
+ }
+ logger := s.logger
+ if logger == nil {
+ logger = slog.Default()
+ }
+ go func() {
+ select {
+ case <-ctx.Done():
+ return
+ case <-time.After(benchmarksPushInitialDelay):
+ }
+ ticker := time.NewTicker(benchmarksPushInterval)
+ defer ticker.Stop()
+ for {
+ s.pushBenchmarks(ctx, hubURL, logger)
+ select {
+ case <-ctx.Done():
+ return
+ case <-ticker.C:
+ }
+ }
+ }()
+}
+
+// benchmarksIngestPayload is the body of POST /api/benchmarks/ingest.
+type benchmarksIngestPayload struct {
+ HiveID string `json:"hive_id"`
+ HiveHash string `json:"hive_hash"`
+ Watermark string `json:"watermark"`
+ Records []benchmarks.Record `json:"records"`
+}
+
+// pushBenchmarks sends one batch and advances the watermark on 2xx.
+func (s *Server) pushBenchmarks(ctx context.Context, hubURL string, logger *slog.Logger) {
+ cfg := s.deps.Config
+ p := s.benchmarks()
+ if !cfg.Benchmarks.PublishEnabled() {
+ return
+ }
+ bearer := spoke.SpokeHeartbeatKey()
+ if bearer == "" {
+ return
+ }
+ p.mu.Lock()
+ since := p.state.Watermark
+ p.mu.Unlock()
+ recs, newest, err := s.benchmarksRecords(ctx, since)
+ if err != nil {
+ if !errors.Is(err, os.ErrNotExist) {
+ s.benchmarksFail(p, "load evidence: "+err.Error())
+ }
+ return
+ }
+ if len(recs) == 0 {
+ return
+ }
+ if len(recs) > benchmarksPushMaxRecords {
+ recs = recs[:benchmarksPushMaxRecords]
+ newest = since // partial batch: keep the watermark so the rest follows
+ }
+ body, err := json.Marshal(benchmarksIngestPayload{
+ HiveID: cfg.HiveID,
+ HiveHash: benchmarks.HiveHash(cfg.HiveID),
+ Watermark: newest.UTC().Format(time.RFC3339),
+ Records: recs,
+ })
+ if err != nil {
+ return
+ }
+ reqCtx, cancel := context.WithTimeout(ctx, benchmarksPushTimeout)
+ defer cancel()
+ req, err := http.NewRequestWithContext(reqCtx, http.MethodPost, hubURL+benchmarksHubIngestPath, bytes.NewReader(body))
+ if err != nil {
+ return
+ }
+ req.Header.Set("Content-Type", "application/json")
+ req.Header.Set("Authorization", "Bearer "+bearer)
+ resp, err := http.DefaultClient.Do(req)
+ if err != nil {
+ s.benchmarksFail(p, "post: "+err.Error())
+ return
+ }
+ defer closeHTTPBody(resp.Body)
+ if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices {
+ s.benchmarksFail(p, "hub answered "+resp.Status)
+ return
+ }
+ p.mu.Lock()
+ if newest.After(p.state.Watermark) {
+ p.state.Watermark = newest
+ }
+ p.state.LastPushAt = time.Now()
+ p.state.LastCount = len(recs)
+ p.state.TotalPushed += len(recs)
+ p.state.LastError = ""
+ p.save()
+ p.mu.Unlock()
+ logger.Info("benchmarks: pushed records to hub", "count", len(recs), "watermark", newest.UTC().Format(time.RFC3339))
+}
+
+func (s *Server) benchmarksFail(p *benchmarksPublisher, msg string) {
+ p.mu.Lock()
+ p.state.LastError = msg
+ p.save()
+ p.mu.Unlock()
+ if s.logger != nil {
+ s.logger.Warn("benchmarks: push failed", "error", msg)
+ }
+}
+
+// featuresBenchmarksResponse is the Settings → Features → Benchmarks panel
+// source (config_features.js).
+func (s *Server) featuresBenchmarksResponse(cfg *config.Config) map[string]interface{} {
+ out := map[string]interface{}{
+ "benchmarksPublish": cfg.Benchmarks.PublishEnabled(),
+ "benchmarksRepoConsent": cfg.Benchmarks.RepoConsentEnabled(),
+ "benchmarksHiveHash": benchmarks.HiveHash(cfg.HiveID),
+ }
+ if st := s.BenchmarksStatusBlock(); st != nil {
+ out["benchmarksLastPushAt"] = st.LastPushAt
+ out["benchmarksLastPushCount"] = st.LastPushCount
+ }
+ return out
+}
diff --git a/src/pkg/dashboard/api_benchmarks_test.go b/src/pkg/dashboard/api_benchmarks_test.go
new file mode 100644
index 0000000000..c5f8589a8d
--- /dev/null
+++ b/src/pkg/dashboard/api_benchmarks_test.go
@@ -0,0 +1,167 @@
+package dashboard
+
+import (
+ "encoding/json"
+ "io"
+ "log/slog"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "path/filepath"
+ "testing"
+ "time"
+
+ "github.com/hivecommons/hive/pkg/config"
+ "github.com/hivecommons/hive/pkg/evidence"
+ "github.com/hivecommons/hive/pkg/evidence/benchmarks"
+ ghpkg "github.com/hivecommons/hive/pkg/github"
+ "github.com/hivecommons/hive/pkg/hub/spoke"
+)
+
+func writeBenchmarkFixtureBundle(t *testing.T, root string, b *evidence.Bundle) {
+ t.Helper()
+ if err := evidence.Seal(b, nil); err != nil {
+ t.Fatal(err)
+ }
+ dir := filepath.Join(root, filepath.FromSlash(b.Repo), "1")
+ if err := os.MkdirAll(dir, 0o755); err != nil {
+ t.Fatal(err)
+ }
+ data, _ := json.Marshal(b)
+ if err := os.WriteFile(filepath.Join(dir, b.HeadSHA+".json"), data, 0o644); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func benchmarksTestServer(t *testing.T, publish bool) *Server {
+ t.Helper()
+ root := t.TempDir()
+ prev := ghpkg.ReviewEvidenceRoot
+ ghpkg.ReviewEvidenceRoot = root
+ t.Cleanup(func() { ghpkg.ReviewEvidenceRoot = prev })
+ now := time.Now().UTC().Add(-time.Hour)
+ writeBenchmarkFixtureBundle(t, root, &evidence.Bundle{
+ SchemaVersion: evidence.SchemaVersion, Repo: "o/r", Number: 1, HeadSHA: "abc1234",
+ Author: evidence.Author{Login: "bot", Kind: evidence.AuthorAgent, Model: "claude-fable-5.1", Backend: "copilot", Runtime: "podman", TaskKind: "docs"},
+ CreatedAt: now, UpdatedAt: now,
+ Merge: &evidence.MergeEvent{Actor: "hive", Method: "squash", At: now.Add(30 * time.Minute), OpenedAt: now},
+ })
+ s := newTestServer()
+ s.deps = &Dependencies{Config: &config.Config{HiveID: "h1", Benchmarks: config.BenchmarksConfig{Publish: &publish}}}
+ return s
+}
+
+func TestBenchmarksExportEmitsSchemaRecordsAndHonoursSince(t *testing.T) {
+ s := benchmarksTestServer(t, true)
+ rec := httptest.NewRecorder()
+ s.handleBenchmarksExport(rec, httptest.NewRequest(http.MethodGet, "/api/benchmarks/export", nil))
+ if rec.Code != http.StatusOK {
+ t.Fatalf("code=%d body=%s", rec.Code, rec.Body.String())
+ }
+ var resp struct {
+ HiveHash string `json:"hive_hash"`
+ Count int `json:"count"`
+ Records []benchmarks.Record `json:"records"`
+ }
+ if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil {
+ t.Fatal(err)
+ }
+ if resp.HiveHash != benchmarks.HiveHash("h1") || resp.Count != 1 {
+ t.Fatalf("resp=%+v", resp)
+ }
+ r := resp.Records[0]
+ if r.Source.HiveID != resp.HiveHash || r.Model.Name != "claude-fable-5.1" || r.SandboxRuntime != "podman" || r.Task.Kind != "docs" || !r.Outcome.PRMerged {
+ t.Errorf("record=%+v", r)
+ }
+ if r.Task.Repo != "" {
+ t.Errorf("repo of unknown visibility must be anonymized, got %q", r.Task.Repo)
+ }
+
+ rec = httptest.NewRecorder()
+ s.handleBenchmarksExport(rec, httptest.NewRequest(http.MethodGet, "/api/benchmarks/export?since="+time.Now().UTC().Format(time.RFC3339), nil))
+ if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil || resp.Count != 0 {
+ t.Errorf("since in the future should yield 0 records: count=%d err=%v", resp.Count, err)
+ }
+ rec = httptest.NewRecorder()
+ s.handleBenchmarksExport(rec, httptest.NewRequest(http.MethodGet, "/api/benchmarks/export?since=yesterday", nil))
+ if rec.Code != http.StatusBadRequest {
+ t.Errorf("bad since: code=%d", rec.Code)
+ }
+}
+
+func TestBenchmarksExportRespectsPublishOptOut(t *testing.T) {
+ s := benchmarksTestServer(t, false)
+ rec := httptest.NewRecorder()
+ s.handleBenchmarksExport(rec, httptest.NewRequest(http.MethodGet, "/api/benchmarks/export", nil))
+ var resp struct {
+ Publish bool `json:"publish"`
+ Count int `json:"count"`
+ }
+ if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil || rec.Code != http.StatusOK {
+ t.Fatalf("code=%d err=%v", rec.Code, err)
+ }
+ if resp.Publish || resp.Count != 0 {
+ t.Errorf("opt-out must export nothing: %+v", resp)
+ }
+ if st := s.BenchmarksStatusBlock(); st == nil || st.Publish || st.HiveHash != benchmarks.HiveHash("h1") {
+ t.Errorf("status block = %+v", st)
+ }
+}
+
+func TestPushBenchmarksAdvancesWatermarkAndRecordsFailures(t *testing.T) {
+ s := benchmarksTestServer(t, true)
+ prevState := benchmarksPublishStatePath
+ benchmarksPublishStatePath = filepath.Join(t.TempDir(), "benchmarks-publish.json")
+ t.Cleanup(func() { benchmarksPublishStatePath = prevState })
+ t.Setenv(spoke.EnvHeartbeatKey, "spoke-key")
+ logger := slog.New(slog.NewTextHandler(io.Discard, nil))
+
+ var got struct {
+ HiveID string `json:"hive_id"`
+ Records []benchmarks.Record `json:"records"`
+ }
+ var bearer string
+ status := http.StatusAccepted
+ hub := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ bearer = r.Header.Get("Authorization")
+ _ = json.NewDecoder(r.Body).Decode(&got)
+ w.WriteHeader(status)
+ }))
+ defer hub.Close()
+
+ s.pushBenchmarks(t.Context(), hub.URL, logger)
+ if bearer != "Bearer spoke-key" || got.HiveID != "h1" || len(got.Records) != 1 {
+ t.Fatalf("push: bearer=%q hive=%q n=%d", bearer, got.HiveID, len(got.Records))
+ }
+ st := s.BenchmarksStatusBlock()
+ if st.TotalPushed != 1 || st.LastPushAt == "" || st.WatermarkAt == "" || st.LastError != "" {
+ t.Fatalf("status after push = %+v", st)
+ }
+ if _, err := os.Stat(benchmarksPublishStatePath); err != nil {
+ t.Fatalf("watermark not persisted: %v", err)
+ }
+
+ // Nothing new since the watermark: no second request.
+ got.Records = nil
+ s.pushBenchmarks(t.Context(), hub.URL, logger)
+ if got.Records != nil {
+ t.Fatal("pushed again with nothing newer than the watermark")
+ }
+
+ // A hub error is surfaced and keeps the watermark where it was.
+ s.benchmarks().state.Watermark = time.Time{}
+ status = http.StatusInternalServerError
+ s.pushBenchmarks(t.Context(), hub.URL, logger)
+ if st := s.BenchmarksStatusBlock(); st.LastError == "" || st.TotalPushed != 1 {
+ t.Fatalf("failure not recorded: %+v", st)
+ }
+
+ // Opt-out short-circuits before any request.
+ off := false
+ s.deps.Config.Benchmarks.Publish = &off
+ bearer = ""
+ s.pushBenchmarks(t.Context(), hub.URL, logger)
+ if bearer != "" {
+ t.Fatal("publish:false must not push")
+ }
+}
diff --git a/src/pkg/dashboard/api_governor_features.go b/src/pkg/dashboard/api_governor_features.go
index 9b664a4c90..3dfece4a37 100644
--- a/src/pkg/dashboard/api_governor_features.go
+++ b/src/pkg/dashboard/api_governor_features.go
@@ -497,6 +497,9 @@ func (s *Server) featuresLinkedView() map[string]interface{} {
// which is what the governor config GET serves.
func (s *Server) featuresSectionWithLinked(cfg *config.Config) map[string]interface{} {
out := featuresSectionResponse(cfg)
+ for k, v := range s.featuresBenchmarksResponse(cfg) {
+ out[k] = v
+ }
for k, v := range s.featuresLinkedView() {
out[k] = v
}
diff --git a/src/pkg/dashboard/api_governor_features_platform.go b/src/pkg/dashboard/api_governor_features_platform.go
index 5fe2a5b1c8..1d8c865d73 100644
--- a/src/pkg/dashboard/api_governor_features_platform.go
+++ b/src/pkg/dashboard/api_governor_features_platform.go
@@ -36,6 +36,9 @@ type featuresPlatformBody struct {
EvidenceEnabled *bool `json:"evidenceEnabled"`
+ BenchmarksPublish *bool `json:"benchmarksPublish"`
+ BenchmarksRepoConsent *bool `json:"benchmarksRepoConsent"`
+
GiteaURL *string `json:"giteaUrl"`
GiteaTokenEnv *string `json:"giteaTokenEnv"`
GitLabURL *string `json:"gitlabUrl"`
@@ -174,6 +177,14 @@ func (b featuresPlatformBody) prepare() (func(*config.Config), error) {
v := *b.EvidenceEnabled
steps = append(steps, func(c *config.Config) { c.Evidence.Enabled = &v })
}
+ if b.BenchmarksPublish != nil {
+ v := *b.BenchmarksPublish
+ steps = append(steps, func(c *config.Config) { c.Benchmarks.Publish = &v })
+ }
+ if b.BenchmarksRepoConsent != nil {
+ v := *b.BenchmarksRepoConsent
+ steps = append(steps, func(c *config.Config) { c.Benchmarks.RepoConsent = &v })
+ }
for _, f := range []struct {
raw *string
diff --git a/src/pkg/dashboard/server.go b/src/pkg/dashboard/server.go
index 0787453285..233bebed05 100644
--- a/src/pkg/dashboard/server.go
+++ b/src/pkg/dashboard/server.go
@@ -139,6 +139,12 @@ type Server struct {
logger *slog.Logger
mux *http.ServeMux
deps *Dependencies
+
+ // benchmarksPub is the hourly hivecommons/benchmarks publisher state
+ // (api_benchmarks.go), built lazily by benchmarks().
+ benchmarksOnce sync.Once
+ benchmarksPub *benchmarksPublisher
+
// stageStore is the stage-label memory /api/stage-tracker reads; nil
// means the process-wide stagelabels.Shared(). Tests inject one.
stageStore *stagelabels.Store
@@ -1782,6 +1788,8 @@ func (s *Server) Start() error {
followCtx, cancelFollow := context.WithCancel(context.Background())
defer cancelFollow()
s.StartRebaseMergeFollowThrough(followCtx)
+ // Hourly anonymized benchmark push to the hub (hivecommons/hive#17694).
+ s.StartBenchmarksPublisher(followCtx)
return srv.Serve(ln)
}
@@ -3763,6 +3771,7 @@ func (s *Server) handleStatus(w http.ResponseWriter, r *http.Request) {
payload.UpgradeProgress = s.upgradeProgress(status, now)
payload.Writeback = s.WritebackStatus()
payload.Hotspots = s.HotspotStatusBlock()
+ payload.Benchmarks = s.BenchmarksStatusBlock()
payload.Config = s.configHealthStatus()
if s.deps != nil {
payload.WorkflowApprovals = s.deps.GHClient.WorkflowApprovalsStatus()
@@ -3800,6 +3809,9 @@ type statusWithWebhooks struct {
// WorkflowApprovals counts the hive's approvals of GitHub-gated
// workflow runs (hivecommons/hive#16603).
WorkflowApprovals github.WorkflowApprovalsStatus `json:"workflow_approvals"`
+ // Benchmarks is this hive's leaderboard hash and last publish to
+ // hivecommons/benchmarks (hivecommons/hive#17694).
+ Benchmarks *BenchmarksStatus `json:"benchmarks,omitempty"`
*EvalLoopStatus
// MatrixEnabled reports whether the Matrix block is active (enabled with
// at least one repo, see config.MatrixConfig.Active) so the header shows
diff --git a/src/pkg/dashboard/static-src/js/config-features.js b/src/pkg/dashboard/static-src/js/config-features.js
index f294e2d203..4677f288de 100644
--- a/src/pkg/dashboard/static-src/js/config-features.js
+++ b/src/pkg/dashboard/static-src/js/config-features.js
@@ -633,6 +633,20 @@
Write evidence bundles ${yamlHint('evidence.enabled')} ${settingHelpMark('settings', 'features', 'Write evidence bundles', `On by default. Off stops writing review evidence bundles. Applies to the next review.`)}
+ Benchmarks ${settingHelpMark('settings', 'features', 'Benchmarks', `This hive contributes anonymized per-PR records (model, harness, sandbox runtime, outcome, time-to-merge, ACMM level) to the fleet leaderboard at hivecommons.dev/benchmarks. No logins, prompts, findings or code ever leave the hive; the hive is identified only by the hash below. Repository names are included only for public repositories.`)}
+
+
+
Publish anonymized benchmark records ${yamlHint('benchmarks.publish')} ${settingHelpMark('settings', 'features', 'Publish benchmark records', `On by default. Off stops the hourly push to the hub and empties GET /api/benchmarks/export.`)}
+
+
+
+
Name public repositories ${yamlHint('benchmarks.repo_consent')} ${settingHelpMark('settings', 'features', 'Name public repositories', `On by default: records for PUBLIC repositories carry the repository and org name, since that activity is already public. Off anonymizes every record. Private repositories are never named.`)}
+
+
+
+
+
+
Review evidence signing
`)}
diff --git a/src/pkg/dashboard/static-src/js/config-state.js b/src/pkg/dashboard/static-src/js/config-state.js
index 95017b5b64..6bb1f4d866 100644
--- a/src/pkg/dashboard/static-src/js/config-state.js
+++ b/src/pkg/dashboard/static-src/js/config-state.js
@@ -109,6 +109,7 @@
{ tab: 'Features', section: 'Integrations & observability', label: 'Standalone deployment upgrades', key: 'deployment.podman_mode', text: 'deployment podman quadlet rootless rootful docker compose upgrade helper request dir deployment.podman_mode deployment.upgrade_helper deployment.upgrade_request_dir' },
{ tab: 'Features', section: 'Review gate', label: 'Review gate: comments, dispatch and out-of-scope findings', key: 'review.post_comments', text: 'review swarm post comments fan out parallel perspectives combined all authors fix human prs acknowledge no findings confidence score priority labels fixer agent human decision label self authored decision policy reviews per head revise verdicts webhook event driven debounce pending ttl redispatch out of scope backlog issues recommendations review.post_comments review.self_authored_decision_policy review.event_driven review.event_debounce_s review.pending_ttl review.max_redispatch review.head_max_retries review.max_reviews_per_head review.revise_verdicts_before review.out_of_scope_backlog_disabled review.max_out_of_scope_backlog_issues review.recommendations.enabled review.recommendations.min_ready_to_open' },
{ tab: 'Features', section: 'Review gate', label: 'Review evidence bundles', key: 'evidence.enabled', text: 'review evidence bundles audit signing key ed25519 evidence.enabled evidence.signing_key_file' },
+ { tab: 'Features', section: 'Review gate', label: 'Benchmarks', key: 'benchmarks.publish', text: 'benchmarks leaderboard publish anonymized hive hash repo consent benchmarks.publish benchmarks.repo_consent hivecommons.dev/benchmarks' },
{ tab: 'Features', section: 'Integrations & observability', label: 'GitLab and Gitea forges', key: 'gitlab.gitlab_url', text: 'gitlab gitea forgejo codeberg forge instance url token env gitlab.gitlab_url gitlab.token_env gitea.gitea_url gitea.token_env' },
{ tab: 'Features', section: 'Agents & tooling', label: 'Token mint key and TTL', key: 'mint.max_ttl_seconds', text: 'token mint signing key path max ttl lifetime jwt mint.key_path mint.max_ttl_seconds' },
{ tab: 'Features', section: 'Autonomy & ACMM', label: 'Design-first planning', key: 'planning.design_approved_label', text: 'planning design approved label architect revisions concurrent designs plan from label planning.design_approved_label planning.max_concurrent_designs planning.max_design_revisions planning.plan_from_label' },
diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html
index 68a4810cc5..e280e84772 100644
--- a/src/pkg/dashboard/static/index.html
+++ b/src/pkg/dashboard/static/index.html
@@ -54686,6 +54686,7 @@ Configure LiteLLM
{ tab: 'Features', section: 'Integrations & observability', label: 'Standalone deployment upgrades', key: 'deployment.podman_mode', text: 'deployment podman quadlet rootless rootful docker compose upgrade helper request dir deployment.podman_mode deployment.upgrade_helper deployment.upgrade_request_dir' },
{ tab: 'Features', section: 'Review gate', label: 'Review gate: comments, dispatch and out-of-scope findings', key: 'review.post_comments', text: 'review swarm post comments fan out parallel perspectives combined all authors fix human prs acknowledge no findings confidence score priority labels fixer agent human decision label self authored decision policy reviews per head revise verdicts webhook event driven debounce pending ttl redispatch out of scope backlog issues recommendations review.post_comments review.self_authored_decision_policy review.event_driven review.event_debounce_s review.pending_ttl review.max_redispatch review.head_max_retries review.max_reviews_per_head review.revise_verdicts_before review.out_of_scope_backlog_disabled review.max_out_of_scope_backlog_issues review.recommendations.enabled review.recommendations.min_ready_to_open' },
{ tab: 'Features', section: 'Review gate', label: 'Review evidence bundles', key: 'evidence.enabled', text: 'review evidence bundles audit signing key ed25519 evidence.enabled evidence.signing_key_file' },
+ { tab: 'Features', section: 'Review gate', label: 'Benchmarks', key: 'benchmarks.publish', text: 'benchmarks leaderboard publish anonymized hive hash repo consent benchmarks.publish benchmarks.repo_consent hivecommons.dev/benchmarks' },
{ tab: 'Features', section: 'Integrations & observability', label: 'GitLab and Gitea forges', key: 'gitlab.gitlab_url', text: 'gitlab gitea forgejo codeberg forge instance url token env gitlab.gitlab_url gitlab.token_env gitea.gitea_url gitea.token_env' },
{ tab: 'Features', section: 'Agents & tooling', label: 'Token mint key and TTL', key: 'mint.max_ttl_seconds', text: 'token mint signing key path max ttl lifetime jwt mint.key_path mint.max_ttl_seconds' },
{ tab: 'Features', section: 'Autonomy & ACMM', label: 'Design-first planning', key: 'planning.design_approved_label', text: 'planning design approved label architect revisions concurrent designs plan from label planning.design_approved_label planning.max_concurrent_designs planning.max_design_revisions planning.plan_from_label' },
@@ -65494,6 +65495,20 @@ ${esc(s.title)}
Write evidence bundles ${yamlHint('evidence.enabled')} ${settingHelpMark('settings', 'features', 'Write evidence bundles', `On by default. Off stops writing review evidence bundles. Applies to the next review.`)}
+ Benchmarks ${settingHelpMark('settings', 'features', 'Benchmarks', `This hive contributes anonymized per-PR records (model, harness, sandbox runtime, outcome, time-to-merge, ACMM level) to the fleet leaderboard at hivecommons.dev/benchmarks. No logins, prompts, findings or code ever leave the hive; the hive is identified only by the hash below. Repository names are included only for public repositories.`)}
+
+
+
Publish anonymized benchmark records ${yamlHint('benchmarks.publish')} ${settingHelpMark('settings', 'features', 'Publish benchmark records', `On by default. Off stops the hourly push to the hub and empties GET /api/benchmarks/export.`)}
+
+
+
+
Name public repositories ${yamlHint('benchmarks.repo_consent')} ${settingHelpMark('settings', 'features', 'Name public repositories', `On by default: records for PUBLIC repositories carry the repository and org name, since that activity is already public. Off anonymizes every record. Private repositories are never named.`)}
+
+
+
+
+
+
Review evidence signing
`)}
diff --git a/src/pkg/evidence/benchmarks/convert.go b/src/pkg/evidence/benchmarks/convert.go
new file mode 100644
index 0000000000..0f9f6c5966
--- /dev/null
+++ b/src/pkg/evidence/benchmarks/convert.go
@@ -0,0 +1,357 @@
+package benchmarks
+
+import (
+ "sort"
+ "strconv"
+ "strings"
+ "time"
+
+ "github.com/hivecommons/hive/pkg/evidence"
+)
+
+// Options parameterise one export.
+type Options struct {
+ HiveID string
+ HiveVersion string
+ Cluster string
+ // Since drops PRs whose newest record is not strictly newer than it (the
+ // pusher passes its last watermark). Zero exports all.
+ Since time.Time
+ // Now stamps published_at; zero means time.Now().
+ Now time.Time
+ // RepoPublic reports whether owner/name is a public repository. nil means
+ // unknown, which anonymizes every repo.
+ RepoPublic func(repo string) bool
+ // RepoConsent is benchmarks.repo_consent: false anonymizes even public repos.
+ RepoConsent bool
+ // SandboxRuntime resolves the runtime an agent's kicks ran under
+ // ("none" when sandboxing is off). nil means "none" for every agent.
+ SandboxRuntime func(agent string) string
+}
+
+// Roles.
+const (
+ RoleAuthor = "author"
+ RoleReviewer = "reviewer"
+)
+
+// SandboxRuntimeNone is the published runtime when agents run in-process.
+const SandboxRuntimeNone = "none"
+
+const (
+ kindPRReview = "pr-review"
+ kindUnknown = "unknown"
+ verdictNone = "none"
+ providerUnknown = "unknown"
+ harnessOther = "other"
+ ciSuccess = "success"
+ ciFailure = "failure"
+ ciPending = "pending"
+ ciUnknown = "unknown"
+ reviewSwarmAgent = "review-swarm"
+)
+
+var knownHarnesses = map[string]bool{"agy": true, "aider": true, "bob": true, "claude": true, "cline": true, "codex": true,
+ "copilot": true, "gemini": true, "goose": true, "kilo": true, "muse": true, "omnigent": true, "omp": true,
+ "opencode": true, "openhands": true, "pi": true, "clanker": true}
+
+var verdictRank = map[string]int{"reject": 4, "changes_requested": 3, "requires_human": 2, "comment": 1, "approve": 0}
+
+var kindHints = []struct{ hint, kind string }{
+ {"doc", "docs"}, {"sec", "security-fix"}, {"ci", "ci-repair"}, {"release", "release"},
+ {"dep", "dependency-update"}, {"renovate", "dependency-update"}, {"triage", "triage"}, {"refactor", "refactor"},
+}
+
+var providerPrefixes = []struct{ prefix, provider string }{
+ {"claude", "anthropic"}, {"gpt", "openai"}, {"o1", "openai"}, {"o3", "openai"}, {"o4", "openai"},
+ {"gemini", "google"}, {"granite", "ibm"}, {"llama", "meta"}, {"mistral", "mistral"},
+ {"qwen", "alibaba"}, {"deepseek", "deepseek"}, {"grok", "xai"}, {"kimi", "moonshot"},
+}
+
+// Convert folds bundles per PR and emits an author record for each PR a hive
+// agent opened and a reviewer record for each PR the swarm scored. Output is
+// sorted by id so exports are byte-stable.
+func Convert(bundles []*evidence.Bundle, opts Options) []Record {
+ now := opts.Now
+ if now.IsZero() {
+ now = time.Now()
+ }
+ now = now.UTC()
+ hash := HiveHash(opts.HiveID)
+ version := strings.TrimSpace(opts.HiveVersion)
+ if version == "" {
+ version = kindUnknown
+ }
+ byPR := map[string][]*evidence.Bundle{}
+ for _, b := range bundles {
+ if b == nil || b.Repo == "" || b.Number == 0 {
+ continue
+ }
+ k := evidence.PRKey(b.Repo, b.Number)
+ byPR[k] = append(byPR[k], b)
+ }
+ folded := evidence.FoldPRs(bundles, "", "")
+ var out []Record
+ for key, group := range byPR {
+ pr := folded[key]
+ if pr == nil {
+ continue
+ }
+ latest := pr.Bundle
+ if !opts.Since.IsZero() && !latest.UpdatedAt.After(opts.Since) {
+ continue
+ }
+ public := opts.RepoConsent && opts.RepoPublic != nil && opts.RepoPublic(latest.Repo)
+ base := Record{
+ SchemaVersion: SchemaVersion,
+ PublishedAt: now.Format(time.RFC3339),
+ Source: Source{HiveID: hash, HiveVersion: version, Cluster: opts.Cluster, Exporter: Exporter},
+ Task: Task{RepoConsent: public},
+ Outcome: outcomeOf(pr, group),
+ Quality: qualityOf(latest, pr),
+ }
+ if public {
+ base.Task.Repo = latest.Repo
+ base.Task.RepoVisibility = "public"
+ if owner, _, ok := strings.Cut(latest.Repo, "/"); ok {
+ base.Source.Org = owner
+ }
+ }
+ if lvl := latest.Policy.ACMMLevel; lvl >= 0 && lvl <= maxACMM {
+ l := lvl
+ base.Task.ACMMLevel = &l
+ }
+ a := latest.Author
+ if a.Kind == evidence.AuthorAgent && strings.TrimSpace(a.Model) != "" {
+ rec := base
+ rec.ID = recordID(hash, latest.Repo, latest.Number, RoleAuthor)
+ name := strings.TrimSpace(a.Agent)
+ if name == "" {
+ name = "agent"
+ }
+ rec.Agent = Agent{Name: name, Role: RoleAuthor, Harness: harnessOf(a.Backend)}
+ rec.Model = Model{Provider: providerOf(a.Model), Name: strings.TrimSpace(a.Model)}
+ rec.SandboxRuntime = runtimeOf(opts, a, name)
+ rec.Task.Kind = kindOf(a.TaskKind, name)
+ out = append(out, rec)
+ }
+ if backend, model, ok := dominantReviewer(latest.Verdicts); ok {
+ rec := base
+ rec.ID = recordID(hash, latest.Repo, latest.Number, RoleReviewer)
+ rec.Agent = Agent{Name: reviewSwarmAgent, Role: RoleReviewer, Harness: harnessOf(backend)}
+ rec.Model = Model{Provider: providerOf(model), Name: model}
+ rec.SandboxRuntime = SandboxRuntimeNone
+ rec.Task.Kind = kindPRReview
+ out = append(out, rec)
+ }
+ }
+ sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID })
+ return out
+}
+
+// maxACMM is the highest ACMM level the schema accepts.
+const maxACMM = 6
+
+func runtimeOf(opts Options, a evidence.Author, agent string) string {
+ if rt := strings.TrimSpace(a.Runtime); rt != "" {
+ return rt
+ }
+ if opts.SandboxRuntime != nil {
+ if rt := strings.TrimSpace(opts.SandboxRuntime(agent)); rt != "" {
+ return rt
+ }
+ }
+ return SandboxRuntimeNone
+}
+
+func outcomeOf(pr *evidence.PR, group []*evidence.Bundle) Outcome {
+ latest := pr.Bundle
+ first := group[0]
+ heads := map[string]bool{}
+ for _, b := range group {
+ if b.CreatedAt.Before(first.CreatedAt) {
+ first = b
+ }
+ if b.HeadSHA != "" {
+ heads[b.HeadSHA] = true
+ }
+ }
+ merged := pr.Merge != nil || (pr.Outcome != nil && pr.Outcome.State == evidence.BackfillStateMerged) ||
+ (latest.Backfill != nil && latest.Backfill.State == evidence.BackfillStateMerged)
+ closed := !merged && pr.Closed
+ o := Outcome{
+ PROpened: true,
+ PRMerged: merged,
+ PRClosedUnmerged: closed,
+ CIStatusAtMerge: ciStatus(latest.CI.Status),
+ ReviewVerdict: worstVerdict(latest.Verdicts),
+ HumanInterventions: len(latest.HumanActions),
+ Retries: max(len(heads)-1, 0),
+ }
+ switch first.CI.Status {
+ case ciSuccess:
+ t := true
+ o.CIGreenFirstTry = &t
+ case ciFailure:
+ f := false
+ o.CIGreenFirstTry = &f
+ }
+ opened := pr.OpenedAt
+ var end time.Time
+ if merged {
+ if pr.Merge != nil {
+ end = pr.Merge.At
+ switch pr.Merge.Method {
+ case "merge", "squash", "rebase":
+ o.MergeMethod = pr.Merge.Method
+ default:
+ o.MergeMethod = evidence.MergeUnrecorded
+ }
+ } else {
+ o.MergeMethod = evidence.MergeUnrecorded
+ if pr.Outcome != nil {
+ end = pr.Outcome.At
+ }
+ }
+ if !end.IsZero() {
+ o.MergedAt = end.UTC().Format(time.RFC3339)
+ }
+ } else if closed && pr.Outcome != nil {
+ end = pr.Outcome.At
+ }
+ if !opened.IsZero() {
+ o.OpenedAt = opened.UTC().Format(time.RFC3339)
+ if !end.IsZero() && !end.Before(opened) {
+ secs := int64(end.Sub(opened).Seconds())
+ o.WallClockS = &secs
+ }
+ }
+ if len(latest.Verdicts) > 0 {
+ minConf, seen := 1.0, false
+ models := map[string]bool{}
+ for _, v := range latest.Verdicts {
+ if v.Confidence < minConf {
+ minConf = v.Confidence
+ }
+ seen = true
+ if m := strings.TrimSpace(v.Model); m != "" {
+ models[m] = true
+ }
+ }
+ if seen {
+ c := min(max(minConf, 0), 1)
+ o.ReviewConfidence = &c
+ }
+ for m := range models {
+ o.ReviewerModels = append(o.ReviewerModels, m)
+ }
+ sort.Strings(o.ReviewerModels)
+ }
+ return o
+}
+
+func qualityOf(latest *evidence.Bundle, pr *evidence.PR) *Quality {
+ q := &Quality{}
+ any := false
+ if pr.Merge != nil || (pr.Outcome != nil && pr.Outcome.State == evidence.BackfillStateMerged) {
+ rv, fu, ro := len(pr.RevertedBy) > 0, len(pr.FollowedUpBy) > 0, len(pr.IssueReopens) > 0
+ q.RevertedWithin14d, q.FollowedUpWithin14d, q.IssueReopened = &rv, &fu, &ro
+ any = true
+ }
+ if lvl := latest.Policy.ACMMLevel; lvl >= 0 && lvl <= maxACMM {
+ l := lvl
+ q.ACMMLevel = &l
+ any = true
+ }
+ if n := len(latest.Sentinel); n > 0 {
+ q.SentinelFindings = &n
+ any = true
+ }
+ if !any {
+ return nil
+ }
+ return q
+}
+
+func ciStatus(s string) string {
+ switch s {
+ case ciSuccess, ciFailure, ciPending:
+ return s
+ }
+ return ciUnknown
+}
+
+func worstVerdict(vs []evidence.Verdict) string {
+ best, bestRank := verdictNone, -1
+ for _, v := range vs {
+ if r, ok := verdictRank[v.Verdict]; ok && r > bestRank {
+ best, bestRank = v.Verdict, r
+ }
+ }
+ return best
+}
+
+// dominantReviewer is the (backend, model) pair that cast most verdicts.
+func dominantReviewer(vs []evidence.Verdict) (backend, model string, ok bool) {
+ type pair struct{ b, m string }
+ counts := map[pair]int{}
+ for _, v := range vs {
+ m := strings.TrimSpace(v.Model)
+ if m == "" {
+ continue
+ }
+ counts[pair{strings.TrimSpace(v.Backend), m}]++
+ }
+ best := pair{}
+ for p, n := range counts {
+ if n > counts[best] || (n == counts[best] && (best.m == "" || p.m < best.m)) {
+ best = p
+ }
+ }
+ return best.b, best.m, best.m != ""
+}
+
+func harnessOf(backend string) string {
+ b := strings.ToLower(strings.TrimSpace(backend))
+ if knownHarnesses[b] {
+ return b
+ }
+ return harnessOther
+}
+
+func providerOf(model string) string {
+ m := strings.ToLower(strings.TrimSpace(model))
+ if p, _, ok := strings.Cut(m, "/"); ok && p != "" {
+ return p
+ }
+ for _, pp := range providerPrefixes {
+ if strings.HasPrefix(m, pp.prefix) {
+ return pp.provider
+ }
+ }
+ return providerUnknown
+}
+
+// kindOf prefers the task kind stamped on the bundle and otherwise guesses
+// from the agent's name.
+func kindOf(stamped, agent string) string {
+ if k := strings.TrimSpace(stamped); k != "" {
+ return k
+ }
+ return TaskKindFor(agent)
+}
+
+// TaskKindFor guesses the benchmark task kind from an agent's name
+// ("docs-writer" → docs); "unknown" when nothing matches. It is what gets
+// stamped on new bundles until agents declare a kind.
+func TaskKindFor(agent string) string {
+ n := strings.ToLower(agent)
+ for _, h := range kindHints {
+ if strings.Contains(n, h.hint) {
+ return h.kind
+ }
+ }
+ return kindUnknown
+}
+
+func itoa(n int) string { return strconv.Itoa(n) }
diff --git a/src/pkg/evidence/benchmarks/convert_test.go b/src/pkg/evidence/benchmarks/convert_test.go
new file mode 100644
index 0000000000..b70a87d042
--- /dev/null
+++ b/src/pkg/evidence/benchmarks/convert_test.go
@@ -0,0 +1,272 @@
+package benchmarks
+
+import (
+ "encoding/json"
+ "os"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/hivecommons/hive/pkg/evidence"
+)
+
+func t0(d time.Duration) time.Time { return time.Date(2026, 10, 3, 12, 0, 0, 0, time.UTC).Add(d) }
+
+func fixtureBundles() []*evidence.Bundle {
+ first := &evidence.Bundle{
+ SchemaVersion: evidence.SchemaVersion, ID: "a", Repo: "hivecommons/hive", Number: 42,
+ Author: evidence.Author{Login: "hive[bot]", Kind: evidence.AuthorAgent, Agent: "docs-writer", Backend: "copilot", Model: "claude-fable-5.1"},
+ HeadSHA: "h1", Policy: evidence.Policy{ACMMLevel: 3},
+ CI: evidence.CI{Status: "failure"},
+ CreatedAt: t0(0), UpdatedAt: t0(time.Hour),
+ }
+ latest := &evidence.Bundle{
+ SchemaVersion: evidence.SchemaVersion, ID: "b", Repo: "hivecommons/hive", Number: 42,
+ Author: first.Author, HeadSHA: "h2", Policy: evidence.Policy{ACMMLevel: 3},
+ CI: evidence.CI{Status: "success"},
+ Verdicts: []evidence.Verdict{
+ {Perspective: "security", Model: "claude-fable-5.1", Backend: "copilot", Verdict: "approve", Confidence: 0.9},
+ {Perspective: "style", Model: "claude-fable-5.1", Backend: "copilot", Verdict: "comment", Confidence: 0.7},
+ },
+ HumanActions: []evidence.Action{{Actor: "andy", Kind: "label", At: t0(2 * time.Hour)}},
+ Merge: &evidence.MergeEvent{Actor: "hive", Method: "squash", At: t0(3 * time.Hour), OpenedAt: t0(-time.Hour)},
+ CreatedAt: t0(2 * time.Hour), UpdatedAt: t0(3 * time.Hour),
+ }
+ private := &evidence.Bundle{
+ SchemaVersion: evidence.SchemaVersion, ID: "c", Repo: "acme/secret", Number: 7,
+ Author: evidence.Author{Login: "x", Kind: evidence.AuthorAgent, Agent: "fixer", Backend: "pi", Model: "gpt-5.3-codex", Runtime: "podman", TaskKind: "issue-fix"},
+ HeadSHA: "p1", Policy: evidence.Policy{ACMMLevel: 2},
+ Outcome: &evidence.Outcome{Source: "ledger", State: evidence.BackfillStateClosed, At: t0(5 * time.Hour)},
+ CreatedAt: t0(0), UpdatedAt: t0(5 * time.Hour),
+ }
+ human := &evidence.Bundle{
+ SchemaVersion: evidence.SchemaVersion, ID: "d", Repo: "hivecommons/hive", Number: 43,
+ Author: evidence.Author{Login: "someone", Kind: evidence.AuthorHuman},
+ HeadSHA: "q1", CreatedAt: t0(0), UpdatedAt: t0(time.Hour),
+ }
+ return []*evidence.Bundle{first, latest, private, human}
+}
+
+func testOptions() Options {
+ return Options{
+ HiveID: "hosted-test", HiveVersion: "v5.1.0", Now: t0(6 * time.Hour),
+ RepoConsent: true,
+ RepoPublic: func(repo string) bool { return strings.HasPrefix(repo, "hivecommons/") },
+ SandboxRuntime: func(agent string) string { return "none" },
+ }
+}
+
+func byRole(recs []Record, repo string, role string) *Record {
+ for i := range recs {
+ if recs[i].Agent.Role == role && (recs[i].Task.Repo == repo || (repo == "" && recs[i].Task.Repo == "")) {
+ return &recs[i]
+ }
+ }
+ return nil
+}
+
+func TestConvertFoldsPRAndRedacts(t *testing.T) {
+ recs := Convert(fixtureBundles(), testOptions())
+ if len(recs) != 3 {
+ t.Fatalf("want 3 records (author+reviewer for #42, author for private #7), got %d", len(recs))
+ }
+ a := byRole(recs, "hivecommons/hive", RoleAuthor)
+ if a == nil {
+ t.Fatal("missing author record for hivecommons/hive#42")
+ }
+ if a.Source.HiveID != HiveHash("hosted-test") || a.Source.Org != "hivecommons" || !a.Task.RepoConsent {
+ t.Errorf("source/task attribution wrong: %+v %+v", a.Source, a.Task)
+ }
+ if a.Task.Kind != "docs" || a.Agent.Harness != "copilot" || a.Model.Provider != "anthropic" {
+ t.Errorf("classification wrong: kind=%s harness=%s provider=%s", a.Task.Kind, a.Agent.Harness, a.Model.Provider)
+ }
+ o := a.Outcome
+ if !o.PRMerged || o.MergeMethod != "squash" || o.Retries != 1 || o.HumanInterventions != 1 {
+ t.Errorf("outcome wrong: %+v", o)
+ }
+ if o.CIGreenFirstTry == nil || *o.CIGreenFirstTry {
+ t.Errorf("first head failed CI, want ci_green_first_try=false, got %v", o.CIGreenFirstTry)
+ }
+ if o.WallClockS == nil || *o.WallClockS != int64(4*time.Hour/time.Second) {
+ t.Errorf("wall clock: opened T-1h, merged T+3h, want 14400 got %v", o.WallClockS)
+ }
+ if o.ReviewVerdict != "comment" || o.ReviewConfidence == nil || *o.ReviewConfidence != 0.7 {
+ t.Errorf("review summary wrong: %+v", o)
+ }
+ if a.Quality == nil || a.Quality.ACMMLevel == nil || *a.Quality.ACMMLevel != 3 || a.Quality.RevertedWithin14d == nil || *a.Quality.RevertedWithin14d {
+ t.Errorf("quality wrong: %+v", a.Quality)
+ }
+
+ r := byRole(recs, "hivecommons/hive", RoleReviewer)
+ if r == nil || r.Task.Kind != "pr-review" || r.Agent.Name != reviewSwarmAgent || r.Model.Name != "claude-fable-5.1" {
+ t.Errorf("reviewer record wrong: %+v", r)
+ }
+
+ p := byRole(recs, "", RoleAuthor)
+ if p == nil {
+ t.Fatal("missing anonymized record for private repo")
+ }
+ if p.Task.Repo != "" || p.Source.Org != "" || p.Task.RepoConsent || p.Task.RepoVisibility != "" {
+ t.Errorf("private repo leaked: %+v %+v", p.Task, p.Source)
+ }
+ if p.SandboxRuntime != "podman" || p.Task.Kind != "issue-fix" {
+ t.Errorf("stamped runtime/kind not honoured: %s %s", p.SandboxRuntime, p.Task.Kind)
+ }
+ if !p.Outcome.PRClosedUnmerged || p.Outcome.PRMerged {
+ t.Errorf("ledger-closed PR should be closed unmerged: %+v", p.Outcome)
+ }
+ for _, rec := range recs {
+ if strings.Contains(rec.Agent.Name, "andy") || strings.Contains(rec.Agent.Name, "[bot]") {
+ t.Errorf("login leaked into %+v", rec.Agent)
+ }
+ }
+}
+
+func TestConvertRepoConsentOffAnonymizesPublic(t *testing.T) {
+ opts := testOptions()
+ opts.RepoConsent = false
+ for _, r := range Convert(fixtureBundles(), opts) {
+ if r.Task.Repo != "" || r.Source.Org != "" || r.Task.RepoConsent {
+ t.Fatalf("repo_consent=false must drop names: %+v", r.Task)
+ }
+ }
+}
+
+func TestConvertSinceAndStableIDs(t *testing.T) {
+ opts := testOptions()
+ all := Convert(fixtureBundles(), opts)
+ opts.Since = t0(4 * time.Hour)
+ recent := Convert(fixtureBundles(), opts)
+ if len(recent) != 1 || recent[0].Task.Repo != "" {
+ t.Fatalf("since=T+4h should keep only the private PR updated at T+5h, got %d", len(recent))
+ }
+ again := Convert(fixtureBundles(), testOptions())
+ for i := range all {
+ if all[i].ID != again[i].ID {
+ t.Fatalf("ids must be stable across runs")
+ }
+ }
+ if HiveHash("hosted-test") == HiveHash("hosted-test2") || len(HiveHash("x")) != hiveHashLen {
+ t.Fatal("hive hash shape")
+ }
+}
+
+// TestConvertOutputMatchesSchema checks every emitted record against the
+// vendored hivecommons/benchmarks schema: required keys, enum values and
+// no unknown keys at each object the schema closes.
+func TestConvertOutputMatchesSchema(t *testing.T) {
+ raw, err := os.ReadFile("testdata/result.schema.json")
+ if err != nil {
+ t.Fatal(err)
+ }
+ var schema map[string]any
+ if err := json.Unmarshal(raw, &schema); err != nil {
+ t.Fatal(err)
+ }
+ for _, rec := range Convert(fixtureBundles(), testOptions()) {
+ data, _ := json.Marshal(rec)
+ var obj map[string]any
+ _ = json.Unmarshal(data, &obj)
+ checkSchema(t, "", schema, obj)
+ }
+}
+
+func checkSchema(t *testing.T, path string, schema map[string]any, v any) {
+ t.Helper()
+ if enum, ok := schema["enum"].([]any); ok {
+ found := false
+ for _, e := range enum {
+ if e == v {
+ found = true
+ }
+ }
+ if !found {
+ t.Errorf("%s: %v not in enum %v", path, v, enum)
+ }
+ }
+ props, _ := schema["properties"].(map[string]any)
+ obj, isObj := v.(map[string]any)
+ if !isObj {
+ return
+ }
+ if req, ok := schema["required"].([]any); ok {
+ for _, r := range req {
+ if _, present := obj[r.(string)]; !present {
+ t.Errorf("%s: missing required %s", path, r)
+ }
+ }
+ }
+ for k, val := range obj {
+ sub, ok := props[k].(map[string]any)
+ if !ok {
+ if ap, closed := schema["additionalProperties"].(bool); closed && !ap {
+ t.Errorf("%s: unexpected key %s", path, k)
+ }
+ continue
+ }
+ if val == nil {
+ continue
+ }
+ checkSchema(t, path+"."+k, sub, val)
+ }
+}
+
+func TestConvertEdgeCases(t *testing.T) {
+ // Backfilled merge (no Merge event), first bundle green, sentinel
+ // findings, nameless agent on an unknown backend/model, and a bundle
+ // that cannot be keyed — all with zero-value Options.
+ ledgerMerged := &evidence.Bundle{
+ SchemaVersion: evidence.SchemaVersion, ID: "e", Repo: "o/r", Number: 9, HeadSHA: "s1",
+ Author: evidence.Author{Login: "bot", Kind: evidence.AuthorAgent, Backend: "mystery", Model: "zeta-9000"},
+ CI: evidence.CI{Status: "success"},
+ Sentinel: []evidence.SentinelFinding{{}},
+ Outcome: &evidence.Outcome{Source: "ledger", State: evidence.BackfillStateMerged, At: t0(4 * time.Hour)},
+ Verdicts: []evidence.Verdict{{Perspective: "security", Verdict: "approve"}}, // no model → no reviewer record
+ CreatedAt: t0(2 * time.Hour), UpdatedAt: t0(4 * time.Hour),
+ }
+ earlier := &evidence.Bundle{
+ SchemaVersion: evidence.SchemaVersion, ID: "f", Repo: "o/r", Number: 9, HeadSHA: "s0",
+ Author: ledgerMerged.Author, CI: evidence.CI{Status: "success"},
+ CreatedAt: t0(0), UpdatedAt: t0(time.Hour),
+ }
+ unkeyed := &evidence.Bundle{SchemaVersion: evidence.SchemaVersion, ID: "g"}
+ recs := Convert([]*evidence.Bundle{ledgerMerged, nil, unkeyed, earlier}, Options{HiveID: "x"})
+ if len(recs) != 1 {
+ t.Fatalf("want 1 author record, got %d: %+v", len(recs), recs)
+ }
+ r := recs[0]
+ if r.Agent.Name != "agent" || r.Agent.Harness != "other" || r.Model.Provider != "unknown" || r.Task.Kind != "unknown" {
+ t.Errorf("defaults: %+v %+v %+v", r.Agent, r.Model, r.Task)
+ }
+ if r.SandboxRuntime != SandboxRuntimeNone || r.Source.HiveVersion == "" || r.PublishedAt == "" {
+ t.Errorf("fallbacks: runtime=%q version=%q published=%q", r.SandboxRuntime, r.Source.HiveVersion, r.PublishedAt)
+ }
+ if !r.Outcome.PRMerged || r.Outcome.MergeMethod != evidence.MergeUnrecorded || r.Outcome.MergedAt == "" || r.Outcome.Retries != 1 {
+ t.Errorf("ledger-merged outcome: %+v", r.Outcome)
+ }
+ if r.Outcome.CIGreenFirstTry == nil || !*r.Outcome.CIGreenFirstTry {
+ t.Errorf("first bundle was green: %+v", r.Outcome.CIGreenFirstTry)
+ }
+ if r.Quality == nil || r.Quality.SentinelFindings == nil || *r.Quality.SentinelFindings != 1 {
+ t.Errorf("quality: %+v", r.Quality)
+ }
+
+ // A merge event with an unknown method is recorded as unrecorded.
+ odd := &evidence.Bundle{
+ SchemaVersion: evidence.SchemaVersion, ID: "h", Repo: "o/r", Number: 10, HeadSHA: "t1",
+ Author: evidence.Author{Kind: evidence.AuthorAgent, Model: "openai/gpt-5", Agent: "ci-fixer"},
+ Policy: evidence.Policy{ACMMLevel: 99},
+ Merge: &evidence.MergeEvent{Actor: "x", Method: "teleport", At: t0(time.Hour)},
+ CreatedAt: t0(0), UpdatedAt: t0(time.Hour),
+ }
+ recs = Convert([]*evidence.Bundle{odd}, Options{HiveID: "x", Now: t0(2 * time.Hour)})
+ if len(recs) != 1 || recs[0].Outcome.MergeMethod != evidence.MergeUnrecorded || recs[0].Model.Provider != "openai" || recs[0].Task.Kind != "ci-repair" {
+ t.Errorf("odd merge: %+v", recs)
+ }
+ if recs[0].Quality != nil && recs[0].Quality.ACMMLevel != nil {
+ t.Errorf("out-of-range ACMM level must be dropped: %+v", recs[0].Quality)
+ }
+ if got := qualityOf(&evidence.Bundle{Policy: evidence.Policy{ACMMLevel: 99}}, &evidence.PR{}); got != nil {
+ t.Errorf("qualityOf with nothing to report = %+v, want nil", got)
+ }
+}
diff --git a/src/pkg/evidence/benchmarks/record.go b/src/pkg/evidence/benchmarks/record.go
new file mode 100644
index 0000000000..50b2aaf0f9
--- /dev/null
+++ b/src/pkg/evidence/benchmarks/record.go
@@ -0,0 +1,117 @@
+// Package benchmarks turns review evidence bundles into the records published
+// at https://hivecommons.dev/benchmarks/ (schema: hivecommons/benchmarks
+// schema/result.schema.json; RFC hivecommons/hive#17694).
+//
+// Privacy contract: a record never carries a login, a prompt, finding text or
+// code. The hive is identified only by HiveHash. Repository and org names are
+// present only when Options.RepoPublic reports the repo public AND the owner
+// has not set benchmarks.repo_consent: false; otherwise task.repo is absent.
+package benchmarks
+
+import (
+ "crypto/sha256"
+ "encoding/hex"
+ "strings"
+)
+
+// SchemaVersion is the published record schema version.
+const SchemaVersion = "1"
+
+// hashDomain salts the hive id so the published hash cannot be joined with
+// any other sha256(hive_id) in the wild.
+const hashDomain = "hivecommons-benchmarks/v1/"
+
+// hiveHashLen is the published hex prefix length of the hive hash.
+const hiveHashLen = 16
+
+// recordIDLen is the hex length of a record id.
+const recordIDLen = 24
+
+// Exporter identifies the producer in source.exporter.
+const Exporter = "hive"
+
+// HiveHash is the anonymous, stable identifier of a hive on the leaderboard.
+// Owners see theirs in Settings → Benchmarks and in /api/status.
+func HiveHash(hiveID string) string {
+ sum := sha256.Sum256([]byte(hashDomain + strings.TrimSpace(hiveID)))
+ return hex.EncodeToString(sum[:])[:hiveHashLen]
+}
+
+// Record is one benchmark result: one agent role on one PR.
+type Record struct {
+ SchemaVersion string `json:"schema_version"`
+ ID string `json:"id"`
+ PublishedAt string `json:"published_at"`
+ Source Source `json:"source"`
+ Agent Agent `json:"agent"`
+ Model Model `json:"model"`
+ SandboxRuntime string `json:"sandbox_runtime"`
+ Task Task `json:"task"`
+ Outcome Outcome `json:"outcome"`
+ Quality *Quality `json:"quality,omitempty"`
+}
+
+// Source says which hive emitted the record.
+type Source struct {
+ HiveID string `json:"hive_id"`
+ Org string `json:"org,omitempty"`
+ Cluster string `json:"cluster,omitempty"`
+ HiveVersion string `json:"hive_version"`
+ Exporter string `json:"exporter,omitempty"`
+}
+
+// Agent is the hive agent (or the review swarm) the record scores.
+type Agent struct {
+ Name string `json:"name"`
+ Role string `json:"role,omitempty"`
+ Harness string `json:"harness"`
+}
+
+// Model is the model the agent ran on.
+type Model struct {
+ Provider string `json:"provider"`
+ Name string `json:"name"`
+}
+
+// Task describes the unit of work.
+type Task struct {
+ Kind string `json:"kind"`
+ Repo string `json:"repo,omitempty"`
+ RepoConsent bool `json:"repo_consent"`
+ RepoVisibility string `json:"repo_visibility,omitempty"`
+ ACMMLevel *int `json:"acmm_level,omitempty"`
+}
+
+// Outcome is what happened to the PR.
+type Outcome struct {
+ PROpened bool `json:"pr_opened"`
+ PRMerged bool `json:"pr_merged"`
+ PRClosedUnmerged bool `json:"pr_closed_unmerged"`
+ MergeMethod string `json:"merge_method,omitempty"`
+ CIGreenFirstTry *bool `json:"ci_green_first_try"`
+ CIStatusAtMerge string `json:"ci_status_at_merge"`
+ ReviewVerdict string `json:"review_verdict"`
+ ReviewConfidence *float64 `json:"review_confidence,omitempty"`
+ ReviewerModels []string `json:"reviewer_models,omitempty"`
+ HumanInterventions int `json:"human_interventions"`
+ Retries int `json:"retries"`
+ WallClockS *int64 `json:"wall_clock_s,omitempty"`
+ OpenedAt string `json:"opened_at,omitempty"`
+ MergedAt string `json:"merged_at,omitempty"`
+}
+
+// Quality carries post-merge signals and the governing ACMM level.
+type Quality struct {
+ RevertedWithin14d *bool `json:"reverted_within_14d,omitempty"`
+ FollowedUpWithin14d *bool `json:"followed_up_within_14d,omitempty"`
+ IssueReopened *bool `json:"issue_reopened,omitempty"`
+ ACMMLevel *int `json:"acmm_level,omitempty"`
+ SentinelFindings *int `json:"sentinel_findings,omitempty"`
+}
+
+// recordID derives the stable id of one (hive, PR, role) triple so re-exports
+// overwrite rather than duplicate.
+func recordID(hiveHash, repo string, number int, role string) string {
+ sum := sha256.Sum256([]byte(hiveHash + "|" + repo + "|" + itoa(number) + "|" + role))
+ return hex.EncodeToString(sum[:])[:recordIDLen]
+}
diff --git a/src/pkg/evidence/benchmarks/testdata/result.schema.json b/src/pkg/evidence/benchmarks/testdata/result.schema.json
new file mode 100644
index 0000000000..c4ecd39ec6
--- /dev/null
+++ b/src/pkg/evidence/benchmarks/testdata/result.schema.json
@@ -0,0 +1,449 @@
+{
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ "$id": "https://hivecommons.dev/benchmarks/schema/result.schema.json",
+ "title": "Hive Commons benchmark result",
+ "description": "One benchmark record: a single unit of agent work (usually one pull request) with its outcome, cost and quality signals. Field names follow the hive evidence bundle (pkg/evidence.Bundle) and GET /api/cost so a spoke can emit records directly.",
+ "type": "object",
+ "additionalProperties": false,
+ "required": [
+ "schema_version",
+ "id",
+ "published_at",
+ "source",
+ "agent",
+ "model",
+ "sandbox_runtime",
+ "task",
+ "outcome"
+ ],
+ "properties": {
+ "schema_version": {
+ "const": "1"
+ },
+ "id": {
+ "type": "string",
+ "description": "Stable unique id. Convention: sha256 of (hive_id, repo, pr number, head sha) hex-encoded, or any URN.",
+ "minLength": 8
+ },
+ "published_at": {
+ "type": "string",
+ "format": "date-time"
+ },
+ "synthetic": {
+ "type": "boolean",
+ "default": false,
+ "description": "True for hand-written sample records. The site excludes these from every leaderboard."
+ },
+ "source": {
+ "type": "object",
+ "additionalProperties": false,
+ "required": [
+ "hive_id",
+ "hive_version"
+ ],
+ "properties": {
+ "hive_id": {
+ "type": "string",
+ "description": "Stable anonymous id: sha256('hivecommons-benchmarks/v1/' + hive id)[:16]. Shown to the owner in their dashboard (Settings \u2192 Benchmarks)."
+ },
+ "hive_name": {
+ "type": "string",
+ "description": "Only present when the owner opted in (benchmarks.publish_name)."
+ },
+ "org": {
+ "type": "string",
+ "description": "GitHub org of the repository. Present only when task.repo_consent is true (public repo)."
+ },
+ "cluster": {
+ "type": "string",
+ "description": "Free-form cluster / environment label (e.g. kind, openshift, bare-metal)."
+ },
+ "hive_version": {
+ "type": "string"
+ },
+ "exporter": {
+ "type": "string",
+ "description": "What produced the record, e.g. hive/api/benchmarks/export, hub/aggregate, manual.",
+ "examples": [
+ "hive/api/benchmarks/export",
+ "hub/publisher",
+ "backfill-from-cluster",
+ "manual"
+ ]
+ }
+ }
+ },
+ "agent": {
+ "type": "object",
+ "additionalProperties": false,
+ "required": [
+ "name",
+ "harness"
+ ],
+ "properties": {
+ "name": {
+ "type": "string",
+ "description": "Agent name as configured in hive.yaml (evidence author.agent)."
+ },
+ "role": {
+ "type": "string",
+ "description": "author = the agent opened the PR; reviewer = the agent reviewed someone else's PR. Other values are free-form lane names.",
+ "examples": [
+ "author",
+ "reviewer",
+ "fix",
+ "review",
+ "triage",
+ "ci-repair"
+ ]
+ },
+ "harness": {
+ "type": "string",
+ "description": "CLI harness / backend (evidence author.backend, crux IsCLIBackend list).",
+ "enum": [
+ "agy",
+ "aider",
+ "bob",
+ "claude",
+ "cline",
+ "codex",
+ "copilot",
+ "gemini",
+ "goose",
+ "kilo",
+ "muse",
+ "omnigent",
+ "omp",
+ "opencode",
+ "openhands",
+ "pi",
+ "clanker",
+ "other"
+ ]
+ },
+ "harness_version": {
+ "type": "string"
+ }
+ }
+ },
+ "model": {
+ "type": "object",
+ "additionalProperties": false,
+ "required": [
+ "provider",
+ "name"
+ ],
+ "properties": {
+ "provider": {
+ "type": "string",
+ "description": "Vendor family (pkg/modelfamily), e.g. anthropic, openai, google, ibm, local."
+ },
+ "name": {
+ "type": "string",
+ "description": "Exact model id as reported by the harness (evidence author.model)."
+ },
+ "version": {
+ "type": "string"
+ },
+ "effort": {
+ "type": "string",
+ "description": "Reasoning effort / thinking budget label if the harness exposes one."
+ }
+ }
+ },
+ "sandbox_runtime": {
+ "type": "string",
+ "description": "Where the agent process ran (crux/config SandboxRuntime*). Open enum.",
+ "anyOf": [
+ {
+ "enum": [
+ "podman",
+ "kubernetes-job",
+ "openshell",
+ "substrate",
+ "none"
+ ]
+ },
+ {
+ "type": "string",
+ "pattern": "^x-[a-z0-9-]+$"
+ }
+ ]
+ },
+ "task": {
+ "type": "object",
+ "additionalProperties": false,
+ "required": [
+ "kind"
+ ],
+ "properties": {
+ "kind": {
+ "type": "string",
+ "enum": [
+ "issue-fix",
+ "pr-review",
+ "ci-repair",
+ "triage",
+ "release",
+ "docs",
+ "security-fix",
+ "dependency-update",
+ "refactor",
+ "experiment",
+ "other",
+ "unknown"
+ ]
+ },
+ "repo": {
+ "type": "string",
+ "description": "owner/name. MUST be omitted unless repo_consent is true.",
+ "pattern": "^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$"
+ },
+ "repo_consent": {
+ "type": "boolean",
+ "description": "Repository owner consented to being named in published results."
+ },
+ "repo_visibility": {
+ "type": "string",
+ "enum": [
+ "public",
+ "private"
+ ]
+ },
+ "language": {
+ "type": "string",
+ "description": "Primary language of the touched files."
+ },
+ "size": {
+ "type": "object",
+ "additionalProperties": false,
+ "properties": {
+ "files_changed": {
+ "type": "integer",
+ "minimum": 0
+ },
+ "additions": {
+ "type": "integer",
+ "minimum": 0
+ },
+ "deletions": {
+ "type": "integer",
+ "minimum": 0
+ },
+ "bucket": {
+ "type": "string",
+ "enum": [
+ "xs",
+ "s",
+ "m",
+ "l",
+ "xl"
+ ]
+ }
+ }
+ },
+ "acmm_level": {
+ "type": "integer",
+ "minimum": 0,
+ "maximum": 6,
+ "description": "Policy ACMM level the hive ran at (evidence policy.acmm_level)."
+ }
+ },
+ "if": {
+ "properties": {
+ "repo_consent": {
+ "const": false
+ }
+ }
+ },
+ "then": {
+ "not": {
+ "required": [
+ "repo"
+ ]
+ }
+ }
+ },
+ "outcome": {
+ "type": "object",
+ "additionalProperties": false,
+ "required": [
+ "pr_opened"
+ ],
+ "properties": {
+ "pr_opened": {
+ "type": "boolean"
+ },
+ "pr_merged": {
+ "type": "boolean"
+ },
+ "pr_closed_unmerged": {
+ "type": "boolean"
+ },
+ "merge_method": {
+ "type": "string",
+ "enum": [
+ "merge",
+ "squash",
+ "rebase",
+ "unrecorded"
+ ]
+ },
+ "ci_green_first_try": {
+ "type": [
+ "boolean",
+ "null"
+ ],
+ "description": "CI status was success on the first head (evidence ci.status on the bundle with no previous_bundle_id). null = no CI captured."
+ },
+ "ci_status_at_merge": {
+ "type": "string",
+ "enum": [
+ "success",
+ "failure",
+ "pending",
+ "unknown"
+ ]
+ },
+ "review_verdict": {
+ "type": "string",
+ "description": "Worst swarm verdict on the final head (evidence verdicts[].verdict); reject > changes_requested > requires_human > comment > approve.",
+ "enum": [
+ "approve",
+ "comment",
+ "requires_human",
+ "changes_requested",
+ "reject",
+ "none"
+ ]
+ },
+ "review_confidence": {
+ "type": "number",
+ "minimum": 0,
+ "maximum": 1
+ },
+ "reviewer_models": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ },
+ "description": "Exact model ids of the reviewing perspectives."
+ },
+ "human_interventions": {
+ "type": "integer",
+ "minimum": 0,
+ "description": "len(evidence human_actions)."
+ },
+ "wall_clock_s": {
+ "type": "number",
+ "minimum": 0,
+ "description": "merge.at - merge.opened_at, or close time for unmerged."
+ },
+ "retries": {
+ "type": "integer",
+ "minimum": 0,
+ "description": "Number of additional heads pushed (bundles chained by previous_bundle_id)."
+ },
+ "opened_at": {
+ "type": "string",
+ "format": "date-time"
+ },
+ "merged_at": {
+ "type": "string",
+ "format": "date-time"
+ }
+ }
+ },
+ "cost": {
+ "type": "object",
+ "additionalProperties": false,
+ "properties": {
+ "input_tokens": {
+ "type": "integer",
+ "minimum": 0
+ },
+ "output_tokens": {
+ "type": "integer",
+ "minimum": 0
+ },
+ "cache_read_tokens": {
+ "type": "integer",
+ "minimum": 0
+ },
+ "cache_write_tokens": {
+ "type": "integer",
+ "minimum": 0
+ },
+ "usd": {
+ "type": "number",
+ "minimum": 0
+ },
+ "basis": {
+ "type": "string",
+ "description": "How usd was computed (GET /api/cost: estimated from public price table, or apportioned from per-agent totals).",
+ "enum": [
+ "estimated",
+ "apportioned",
+ "billed",
+ "unpriced"
+ ]
+ },
+ "price_table_date": {
+ "type": "string",
+ "format": "date"
+ }
+ }
+ },
+ "quality": {
+ "type": "object",
+ "additionalProperties": false,
+ "properties": {
+ "reverted_within_14d": {
+ "type": [
+ "boolean",
+ "null"
+ ]
+ },
+ "followed_up_within_14d": {
+ "type": [
+ "boolean",
+ "null"
+ ]
+ },
+ "issue_reopened": {
+ "type": [
+ "boolean",
+ "null"
+ ]
+ },
+ "acmm_level": {
+ "type": "integer",
+ "minimum": 0,
+ "maximum": 6,
+ "description": "Repo overall ACMM level at the time (GET /api/acmm/evaluation overall_level); context, not an agent score."
+ },
+ "acmm_scores": {
+ "type": "object",
+ "description": "Per-level detected_score 0..1 keyed by level number, from GET /api/acmm/evaluation levels[].",
+ "patternProperties": {
+ "^[0-6]$": {
+ "type": "number",
+ "minimum": 0,
+ "maximum": 1
+ }
+ },
+ "additionalProperties": false
+ },
+ "sentinel_findings": {
+ "type": "integer",
+ "minimum": 0
+ }
+ }
+ },
+ "notes": {
+ "type": "string",
+ "maxLength": 2000,
+ "description": "Free text. MUST NOT contain prompts, logins, or code excerpts."
+ }
+ }
+}
diff --git a/src/pkg/evidence/evidence.go b/src/pkg/evidence/evidence.go
index 8818e39c44..a3c27e11f1 100644
--- a/src/pkg/evidence/evidence.go
+++ b/src/pkg/evidence/evidence.go
@@ -91,13 +91,19 @@ type Bundle struct {
// Author identifies the PR author. Kind is one of human, agent or bot. For
// an agent-authored PR, Agent, Backend and Model are the launch metadata the
-// hive recorded when the agent opened it; empty when unknown.
+// hive recorded when the agent opened it; empty when unknown. Runtime is the
+// sandbox runtime the agent's kicks ran under (podman, job, openshell, or
+// "none" in-process) and TaskKind the benchmark task class
+// (hivecommons/hive#17694); both empty on records written before they were
+// stamped.
type Author struct {
- Login string `json:"login"`
- Kind string `json:"kind"`
- Agent string `json:"agent,omitempty"`
- Backend string `json:"backend,omitempty"`
- Model string `json:"model,omitempty"`
+ Login string `json:"login"`
+ Kind string `json:"kind"`
+ Agent string `json:"agent,omitempty"`
+ Backend string `json:"backend,omitempty"`
+ Model string `json:"model,omitempty"`
+ Runtime string `json:"runtime,omitempty"`
+ TaskKind string `json:"task_kind,omitempty"`
}
// Policy snapshots the review policy in force when the bundle was written.
diff --git a/src/pkg/github/review_evidence.go b/src/pkg/github/review_evidence.go
index 5b4574be5c..b0c5f6b327 100644
--- a/src/pkg/github/review_evidence.go
+++ b/src/pkg/github/review_evidence.go
@@ -803,6 +803,13 @@ func (c *Client) reviewEvidencePolicy(p evidence.Policy) evidence.Policy {
return p
}
+// EvidenceAgentContext resolves the sandbox runtime and benchmark task kind
+// stamped on agent-authored bundles (Author.Runtime / Author.TaskKind,
+// hivecommons/hive#17694). The composition root installs the config-backed
+// resolver; the default leaves both empty, which the exporter reads as
+// "none" / "unknown".
+var EvidenceAgentContext = func(agent string) (runtime, taskKind string) { return "", "" }
+
// evidenceAuthor classifies the PR author. A PR the audit log records as
// opened by one of this hive's agents is "agent" even though GitHub shows the
// App bot as its author, and carries the agent, backend and model that audit
@@ -810,7 +817,8 @@ func (c *Client) reviewEvidencePolicy(p evidence.Policy) evidence.Policy {
func evidenceAuthor(repo string, number int, user *gh.User) evidence.Author {
login := strings.TrimSpace(user.GetLogin())
if m := auditAuthorMetaForPR(repo, number, reviewVerdictAuditPath, time.Now().UTC().Add(-reviewVerdictAuditLookback)); m.Agent != "" {
- return evidence.Author{Login: login, Kind: evidence.AuthorAgent, Agent: m.Agent, Backend: m.Backend, Model: m.Model}
+ runtime, kind := EvidenceAgentContext(m.Agent)
+ return evidence.Author{Login: login, Kind: evidence.AuthorAgent, Agent: m.Agent, Backend: m.Backend, Model: m.Model, Runtime: runtime, TaskKind: kind}
}
switch {
case strings.EqualFold(user.GetType(), "Bot") || strings.HasSuffix(strings.ToLower(login), "[bot]"):
diff --git a/src/pkg/hub/background_pollers.go b/src/pkg/hub/background_pollers.go
index 8f9021dabd..4bcd0f8043 100644
--- a/src/pkg/hub/background_pollers.go
+++ b/src/pkg/hub/background_pollers.go
@@ -48,6 +48,9 @@ func (s *HubServer) StartBackgroundPollers(ctx context.Context) <-chan struct{}
start(s.startAlertStatePersistence)
// Live-vs-approved org reconciliation for assigned hives (#11847).
start(s.startPlaceholderOrgSweep)
+ // Hourly commit of spoke benchmark records to hivecommons/benchmarks
+ // (hivecommons/hive#17694).
+ start(s.StartBenchmarksPublisher)
s.githubActivityMu.Lock()
s.githubActivityCtx = ctx
feed := s.githubActivityFeed
diff --git a/src/pkg/hub/benchmarks.go b/src/pkg/hub/benchmarks.go
new file mode 100644
index 0000000000..89f7b4d564
--- /dev/null
+++ b/src/pkg/hub/benchmarks.go
@@ -0,0 +1,457 @@
+package hub
+
+import (
+ "bytes"
+ "context"
+ "encoding/base64"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "log/slog"
+ "net/http"
+ "os"
+ "path/filepath"
+ "regexp"
+ "sort"
+ "strings"
+ "sync"
+ "time"
+
+ "github.com/hivecommons/hive/pkg/evidence/benchmarks"
+)
+
+// Fleet benchmark publication (hivecommons/hive#17694). Spokes POST their
+// anonymized per-PR records here hourly (pkg/dashboard/api_benchmarks.go);
+// the hub merges them by record id into /data/benchmarks//.json
+// and, hourly, commits every changed month file to the data repo through the
+// GitHub Contents API as the hub's own identity. Direct pushes to main: the
+// target is a data-only repo whose Pages workflow rebuilds on push.
+
+const (
+ benchmarksIngestPath = "/api/benchmarks/ingest"
+ benchmarksStatusPath = "/api/hub/benchmarks"
+ // benchmarksIngestMaxBodyBytes caps one spoke batch (2000 records ≈ 2 MB).
+ benchmarksIngestMaxBodyBytes = 8 << 20
+ // benchmarksPublishInterval is the hub→GitHub cadence.
+ benchmarksPublishInterval = time.Hour
+ // benchmarksPublishInitialDelay lets the first spoke batches land.
+ benchmarksPublishInitialDelay = 10 * time.Minute
+ // benchmarksCommitTimeout bounds one Contents API call.
+ benchmarksCommitTimeout = 45 * time.Second
+ // benchmarksRepoEnv overrides the target repo (owner/name).
+ benchmarksRepoEnv = "HIVE_BENCHMARKS_REPO"
+ // benchmarksRepoDefault is the public leaderboard data repo.
+ benchmarksRepoDefault = "hivecommons/benchmarks"
+ // benchmarksDataPrefix is the path inside the repo.
+ benchmarksDataPrefix = "data/fleet"
+ // benchmarksDisableEnv turns the GitHub publisher off (ingest still stores).
+ benchmarksDisableEnv = "HIVE_BENCHMARKS_PUBLISH_DISABLED"
+ // benchmarksMaxFilesPerRun bounds commits per hour so a cold start does
+ // not burn the installation's write budget in one go.
+ benchmarksMaxFilesPerRun = 60
+)
+
+var (
+ // benchmarksStoreDir is the on-PVC mirror of the repo's data/fleet tree.
+ benchmarksStoreDir = "/data/benchmarks"
+ // benchmarksStatePath remembers the last publish for the status page.
+ benchmarksStatePath = "/data/benchmarks-publish.json"
+ // benchmarksGitHubAPI is the Contents API base; a var for tests.
+ benchmarksGitHubAPI = "https://api.github.com"
+ hiveHashRe = regexp.MustCompile(`^[0-9a-f]{16}$`)
+ monthRe = regexp.MustCompile(`^\d{4}-\d{2}$`)
+)
+
+// BenchmarksPublishState is the hub status block.
+type BenchmarksPublishState struct {
+ LastIngestAt string `json:"last_ingest_at,omitempty"`
+ LastPublishAt string `json:"last_publish_at,omitempty"`
+ LastPublishFiles int `json:"last_publish_files"`
+ LastError string `json:"last_error,omitempty"`
+ Repo string `json:"repo"`
+ PendingFiles int `json:"pending_files"`
+ HiveCount int `json:"hive_count"`
+ RecordCount int `json:"record_count"`
+ Hives map[string]int `json:"hives,omitempty"`
+ Enabled bool `json:"enabled"`
+}
+
+type benchmarksHub struct {
+ mu sync.Mutex
+ pending map[string]bool // relative file paths (hash/month.json) changed since last publish
+ state BenchmarksPublishState
+ loaded bool
+}
+
+var hubBenchmarks = &benchmarksHub{pending: map[string]bool{}}
+
+func (b *benchmarksHub) load() {
+ if b.loaded {
+ return
+ }
+ b.loaded = true
+ data, err := os.ReadFile(benchmarksStatePath)
+ if err != nil {
+ return
+ }
+ var st struct {
+ BenchmarksPublishState
+ Pending []string `json:"pending"`
+ }
+ if json.Unmarshal(data, &st) == nil {
+ b.state = st.BenchmarksPublishState
+ for _, p := range st.Pending {
+ b.pending[p] = true
+ }
+ }
+}
+
+func (b *benchmarksHub) save() {
+ var pending []string
+ for p := range b.pending {
+ pending = append(pending, p)
+ }
+ sort.Strings(pending)
+ data, err := json.Marshal(struct {
+ BenchmarksPublishState
+ Pending []string `json:"pending"`
+ }{b.state, pending})
+ if err != nil {
+ return
+ }
+ _ = os.MkdirAll(filepath.Dir(benchmarksStatePath), 0o755)
+ _ = os.WriteFile(benchmarksStatePath, data, 0o644)
+}
+
+type benchmarksIngestBody struct {
+ HiveID string `json:"hive_id"`
+ HiveHash string `json:"hive_hash"`
+ Watermark string `json:"watermark"`
+ Records []benchmarks.Record `json:"records"`
+}
+
+// handleBenchmarksIngest accepts one spoke batch. Auth is the per-hive
+// heartbeat bearer bound to hive_id, exactly as NPS ingest. The hive hash is
+// recomputed from hive_id, never trusted from the body.
+func (s *HubServer) handleBenchmarksIngest(w http.ResponseWriter, r *http.Request) {
+ if s.hubSecret == "" {
+ npsJSONError(w, "benchmarks ingest requires a configured hub secret", http.StatusServiceUnavailable)
+ return
+ }
+ auth := r.Header.Get("Authorization")
+ if !strings.HasPrefix(auth, "Bearer ") {
+ npsJSONError(w, "unauthorized", http.StatusUnauthorized)
+ return
+ }
+ body, err := io.ReadAll(io.LimitReader(r.Body, benchmarksIngestMaxBodyBytes+1))
+ if err != nil {
+ npsJSONError(w, "read error", http.StatusBadRequest)
+ return
+ }
+ if len(body) > benchmarksIngestMaxBodyBytes {
+ npsJSONError(w, "payload too large", http.StatusRequestEntityTooLarge)
+ return
+ }
+ var p benchmarksIngestBody
+ if err := json.Unmarshal(body, &p); err != nil {
+ npsJSONError(w, "invalid payload", http.StatusBadRequest)
+ return
+ }
+ hiveID := sanitizeHeartbeatField(p.HiveID)
+ if hiveID == "" || hiveID != p.HiveID || !isValidName(hiveID) {
+ npsJSONError(w, "invalid hive_id", http.StatusBadRequest)
+ return
+ }
+ if !s.verifyHeartbeatBearer(strings.TrimPrefix(auth, "Bearer "), hiveID) {
+ npsJSONError(w, "unauthorized", http.StatusUnauthorized)
+ return
+ }
+ if !s.npsHiveRegistered(hiveID) {
+ npsJSONError(w, "unknown hive - heartbeat first", http.StatusForbidden)
+ return
+ }
+ hash := benchmarks.HiveHash(hiveID)
+ stored, rejected, err := storeBenchmarkRecords(hash, p.Records)
+ if err != nil {
+ if s.logger != nil {
+ s.logger.Error("benchmarks: store failed", "hive_hash", hash, "error", err)
+ }
+ npsJSONError(w, "store failed", http.StatusInternalServerError)
+ return
+ }
+ hubBenchmarks.mu.Lock()
+ hubBenchmarks.load()
+ hubBenchmarks.state.LastIngestAt = time.Now().UTC().Format(time.RFC3339)
+ for _, f := range stored {
+ hubBenchmarks.pending[f] = true
+ }
+ hubBenchmarks.save()
+ hubBenchmarks.mu.Unlock()
+ w.Header().Set("Content-Type", "application/json")
+ w.WriteHeader(http.StatusAccepted)
+ _ = json.NewEncoder(w).Encode(map[string]any{"ok": true, "accepted": len(p.Records) - rejected, "rejected": rejected, "files": len(stored)})
+}
+
+// storeBenchmarkRecords merges records by id into /.json files
+// and returns the relative paths it changed. A record whose source.hive_id
+// is not the authenticated hash is rejected: a spoke publishes only as itself.
+func storeBenchmarkRecords(hash string, recs []benchmarks.Record) (changed []string, rejected int, err error) {
+ byMonth := map[string][]benchmarks.Record{}
+ for _, r := range recs {
+ if r.Source.HiveID != hash || r.ID == "" {
+ rejected++
+ continue
+ }
+ r.Source.HiveID = hash
+ byMonth[benchmarkMonth(r)] = append(byMonth[benchmarkMonth(r)], r)
+ }
+ for month, add := range byMonth {
+ rel := filepath.Join(hash, month+".json")
+ path := filepath.Join(benchmarksStoreDir, rel)
+ existing := map[string]benchmarks.Record{}
+ if data, rerr := os.ReadFile(path); rerr == nil {
+ var cur []benchmarks.Record
+ if json.Unmarshal(data, &cur) == nil {
+ for _, r := range cur {
+ existing[r.ID] = r
+ }
+ }
+ }
+ for _, r := range add {
+ existing[r.ID] = r
+ }
+ merged := make([]benchmarks.Record, 0, len(existing))
+ for _, r := range existing {
+ merged = append(merged, r)
+ }
+ sort.Slice(merged, func(i, j int) bool { return merged[i].ID < merged[j].ID })
+ data, merr := json.Marshal(merged)
+ if merr != nil {
+ return changed, rejected, merr
+ }
+ if werr := os.MkdirAll(filepath.Dir(path), 0o755); werr != nil {
+ return changed, rejected, werr
+ }
+ if werr := os.WriteFile(path, append(data, '\n'), 0o644); werr != nil {
+ return changed, rejected, werr
+ }
+ changed = append(changed, filepath.ToSlash(rel))
+ }
+ sort.Strings(changed)
+ return changed, rejected, nil
+}
+
+// benchmarkMonth files a record under the month it merged, else opened,
+// else was published.
+func benchmarkMonth(r benchmarks.Record) string {
+ for _, ts := range []string{r.Outcome.MergedAt, r.Outcome.OpenedAt, r.PublishedAt} {
+ if t, err := time.Parse(time.RFC3339, ts); err == nil {
+ return t.UTC().Format("2006-01")
+ }
+ }
+ return time.Now().UTC().Format("2006-01")
+}
+
+// benchmarksTargetRepo is owner/name of the data repo.
+func benchmarksTargetRepo() string {
+ if v := strings.TrimSpace(os.Getenv(benchmarksRepoEnv)); v != "" {
+ return v
+ }
+ return benchmarksRepoDefault
+}
+
+func benchmarksPublishEnabled() bool {
+ return strings.TrimSpace(os.Getenv(benchmarksDisableEnv)) == ""
+}
+
+// StartBenchmarksPublisher commits pending month files to the data repo
+// every hour. Registered from StartBackgroundPollers.
+func (s *HubServer) StartBenchmarksPublisher(ctx context.Context) {
+ select {
+ case <-ctx.Done():
+ return
+ case <-time.After(benchmarksPublishInitialDelay):
+ }
+ ticker := time.NewTicker(benchmarksPublishInterval)
+ defer ticker.Stop()
+ for {
+ s.publishBenchmarks(ctx)
+ select {
+ case <-ctx.Done():
+ return
+ case <-ticker.C:
+ }
+ }
+}
+
+// publishBenchmarks pushes every pending file; a file that fails stays
+// pending for the next run.
+func (s *HubServer) publishBenchmarks(ctx context.Context) {
+ logger := s.logger
+ if logger == nil {
+ logger = slog.Default()
+ }
+ if !benchmarksPublishEnabled() {
+ return
+ }
+ hubBenchmarks.mu.Lock()
+ hubBenchmarks.load()
+ var files []string
+ for f := range hubBenchmarks.pending {
+ files = append(files, f)
+ }
+ hubBenchmarks.mu.Unlock()
+ if len(files) == 0 {
+ return
+ }
+ sort.Strings(files)
+ if len(files) > benchmarksMaxFilesPerRun {
+ files = files[:benchmarksMaxFilesPerRun]
+ }
+ token := hubGitHubAuthTokenForRequest(ctx)
+ if token == "" {
+ s.benchmarksSetError("no hub GitHub identity (App or token) configured")
+ return
+ }
+ client := hubGitHubHTTPClient()
+ repo := benchmarksTargetRepo()
+ done := 0
+ var lastErr error
+ for _, rel := range files {
+ if err := ctx.Err(); err != nil {
+ return
+ }
+ if err := commitBenchmarkFile(ctx, client, token, repo, rel); err != nil {
+ lastErr = err
+ logger.Warn("benchmarks: publish failed", "file", rel, "error", err)
+ continue
+ }
+ done++
+ hubBenchmarks.mu.Lock()
+ delete(hubBenchmarks.pending, rel)
+ hubBenchmarks.mu.Unlock()
+ }
+ hubBenchmarks.mu.Lock()
+ hubBenchmarks.state.LastPublishAt = time.Now().UTC().Format(time.RFC3339)
+ hubBenchmarks.state.LastPublishFiles = done
+ hubBenchmarks.state.LastError = ""
+ if lastErr != nil {
+ hubBenchmarks.state.LastError = lastErr.Error()
+ }
+ hubBenchmarks.save()
+ hubBenchmarks.mu.Unlock()
+ logger.Info("benchmarks: published to data repo", "repo", repo, "files", done, "failed", len(files)-done)
+}
+
+func (s *HubServer) benchmarksSetError(msg string) {
+ hubBenchmarks.mu.Lock()
+ hubBenchmarks.load()
+ hubBenchmarks.state.LastError = msg
+ hubBenchmarks.save()
+ hubBenchmarks.mu.Unlock()
+}
+
+// commitBenchmarkFile PUTs one file through the Contents API, reading the
+// current blob sha first so the write is an update, not a conflict.
+func commitBenchmarkFile(ctx context.Context, client *http.Client, token, repo, rel string) error {
+ if !validBenchmarkRel(rel) {
+ return fmt.Errorf("refusing to publish path %q", rel)
+ }
+ data, err := os.ReadFile(filepath.Join(benchmarksStoreDir, filepath.FromSlash(rel)))
+ if err != nil {
+ return err
+ }
+ url := fmt.Sprintf("%s/repos/%s/contents/%s/%s", benchmarksGitHubAPI, repo, benchmarksDataPrefix, rel)
+ cctx, cancel := context.WithTimeout(hubGitHubCallerContext(ctx, "benchmarks"), benchmarksCommitTimeout)
+ defer cancel()
+ sha := ""
+ req, _ := http.NewRequestWithContext(cctx, http.MethodGet, url, nil)
+ req.Header.Set("Authorization", "Bearer "+token)
+ req.Header.Set("Accept", "application/vnd.github+json")
+ if resp, gerr := client.Do(req); gerr == nil {
+ var cur struct {
+ SHA string `json:"sha"`
+ Content string `json:"content"`
+ }
+ _ = json.NewDecoder(io.LimitReader(resp.Body, 16<<20)).Decode(&cur)
+ _ = resp.Body.Close()
+ if resp.StatusCode == http.StatusOK {
+ sha = cur.SHA
+ if existing, derr := base64.StdEncoding.DecodeString(strings.ReplaceAll(cur.Content, "\n", "")); derr == nil && bytes.Equal(existing, data) {
+ return nil // already published
+ }
+ }
+ }
+ hash := strings.SplitN(rel, "/", 2)[0]
+ payload := map[string]any{
+ "message": fmt.Sprintf("🌱 fleet: hourly benchmark records for %s (%s)", hash, strings.TrimSuffix(filepath.Base(rel), ".json")),
+ "content": base64.StdEncoding.EncodeToString(data),
+ }
+ if sha != "" {
+ payload["sha"] = sha
+ }
+ body, _ := json.Marshal(payload)
+ req, err = http.NewRequestWithContext(cctx, http.MethodPut, url, bytes.NewReader(body))
+ if err != nil {
+ return err
+ }
+ req.Header.Set("Authorization", "Bearer "+token)
+ req.Header.Set("Accept", "application/vnd.github+json")
+ req.Header.Set("Content-Type", "application/json")
+ resp, err := client.Do(req)
+ if err != nil {
+ return err
+ }
+ defer func() { _ = resp.Body.Close() }()
+ if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusCreated {
+ msg, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
+ return fmt.Errorf("contents API %s: %s", resp.Status, strings.TrimSpace(string(msg)))
+ }
+ return nil
+}
+
+func validBenchmarkRel(rel string) bool {
+ hash, file, ok := strings.Cut(rel, "/")
+ return ok && hiveHashRe.MatchString(hash) && strings.HasSuffix(file, ".json") && monthRe.MatchString(strings.TrimSuffix(file, ".json"))
+}
+
+// handleBenchmarksStatus serves GET /api/hub/benchmarks: last ingest and
+// publish times plus per-hive record counts from the on-disk mirror.
+func (s *HubServer) handleBenchmarksStatus(w http.ResponseWriter, _ *http.Request) {
+ hubBenchmarks.mu.Lock()
+ hubBenchmarks.load()
+ st := hubBenchmarks.state
+ st.PendingFiles = len(hubBenchmarks.pending)
+ hubBenchmarks.mu.Unlock()
+ st.Repo = benchmarksTargetRepo()
+ st.Enabled = benchmarksPublishEnabled()
+ st.Hives = map[string]int{}
+ entries, err := os.ReadDir(benchmarksStoreDir)
+ if err != nil && !errors.Is(err, os.ErrNotExist) {
+ npsJSONError(w, "read store", http.StatusInternalServerError)
+ return
+ }
+ for _, e := range entries {
+ if !e.IsDir() || !hiveHashRe.MatchString(e.Name()) {
+ continue
+ }
+ n := 0
+ months, _ := os.ReadDir(filepath.Join(benchmarksStoreDir, e.Name()))
+ for _, m := range months {
+ data, rerr := os.ReadFile(filepath.Join(benchmarksStoreDir, e.Name(), m.Name()))
+ if rerr != nil {
+ continue
+ }
+ var recs []json.RawMessage
+ if json.Unmarshal(data, &recs) == nil {
+ n += len(recs)
+ }
+ }
+ st.Hives[e.Name()] = n
+ st.RecordCount += n
+ }
+ st.HiveCount = len(st.Hives)
+ w.Header().Set("Content-Type", "application/json")
+ _ = json.NewEncoder(w).Encode(st)
+}
diff --git a/src/pkg/hub/benchmarks_test.go b/src/pkg/hub/benchmarks_test.go
new file mode 100644
index 0000000000..73f5a50c55
--- /dev/null
+++ b/src/pkg/hub/benchmarks_test.go
@@ -0,0 +1,253 @@
+package hub
+
+import (
+ "context"
+ "encoding/base64"
+ "encoding/json"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "path/filepath"
+ "strings"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/hivecommons/hive/pkg/evidence/benchmarks"
+)
+
+func useTempBenchmarksStore(t *testing.T) string {
+ t.Helper()
+ dir := t.TempDir()
+ oldDir, oldState := benchmarksStoreDir, benchmarksStatePath
+ benchmarksStoreDir = filepath.Join(dir, "benchmarks")
+ benchmarksStatePath = filepath.Join(dir, "state.json")
+ hubBenchmarks.mu.Lock()
+ hubBenchmarks.pending = map[string]bool{}
+ hubBenchmarks.state = BenchmarksPublishState{}
+ hubBenchmarks.loaded = true
+ hubBenchmarks.mu.Unlock()
+ t.Cleanup(func() {
+ benchmarksStoreDir, benchmarksStatePath = oldDir, oldState
+ hubBenchmarks.mu.Lock()
+ hubBenchmarks.pending = map[string]bool{}
+ hubBenchmarks.state = BenchmarksPublishState{}
+ hubBenchmarks.loaded = false
+ hubBenchmarks.mu.Unlock()
+ })
+ return dir
+}
+
+func benchRecord(hash, id, mergedAt string) benchmarks.Record {
+ return benchmarks.Record{
+ SchemaVersion: benchmarks.SchemaVersion, ID: id, PublishedAt: "2026-11-11T00:00:00Z",
+ Source: benchmarks.Source{HiveID: hash, HiveVersion: "v5"},
+ Agent: benchmarks.Agent{Name: "a", Role: "author", Harness: "copilot"},
+ Model: benchmarks.Model{Provider: "anthropic", Name: "claude-fable-5.1"},
+ SandboxRuntime: "none", Task: benchmarks.Task{Kind: "docs"},
+ Outcome: benchmarks.Outcome{PROpened: true, PRMerged: mergedAt != "", MergedAt: mergedAt, CIStatusAtMerge: "unknown", ReviewVerdict: "none"},
+ }
+}
+
+func benchIngest(s *HubServer, body string, bearer string) *httptest.ResponseRecorder {
+ req := httptest.NewRequest(http.MethodPost, benchmarksIngestPath, strings.NewReader(body))
+ req.Header.Set("Content-Type", "application/json")
+ if bearer != "" {
+ req.Header.Set("Authorization", "Bearer "+bearer)
+ }
+ rec := httptest.NewRecorder()
+ s.handleBenchmarksIngest(rec, req)
+ return rec
+}
+
+func TestBenchmarksIngestStoresByMonthAndRejectsForeignHash(t *testing.T) {
+ useTempBenchmarksStore(t)
+ s := npsTestHub("h1")
+ hash := benchmarks.HiveHash("h1")
+ payload := benchmarksIngestBody{HiveID: "h1", Records: []benchmarks.Record{
+ benchRecord(hash, "aaa", "2026-10-02T00:00:00Z"),
+ benchRecord(hash, "bbb", "2026-11-05T00:00:00Z"),
+ benchRecord(benchmarks.HiveHash("h2"), "ccc", "2026-11-05T00:00:00Z"),
+ }}
+ body, _ := json.Marshal(payload)
+ rec := benchIngest(s, string(body), s.heartbeatKeyFor("h1"))
+ if rec.Code != http.StatusAccepted {
+ t.Fatalf("code=%d body=%s", rec.Code, rec.Body.String())
+ }
+ if !strings.Contains(rec.Body.String(), `"accepted":2`) || !strings.Contains(rec.Body.String(), `"rejected":1`) {
+ t.Errorf("response: %s", rec.Body.String())
+ }
+ for _, f := range []string{"2026-10.json", "2026-11.json"} {
+ if _, err := os.Stat(filepath.Join(benchmarksStoreDir, hash, f)); err != nil {
+ t.Errorf("missing %s: %v", f, err)
+ }
+ }
+ if _, err := os.Stat(filepath.Join(benchmarksStoreDir, benchmarks.HiveHash("h2"))); err == nil {
+ t.Error("a record claiming another hive's hash was stored")
+ }
+ // Re-ingesting the same id merges rather than duplicates.
+ benchIngest(s, string(body), s.heartbeatKeyFor("h1"))
+ data, _ := os.ReadFile(filepath.Join(benchmarksStoreDir, hash, "2026-11.json"))
+ var recs []benchmarks.Record
+ _ = json.Unmarshal(data, &recs)
+ if len(recs) != 1 {
+ t.Errorf("want 1 record after re-ingest, got %d", len(recs))
+ }
+ hubBenchmarks.mu.Lock()
+ pending := len(hubBenchmarks.pending)
+ hubBenchmarks.mu.Unlock()
+ if pending != 2 {
+ t.Errorf("pending files = %d, want 2", pending)
+ }
+
+ // Status reflects the store.
+ st := httptest.NewRecorder()
+ s.handleBenchmarksStatus(st, httptest.NewRequest(http.MethodGet, benchmarksStatusPath, nil))
+ var status BenchmarksPublishState
+ _ = json.Unmarshal(st.Body.Bytes(), &status)
+ if status.HiveCount != 1 || status.RecordCount != 2 || status.PendingFiles != 2 || status.Hives[hash] != 2 {
+ t.Errorf("status = %+v", status)
+ }
+}
+
+func TestBenchmarksIngestAuth(t *testing.T) {
+ useTempBenchmarksStore(t)
+ s := npsTestHub("h1", "h2")
+ body := `{"hive_id":"h1","records":[]}`
+ for name, bearer := range map[string]string{"none": "", "other hive": s.heartbeatKeyFor("h2"), "fleet key": s.heartbeatKey()} {
+ if rec := benchIngest(s, body, bearer); rec.Code != http.StatusUnauthorized {
+ t.Errorf("%s: code=%d want 401", name, rec.Code)
+ }
+ }
+ if rec := benchIngest(s, `{"hive_id":"h3","records":[]}`, s.heartbeatKeyFor("h3")); rec.Code != http.StatusForbidden {
+ t.Errorf("unregistered hive: code=%d want 403", rec.Code)
+ }
+}
+
+func TestPublishBenchmarksCommitsPendingFiles(t *testing.T) {
+ useTempBenchmarksStore(t)
+ hash := benchmarks.HiveHash("h1")
+ changed, _, err := storeBenchmarkRecords(hash, []benchmarks.Record{benchRecord(hash, "aaa", "2026-11-05T00:00:00Z")})
+ if err != nil || len(changed) != 1 {
+ t.Fatalf("store: %v %v", changed, err)
+ }
+ hubBenchmarks.mu.Lock()
+ hubBenchmarks.pending[changed[0]] = true
+ hubBenchmarks.mu.Unlock()
+
+ var puts atomic.Int32
+ var gotPath, gotSHA string
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if !strings.HasPrefix(r.Header.Get("Authorization"), "Bearer tok") {
+ w.WriteHeader(http.StatusUnauthorized)
+ return
+ }
+ switch r.Method {
+ case http.MethodGet:
+ _ = json.NewEncoder(w).Encode(map[string]string{"sha": "oldsha", "content": base64.StdEncoding.EncodeToString([]byte("[]"))})
+ case http.MethodPut:
+ puts.Add(1)
+ var p map[string]any
+ _ = json.NewDecoder(r.Body).Decode(&p)
+ gotPath = r.URL.Path
+ gotSHA, _ = p["sha"].(string)
+ w.WriteHeader(http.StatusOK)
+ }
+ }))
+ defer srv.Close()
+ oldAPI, oldTok := benchmarksGitHubAPI, hubGitHubAuthTokenForRequest
+ benchmarksGitHubAPI = srv.URL
+ hubGitHubAuthTokenForRequest = func(ctx context.Context) string { return "tok" }
+ t.Cleanup(func() { benchmarksGitHubAPI, hubGitHubAuthTokenForRequest = oldAPI, oldTok })
+ t.Setenv(benchmarksRepoEnv, "example/data")
+
+ s := newTestHubServer("secret")
+ s.publishBenchmarks(t.Context())
+ if puts.Load() != 1 {
+ t.Fatalf("puts = %d, want 1", puts.Load())
+ }
+ if want := "/repos/example/data/contents/data/fleet/" + hash + "/2026-11.json"; gotPath != want {
+ t.Errorf("path = %s, want %s", gotPath, want)
+ }
+ if gotSHA != "oldsha" {
+ t.Errorf("update must carry the existing blob sha, got %q", gotSHA)
+ }
+ hubBenchmarks.mu.Lock()
+ defer hubBenchmarks.mu.Unlock()
+ if len(hubBenchmarks.pending) != 0 || hubBenchmarks.state.LastPublishFiles != 1 || hubBenchmarks.state.LastPublishAt == "" {
+ t.Errorf("state after publish: pending=%v state=%+v", hubBenchmarks.pending, hubBenchmarks.state)
+ }
+}
+
+func TestValidBenchmarkRel(t *testing.T) {
+ for rel, ok := range map[string]bool{"0123456789abcdef/2026-11.json": true, "../x/2026-11.json": false, "0123456789abcdef/evil.json": false, "0123456789abcdef/2026-11.json/..": false} {
+ if validBenchmarkRel(rel) != ok {
+ t.Errorf("%q: want %v", rel, ok)
+ }
+ }
+}
+
+func TestBenchmarksIngestRejectsMalformedBodies(t *testing.T) {
+ useTempBenchmarksStore(t)
+ s := npsTestHub("h1")
+ for name, tc := range map[string]struct {
+ body string
+ want int
+ }{
+ "not json": {"{", http.StatusBadRequest},
+ "bad hive id": {`{"hive_id":"../x","records":[]}`, http.StatusBadRequest},
+ "too large": {`{"hive_id":"h1","records":[` + strings.Repeat(`{},`, benchmarksIngestMaxBodyBytes/3) + `{}]}`, http.StatusRequestEntityTooLarge},
+ "publish is off": {`{"hive_id":"h1","records":[]}`, http.StatusAccepted},
+ } {
+ if rec := benchIngest(s, tc.body, s.heartbeatKeyFor("h1")); rec.Code != tc.want {
+ t.Errorf("%s: code=%d want %d (%s)", name, rec.Code, tc.want, rec.Body.String())
+ }
+ }
+ nosecret := newTestHubServer("")
+ if rec := benchIngest(nosecret, `{"hive_id":"h1","records":[]}`, "x"); rec.Code != http.StatusServiceUnavailable {
+ t.Errorf("no hub secret: code=%d", rec.Code)
+ }
+}
+
+func TestBenchmarksStatePersistsAcrossLoad(t *testing.T) {
+ useTempBenchmarksStore(t)
+ hash := benchmarks.HiveHash("h1")
+ changed, _, err := storeBenchmarkRecords(hash, []benchmarks.Record{benchRecord(hash, "aaa", "2026-11-05T00:00:00Z")})
+ if err != nil {
+ t.Fatal(err)
+ }
+ hubBenchmarks.mu.Lock()
+ hubBenchmarks.pending[changed[0]] = true
+ hubBenchmarks.state.LastIngestAt = "2026-11-11T00:00:00Z"
+ hubBenchmarks.save()
+ hubBenchmarks.pending = map[string]bool{}
+ hubBenchmarks.state = BenchmarksPublishState{}
+ hubBenchmarks.loaded = false
+ hubBenchmarks.load()
+ defer hubBenchmarks.mu.Unlock()
+ if !hubBenchmarks.pending[changed[0]] || hubBenchmarks.state.LastIngestAt != "2026-11-11T00:00:00Z" {
+ t.Fatalf("state did not round-trip: pending=%v state=%+v", hubBenchmarks.pending, hubBenchmarks.state)
+ }
+}
+
+func TestPublishBenchmarksRecordsMissingIdentity(t *testing.T) {
+ useTempBenchmarksStore(t)
+ hash := benchmarks.HiveHash("h1")
+ changed, _, _ := storeBenchmarkRecords(hash, []benchmarks.Record{benchRecord(hash, "aaa", "")})
+ hubBenchmarks.mu.Lock()
+ hubBenchmarks.pending[changed[0]] = true
+ hubBenchmarks.mu.Unlock()
+ oldTok := hubGitHubAuthTokenForRequest
+ hubGitHubAuthTokenForRequest = func(ctx context.Context) string { return "" }
+ t.Cleanup(func() { hubGitHubAuthTokenForRequest = oldTok })
+ s := newTestHubServer("secret")
+ s.publishBenchmarks(t.Context())
+ hubBenchmarks.mu.Lock()
+ defer hubBenchmarks.mu.Unlock()
+ if hubBenchmarks.state.LastError == "" || len(hubBenchmarks.pending) != 1 {
+ t.Fatalf("missing identity must be surfaced and keep files pending: %+v pending=%v", hubBenchmarks.state, hubBenchmarks.pending)
+ }
+ if benchmarkMonth(benchmarks.Record{}) != time.Now().UTC().Format("2006-01") {
+ t.Error("record without timestamps must land in the current month")
+ }
+}
diff --git a/src/pkg/hub/server.go b/src/pkg/hub/server.go
index d107147380..724c4c15e1 100644
--- a/src/pkg/hub/server.go
+++ b/src/pkg/hub/server.go
@@ -1721,6 +1721,9 @@ func NewHubServer(port int, logger *slog.Logger, gitHash, gitBranch string) *Hub
// same credential as /api/task-status. The only read path is the admin-gated
// GET registered in registerSaaSRoutes.
s.mux.HandleFunc("POST "+npsIngestPath, s.handleNPSIngest)
+ // Fleet benchmark records from spokes (hivecommons/hive#17694).
+ s.mux.HandleFunc("POST "+benchmarksIngestPath, s.handleBenchmarksIngest)
+ s.mux.HandleFunc("GET "+benchmarksStatusPath, s.handleBenchmarksStatus)
s.mux.HandleFunc("POST "+feedbackIngestPath, s.handleFeedbackIngest)
s.mux.HandleFunc("POST "+selfReportIngestPath, s.handleSelfReportIngest)
s.mux.HandleFunc("GET "+feedbackIssuesPath, s.handleFeedbackIssues)