From c2888b41c31582f20d3eb4465adb97a5be4e691f Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Wed, 18 Nov 2026 09:19:06 -0500 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20hivectl=20benchmarks=20export:=20of?= =?UTF-8?q?fline=20schema=20records=20from=20local=20evidence=20(#18990)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the offline variant of GET /api/benchmarks/export: reads the local evidence tree, converts with the shared pkg/evidence/benchmarks converter, and writes the same anonymized records as a JSON array or as data/fleet//.json month files. The hive is named by hash only; repositories are named only when passed with --public-repo. Tracks hivecommons/benchmarks#1. Signed-off-by: Andy Anderson --- .../added-18990-hivectl-benchmarks-export.md | 1 + src/docs/benchmarks.md | 4 + src/docs/hivectl.md | 16 +- src/pkg/evidence/benchmarks/convert.go | 11 +- src/pkg/evidence/benchmarks/convert_test.go | 10 + src/pkg/hivectl/commands/benchmarks.go | 278 ++++++++++++++++++ src/pkg/hivectl/commands/benchmarks_test.go | 269 +++++++++++++++++ src/pkg/hivectl/commands/evidence.go | 21 +- src/pkg/hivectl/commands/root.go | 1 + 9 files changed, 603 insertions(+), 8 deletions(-) create mode 100644 changelog.d/added-18990-hivectl-benchmarks-export.md create mode 100644 src/pkg/hivectl/commands/benchmarks.go create mode 100644 src/pkg/hivectl/commands/benchmarks_test.go diff --git a/changelog.d/added-18990-hivectl-benchmarks-export.md b/changelog.d/added-18990-hivectl-benchmarks-export.md new file mode 100644 index 0000000000..63d794a843 --- /dev/null +++ b/changelog.d/added-18990-hivectl-benchmarks-export.md @@ -0,0 +1 @@ +- `hivectl benchmarks export` converts a local evidence tree into the same anonymized benchmark records as `GET /api/benchmarks/export`, offline, so an owner can publish to hivecommons.dev/benchmarks by hand. Name the hive with `--hive-id` or `--hive-hash`, repositories are named only when listed with `--public-repo`, and `--by-month` writes the `data/fleet//.json` layout. See `src/docs/hivectl.md` (#18990) diff --git a/src/docs/benchmarks.md b/src/docs/benchmarks.md index fd27ff4475..2eb668fcfa 100644 --- a/src/docs/benchmarks.md +++ b/src/docs/benchmarks.md @@ -102,6 +102,10 @@ Until the hub publisher is rolled out, the data repo's `scripts/backfill-from-cluster.sh` pulls `/data/evidence` straight from spoke pods and a nightly workflow rebuilds the index. +## Publishing by hand: `hivectl benchmarks export` + +An owner who does not want the hourly push, or wants to review the records first, can run `hivectl benchmarks export` over a copy of `/data/evidence`. It uses the same converter and privacy rules as the endpoint below, identifies the hive only by its hash, names repositories only when listed with `--public-repo`, and with `--by-month` writes the `data/fleet//.json` files ready to commit to the data repo. See [hivectl](hivectl.md#benchmarks-export-leaderboard-records-offline). + ## Endpoints | Endpoint | Where | Auth | Purpose | diff --git a/src/docs/hivectl.md b/src/docs/hivectl.md index 2c8a1e3097..9a58fe11fe 100644 --- a/src/docs/hivectl.md +++ b/src/docs/hivectl.md @@ -12,7 +12,7 @@ Choose the task before choosing credentials: | Watch agents in a terminal | [Launch and endpoint selection](#launch-and-endpoint-selection) | A real terminal; the TUI uses `HIVE_DASHBOARD_URL`, not `--server`. | | Contribute my worker to Hives | [Contributor profiles](#hives--named-profiles-for-the-hives-you-contribute-to) | Local contributor setup; profiles are separate from dashboard login. | | Enroll a repository in hosted lite mode | [`enroll`](#enroll--spoke-based-lite-repo-enrollment) | Local `gh` authentication and repository access. | -| Export or analyze evidence locally | [`evidence`](#evidence--export-stored-bundles) and [benchmarks](#evidence--benchmark) | The inputs named by that command; these are not agent-control procedures. | +| Export or analyze evidence locally | [`evidence`](#evidence--export-stored-bundles), [benchmarks](#evidence--benchmark) and [leaderboard records](#benchmarks-export-leaderboard-records-offline) | The inputs named by that command; these are not agent-control procedures. | | Develop or record the TUI | [Seeing it run](#seeing-it-run) | A source checkout and the developer tools listed there. | A **spoke** is a Hive connected to a coordinating **hub**. These deployment names do not choose your permissions: the dashboard's authentication policy does. See the [glossary](dashboard-glossary.md#terms) for other terms. @@ -361,6 +361,20 @@ Columns: `repo`, `pr`, `head_sha`, `bundle_id`, `author_login`, `author_kind`, ` Reporting only: nothing here reads into or alters merge eligibility. See [Review evidence bundle](review-evidence.md#exporting). +### benchmarks: export leaderboard records offline + +Use this to publish to [hivecommons.dev/benchmarks](https://hivecommons.dev/benchmarks/) by hand: it is the offline twin of the spoke's `GET /api/benchmarks/export` and runs the same converter over a local copy of the evidence tree. No dashboard session or network is needed. + +```bash +hivectl benchmarks export --hive-hash 0123456789abcdef --out records.json # one JSON array +hivectl benchmarks export --hive-id my-hive --since 2026-11-01 --by-month ./data/fleet +hivectl benchmarks export --hive-id my-hive --public-repo my-org/app --public-repo my-org/docs +``` + +`--root` (default `/data/evidence`) holds the schema v1 bundles. Exactly one of `--hive-id` (hashed locally and never written out) or `--hive-hash` (the 16 hex digits from Settings > Benchmarks or `/api/status`) is required. Output is a JSON array of records sorted by id, one author record and one reviewer record per PR, matching `schema/result.schema.json` in hivecommons/benchmarks. `--since` and `--until` take an RFC3339 time, `YYYY-MM-DD` or a Go duration and window on the bundle update time. `--by-month DIR` writes `DIR//.json` (the `data/fleet/` layout), merging by record id into any file already there and refusing to overwrite a file that is not a record array; commit those files to hivecommons/benchmarks as a pull request. + +Privacy matches the HTTP exporter and is anonymized by default: records never carry logins, PR numbers, commit SHAs, prompts, finding text, paths or the hive id. `task.repo` and `source.org` appear only for repositories you list with `--public-repo owner/name`, because offline there is no GitHub client to look visibility up; list only repositories that are public. Differences from the HTTP exporter: bundle signatures are not verified, `cost.basis` is `unknown` (the launch-cost store lives on the running hive), and bundles that predate the `sandbox_runtime` stamp take `--sandbox-runtime` (default `none`). See [Benchmarks](benchmarks.md). + ### evidence — benchmark Use a benchmark when you want to compare PRs reviewed by Hive with those merged without it. You need a readable local evidence tree or a saved report for `--from-json`; the command does not log in to a Hive or send telemetry. It writes report and image files. `--readme FILE` supports legacy marked documents, not the hand-written root README. diff --git a/src/pkg/evidence/benchmarks/convert.go b/src/pkg/evidence/benchmarks/convert.go index 6a9330f6ae..d01b587a3c 100644 --- a/src/pkg/evidence/benchmarks/convert.go +++ b/src/pkg/evidence/benchmarks/convert.go @@ -12,7 +12,11 @@ import ( // Options parameterise one export. type Options struct { - HiveID string + HiveID string + // HiveHash, when set, is used as source.hive_id as given and HiveID is + // ignored. Offline exports use it for owners who know their hash (shown in + // Settings > Benchmarks) but do not want the hive id on a command line. + HiveHash string HiveVersion string Cluster string // Since drops PRs whose newest record is not strictly newer than it (the @@ -75,7 +79,10 @@ func Convert(bundles []*evidence.Bundle, opts Options) []Record { now = time.Now() } now = now.UTC() - hash := HiveHash(opts.HiveID) + hash := opts.HiveHash + if hash == "" { + hash = HiveHash(opts.HiveID) + } version := strings.TrimSpace(opts.HiveVersion) if version == "" { version = kindUnknown diff --git a/src/pkg/evidence/benchmarks/convert_test.go b/src/pkg/evidence/benchmarks/convert_test.go index b9c91c8e27..cab0e8bd16 100644 --- a/src/pkg/evidence/benchmarks/convert_test.go +++ b/src/pkg/evidence/benchmarks/convert_test.go @@ -348,3 +348,13 @@ func TestConvertCanonicalisesReportedModel(t *testing.T) { t.Errorf("reviewer model not canonicalised: %+v", r) } } + +func TestConvertHiveHashOverridesHiveID(t *testing.T) { + opts := testOptions() + opts.HiveHash = "0123456789abcdef" + for _, r := range Convert(fixtureBundles(), opts) { + if r.Source.HiveID != "0123456789abcdef" { + t.Fatalf("source.hive_id = %q, want the supplied hash", r.Source.HiveID) + } + } +} diff --git a/src/pkg/hivectl/commands/benchmarks.go b/src/pkg/hivectl/commands/benchmarks.go new file mode 100644 index 0000000000..b7ffd40fd3 --- /dev/null +++ b/src/pkg/hivectl/commands/benchmarks.go @@ -0,0 +1,278 @@ +package commands + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "regexp" + "sort" + "strings" + "time" + + "github.com/spf13/cobra" + + "github.com/hivecommons/hive/pkg/evidence/benchmarks" + ghpkg "github.com/hivecommons/hive/pkg/github" +) + +// `hivectl benchmarks export` (hivecommons/hive#18990, hivecommons/benchmarks#1 +// scope item 3, RFC #17694): the offline twin of GET /api/benchmarks/export. +// It runs the same converter (pkg/evidence/benchmarks.Convert) over the local +// evidence tree, so the records carry the same fields and the same privacy +// rules; only the inputs that need a live hive differ (see the command help). + +const ( + // benchmarksHashLen is the length of a published hive hash in hex digits. + benchmarksHashLen = 16 + // benchmarksDateLayout is the YYYY-MM-DD form --since and --until accept, + // matching the HTTP exporter's ?since=. + benchmarksDateLayout = "2006-01-02" + // benchmarksMonthLayout names the month files under /. + benchmarksMonthLayout = "2006-01" + // benchmarksMonthDirPerm and benchmarksMonthFilePerm match the hub's + // month store. + benchmarksMonthDirPerm = 0o755 + benchmarksMonthFilePerm = 0o644 +) + +var benchmarksHashRe = regexp.MustCompile(`^[0-9a-f]{16}$`) + +// benchmarksNow is overridable in tests; it stamps published_at and anchors +// duration forms of --since and --until. +var benchmarksNow = func() time.Time { return time.Now().UTC() } + +func newBenchmarksCommand(env *commandEnv) *cobra.Command { + cmd := &cobra.Command{ + Use: "benchmarks", + Short: "Benchmark records for hivecommons.dev/benchmarks: export them offline from local evidence", + Long: "Work with the per-PR benchmark records Hive publishes to https://hivecommons.dev/benchmarks/ (docs/benchmarks.md).\nexport reads the local evidence tree only; it does not contact a hive or the leaderboard.", + Example: " hivectl benchmarks export --hive-hash 0123456789abcdef --out records.json\n hivectl benchmarks export --hive-id my-hive --by-month ./data/fleet --public-repo my-org/my-repo", + } + cmd.AddCommand(newBenchmarksExportCommand(env)) + return cmd +} + +func newBenchmarksExportCommand(_ *commandEnv) *cobra.Command { + var root, hiveID, hiveHash, hiveVersion, runtime, since, until, out, byMonth string + var publicRepos []string + cmd := &cobra.Command{ + Use: "export", + Short: "Convert local evidence bundles into anonymized benchmark records (offline GET /api/benchmarks/export)", + Long: "Convert the schema v1 evidence bundles under --root (////.json) into the records\n" + + "the leaderboard publishes: one author record and one reviewer record per PR, validating against\n" + + "hivecommons/benchmarks schema/result.schema.json. It runs the same converter as the spoke's\n" + + "GET /api/benchmarks/export, so an owner can publish by hand from a copy of /data/evidence.\n\n" + + "The hive is identified only by its salted hash. Pass --hive-id (hashed locally, never written out) or\n" + + "--hive-hash (the 16 hex digits Settings > Benchmarks and /api/status show); exactly one is required.\n\n" + + "Privacy is the HTTP exporter's, anonymized by default: a record never carries a login, PR number, commit SHA,\n" + + "prompt, finding text, file path or code. task.repo and source.org appear only for repositories you list with\n" + + "--public-repo owner/name (repeatable); offline there is no GitHub client to look visibility up, so list\n" + + "only repositories that are public. Every other repository stays anonymous.\n\n" + + "Differences from the HTTP exporter: bundle signatures are not verified (like `evidence export`); cost.basis is\n" + + "\"unknown\" because the launch-cost store lives on the running hive; sandbox_runtime comes from the bundle\n" + + "stamp and falls back to --sandbox-runtime for bundles that predate the stamp; hive_version is --hive-version.\n\n" + + "--since and --until take an RFC3339 time, YYYY-MM-DD, or (since/until back from now) a Go duration such as\n" + + "72h, and window on the evidence bundle's update time, as the hourly push does. Output is a JSON array sorted by\n" + + "record id, on stdout or --out. --by-month DIR instead writes DIR//.json (the data/fleet layout\n" + + "of hivecommons/benchmarks), merging by record id into any file already there, so the files can be committed\n" + + "as a pull request to that repository.", + Example: " hivectl benchmarks export --root ./evidence --hive-hash 0123456789abcdef --out records.json\n" + + " hivectl benchmarks export --hive-id my-hive --since 2026-11-01 --by-month ./data/fleet\n" + + " hivectl benchmarks export --hive-id my-hive --public-repo my-org/app --public-repo my-org/docs", + Args: argsNone(), + RunE: func(cmd *cobra.Command, _ []string) error { + hiveID = strings.TrimSpace(hiveID) + hiveHash = strings.ToLower(strings.TrimSpace(hiveHash)) + switch { + case hiveID != "" && hiveHash != "": + return &usageError{message: "--hive-id and --hive-hash are mutually exclusive"} + case hiveID == "" && hiveHash == "": + return &usageError{message: "one of --hive-id or --hive-hash is required"} + case hiveHash != "" && !benchmarksHashRe.MatchString(hiveHash): + return &usageError{message: fmt.Sprintf("invalid --hive-hash %q; expected %d lowercase hex digits", hiveHash, benchmarksHashLen)} + } + if byMonth != "" && out != "" { + return &usageError{message: "--by-month and --out are mutually exclusive"} + } + now := benchmarksNow() + var sinceT, untilT time.Time + var err error + if sinceT, err = parseBenchmarksTime("--since", since, now); err != nil { + return &usageError{message: err.Error()} + } + if untilT, err = parseBenchmarksTime("--until", until, now); err != nil { + return &usageError{message: err.Error()} + } + if !sinceT.IsZero() && !untilT.IsZero() && !untilT.After(sinceT) { + return &usageError{message: "--until must be after --since"} + } + public := map[string]bool{} + for _, r := range publicRepos { + r = strings.TrimSpace(r) + if strings.Count(r, "/") != 1 || strings.HasPrefix(r, "/") || strings.HasSuffix(r, "/") { + return &usageError{message: fmt.Sprintf("invalid --public-repo %q; expected owner/name", r)} + } + public[strings.ToLower(r)] = true + } + + bundles, unreadable, err := readEvidenceBundles(root) + if err != nil { + return err + } + if unreadable > 0 { + fmt.Fprintf(cmd.ErrOrStderr(), "skipped %d unreadable or non-v1 bundle(s) under %s\n", unreadable, root) + } + recs := benchmarks.Convert(bundles, benchmarks.Options{ + HiveID: hiveID, + HiveHash: hiveHash, + HiveVersion: hiveVersion, + Since: sinceT, + Now: now, + RepoConsent: len(public) > 0, + RepoPublic: func(repo string) bool { return public[strings.ToLower(repo)] }, + SandboxRuntime: func(string) string { return runtime }, + }) + if !untilT.IsZero() { + kept := recs[:0] + for _, r := range recs { + if !r.UpdatedAt.After(untilT) { + kept = append(kept, r) + } + } + recs = kept + } + + if byMonth != "" { + paths, err := writeBenchmarkMonths(byMonth, recs) + if err != nil { + return err + } + for _, p := range paths { + fmt.Fprintf(cmd.ErrOrStderr(), "wrote %s\n", p) + } + fmt.Fprintf(cmd.ErrOrStderr(), "exported %d record(s) from %d bundle(s) into %d file(s)\n", len(recs), len(bundles), len(paths)) + return nil + } + data, err := marshalBenchmarkRecords(recs) + if err != nil { + return err + } + if out == "" || out == "-" { + _, err = cmd.OutOrStdout().Write(data) + return err + } + if err := os.WriteFile(out, data, benchmarksMonthFilePerm); err != nil { + return fmt.Errorf("writing %s: %w", out, err) + } + fmt.Fprintf(cmd.ErrOrStderr(), "wrote %d record(s) from %d bundle(s) to %s\n", len(recs), len(bundles), out) + return nil + }, + } + cmd.Flags().StringVar(&root, "root", ghpkg.ReviewEvidenceRoot, "evidence root holding ///.json bundles") + cmd.Flags().StringVar(&hiveID, "hive-id", "", "this hive's id (hive_id in hive.yaml); hashed locally, never written out") + cmd.Flags().StringVar(&hiveHash, "hive-hash", "", "this hive's published hash (16 hex digits, from Settings > Benchmarks); alternative to --hive-id") + cmd.Flags().StringVar(&hiveVersion, "hive-version", "", "source.hive_version to stamp (default \"unknown\")") + cmd.Flags().StringVar(&runtime, "sandbox-runtime", benchmarks.SandboxRuntimeNone, "sandbox_runtime for bundles that predate the runtime stamp") + cmd.Flags().StringSliceVar(&publicRepos, "public-repo", nil, "owner/name of a public repository that may be named in records (repeatable); all others stay anonymous") + cmd.Flags().StringVar(&since, "since", "", "only bundles updated after this RFC3339 time, YYYY-MM-DD, or within this Go duration (e.g. 72h)") + cmd.Flags().StringVar(&until, "until", "", "only bundles updated at or before this RFC3339 time, YYYY-MM-DD, or Go duration back from now") + cmd.Flags().StringVar(&out, "out", "", "write the JSON array to this file instead of stdout (\"-\" is stdout)") + cmd.Flags().StringVar(&byMonth, "by-month", "", "write DIR//.json month files (data/fleet layout), merging by record id, instead of one array") + return cmd +} + +// parseBenchmarksTime accepts RFC3339, YYYY-MM-DD, or a Go duration back from +// now. Empty means unset. +func parseBenchmarksTime(flag, s string, now time.Time) (time.Time, error) { + s = strings.TrimSpace(s) + if s == "" { + return time.Time{}, nil + } + if t, err := time.Parse(benchmarksDateLayout, s); err == nil { + return t, nil + } + if t, err := time.Parse(time.RFC3339, s); err == nil { + return t, nil + } + d, err := time.ParseDuration(s) + if err != nil || d < 0 { + return time.Time{}, fmt.Errorf("invalid %s %q; expected an RFC3339 time, YYYY-MM-DD, or a Go duration such as 72h", flag, s) + } + return now.Add(-d), nil +} + +// marshalBenchmarkRecords renders recs as one JSON array plus a newline; an +// empty set is [] rather than null. +func marshalBenchmarkRecords(recs []benchmarks.Record) ([]byte, error) { + if recs == nil { + recs = []benchmarks.Record{} + } + data, err := json.Marshal(recs) + if err != nil { + return nil, fmt.Errorf("encoding records: %w", err) + } + return append(data, '\n'), nil +} + +// benchmarkMonthOf files a record under the month it merged, else opened, +// else was published: the rule the hub's month store applies. +func benchmarkMonthOf(r benchmarks.Record, now time.Time) string { + for _, ts := range []string{r.Outcome.MergedAt, r.Outcome.OpenedAt, r.PublishedAt} { + if t, err := time.Parse(time.RFC3339, ts); err == nil { + return t.UTC().Format(benchmarksMonthLayout) + } + } + return now.UTC().Format(benchmarksMonthLayout) +} + +// writeBenchmarkMonths writes recs to dir//.json, merging by +// record id into a file already present (new records win), and returns the +// paths written in sorted order. An existing file that is not a JSON array of +// records is an error: it is never overwritten. +func writeBenchmarkMonths(dir string, recs []benchmarks.Record) ([]string, error) { + type key struct{ hash, month string } + groups := map[key][]benchmarks.Record{} + now := benchmarksNow() + for _, r := range recs { + k := key{hash: r.Source.HiveID, month: benchmarkMonthOf(r, now)} + groups[k] = append(groups[k], r) + } + var paths []string + for k, add := range groups { + path := filepath.Join(dir, k.hash, k.month+".json") + byID := map[string]benchmarks.Record{} + if data, err := os.ReadFile(path); err == nil { + var cur []benchmarks.Record + if err := json.Unmarshal(data, &cur); err != nil { + return nil, fmt.Errorf("%s exists but is not a JSON array of records: %w", path, err) + } + for _, r := range cur { + byID[r.ID] = r + } + } else if !os.IsNotExist(err) { + return nil, fmt.Errorf("reading %s: %w", path, err) + } + for _, r := range add { + byID[r.ID] = r + } + merged := make([]benchmarks.Record, 0, len(byID)) + for _, r := range byID { + merged = append(merged, r) + } + sort.Slice(merged, func(i, j int) bool { return merged[i].ID < merged[j].ID }) + data, err := marshalBenchmarkRecords(merged) + if err != nil { + return nil, err + } + if err := os.MkdirAll(filepath.Dir(path), benchmarksMonthDirPerm); err != nil { + return nil, fmt.Errorf("creating %s: %w", filepath.Dir(path), err) + } + if err := os.WriteFile(path, data, benchmarksMonthFilePerm); err != nil { + return nil, fmt.Errorf("writing %s: %w", path, err) + } + paths = append(paths, path) + } + sort.Strings(paths) + return paths, nil +} diff --git a/src/pkg/hivectl/commands/benchmarks_test.go b/src/pkg/hivectl/commands/benchmarks_test.go new file mode 100644 index 0000000000..4a76b1a537 --- /dev/null +++ b/src/pkg/hivectl/commands/benchmarks_test.go @@ -0,0 +1,269 @@ +package commands + +import ( + "encoding/json" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/evidence" + "github.com/hivecommons/hive/pkg/evidence/benchmarks" + ghpkg "github.com/hivecommons/hive/pkg/github" +) + +const ( + benchTestHiveID = "bench-test-hive" + benchTestPublic = "pub-org/app" + benchTestSecret = "acme/secret-repo" +) + +// writeBenchmarkFixtures stores a merged agent PR in a public repo, an agent PR +// in a private repo, a human PR (no record), and a corrupt file. It returns +// the update time of the newest bundle. +func writeBenchmarkFixtures(t *testing.T, root string) time.Time { + t.Helper() + t0 := time.Date(2026, 10, 30, 12, 0, 0, 0, time.UTC) + write := func(b *evidence.Bundle) { + t.Helper() + if err := evidence.Seal(b, nil); err != nil { + t.Fatal(err) + } + path := ghpkg.ReviewEvidencePath(root, b.Repo, b.Number, b.HeadSHA) + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, mustJSON(t, b), 0o600); err != nil { + t.Fatal(err) + } + } + author := evidence.Author{Login: "hive-agent-login", Kind: evidence.AuthorAgent, Agent: "fixer", Backend: "copilot", Model: "claude-sonnet-5-5"} + write(&evidence.Bundle{ + SchemaVersion: evidence.SchemaVersion, ID: evidence.BundleID(benchTestPublic, 11, "aaa1111"), Repo: benchTestPublic, Number: 11, + Author: author, BaseSHA: "base", HeadSHA: "aaa1111", Policy: evidence.Policy{ACMMLevel: 3}, + CI: evidence.CI{Status: "success"}, + Verdicts: []evidence.Verdict{ + {Perspective: "security", Model: "claude-sonnet-5-5", Backend: "copilot", Verdict: "approve", Confidence: 0.9, RecordedAt: t0, Findings: []evidence.Finding{{Path: "internal/secret.go", Line: 7, Severity: "low", Summary: "SECRET-FINDING-TEXT"}}}, + }, + Merge: &evidence.MergeEvent{Actor: "hive-merger-login", Method: "squash", At: t0.Add(time.Hour), SHA: "mergedsha1", OpenedAt: t0}, + CreatedAt: t0, UpdatedAt: t0.Add(time.Hour), + }) + newest := t0.Add(48 * time.Hour) + write(&evidence.Bundle{ + SchemaVersion: evidence.SchemaVersion, ID: evidence.BundleID(benchTestSecret, 7, "bbb2222"), Repo: benchTestSecret, Number: 7, + Author: author, BaseSHA: "base", HeadSHA: "bbb2222", + CreatedAt: t0.Add(24 * time.Hour), UpdatedAt: newest, + }) + write(&evidence.Bundle{ + SchemaVersion: evidence.SchemaVersion, ID: evidence.BundleID(benchTestPublic, 12, "ccc3333"), Repo: benchTestPublic, Number: 12, + Author: evidence.Author{Login: "alice", Kind: evidence.AuthorHuman}, BaseSHA: "base", HeadSHA: "ccc3333", + CreatedAt: t0, UpdatedAt: t0, + }) + bad := filepath.Join(root, "x", "y", "1", "bad.json") + if err := os.MkdirAll(filepath.Dir(bad), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(bad, []byte("{nope"), 0o600); err != nil { + t.Fatal(err) + } + return newest +} + +func runBenchmarksExport(t *testing.T, args ...string) (string, string, error) { + t.Helper() + server := httptest.NewServer(nil) + t.Cleanup(server.Close) + prev := benchmarksNow + benchmarksNow = func() time.Time { return time.Date(2026, 11, 18, 0, 0, 0, 0, time.UTC) } + t.Cleanup(func() { benchmarksNow = prev }) + return execute(t, server, "", append([]string{"benchmarks", "export"}, args...)...) +} + +func decodeBenchmarkRecords(t *testing.T, s string) []map[string]any { + t.Helper() + var recs []map[string]any + if err := json.Unmarshal([]byte(s), &recs); err != nil { + t.Fatalf("output is not a JSON array: %v\n%s", err, s) + } + return recs +} + +func TestBenchmarksExportAnonymizedByDefault(t *testing.T) { + root := t.TempDir() + writeBenchmarkFixtures(t, root) + stdout, stderr, err := runBenchmarksExport(t, "--root", root, "--hive-id", benchTestHiveID) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(stderr, "skipped 1 unreadable or non-v1 bundle(s)") { + t.Fatalf("stderr = %q", stderr) + } + recs := decodeBenchmarkRecords(t, stdout) + // Two agent PRs yield an author record each, the reviewed one a reviewer record, the human PR none. + if len(recs) != 3 { + t.Fatalf("want 3 records, got %d:\n%s", len(recs), stdout) + } + wantHash := benchmarks.HiveHash(benchTestHiveID) + for _, r := range recs { + src := r["source"].(map[string]any) + if src["hive_id"] != wantHash { + t.Fatalf("source.hive_id = %v, want the salted hash %s", src["hive_id"], wantHash) + } + } + // Privacy invariants: no identity, repo, number, sha or finding text anywhere in the output. + for _, banned := range []string{benchTestHiveID, benchTestPublic, benchTestSecret, "pub-org", "acme", "hive-agent-login", "hive-merger-login", "alice", "aaa1111", "mergedsha1", "SECRET-FINDING-TEXT", "internal/secret.go"} { + if strings.Contains(stdout, banned) { + t.Fatalf("output leaks %q:\n%s", banned, stdout) + } + } + if !strings.HasSuffix(stdout, "]\n") { + t.Fatalf("output must end with a newline: %q", stdout[len(stdout)-5:]) + } +} + +func TestBenchmarksExportNamesOnlyListedPublicRepos(t *testing.T) { + root := t.TempDir() + writeBenchmarkFixtures(t, root) + stdout, _, err := runBenchmarksExport(t, "--root", root, "--hive-id", benchTestHiveID, "--public-repo", "PUB-ORG/app") + if err != nil { + t.Fatal(err) + } + if strings.Contains(stdout, benchTestSecret) || strings.Contains(stdout, "acme") { + t.Fatalf("unlisted repo leaked:\n%s", stdout) + } + named := 0 + for _, r := range decodeBenchmarkRecords(t, stdout) { + task := r["task"].(map[string]any) + if task["repo"] == benchTestPublic { + named++ + if r["source"].(map[string]any)["org"] != "pub-org" || task["repo_visibility"] != "public" || task["repo_consent"] != true { + t.Fatalf("public record = %v", r) + } + } + } + if named != 2 { + t.Fatalf("want author and reviewer record naming %s, got %d", benchTestPublic, named) + } +} + +func TestBenchmarksExportHiveHashMatchesHiveID(t *testing.T) { + root := t.TempDir() + writeBenchmarkFixtures(t, root) + byID, _, err := runBenchmarksExport(t, "--root", root, "--hive-id", benchTestHiveID) + if err != nil { + t.Fatal(err) + } + byHash, _, err := runBenchmarksExport(t, "--root", root, "--hive-hash", strings.ToUpper(benchmarks.HiveHash(benchTestHiveID))) + if err != nil { + t.Fatal(err) + } + if byID != byHash { + t.Fatalf("--hive-id and --hive-hash outputs differ:\n%s\n%s", byID, byHash) + } + again, _, err := runBenchmarksExport(t, "--root", root, "--hive-id", benchTestHiveID) + if err != nil || again != byID { + t.Fatalf("export is not deterministic (err %v)", err) + } +} + +func TestBenchmarksExportWindow(t *testing.T) { + root := t.TempDir() + newest := writeBenchmarkFixtures(t, root) + // Strictly after the first bundle's update time keeps only the private-repo PR (author record). + after := newest.Add(-24 * time.Hour).Format(time.RFC3339) + stdout, _, err := runBenchmarksExport(t, "--root", root, "--hive-id", benchTestHiveID, "--since", after) + if err != nil { + t.Fatal(err) + } + if got := len(decodeBenchmarkRecords(t, stdout)); got != 1 { + t.Fatalf("--since kept %d records, want 1:\n%s", got, stdout) + } + stdout, _, err = runBenchmarksExport(t, "--root", root, "--hive-id", benchTestHiveID, "--until", "2026-10-31") + if err != nil { + t.Fatal(err) + } + if got := len(decodeBenchmarkRecords(t, stdout)); got != 2 { + t.Fatalf("--until kept %d records, want 2:\n%s", got, stdout) + } + stdout, _, err = runBenchmarksExport(t, "--root", t.TempDir(), "--hive-id", benchTestHiveID) + if err != nil || stdout != "[]\n" { + t.Fatalf("empty tree = %q, %v; want []", stdout, err) + } +} + +func TestBenchmarksExportByMonthMergesAndOutWrites(t *testing.T) { + root := t.TempDir() + writeBenchmarkFixtures(t, root) + dir := t.TempDir() + hash := benchmarks.HiveHash(benchTestHiveID) + // A stale file with an old record and an unrelated one: the unrelated one survives, the overlap is replaced. + monthFile := filepath.Join(dir, hash, "2026-10.json") + if err := os.MkdirAll(filepath.Dir(monthFile), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(monthFile, []byte(`[{"id":"keep-me","schema_version":"1"}]`), 0o644); err != nil { + t.Fatal(err) + } + if _, stderr, err := runBenchmarksExport(t, "--root", root, "--hive-id", benchTestHiveID, "--by-month", dir); err != nil || !strings.Contains(stderr, "into 1 file(s)") { + t.Fatalf("by-month: %v %q", err, stderr) + } + data, err := os.ReadFile(monthFile) + if err != nil { + t.Fatal(err) + } + recs := decodeBenchmarkRecords(t, string(data)) + if len(recs) != 4 || recs[0]["id"] == nil { + t.Fatalf("want 3 exported + 1 kept record, got %d:\n%s", len(recs), data) + } + kept := false + for _, r := range recs { + kept = kept || r["id"] == "keep-me" + } + if !kept { + t.Fatalf("existing record dropped:\n%s", data) + } + // Re-running is idempotent. + if _, _, err := runBenchmarksExport(t, "--root", root, "--hive-id", benchTestHiveID, "--by-month", dir); err != nil { + t.Fatal(err) + } + again, _ := os.ReadFile(monthFile) + if string(again) != string(data) { + t.Fatalf("by-month is not idempotent") + } + // A non-array file is refused, never overwritten. + if err := os.WriteFile(monthFile, []byte("{not records}"), 0o644); err != nil { + t.Fatal(err) + } + if _, _, err := runBenchmarksExport(t, "--root", root, "--hive-id", benchTestHiveID, "--by-month", dir); err == nil || !strings.Contains(err.Error(), "not a JSON array") { + t.Fatalf("want refusal, got %v", err) + } + if got, _ := os.ReadFile(monthFile); string(got) != "{not records}" { + t.Fatalf("file was overwritten: %s", got) + } + // --out writes the array to a file and reports on stderr. + out := filepath.Join(t.TempDir(), "records.json") + stdout, stderr, err := runBenchmarksExport(t, "--root", root, "--hive-id", benchTestHiveID, "--out", out) + if err != nil || stdout != "" || !strings.Contains(stderr, "wrote 3 record(s) from 3 bundle(s) to "+out) { + t.Fatalf("out: %v stdout=%q stderr=%q", err, stdout, stderr) + } +} + +func TestBenchmarksExportUsageErrors(t *testing.T) { + root := t.TempDir() + for name, args := range map[string][]string{ + "no hive": {"--root", root}, + "both": {"--root", root, "--hive-id", "a", "--hive-hash", "0123456789abcdef"}, + "short hash": {"--root", root, "--hive-hash", "abc"}, + "bad since": {"--root", root, "--hive-id", "a", "--since", "yesterday"}, + "until before": {"--root", root, "--hive-id", "a", "--since", "2026-11-02", "--until", "2026-11-01"}, + "bad public repo": {"--root", root, "--hive-id", "a", "--public-repo", "norepo"}, + "out and by-month": {"--root", root, "--hive-id", "a", "--out", "x", "--by-month", "y"}, + } { + _, _, err := runBenchmarksExport(t, args...) + if _, ok := err.(*usageError); !ok { + t.Fatalf("%s: want usage error, got %v", name, err) + } + } +} diff --git a/src/pkg/hivectl/commands/evidence.go b/src/pkg/hivectl/commands/evidence.go index a0ca13d830..17cf87e4cf 100644 --- a/src/pkg/hivectl/commands/evidence.go +++ b/src/pkg/hivectl/commands/evidence.go @@ -209,11 +209,10 @@ func parseEvidenceSince(s string, now time.Time) (time.Time, error) { return now.Add(-d), nil } -// exportEvidenceRows walks root, folds the readable v1 bundles in repo -// (empty = all) per PR, keeps the PRs opened at or after cutoff (zero = all -// time), and returns rows ordered by repo then PR number plus the unreadable -// count. -func exportEvidenceRows(root string, cutoff time.Time, repo string) ([]evidenceExportRow, int, error) { +// readEvidenceBundles walks root and returns every readable schema v1 bundle +// plus the count of files that were skipped as unreadable or non-v1. A missing +// root is an empty tree, not an error. +func readEvidenceBundles(root string) ([]*evidence.Bundle, int, error) { var bundles []*evidence.Bundle unreadable := 0 err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { @@ -237,6 +236,18 @@ func exportEvidenceRows(root string, cutoff time.Time, repo string) ([]evidenceE if err != nil { return nil, unreadable, fmt.Errorf("reading evidence under %s: %w", root, err) } + return bundles, unreadable, nil +} + +// exportEvidenceRows walks root, folds the readable v1 bundles in repo +// (empty = all) per PR, keeps the PRs opened at or after cutoff (zero = all +// time), and returns rows ordered by repo then PR number plus the unreadable +// count. +func exportEvidenceRows(root string, cutoff time.Time, repo string) ([]evidenceExportRow, int, error) { + bundles, unreadable, err := readEvidenceBundles(root) + if err != nil { + return nil, unreadable, err + } prs := evidence.FoldPRs(bundles, repo, "") rows := make([]evidenceExportRow, 0, len(prs)) for _, p := range prs { diff --git a/src/pkg/hivectl/commands/root.go b/src/pkg/hivectl/commands/root.go index b2de18f691..003483e597 100644 --- a/src/pkg/hivectl/commands/root.go +++ b/src/pkg/hivectl/commands/root.go @@ -80,6 +80,7 @@ func NewRootCommand(in io.Reader, out, errOut io.Writer) *cobra.Command { root.AddCommand(newClaimsCommand(env)) root.AddCommand(newReviewCommand(env)) root.AddCommand(newEvidenceCommand(env)) + root.AddCommand(newBenchmarksCommand(env)) root.AddCommand(newTUICommand(env)) root.AddCommand(newLoginCommand(env)) root.AddCommand(newLogoutCommand(env))