diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c94c4d198..da3f014526 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -85,6 +85,19 @@ Hive did not historically maintain a complete changelog. This file starts a prag ### Fixed - release: cap GitHub Release notes at `RELEASE_NOTES_MAX_CHARS` (120000) before the SBOM/NOTICE trailer, cutting the changelog section at a bullet boundary and linking the tag's full `CHANGELOG.md` section, so a first-on-the-line release such as v6.0.0 no longer fails `gh release create` with HTTP 422 "body is too long" after the tag and image retags have landed. + +## 2026-11-08 (v5.204.1) + +### Fixed + +- Fleet report: repeated `copilot sniff: upstream dial failed` ERRORs — the proxy could not reach a `*.githubcopilot.com` host it was usage-sniffing, such as a dial timeout to `telemetry.enterprise.githubcopilot.com` — are now classified as `proxy_connect_dial_failed` (medium, component `github-proxy`, remedy "restore the spoke's outbound proxy or network path") alongside `proxy: CONNECT dial failed`, instead of surfacing as `unclassified_error_repeated` ([#17262](https://github.com/hivecommons/hive/issues/17262)). + +## 2026-11-08 (v5.204.0) + +### Added + +- crux: agent step service, the sixth work service of the Crux work-services RFC ([hivecommons/crux#114](https://github.com/hivecommons/crux/issues/114) S6). `POST /v1/runs` with `"profile": "agent"` now opens a step session — a contained run that provisions its checkout and waits — and `POST /v1/runs/{id}/steps` runs one agent step inside it: a `claude`, `codex` or `process` backend reads the prompt in the sandbox and its JSON answer is checked against the request's `output_schema`, with bounded re-asks on a mismatch, a per-attempt timeout, tool and turn limits where the backend can enforce them, and a recorded `unsupported` refusal where it cannot. Each step is logged in the S4 command log, announced as a `step` turn signal, and read back with `GET /v1/runs/{id}/steps[/{n}]` or `crux --server … step run|list|show`. + ## 2026-11-08 (v5.203.0) ### Added diff --git a/changelog.d/added-crux-s7-saved-run-results.md b/changelog.d/added-crux-s7-saved-run-results.md new file mode 100644 index 0000000000..a425d23083 --- /dev/null +++ b/changelog.d/added-crux-s7-saved-run-results.md @@ -0,0 +1 @@ +- crux: saved run results complete, the seventh work service of the Crux work-services RFC ([hivecommons/crux#114](https://github.com/hivecommons/crux/issues/114) S7). A finished run can now be re-run with `POST /v1/runs/{id}/restart` (`crux --server … runs restart`): the server re-validates the request the run was accepted with as its original owner against the current configuration, links the two records (`restarted_from` / `restarted_by`), audits the restart with its reason, and refuses with `409 conflict` while the old run is still live so two drivers never resume one task. `GET /v1/runs` lists the caller's runs newest first (the operator's view is every run; `?status=` and `?limit=` narrow it), `GET /v1/runs/{id}/history` is the record's status transitions, and a `crux serve` restart now settles the `running` steps of an unfinished session to `unfinished` alongside the run. `crux runs list|restart|history` round out the CLI. diff --git a/changelog.d/added-etag-cache-size-setting.md b/changelog.d/added-etag-cache-size-setting.md new file mode 100644 index 0000000000..edabb9259f --- /dev/null +++ b/changelog.d/added-etag-cache-size-setting.md @@ -0,0 +1 @@ +- The GitHub conditional-request (ETag / 304) response cache is now sized by `github.etag_cache_max_mb` and `github.etag_cache_max_entries` (env `HIVE_GITHUB_ETAG_CACHE_MAX_MB` / `HIVE_GITHUB_ETAG_CACHE_MAX_ENTRIES`), editable under Settings → Forge App and applied without a restart. The built-in default rises from 64 MiB / 4096 entries to 128 MiB / 8192 entries, cached bodies are stored gzip-compressed so the cap holds several times as many responses, and `/api/gh-rate-limits` plus the Debug → GitHub API budget panel show the cache's fill level (`bytes` / `max_bytes` / `max_entries`) so an operator can see eviction pressure before the hit rate slides. diff --git a/src/cmd/hive/github_client_config.go b/src/cmd/hive/github_client_config.go index 6d2cac5323..b5b802fbcc 100644 --- a/src/cmd/hive/github_client_config.go +++ b/src/cmd/hive/github_client_config.go @@ -120,6 +120,10 @@ func (b *boot) applyGitHubClientConfigHooks(client *github.Client) { // open PR nor blanks the Matrix feed until the next sweep (#16276). client.SetSentinelHeadsPath(sentinelHeadsStatePath) client.SetAPIBudgetThresholds(b.cfg.GitHub.APIReserve, b.cfg.GitHub.APICritical) + // The conditional-request cache is process-wide, not per client; size it + // from github.etag_cache_max_mb / _max_entries so a hive that evicts + // within hours can be given more room without a rebuild. + github.SetETagCacheLimits(b.cfg.GitHub.EffectiveETagCacheMaxBytes(), b.cfg.GitHub.EffectiveETagCacheMaxEntries()) // Per-repo pause (#6203). A live predicate over the shared config, so a // pause taken in the dashboard narrows the very next enumeration and // automerge sweep without a restart. diff --git a/src/crux/agentstep/agentstep_test.go b/src/crux/agentstep/agentstep_test.go index e59bedd78f..35bd5eec52 100644 --- a/src/crux/agentstep/agentstep_test.go +++ b/src/crux/agentstep/agentstep_test.go @@ -362,3 +362,36 @@ func TestHostWithoutLogOrSignalsStillRuns(t *testing.T) { t.Fatalf("step = %+v", step) } } + +func TestReconcileMarksRunningStepsUnfinished(t *testing.T) { + store := Store{RunDir: t.TempDir()} + now := time.Date(2026, 11, 8, 12, 0, 0, 0, time.UTC) + for _, st := range []string{api.StepSucceeded, api.StepRunning, api.StepRunning} { + if _, err := store.Create(api.Step{Run: "run_1", Status: st, CreatedAt: now}); err != nil { + t.Fatal(err) + } + } + // A corrupt record is skipped so the rest still settle. + if err := os.MkdirAll(store.Dir(4), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(store.Dir(4), RecordName), []byte("{"), 0o600); err != nil { + t.Fatal(err) + } + changed, err := store.Reconcile("serve restarted", now) + if err != nil || len(changed) != 2 || changed[0].N != 2 || changed[1].N != 3 { + t.Fatalf("Reconcile = %+v, %v", changed, err) + } + for _, n := range []int{2, 3} { + got, err := store.Get(n) + if err != nil || got.Status != api.StepUnfinished || got.Reason != "serve restarted" || got.FinishedAt == nil || !got.FinishedAt.Equal(now) { + t.Fatalf("step %d = %+v, %v", n, got, err) + } + } + if got, _ := store.Get(1); got.Status != api.StepSucceeded { + t.Fatalf("finished step touched: %+v", got) + } + if again, err := store.Reconcile("x", now); err != nil || len(again) != 0 { + t.Fatalf("second reconcile = %+v, %v", again, err) + } +} diff --git a/src/crux/agentstep/store.go b/src/crux/agentstep/store.go index 876fc23815..98a715275c 100644 --- a/src/crux/agentstep/store.go +++ b/src/crux/agentstep/store.go @@ -9,6 +9,7 @@ import ( "sort" "strconv" "sync" + "time" "github.com/hivecommons/hive/crux/api" ) @@ -157,3 +158,28 @@ func (s Store) AppendTranscript(n int, a attempt) error { } return werr } + +// Reconcile marks every step still recorded as running unfinished with +// reason and returns the steps it changed. It runs when the run that owns +// the steps is itself reconciled at `crux serve` start: the process that +// ran the step is gone, so its outcome is not knowable and is never +// guessed. A corrupt record is skipped, not an error, so the rest settle. +func (s Store) Reconcile(reason string, now time.Time) ([]api.Step, error) { + steps, err := s.List() + if err != nil && !errors.Is(err, ErrCorrupt) { + return nil, err + } + var changed []api.Step + for _, step := range steps { + if step.Status != api.StepRunning { + continue + } + at := now + step.Status, step.Reason, step.FinishedAt = api.StepUnfinished, reason, &at + if err := s.Write(step); err != nil { + return changed, err + } + changed = append(changed, step) + } + return changed, nil +} diff --git a/src/crux/api/runs.go b/src/crux/api/runs.go index dd017cc5d1..1acf4f9d7c 100644 --- a/src/crux/api/runs.go +++ b/src/crux/api/runs.go @@ -1,6 +1,10 @@ package api -import "time" +import ( + "net/url" + "strconv" + "time" +) // Contained runs and saved run results (hivecommons/crux#114, S2 and S7): // @@ -8,9 +12,14 @@ import "time" // GET /v1/runs/{id} the saved record, whatever the process state // POST /v1/runs/{id}/cancel cancel; after completion returns the result // unchanged with already_finished: true +// GET /v1/runs the caller's saved records (all, for the +// operator), newest first, ?status= ?limit= +// POST /v1/runs/{id}/restart explicit, audited re-run of a finished run as +// a new record linked to the old one; 409 +// conflict while the saved state is still live +// GET /v1/runs/{id}/history every status the record passed through // -// Routes beyond these (restart, listing) are deferred; see -// src/docs/design/crux-work-services.md. +// See src/docs/design/crux-work-services.md (S7). const PathRuns = "/" + Version + "/runs" // RunPath is the route of one run's saved record. @@ -19,6 +28,29 @@ func RunPath(id string) string { return PathRuns + "/" + id } // RunCancelPath is the route that cancels one run. func RunCancelPath(id string) string { return RunPath(id) + "/cancel" } +// RunRestartPath is the route that re-runs one finished run (S7). +func RunRestartPath(id string) string { return RunPath(id) + "/restart" } + +// RunHistoryPath is the route of one run's status history (S7). +func RunHistoryPath(id string) string { return RunPath(id) + "/history" } + +// RunsListQuery is the query string of GET /v1/runs: status filters on one +// status of the closed set, limit caps the page (newest first); either may +// be empty or zero for no filter and the server default. +func RunsListQuery(status string, limit int) string { + q := url.Values{} + if status != "" { + q.Set("status", status) + } + if limit > 0 { + q.Set("limit", strconv.Itoa(limit)) + } + if len(q) == 0 { + return PathRuns + } + return PathRuns + "?" + q.Encode() +} + // The runs routes answer CodeUnsupported (501, shared with the checkout // routes) when the request asks for something the configured backend cannot // enforce (a network mode, a profile, a workspace form); the message names it @@ -124,4 +156,41 @@ type ContainedRun struct { // AlreadyFinished is set on a cancel response when the run had already // reached a terminal status; the record is returned unchanged. AlreadyFinished bool `json:"already_finished,omitempty"` + // RestartedFrom is the finished run this one re-ran (S7); RestartedBy + // is the run that re-ran this one. Both are saved, so the chain is + // readable from either end after a restart of the serving process. + RestartedFrom string `json:"restarted_from,omitempty"` + RestartedBy string `json:"restarted_by,omitempty"` +} + +// RunRestartRequest is POST /v1/runs/{id}/restart (S7). The run re-runs +// with the request it was created from; only the bookkeeping is new. +type RunRestartRequest struct { + // Reason is recorded with the new run and in the audit record. + Reason string `json:"reason,omitempty"` + // IdempotencyKey makes a repeated restart return the run the first one + // started instead of a third run. + IdempotencyKey string `json:"idempotency_key,omitempty"` + // Detach returns the accepted record instead of waiting for the result. + Detach bool `json:"detach,omitempty"` +} + +// ContainedRunList is GET /v1/runs: the caller's saved records (every +// record for the operator), newest first, without the command output. +type ContainedRunList struct { + Runs []ContainedRun `json:"runs"` +} + +// RunTransition is one status the record passed through, in order. +type RunTransition struct { + At time.Time `json:"at"` + Status string `json:"status"` + Reason string `json:"reason,omitempty"` +} + +// RunHistory is GET /v1/runs/{id}/history: the saved record's statuses in +// the order they were written, the proof of what happened to the run. +type RunHistory struct { + Run string `json:"run"` + Transitions []RunTransition `json:"transitions"` } diff --git a/src/crux/api/runs_client.go b/src/crux/api/runs_client.go index 446f05420a..7f4f1973cd 100644 --- a/src/crux/api/runs_client.go +++ b/src/crux/api/runs_client.go @@ -25,3 +25,25 @@ func (c *Client) CancelRun(ctx context.Context, id string) (ContainedRun, error) err := c.Do(ctx, http.MethodPost, RunCancelPath(id), nil, &out) return out, err } + +// ListRuns is GET /v1/runs. +func (c *Client) ListRuns(ctx context.Context, status string, limit int) (ContainedRunList, error) { + var out ContainedRunList + err := c.Do(ctx, http.MethodGet, RunsListQuery(status, limit), nil, &out) + return out, err +} + +// RestartRun is POST /v1/runs/{id}/restart: the new run's record, final +// unless req.Detach. +func (c *Client) RestartRun(ctx context.Context, id string, req RunRestartRequest) (ContainedRun, error) { + var out ContainedRun + err := c.Do(ctx, http.MethodPost, RunRestartPath(id), req, &out) + return out, err +} + +// RunHistory is GET /v1/runs/{id}/history. +func (c *Client) RunHistory(ctx context.Context, id string) (RunHistory, error) { + var out RunHistory + err := c.Do(ctx, http.MethodGet, RunHistoryPath(id), nil, &out) + return out, err +} diff --git a/src/crux/api/runs_test.go b/src/crux/api/runs_test.go index efe0433393..6687732c18 100644 --- a/src/crux/api/runs_test.go +++ b/src/crux/api/runs_test.go @@ -1,6 +1,14 @@ package api -import "testing" +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "strings" + "testing" +) func TestRunPaths(t *testing.T) { if PathRuns != "/v1/runs" || RunPath("run_a") != "/v1/runs/run_a" || RunCancelPath("run_a") != "/v1/runs/run_a/cancel" { @@ -10,3 +18,76 @@ func TestRunPaths(t *testing.T) { t.Fatal("unsupported-limit refusals must be distinguishable from a not-configured server") } } + +func TestSavedRunPathsAndQuery(t *testing.T) { + if RunRestartPath("run_a") != "/v1/runs/run_a/restart" || RunHistoryPath("run_a") != "/v1/runs/run_a/history" { + t.Fatalf("paths: %s %s", RunRestartPath("run_a"), RunHistoryPath("run_a")) + } + for _, tc := range []struct { + status string + limit int + want string + }{ + {"", 0, "/v1/runs"}, + {"", -1, "/v1/runs"}, + {RunUnfinished, 0, "/v1/runs?status=unfinished"}, + {"", 7, "/v1/runs?limit=7"}, + {"a b", 2, "/v1/runs?limit=2&status=a+b"}, + } { + if got := RunsListQuery(tc.status, tc.limit); got != tc.want { + t.Fatalf("RunsListQuery(%q, %d) = %q want %q", tc.status, tc.limit, got, tc.want) + } + } +} + +func TestSavedRunClientRoutesAndDecodes(t *testing.T) { + var seen []string + mux := http.NewServeMux() + mux.HandleFunc("GET "+PathRuns, func(w http.ResponseWriter, r *http.Request) { + seen = append(seen, r.Method+" "+r.URL.RequestURI()) + _ = json.NewEncoder(w).Encode(ContainedRunList{Runs: []ContainedRun{{ID: "run_1", Status: RunUnfinished}}}) + }) + mux.HandleFunc("POST "+PathRuns+"/{id}/restart", func(w http.ResponseWriter, r *http.Request) { + var req RunRestartRequest + _ = json.NewDecoder(r.Body).Decode(&req) + seen = append(seen, r.Method+" "+r.URL.RequestURI()+" "+req.Reason) + if r.PathValue("id") == "run_live" { + w.WriteHeader(http.StatusConflict) + _ = json.NewEncoder(w).Encode(Error{Error: "run run_live is running", Code: CodeConflict}) + return + } + _ = json.NewEncoder(w).Encode(ContainedRun{ID: "run_2", Status: RunAccepted, RestartedFrom: r.PathValue("id")}) + }) + mux.HandleFunc("GET "+PathRuns+"/{id}/history", func(w http.ResponseWriter, r *http.Request) { + seen = append(seen, r.Method+" "+r.URL.RequestURI()) + _ = json.NewEncoder(w).Encode(RunHistory{Run: r.PathValue("id"), Transitions: []RunTransition{{Status: RunAccepted}, {Status: RunRunning}}}) + }) + srv := httptest.NewServer(mux) + t.Cleanup(srv.Close) + c, err := NewClient(srv.URL, "tok") + if err != nil { + t.Fatal(err) + } + ctx := context.Background() + + list, err := c.ListRuns(ctx, RunUnfinished, 5) + if err != nil || len(list.Runs) != 1 || list.Runs[0].ID != "run_1" { + t.Fatalf("ListRuns = %+v, %v", list, err) + } + again, err := c.RestartRun(ctx, "run_1", RunRestartRequest{Reason: "flaky"}) + if err != nil || again.ID != "run_2" || again.RestartedFrom != "run_1" { + t.Fatalf("RestartRun = %+v, %v", again, err) + } + var se *StatusError + if _, err := c.RestartRun(ctx, "run_live", RunRestartRequest{}); !errors.As(err, &se) || se.Status != http.StatusConflict || se.Body.Code != CodeConflict { + t.Fatalf("live restart error = %v", err) + } + hist, err := c.RunHistory(ctx, "run_1") + if err != nil || hist.Run != "run_1" || len(hist.Transitions) != 2 { + t.Fatalf("RunHistory = %+v, %v", hist, err) + } + want := []string{"GET /v1/runs?limit=5&status=unfinished", "POST /v1/runs/run_1/restart flaky", "POST /v1/runs/run_live/restart ", "GET /v1/runs/run_1/history"} + if strings.Join(seen, "|") != strings.Join(want, "|") { + t.Fatalf("requests = %q", seen) + } +} diff --git a/src/crux/api/steps.go b/src/crux/api/steps.go index d17499d446..c315d09264 100644 --- a/src/crux/api/steps.go +++ b/src/crux/api/steps.go @@ -55,6 +55,9 @@ const ( // StepUnsupported: the request asked for something the backend cannot // enforce (a tool allowlist, a turn limit); the step never ran. StepUnsupported = "unsupported" + // StepUnfinished: the serving process stopped while the step ran, so + // its outcome is not knowable (S7 reconciliation); never a guess. + StepUnfinished = "unfinished" ) // Step backends shipped with crux. Process is the generic adapter: the diff --git a/src/crux/cli/cli.go b/src/crux/cli/cli.go index 61e8058737..6ae30633ee 100644 --- a/src/crux/cli/cli.go +++ b/src/crux/cli/cli.go @@ -363,8 +363,10 @@ Commands (each maps onto crux.Runtime): runtime that spawned it, not run by hand cline-turn one bounded Cline CLI turn (stdin prompt); --model provider/model, --check for preflight - runs start|status|cancel contained command runs on a crux serve - (--server only; 'crux runs help' for flags) + runs start|status|cancel|list|restart|history + contained command runs on a crux serve and + their saved results (--server only; + 'crux runs help' for flags) Work services (no runtime; hivecommons/crux#114): checkout add|rm|list pinned checkouts under /checkouts diff --git a/src/crux/cli/runs.go b/src/crux/cli/runs.go index 9f0825b7d1..236173c22d 100644 --- a/src/crux/cli/runs.go +++ b/src/crux/cli/runs.go @@ -14,6 +14,9 @@ import ( const runsUsage = `usage: crux --server ADDR runs start [flags] -- [args…] crux --server ADDR runs status crux --server ADDR runs cancel + crux --server ADDR runs list [--status S] [--limit N] [--json] + crux --server ADDR runs restart [--reason TEXT] [--idempotency-key KEY] [--detach] [--json] + crux --server ADDR runs history [--json] start flags: --checkout ID run in a pinned checkout you own (crux checkout add) @@ -46,6 +49,12 @@ func (e *Env) remoteRuns(ctx context.Context, c *api.Client, args []string) int return e.remoteRunsGet(ctx, c, args[1:]) case "cancel": return e.remoteRunsCancel(ctx, c, args[1:]) + case "list": + return e.remoteRunsList(ctx, c, args[1:]) + case "restart": + return e.remoteRunsRestart(ctx, c, args[1:]) + case "history": + return e.remoteRunsHistory(ctx, c, args[1:]) case "help", "-h", "--help": fmt.Fprintln(e.Stderr, runsUsage) return ExitOK diff --git a/src/crux/cli/runs_saved.go b/src/crux/cli/runs_saved.go new file mode 100644 index 0000000000..476720186a --- /dev/null +++ b/src/crux/cli/runs_saved.go @@ -0,0 +1,99 @@ +package cli + +import ( + "context" + "fmt" + "strings" + "text/tabwriter" + "time" + + "github.com/hivecommons/hive/crux/api" +) + +// `crux runs list|restart|history`: the saved-run-results verbs (RFC +// crux#114 S7). Like the rest of the family they exist only against a +// server. + +func (e *Env) remoteRunsList(ctx context.Context, c *api.Client, args []string) int { + fs := e.subFlags("runs list") + status := fs.String("status", "", "only runs in this status") + limit := fs.Int("limit", 0, "at most this many runs, newest first (server default when 0)") + asJSON := fs.Bool("json", false, "print the list as JSON") + if code, ok := parseSub(fs, args); !ok { + return code + } + if fs.NArg() != 0 { + fmt.Fprintln(e.Stderr, "usage: crux --server ADDR runs list [--status S] [--limit N] [--json]") + return ExitUsage + } + list, err := c.ListRuns(ctx, *status, *limit) + if err != nil { + return e.remoteErr("runs list", "", err) + } + if *asJSON { + return e.printJSON(list) + } + tw := tabwriter.NewWriter(e.Stdout, 0, 8, 2, ' ', 0) + fmt.Fprintln(tw, "ID\tSTATUS\tOWNER\tPROFILE\tCREATED\tCOMMAND") + for _, run := range list.Runs { + fmt.Fprintf(tw, "%s\t%s\t%s\t%s\t%s\t%s\n", run.ID, run.Status, run.Owner, run.Profile, run.CreatedAt.UTC().Format(time.RFC3339), strings.Join(run.Command, " ")) + } + if err := tw.Flush(); err != nil { + fmt.Fprintln(e.Stderr, "crux:", err) + return ExitError + } + return ExitOK +} + +func (e *Env) remoteRunsRestart(ctx context.Context, c *api.Client, args []string) int { + fs := e.subFlags("runs restart") + var req api.RunRestartRequest + fs.StringVar(&req.Reason, "reason", "", "why the run is re-run; recorded and audited") + fs.StringVar(&req.IdempotencyKey, "idempotency-key", "", "return the restart an earlier identical request started") + fs.BoolVar(&req.Detach, "detach", false, "return the accepted record instead of waiting") + asJSON := fs.Bool("json", false, "print the record as JSON") + if code, ok := parseSub(fs, args); !ok { + return code + } + if fs.NArg() != 1 { + fmt.Fprintln(e.Stderr, "usage: crux --server ADDR runs restart [--reason TEXT] [--idempotency-key KEY] [--detach] [--json] ") + return ExitUsage + } + run, err := c.RestartRun(ctx, fs.Arg(0), req) + if err != nil { + return e.remoteErr("runs restart", fs.Arg(0), err) + } + if !*asJSON { + fmt.Fprintf(e.Stderr, "crux: run %s restarted as %s\n", fs.Arg(0), run.ID) + } + return e.printRun(run, *asJSON) +} + +func (e *Env) remoteRunsHistory(ctx context.Context, c *api.Client, args []string) int { + fs := e.subFlags("runs history") + asJSON := fs.Bool("json", false, "print the history as JSON") + if code, ok := parseSub(fs, args); !ok { + return code + } + if fs.NArg() != 1 { + fmt.Fprintln(e.Stderr, "usage: crux --server ADDR runs history [--json] ") + return ExitUsage + } + hist, err := c.RunHistory(ctx, fs.Arg(0)) + if err != nil { + return e.remoteErr("runs history", fs.Arg(0), err) + } + if *asJSON { + return e.printJSON(hist) + } + tw := tabwriter.NewWriter(e.Stdout, 0, 8, 2, ' ', 0) + fmt.Fprintln(tw, "AT\tSTATUS\tREASON") + for _, tr := range hist.Transitions { + fmt.Fprintf(tw, "%s\t%s\t%s\n", tr.At.UTC().Format(time.RFC3339), tr.Status, tr.Reason) + } + if err := tw.Flush(); err != nil { + fmt.Fprintln(e.Stderr, "crux:", err) + return ExitError + } + return ExitOK +} diff --git a/src/crux/cli/runs_test.go b/src/crux/cli/runs_test.go index ae548c16df..6377eed720 100644 --- a/src/crux/cli/runs_test.go +++ b/src/crux/cli/runs_test.go @@ -8,6 +8,7 @@ import ( "strings" "sync" "testing" + "time" "github.com/hivecommons/hive/crux/api" "github.com/hivecommons/hive/crux/config" @@ -20,6 +21,8 @@ type fakeRunsAPI struct { mu sync.Mutex created []api.ContainedRunRequest cancelled []string + listed []string + restarted []api.RunRestartRequest } func (f *fakeRunsAPI) handler() http.Handler { @@ -81,9 +84,142 @@ func (f *fakeRunsAPI) handler() http.Handler { } _ = json.NewEncoder(w).Encode(rec) }) + mux.HandleFunc("GET "+api.PathRuns, func(w http.ResponseWriter, r *http.Request) { + f.mu.Lock() + f.listed = append(f.listed, r.URL.RawQuery) + f.mu.Unlock() + list := api.ContainedRunList{Runs: []api.ContainedRun{}} + for _, id := range []string{"run_sleep", "run_lost", "run_ok"} { + rec := records[id] + if q := r.URL.Query().Get("status"); q != "" && rec.Status != q { + continue + } + rec.Stdout, rec.Stderr, rec.Owner, rec.Command = "", "", "operator", []string{"make", "test"} + list.Runs = append(list.Runs, rec) + } + _ = json.NewEncoder(w).Encode(list) + }) + mux.HandleFunc("POST "+api.PathRuns+"/{id}/restart", func(w http.ResponseWriter, r *http.Request) { + var req api.RunRestartRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + w.WriteHeader(http.StatusBadRequest) + _ = json.NewEncoder(w).Encode(api.Error{Error: err.Error(), Code: api.CodeBadRequest}) + return + } + f.mu.Lock() + f.restarted = append(f.restarted, req) + f.mu.Unlock() + rec, ok := records[r.PathValue("id")] + if !ok { + w.WriteHeader(http.StatusNotFound) + _ = json.NewEncoder(w).Encode(api.Error{Error: "run not found", Code: api.CodeNotFound}) + return + } + if rec.Status == api.RunRunning { + w.WriteHeader(http.StatusConflict) + _ = json.NewEncoder(w).Encode(api.Error{Error: "run is running", Code: api.CodeConflict}) + return + } + again := records["run_ok"] + again.ID, again.RestartedFrom = "run_again", rec.ID + if req.Detach { + again.Status, again.Stdout, again.ExitCode = api.RunAccepted, "", nil + w.WriteHeader(http.StatusAccepted) + } + _ = json.NewEncoder(w).Encode(again) + }) + mux.HandleFunc("GET "+api.PathRuns+"/{id}/history", func(w http.ResponseWriter, r *http.Request) { + id := r.PathValue("id") + if _, ok := records[id]; !ok { + w.WriteHeader(http.StatusNotFound) + _ = json.NewEncoder(w).Encode(api.Error{Error: "run not found", Code: api.CodeNotFound}) + return + } + at := time.Date(2026, 11, 9, 10, 0, 0, 0, time.UTC) + _ = json.NewEncoder(w).Encode(api.RunHistory{Run: id, Transitions: []api.RunTransition{ + {At: at, Status: api.RunAccepted}, {At: at.Add(time.Second), Status: api.RunRunning}, {At: at.Add(2 * time.Second), Status: api.RunUnfinished, Reason: "pod vanished"}, + }}) + }) return mux } +func TestRunsListRestartHistory(t *testing.T) { + t.Setenv(config.ServerEnvVar, "") + t.Setenv(config.ServeTokenEnvVar, "") + f := &fakeRunsAPI{} + ts := httptest.NewServer(f.handler()) + t.Cleanup(ts.Close) + server := ts.URL + + h := newHarness(t) + if code := h.run("--server", server, "runs", "list"); code != ExitOK { + t.Fatalf("list: %d %s", code, h.stderr) + } + out := h.stdout.String() + if !strings.HasPrefix(out, "ID") || !strings.Contains(out, "run_sleep") || !strings.Contains(out, "run_lost") || !strings.Contains(out, "make test") || strings.Contains(out, "hello") { + t.Fatalf("list output:\n%s", out) + } + h = newHarness(t) + if code := h.run("--server", server, "runs", "list", "--status", api.RunUnfinished, "--limit", "7", "--json"); code != ExitOK { + t.Fatalf("list --json: %d %s", code, h.stderr) + } + var list api.ContainedRunList + if err := json.Unmarshal(h.stdout.Bytes(), &list); err != nil || len(list.Runs) != 1 || list.Runs[0].ID != "run_lost" { + t.Fatalf("list --json = %s (%v)", h.stdout, err) + } + if q := f.listed[len(f.listed)-1]; !strings.Contains(q, "status=unfinished") || !strings.Contains(q, "limit=7") { + t.Fatalf("list query = %q", q) + } + + h = newHarness(t) + if code := h.run("--server", server, "runs", "restart", "--reason", "flaky", "--idempotency-key", "r1", "run_lost"); code != ExitOK { + t.Fatalf("restart: %d %s", code, h.stderr) + } + if !strings.Contains(h.stderr.String(), "run run_lost restarted as run_again") || !strings.Contains(h.stdout.String(), "hello") { + t.Fatalf("restart output: %s / %s", h.stderr, h.stdout) + } + if len(f.restarted) != 1 || f.restarted[0].Reason != "flaky" || f.restarted[0].IdempotencyKey != "r1" || f.restarted[0].Detach { + t.Fatalf("restart request = %+v", f.restarted) + } + h = newHarness(t) + if code := h.run("--server", server, "runs", "restart", "--detach", "--json", "run_ok"); code != ExitOK { + t.Fatalf("restart --detach: %d %s", code, h.stderr) + } + var again api.ContainedRun + if err := json.Unmarshal(h.stdout.Bytes(), &again); err != nil || again.ID != "run_again" || again.Status != api.RunAccepted || again.RestartedFrom != "run_ok" { + t.Fatalf("restart --detach --json = %s (%v)", h.stdout, err) + } + h = newHarness(t) + if code := h.run("--server", server, "runs", "restart", "run_sleep"); code != ExitError || !strings.Contains(h.stderr.String(), "run is running") { + t.Fatalf("restart of a live run: %d %s", code, h.stderr) + } + h = newHarness(t) + if code := h.run("--server", server, "runs", "restart", "run_nope"); code != ExitError || !strings.Contains(h.stderr.String(), "not found") { + t.Fatalf("restart of a missing run: %d %s", code, h.stderr) + } + + h = newHarness(t) + if code := h.run("--server", server, "runs", "history", "run_lost"); code != ExitOK { + t.Fatalf("history: %d %s", code, h.stderr) + } + out = h.stdout.String() + if !strings.HasPrefix(out, "AT") || !strings.Contains(out, "2026-11-09T10:00:02Z") || !strings.Contains(out, "unfinished") || !strings.Contains(out, "pod vanished") { + t.Fatalf("history output:\n%s", out) + } + h = newHarness(t) + if code := h.run("--server", server, "runs", "history", "--json", "run_lost"); code != ExitOK { + t.Fatalf("history --json: %d %s", code, h.stderr) + } + var hist api.RunHistory + if err := json.Unmarshal(h.stdout.Bytes(), &hist); err != nil || hist.Run != "run_lost" || len(hist.Transitions) != 3 { + t.Fatalf("history --json = %s (%v)", h.stdout, err) + } + h = newHarness(t) + if code := h.run("--server", server, "runs", "history", "run_nope"); code != ExitError || !strings.Contains(h.stderr.String(), "not found") { + t.Fatalf("history of a missing run: %d %s", code, h.stderr) + } +} + func TestRunsIsRemoteOnly(t *testing.T) { t.Setenv(config.ServerEnvVar, "") h := newHarness(t) @@ -206,6 +342,12 @@ func TestRunsUsage(t *testing.T) { {[]string{"runs", "status", "-bogus"}, ExitUsage, "bogus"}, {[]string{"runs", "cancel", "a", "b"}, ExitUsage, "usage: crux --server ADDR runs cancel"}, {[]string{"runs", "cancel", "-bogus"}, ExitUsage, "bogus"}, + {[]string{"runs", "list", "extra"}, ExitUsage, "usage: crux --server ADDR runs list"}, + {[]string{"runs", "list", "-bogus"}, ExitUsage, "bogus"}, + {[]string{"runs", "restart"}, ExitUsage, "usage: crux --server ADDR runs restart"}, + {[]string{"runs", "restart", "-h"}, ExitOK, "-idempotency-key"}, + {[]string{"runs", "history"}, ExitUsage, "usage: crux --server ADDR runs history"}, + {[]string{"runs", "history", "a", "b"}, ExitUsage, "usage: crux --server ADDR runs history"}, } { h := newHarness(t) if code := h.run(append([]string{"--server", ts.URL}, tc.args...)...); code != tc.code || !strings.Contains(h.stderr.String(), tc.stderr) { diff --git a/src/crux/runs/restart.go b/src/crux/runs/restart.go new file mode 100644 index 0000000000..f1c31790cd --- /dev/null +++ b/src/crux/runs/restart.go @@ -0,0 +1,126 @@ +package runs + +import ( + "context" + "encoding/json" + "errors" + "fmt" + + "github.com/hivecommons/hive/crux/api" + "github.com/hivecommons/hive/crux/runstate" +) + +// Saved-run-results routes beyond the record itself (S7): an explicit, +// audited restart, the caller's listing and the record's status history. + +// Typed errors the serve routes map onto api.Code* values. +var ( + // ErrLive: a restart was asked for while the saved state is accepted, + // running or waiting. The runner may still be alive, so two drivers + // cannot both resume the same task; cancel first, or wait. + ErrLive = errors.New("runs: the run is still live; cancel it or wait for it to finish before restarting") + // ErrNoRequest: the record predates saved requests and holds nothing a + // restart could re-run; create the run again instead. + ErrNoRequest = errors.New("runs: the record does not carry the request it was created from; start a new run instead") +) + +// DefaultListLimit caps GET /v1/runs when the caller names no limit. +const DefaultListLimit = 100 + +// ListFilter narrows List. Owner is enforced by the caller of List (the +// serve routes pass the principal's name unless it is the operator); an +// empty Owner lists every record. Status is one of the closed set or empty. +type ListFilter struct { + Owner string + Status string + Limit int +} + +// Restart re-runs finished run id as a new record owned by the original +// owner and linked both ways (RestartedFrom on the new record, RestartedBy +// on the old). The request the run was created from is validated again +// against the current configuration and checkouts, so a workspace that no +// longer exists or a checkout that was deleted is refused rather than +// guessed around. A run whose saved state is not terminal is ErrLive; +// created is false when req.IdempotencyKey names an earlier restart. +func (s *Service) Restart(launchCtx context.Context, caller Caller, id string, req api.RunRestartRequest) (api.ContainedRun, bool, error) { + if !s.Configured() { + return api.ContainedRun{}, false, ErrNotConfigured + } + rec, err := s.Store.Get(id) + if err != nil { + return api.ContainedRun{}, false, err + } + if !runstate.Terminal(rec.Status) { + return api.ContainedRun{}, false, fmt.Errorf("%w (run %s is %s)", ErrLive, id, rec.Status) + } + var old detail + _ = json.Unmarshal(rec.Detail, &old) + if old.Request == nil { + return api.ContainedRun{}, false, ErrNoRequest + } + // The new run is validated as its owner: the operator restarting a + // project's run does not widen what the run may touch. + p, err := s.validate(Caller{Name: rec.Owner, Operator: caller.Operator}, *old.Request) + if err != nil { + return api.ContainedRun{}, false, err + } + d := s.detailFor(p, *old.Request) + d.RestartedFrom, d.RestartReason = id, req.Reason + run, created, err := s.accept(launchCtx, rec.Owner, req.IdempotencyKey, p, d) + if err != nil || !created { + return run, created, err + } + // Link back. The old record is terminal; its status does not change, + // so the store permits the detail update. + _, _ = s.Store.Update(id, func(r *runstate.Record) error { + var od detail + _ = json.Unmarshal(r.Detail, &od) + od.RestartedBy = run.ID + raw, err := json.Marshal(od) + r.Detail = raw + return err + }) + run.RestartedFrom = id + return run, true, nil +} + +// List is the saved records matching f, newest first, without the command +// output (the record route carries it). +func (s *Service) List(f ListFilter) (api.ContainedRunList, error) { + if f.Status != "" && !runstate.Valid(f.Status) { + return api.ContainedRunList{}, &RequestError{fmt.Sprintf("status %q is not a run status", f.Status)} + } + limit := f.Limit + if limit <= 0 { + limit = DefaultListLimit + } + recs, err := s.Store.List() + if err != nil { + return api.ContainedRunList{}, err + } + out := api.ContainedRunList{Runs: []api.ContainedRun{}} + for i := len(recs) - 1; i >= 0 && len(out.Runs) < limit; i-- { + rec := recs[i] + if (f.Owner != "" && rec.Owner != f.Owner) || (f.Status != "" && rec.Status != f.Status) { + continue + } + run := toWire(rec) + run.Stdout, run.Stderr, run.StdoutTruncatedAt, run.StderrTruncatedAt = "", "", nil, nil + out.Runs = append(out.Runs, run) + } + return out, nil +} + +// History is every status run id passed through, oldest first. +func (s *Service) History(id string) (api.RunHistory, error) { + entries, err := s.Store.History(id) + if err != nil { + return api.RunHistory{}, err + } + out := api.RunHistory{Run: id, Transitions: make([]api.RunTransition, 0, len(entries))} + for _, e := range entries { + out.Transitions = append(out.Transitions, api.RunTransition{At: e.At, Status: e.Status, Reason: e.Reason}) + } + return out, nil +} diff --git a/src/crux/runs/restart_test.go b/src/crux/runs/restart_test.go new file mode 100644 index 0000000000..7c9d00f038 --- /dev/null +++ b/src/crux/runs/restart_test.go @@ -0,0 +1,188 @@ +package runs + +import ( + "context" + "errors" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/hivecommons/hive/crux/agentstep" + "github.com/hivecommons/hive/crux/api" + "github.com/hivecommons/hive/crux/runstate" + "github.com/hivecommons/hive/crux/sandbox" +) + +func TestRestartReRunsTheSavedRequestAndLinksBothRecords(t *testing.T) { + f := newFixture(t, newFakeLauncher(func(_ context.Context, spec sandbox.LaunchSpec) (sandbox.Result, error) { + return sandbox.Result{Stdout: strings.Join(spec.Command, " ")}, nil + })) + req := api.ContainedRunRequest{WorkspacePath: f.workspace(), Command: []string{"make", "test"}, EnvAllowlist: []string{"KEEP"}, TimeoutMs: 5000, IdempotencyKey: "orig", Detach: true} + first := f.wait(f.start(req).ID) + + again, created, err := f.svc.Restart(context.Background(), operator, first.ID, api.RunRestartRequest{Reason: "flaky", IdempotencyKey: "r1"}) + if err != nil || !created || again.ID == first.ID || again.RestartedFrom != first.ID || again.Status != api.RunAccepted { + t.Fatalf("Restart = %+v created=%v err=%v", again, created, err) + } + done := f.wait(again.ID) + if done.Stdout != "make test" || done.Owner != first.Owner || done.RestartedFrom != first.ID { + t.Fatalf("restarted run = %+v", done) + } + f.spec() // the original launch + if spec := f.spec(); spec.Env["KEEP"] != "yes" || strings.Join(spec.Command, " ") != "make test" { + t.Fatalf("restart launched %+v", spec) + } + if old, _ := f.svc.Get(first.ID); old.RestartedBy != again.ID || old.Status != api.RunCompleted { + t.Fatalf("old record = %+v", old) + } + // The old record's idempotency key still names the old run; the + // restart's own key names the restart. + if replay, created, _ := f.svc.Create(context.Background(), operator, req); created || replay.ID != first.ID { + t.Fatalf("original key replay = %+v created=%v", replay, created) + } + if third, created, err := f.svc.Restart(context.Background(), operator, first.ID, api.RunRestartRequest{IdempotencyKey: "r1"}); err != nil || created || third.ID != again.ID { + t.Fatalf("restart replay = %+v created=%v err=%v", third, created, err) + } + if len(f.launcher.specs) != 0 { + t.Fatalf("replay launched again") + } + hist, err := f.svc.History(again.ID) + if err != nil || hist.Run != again.ID || len(hist.Transitions) != 3 || hist.Transitions[2].Status != api.RunCompleted { + t.Fatalf("history = %+v, %v", hist, err) + } + if _, err := f.svc.History("run_missing"); !errors.Is(err, ErrNotFound) { + t.Fatalf("missing history = %v", err) + } +} + +func TestRestartRefusesALiveRunAndRecordsWithoutARequest(t *testing.T) { + f := newFixture(t, blocking()) + live := f.start(api.ContainedRunRequest{WorkspacePath: f.workspace(), Command: []string{"sleep"}}) + f.spec() + if _, _, err := f.svc.Restart(context.Background(), operator, live.ID, api.RunRestartRequest{}); !errors.Is(err, ErrLive) { + t.Fatalf("restart of a live run = %v", err) + } + cancelled, err := f.svc.Cancel(context.Background(), live.ID) + if err != nil || cancelled.Status != api.RunCancelled { + t.Fatalf("cancel = %+v, %v", cancelled, err) + } + again, created, err := f.svc.Restart(context.Background(), operator, live.ID, api.RunRestartRequest{Detach: true}) + if err != nil || !created { + t.Fatalf("restart after cancel = created=%v err=%v", created, err) + } + f.spec() + if _, err := f.svc.Cancel(context.Background(), again.ID); err != nil { + t.Fatal(err) + } + + legacy, _, err := f.svc.Store.Create(runstate.Record{Owner: "operator", Detail: []byte(`{"command":["x"]}`)}) + if err != nil { + t.Fatal(err) + } + if _, err := f.svc.Store.Update(legacy.ID, func(r *runstate.Record) error { r.Status = runstate.StatusUnfinished; return nil }); err != nil { + t.Fatal(err) + } + if _, _, err := f.svc.Restart(context.Background(), operator, legacy.ID, api.RunRestartRequest{}); !errors.Is(err, ErrNoRequest) { + t.Fatalf("restart of a legacy record = %v", err) + } + if _, _, err := f.svc.Restart(context.Background(), operator, "run_missing", api.RunRestartRequest{}); !errors.Is(err, ErrNotFound) { + t.Fatalf("restart of a missing run = %v", err) + } + unconfigured := New(f.svc.Store, nil, Config{}) + if _, _, err := unconfigured.Restart(context.Background(), operator, legacy.ID, api.RunRestartRequest{}); !errors.Is(err, ErrNotConfigured) { + t.Fatalf("restart without a backend = %v", err) + } +} + +func TestRestartRevalidatesAgainstTheCurrentConfiguration(t *testing.T) { + f := newFixture(t, newFakeLauncher(func(_ context.Context, _ sandbox.LaunchSpec) (sandbox.Result, error) { + return sandbox.Result{}, nil + })) + gone := filepath.Join(f.root, "gone") + if err := os.MkdirAll(gone, 0o755); err != nil { + t.Fatal(err) + } + run := f.wait(f.start(api.ContainedRunRequest{WorkspacePath: gone, Command: []string{"x"}}).ID) + f.spec() + if err := os.RemoveAll(gone); err != nil { + t.Fatal(err) + } + var reqErr *RequestError + if _, _, err := f.svc.Restart(context.Background(), operator, run.ID, api.RunRestartRequest{}); !errors.As(err, &reqErr) || !strings.Contains(err.Error(), "not a directory") { + t.Fatalf("restart into a removed workspace = %v", err) + } + // The checkout owner, not the restarting operator, is who the request + // is validated as. + f.svc.Checkouts = fakeCheckouts{} + project := Caller{Name: "project:atlas"} + rec, _, err := f.svc.Store.Create(runstate.Record{Owner: project.Name, Detail: []byte(`{"request":{"checkout":"co_1","command":["x"]}}`)}) + if err != nil { + t.Fatal(err) + } + f.svc.Store.Update(rec.ID, func(r *runstate.Record) error { r.Status = runstate.StatusUnfinished; return nil }) + if _, _, err := f.svc.Restart(context.Background(), operator, rec.ID, api.RunRestartRequest{}); !errors.As(err, &reqErr) || !strings.Contains(err.Error(), `checkout "co_1" not found`) { + t.Fatalf("restart with a deleted checkout = %v", err) + } +} + +func TestListIsNewestFirstScopedAndFiltered(t *testing.T) { + f := newFixture(t, newFakeLauncher(func(_ context.Context, _ sandbox.LaunchSpec) (sandbox.Result, error) { + return sandbox.Result{Stdout: "out"}, nil + })) + a := f.wait(f.start(api.ContainedRunRequest{WorkspacePath: f.workspace(), Command: []string{"a"}}).ID) + b, _, err := f.svc.Create(context.Background(), Caller{Name: "project:atlas"}, api.ContainedRunRequest{WorkspacePath: f.workspace(), Command: []string{"b"}}) + if err != nil { + t.Fatal(err) + } + f.wait(b.ID) + stale, _, _ := f.svc.Store.Create(runstate.Record{Owner: "operator"}) + f.svc.Store.Update(stale.ID, func(r *runstate.Record) error { r.Status = runstate.StatusUnfinished; return nil }) + + all, err := f.svc.List(ListFilter{}) + if err != nil || len(all.Runs) != 3 || all.Runs[0].ID != stale.ID || all.Runs[1].ID != b.ID || all.Runs[2].ID != a.ID { + t.Fatalf("List = %+v, %v", all, err) + } + if all.Runs[2].Stdout != "" { + t.Fatalf("list carries output: %+v", all.Runs[2]) + } + if mine, _ := f.svc.List(ListFilter{Owner: "project:atlas"}); len(mine.Runs) != 1 || mine.Runs[0].ID != b.ID { + t.Fatalf("owner filter = %+v", mine) + } + if lost, _ := f.svc.List(ListFilter{Status: api.RunUnfinished}); len(lost.Runs) != 1 || lost.Runs[0].ID != stale.ID { + t.Fatalf("status filter = %+v", lost) + } + if one, _ := f.svc.List(ListFilter{Limit: 1}); len(one.Runs) != 1 || one.Runs[0].ID != stale.ID { + t.Fatalf("limit = %+v", one) + } + var reqErr *RequestError + if _, err := f.svc.List(ListFilter{Status: "done"}); !errors.As(err, &reqErr) { + t.Fatalf("bad status = %v", err) + } + if none, err := New(f.svc.Store, nil, Config{}).List(ListFilter{Owner: "nobody"}); err != nil || len(none.Runs) != 0 || none.Runs == nil { + t.Fatalf("empty list = %+v, %v", none, err) + } +} + +func TestReconcileSettlesRunningStepsToo(t *testing.T) { + f := newFixture(t, blocking()) + rec, _, err := f.svc.Store.Create(runstate.Record{Owner: "operator"}) + if err != nil { + t.Fatal(err) + } + f.svc.Store.Update(rec.ID, func(r *runstate.Record) error { r.Status = runstate.StatusRunning; return nil }) + steps := agentstep.Store{RunDir: f.svc.Store.Path(rec.ID)} + for _, st := range []string{api.StepSucceeded, api.StepRunning} { + if _, err := steps.Create(api.Step{Run: rec.ID, Status: st}); err != nil { + t.Fatal(err) + } + } + changed, err := f.svc.Reconcile() + if err != nil || len(changed) != 1 { + t.Fatalf("Reconcile = %+v, %v", changed, err) + } + list, err := f.svc.Steps(rec.ID) + if err != nil || len(list.Steps) != 2 || list.Steps[0].Status != api.StepSucceeded || list.Steps[1].Status != api.StepUnfinished || list.Steps[1].Reason != ReconcileReason { + t.Fatalf("steps after reconcile = %+v, %v", list, err) + } +} diff --git a/src/crux/runs/service.go b/src/crux/runs/service.go index fb524c2496..bb35bab36d 100644 --- a/src/crux/runs/service.go +++ b/src/crux/runs/service.go @@ -12,6 +12,7 @@ import ( "sync" "time" + "github.com/hivecommons/hive/crux/agentstep" "github.com/hivecommons/hive/crux/api" "github.com/hivecommons/hive/crux/checkout" "github.com/hivecommons/hive/crux/cmdlog" @@ -83,6 +84,14 @@ type detail struct { Stderr string `json:"stderr,omitempty"` StdoutTruncatedAt *int64 `json:"stdout_truncated_at,omitempty"` StderrTruncatedAt *int64 `json:"stderr_truncated_at,omitempty"` + // Request is what the run was created from, less the idempotency key + // and detach, so a restart (S7) re-validates and re-runs the same thing. + Request *api.ContainedRunRequest `json:"request,omitempty"` + // RestartedFrom and RestartedBy link a restart to the run it re-ran and + // back; RestartReason is the caller's stated reason for the restart. + RestartedFrom string `json:"restarted_from,omitempty"` + RestartedBy string `json:"restarted_by,omitempty"` + RestartReason string `json:"restart_reason,omitempty"` } // plan is a validated request. @@ -165,13 +174,17 @@ func New(store *runstate.Store, backend *Backend, cfg Config) *Service { // Configured reports whether runs can be executed here. func (s *Service) Configured() bool { return s.Backend != nil } -// Reconcile marks every run left accepted or running by a previous process -// unfinished. Call it once at serve start, before accepting requests. +// Reconcile marks every run left accepted, running or waiting by a previous +// process unfinished, and with it any step (S6) that run still records as +// running. Call it once at serve start, before accepting requests. func (s *Service) Reconcile() ([]runstate.Record, error) { recs, err := s.Store.Reconcile(ReconcileReason) // A run the previous process left in flight ended unobserved; its done // signal says so, so a follower never waits on it forever. for _, rec := range recs { + if _, serr := (agentstep.Store{RunDir: s.Store.Path(rec.ID)}).Reconcile(ReconcileReason, s.Store.Now()); serr != nil && err == nil { + err = serr + } s.signalDone(rec.ID, api.SignalDoneData{Status: rec.Status}, rec.Reason) } return recs, err @@ -189,8 +202,20 @@ func (s *Service) Create(launchCtx context.Context, caller Caller, req api.Conta if err != nil { return api.ContainedRun{}, false, err } - owner := caller.Name - d := detail{Profile: p.profile, Network: p.network, Backend: s.Backend.Name, Command: p.command, Checkout: p.checkout, Workspace: p.workspace, EnvAllowlist: p.allowlist} + return s.accept(launchCtx, caller.Name, req.IdempotencyKey, p, s.detailFor(p, req)) +} + +// detailFor is the accepted record's payload: the plan and the request it +// came from, with the per-call fields (idempotency key, detach) dropped so +// a restart cannot inherit them. +func (s *Service) detailFor(p plan, req api.ContainedRunRequest) detail { + req.IdempotencyKey, req.Detach = "", false + return detail{Profile: p.profile, Network: p.network, Backend: s.Backend.Name, Command: p.command, Checkout: p.checkout, Workspace: p.workspace, EnvAllowlist: p.allowlist, Request: &req} +} + +// accept saves d as owner's accepted record (or returns the record key +// already names) and starts the run under launchCtx. +func (s *Service) accept(launchCtx context.Context, owner, key string, p plan, d detail) (api.ContainedRun, bool, error) { raw, err := json.Marshal(d) if err != nil { return api.ContainedRun{}, false, err @@ -198,7 +223,7 @@ func (s *Service) Create(launchCtx context.Context, caller Caller, req api.Conta rec, created, err := s.Store.Create(runstate.Record{ Kind: runstate.KindRun, Owner: owner, - IdempotencyKey: runstate.DeriveIdempotencyKey(owner, req.IdempotencyKey), + IdempotencyKey: runstate.DeriveIdempotencyKey(owner, key), Detail: raw, }) if err != nil { @@ -721,5 +746,6 @@ func toWire(rec runstate.Record) api.ContainedRun { Profile: d.Profile, Network: d.Network, Backend: d.Backend, Command: d.Command, Checkout: d.Checkout, Workdir: d.Workspace, ExitCode: d.ExitCode, TimedOut: d.TimedOut, Stdout: d.Stdout, Stderr: d.Stderr, StdoutTruncatedAt: d.StdoutTruncatedAt, StderrTruncatedAt: d.StderrTruncatedAt, + RestartedFrom: d.RestartedFrom, RestartedBy: d.RestartedBy, } } diff --git a/src/crux/runstate/runstate.go b/src/crux/runstate/runstate.go index 3115d41539..ba5851c2bd 100644 --- a/src/crux/runstate/runstate.go +++ b/src/crux/runstate/runstate.go @@ -12,6 +12,7 @@ package runstate import ( + "bufio" "crypto/rand" "crypto/sha256" "encoding/hex" @@ -52,8 +53,10 @@ const ( idRandomBytes = 6 maxIDLen = 64 - dirMode = 0o770 - fileMode = 0o660 + // indexLineBuf bounds one index line; a reason is a sentence, not a log. + indexLineBuf = 64 * 1024 + dirMode = 0o770 + fileMode = 0o660 ) // ErrNotFound is returned by Get and Update for an unknown ID. @@ -178,7 +181,7 @@ func (s *Store) Create(rec Record) (Record, bool, error) { if err := os.MkdirAll(s.Path(rec.ID), dirMode); err != nil { return Record{}, false, fmt.Errorf("runstate: %w", err) } - if err := s.write(rec); err != nil { + if err := s.write(rec, true); err != nil { return Record{}, false, err } if rec.IdempotencyKey != "" { @@ -226,7 +229,9 @@ func (s *Store) Update(id string, fn func(*Record) error) (Record, error) { rec.FinishedAt = &now } } - if err := s.write(rec); err != nil { + // The index is the status history: a write that only touched the + // payload (output, a restart link) is not a transition. + if err := s.write(rec, rec.Status != before); err != nil { return Record{}, err } return rec, nil @@ -291,6 +296,42 @@ func (s *Store) Reconcile(reason string) ([]Record, error) { return changed, nil } +// History is every status id passed through, in the order the index +// recorded them: the proof behind the record's current status. An unknown +// id is ErrNotFound; a record whose index lines are missing (the index was +// pruned) answers an empty history, not an error. +func (s *Store) History(id string) ([]IndexEntry, error) { + s.mu.Lock() + defer s.mu.Unlock() + if _, err := s.read(id); err != nil { + return nil, err + } + f, err := os.Open(filepath.Join(s.dir, IndexName)) + if err != nil { + if os.IsNotExist(err) { + return []IndexEntry{}, nil + } + return nil, fmt.Errorf("runstate: %w", err) + } + defer func() { _ = f.Close() }() + out := []IndexEntry{} + sc := bufio.NewScanner(f) + sc.Buffer(make([]byte, 0, indexLineBuf), indexLineBuf) + for sc.Scan() { + var e IndexEntry + if err := json.Unmarshal(sc.Bytes(), &e); err != nil { + return nil, fmt.Errorf("runstate: %s: corrupt index line: %w", id, err) + } + if e.ID == id { + out = append(out, e) + } + } + if err := sc.Err(); err != nil { + return nil, fmt.Errorf("runstate: %w", err) + } + return out, nil +} + func (s *Store) read(id string) (Record, error) { if !validID(id) { return Record{}, ErrNotFound @@ -309,8 +350,9 @@ func (s *Store) read(id string) (Record, error) { return rec, nil } -// write replaces the record atomically and appends the status to the index. -func (s *Store) write(rec Record) error { +// write replaces the record atomically and, when index is set, appends the +// status to the index. +func (s *Store) write(rec Record, index bool) error { data, err := json.Marshal(rec) if err != nil { return fmt.Errorf("runstate: %w", err) @@ -323,6 +365,9 @@ func (s *Store) write(rec Record) error { if err := os.Rename(tmp, path); err != nil { return fmt.Errorf("runstate: %w", err) } + if !index { + return nil + } line, err := json.Marshal(IndexEntry{At: s.Now().UTC(), ID: rec.ID, Kind: rec.Kind, Status: rec.Status, Reason: rec.Reason}) if err != nil { return fmt.Errorf("runstate: %w", err) diff --git a/src/crux/runstate/runstate_test.go b/src/crux/runstate/runstate_test.go index 7ecdfe7e55..657955b257 100644 --- a/src/crux/runstate/runstate_test.go +++ b/src/crux/runstate/runstate_test.go @@ -99,7 +99,7 @@ func TestUpdateTimestampsAndTerminalGuard(t *testing.T) { for _, e := range readIndex(t, dir) { statuses = append(statuses, e.Status) } - if want := "accepted running completed completed"; strings.Join(statuses, " ") != want { + if want := "accepted running completed"; strings.Join(statuses, " ") != want { t.Fatalf("index statuses = %v", statuses) } } @@ -379,3 +379,46 @@ func TestNewIDShape(t *testing.T) { t.Fatalf("newID = %q, %q", a, b) } } + +func TestHistoryIsTheRecordsTransitionsOnly(t *testing.T) { + s, dir := newStore(t) + a, _, _ := s.Create(Record{}) + b, _, _ := s.Create(Record{}) + s.Update(a.ID, func(r *Record) error { r.Status = StatusRunning; return nil }) + s.Update(b.ID, func(r *Record) error { r.Status = StatusFailed; r.Reason = "boom"; return nil }) + s.Update(a.ID, func(r *Record) error { r.Status = StatusCompleted; return nil }) + // A payload-only write is not a transition and leaves no index line. + s.Update(a.ID, func(r *Record) error { r.Detail = json.RawMessage(`{"restarted_by":"run_x"}`); return nil }) + hist, err := s.History(a.ID) + if err != nil { + t.Fatal(err) + } + var statuses []string + for _, e := range hist { + if e.ID != a.ID { + t.Fatalf("foreign entry %+v", e) + } + statuses = append(statuses, e.Status) + } + if strings.Join(statuses, " ") != "accepted running completed" { + t.Fatalf("history = %v", statuses) + } + if hb, _ := s.History(b.ID); len(hb) != 2 || hb[1].Reason != "boom" { + t.Fatalf("b history = %+v", hb) + } + if _, err := s.History("missing"); !errors.Is(err, ErrNotFound) { + t.Fatalf("missing = %v", err) + } + if err := os.Remove(filepath.Join(Dir(dir), IndexName)); err != nil { + t.Fatal(err) + } + if hist, err := s.History(a.ID); err != nil || len(hist) != 0 { + t.Fatalf("pruned index = %+v, %v", hist, err) + } + if err := os.WriteFile(filepath.Join(Dir(dir), IndexName), []byte("{not json\n"), 0o660); err != nil { + t.Fatal(err) + } + if _, err := s.History(a.ID); err == nil || !strings.Contains(err.Error(), "corrupt index") { + t.Fatalf("corrupt index = %v", err) + } +} diff --git a/src/crux/serve/runs.go b/src/crux/serve/runs.go index 0d875ef840..5b079467c9 100644 --- a/src/crux/serve/runs.go +++ b/src/crux/serve/runs.go @@ -37,6 +37,7 @@ func (s *Server) registerRuns(mux *http.ServeMux) { mux.HandleFunc("GET "+api.PathRuns+"/{id}/cmdlog", s.handleRunsCmdlog) mux.HandleFunc("GET "+api.PathRuns+"/{id}/signals", s.handleRunsSignalsList) mux.HandleFunc("POST "+api.PathRuns+"/{id}/signals", s.handleRunsSignalsEmit) + s.registerRunsSaved(mux) s.workService(api.PathRuns) if s.Runs != nil && s.Runs.Signals != nil { s.Runs.Signals.SetObserver(s.publishRunSignal) @@ -263,6 +264,8 @@ func (s *Server) writeRunsError(w http.ResponseWriter, err error) { s.writeError(w, http.StatusNotImplemented, api.CodeNotSupported, err.Error()) case errors.Is(err, runs.ErrNotFound): s.writeError(w, http.StatusNotFound, api.CodeNotFound, "run not found") + case errors.Is(err, runs.ErrLive), errors.Is(err, runs.ErrNoRequest): + s.writeError(w, http.StatusConflict, api.CodeConflict, err.Error()) case errors.As(err, &reqErr): s.writeError(w, http.StatusBadRequest, api.CodeBadRequest, err.Error()) case errors.As(err, &forbidden): diff --git a/src/crux/serve/runs_saved.go b/src/crux/serve/runs_saved.go new file mode 100644 index 0000000000..c75285be18 --- /dev/null +++ b/src/crux/serve/runs_saved.go @@ -0,0 +1,115 @@ +package serve + +import ( + "context" + "errors" + "net/http" + "strconv" + + "github.com/hivecommons/hive/crux/api" + "github.com/hivecommons/hive/crux/runs" +) + +// Saved-run-results routes (RFC crux#114 S7) beyond the record itself: +// listing, the explicit restart and the status history. Registered from +// registerRuns; the same ownership rule as the record applies. + +// auditRunRestart is the audit action of every restart: who re-ran which +// run as which new one, and why. +const auditRunRestart = "run.restart" + +func (s *Server) registerRunsSaved(mux *http.ServeMux) { + mux.HandleFunc("GET "+api.PathRuns, s.handleRunsList) + mux.HandleFunc("POST "+api.PathRuns+"/{id}/restart", s.handleRunsRestart) + mux.HandleFunc("GET "+api.PathRuns+"/{id}/history", s.handleRunsHistory) +} + +// handleRunsList is GET /v1/runs: the caller's records, every record for +// the operator, newest first; ?status= narrows to one status, ?limit= caps +// the page. +func (s *Server) handleRunsList(w http.ResponseWriter, r *http.Request) { + if !s.runsReady(w) { + return + } + f := runs.ListFilter{Status: r.URL.Query().Get("status")} + if caller := s.runsCaller(r); !caller.Operator { + f.Owner = caller.Name + } + if v := r.URL.Query().Get("limit"); v != "" { + n, err := strconv.Atoi(v) + if err != nil || n <= 0 { + s.writeError(w, http.StatusBadRequest, api.CodeBadRequest, "limit must be a positive integer") + return + } + f.Limit = n + } + list, err := s.Runs.List(f) + if err != nil { + s.writeRunsError(w, err) + return + } + s.writeJSON(w, http.StatusOK, list) +} + +// handleRunsRestart is POST /v1/runs/{id}/restart: an explicit, audited +// re-run of a finished run by its owner or the operator. A run whose saved +// state is still live is 409 conflict: the runner may be alive, and two +// drivers must not both resume one task. +func (s *Server) handleRunsRestart(w http.ResponseWriter, r *http.Request) { + if !s.runsReady(w) { + return + } + old, ok := s.ownedRun(w, r) + if !ok { + return + } + // An empty body is a plain restart; curl -X POST needs no "{}". + var req api.RunRestartRequest + if r.ContentLength != 0 && !s.readJSON(w, r, &req) { + return + } + caller := s.runsCaller(r) + run, created, err := s.Runs.Restart(s.LaunchContext, caller, old.ID, req) + if err != nil { + s.writeRunsError(w, err) + return + } + if !created { + s.writeJSON(w, http.StatusOK, run) + return + } + auditTee{s}.Record(caller.Name, auditRunRestart, "", map[string]any{ + "run": run.ID, "restarted_from": old.ID, "was": old.Status, "reason": req.Reason, "owner": run.Owner, + }) + if req.Detach || run.Profile == api.RunProfileAgent { + s.writeJSON(w, http.StatusAccepted, run) + return + } + run, err = s.Runs.Wait(r.Context(), run.ID) + if err != nil { + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return + } + s.writeRunsError(w, err) + return + } + s.writeJSON(w, http.StatusOK, run) +} + +// handleRunsHistory is GET /v1/runs/{id}/history: every status the record +// passed through, for its owner or the operator. +func (s *Server) handleRunsHistory(w http.ResponseWriter, r *http.Request) { + if !s.runsReady(w) { + return + } + run, ok := s.ownedRun(w, r) + if !ok { + return + } + hist, err := s.Runs.History(run.ID) + if err != nil { + s.writeRunsError(w, err) + return + } + s.writeJSON(w, http.StatusOK, hist) +} diff --git a/src/crux/serve/runs_saved_test.go b/src/crux/serve/runs_saved_test.go new file mode 100644 index 0000000000..44d1f17531 --- /dev/null +++ b/src/crux/serve/runs_saved_test.go @@ -0,0 +1,183 @@ +package serve + +import ( + "context" + "net/http" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/crux/api" + "github.com/hivecommons/hive/crux/lanes" + "github.com/hivecommons/hive/crux/runs" + "github.com/hivecommons/hive/crux/runstate" + "github.com/hivecommons/hive/crux/sandbox" +) + +// newSavedRunsFixture is a token-bearing server with two lane projects and +// a run service over a fake launcher that echoes its command; the call +// helper is the checkout fixture's. +func newSavedRunsFixture(t *testing.T) (*checkoutFixture, string) { + t.Helper() + tokens := map[string]string{"atlas": strings.Repeat("a", 32), "beacon": strings.Repeat("b", 32), "operator": "agent-admin"} + cfg := lanes.Config{Principals: []lanes.Principal{{Project: "atlas", TokenEnv: "ATLAS"}, {Project: "beacon", TokenEnv: "BEACON"}}} + auth, err := lanes.Bind(cfg, func(env string) string { return tokens[strings.ToLower(env)] }, tokens["operator"]) + if err != nil { + t.Fatal(err) + } + store, err := runstate.Open(t.TempDir()) + if err != nil { + t.Fatal(err) + } + root := t.TempDir() + ws := filepath.Join(root, "proj") + if err := os.MkdirAll(ws, 0o755); err != nil { + t.Fatal(err) + } + launcher := runsLauncher(func(_ context.Context, spec sandbox.LaunchSpec) (sandbox.Result, error) { + return sandbox.Result{Stdout: strings.Join(spec.Command, " ")}, nil + }) + f := &checkoutFixture{t: t, inUse: map[string]bool{}, tokens: tokens} + f.s = New(newFakeRuntime()) + f.s.Token, f.s.LaneAuth = tokens["operator"], auth + f.s.Runs = runs.New(store, &runs.Backend{Name: "fake", Launcher: launcher, Capabilities: runs.Capabilities{NetworkNone: true}}, runs.Config{Image: "img", WorkspaceRoots: []string{root}}) + f.s.LaunchContext = context.Background() + f.h = f.s.Handler() + return f, ws +} + +func TestRunsSavedRoutesAnswerNotSupportedWithoutAService(t *testing.T) { + f := newFamServer(t) + f.expectCode(http.MethodGet, api.PathRuns, nil, http.StatusNotImplemented, api.CodeNotSupported) + f.expectCode(http.MethodPost, api.RunRestartPath("run_x"), nil, http.StatusNotImplemented, api.CodeNotSupported) + f.expectCode(http.MethodGet, api.RunHistoryPath("run_x"), nil, http.StatusNotImplemented, api.CodeNotSupported) +} + +func TestRunsListIsScopedToTheCallerUnlessOperator(t *testing.T) { + f, ws := newSavedRunsFixture(t) + var a, b api.ContainedRun + if code, e := f.call("atlas", http.MethodPost, api.PathRuns, api.ContainedRunRequest{WorkspacePath: ws, Command: []string{"a"}}, &a); code != http.StatusOK { + t.Fatalf("atlas create: %d %+v", code, e) + } + if code, e := f.call("beacon", http.MethodPost, api.PathRuns, api.ContainedRunRequest{WorkspacePath: ws, Command: []string{"b"}}, &b); code != http.StatusOK { + t.Fatalf("beacon create: %d %+v", code, e) + } + var mine api.ContainedRunList + if code, e := f.call("atlas", http.MethodGet, api.PathRuns, nil, &mine); code != http.StatusOK || len(mine.Runs) != 1 || mine.Runs[0].ID != a.ID || mine.Runs[0].Stdout != "" { + t.Fatalf("atlas list: %d %+v %+v", code, e, mine) + } + var all api.ContainedRunList + if code, _ := f.call("operator", http.MethodGet, api.PathRuns, nil, &all); code != http.StatusOK || len(all.Runs) != 2 || all.Runs[0].ID != b.ID { + t.Fatalf("operator list: %d %+v", code, all) + } + var none api.ContainedRunList + if code, _ := f.call("operator", http.MethodGet, api.RunsListQuery(api.RunUnfinished, 0), nil, &none); code != http.StatusOK || len(none.Runs) != 0 { + t.Fatalf("status filter: %d %+v", code, none) + } + var one api.ContainedRunList + if code, _ := f.call("operator", http.MethodGet, api.RunsListQuery("", 1), nil, &one); code != http.StatusOK || len(one.Runs) != 1 { + t.Fatalf("limit: %d %+v", code, one) + } + if code, e := f.call("operator", http.MethodGet, api.PathRuns+"?limit=0", nil, nil); code != http.StatusBadRequest || e.Code != api.CodeBadRequest { + t.Fatalf("limit=0: %d %+v", code, e) + } + if code, e := f.call("operator", http.MethodGet, api.PathRuns+"?status=done", nil, nil); code != http.StatusBadRequest || e.Code != api.CodeBadRequest { + t.Fatalf("bad status: %d %+v", code, e) + } +} + +func TestRunsRestartIsOwnedAuditedAndRefusedWhileLive(t *testing.T) { + f, ws := newSavedRunsFixture(t) + events := f.s.subscribe() + defer f.s.unsubscribe(events) + var first api.ContainedRun + if code, e := f.call("atlas", http.MethodPost, api.PathRuns, api.ContainedRunRequest{WorkspacePath: ws, Command: []string{"make", "test"}}, &first); code != http.StatusOK { + t.Fatalf("create: %d %+v", code, e) + } + if code, e := f.call("beacon", http.MethodPost, api.RunRestartPath(first.ID), api.RunRestartRequest{}, nil); code != http.StatusForbidden || e.Code != api.CodeForbidden { + t.Fatalf("beacon restart: %d %+v", code, e) + } + if code, e := f.call("operator", http.MethodPost, api.RunRestartPath("run_missing"), api.RunRestartRequest{}, nil); code != http.StatusNotFound || e.Code != api.CodeNotFound { + t.Fatalf("missing restart: %d %+v", code, e) + } + var again api.ContainedRun + if code, e := f.call("operator", http.MethodPost, api.RunRestartPath(first.ID), api.RunRestartRequest{Reason: "flaky", IdempotencyKey: "r1"}, &again); code != http.StatusOK { + t.Fatalf("restart: %d %+v", code, e) + } + if again.ID == first.ID || again.RestartedFrom != first.ID || again.Owner != "atlas" || again.Status != api.RunCompleted || again.Stdout != "make test" { + t.Fatalf("restarted = %+v", again) + } + var replay api.ContainedRun + if code, _ := f.call("operator", http.MethodPost, api.RunRestartPath(first.ID), api.RunRestartRequest{IdempotencyKey: "r1"}, &replay); code != http.StatusOK || replay.ID != again.ID { + t.Fatalf("replay: %d %+v", code, replay) + } + var old api.ContainedRun + if code, _ := f.call("atlas", http.MethodGet, api.RunPath(first.ID), nil, &old); code != http.StatusOK || old.RestartedBy != again.ID { + t.Fatalf("old record: %d %+v", code, old) + } + var hist api.RunHistory + if code, _ := f.call("atlas", http.MethodGet, api.RunHistoryPath(again.ID), nil, &hist); code != http.StatusOK || hist.Run != again.ID || len(hist.Transitions) != 3 || hist.Transitions[0].Status != api.RunAccepted || hist.Transitions[2].Status != api.RunCompleted { + t.Fatalf("history: %d %+v", code, hist) + } + if code, e := f.call("beacon", http.MethodGet, api.RunHistoryPath(again.ID), nil, nil); code != http.StatusForbidden || e.Code != api.CodeForbidden { + t.Fatalf("beacon history: %d %+v", code, e) + } + + var audits []string + deadline := time.After(10 * time.Second) + for len(audits) == 0 { + select { + case ev := <-events: + if ev.Type == api.EventAudit { + audits = append(audits, string(ev.Data)) + } + case <-deadline: + t.Fatal("no audit event for the restart") + } + } + if a := audits[0]; !strings.Contains(a, auditRunRestart) || !strings.Contains(a, first.ID) || !strings.Contains(a, again.ID) || !strings.Contains(a, "flaky") { + t.Fatalf("audit = %s", a) + } + + // A record whose saved state is still live refuses the restart. + rec, _, err := f.s.Runs.Store.Create(runstate.Record{Owner: "atlas", Detail: []byte(`{"request":{"command":["x"]}}`)}) + if err != nil { + t.Fatal(err) + } + if _, err := f.s.Runs.Store.Update(rec.ID, func(r *runstate.Record) error { r.Status = runstate.StatusRunning; return nil }); err != nil { + t.Fatal(err) + } + if code, e := f.call("atlas", http.MethodPost, api.RunRestartPath(rec.ID), api.RunRestartRequest{}, nil); code != http.StatusConflict || e.Code != api.CodeConflict { + t.Fatalf("live restart: %d %+v", code, e) + } + // One that predates the saved request cannot be re-run either. + legacy, _, _ := f.s.Runs.Store.Create(runstate.Record{Owner: "atlas"}) + f.s.Runs.Store.Update(legacy.ID, func(r *runstate.Record) error { r.Status = runstate.StatusUnfinished; return nil }) + if code, e := f.call("atlas", http.MethodPost, api.RunRestartPath(legacy.ID), api.RunRestartRequest{}, nil); code != http.StatusConflict || e.Code != api.CodeConflict { + t.Fatalf("legacy restart: %d %+v", code, e) + } + if code, e := f.call("atlas", http.MethodPost, api.RunRestartPath(legacy.ID), map[string]any{"bogus": 1}, nil); code != http.StatusBadRequest || e.Code != api.CodeBadRequest { + t.Fatalf("bad body: %d %+v", code, e) + } +} + +func TestRunsRestartDetachedReturnsAccepted(t *testing.T) { + f, ws := newRunsServer(t, func(ctx context.Context, _ sandbox.LaunchSpec) (sandbox.Result, error) { + return sandbox.Result{}, nil + }) + f.srv.LaunchContext = context.Background() + var first api.ContainedRun + f.expect(http.MethodPost, api.PathRuns, api.ContainedRunRequest{WorkspacePath: ws, Command: []string{"x"}}, http.StatusOK, &first) + var again api.ContainedRun + f.expect(http.MethodPost, api.RunRestartPath(first.ID), api.RunRestartRequest{Detach: true}, http.StatusAccepted, &again) + if again.Status != api.RunAccepted || again.RestartedFrom != first.ID { + t.Fatalf("detached restart = %+v", again) + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if done, err := f.srv.Runs.Wait(ctx, again.ID); err != nil || done.Status != api.RunCompleted { + t.Fatalf("wait = %+v, %v", done, err) + } +} diff --git a/src/docs/crux-integration-guide.md b/src/docs/crux-integration-guide.md new file mode 100644 index 0000000000..51c5d35cb3 --- /dev/null +++ b/src/docs/crux-integration-guide.md @@ -0,0 +1,403 @@ +# Crux work services: integration guide + +**Audience:** a developer wiring an orchestrator — hive, a Nous v2 session +backend, Spotlights, or a fourth consumer — onto the Crux work services of +RFC [hivecommons/crux#114](https://github.com/hivecommons/crux/issues/114). +It walks the seven slices in the order a driver composes them, with the +`curl` and `crux` CLI form of every call, the server configuration each +one needs, authentication in both flavours, the failure drills from the +RFC's acceptance sections, and a symptom → saved-record troubleshooting +map. The slice-by-slice semantics, status codes and storage layout are in +the [design record](design/crux-work-services.md); this guide does not +repeat them, it sequences them. + +The rule every slice shares, and the reason the guide ends at S7: **crux +records what happened; the driver decides what it means.** A `done` +signal is a notification, a saved record is proof, and after any +disconnect the driver asks for the record before it acts again. + +## The composition at a glance + +```mermaid +flowchart LR + Card[Driver claims work\ncommit, repo, task] --> S1[S1 checkout\nPOST /v1/checkouts] + S1 --> S2[S2 run or session\nPOST /v1/runs] + S2 --> S6[S6 agent step\nPOST /v1/runs/id/steps] + S6 --> S4[S4 command log\nGET /v1/runs/id/cmdlog] + S6 --> S5[S5 turn signals\nGET /v1/runs/id/signals] + S5 --> S3[S3 human gate\nPOST /v1/gates run=id] + S3 --> S6 + S2 --> S7[S7 saved record\nGET /v1/runs/id\nhistory, restart] + S6 --> S7 +``` + +| Step | Slice | Verb | What the driver gets | +|---|---|---|---| +| 1 | S1 | `POST /v1/checkouts` | a worktree pinned at a full SHA, owned by the caller, capped, deletable | +| 2 | S2 | `POST /v1/runs` | a sandboxed command's real exit code and output — or, with `profile: agent`, an open **session** that steps run inside | +| 3 | S6 | `POST /v1/runs/{id}/steps` | one agent step whose JSON answer was checked against the driver's schema | +| 4 | S4 | `GET /v1/runs/{id}/cmdlog` | the commands the runtime actually dispatched for that run | +| 4 | S5 | `GET /v1/runs/{id}/signals` | where the turn is and how it ended, append-only | +| 5 | S3 | `POST /v1/gates` + `GET /v1/gates/{id}/wait` | a recorded human decision before the driver continues | +| 6 | S7 | `GET /v1/runs/{id}`, `/history`, `GET /v1/runs`, `POST /v1/runs/{id}/restart` | the truth after a disconnect, and an explicit, audited re-run | + +S0 (caller principals) is not a step; it is what every row above means by +"the caller". + +## Before the first call + +### Start a server that has every slice on + +```sh +export CRUX_TOKEN="$(openssl rand -hex 32)" # control token (S0 operator) +export CRUX_CHECKOUT_ALLOWLIST=https://github.com/hivecommons/ # S1 on +export CRUX_RUNS_CONFIG=./runs.json # S2/S6 on +export CRUX_LANES_CONFIG=./lanes.json # optional: lane principals +crux serve --workdir ./crux-work --listen 127.0.0.1:7777 --operator-name kelly +``` + +`runs.json` is the contained-run backend (`crux/runs/config.go`); the +keys a driver's operator will touch: + +```json +{ + "backend": "podman", + "image": "ghcr.io/hivecommons/hive-sandbox:latest", + "repo_allowlist": ["https://github.com/hivecommons/"], + "workspace_roots": ["/srv/scratch"], + "default_timeout_ms": 600000, + "max_timeout_ms": 3600000, + "default_max_output_bytes": 65536, + "max_output_bytes_cap": 1048576, + "network_restricted": false +} +``` + +| Key | Slice | Meaning | +|---|---|---| +| `backend` | S2 | `podman` (default) or `job` (Kubernetes Job; `job`, `job_network`, `job_network_policy` then apply) | +| `image` | S2/S6 | the container image every run and step runs in; **required** | +| `repo_allowlist` | S2 | URL prefixes the ad-hoc `worktree` form may clone; empty refuses it (use S1 instead) | +| `workspace_roots` | S2 | host directories a plain `workspace_path` may lie under; empty refuses it | +| `default_timeout_ms`, `max_timeout_ms` | S2/S6 | bounds on `timeout_ms` | +| `default_max_output_bytes`, `max_output_bytes_cap` | S2 | bounds on `max_output_bytes`; the full output is kept as an artifact regardless | +| `network_restricted` | S2 | operator's assertion that a podman network named `restricted` exists | + +Server flags and env the slices read: + +| Flag / env | Slice | Meaning | +|---|---|---| +| `--token`, `CRUX_TOKEN` | S0 | the control token; required on `--listen`; its principal is the **operator** | +| `--operator-name`, `CRUX_OPERATOR_NAME` | S0 | the name saved as `owner`/`opened_by_principal` for control-token calls (default `operator`) | +| `--lanes-config`, `CRUX_LANES_CONFIG` | S0 | lane principals: `{"principals":[{"project":"atlas","token_env":"ATLAS_LANE_TOKEN"}]}`; each project's token is a bearer token too | +| `--checkout-allow`, `CRUX_CHECKOUT_ALLOWLIST` | S1 | repo URL prefixes `POST /v1/checkouts` may clone; **unset means no checkout service (501)** | +| `--checkout-dir`, `--checkout-max-bytes`, `CRUX_CHECKOUT_TOKEN` | S1 | where checkouts live, the byte ceiling, the private-repo HTTPS token (env only) | +| `--runs-config`, `CRUX_RUNS_CONFIG` | S2/S6 | the file above; **unset means `POST /v1/runs` is 501**, saved records still serve | +| `--workdir` | S3–S7 | gates, run state, command logs, signals and steps all live under it | + +### Authentication, two flavours + +Every request carries `Authorization: Bearer …`. Which token decides who +the caller *is*, and that is the whole S0 model: + +| Token | Principal | Use it for | +|---|---|---| +| the control token (`CRUX_TOKEN`) | the **operator** (`--operator-name`) | S1–S7 as the single driver: it owns what it creates, sees every record, and is the only principal that may answer a gate without an override | +| a lane principal's token (`lanes-config`) | the **project** (`atlas`) | one driver per project: each sees and may act on only its own checkouts, runs, steps and gates; a gate it opened it cannot answer | + +Pick one and stay with it per driver: a run created as `atlas` is +invisible to `beacon` (403 `forbidden`, the message names the owner), and +an operator restart of a project's run keeps the run owned by the project +— it does not widen what the run may do. + +```sh +export CRUX_SERVER=http://127.0.0.1:7777 # the CLI reads these +export CRUX_TOKEN=… # control token or a lane token +H=(-H "Authorization: Bearer $CRUX_TOKEN" -H "Content-Type: application/json") +``` + +Unix-socket servers without a token treat every caller as the operator; +that is what the transport already establishes. + +## Step 1 — S1: pin the checkout + +```sh +curl -s "${H[@]}" -X POST $CRUX_SERVER/v1/checkouts -d '{ + "repo": "https://github.com/hivecommons/hive", + "commit": "84b42eabe0a1d2c3b4e5f60718293a4b5c6d7e8f", + "ttl": "72h" +}' +# → 201 {"id":"co_…","path":"/…/checkouts/co_…","commit":"84b42…","owner":"kelly",…} + +crux checkout add -repo https://github.com/hivecommons/hive -commit 84b42eabe… -ttl 72h -json +crux checkout list +crux checkout rm co_… +``` + +What to rely on: `commit` must be the full 40-hex SHA (a branch or tag is +400 — the pin cannot move under you); an off-allowlist repo is 403 before +anything is cloned; the response `path` is on the server and is what S2 +runs in; `DELETE` is 409 while a run uses the checkout. Save the `id` with +the work item — it is the thing you hand to S2. + +## Step 2 — S2: run the command, or open the session + +A one-shot command: + +```sh +curl -s "${H[@]}" -X POST $CRUX_SERVER/v1/runs -d '{ + "checkout": "co_…", + "command": ["make", "test"], + "env_allowlist": ["CI"], + "network": "none", + "timeout_ms": 600000, + "idempotency_key": "card-42/baseline/1" +}' +# → 200 {"id":"run_…","status":"completed","exit_code":0,"stdout":"…","owner":"kelly",…} + +crux runs start --checkout co_… --env CI --timeout 10m --idempotency-key card-42/baseline/1 -- make test +``` + +Always send `idempotency_key`: it is what makes a retried `POST` return +the run you already started instead of a second one. The response carries +the `id` the moment the run is **accepted**, before anything executes; +with `"detach": true` the call returns `202` with that accepted record and +the driver polls or waits (step 6). + +A step **session** for S6 is the same route with no command: + +```sh +curl -s "${H[@]}" -X POST $CRUX_SERVER/v1/runs -d '{ + "checkout": "co_…", "profile": "agent", "idempotency_key": "card-42/session" +}' +# → 202 {"id":"run_…","status":"accepted","profile":"agent",…} then "waiting" once provisioned + +crux runs start --checkout co_… --profile agent --idempotency-key card-42/session +``` + +The session provisions its sandbox and sits at `waiting`; it closes with +`POST /v1/runs/{id}/cancel` (`crux runs cancel`). Nothing runs in it until +step 3. + +## Step 3 — S6: run one schema-checked agent step + +```sh +curl -s "${H[@]}" -X POST $CRUX_SERVER/v1/runs/run_…/steps -d '{ + "backend": "claude", + "model": "claude-sonnet-4-5", + "prompt": "Measure the baseline: run make test three times and report pass counts as JSON.", + "output_schema": {"type":"object","required":["runs","passed"], + "properties":{"runs":{"type":"integer"},"passed":{"type":"integer"}}}, + "tools": ["Bash"], + "turn_limit": 20, + "timeout_ms": 900000, + "retry_on_mismatch": 2 +}' +# → 200 {"run":"run_…","n":1,"status":"succeeded","answer":{"runs":3,"passed":3},"attempts":1, +# "usage":{"input_tokens":…,"output_tokens":…},"transcript_path":"steps/1/transcript.jsonl"} + +crux step run run_… --backend claude --prompt-file prompts/baseline.md --schema schemas/baseline.json \ + --tool Bash --turn-limit 20 --retry 2 +crux step list run_… +crux step show run_… 1 +``` + +Statuses a step ends in: `succeeded`; `schema_mismatch` (the answer +never matched after `retry_on_mismatch` re-asks — `violations` says +where); `timeout`, `backend_error`, `tool_denied`, `turn_limit_exceeded`, +`cancelled`; `unsupported` (the backend cannot enforce a limit you asked +for; recorded, never faked); `unfinished` (step 6). `answer` is only ever +present on `succeeded`, and it has already been validated against +`output_schema` — the driver does not re-check shape, it checks meaning. +One step at a time per session: a second `POST` while one runs is 409 +`step_busy`. + +The `process` backend (`"backend":"process","command":["./my-agent"]`) +is the way to put a non-LLM tool, or a test double, behind the same +contract. + +## Step 4 — S4 + S5: verify what happened, follow where the turn is + +The command log is the runtime's record of what the step actually +dispatched. The driver uses it to check an agent's claims, never the +agent's narration: + +```sh +curl -s "${H[@]}" "$CRUX_SERVER/v1/runs/run_…/cmdlog?since_seq=0&limit=100" +# → {"run":"run_…","commands":[{"seq":1,"status":"completed","command":["make","test"],"exit_code":0,"capture":…,…}],"next_seq":2} + +crux cmdlog list run_… +crux cmdlog show run_… 1 # one record with its captured output +``` + +"Did `make test` run at least three times?" is `cmdlog list --json | jq` +over `command` and `status`, not a question for the agent. + +Turn signals are the live progress stream and the saved outcome of each +turn. The runner writes `started` and `done`; the driver (as owner) may +post the middle kinds for its own bookkeeping; a gate opened with `run` +(step 5) appears here as `needs-input` → `input`: + +```sh +curl -s "${H[@]}" "$CRUX_SERVER/v1/runs/run_…/signals?since_seq=0" +# → {"run":"run_…","signals":[{"seq":1,"kind":"started","source":"runner",…},{"seq":2,"kind":"step","data":{"n":1,"status":"succeeded"}},…],"next_seq":3} + +curl -s "${H[@]}" -X POST $CRUX_SERVER/v1/runs/run_…/signals -d '{"kind":"thinking","message":"planning the control arm"}' + +crux signal list run_… +crux signal emit run_… --kind thinking --message "planning the control arm" +``` + +For push instead of poll, `GET /v1/events` is the server's SSE stream; +`run.signal` events carry the same records, `gate.*` events the gates. +Treat an event as a hint to read the saved record, not as the record. + +## Step 5 — S3: ask a human before continuing + +```sh +curl -s "${H[@]}" -X POST $CRUX_SERVER/v1/gates -d '{ + "subject": "Promote finding card-42 to the report?", + "choices": ["approve", "reject", "rerun"], + "timeout": "24h", + "context": {"runs": 3, "passed": 3, "control_passed": 1}, + "idempotency_key": "card-42/promote", + "run": "run_…" +}' +# → 201 {"id":"gate_…","status":"waiting","opened_by_principal":"atlas",…} + +curl -s "${H[@]}" "$CRUX_SERVER/v1/gates/gate_…/wait?timeout=5m" +# → 200 the gate: answered or expired if that happened within the window (max 5m), still "waiting" otherwise — loop + +# the operator answers, from anywhere with the control token: +curl -s "${H[@]}" -X POST $CRUX_SERVER/v1/gates/gate_…/answer -d '{"answer":"approve","note":"control arm looks right"}' + +crux gate open --subject "Promote finding card-42?" --choice approve --choice reject --timeout 24h \ + --idempotency-key card-42/promote --run run_… --json +crux gate wait gate_… --timeout 30m +crux gate answer gate_… --choice approve --note "control arm looks right" +``` + +The opener cannot answer its own gate (403 `gate_self_answer`); the +operator may override with `"as_operator": true`, and that override is +audited. `answered` and `expired` are different statuses — an expiry is +never a denial. `context` is yours; keep secrets out of it, it is saved +and shown to the person deciding. + +## Step 6 — S7: the saved record is the truth + +Every S2 run and S6 step has an id from the moment it was accepted and a +record that outlives the connection, the request and the `crux serve` +process. After a timeout, a dropped connection or a restart, the driver's +first move is to read it: + +```sh +curl -s "${H[@]}" $CRUX_SERVER/v1/runs/run_… # the run +curl -s "${H[@]}" $CRUX_SERVER/v1/runs/run_…/steps/1 # the step (RFC's GET /v1/steps/{id}, numbered per run) +curl -s "${H[@]}" $CRUX_SERVER/v1/runs/run_…/history +# → {"run":"run_…","transitions":[{"at":…,"status":"accepted"},{"at":…,"status":"running"}, +# {"at":…,"status":"unfinished","reason":"crux serve restarted while the run was in flight; the exit is not knowable"}]} +curl -s "${H[@]}" "$CRUX_SERVER/v1/runs?status=unfinished&limit=50" # everything that needs a decision + +crux runs status run_… +crux step show run_… 1 +crux runs history run_… +crux runs list --status unfinished +``` + +Run statuses are a closed set: `accepted | running | waiting | completed | +failed | cancelled | unfinished`; steps add `succeeded` and +`unsupported`. `unfinished` means exactly one thing: the process that +would have known the exit is gone. Crux never upgrades it to success and +never silently re-runs it — the driver decides, and the record shows why. + +When the decision is "run it again", say so explicitly: + +```sh +curl -s "${H[@]}" -X POST $CRUX_SERVER/v1/runs/run_…/restart -d '{ + "reason": "serve restarted mid-run; baseline must be measured whole", + "idempotency_key": "card-42/baseline/2" +}' +# → 200 the new run (202 with "detach": true), "restarted_from":"run_…"; the old record gains "restarted_by" +# → 409 conflict while the old run is accepted/running/waiting: the runner may be alive + +crux runs restart --reason "serve restarted mid-run" --idempotency-key card-42/baseline/2 run_… +``` + +A restart re-validates the *saved* request as the *original* owner +against the *current* server configuration, so a removed checkout or a +shrunk allowlist refuses it the same way a fresh create would. It is +audited (`run.restart`: who, from which run, to which, why). It is a new +run with its own id, never a mutation of the finished one — `history` of +the old run still ends where it ended. + +## Failure drills + +Run these against a scratch project before trusting the integration. Each +is one of the RFC's "done when" conditions; the expected observation is +what the saved records show. + +| Drill | Do | Expect | +|---|---|---| +| Lost connection after `POST /v1/runs` | send the create, kill the client before the response | `GET /v1/runs?status=…` or the same `POST` with the same `idempotency_key` returns the run that was started; nothing ran twice | +| Duplicate create | send the same body with the same key twice | one run, one id, `created` only the first time; `cmdlog` shows one dispatch | +| Server restart mid-run | start a long run or a step, restart `crux serve` | run `unfinished` with the restart reason; its `running` step `unfinished` with the same reason; `signals` ends in `done` with that status; `GET /v1/runs?status=unfinished` lists it | +| Restart while live | `POST …/restart` on a run that is `running` | 409 `conflict`; cancel it first (or let it end), then restart succeeds and the two records link | +| Pin that moves | `POST /v1/checkouts` with a branch name | 400 `bad_request`; nothing cloned | +| Off-list repository | checkout or `worktree` run against a repo not on the allowlist | 403 `forbidden`, message names the allowed prefixes | +| Agent lies about its work | step prompt says "run the suite three times"; agent answers `runs: 3` | `cmdlog list` is the count that matters; compare, do not trust the answer | +| Answer outside the schema | step with a strict `output_schema`, prompt that invites extra fields | re-asks up to `retry_on_mismatch`; then `schema_mismatch` with `violations`, no `answer` | +| Self-answered gate | open a gate as `atlas`, answer as `atlas` | 403 `gate_self_answer`; with `as_operator: true` 200 and an audit record | +| Gate expiry vs denial | open with `timeout: "1m"`, wait | `expired`, `expired_at` set, no `answer`; distinguishable from `answered: reject` | +| Cross-project read | create as `atlas`, read as `beacon` | 403 naming the owner, for runs, steps, checkouts and gates alike; `GET /v1/runs` as `beacon` omits it | + +## Symptom → saved record + +| You see | Read | It tells you | +|---|---|---| +| the create call hung or errored | `GET /v1/runs?limit=…` or replay the `POST` with its `idempotency_key` | whether the run exists and what state it is in | +| `status: unfinished` | `GET /v1/runs/{id}/history` | when it went in flight and the reason it was settled (serve restart) | +| a step `schema_mismatch` with no `answer` | `GET /v1/runs/{id}/steps/{n}` → `violations`, `attempts`, `transcript_path` | what failed to match per attempt; the transcript is under the run's work dir | +| a step `unsupported` | the step's `reason` | which limit the backend could not enforce; pick another backend or drop the limit | +| the agent says it ran X | `GET /v1/runs/{id}/cmdlog` | what was actually dispatched, with exit codes and captured output | +| "nothing is happening" | `GET /v1/runs/{id}/signals` | the last signal: `needs-input` on a gate (its id is in `data`), `thinking`, or no `started` yet (still provisioning) | +| a gate the driver is waiting on | `GET /v1/gates/{id}` | `waiting`, `answered` (by whom, with what, overridden or not) or `expired` | +| 409 on restart | `GET /v1/runs/{id}` | its status is live; `cancel` or wait | +| 409 on `DELETE /v1/checkouts/{id}` | `GET /v1/runs?status=running` / `waiting` | which run still uses it | +| 501 on `POST /v1/runs` or `/v1/checkouts` | server flags | `--runs-config` / `--checkout-allow` is unset on that server; saved records still serve | +| 403 on your own record | the token you sent | you are a different principal than the one that created it | + +## What hive uses today vs. post-cutover + +Hive is one driver among the ones the RFC names, and today a partial one. +The code of every slice lives in hive `src/crux` and reaches +[hivecommons/crux](https://github.com/hivecommons/crux) through the +[publish sync](crux-publish.md) until the **2026-11-16 cutover**, after +which the crux repository is the source of truth and hive pins a tagged +release. + +| Concern | Hive today (`v5`) | After the cutover | +|---|---|---| +| Where the code lives | `src/crux`, same module as hive; the crux repo is a generated copy | `hivecommons/crux` at a tag; hive depends on it like any consumer | +| Running agents | in-process `crux.Runtime` behind `cmd/hive` (`agent_runtime.go`); no `crux serve` between hive and its agents | unchanged for hive's own agents — the work services are for *work*, not for hosting hive's agents | +| Stage worktrees (S1) | hive's own worktree management and `nousdeps` trust rules, which S1 was ported from | drivers that need a pinned checkout call `POST /v1/checkouts`; hive may keep its own path or adopt S1 for stage worktrees | +| Untrusted commands (S2) | hive's agent sandbox (`config.SandboxJobConfig`, podman/Job) runs the agent itself | the same backends behind `runs.Service`; a driver asks for a contained run instead of owning a sandbox | +| Human decisions (S3) | hive's maintainer commands and park/un-park on GitHub issues | an S3 gate is the runtime-side record; a product adapter can mirror it to a forge — crux never does | +| Verifying agent claims (S4) | not verified; the agent's narration is what the stage sees | `cmdlog` is the check; the Nous v2 backend is the first driver built on it | +| Turn outcomes (S5) | `GET /v1/agents/{name}/kick-outcome` for kicked agents; `run.signal` events | the same signals for every contained run | +| Agent steps (S6) | none; hive stages are whole agent turns | the Nous v2 backend's unit of work; hive stages could become steps in a session | +| After a restart (S7) | hive reconciles its own leases; crux runs were not hive's | `unfinished` records, `history`, explicit `restart`; hive's lanes (`cmd/hive/lanes.go`) already use the crux API client and are the natural place to read them | +| Lanes / work cards | hive is a lane participant via `crux/api` lanes client | unchanged; the work-card contracts are a companion spec, parked | + +The first driver built end to end on these surfaces is the Nous v2 +session backend — out of crux, built against a **tagged** Nous v2 release +and a **tagged** crux release (v0.3.0 is the floor). If this guide cannot +get that driver, or a fourth consumer, wired in days, the surfaces need +fixing, not the guide. + +## See also + +- [Design record: crux work services](design/crux-work-services.md) — routes, codes, semantics and storage per slice, as implemented +- [`crux serve`](crux-serve.md) — the server, its routes, events and the Go client +- [`crux` CLI](crux-cli.md) — the local runtime verbs the work-service verbs sit beside +- [Crux durable lanes](crux-lanes.md) — the transport the work-card contracts will ride on +- [Publishing Crux](crux-publish.md) — how `src/crux` becomes `hivecommons/crux` diff --git a/src/docs/crux-serve.md b/src/docs/crux-serve.md index b6cdba81c3..df25c7574d 100644 --- a/src/docs/crux-serve.md +++ b/src/docs/crux-serve.md @@ -98,7 +98,7 @@ and [lane error codes and failure behavior](crux-lanes.md#bounds-and-failure-beh | `POST /v1/agents/{name}/stop` | | `Lifecycle.Stop` | `{"agent","process"}` | | `GET /v1/events` | | watcher + `Kicker.SetKickObserver` + the `crux.Host` observers | `text/event-stream` of `Event` records | | `POST /v1/checkouts`, `GET /v1/checkouts[?owner=]`, `GET`/`DELETE /v1/checkouts/{id}` | `{"repo","commit","owner"?,"branches"?,"max_bytes"?,"ttl"?}` | `checkout.Service` | Work service (S1): a worktree pinned at a full SHA, owned by the caller principal; `501` without `--checkout-allow`; [routes, codes and semantics](design/crux-work-services.md) | -| `POST /v1/runs`, `GET /v1/runs/{id}`, `POST /v1/runs/{id}/cancel` | contained-run request (see `ContainedRunRequest` in `crux/api/runs.go`) | `runs.Service` | Work service (S2/S7): run an untrusted command in a sandbox and read the saved record whatever the process state; cancel after completion returns the result; `501` on `POST` without `--runs-config`; restart and listing routes are [deferred](design/crux-work-services.md#deferred-runs-routes) | +| `POST /v1/runs`, `GET /v1/runs[?status=&limit=]`, `GET /v1/runs/{id}`, `POST /v1/runs/{id}/cancel`, `POST /v1/runs/{id}/restart`, `GET /v1/runs/{id}/history` | contained-run request (see `ContainedRunRequest` in `crux/api/runs.go`) | `runs.Service` | Work service (S2/S7): run an untrusted command in a sandbox and read the saved record whatever the process state; cancel after completion returns the result; `501` on `POST` without `--runs-config`; listing is owner-scoped (the operator sees all), restart re-runs the saved request and is audited, refused with `409` while the run is live; [S7 routes, codes and semantics](design/crux-work-services.md#s7-saved-run-results) | | `POST /v1/gates`, `GET /v1/gates[?status=]`, `GET /v1/gates/{id}`, `POST /v1/gates/{id}/answer`, `GET /v1/gates/{id}/wait[?timeout=]` | `{"subject","context"?,"choices"?,"timeout"?,"opened_by"?,"idempotency_key"?,"run"?}`; answer `{"answer","note"?,"as_operator"?}` | `gates.Service` | Work service (S3): human gates under `/gates`, always on; re-opening the same gate returns the same id and recorded answer; only the operator answers, the opener only with the audited `as_operator` override; a gate opened with `run` becomes that run's turn signals (S5); [routes, codes and semantics](design/crux-work-services.md#s3-human-gate) | | `GET /v1/runs/{id}/signals[?since_seq=&limit=]`, `POST /v1/runs/{id}/signals` | `{"kind","message"?,"data"?}` | `runs.Service` + `signals.Book` | Work service (S5): a run's append-only turn signals (`started`, `thinking`, `tool`, `blocked`, `needs-input`, `input`, `done`), owner-visible; the owner posts the middle kinds, the runner writes `started` and `done`; [routes, codes and semantics](design/crux-work-services.md#s5-turn-signals) | | `GET /v1/agents/{name}/kick-outcome` | | `signals.AgentOutcomes` + `Observer.GetStatusFast` | S5: the saved outcome of the agent's last kicked turn with `settled_for_last_kick`; [semantics](design/crux-work-services.md#s5-turn-signals) | diff --git a/src/docs/design/crux-work-services.md b/src/docs/design/crux-work-services.md index 8e9f6734c8..d948e5ae55 100644 --- a/src/docs/design/crux-work-services.md +++ b/src/docs/design/crux-work-services.md @@ -1,7 +1,7 @@ # Crux work services: S1 pinned checkout (hivecommons/crux#114) Status: **S0 caller principals and S1 pinned checkout shipped in hive -`src/crux` (v5).** S2 contained run and S7 minimum shipped in #17014; S3 human gate is [below](#s3-human-gate); S4 command log is [below](#s4-command-log); S5 turn signals are [below](#s5-turn-signals); S6 agent step is [below](#s6-agent-step); everything else in the RFC is unscheduled. This records +`src/crux` (v5).** S2 contained run and the S7 minimum shipped in #17014; S3 human gate is [below](#s3-human-gate); S4 command log is [below](#s4-command-log); S5 turn signals are [below](#s5-turn-signals); S6 agent step is [below](#s6-agent-step); S7 saved run results is complete and [below](#s7-saved-run-results). How an orchestrator composes the slices end to end is the [integration guide](../crux-integration-guide.md). This records S1 *as implemented*, including where it is weaker than the RFC text. The RFC asks for "a working copy of this repo at this commit, owned by @@ -588,11 +588,69 @@ next one sees. the service lives in hive `src/crux` and ships through the existing mirror unchanged. -## Deferred runs routes +## S7. Saved run results -The runs routes (`POST /v1/runs`, `GET /v1/runs/{id}`, -`POST /v1/runs/{id}/cancel`) shipped in #17014. Restart and listing routes -for runs are deferred and not scheduled. +The RFC's S7 is the promise that a run's outcome outlives the process that +produced it: stable ids at accept, idempotent create, a closed status set, +the saved record readable later, in-flight runs settled to `unfinished` +when `crux serve` restarts, and an explicit, audited restart that refuses +a run whose saved state is still live. #17014 shipped the minimum (ids, +idempotency, `GET /v1/runs/{id}`, reconcile of runs); this section is the +rest, *as implemented*. + +### Surface + +| Route | Who | Answer | +|---|---|---| +| `GET /v1/runs?status=S&limit=N` | owner sees own runs, operator sees all | `ContainedRunList`, newest first, stdout/stderr stripped; `limit` must be a positive integer (server default 100); `status` must be a known status | +| `POST /v1/runs/{id}/restart` | owner or operator | the new run, `200` once it has finished or `202` with `detach: true` (sessions are always `202`); `409 conflict` while the old run is `accepted`/`running`/`waiting` or when its record predates the saved request; an empty body is a plain restart | +| `GET /v1/runs/{id}/history` | owner or operator | `RunHistory`: every status the record passed through with its time and reason | +| `GET /v1/runs/{id}/steps/{n}` | owner or operator | the RFC's `GET /v1/steps/{id}`: steps are numbered per run (S6), so a step's saved record lives under its run | + +CLI: `crux --server ADDR runs list [--status S] [--limit N] [--json]`, +`runs restart [--reason TEXT] [--idempotency-key KEY] [--detach] [--json] `, +`runs history [--json] `. + +### Semantics + +- **Restart re-runs the saved request.** The record keeps the + `ContainedRunRequest` it was accepted with (minus `idempotency_key` and + `detach`); restart validates it again as the *original owner* against + the *current* configuration, so a removed workspace root, a deleted + checkout or a shrunk allowlist refuse the restart with the same + `bad_request` the original would get now. An operator restarting a + project's run does not widen what the run may do, and the new run is + owned by the project. +- **Two records, linked both ways.** The new run carries + `restarted_from`; the old record gains `restarted_by` once the restart + is accepted. Its status does not change — a restart is a new run, not + a mutation of the finished one. `reason` is saved on the new record and + audited. +- **Refused while live.** Any non-terminal status is `ErrLive` → `409`: + the runner may still be alive, and two drivers must not both resume one + task. The only ways out of a live state are the run ending, `cancel`, + or the reconcile at serve restart. +- **Idempotent like create.** `idempotency_key` on the restart request is + scoped the same way as on create; a replay returns the restart it + already started, without a second launch. +- **Audited.** Every accepted restart records `run.restart` with the new + run, `restarted_from`, the old status, the reason and the owner, through + the same sink as the gate override. +- **History is the transition log.** `runstate` appends to `index.jsonl` + on create and on every status change, and only then — a payload-only + update (output, cmdlog pointer, restart back-link) is not a transition. + `History(id)` is that file filtered by id. +- **Reconcile settles steps too.** When serve restarts, every run that + goes `unfinished` has its `running` steps marked `unfinished` with the + same reason, so `GET /v1/runs/{id}/steps/{n}` never shows a step that is + "running" in a run that is not. + +### Not in S7 + +- Pruning or archival of saved records. +- Restart of a run whose record predates the saved request (shipped + before this slice): `409 conflict` with a message naming the cause. +- A `GET /v1/steps/{id}` route distinct from the per-run one. ## Package map @@ -626,3 +684,9 @@ for runs are deferred and not scheduled. | `src/crux/runs/service.go` | S6 session: `profile: agent` validation, `session` runner, `RunStep`, `Steps`, `Step`, `ErrNotSession`, `ErrStepBusy` | | `src/crux/serve/steps.go` | `POST`/`GET /v1/runs/{id}/steps`, `GET /v1/runs/{id}/steps/{n}`, ownership check, error mapping | | `src/crux/cli/step.go` | `crux step run|list|show`, remote only | +| `src/crux/api/runs.go` | S7: `RunRestartPath`, `RunHistoryPath`, `RunsListQuery`, `ContainedRun.RestartedFrom/RestartedBy`, `RunRestartRequest`, `ContainedRunList`, `RunTransition`, `RunHistory`, `Client.{ListRuns,RestartRun,RunHistory}` | +| `src/crux/runstate/runstate.go` | S7: `History`; the index is the transition log | +| `src/crux/runs/restart.go` | S7: `Restart`, `List`, `ListFilter`, `History`, `ErrLive`, `ErrNoRequest`, `DefaultListLimit`; `service.go` saves the request on the record and reconciles steps | +| `src/crux/agentstep/store.go` | S7: `Store.Reconcile` marks running steps `unfinished` | +| `src/crux/serve/runs_saved.go` | `GET /v1/runs`, `POST /v1/runs/{id}/restart`, `GET /v1/runs/{id}/history`, `run.restart` audit | +| `src/crux/cli/runs_saved.go` | `crux runs list|restart|history`, remote only | diff --git a/src/docs/documentation-map.md b/src/docs/documentation-map.md index 48f81f8011..572ca3ee3e 100644 --- a/src/docs/documentation-map.md +++ b/src/docs/documentation-map.md @@ -126,6 +126,7 @@ Documentation for the current Hive line (branch `v5`; the code and docs live und - [`bd` beads CLI](beads-cli.md) — work-ledger and knowledge command reference for operators and contributors. - [`crux` CLI](crux-cli.md) — run, list, tail, kick and stop agents through `crux.Runtime` with no governor or dashboard; definition formats, the control socket served by `crux run`, and why the runtime being in-process shapes it. - [`crux serve`: the runtime over a local API](crux-serve.md) — **Developer:** operate the runtime server through its authenticated API, event stream, CLI, or Go client. +- [Crux work services: integration guide](crux-integration-guide.md) — **Developer:** wire an orchestrator (hive, Nous v2, a fourth consumer) onto the work services of hivecommons/crux#114 end to end — checkout → run/session → agent step → command log and signals → gate → saved results and restart — with `curl` and `crux` forms, config keys, both auth flavours, failure drills and a symptom → saved-record map. - [Crux durable lanes](crux-lanes.md) — **Developer/operator; opt-in:** configure forge-neutral JetStream-backed events and work, project capabilities, replay, leases and explicit recovery through the Crux API; connect Hive as an ordinary participant. - [Publishing Crux](crux-publish.md) — how `hivecommons/crux` is produced from `src/crux` (plain copy, module path rewritten, fast-forward commits), what is preserved there, the `CRUX_SYNC_TOKEN` credential, manual sync and how a `crux/vX.Y.Z` tag becomes a release. ### Backup and API reference diff --git a/src/docs/env-vars.md b/src/docs/env-vars.md index 55e19229c0..a1a09b6868 100644 --- a/src/docs/env-vars.md +++ b/src/docs/env-vars.md @@ -57,6 +57,8 @@ behavior. [Maintenance limitations](#keeping-this-reference-current) still apply | `HIVE_DASHBOARD_COOKIE` | No | none | **Client-side only** - read by `hivectl tui`, never by the server. Cookie header value (e.g. `hive_session=...`) carrying a per-user session, for hives that do not accept the shared token: hub-hosted ones, and spokes with an `authorized_users` allowlist. See [hivectl.md, Credentials](hivectl.md#credentials). | | `HIVE_DASHBOARD_BIND` | No | `127.0.0.1` for the legacy `dashboard/server.js` | Legacy Node dashboard listen address. Leave unset for loopback-only binding; set `HIVE_DASHBOARD_BIND=0.0.0.0` only when an authenticated reverse proxy or equivalent network control protects the unauthenticated legacy control endpoints. | | `HIVE_AUTHORIZED_USERS` | No | none | Comma-separated direct-route dashboard allowlist, with optional `user:role` entries. Used when `dashboard.authorized_users` is empty. | +| `HIVE_GITHUB_ETAG_CACHE_MAX_MB` | No | `github.etag_cache_max_mb`, else `128` | Memory cap, in MiB of gzip-compressed bodies, for the GitHub conditional-request (ETag / `304`) response cache shared by every GitHub client in the process. Wins over the YAML key; `1`–`2048`, `0` means the default. Raise it when **Debug → GitHub API budget → ETag cache** sits at the cap and the hit rate slides — evicted bodies turn free `304`s back into charged fetches. See [Conditional requests](github-app-setup.md#api-quota-conditional-requests-and-post-reset-pacing). | +| `HIVE_GITHUB_ETAG_CACHE_MAX_ENTRIES` | No | `github.etag_cache_max_entries`, else `8192` | Entry cap for the same cache, whatever the bodies' size. Wins over the YAML key; `0` means the default. | | `HIVE_SELF_AUTHORIZATION_HOLD` | No | `github.self_authorization_hold` / `project.repo_policies[].self_authorization_hold`; when all are unset, default `true` through ACMM L5 and `false` at ACMM L6 (Autonomous) | Process-level override for the #5117 self-authorization hold. Set `false` to let this hive skip and release self-authorization holds for every repo while preserving unrelated human holds; set `true` to keep the hold enabled even at L6. | | `HIVE_SELF_EXPERIMENTS_OPT_IN` | No | unset (off) | Opts this hive in to receiving hub-delivered Hive-self feature-flag assignments over heartbeat (`pkg/hub/spoke/self_experiment_flags.go`). Only the exact string `true` enables it; `1`, `yes`, `on` and any other value leave it off. Consent is local: the hub cannot opt a spoke in. Unsetting it withdraws the flags on the next heartbeat. Separate from the Experiment Lab/governor settings. | | `HIVE_REPORTER_TRUST_HOLD` | No | `github.reporter_trust_hold` / `project.repo_policies[].reporter_trust_hold`; when all are unset, follows `project.issue_filter.reporter_trust.enabled` | Process-level override for the #9665 reporter-trust hold (a PR whose rationale traces to an issue from an untrusted reporter is held at every ACMM level). When set, the dashboard toggle and per-repo overrides are locked and say so. | diff --git a/src/docs/fleet-report.md b/src/docs/fleet-report.md index 3af866310b..02aac3963b 100644 --- a/src/docs/fleet-report.md +++ b/src/docs/fleet-report.md @@ -155,7 +155,7 @@ title is `[selfreport] : `. | `github_fetch_transient` | a non-critical GitHub fetch failed and the feature degrades gracefully (`channel commit date: fetch failed — the row will render without a timestamp`, `pkg/hub/channel_commit_date.go`) — typically a rate limit or transient API error, not a hive defect. Reported only when persistent: log-fed, same ≥5-in-10-min repeat threshold and idle clear as `agent_pane_timeout`. The sibling `SHA poll: commit message fetch non-200` has its own class, `sha_poll_commit_message_unavailable` | low | | `label_provisioning_failed` | provisioning the issue stage labels on a repo failed (`issue stage labels: provisioning failed`, `pkg/github/issue_stage_labels.go`) — typically the App lacking label permission on that repo or a transient API error; diagnostics carry the repo and scrubbed error. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | low | | `agent_cli_restart_loop` | the agent CLI crashed (bare shell) or sat on a consent screen and the harness restarted it before the kick (`agent CLI crashed or stuck on consent screen, restarting before kick`, `crux/agent/manager_kick.go`, `kick_async.go`), repeated ≥5 times in 10 min — a restart loop, not transient noise. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | -| `proxy_connect_dial_failed` | the hive proxy could not dial the upstream host of a CONNECT request (`proxy: CONNECT dial failed`, `crux/proxy/github_proxy_connect.go`) — typically DNS or network unreachability of an agent-requested host, repeated ≥5 times in 10 min. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | +| `proxy_connect_dial_failed` | the hive proxy could not dial the upstream host of a CONNECT request (`proxy: CONNECT dial failed`, `crux/proxy/github_proxy_connect.go`) or of a Copilot usage-sniffed connection (`copilot sniff: upstream dial failed`, `crux/proxy/github_proxy_copilot.go`) — typically DNS or network unreachability of an agent-requested host (for example a dial timeout to `telemetry.enterprise.githubcopilot.com`), repeated ≥5 times in 10 min. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | | `proxy_tls_handshake_failed` | the hive proxy's TLS handshake with an upstream host failed (`proxy client tls handshake failed`, `transparent proxy tls handshake failed`, `copilot sniff: client tls handshake failed`, `crux/proxy/github_proxy_connect.go`, `crux/proxy/github_proxy_copilot.go`) — typically a certificate validation failure or encryption mismatch on an agent-requested host, repeated ≥5 times in 10 min. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | | `review_evidence_write_failed` | the github-client failed to write a review evidence bundle to persistent storage (`review evidence: could not write bundle`, `pkg/github`), repeated ≥5 times in 10 min. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | | `selfreport_cleared_comment_failed` | the fleet report publisher failed to post the "condition cleared" comment on a self-report issue (`fleet report: cleared comment failed`, `pkg/github`), repeated ≥5 times in 10 min. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | diff --git a/src/docs/github-app-setup.md b/src/docs/github-app-setup.md index 7c1f10e4fc..282c616b40 100644 --- a/src/docs/github-app-setup.md +++ b/src/docs/github-app-setup.md @@ -311,7 +311,7 @@ To rotate a private key, generate a new key in GitHub, mount it at the configure An installation token has an hourly REST allowance (about 6,650 requests for a 16-repo installation) shared by everything that uses that installation — the hive's own sweeps and every agent's scoped token. Two behaviours in the shared HTTP transport keep the hive's share small ([#7430](https://github.com/hivecommons/hive/issues/7430)): -- **Conditional requests.** Every repeat `GET` of a resource the hive has already fetched carries `If-None-Match` with the remembered `ETag`; GitHub answers `304 Not Modified`, which is **not charged** against the quota, and the hive replays the remembered body. Unchanged issue and PR lists — the common case between two governor cycles — cost nothing. Cached bodies are keyed by the requesting identity as well as the URL, so an agent's scoped token never reads a body the App fetched. A replayed response carries `X-Hive-Cache: revalidated`. +- **Conditional requests.** Every repeat `GET` of a resource the hive has already fetched carries `If-None-Match` with the remembered `ETag`; GitHub answers `304 Not Modified`, which is **not charged** against the quota, and the hive replays the remembered body. Unchanged issue and PR lists — the common case between two governor cycles — cost nothing. Cached bodies are keyed by the requesting identity as well as the URL, so an agent's scoped token never reads a body the App fetched. A replayed response carries `X-Hive-Cache: revalidated`. The cache is bounded — `github.etag_cache_max_mb` (default `128` MiB of gzip-compressed bodies, env `HIVE_GITHUB_ETAG_CACHE_MAX_MB`) and `github.etag_cache_max_entries` (default `8192`) — and evicts least-recently-used bodies past either bound. A body evicted before its next poll costs a charged fetch instead of a free `304`, so a hive whose hit rate slides while **Debug → GitHub API budget → ETag cache** sits at the cap should raise the cap; the change applies from Settings → Forge App without a restart. - **Post-reset pacing.** When GitHub answers a primary-limit `403` (`X-RateLimit-Remaining: 0`), the hive does not fire everything it had queued the instant the window resets — that is how a spoke re-exhausted its quota one second after every reset. For ten minutes after the reset, requests are spaced ~2 s apart (the same slow start already used after a secondary limit), so the new window is spent gradually. If a hive is still exhausting its quota, look for consumers outside the hive process on the same installation: agents' `gh` calls and the API proxy count against it too. diff --git a/src/docs/integration-guide.md b/src/docs/integration-guide.md index f75112ebe0..5ee54334af 100644 --- a/src/docs/integration-guide.md +++ b/src/docs/integration-guide.md @@ -8,6 +8,7 @@ Use this guide to choose a connection to Hive v5. Configuring an existing integr | Lend your machine's AI coding tool | [Contributor relay: basic setup](contributor-relay.md#basic-setup) | The relay receives work from Hive and reports its result. | | Have an external workflow engine run work | [ClankeR and Flue-style execution](integrations/clanker-flue.md) | Hive still decides what may run and verifies reported evidence; Flue dispatch is report-only. | | Turn a specification into staged work | [Spektacular integration](integrations/spektacular.md) | The planning tool must implement the documented Spektacular CLI contract; there is no general planning-engine registry. | +| Drive agent work through the Crux work services | [Crux work services: integration guide](crux-integration-guide.md) | Pinned checkouts, contained runs, schema-checked agent steps, command log, turn signals, human gates and saved results over the `crux serve` API; the orchestrator decides what the records mean. | In the diagram, the **work source** supplies tasks, the **governor** schedules agents, and a **run lease** records who owns a stage and when it expires. An *additive* source appends tasks alongside the selected primary source rather than replacing it. diff --git a/src/docs/operator-reference/README.md b/src/docs/operator-reference/README.md index 7eb9453b99..fcd00228c0 100644 --- a/src/docs/operator-reference/README.md +++ b/src/docs/operator-reference/README.md @@ -70,7 +70,8 @@ agents: ## GitHub API quota — who is spending it? Every hive exposes `GET /api/gh-rate-limits` behind dashboard auth. The payload -contains GitHub's core/search/GraphQL rate-limit windows, ETag cache counters, +contains GitHub's core/search/GraphQL rate-limit windows, ETag cache counters +and fill level (`etag_cache.hits/misses/entries/bytes/max_bytes/max_entries`), and a one-hour REST top-consumer list grouped by caller, method, and normalized endpoint. Operators can collect a hosted fleet view with: @@ -257,6 +258,8 @@ Go callers keep reading `cfg.Agents`, `cfg.Governor.Gateways` and so on through | `nous.observations` | `enabled: false`, `max_per_week: 5` (minimum `1`). | Opt-in filing of hive-state detector findings through the hub self-report relay. When enabled the detectors run once per eval cycle after the snapshot is recorded; at most `max_per_week` new observations are filed per rolling 7 days (ledger `observations-budget.json` under `data.nous_dir`, survives restarts), drops over budget are logged, and nothing is filed while a self-upgrade is draining. See [experiment-lab.md](../experiment-lab.md#filing-observations). Editable under **Settings → Features → Nous / Experiment Lab**. | | `agents..engine` | `nous` for an agent whose `role` is `experimenter`; empty (and rejected if set) for every other role. Valid: `nous`. | Which experimentation engine (`pkg/nousengine`) an experimenter runs. Nothing else reads the field, so setting it on a strategist or scanner is a load-time error rather than a silent no-op. Editable in the agent dialog (General → Experiment engine). | | `github.pr_detail_ttl_s` | `1800` seconds. `HIVE_GITHUB_PR_DETAIL_TTL` overrides for tests/experiments. | Reuses `GET /pulls/{number}` detail responses while the cheap PR list still reports the same `head.sha` and `updated_at`, and GitHub has resolved `mergeable_state`. | +| `github.etag_cache_max_mb` | `128` MiB. `HIVE_GITHUB_ETAG_CACHE_MAX_MB` overrides the file. Settings → Forge App → **ETag cache size (MB)**; applies without a restart. | Memory cap for the conditional-request (ETag / `304 Not Modified`) response cache every GitHub client in the process shares. Bodies are stored gzip-compressed, so a MiB holds several MiB of JSON. A cache that fills before its entries are revalidated evicts bodies other pollers are about to re-request, and those become charged fetches again — a 16-repo hive filled the former 64 MiB cap within hours and its hourly REST usage crept from ~2.6k back toward the installation limit. `GET /api/gh-rate-limits` → `etag_cache.bytes` / `max_bytes` and the **ETag cache** row under Debug → GitHub API budget show the fill level. `1`–`2048`; `0` restores the default. | +| `github.etag_cache_max_entries` | `8192`. `HIVE_GITHUB_ETAG_CACHE_MAX_ENTRIES` overrides the file. Settings → Forge App → **ETag cache entries**; applies without a restart. | Entry cap for the same cache, whichever bound is hit first evicts least-recently-used bodies. `0` restores the default. | | `github.graphql_pr_batch` | `true` | Uses one paginated GraphQL query per repository scan to populate the PR detail cache and CI rollup cache, replacing most per-PR `GET /pulls/{number}` and `check-runs` reads. Set `false` to return to the REST-only scan path. | | `github.graphql_pr_batch_page_size` | `50` | Page size for the GraphQL `pullRequests(first:)` batch. Values above GitHub's `100` maximum are rejected. | | `github.app_signed_commits` | Unset (off). App-authenticated hives only; PAT-authenticated hives ignore it. Settings → Forge App → **Re-author agent PRs as GitHub-signed commits** (Owner role). | Opt-in for repositories whose ruleset requires signed commits (`required_signatures`). Agents commit with plain `git` in their pane and cannot sign, so without it every agent PR is unmergeable there however many approvals it has. With it on, the PR-request watcher re-authors the head branch before the PR opens as one commit created through the GraphQL `createCommitOnBranch` mutation with the App installation token — same tree, the agents' messages and DCO trailers as the message — which GitHub signs and shows as **Verified**, authored by `[bot]`. A reconcile pass (at most once a minute) re-signs the hive's own unsigned follow-up pushes on open App-authored PRs the same way. Falls back to the plain unsigned push (PR still opens; `signed_skipped` in the request's `.result.json` and a WARN in the hive log) for changes the mutation cannot express (executable bits, symlinks, submodule pointers), more than 20 MiB of content or 300 files, a head that moved mid-rewrite, or a head that already has an open PR. A person's commit or a merge commit in the range stops the reconcile with one `hive:signed-commits-blocked` comment. See [github-app-setup.md](../github-app-setup.md#signed-commits-on-agent-prs-app_signed_commits). | diff --git a/src/docs/operator-reference/runtime-and-quota.md b/src/docs/operator-reference/runtime-and-quota.md index 121071b6ae..0587cf4e56 100644 --- a/src/docs/operator-reference/runtime-and-quota.md +++ b/src/docs/operator-reference/runtime-and-quota.md @@ -181,6 +181,16 @@ loops, so the proxy enforces two safeguards: mode follow that window, not the process's first `GET /rate_limit` reading after a restart. +- `github.etag_cache_max_mb` (default `128`, env `HIVE_GITHUB_ETAG_CACHE_MAX_MB`) + and `github.etag_cache_max_entries` (default `8192`) bound the shared + conditional-request cache that turns repeat `GET`s into free `304`s. When it + is full, every new store evicts a body some poller is about to revalidate, + and that poll is charged again. The **ETag cache** row under Debug → GitHub + API budget (and `etag_cache.bytes` / `max_bytes` in `/api/gh-rate-limits`) + shows the fill level; raising the cap from Settings → Forge App applies at + once. Bodies are stored gzip-compressed, so the cap is several times its + size in raw JSON. + The proxy returns JSON shaped for the GitHub CLI: ```json @@ -783,6 +793,7 @@ curl -s 'http://127.0.0.1:6060/debug/pprof/goroutine?debug=1' | head # parked g |---|---| | `HIVE_GITHUB_TOKEN` | Main PAT fallback when `github.token` is empty; also used for token identity/fleet stats fallback. Wrong-scope tokens fail with generic 403s at request time — see [github-app-setup.md](../github-app-setup.md#personal-access-token-pat-scopes) for the required scopes per ACMM tier. | | `GH_APP_KEY_FILE` | GitHub App private-key path fallback when `github.key_file` is empty. | +| `HIVE_GITHUB_ETAG_CACHE_MAX_MB` / `HIVE_GITHUB_ETAG_CACHE_MAX_ENTRIES` | Size the shared GitHub ETag response cache (defaults `128` MiB compressed / `8192` entries); win over `github.etag_cache_max_mb` / `github.etag_cache_max_entries`. See [GitHub API quota](#github-api-quota). | | `HIVE_DASHBOARD_TOKEN` | Shared dashboard/API token fallback for `dashboard.auth_token`. Any non-empty string is accepted with no strength check — generate one with `openssl rand -hex 32`; see [env-vars.md](../env-vars.md#generating-and-rotating-hive_dashboard_token). | ### Hosted-spoke and fleet metadata diff --git a/src/docs/scale-envelope.md b/src/docs/scale-envelope.md index 27ff1e3aa5..effd14c6aa 100644 --- a/src/docs/scale-envelope.md +++ b/src/docs/scale-envelope.md @@ -34,7 +34,7 @@ ones to: 2. **Truncation must preserve the prioritization signal, not cut an arbitrary slice.** This holds today and is easy to assume backwards: - Issues are sorted by actionable rank tier, then **oldest first** within each tier before any cap is applied - (`RankActionableIssues`, `src/pkg/github/client.go:5480-5490`, descending `AgeMinutes` within each tier). + (`RankActionableIssues`, `src/pkg/github/client.go:5489-5499`, descending `AgeMinutes` within each tier). - PRs are sorted by **review class then oldest-first within class** (`SortPullRequestsForReview`, `src/pkg/github/review_priority.go:334`; fixes → refactors/docs → tests). diff --git a/src/hive.yaml.example b/src/hive.yaml.example index 1a5b6a6079..72be3e60ee 100644 --- a/src/hive.yaml.example +++ b/src/hive.yaml.example @@ -1093,6 +1093,13 @@ github: token: ${HIVE_GITHUB_TOKEN} # Reuse per-PR detail GETs while list head.sha/updated_at still match. pr_detail_ttl_s: 1800 + # Memory for the conditional-request (ETag / 304) response cache, in MiB of + # gzip-compressed bodies (default 128, max 2048), and its entry cap (default + # 8192). A cache that fills within hours evicts bodies other pollers are + # about to revalidate and charged fetches climb; raise it. Env overrides: + # HIVE_GITHUB_ETAG_CACHE_MAX_MB / HIVE_GITHUB_ETAG_CACHE_MAX_ENTRIES. + etag_cache_max_mb: 128 + etag_cache_max_entries: 8192 # Batch open PR detail and CI rollups with GraphQL during scans (default true). graphql_pr_batch: true graphql_pr_batch_page_size: 50 diff --git a/src/pkg/config/forge_config.go b/src/pkg/config/forge_config.go index 227e879baa..6cd6ee0fa9 100644 --- a/src/pkg/config/forge_config.go +++ b/src/pkg/config/forge_config.go @@ -95,6 +95,16 @@ type GitHubConfig struct { APIReserve int `yaml:"api_reserve,omitempty"` // APICritical is the lower core REST quota floor where Hive enters critical mode. APICritical int `yaml:"api_critical,omitempty"` + // ETagCacheMaxMB bounds the memory (MiB of gzip-compressed bodies) the + // shared GitHub conditional-request (ETag / 304) cache may hold. Zero + // uses DefaultGitHubETagCacheMaxMB; HIVE_GITHUB_ETAG_CACHE_MAX_MB + // overrides the file. A full cache evicts bodies other pollers are about + // to revalidate, turning free 304s back into charged fetches. + ETagCacheMaxMB int `yaml:"etag_cache_max_mb,omitempty" json:"etag_cache_max_mb,omitempty"` + // ETagCacheMaxEntries bounds how many responses the ETag cache holds + // regardless of size. Zero uses DefaultGitHubETagCacheMaxEntries; + // HIVE_GITHUB_ETAG_CACHE_MAX_ENTRIES overrides the file. + ETagCacheMaxEntries int `yaml:"etag_cache_max_entries,omitempty" json:"etag_cache_max_entries,omitempty"` // BaseURL is the GitHub web base URL. Defaults to DefaultGitHubBaseURL. // For GitHub Enterprise, set to e.g. "https://github.ibm.com". BaseURL string `yaml:"base_url"` @@ -189,6 +199,52 @@ const ( DefaultGraphQLPRBatchPageSize = 50 ) +// GitHub ETag response cache bounds (github.etag_cache_max_mb / +// github.etag_cache_max_entries). The defaults were raised from 64 MiB / +// 4096 entries: a busy multi-repo hive filled 64 MiB within hours and the +// evictions pushed its hourly REST usage back toward the installation limit. +// Bodies are stored gzip-compressed, so a MiB holds several MiB of JSON. +const ( + DefaultGitHubETagCacheMaxMB = 128 + MaxGitHubETagCacheMaxMB = 2048 + DefaultGitHubETagCacheMaxEntries = 8192 + MaxGitHubETagCacheMaxEntries = 1 << 20 + GitHubETagCacheMaxMBEnv = "HIVE_GITHUB_ETAG_CACHE_MAX_MB" + GitHubETagCacheMaxEntriesEnv = "HIVE_GITHUB_ETAG_CACHE_MAX_ENTRIES" +) + +// EffectiveETagCacheMaxBytes is the ETag cache byte bound in bytes: the +// configured MiB, or the default when unset. +func (g GitHubConfig) EffectiveETagCacheMaxBytes() int { + mb := g.ETagCacheMaxMB + if mb <= 0 { + mb = DefaultGitHubETagCacheMaxMB + } + return mb << 20 +} + +// EffectiveETagCacheMaxEntries is the ETag cache entry bound, or the default +// when unset. +func (g GitHubConfig) EffectiveETagCacheMaxEntries() int { + if g.ETagCacheMaxEntries <= 0 { + return DefaultGitHubETagCacheMaxEntries + } + return g.ETagCacheMaxEntries +} + +// ValidateETagCacheLimits applies the bounds config.Validate enforces at boot +// to a (possibly partial) edit, so the Settings dialog cannot save a value the +// next start would reject. +func ValidateETagCacheLimits(maxMB, maxEntries int) error { + if maxMB < 0 || maxMB > MaxGitHubETagCacheMaxMB { + return fmt.Errorf("github.etag_cache_max_mb must be between 1 and %d, or 0 for the default (%d)", MaxGitHubETagCacheMaxMB, DefaultGitHubETagCacheMaxMB) + } + if maxEntries < 0 || maxEntries > MaxGitHubETagCacheMaxEntries { + return fmt.Errorf("github.etag_cache_max_entries must be between 1 and %d, or 0 for the default (%d)", MaxGitHubETagCacheMaxEntries, DefaultGitHubETagCacheMaxEntries) + } + return nil +} + func (g GitHubConfig) PRDetailTTL() time.Duration { if raw := strings.TrimSpace(os.Getenv(GitHubPRDetailTTLEnv)); raw != "" { if d, err := time.ParseDuration(raw); err == nil && d >= 0 { diff --git a/src/pkg/config/github_etag_cache_test.go b/src/pkg/config/github_etag_cache_test.go new file mode 100644 index 0000000000..e3dcc66a09 --- /dev/null +++ b/src/pkg/config/github_etag_cache_test.go @@ -0,0 +1,123 @@ +package config + +import ( + "strings" + "testing" +) + +const etagCacheTestYAML = ` +project: + org: my-org + repos: [repo-a] +github: + token: ghp_tok +agents: + w: + backend: claude +` + +func TestGitHubETagCacheLimitsDefaults(t *testing.T) { + cfg, err := Load(writeTempConfig(t, etagCacheTestYAML)) + if err != nil { + t.Fatalf("Load() error = %v", err) + } + if cfg.GitHub.ETagCacheMaxMB != 0 || cfg.GitHub.ETagCacheMaxEntries != 0 { + t.Fatalf("unset etag cache bounds must stay 0 in the file, got mb=%d entries=%d", cfg.GitHub.ETagCacheMaxMB, cfg.GitHub.ETagCacheMaxEntries) + } + if got := cfg.GitHub.EffectiveETagCacheMaxBytes(); got != DefaultGitHubETagCacheMaxMB<<20 { + t.Fatalf("EffectiveETagCacheMaxBytes() = %d, want %d MiB", got, DefaultGitHubETagCacheMaxMB) + } + if got := cfg.GitHub.EffectiveETagCacheMaxEntries(); got != DefaultGitHubETagCacheMaxEntries { + t.Fatalf("EffectiveETagCacheMaxEntries() = %d, want %d", got, DefaultGitHubETagCacheMaxEntries) + } + if got := (GitHubConfig{ETagCacheMaxMB: 256, ETagCacheMaxEntries: 20000}).EffectiveETagCacheMaxBytes(); got != 256<<20 { + t.Fatalf("EffectiveETagCacheMaxBytes(256) = %d", got) + } + if got := (GitHubConfig{ETagCacheMaxEntries: 20000}).EffectiveETagCacheMaxEntries(); got != 20000 { + t.Fatalf("EffectiveETagCacheMaxEntries(20000) = %d", got) + } +} + +func TestGitHubETagCacheLimitsYAML(t *testing.T) { + cfg, err := Load(writeTempConfig(t, strings.Replace(etagCacheTestYAML, " token: ghp_tok\n", " token: ghp_tok\n etag_cache_max_mb: 256\n etag_cache_max_entries: 16384\n", 1))) + if err != nil { + t.Fatalf("Load() error = %v", err) + } + if cfg.GitHub.ETagCacheMaxMB != 256 || cfg.GitHub.ETagCacheMaxEntries != 16384 { + t.Fatalf("yaml etag cache bounds not read: mb=%d entries=%d", cfg.GitHub.ETagCacheMaxMB, cfg.GitHub.ETagCacheMaxEntries) + } +} + +// HIVE_GITHUB_ETAG_CACHE_MAX_MB is how an operator sizes the cache per +// deployment without editing the file; it wins over the YAML value. +func TestGitHubETagCacheLimitsEnvOverride(t *testing.T) { + t.Setenv(GitHubETagCacheMaxMBEnv, "512") + t.Setenv(GitHubETagCacheMaxEntriesEnv, "32768") + cfg, err := Load(writeTempConfig(t, strings.Replace(etagCacheTestYAML, " token: ghp_tok\n", " token: ghp_tok\n etag_cache_max_mb: 256\n", 1))) + if err != nil { + t.Fatalf("Load() error = %v", err) + } + if cfg.GitHub.ETagCacheMaxMB != 512 { + t.Fatalf("%s=512 did not override etag_cache_max_mb: %d", GitHubETagCacheMaxMBEnv, cfg.GitHub.ETagCacheMaxMB) + } + if cfg.GitHub.ETagCacheMaxEntries != 32768 { + t.Fatalf("%s=32768 not applied: %d", GitHubETagCacheMaxEntriesEnv, cfg.GitHub.ETagCacheMaxEntries) + } + + // A malformed value is ignored rather than failing the boot. + t.Setenv(GitHubETagCacheMaxMBEnv, "lots") + t.Setenv(GitHubETagCacheMaxEntriesEnv, "") + cfg, err = Load(writeTempConfig(t, etagCacheTestYAML)) + if err != nil { + t.Fatalf("Load() with malformed env: %v", err) + } + if cfg.GitHub.ETagCacheMaxMB != 0 { + t.Fatalf("malformed %s should be ignored, got %d", GitHubETagCacheMaxMBEnv, cfg.GitHub.ETagCacheMaxMB) + } + + // An out-of-range env value is rejected exactly like a file value. + t.Setenv(GitHubETagCacheMaxMBEnv, "4096") + if _, err := Load(writeTempConfig(t, etagCacheTestYAML)); err == nil || !strings.Contains(err.Error(), "etag_cache_max_mb") { + t.Fatalf("out-of-range env value should fail validation, got %v", err) + } +} + +func TestGitHubETagCacheLimitsValidation(t *testing.T) { + for _, tc := range []struct { + mb, entries int + wantErr string + }{ + {0, 0, ""}, + {1, 1, ""}, + {MaxGitHubETagCacheMaxMB, MaxGitHubETagCacheMaxEntries, ""}, + {-1, 0, "etag_cache_max_mb"}, + {MaxGitHubETagCacheMaxMB + 1, 0, "etag_cache_max_mb"}, + {0, -1, "etag_cache_max_entries"}, + {0, MaxGitHubETagCacheMaxEntries + 1, "etag_cache_max_entries"}, + } { + err := ValidateETagCacheLimits(tc.mb, tc.entries) + if tc.wantErr == "" { + if err != nil { + t.Errorf("ValidateETagCacheLimits(%d, %d) = %v, want nil", tc.mb, tc.entries, err) + } + continue + } + if err == nil || !strings.Contains(err.Error(), tc.wantErr) { + t.Errorf("ValidateETagCacheLimits(%d, %d) = %v, want error mentioning %s", tc.mb, tc.entries, err, tc.wantErr) + } + } + + cfg, err := Load(writeTempConfig(t, etagCacheTestYAML)) + if err != nil { + t.Fatalf("Load() error = %v", err) + } + cfg.GitHub.ETagCacheMaxMB = -5 + if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "github.etag_cache_max_mb") { + t.Fatalf("Validate() with negative etag_cache_max_mb = %v", err) + } + cfg.GitHub.ETagCacheMaxMB = 0 + cfg.GitHub.ETagCacheMaxEntries = MaxGitHubETagCacheMaxEntries + 1 + if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "github.etag_cache_max_entries") { + t.Fatalf("Validate() with oversized etag_cache_max_entries = %v", err) + } +} diff --git a/src/pkg/config/load.go b/src/pkg/config/load.go index 3dd1b4657f..c09dce089b 100644 --- a/src/pkg/config/load.go +++ b/src/pkg/config/load.go @@ -444,6 +444,19 @@ func (c *Config) applyBootstrapEnv() { if v := strings.TrimSpace(os.Getenv(ContributeSkipLabelsEnvVar)); v != "" { c.Hub.ContributeSkipLabels = parseContributeSkipLabels(v) } + // Operators size the ETag cache per deployment from the environment; a + // value that fails to parse is ignored so a typo cannot stop the boot, + // and an out-of-range one is caught by Validate like a file value. + if v := strings.TrimSpace(os.Getenv(GitHubETagCacheMaxMBEnv)); v != "" { + if n, err := strconv.Atoi(v); err == nil { + c.GitHub.ETagCacheMaxMB = n + } + } + if v := strings.TrimSpace(os.Getenv(GitHubETagCacheMaxEntriesEnv)); v != "" { + if n, err := strconv.Atoi(v); err == nil { + c.GitHub.ETagCacheMaxEntries = n + } + } } func readDashboardAuthTokenFile() string { diff --git a/src/pkg/config/validate.go b/src/pkg/config/validate.go index 25dd7c9ee1..b31d32f6f2 100644 --- a/src/pkg/config/validate.go +++ b/src/pkg/config/validate.go @@ -159,6 +159,9 @@ func (c *Config) ValidateWithOptions(opts ValidateOptions) error { if c.GitHub.GraphQLPRBatchPageSize < 0 || c.GitHub.GraphQLPRBatchPageSize > 100 { return fmt.Errorf("github.graphql_pr_batch_page_size must be between 1 and 100, or 0 for the default") } + if err := ValidateETagCacheLimits(c.GitHub.ETagCacheMaxMB, c.GitHub.ETagCacheMaxEntries); err != nil { + return err + } if !c.GitHub.Notifications.Valid() { return fmt.Errorf("github.notifications.mode must be %q or %q", NotificationsModeQuiet, NotificationsModeNormal) } diff --git a/src/pkg/dashboard/api_config.go b/src/pkg/dashboard/api_config.go index c93dd730ee..2c90f31b98 100644 --- a/src/pkg/dashboard/api_config.go +++ b/src/pkg/dashboard/api_config.go @@ -121,6 +121,8 @@ type githubAPISettings struct { AgentReserveFloor *int `json:"agent_reserve_floor"` APIReserve *int `json:"api_reserve"` APICritical *int `json:"api_critical"` + ETagCacheMaxMB *int `json:"etag_cache_max_mb"` + ETagCacheMaxEntries *int `json:"etag_cache_max_entries"` PRDetailTTLS *int `json:"pr_detail_ttl_s"` GraphQLPRBatch *bool `json:"graphql_pr_batch"` GraphQLPRBatchPageSize *int `json:"graphql_pr_batch_page_size"` @@ -130,6 +132,7 @@ type githubAPISettings struct { func (b githubAPISettings) empty() bool { return b.AgentReserveFloor == nil && b.APIReserve == nil && b.APICritical == nil && b.PRDetailTTLS == nil && + b.ETagCacheMaxMB == nil && b.ETagCacheMaxEntries == nil && b.GraphQLPRBatch == nil && b.GraphQLPRBatchPageSize == nil && b.AppAuthoredPRs == nil && b.AppSignedCommits == nil } @@ -152,6 +155,18 @@ func (b githubAPISettings) validate(cur config.GitHubConfig) error { if b.GraphQLPRBatchPageSize != nil && *b.GraphQLPRBatchPageSize > 100 { return fmt.Errorf("graphql_pr_batch_page_size must be between 1 and 100, or 0 for the default") } + if b.ETagCacheMaxMB != nil || b.ETagCacheMaxEntries != nil { + mb, entries := cur.ETagCacheMaxMB, cur.ETagCacheMaxEntries + if b.ETagCacheMaxMB != nil { + mb = *b.ETagCacheMaxMB + } + if b.ETagCacheMaxEntries != nil { + entries = *b.ETagCacheMaxEntries + } + if err := config.ValidateETagCacheLimits(mb, entries); err != nil { + return fmt.Errorf("%s", strings.TrimPrefix(err.Error(), "github.")) + } + } for _, f := range []struct { v *int name string @@ -201,6 +216,12 @@ func (b githubAPISettings) apply(g *config.GitHubConfig) { g.APICritical = config.DefaultGitHubAPICritical } } + if b.ETagCacheMaxMB != nil { + g.ETagCacheMaxMB = *b.ETagCacheMaxMB + } + if b.ETagCacheMaxEntries != nil { + g.ETagCacheMaxEntries = *b.ETagCacheMaxEntries + } if b.PRDetailTTLS != nil { g.PRDetailTTLS = *b.PRDetailTTLS } @@ -251,7 +272,7 @@ func (s *Server) handleConfigGitHub(w http.ResponseWriter, r *http.Request) { } if body.AppID == nil && body.InstallationID == nil && body.KeyFile == "" && body.PrivateKey == "" && body.SelfAuthorizationHold == nil && body.githubAPISettings.empty() { - jsonError(w, "at least one field required: app_id, installation_id, key_file, private_key, self_authorization_hold, agent_reserve_floor, api_reserve, api_critical, pr_detail_ttl_s, graphql_pr_batch, graphql_pr_batch_page_size, app_authored_prs, app_signed_commits", http.StatusBadRequest) + jsonError(w, "at least one field required: app_id, installation_id, key_file, private_key, self_authorization_hold, agent_reserve_floor, api_reserve, api_critical, etag_cache_max_mb, etag_cache_max_entries, pr_detail_ttl_s, graphql_pr_batch, graphql_pr_batch_page_size, app_authored_prs, app_signed_commits", http.StatusBadRequest) return } @@ -330,6 +351,12 @@ func (s *Server) applyGitHubConfigUpdate(r *http.Request, body githubConfigUpdat } body.githubAPISettings.apply(g) }) + // The ETag cache is process-wide and reads no config of its own, so a + // resize is pushed to it here; shrinking evicts immediately, growing + // stops evictions on the next store — no restart either way. + if body.ETagCacheMaxMB != nil || body.ETagCacheMaxEntries != nil { + github.SetETagCacheLimits(cfg.GitHub.EffectiveETagCacheMaxBytes(), cfg.GitHub.EffectiveETagCacheMaxEntries()) + } result, err := s.finishGitHubConfigUpdateLocked(requestAuditUser(r), "") if err != nil { diff --git a/src/pkg/dashboard/forge_apps.go b/src/pkg/dashboard/forge_apps.go index fa5edae041..c4e0070c9d 100644 --- a/src/pkg/dashboard/forge_apps.go +++ b/src/pkg/dashboard/forge_apps.go @@ -87,6 +87,8 @@ type forgeGitHubSettings struct { AgentReserveFloor int `json:"agent_reserve_floor"` APIReserve int `json:"api_reserve"` APICritical int `json:"api_critical"` + ETagCacheMaxMB int `json:"etag_cache_max_mb"` + ETagCacheMaxEntries int `json:"etag_cache_max_entries"` PRDetailTTLS int `json:"pr_detail_ttl_s"` GraphQLPRBatch bool `json:"graphql_pr_batch"` GraphQLPRBatchPageSize int `json:"graphql_pr_batch_page_size"` @@ -108,6 +110,8 @@ func forgeGitHubSettingsFor(cfg *config.Config, hubManaged bool) forgeGitHubSett AgentReserveFloor: cfg.GitHubAgentReserveFloor(), APIReserve: g.APIReserve, APICritical: g.APICritical, + ETagCacheMaxMB: g.ETagCacheMaxMB, + ETagCacheMaxEntries: g.ETagCacheMaxEntries, PRDetailTTLS: g.PRDetailTTLS, GraphQLPRBatch: g.GraphQLPRBatchEnabled(), GraphQLPRBatchPageSize: g.GraphQLPRBatchPageSize, @@ -115,6 +119,8 @@ func forgeGitHubSettingsFor(cfg *config.Config, hubManaged bool) forgeGitHubSett AppSignedCommits: g.AppSignedCommitsEnabled(), Defaults: map[string]int{ "agent_reserve_floor": config.DefaultGitHubAgentReserveFloor, + "etag_cache_max_mb": config.DefaultGitHubETagCacheMaxMB, + "etag_cache_max_entries": config.DefaultGitHubETagCacheMaxEntries, "pr_detail_ttl_s": int(config.DefaultGitHubPRDetailTTL / time.Second), "graphql_pr_batch_page_size": config.DefaultGraphQLPRBatchPageSize, }, diff --git a/src/pkg/dashboard/github_settings_parity_test.go b/src/pkg/dashboard/github_settings_parity_test.go index 47d083c322..ac31266870 100644 --- a/src/pkg/dashboard/github_settings_parity_test.go +++ b/src/pkg/dashboard/github_settings_parity_test.go @@ -7,6 +7,9 @@ import ( "strings" "testing" "time" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/github" ) // The github.* quota, scan and authorship keys round-trip through @@ -27,6 +30,10 @@ func TestConfigGitHubParityBatch6RoundTrip(t *testing.T) { {"graphql_pr_batch_page_size": -1}, {"graphql_pr_batch_page_size": 101}, {"api_reserve": -1}, + {"etag_cache_max_mb": -1}, + {"etag_cache_max_mb": 2049}, + {"etag_cache_max_entries": -1}, + {"etag_cache_max_entries": 1 << 21}, {"api_critical": 900}, {"api_reserve": 200}, {"api_reserve": 100, "api_critical": 100}, @@ -36,14 +43,19 @@ func TestConfigGitHubParityBatch6RoundTrip(t *testing.T) { t.Fatalf("%v: expected 400, got %d — %s", bad, rec.Code, rec.Body.String()) } } - if g := s.deps.Config.GitHub; g.APIReserve != before.APIReserve || g.APICritical != before.APICritical || g.AppSignedCommits != nil { + if g := s.deps.Config.GitHub; g.APIReserve != before.APIReserve || g.APICritical != before.APICritical || g.AppSignedCommits != nil || + g.ETagCacheMaxMB != 0 || g.ETagCacheMaxEntries != 0 { t.Fatal("a rejected PUT changed the config") } + _, maxBytes0, maxEntries0 := github.ETagCacheUsage() + t.Cleanup(func() { github.SetETagCacheLimits(maxBytes0, maxEntries0) }) if rec := doPut(s, "/api/config/github", map[string]any{ "agent_reserve_floor": 500, "api_reserve": 1000, "api_critical": 300, + "etag_cache_max_mb": 256, + "etag_cache_max_entries": 16384, "pr_detail_ttl_s": 600, "graphql_pr_batch": false, "graphql_pr_batch_page_size": 25, @@ -54,9 +66,24 @@ func TestConfigGitHubParityBatch6RoundTrip(t *testing.T) { } g := s.deps.Config.GitHub if g.AgentReserveFloor != 500 || g.APIReserve != 1000 || g.APICritical != 300 || g.PRDetailTTLS != 600 || + g.ETagCacheMaxMB != 256 || g.ETagCacheMaxEntries != 16384 || g.GraphQLPRBatchEnabled() || g.GraphQLPRBatchPageSize != 25 || g.AppAuthoredPRsEnabled() || !g.AppSignedCommitsEnabled() { t.Fatalf("github settings not applied: %+v", g) } + // The ETag cache resize applies to the live process-wide cache at once. + if _, maxBytes, maxEntries := github.ETagCacheUsage(); maxBytes != 256<<20 || maxEntries != 16384 { + t.Fatalf("etag cache limits not applied live: %d bytes / %d entries", maxBytes, maxEntries) + } + // Clearing restores the built-in defaults on the live cache too. + if rec := doPut(s, "/api/config/github", map[string]any{"etag_cache_max_mb": 0, "etag_cache_max_entries": 0}); rec.Code != http.StatusOK { + t.Fatalf("clearing etag cache bounds: %d — %s", rec.Code, rec.Body.String()) + } + if _, maxBytes, maxEntries := github.ETagCacheUsage(); maxBytes != config.DefaultGitHubETagCacheMaxMB<<20 || maxEntries != config.DefaultGitHubETagCacheMaxEntries { + t.Fatalf("cleared etag cache bounds did not restore defaults: %d bytes / %d entries", maxBytes, maxEntries) + } + if rec := doPut(s, "/api/config/github", map[string]any{"etag_cache_max_mb": 256, "etag_cache_max_entries": 16384}); rec.Code != http.StatusOK { + t.Fatalf("restore etag cache bounds: %d — %s", rec.Code, rec.Body.String()) + } // Clearing a quota field restores the default, as config.Validate and // applyDefaults treat 0 at boot; the bound is checked against the default. @@ -100,6 +127,7 @@ func TestConfigGitHubParityBatch6RoundTrip(t *testing.T) { } st := got.Settings if !st.TokenSet || st.AgentReserveFloor != 500 || st.APIReserve != 1000 || st.APICritical != 300 || st.PRDetailTTLS != 600 || + st.ETagCacheMaxMB != 256 || st.ETagCacheMaxEntries != 16384 || st.Defaults["etag_cache_max_mb"] != config.DefaultGitHubETagCacheMaxMB || !st.GraphQLPRBatch || st.GraphQLPRBatchPageSize != 25 || st.AppAuthoredPRs || !st.AppSignedCommits { t.Fatalf("forge-apps settings: %+v", st) } @@ -202,6 +230,8 @@ func TestParityBatch6SettingsRenderEveryKey(t *testing.T) { {"github.api_critical", `num('api_critical'`}, {"github.agent_reserve_floor", `num('agent_reserve_floor'`}, {"github.pr_detail_ttl_s", `num('pr_detail_ttl_s'`}, + {"github.etag_cache_max_mb", `num('etag_cache_max_mb'`}, + {"github.etag_cache_max_entries", `num('etag_cache_max_entries'`}, {"github.graphql_pr_batch_page_size", `num('graphql_pr_batch_page_size'`}, {"github.graphql_pr_batch", `toggle('graphql_pr_batch'`}, {"github.app_authored_prs", `toggle('app_authored_prs'`}, diff --git a/src/pkg/dashboard/rate_limit_static_test.go b/src/pkg/dashboard/rate_limit_static_test.go index d8b76f0512..29b1dfa8eb 100644 --- a/src/pkg/dashboard/rate_limit_static_test.go +++ b/src/pkg/dashboard/rate_limit_static_test.go @@ -69,6 +69,9 @@ func TestDashboardGitHubAPIBudgetRowsHaveHelpMarks(t *testing.T) { "const GH_API_BUDGET_HELP = {", "Core remaining${ghApiBudgetHelpMark('core', 'Core remaining')}", "ETag hit rate${ghApiBudgetHelpMark('etag', 'ETag hit rate')}", + "ETag cache${ghApiBudgetHelpMark('etagFill', 'ETag cache')}${ghEtagCacheFillText(etag)}", + "function ghEtagCacheFillText(etag)", + "github.etag_cache_max_mb, default 128 MB", `id="${id}" class="config-tooltip" role="tooltip">${esc(GH_API_BUDGET_HELP[key] || '')}`, "answered 304 Not Modified because the hive re-sent a remembered ETag (If-None-Match)", "GitHub does not charge a 304 against the core rate limit", diff --git a/src/pkg/dashboard/static-src/js/config-state.js b/src/pkg/dashboard/static-src/js/config-state.js index c369d36cd1..b222ff690f 100644 --- a/src/pkg/dashboard/static-src/js/config-state.js +++ b/src/pkg/dashboard/static-src/js/config-state.js @@ -130,6 +130,6 @@ { tab: 'Knowledge', section: 'Knowledge Integrations', label: 'Context7 connector', key: 'knowledge.connectors', lazy: true, text: 'context7 connector api key env file reference library ids topic validate integrations knowledge.connectors' }, { tab: 'Knowledge', section: 'Knowledge Integrations', label: 'Context7 auto-import', key: 'knowledge.context7.auto_import', lazy: true, text: 'context7 auto import startup libraries knowledge.context7.auto_import[].id knowledge.context7.auto_import[].query' }, { tab: 'Knowledge', section: 'Knowledge pipeline', label: 'Bead synthesizer, curator, code maps and primer', key: 'knowledge.bead_synthesizer.enabled', lazy: true, text: 'knowledge pipeline bead synthesizer wiki facts schedule hourly daily confidence target layer facts per cycle api calls vault retention archive beads high priority dependents curator auto promote threshold promotion from to code maps repository layer max bytes primer facts merge strategy engine public read-only mcp knowledge.engine knowledge.public.enabled knowledge.bead_synthesizer.enabled knowledge.bead_synthesizer.schedule knowledge.bead_synthesizer.min_confidence knowledge.bead_synthesizer.target_layer knowledge.bead_synthesizer.max_facts_per_cycle knowledge.bead_synthesizer.max_api_calls_per_cycle knowledge.bead_synthesizer.vault_path publish to repository knowledge.bead_synthesizer.publish.enabled knowledge.bead_synthesizer.publish.repo knowledge.bead_synthesizer.publish.path knowledge.bead_synthesizer.publish.base_branch knowledge.bead_synthesizer.publish.mode knowledge.bead_synthesizer.publish.cadence knowledge.bead_synthesizer.retention_policy.max_beads knowledge.bead_synthesizer.retention_policy.archive_after_synth_days knowledge.bead_synthesizer.retention_policy.high_priority_retain_days knowledge.bead_synthesizer.retention_policy.preserve_with_deps knowledge.curator.enabled knowledge.curator.schedule knowledge.curator.auto_promote_threshold knowledge.curator.promote_from knowledge.curator.promote_to knowledge.code_maps.enabled knowledge.code_maps.schedule knowledge.code_maps.layer knowledge.code_maps.max_bytes knowledge.code_maps.vault_path knowledge.primer.max_facts knowledge.primer.max_bytes knowledge.primer.merge_strategy' }, - { tab: 'Forge App', section: 'Forge App', label: 'GitHub API quota, PR scans and authorship', key: 'github.api_reserve', lazy: true, text: 'github api quota reserve critical agent reserve floor pr detail cache ttl graphql batch page size app authored prs bot signed commits forge api url docs installation oauth client token github.api_reserve github.api_critical github.agent_reserve_floor github.pr_detail_ttl_s github.graphql_pr_batch github.graphql_pr_batch_page_size github.app_authored_prs github.app_signed_commits github.forge github.api_url github.docs_installation_id github.oauth_client_id github.token github.key_file' }, + { tab: 'Forge App', section: 'Forge App', label: 'GitHub API quota, PR scans and authorship', key: 'github.api_reserve', lazy: true, text: 'github api quota reserve critical agent reserve floor pr detail cache ttl etag cache size mb entries conditional request 304 graphql batch page size app authored prs bot signed commits forge api url docs installation oauth client token github.api_reserve github.api_critical github.agent_reserve_floor github.etag_cache_max_mb github.etag_cache_max_entries github.pr_detail_ttl_s github.graphql_pr_batch github.graphql_pr_batch_page_size github.app_authored_prs github.app_signed_commits github.forge github.api_url github.docs_installation_id github.oauth_client_id github.token github.key_file' }, ]; let _settingsSearch = { active: false, query: '', previousTab: '', index: null, bound: false }; diff --git a/src/pkg/dashboard/static-src/js/config-tabs.js b/src/pkg/dashboard/static-src/js/config-tabs.js index 9b9608e828..b12d8bc918 100644 --- a/src/pkg/dashboard/static-src/js/config-tabs.js +++ b/src/pkg/dashboard/static-src/js/config-tabs.js @@ -1899,6 +1899,10 @@ ${num('pr_detail_ttl_s', 'PR detail cache TTL (seconds)', 'github.pr_detail_ttl_s', d.pr_detail_ttl_s || 1800, 'Reuses a PR detail response while the PR list still reports the same head SHA and updated_at. 0 uses the default.', false, settingHelpMark('operator-reference/README', 'notable-fields', 'PR detail cache TTL', 'How long a GET /pulls/{number} detail response is reused while the cheap PR list still reports the same head SHA and updated_at and GitHub has resolved mergeable_state. 0 uses the default.'))} ${num('graphql_pr_batch_page_size', 'GraphQL PR batch page size', 'github.graphql_pr_batch_page_size', d.graphql_pr_batch_page_size || 50, 'Page size for the batched pullRequests query, 1–100. 0 uses the default.', false, settingHelpMark('operator-reference/README', 'notable-fields', 'GraphQL PR batch page size', 'Page size for the GraphQL pullRequests(first:) batch query. Values above GitHub's maximum of 100 are rejected; 0 uses the default.'))} +
+ ${num('etag_cache_max_mb', 'ETag cache size (MB)', 'github.etag_cache_max_mb', d.etag_cache_max_mb || 128, 'Memory for remembered GitHub responses so repeat GETs can be answered by a free 304. Bodies are gzip-compressed (about 7× effective). A cache that fills within hours evicts bodies other pollers are about to revalidate and charged fetches climb; raise it. 0 uses the default (128); up to 2048.', false, settingHelpMark('operator-reference/README', 'notable-fields', 'ETag cache size (MB)', 'Upper bound on the gzip-compressed bodies the GitHub conditional-request cache keeps in memory. Each remembered body lets the next GET of the same resource be answered 304 Not Modified, which GitHub does not charge. Watch the fill level under Debug → GitHub API budget: at the cap, evictions turn free revalidations back into charged fetches. 0 restores the default (128 MB); the maximum is 2048. HIVE_GITHUB_ETAG_CACHE_MAX_MB overrides the file.'))} + ${num('etag_cache_max_entries', 'ETag cache entries', 'github.etag_cache_max_entries', d.etag_cache_max_entries || 8192, 'Maximum number of remembered responses regardless of size. 0 uses the default (8192).', false, settingHelpMark('operator-reference/README', 'notable-fields', 'ETag cache entries', 'How many GitHub responses the conditional-request cache holds at most, whatever their size. A multi-repo hive with many small list pages can hit this before the MB bound. 0 restores the default (8192). HIVE_GITHUB_ETAG_CACHE_MAX_ENTRIES overrides the file.'))} +
${toggle('graphql_pr_batch', 'Batch open-PR scans with GraphQL', 'github.graphql_pr_batch', 'one paginated query per repo instead of per-PR detail and check-run reads.', false, settingHelpMark('operator-reference/README', 'notable-fields', 'Batch open-PR scans with GraphQL', 'On by default. One paginated GraphQL query per repository scan fills the PR detail and CI rollup caches, replacing most per-PR detail and check-run reads. Off returns to the REST-only scan path.'))} ${toggle('app_authored_prs', 'Author agent PRs as the App bot', 'github.app_authored_prs', 'agents open PRs and commit as <slug>[bot] with the App installation token when project.ai_author is empty.', true, settingHelpMark('github-app-setup', '', 'Author agent PRs as the App bot', 'With a GitHub App installed, agents open PRs and commit as <slug>[bot] using the installation token whenever project.ai_author is empty, so PR authorship is the App rather than a personal account.'))} ${toggle('app_signed_commits', 'Re-author agent PRs as GitHub-signed commits', 'github.app_signed_commits', 'needed when a base branch requires signed commits; rewrites the agent branch before the PR opens.', false, settingHelpMark('github-app-setup', 'signed-commits-on-agent-prs-app_signed_commits', 'Re-author agent PRs as GitHub-signed commits', 'Opt-in; needs an installed App. Before a PR opens, the watcher squashes the agent branch into one commit created through the GraphQL createCommitOnBranch mutation, which GitHub signs and shows as Verified, so a required_signatures ruleset accepts it. Unsigned follow-up commits are reconciled the same way.'))} @@ -1915,7 +1919,7 @@ async function saveForgeGitHubSettings() { const st = _forgeGitHubSettings || {}; const body = {}; - for (const key of ['api_reserve', 'api_critical', 'agent_reserve_floor', 'pr_detail_ttl_s', 'graphql_pr_batch_page_size']) { + for (const key of ['api_reserve', 'api_critical', 'agent_reserve_floor', 'pr_detail_ttl_s', 'graphql_pr_batch_page_size', 'etag_cache_max_mb', 'etag_cache_max_entries']) { const el = document.getElementById('forge-gh-' + key); if (!el) continue; const raw = (el.value || '').trim(); diff --git a/src/pkg/dashboard/static-src/js/debug-logs.js b/src/pkg/dashboard/static-src/js/debug-logs.js index 819e37b69e..54e66603c6 100644 --- a/src/pkg/dashboard/static-src/js/debug-logs.js +++ b/src/pkg/dashboard/static-src/js/debug-logs.js @@ -131,7 +131,18 @@ const GH_API_BUDGET_HELP = { core: 'Requests left in GitHub’s core REST rate limit for the current hour, out of the hourly limit for this credential (5,000 for a token; App installations may get more). GitHub resets the counter once an hour. The bar turns yellow at 25% free and red at 10%. Below the configured API reserve (default 800 requests) the governor enters conserve mode — it stretches the eval interval and skips optional steps such as recommendations and advisories; below the API critical floor (default 250) it stretches the interval further.', etag: 'Share of cacheable GitHub REST GETs answered 304 Not Modified because the hive re-sent a remembered ETag (If-None-Match). GitHub does not charge a 304 against the core rate limit, so a higher rate means more of the hive’s polling is free. Counted since the process started, not over the 1h window — a low rate right after a restart is normal while the cache warms. The 304 column below shows the same hits per caller.', + etagFill: 'How full the ETag response cache is: remembered responses held, and gzip-compressed bytes against the configured cap (Settings → Forge App → ETag cache size, github.etag_cache_max_mb, default 128 MB). At the cap, each new store evicts the least recently used body, and a body evicted before its next poll costs a charged fetch instead of a free 304 — if the hit rate slides while this sits at the cap, raise the cap.', }; + function ghEtagCacheFillText(etag) { + etag = etag || {}; + const entries = Number(etag.entries) || 0; + const bytes = Number(etag.bytes) || 0; + const maxBytes = Number(etag.max_bytes || etag.maxBytes) || 0; + const MB = 1024 * 1024; + const mb = v => (v / MB).toFixed(v >= 100 * MB ? 0 : 1); + const size = maxBytes > 0 ? `${mb(bytes)} / ${mb(maxBytes)} MB` : `${mb(bytes)} MB`; + return `${formatNumber(entries)} entries · ${size}`; + } function ghApiBudgetHelpMark(key, label) { const id = 'gh-api-budget-' + key + '-tip'; return `ⓘ${esc(GH_API_BUDGET_HELP[key] || '')}`; @@ -157,6 +168,7 @@
GitHub API budget (1h)
Core remaining${ghApiBudgetHelpMark('core', 'Core remaining')}${core.remaining} / ${core.limit || '—'} (${core.remainingPct}% free)
ETag hit rate${ghApiBudgetHelpMark('etag', 'ETag hit rate')}${etagPct}%
+
ETag cache${ghApiBudgetHelpMark('etagFill', 'ETag cache')}${ghEtagCacheFillText(etag)}
${bar}

Scroll horizontally for more columns.

diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index df7ebc9bcd..f0837c9c4c 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -51041,7 +51041,18 @@

GitHub Projects sync

const GH_API_BUDGET_HELP = { core: 'Requests left in GitHub’s core REST rate limit for the current hour, out of the hourly limit for this credential (5,000 for a token; App installations may get more). GitHub resets the counter once an hour. The bar turns yellow at 25% free and red at 10%. Below the configured API reserve (default 800 requests) the governor enters conserve mode — it stretches the eval interval and skips optional steps such as recommendations and advisories; below the API critical floor (default 250) it stretches the interval further.', etag: 'Share of cacheable GitHub REST GETs answered 304 Not Modified because the hive re-sent a remembered ETag (If-None-Match). GitHub does not charge a 304 against the core rate limit, so a higher rate means more of the hive’s polling is free. Counted since the process started, not over the 1h window — a low rate right after a restart is normal while the cache warms. The 304 column below shows the same hits per caller.', + etagFill: 'How full the ETag response cache is: remembered responses held, and gzip-compressed bytes against the configured cap (Settings → Forge App → ETag cache size, github.etag_cache_max_mb, default 128 MB). At the cap, each new store evicts the least recently used body, and a body evicted before its next poll costs a charged fetch instead of a free 304 — if the hit rate slides while this sits at the cap, raise the cap.', }; + function ghEtagCacheFillText(etag) { + etag = etag || {}; + const entries = Number(etag.entries) || 0; + const bytes = Number(etag.bytes) || 0; + const maxBytes = Number(etag.max_bytes || etag.maxBytes) || 0; + const MB = 1024 * 1024; + const mb = v => (v / MB).toFixed(v >= 100 * MB ? 0 : 1); + const size = maxBytes > 0 ? `${mb(bytes)} / ${mb(maxBytes)} MB` : `${mb(bytes)} MB`; + return `${formatNumber(entries)} entries · ${size}`; + } function ghApiBudgetHelpMark(key, label) { const id = 'gh-api-budget-' + key + '-tip'; return `ⓘ${esc(GH_API_BUDGET_HELP[key] || '')}`; @@ -51067,6 +51078,7 @@

GitHub Projects sync

GitHub API budget (1h)
Core remaining${ghApiBudgetHelpMark('core', 'Core remaining')}${core.remaining} / ${core.limit || '—'} (${core.remainingPct}% free)
ETag hit rate${ghApiBudgetHelpMark('etag', 'ETag hit rate')}${etagPct}%
+
ETag cache${ghApiBudgetHelpMark('etagFill', 'ETag cache')}${ghEtagCacheFillText(etag)}
${bar}

Scroll horizontally for more columns.

@@ -56528,7 +56540,7 @@

Configure LiteLLM

{ tab: 'Knowledge', section: 'Knowledge Integrations', label: 'Context7 connector', key: 'knowledge.connectors', lazy: true, text: 'context7 connector api key env file reference library ids topic validate integrations knowledge.connectors' }, { tab: 'Knowledge', section: 'Knowledge Integrations', label: 'Context7 auto-import', key: 'knowledge.context7.auto_import', lazy: true, text: 'context7 auto import startup libraries knowledge.context7.auto_import[].id knowledge.context7.auto_import[].query' }, { tab: 'Knowledge', section: 'Knowledge pipeline', label: 'Bead synthesizer, curator, code maps and primer', key: 'knowledge.bead_synthesizer.enabled', lazy: true, text: 'knowledge pipeline bead synthesizer wiki facts schedule hourly daily confidence target layer facts per cycle api calls vault retention archive beads high priority dependents curator auto promote threshold promotion from to code maps repository layer max bytes primer facts merge strategy engine public read-only mcp knowledge.engine knowledge.public.enabled knowledge.bead_synthesizer.enabled knowledge.bead_synthesizer.schedule knowledge.bead_synthesizer.min_confidence knowledge.bead_synthesizer.target_layer knowledge.bead_synthesizer.max_facts_per_cycle knowledge.bead_synthesizer.max_api_calls_per_cycle knowledge.bead_synthesizer.vault_path publish to repository knowledge.bead_synthesizer.publish.enabled knowledge.bead_synthesizer.publish.repo knowledge.bead_synthesizer.publish.path knowledge.bead_synthesizer.publish.base_branch knowledge.bead_synthesizer.publish.mode knowledge.bead_synthesizer.publish.cadence knowledge.bead_synthesizer.retention_policy.max_beads knowledge.bead_synthesizer.retention_policy.archive_after_synth_days knowledge.bead_synthesizer.retention_policy.high_priority_retain_days knowledge.bead_synthesizer.retention_policy.preserve_with_deps knowledge.curator.enabled knowledge.curator.schedule knowledge.curator.auto_promote_threshold knowledge.curator.promote_from knowledge.curator.promote_to knowledge.code_maps.enabled knowledge.code_maps.schedule knowledge.code_maps.layer knowledge.code_maps.max_bytes knowledge.code_maps.vault_path knowledge.primer.max_facts knowledge.primer.max_bytes knowledge.primer.merge_strategy' }, - { tab: 'Forge App', section: 'Forge App', label: 'GitHub API quota, PR scans and authorship', key: 'github.api_reserve', lazy: true, text: 'github api quota reserve critical agent reserve floor pr detail cache ttl graphql batch page size app authored prs bot signed commits forge api url docs installation oauth client token github.api_reserve github.api_critical github.agent_reserve_floor github.pr_detail_ttl_s github.graphql_pr_batch github.graphql_pr_batch_page_size github.app_authored_prs github.app_signed_commits github.forge github.api_url github.docs_installation_id github.oauth_client_id github.token github.key_file' }, + { tab: 'Forge App', section: 'Forge App', label: 'GitHub API quota, PR scans and authorship', key: 'github.api_reserve', lazy: true, text: 'github api quota reserve critical agent reserve floor pr detail cache ttl etag cache size mb entries conditional request 304 graphql batch page size app authored prs bot signed commits forge api url docs installation oauth client token github.api_reserve github.api_critical github.agent_reserve_floor github.etag_cache_max_mb github.etag_cache_max_entries github.pr_detail_ttl_s github.graphql_pr_batch github.graphql_pr_batch_page_size github.app_authored_prs github.app_signed_commits github.forge github.api_url github.docs_installation_id github.oauth_client_id github.token github.key_file' }, ]; let _settingsSearch = { active: false, query: '', previousTab: '', index: null, bound: false }; function settingsSearchControls() { @@ -59421,6 +59433,10 @@

Configure LiteLLM

${num('pr_detail_ttl_s', 'PR detail cache TTL (seconds)', 'github.pr_detail_ttl_s', d.pr_detail_ttl_s || 1800, 'Reuses a PR detail response while the PR list still reports the same head SHA and updated_at. 0 uses the default.', false, settingHelpMark('operator-reference/README', 'notable-fields', 'PR detail cache TTL', 'How long a GET /pulls/{number} detail response is reused while the cheap PR list still reports the same head SHA and updated_at and GitHub has resolved mergeable_state. 0 uses the default.'))} ${num('graphql_pr_batch_page_size', 'GraphQL PR batch page size', 'github.graphql_pr_batch_page_size', d.graphql_pr_batch_page_size || 50, 'Page size for the batched pullRequests query, 1–100. 0 uses the default.', false, settingHelpMark('operator-reference/README', 'notable-fields', 'GraphQL PR batch page size', 'Page size for the GraphQL pullRequests(first:) batch query. Values above GitHub's maximum of 100 are rejected; 0 uses the default.'))}
+
+ ${num('etag_cache_max_mb', 'ETag cache size (MB)', 'github.etag_cache_max_mb', d.etag_cache_max_mb || 128, 'Memory for remembered GitHub responses so repeat GETs can be answered by a free 304. Bodies are gzip-compressed (about 7× effective). A cache that fills within hours evicts bodies other pollers are about to revalidate and charged fetches climb; raise it. 0 uses the default (128); up to 2048.', false, settingHelpMark('operator-reference/README', 'notable-fields', 'ETag cache size (MB)', 'Upper bound on the gzip-compressed bodies the GitHub conditional-request cache keeps in memory. Each remembered body lets the next GET of the same resource be answered 304 Not Modified, which GitHub does not charge. Watch the fill level under Debug → GitHub API budget: at the cap, evictions turn free revalidations back into charged fetches. 0 restores the default (128 MB); the maximum is 2048. HIVE_GITHUB_ETAG_CACHE_MAX_MB overrides the file.'))} + ${num('etag_cache_max_entries', 'ETag cache entries', 'github.etag_cache_max_entries', d.etag_cache_max_entries || 8192, 'Maximum number of remembered responses regardless of size. 0 uses the default (8192).', false, settingHelpMark('operator-reference/README', 'notable-fields', 'ETag cache entries', 'How many GitHub responses the conditional-request cache holds at most, whatever their size. A multi-repo hive with many small list pages can hit this before the MB bound. 0 restores the default (8192). HIVE_GITHUB_ETAG_CACHE_MAX_ENTRIES overrides the file.'))} +
${toggle('graphql_pr_batch', 'Batch open-PR scans with GraphQL', 'github.graphql_pr_batch', 'one paginated query per repo instead of per-PR detail and check-run reads.', false, settingHelpMark('operator-reference/README', 'notable-fields', 'Batch open-PR scans with GraphQL', 'On by default. One paginated GraphQL query per repository scan fills the PR detail and CI rollup caches, replacing most per-PR detail and check-run reads. Off returns to the REST-only scan path.'))} ${toggle('app_authored_prs', 'Author agent PRs as the App bot', 'github.app_authored_prs', 'agents open PRs and commit as <slug>[bot] with the App installation token when project.ai_author is empty.', true, settingHelpMark('github-app-setup', '', 'Author agent PRs as the App bot', 'With a GitHub App installed, agents open PRs and commit as <slug>[bot] using the installation token whenever project.ai_author is empty, so PR authorship is the App rather than a personal account.'))} ${toggle('app_signed_commits', 'Re-author agent PRs as GitHub-signed commits', 'github.app_signed_commits', 'needed when a base branch requires signed commits; rewrites the agent branch before the PR opens.', false, settingHelpMark('github-app-setup', 'signed-commits-on-agent-prs-app_signed_commits', 'Re-author agent PRs as GitHub-signed commits', 'Opt-in; needs an installed App. Before a PR opens, the watcher squashes the agent branch into one commit created through the GraphQL createCommitOnBranch mutation, which GitHub signs and shows as Verified, so a required_signatures ruleset accepts it. Unsigned follow-up commits are reconciled the same way.'))} @@ -59437,7 +59453,7 @@

Configure LiteLLM

async function saveForgeGitHubSettings() { const st = _forgeGitHubSettings || {}; const body = {}; - for (const key of ['api_reserve', 'api_critical', 'agent_reserve_floor', 'pr_detail_ttl_s', 'graphql_pr_batch_page_size']) { + for (const key of ['api_reserve', 'api_critical', 'agent_reserve_floor', 'pr_detail_ttl_s', 'graphql_pr_batch_page_size', 'etag_cache_max_mb', 'etag_cache_max_entries']) { const el = document.getElementById('forge-gh-' + key); if (!el) continue; const raw = (el.value || '').trim(); diff --git a/src/pkg/dashboard/status_builder.go b/src/pkg/dashboard/status_builder.go index 616fab52e9..e251eaa6a1 100644 --- a/src/pkg/dashboard/status_builder.go +++ b/src/pkg/dashboard/status_builder.go @@ -3215,12 +3215,13 @@ func buildGHRateLimits(ghClient *github.Client, ctx context.Context, cfg *config } result["core"] = core result["etag_cache"] = map[string]any{ - "hits": limits.ETagCache.Hits, - "misses": limits.ETagCache.Misses, - "entries": limits.ETagCache.Entries, - "bytes": limits.ETagCache.Bytes, - "max_bytes": limits.ETagCache.MaxBytes, - "evictions": limits.ETagCache.Evictions, + "hits": limits.ETagCache.Hits, + "misses": limits.ETagCache.Misses, + "entries": limits.ETagCache.Entries, + "bytes": limits.ETagCache.Bytes, + "max_bytes": limits.ETagCache.MaxBytes, + "max_entries": limits.ETagCache.MaxEntries, + "evictions": limits.ETagCache.Evictions, } top := make([]map[string]any, 0, len(limits.TopConsumers)) for _, c := range limits.TopConsumers { diff --git a/src/pkg/fleetreport/runtime.go b/src/pkg/fleetreport/runtime.go index 379d766537..07ee7551a0 100644 --- a/src/pkg/fleetreport/runtime.go +++ b/src/pkg/fleetreport/runtime.go @@ -196,7 +196,7 @@ var Classes = map[string]ClassSpec{ ClassBeadFactIngestFailed: {Component: "knowledge", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"failed to ingest bead fact"}}, ClassSelfReportClearedFailed: {Component: "github-client", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"fleet report: cleared comment failed"}}, ClassGitHubAppRepoNotCovered: {Component: "github-client", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"installation does not cover every configured repo", "state=repo-not-covered"}}, - ClassProxyConnectDialFailed: {Component: "github-proxy", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"proxy: connect dial failed"}}, + ClassProxyConnectDialFailed: {Component: "github-proxy", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"proxy: connect dial failed", "copilot sniff: upstream dial failed"}}, ClassProxyPlainForwardRefused: {Component: "github-proxy", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"proxy: plain http forward refused"}}, ClassInferenceCallerUnknown: {Component: "inference-gateway", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"inference request from unidentified caller refused"}}, ClassInferenceUpstreamError: {Component: "inference-gateway", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"inference upstream error", "inference upstream failed", "inference translate request failed"}}, diff --git a/src/pkg/fleetreport/runtime_test.go b/src/pkg/fleetreport/runtime_test.go index 0a60e6607e..ba6511a53e 100644 --- a/src/pkg/fleetreport/runtime_test.go +++ b/src/pkg/fleetreport/runtime_test.go @@ -1139,6 +1139,7 @@ func TestRecurringRuntimeMessagesAreClassified(t *testing.T) { "github app recheck: installation does not cover every configured repo": ClassGitHubAppRepoNotCovered, "proxy: plain HTTP forward refused — destination is not globally routable": ClassProxyPlainForwardRefused, "proxy: CONNECT dial failed": ClassProxyConnectDialFailed, + "copilot sniff: upstream dial failed": ClassProxyConnectDialFailed, "inference upstream error": ClassInferenceUpstreamError, "copilot transcript activity scan outlived its interval; agent liveness reads may lag": ClassTranscriptScanSlow, "inference upstream failed": ClassInferenceUpstreamError, diff --git a/src/pkg/github/client.go b/src/pkg/github/client.go index 2d61bd742a..74ae0c6d5e 100644 --- a/src/pkg/github/client.go +++ b/src/pkg/github/client.go @@ -4413,14 +4413,23 @@ type ETagCacheInfo struct { Hits int64 `json:"hits"` Misses int64 `json:"misses"` Entries int64 `json:"entries"` - // Bytes / MaxBytes / Evictions are the conditional-request cache's - // memory against its bound and how many entries the bounds pushed out - // (ETagCacheSnapshot). Unset for caches that do not report them. - Bytes int64 `json:"bytes,omitempty"` - MaxBytes int64 `json:"max_bytes,omitempty"` - Evictions int64 `json:"evictions,omitempty"` + // Bytes / MaxBytes / MaxEntries show the fill level of a size-bounded + // cache so an operator can see eviction pressure, and Evictions how many + // entries the bounds pushed out (ETagCacheSnapshot). The PR detail cache + // has no byte bound and leaves them zero. + Bytes int64 `json:"bytes,omitempty"` + MaxBytes int64 `json:"max_bytes,omitempty"` + MaxEntries int64 `json:"max_entries,omitempty"` + Evictions int64 `json:"evictions,omitempty"` } +// etagCacheInfo snapshots the shared ETag cache's counters and fill level. +func etagCacheInfo() ETagCacheInfo { return ETagCacheSnapshot() } + +// ETagCacheInfoSnapshot is etagCacheInfo for callers outside the package +// that assemble their own RateLimitInfo (the hub's token-less poller). +func ETagCacheInfoSnapshot() ETagCacheInfo { return etagCacheInfo() } + type RateLimitEntry struct { Limit int `json:"limit"` Remaining int `json:"remaining"` @@ -4487,7 +4496,7 @@ func (c *Client) RateLimits(ctx context.Context) (*RateLimitInfo, error) { info.GraphQLPRBatch = c.prBatchStats c.prBatchMu.Unlock() } - info.ETagCache = ETagCacheSnapshot() + info.ETagCache = etagCacheInfo() hits, misses, entries := PRDetailCacheStats() info.PRDetailCache = ETagCacheInfo{Hits: hits, Misses: misses, Entries: entries} info.Webhooks = WebhookHealthSnapshot() diff --git a/src/pkg/github/etagcache.go b/src/pkg/github/etagcache.go index 394fa31f2b..d0c47002fd 100644 --- a/src/pkg/github/etagcache.go +++ b/src/pkg/github/etagcache.go @@ -32,17 +32,6 @@ package github // // Rate-limit headers on a replayed response are the 304's, not the stale // 200's, so go-github's own quota bookkeeping stays exact. -// -// Bodies of etagCacheCompressMin bytes or more are held gzip-compressed and -// the byte bound counts what is held. A closed-PR list page is ~2 MiB of -// JSON and the hive re-lists such pages for every watched repo in several -// query variants each cycle: on the projectbluefin hive (2026-11-08, 20 -// repos) one variant alone was 36 MiB, so a raw 64 MiB bound let each sweep -// push the rest of the working set out, and a sweep that cycles through more -// bytes than the bound gets no 304 at all under least-recently-used eviction. -// GitHub JSON compresses about 7x at gzip's fastest level, so the same -// memory now holds the working set. HIVE_GITHUB_ETAG_CACHE_MAX_MB sets the -// bound for a hive that watches more than that. import ( "bytes" @@ -51,12 +40,13 @@ import ( "encoding/hex" "io" "net/http" - "os" "strconv" "strings" "sync" "sync/atomic" "time" + + "github.com/hivecommons/hive/pkg/config" ) const ( @@ -65,15 +55,27 @@ const ( // tarball or diff response into memory. etagCacheMaxBody = 4 << 20 // etagCacheMaxEntries / etagCacheMaxBytes bound the whole cache; least - // recently used entries are evicted past either. The byte bound counts - // what is held: compressed bodies plus the remembered headers. - // etagCacheMaxBytesEnv overrides the byte bound, in MiB. - etagCacheMaxEntries = 16384 - etagCacheMaxBytes = 64 << 20 - etagCacheMaxBytesEnv = "HIVE_GITHUB_ETAG_CACHE_MAX_MB" - // etagCacheCompressMin is the smallest body held compressed; below it - // the gzip framing costs more than it saves. - etagCacheCompressMin = 1 << 10 + // recently used entries are evicted past either. Both are the built-in + // defaults of github.etag_cache_max_entries / github.etag_cache_max_mb + // (config.DefaultGitHubETagCacheMax*) and are replaced at boot and on a + // Settings save through SetETagCacheLimits. + // + // They started at 4096 entries / 64 MiB. A busy multi-repo hive (a + // 16-repo App installation re-listing issues, PRs, check runs and + // comments every cycle) filled 64 MiB within hours of a restart; from + // then on every store evicted a body some other poller was about to + // revalidate, those revalidations became charged fetches again, and the + // hourly usage crept from ~2.6k back toward the 7k installation limit. + // Bodies are gzip-compressed on store (see etagCacheCompressMin), so the + // byte cap counts compressed JSON — roughly 5–8x more responses per MiB + // than the raw bodies would allow — and 128 MiB of compressed bodies is + // a working set that stays warm for the whole TTL on such a hive. + etagCacheMaxEntries = config.DefaultGitHubETagCacheMaxEntries + etagCacheMaxBytes = config.DefaultGitHubETagCacheMaxMB << 20 + // etagCacheCompressMin is the smallest body worth gzipping. Below it the + // gzip framing costs more than it saves; a stored body is only kept + // compressed when that actually made it smaller. + etagCacheCompressMin = 512 // etagCacheTTL is how long an entry may be reused for revalidation. An // ETag never goes stale on GitHub's side, but an entry nobody has asked // about in this long is just memory. @@ -86,62 +88,53 @@ const ( type etagEntry struct { etag string header http.Header - // body is what is held: the response body, gzip-compressed when gz is - // set. rawLen is the body's length as GitHub sent it. + // body is the remembered 200 body, gzip-compressed when gzipped is set. + // size is the uncompressed length; len(body) is what the cache's byte + // bound counts. body []byte - gz bool - rawLen int size int + gzipped bool storedAt time.Time lastUsed time.Time } -// newETagEntry remembers a 200's body and headers, compressing the body when -// that is worth it. -func newETagEntry(etag string, header http.Header, raw []byte) *etagEntry { - e := &etagEntry{etag: etag, header: header, body: raw, rawLen: len(raw)} - if len(raw) >= etagCacheCompressMin { - var buf bytes.Buffer - buf.Grow(len(raw) / 4) - zw, err := gzip.NewWriterLevel(&buf, gzip.BestSpeed) - if err == nil { - _, err = zw.Write(raw) - if cerr := zw.Close(); err == nil { - err = cerr - } - } - if err == nil && buf.Len() < len(raw) { - e.body, e.gz = bytes.Clone(buf.Bytes()), true - } +// newETagEntry remembers body, compressing it when that saves memory. +func newETagEntry(etag string, header http.Header, body []byte) *etagEntry { + e := &etagEntry{etag: etag, header: header, body: body, size: len(body)} + if len(body) < etagCacheCompressMin { + return e } - e.size = len(e.body) - for k, vs := range header { - e.size += len(k) - for _, v := range vs { - e.size += len(v) - } + var buf bytes.Buffer + zw, err := gzip.NewWriterLevel(&buf, gzip.BestSpeed) + if err != nil { + return e + } + if _, err := zw.Write(body); err != nil || zw.Close() != nil { + return e + } + if buf.Len() < len(body) { + e.body, e.gzipped = buf.Bytes(), true } return e } -// rawBody returns the body as GitHub sent it. -func (e *etagEntry) rawBody() ([]byte, error) { - if !e.gz { - return e.body, nil +// plainBody returns the remembered body uncompressed. A body that fails to +// inflate (impossible for one this cache wrote, but the caller must not get +// garbage) comes back nil so the caller can treat the entry as a miss. +func (e *etagEntry) plainBody() []byte { + if !e.gzipped { + return e.body } zr, err := gzip.NewReader(bytes.NewReader(e.body)) if err != nil { - return nil, err - } - out := make([]byte, 0, e.rawLen) - buf := bytes.NewBuffer(out) - if _, err := io.Copy(buf, io.LimitReader(zr, int64(e.rawLen)+1)); err != nil { - return nil, err + return nil } - if buf.Len() != e.rawLen { - return nil, io.ErrUnexpectedEOF + out, err := io.ReadAll(io.LimitReader(zr, int64(e.size)+1)) + _ = zr.Close() + if err != nil || len(out) != e.size { + return nil } - return buf.Bytes(), nil + return out } // etagCache is the process-wide store. Separate from the transport wrapper @@ -167,27 +160,13 @@ func newETagCache() *etagCache { return &etagCache{ entries: map[string]*etagEntry{}, maxEntries: etagCacheMaxEntries, - maxBytes: etagCacheMaxBytesFromEnv(), + maxBytes: etagCacheMaxBytes, maxBody: etagCacheMaxBody, ttl: etagCacheTTL, now: time.Now, } } -// etagCacheMaxBytesFromEnv is the byte bound: HIVE_GITHUB_ETAG_CACHE_MAX_MB -// MiB when that is a positive integer, etagCacheMaxBytes otherwise. -func etagCacheMaxBytesFromEnv() int { - raw := strings.TrimSpace(os.Getenv(etagCacheMaxBytesEnv)) - if raw == "" { - return etagCacheMaxBytes - } - mb, err := strconv.Atoi(raw) - if err != nil || mb <= 0 || mb > 1<<20 { - return etagCacheMaxBytes - } - return mb << 20 -} - // sharedETagCache is the store every wrapped client shares. var sharedETagCache = newETagCache() @@ -199,22 +178,58 @@ func ETagCacheStats() (hits, misses, entries int64) { } // ETagCacheSnapshot is ETagCacheStats plus how many bytes are held against -// which bound, and how many entries a bound has pushed out. A rising +// which bounds, and how many entries a bound has pushed out. A rising // evictions count with a byte count at the bound means the working set does -// not fit (HIVE_GITHUB_ETAG_CACHE_MAX_MB). +// not fit (github.etag_cache_max_mb / HIVE_GITHUB_ETAG_CACHE_MAX_MB). func ETagCacheSnapshot() ETagCacheInfo { c := sharedETagCache c.mu.Lock() - n, held, limit := len(c.entries), c.totalBytes, c.maxBytes + n, held, limit, maxEntries := len(c.entries), c.totalBytes, c.maxBytes, c.maxEntries c.mu.Unlock() return ETagCacheInfo{ - Hits: c.hits.Load(), - Misses: c.misses.Load(), - Entries: int64(n), - Bytes: int64(held), - MaxBytes: int64(limit), - Evictions: c.evictions.Load(), + Hits: c.hits.Load(), + Misses: c.misses.Load(), + Entries: int64(n), + Bytes: int64(held), + MaxBytes: int64(limit), + MaxEntries: int64(maxEntries), + Evictions: c.evictions.Load(), + } +} + +// ETagCacheUsage reports how full the shared cache is against its bounds: +// bytes held (compressed), the byte cap and the entry cap. An operator whose +// hit rate is sliding can tell from bytes ≈ maxBytes that evictions, not +// churn, are the cause and raise github.etag_cache_max_mb. +func ETagCacheUsage() (bytes, maxBytes, maxEntries int) { + c := sharedETagCache + c.mu.Lock() + defer c.mu.Unlock() + return c.totalBytes, c.maxBytes, c.maxEntries +} + +// SetETagCacheLimits resizes the shared cache. A value ≤ 0 restores that +// bound's built-in default. Shrinking evicts least-recently-used entries +// down to the new bounds immediately; growing takes effect on the next +// store. Called at boot from the GitHub client wiring and again from the +// Settings → Forge App save, so a changed github.etag_cache_max_mb applies +// without a restart. +func SetETagCacheLimits(maxBytes, maxEntries int) { + sharedETagCache.setLimits(maxBytes, maxEntries) +} + +func (c *etagCache) setLimits(maxBytes, maxEntries int) { + if maxBytes <= 0 { + maxBytes = etagCacheMaxBytes + } + if maxEntries <= 0 { + maxEntries = etagCacheMaxEntries } + c.mu.Lock() + defer c.mu.Unlock() + c.maxBytes = maxBytes + c.maxEntries = maxEntries + c.evictLocked() } type etagCacheTransport struct { @@ -331,7 +346,7 @@ func (c *etagCache) get(key string) *etagEntry { } now := c.now() if now.Sub(e.lastUsed) > c.ttl { - c.totalBytes -= e.size + c.totalBytes -= len(e.body) delete(c.entries, key) return nil } @@ -343,12 +358,12 @@ func (c *etagCache) put(key string, e *etagEntry) { c.mu.Lock() defer c.mu.Unlock() if old, ok := c.entries[key]; ok { - c.totalBytes -= old.size + c.totalBytes -= len(old.body) } e.storedAt = c.now() e.lastUsed = e.storedAt c.entries[key] = e - c.totalBytes += e.size + c.totalBytes += len(e.body) c.stores.Add(1) c.evictLocked() } @@ -357,7 +372,7 @@ func (c *etagCache) drop(key string) { c.mu.Lock() defer c.mu.Unlock() if old, ok := c.entries[key]; ok { - c.totalBytes -= old.size + c.totalBytes -= len(old.body) delete(c.entries, key) } } @@ -376,7 +391,7 @@ func (c *etagCache) evictLocked() { if oldest == nil { return } - c.totalBytes -= oldest.size + c.totalBytes -= len(oldest.body) delete(c.entries, oldestKey) c.evictions.Add(1) } @@ -395,6 +410,13 @@ func (t *etagCacheTransport) RoundTrip(req *http.Request) (*http.Response, error } key := etagCacheKey(req) entry := t.cache.get(key) + var entryBody []byte + if entry != nil { + if entryBody = entry.plainBody(); entryBody == nil { + t.cache.drop(key) + entry = nil + } + } sent := req if entry != nil { sent = req.Clone(req.Context()) @@ -413,14 +435,6 @@ func (t *etagCacheTransport) RoundTrip(req *http.Request) (*http.Response, error _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, etagCacheMaxBody)) _ = resp.Body.Close() } - body, berr := entry.rawBody() - if berr != nil { - // The held copy cannot be read back: forget it and fetch the - // resource plainly rather than hand the caller a broken body. - t.cache.drop(key) - t.cache.misses.Add(1) - return t.inner.RoundTrip(req) - } t.cache.hits.Add(1) hdr := entry.header.Clone() for _, h := range replayHeaders { @@ -429,7 +443,7 @@ func (t *etagCacheTransport) RoundTrip(req *http.Request) (*http.Response, error } } hdr.Set(etagCacheHeader, "revalidated") - hdr.Set("Content-Length", strconv.Itoa(len(body))) + hdr.Set("Content-Length", strconv.Itoa(len(entryBody))) return &http.Response{ Status: "200 OK", StatusCode: http.StatusOK, @@ -437,8 +451,8 @@ func (t *etagCacheTransport) RoundTrip(req *http.Request) (*http.Response, error ProtoMajor: resp.ProtoMajor, ProtoMinor: resp.ProtoMinor, Header: hdr, - Body: io.NopCloser(bytes.NewReader(body)), - ContentLength: int64(len(body)), + Body: io.NopCloser(bytes.NewReader(entryBody)), + ContentLength: int64(len(entryBody)), Request: req, }, nil } diff --git a/src/pkg/github/etagcache_limits_test.go b/src/pkg/github/etagcache_limits_test.go new file mode 100644 index 0000000000..fdfb20d8ea --- /dev/null +++ b/src/pkg/github/etagcache_limits_test.go @@ -0,0 +1,142 @@ +package github + +import ( + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/hivecommons/hive/pkg/config" +) + +// The built-in bounds are the config package's defaults, so the Settings +// dialog's "0 restores the default" and the cache agree on what that is. +func TestETagCache_DefaultsMatchConfig(t *testing.T) { + c := newETagCache() + if c.maxBytes != config.DefaultGitHubETagCacheMaxMB<<20 { + t.Fatalf("maxBytes = %d, want %d MiB", c.maxBytes, config.DefaultGitHubETagCacheMaxMB) + } + if c.maxEntries != config.DefaultGitHubETagCacheMaxEntries { + t.Fatalf("maxEntries = %d, want %d", c.maxEntries, config.DefaultGitHubETagCacheMaxEntries) + } +} + +// Shrinking the bounds evicts least-recently-used entries at once; a +// non-positive bound restores the default. +func TestETagCache_SetLimitsEvictsDown(t *testing.T) { + n := 0 + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + n++ + w.Header().Set("ETag", fmt.Sprintf(`"e%d"`, n)) + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, `{"n":1}`) + })) + defer ts.Close() + tr, cache := newETagTransport() + for i := 0; i < 6; i++ { + etagGet(t, tr, fmt.Sprintf("%s/r/%d", ts.URL, i), "t") + } + // Touch entry 0 so it is the most recently used and survives. + etagGet(t, tr, fmt.Sprintf("%s/r/0", ts.URL), "t") + if len(cache.entries) != 6 { + t.Fatalf("entries = %d, want 6 before resize", len(cache.entries)) + } + + cache.setLimits(cache.maxBytes, 2) + if len(cache.entries) != 2 { + t.Fatalf("entries = %d, want 2 after shrinking maxEntries", len(cache.entries)) + } + if cache.get(etagCacheKeyFor(t, ts.URL+"/r/0", "t")) == nil { + t.Error("most recently used entry was evicted") + } + for i := 1; i <= 4; i++ { + if cache.get(etagCacheKeyFor(t, fmt.Sprintf("%s/r/%d", ts.URL, i), "t")) != nil { + t.Errorf("entry %d survived shrinking to 2", i) + } + } + + // Shrink by bytes: one body's worth keeps exactly one entry. + cache.setLimits(len(`{"n":1}`), 0) + if cache.maxEntries != etagCacheMaxEntries { + t.Fatalf("maxEntries = %d, want default %d restored", cache.maxEntries, etagCacheMaxEntries) + } + if len(cache.entries) != 1 || cache.totalBytes != len(`{"n":1}`) { + t.Fatalf("entries = %d bytes = %d after byte shrink, want 1 / %d", len(cache.entries), cache.totalBytes, len(`{"n":1}`)) + } + + cache.setLimits(0, 0) + if cache.maxBytes != etagCacheMaxBytes || cache.maxEntries != etagCacheMaxEntries { + t.Fatalf("setLimits(0, 0) = %d/%d, want defaults", cache.maxBytes, cache.maxEntries) + } +} + +// SetETagCacheLimits / ETagCacheUsage act on the shared cache every wrapped +// client uses, and the usage snapshot reflects the new bounds. +func TestSetETagCacheLimits_SharedCache(t *testing.T) { + _, maxBytes0, maxEntries0 := ETagCacheUsage() + t.Cleanup(func() { SetETagCacheLimits(maxBytes0, maxEntries0) }) + + SetETagCacheLimits(3<<20, 77) + _, maxBytes, maxEntries := ETagCacheUsage() + if maxBytes != 3<<20 || maxEntries != 77 { + t.Fatalf("ETagCacheUsage() bounds = %d/%d, want 3 MiB / 77", maxBytes, maxEntries) + } + info := ETagCacheInfoSnapshot() + if info.MaxBytes != 3<<20 || info.MaxEntries != 77 { + t.Fatalf("ETagCacheInfoSnapshot() bounds = %d/%d", info.MaxBytes, info.MaxEntries) + } + if info.Bytes < 0 || info.Entries < 0 { + t.Fatalf("negative usage: %+v", info) + } +} + +// Bodies over etagCacheCompressMin are stored gzip-compressed — the byte +// bound counts compressed bytes — and replayed byte-for-byte. +func TestETagCache_CompressesLargeBodies(t *testing.T) { + body := `[` + strings.Repeat(`{"number":7,"title":"the same issue again","state":"open"},`, 200) + `{}]` + srv := &etagServer{body: body, etag: `W/"big"`, remaining: 100} + ts := httptest.NewServer(http.HandlerFunc(srv.handler)) + defer ts.Close() + tr, cache := newETagTransport() + + resp, got := etagGet(t, tr, ts.URL+"/repos/o/r/issues", "t") + if resp.StatusCode != http.StatusOK || got != body { + t.Fatalf("first fetch: %d, body mismatch=%v", resp.StatusCode, got != body) + } + e := cache.get(etagCacheKeyFor(t, ts.URL+"/repos/o/r/issues", "t")) + if e == nil || !e.gzipped { + t.Fatalf("large JSON body was not stored compressed: %+v", e != nil) + } + if e.size != len(body) || len(e.body) >= len(body) || cache.totalBytes != len(e.body) { + t.Fatalf("size=%d stored=%d total=%d raw=%d", e.size, len(e.body), cache.totalBytes, len(body)) + } + + resp, got = etagGet(t, tr, ts.URL+"/repos/o/r/issues", "t") + if resp.Header.Get(etagCacheHeader) != "revalidated" || got != body { + t.Fatalf("replay: %s=%q, body mismatch=%v", etagCacheHeader, resp.Header.Get(etagCacheHeader), got != body) + } + if resp.ContentLength != int64(len(body)) { + t.Fatalf("replayed ContentLength = %d, want %d (uncompressed)", resp.ContentLength, len(body)) + } + + // A small body stays raw: gzip framing would only cost memory. + small := &etagServer{body: `{}`, etag: `"s"`, remaining: 100} + ts2 := httptest.NewServer(http.HandlerFunc(small.handler)) + defer ts2.Close() + etagGet(t, tr, ts2.URL+"/x", "t") + if e := cache.get(etagCacheKeyFor(t, ts2.URL+"/x", "t")); e == nil || e.gzipped { + t.Fatalf("small body should be stored raw") + } + + // A corrupt compressed body is dropped and refetched, never replayed. + e.body = []byte("not gzip") + resp, got = etagGet(t, tr, ts.URL+"/repos/o/r/issues", "t") + if resp.Header.Get(etagCacheHeader) != "" || got != body { + t.Fatalf("corrupt entry was replayed: %s=%q", etagCacheHeader, resp.Header.Get(etagCacheHeader)) + } + if e2 := cache.get(etagCacheKeyFor(t, ts.URL+"/repos/o/r/issues", "t")); e2 == nil || !e2.gzipped || string(e2.body) == "not gzip" { + t.Fatal("corrupt entry was not replaced by the refetched body") + } +} diff --git a/src/pkg/github/etagcache_workingset_test.go b/src/pkg/github/etagcache_workingset_test.go index 90508f8f84..3e3f985055 100644 --- a/src/pkg/github/etagcache_workingset_test.go +++ b/src/pkg/github/etagcache_workingset_test.go @@ -152,27 +152,6 @@ func TestETagCache_WorkingSetLargerThanOldBoundKeepsHits(t *testing.T) { } } -// HIVE_GITHUB_ETAG_CACHE_MAX_MB sets the byte bound for a hive whose working -// set is larger still; anything but a positive whole number keeps the default. -func TestETagCache_MaxBytesFromEnv(t *testing.T) { - for _, tc := range []struct { - env string - want int - }{ - {"", 64 << 20}, - {"256", 256 << 20}, - {" 8 ", 8 << 20}, - {"0", 64 << 20}, - {"-5", 64 << 20}, - {"lots", 64 << 20}, - } { - t.Setenv("HIVE_GITHUB_ETAG_CACHE_MAX_MB", tc.env) - if got := newETagCache().maxBytes; got != tc.want { - t.Errorf("HIVE_GITHUB_ETAG_CACHE_MAX_MB=%q: maxBytes = %d, want %d", tc.env, got, tc.want) - } - } -} - // A held body that cannot be read back is never handed to the caller: the // entry is dropped and the resource fetched plainly. func TestETagCache_UnreadableHeldBodyFetchesPlainly(t *testing.T) { @@ -198,8 +177,8 @@ func TestETagCache_UnreadableHeldBodyFetchesPlainly(t *testing.T) { t.Fatalf("entries = %d, want 1", len(cache.entries)) } for _, e := range cache.entries { - if !e.gz || len(e.body) >= e.rawLen { - t.Errorf("a %d-byte JSON body is held as %d bytes (gz=%v), want it compressed", e.rawLen, len(e.body), e.gz) + if !e.gzipped || len(e.body) >= e.size { + t.Errorf("a %d-byte JSON body is held as %d bytes (gzipped=%v), want it compressed", e.size, len(e.body), e.gzipped) } e.body = []byte("not gzip") } diff --git a/src/pkg/hub/github_http.go b/src/pkg/hub/github_http.go index acfb5554b5..c09256c902 100644 --- a/src/pkg/hub/github_http.go +++ b/src/pkg/hub/github_http.go @@ -234,6 +234,6 @@ func hubGitHubRateLimits(ctx context.Context) (*hgithub.RateLimitInfo, error) { Search: toEntry(body.Resources.Search), GraphQL: toEntry(body.Resources.GraphQL), TopConsumers: hgithub.RESTTopConsumers(10), - ETagCache: hgithub.ETagCacheSnapshot(), + ETagCache: hgithub.ETagCacheInfoSnapshot(), }, nil }