Repository navigation
✨ Sandbox Jobs: authenticated GitHub reads via a read-only hive relay #18868
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
772d8ee
ce083e5
e3bc0a4
4486b74
dc8bf36
5e0333d
f05bcfd
fdf97f0
764b8eb
cc7804e
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| - Sandbox Jobs can now make authenticated GitHub reads through a hive-hosted read relay instead of hitting GitHub's 60-requests-per-hour anonymous cap (so `uv lock --upgrade` resolves git dependencies without falling back to full clones). Opt in with `agent_sandbox.job.github_read_proxy` (`enabled`, `listen_addr` and `endpoint`, optional `no_proxy`); default off, with no default bind: `listen_addr` is required when enabled, and `endpoint` must be `https://` because the listener serves TLS only, with a certificate from hive's proxy CA for the endpoint host. Pair it with a NetworkPolicy (example in the docs). The relay attaches a read-only, repo-scoped App token (the existing `shared-uid-read` tier) to `GET`/`HEAD` for `api.github.com`, `codeload.github.com` and `raw.githubusercontent.com` and to git fetch on `github.com`, refuses every write, GraphQL and push discovery with a 403, tunnels every other host untouched, and never puts a GitHub credential in the Job: the Job holds only a per-kick relay key that is revoked when the kick ends. See [Sandbox isolation](src/docs/sandbox-isolation.md) ([#18865](https://github.com/hivecommons/hive/issues/18865)). |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,93 @@ | ||
| package main | ||
|
|
||
| import ( | ||
| "context" | ||
| "crypto/tls" | ||
| "errors" | ||
| "net" | ||
| "os" | ||
| "time" | ||
|
|
||
| "github.com/hivecommons/hive/crux" | ||
| "github.com/hivecommons/hive/crux/proxy" | ||
| ) | ||
|
|
||
| // sandboxReadTokenTimeout bounds one read-token mint for the sandbox read | ||
| // relay. | ||
| const sandboxReadTokenTimeout = 20 * time.Second | ||
|
|
||
| // systemRootBundles are the PEM trust bundles hive looks for, first match | ||
| // wins. The relay hands one to each Job with the hive CA appended, so a Job | ||
| // that sets SSL_CERT_FILE still trusts the real internet. | ||
| var systemRootBundles = []string{ | ||
| "/etc/ssl/certs/ca-certificates.crt", | ||
| "/etc/pki/tls/certs/ca-bundle.crt", | ||
| "/etc/ssl/cert.pem", | ||
| } | ||
|
|
||
| func readSystemRootsPEM() []byte { | ||
| for _, p := range systemRootBundles { | ||
| if data, err := os.ReadFile(p); err == nil && len(data) > 0 { | ||
| return data | ||
| } | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| // wireSandboxGitHubReadRelay starts the sandbox GitHub read relay when | ||
| // agent_sandbox.job.github_read_proxy.enabled is true (#18865). Default off. | ||
| // Any failure here is logged and leaves Jobs on their existing network path; | ||
| // it never stops boot. | ||
| func (b *boot) wireSandboxGitHubReadRelay() { | ||
| cfg := b.cfg.AgentSandbox.Job | ||
| if b.githubProxy == nil || cfg == nil || cfg.GitHubReadProxy == nil || !cfg.GitHubReadProxy.Enabled { | ||
| return | ||
| } | ||
| rp := cfg.GitHubReadProxy | ||
| if err := rp.Validate(); err != nil { | ||
| b.logger.Error("sandbox GitHub read relay not started", "error", err.Error()) | ||
| return | ||
| } | ||
| if b.agentMgr == nil { | ||
| b.logger.Error("sandbox GitHub read relay not started: no agent manager") | ||
| return | ||
| } | ||
| relay, err := proxy.NewSandboxReadRelay(b.githubProxy, proxy.SandboxReadRelayOptions{ | ||
| Endpoint: rp.Endpoint, | ||
| Token: b.sandboxReadToken, | ||
| RootsPEM: readSystemRootsPEM(), | ||
| }) | ||
| if err != nil { | ||
| b.logger.Error("sandbox GitHub read relay not started", "error", err.Error()) | ||
| return | ||
| } | ||
| ln, err := net.Listen("tcp", rp.Listen()) | ||
| if err != nil { | ||
| b.logger.Error("sandbox GitHub read relay not started: cannot listen", "addr", rp.Listen(), "error", err.Error()) | ||
| return | ||
| } | ||
| // TLS only: the per-kick key is carried in Proxy-Authorization, so a | ||
| // plain-HTTP connection is never served. The certificate comes from the | ||
| // hive proxy CA for the endpoint's host. | ||
| tlsLn := tls.NewListener(ln, relay.TLSConfig()) | ||
| go func() { | ||
| // Serve only returns when the listener fails, always with an error. | ||
| b.logger.Error("sandbox GitHub read relay stopped", "error", relay.Serve(tlsLn).Error()) | ||
| }() | ||
| b.agentMgr.SetSandboxGitHubReadRelay(relay, rp.NoProxy) | ||
| b.logger.Info("sandbox GitHub read relay enabled (TLS): Job GitHub reads use a read-only, repo-scoped token held by hive; writes are refused", "listen", rp.Listen(), "endpoint", rp.Endpoint) | ||
| } | ||
|
|
||
| // sandboxReadToken mints the READ identity the relay attaches: the | ||
| // shared-uid-read tier (metadata, contents, actions, checks, pull requests and | ||
| // issues, all read), narrowed to the project's repositories. It is never the | ||
| // write-capable credential. | ||
| func (b *boot) sandboxReadToken() (string, error) { | ||
| auth := b.appAuth | ||
| if auth == nil { | ||
| return "", errors.New("no GitHub App auth to mint a read token") | ||
| } | ||
| ctx, cancel := context.WithTimeout(context.Background(), sandboxReadTokenTimeout) | ||
| defer cancel() | ||
| return auth.ScopedTokenForRepos(ctx, crux.SharedUIDReadTokenTier, b.cfg.Project.Repos) | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This token is scoped to all |
||
| } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,36 @@ | ||
| package main | ||
|
|
||
| import ( | ||
| "testing" | ||
|
|
||
| "github.com/hivecommons/hive/pkg/config" | ||
| ) | ||
|
|
||
| // The sandbox GitHub read relay is default off (#18865): with no block, or a | ||
| // disabled one, boot wires nothing and binds nothing. | ||
| func TestWireSandboxGitHubReadRelayDefaultOff(t *testing.T) { | ||
| withJob := func(job *config.SandboxJobConfig) *config.Config { | ||
| c := &config.Config{} | ||
| c.AgentSandbox = config.AgentSandboxConfig{Job: job} | ||
| return c | ||
| } | ||
| cfgs := []*config.Config{ | ||
| {}, | ||
| withJob(&config.SandboxJobConfig{}), | ||
| withJob(&config.SandboxJobConfig{GitHubReadProxy: &config.SandboxGitHubReadProxy{Enabled: false, Endpoint: "http://h:1"}}), | ||
| } | ||
| for i, cfg := range cfgs { | ||
| b := &boot{cfg: cfg, logger: quietLogger()} | ||
| b.wireSandboxGitHubReadRelay() // nil proxy and nil manager: must be a silent no-op | ||
| if b.agentMgr != nil { | ||
| t.Fatalf("case %d wired a manager", i) | ||
| } | ||
| } | ||
| } | ||
|
|
||
| func TestSandboxReadTokenNeedsAppAuth(t *testing.T) { | ||
| b := &boot{cfg: &config.Config{}, logger: quietLogger()} | ||
| if tok, err := b.sandboxReadToken(); err == nil || tok != "" { | ||
| t.Fatalf("token = %q, err = %v; want an error and no token", tok, err) | ||
| } | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,16 @@ | ||
| package agent | ||
|
|
||
| // SetACMMLevel updates the cached ACMM level used by agentMode() when | ||
| // launching agents. Call this whenever the ACMM level changes. | ||
| func (m *Manager) SetACMMLevel(level int) { | ||
| m.mu.Lock() | ||
| defer m.mu.Unlock() | ||
| m.project.ACMMLevel = level | ||
| m.removeAgentsBelowACMMGateLocked(level) | ||
| } | ||
|
|
||
| func (m *Manager) GetACMMLevel() int { | ||
| m.mu.RLock() | ||
| defer m.mu.RUnlock() | ||
| return m.project.ACMMLevel | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,136 @@ | ||
| package agent | ||
|
|
||
| import ( | ||
| "log/slog" | ||
| "os" | ||
| "path" | ||
| "path/filepath" | ||
| "strings" | ||
| "sync/atomic" | ||
|
|
||
| "github.com/hivecommons/hive/crux" | ||
| "github.com/hivecommons/hive/crux/config" | ||
| "github.com/hivecommons/hive/crux/sandbox" | ||
| ) | ||
|
|
||
| // SandboxGitHubReadRelay is the relay a sandbox Job sends its GitHub reads | ||
| // through (#18865); see crux.SandboxGitHubReadRelay. | ||
| type SandboxGitHubReadRelay = crux.SandboxGitHubReadRelay | ||
|
|
||
| // sandboxReadRelayState is the Manager's relay wiring; the zero value is off. | ||
| // The relay and its extra NO_PROXY hosts are stored as one value so a reader | ||
| // never sees one from a previous install paired with the other. | ||
| type sandboxReadRelayState struct { | ||
| cur atomic.Pointer[sandboxReadRelayInstall] | ||
| } | ||
|
|
||
| type sandboxReadRelayInstall struct { | ||
| relay SandboxGitHubReadRelay | ||
| noProxy []string | ||
| } | ||
|
|
||
| // SetSandboxGitHubReadRelay installs (or, with nil, removes) the relay that | ||
| // carries sandbox Job GitHub reads (#18865). noProxy adds hosts Jobs reach | ||
| // directly. | ||
| func (m *Manager) SetSandboxGitHubReadRelay(relay SandboxGitHubReadRelay, noProxy []string) { | ||
| if relay == nil { | ||
| m.sandboxReadRelay.cur.Store(nil) | ||
| return | ||
| } | ||
| m.sandboxReadRelay.cur.Store(&sandboxReadRelayInstall{relay: relay, noProxy: append([]string(nil), noProxy...)}) | ||
| } | ||
|
|
||
| const ( | ||
| // sandboxReadCARelPath is where the trust bundle is written in the kick's | ||
| // workspace; the Job sees it under sandbox.DefaultWorkspaceMount. | ||
| sandboxReadCARelPath = ".hive/github-read-ca.pem" | ||
| sandboxReadCAFileMode = 0o644 | ||
| sandboxReadCADirMode = 0o755 | ||
| // sandboxReadBaseNoProxy keeps cluster-local traffic off the relay. | ||
| sandboxReadBaseNoProxy = "localhost,127.0.0.1,::1,.svc,.cluster.local" | ||
| ) | ||
|
|
||
| // sandboxReadEnv is the environment that points a Job's HTTPS clients at the | ||
| // relay and at the CA bundle. It carries only the proxy URL (relay key) and a | ||
| // file path: no GitHub credential exists on this path, and | ||
| // TestSandboxReadEnvCarriesNoGitHubToken pins that. | ||
| func sandboxReadEnv(proxyURL, caPath string, extraNoProxy []string) map[string]string { | ||
| noProxy := sandboxReadBaseNoProxy | ||
| for _, h := range extraNoProxy { | ||
| if h = strings.TrimSpace(h); h != "" { | ||
| noProxy += "," + h | ||
| } | ||
| } | ||
| return map[string]string{ | ||
| "HTTPS_PROXY": proxyURL, | ||
| "https_proxy": proxyURL, | ||
| "NO_PROXY": noProxy, | ||
| "no_proxy": noProxy, | ||
| "SSL_CERT_FILE": caPath, | ||
| "GIT_SSL_CAINFO": caPath, | ||
| "REQUESTS_CA_BUNDLE": caPath, | ||
| "CURL_CA_BUNDLE": caPath, | ||
| "NODE_EXTRA_CA_CERTS": caPath, | ||
| // libcurl (so git) verifies an https PROXY against its own CA setting, | ||
| // separate from the one for the origin server: GIT_SSL_CAINFO alone | ||
| // does not cover it. Git reads this entry from the environment. | ||
| "GIT_CONFIG_COUNT": "1", | ||
| "GIT_CONFIG_KEY_0": "http.proxySSLCAInfo", | ||
| "GIT_CONFIG_VALUE_0": caPath, | ||
| } | ||
| } | ||
|
|
||
| // mergeSandboxEnv returns base overlaid with extra; neither input is mutated. | ||
| // GIT_CONFIG_* entries in extra are dropped when base already defines | ||
| // GIT_CONFIG_COUNT, so the relay never rewrites a git config the Job has set. | ||
| func mergeSandboxEnv(base, extra map[string]string) map[string]string { | ||
| if len(extra) == 0 { | ||
| return base | ||
| } | ||
| out := make(map[string]string, len(base)+len(extra)) | ||
| for k, v := range base { | ||
| out[k] = v | ||
| } | ||
| _, baseGitConfig := base["GIT_CONFIG_COUNT"] | ||
| for k, v := range extra { | ||
| if baseGitConfig && strings.HasPrefix(k, "GIT_CONFIG_") { | ||
| continue | ||
| } | ||
| out[k] = v | ||
| } | ||
| return out | ||
| } | ||
|
|
||
| // enrollGitHubReadRelay routes this kick's Job through the read relay. It is | ||
| // best effort by design: the relay only improves rate limits, so any failure | ||
| // is logged and the kick runs as it did before, never failing the work. The | ||
| // returned release must be called when the kick ends; it is never nil. | ||
| func (e *SandboxExecutor) enrollGitHubReadRelay(spec SandboxKickSpec, workspace string) (map[string]string, func()) { | ||
| noop := func() {} | ||
| if e.GitHubReadRelay == nil || spec.Runtime != config.SandboxRuntimeJob { | ||
| return nil, noop | ||
| } | ||
| proxyURL, bundle, release, err := e.GitHubReadRelay.Enroll(spec.Agent) | ||
| if err != nil { | ||
| e.warnReadRelay(spec.Agent, "enroll", err) | ||
| return nil, noop | ||
| } | ||
| caHost := filepath.Join(workspace, filepath.FromSlash(sandboxReadCARelPath)) | ||
| writeErr := os.MkdirAll(filepath.Dir(caHost), sandboxReadCADirMode) | ||
| if writeErr == nil { | ||
| writeErr = os.WriteFile(caHost, bundle, sandboxReadCAFileMode) | ||
| } | ||
| if err = writeErr; err != nil { | ||
| release() | ||
| e.warnReadRelay(spec.Agent, "write CA bundle", err) | ||
| return nil, noop | ||
| } | ||
| caJob := path.Join(sandbox.DefaultWorkspaceMount, sandboxReadCARelPath) | ||
| return sandboxReadEnv(proxyURL, caJob, e.GitHubReadRelayNoProxy), release | ||
| } | ||
|
|
||
| func (e *SandboxExecutor) warnReadRelay(agentName, step string, err error) { | ||
| if e.Logger != nil { | ||
| e.Logger.Warn("sandbox GitHub read relay unavailable; this kick's Job reads GitHub anonymously", slog.String("agent", agentName), slog.String("step", step), slog.String("error", err.Error())) | ||
| } | ||
| } |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Build break.
b.agentMgrishostedRuntime(crux.Runtime+crux.Host).SetSandboxGitHubReadRelayexists only on*agent.Manager; it is not incrux.Host(src/crux/host_wiring.go) and the PR doesn't touch that file,sidecarHost, or the cruxtest fakes. CI build-go/build-and-test/golangci-lint are red. Add the method tocrux.Hostand implement it on every Host (sidecarHost: forward or no-op with a warning, since sidecar-mode Job kicks run out of process).