From 772d8ee34dc9ace94c761a7c1a50e5ea3402f9ce Mon Sep 17 00:00:00 2001 From: Andrew Anderson Date: Tue, 17 Nov 2026 22:24:02 -0500 Subject: [PATCH 1/8] =?UTF-8?q?=E2=9C=A8=20sandbox=20Jobs:=20authenticated?= =?UTF-8?q?=20GitHub=20reads=20through=20a=20read-only=20hive=20relay=20(#?= =?UTF-8?q?18865)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds an opt-in relay (agent_sandbox.job.github_read_proxy, default off) that sandbox Jobs reach through HTTPS_PROXY. It attaches a read-only, repo-scoped App token (shared-uid-read tier) to GET/HEAD for api.github.com, codeload.github.com, raw.githubusercontent.com and git fetch on github.com; refuses writes, GraphQL and push discovery; tunnels other hosts untouched. The Job holds only a per-kick relay key, never a GitHub credential. Signed-off-by: Andrew Anderson --- .../added-18865-sandbox-github-read-relay.md | 1 + src/cmd/hive/main.go | 1 + src/cmd/hive/sandbox_read_relay_boot.go | 89 ++++ src/cmd/hive/sandbox_read_relay_boot_test.go | 27 ++ src/crux/agent/manager.go | 4 + src/crux/agent/manager_sandbox.go | 20 + src/crux/agent/sandbox_executor.go | 10 +- src/crux/agent/sandbox_github_read_relay.go | 105 +++++ .../agent/sandbox_github_read_relay_test.go | 111 +++++ src/crux/config/agent_config.go | 67 +++ .../config/sandbox_github_read_proxy_test.go | 43 ++ .../kubejob/github_read_relay_env_test.go | 42 ++ src/crux/proxy/sandbox_read_relay.go | 419 ++++++++++++++++++ src/crux/proxy/sandbox_read_relay_test.go | 384 ++++++++++++++++ src/docs/sandbox-isolation.md | 28 ++ src/docs/security-model.md | 1 + src/pkg/config/crux_aliases.go | 1 + .../dashboard/settings_parity_allowlist.txt | 8 + 18 files changed, 1360 insertions(+), 1 deletion(-) create mode 100644 changelog.d/added-18865-sandbox-github-read-relay.md create mode 100644 src/cmd/hive/sandbox_read_relay_boot.go create mode 100644 src/cmd/hive/sandbox_read_relay_boot_test.go create mode 100644 src/crux/agent/sandbox_github_read_relay.go create mode 100644 src/crux/agent/sandbox_github_read_relay_test.go create mode 100644 src/crux/config/sandbox_github_read_proxy_test.go create mode 100644 src/crux/kubejob/github_read_relay_env_test.go create mode 100644 src/crux/proxy/sandbox_read_relay.go create mode 100644 src/crux/proxy/sandbox_read_relay_test.go diff --git a/changelog.d/added-18865-sandbox-github-read-relay.md b/changelog.d/added-18865-sandbox-github-read-relay.md new file mode 100644 index 0000000000..5dff0d98a2 --- /dev/null +++ b/changelog.d/added-18865-sandbox-github-read-relay.md @@ -0,0 +1 @@ +- Sandbox Jobs can now make authenticated GitHub reads through a hive-hosted read relay instead of hitting GitHub's 60-requests-per-hour anonymous cap (so `uv lock --upgrade` resolves git dependencies without falling back to full clones). Opt in with `agent_sandbox.job.github_read_proxy` (`enabled`, `endpoint`, optional `listen_addr` and `no_proxy`); default off. The relay attaches a read-only, repo-scoped App token (the existing `shared-uid-read` tier) to `GET`/`HEAD` for `api.github.com`, `codeload.github.com` and `raw.githubusercontent.com` and to git fetch on `github.com`, refuses every write, GraphQL and push discovery with a 403, tunnels every other host untouched, and never puts a GitHub credential in the Job: the Job holds only a per-kick relay key that is revoked when the kick ends. See [Sandbox isolation](src/docs/sandbox-isolation.md) ([#18865](https://github.com/hivecommons/hive/issues/18865)). diff --git a/src/cmd/hive/main.go b/src/cmd/hive/main.go index 06ff7bcc08..6b2b26e0e7 100644 --- a/src/cmd/hive/main.go +++ b/src/cmd/hive/main.go @@ -4515,6 +4515,7 @@ func (b *boot) bootProxyWith(deps bootProxyDeps) { ) deps.startProxy(b.githubProxy, b.logger) + b.wireSandboxGitHubReadRelay() // Jev decision endpoint (hivecommons/hive#8939). Always bound so a // jev_mode toggle in the dashboard takes effect on the agent's next // call; every predicate reads the LIVE config, and an agent whose diff --git a/src/cmd/hive/sandbox_read_relay_boot.go b/src/cmd/hive/sandbox_read_relay_boot.go new file mode 100644 index 0000000000..18f2a652ba --- /dev/null +++ b/src/cmd/hive/sandbox_read_relay_boot.go @@ -0,0 +1,89 @@ +package main + +import ( + "context" + "errors" + "net" + "os" + "time" + + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/proxy" +) + +// sandboxReadTokenTimeout bounds one read-token mint for the sandbox read +// relay. +const sandboxReadTokenTimeout = 20 * time.Second + +// systemRootBundles are the PEM trust bundles hive looks for, first match +// wins. The relay hands one to each Job with the hive CA appended, so a Job +// that sets SSL_CERT_FILE still trusts the real internet. +var systemRootBundles = []string{ + "/etc/ssl/certs/ca-certificates.crt", + "/etc/pki/tls/certs/ca-bundle.crt", + "/etc/ssl/cert.pem", +} + +func readSystemRootsPEM() []byte { + for _, p := range systemRootBundles { + if data, err := os.ReadFile(p); err == nil && len(data) > 0 { + return data + } + } + return nil +} + +// wireSandboxGitHubReadRelay starts the sandbox GitHub read relay when +// agent_sandbox.job.github_read_proxy.enabled is true (#18865). Default off. +// Any failure here is logged and leaves Jobs on their existing network path; +// it never stops boot. +func (b *boot) wireSandboxGitHubReadRelay() { + cfg := b.cfg.AgentSandbox.Job + if b.githubProxy == nil || cfg == nil || cfg.GitHubReadProxy == nil || !cfg.GitHubReadProxy.Enabled { + return + } + rp := cfg.GitHubReadProxy + if err := rp.Validate(); err != nil { + b.logger.Error("sandbox GitHub read relay not started", "error", err.Error()) + return + } + if b.agentMgr == nil { + b.logger.Error("sandbox GitHub read relay not started: no agent manager") + return + } + relay, err := proxy.NewSandboxReadRelay(b.githubProxy, proxy.SandboxReadRelayOptions{ + Endpoint: rp.Endpoint, + Token: b.sandboxReadToken, + RootsPEM: readSystemRootsPEM(), + }) + if err != nil { + b.logger.Error("sandbox GitHub read relay not started", "error", err.Error()) + return + } + ln, err := net.Listen("tcp", rp.Listen()) + if err != nil { + b.logger.Error("sandbox GitHub read relay not started: cannot listen", "addr", rp.Listen(), "error", err.Error()) + return + } + go func() { + if err := relay.Serve(ln); err != nil { + b.logger.Error("sandbox GitHub read relay stopped", "error", err.Error()) + } + }() + b.agentMgr.SetSandboxGitHubReadRelay(relay, rp.NoProxy) + b.logger.Info("sandbox GitHub read relay enabled: Job GitHub reads use a read-only, repo-scoped token held by hive; writes are refused", "listen", rp.Listen()) +} + +// sandboxReadToken mints the READ identity the relay attaches: the +// shared-uid-read tier (metadata, contents, actions, checks, pull requests and +// issues, all read), narrowed to the project's repositories. It is never the +// write-capable credential. +func (b *boot) sandboxReadToken() (string, error) { + auth := b.appAuth + if auth == nil { + return "", errors.New("no GitHub App auth to mint a read token") + } + ctx, cancel := context.WithTimeout(context.Background(), sandboxReadTokenTimeout) + defer cancel() + return auth.ScopedTokenForRepos(ctx, crux.SharedUIDReadTokenTier, b.cfg.Project.Repos) +} diff --git a/src/cmd/hive/sandbox_read_relay_boot_test.go b/src/cmd/hive/sandbox_read_relay_boot_test.go new file mode 100644 index 0000000000..10eb608c73 --- /dev/null +++ b/src/cmd/hive/sandbox_read_relay_boot_test.go @@ -0,0 +1,27 @@ +package main + +import ( + "testing" + + "github.com/hivecommons/hive/pkg/config" +) + +// The sandbox GitHub read relay is default off (#18865): with no block, or a +// disabled one, boot wires nothing and binds nothing. +func TestWireSandboxGitHubReadRelayDefaultOff(t *testing.T) { + cfgs := []*config.Config{{}, {AgentSandbox: config.AgentSandboxConfig{Job: &config.SandboxJobConfig{}}}, {AgentSandbox: config.AgentSandboxConfig{Job: &config.SandboxJobConfig{GitHubReadProxy: &config.SandboxGitHubReadProxy{Enabled: false, Endpoint: "http://h:1"}}}}} + for i, cfg := range cfgs { + b := &boot{cfg: cfg, logger: quietLogger()} + b.wireSandboxGitHubReadRelay() // nil proxy and nil manager: must be a silent no-op + if b.agentMgr != nil { + t.Fatalf("case %d wired a manager", i) + } + } +} + +func TestSandboxReadTokenNeedsAppAuth(t *testing.T) { + b := &boot{cfg: &config.Config{}, logger: quietLogger()} + if tok, err := b.sandboxReadToken(); err == nil || tok != "" { + t.Fatalf("token = %q, err = %v; want an error and no token", tok, err) + } +} diff --git a/src/crux/agent/manager.go b/src/crux/agent/manager.go index a111818fe3..88865a5570 100644 --- a/src/crux/agent/manager.go +++ b/src/crux/agent/manager.go @@ -812,6 +812,10 @@ type Manager struct { sandboxPRClient PRCreator sandboxMutation effects.Boundary sandboxAuditCallback atomic.Pointer[func(agent, action, detail string)] + // sandboxReadRelay and sandboxReadRelayNoProxy route Job-runtime kicks' + // GitHub reads through the hive read relay (#18865); nil is off. + sandboxReadRelay atomic.Pointer[SandboxGitHubReadRelay] + sandboxReadRelayNoProxy atomic.Pointer[[]string] // terminal is the nil-safe pane/tmux boundary. Nil means use the real tmux // implementation, preserving zero-value Manager behavior in tests. diff --git a/src/crux/agent/manager_sandbox.go b/src/crux/agent/manager_sandbox.go index 8704ad0c76..cd79e14f02 100644 --- a/src/crux/agent/manager_sandbox.go +++ b/src/crux/agent/manager_sandbox.go @@ -57,6 +57,20 @@ func (m *Manager) SetSandboxConfig(cfg config.AgentSandboxConfig) { } } +// SetSandboxGitHubReadRelay installs (or, with nil, removes) the relay that +// carries sandbox Job GitHub reads (#18865). noProxy adds hosts Jobs reach +// directly. +func (m *Manager) SetSandboxGitHubReadRelay(relay SandboxGitHubReadRelay, noProxy []string) { + if relay == nil { + m.sandboxReadRelay.Store(nil) + m.sandboxReadRelayNoProxy.Store(nil) + return + } + np := append([]string(nil), noProxy...) + m.sandboxReadRelayNoProxy.Store(&np) + m.sandboxReadRelay.Store(&relay) +} + func (m *Manager) SetSandboxLauncher(l sandbox.Launcher) { m.mu.Lock() defer m.mu.Unlock() @@ -309,6 +323,12 @@ func (m *Manager) runSandboxKick(ctx context.Context, name, runtime string, spec Mutation: mutationBoundary, Logger: m.logger, } + if relay := m.sandboxReadRelay.Load(); relay != nil { + exec.GitHubReadRelay = *relay + if np := m.sandboxReadRelayNoProxy.Load(); np != nil { + exec.GitHubReadRelayNoProxy = *np + } + } res, err := exec.Run(ctx, spec) ended := time.Now() // The run record is written to disk after m.mu is released (defers run diff --git a/src/crux/agent/sandbox_executor.go b/src/crux/agent/sandbox_executor.go index 9918cd5b8a..336fdb69e3 100644 --- a/src/crux/agent/sandbox_executor.go +++ b/src/crux/agent/sandbox_executor.go @@ -187,6 +187,12 @@ type SandboxExecutor struct { Logger *slog.Logger Now func() time.Time Mutation effects.Boundary + // GitHubReadRelay, when set, routes a Job-runtime kick's GitHub reads + // through hive's read relay (#18865). Nil leaves the Job on its own + // network path, exactly as before. + GitHubReadRelay SandboxGitHubReadRelay + // GitHubReadRelayNoProxy is extra NO_PROXY entries for relayed Jobs. + GitHubReadRelayNoProxy []string } func (e *SandboxExecutor) Run(ctx context.Context, spec SandboxKickSpec) (SandboxKickResult, error) { @@ -252,6 +258,8 @@ func (e *SandboxExecutor) Run(ctx context.Context, spec SandboxKickSpec) (Sandbo e.Logger.Warn("sandbox review spool: could not mark the kick ended", "agent", spec.Agent, "error", err) } }() + relayEnv, releaseRelay := e.enrollGitHubReadRelay(spec, workspace) + defer releaseRelay() runCtx, cancel := context.WithTimeout(ctx, spec.Timeout) defer cancel() sres, runErr := e.launcher().Run(runCtx, sandbox.LaunchSpec{ @@ -261,7 +269,7 @@ func (e *SandboxExecutor) Run(ctx context.Context, spec SandboxKickSpec) (Sandbo WorkspaceMount: sandbox.DefaultWorkspaceMount, WorkDir: sandbox.DefaultWorkspaceMount, Command: command, - Env: spool.sandboxEnv(), + Env: mergeSandboxEnv(spool.sandboxEnv(), relayEnv), EnvAllowlist: spec.EnvAllowlist, NetworkMode: spec.NetworkMode, ClineProvider: provider, diff --git a/src/crux/agent/sandbox_github_read_relay.go b/src/crux/agent/sandbox_github_read_relay.go new file mode 100644 index 0000000000..cac55bfab4 --- /dev/null +++ b/src/crux/agent/sandbox_github_read_relay.go @@ -0,0 +1,105 @@ +package agent + +import ( + "log/slog" + "os" + "path" + "path/filepath" + "strings" + + "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/sandbox" +) + +// SandboxGitHubReadRelay is the hive-hosted relay a sandbox Job sends its +// GitHub reads through (#18865). Enroll admits one kick and returns the proxy +// URL for it (carrying a per-kick relay key, never a GitHub credential), the +// PEM trust bundle the Job needs to accept the relay's certificates, and a +// release func that revokes the key. It lives behind an interface so this +// package does not depend on the proxy. +type SandboxGitHubReadRelay interface { + Enroll(agentName string) (proxyURL string, caBundlePEM []byte, release func(), err error) +} + +const ( + // sandboxReadCARelPath is where the trust bundle is written in the kick's + // workspace; the Job sees it under sandbox.DefaultWorkspaceMount. + sandboxReadCARelPath = ".hive/github-read-ca.pem" + sandboxReadCAFileMode = 0o644 + sandboxReadCADirMode = 0o755 + // sandboxReadBaseNoProxy keeps cluster-local traffic off the relay. + sandboxReadBaseNoProxy = "localhost,127.0.0.1,::1,.svc,.cluster.local" +) + +// sandboxReadEnv is the environment that points a Job's HTTPS clients at the +// relay and at the CA bundle. It carries only the proxy URL (relay key) and a +// file path: no GitHub credential exists on this path, and +// TestSandboxReadEnvCarriesNoGitHubToken pins that. +func sandboxReadEnv(proxyURL, caPath string, extraNoProxy []string) map[string]string { + noProxy := sandboxReadBaseNoProxy + for _, h := range extraNoProxy { + if h = strings.TrimSpace(h); h != "" { + noProxy += "," + h + } + } + return map[string]string{ + "HTTPS_PROXY": proxyURL, + "https_proxy": proxyURL, + "NO_PROXY": noProxy, + "no_proxy": noProxy, + "SSL_CERT_FILE": caPath, + "GIT_SSL_CAINFO": caPath, + "REQUESTS_CA_BUNDLE": caPath, + "CURL_CA_BUNDLE": caPath, + "NODE_EXTRA_CA_CERTS": caPath, + } +} + +// mergeSandboxEnv returns base overlaid with extra; neither input is mutated. +func mergeSandboxEnv(base, extra map[string]string) map[string]string { + if len(extra) == 0 { + return base + } + out := make(map[string]string, len(base)+len(extra)) + for k, v := range base { + out[k] = v + } + for k, v := range extra { + out[k] = v + } + return out +} + +// enrollGitHubReadRelay routes this kick's Job through the read relay. It is +// best effort by design: the relay only improves rate limits, so any failure +// is logged and the kick runs as it did before, never failing the work. The +// returned release must be called when the kick ends; it is never nil. +func (e *SandboxExecutor) enrollGitHubReadRelay(spec SandboxKickSpec, workspace string) (map[string]string, func()) { + noop := func() {} + if e.GitHubReadRelay == nil || spec.Runtime != config.SandboxRuntimeJob { + return nil, noop + } + proxyURL, bundle, release, err := e.GitHubReadRelay.Enroll(spec.Agent) + if err != nil { + e.warnReadRelay(spec.Agent, "enroll", err) + return nil, noop + } + caHost := filepath.Join(workspace, filepath.FromSlash(sandboxReadCARelPath)) + writeErr := os.MkdirAll(filepath.Dir(caHost), sandboxReadCADirMode) + if writeErr == nil { + writeErr = os.WriteFile(caHost, bundle, sandboxReadCAFileMode) + } + if err = writeErr; err != nil { + release() + e.warnReadRelay(spec.Agent, "write CA bundle", err) + return nil, noop + } + caJob := path.Join(sandbox.DefaultWorkspaceMount, sandboxReadCARelPath) + return sandboxReadEnv(proxyURL, caJob, e.GitHubReadRelayNoProxy), release +} + +func (e *SandboxExecutor) warnReadRelay(agentName, step string, err error) { + if e.Logger != nil { + e.Logger.Warn("sandbox GitHub read relay unavailable; this kick's Job reads GitHub anonymously", slog.String("agent", agentName), slog.String("step", step), slog.String("error", err.Error())) + } +} diff --git a/src/crux/agent/sandbox_github_read_relay_test.go b/src/crux/agent/sandbox_github_read_relay_test.go new file mode 100644 index 0000000000..5f60b58957 --- /dev/null +++ b/src/crux/agent/sandbox_github_read_relay_test.go @@ -0,0 +1,111 @@ +package agent + +import ( + "errors" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/hivecommons/hive/crux/config" +) + +const relayTestGitHubToken = "ghs_SHOULD_NEVER_APPEAR" + +type fakeReadRelay struct { + enrolled []string + released int + err error +} + +func (f *fakeReadRelay) Enroll(agentName string) (string, []byte, func(), error) { + if f.err != nil { + return "", nil, nil, f.err + } + f.enrolled = append(f.enrolled, agentName) + return "http://hive-sandbox:kickkey@hive.ns.svc:18445", []byte("-----BEGIN CERTIFICATE-----\nX\n-----END CERTIFICATE-----\n"), func() { f.released++ }, nil +} + +func TestSandboxReadEnvCarriesNoGitHubToken(t *testing.T) { + env := sandboxReadEnv("http://hive-sandbox:kickkey@hive.ns.svc:18445", "/workspace/.hive/github-read-ca.pem", []string{"internal.example"}) + for k, v := range env { + if strings.Contains(v, relayTestGitHubToken) || strings.Contains(strings.ToUpper(k), "TOKEN") { + t.Errorf("env %s=%s looks like a GitHub credential", k, v) + } + } + for _, k := range []string{"HTTPS_PROXY", "https_proxy", "SSL_CERT_FILE", "GIT_SSL_CAINFO", "REQUESTS_CA_BUNDLE", "CURL_CA_BUNDLE"} { + if env[k] == "" { + t.Errorf("env %s missing", k) + } + } + if !strings.Contains(env["NO_PROXY"], ".svc") || !strings.Contains(env["NO_PROXY"], "internal.example") { + t.Errorf("NO_PROXY = %q", env["NO_PROXY"]) + } + if _, ok := env["HTTP_PROXY"]; ok { + t.Error("plain HTTP must not be proxied") + } +} + +func TestEnrollGitHubReadRelayOffByDefault(t *testing.T) { + e := &SandboxExecutor{} + env, release := e.enrollGitHubReadRelay(SandboxKickSpec{Agent: "deps", Runtime: config.SandboxRuntimeJob}, t.TempDir()) + release() + if env != nil { + t.Fatalf("no relay configured but env = %v", env) + } +} + +func TestEnrollGitHubReadRelayOnlyForJobRuntime(t *testing.T) { + relay := &fakeReadRelay{} + e := &SandboxExecutor{GitHubReadRelay: relay} + for _, rt := range []string{"", config.SandboxRuntimePodman, config.SandboxRuntimeOpenShell, config.SandboxRuntimeSubstrate} { + env, release := e.enrollGitHubReadRelay(SandboxKickSpec{Agent: "deps", Runtime: rt}, t.TempDir()) + release() + if env != nil { + t.Errorf("runtime %q must not be routed through the relay", rt) + } + } + if len(relay.enrolled) != 0 { + t.Fatalf("relay enrolled %v for non-Job runtimes", relay.enrolled) + } +} + +func TestEnrollGitHubReadRelayJobRuntime(t *testing.T) { + relay := &fakeReadRelay{} + e := &SandboxExecutor{GitHubReadRelay: relay} + ws := t.TempDir() + env, release := e.enrollGitHubReadRelay(SandboxKickSpec{Agent: "deps", Runtime: config.SandboxRuntimeJob}, ws) + if env["HTTPS_PROXY"] == "" || env["SSL_CERT_FILE"] != "/workspace/.hive/github-read-ca.pem" { + t.Fatalf("env = %v", env) + } + if _, err := os.Stat(filepath.Join(ws, ".hive", "github-read-ca.pem")); err != nil { + t.Fatalf("CA bundle not written: %v", err) + } + if relay.released != 0 { + t.Fatal("key released before the kick ended") + } + release() + if relay.released != 1 { + t.Fatal("key not released when the kick ended") + } +} + +func TestEnrollGitHubReadRelayFailureIsNotFatal(t *testing.T) { + e := &SandboxExecutor{GitHubReadRelay: &fakeReadRelay{err: errors.New("boom")}} + env, release := e.enrollGitHubReadRelay(SandboxKickSpec{Agent: "deps", Runtime: config.SandboxRuntimeJob}, t.TempDir()) + release() + if env != nil { + t.Fatalf("env = %v, want none so the kick runs as before", env) + } +} + +func TestMergeSandboxEnvDoesNotMutateInputs(t *testing.T) { + base := map[string]string{"A": "1"} + out := mergeSandboxEnv(base, map[string]string{"B": "2"}) + if len(base) != 1 || out["A"] != "1" || out["B"] != "2" { + t.Fatalf("base=%v out=%v", base, out) + } + if got := mergeSandboxEnv(nil, nil); got != nil { + t.Fatalf("merge of nothing = %v", got) + } +} diff --git a/src/crux/config/agent_config.go b/src/crux/config/agent_config.go index a9d25168ba..1623a0052a 100644 --- a/src/crux/config/agent_config.go +++ b/src/crux/config/agent_config.go @@ -1,6 +1,8 @@ package config import ( + "errors" + "net/url" "strings" "time" ) @@ -55,6 +57,69 @@ type SandboxJobConfig struct { // and uses 65534 elsewhere, since the hive image's default user is root. RelayRunAsUser *int64 `yaml:"relay_run_as_user,omitempty" json:"relay_run_as_user,omitempty"` ImagePullSecrets []string `yaml:"image_pull_secrets,omitempty" json:"image_pull_secrets,omitempty"` + // GitHubReadProxy routes a Job's GitHub READS through a hive-hosted relay + // that attaches a read-only credential the Job never sees (#18865). It is + // read from the hive-wide agent_sandbox.job block only; a per-agent job + // block cannot enable, disable or repoint it. + GitHubReadProxy *SandboxGitHubReadProxy `yaml:"github_read_proxy,omitempty" json:"github_read_proxy,omitempty"` +} + +// Defaults and limits for SandboxGitHubReadProxy. +const ( + // DefaultSandboxGitHubReadProxyListen is where the relay listens when + // listen_addr is unset. All interfaces: a Job pod reaches hive over the pod + // network, not loopback. Reachability is the operator's Service and + // NetworkPolicy to grant. + DefaultSandboxGitHubReadProxyListen = ":18445" +) + +// SandboxGitHubReadProxy configures the sandbox GitHub read relay (#18865). +// Default off. With enabled: true, hive listens on listen_addr and every +// sandbox.runtime: job kick gets HTTPS_PROXY pointing at endpoint plus a +// per-kick relay key. The relay attaches a read-only, repo-scoped GitHub App +// token to GET/HEAD requests for api.github.com, codeload.github.com, +// raw.githubusercontent.com and the git fetch paths of github.com. Writes are +// refused and the token never enters the Job. +type SandboxGitHubReadProxy struct { + Enabled bool `yaml:"enabled,omitempty" json:"enabled,omitempty"` + // ListenAddr is the host:port hive binds. Empty means + // DefaultSandboxGitHubReadProxyListen. + ListenAddr string `yaml:"listen_addr,omitempty" json:"listen_addr,omitempty"` + // Endpoint is the http://host:port Job pods use to reach the relay, for + // example the in-namespace Service name. Required when enabled; it carries + // no credentials (hive adds a per-kick key itself). + Endpoint string `yaml:"endpoint,omitempty" json:"endpoint,omitempty"` + // NoProxy lists extra hosts or domains a Job reaches directly, added to + // the built-in cluster-local list (localhost, .svc, .cluster.local). + NoProxy []string `yaml:"no_proxy,omitempty" json:"no_proxy,omitempty"` +} + +// Listen returns the bind address with the default applied. +func (c *SandboxGitHubReadProxy) Listen() string { + if c == nil || strings.TrimSpace(c.ListenAddr) == "" { + return DefaultSandboxGitHubReadProxyListen + } + return strings.TrimSpace(c.ListenAddr) +} + +// Validate reports why an enabled block cannot work. A disabled or nil block +// is always valid. +func (c *SandboxGitHubReadProxy) Validate() error { + if c == nil || !c.Enabled { + return nil + } + raw := strings.TrimSpace(c.Endpoint) + if raw == "" { + return errors.New("agent_sandbox.job.github_read_proxy.endpoint is required when enabled (the http://host:port Job pods use to reach hive)") + } + u, err := url.Parse(raw) + if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" { + return errors.New("agent_sandbox.job.github_read_proxy.endpoint must be an http(s)://host:port URL") + } + if u.User != nil { + return errors.New("agent_sandbox.job.github_read_proxy.endpoint must not carry credentials; hive adds a per-kick key itself") + } + return nil } // SandboxJobToleration mirrors the subset of a Kubernetes toleration an @@ -110,6 +175,8 @@ func (a *AgentConfig) SandboxJob(global AgentSandboxConfig) SandboxJobConfig { return g } out := *a.Sandbox.Job + // The read relay is hive-wide: a per-agent block cannot enable it. + out.GitHubReadProxy = g.GitHubReadProxy if strings.TrimSpace(out.WorkspaceClaim) == "" { out.WorkspaceClaim = g.WorkspaceClaim } diff --git a/src/crux/config/sandbox_github_read_proxy_test.go b/src/crux/config/sandbox_github_read_proxy_test.go new file mode 100644 index 0000000000..c3059f3b7b --- /dev/null +++ b/src/crux/config/sandbox_github_read_proxy_test.go @@ -0,0 +1,43 @@ +package config + +import "testing" + +func TestSandboxGitHubReadProxyDefaultOff(t *testing.T) { + var nilCfg *SandboxGitHubReadProxy + if err := nilCfg.Validate(); err != nil { + t.Fatalf("nil block must be valid: %v", err) + } + if err := (&SandboxGitHubReadProxy{}).Validate(); err != nil { + t.Fatalf("disabled block must be valid: %v", err) + } + if got := nilCfg.Listen(); got != DefaultSandboxGitHubReadProxyListen { + t.Fatalf("Listen = %q", got) + } +} + +func TestSandboxGitHubReadProxyValidate(t *testing.T) { + for _, tc := range []struct { + endpoint string + ok bool + }{ + {"http://hive.ns.svc:18445", true}, + {"https://hive.example.com:18445", true}, + {"", false}, + {"hive.ns.svc:18445", false}, + {"ftp://hive:1", false}, + {"http://user:pw@hive:1", false}, + } { + err := (&SandboxGitHubReadProxy{Enabled: true, Endpoint: tc.endpoint}).Validate() + if (err == nil) != tc.ok { + t.Errorf("endpoint %q: err = %v, want ok=%v", tc.endpoint, err, tc.ok) + } + } +} + +func TestSandboxJobGitHubReadProxyIsHiveWide(t *testing.T) { + global := AgentSandboxConfig{Job: &SandboxJobConfig{GitHubReadProxy: &SandboxGitHubReadProxy{Enabled: true, Endpoint: "http://h:1"}}} + agent := &AgentConfig{Sandbox: &AgentSandboxOverride{Job: &SandboxJobConfig{GitHubReadProxy: &SandboxGitHubReadProxy{Enabled: false}}}} + if got := agent.SandboxJob(global); got.GitHubReadProxy == nil || !got.GitHubReadProxy.Enabled { + t.Fatalf("a per-agent block must not override the hive-wide relay setting: %+v", got.GitHubReadProxy) + } +} diff --git a/src/crux/kubejob/github_read_relay_env_test.go b/src/crux/kubejob/github_read_relay_env_test.go new file mode 100644 index 0000000000..c711583067 --- /dev/null +++ b/src/crux/kubejob/github_read_relay_env_test.go @@ -0,0 +1,42 @@ +package kubejob + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/hivecommons/hive/crux/sandbox" +) + +// A Job routed through the sandbox GitHub read relay (#18865) carries the +// proxy URL and CA paths, and never a GitHub credential, even when one sits in +// hive's own environment or in the launch spec. +func TestManifestRelayEnvCarriesNoGitHubToken(t *testing.T) { + const secret = "ghs_MUST_NOT_REACH_THE_JOB" + t.Setenv("GITHUB_TOKEN", secret) + t.Setenv("GH_TOKEN", secret) + t.Setenv("HIVE_GITHUB_TOKEN", secret) + + l := &Launcher{Options: Options{WorkspaceClaim: "c"}} + m := l.manifest("ns", "job", "sub", sandbox.LaunchSpec{ + Image: "i", + EnvAllowlist: []string{"GITHUB_TOKEN", "GH_TOKEN", "HIVE_GITHUB_TOKEN"}, + Env: map[string]string{ + "HTTPS_PROXY": "http://hive-sandbox:relaykey@hive.ns.svc:18445", + "SSL_CERT_FILE": "/workspace/.hive/github-read-ca.pem", + "GIT_SSL_CAINFO": "/workspace/.hive/github-read-ca.pem", + "GITHUB_TOKEN": secret, + "GH_TOKEN": secret, + }, + }) + raw, _ := json.Marshal(m) + s := string(raw) + if strings.Contains(s, secret) { + t.Fatalf("the Job manifest leaks a GitHub token:\n%s", s) + } + for _, want := range []string{`"name":"HTTPS_PROXY"`, `"name":"SSL_CERT_FILE"`, `"name":"GIT_SSL_CAINFO"`, "relaykey"} { + if !strings.Contains(s, want) { + t.Errorf("manifest missing %s\n%s", want, s) + } + } +} diff --git a/src/crux/proxy/sandbox_read_relay.go b/src/crux/proxy/sandbox_read_relay.go new file mode 100644 index 0000000000..5ded360cbf --- /dev/null +++ b/src/crux/proxy/sandbox_read_relay.go @@ -0,0 +1,419 @@ +package proxy + +import ( + "bufio" + "crypto/rand" + "crypto/sha256" + "crypto/subtle" + "crypto/tls" + "encoding/base64" + "encoding/hex" + "encoding/json" + "encoding/pem" + "errors" + "fmt" + "net" + "net/http" + "net/url" + "strings" + "sync" + "time" +) + +// Sandbox GitHub read relay (#18865). +// +// Sandbox Jobs run in their own pods, so they cannot reach the loopback proxy +// the hive's own agents use, and an anonymous Job is capped at GitHub's 60 +// requests/hour per source IP, shared by every agent on the host. This relay +// is a second, deliberately narrow listener that a Job points HTTPS_PROXY at. +// +// The invariants, each pinned by a test: +// +// - READ ONLY. A GitHub credential is attached only to GET/HEAD requests to +// the read hosts, plus the one POST git itself needs to fetch +// (git-upload-pack). Every other method, GraphQL included, is refused with +// a 403 and never reaches GitHub. +// - NO INJECTION OFF GITHUB. Any other host is an opaque, destination-guarded +// tunnel: no TLS interception, no header touched, no credential attached. +// - THE CREDENTIAL STAYS HERE. The Job holds only a per-kick relay key, which +// authenticates it to this listener and is worth nothing to GitHub. The +// GitHub token is fetched, attached and discarded in this process; an +// Authorization header the Job sends is always stripped first. +// - A READ IDENTITY. The token source is supplied by the host and is expected +// to be the read-only, repo-scoped tier, never the App write credential. + +// SandboxReadProxyUser is the Proxy-Authorization user a Job presents; the +// password is the per-kick relay key. +const SandboxReadProxyUser = "hive-sandbox" + +const ( + sandboxReadKeyBytes = 32 + // sandboxReadTokenTTL is how long a fetched read token is reused. GitHub + // installation tokens live an hour; refreshing at 45 minutes keeps a + // margin for a request already in flight. + sandboxReadTokenTTL = 45 * time.Minute + sandboxReadHTTPSPort = "443" + sandboxReadRefsSuffix = "/info/refs" + sandboxReadUploadPackSuffix = "/git-upload-pack" + sandboxReadUploadPackName = "git-upload-pack" +) + +// sandboxReadHosts are the hosts whose traffic the relay intercepts. Fixed on +// purpose: not derived from the registered GitHub-family hosts, so enterprise +// hosts added elsewhere never gain relay credential injection. +var sandboxReadHosts = map[string]bool{ + "api.github.com": true, + "codeload.github.com": true, + "raw.githubusercontent.com": true, + "github.com": true, +} + +// SandboxReadHost reports whether host is one the relay terminates TLS for. +func SandboxReadHost(host string) bool { return sandboxReadHosts[canonicalHost(host)] } + +// SandboxReadDecision is the relay's verdict on one intercepted request. +type SandboxReadDecision struct { + // Allowed is false for anything the relay refuses to forward. + Allowed bool + // Inject is true when a read credential may be attached. A request can be + // allowed without it (a github.com page GET), and then goes out with no + // Authorization at all. + Inject bool + // Basic selects git smart-HTTP Basic auth over the "token" scheme. + Basic bool + Reason string +} + +// SandboxReadPolicy classifies one request to an intercepted host. It is a +// pure function so the read-only invariant is testable without a network. +func SandboxReadPolicy(host, method, path, rawQuery string) SandboxReadDecision { + host = canonicalHost(host) + if !sandboxReadHosts[host] { + return SandboxReadDecision{Reason: "host is not a GitHub read host"} + } + method = strings.ToUpper(method) + isRead := method == http.MethodGet || method == http.MethodHead + + if host == "github.com" { + switch { + case isRead && strings.HasSuffix(path, sandboxReadRefsSuffix): + // Smart-HTTP ref discovery. service=git-receive-pack is push + // discovery and is refused; no service is a dumb-protocol read. + service := "" + if q, err := url.ParseQuery(rawQuery); err == nil { + service = q.Get("service") + } + if service != "" && service != sandboxReadUploadPackName { + return SandboxReadDecision{Reason: "only git fetch (git-upload-pack) is allowed through the sandbox read relay"} + } + return SandboxReadDecision{Allowed: true, Inject: true, Basic: true} + case method == http.MethodPost && strings.HasSuffix(path, sandboxReadUploadPackSuffix): + return SandboxReadDecision{Allowed: true, Inject: true, Basic: true} + case isRead && !strings.HasSuffix(path, "/git-receive-pack"): + // Archives, release downloads and the like redirect to the + // download hosts. No credential is needed or attached here. + return SandboxReadDecision{Allowed: true} + } + return SandboxReadDecision{Reason: "the sandbox read relay forwards reads only; writes go through hive's relays"} + } + if !isRead { + return SandboxReadDecision{Reason: "the sandbox read relay forwards GET and HEAD only; writes go through hive's relays"} + } + return SandboxReadDecision{Allowed: true, Inject: true} +} + +// SandboxReadTokenFunc returns the read-only GitHub credential the relay +// attaches. It must be a read identity (the shared-uid-read tier), never the +// write credential. +type SandboxReadTokenFunc func() (string, error) + +// SandboxReadRelayOptions configure NewSandboxReadRelay. +type SandboxReadRelayOptions struct { + // Endpoint is the http://host:port Jobs use to reach the relay's listener. + Endpoint string + // Token supplies the read credential. Nil sends reads unauthenticated. + Token SandboxReadTokenFunc + // RootsPEM is the trust bundle handed to a Job alongside the hive CA, so + // the Job still trusts the real internet. Empty means the hive CA alone. + RootsPEM []byte +} + +// SandboxReadRelay is the Job-facing read relay. +type SandboxReadRelay struct { + p *GitHubProxy + opts SandboxReadRelayOptions + endpoint *url.URL + + keysMu sync.RWMutex + keys map[[sha256.Size]byte]string // sha256(key) -> agent + + tokMu sync.Mutex + tok string + tokFetched time.Time + now func() time.Time + // dial overrides the upstream TLS dial (tests). + dial func(host string) (net.Conn, error) +} + +// NewSandboxReadRelay builds the relay on p (which supplies the CA, the +// destination-guarded dialers and the upstream chain). +func NewSandboxReadRelay(p *GitHubProxy, opts SandboxReadRelayOptions) (*SandboxReadRelay, error) { + if p == nil { + return nil, errors.New("sandbox read relay: no proxy") + } + u, err := url.Parse(strings.TrimSpace(opts.Endpoint)) + if err != nil || u.Host == "" || (u.Scheme != "http" && u.Scheme != "https") || u.User != nil { + return nil, errors.New("sandbox read relay: endpoint must be an http(s)://host:port URL without credentials") + } + return &SandboxReadRelay{p: p, opts: opts, endpoint: u, keys: map[[sha256.Size]byte]string{}, now: time.Now}, nil +} + +// Enroll admits one kick: it mints a per-kick relay key and returns the proxy +// URL the Job should use, the CA bundle it must trust and a release func that +// revokes the key. The key authenticates the Job to this listener only. +func (r *SandboxReadRelay) Enroll(agentName string) (proxyURL string, caBundlePEM []byte, release func(), err error) { + raw := make([]byte, sandboxReadKeyBytes) + if _, err := rand.Read(raw); err != nil { + return "", nil, nil, fmt.Errorf("sandbox read relay: minting kick key: %w", err) + } + key := hex.EncodeToString(raw) + digest := sha256.Sum256([]byte(key)) + r.keysMu.Lock() + r.keys[digest] = agentName + r.keysMu.Unlock() + + u := *r.endpoint + u.User = url.UserPassword(SandboxReadProxyUser, key) + ca := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: r.p.caX509.Raw}) + bundle := append(append([]byte{}, r.opts.RootsPEM...), ca...) + return u.String(), bundle, func() { + r.keysMu.Lock() + delete(r.keys, digest) + r.keysMu.Unlock() + }, nil +} + +// authenticate returns the agent a Proxy-Authorization header belongs to. +func (r *SandboxReadRelay) authenticate(req *http.Request) (string, bool) { + auth := req.Header.Get("Proxy-Authorization") + if !strings.HasPrefix(auth, "Basic ") { + return "", false + } + decoded, err := base64.StdEncoding.DecodeString(strings.TrimPrefix(auth, "Basic ")) + if err != nil { + return "", false + } + user, key, ok := strings.Cut(string(decoded), ":") + if !ok || subtle.ConstantTimeCompare([]byte(user), []byte(SandboxReadProxyUser)) != 1 { + return "", false + } + digest := sha256.Sum256([]byte(key)) + r.keysMu.RLock() + agentName, found := r.keys[digest] + r.keysMu.RUnlock() + return agentName, found +} + +// readToken returns the cached read credential, fetching a fresh one when it +// is older than sandboxReadTokenTTL. An error or empty result means "send the +// request unauthenticated": the relay never substitutes another credential. +func (r *SandboxReadRelay) readToken() string { + if r.opts.Token == nil { + return "" + } + r.tokMu.Lock() + defer r.tokMu.Unlock() + if r.tok != "" && r.now().Sub(r.tokFetched) < sandboxReadTokenTTL { + return r.tok + } + tok, err := r.opts.Token() + if err != nil || tok == "" { + reason := "empty token" + if err != nil { + reason = err.Error() + } + r.p.logger.Warn("sandbox read relay: no read credential available; forwarding reads unauthenticated", "reason", reason) + r.tok = "" + return "" + } + r.tok, r.tokFetched = tok, r.now() + return tok +} + +// applySandboxReadAuth rewrites a request's Authorization per decision d. Any +// credential the Job supplied is removed unconditionally; the read token is +// attached only when d allows injection. +func applySandboxReadAuth(req *http.Request, d SandboxReadDecision, token string) { + req.Header.Del("Authorization") + req.Header.Del("Proxy-Authorization") + req.Header.Del("Expect") + if !d.Allowed || !d.Inject || token == "" { + return + } + if d.Basic { + req.Header.Set("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(gitInjectBasicUser+":"+token))) + return + } + req.Header.Set("Authorization", "token "+token) +} + +// Serve accepts Job connections on ln until it is closed. +func (r *SandboxReadRelay) Serve(ln net.Listener) error { + for { + conn, err := ln.Accept() + if err != nil { + return fmt.Errorf("sandbox read relay accept: %w", err) + } + go r.handleConn(conn) + } +} + +func (r *SandboxReadRelay) handleConn(conn net.Conn) { + defer func() { _ = conn.Close() }() + _ = conn.SetReadDeadline(time.Now().Add(httpReadTimeout)) + br := bufio.NewReader(conn) + req, err := http.ReadRequest(br) + _ = conn.SetReadDeadline(time.Time{}) + if err != nil { + return + } + agentName, ok := r.authenticate(req) + if !ok { + writeRelayStatus(conn, http.StatusProxyAuthRequired, "proxy authentication required", "Proxy-Authenticate: Basic realm=\"hive-sandbox\"\r\n") + return + } + if req.Method != http.MethodConnect { + // Plain-HTTP proxying is not offered: every GitHub read host is HTTPS. + writeRelayStatus(conn, http.StatusForbidden, "the sandbox read relay carries HTTPS CONNECT only", "") + return + } + host, port, err := net.SplitHostPort(req.Host) + if err != nil { + host, port = req.Host, sandboxReadHTTPSPort + } + if port != sandboxReadHTTPSPort { + writeRelayStatus(conn, http.StatusForbidden, "the sandbox read relay carries port 443 only", "") + return + } + if !SandboxReadHost(host) { + r.tunnel(conn, host, agentName) + return + } + r.intercept(conn, host, agentName) +} + +// tunnel is the opaque path for every non-GitHub host: no interception, no +// header touched, no credential. The destination guard keeps it from reaching +// loopback, link-local metadata or the private estate. +func (r *SandboxReadRelay) tunnel(conn net.Conn, host, agentName string) { + const dialTimeout = 10 * time.Second + upstream, err := r.p.dialUpstreamRaw(net.JoinHostPort(host, sandboxReadHTTPSPort), dialTimeout) + if err != nil { + r.p.logger.Debug("sandbox read relay: tunnel refused", "agent", agentName, "host", host, "error", err) + writeRelayStatus(conn, http.StatusBadGateway, "connection failed", "") + return + } + defer func() { _ = upstream.Close() }() + _, _ = fmt.Fprint(conn, "HTTP/1.1 200 Connection established\r\n\r\n") + relayTunnel(conn, upstream) +} + +// intercept terminates TLS for a GitHub read host and relays each request +// through SandboxReadPolicy. +func (r *SandboxReadRelay) intercept(conn net.Conn, host, agentName string) { + cert, err := r.p.forgeCert(host) + if err != nil { + r.p.logger.Error("sandbox read relay: forge cert failed", "host", host, "error", err) + writeRelayStatus(conn, http.StatusBadGateway, "certificate error", "") + return + } + if _, err := fmt.Fprint(conn, "HTTP/1.1 200 Connection established\r\n\r\n"); err != nil { + return + } + client := tls.Server(conn, &tls.Config{Certificates: []tls.Certificate{cert}}) + _ = conn.SetDeadline(time.Now().Add(clientTLSHandshakeTimeout)) + err = client.Handshake() + _ = conn.SetDeadline(time.Time{}) + if err != nil { + return + } + defer func() { _ = client.Close() }() + + // The upstream leg is dialed on the first ALLOWED request, so a refused + // request never opens a connection to GitHub. + var upstream net.Conn + var upstreamBuf *bufio.Reader + defer func() { + if upstream != nil { + _ = upstream.Close() + } + }() + + clientBuf := bufio.NewReader(client) + for { + _ = client.SetReadDeadline(time.Now().Add(httpReadTimeout)) + req, err := http.ReadRequest(clientBuf) + if err != nil { + return + } + d := SandboxReadPolicy(host, req.Method, req.URL.Path, req.URL.RawQuery) + if !d.Allowed { + r.p.logger.Info("sandbox read relay: request refused", "agent", agentName, "host", host, "method", req.Method) + writeRelayRefusal(client, d.Reason) + return + } + if upstream == nil { + up, derr := r.dialUpstream(host) + if derr != nil { + r.p.logger.Warn("sandbox read relay: upstream dial failed", "host", host, "error", derr) + return + } + upstream, upstreamBuf = up, bufio.NewReader(up) + } + token := "" + if d.Inject { + token = r.readToken() + } + applySandboxReadAuth(req, d, token) + // Log the decision, never the credential. + r.p.logger.Debug("sandbox read relay: request", "agent", agentName, "host", host, "method", req.Method, "injected", token != "") + + _ = upstream.SetWriteDeadline(time.Now().Add(httpWriteTimeout)) + if err := req.Write(upstream); err != nil { + return + } + _ = upstream.SetWriteDeadline(time.Time{}) + _ = client.SetReadDeadline(time.Time{}) + _ = upstream.SetReadDeadline(time.Now().Add(httpReadTimeout)) + resp, err := http.ReadResponse(upstreamBuf, req) + _ = upstream.SetReadDeadline(time.Time{}) + if err != nil { + return + } + resp.Body = &stallBoundedBody{body: resp.Body, conn: upstream, idle: r.p.bodyStallTimeoutDuration()} + werr := resp.Write(&stallBoundedWriter{conn: client, idle: httpWriteTimeout}) + _ = resp.Body.Close() + _ = client.SetWriteDeadline(time.Time{}) + if werr != nil || resp.Close || req.Close { + return + } + } +} + +// dialUpstream opens the TLS leg to a read host. It is a method so +// tests can drive the relay without network access by replacing r.dial. +func (r *SandboxReadRelay) dialUpstream(host string) (net.Conn, error) { + if r.dial != nil { + return r.dial(host) + } + return r.p.dialUpstreamTLS(net.JoinHostPort(host, sandboxReadHTTPSPort), host, upstreamDialTimeout) +} + +func writeRelayStatus(conn net.Conn, status int, msg, extraHeaders string) { + _, _ = fmt.Fprintf(conn, "HTTP/1.1 %d %s\r\n%sContent-Length: %d\r\nConnection: close\r\n\r\n%s", status, http.StatusText(status), extraHeaders, len(msg), msg) +} + +func writeRelayRefusal(w net.Conn, reason string) { + body, _ := json.Marshal(map[string]string{"message": "blocked by hive: " + reason}) + _, _ = fmt.Fprintf(w, "HTTP/1.1 403 Forbidden\r\nContent-Type: application/json\r\nContent-Length: %d\r\nConnection: close\r\n\r\n%s", len(body), body) +} diff --git a/src/crux/proxy/sandbox_read_relay_test.go b/src/crux/proxy/sandbox_read_relay_test.go new file mode 100644 index 0000000000..9ed48e8cb2 --- /dev/null +++ b/src/crux/proxy/sandbox_read_relay_test.go @@ -0,0 +1,384 @@ +package proxy + +import ( + "bufio" + "crypto/tls" + "crypto/x509" + "encoding/base64" + "fmt" + "net" + "net/http" + "strings" + "sync" + "sync/atomic" + "testing" + "time" +) + +const testReadToken = "ghs_READONLYTOKEN" + +func TestSandboxReadPolicy(t *testing.T) { + cases := []struct { + name, host, method, path, query string + allowed, inject, basic bool + }{ + {"api GET", "api.github.com", "GET", "/repos/o/r/tags", "", true, true, false}, + {"api HEAD", "api.github.com", "HEAD", "/repos/o/r", "", true, true, false}, + {"codeload GET", "codeload.github.com", "GET", "/o/r/tar.gz/main", "", true, true, false}, + {"raw GET", "raw.githubusercontent.com", "GET", "/o/r/main/a.txt", "", true, true, false}, + {"api POST refused", "api.github.com", "POST", "/repos/o/r/issues", "", false, false, false}, + {"api GraphQL POST refused", "api.github.com", "POST", "/graphql", "", false, false, false}, + {"api PUT refused", "api.github.com", "PUT", "/repos/o/r/pulls/1/merge", "", false, false, false}, + {"api PATCH refused", "api.github.com", "PATCH", "/repos/o/r/issues/1", "", false, false, false}, + {"api DELETE refused", "api.github.com", "DELETE", "/repos/o/r/git/refs/heads/x", "", false, false, false}, + {"codeload POST refused", "codeload.github.com", "POST", "/o/r", "", false, false, false}, + {"raw PUT refused", "raw.githubusercontent.com", "PUT", "/o/r/main/a", "", false, false, false}, + {"git discovery fetch", "github.com", "GET", "/o/r.git/info/refs", "service=git-upload-pack", true, true, true}, + {"git upload-pack POST", "github.com", "POST", "/o/r.git/git-upload-pack", "", true, true, true}, + {"git push discovery refused", "github.com", "GET", "/o/r.git/info/refs", "service=git-receive-pack", false, false, false}, + {"git receive-pack POST refused", "github.com", "POST", "/o/r.git/git-receive-pack", "", false, false, false}, + {"git receive-pack GET refused", "github.com", "GET", "/o/r.git/git-receive-pack", "", false, false, false}, + {"github.com page GET no credential", "github.com", "GET", "/o/r/archive/refs/heads/main.tar.gz", "", true, false, false}, + {"github.com other POST refused", "github.com", "POST", "/o/r/issues", "", false, false, false}, + {"non-GitHub host", "pypi.org", "GET", "/simple/x", "", false, false, false}, + {"lookalike host", "api.github.com.evil.example", "GET", "/", "", false, false, false}, + {"GHE host not injected", "ghe.example.com", "GET", "/api/v3/user", "", false, false, false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + d := SandboxReadPolicy(tc.host, tc.method, tc.path, tc.query) + if d.Allowed != tc.allowed || d.Inject != tc.inject || d.Basic != tc.basic { + t.Fatalf("SandboxReadPolicy(%s %s %s) = %+v, want allowed=%v inject=%v basic=%v", tc.method, tc.host, tc.path, d, tc.allowed, tc.inject, tc.basic) + } + if !d.Allowed && d.Inject { + t.Fatal("a refused request must never be marked for injection") + } + }) + } +} + +func TestSandboxReadPolicyNeverInjectsForAnyWriteMethod(t *testing.T) { + for _, host := range []string{"api.github.com", "codeload.github.com", "raw.githubusercontent.com", "github.com"} { + for _, method := range []string{"POST", "PUT", "PATCH", "DELETE", "OPTIONS", "TRACE", "CONNECT"} { + for _, path := range []string{"/", "/graphql", "/o/r.git/git-receive-pack", "/o/r/issues"} { + d := SandboxReadPolicy(host, method, path, "") + if d.Inject && !(host == "github.com" && method == "POST" && strings.HasSuffix(path, "/git-upload-pack")) { + t.Errorf("%s %s%s must not receive a credential: %+v", method, host, path, d) + } + } + } + } +} + +func TestApplySandboxReadAuth(t *testing.T) { + newReq := func() *http.Request { + r, _ := http.NewRequest("GET", "https://api.github.com/x", nil) + r.Header.Set("Authorization", "token AGENT_SUPPLIED") + r.Header.Set("Proxy-Authorization", "Basic abc") + r.Header.Set("Expect", "100-continue") + return r + } + t.Run("token scheme", func(t *testing.T) { + r := newReq() + applySandboxReadAuth(r, SandboxReadDecision{Allowed: true, Inject: true}, testReadToken) + if got := r.Header.Get("Authorization"); got != "token "+testReadToken { + t.Fatalf("Authorization = %q", got) + } + if r.Header.Get("Proxy-Authorization") != "" || r.Header.Get("Expect") != "" { + t.Fatal("hop headers must be removed") + } + }) + t.Run("git basic", func(t *testing.T) { + r := newReq() + applySandboxReadAuth(r, SandboxReadDecision{Allowed: true, Inject: true, Basic: true}, testReadToken) + want := "Basic " + base64.StdEncoding.EncodeToString([]byte("x-access-token:"+testReadToken)) + if got := r.Header.Get("Authorization"); got != want { + t.Fatalf("Authorization = %q", got) + } + }) + t.Run("agent credential stripped when not injecting", func(t *testing.T) { + r := newReq() + applySandboxReadAuth(r, SandboxReadDecision{Allowed: true}, testReadToken) + if got := r.Header.Get("Authorization"); got != "" { + t.Fatalf("Authorization = %q, want none", got) + } + }) + t.Run("no token means anonymous, not the agent's header", func(t *testing.T) { + r := newReq() + applySandboxReadAuth(r, SandboxReadDecision{Allowed: true, Inject: true}, "") + if got := r.Header.Get("Authorization"); got != "" { + t.Fatalf("Authorization = %q, want none", got) + } + }) + t.Run("refused request gets no credential", func(t *testing.T) { + r := newReq() + applySandboxReadAuth(r, SandboxReadDecision{}, testReadToken) + if got := r.Header.Get("Authorization"); got != "" { + t.Fatalf("Authorization = %q, want none", got) + } + }) +} + +// relayHarness serves a relay on loopback and fakes the GitHub upstream. +type relayHarness struct { + relay *SandboxReadRelay + addr string + pool *x509.CertPool + dials atomic.Int32 + + mu sync.Mutex + upstream []*http.Request +} + +func newRelayHarness(t *testing.T, token SandboxReadTokenFunc) *relayHarness { + t.Helper() + p, _ := newTestProxyWithCA(t) + relay, err := NewSandboxReadRelay(p, SandboxReadRelayOptions{Endpoint: "http://hive.example.svc:18445", Token: token}) + if err != nil { + t.Fatal(err) + } + h := &relayHarness{relay: relay, pool: x509.NewCertPool()} + h.pool.AddCert(p.caX509) + relay.dial = func(host string) (net.Conn, error) { + h.dials.Add(1) + client, server := net.Pipe() + go func() { + defer func() { _ = server.Close() }() + br := bufio.NewReader(server) + for { + req, err := http.ReadRequest(br) + if err != nil { + return + } + h.mu.Lock() + h.upstream = append(h.upstream, req) + h.mu.Unlock() + _, _ = fmt.Fprint(server, "HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok") + } + }() + return client, nil + } + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = ln.Close() }) + go func() { _ = relay.Serve(ln) }() + h.addr = ln.Addr().String() + return h +} + +func (h *relayHarness) seen() []*http.Request { + h.mu.Lock() + defer h.mu.Unlock() + return append([]*http.Request(nil), h.upstream...) +} + +// connect opens a CONNECT tunnel through the relay and returns the TLS client +// conn, or the raw HTTP status when the relay refused the CONNECT. +func (h *relayHarness) connect(t *testing.T, proxyURLUser, target string) (*tls.Conn, int) { + t.Helper() + conn, err := net.DialTimeout("tcp", h.addr, 5*time.Second) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = conn.Close() }) + _ = conn.SetDeadline(time.Now().Add(10 * time.Second)) + hdr := "" + if proxyURLUser != "" { + hdr = "Proxy-Authorization: Basic " + base64.StdEncoding.EncodeToString([]byte(proxyURLUser)) + "\r\n" + } + _, _ = fmt.Fprintf(conn, "CONNECT %s HTTP/1.1\r\nHost: %s\r\n%s\r\n", target, target, hdr) + br := bufio.NewReader(conn) + resp, err := http.ReadResponse(br, &http.Request{Method: "CONNECT"}) + if err != nil { + t.Fatal(err) + } + if resp.StatusCode != 200 { + return nil, resp.StatusCode + } + host, _, _ := net.SplitHostPort(target) + tc := tls.Client(conn, &tls.Config{RootCAs: h.pool, ServerName: host}) + if err := tc.Handshake(); err != nil { + t.Fatalf("TLS handshake through the relay: %v", err) + } + return tc, 200 +} + +func roundTrip(t *testing.T, tc *tls.Conn, method, host, path string, headers map[string]string) *http.Response { + t.Helper() + req, _ := http.NewRequest(method, "https://"+host+path, nil) + for k, v := range headers { + req.Header.Set(k, v) + } + if err := req.Write(tc); err != nil { + t.Fatal(err) + } + resp, err := http.ReadResponse(bufio.NewReader(tc), req) + if err != nil { + t.Fatal(err) + } + return resp +} + +func enrollKey(t *testing.T, r *SandboxReadRelay, agentName string) (userinfo string, release func()) { + t.Helper() + proxyURL, _, rel, err := r.Enroll(agentName) + if err != nil { + t.Fatal(err) + } + rest := strings.TrimPrefix(proxyURL, "http://") + userinfo, _, _ = strings.Cut(rest, "@") + return userinfo, rel +} + +func TestSandboxReadRelayInjectsOnReadAndRefusesWrites(t *testing.T) { + h := newRelayHarness(t, func() (string, error) { return testReadToken, nil }) + user, release := enrollKey(t, h.relay, "deps") + defer release() + + tc, status := h.connect(t, user, "api.github.com:443") + if status != 200 { + t.Fatalf("CONNECT status = %d", status) + } + resp := roundTrip(t, tc, "GET", "api.github.com", "/repos/o/r/tags", map[string]string{"Authorization": "token AGENT_SUPPLIED"}) + if resp.StatusCode != 200 { + t.Fatalf("GET status = %d", resp.StatusCode) + } + seen := h.seen() + if len(seen) != 1 { + t.Fatalf("upstream saw %d requests, want 1", len(seen)) + } + if got := seen[0].Header.Get("Authorization"); got != "token "+testReadToken { + t.Fatalf("upstream Authorization = %q, want the relay's read token (the agent's own header must be replaced)", got) + } + + // A write is refused by the relay and never reaches GitHub, whatever the + // agent put in the request. + tc2, _ := h.connect(t, user, "api.github.com:443") + before := h.dials.Load() + resp = roundTrip(t, tc2, "POST", "api.github.com", "/repos/o/r/issues", map[string]string{"Authorization": "token AGENT_SUPPLIED"}) + if resp.StatusCode != http.StatusForbidden { + t.Fatalf("POST status = %d, want 403", resp.StatusCode) + } + if h.dials.Load() != before { + t.Fatal("a refused write opened an upstream connection") + } + if len(h.seen()) != 1 { + t.Fatal("a refused write reached GitHub") + } +} + +func TestSandboxReadRelayGitPushDiscoveryRefused(t *testing.T) { + h := newRelayHarness(t, func() (string, error) { return testReadToken, nil }) + user, release := enrollKey(t, h.relay, "deps") + defer release() + tc, _ := h.connect(t, user, "github.com:443") + resp := roundTrip(t, tc, "GET", "github.com", "/o/r.git/info/refs?service=git-receive-pack", nil) + if resp.StatusCode != http.StatusForbidden { + t.Fatalf("status = %d, want 403", resp.StatusCode) + } + if len(h.seen()) != 0 { + t.Fatal("push discovery reached GitHub") + } +} + +func TestSandboxReadRelayGitFetchUsesBasicWithReadToken(t *testing.T) { + h := newRelayHarness(t, func() (string, error) { return testReadToken, nil }) + user, release := enrollKey(t, h.relay, "deps") + defer release() + tc, _ := h.connect(t, user, "github.com:443") + resp := roundTrip(t, tc, "GET", "github.com", "/o/r.git/info/refs?service=git-upload-pack", nil) + if resp.StatusCode != 200 { + t.Fatalf("status = %d", resp.StatusCode) + } + want := "Basic " + base64.StdEncoding.EncodeToString([]byte("x-access-token:"+testReadToken)) + if got := h.seen()[0].Header.Get("Authorization"); got != want { + t.Fatalf("upstream Authorization = %q", got) + } +} + +func TestSandboxReadRelayRequiresLiveKey(t *testing.T) { + h := newRelayHarness(t, func() (string, error) { return testReadToken, nil }) + if _, status := h.connect(t, "", "api.github.com:443"); status != http.StatusProxyAuthRequired { + t.Fatalf("no credentials: status = %d, want 407", status) + } + if _, status := h.connect(t, SandboxReadProxyUser+":not-a-key", "api.github.com:443"); status != http.StatusProxyAuthRequired { + t.Fatalf("wrong key: status = %d, want 407", status) + } + user, release := enrollKey(t, h.relay, "deps") + if _, status := h.connect(t, user, "api.github.com:443"); status != 200 { + t.Fatalf("live key: status = %d, want 200", status) + } + release() + if _, status := h.connect(t, user, "api.github.com:443"); status != http.StatusProxyAuthRequired { + t.Fatalf("released key: status = %d, want 407", status) + } + if h.dials.Load() != 0 { + t.Fatal("no request was made, so no upstream dial was expected") + } +} + +func TestSandboxReadRelayRefusesNon443Connect(t *testing.T) { + h := newRelayHarness(t, nil) + user, release := enrollKey(t, h.relay, "deps") + defer release() + if _, status := h.connect(t, user, "api.github.com:8080"); status != http.StatusForbidden { + t.Fatalf("status = %d, want 403", status) + } +} + +func TestSandboxReadRelayNoTokenForwardsAnonymously(t *testing.T) { + h := newRelayHarness(t, func() (string, error) { return "", fmt.Errorf("mint failed") }) + user, release := enrollKey(t, h.relay, "deps") + defer release() + tc, _ := h.connect(t, user, "api.github.com:443") + _ = roundTrip(t, tc, "GET", "api.github.com", "/rate_limit", map[string]string{"Authorization": "token AGENT_SUPPLIED"}) + if got := h.seen()[0].Header.Get("Authorization"); got != "" { + t.Fatalf("upstream Authorization = %q, want none: the agent's header must not ride through", got) + } +} + +func TestSandboxReadRelayEnrollCarriesNoGitHubToken(t *testing.T) { + h := newRelayHarness(t, func() (string, error) { return testReadToken, nil }) + proxyURL, bundle, release, err := h.relay.Enroll("deps") + if err != nil { + t.Fatal(err) + } + defer release() + if strings.Contains(proxyURL, testReadToken) || strings.Contains(string(bundle), testReadToken) { + t.Fatal("the GitHub token must never be in what a Job receives") + } + if !strings.HasPrefix(proxyURL, "http://"+SandboxReadProxyUser+":") || !strings.Contains(string(bundle), "BEGIN CERTIFICATE") { + t.Fatalf("unexpected enrollment: %q", proxyURL) + } +} + +func TestSandboxReadRelayTokenIsCached(t *testing.T) { + var calls atomic.Int32 + h := newRelayHarness(t, func() (string, error) { calls.Add(1); return testReadToken, nil }) + now := time.Now() + h.relay.now = func() time.Time { return now } + for i := 0; i < 3; i++ { + if h.relay.readToken() != testReadToken { + t.Fatal("no token") + } + } + if calls.Load() != 1 { + t.Fatalf("minted %d times, want 1", calls.Load()) + } + now = now.Add(sandboxReadTokenTTL + time.Second) + _ = h.relay.readToken() + if calls.Load() != 2 { + t.Fatalf("minted %d times after TTL, want 2", calls.Load()) + } +} + +func TestNewSandboxReadRelayRejectsCredentialedEndpoint(t *testing.T) { + p, _ := newTestProxyWithCA(t) + for _, ep := range []string{"", "ftp://x:1", "http://u:p@host:1", "host:1"} { + if _, err := NewSandboxReadRelay(p, SandboxReadRelayOptions{Endpoint: ep}); err == nil { + t.Errorf("endpoint %q accepted", ep) + } + } +} diff --git a/src/docs/sandbox-isolation.md b/src/docs/sandbox-isolation.md index 1251b2ec41..329c2da9dd 100644 --- a/src/docs/sandbox-isolation.md +++ b/src/docs/sandbox-isolation.md @@ -280,6 +280,34 @@ agent_sandbox: image_pull_secrets: [ registry-pull ] ``` +**Authenticated GitHub reads for Jobs (#18865).** A Job runs in its own pod, so by default every `api.github.com` call it makes is anonymous: GitHub allows 60 requests per hour per source IP, shared by every agent on the node, and `uv lock --upgrade` on a repo with git dependencies falls back to full clones when the API call fails. The opt-in GitHub read relay fixes that without giving the Job a credential. Default off: + +```yaml +agent_sandbox: + job: + github_read_proxy: + enabled: true + endpoint: http://hive.hive-system.svc:18445 # how Job pods reach hive; required + listen_addr: ":18445" # optional, this is the default + no_proxy: [ models.internal.example ] # optional extra direct hosts +``` + +With it on, hive listens on `listen_addr` and every `sandbox.runtime: job` kick (the agent run, not the `publish.verify` preflights) gets `HTTPS_PROXY` set to `endpoint` with a **per-kick relay key**, plus `SSL_CERT_FILE`, `GIT_SSL_CAINFO`, `REQUESTS_CA_BUNDLE`, `CURL_CA_BUNDLE` and `NODE_EXTRA_CA_CERTS` pointing at a trust bundle hive writes to `/.hive/github-read-ca.pem` (the system roots plus hive's proxy CA, so the Job still trusts the real internet). `localhost`, `.svc` and `.cluster.local` bypass the relay. The block is read from `agent_sandbox.job` only; a per-agent `job` block cannot enable or repoint it. You must expose the listener to the Job pods yourself: a Service for `listen_addr`, and a NetworkPolicy allowing the Job pods to reach it. + +What the relay does: + +| Host | Request | What hive does | +|---|---|---| +| `api.github.com`, `codeload.github.com`, `raw.githubusercontent.com` | `GET`, `HEAD` | Strips any `Authorization` the Job sent and attaches hive's read token. | +| `github.com` | `GET /.git/info/refs?service=git-upload-pack`, `POST /.git/git-upload-pack` (git fetch and clone, uv's git resolution) | Same, as Basic `x-access-token`. | +| `github.com` | other `GET`/`HEAD` (archives, release downloads) | Forwarded with no credential. | +| the four hosts above | every other method, GraphQL, `git-receive-pack` (push discovery included) | Refused with a 403 and never sent to GitHub. | +| any other host | `CONNECT` on 443 | Opaque tunnel, no interception and no credential, through the same destination guard as the agent proxy (loopback, link-local metadata and private ranges are refused). | + +The credential is the existing **read tier** (`shared-uid-read`: metadata, contents, actions, checks, pull requests and issues, all read), minted from the Hive App and narrowed to `project.repos`. It is never the write token and never enters the Job: the Job holds only the relay key, which authenticates it to hive's listener and is worthless to GitHub. Hive caches the token for 45 minutes and logs decisions, never token bytes. The key is revoked when the kick ends. When no read token can be minted (no App auth, a repo outside the installation), reads go out anonymously, as they did before; the Job never receives an error it would not have had without the relay. Public repositories outside `project.repos`, such as a git dependency, are readable with that token because GitHub serves public data to any authenticated caller. + +This is separate from `HIVE_PROXY_INJECT_GH_AUTH`, which governs the placeholder token of agents running on the hive pod; the relay does not need it and is not affected by it. Limits to know: a Job that sets its own `HTTPS_PROXY` or `NO_PROXY` can route around the relay and is back to anonymous reads; `publish.verify` Jobs do not use it; GraphQL reads from a Job are refused because the relay cannot prove a query is read-only. A shared git dependency cache ([#18865](https://github.com/hivecommons/hive/issues/18865), option 3) is a separate, complementary change. + **RBAC.** The spoke's ServiceAccount needs a Role for `batch/jobs` (create, get, list, delete), `pods` plus `pods/log` (get, list) and, optionally, `serviceaccounts` (get) so the Job service account's pull secrets are merged; the terminal viewer needs nothing beyond that same `pods/log` grant. The standalone kustomize base ships it as `sandbox-job-rbac.yaml`, and the hub provisioner grants it to every hosted spoke as `hive-sandbox-jobs`. Hive warns at boot and on reload (`config.AgentSandboxGateWarnings`) when a job-runtime agent resolves no `workspace_claim` or names a runtime hive does not have, and refuses the kick with the same message rather than failing it after the agent is marked running. diff --git a/src/docs/security-model.md b/src/docs/security-model.md index eb6d8f07ee..5332b35d52 100644 --- a/src/docs/security-model.md +++ b/src/docs/security-model.md @@ -73,6 +73,7 @@ are enabled on a given hive. | Forced egress through the proxy (iptables) | proxy bypass | `src/deploy/entrypoint.sh` | boot log; no `HIVE_PROXY_ADVISORY_OK` | | Tiered GitHub tokens per agent mode | blast radius of a leaked token | `src/crux/agentmode`, `src/crux/agent/manager_tokens.go` | token tier on the agent card | | Proxy-side credential injection (opt-in) | token theft | `HIVE_PROXY_INJECT_GH_AUTH` | `security.credentialInjection` in governor config | +| Sandbox Job GitHub read relay (opt-in) | anonymous 60/hr API cap; token in the Job | `agent_sandbox.job.github_read_proxy` | `sandbox GitHub read relay enabled` boot line | | Inference-key placeholder swap | provider key theft | `src/pkg/apiproxy` | agent env shows `sk-hive-` | | `gh` wrapper with per-agent restrictions | denied commands | `bin/gh-wrapper.sh` | `gh` resolves to `/usr/local/bin/gh` | | `hive-open-pr` / `hive-merge` relays | forged authorship, stale-commit merges | `bin/hive-merge.sh`, `bin/hive-open-pr.sh` | `POST /pulls` returns 403 for agents | diff --git a/src/pkg/config/crux_aliases.go b/src/pkg/config/crux_aliases.go index e86b3f3c0f..22ce1b1c37 100644 --- a/src/pkg/config/crux_aliases.go +++ b/src/pkg/config/crux_aliases.go @@ -36,6 +36,7 @@ type ( ReviewModelsConfig = cruxconfig.ReviewModelsConfig RotationConfig = cruxconfig.RotationConfig SandboxJobConfig = cruxconfig.SandboxJobConfig + SandboxGitHubReadProxy = cruxconfig.SandboxGitHubReadProxy SandboxPublishConfig = cruxconfig.SandboxPublishConfig SandboxJobResources = cruxconfig.SandboxJobResources SandboxJobToleration = cruxconfig.SandboxJobToleration diff --git a/src/pkg/dashboard/settings_parity_allowlist.txt b/src/pkg/dashboard/settings_parity_allowlist.txt index acf9474de7..959df25222 100644 --- a/src/pkg/dashboard/settings_parity_allowlist.txt +++ b/src/pkg/dashboard/settings_parity_allowlist.txt @@ -8,6 +8,10 @@ agent_sandbox.job.affinity.nodeAffinity.preferredDuringSchedulingIgnoredDuringEx agent_sandbox.job.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[].matchExpressions[].key # TODO #18809 agent_sandbox.job.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[].matchExpressions[].operator # TODO #18809 agent_sandbox.job.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[].matchExpressions[].values[] # TODO #18809 +agent_sandbox.job.github_read_proxy.enabled # TODO #18865 +agent_sandbox.job.github_read_proxy.endpoint # TODO #18865 +agent_sandbox.job.github_read_proxy.listen_addr # TODO #18865 +agent_sandbox.job.github_read_proxy.no_proxy[] # TODO #18865 agent_sandbox.job.image_pull_secrets[] # TODO #11854 agent_sandbox.job.node_selector. # TODO #11854 agent_sandbox.job.relay_image # TODO #11854 @@ -131,6 +135,10 @@ agents..sandbox.job.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDu agents..sandbox.job.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[].matchExpressions[].operator # TODO #18809 agents..sandbox.job.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[].matchExpressions[].values[] # TODO #18809 agents..sandbox.job.env_from_secrets[] # TODO #11854 +agents..sandbox.job.github_read_proxy.enabled # TODO #18865 +agents..sandbox.job.github_read_proxy.endpoint # TODO #18865 +agents..sandbox.job.github_read_proxy.listen_addr # TODO #18865 +agents..sandbox.job.github_read_proxy.no_proxy[] # TODO #18865 agents..sandbox.job.image_pull_secrets[] # TODO #11854 agents..sandbox.job.node_selector. # TODO #11854 agents..sandbox.job.relay_image # TODO #11854 From ce083e50156302425d4a4e2dc8ca79c96a3e479c Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Tue, 17 Nov 2026 22:39:01 -0500 Subject: [PATCH 2/8] =?UTF-8?q?=F0=9F=90=9B=20sandbox=20read=20relay:=20fi?= =?UTF-8?q?x=20compile,=20file-size=20ratchets,=20harden=20tunnel=20host?= =?UTF-8?q?=20check=20(#18865)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add SetSandboxGitHubReadRelay to crux.Host and implement it on every Host: Manager, the sidecar host (in-process-only), the cruxtest recorder, and the api HostSetters table. boot calls it through the interface. - Move SandboxGitHubReadRelay into the crux package (agent aliases it) and keep the Manager's relay wiring in one atomic value so the relay and its NO_PROXY hosts can never be read from different installs. - Keep agent/manager.go and config/agent_config.go under their ratchets by moving code out (ACMM level accessors, sandbox job types and the relay config) instead of raising any limit. - Refuse a request inside a tunnel whose Host or absolute URL names a host other than the CONNECT target, so the credential decision cannot be borrowed for another host. - Fix the boot test's Config literal (AgentSandbox is promoted). - Tests: relay logs never contain the token, relay key or agent header; keys are per kick; Host mismatch is refused without an upstream dial; manager install/clear copies noProxy. Signed-off-by: Andy Anderson --- src/cmd/hive/agent_runtime_sidecar.go | 8 ++ src/cmd/hive/sandbox_read_relay_boot_test.go | 11 +- src/crux/agent/manager.go | 20 +--- src/crux/agent/manager_acmm_level.go | 16 +++ src/crux/agent/manager_sandbox.go | 22 +--- src/crux/agent/sandbox_github_read_relay.go | 35 ++++-- .../agent/sandbox_github_read_relay_test.go | 24 ++++ src/crux/api/client/client_test.go | 2 +- src/crux/api/runtime.go | 5 +- src/crux/config/agent_config.go | 85 ------------- src/crux/config/sandbox_job_types.go | 90 ++++++++++++++ src/crux/cruxtest/recorder_knowledge.go | 7 ++ src/crux/host_wiring.go | 6 + src/crux/proxy/sandbox_read_relay.go | 24 ++++ src/crux/proxy/sandbox_read_relay_test.go | 112 ++++++++++++++++++ src/crux/sandbox_read_relay.go | 11 ++ 16 files changed, 343 insertions(+), 135 deletions(-) create mode 100644 src/crux/agent/manager_acmm_level.go create mode 100644 src/crux/config/sandbox_job_types.go create mode 100644 src/crux/sandbox_read_relay.go diff --git a/src/cmd/hive/agent_runtime_sidecar.go b/src/cmd/hive/agent_runtime_sidecar.go index 016e2f4723..2f4982c134 100644 --- a/src/cmd/hive/agent_runtime_sidecar.go +++ b/src/cmd/hive/agent_runtime_sidecar.go @@ -339,6 +339,14 @@ func (h *sidecarHost) SetKnowledgeAdviser(crux.KnowledgeAdviser) { h.inProcessOnly("SetKnowledgeAdviser") } +// SetSandboxGitHubReadRelay is in-process-only (#18865): the relay holds the +// read credential and Enroll is a Go call into hive's proxy, so neither +// crosses into a sidecar runtime. A sidecar runtime's Job kicks run without +// the relay and read GitHub as they did before. +func (h *sidecarHost) SetSandboxGitHubReadRelay(crux.SandboxGitHubReadRelay, []string) { + h.inProcessOnly("SetSandboxGitHubReadRelay") +} + // SetInferenceRoute and ClearInferenceRoute are cruxhost.InferenceRouter: // the callback's end of SetInferenceCallbacks. A route that arrives before // boot has installed the closures is dropped and logged; the runtime's diff --git a/src/cmd/hive/sandbox_read_relay_boot_test.go b/src/cmd/hive/sandbox_read_relay_boot_test.go index 10eb608c73..d18fdbc955 100644 --- a/src/cmd/hive/sandbox_read_relay_boot_test.go +++ b/src/cmd/hive/sandbox_read_relay_boot_test.go @@ -9,7 +9,16 @@ import ( // The sandbox GitHub read relay is default off (#18865): with no block, or a // disabled one, boot wires nothing and binds nothing. func TestWireSandboxGitHubReadRelayDefaultOff(t *testing.T) { - cfgs := []*config.Config{{}, {AgentSandbox: config.AgentSandboxConfig{Job: &config.SandboxJobConfig{}}}, {AgentSandbox: config.AgentSandboxConfig{Job: &config.SandboxJobConfig{GitHubReadProxy: &config.SandboxGitHubReadProxy{Enabled: false, Endpoint: "http://h:1"}}}}} + withJob := func(job *config.SandboxJobConfig) *config.Config { + c := &config.Config{} + c.AgentSandbox = config.AgentSandboxConfig{Job: job} + return c + } + cfgs := []*config.Config{ + {}, + withJob(&config.SandboxJobConfig{}), + withJob(&config.SandboxJobConfig{GitHubReadProxy: &config.SandboxGitHubReadProxy{Enabled: false, Endpoint: "http://h:1"}}), + } for i, cfg := range cfgs { b := &boot{cfg: cfg, logger: quietLogger()} b.wireSandboxGitHubReadRelay() // nil proxy and nil manager: must be a silent no-op diff --git a/src/crux/agent/manager.go b/src/crux/agent/manager.go index 88865a5570..cca02b9785 100644 --- a/src/crux/agent/manager.go +++ b/src/crux/agent/manager.go @@ -812,10 +812,7 @@ type Manager struct { sandboxPRClient PRCreator sandboxMutation effects.Boundary sandboxAuditCallback atomic.Pointer[func(agent, action, detail string)] - // sandboxReadRelay and sandboxReadRelayNoProxy route Job-runtime kicks' - // GitHub reads through the hive read relay (#18865); nil is off. - sandboxReadRelay atomic.Pointer[SandboxGitHubReadRelay] - sandboxReadRelayNoProxy atomic.Pointer[[]string] + sandboxReadRelay sandboxReadRelayState // terminal is the nil-safe pane/tmux boundary. Nil means use the real tmux // implementation, preserving zero-value Manager behavior in tests. @@ -840,21 +837,6 @@ type Manager struct { buildInfo func() (version, commit string) } -// SetACMMLevel updates the cached ACMM level used by agentMode() when -// launching agents. Call this whenever the ACMM level changes. -func (m *Manager) SetACMMLevel(level int) { - m.mu.Lock() - defer m.mu.Unlock() - m.project.ACMMLevel = level - m.removeAgentsBelowACMMGateLocked(level) -} - -func (m *Manager) GetACMMLevel() int { - m.mu.RLock() - defer m.mu.RUnlock() - return m.project.ACMMLevel -} - func NewManager(agents map[string]config.AgentConfig, logger *slog.Logger, project ProjectContext) *Manager { return NewManagerWithOptions(agents, logger, project) } diff --git a/src/crux/agent/manager_acmm_level.go b/src/crux/agent/manager_acmm_level.go new file mode 100644 index 0000000000..7972684d9e --- /dev/null +++ b/src/crux/agent/manager_acmm_level.go @@ -0,0 +1,16 @@ +package agent + +// SetACMMLevel updates the cached ACMM level used by agentMode() when +// launching agents. Call this whenever the ACMM level changes. +func (m *Manager) SetACMMLevel(level int) { + m.mu.Lock() + defer m.mu.Unlock() + m.project.ACMMLevel = level + m.removeAgentsBelowACMMGateLocked(level) +} + +func (m *Manager) GetACMMLevel() int { + m.mu.RLock() + defer m.mu.RUnlock() + return m.project.ACMMLevel +} diff --git a/src/crux/agent/manager_sandbox.go b/src/crux/agent/manager_sandbox.go index cd79e14f02..7532deeae4 100644 --- a/src/crux/agent/manager_sandbox.go +++ b/src/crux/agent/manager_sandbox.go @@ -57,20 +57,6 @@ func (m *Manager) SetSandboxConfig(cfg config.AgentSandboxConfig) { } } -// SetSandboxGitHubReadRelay installs (or, with nil, removes) the relay that -// carries sandbox Job GitHub reads (#18865). noProxy adds hosts Jobs reach -// directly. -func (m *Manager) SetSandboxGitHubReadRelay(relay SandboxGitHubReadRelay, noProxy []string) { - if relay == nil { - m.sandboxReadRelay.Store(nil) - m.sandboxReadRelayNoProxy.Store(nil) - return - } - np := append([]string(nil), noProxy...) - m.sandboxReadRelayNoProxy.Store(&np) - m.sandboxReadRelay.Store(&relay) -} - func (m *Manager) SetSandboxLauncher(l sandbox.Launcher) { m.mu.Lock() defer m.mu.Unlock() @@ -323,11 +309,9 @@ func (m *Manager) runSandboxKick(ctx context.Context, name, runtime string, spec Mutation: mutationBoundary, Logger: m.logger, } - if relay := m.sandboxReadRelay.Load(); relay != nil { - exec.GitHubReadRelay = *relay - if np := m.sandboxReadRelayNoProxy.Load(); np != nil { - exec.GitHubReadRelayNoProxy = *np - } + if in := m.sandboxReadRelay.cur.Load(); in != nil { + exec.GitHubReadRelay = in.relay + exec.GitHubReadRelayNoProxy = in.noProxy } res, err := exec.Run(ctx, spec) ended := time.Now() diff --git a/src/crux/agent/sandbox_github_read_relay.go b/src/crux/agent/sandbox_github_read_relay.go index cac55bfab4..a9160c4de1 100644 --- a/src/crux/agent/sandbox_github_read_relay.go +++ b/src/crux/agent/sandbox_github_read_relay.go @@ -6,19 +6,38 @@ import ( "path" "path/filepath" "strings" + "sync/atomic" + "github.com/hivecommons/hive/crux" "github.com/hivecommons/hive/crux/config" "github.com/hivecommons/hive/crux/sandbox" ) -// SandboxGitHubReadRelay is the hive-hosted relay a sandbox Job sends its -// GitHub reads through (#18865). Enroll admits one kick and returns the proxy -// URL for it (carrying a per-kick relay key, never a GitHub credential), the -// PEM trust bundle the Job needs to accept the relay's certificates, and a -// release func that revokes the key. It lives behind an interface so this -// package does not depend on the proxy. -type SandboxGitHubReadRelay interface { - Enroll(agentName string) (proxyURL string, caBundlePEM []byte, release func(), err error) +// SandboxGitHubReadRelay is the relay a sandbox Job sends its GitHub reads +// through (#18865); see crux.SandboxGitHubReadRelay. +type SandboxGitHubReadRelay = crux.SandboxGitHubReadRelay + +// sandboxReadRelayState is the Manager's relay wiring; the zero value is off. +// The relay and its extra NO_PROXY hosts are stored as one value so a reader +// never sees one from a previous install paired with the other. +type sandboxReadRelayState struct { + cur atomic.Pointer[sandboxReadRelayInstall] +} + +type sandboxReadRelayInstall struct { + relay SandboxGitHubReadRelay + noProxy []string +} + +// SetSandboxGitHubReadRelay installs (or, with nil, removes) the relay that +// carries sandbox Job GitHub reads (#18865). noProxy adds hosts Jobs reach +// directly. +func (m *Manager) SetSandboxGitHubReadRelay(relay SandboxGitHubReadRelay, noProxy []string) { + if relay == nil { + m.sandboxReadRelay.cur.Store(nil) + return + } + m.sandboxReadRelay.cur.Store(&sandboxReadRelayInstall{relay: relay, noProxy: append([]string(nil), noProxy...)}) } const ( diff --git a/src/crux/agent/sandbox_github_read_relay_test.go b/src/crux/agent/sandbox_github_read_relay_test.go index 5f60b58957..14f8d1db74 100644 --- a/src/crux/agent/sandbox_github_read_relay_test.go +++ b/src/crux/agent/sandbox_github_read_relay_test.go @@ -7,6 +7,7 @@ import ( "strings" "testing" + "github.com/hivecommons/hive/crux" "github.com/hivecommons/hive/crux/config" ) @@ -109,3 +110,26 @@ func TestMergeSandboxEnvDoesNotMutateInputs(t *testing.T) { t.Fatalf("merge of nothing = %v", got) } } + +// The relay is installed through crux.Host like every other host capability +// (no type assertion in boot). +func TestManagerSetSandboxGitHubReadRelayInstallsAndClears(t *testing.T) { + m := &Manager{} + var host crux.Host = m + relay := &fakeReadRelay{} + noProxy := []string{"internal.example"} + host.SetSandboxGitHubReadRelay(relay, noProxy) + noProxy[0] = "mutated.example" + + in := m.sandboxReadRelay.cur.Load() + if in == nil || in.relay != SandboxGitHubReadRelay(relay) { + t.Fatal("relay was not installed") + } + if len(in.noProxy) != 1 || in.noProxy[0] != "internal.example" { + t.Fatalf("noProxy = %v; the caller's slice must be copied", in.noProxy) + } + host.SetSandboxGitHubReadRelay(nil, []string{"ignored.example"}) + if m.sandboxReadRelay.cur.Load() != nil { + t.Fatal("a nil relay must switch the feature off") + } +} diff --git a/src/crux/api/client/client_test.go b/src/crux/api/client/client_test.go index 3e2b2443a4..8675983219 100644 --- a/src/crux/api/client/client_test.go +++ b/src/crux/api/client/client_test.go @@ -139,7 +139,7 @@ func TestParityHostVersusSetterTable(t *testing.T) { t.Errorf("api.HostSetters[%q] has unknown kind %q", name, row.Kind) } } - for _, name := range []string{"SetSpanStarter", "SetKnowledgeAdviser"} { + for _, name := range []string{"SetSpanStarter", "SetKnowledgeAdviser", "SetSandboxGitHubReadRelay"} { if api.HostSetters[name].Kind != api.HostSetterInProcessOnly { t.Errorf("%s is an in-process-only setter", name) } diff --git a/src/crux/api/runtime.go b/src/crux/api/runtime.go index a9e59524b9..aa1f868c69 100644 --- a/src/crux/api/runtime.go +++ b/src/crux/api/runtime.go @@ -826,6 +826,7 @@ var HostSetters = map[string]HostSetter{ "SetAuditSink": {HostSetterEvent, EventAudit}, "SetConditionReporter": {HostSetterEvent, EventCondition + ", " + hostViaConditions}, // In-process only. - "SetSpanStarter": {HostSetterInProcessOnly, "OpenTelemetry global tracer in the runtime process; trajectory events are independent of spans"}, - "SetKnowledgeAdviser": {HostSetterInProcessOnly, "the knowledge store lives in the hive process; a sidecar runtime runs without the mid-task knowledge advisor (hivecommons/hive#17052 phase 1)"}, + "SetSpanStarter": {HostSetterInProcessOnly, "OpenTelemetry global tracer in the runtime process; trajectory events are independent of spans"}, + "SetSandboxGitHubReadRelay": {HostSetterInProcessOnly, "the relay holds the read-only GitHub credential and Enroll is a Go call into hive's proxy; neither crosses into a runtime process, so a sidecar runtime's Job kicks read GitHub without the relay (hivecommons/hive#18865)"}, + "SetKnowledgeAdviser": {HostSetterInProcessOnly, "the knowledge store lives in the hive process; a sidecar runtime runs without the mid-task knowledge advisor (hivecommons/hive#17052 phase 1)"}, } diff --git a/src/crux/config/agent_config.go b/src/crux/config/agent_config.go index 1623a0052a..c30c99544f 100644 --- a/src/crux/config/agent_config.go +++ b/src/crux/config/agent_config.go @@ -1,8 +1,6 @@ package config import ( - "errors" - "net/url" "strings" "time" ) @@ -64,89 +62,6 @@ type SandboxJobConfig struct { GitHubReadProxy *SandboxGitHubReadProxy `yaml:"github_read_proxy,omitempty" json:"github_read_proxy,omitempty"` } -// Defaults and limits for SandboxGitHubReadProxy. -const ( - // DefaultSandboxGitHubReadProxyListen is where the relay listens when - // listen_addr is unset. All interfaces: a Job pod reaches hive over the pod - // network, not loopback. Reachability is the operator's Service and - // NetworkPolicy to grant. - DefaultSandboxGitHubReadProxyListen = ":18445" -) - -// SandboxGitHubReadProxy configures the sandbox GitHub read relay (#18865). -// Default off. With enabled: true, hive listens on listen_addr and every -// sandbox.runtime: job kick gets HTTPS_PROXY pointing at endpoint plus a -// per-kick relay key. The relay attaches a read-only, repo-scoped GitHub App -// token to GET/HEAD requests for api.github.com, codeload.github.com, -// raw.githubusercontent.com and the git fetch paths of github.com. Writes are -// refused and the token never enters the Job. -type SandboxGitHubReadProxy struct { - Enabled bool `yaml:"enabled,omitempty" json:"enabled,omitempty"` - // ListenAddr is the host:port hive binds. Empty means - // DefaultSandboxGitHubReadProxyListen. - ListenAddr string `yaml:"listen_addr,omitempty" json:"listen_addr,omitempty"` - // Endpoint is the http://host:port Job pods use to reach the relay, for - // example the in-namespace Service name. Required when enabled; it carries - // no credentials (hive adds a per-kick key itself). - Endpoint string `yaml:"endpoint,omitempty" json:"endpoint,omitempty"` - // NoProxy lists extra hosts or domains a Job reaches directly, added to - // the built-in cluster-local list (localhost, .svc, .cluster.local). - NoProxy []string `yaml:"no_proxy,omitempty" json:"no_proxy,omitempty"` -} - -// Listen returns the bind address with the default applied. -func (c *SandboxGitHubReadProxy) Listen() string { - if c == nil || strings.TrimSpace(c.ListenAddr) == "" { - return DefaultSandboxGitHubReadProxyListen - } - return strings.TrimSpace(c.ListenAddr) -} - -// Validate reports why an enabled block cannot work. A disabled or nil block -// is always valid. -func (c *SandboxGitHubReadProxy) Validate() error { - if c == nil || !c.Enabled { - return nil - } - raw := strings.TrimSpace(c.Endpoint) - if raw == "" { - return errors.New("agent_sandbox.job.github_read_proxy.endpoint is required when enabled (the http://host:port Job pods use to reach hive)") - } - u, err := url.Parse(raw) - if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" { - return errors.New("agent_sandbox.job.github_read_proxy.endpoint must be an http(s)://host:port URL") - } - if u.User != nil { - return errors.New("agent_sandbox.job.github_read_proxy.endpoint must not carry credentials; hive adds a per-kick key itself") - } - return nil -} - -// SandboxJobToleration mirrors the subset of a Kubernetes toleration an -// operator writes by hand. -type SandboxJobToleration struct { - Key string `yaml:"key,omitempty" json:"key,omitempty"` - Operator string `yaml:"operator,omitempty" json:"operator,omitempty"` - Value string `yaml:"value,omitempty" json:"value,omitempty"` - Effect string `yaml:"effect,omitempty" json:"effect,omitempty"` -} - -// SandboxJobResources mirrors Kubernetes resource requirements with string -// quantities so a device limit (`.com/: "2"`) passes through. -type SandboxJobResources struct { - Limits map[string]string `yaml:"limits,omitempty" json:"limits,omitempty"` - Requests map[string]string `yaml:"requests,omitempty" json:"requests,omitempty"` -} - -// SandboxJobVolume is an additional PVC mounted into the Job (a compile -// cache, a model store). -type SandboxJobVolume struct { - Name string `yaml:"name,omitempty" json:"name,omitempty"` - Claim string `yaml:"claim" json:"claim"` - MountPath string `yaml:"mount_path" json:"mount_path"` - ReadOnly bool `yaml:"read_only,omitempty" json:"read_only,omitempty"` -} - // SandboxRuntime returns the per-agent runtime override, then the global // runtime, then SandboxRuntimePodman. The value is returned as written; the // manager rejects one outside ValidSandboxRuntimes at kick time. diff --git a/src/crux/config/sandbox_job_types.go b/src/crux/config/sandbox_job_types.go new file mode 100644 index 0000000000..6b20aae562 --- /dev/null +++ b/src/crux/config/sandbox_job_types.go @@ -0,0 +1,90 @@ +package config + +import ( + "errors" + "net/url" + "strings" +) + +// SandboxJobToleration mirrors the subset of a Kubernetes toleration an +// operator writes by hand. +type SandboxJobToleration struct { + Key string `yaml:"key,omitempty" json:"key,omitempty"` + Operator string `yaml:"operator,omitempty" json:"operator,omitempty"` + Value string `yaml:"value,omitempty" json:"value,omitempty"` + Effect string `yaml:"effect,omitempty" json:"effect,omitempty"` +} + +// SandboxJobResources mirrors Kubernetes resource requirements with string +// quantities so a device limit (`.com/: "2"`) passes through. +type SandboxJobResources struct { + Limits map[string]string `yaml:"limits,omitempty" json:"limits,omitempty"` + Requests map[string]string `yaml:"requests,omitempty" json:"requests,omitempty"` +} + +// SandboxJobVolume is an additional PVC mounted into the Job (a compile +// cache, a model store). +type SandboxJobVolume struct { + Name string `yaml:"name,omitempty" json:"name,omitempty"` + Claim string `yaml:"claim" json:"claim"` + MountPath string `yaml:"mount_path" json:"mount_path"` + ReadOnly bool `yaml:"read_only,omitempty" json:"read_only,omitempty"` +} + +// Defaults and limits for SandboxGitHubReadProxy. +const ( + // DefaultSandboxGitHubReadProxyListen is where the relay listens when + // listen_addr is unset. All interfaces: a Job pod reaches hive over the pod + // network, not loopback. Reachability is the operator's Service and + // NetworkPolicy to grant. + DefaultSandboxGitHubReadProxyListen = ":18445" +) + +// SandboxGitHubReadProxy configures the sandbox GitHub read relay (#18865). +// Default off. With enabled: true, hive listens on listen_addr and every +// sandbox.runtime: job kick gets HTTPS_PROXY pointing at endpoint plus a +// per-kick relay key. The relay attaches a read-only, repo-scoped GitHub App +// token to GET/HEAD requests for api.github.com, codeload.github.com, +// raw.githubusercontent.com and the git fetch paths of github.com. Writes are +// refused and the token never enters the Job. +type SandboxGitHubReadProxy struct { + Enabled bool `yaml:"enabled,omitempty" json:"enabled,omitempty"` + // ListenAddr is the host:port hive binds. Empty means + // DefaultSandboxGitHubReadProxyListen. + ListenAddr string `yaml:"listen_addr,omitempty" json:"listen_addr,omitempty"` + // Endpoint is the http://host:port Job pods use to reach the relay, for + // example the in-namespace Service name. Required when enabled; it carries + // no credentials (hive adds a per-kick key itself). + Endpoint string `yaml:"endpoint,omitempty" json:"endpoint,omitempty"` + // NoProxy lists extra hosts or domains a Job reaches directly, added to + // the built-in cluster-local list (localhost, .svc, .cluster.local). + NoProxy []string `yaml:"no_proxy,omitempty" json:"no_proxy,omitempty"` +} + +// Listen returns the bind address with the default applied. +func (c *SandboxGitHubReadProxy) Listen() string { + if c == nil || strings.TrimSpace(c.ListenAddr) == "" { + return DefaultSandboxGitHubReadProxyListen + } + return strings.TrimSpace(c.ListenAddr) +} + +// Validate reports why an enabled block cannot work. A disabled or nil block +// is always valid. +func (c *SandboxGitHubReadProxy) Validate() error { + if c == nil || !c.Enabled { + return nil + } + raw := strings.TrimSpace(c.Endpoint) + if raw == "" { + return errors.New("agent_sandbox.job.github_read_proxy.endpoint is required when enabled (the http://host:port Job pods use to reach hive)") + } + u, err := url.Parse(raw) + if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" { + return errors.New("agent_sandbox.job.github_read_proxy.endpoint must be an http(s)://host:port URL") + } + if u.User != nil { + return errors.New("agent_sandbox.job.github_read_proxy.endpoint must not carry credentials; hive adds a per-kick key itself") + } + return nil +} diff --git a/src/crux/cruxtest/recorder_knowledge.go b/src/crux/cruxtest/recorder_knowledge.go index 01ed6d9085..23ee0f5eeb 100644 --- a/src/crux/cruxtest/recorder_knowledge.go +++ b/src/crux/cruxtest/recorder_knowledge.go @@ -8,3 +8,10 @@ import "github.com/hivecommons/hive/crux" func (r *Recorder) SetKnowledgeAdviser(adviser crux.KnowledgeAdviser) { r.record("SetKnowledgeAdviser", adviser != nil) } + +// SetSandboxGitHubReadRelay records the sandbox GitHub read relay seam +// (hivecommons/hive#18865): whether one was installed, and its extra NO_PROXY +// hosts. It lives here for the same size-ratchet reason as above. +func (r *Recorder) SetSandboxGitHubReadRelay(relay crux.SandboxGitHubReadRelay, noProxy []string) { + r.record("SetSandboxGitHubReadRelay", relay != nil, noProxy) +} diff --git a/src/crux/host_wiring.go b/src/crux/host_wiring.go index f87cc17c7a..cc1c3ac2d4 100644 --- a/src/crux/host_wiring.go +++ b/src/crux/host_wiring.go @@ -63,4 +63,10 @@ type Host interface { // (hivecommons/hive#17052). In-process-only in phase 1: the knowledge // store lives in the hive process and the adviser is a Go call into it. SetKnowledgeAdviser(adviser KnowledgeAdviser) + // SetSandboxGitHubReadRelay installs (nil removes) the relay that carries + // sandbox Job GitHub reads (hivecommons/hive#18865); noProxy adds hosts + // Jobs reach directly. In-process-only: Enroll is a Go call into hive's + // proxy, and the relay holds the read credential, which must not cross + // into a runtime process. + SetSandboxGitHubReadRelay(relay SandboxGitHubReadRelay, noProxy []string) } diff --git a/src/crux/proxy/sandbox_read_relay.go b/src/crux/proxy/sandbox_read_relay.go index 5ded360cbf..150420d88e 100644 --- a/src/crux/proxy/sandbox_read_relay.go +++ b/src/crux/proxy/sandbox_read_relay.go @@ -357,6 +357,12 @@ func (r *SandboxReadRelay) intercept(conn net.Conn, host, agentName string) { return } d := SandboxReadPolicy(host, req.Method, req.URL.Path, req.URL.RawQuery) + if d.Allowed && !sandboxReadRequestTargetsHost(req, host) { + // The policy and the credential decision were made for the CONNECT + // host; a request naming a different Host or an absolute URL for + // another host must not ride that decision. + d = SandboxReadDecision{Reason: "the request names a different host than the CONNECT target"} + } if !d.Allowed { r.p.logger.Info("sandbox read relay: request refused", "agent", agentName, "host", host, "method", req.Method) writeRelayRefusal(client, d.Reason) @@ -400,6 +406,24 @@ func (r *SandboxReadRelay) intercept(conn net.Conn, host, agentName string) { } } +// sandboxReadRequestTargetsHost reports whether the Host header and any +// absolute request URL name the host the tunnel was opened for. +func sandboxReadRequestTargetsHost(req *http.Request, host string) bool { + want := canonicalHost(host) + for _, got := range []string{req.Host, req.URL.Host} { + if got == "" { + continue + } + if h, _, err := net.SplitHostPort(got); err == nil { + got = h + } + if canonicalHost(got) != want { + return false + } + } + return true +} + // dialUpstream opens the TLS leg to a read host. It is a method so // tests can drive the relay without network access by replacing r.dial. func (r *SandboxReadRelay) dialUpstream(host string) (net.Conn, error) { diff --git a/src/crux/proxy/sandbox_read_relay_test.go b/src/crux/proxy/sandbox_read_relay_test.go index 9ed48e8cb2..23d7f5691b 100644 --- a/src/crux/proxy/sandbox_read_relay_test.go +++ b/src/crux/proxy/sandbox_read_relay_test.go @@ -6,6 +6,7 @@ import ( "crypto/x509" "encoding/base64" "fmt" + "log/slog" "net" "net/http" "strings" @@ -131,8 +132,18 @@ type relayHarness struct { } func newRelayHarness(t *testing.T, token SandboxReadTokenFunc) *relayHarness { + t.Helper() + return newRelayHarnessWithLogger(t, token, nil) +} + +// newRelayHarnessWithLogger is newRelayHarness with the proxy's logger replaced +// before the listener starts, so a test can read back everything the relay logs. +func newRelayHarnessWithLogger(t *testing.T, token SandboxReadTokenFunc, logger *slog.Logger) *relayHarness { t.Helper() p, _ := newTestProxyWithCA(t) + if logger != nil { + p.logger = logger + } relay, err := NewSandboxReadRelay(p, SandboxReadRelayOptions{Endpoint: "http://hive.example.svc:18445", Token: token}) if err != nil { t.Fatal(err) @@ -382,3 +393,104 @@ func TestNewSandboxReadRelayRejectsCredentialedEndpoint(t *testing.T) { } } } + +// A request inside the tunnel that names another host (Host header or absolute +// URL) must not borrow the CONNECT host's credential decision. +func TestSandboxReadRelayRefusesHostMismatchInsideTunnel(t *testing.T) { + h := newRelayHarness(t, func() (string, error) { return testReadToken, nil }) + user, release := enrollKey(t, h.relay, "deps") + defer release() + + for _, tc := range []struct{ name, hostHeader, target string }{ + {"host header", "github.com", "/repos/o/r"}, + {"absolute url", "api.github.com", "https://evil.example/repos/o/r"}, + } { + t.Run(tc.name, func(t *testing.T) { + conn, status := h.connect(t, user, "api.github.com:443") + if status != 200 { + t.Fatalf("CONNECT status = %d", status) + } + _, err := fmt.Fprintf(conn, "GET %s HTTP/1.1\r\nHost: %s\r\n\r\n", tc.target, tc.hostHeader) + if err != nil { + t.Fatal(err) + } + resp, err := http.ReadResponse(bufio.NewReader(conn), nil) + if err != nil { + t.Fatal(err) + } + if resp.StatusCode != http.StatusForbidden { + t.Fatalf("status = %d, want 403", resp.StatusCode) + } + if len(h.seen()) != 0 { + t.Fatal("a mismatched-host request reached upstream with the CONNECT host's credential") + } + }) + } +} + +type lockedBuffer struct { + mu sync.Mutex + b strings.Builder +} + +func (l *lockedBuffer) Write(p []byte) (int, error) { + l.mu.Lock() + defer l.mu.Unlock() + return l.b.Write(p) +} + +func (l *lockedBuffer) String() string { + l.mu.Lock() + defer l.mu.Unlock() + return l.b.String() +} + +// The read token and the per-kick relay key must never appear in the relay's +// logs, at the most verbose level, across injected, refused and anonymous +// requests. +func TestSandboxReadRelayNeverLogsTokenOrKey(t *testing.T) { + var logs lockedBuffer + logger := slog.New(slog.NewTextHandler(&logs, &slog.HandlerOptions{Level: slog.LevelDebug})) + h := newRelayHarnessWithLogger(t, func() (string, error) { return testReadToken, nil }, logger) + user, release := enrollKey(t, h.relay, "deps") + defer release() + _, key, _ := strings.Cut(user, ":") + + tc, _ := h.connect(t, user, "api.github.com:443") + _ = roundTrip(t, tc, "GET", "api.github.com", "/repos/o/r", map[string]string{"Authorization": "token AGENT_SUPPLIED"}) + tc2, _ := h.connect(t, user, "api.github.com:443") + _ = roundTrip(t, tc2, "POST", "api.github.com", "/repos/o/r/issues", map[string]string{"Authorization": "token AGENT_SUPPLIED"}) + _, _ = h.connect(t, SandboxReadProxyUser+":"+key+"x", "api.github.com:443") + + out := logs.String() + if out == "" { + t.Fatal("expected the relay to log its decisions") + } + for _, secret := range []string{testReadToken, key, "AGENT_SUPPLIED"} { + if strings.Contains(out, secret) { + t.Fatalf("relay logs contain a secret (%q):\n%s", secret, out) + } + } +} + +// Each kick gets its own key, keys are unguessable, and releasing one kick +// leaves the others live. +func TestSandboxReadRelayKeysArePerKick(t *testing.T) { + h := newRelayHarness(t, nil) + u1, rel1 := enrollKey(t, h.relay, "deps") + u2, rel2 := enrollKey(t, h.relay, "deps") + defer rel2() + if u1 == u2 { + t.Fatal("two kicks of one agent shared a relay key") + } + if _, key, _ := strings.Cut(u1, ":"); len(key) < 2*sandboxReadKeyBytes { + t.Fatalf("relay key too short: %d chars", len(key)) + } + rel1() + if _, status := h.connect(t, u1, "api.github.com:443"); status != http.StatusProxyAuthRequired { + t.Fatalf("released kick key: status = %d, want 407", status) + } + if _, status := h.connect(t, u2, "api.github.com:443"); status != 200 { + t.Fatalf("other kick key: status = %d, want 200", status) + } +} diff --git a/src/crux/sandbox_read_relay.go b/src/crux/sandbox_read_relay.go new file mode 100644 index 0000000000..77d048172d --- /dev/null +++ b/src/crux/sandbox_read_relay.go @@ -0,0 +1,11 @@ +package crux + +// SandboxGitHubReadRelay is the hive-hosted relay a sandbox Job sends its +// GitHub reads through (hivecommons/hive#18865). Enroll admits one kick and +// returns the proxy URL for it (carrying a per-kick relay key, never a GitHub +// credential), the PEM trust bundle the Job needs to accept the relay's +// certificates, and a release func that revokes the key. It is an interface so +// the runtime does not depend on hive's proxy. +type SandboxGitHubReadRelay interface { + Enroll(agentName string) (proxyURL string, caBundlePEM []byte, release func(), err error) +} From e3bc0a4c96af35f0f145ff5ce42b13dac2ed13f8 Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Tue, 17 Nov 2026 22:45:10 -0500 Subject: [PATCH 3/8] =?UTF-8?q?=F0=9F=90=9B=20crux/api:=20move=20HostSette?= =?UTF-8?q?r=20type=20out=20of=20runtime.go=20to=20stay=20under=20its=20ra?= =?UTF-8?q?tchet=20(#18865)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Andy Anderson --- src/crux/api/host_setter.go | 7 +++++++ src/crux/api/runtime.go | 6 ------ 2 files changed, 7 insertions(+), 6 deletions(-) create mode 100644 src/crux/api/host_setter.go diff --git a/src/crux/api/host_setter.go b/src/crux/api/host_setter.go new file mode 100644 index 0000000000..804fb28907 --- /dev/null +++ b/src/crux/api/host_setter.go @@ -0,0 +1,7 @@ +package api + +// HostSetter is one row of HostSetters. +type HostSetter struct { + Kind HostSetterKind + Via string +} diff --git a/src/crux/api/runtime.go b/src/crux/api/runtime.go index aa1f868c69..a38c795725 100644 --- a/src/crux/api/runtime.go +++ b/src/crux/api/runtime.go @@ -766,12 +766,6 @@ const ( HostSetterInProcessOnly HostSetterKind = "inprocess" ) -// HostSetter is one row of HostSetters. -type HostSetter struct { - Kind HostSetterKind - Via string -} - // Callback endpoint routes the table refers to, spelled here because // crux/api/callback imports this package; its tests hold the two in step. const ( From 4486b7416b6beea519f0518db4f5642a612d308f Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Tue, 17 Nov 2026 22:52:14 -0500 Subject: [PATCH 4/8] =?UTF-8?q?=F0=9F=90=9B=20sandbox=20read=20relay=20boo?= =?UTF-8?q?t:=20Serve=20always=20returns=20an=20error=20(staticcheck=20SA4?= =?UTF-8?q?023)=20(#18865)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Andy Anderson --- src/cmd/hive/sandbox_read_relay_boot.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/cmd/hive/sandbox_read_relay_boot.go b/src/cmd/hive/sandbox_read_relay_boot.go index 18f2a652ba..5cff3a185e 100644 --- a/src/cmd/hive/sandbox_read_relay_boot.go +++ b/src/cmd/hive/sandbox_read_relay_boot.go @@ -66,9 +66,8 @@ func (b *boot) wireSandboxGitHubReadRelay() { return } go func() { - if err := relay.Serve(ln); err != nil { - b.logger.Error("sandbox GitHub read relay stopped", "error", err.Error()) - } + // Serve only returns when the listener fails, always with an error. + b.logger.Error("sandbox GitHub read relay stopped", "error", relay.Serve(ln).Error()) }() b.agentMgr.SetSandboxGitHubReadRelay(relay, rp.NoProxy) b.logger.Info("sandbox GitHub read relay enabled: Job GitHub reads use a read-only, repo-scoped token held by hive; writes are refused", "listen", rp.Listen()) From dc8bf36301dcbb79b50422c6ef067dcb3bff5c6c Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Tue, 17 Nov 2026 23:08:04 -0500 Subject: [PATCH 5/8] =?UTF-8?q?=F0=9F=90=9B=20sandbox=20read=20relay:=20re?= =?UTF-8?q?member=20a=20failed=20token=20mint=20for=2030s=20instead=20of?= =?UTF-8?q?=20retrying=20per=20request=20(#18865)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Andy Anderson --- src/crux/proxy/sandbox_read_relay.go | 14 ++++++++--- src/crux/proxy/sandbox_read_relay_test.go | 30 +++++++++++++++++++++++ 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/src/crux/proxy/sandbox_read_relay.go b/src/crux/proxy/sandbox_read_relay.go index 150420d88e..bf0d8ed5ee 100644 --- a/src/crux/proxy/sandbox_read_relay.go +++ b/src/crux/proxy/sandbox_read_relay.go @@ -51,7 +51,11 @@ const ( // sandboxReadTokenTTL is how long a fetched read token is reused. GitHub // installation tokens live an hour; refreshing at 45 minutes keeps a // margin for a request already in flight. - sandboxReadTokenTTL = 45 * time.Minute + sandboxReadTokenTTL = 45 * time.Minute + // sandboxReadTokenRetryAfter is how long a failed mint is remembered, so a + // token-endpoint outage costs one attempt per interval rather than one + // (up to 20s, serialized) attempt per request. + sandboxReadTokenRetryAfter = 30 * time.Second sandboxReadHTTPSPort = "443" sandboxReadRefsSuffix = "/info/refs" sandboxReadUploadPackSuffix = "/git-upload-pack" @@ -150,6 +154,7 @@ type SandboxReadRelay struct { tokMu sync.Mutex tok string tokFetched time.Time + tokFailed time.Time // last failed mint; zero when the last mint succeeded now func() time.Time // dial overrides the upstream TLS dial (tests). dial func(host string) (net.Conn, error) @@ -226,6 +231,9 @@ func (r *SandboxReadRelay) readToken() string { if r.tok != "" && r.now().Sub(r.tokFetched) < sandboxReadTokenTTL { return r.tok } + if r.tok == "" && !r.tokFailed.IsZero() && r.now().Sub(r.tokFailed) < sandboxReadTokenRetryAfter { + return "" + } tok, err := r.opts.Token() if err != nil || tok == "" { reason := "empty token" @@ -233,10 +241,10 @@ func (r *SandboxReadRelay) readToken() string { reason = err.Error() } r.p.logger.Warn("sandbox read relay: no read credential available; forwarding reads unauthenticated", "reason", reason) - r.tok = "" + r.tok, r.tokFailed = "", r.now() return "" } - r.tok, r.tokFetched = tok, r.now() + r.tok, r.tokFetched, r.tokFailed = tok, r.now(), time.Time{} return tok } diff --git a/src/crux/proxy/sandbox_read_relay_test.go b/src/crux/proxy/sandbox_read_relay_test.go index 23d7f5691b..d2acb0452b 100644 --- a/src/crux/proxy/sandbox_read_relay_test.go +++ b/src/crux/proxy/sandbox_read_relay_test.go @@ -494,3 +494,33 @@ func TestSandboxReadRelayKeysArePerKick(t *testing.T) { t.Fatalf("other kick key: status = %d, want 200", status) } } + +// A failing token endpoint is not hammered: one mint attempt per retry +// interval, requests in between go out anonymously, and recovery is picked up. +func TestSandboxReadRelayFailedMintIsNotRetriedPerRequest(t *testing.T) { + var calls atomic.Int32 + var fail atomic.Bool + fail.Store(true) + h := newRelayHarness(t, func() (string, error) { + calls.Add(1) + if fail.Load() { + return "", fmt.Errorf("mint failed") + } + return testReadToken, nil + }) + now := time.Now() + h.relay.now = func() time.Time { return now } + for i := 0; i < 5; i++ { + if tok := h.relay.readToken(); tok != "" { + t.Fatalf("token = %q during an outage", tok) + } + } + if calls.Load() != 1 { + t.Fatalf("mint attempted %d times for 5 requests during an outage, want 1", calls.Load()) + } + fail.Store(false) + now = now.Add(sandboxReadTokenRetryAfter + time.Second) + if h.relay.readToken() != testReadToken { + t.Fatal("relay did not recover after the retry interval") + } +} From 5e0333d77ccdf339fb71064d3fac702191d5b67a Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Tue, 17 Nov 2026 23:19:39 -0500 Subject: [PATCH 6/8] =?UTF-8?q?=F0=9F=90=9B=20sandbox=20read=20relay=20tes?= =?UTF-8?q?t:=20reuse=20the=20package's=20lockedBuffer=20after=20the=20v5?= =?UTF-8?q?=20merge=20(#18865)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Andy Anderson --- src/crux/proxy/sandbox_read_relay_test.go | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/src/crux/proxy/sandbox_read_relay_test.go b/src/crux/proxy/sandbox_read_relay_test.go index d2acb0452b..cbff9d3814 100644 --- a/src/crux/proxy/sandbox_read_relay_test.go +++ b/src/crux/proxy/sandbox_read_relay_test.go @@ -428,23 +428,6 @@ func TestSandboxReadRelayRefusesHostMismatchInsideTunnel(t *testing.T) { } } -type lockedBuffer struct { - mu sync.Mutex - b strings.Builder -} - -func (l *lockedBuffer) Write(p []byte) (int, error) { - l.mu.Lock() - defer l.mu.Unlock() - return l.b.Write(p) -} - -func (l *lockedBuffer) String() string { - l.mu.Lock() - defer l.mu.Unlock() - return l.b.String() -} - // The read token and the per-kick relay key must never appear in the relay's // logs, at the most verbose level, across injected, refused and anonymous // requests. From f05bcfdbcc9876fc76d23cdde6afd824ee583e79 Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Tue, 17 Nov 2026 23:41:02 -0500 Subject: [PATCH 7/8] =?UTF-8?q?=F0=9F=94=92=20sandbox=20read=20relay:=20TL?= =?UTF-8?q?S-only=20listener,=20no=20default=20bind,=20https=20endpoint,?= =?UTF-8?q?=20NetworkPolicy=20docs=20(#18865)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Owner decision on the needs-decision review item: - listen_addr has no default; enabling without it fails config validation with an error naming the key (default off is unchanged). - The listener serves TLS only, with a certificate issued by the existing hive proxy CA for the endpoint host (DNS or IP SAN); endpoint must be https:// and http:// is rejected at validation. No new CA or key on disk. - Jobs verify the https proxy against the hive CA bundle, including libcurl's separate proxy CA setting for git (http.proxySSLCAInfo). - Tests: plain HTTP is refused and its key never logged, no key still gets 407 over TLS, certificate SANs, config validation, env. - Docs: example NetworkPolicy, two-layer statement, client TLS-to-proxy limits, example config, changelog fragment. Follow-up for the tunnel that outlives its key: #18895. Signed-off-by: Andy Anderson --- .../added-18865-sandbox-github-read-relay.md | 2 +- src/cmd/hive/sandbox_read_relay_boot.go | 9 +- src/crux/agent/sandbox_github_read_relay.go | 12 +++ .../agent/sandbox_github_read_relay_test.go | 21 ++++- .../config/sandbox_github_read_proxy_test.go | 48 +++++++++-- src/crux/config/sandbox_job_types.go | 48 ++++++----- .../kubejob/github_read_relay_env_test.go | 2 +- src/crux/proxy/github_proxy_ca.go | 6 ++ src/crux/proxy/sandbox_read_relay.go | 24 +++++- src/crux/proxy/sandbox_read_relay_test.go | 84 +++++++++++++++++-- src/docs/sandbox-isolation.md | 34 +++++++- src/hive.yaml.example | 17 ++++ ...sandbox_github_read_proxy_validate_test.go | 40 +++++++++ src/pkg/config/validate.go | 5 ++ 14 files changed, 304 insertions(+), 48 deletions(-) create mode 100644 src/pkg/config/sandbox_github_read_proxy_validate_test.go diff --git a/changelog.d/added-18865-sandbox-github-read-relay.md b/changelog.d/added-18865-sandbox-github-read-relay.md index 5dff0d98a2..af3b658be5 100644 --- a/changelog.d/added-18865-sandbox-github-read-relay.md +++ b/changelog.d/added-18865-sandbox-github-read-relay.md @@ -1 +1 @@ -- Sandbox Jobs can now make authenticated GitHub reads through a hive-hosted read relay instead of hitting GitHub's 60-requests-per-hour anonymous cap (so `uv lock --upgrade` resolves git dependencies without falling back to full clones). Opt in with `agent_sandbox.job.github_read_proxy` (`enabled`, `endpoint`, optional `listen_addr` and `no_proxy`); default off. The relay attaches a read-only, repo-scoped App token (the existing `shared-uid-read` tier) to `GET`/`HEAD` for `api.github.com`, `codeload.github.com` and `raw.githubusercontent.com` and to git fetch on `github.com`, refuses every write, GraphQL and push discovery with a 403, tunnels every other host untouched, and never puts a GitHub credential in the Job: the Job holds only a per-kick relay key that is revoked when the kick ends. See [Sandbox isolation](src/docs/sandbox-isolation.md) ([#18865](https://github.com/hivecommons/hive/issues/18865)). +- Sandbox Jobs can now make authenticated GitHub reads through a hive-hosted read relay instead of hitting GitHub's 60-requests-per-hour anonymous cap (so `uv lock --upgrade` resolves git dependencies without falling back to full clones). Opt in with `agent_sandbox.job.github_read_proxy` (`enabled`, `listen_addr` and `endpoint`, optional `no_proxy`); default off, with no default bind: `listen_addr` is required when enabled, and `endpoint` must be `https://` because the listener serves TLS only, with a certificate from hive's proxy CA for the endpoint host. Pair it with a NetworkPolicy (example in the docs). The relay attaches a read-only, repo-scoped App token (the existing `shared-uid-read` tier) to `GET`/`HEAD` for `api.github.com`, `codeload.github.com` and `raw.githubusercontent.com` and to git fetch on `github.com`, refuses every write, GraphQL and push discovery with a 403, tunnels every other host untouched, and never puts a GitHub credential in the Job: the Job holds only a per-kick relay key that is revoked when the kick ends. See [Sandbox isolation](src/docs/sandbox-isolation.md) ([#18865](https://github.com/hivecommons/hive/issues/18865)). diff --git a/src/cmd/hive/sandbox_read_relay_boot.go b/src/cmd/hive/sandbox_read_relay_boot.go index 5cff3a185e..00a8f8ee78 100644 --- a/src/cmd/hive/sandbox_read_relay_boot.go +++ b/src/cmd/hive/sandbox_read_relay_boot.go @@ -2,6 +2,7 @@ package main import ( "context" + "crypto/tls" "errors" "net" "os" @@ -65,12 +66,16 @@ func (b *boot) wireSandboxGitHubReadRelay() { b.logger.Error("sandbox GitHub read relay not started: cannot listen", "addr", rp.Listen(), "error", err.Error()) return } + // TLS only: the per-kick key is carried in Proxy-Authorization, so a + // plain-HTTP connection is never served. The certificate comes from the + // hive proxy CA for the endpoint's host. + tlsLn := tls.NewListener(ln, relay.TLSConfig()) go func() { // Serve only returns when the listener fails, always with an error. - b.logger.Error("sandbox GitHub read relay stopped", "error", relay.Serve(ln).Error()) + b.logger.Error("sandbox GitHub read relay stopped", "error", relay.Serve(tlsLn).Error()) }() b.agentMgr.SetSandboxGitHubReadRelay(relay, rp.NoProxy) - b.logger.Info("sandbox GitHub read relay enabled: Job GitHub reads use a read-only, repo-scoped token held by hive; writes are refused", "listen", rp.Listen()) + b.logger.Info("sandbox GitHub read relay enabled (TLS): Job GitHub reads use a read-only, repo-scoped token held by hive; writes are refused", "listen", rp.Listen(), "endpoint", rp.Endpoint) } // sandboxReadToken mints the READ identity the relay attaches: the diff --git a/src/crux/agent/sandbox_github_read_relay.go b/src/crux/agent/sandbox_github_read_relay.go index a9160c4de1..fbac649096 100644 --- a/src/crux/agent/sandbox_github_read_relay.go +++ b/src/crux/agent/sandbox_github_read_relay.go @@ -71,10 +71,18 @@ func sandboxReadEnv(proxyURL, caPath string, extraNoProxy []string) map[string]s "REQUESTS_CA_BUNDLE": caPath, "CURL_CA_BUNDLE": caPath, "NODE_EXTRA_CA_CERTS": caPath, + // libcurl (so git) verifies an https PROXY against its own CA setting, + // separate from the one for the origin server: GIT_SSL_CAINFO alone + // does not cover it. Git reads this entry from the environment. + "GIT_CONFIG_COUNT": "1", + "GIT_CONFIG_KEY_0": "http.proxySSLCAInfo", + "GIT_CONFIG_VALUE_0": caPath, } } // mergeSandboxEnv returns base overlaid with extra; neither input is mutated. +// GIT_CONFIG_* entries in extra are dropped when base already defines +// GIT_CONFIG_COUNT, so the relay never rewrites a git config the Job has set. func mergeSandboxEnv(base, extra map[string]string) map[string]string { if len(extra) == 0 { return base @@ -83,7 +91,11 @@ func mergeSandboxEnv(base, extra map[string]string) map[string]string { for k, v := range base { out[k] = v } + _, baseGitConfig := base["GIT_CONFIG_COUNT"] for k, v := range extra { + if baseGitConfig && strings.HasPrefix(k, "GIT_CONFIG_") { + continue + } out[k] = v } return out diff --git a/src/crux/agent/sandbox_github_read_relay_test.go b/src/crux/agent/sandbox_github_read_relay_test.go index 14f8d1db74..3794ec9167 100644 --- a/src/crux/agent/sandbox_github_read_relay_test.go +++ b/src/crux/agent/sandbox_github_read_relay_test.go @@ -24,11 +24,11 @@ func (f *fakeReadRelay) Enroll(agentName string) (string, []byte, func(), error) return "", nil, nil, f.err } f.enrolled = append(f.enrolled, agentName) - return "http://hive-sandbox:kickkey@hive.ns.svc:18445", []byte("-----BEGIN CERTIFICATE-----\nX\n-----END CERTIFICATE-----\n"), func() { f.released++ }, nil + return "https://hive-sandbox:kickkey@hive.ns.svc:18445", []byte("-----BEGIN CERTIFICATE-----\nX\n-----END CERTIFICATE-----\n"), func() { f.released++ }, nil } func TestSandboxReadEnvCarriesNoGitHubToken(t *testing.T) { - env := sandboxReadEnv("http://hive-sandbox:kickkey@hive.ns.svc:18445", "/workspace/.hive/github-read-ca.pem", []string{"internal.example"}) + env := sandboxReadEnv("https://hive-sandbox:kickkey@hive.ns.svc:18445", "/workspace/.hive/github-read-ca.pem", []string{"internal.example"}) for k, v := range env { if strings.Contains(v, relayTestGitHubToken) || strings.Contains(strings.ToUpper(k), "TOKEN") { t.Errorf("env %s=%s looks like a GitHub credential", k, v) @@ -45,6 +45,23 @@ func TestSandboxReadEnvCarriesNoGitHubToken(t *testing.T) { if _, ok := env["HTTP_PROXY"]; ok { t.Error("plain HTTP must not be proxied") } + if !strings.HasPrefix(env["HTTPS_PROXY"], "https://") { + t.Errorf("HTTPS_PROXY = %q, want an https:// proxy URL so the key is not sent in cleartext", env["HTTPS_PROXY"]) + } + if env["GIT_CONFIG_KEY_0"] != "http.proxySSLCAInfo" || env["GIT_CONFIG_VALUE_0"] != env["GIT_SSL_CAINFO"] { + t.Errorf("git must verify the https proxy against the hive CA bundle: %v", env) + } +} + +func TestMergeSandboxEnvKeepsJobGitConfig(t *testing.T) { + base := map[string]string{"GIT_CONFIG_COUNT": "2", "GIT_CONFIG_KEY_0": "a.b", "GIT_CONFIG_VALUE_0": "c"} + got := mergeSandboxEnv(base, sandboxReadEnv("https://u:k@h:1", "/ca.pem", nil)) + if got["GIT_CONFIG_COUNT"] != "2" || got["GIT_CONFIG_KEY_0"] != "a.b" { + t.Fatalf("the relay overwrote the Job's git config: %v", got) + } + if got["HTTPS_PROXY"] == "" { + t.Fatal("the proxy env must still be applied") + } } func TestEnrollGitHubReadRelayOffByDefault(t *testing.T) { diff --git a/src/crux/config/sandbox_github_read_proxy_test.go b/src/crux/config/sandbox_github_read_proxy_test.go index c3059f3b7b..e8eb90a470 100644 --- a/src/crux/config/sandbox_github_read_proxy_test.go +++ b/src/crux/config/sandbox_github_read_proxy_test.go @@ -1,6 +1,9 @@ package config -import "testing" +import ( + "strings" + "testing" +) func TestSandboxGitHubReadProxyDefaultOff(t *testing.T) { var nilCfg *SandboxGitHubReadProxy @@ -10,8 +13,27 @@ func TestSandboxGitHubReadProxyDefaultOff(t *testing.T) { if err := (&SandboxGitHubReadProxy{}).Validate(); err != nil { t.Fatalf("disabled block must be valid: %v", err) } - if got := nilCfg.Listen(); got != DefaultSandboxGitHubReadProxyListen { - t.Fatalf("Listen = %q", got) + // Disabled stays valid however the other keys look: the feature is off. + if err := (&SandboxGitHubReadProxy{Endpoint: "http://h:1"}).Validate(); err != nil { + t.Fatalf("disabled block must be valid: %v", err) + } +} + +// There is no default bind: enabling without listen_addr is a config error +// that names the key. +func TestSandboxGitHubReadProxyHasNoDefaultListen(t *testing.T) { + if got := (*SandboxGitHubReadProxy)(nil).Listen(); got != "" { + t.Fatalf("Listen = %q, want no default", got) + } + err := (&SandboxGitHubReadProxy{Enabled: true, Endpoint: "https://hive.ns.svc:8443"}).Validate() + if err == nil || !strings.Contains(err.Error(), "agent_sandbox.job.github_read_proxy.listen_addr") { + t.Fatalf("err = %v, want one naming listen_addr", err) + } + if err := (&SandboxGitHubReadProxy{Enabled: true, ListenAddr: " ", Endpoint: "https://h:1"}).Validate(); err == nil { + t.Fatal("a blank listen_addr must be refused") + } + if err := (&SandboxGitHubReadProxy{Enabled: true, ListenAddr: "8443", Endpoint: "https://h:1"}).Validate(); err == nil { + t.Fatal("listen_addr without a port must be refused") } } @@ -20,22 +42,30 @@ func TestSandboxGitHubReadProxyValidate(t *testing.T) { endpoint string ok bool }{ - {"http://hive.ns.svc:18445", true}, - {"https://hive.example.com:18445", true}, + {"https://hive.ns.svc:8443", true}, + {"https://10.20.30.40:8443", true}, + {"http://hive.ns.svc:8443", false}, // cleartext is refused {"", false}, - {"hive.ns.svc:18445", false}, + {"hive.ns.svc:8443", false}, {"ftp://hive:1", false}, - {"http://user:pw@hive:1", false}, + {"https://user:pw@hive:1", false}, } { - err := (&SandboxGitHubReadProxy{Enabled: true, Endpoint: tc.endpoint}).Validate() + err := (&SandboxGitHubReadProxy{Enabled: true, ListenAddr: ":8443", Endpoint: tc.endpoint}).Validate() if (err == nil) != tc.ok { t.Errorf("endpoint %q: err = %v, want ok=%v", tc.endpoint, err, tc.ok) } + if err != nil && !strings.Contains(err.Error(), "agent_sandbox.job.github_read_proxy.endpoint") { + t.Errorf("endpoint %q: error %q does not name the key", tc.endpoint, err) + } + } + err := (&SandboxGitHubReadProxy{Enabled: true, ListenAddr: ":8443", Endpoint: "http://h:1"}).Validate() + if err == nil || !strings.Contains(err.Error(), "https") { + t.Fatalf("http endpoint error = %v, want one saying https is required", err) } } func TestSandboxJobGitHubReadProxyIsHiveWide(t *testing.T) { - global := AgentSandboxConfig{Job: &SandboxJobConfig{GitHubReadProxy: &SandboxGitHubReadProxy{Enabled: true, Endpoint: "http://h:1"}}} + global := AgentSandboxConfig{Job: &SandboxJobConfig{GitHubReadProxy: &SandboxGitHubReadProxy{Enabled: true, ListenAddr: ":8443", Endpoint: "https://h:1"}}} agent := &AgentConfig{Sandbox: &AgentSandboxOverride{Job: &SandboxJobConfig{GitHubReadProxy: &SandboxGitHubReadProxy{Enabled: false}}}} if got := agent.SandboxJob(global); got.GitHubReadProxy == nil || !got.GitHubReadProxy.Enabled { t.Fatalf("a per-agent block must not override the hive-wide relay setting: %+v", got.GitHubReadProxy) diff --git a/src/crux/config/sandbox_job_types.go b/src/crux/config/sandbox_job_types.go index 6b20aae562..b70a78a6f6 100644 --- a/src/crux/config/sandbox_job_types.go +++ b/src/crux/config/sandbox_job_types.go @@ -2,6 +2,7 @@ package config import ( "errors" + "net" "net/url" "strings" ) @@ -31,17 +32,8 @@ type SandboxJobVolume struct { ReadOnly bool `yaml:"read_only,omitempty" json:"read_only,omitempty"` } -// Defaults and limits for SandboxGitHubReadProxy. -const ( - // DefaultSandboxGitHubReadProxyListen is where the relay listens when - // listen_addr is unset. All interfaces: a Job pod reaches hive over the pod - // network, not loopback. Reachability is the operator's Service and - // NetworkPolicy to grant. - DefaultSandboxGitHubReadProxyListen = ":18445" -) - // SandboxGitHubReadProxy configures the sandbox GitHub read relay (#18865). -// Default off. With enabled: true, hive listens on listen_addr and every +// Default off. With enabled: true, hive listens (TLS) on listen_addr and every // sandbox.runtime: job kick gets HTTPS_PROXY pointing at endpoint plus a // per-kick relay key. The relay attaches a read-only, repo-scoped GitHub App // token to GET/HEAD requests for api.github.com, codeload.github.com, @@ -49,22 +41,26 @@ const ( // refused and the token never enters the Job. type SandboxGitHubReadProxy struct { Enabled bool `yaml:"enabled,omitempty" json:"enabled,omitempty"` - // ListenAddr is the host:port hive binds. Empty means - // DefaultSandboxGitHubReadProxyListen. + // ListenAddr is the host:port hive binds. There is deliberately NO + // default: binding an unrequested port on every interface is a decision + // the operator makes. Required when enabled. ListenAddr string `yaml:"listen_addr,omitempty" json:"listen_addr,omitempty"` - // Endpoint is the http://host:port Job pods use to reach the relay, for - // example the in-namespace Service name. Required when enabled; it carries - // no credentials (hive adds a per-kick key itself). + // Endpoint is the https://host:port Job pods use to reach the relay, for + // example the in-namespace Service name. Required when enabled and must + // be https: the listener serves TLS with a certificate issued by hive's + // proxy CA for this host. It carries no credentials (hive adds a per-kick + // key itself). Endpoint string `yaml:"endpoint,omitempty" json:"endpoint,omitempty"` // NoProxy lists extra hosts or domains a Job reaches directly, added to // the built-in cluster-local list (localhost, .svc, .cluster.local). NoProxy []string `yaml:"no_proxy,omitempty" json:"no_proxy,omitempty"` } -// Listen returns the bind address with the default applied. +// Listen returns the trimmed bind address; empty when unset (there is no +// default). func (c *SandboxGitHubReadProxy) Listen() string { - if c == nil || strings.TrimSpace(c.ListenAddr) == "" { - return DefaultSandboxGitHubReadProxyListen + if c == nil { + return "" } return strings.TrimSpace(c.ListenAddr) } @@ -75,13 +71,23 @@ func (c *SandboxGitHubReadProxy) Validate() error { if c == nil || !c.Enabled { return nil } + listen := c.Listen() + if listen == "" { + return errors.New("agent_sandbox.job.github_read_proxy.listen_addr is required when enabled (there is no default; set the host:port hive should bind, for example :8443)") + } + if _, port, err := net.SplitHostPort(listen); err != nil || port == "" { + return errors.New("agent_sandbox.job.github_read_proxy.listen_addr must be host:port (the host may be empty to bind every interface)") + } raw := strings.TrimSpace(c.Endpoint) if raw == "" { - return errors.New("agent_sandbox.job.github_read_proxy.endpoint is required when enabled (the http://host:port Job pods use to reach hive)") + return errors.New("agent_sandbox.job.github_read_proxy.endpoint is required when enabled (the https://host:port Job pods use to reach hive)") } u, err := url.Parse(raw) - if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" { - return errors.New("agent_sandbox.job.github_read_proxy.endpoint must be an http(s)://host:port URL") + if err != nil || u.Host == "" || u.Hostname() == "" { + return errors.New("agent_sandbox.job.github_read_proxy.endpoint must be an https://host:port URL") + } + if u.Scheme != "https" { + return errors.New("agent_sandbox.job.github_read_proxy.endpoint must use https:// (the per-kick key must not cross the pod network in cleartext; hive serves TLS with a certificate from its proxy CA)") } if u.User != nil { return errors.New("agent_sandbox.job.github_read_proxy.endpoint must not carry credentials; hive adds a per-kick key itself") diff --git a/src/crux/kubejob/github_read_relay_env_test.go b/src/crux/kubejob/github_read_relay_env_test.go index c711583067..76ef5940b5 100644 --- a/src/crux/kubejob/github_read_relay_env_test.go +++ b/src/crux/kubejob/github_read_relay_env_test.go @@ -22,7 +22,7 @@ func TestManifestRelayEnvCarriesNoGitHubToken(t *testing.T) { Image: "i", EnvAllowlist: []string{"GITHUB_TOKEN", "GH_TOKEN", "HIVE_GITHUB_TOKEN"}, Env: map[string]string{ - "HTTPS_PROXY": "http://hive-sandbox:relaykey@hive.ns.svc:18445", + "HTTPS_PROXY": "https://hive-sandbox:relaykey@hive.ns.svc:18445", "SSL_CERT_FILE": "/workspace/.hive/github-read-ca.pem", "GIT_SSL_CAINFO": "/workspace/.hive/github-read-ca.pem", "GITHUB_TOKEN": secret, diff --git a/src/crux/proxy/github_proxy_ca.go b/src/crux/proxy/github_proxy_ca.go index 05818270a0..34425808b8 100644 --- a/src/crux/proxy/github_proxy_ca.go +++ b/src/crux/proxy/github_proxy_ca.go @@ -11,6 +11,7 @@ import ( "fmt" "log/slog" "math/big" + "net" "os" "path/filepath" "time" @@ -50,6 +51,11 @@ func (p *GitHubProxy) forgeCert(host string) (tls.Certificate, error) { ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, DNSNames: []string{host}, } + if ip := net.ParseIP(host); ip != nil { + // An IP-literal name is only valid as an IP SAN. + template.DNSNames = nil + template.IPAddresses = []net.IP{ip} + } caKey, ok := p.caCert.PrivateKey.(*ecdsa.PrivateKey) if !ok { diff --git a/src/crux/proxy/sandbox_read_relay.go b/src/crux/proxy/sandbox_read_relay.go index bf0d8ed5ee..b8994db3eb 100644 --- a/src/crux/proxy/sandbox_read_relay.go +++ b/src/crux/proxy/sandbox_read_relay.go @@ -167,12 +167,32 @@ func NewSandboxReadRelay(p *GitHubProxy, opts SandboxReadRelayOptions) (*Sandbox return nil, errors.New("sandbox read relay: no proxy") } u, err := url.Parse(strings.TrimSpace(opts.Endpoint)) - if err != nil || u.Host == "" || (u.Scheme != "http" && u.Scheme != "https") || u.User != nil { - return nil, errors.New("sandbox read relay: endpoint must be an http(s)://host:port URL without credentials") + if err != nil || u.Hostname() == "" || u.Scheme != "https" || u.User != nil { + return nil, errors.New("sandbox read relay: endpoint must be an https://host:port URL without credentials") } return &SandboxReadRelay{p: p, opts: opts, endpoint: u, keys: map[[sha256.Size]byte]string{}, now: time.Now}, nil } +// TLSConfig is the server configuration for the relay's listener. The relay +// is only ever served over TLS: the per-kick key travels in Proxy-Authorization, +// so a plain-HTTP connection is never accepted. The certificate is issued by +// the hive proxy CA (the same CA and forging the GitHub interception uses, no +// new CA and no new key material on disk) for the endpoint's host, as a DNS or +// IP SAN. Jobs already trust that CA through the bundle Enroll returns. +func (r *SandboxReadRelay) TLSConfig() *tls.Config { + host := canonicalHost(r.endpoint.Hostname()) + return &tls.Config{ + MinVersion: tls.VersionTLS12, + GetCertificate: func(*tls.ClientHelloInfo) (*tls.Certificate, error) { + cert, err := r.p.forgeCert(host) + if err != nil { + return nil, fmt.Errorf("sandbox read relay: issuing listener certificate for %q: %w", host, err) + } + return &cert, nil + }, + } +} + // Enroll admits one kick: it mints a per-kick relay key and returns the proxy // URL the Job should use, the CA bundle it must trust and a release func that // revokes the key. The key authenticates the Job to this listener only. diff --git a/src/crux/proxy/sandbox_read_relay_test.go b/src/crux/proxy/sandbox_read_relay_test.go index cbff9d3814..f4411f173b 100644 --- a/src/crux/proxy/sandbox_read_relay_test.go +++ b/src/crux/proxy/sandbox_read_relay_test.go @@ -16,7 +16,11 @@ import ( "time" ) -const testReadToken = "ghs_READONLYTOKEN" +const ( + testReadToken = "ghs_READONLYTOKEN" + // relayTestHost is the endpoint host the test relay's certificate is for. + relayTestHost = "hive.example.svc" +) func TestSandboxReadPolicy(t *testing.T) { cases := []struct { @@ -144,7 +148,7 @@ func newRelayHarnessWithLogger(t *testing.T, token SandboxReadTokenFunc, logger if logger != nil { p.logger = logger } - relay, err := NewSandboxReadRelay(p, SandboxReadRelayOptions{Endpoint: "http://hive.example.svc:18445", Token: token}) + relay, err := NewSandboxReadRelay(p, SandboxReadRelayOptions{Endpoint: "https://" + relayTestHost + ":18445", Token: token}) if err != nil { t.Fatal(err) } @@ -174,7 +178,8 @@ func newRelayHarnessWithLogger(t *testing.T, token SandboxReadTokenFunc, logger t.Fatal(err) } t.Cleanup(func() { _ = ln.Close() }) - go func() { _ = relay.Serve(ln) }() + // The relay is only ever served over TLS, exactly as boot does it. + go func() { _ = relay.Serve(tls.NewListener(ln, relay.TLSConfig())) }() h.addr = ln.Addr().String() return h } @@ -189,10 +194,13 @@ func (h *relayHarness) seen() []*http.Request { // conn, or the raw HTTP status when the relay refused the CONNECT. func (h *relayHarness) connect(t *testing.T, proxyURLUser, target string) (*tls.Conn, int) { t.Helper() - conn, err := net.DialTimeout("tcp", h.addr, 5*time.Second) + raw, err := net.DialTimeout("tcp", h.addr, 5*time.Second) if err != nil { t.Fatal(err) } + // Leg 1: TLS to the relay itself, verified against the hive CA for the + // endpoint host. Leg 2 (below) is the CONNECT tunnel's own TLS. + var conn net.Conn = tls.Client(raw, &tls.Config{RootCAs: h.pool, ServerName: relayTestHost}) t.Cleanup(func() { _ = conn.Close() }) _ = conn.SetDeadline(time.Now().Add(10 * time.Second)) hdr := "" @@ -238,7 +246,7 @@ func enrollKey(t *testing.T, r *SandboxReadRelay, agentName string) (userinfo st if err != nil { t.Fatal(err) } - rest := strings.TrimPrefix(proxyURL, "http://") + rest := strings.TrimPrefix(proxyURL, "https://") userinfo, _, _ = strings.Cut(rest, "@") return userinfo, rel } @@ -360,7 +368,7 @@ func TestSandboxReadRelayEnrollCarriesNoGitHubToken(t *testing.T) { if strings.Contains(proxyURL, testReadToken) || strings.Contains(string(bundle), testReadToken) { t.Fatal("the GitHub token must never be in what a Job receives") } - if !strings.HasPrefix(proxyURL, "http://"+SandboxReadProxyUser+":") || !strings.Contains(string(bundle), "BEGIN CERTIFICATE") { + if !strings.HasPrefix(proxyURL, "https://"+SandboxReadProxyUser+":") || !strings.Contains(string(bundle), "BEGIN CERTIFICATE") { t.Fatalf("unexpected enrollment: %q", proxyURL) } } @@ -387,7 +395,7 @@ func TestSandboxReadRelayTokenIsCached(t *testing.T) { func TestNewSandboxReadRelayRejectsCredentialedEndpoint(t *testing.T) { p, _ := newTestProxyWithCA(t) - for _, ep := range []string{"", "ftp://x:1", "http://u:p@host:1", "host:1"} { + for _, ep := range []string{"", "ftp://x:1", "http://host:1", "https://u:p@host:1", "host:1"} { if _, err := NewSandboxReadRelay(p, SandboxReadRelayOptions{Endpoint: ep}); err == nil { t.Errorf("endpoint %q accepted", ep) } @@ -507,3 +515,65 @@ func TestSandboxReadRelayFailedMintIsNotRetriedPerRequest(t *testing.T) { t.Fatal("relay did not recover after the retry interval") } } + +// The listener serves TLS only. A plain-HTTP client, even one holding a live +// key, gets no tunnel and no proxy handling, and the key it sent in the clear +// is never logged. +func TestSandboxReadRelayRefusesPlainHTTP(t *testing.T) { + var logs lockedBuffer + logger := slog.New(slog.NewTextHandler(&logs, &slog.HandlerOptions{Level: slog.LevelDebug})) + h := newRelayHarnessWithLogger(t, func() (string, error) { return testReadToken, nil }, logger) + user, release := enrollKey(t, h.relay, "deps") + defer release() + _, key, _ := strings.Cut(user, ":") + + conn, err := net.DialTimeout("tcp", h.addr, 5*time.Second) + if err != nil { + t.Fatal(err) + } + defer func() { _ = conn.Close() }() + _ = conn.SetDeadline(time.Now().Add(5 * time.Second)) + auth := base64.StdEncoding.EncodeToString([]byte(user)) + _, _ = fmt.Fprintf(conn, "CONNECT api.github.com:443 HTTP/1.1\r\nHost: api.github.com:443\r\nProxy-Authorization: Basic %s\r\n\r\n", auth) + reply, _ := bufio.NewReader(conn).ReadString('\n') + if strings.Contains(reply, "200") || strings.Contains(reply, "407") { + t.Fatalf("plain HTTP reached the relay's proxy handler: %q", reply) + } + if h.dials.Load() != 0 || len(h.seen()) != 0 { + t.Fatal("a plain-HTTP request produced upstream traffic") + } + if strings.Contains(logs.String(), key) || strings.Contains(logs.String(), auth) { + t.Fatalf("the key sent over plain HTTP was logged:\n%s", logs.String()) + } +} + +// The listener certificate is issued by the hive CA for the endpoint host, as a +// DNS or IP SAN, and for nothing else. +func TestSandboxReadRelayCertificateHasEndpointSAN(t *testing.T) { + for _, tc := range []struct{ endpoint, name string }{ + {"https://hive.example.svc:18445", "hive.example.svc"}, + {"https://10.20.30.40:18445", "10.20.30.40"}, + } { + p, _ := newTestProxyWithCA(t) + relay, err := NewSandboxReadRelay(p, SandboxReadRelayOptions{Endpoint: tc.endpoint}) + if err != nil { + t.Fatal(err) + } + cert, err := relay.TLSConfig().GetCertificate(nil) + if err != nil { + t.Fatal(err) + } + leaf, err := x509.ParseCertificate(cert.Certificate[0]) + if err != nil { + t.Fatal(err) + } + pool := x509.NewCertPool() + pool.AddCert(p.caX509) + if _, err := leaf.Verify(x509.VerifyOptions{DNSName: tc.name, Roots: pool}); err != nil { + t.Errorf("%s: certificate does not verify for the endpoint host against the hive CA: %v", tc.endpoint, err) + } + if _, err := leaf.Verify(x509.VerifyOptions{DNSName: "other.example", Roots: pool}); err == nil { + t.Errorf("%s: certificate verifies for an unrelated host", tc.endpoint) + } + } +} diff --git a/src/docs/sandbox-isolation.md b/src/docs/sandbox-isolation.md index 329c2da9dd..c75f147d48 100644 --- a/src/docs/sandbox-isolation.md +++ b/src/docs/sandbox-isolation.md @@ -287,12 +287,40 @@ agent_sandbox: job: github_read_proxy: enabled: true - endpoint: http://hive.hive-system.svc:18445 # how Job pods reach hive; required - listen_addr: ":18445" # optional, this is the default + listen_addr: ":8443" # required when enabled; there is no default + endpoint: https://hive.hive-system.svc:8443 # how Job pods reach hive; required, https only no_proxy: [ models.internal.example ] # optional extra direct hosts ``` -With it on, hive listens on `listen_addr` and every `sandbox.runtime: job` kick (the agent run, not the `publish.verify` preflights) gets `HTTPS_PROXY` set to `endpoint` with a **per-kick relay key**, plus `SSL_CERT_FILE`, `GIT_SSL_CAINFO`, `REQUESTS_CA_BUNDLE`, `CURL_CA_BUNDLE` and `NODE_EXTRA_CA_CERTS` pointing at a trust bundle hive writes to `/.hive/github-read-ca.pem` (the system roots plus hive's proxy CA, so the Job still trusts the real internet). `localhost`, `.svc` and `.cluster.local` bypass the relay. The block is read from `agent_sandbox.job` only; a per-agent `job` block cannot enable or repoint it. You must expose the listener to the Job pods yourself: a Service for `listen_addr`, and a NetworkPolicy allowing the Job pods to reach it. +The feature is **off by default**. When `enabled` is true, `listen_addr` is required (there is no default bind, so hive never opens a port you did not name) and `endpoint` must be `https://`; config validation fails with an error naming the key otherwise. With it on, hive listens on `listen_addr`, **serving TLS only** with a certificate issued by hive's existing proxy CA for the `endpoint` host (a DNS or IP SAN; no new CA, and no CA key is written anywhere new), and every `sandbox.runtime: job` kick (the agent run, not the `publish.verify` preflights) gets `HTTPS_PROXY` set to `endpoint` (an `https://` proxy URL) with a **per-kick relay key**, plus `SSL_CERT_FILE`, `GIT_SSL_CAINFO`, `REQUESTS_CA_BUNDLE`, `CURL_CA_BUNDLE` and `NODE_EXTRA_CA_CERTS` pointing at a trust bundle hive writes to `/.hive/github-read-ca.pem` (the system roots plus hive's proxy CA, so the Job still trusts the real internet). `localhost`, `.svc` and `.cluster.local` bypass the relay. The block is read from `agent_sandbox.job` only; a per-agent `job` block cannot enable or repoint it. You must expose the listener to the Job pods yourself: a Service for `listen_addr`, and a NetworkPolicy allowing only the Job pods to reach it. + +**Two layers guard the listener.** The first is the NetworkPolicy: it decides which pods can open a connection to the relay port at all. The second is the per-kick relay key: a connection that gets through must still present a live key, or it gets a 407 and nothing else. The key is random per kick, held by hive only as a hash, never logged, and revoked when the kick ends. Because the listener serves TLS only, the key never crosses the pod network in cleartext, and a plain-HTTP connection is dropped before any proxy handling. Example NetworkPolicy, applied in hive's namespace, allowing ingress on the relay port only from pods in the Jobs' namespace (replace the placeholders): + +```yaml +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: hive-github-read-relay + namespace: +spec: + podSelector: + matchLabels: + : + policyTypes: [ Ingress ] + ingress: + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: + podSelector: + matchLabels: + app.kubernetes.io/managed-by: hive-kubejob + ports: + - protocol: TCP + port: 8443 # your listen_addr port +``` + +Client support for an `https://` proxy URL varies. Go, Node (through its proxy agents) and libcurl 7.52 or newer (curl, and git built on it) can speak TLS to a proxy. libcurl checks an https proxy against its own CA setting rather than the origin's, so hive also sets `http.proxySSLCAInfo` through `GIT_CONFIG_COUNT`/`GIT_CONFIG_KEY_0`/`GIT_CONFIG_VALUE_0` (skipped when the Job already defines `GIT_CONFIG_COUNT`). Some clients, for example Python HTTP stacks built on urllib3 older than 2.0, cannot speak TLS to a proxy; for those the Job's reads go out anonymously, as without the relay. Hive does not downgrade to a cleartext proxy for them. What the relay does: diff --git a/src/hive.yaml.example b/src/hive.yaml.example index 33d2522128..7c211822cd 100644 --- a/src/hive.yaml.example +++ b/src/hive.yaml.example @@ -1369,5 +1369,22 @@ data: # enabled: true # false (or env HIVE_CUSTOMERS_ENABLED=false) opens no store and hides the section # inference: # enabled: false # true (or env HIVE_CUSTOMERS_INFERENCE_ENABLED=true) suggests goals and satisfaction with the meetings model; default off +<<<<<<< HEAD # avatars: # enabled: true # false (or env HIVE_CUSTOMERS_AVATARS_ENABLED=false) shows initials and makes no browser request to github.com for pictures; default on +======= + +# Sandbox Job GitHub read relay (docs/sandbox-isolation.md): lets sandbox Jobs +# make authenticated, read-only GitHub reads without ever holding a credential. +# Default off. There is NO default bind: with enabled: true, listen_addr is +# required, and endpoint must be https:// (hive serves TLS with a certificate +# from its proxy CA for the endpoint host). Hive-wide only; a per-agent job +# block cannot enable or repoint it. Pair it with a NetworkPolicy. +# agent_sandbox: +# job: +# github_read_proxy: +# enabled: false +# listen_addr: ":8443" # required when enabled; no default +# endpoint: https://hive.hive-system.svc:8443 # how Job pods reach hive; required when enabled +# no_proxy: [] # optional extra hosts Jobs reach directly +>>>>>>> a0579d902 (🔒 sandbox read relay: TLS-only listener, no default bind, https endpoint, NetworkPolicy docs (#18865)) diff --git a/src/pkg/config/sandbox_github_read_proxy_validate_test.go b/src/pkg/config/sandbox_github_read_proxy_validate_test.go new file mode 100644 index 0000000000..f2a87bc5fa --- /dev/null +++ b/src/pkg/config/sandbox_github_read_proxy_validate_test.go @@ -0,0 +1,40 @@ +package config + +import ( + "strings" + "testing" +) + +// Config validation (the load and save paths) enforces the read relay's +// contract: default off, no default bind, https endpoint only (#18865). +func TestConfigValidateSandboxGitHubReadProxy(t *testing.T) { + withProxy := func(rp *SandboxGitHubReadProxy) *Config { + c := &Config{} + c.AgentSandbox = AgentSandboxConfig{Job: &SandboxJobConfig{GitHubReadProxy: rp}} + return c + } + for _, tc := range []struct { + name string + cfg *Config + wantErr string + }{ + {"no block", &Config{}, ""}, + {"disabled", withProxy(&SandboxGitHubReadProxy{Endpoint: "http://h:1"}), ""}, + {"enabled and complete", withProxy(&SandboxGitHubReadProxy{Enabled: true, ListenAddr: ":8443", Endpoint: "https://hive.ns.svc:8443"}), ""}, + {"enabled without listen_addr", withProxy(&SandboxGitHubReadProxy{Enabled: true, Endpoint: "https://hive.ns.svc:8443"}), "github_read_proxy.listen_addr"}, + {"enabled with http endpoint", withProxy(&SandboxGitHubReadProxy{Enabled: true, ListenAddr: ":8443", Endpoint: "http://hive.ns.svc:8443"}), "https"}, + } { + t.Run(tc.name, func(t *testing.T) { + err := tc.cfg.validateJobAffinities() + if tc.wantErr == "" { + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + return + } + if err == nil || !strings.Contains(err.Error(), tc.wantErr) { + t.Fatalf("err = %v, want one containing %q", err, tc.wantErr) + } + }) + } +} diff --git a/src/pkg/config/validate.go b/src/pkg/config/validate.go index 10e1718e9f..bdcf616579 100644 --- a/src/pkg/config/validate.go +++ b/src/pkg/config/validate.go @@ -410,6 +410,11 @@ func (c *Config) ValidateWithOptions(opts ValidateOptions) error { } func (c *Config) validateJobAffinities() error { + if c.AgentSandbox.Job != nil { + if err := c.AgentSandbox.Job.GitHubReadProxy.Validate(); err != nil { + return err + } + } if c.AgentSandbox.Job != nil { if err := cruxconfig.ValidateSandboxJobAffinity(c.AgentSandbox.Job.Affinity); err != nil { return fmt.Errorf("agent_sandbox.job.%w", err) From fdf97f05b1bf977537d511b0fbf53288586edd01 Mon Sep 17 00:00:00 2001 From: Andrew Anderson Date: Tue, 17 Nov 2026 23:51:12 -0500 Subject: [PATCH 8/8] =?UTF-8?q?=F0=9F=90=9B=20hive.yaml.example:=20resolve?= =?UTF-8?q?=20merge=20markers=20(#18865)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Andrew Anderson --- src/hive.yaml.example | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/hive.yaml.example b/src/hive.yaml.example index 7c211822cd..065525558b 100644 --- a/src/hive.yaml.example +++ b/src/hive.yaml.example @@ -1369,10 +1369,9 @@ data: # enabled: true # false (or env HIVE_CUSTOMERS_ENABLED=false) opens no store and hides the section # inference: # enabled: false # true (or env HIVE_CUSTOMERS_INFERENCE_ENABLED=true) suggests goals and satisfaction with the meetings model; default off -<<<<<<< HEAD # avatars: # enabled: true # false (or env HIVE_CUSTOMERS_AVATARS_ENABLED=false) shows initials and makes no browser request to github.com for pictures; default on -======= + # Sandbox Job GitHub read relay (docs/sandbox-isolation.md): lets sandbox Jobs # make authenticated, read-only GitHub reads without ever holding a credential. @@ -1387,4 +1386,3 @@ data: # listen_addr: ":8443" # required when enabled; no default # endpoint: https://hive.hive-system.svc:8443 # how Job pods reach hive; required when enabled # no_proxy: [] # optional extra hosts Jobs reach directly ->>>>>>> a0579d902 (🔒 sandbox read relay: TLS-only listener, no default bind, https endpoint, NetworkPolicy docs (#18865))