diff --git a/changelog.d/added-18865-sandbox-github-read-relay.md b/changelog.d/added-18865-sandbox-github-read-relay.md new file mode 100644 index 0000000000..af3b658be5 --- /dev/null +++ b/changelog.d/added-18865-sandbox-github-read-relay.md @@ -0,0 +1 @@ +- Sandbox Jobs can now make authenticated GitHub reads through a hive-hosted read relay instead of hitting GitHub's 60-requests-per-hour anonymous cap (so `uv lock --upgrade` resolves git dependencies without falling back to full clones). Opt in with `agent_sandbox.job.github_read_proxy` (`enabled`, `listen_addr` and `endpoint`, optional `no_proxy`); default off, with no default bind: `listen_addr` is required when enabled, and `endpoint` must be `https://` because the listener serves TLS only, with a certificate from hive's proxy CA for the endpoint host. Pair it with a NetworkPolicy (example in the docs). The relay attaches a read-only, repo-scoped App token (the existing `shared-uid-read` tier) to `GET`/`HEAD` for `api.github.com`, `codeload.github.com` and `raw.githubusercontent.com` and to git fetch on `github.com`, refuses every write, GraphQL and push discovery with a 403, tunnels every other host untouched, and never puts a GitHub credential in the Job: the Job holds only a per-kick relay key that is revoked when the kick ends. See [Sandbox isolation](src/docs/sandbox-isolation.md) ([#18865](https://github.com/hivecommons/hive/issues/18865)). diff --git a/src/cmd/hive/agent_runtime_sidecar.go b/src/cmd/hive/agent_runtime_sidecar.go index 016e2f4723..2f4982c134 100644 --- a/src/cmd/hive/agent_runtime_sidecar.go +++ b/src/cmd/hive/agent_runtime_sidecar.go @@ -339,6 +339,14 @@ func (h *sidecarHost) SetKnowledgeAdviser(crux.KnowledgeAdviser) { h.inProcessOnly("SetKnowledgeAdviser") } +// SetSandboxGitHubReadRelay is in-process-only (#18865): the relay holds the +// read credential and Enroll is a Go call into hive's proxy, so neither +// crosses into a sidecar runtime. A sidecar runtime's Job kicks run without +// the relay and read GitHub as they did before. +func (h *sidecarHost) SetSandboxGitHubReadRelay(crux.SandboxGitHubReadRelay, []string) { + h.inProcessOnly("SetSandboxGitHubReadRelay") +} + // SetInferenceRoute and ClearInferenceRoute are cruxhost.InferenceRouter: // the callback's end of SetInferenceCallbacks. A route that arrives before // boot has installed the closures is dropped and logged; the runtime's diff --git a/src/cmd/hive/main.go b/src/cmd/hive/main.go index e0391c386c..edd1285b69 100644 --- a/src/cmd/hive/main.go +++ b/src/cmd/hive/main.go @@ -4518,6 +4518,7 @@ func (b *boot) bootProxyWith(deps bootProxyDeps) { ) deps.startProxy(b.githubProxy, b.logger) + b.wireSandboxGitHubReadRelay() // Jev decision endpoint (hivecommons/hive#8939). Always bound so a // jev_mode toggle in the dashboard takes effect on the agent's next // call; every predicate reads the LIVE config, and an agent whose diff --git a/src/cmd/hive/sandbox_read_relay_boot.go b/src/cmd/hive/sandbox_read_relay_boot.go new file mode 100644 index 0000000000..00a8f8ee78 --- /dev/null +++ b/src/cmd/hive/sandbox_read_relay_boot.go @@ -0,0 +1,93 @@ +package main + +import ( + "context" + "crypto/tls" + "errors" + "net" + "os" + "time" + + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/proxy" +) + +// sandboxReadTokenTimeout bounds one read-token mint for the sandbox read +// relay. +const sandboxReadTokenTimeout = 20 * time.Second + +// systemRootBundles are the PEM trust bundles hive looks for, first match +// wins. The relay hands one to each Job with the hive CA appended, so a Job +// that sets SSL_CERT_FILE still trusts the real internet. +var systemRootBundles = []string{ + "/etc/ssl/certs/ca-certificates.crt", + "/etc/pki/tls/certs/ca-bundle.crt", + "/etc/ssl/cert.pem", +} + +func readSystemRootsPEM() []byte { + for _, p := range systemRootBundles { + if data, err := os.ReadFile(p); err == nil && len(data) > 0 { + return data + } + } + return nil +} + +// wireSandboxGitHubReadRelay starts the sandbox GitHub read relay when +// agent_sandbox.job.github_read_proxy.enabled is true (#18865). Default off. +// Any failure here is logged and leaves Jobs on their existing network path; +// it never stops boot. +func (b *boot) wireSandboxGitHubReadRelay() { + cfg := b.cfg.AgentSandbox.Job + if b.githubProxy == nil || cfg == nil || cfg.GitHubReadProxy == nil || !cfg.GitHubReadProxy.Enabled { + return + } + rp := cfg.GitHubReadProxy + if err := rp.Validate(); err != nil { + b.logger.Error("sandbox GitHub read relay not started", "error", err.Error()) + return + } + if b.agentMgr == nil { + b.logger.Error("sandbox GitHub read relay not started: no agent manager") + return + } + relay, err := proxy.NewSandboxReadRelay(b.githubProxy, proxy.SandboxReadRelayOptions{ + Endpoint: rp.Endpoint, + Token: b.sandboxReadToken, + RootsPEM: readSystemRootsPEM(), + }) + if err != nil { + b.logger.Error("sandbox GitHub read relay not started", "error", err.Error()) + return + } + ln, err := net.Listen("tcp", rp.Listen()) + if err != nil { + b.logger.Error("sandbox GitHub read relay not started: cannot listen", "addr", rp.Listen(), "error", err.Error()) + return + } + // TLS only: the per-kick key is carried in Proxy-Authorization, so a + // plain-HTTP connection is never served. The certificate comes from the + // hive proxy CA for the endpoint's host. + tlsLn := tls.NewListener(ln, relay.TLSConfig()) + go func() { + // Serve only returns when the listener fails, always with an error. + b.logger.Error("sandbox GitHub read relay stopped", "error", relay.Serve(tlsLn).Error()) + }() + b.agentMgr.SetSandboxGitHubReadRelay(relay, rp.NoProxy) + b.logger.Info("sandbox GitHub read relay enabled (TLS): Job GitHub reads use a read-only, repo-scoped token held by hive; writes are refused", "listen", rp.Listen(), "endpoint", rp.Endpoint) +} + +// sandboxReadToken mints the READ identity the relay attaches: the +// shared-uid-read tier (metadata, contents, actions, checks, pull requests and +// issues, all read), narrowed to the project's repositories. It is never the +// write-capable credential. +func (b *boot) sandboxReadToken() (string, error) { + auth := b.appAuth + if auth == nil { + return "", errors.New("no GitHub App auth to mint a read token") + } + ctx, cancel := context.WithTimeout(context.Background(), sandboxReadTokenTimeout) + defer cancel() + return auth.ScopedTokenForRepos(ctx, crux.SharedUIDReadTokenTier, b.cfg.Project.Repos) +} diff --git a/src/cmd/hive/sandbox_read_relay_boot_test.go b/src/cmd/hive/sandbox_read_relay_boot_test.go new file mode 100644 index 0000000000..d18fdbc955 --- /dev/null +++ b/src/cmd/hive/sandbox_read_relay_boot_test.go @@ -0,0 +1,36 @@ +package main + +import ( + "testing" + + "github.com/hivecommons/hive/pkg/config" +) + +// The sandbox GitHub read relay is default off (#18865): with no block, or a +// disabled one, boot wires nothing and binds nothing. +func TestWireSandboxGitHubReadRelayDefaultOff(t *testing.T) { + withJob := func(job *config.SandboxJobConfig) *config.Config { + c := &config.Config{} + c.AgentSandbox = config.AgentSandboxConfig{Job: job} + return c + } + cfgs := []*config.Config{ + {}, + withJob(&config.SandboxJobConfig{}), + withJob(&config.SandboxJobConfig{GitHubReadProxy: &config.SandboxGitHubReadProxy{Enabled: false, Endpoint: "http://h:1"}}), + } + for i, cfg := range cfgs { + b := &boot{cfg: cfg, logger: quietLogger()} + b.wireSandboxGitHubReadRelay() // nil proxy and nil manager: must be a silent no-op + if b.agentMgr != nil { + t.Fatalf("case %d wired a manager", i) + } + } +} + +func TestSandboxReadTokenNeedsAppAuth(t *testing.T) { + b := &boot{cfg: &config.Config{}, logger: quietLogger()} + if tok, err := b.sandboxReadToken(); err == nil || tok != "" { + t.Fatalf("token = %q, err = %v; want an error and no token", tok, err) + } +} diff --git a/src/crux/agent/manager.go b/src/crux/agent/manager.go index a111818fe3..cca02b9785 100644 --- a/src/crux/agent/manager.go +++ b/src/crux/agent/manager.go @@ -812,6 +812,7 @@ type Manager struct { sandboxPRClient PRCreator sandboxMutation effects.Boundary sandboxAuditCallback atomic.Pointer[func(agent, action, detail string)] + sandboxReadRelay sandboxReadRelayState // terminal is the nil-safe pane/tmux boundary. Nil means use the real tmux // implementation, preserving zero-value Manager behavior in tests. @@ -836,21 +837,6 @@ type Manager struct { buildInfo func() (version, commit string) } -// SetACMMLevel updates the cached ACMM level used by agentMode() when -// launching agents. Call this whenever the ACMM level changes. -func (m *Manager) SetACMMLevel(level int) { - m.mu.Lock() - defer m.mu.Unlock() - m.project.ACMMLevel = level - m.removeAgentsBelowACMMGateLocked(level) -} - -func (m *Manager) GetACMMLevel() int { - m.mu.RLock() - defer m.mu.RUnlock() - return m.project.ACMMLevel -} - func NewManager(agents map[string]config.AgentConfig, logger *slog.Logger, project ProjectContext) *Manager { return NewManagerWithOptions(agents, logger, project) } diff --git a/src/crux/agent/manager_acmm_level.go b/src/crux/agent/manager_acmm_level.go new file mode 100644 index 0000000000..7972684d9e --- /dev/null +++ b/src/crux/agent/manager_acmm_level.go @@ -0,0 +1,16 @@ +package agent + +// SetACMMLevel updates the cached ACMM level used by agentMode() when +// launching agents. Call this whenever the ACMM level changes. +func (m *Manager) SetACMMLevel(level int) { + m.mu.Lock() + defer m.mu.Unlock() + m.project.ACMMLevel = level + m.removeAgentsBelowACMMGateLocked(level) +} + +func (m *Manager) GetACMMLevel() int { + m.mu.RLock() + defer m.mu.RUnlock() + return m.project.ACMMLevel +} diff --git a/src/crux/agent/manager_sandbox.go b/src/crux/agent/manager_sandbox.go index 6b91268843..ae0b359879 100644 --- a/src/crux/agent/manager_sandbox.go +++ b/src/crux/agent/manager_sandbox.go @@ -310,6 +310,10 @@ func (m *Manager) runSandboxKick(ctx context.Context, name, runtime string, spec Mutation: mutationBoundary, Logger: m.logger, } + if in := m.sandboxReadRelay.cur.Load(); in != nil { + exec.GitHubReadRelay = in.relay + exec.GitHubReadRelayNoProxy = in.noProxy + } res, err := exec.Run(ctx, spec) ended := time.Now() // The run record is written to disk after m.mu is released (defers run diff --git a/src/crux/agent/sandbox_executor.go b/src/crux/agent/sandbox_executor.go index 9918cd5b8a..336fdb69e3 100644 --- a/src/crux/agent/sandbox_executor.go +++ b/src/crux/agent/sandbox_executor.go @@ -187,6 +187,12 @@ type SandboxExecutor struct { Logger *slog.Logger Now func() time.Time Mutation effects.Boundary + // GitHubReadRelay, when set, routes a Job-runtime kick's GitHub reads + // through hive's read relay (#18865). Nil leaves the Job on its own + // network path, exactly as before. + GitHubReadRelay SandboxGitHubReadRelay + // GitHubReadRelayNoProxy is extra NO_PROXY entries for relayed Jobs. + GitHubReadRelayNoProxy []string } func (e *SandboxExecutor) Run(ctx context.Context, spec SandboxKickSpec) (SandboxKickResult, error) { @@ -252,6 +258,8 @@ func (e *SandboxExecutor) Run(ctx context.Context, spec SandboxKickSpec) (Sandbo e.Logger.Warn("sandbox review spool: could not mark the kick ended", "agent", spec.Agent, "error", err) } }() + relayEnv, releaseRelay := e.enrollGitHubReadRelay(spec, workspace) + defer releaseRelay() runCtx, cancel := context.WithTimeout(ctx, spec.Timeout) defer cancel() sres, runErr := e.launcher().Run(runCtx, sandbox.LaunchSpec{ @@ -261,7 +269,7 @@ func (e *SandboxExecutor) Run(ctx context.Context, spec SandboxKickSpec) (Sandbo WorkspaceMount: sandbox.DefaultWorkspaceMount, WorkDir: sandbox.DefaultWorkspaceMount, Command: command, - Env: spool.sandboxEnv(), + Env: mergeSandboxEnv(spool.sandboxEnv(), relayEnv), EnvAllowlist: spec.EnvAllowlist, NetworkMode: spec.NetworkMode, ClineProvider: provider, diff --git a/src/crux/agent/sandbox_github_read_relay.go b/src/crux/agent/sandbox_github_read_relay.go new file mode 100644 index 0000000000..fbac649096 --- /dev/null +++ b/src/crux/agent/sandbox_github_read_relay.go @@ -0,0 +1,136 @@ +package agent + +import ( + "log/slog" + "os" + "path" + "path/filepath" + "strings" + "sync/atomic" + + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/sandbox" +) + +// SandboxGitHubReadRelay is the relay a sandbox Job sends its GitHub reads +// through (#18865); see crux.SandboxGitHubReadRelay. +type SandboxGitHubReadRelay = crux.SandboxGitHubReadRelay + +// sandboxReadRelayState is the Manager's relay wiring; the zero value is off. +// The relay and its extra NO_PROXY hosts are stored as one value so a reader +// never sees one from a previous install paired with the other. +type sandboxReadRelayState struct { + cur atomic.Pointer[sandboxReadRelayInstall] +} + +type sandboxReadRelayInstall struct { + relay SandboxGitHubReadRelay + noProxy []string +} + +// SetSandboxGitHubReadRelay installs (or, with nil, removes) the relay that +// carries sandbox Job GitHub reads (#18865). noProxy adds hosts Jobs reach +// directly. +func (m *Manager) SetSandboxGitHubReadRelay(relay SandboxGitHubReadRelay, noProxy []string) { + if relay == nil { + m.sandboxReadRelay.cur.Store(nil) + return + } + m.sandboxReadRelay.cur.Store(&sandboxReadRelayInstall{relay: relay, noProxy: append([]string(nil), noProxy...)}) +} + +const ( + // sandboxReadCARelPath is where the trust bundle is written in the kick's + // workspace; the Job sees it under sandbox.DefaultWorkspaceMount. + sandboxReadCARelPath = ".hive/github-read-ca.pem" + sandboxReadCAFileMode = 0o644 + sandboxReadCADirMode = 0o755 + // sandboxReadBaseNoProxy keeps cluster-local traffic off the relay. + sandboxReadBaseNoProxy = "localhost,127.0.0.1,::1,.svc,.cluster.local" +) + +// sandboxReadEnv is the environment that points a Job's HTTPS clients at the +// relay and at the CA bundle. It carries only the proxy URL (relay key) and a +// file path: no GitHub credential exists on this path, and +// TestSandboxReadEnvCarriesNoGitHubToken pins that. +func sandboxReadEnv(proxyURL, caPath string, extraNoProxy []string) map[string]string { + noProxy := sandboxReadBaseNoProxy + for _, h := range extraNoProxy { + if h = strings.TrimSpace(h); h != "" { + noProxy += "," + h + } + } + return map[string]string{ + "HTTPS_PROXY": proxyURL, + "https_proxy": proxyURL, + "NO_PROXY": noProxy, + "no_proxy": noProxy, + "SSL_CERT_FILE": caPath, + "GIT_SSL_CAINFO": caPath, + "REQUESTS_CA_BUNDLE": caPath, + "CURL_CA_BUNDLE": caPath, + "NODE_EXTRA_CA_CERTS": caPath, + // libcurl (so git) verifies an https PROXY against its own CA setting, + // separate from the one for the origin server: GIT_SSL_CAINFO alone + // does not cover it. Git reads this entry from the environment. + "GIT_CONFIG_COUNT": "1", + "GIT_CONFIG_KEY_0": "http.proxySSLCAInfo", + "GIT_CONFIG_VALUE_0": caPath, + } +} + +// mergeSandboxEnv returns base overlaid with extra; neither input is mutated. +// GIT_CONFIG_* entries in extra are dropped when base already defines +// GIT_CONFIG_COUNT, so the relay never rewrites a git config the Job has set. +func mergeSandboxEnv(base, extra map[string]string) map[string]string { + if len(extra) == 0 { + return base + } + out := make(map[string]string, len(base)+len(extra)) + for k, v := range base { + out[k] = v + } + _, baseGitConfig := base["GIT_CONFIG_COUNT"] + for k, v := range extra { + if baseGitConfig && strings.HasPrefix(k, "GIT_CONFIG_") { + continue + } + out[k] = v + } + return out +} + +// enrollGitHubReadRelay routes this kick's Job through the read relay. It is +// best effort by design: the relay only improves rate limits, so any failure +// is logged and the kick runs as it did before, never failing the work. The +// returned release must be called when the kick ends; it is never nil. +func (e *SandboxExecutor) enrollGitHubReadRelay(spec SandboxKickSpec, workspace string) (map[string]string, func()) { + noop := func() {} + if e.GitHubReadRelay == nil || spec.Runtime != config.SandboxRuntimeJob { + return nil, noop + } + proxyURL, bundle, release, err := e.GitHubReadRelay.Enroll(spec.Agent) + if err != nil { + e.warnReadRelay(spec.Agent, "enroll", err) + return nil, noop + } + caHost := filepath.Join(workspace, filepath.FromSlash(sandboxReadCARelPath)) + writeErr := os.MkdirAll(filepath.Dir(caHost), sandboxReadCADirMode) + if writeErr == nil { + writeErr = os.WriteFile(caHost, bundle, sandboxReadCAFileMode) + } + if err = writeErr; err != nil { + release() + e.warnReadRelay(spec.Agent, "write CA bundle", err) + return nil, noop + } + caJob := path.Join(sandbox.DefaultWorkspaceMount, sandboxReadCARelPath) + return sandboxReadEnv(proxyURL, caJob, e.GitHubReadRelayNoProxy), release +} + +func (e *SandboxExecutor) warnReadRelay(agentName, step string, err error) { + if e.Logger != nil { + e.Logger.Warn("sandbox GitHub read relay unavailable; this kick's Job reads GitHub anonymously", slog.String("agent", agentName), slog.String("step", step), slog.String("error", err.Error())) + } +} diff --git a/src/crux/agent/sandbox_github_read_relay_test.go b/src/crux/agent/sandbox_github_read_relay_test.go new file mode 100644 index 0000000000..3794ec9167 --- /dev/null +++ b/src/crux/agent/sandbox_github_read_relay_test.go @@ -0,0 +1,152 @@ +package agent + +import ( + "errors" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/config" +) + +const relayTestGitHubToken = "ghs_SHOULD_NEVER_APPEAR" + +type fakeReadRelay struct { + enrolled []string + released int + err error +} + +func (f *fakeReadRelay) Enroll(agentName string) (string, []byte, func(), error) { + if f.err != nil { + return "", nil, nil, f.err + } + f.enrolled = append(f.enrolled, agentName) + return "https://hive-sandbox:kickkey@hive.ns.svc:18445", []byte("-----BEGIN CERTIFICATE-----\nX\n-----END CERTIFICATE-----\n"), func() { f.released++ }, nil +} + +func TestSandboxReadEnvCarriesNoGitHubToken(t *testing.T) { + env := sandboxReadEnv("https://hive-sandbox:kickkey@hive.ns.svc:18445", "/workspace/.hive/github-read-ca.pem", []string{"internal.example"}) + for k, v := range env { + if strings.Contains(v, relayTestGitHubToken) || strings.Contains(strings.ToUpper(k), "TOKEN") { + t.Errorf("env %s=%s looks like a GitHub credential", k, v) + } + } + for _, k := range []string{"HTTPS_PROXY", "https_proxy", "SSL_CERT_FILE", "GIT_SSL_CAINFO", "REQUESTS_CA_BUNDLE", "CURL_CA_BUNDLE"} { + if env[k] == "" { + t.Errorf("env %s missing", k) + } + } + if !strings.Contains(env["NO_PROXY"], ".svc") || !strings.Contains(env["NO_PROXY"], "internal.example") { + t.Errorf("NO_PROXY = %q", env["NO_PROXY"]) + } + if _, ok := env["HTTP_PROXY"]; ok { + t.Error("plain HTTP must not be proxied") + } + if !strings.HasPrefix(env["HTTPS_PROXY"], "https://") { + t.Errorf("HTTPS_PROXY = %q, want an https:// proxy URL so the key is not sent in cleartext", env["HTTPS_PROXY"]) + } + if env["GIT_CONFIG_KEY_0"] != "http.proxySSLCAInfo" || env["GIT_CONFIG_VALUE_0"] != env["GIT_SSL_CAINFO"] { + t.Errorf("git must verify the https proxy against the hive CA bundle: %v", env) + } +} + +func TestMergeSandboxEnvKeepsJobGitConfig(t *testing.T) { + base := map[string]string{"GIT_CONFIG_COUNT": "2", "GIT_CONFIG_KEY_0": "a.b", "GIT_CONFIG_VALUE_0": "c"} + got := mergeSandboxEnv(base, sandboxReadEnv("https://u:k@h:1", "/ca.pem", nil)) + if got["GIT_CONFIG_COUNT"] != "2" || got["GIT_CONFIG_KEY_0"] != "a.b" { + t.Fatalf("the relay overwrote the Job's git config: %v", got) + } + if got["HTTPS_PROXY"] == "" { + t.Fatal("the proxy env must still be applied") + } +} + +func TestEnrollGitHubReadRelayOffByDefault(t *testing.T) { + e := &SandboxExecutor{} + env, release := e.enrollGitHubReadRelay(SandboxKickSpec{Agent: "deps", Runtime: config.SandboxRuntimeJob}, t.TempDir()) + release() + if env != nil { + t.Fatalf("no relay configured but env = %v", env) + } +} + +func TestEnrollGitHubReadRelayOnlyForJobRuntime(t *testing.T) { + relay := &fakeReadRelay{} + e := &SandboxExecutor{GitHubReadRelay: relay} + for _, rt := range []string{"", config.SandboxRuntimePodman, config.SandboxRuntimeOpenShell, config.SandboxRuntimeSubstrate} { + env, release := e.enrollGitHubReadRelay(SandboxKickSpec{Agent: "deps", Runtime: rt}, t.TempDir()) + release() + if env != nil { + t.Errorf("runtime %q must not be routed through the relay", rt) + } + } + if len(relay.enrolled) != 0 { + t.Fatalf("relay enrolled %v for non-Job runtimes", relay.enrolled) + } +} + +func TestEnrollGitHubReadRelayJobRuntime(t *testing.T) { + relay := &fakeReadRelay{} + e := &SandboxExecutor{GitHubReadRelay: relay} + ws := t.TempDir() + env, release := e.enrollGitHubReadRelay(SandboxKickSpec{Agent: "deps", Runtime: config.SandboxRuntimeJob}, ws) + if env["HTTPS_PROXY"] == "" || env["SSL_CERT_FILE"] != "/workspace/.hive/github-read-ca.pem" { + t.Fatalf("env = %v", env) + } + if _, err := os.Stat(filepath.Join(ws, ".hive", "github-read-ca.pem")); err != nil { + t.Fatalf("CA bundle not written: %v", err) + } + if relay.released != 0 { + t.Fatal("key released before the kick ended") + } + release() + if relay.released != 1 { + t.Fatal("key not released when the kick ended") + } +} + +func TestEnrollGitHubReadRelayFailureIsNotFatal(t *testing.T) { + e := &SandboxExecutor{GitHubReadRelay: &fakeReadRelay{err: errors.New("boom")}} + env, release := e.enrollGitHubReadRelay(SandboxKickSpec{Agent: "deps", Runtime: config.SandboxRuntimeJob}, t.TempDir()) + release() + if env != nil { + t.Fatalf("env = %v, want none so the kick runs as before", env) + } +} + +func TestMergeSandboxEnvDoesNotMutateInputs(t *testing.T) { + base := map[string]string{"A": "1"} + out := mergeSandboxEnv(base, map[string]string{"B": "2"}) + if len(base) != 1 || out["A"] != "1" || out["B"] != "2" { + t.Fatalf("base=%v out=%v", base, out) + } + if got := mergeSandboxEnv(nil, nil); got != nil { + t.Fatalf("merge of nothing = %v", got) + } +} + +// The relay is installed through crux.Host like every other host capability +// (no type assertion in boot). +func TestManagerSetSandboxGitHubReadRelayInstallsAndClears(t *testing.T) { + m := &Manager{} + var host crux.Host = m + relay := &fakeReadRelay{} + noProxy := []string{"internal.example"} + host.SetSandboxGitHubReadRelay(relay, noProxy) + noProxy[0] = "mutated.example" + + in := m.sandboxReadRelay.cur.Load() + if in == nil || in.relay != SandboxGitHubReadRelay(relay) { + t.Fatal("relay was not installed") + } + if len(in.noProxy) != 1 || in.noProxy[0] != "internal.example" { + t.Fatalf("noProxy = %v; the caller's slice must be copied", in.noProxy) + } + host.SetSandboxGitHubReadRelay(nil, []string{"ignored.example"}) + if m.sandboxReadRelay.cur.Load() != nil { + t.Fatal("a nil relay must switch the feature off") + } +} diff --git a/src/crux/api/client/client_test.go b/src/crux/api/client/client_test.go index 3e2b2443a4..8675983219 100644 --- a/src/crux/api/client/client_test.go +++ b/src/crux/api/client/client_test.go @@ -139,7 +139,7 @@ func TestParityHostVersusSetterTable(t *testing.T) { t.Errorf("api.HostSetters[%q] has unknown kind %q", name, row.Kind) } } - for _, name := range []string{"SetSpanStarter", "SetKnowledgeAdviser"} { + for _, name := range []string{"SetSpanStarter", "SetKnowledgeAdviser", "SetSandboxGitHubReadRelay"} { if api.HostSetters[name].Kind != api.HostSetterInProcessOnly { t.Errorf("%s is an in-process-only setter", name) } diff --git a/src/crux/api/host_setter.go b/src/crux/api/host_setter.go new file mode 100644 index 0000000000..804fb28907 --- /dev/null +++ b/src/crux/api/host_setter.go @@ -0,0 +1,7 @@ +package api + +// HostSetter is one row of HostSetters. +type HostSetter struct { + Kind HostSetterKind + Via string +} diff --git a/src/crux/api/runtime.go b/src/crux/api/runtime.go index a9e59524b9..a38c795725 100644 --- a/src/crux/api/runtime.go +++ b/src/crux/api/runtime.go @@ -766,12 +766,6 @@ const ( HostSetterInProcessOnly HostSetterKind = "inprocess" ) -// HostSetter is one row of HostSetters. -type HostSetter struct { - Kind HostSetterKind - Via string -} - // Callback endpoint routes the table refers to, spelled here because // crux/api/callback imports this package; its tests hold the two in step. const ( @@ -826,6 +820,7 @@ var HostSetters = map[string]HostSetter{ "SetAuditSink": {HostSetterEvent, EventAudit}, "SetConditionReporter": {HostSetterEvent, EventCondition + ", " + hostViaConditions}, // In-process only. - "SetSpanStarter": {HostSetterInProcessOnly, "OpenTelemetry global tracer in the runtime process; trajectory events are independent of spans"}, - "SetKnowledgeAdviser": {HostSetterInProcessOnly, "the knowledge store lives in the hive process; a sidecar runtime runs without the mid-task knowledge advisor (hivecommons/hive#17052 phase 1)"}, + "SetSpanStarter": {HostSetterInProcessOnly, "OpenTelemetry global tracer in the runtime process; trajectory events are independent of spans"}, + "SetSandboxGitHubReadRelay": {HostSetterInProcessOnly, "the relay holds the read-only GitHub credential and Enroll is a Go call into hive's proxy; neither crosses into a runtime process, so a sidecar runtime's Job kicks read GitHub without the relay (hivecommons/hive#18865)"}, + "SetKnowledgeAdviser": {HostSetterInProcessOnly, "the knowledge store lives in the hive process; a sidecar runtime runs without the mid-task knowledge advisor (hivecommons/hive#17052 phase 1)"}, } diff --git a/src/crux/config/agent_config.go b/src/crux/config/agent_config.go index 9524f8db94..44f54c03ce 100644 --- a/src/crux/config/agent_config.go +++ b/src/crux/config/agent_config.go @@ -58,6 +58,11 @@ type SandboxJobConfig struct { // and uses 65534 elsewhere, since the hive image's default user is root. RelayRunAsUser *int64 `yaml:"relay_run_as_user,omitempty" json:"relay_run_as_user,omitempty"` ImagePullSecrets []string `yaml:"image_pull_secrets,omitempty" json:"image_pull_secrets,omitempty"` + // GitHubReadProxy routes a Job's GitHub READS through a hive-hosted relay + // that attaches a read-only credential the Job never sees (#18865). It is + // read from the hive-wide agent_sandbox.job block only; a per-agent job + // block cannot enable, disable or repoint it. + GitHubReadProxy *SandboxGitHubReadProxy `yaml:"github_read_proxy,omitempty" json:"github_read_proxy,omitempty"` } // SandboxRuntime returns the per-agent runtime override, then the global @@ -88,6 +93,8 @@ func (a *AgentConfig) SandboxJob(global AgentSandboxConfig) SandboxJobConfig { return g } out := *a.Sandbox.Job + // The read relay is hive-wide: a per-agent block cannot enable it. + out.GitHubReadProxy = g.GitHubReadProxy if strings.TrimSpace(out.WorkspaceClaim) == "" { out.WorkspaceClaim = g.WorkspaceClaim } diff --git a/src/crux/config/sandbox_github_read_proxy_test.go b/src/crux/config/sandbox_github_read_proxy_test.go new file mode 100644 index 0000000000..e8eb90a470 --- /dev/null +++ b/src/crux/config/sandbox_github_read_proxy_test.go @@ -0,0 +1,73 @@ +package config + +import ( + "strings" + "testing" +) + +func TestSandboxGitHubReadProxyDefaultOff(t *testing.T) { + var nilCfg *SandboxGitHubReadProxy + if err := nilCfg.Validate(); err != nil { + t.Fatalf("nil block must be valid: %v", err) + } + if err := (&SandboxGitHubReadProxy{}).Validate(); err != nil { + t.Fatalf("disabled block must be valid: %v", err) + } + // Disabled stays valid however the other keys look: the feature is off. + if err := (&SandboxGitHubReadProxy{Endpoint: "http://h:1"}).Validate(); err != nil { + t.Fatalf("disabled block must be valid: %v", err) + } +} + +// There is no default bind: enabling without listen_addr is a config error +// that names the key. +func TestSandboxGitHubReadProxyHasNoDefaultListen(t *testing.T) { + if got := (*SandboxGitHubReadProxy)(nil).Listen(); got != "" { + t.Fatalf("Listen = %q, want no default", got) + } + err := (&SandboxGitHubReadProxy{Enabled: true, Endpoint: "https://hive.ns.svc:8443"}).Validate() + if err == nil || !strings.Contains(err.Error(), "agent_sandbox.job.github_read_proxy.listen_addr") { + t.Fatalf("err = %v, want one naming listen_addr", err) + } + if err := (&SandboxGitHubReadProxy{Enabled: true, ListenAddr: " ", Endpoint: "https://h:1"}).Validate(); err == nil { + t.Fatal("a blank listen_addr must be refused") + } + if err := (&SandboxGitHubReadProxy{Enabled: true, ListenAddr: "8443", Endpoint: "https://h:1"}).Validate(); err == nil { + t.Fatal("listen_addr without a port must be refused") + } +} + +func TestSandboxGitHubReadProxyValidate(t *testing.T) { + for _, tc := range []struct { + endpoint string + ok bool + }{ + {"https://hive.ns.svc:8443", true}, + {"https://10.20.30.40:8443", true}, + {"http://hive.ns.svc:8443", false}, // cleartext is refused + {"", false}, + {"hive.ns.svc:8443", false}, + {"ftp://hive:1", false}, + {"https://user:pw@hive:1", false}, + } { + err := (&SandboxGitHubReadProxy{Enabled: true, ListenAddr: ":8443", Endpoint: tc.endpoint}).Validate() + if (err == nil) != tc.ok { + t.Errorf("endpoint %q: err = %v, want ok=%v", tc.endpoint, err, tc.ok) + } + if err != nil && !strings.Contains(err.Error(), "agent_sandbox.job.github_read_proxy.endpoint") { + t.Errorf("endpoint %q: error %q does not name the key", tc.endpoint, err) + } + } + err := (&SandboxGitHubReadProxy{Enabled: true, ListenAddr: ":8443", Endpoint: "http://h:1"}).Validate() + if err == nil || !strings.Contains(err.Error(), "https") { + t.Fatalf("http endpoint error = %v, want one saying https is required", err) + } +} + +func TestSandboxJobGitHubReadProxyIsHiveWide(t *testing.T) { + global := AgentSandboxConfig{Job: &SandboxJobConfig{GitHubReadProxy: &SandboxGitHubReadProxy{Enabled: true, ListenAddr: ":8443", Endpoint: "https://h:1"}}} + agent := &AgentConfig{Sandbox: &AgentSandboxOverride{Job: &SandboxJobConfig{GitHubReadProxy: &SandboxGitHubReadProxy{Enabled: false}}}} + if got := agent.SandboxJob(global); got.GitHubReadProxy == nil || !got.GitHubReadProxy.Enabled { + t.Fatalf("a per-agent block must not override the hive-wide relay setting: %+v", got.GitHubReadProxy) + } +} diff --git a/src/crux/config/sandbox_job_types.go b/src/crux/config/sandbox_job_types.go index d86ef4aeb5..b70a78a6f6 100644 --- a/src/crux/config/sandbox_job_types.go +++ b/src/crux/config/sandbox_job_types.go @@ -1,5 +1,12 @@ package config +import ( + "errors" + "net" + "net/url" + "strings" +) + // SandboxJobToleration mirrors the subset of a Kubernetes toleration an // operator writes by hand. type SandboxJobToleration struct { @@ -24,3 +31,66 @@ type SandboxJobVolume struct { MountPath string `yaml:"mount_path" json:"mount_path"` ReadOnly bool `yaml:"read_only,omitempty" json:"read_only,omitempty"` } + +// SandboxGitHubReadProxy configures the sandbox GitHub read relay (#18865). +// Default off. With enabled: true, hive listens (TLS) on listen_addr and every +// sandbox.runtime: job kick gets HTTPS_PROXY pointing at endpoint plus a +// per-kick relay key. The relay attaches a read-only, repo-scoped GitHub App +// token to GET/HEAD requests for api.github.com, codeload.github.com, +// raw.githubusercontent.com and the git fetch paths of github.com. Writes are +// refused and the token never enters the Job. +type SandboxGitHubReadProxy struct { + Enabled bool `yaml:"enabled,omitempty" json:"enabled,omitempty"` + // ListenAddr is the host:port hive binds. There is deliberately NO + // default: binding an unrequested port on every interface is a decision + // the operator makes. Required when enabled. + ListenAddr string `yaml:"listen_addr,omitempty" json:"listen_addr,omitempty"` + // Endpoint is the https://host:port Job pods use to reach the relay, for + // example the in-namespace Service name. Required when enabled and must + // be https: the listener serves TLS with a certificate issued by hive's + // proxy CA for this host. It carries no credentials (hive adds a per-kick + // key itself). + Endpoint string `yaml:"endpoint,omitempty" json:"endpoint,omitempty"` + // NoProxy lists extra hosts or domains a Job reaches directly, added to + // the built-in cluster-local list (localhost, .svc, .cluster.local). + NoProxy []string `yaml:"no_proxy,omitempty" json:"no_proxy,omitempty"` +} + +// Listen returns the trimmed bind address; empty when unset (there is no +// default). +func (c *SandboxGitHubReadProxy) Listen() string { + if c == nil { + return "" + } + return strings.TrimSpace(c.ListenAddr) +} + +// Validate reports why an enabled block cannot work. A disabled or nil block +// is always valid. +func (c *SandboxGitHubReadProxy) Validate() error { + if c == nil || !c.Enabled { + return nil + } + listen := c.Listen() + if listen == "" { + return errors.New("agent_sandbox.job.github_read_proxy.listen_addr is required when enabled (there is no default; set the host:port hive should bind, for example :8443)") + } + if _, port, err := net.SplitHostPort(listen); err != nil || port == "" { + return errors.New("agent_sandbox.job.github_read_proxy.listen_addr must be host:port (the host may be empty to bind every interface)") + } + raw := strings.TrimSpace(c.Endpoint) + if raw == "" { + return errors.New("agent_sandbox.job.github_read_proxy.endpoint is required when enabled (the https://host:port Job pods use to reach hive)") + } + u, err := url.Parse(raw) + if err != nil || u.Host == "" || u.Hostname() == "" { + return errors.New("agent_sandbox.job.github_read_proxy.endpoint must be an https://host:port URL") + } + if u.Scheme != "https" { + return errors.New("agent_sandbox.job.github_read_proxy.endpoint must use https:// (the per-kick key must not cross the pod network in cleartext; hive serves TLS with a certificate from its proxy CA)") + } + if u.User != nil { + return errors.New("agent_sandbox.job.github_read_proxy.endpoint must not carry credentials; hive adds a per-kick key itself") + } + return nil +} diff --git a/src/crux/cruxtest/recorder_knowledge.go b/src/crux/cruxtest/recorder_knowledge.go index 01ed6d9085..23ee0f5eeb 100644 --- a/src/crux/cruxtest/recorder_knowledge.go +++ b/src/crux/cruxtest/recorder_knowledge.go @@ -8,3 +8,10 @@ import "github.com/hivecommons/hive/crux" func (r *Recorder) SetKnowledgeAdviser(adviser crux.KnowledgeAdviser) { r.record("SetKnowledgeAdviser", adviser != nil) } + +// SetSandboxGitHubReadRelay records the sandbox GitHub read relay seam +// (hivecommons/hive#18865): whether one was installed, and its extra NO_PROXY +// hosts. It lives here for the same size-ratchet reason as above. +func (r *Recorder) SetSandboxGitHubReadRelay(relay crux.SandboxGitHubReadRelay, noProxy []string) { + r.record("SetSandboxGitHubReadRelay", relay != nil, noProxy) +} diff --git a/src/crux/host_wiring.go b/src/crux/host_wiring.go index f87cc17c7a..cc1c3ac2d4 100644 --- a/src/crux/host_wiring.go +++ b/src/crux/host_wiring.go @@ -63,4 +63,10 @@ type Host interface { // (hivecommons/hive#17052). In-process-only in phase 1: the knowledge // store lives in the hive process and the adviser is a Go call into it. SetKnowledgeAdviser(adviser KnowledgeAdviser) + // SetSandboxGitHubReadRelay installs (nil removes) the relay that carries + // sandbox Job GitHub reads (hivecommons/hive#18865); noProxy adds hosts + // Jobs reach directly. In-process-only: Enroll is a Go call into hive's + // proxy, and the relay holds the read credential, which must not cross + // into a runtime process. + SetSandboxGitHubReadRelay(relay SandboxGitHubReadRelay, noProxy []string) } diff --git a/src/crux/kubejob/github_read_relay_env_test.go b/src/crux/kubejob/github_read_relay_env_test.go new file mode 100644 index 0000000000..76ef5940b5 --- /dev/null +++ b/src/crux/kubejob/github_read_relay_env_test.go @@ -0,0 +1,42 @@ +package kubejob + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/hivecommons/hive/crux/sandbox" +) + +// A Job routed through the sandbox GitHub read relay (#18865) carries the +// proxy URL and CA paths, and never a GitHub credential, even when one sits in +// hive's own environment or in the launch spec. +func TestManifestRelayEnvCarriesNoGitHubToken(t *testing.T) { + const secret = "ghs_MUST_NOT_REACH_THE_JOB" + t.Setenv("GITHUB_TOKEN", secret) + t.Setenv("GH_TOKEN", secret) + t.Setenv("HIVE_GITHUB_TOKEN", secret) + + l := &Launcher{Options: Options{WorkspaceClaim: "c"}} + m := l.manifest("ns", "job", "sub", sandbox.LaunchSpec{ + Image: "i", + EnvAllowlist: []string{"GITHUB_TOKEN", "GH_TOKEN", "HIVE_GITHUB_TOKEN"}, + Env: map[string]string{ + "HTTPS_PROXY": "https://hive-sandbox:relaykey@hive.ns.svc:18445", + "SSL_CERT_FILE": "/workspace/.hive/github-read-ca.pem", + "GIT_SSL_CAINFO": "/workspace/.hive/github-read-ca.pem", + "GITHUB_TOKEN": secret, + "GH_TOKEN": secret, + }, + }) + raw, _ := json.Marshal(m) + s := string(raw) + if strings.Contains(s, secret) { + t.Fatalf("the Job manifest leaks a GitHub token:\n%s", s) + } + for _, want := range []string{`"name":"HTTPS_PROXY"`, `"name":"SSL_CERT_FILE"`, `"name":"GIT_SSL_CAINFO"`, "relaykey"} { + if !strings.Contains(s, want) { + t.Errorf("manifest missing %s\n%s", want, s) + } + } +} diff --git a/src/crux/proxy/github_proxy_ca.go b/src/crux/proxy/github_proxy_ca.go index 05818270a0..34425808b8 100644 --- a/src/crux/proxy/github_proxy_ca.go +++ b/src/crux/proxy/github_proxy_ca.go @@ -11,6 +11,7 @@ import ( "fmt" "log/slog" "math/big" + "net" "os" "path/filepath" "time" @@ -50,6 +51,11 @@ func (p *GitHubProxy) forgeCert(host string) (tls.Certificate, error) { ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, DNSNames: []string{host}, } + if ip := net.ParseIP(host); ip != nil { + // An IP-literal name is only valid as an IP SAN. + template.DNSNames = nil + template.IPAddresses = []net.IP{ip} + } caKey, ok := p.caCert.PrivateKey.(*ecdsa.PrivateKey) if !ok { diff --git a/src/crux/proxy/sandbox_read_relay.go b/src/crux/proxy/sandbox_read_relay.go new file mode 100644 index 0000000000..b8994db3eb --- /dev/null +++ b/src/crux/proxy/sandbox_read_relay.go @@ -0,0 +1,471 @@ +package proxy + +import ( + "bufio" + "crypto/rand" + "crypto/sha256" + "crypto/subtle" + "crypto/tls" + "encoding/base64" + "encoding/hex" + "encoding/json" + "encoding/pem" + "errors" + "fmt" + "net" + "net/http" + "net/url" + "strings" + "sync" + "time" +) + +// Sandbox GitHub read relay (#18865). +// +// Sandbox Jobs run in their own pods, so they cannot reach the loopback proxy +// the hive's own agents use, and an anonymous Job is capped at GitHub's 60 +// requests/hour per source IP, shared by every agent on the host. This relay +// is a second, deliberately narrow listener that a Job points HTTPS_PROXY at. +// +// The invariants, each pinned by a test: +// +// - READ ONLY. A GitHub credential is attached only to GET/HEAD requests to +// the read hosts, plus the one POST git itself needs to fetch +// (git-upload-pack). Every other method, GraphQL included, is refused with +// a 403 and never reaches GitHub. +// - NO INJECTION OFF GITHUB. Any other host is an opaque, destination-guarded +// tunnel: no TLS interception, no header touched, no credential attached. +// - THE CREDENTIAL STAYS HERE. The Job holds only a per-kick relay key, which +// authenticates it to this listener and is worth nothing to GitHub. The +// GitHub token is fetched, attached and discarded in this process; an +// Authorization header the Job sends is always stripped first. +// - A READ IDENTITY. The token source is supplied by the host and is expected +// to be the read-only, repo-scoped tier, never the App write credential. + +// SandboxReadProxyUser is the Proxy-Authorization user a Job presents; the +// password is the per-kick relay key. +const SandboxReadProxyUser = "hive-sandbox" + +const ( + sandboxReadKeyBytes = 32 + // sandboxReadTokenTTL is how long a fetched read token is reused. GitHub + // installation tokens live an hour; refreshing at 45 minutes keeps a + // margin for a request already in flight. + sandboxReadTokenTTL = 45 * time.Minute + // sandboxReadTokenRetryAfter is how long a failed mint is remembered, so a + // token-endpoint outage costs one attempt per interval rather than one + // (up to 20s, serialized) attempt per request. + sandboxReadTokenRetryAfter = 30 * time.Second + sandboxReadHTTPSPort = "443" + sandboxReadRefsSuffix = "/info/refs" + sandboxReadUploadPackSuffix = "/git-upload-pack" + sandboxReadUploadPackName = "git-upload-pack" +) + +// sandboxReadHosts are the hosts whose traffic the relay intercepts. Fixed on +// purpose: not derived from the registered GitHub-family hosts, so enterprise +// hosts added elsewhere never gain relay credential injection. +var sandboxReadHosts = map[string]bool{ + "api.github.com": true, + "codeload.github.com": true, + "raw.githubusercontent.com": true, + "github.com": true, +} + +// SandboxReadHost reports whether host is one the relay terminates TLS for. +func SandboxReadHost(host string) bool { return sandboxReadHosts[canonicalHost(host)] } + +// SandboxReadDecision is the relay's verdict on one intercepted request. +type SandboxReadDecision struct { + // Allowed is false for anything the relay refuses to forward. + Allowed bool + // Inject is true when a read credential may be attached. A request can be + // allowed without it (a github.com page GET), and then goes out with no + // Authorization at all. + Inject bool + // Basic selects git smart-HTTP Basic auth over the "token" scheme. + Basic bool + Reason string +} + +// SandboxReadPolicy classifies one request to an intercepted host. It is a +// pure function so the read-only invariant is testable without a network. +func SandboxReadPolicy(host, method, path, rawQuery string) SandboxReadDecision { + host = canonicalHost(host) + if !sandboxReadHosts[host] { + return SandboxReadDecision{Reason: "host is not a GitHub read host"} + } + method = strings.ToUpper(method) + isRead := method == http.MethodGet || method == http.MethodHead + + if host == "github.com" { + switch { + case isRead && strings.HasSuffix(path, sandboxReadRefsSuffix): + // Smart-HTTP ref discovery. service=git-receive-pack is push + // discovery and is refused; no service is a dumb-protocol read. + service := "" + if q, err := url.ParseQuery(rawQuery); err == nil { + service = q.Get("service") + } + if service != "" && service != sandboxReadUploadPackName { + return SandboxReadDecision{Reason: "only git fetch (git-upload-pack) is allowed through the sandbox read relay"} + } + return SandboxReadDecision{Allowed: true, Inject: true, Basic: true} + case method == http.MethodPost && strings.HasSuffix(path, sandboxReadUploadPackSuffix): + return SandboxReadDecision{Allowed: true, Inject: true, Basic: true} + case isRead && !strings.HasSuffix(path, "/git-receive-pack"): + // Archives, release downloads and the like redirect to the + // download hosts. No credential is needed or attached here. + return SandboxReadDecision{Allowed: true} + } + return SandboxReadDecision{Reason: "the sandbox read relay forwards reads only; writes go through hive's relays"} + } + if !isRead { + return SandboxReadDecision{Reason: "the sandbox read relay forwards GET and HEAD only; writes go through hive's relays"} + } + return SandboxReadDecision{Allowed: true, Inject: true} +} + +// SandboxReadTokenFunc returns the read-only GitHub credential the relay +// attaches. It must be a read identity (the shared-uid-read tier), never the +// write credential. +type SandboxReadTokenFunc func() (string, error) + +// SandboxReadRelayOptions configure NewSandboxReadRelay. +type SandboxReadRelayOptions struct { + // Endpoint is the http://host:port Jobs use to reach the relay's listener. + Endpoint string + // Token supplies the read credential. Nil sends reads unauthenticated. + Token SandboxReadTokenFunc + // RootsPEM is the trust bundle handed to a Job alongside the hive CA, so + // the Job still trusts the real internet. Empty means the hive CA alone. + RootsPEM []byte +} + +// SandboxReadRelay is the Job-facing read relay. +type SandboxReadRelay struct { + p *GitHubProxy + opts SandboxReadRelayOptions + endpoint *url.URL + + keysMu sync.RWMutex + keys map[[sha256.Size]byte]string // sha256(key) -> agent + + tokMu sync.Mutex + tok string + tokFetched time.Time + tokFailed time.Time // last failed mint; zero when the last mint succeeded + now func() time.Time + // dial overrides the upstream TLS dial (tests). + dial func(host string) (net.Conn, error) +} + +// NewSandboxReadRelay builds the relay on p (which supplies the CA, the +// destination-guarded dialers and the upstream chain). +func NewSandboxReadRelay(p *GitHubProxy, opts SandboxReadRelayOptions) (*SandboxReadRelay, error) { + if p == nil { + return nil, errors.New("sandbox read relay: no proxy") + } + u, err := url.Parse(strings.TrimSpace(opts.Endpoint)) + if err != nil || u.Hostname() == "" || u.Scheme != "https" || u.User != nil { + return nil, errors.New("sandbox read relay: endpoint must be an https://host:port URL without credentials") + } + return &SandboxReadRelay{p: p, opts: opts, endpoint: u, keys: map[[sha256.Size]byte]string{}, now: time.Now}, nil +} + +// TLSConfig is the server configuration for the relay's listener. The relay +// is only ever served over TLS: the per-kick key travels in Proxy-Authorization, +// so a plain-HTTP connection is never accepted. The certificate is issued by +// the hive proxy CA (the same CA and forging the GitHub interception uses, no +// new CA and no new key material on disk) for the endpoint's host, as a DNS or +// IP SAN. Jobs already trust that CA through the bundle Enroll returns. +func (r *SandboxReadRelay) TLSConfig() *tls.Config { + host := canonicalHost(r.endpoint.Hostname()) + return &tls.Config{ + MinVersion: tls.VersionTLS12, + GetCertificate: func(*tls.ClientHelloInfo) (*tls.Certificate, error) { + cert, err := r.p.forgeCert(host) + if err != nil { + return nil, fmt.Errorf("sandbox read relay: issuing listener certificate for %q: %w", host, err) + } + return &cert, nil + }, + } +} + +// Enroll admits one kick: it mints a per-kick relay key and returns the proxy +// URL the Job should use, the CA bundle it must trust and a release func that +// revokes the key. The key authenticates the Job to this listener only. +func (r *SandboxReadRelay) Enroll(agentName string) (proxyURL string, caBundlePEM []byte, release func(), err error) { + raw := make([]byte, sandboxReadKeyBytes) + if _, err := rand.Read(raw); err != nil { + return "", nil, nil, fmt.Errorf("sandbox read relay: minting kick key: %w", err) + } + key := hex.EncodeToString(raw) + digest := sha256.Sum256([]byte(key)) + r.keysMu.Lock() + r.keys[digest] = agentName + r.keysMu.Unlock() + + u := *r.endpoint + u.User = url.UserPassword(SandboxReadProxyUser, key) + ca := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: r.p.caX509.Raw}) + bundle := append(append([]byte{}, r.opts.RootsPEM...), ca...) + return u.String(), bundle, func() { + r.keysMu.Lock() + delete(r.keys, digest) + r.keysMu.Unlock() + }, nil +} + +// authenticate returns the agent a Proxy-Authorization header belongs to. +func (r *SandboxReadRelay) authenticate(req *http.Request) (string, bool) { + auth := req.Header.Get("Proxy-Authorization") + if !strings.HasPrefix(auth, "Basic ") { + return "", false + } + decoded, err := base64.StdEncoding.DecodeString(strings.TrimPrefix(auth, "Basic ")) + if err != nil { + return "", false + } + user, key, ok := strings.Cut(string(decoded), ":") + if !ok || subtle.ConstantTimeCompare([]byte(user), []byte(SandboxReadProxyUser)) != 1 { + return "", false + } + digest := sha256.Sum256([]byte(key)) + r.keysMu.RLock() + agentName, found := r.keys[digest] + r.keysMu.RUnlock() + return agentName, found +} + +// readToken returns the cached read credential, fetching a fresh one when it +// is older than sandboxReadTokenTTL. An error or empty result means "send the +// request unauthenticated": the relay never substitutes another credential. +func (r *SandboxReadRelay) readToken() string { + if r.opts.Token == nil { + return "" + } + r.tokMu.Lock() + defer r.tokMu.Unlock() + if r.tok != "" && r.now().Sub(r.tokFetched) < sandboxReadTokenTTL { + return r.tok + } + if r.tok == "" && !r.tokFailed.IsZero() && r.now().Sub(r.tokFailed) < sandboxReadTokenRetryAfter { + return "" + } + tok, err := r.opts.Token() + if err != nil || tok == "" { + reason := "empty token" + if err != nil { + reason = err.Error() + } + r.p.logger.Warn("sandbox read relay: no read credential available; forwarding reads unauthenticated", "reason", reason) + r.tok, r.tokFailed = "", r.now() + return "" + } + r.tok, r.tokFetched, r.tokFailed = tok, r.now(), time.Time{} + return tok +} + +// applySandboxReadAuth rewrites a request's Authorization per decision d. Any +// credential the Job supplied is removed unconditionally; the read token is +// attached only when d allows injection. +func applySandboxReadAuth(req *http.Request, d SandboxReadDecision, token string) { + req.Header.Del("Authorization") + req.Header.Del("Proxy-Authorization") + req.Header.Del("Expect") + if !d.Allowed || !d.Inject || token == "" { + return + } + if d.Basic { + req.Header.Set("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(gitInjectBasicUser+":"+token))) + return + } + req.Header.Set("Authorization", "token "+token) +} + +// Serve accepts Job connections on ln until it is closed. +func (r *SandboxReadRelay) Serve(ln net.Listener) error { + for { + conn, err := ln.Accept() + if err != nil { + return fmt.Errorf("sandbox read relay accept: %w", err) + } + go r.handleConn(conn) + } +} + +func (r *SandboxReadRelay) handleConn(conn net.Conn) { + defer func() { _ = conn.Close() }() + _ = conn.SetReadDeadline(time.Now().Add(httpReadTimeout)) + br := bufio.NewReader(conn) + req, err := http.ReadRequest(br) + _ = conn.SetReadDeadline(time.Time{}) + if err != nil { + return + } + agentName, ok := r.authenticate(req) + if !ok { + writeRelayStatus(conn, http.StatusProxyAuthRequired, "proxy authentication required", "Proxy-Authenticate: Basic realm=\"hive-sandbox\"\r\n") + return + } + if req.Method != http.MethodConnect { + // Plain-HTTP proxying is not offered: every GitHub read host is HTTPS. + writeRelayStatus(conn, http.StatusForbidden, "the sandbox read relay carries HTTPS CONNECT only", "") + return + } + host, port, err := net.SplitHostPort(req.Host) + if err != nil { + host, port = req.Host, sandboxReadHTTPSPort + } + if port != sandboxReadHTTPSPort { + writeRelayStatus(conn, http.StatusForbidden, "the sandbox read relay carries port 443 only", "") + return + } + if !SandboxReadHost(host) { + r.tunnel(conn, host, agentName) + return + } + r.intercept(conn, host, agentName) +} + +// tunnel is the opaque path for every non-GitHub host: no interception, no +// header touched, no credential. The destination guard keeps it from reaching +// loopback, link-local metadata or the private estate. +func (r *SandboxReadRelay) tunnel(conn net.Conn, host, agentName string) { + const dialTimeout = 10 * time.Second + upstream, err := r.p.dialUpstreamRaw(net.JoinHostPort(host, sandboxReadHTTPSPort), dialTimeout) + if err != nil { + r.p.logger.Debug("sandbox read relay: tunnel refused", "agent", agentName, "host", host, "error", err) + writeRelayStatus(conn, http.StatusBadGateway, "connection failed", "") + return + } + defer func() { _ = upstream.Close() }() + _, _ = fmt.Fprint(conn, "HTTP/1.1 200 Connection established\r\n\r\n") + relayTunnel(conn, upstream) +} + +// intercept terminates TLS for a GitHub read host and relays each request +// through SandboxReadPolicy. +func (r *SandboxReadRelay) intercept(conn net.Conn, host, agentName string) { + cert, err := r.p.forgeCert(host) + if err != nil { + r.p.logger.Error("sandbox read relay: forge cert failed", "host", host, "error", err) + writeRelayStatus(conn, http.StatusBadGateway, "certificate error", "") + return + } + if _, err := fmt.Fprint(conn, "HTTP/1.1 200 Connection established\r\n\r\n"); err != nil { + return + } + client := tls.Server(conn, &tls.Config{Certificates: []tls.Certificate{cert}}) + _ = conn.SetDeadline(time.Now().Add(clientTLSHandshakeTimeout)) + err = client.Handshake() + _ = conn.SetDeadline(time.Time{}) + if err != nil { + return + } + defer func() { _ = client.Close() }() + + // The upstream leg is dialed on the first ALLOWED request, so a refused + // request never opens a connection to GitHub. + var upstream net.Conn + var upstreamBuf *bufio.Reader + defer func() { + if upstream != nil { + _ = upstream.Close() + } + }() + + clientBuf := bufio.NewReader(client) + for { + _ = client.SetReadDeadline(time.Now().Add(httpReadTimeout)) + req, err := http.ReadRequest(clientBuf) + if err != nil { + return + } + d := SandboxReadPolicy(host, req.Method, req.URL.Path, req.URL.RawQuery) + if d.Allowed && !sandboxReadRequestTargetsHost(req, host) { + // The policy and the credential decision were made for the CONNECT + // host; a request naming a different Host or an absolute URL for + // another host must not ride that decision. + d = SandboxReadDecision{Reason: "the request names a different host than the CONNECT target"} + } + if !d.Allowed { + r.p.logger.Info("sandbox read relay: request refused", "agent", agentName, "host", host, "method", req.Method) + writeRelayRefusal(client, d.Reason) + return + } + if upstream == nil { + up, derr := r.dialUpstream(host) + if derr != nil { + r.p.logger.Warn("sandbox read relay: upstream dial failed", "host", host, "error", derr) + return + } + upstream, upstreamBuf = up, bufio.NewReader(up) + } + token := "" + if d.Inject { + token = r.readToken() + } + applySandboxReadAuth(req, d, token) + // Log the decision, never the credential. + r.p.logger.Debug("sandbox read relay: request", "agent", agentName, "host", host, "method", req.Method, "injected", token != "") + + _ = upstream.SetWriteDeadline(time.Now().Add(httpWriteTimeout)) + if err := req.Write(upstream); err != nil { + return + } + _ = upstream.SetWriteDeadline(time.Time{}) + _ = client.SetReadDeadline(time.Time{}) + _ = upstream.SetReadDeadline(time.Now().Add(httpReadTimeout)) + resp, err := http.ReadResponse(upstreamBuf, req) + _ = upstream.SetReadDeadline(time.Time{}) + if err != nil { + return + } + resp.Body = &stallBoundedBody{body: resp.Body, conn: upstream, idle: r.p.bodyStallTimeoutDuration()} + werr := resp.Write(&stallBoundedWriter{conn: client, idle: httpWriteTimeout}) + _ = resp.Body.Close() + _ = client.SetWriteDeadline(time.Time{}) + if werr != nil || resp.Close || req.Close { + return + } + } +} + +// sandboxReadRequestTargetsHost reports whether the Host header and any +// absolute request URL name the host the tunnel was opened for. +func sandboxReadRequestTargetsHost(req *http.Request, host string) bool { + want := canonicalHost(host) + for _, got := range []string{req.Host, req.URL.Host} { + if got == "" { + continue + } + if h, _, err := net.SplitHostPort(got); err == nil { + got = h + } + if canonicalHost(got) != want { + return false + } + } + return true +} + +// dialUpstream opens the TLS leg to a read host. It is a method so +// tests can drive the relay without network access by replacing r.dial. +func (r *SandboxReadRelay) dialUpstream(host string) (net.Conn, error) { + if r.dial != nil { + return r.dial(host) + } + return r.p.dialUpstreamTLS(net.JoinHostPort(host, sandboxReadHTTPSPort), host, upstreamDialTimeout) +} + +func writeRelayStatus(conn net.Conn, status int, msg, extraHeaders string) { + _, _ = fmt.Fprintf(conn, "HTTP/1.1 %d %s\r\n%sContent-Length: %d\r\nConnection: close\r\n\r\n%s", status, http.StatusText(status), extraHeaders, len(msg), msg) +} + +func writeRelayRefusal(w net.Conn, reason string) { + body, _ := json.Marshal(map[string]string{"message": "blocked by hive: " + reason}) + _, _ = fmt.Fprintf(w, "HTTP/1.1 403 Forbidden\r\nContent-Type: application/json\r\nContent-Length: %d\r\nConnection: close\r\n\r\n%s", len(body), body) +} diff --git a/src/crux/proxy/sandbox_read_relay_test.go b/src/crux/proxy/sandbox_read_relay_test.go new file mode 100644 index 0000000000..f4411f173b --- /dev/null +++ b/src/crux/proxy/sandbox_read_relay_test.go @@ -0,0 +1,579 @@ +package proxy + +import ( + "bufio" + "crypto/tls" + "crypto/x509" + "encoding/base64" + "fmt" + "log/slog" + "net" + "net/http" + "strings" + "sync" + "sync/atomic" + "testing" + "time" +) + +const ( + testReadToken = "ghs_READONLYTOKEN" + // relayTestHost is the endpoint host the test relay's certificate is for. + relayTestHost = "hive.example.svc" +) + +func TestSandboxReadPolicy(t *testing.T) { + cases := []struct { + name, host, method, path, query string + allowed, inject, basic bool + }{ + {"api GET", "api.github.com", "GET", "/repos/o/r/tags", "", true, true, false}, + {"api HEAD", "api.github.com", "HEAD", "/repos/o/r", "", true, true, false}, + {"codeload GET", "codeload.github.com", "GET", "/o/r/tar.gz/main", "", true, true, false}, + {"raw GET", "raw.githubusercontent.com", "GET", "/o/r/main/a.txt", "", true, true, false}, + {"api POST refused", "api.github.com", "POST", "/repos/o/r/issues", "", false, false, false}, + {"api GraphQL POST refused", "api.github.com", "POST", "/graphql", "", false, false, false}, + {"api PUT refused", "api.github.com", "PUT", "/repos/o/r/pulls/1/merge", "", false, false, false}, + {"api PATCH refused", "api.github.com", "PATCH", "/repos/o/r/issues/1", "", false, false, false}, + {"api DELETE refused", "api.github.com", "DELETE", "/repos/o/r/git/refs/heads/x", "", false, false, false}, + {"codeload POST refused", "codeload.github.com", "POST", "/o/r", "", false, false, false}, + {"raw PUT refused", "raw.githubusercontent.com", "PUT", "/o/r/main/a", "", false, false, false}, + {"git discovery fetch", "github.com", "GET", "/o/r.git/info/refs", "service=git-upload-pack", true, true, true}, + {"git upload-pack POST", "github.com", "POST", "/o/r.git/git-upload-pack", "", true, true, true}, + {"git push discovery refused", "github.com", "GET", "/o/r.git/info/refs", "service=git-receive-pack", false, false, false}, + {"git receive-pack POST refused", "github.com", "POST", "/o/r.git/git-receive-pack", "", false, false, false}, + {"git receive-pack GET refused", "github.com", "GET", "/o/r.git/git-receive-pack", "", false, false, false}, + {"github.com page GET no credential", "github.com", "GET", "/o/r/archive/refs/heads/main.tar.gz", "", true, false, false}, + {"github.com other POST refused", "github.com", "POST", "/o/r/issues", "", false, false, false}, + {"non-GitHub host", "pypi.org", "GET", "/simple/x", "", false, false, false}, + {"lookalike host", "api.github.com.evil.example", "GET", "/", "", false, false, false}, + {"GHE host not injected", "ghe.example.com", "GET", "/api/v3/user", "", false, false, false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + d := SandboxReadPolicy(tc.host, tc.method, tc.path, tc.query) + if d.Allowed != tc.allowed || d.Inject != tc.inject || d.Basic != tc.basic { + t.Fatalf("SandboxReadPolicy(%s %s %s) = %+v, want allowed=%v inject=%v basic=%v", tc.method, tc.host, tc.path, d, tc.allowed, tc.inject, tc.basic) + } + if !d.Allowed && d.Inject { + t.Fatal("a refused request must never be marked for injection") + } + }) + } +} + +func TestSandboxReadPolicyNeverInjectsForAnyWriteMethod(t *testing.T) { + for _, host := range []string{"api.github.com", "codeload.github.com", "raw.githubusercontent.com", "github.com"} { + for _, method := range []string{"POST", "PUT", "PATCH", "DELETE", "OPTIONS", "TRACE", "CONNECT"} { + for _, path := range []string{"/", "/graphql", "/o/r.git/git-receive-pack", "/o/r/issues"} { + d := SandboxReadPolicy(host, method, path, "") + if d.Inject && !(host == "github.com" && method == "POST" && strings.HasSuffix(path, "/git-upload-pack")) { + t.Errorf("%s %s%s must not receive a credential: %+v", method, host, path, d) + } + } + } + } +} + +func TestApplySandboxReadAuth(t *testing.T) { + newReq := func() *http.Request { + r, _ := http.NewRequest("GET", "https://api.github.com/x", nil) + r.Header.Set("Authorization", "token AGENT_SUPPLIED") + r.Header.Set("Proxy-Authorization", "Basic abc") + r.Header.Set("Expect", "100-continue") + return r + } + t.Run("token scheme", func(t *testing.T) { + r := newReq() + applySandboxReadAuth(r, SandboxReadDecision{Allowed: true, Inject: true}, testReadToken) + if got := r.Header.Get("Authorization"); got != "token "+testReadToken { + t.Fatalf("Authorization = %q", got) + } + if r.Header.Get("Proxy-Authorization") != "" || r.Header.Get("Expect") != "" { + t.Fatal("hop headers must be removed") + } + }) + t.Run("git basic", func(t *testing.T) { + r := newReq() + applySandboxReadAuth(r, SandboxReadDecision{Allowed: true, Inject: true, Basic: true}, testReadToken) + want := "Basic " + base64.StdEncoding.EncodeToString([]byte("x-access-token:"+testReadToken)) + if got := r.Header.Get("Authorization"); got != want { + t.Fatalf("Authorization = %q", got) + } + }) + t.Run("agent credential stripped when not injecting", func(t *testing.T) { + r := newReq() + applySandboxReadAuth(r, SandboxReadDecision{Allowed: true}, testReadToken) + if got := r.Header.Get("Authorization"); got != "" { + t.Fatalf("Authorization = %q, want none", got) + } + }) + t.Run("no token means anonymous, not the agent's header", func(t *testing.T) { + r := newReq() + applySandboxReadAuth(r, SandboxReadDecision{Allowed: true, Inject: true}, "") + if got := r.Header.Get("Authorization"); got != "" { + t.Fatalf("Authorization = %q, want none", got) + } + }) + t.Run("refused request gets no credential", func(t *testing.T) { + r := newReq() + applySandboxReadAuth(r, SandboxReadDecision{}, testReadToken) + if got := r.Header.Get("Authorization"); got != "" { + t.Fatalf("Authorization = %q, want none", got) + } + }) +} + +// relayHarness serves a relay on loopback and fakes the GitHub upstream. +type relayHarness struct { + relay *SandboxReadRelay + addr string + pool *x509.CertPool + dials atomic.Int32 + + mu sync.Mutex + upstream []*http.Request +} + +func newRelayHarness(t *testing.T, token SandboxReadTokenFunc) *relayHarness { + t.Helper() + return newRelayHarnessWithLogger(t, token, nil) +} + +// newRelayHarnessWithLogger is newRelayHarness with the proxy's logger replaced +// before the listener starts, so a test can read back everything the relay logs. +func newRelayHarnessWithLogger(t *testing.T, token SandboxReadTokenFunc, logger *slog.Logger) *relayHarness { + t.Helper() + p, _ := newTestProxyWithCA(t) + if logger != nil { + p.logger = logger + } + relay, err := NewSandboxReadRelay(p, SandboxReadRelayOptions{Endpoint: "https://" + relayTestHost + ":18445", Token: token}) + if err != nil { + t.Fatal(err) + } + h := &relayHarness{relay: relay, pool: x509.NewCertPool()} + h.pool.AddCert(p.caX509) + relay.dial = func(host string) (net.Conn, error) { + h.dials.Add(1) + client, server := net.Pipe() + go func() { + defer func() { _ = server.Close() }() + br := bufio.NewReader(server) + for { + req, err := http.ReadRequest(br) + if err != nil { + return + } + h.mu.Lock() + h.upstream = append(h.upstream, req) + h.mu.Unlock() + _, _ = fmt.Fprint(server, "HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok") + } + }() + return client, nil + } + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = ln.Close() }) + // The relay is only ever served over TLS, exactly as boot does it. + go func() { _ = relay.Serve(tls.NewListener(ln, relay.TLSConfig())) }() + h.addr = ln.Addr().String() + return h +} + +func (h *relayHarness) seen() []*http.Request { + h.mu.Lock() + defer h.mu.Unlock() + return append([]*http.Request(nil), h.upstream...) +} + +// connect opens a CONNECT tunnel through the relay and returns the TLS client +// conn, or the raw HTTP status when the relay refused the CONNECT. +func (h *relayHarness) connect(t *testing.T, proxyURLUser, target string) (*tls.Conn, int) { + t.Helper() + raw, err := net.DialTimeout("tcp", h.addr, 5*time.Second) + if err != nil { + t.Fatal(err) + } + // Leg 1: TLS to the relay itself, verified against the hive CA for the + // endpoint host. Leg 2 (below) is the CONNECT tunnel's own TLS. + var conn net.Conn = tls.Client(raw, &tls.Config{RootCAs: h.pool, ServerName: relayTestHost}) + t.Cleanup(func() { _ = conn.Close() }) + _ = conn.SetDeadline(time.Now().Add(10 * time.Second)) + hdr := "" + if proxyURLUser != "" { + hdr = "Proxy-Authorization: Basic " + base64.StdEncoding.EncodeToString([]byte(proxyURLUser)) + "\r\n" + } + _, _ = fmt.Fprintf(conn, "CONNECT %s HTTP/1.1\r\nHost: %s\r\n%s\r\n", target, target, hdr) + br := bufio.NewReader(conn) + resp, err := http.ReadResponse(br, &http.Request{Method: "CONNECT"}) + if err != nil { + t.Fatal(err) + } + if resp.StatusCode != 200 { + return nil, resp.StatusCode + } + host, _, _ := net.SplitHostPort(target) + tc := tls.Client(conn, &tls.Config{RootCAs: h.pool, ServerName: host}) + if err := tc.Handshake(); err != nil { + t.Fatalf("TLS handshake through the relay: %v", err) + } + return tc, 200 +} + +func roundTrip(t *testing.T, tc *tls.Conn, method, host, path string, headers map[string]string) *http.Response { + t.Helper() + req, _ := http.NewRequest(method, "https://"+host+path, nil) + for k, v := range headers { + req.Header.Set(k, v) + } + if err := req.Write(tc); err != nil { + t.Fatal(err) + } + resp, err := http.ReadResponse(bufio.NewReader(tc), req) + if err != nil { + t.Fatal(err) + } + return resp +} + +func enrollKey(t *testing.T, r *SandboxReadRelay, agentName string) (userinfo string, release func()) { + t.Helper() + proxyURL, _, rel, err := r.Enroll(agentName) + if err != nil { + t.Fatal(err) + } + rest := strings.TrimPrefix(proxyURL, "https://") + userinfo, _, _ = strings.Cut(rest, "@") + return userinfo, rel +} + +func TestSandboxReadRelayInjectsOnReadAndRefusesWrites(t *testing.T) { + h := newRelayHarness(t, func() (string, error) { return testReadToken, nil }) + user, release := enrollKey(t, h.relay, "deps") + defer release() + + tc, status := h.connect(t, user, "api.github.com:443") + if status != 200 { + t.Fatalf("CONNECT status = %d", status) + } + resp := roundTrip(t, tc, "GET", "api.github.com", "/repos/o/r/tags", map[string]string{"Authorization": "token AGENT_SUPPLIED"}) + if resp.StatusCode != 200 { + t.Fatalf("GET status = %d", resp.StatusCode) + } + seen := h.seen() + if len(seen) != 1 { + t.Fatalf("upstream saw %d requests, want 1", len(seen)) + } + if got := seen[0].Header.Get("Authorization"); got != "token "+testReadToken { + t.Fatalf("upstream Authorization = %q, want the relay's read token (the agent's own header must be replaced)", got) + } + + // A write is refused by the relay and never reaches GitHub, whatever the + // agent put in the request. + tc2, _ := h.connect(t, user, "api.github.com:443") + before := h.dials.Load() + resp = roundTrip(t, tc2, "POST", "api.github.com", "/repos/o/r/issues", map[string]string{"Authorization": "token AGENT_SUPPLIED"}) + if resp.StatusCode != http.StatusForbidden { + t.Fatalf("POST status = %d, want 403", resp.StatusCode) + } + if h.dials.Load() != before { + t.Fatal("a refused write opened an upstream connection") + } + if len(h.seen()) != 1 { + t.Fatal("a refused write reached GitHub") + } +} + +func TestSandboxReadRelayGitPushDiscoveryRefused(t *testing.T) { + h := newRelayHarness(t, func() (string, error) { return testReadToken, nil }) + user, release := enrollKey(t, h.relay, "deps") + defer release() + tc, _ := h.connect(t, user, "github.com:443") + resp := roundTrip(t, tc, "GET", "github.com", "/o/r.git/info/refs?service=git-receive-pack", nil) + if resp.StatusCode != http.StatusForbidden { + t.Fatalf("status = %d, want 403", resp.StatusCode) + } + if len(h.seen()) != 0 { + t.Fatal("push discovery reached GitHub") + } +} + +func TestSandboxReadRelayGitFetchUsesBasicWithReadToken(t *testing.T) { + h := newRelayHarness(t, func() (string, error) { return testReadToken, nil }) + user, release := enrollKey(t, h.relay, "deps") + defer release() + tc, _ := h.connect(t, user, "github.com:443") + resp := roundTrip(t, tc, "GET", "github.com", "/o/r.git/info/refs?service=git-upload-pack", nil) + if resp.StatusCode != 200 { + t.Fatalf("status = %d", resp.StatusCode) + } + want := "Basic " + base64.StdEncoding.EncodeToString([]byte("x-access-token:"+testReadToken)) + if got := h.seen()[0].Header.Get("Authorization"); got != want { + t.Fatalf("upstream Authorization = %q", got) + } +} + +func TestSandboxReadRelayRequiresLiveKey(t *testing.T) { + h := newRelayHarness(t, func() (string, error) { return testReadToken, nil }) + if _, status := h.connect(t, "", "api.github.com:443"); status != http.StatusProxyAuthRequired { + t.Fatalf("no credentials: status = %d, want 407", status) + } + if _, status := h.connect(t, SandboxReadProxyUser+":not-a-key", "api.github.com:443"); status != http.StatusProxyAuthRequired { + t.Fatalf("wrong key: status = %d, want 407", status) + } + user, release := enrollKey(t, h.relay, "deps") + if _, status := h.connect(t, user, "api.github.com:443"); status != 200 { + t.Fatalf("live key: status = %d, want 200", status) + } + release() + if _, status := h.connect(t, user, "api.github.com:443"); status != http.StatusProxyAuthRequired { + t.Fatalf("released key: status = %d, want 407", status) + } + if h.dials.Load() != 0 { + t.Fatal("no request was made, so no upstream dial was expected") + } +} + +func TestSandboxReadRelayRefusesNon443Connect(t *testing.T) { + h := newRelayHarness(t, nil) + user, release := enrollKey(t, h.relay, "deps") + defer release() + if _, status := h.connect(t, user, "api.github.com:8080"); status != http.StatusForbidden { + t.Fatalf("status = %d, want 403", status) + } +} + +func TestSandboxReadRelayNoTokenForwardsAnonymously(t *testing.T) { + h := newRelayHarness(t, func() (string, error) { return "", fmt.Errorf("mint failed") }) + user, release := enrollKey(t, h.relay, "deps") + defer release() + tc, _ := h.connect(t, user, "api.github.com:443") + _ = roundTrip(t, tc, "GET", "api.github.com", "/rate_limit", map[string]string{"Authorization": "token AGENT_SUPPLIED"}) + if got := h.seen()[0].Header.Get("Authorization"); got != "" { + t.Fatalf("upstream Authorization = %q, want none: the agent's header must not ride through", got) + } +} + +func TestSandboxReadRelayEnrollCarriesNoGitHubToken(t *testing.T) { + h := newRelayHarness(t, func() (string, error) { return testReadToken, nil }) + proxyURL, bundle, release, err := h.relay.Enroll("deps") + if err != nil { + t.Fatal(err) + } + defer release() + if strings.Contains(proxyURL, testReadToken) || strings.Contains(string(bundle), testReadToken) { + t.Fatal("the GitHub token must never be in what a Job receives") + } + if !strings.HasPrefix(proxyURL, "https://"+SandboxReadProxyUser+":") || !strings.Contains(string(bundle), "BEGIN CERTIFICATE") { + t.Fatalf("unexpected enrollment: %q", proxyURL) + } +} + +func TestSandboxReadRelayTokenIsCached(t *testing.T) { + var calls atomic.Int32 + h := newRelayHarness(t, func() (string, error) { calls.Add(1); return testReadToken, nil }) + now := time.Now() + h.relay.now = func() time.Time { return now } + for i := 0; i < 3; i++ { + if h.relay.readToken() != testReadToken { + t.Fatal("no token") + } + } + if calls.Load() != 1 { + t.Fatalf("minted %d times, want 1", calls.Load()) + } + now = now.Add(sandboxReadTokenTTL + time.Second) + _ = h.relay.readToken() + if calls.Load() != 2 { + t.Fatalf("minted %d times after TTL, want 2", calls.Load()) + } +} + +func TestNewSandboxReadRelayRejectsCredentialedEndpoint(t *testing.T) { + p, _ := newTestProxyWithCA(t) + for _, ep := range []string{"", "ftp://x:1", "http://host:1", "https://u:p@host:1", "host:1"} { + if _, err := NewSandboxReadRelay(p, SandboxReadRelayOptions{Endpoint: ep}); err == nil { + t.Errorf("endpoint %q accepted", ep) + } + } +} + +// A request inside the tunnel that names another host (Host header or absolute +// URL) must not borrow the CONNECT host's credential decision. +func TestSandboxReadRelayRefusesHostMismatchInsideTunnel(t *testing.T) { + h := newRelayHarness(t, func() (string, error) { return testReadToken, nil }) + user, release := enrollKey(t, h.relay, "deps") + defer release() + + for _, tc := range []struct{ name, hostHeader, target string }{ + {"host header", "github.com", "/repos/o/r"}, + {"absolute url", "api.github.com", "https://evil.example/repos/o/r"}, + } { + t.Run(tc.name, func(t *testing.T) { + conn, status := h.connect(t, user, "api.github.com:443") + if status != 200 { + t.Fatalf("CONNECT status = %d", status) + } + _, err := fmt.Fprintf(conn, "GET %s HTTP/1.1\r\nHost: %s\r\n\r\n", tc.target, tc.hostHeader) + if err != nil { + t.Fatal(err) + } + resp, err := http.ReadResponse(bufio.NewReader(conn), nil) + if err != nil { + t.Fatal(err) + } + if resp.StatusCode != http.StatusForbidden { + t.Fatalf("status = %d, want 403", resp.StatusCode) + } + if len(h.seen()) != 0 { + t.Fatal("a mismatched-host request reached upstream with the CONNECT host's credential") + } + }) + } +} + +// The read token and the per-kick relay key must never appear in the relay's +// logs, at the most verbose level, across injected, refused and anonymous +// requests. +func TestSandboxReadRelayNeverLogsTokenOrKey(t *testing.T) { + var logs lockedBuffer + logger := slog.New(slog.NewTextHandler(&logs, &slog.HandlerOptions{Level: slog.LevelDebug})) + h := newRelayHarnessWithLogger(t, func() (string, error) { return testReadToken, nil }, logger) + user, release := enrollKey(t, h.relay, "deps") + defer release() + _, key, _ := strings.Cut(user, ":") + + tc, _ := h.connect(t, user, "api.github.com:443") + _ = roundTrip(t, tc, "GET", "api.github.com", "/repos/o/r", map[string]string{"Authorization": "token AGENT_SUPPLIED"}) + tc2, _ := h.connect(t, user, "api.github.com:443") + _ = roundTrip(t, tc2, "POST", "api.github.com", "/repos/o/r/issues", map[string]string{"Authorization": "token AGENT_SUPPLIED"}) + _, _ = h.connect(t, SandboxReadProxyUser+":"+key+"x", "api.github.com:443") + + out := logs.String() + if out == "" { + t.Fatal("expected the relay to log its decisions") + } + for _, secret := range []string{testReadToken, key, "AGENT_SUPPLIED"} { + if strings.Contains(out, secret) { + t.Fatalf("relay logs contain a secret (%q):\n%s", secret, out) + } + } +} + +// Each kick gets its own key, keys are unguessable, and releasing one kick +// leaves the others live. +func TestSandboxReadRelayKeysArePerKick(t *testing.T) { + h := newRelayHarness(t, nil) + u1, rel1 := enrollKey(t, h.relay, "deps") + u2, rel2 := enrollKey(t, h.relay, "deps") + defer rel2() + if u1 == u2 { + t.Fatal("two kicks of one agent shared a relay key") + } + if _, key, _ := strings.Cut(u1, ":"); len(key) < 2*sandboxReadKeyBytes { + t.Fatalf("relay key too short: %d chars", len(key)) + } + rel1() + if _, status := h.connect(t, u1, "api.github.com:443"); status != http.StatusProxyAuthRequired { + t.Fatalf("released kick key: status = %d, want 407", status) + } + if _, status := h.connect(t, u2, "api.github.com:443"); status != 200 { + t.Fatalf("other kick key: status = %d, want 200", status) + } +} + +// A failing token endpoint is not hammered: one mint attempt per retry +// interval, requests in between go out anonymously, and recovery is picked up. +func TestSandboxReadRelayFailedMintIsNotRetriedPerRequest(t *testing.T) { + var calls atomic.Int32 + var fail atomic.Bool + fail.Store(true) + h := newRelayHarness(t, func() (string, error) { + calls.Add(1) + if fail.Load() { + return "", fmt.Errorf("mint failed") + } + return testReadToken, nil + }) + now := time.Now() + h.relay.now = func() time.Time { return now } + for i := 0; i < 5; i++ { + if tok := h.relay.readToken(); tok != "" { + t.Fatalf("token = %q during an outage", tok) + } + } + if calls.Load() != 1 { + t.Fatalf("mint attempted %d times for 5 requests during an outage, want 1", calls.Load()) + } + fail.Store(false) + now = now.Add(sandboxReadTokenRetryAfter + time.Second) + if h.relay.readToken() != testReadToken { + t.Fatal("relay did not recover after the retry interval") + } +} + +// The listener serves TLS only. A plain-HTTP client, even one holding a live +// key, gets no tunnel and no proxy handling, and the key it sent in the clear +// is never logged. +func TestSandboxReadRelayRefusesPlainHTTP(t *testing.T) { + var logs lockedBuffer + logger := slog.New(slog.NewTextHandler(&logs, &slog.HandlerOptions{Level: slog.LevelDebug})) + h := newRelayHarnessWithLogger(t, func() (string, error) { return testReadToken, nil }, logger) + user, release := enrollKey(t, h.relay, "deps") + defer release() + _, key, _ := strings.Cut(user, ":") + + conn, err := net.DialTimeout("tcp", h.addr, 5*time.Second) + if err != nil { + t.Fatal(err) + } + defer func() { _ = conn.Close() }() + _ = conn.SetDeadline(time.Now().Add(5 * time.Second)) + auth := base64.StdEncoding.EncodeToString([]byte(user)) + _, _ = fmt.Fprintf(conn, "CONNECT api.github.com:443 HTTP/1.1\r\nHost: api.github.com:443\r\nProxy-Authorization: Basic %s\r\n\r\n", auth) + reply, _ := bufio.NewReader(conn).ReadString('\n') + if strings.Contains(reply, "200") || strings.Contains(reply, "407") { + t.Fatalf("plain HTTP reached the relay's proxy handler: %q", reply) + } + if h.dials.Load() != 0 || len(h.seen()) != 0 { + t.Fatal("a plain-HTTP request produced upstream traffic") + } + if strings.Contains(logs.String(), key) || strings.Contains(logs.String(), auth) { + t.Fatalf("the key sent over plain HTTP was logged:\n%s", logs.String()) + } +} + +// The listener certificate is issued by the hive CA for the endpoint host, as a +// DNS or IP SAN, and for nothing else. +func TestSandboxReadRelayCertificateHasEndpointSAN(t *testing.T) { + for _, tc := range []struct{ endpoint, name string }{ + {"https://hive.example.svc:18445", "hive.example.svc"}, + {"https://10.20.30.40:18445", "10.20.30.40"}, + } { + p, _ := newTestProxyWithCA(t) + relay, err := NewSandboxReadRelay(p, SandboxReadRelayOptions{Endpoint: tc.endpoint}) + if err != nil { + t.Fatal(err) + } + cert, err := relay.TLSConfig().GetCertificate(nil) + if err != nil { + t.Fatal(err) + } + leaf, err := x509.ParseCertificate(cert.Certificate[0]) + if err != nil { + t.Fatal(err) + } + pool := x509.NewCertPool() + pool.AddCert(p.caX509) + if _, err := leaf.Verify(x509.VerifyOptions{DNSName: tc.name, Roots: pool}); err != nil { + t.Errorf("%s: certificate does not verify for the endpoint host against the hive CA: %v", tc.endpoint, err) + } + if _, err := leaf.Verify(x509.VerifyOptions{DNSName: "other.example", Roots: pool}); err == nil { + t.Errorf("%s: certificate verifies for an unrelated host", tc.endpoint) + } + } +} diff --git a/src/crux/sandbox_read_relay.go b/src/crux/sandbox_read_relay.go new file mode 100644 index 0000000000..77d048172d --- /dev/null +++ b/src/crux/sandbox_read_relay.go @@ -0,0 +1,11 @@ +package crux + +// SandboxGitHubReadRelay is the hive-hosted relay a sandbox Job sends its +// GitHub reads through (hivecommons/hive#18865). Enroll admits one kick and +// returns the proxy URL for it (carrying a per-kick relay key, never a GitHub +// credential), the PEM trust bundle the Job needs to accept the relay's +// certificates, and a release func that revokes the key. It is an interface so +// the runtime does not depend on hive's proxy. +type SandboxGitHubReadRelay interface { + Enroll(agentName string) (proxyURL string, caBundlePEM []byte, release func(), err error) +} diff --git a/src/docs/sandbox-isolation.md b/src/docs/sandbox-isolation.md index 26c83a7997..a1e94d3c9f 100644 --- a/src/docs/sandbox-isolation.md +++ b/src/docs/sandbox-isolation.md @@ -312,6 +312,62 @@ agent_sandbox: image_pull_secrets: [ registry-pull ] ``` +**Authenticated GitHub reads for Jobs (#18865).** A Job runs in its own pod, so by default every `api.github.com` call it makes is anonymous: GitHub allows 60 requests per hour per source IP, shared by every agent on the node, and `uv lock --upgrade` on a repo with git dependencies falls back to full clones when the API call fails. The opt-in GitHub read relay fixes that without giving the Job a credential. Default off: + +```yaml +agent_sandbox: + job: + github_read_proxy: + enabled: true + listen_addr: ":8443" # required when enabled; there is no default + endpoint: https://hive.hive-system.svc:8443 # how Job pods reach hive; required, https only + no_proxy: [ models.internal.example ] # optional extra direct hosts +``` + +The feature is **off by default**. When `enabled` is true, `listen_addr` is required (there is no default bind, so hive never opens a port you did not name) and `endpoint` must be `https://`; config validation fails with an error naming the key otherwise. With it on, hive listens on `listen_addr`, **serving TLS only** with a certificate issued by hive's existing proxy CA for the `endpoint` host (a DNS or IP SAN; no new CA, and no CA key is written anywhere new), and every `sandbox.runtime: job` kick (the agent run, not the `publish.verify` preflights) gets `HTTPS_PROXY` set to `endpoint` (an `https://` proxy URL) with a **per-kick relay key**, plus `SSL_CERT_FILE`, `GIT_SSL_CAINFO`, `REQUESTS_CA_BUNDLE`, `CURL_CA_BUNDLE` and `NODE_EXTRA_CA_CERTS` pointing at a trust bundle hive writes to `/.hive/github-read-ca.pem` (the system roots plus hive's proxy CA, so the Job still trusts the real internet). `localhost`, `.svc` and `.cluster.local` bypass the relay. The block is read from `agent_sandbox.job` only; a per-agent `job` block cannot enable or repoint it. You must expose the listener to the Job pods yourself: a Service for `listen_addr`, and a NetworkPolicy allowing only the Job pods to reach it. + +**Two layers guard the listener.** The first is the NetworkPolicy: it decides which pods can open a connection to the relay port at all. The second is the per-kick relay key: a connection that gets through must still present a live key, or it gets a 407 and nothing else. The key is random per kick, held by hive only as a hash, never logged, and revoked when the kick ends. Because the listener serves TLS only, the key never crosses the pod network in cleartext, and a plain-HTTP connection is dropped before any proxy handling. Example NetworkPolicy, applied in hive's namespace, allowing ingress on the relay port only from pods in the Jobs' namespace (replace the placeholders): + +```yaml +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: hive-github-read-relay + namespace: +spec: + podSelector: + matchLabels: + : + policyTypes: [ Ingress ] + ingress: + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: + podSelector: + matchLabels: + app.kubernetes.io/managed-by: hive-kubejob + ports: + - protocol: TCP + port: 8443 # your listen_addr port +``` + +Client support for an `https://` proxy URL varies. Go, Node (through its proxy agents) and libcurl 7.52 or newer (curl, and git built on it) can speak TLS to a proxy. libcurl checks an https proxy against its own CA setting rather than the origin's, so hive also sets `http.proxySSLCAInfo` through `GIT_CONFIG_COUNT`/`GIT_CONFIG_KEY_0`/`GIT_CONFIG_VALUE_0` (skipped when the Job already defines `GIT_CONFIG_COUNT`). Some clients, for example Python HTTP stacks built on urllib3 older than 2.0, cannot speak TLS to a proxy; for those the Job's reads go out anonymously, as without the relay. Hive does not downgrade to a cleartext proxy for them. + +What the relay does: + +| Host | Request | What hive does | +|---|---|---| +| `api.github.com`, `codeload.github.com`, `raw.githubusercontent.com` | `GET`, `HEAD` | Strips any `Authorization` the Job sent and attaches hive's read token. | +| `github.com` | `GET /.git/info/refs?service=git-upload-pack`, `POST /.git/git-upload-pack` (git fetch and clone, uv's git resolution) | Same, as Basic `x-access-token`. | +| `github.com` | other `GET`/`HEAD` (archives, release downloads) | Forwarded with no credential. | +| the four hosts above | every other method, GraphQL, `git-receive-pack` (push discovery included) | Refused with a 403 and never sent to GitHub. | +| any other host | `CONNECT` on 443 | Opaque tunnel, no interception and no credential, through the same destination guard as the agent proxy (loopback, link-local metadata and private ranges are refused). | + +The credential is the existing **read tier** (`shared-uid-read`: metadata, contents, actions, checks, pull requests and issues, all read), minted from the Hive App and narrowed to `project.repos`. It is never the write token and never enters the Job: the Job holds only the relay key, which authenticates it to hive's listener and is worthless to GitHub. Hive caches the token for 45 minutes and logs decisions, never token bytes. The key is revoked when the kick ends. When no read token can be minted (no App auth, a repo outside the installation), reads go out anonymously, as they did before; the Job never receives an error it would not have had without the relay. Public repositories outside `project.repos`, such as a git dependency, are readable with that token because GitHub serves public data to any authenticated caller. + +This is separate from `HIVE_PROXY_INJECT_GH_AUTH`, which governs the placeholder token of agents running on the hive pod; the relay does not need it and is not affected by it. Limits to know: a Job that sets its own `HTTPS_PROXY` or `NO_PROXY` can route around the relay and is back to anonymous reads; `publish.verify` Jobs do not use it; GraphQL reads from a Job are refused because the relay cannot prove a query is read-only. A shared git dependency cache ([#18865](https://github.com/hivecommons/hive/issues/18865), option 3) is a separate, complementary change. + **RBAC.** The spoke's ServiceAccount needs a Role for `batch/jobs` (create, get, list, delete), `pods` plus `pods/log` (get, list) and, optionally, `serviceaccounts` (get) so the Job service account's pull secrets are merged; the terminal viewer needs nothing beyond that same `pods/log` grant. The standalone kustomize base ships it as `sandbox-job-rbac.yaml`, and the hub provisioner grants it to every hosted spoke as `hive-sandbox-jobs`. Hive warns at boot and on reload (`config.AgentSandboxGateWarnings`) when a job-runtime agent resolves no `workspace_claim` or names a runtime hive does not have, and refuses the kick with the same message rather than failing it after the agent is marked running. diff --git a/src/docs/security-model.md b/src/docs/security-model.md index ab0343935f..6914d12476 100644 --- a/src/docs/security-model.md +++ b/src/docs/security-model.md @@ -73,6 +73,7 @@ are enabled on a given hive. | Forced egress through the proxy (iptables) | proxy bypass | `src/deploy/entrypoint.sh` | boot log; no `HIVE_PROXY_ADVISORY_OK` | | Tiered GitHub tokens per agent mode | blast radius of a leaked token | `src/crux/agentmode`, `src/crux/agent/manager_tokens.go` | token tier on the agent card | | Proxy-side credential injection (opt-in) | token theft | `HIVE_PROXY_INJECT_GH_AUTH` | `security.credentialInjection` in governor config | +| Sandbox Job GitHub read relay (opt-in) | anonymous 60/hr API cap; token in the Job | `agent_sandbox.job.github_read_proxy` | `sandbox GitHub read relay enabled` boot line | | Inference-key placeholder swap | provider key theft | `src/pkg/apiproxy` | agent env shows `sk-hive-` | | `gh` wrapper with per-agent restrictions | denied commands | `bin/gh-wrapper.sh` | `gh` resolves to `/usr/local/bin/gh` | | `hive-open-pr` / `hive-merge` relays | forged authorship, stale-commit merges | `bin/hive-merge.sh`, `bin/hive-open-pr.sh` | `POST /pulls` returns 403 for agents | diff --git a/src/hive.yaml.example b/src/hive.yaml.example index d5b6198e59..4dcf6d51e9 100644 --- a/src/hive.yaml.example +++ b/src/hive.yaml.example @@ -1374,3 +1374,18 @@ data: # mention_search: # enabled: false # true (or env HIVE_CUSTOMERS_MENTION_SEARCH_ENABLED=true) runs one budgeted GitHub search for mentions and comments when an owner accepts a suggested GitHub login; default off # max_requests_per_hour: 10 # search requests all accepts may spend together per hour (default 10, at most 60); each accept spends at most 2 + + +# Sandbox Job GitHub read relay (docs/sandbox-isolation.md): lets sandbox Jobs +# make authenticated, read-only GitHub reads without ever holding a credential. +# Default off. There is NO default bind: with enabled: true, listen_addr is +# required, and endpoint must be https:// (hive serves TLS with a certificate +# from its proxy CA for the endpoint host). Hive-wide only; a per-agent job +# block cannot enable or repoint it. Pair it with a NetworkPolicy. +# agent_sandbox: +# job: +# github_read_proxy: +# enabled: false +# listen_addr: ":8443" # required when enabled; no default +# endpoint: https://hive.hive-system.svc:8443 # how Job pods reach hive; required when enabled +# no_proxy: [] # optional extra hosts Jobs reach directly diff --git a/src/pkg/config/crux_aliases.go b/src/pkg/config/crux_aliases.go index 5b6cd3254a..debd9fe21e 100644 --- a/src/pkg/config/crux_aliases.go +++ b/src/pkg/config/crux_aliases.go @@ -36,6 +36,7 @@ type ( ReviewModelsConfig = cruxconfig.ReviewModelsConfig RotationConfig = cruxconfig.RotationConfig SandboxJobConfig = cruxconfig.SandboxJobConfig + SandboxGitHubReadProxy = cruxconfig.SandboxGitHubReadProxy SandboxPublishConfig = cruxconfig.SandboxPublishConfig SandboxJobResources = cruxconfig.SandboxJobResources SandboxJobToleration = cruxconfig.SandboxJobToleration diff --git a/src/pkg/config/sandbox_github_read_proxy_validate_test.go b/src/pkg/config/sandbox_github_read_proxy_validate_test.go new file mode 100644 index 0000000000..f2a87bc5fa --- /dev/null +++ b/src/pkg/config/sandbox_github_read_proxy_validate_test.go @@ -0,0 +1,40 @@ +package config + +import ( + "strings" + "testing" +) + +// Config validation (the load and save paths) enforces the read relay's +// contract: default off, no default bind, https endpoint only (#18865). +func TestConfigValidateSandboxGitHubReadProxy(t *testing.T) { + withProxy := func(rp *SandboxGitHubReadProxy) *Config { + c := &Config{} + c.AgentSandbox = AgentSandboxConfig{Job: &SandboxJobConfig{GitHubReadProxy: rp}} + return c + } + for _, tc := range []struct { + name string + cfg *Config + wantErr string + }{ + {"no block", &Config{}, ""}, + {"disabled", withProxy(&SandboxGitHubReadProxy{Endpoint: "http://h:1"}), ""}, + {"enabled and complete", withProxy(&SandboxGitHubReadProxy{Enabled: true, ListenAddr: ":8443", Endpoint: "https://hive.ns.svc:8443"}), ""}, + {"enabled without listen_addr", withProxy(&SandboxGitHubReadProxy{Enabled: true, Endpoint: "https://hive.ns.svc:8443"}), "github_read_proxy.listen_addr"}, + {"enabled with http endpoint", withProxy(&SandboxGitHubReadProxy{Enabled: true, ListenAddr: ":8443", Endpoint: "http://hive.ns.svc:8443"}), "https"}, + } { + t.Run(tc.name, func(t *testing.T) { + err := tc.cfg.validateJobAffinities() + if tc.wantErr == "" { + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + return + } + if err == nil || !strings.Contains(err.Error(), tc.wantErr) { + t.Fatalf("err = %v, want one containing %q", err, tc.wantErr) + } + }) + } +} diff --git a/src/pkg/config/validate.go b/src/pkg/config/validate.go index d141eb37be..b4d056e8e0 100644 --- a/src/pkg/config/validate.go +++ b/src/pkg/config/validate.go @@ -413,6 +413,11 @@ func (c *Config) ValidateWithOptions(opts ValidateOptions) error { } func (c *Config) validateJobAffinities() error { + if c.AgentSandbox.Job != nil { + if err := c.AgentSandbox.Job.GitHubReadProxy.Validate(); err != nil { + return err + } + } if c.AgentSandbox.Job != nil { if err := cruxconfig.ValidateSandboxJobAffinity(c.AgentSandbox.Job.Affinity); err != nil { return fmt.Errorf("agent_sandbox.job.%w", err) diff --git a/src/pkg/dashboard/settings_parity_allowlist.txt b/src/pkg/dashboard/settings_parity_allowlist.txt index b530de75ae..b10a9e5e31 100644 --- a/src/pkg/dashboard/settings_parity_allowlist.txt +++ b/src/pkg/dashboard/settings_parity_allowlist.txt @@ -8,6 +8,10 @@ agent_sandbox.job.affinity.nodeAffinity.preferredDuringSchedulingIgnoredDuringEx agent_sandbox.job.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[].matchExpressions[].key # TODO #18809 agent_sandbox.job.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[].matchExpressions[].operator # TODO #18809 agent_sandbox.job.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[].matchExpressions[].values[] # TODO #18809 +agent_sandbox.job.github_read_proxy.enabled # TODO #18865 +agent_sandbox.job.github_read_proxy.endpoint # TODO #18865 +agent_sandbox.job.github_read_proxy.listen_addr # TODO #18865 +agent_sandbox.job.github_read_proxy.no_proxy[] # TODO #18865 agent_sandbox.job.image_pull_secrets[] # TODO #11854 agent_sandbox.job.node_selector. # TODO #11854 agent_sandbox.job.relay_image # TODO #11854 @@ -136,6 +140,10 @@ agents..sandbox.job.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDu agents..sandbox.job.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[].matchExpressions[].operator # TODO #18809 agents..sandbox.job.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[].matchExpressions[].values[] # TODO #18809 agents..sandbox.job.env_from_secrets[] # TODO #11854 +agents..sandbox.job.github_read_proxy.enabled # TODO #18865 +agents..sandbox.job.github_read_proxy.endpoint # TODO #18865 +agents..sandbox.job.github_read_proxy.listen_addr # TODO #18865 +agents..sandbox.job.github_read_proxy.no_proxy[] # TODO #18865 agents..sandbox.job.image_pull_secrets[] # TODO #11854 agents..sandbox.job.node_selector. # TODO #11854 agents..sandbox.job.relay_image # TODO #11854