Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/added-18512-sandbox-relays.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- Sandboxed agent kicks now get hive's own `hive-review` relay on `PATH`. The Job runtime copies it in from the running hive image by digest, through a restricted-SCC-safe init container. Podman bind-mounts it read-only. Project images no longer need to include hive's scripts, only bash and python3, and a kick on an image without them fails with a clear message (#18512).
2 changes: 2 additions & 0 deletions src/crux/agent/manager_sandbox.go
Original file line number Diff line number Diff line change
Expand Up @@ -447,6 +447,8 @@ func JobOptionsFromConfig(job config.SandboxJobConfig) kubejob.Options {
SchedulerName: job.SchedulerName,
EnvFromSecrets: append([]string(nil), job.EnvFromSecrets...),
TTLSeconds: job.TTLSeconds,
RelayImage: job.RelayImage,
RelayRunAsUser: job.RelayRunAsUser,
Resources: kubejob.Resources{
Limits: job.Resources.Limits,
Requests: job.Resources.Requests,
Expand Down
3 changes: 3 additions & 0 deletions src/crux/agent/sandbox_executor.go
Original file line number Diff line number Diff line change
Expand Up @@ -229,6 +229,9 @@ func (e *SandboxExecutor) Run(ctx context.Context, spec SandboxKickSpec) (Sandbo
EnvAllowlist: spec.EnvAllowlist,
NetworkMode: spec.NetworkMode,
ClineProvider: provider,
// hive-review and the other sandbox-safe relays come from hive, not
// the project image (#18512).
Relays: true,
})
if provider != nil {
sres.Stdout = provider.Redact(sres.Stdout)
Expand Down
9 changes: 9 additions & 0 deletions src/crux/config/agent_config.go
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,15 @@ type SandboxJobConfig struct {
EnvFromSecrets []string `yaml:"env_from_secrets,omitempty" json:"env_from_secrets,omitempty"`
Volumes []SandboxJobVolume `yaml:"volumes,omitempty" json:"volumes,omitempty"`
TTLSeconds int `yaml:"ttl_seconds,omitempty" json:"ttl_seconds,omitempty"`
// RelayImage is the image of the init container that copies hive's
// sandbox relays (hive-review) into the Job (#18512). Empty resolves the
// running hive image by digest from hive's own pod; "none" skips the
// relays entirely.
RelayImage string `yaml:"relay_image,omitempty" json:"relay_image,omitempty"`
// RelayRunAsUser pins the relay init container's uid. Unset leaves it to
// the cluster on OpenShift (the SCC assigns one from the namespace range)
// and uses 65534 elsewhere, since the hive image's default user is root.
RelayRunAsUser *int64 `yaml:"relay_run_as_user,omitempty" json:"relay_run_as_user,omitempty"`
}

// SandboxJobToleration mirrors the subset of a Kubernetes toleration an
Expand Down
108 changes: 108 additions & 0 deletions src/crux/kubejob/client.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
package kubejob

import (
"bytes"
"context"
"crypto/tls"
"crypto/x509"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"os"
"strings"

"github.com/hivecommons/hive/crux/egressmark"
)

// do performs one authenticated API call and returns the response body.
// Non-2xx responses are errors carrying the status and the API's message.
func (l *Launcher) do(ctx context.Context, method, path string, body []byte) ([]byte, error) {
client, err := l.client()
if err != nil {
return nil, err
}
server := l.APIServer
if server == "" {
server = DefaultAPIServer
}
reqCtx, cancel := context.WithTimeout(ctx, requestTimeout)
defer cancel()
var rdr io.Reader
if body != nil {
rdr = bytes.NewReader(body)
}
req, err := http.NewRequestWithContext(reqCtx, method, server+path, rdr)
if err != nil {
return nil, err
}
if tok := l.token(); tok != "" {
req.Header.Set("Authorization", "Bearer "+tok)
}
if body != nil {
req.Header.Set("Content-Type", "application/json")
}
resp, err := client.Do(req)
if err != nil {
return nil, err
}
defer func() { _ = resp.Body.Close() }()
out, err := io.ReadAll(io.LimitReader(resp.Body, maxLogBytes+1))
if err != nil {
return nil, err
}
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return nil, fmt.Errorf("%s %s: HTTP %d: %s", method, path, resp.StatusCode, apiMessage(out))
}
return out, nil
}

// apiMessage pulls the human message out of a Kubernetes Status body, falling
// back to a trimmed raw body.
func apiMessage(raw []byte) string {
var st struct {
Message string `json:"message"`
}
if json.Unmarshal(raw, &st) == nil && st.Message != "" {
return st.Message
}
s := strings.TrimSpace(string(raw))
if len(s) > 200 {
s = s[:200]
}
return s
}

func (l *Launcher) token() string {
path := l.TokenPath
if path == "" {
path = DefaultTokenPath
}
raw, err := os.ReadFile(path)
if err != nil {
return ""
}
return strings.TrimSpace(string(raw))
}

// client returns the injected HTTP client, or one trusting the in-cluster CA.
func (l *Launcher) client() (*http.Client, error) {
if l.HTTPClient != nil {
return l.HTTPClient, nil
}
path := l.CAPath
if path == "" {
path = DefaultCAPath
}
pem, err := os.ReadFile(path)
if err != nil {
return nil, fmt.Errorf("kubejob: reading in-cluster CA: %w", err)
}
pool := x509.NewCertPool()
if !pool.AppendCertsFromPEM(pem) {
return nil, errors.New("kubejob: in-cluster CA bundle contained no certificates")
}
l.HTTPClient = &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}, DialContext: egressmark.Dialer(0).DialContext}}
return l.HTTPClient, nil
}
144 changes: 45 additions & 99 deletions src/crux/kubejob/kubejob.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,22 +26,17 @@
package kubejob

import (
"bytes"
"context"
"crypto/tls"
"crypto/x509"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"os"
"path/filepath"
"regexp"
"strings"
"time"

"github.com/hivecommons/hive/crux/egressmark"
"github.com/hivecommons/hive/crux/sandbox"
)

Expand Down Expand Up @@ -135,6 +130,14 @@ type Options struct {
Network string
// PollInterval overrides how often the Job status is read (tests).
PollInterval time.Duration
// RelayImage is the image of the init container that supplies hive's
// sandbox relays to a LaunchSpec with Relays set (#18512). Empty resolves
// the running hive image, by digest, from hive's own pod; RelayImageNone
// skips the relays.
RelayImage string
// RelayRunAsUser pins the relay init container's uid. Nil leaves it to
// the cluster on OpenShift and uses DefaultRelayRunAsUser elsewhere.
RelayRunAsUser *int64
}

// network is Options.Network with the empty default applied.
Expand Down Expand Up @@ -196,6 +199,9 @@ type Launcher struct {
NamespacePath string
HTTPClient *http.Client
Now func() time.Time
// PodName is hive's own pod, read to resolve the relay image; empty
// falls back to $POD_NAME, then $HOSTNAME, then os.Hostname.
PodName string
}

var _ sandbox.Launcher = (*Launcher)(nil)
Expand All @@ -214,6 +220,9 @@ func (o Options) Validate() error {
return fmt.Errorf("kubejob: volumes[%d] needs both claim and mount_path", i)
}
}
if o.RelayRunAsUser != nil && *o.RelayRunAsUser <= 0 {
return errors.New("kubejob: relay_run_as_user must be a positive (non-root) uid")
}
switch o.network() {
case NetworkDefault, NetworkNone, NetworkHost:
default:
Expand Down Expand Up @@ -248,8 +257,14 @@ func (l *Launcher) Run(ctx context.Context, spec sandbox.LaunchSpec) (sandbox.Re
if err != nil {
return res, err
}
var relays *relayPlan
if spec.Relays {
if relays, err = l.resolveRelays(ctx, ns); err != nil {
return res, err
}
}
name := jobName(spec.Name, l.now())
body, err := json.Marshal(l.manifest(ns, name, subPath, spec))
body, err := json.Marshal(l.jobManifest(ns, name, subPath, spec, relays))
if err != nil {
return res, err
}
Expand Down Expand Up @@ -493,6 +508,12 @@ func (l *Launcher) delete(ctx context.Context, jobPath string) error {
// manifest renders the batch/v1 Job. Kept as plain maps rather than client-go
// types: the repo has no k8s.io dependency and this is the only writer.
func (l *Launcher) manifest(ns, name, subPath string, spec sandbox.LaunchSpec) map[string]any {
return l.jobManifest(ns, name, subPath, spec, nil)
}

// jobManifest is manifest plus, when relays is non-nil, the relay init
// container, its emptyDir and the agent container's read-only mount of it.
func (l *Launcher) jobManifest(ns, name, subPath string, spec sandbox.LaunchSpec, relays *relayPlan) map[string]any {
mount := spec.WorkspaceMount
if mount == "" {
mount = sandbox.DefaultWorkspaceMount
Expand Down Expand Up @@ -557,8 +578,18 @@ func (l *Launcher) manifest(ns, name, subPath string, spec sandbox.LaunchSpec) m
"envFrom": envFrom,
"volumeMounts": mounts,
}
if len(spec.Command) > 0 {
container["command"] = spec.Command
command := spec.Command
if relays != nil {
command = sandbox.RelayCommand(command)
mounts = append(mounts, map[string]any{"name": relayVolumeName, "mountPath": sandbox.RelayMountPath, "readOnly": true})
container["volumeMounts"] = mounts
volumes = append(volumes, map[string]any{
"name": relayVolumeName,
"emptyDir": map[string]string{"sizeLimit": relayVolumeSizeLimit},
})
}
if len(command) > 0 {
container["command"] = command
}
if len(l.Options.Resources.Limits) > 0 || len(l.Options.Resources.Requests) > 0 {
container["resources"] = l.Options.Resources
Expand All @@ -574,6 +605,12 @@ func (l *Launcher) manifest(ns, name, subPath string, spec sandbox.LaunchSpec) m
"seccompProfile": map[string]string{"type": "RuntimeDefault"},
},
}
if relays != nil {
podSpec["initContainers"] = []map[string]any{relays.initContainer()}
if len(relays.pullSecrets) > 0 {
podSpec["imagePullSecrets"] = relays.pullSecrets
}
}
if len(l.Options.NodeSelector) > 0 {
podSpec["nodeSelector"] = l.Options.NodeSelector
}
Expand Down Expand Up @@ -692,94 +729,3 @@ func (l *Launcher) namespace() (string, error) {
}
return ns, nil
}

// do performs one authenticated API call and returns the response body.
// Non-2xx responses are errors carrying the status and the API's message.
func (l *Launcher) do(ctx context.Context, method, path string, body []byte) ([]byte, error) {
client, err := l.client()
if err != nil {
return nil, err
}
server := l.APIServer
if server == "" {
server = DefaultAPIServer
}
reqCtx, cancel := context.WithTimeout(ctx, requestTimeout)
defer cancel()
var rdr io.Reader
if body != nil {
rdr = bytes.NewReader(body)
}
req, err := http.NewRequestWithContext(reqCtx, method, server+path, rdr)
if err != nil {
return nil, err
}
if tok := l.token(); tok != "" {
req.Header.Set("Authorization", "Bearer "+tok)
}
if body != nil {
req.Header.Set("Content-Type", "application/json")
}
resp, err := client.Do(req)
if err != nil {
return nil, err
}
defer func() { _ = resp.Body.Close() }()
out, err := io.ReadAll(io.LimitReader(resp.Body, maxLogBytes+1))
if err != nil {
return nil, err
}
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return nil, fmt.Errorf("%s %s: HTTP %d: %s", method, path, resp.StatusCode, apiMessage(out))
}
return out, nil
}

// apiMessage pulls the human message out of a Kubernetes Status body, falling
// back to a trimmed raw body.
func apiMessage(raw []byte) string {
var st struct {
Message string `json:"message"`
}
if json.Unmarshal(raw, &st) == nil && st.Message != "" {
return st.Message
}
s := strings.TrimSpace(string(raw))
if len(s) > 200 {
s = s[:200]
}
return s
}

func (l *Launcher) token() string {
path := l.TokenPath
if path == "" {
path = DefaultTokenPath
}
raw, err := os.ReadFile(path)
if err != nil {
return ""
}
return strings.TrimSpace(string(raw))
}

// client returns the injected HTTP client, or one trusting the in-cluster CA.
func (l *Launcher) client() (*http.Client, error) {
if l.HTTPClient != nil {
return l.HTTPClient, nil
}
path := l.CAPath
if path == "" {
path = DefaultCAPath
}
pem, err := os.ReadFile(path)
if err != nil {
return nil, fmt.Errorf("kubejob: reading in-cluster CA: %w", err)
}
pool := x509.NewCertPool()
if !pool.AppendCertsFromPEM(pem) {
return nil, errors.New("kubejob: in-cluster CA bundle contained no certificates")
}
l.HTTPClient = &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}, DialContext: egressmark.Dialer(0).DialContext}}
return l.HTTPClient, nil
}
11 changes: 11 additions & 0 deletions src/crux/kubejob/kubejob_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,8 @@ type fakeAPI struct {
policyErr int // when >0, the NetworkPolicy POST fails with this status
policyDeleted bool
order []string // "policy", "job" in creation order
selfPod string // JSON body of hive's own pod (testSelfPod), "" = 404
selfPodGets int
}

func newFakeAPI() *fakeAPI {
Expand Down Expand Up @@ -120,6 +122,14 @@ func (f *fakeAPI) handler(t *testing.T) http.HandlerFunc {
f.onPodLogAfterComplete()
}
_, _ = w.Write([]byte(f.logBody))
case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/pods/"+testSelfPod):
f.selfPodGets++
if f.selfPod == "" {
w.WriteHeader(http.StatusNotFound)
_, _ = w.Write([]byte(`{"kind":"Status","message":"pods \"` + testSelfPod + `\" not found"}`))
return
}
_, _ = w.Write([]byte(f.selfPod))
case r.Method == http.MethodGet && strings.Contains(r.URL.Path, "/pods/"):
state := `{"terminated":{"exitCode":` + itoa(f.exitCode) + `}}`
if f.podStatus != "terminated" {
Expand Down Expand Up @@ -155,6 +165,7 @@ func newLauncher(srv *httptest.Server, opts Options) *Launcher {
HTTPClient: srv.Client(),
TokenPath: filepath.Join("/nonexistent", "token"),
Now: func() time.Time { return time.Unix(1700000000, 0) },
PodName: testSelfPod,
}
}

Expand Down
Loading
Loading