Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 19 additions & 4 deletions .github/scripts/native-queue-policy.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -237,10 +237,12 @@ function isCosmeticLabelFlip({ eventName, payload }) {
// A cosmetic flip skips the evidence wait but must not mint a fresh passing
// required check: replay the latest completed `success` run of this check on
// the head (rejecting one not from this workflow) and still read live PR
// eligibility. Returns null when there is nothing to replay — no completed
// run, or the latest one did not succeed (a failed, cancelled or timed-out
// wait) — so the caller falls through to the full evidence wait instead of
// pinning that earlier verdict onto every later stage-label flip.
// eligibility and the POLICY_CHECKS evidence. Returns null when there is
// nothing to replay — no completed run, or the latest one did not succeed (a
// failed, cancelled or timed-out wait), or a policy provider is missing,
// still running or reporting UNKNOWN — so the caller falls through to the
// full evidence wait instead of pinning that earlier verdict onto every later
// stage-label flip.
async function replayLastVerdict({ github, context, currentCheckRunId }) {
const repo = context.repo;
const event = context.payload.pull_request;
Expand All @@ -259,6 +261,19 @@ async function replayLastVerdict({ github, context, currentCheckRunId }) {
// a run of this workflow file on this event may be replayed.
await assertActionsProvenance(github, repo, pr.number, pr.head.sha, prior, POLICY_CHECK_SOURCE, new Map());
if (prior.conclusion !== 'success') return null;
// The prior success is only valid while the providers it validated are
// still green: one re-run red on this head must not be papered over. A
// provider that is missing or still running is not a verdict either way;
// fall through so the caller runs the full polling wait instead of failing.
// A DCO enumeration UNKNOWN also falls through: the full wait re-raises it
// once and renders the runbook job summary, so operators see the same
// surface whichever event fired.
try {
await assertEvidence(github, repo, pr.number, pr.head.sha, POLICY_CHECKS);
} catch (error) {
if (error instanceof PendingEvidence || error instanceof ProviderUnavailable) return null;
throw error;
}
return `PR #${pr.number}: replayed ${POLICY_CHECK_NAME} run ${prior.id} (success) on ${pr.head.sha}`;
}

Expand Down
40 changes: 39 additions & 1 deletion .github/scripts/native-queue-policy.test.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -409,11 +409,16 @@ function replayFixture(priorRuns, labels = [], workflowOverride = {}) {
check_suite: { id: 5000 + r.id },
details_url: `https://github.com/acme/widget/actions/runs/${3000 + r.id}/job/1`, ...r,
}));
f.github.rest.actions.getWorkflowRun = async ({ run_id: id }) => ({ data: {
const baseGetWorkflowRun = f.github.rest.actions.getWorkflowRun;
const basePaginate = f.github.paginate;
f.github.rest.actions.getWorkflowRun = async ({ run_id: id }) => (id < 3000 ? baseGetWorkflowRun({ run_id: id }) : { data: {
id, check_suite_id: id + 2000, path: '.github/workflows/native-queue-policy.yml',
event: 'pull_request_target', head_sha: head, ...workflowOverride,
} });
f.github.paginate = async (method, args) => {
// Policy provider evidence comes from the base fixture (all green).
if (method === 'jobs' && args.run_id < 3000) return basePaginate(method, args);
if (method === 'checks' && !args.check_name) return basePaginate(method, args);
if (method === 'jobs') {
const run = runs.find(r => 3000 + r.id === args.run_id);
if (!run) return [];
Expand All @@ -440,6 +445,39 @@ test('cosmetic flip replays the latest completed success on the same head', asyn
assert.equal(f.state.listed[0].ref, sha(11));
});

test('cosmetic flip does not replay a success when a policy provider re-ran red on the head', async () => {
for (const name of Object.keys(POLICY_CHECKS)) {
const f = replayFixture([{ id: 7, status: 'completed', conclusion: 'success' }]);
f.state.runs.get(sha(11)).find(r => r.name === name).conclusion = 'failure';
await assert.rejects(replayLastVerdict(f), new RegExp(`${name}: failure`));
}
});

test('cosmetic flip falls through to the full wait while a policy provider is re-running or missing', async () => {
// PendingEvidence is not a verdict: returning null sends the caller to the
// polling wait instead of failing the check with no retry.
for (const name of Object.keys(POLICY_CHECKS)) {
for (const change of [{ status: 'queued', conclusion: null }, { status: 'in_progress', conclusion: null }]) {
const f = replayFixture([{ id: 7, status: 'completed', conclusion: 'success' }]);
Object.assign(f.state.runs.get(sha(11)).find(r => r.name === name), change);
assert.equal(await replayLastVerdict(f), null);
}
const f = replayFixture([{ id: 7, status: 'completed', conclusion: 'success' }]);
const runs = f.state.runs.get(sha(11));
runs.splice(runs.findIndex(r => r.name === name), 1);
assert.equal(await replayLastVerdict(f), null);
}
});

test('cosmetic flip falls through to the full wait when the DCO provider reports UNKNOWN', async () => {
// The full wait re-raises ProviderUnavailable once and renders the runbook
// job summary; failing here would skip that operator-facing surface.
const f = replayFixture([{ id: 7, status: 'completed', conclusion: 'success' }]);
Object.assign(f.state.runs.get(sha(11)).find(r => r.name === 'DCO'),
{ conclusion: 'failure', output: { summary: unavailableSummary } });
assert.equal(await replayLastVerdict(f), null);
});

test('cosmetic flip has nothing to replay without a prior success and falls through', async () => {
// A failed, cancelled (timed-out) or absent prior run must not be pinned
// onto every later stage-label flip: null sends the caller to the full wait.
Expand Down
1 change: 1 addition & 0 deletions changelog.d/added-crux-contained-run-service.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- crux: contained run service (`crux runs`) on top of the sandbox launcher (Podman + kubejob) with saved run state under `runstate/` and `/v1/runs` serve routes (RFC hivecommons/crux#114, S2 + S7 minimum).
1 change: 1 addition & 0 deletions changelog.d/changed-16567.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- build(deps): bump ws from 8.21.3 to 8.22.1 in /src/proxy (#16567)
1 change: 1 addition & 0 deletions changelog.d/changed-16785-proxy-override-docs.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- `src/docs/env-vars.md` and `src/docs/security-model.md` now describe `HIVE_PROXY_ALLOW_PRIVATE_FORWARD` as it behaves since [#16785](https://github.com/hivecommons/hive/pull/16785): the destination guard covers plain-HTTP forwards, uninspected CONNECT tunnels and transparent-TLS passthrough, and the override lifts it for all three legs at once. The docs previously described the guard and its override as plain-HTTP-only.
1 change: 1 addition & 0 deletions changelog.d/fixed-autopick-pool-fallback.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- hub: Auto-pick on "Approve provision" now falls back to any other registered non-GPU pool (most available first, tie-break by cluster ID) when the request's preferred pool has no available placeholder, instead of returning 409 while capacity exists elsewhere; private requests stay pinned to the GPU pool and an admin's explicit placeholder choice remains strict.
1 change: 1 addition & 0 deletions changelog.d/fixed-kubejob-sa-token-exit-code.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- Sandbox Job pods no longer receive the ServiceAccount token and now run with a restrictive security context, and an unreadable completed pod no longer reports a guessed exit code 0 ([#17021](https://github.com/hivecommons/hive/issues/17021), [#17022](https://github.com/hivecommons/hive/issues/17022)). Previously the kubejob manifest left `automountServiceAccountToken` at its default and set no `securityContext`, so a contained run could read the cluster token; it now sets `automountServiceAccountToken: false`, `allowPrivilegeEscalation: false`, drops all capabilities and uses the `RuntimeDefault` seccomp profile. When a Job completes but its pod can no longer be read, the result keeps the unknown exit code (`-1`) instead of claiming 0; the kick log shows `exit_code: unknown`, the transcript's trailing status line and the dashboard's job terminal show the code as `-1`, and neither the publish gate nor `publish.verify` withholds the PR for it.
17 changes: 14 additions & 3 deletions src/crux/agent/sandbox_executor.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import (
"github.com/hivecommons/hive/crux/config"
"github.com/hivecommons/hive/crux/effects"
"github.com/hivecommons/hive/crux/internal/textutil"
"github.com/hivecommons/hive/crux/kubejob"
"github.com/hivecommons/hive/crux/logscrub"
"github.com/hivecommons/hive/crux/outputschema"
"github.com/hivecommons/hive/crux/pushbroker"
Expand Down Expand Up @@ -129,8 +130,16 @@ type SandboxVerifyResult struct {
TimedOut bool
}

// Passed reports whether the command ran to completion with exit 0.
func (v SandboxVerifyResult) Passed() bool { return v.Err == "" && v.ExitCode == 0 && !v.TimedOut }
// Passed reports whether the command ran to completion with exit 0. A
// launcher that returned no error but could not read the exit code back
// (a Complete Job whose pod is already gone) counts as passed too: the
// Job's own success condition is the fact, the missing code is not a failure.
func (v SandboxVerifyResult) Passed() bool {
if v.Err != "" || v.TimedOut {
return false
}
return v.ExitCode == 0 || v.ExitCode == kubejob.ExitCodeUnknown
}

type SandboxExecutor struct {
Launcher sandbox.Launcher
Expand Down Expand Up @@ -622,7 +631,9 @@ func sandboxPublishWithheld(res SandboxKickResult) string {
}
return reason
}
if res.Sandbox.ExitCode != 0 {
// A completed Job whose pod could not be read has no exit code; that is
// not a failure, and the guessed 0 it used to carry was not a fact either.
if code := res.Sandbox.ExitCode; code != 0 && code != kubejob.ExitCodeUnknown {
return fmt.Sprintf("sandbox exited %d", res.Sandbox.ExitCode)
}
return ""
Expand Down
9 changes: 8 additions & 1 deletion src/crux/agent/sandbox_kick_log.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,11 @@ import (
"fmt"
"os"
"path/filepath"
"strconv"
"strings"
"time"

"github.com/hivecommons/hive/crux/kubejob"
)

const (
Expand All @@ -25,7 +28,11 @@ const (
// text from the agent.
func sandboxKickLogBody(runtime string, res SandboxKickResult, runErr error) string {
var b strings.Builder
fmt.Fprintf(&b, "runtime: %s\nexit_code: %d\ncommits: %d\n", runtime, res.Sandbox.ExitCode, res.CommitCount)
exitCode := strconv.Itoa(res.Sandbox.ExitCode)
if res.Sandbox.ExitCode == kubejob.ExitCodeUnknown {
exitCode = "unknown"
}
fmt.Fprintf(&b, "runtime: %s\nexit_code: %s\ncommits: %d\n", runtime, exitCode, res.CommitCount)
if res.Branch != "" {
fmt.Fprintf(&b, "branch: %s\n", res.Branch)
}
Expand Down
37 changes: 37 additions & 0 deletions src/crux/agent/sandbox_kick_log_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (

"github.com/hivecommons/hive/crux"
"github.com/hivecommons/hive/crux/config"
"github.com/hivecommons/hive/crux/kubejob"
"github.com/hivecommons/hive/crux/outputschema"
"github.com/hivecommons/hive/crux/sandbox"
)
Expand Down Expand Up @@ -146,3 +147,39 @@ func TestBuildSandboxRunCarriesPRWithheld(t *testing.T) {
t.Errorf("PRWithheld = %q (len %d), want the bounded reason", run.PRWithheld, len([]rune(run.PRWithheld)))
}
}

func TestSandboxKickLogBodyUnknownExitCode(t *testing.T) {
res := SandboxKickResult{Sandbox: sandbox.Result{ExitCode: kubejob.ExitCodeUnknown}}
body := sandboxKickLogBody("job", res, nil)
if !strings.Contains(body, "exit_code: unknown\n") {
t.Errorf("unreadable-pod run must not render a guessed exit code:\n%s", body)
}
}

func TestSandboxPublishWithheldIgnoresUnknownExitCode(t *testing.T) {
unknown := SandboxKickResult{Sandbox: sandbox.Result{ExitCode: kubejob.ExitCodeUnknown}}
if got := sandboxPublishWithheld(unknown); got != "" {
t.Errorf("completed-without-code must not withhold the PR, got %q", got)
}
failed := SandboxKickResult{Sandbox: sandbox.Result{ExitCode: 3}}
if got := sandboxPublishWithheld(failed); got != "sandbox exited 3" {
t.Errorf("got %q, want sandbox exited 3", got)
}
}

func TestSandboxVerifyResultPassedUnknownExitCode(t *testing.T) {
// A Complete verify Job whose pod could not be read back carries no exit
// code and no error; that must not withhold the PR.
if !(SandboxVerifyResult{ExitCode: kubejob.ExitCodeUnknown}).Passed() {
t.Error("nil-error verify with unknown exit code must pass")
}
if (SandboxVerifyResult{ExitCode: kubejob.ExitCodeUnknown, Err: "boom"}).Passed() {
t.Error("unknown exit code with an error must not pass")
}
if (SandboxVerifyResult{ExitCode: kubejob.ExitCodeUnknown, TimedOut: true}).Passed() {
t.Error("unknown exit code after a timeout must not pass")
}
if (SandboxVerifyResult{ExitCode: 1}).Passed() {
t.Error("exit 1 must not pass")
}
}
122 changes: 122 additions & 0 deletions src/crux/api/runs.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,122 @@
package api

import "time"

// Contained runs and saved run results (hivecommons/crux#114, S2 and S7):
//
// POST /v1/runs run an untrusted command in a sandbox
// GET /v1/runs/{id} the saved record, whatever the process state
// POST /v1/runs/{id}/cancel cancel; after completion returns the result
// unchanged with already_finished: true
//
// Routes beyond these (restart, listing) are deferred; see
// src/docs/design/crux-work-services.md.
const PathRuns = "/" + Version + "/runs"

// RunPath is the route of one run's saved record.
func RunPath(id string) string { return PathRuns + "/" + id }

// RunCancelPath is the route that cancels one run.
func RunCancelPath(id string) string { return RunPath(id) + "/cancel" }

// The runs routes answer CodeUnsupported (501, shared with the checkout
// routes) when the request asks for something the configured backend cannot
// enforce (a network mode, a profile, a workspace form); the message names it
// and the run never starts with the limit dropped. It differs from
// CodeNotSupported, which is a whole route the server cannot serve (here: a
// server started without a contained-run backend answers POST /v1/runs with
// CodeNotSupported while still serving the saved records).

// Run statuses, the closed set every record reports.
const (
RunAccepted = "accepted"
RunRunning = "running"
RunWaiting = "waiting"
RunCompleted = "completed"
RunFailed = "failed"
RunCancelled = "cancelled"
RunUnfinished = "unfinished"
RunUnsupported = "unsupported"
)

// Run profiles.
const (
// RunProfileExperiment (the default) runs with network none, no
// credentials and time and output limits.
RunProfileExperiment = "experiment"
// RunProfileAgent is S6's execution half and is not yet accepted
// (CodeUnsupported).
RunProfileAgent = "agent"
)

// Network modes a run may request.
const (
RunNetworkNone = "none"
RunNetworkRestricted = "restricted"
)

// ContainedRunWorktree is the ad-hoc workspace form: the service clones repo and
// pins commit (a full 40-hex SHA) under the run's directory.
type ContainedRunWorktree struct {
Repo string `json:"repo"`
Commit string `json:"commit"`
}

// ContainedRunRequest is POST /v1/runs. Exactly one of Checkout, Worktree or
// WorkspacePath names the workspace. Checkout (an S1 id) is accepted by the
// wire shape but answers CodeUnsupported until the checkout service lands.
type ContainedRunRequest struct {
Checkout string `json:"checkout,omitempty"`
Worktree *ContainedRunWorktree `json:"worktree,omitempty"`
WorkspacePath string `json:"workspace_path,omitempty"`
Command []string `json:"command"`
EnvAllowlist []string `json:"env_allowlist,omitempty"`
// Network is RunNetworkNone (default) or RunNetworkRestricted.
Network string `json:"network,omitempty"`
// TimeoutMs bounds the command; zero uses the server default.
TimeoutMs int64 `json:"timeout_ms,omitempty"`
// MaxOutputBytes caps each of stdout and stderr in the response; zero
// uses the server default. The full output is kept as an artifact.
MaxOutputBytes int64 `json:"max_output_bytes,omitempty"`
// Profile is RunProfileExperiment (default) or RunProfileAgent.
Profile string `json:"profile,omitempty"`
// IdempotencyKey, when set, makes a repeated request return the
// existing run instead of starting a second one (S7).
IdempotencyKey string `json:"idempotency_key,omitempty"`
// Detach returns as soon as the run is accepted (202) instead of
// waiting for the result; poll GET /v1/runs/{id}.
Detach bool `json:"detach,omitempty"`
}

// Run is the saved record of one contained run: the response of every runs
// route. Fields that are not yet known are absent; ExitCode in particular is
// present only when the command's exit is known, never guessed.
type ContainedRun struct {
ID string `json:"id"`
Status string `json:"status"`
Owner string `json:"owner,omitempty"`
// Reason explains a failed, unfinished or unsupported status.
Reason string `json:"reason,omitempty"`
CreatedAt time.Time `json:"created_at"`
StartedAt *time.Time `json:"started_at,omitempty"`
FinishedAt *time.Time `json:"finished_at,omitempty"`

Profile string `json:"profile,omitempty"`
Network string `json:"network,omitempty"`
Backend string `json:"backend,omitempty"`
Command []string `json:"command,omitempty"`
Workdir string `json:"workspace,omitempty"`
ExitCode *int `json:"exit_code,omitempty"`
TimedOut bool `json:"timed_out,omitempty"`
Stdout string `json:"stdout,omitempty"`
Stderr string `json:"stderr,omitempty"`
// StdoutTruncatedAt and StderrTruncatedAt are the byte offsets at which
// the response output was cut; absent when nothing was truncated. The
// full, scrubbed output is the stdout.txt / stderr.txt artifact in the
// run's directory.
StdoutTruncatedAt *int64 `json:"stdout_truncated_at,omitempty"`
StderrTruncatedAt *int64 `json:"stderr_truncated_at,omitempty"`
// AlreadyFinished is set on a cancel response when the run had already
// reached a terminal status; the record is returned unchanged.
AlreadyFinished bool `json:"already_finished,omitempty"`
}
27 changes: 27 additions & 0 deletions src/crux/api/runs_client.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
package api

import (
"context"
"net/http"
)

// CreateRun is POST /v1/runs: the saved record, final unless req.Detach.
func (c *Client) CreateRun(ctx context.Context, req ContainedRunRequest) (ContainedRun, error) {
var out ContainedRun
err := c.Do(ctx, http.MethodPost, PathRuns, req, &out)
return out, err
}

// GetRun is GET /v1/runs/{id}.
func (c *Client) GetRun(ctx context.Context, id string) (ContainedRun, error) {
var out ContainedRun
err := c.Do(ctx, http.MethodGet, RunPath(id), nil, &out)
return out, err
}

// CancelRun is POST /v1/runs/{id}/cancel.
func (c *Client) CancelRun(ctx context.Context, id string) (ContainedRun, error) {
var out ContainedRun
err := c.Do(ctx, http.MethodPost, RunCancelPath(id), nil, &out)
return out, err
}
12 changes: 12 additions & 0 deletions src/crux/api/runs_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
package api

import "testing"

func TestRunPaths(t *testing.T) {
if PathRuns != "/v1/runs" || RunPath("run_a") != "/v1/runs/run_a" || RunCancelPath("run_a") != "/v1/runs/run_a/cancel" {
t.Fatalf("paths: %s %s %s", PathRuns, RunPath("run_a"), RunCancelPath("run_a"))
}
if CodeUnsupported == CodeNotSupported {
t.Fatal("unsupported-limit refusals must be distinguishable from a not-configured server")
}
}
Loading
Loading