diff --git a/.gitforge.yml b/.gitforge.yml index 3065bb4d..15490a70 100644 --- a/.gitforge.yml +++ b/.gitforge.yml @@ -2,16 +2,23 @@ # # The project's own gates, run by the project itself: a shellcheck pass over # the release-channel ops scripts, rustfmt --check plus the route-docs drift -# gate, the strict clippy pass, the workspace test suite, the line-coverage -# gate, and a bounded fuzz smoke over the untrusted-input parsers. Pushes to -# any branch run the full chain, so a PR branch is validated here before it -# merges. +# gate, the strict clippy pass, the workspace test suite, the docker-dependent +# tests that `cargo test` skips, the line-coverage gate, and a bounded fuzz +# smoke over the untrusted-input parsers. Pushes to any branch run the full +# chain, so a PR branch is validated here before it merges. # # CHANGE-DETECTION PRUNING: the detect-changes job enumerates files changed # vs the merge-base with origin/main and classifies them. Downstream jobs # use skip_if to skip themselves when their class of file hasn't changed: # - clippy: skips when no Rust source or crates/ changed +# NOTE (2026-11-01, debt): clippy --all-targets also lints tests, but a +# tests-only change (CI_CHANGE_INCLUDE_TESTS=1, CODE=CRATES=0) skips the +# clippy job — caught live on 3dbd166d's parent (48618cf0): an E0425 in +# services/api/tests compiled only by the test job. Acceptable because +# test compiles everything, but clippy-grade lints on tests-only diffs +# are unenforced until the skip_if includes TESTS. # - test: skips when no source, crates/, or tests changed +# - docker-integration: skips when no Rust source or crates/ changed # - coverage: skips when no source, crates/, or tests changed # - fuzz: skips when no source or crates/ changed # A doc-only push (e.g. updating a README) runs detect-changes → lint → fmt @@ -21,7 +28,9 @@ # Documented constraints (same list proven in gods-eye-view's and dsc's # .gitforce.yml): # - The job graph is a LINEAR CHAIN (detect-changes → lint → fmt → clippy → -# test → coverage → fuzz), not a fan-out, +# test → docker-integration → coverage → fuzz), not a fan-out, +# - `environment: {}` is required by the parser (PipelineDefinition has no +# serde default) even though it is never forwarded. # because all jobs of one run share the SAME workspace directory on this # runner. Concurrent jobs would run cargo against the same target/ # directory and corrupt each other's build locks. @@ -55,9 +64,9 @@ # every time; the host directory is keyed by image tag ({image}), so a # rebuilt image starts from an empty cache rather than inheriting # artifacts compiled by a different toolchain. The host location comes -# from the runner's GITFORGE_CACHE_ROOT. A cache mount that cannot be -# prepared degrades to a cold build with a warning — it is an -# optimisation, never a correctness requirement. +# from the runner's GITFORGE_CACHE_ROOT. A mount that cannot be prepared +# fails the job: the runner reports it as an infrastructure failure, so +# the reason reaches the job's own result instead of a log line. name: gitforge-ci version: "1.0" trigger_on: @@ -86,13 +95,38 @@ jobs: # The aegis pattern scan stays host-side (make lint / make aegis): # the binary is a locally-built tool, not distro-packaged, and is # not baked into this image. - run: shellcheck scripts/*.sh scripts/gitforge-* scripts/qualityctl scripts/verify-user-systemd-policy systemd/*.sh + run: shellcheck scripts/*.sh scripts/gitforge-* scripts/qualityctl scripts/verify-user-systemd-policy systemd/*.sh tests/*.sh - name: Change-detection selftest (skip_if classification) # detect-changes.sh is a single point of decision for the whole # quality chain: a misclassification silently skips (or pointlessly # runs) every gated job. These synthetic-repo cases pin the # contract, including the main-push tip-diff fallback. run: bash tests/detect-changes-selftest.sh + - name: Cutover unit-detection selftest + # The release cutover decides which units to stop and start from + # what systemctl reports. A detection gap there is a silent no-op + # release: the symlink flips, nothing restarts, the old binaries + # keep serving, and the cutover reports success. The NAS profile + # runs template INSTANCES, which `systemctl list-unit-files` does + # not enumerate at all - so this selftest pins the instance-aware + # detection against a stubbed systemctl. + run: bash tests/remote-cutover-unit-detection-selftest.sh + - name: Release restart-plan selftest (masked units excluded) + # The release channel restarts a hardcoded unit list. A unit + # masked on this host is a retired component - restarting it made + # every promote roll back (2026-10-10). The skip has to survive + # future edits to that list, and only a test pins it: the failure + # otherwise shows up on the release channel and nowhere else. + run: bash tests/release-auto-restart-plan-selftest.sh + - name: Release busy-gate selftest (store, not container names) + # The release channel defers a cutover while jobs are in flight. + # The busy signal has to come from the authoritative store: the + # runner keeps a warm pool of containers that have no job row, so + # counting containers deferred every cutover forever (GF-16). The + # selftest pins the store-backed count, the stale-row case that + # would otherwise wedge the gate, and the fail-closed behaviour + # when the store cannot be read. + run: bash tests/release-auto-busy-gate-selftest.sh timeout: 10m - name: fmt @@ -172,9 +206,59 @@ jobs: run: cargo test --workspace -- --test-threads=2 timeout: 90m - - name: coverage + - name: docker-integration image: dsc-ci-rust:16 needs: [test] + cache: + # Compiled-artifact cache: the runner bind-mounts a host directory + # (keyed by the image tag) at /cargo-target and exports + # CARGO_TARGET_DIR pointing at it. The mount path inside the + # container is FIXED, so cargo can reuse the dependency artifacts + # an earlier run compiled instead of rebuilding the world every + # time; the host directory is keyed by image tag ({image}), so a + # rebuilt image starts from an empty cache rather than inheriting + # artifacts compiled by a different toolchain. The host location + # comes from the runner's GITFORGE_CACHE_ROOT. + source: "{cache_root}/cargo/{image}" + mount_path: /cargo-target + skip_if: '[[ "$CI_CHANGE_INCLUDE_CODE" == "1" || "$CI_CHANGE_INCLUDE_CRATES" == "1" ]]' + env: + # The daemon socket is mounted at the well-known path inside the + # container, so the tests inside it talk to the daemon there. + DOCKER_HOST: unix:///var/run/docker.sock + volumes: + # The daemon socket, so the job's tests can drive a real daemon. + # `{docker_socket}` expands to the socket of the daemon THIS runner + # uses — a rootless podman runner's socket is not the system-wide + # path. Read-only: the mount grants the daemon API, which is already + # host-root-equivalent, and the socket file itself needs no writes. + # The sandbox detects the socket and runs this container with SELinux + # labeling disabled — the socket's type does not match the + # container's label, so the daemon refuses the connection otherwise. + - source: "{docker_socket}" + destination: /var/run/docker.sock + read_only: true + steps: + - name: Docker-dependent tests (cargo test -- --ignored) + # The `#[ignore]`-gated tests are the only coverage of the runner's + # container timeout reaping and of the sandbox's real container + # lifecycle. They were never run by any gate, so a release could + # ship a broken isolation path and every green run would still + # report success. `cargo test` alone skips them entirely. + # + # Scoped to the two crates that carry them and to test targets + # only: the workspace's doctests accept no `--ignored` filter, and + # the remaining crates have no daemon-dependent tests to run. + # + # The job container talks to the daemon over the socket mounted + # above, so the daemon it drives is the host's — the same daemon + # that runs this job. + run: cargo test -p gitforge-sandbox -p gitforge-runner --lib --tests -- --ignored --test-threads=2 + timeout: 30m + + - name: coverage + image: dsc-ci-rust:16 + needs: [docker-integration] cache: # Compiled-artifact cache: the runner bind-mounts a host directory # (keyed by the image tag) at /cargo-target and exports diff --git a/.githooks/pre-push b/.githooks/pre-push index 58cf2386..21b7d4d2 100755 --- a/.githooks/pre-push +++ b/.githooks/pre-push @@ -31,4 +31,17 @@ if ! bash "$REPO_ROOT/scripts/gitforge-quick-check.sh"; then exit 1 fi +# Advisory branch/commit hygiene. Never blocks the push — it reports a +# dirty tree, a non-kebab branch name, or commits that miss the +# conventional-commit format this repo documents in CLAUDE.md. +# +# The commit range is the upstream tracking ref against HEAD. Git's own +# pre-push arguments ($1=remote ref, $2=local ref) name the same branch +# for an ordinary push, so `git log $1..$2` would compare a ref with +# itself and check nothing. +UPSTREAM="$(git rev-parse --abbrev-ref --symbolic-full-name '@{u}' 2>/dev/null || true)" +if [ -n "$UPSTREAM" ]; then + bash "$REPO_ROOT/.githooks/pre-push.d/20-clean-branch" "$UPSTREAM" HEAD || true +fi + INFO "Pre-push checks passed — pushing..." diff --git a/.githooks/pre-push.d/10-complexity-gate b/.githooks/pre-push.d/10-complexity-gate deleted file mode 100755 index 7f3e4109..00000000 --- a/.githooks/pre-push.d/10-complexity-gate +++ /dev/null @@ -1,254 +0,0 @@ -#!/usr/bin/env bash -# ───────────────────────────────────────────────────────────────────────────── -# Dark Factory Complexity Gate Pre-push Hook -# -# Blocks pushes if: -# - Any function exceeds 50 lines -# - Any file exceeds 500 lines -# - Cyclomatic complexity > 10 (for/if chains) -# -# Usage: Add to .githooks/pre-push -# ───────────────────────────────────────────────────────────────────────────── - -set -euo pipefail - -REPO_ROOT="$(git rev-parse --show-toplevel 2>/dev/null)" -cd "$REPO_ROOT" - -RED='\033[0;31m'; GREEN='\033[0;32m'; YELLOW='\033[1;33m'; NC='\033[0m' -ERROR() { echo -e "${RED}[complexity-gate]${NC} $*"; } -WARN() { echo -e "${YELLOW}[complexity-gate]${NC} $*"; } -INFO() { echo -e "${GREEN}[complexity-gate]${NC} $*"; } - -show_help() { - cat << 'EOF' -Dark Factory Complexity Gate Pre-push Hook - -Blocks push if: - - Any function exceeds 50 lines (Go/Python) - - Any file exceeds 500 lines - - Cyclomatic complexity > 10 (for/if chains) - -Exit codes: - 0 - All checks passed or skipped - 1 - Violation found - -Usage: This hook runs automatically before push. -EOF -} - -for arg in "$@"; do - case "$arg" in - -h|--help) show_help; exit 0 ;; - esac -done - -BLOCKED=0 -VIOLATIONS=() - -# Get list of files being pushed (compare against origin/{branch} or main) -REMOTE_REF="${1:-}" -LOCAL_REF="${2:-}" -LOCAL_COMMIT="${3:-}" - -if [ -z "$LOCAL_REF" ] || [ -z "$LOCAL_COMMIT" ]; then - INFO "No remote configured or nothing to push — skipping complexity gate" - exit 0 -fi - -# Get list of changed/added files from the commits being pushed -if command -v git &>/dev/null && git rev-parse --git-dir &>/dev/null; then - CHANGED_FILES=$(git diff --name-only "$LOCAL_REF".."$LOCAL_COMMIT" 2>/dev/null || echo "") -else - INFO "Not in a git repo or git not available — skipping" - exit 0 -fi - -if [ -z "$CHANGED_FILES" ]; then - INFO "No files changed in this push — skipping complexity gate" - exit 0 -fi - -# ─── 1. File length check (500 lines) ────────────────────────────────────────── -check_file_length() { - local file="$1" - local max_lines=500 - - if [ ! -f "$file" ]; then - return 0 - fi - - # Skip binary and generated files - case "$file" in - *.min.js|*.min.css|*.pb.go|*.pb.gw.go) return 0 ;; - esac - - local line_count - line_count=$(wc -l < "$file" 2>/dev/null || echo 0) - - if [ "$line_count" -gt "$max_lines" ]; then - ERROR "FILE_TOO_LONG: $file has $line_count lines (max: $max_lines)" - VIOLATIONS+=("FILE_TOO_LONG: $file:$line_count") - BLOCKED=1 - fi -} - -# ─── 2. Function length check (50 lines) ────────────────────────────────────── -check_function_length() { - local file="$1" - local max_func_lines=50 - - if [ ! -f "$file" ]; then - return 0 - fi - - # Only check source files - case "$file" in - *.go|*.py) ;; - *) return 0 ;; - esac - - if [ "$file" = "*.go" ]; then - # Go: parse AST to find functions - if command -v gofmt &>/dev/null; then - local in_func=0 - local func_name="" - local func_start=0 - local brace_count=0 - local line_num=0 - - while IFS= read -r line; do - line_num=$((line_num + 1)) - - # Function detection - if grep -qE '^func\s+('"$file"' )?[A-Z][a-zA-Z0-9]*\(' "$file" 2>/dev/null; then - if [ "$in_func" -eq 1 ]; then - # Previous function never closed (syntax error) - func_start=$line_num - else - func_start=$line_num - func_name=$(grep -nE '^func\s+('"$file"' )?[A-Z][a-zA-Z0-9]*\(' "$file" 2>/dev/null | tail -1 | sed 's/.*^func.*(\([^)]*\).*/\1/') - in_func=1 - brace_count=0 - fi - fi - - # Count braces - if [ "$in_func" -eq 1 ]; then - brace_count=$((brace_count + $(echo "$line" | tr -cd '{' | wc -c))) - brace_count=$((brace_count - $(echo "$line" | tr -cd '}' | wc -c))) - - if [ "$brace_count" -le 0 ] && [ "$in_func" -eq 1 ]; then - local func_length=$((line_num - func_start)) - if [ "$func_length" -gt "$max_func_lines" ]; then - ERROR "FUNCTION_TOO_LONG: $file:$func_start function '$func_name' has $func_length lines (max: $max_func_lines)" - VIOLATIONS+=("FUNCTION_TOO_LONG: $file:${func_start}:${func_length}") - BLOCKED=1 - fi - in_func=0 - func_name="" - fi - fi - done < "$file" - fi - elif [ "$file" = "*.py" ]; then - # Python: use awk to track def blocks - awk ' - BEGIN { in_def=0; def_name=""; def_start=0; brace_count=0 } - /^def [a-zA-Z_][a-zA-Z0-9_]*\(/ { - if (in_def && def_start > 0) { - len = NR - def_start - if (len > 50) { - print "FUNCTION_TOO_LONG: FILENAME:" def_start ":" len > "/dev/stderr" - } - } - in_def = 1 - def_start = NR - def_name = $0 - sub(/:.*/, "", def_name) - sub("def ", "", def_name) - } - in_def && /^[ \t]*[^# \t]/ && !/^def / && !/^[ \t]*#/ { - # Track indentation end - if ($0 ~ /^[^ \t]/) { - if (in_def) { - len = NR - def_start - if (len > 50) { - print "FUNCTION_TOO_LONG: FILENAME:" def_start ":" len > "/dev/stderr" - } - } - in_def = 0 - } - } - in_def && /^[^ \t]/ && NR > def_start + 50 { - len = NR - def_start - print "FUNCTION_TOO_LONG: FILENAME:" def_start ":" len > "/dev/stderr" - } - ' "$file" 2>&1 | while read -r line; do - ERROR "$line" - VIOLATIONS+=("FUNCTION_TOO_LONG: $file") - BLOCKED=1 - done - fi -} - -# ─── 3. Cyclomatic complexity check (for/if chains) ─────────────────────────── -check_cyclomatic_complexity() { - local file="$1" - - if [ ! -f "$file" ]; then - return 0 - fi - - # Only check source files - case "$file" in - *.go|*.py|*.js|*.ts) ;; - *) return 0 ;; - esac - - # Simple heuristic: count if/for/while per function - # More than 10 decision points suggests high complexity - - local decision_count=0 - local content - content=$(cat "$file" 2>/dev/null) - - # Count decision keywords (rough cyclomatic complexity estimate) - decision_count=$(grep -cE '\b(if|elif|else if|for|while|case|&&|\|\|)\b' <<< "$content" 2>/dev/null || echo 0) - - # Normalize: if file has more than 10 decision points per 50 lines, flag it - local line_count - line_count=$(wc -l < "$file" 2>/dev/null || echo 1) - local threshold=$((10 + (line_count / 50) * 3)) # Allow more in larger files - - if [ "$decision_count" -gt "$threshold" ]; then - local ratio - ratio=$(echo "scale=1; $decision_count * 100 / $line_count" | bc 2>/dev/null || echo "high") - ERROR "HIGH_COMPLEXITY: $file has $decision_count decision points (ratio: ${ratio}%)" - VIOLATIONS+=("HIGH_COMPLEXITY: $file:$decision_count") - BLOCKED=1 - fi -} - -# ─── Run all checks ──────────────────────────────────────────────────────────── -INFO "Running complexity gate on files being pushed..." -echo "$CHANGED_FILES" | while read -r file; do - [ -z "$file" ] && continue - [ ! -f "$file" ] && continue - - check_file_length "$file" || true - check_function_length "$file" || true - check_cyclomatic_complexity "$file" || true -done - -if [ "$BLOCKED" -eq 1 ]; then - ERROR "Complexity gate blocked push. Fix violations above." - echo "" - WARN "Tip: Break large functions (>50 lines) into smaller helpers." - WARN "Tip: Extract files >500 lines into separate modules." - WARN "Tip: Reduce cyclomatic complexity with early returns or guard clauses." - exit 1 -fi - -INFO "Complexity gate passed" -exit 0 diff --git a/.githooks/pre-push.d/20-clean-branch b/.githooks/pre-push.d/20-clean-branch index 53d941ca..d0dc932e 100755 --- a/.githooks/pre-push.d/20-clean-branch +++ b/.githooks/pre-push.d/20-clean-branch @@ -41,7 +41,7 @@ for arg in "$@"; do case "$arg" in -h|--help) show_help; exit 0 ;; esac -fi +done WARNINGS=0 diff --git a/.githooks/pre-push.d/30-coverage-gate b/.githooks/pre-push.d/30-coverage-gate deleted file mode 100755 index 1a6cbc12..00000000 --- a/.githooks/pre-push.d/30-coverage-gate +++ /dev/null @@ -1,197 +0,0 @@ -#!/usr/bin/env bash -# ───────────────────────────────────────────────────────────────────────────── -# Dark Factory Coverage Gate Pre-push Hook -# -# Fails push if coverage drops below threshold. -# Uses: -# - Go: go tool cover -# - Python: coverage report -# -# Threshold: env COVERAGE_THRESHOLD or default 70% -# -# Usage: Add to .githooks/pre-push -# ───────────────────────────────────────────────────────────────────────────── - -set -euo pipefail - -REPO_ROOT="$(git rev-parse --show-toplevel 2>/dev/null)" -cd "$REPO_ROOT" - -RED='\033[0;31m'; GREEN='\033[0;32m'; YELLOW='\033[1;33m'; NC='\033[0m' -ERROR() { echo -e "${RED}[coverage-gate]${NC} $*"; } -WARN() { echo -e "${YELLOW}[coverage-gate]${NC} $*"; } -INFO() { echo -e "${GREEN}[coverage-gate]${NC} $*"; } - -show_help() { - cat << 'EOF' -Dark Factory Coverage Gate Pre-push Hook - -Fails push if code coverage drops below threshold. - -Supported languages: - - Go: uses 'go tool cover' - - Python: uses 'coverage report' - -Threshold: - - env COVERAGE_THRESHOLD (e.g., export COVERAGE_THRESHOLD=80) - - Default: 70% - -Exit codes: - 0 - Coverage acceptable or no coverage data - 1 - Coverage below threshold -EOF -} - -for arg in "$@"; do - case "$arg" in - -h|--help) show_help; exit 0 ;; - esac -done - -# Get threshold from env or default -THRESHOLD="${COVERAGE_THRESHOLD:-70}" -BLOCKED=0 -SKIPPED=0 - -# ─── Go coverage check ──────────────────────────────────────────────────────── -check_go_coverage() { - if ! command -v go &>/dev/null; then - return 0 - fi - - # Check if there are Go files with tests - if ! ls *.go &>/dev/null && ! find . -name "*.go" -not -path "./vendor/*" 2>/dev/null | head -5 | grep -q .; then - return 0 - fi - - # Look for coverage profiles - local coverage_file="coverage.out" - if [ ! -f "$coverage_file" ]; then - # Try to generate coverage - if go test ./... -coverprofile="$coverage_file" -covermode=atomic &>/dev/null 2>&1; then - INFO "Generated coverage profile: $coverage_file" - else - return 0 # Skip if no tests - fi - fi - - if [ ! -f "$coverage_file" ]; then - return 0 - fi - - # Parse coverage percentage from go tool cover - local coverage_output - coverage_output=$(go tool cover -func="$coverage_file" 2>/dev/null || echo "") - - if [ -z "$coverage_output" ]; then - return 0 - fi - - # Get total coverage (last line usually) - local total_coverage - total_coverage=$(echo "$coverage_output" | grep "^total:" | sed 's/.*\t\([0-9.]*\)%.*/\1/' || echo "") - - if [ -z "$total_coverage" ]; then - # Try parsing differently - total_coverage=$(go tool cover -func="$coverage_file" 2>/dev/null | tail -1 | awk '{print $3}' | tr -d '%' || echo "") - fi - - if [ -z "$total_coverage" ]; then - WARN "Could not parse Go coverage output" - return 0 - fi - - local coverage_int - coverage_int=$(printf "%.0f" "$total_coverage" 2>/dev/null || echo 0) - - INFO "Go coverage: ${coverage_int}% (threshold: ${THRESHOLD}%)" - - if [ "$coverage_int" -lt "$THRESHOLD" ]; then - ERROR "COVERAGE_LOW: Go coverage is ${coverage_int}% (minimum: ${THRESHOLD}%)" - BLOCKED=1 - fi -} - -# ─── Python coverage check ───────────────────────────────────────────────────── -check_python_coverage() { - if ! command -v coverage &>/dev/null; then - # Try with python -m coverage - if ! python3 -m coverage --version &>/dev/null 2>&1; then - return 0 - fi - local coverage_cmd="python3 -m coverage" - else - local coverage_cmd="coverage" - fi - - # Check if there are Python files - if ! ls *.py 2>/dev/null | head -1 | grep -q .; then - if ! find . -name "*.py" -not -path "./venv/*" -not -path "./.venv/*" 2>/dev/null | head -5 | grep -q .; then - return 0 - fi - fi - - # Check for .coverage file - local coverage_file=".coverage" - if [ ! -f "$coverage_file" ]; then - INFO "No Python coverage data found (.coverage) — skipping" - SKIPPED=1 - return 0 - fi - - # Run coverage report - local coverage_output - coverage_output=$($coverage_cmd report --precision=2 2>/dev/null || echo "") - - if [ -z "$coverage_output" ]; then - WARN "Could not generate Python coverage report" - return 0 - fi - - # Parse total coverage (last line with "TOTAL") - local total_coverage - total_coverage=$(echo "$coverage_output" | grep "^TOTAL" | awk '{print $NF}' | tr -d '%' || echo "") - - # Fallback: try last line - if [ -z "$total_coverage" ]; then - total_coverage=$(echo "$coverage_output" | tail -1 | awk '{print $NF}' | tr -d '%' || echo "") - fi - - if [ -z "$total_coverage" ]; then - WARN "Could not parse Python coverage output" - echo "$coverage_output" | tail -5 >&2 - return 0 - fi - - local coverage_int - coverage_int=$(printf "%.0f" "$total_coverage" 2>/dev/null || echo 0) - - INFO "Python coverage: ${coverage_int}% (threshold: ${THRESHOLD}%)" - - if [ "$coverage_int" -lt "$THRESHOLD" ]; then - ERROR "COVERAGE_LOW: Python coverage is ${coverage_int}% (minimum: ${THRESHOLD}%)" - echo "" - echo "$coverage_output" | tail -10 >&2 - BLOCKED=1 - fi -} - -# ─── Run checks ─────────────────────────────────────────────────────────────── -INFO "Running coverage gate (threshold: ${THRESHOLD}%)..." - -check_go_coverage || true -check_python_coverage || true - -if [ "$BLOCKED" -eq 1 ]; then - echo "" - ERROR "Coverage gate blocked push. Improve coverage to at least ${THRESHOLD}%." - exit 1 -fi - -if [ "$SKIPPED" -eq 1 ]; then - INFO "Coverage gate skipped (no coverage data found)" -else - INFO "Coverage gate passed" -fi - -exit 0 diff --git a/.github/workflows/ai-review.yml b/.github/workflows/ai-review.yml index cf2b4bb8..f7767917 100644 --- a/.github/workflows/ai-review.yml +++ b/.github/workflows/ai-review.yml @@ -48,7 +48,7 @@ jobs: fetch-depth: 0 - name: Install Rust - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master + uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # master with: toolchain: stable diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 57064843..0a8c8eb6 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -59,7 +59,7 @@ jobs: - name: Install Rust if: matrix.language == 'rust' - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master + uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # master with: toolchain: stable diff --git a/.github/workflows/rust-ci.yml b/.github/workflows/rust-ci.yml index 7039ccd2..f8f3786b 100644 --- a/.github/workflows/rust-ci.yml +++ b/.github/workflows/rust-ci.yml @@ -30,7 +30,7 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install Rust - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master + uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # master with: toolchain: stable components: rustfmt @@ -44,7 +44,7 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install Rust - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master + uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # master with: toolchain: stable components: clippy @@ -93,7 +93,7 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install Rust - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master + uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # master with: toolchain: stable - name: Run tests @@ -111,7 +111,7 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install Rust - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master + uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # master with: toolchain: stable components: rust-src @@ -137,7 +137,7 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install Rust - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master + uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # master with: toolchain: stable components: rust-src @@ -196,7 +196,7 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install Rust - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master + uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # master with: toolchain: stable - name: Install cargo-audit @@ -213,7 +213,7 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install Rust - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master + uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # master with: toolchain: stable - name: Install cargo-vet @@ -230,7 +230,7 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install Rust - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master + uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # master with: toolchain: stable targets: x86_64-unknown-linux-musl diff --git a/.github/workflows/security.yml b/.github/workflows/security.yml index 644eb349..9e40ac1f 100644 --- a/.github/workflows/security.yml +++ b/.github/workflows/security.yml @@ -37,7 +37,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install Rust - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master + uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # master with: toolchain: stable @@ -197,7 +197,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Install Rust - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master + uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # master with: toolchain: stable components: clippy diff --git a/Cargo.lock b/Cargo.lock index e65cd3b1..fb265a66 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1488,6 +1488,7 @@ dependencies = [ "bcrypt", "chrono", "serde", + "serde_json", "serde_yaml", "thiserror 2.0.21", "tracing", @@ -1636,6 +1637,7 @@ dependencies = [ "gitforge-sandbox", "gitforge-storage", "hex", + "libc", "reqwest", "serde", "serde_json", diff --git a/crates/gitforge-api/src/routes/repo.rs b/crates/gitforge-api/src/routes/repo.rs index bd48dd2c..5ea53102 100644 --- a/crates/gitforge-api/src/routes/repo.rs +++ b/crates/gitforge-api/src/routes/repo.rs @@ -19,6 +19,28 @@ use serde::{Deserialize, Serialize}; use std::sync::Arc; use uuid::Uuid; +/// The filesystem root under which bare repositories live. +/// +/// The root is resolved once at the startup edge (see `ApiServer::build`) +/// and carried into the handlers as data. Reading `GIT_ROOT` inside a +/// handler made every request depend on process-global state that another +/// thread could change underneath it, and left the routes with no way to +/// be pointed at a scratch root in a test. +#[derive(Clone, Debug)] +pub struct RepoStorageRoot(Arc); + +impl RepoStorageRoot { + /// Wrap an already-resolved storage root. + pub fn new(root: impl Into) -> Self { + Self(Arc::new(root.into())) + } + + /// The root as a path string. + pub fn as_str(&self) -> &str { + self.0.as_str() + } +} + /// Validate repository name to prevent path traversal and injection attacks fn validate_repo_name(name: &str) -> Result<(), String> { // Check for empty name @@ -91,7 +113,13 @@ pub struct CreateRepoRequest { } /// Repository routes -pub fn repo_routes() -> Router { +/// +/// `storage_root` is the resolved filesystem root for bare repositories. +/// It is carried in as data rather than read from the environment inside a +/// handler: a request must not depend on process-global state that another +/// thread can change underneath it, and a caller must be able to point the +/// routes at a scratch root. +pub fn repo_routes(storage_root: RepoStorageRoot) -> Router { Router::new() .route("/repos", get(list_repos).post(create_repo)) .route("/repos/{id}", get(get_repo).delete(delete_repo)) @@ -103,6 +131,7 @@ pub fn repo_routes() -> Router { "/repos/{owner}/{repo}/commits/{sha}/status", get(get_commit_status), ) + .layer(Extension(storage_root)) } fn can_access_repo(claims: &Claims, owner_id: UserId) -> bool { @@ -212,6 +241,7 @@ async fn list_repos( async fn create_repo( Extension(pool): Extension>, Extension(claims): Extension, + Extension(storage_root): Extension, Json(req): Json, ) -> impl IntoResponse { let owner_id = claims.user_id; @@ -231,11 +261,7 @@ async fn create_repo( } let repo_id = RepoId::new(); - let storage_root = std::env::var("GIT_ROOT") - .ok() - .filter(|root| !root.trim().is_empty()) - .unwrap_or_else(|| "target/gitforge-repos".to_string()); - let storage = FileStorageBackend::new(&storage_root); + let storage = FileStorageBackend::new(storage_root.as_str()); if let Err(error) = storage.ensure_root().await { tracing::error!(%error, "failed to initialize git storage root"); @@ -458,6 +484,7 @@ async fn get_ref_policy( async fn update_ref_policy( Extension(pool): Extension>, Extension(claims): Extension, + Extension(storage_root): Extension, Path((owner, repo)): Path<(String, String)>, Json(request): Json, ) -> impl IntoResponse { @@ -485,11 +512,7 @@ async fn update_ref_policy( .deny_non_fast_forward .unwrap_or(repository.deny_non_fast_forward); - let storage_root = std::env::var("GIT_ROOT") - .ok() - .filter(|root| !root.trim().is_empty()) - .unwrap_or_else(|| "target/gitforge-repos".to_string()); - let storage = FileStorageBackend::new(&storage_root); + let storage = FileStorageBackend::new(storage_root.as_str()); let tightening = deny_non_fast_forward && !repository.deny_non_fast_forward; if tightening { diff --git a/crates/gitforge-api/src/server.rs b/crates/gitforge-api/src/server.rs index c713d14d..7c4da703 100644 --- a/crates/gitforge-api/src/server.rs +++ b/crates/gitforge-api/src/server.rs @@ -3,7 +3,7 @@ use crate::auth::ApiAuth; use crate::routes::{ artifact_routes, ci_routes, public_runner_routes, repo_routes, review_routes, runner_routes, - ssh_key_routes, user_routes, webhook_routes, + ssh_key_routes, user_routes, webhook_routes, RepoStorageRoot, }; use axum::{ extract::Extension, @@ -24,6 +24,22 @@ use tower_http::cors::{Any, CorsLayer}; use crate::rate_limit::{RateLimitConfig, RateLimitLayer, RateLimiter}; +/// Resolve the bare-repository storage root from an explicit path or the +/// environment. +/// +/// The environment is read exactly once, at the startup edge. The result +/// is a value that is threaded into the request path, so a request never +/// observes a root another thread changed mid-flight. +fn resolve_repo_storage_root(storage_path: Option) -> RepoStorageRoot { + if let Some(path) = storage_path { + return RepoStorageRoot::new(path.to_string_lossy().into_owned()); + } + let from_env = std::env::var("GIT_ROOT") + .ok() + .filter(|root| !root.trim().is_empty()); + RepoStorageRoot::new(from_env.unwrap_or_else(|| "target/gitforge-repos".to_string())) +} + /// API server pub struct ApiServer { pub router: Router, @@ -52,7 +68,12 @@ impl ApiServer { Self::build(jwt_secret, pool, None, registration_token) } - /// Create a new API server with custom storage path + /// Create a new API server with an explicit bare-repository storage root. + /// + /// The root is where bare repositories are provisioned and where the + /// git server reads them from, so it must be the same path the git + /// server is configured with. When it is `None` the root is taken + /// from `GIT_ROOT` in the environment, read once here. pub fn with_storage( jwt_secret: &str, pool: Pool, @@ -69,9 +90,16 @@ impl ApiServer { fn build( jwt_secret: &str, pool: Pool, - _storage_path: Option, + storage_path: Option, registration_token: Option>, ) -> Self { + // The repository storage root is resolved once, here at the + // startup edge, and handed to the routes as data. Reading + // `GIT_ROOT` inside a request handler made every request depend + // on process-global state that another thread could change + // underneath it, and left the API with no way to be pointed at a + // scratch root. + let repo_storage_root = resolve_repo_storage_root(storage_path); let auth = ApiAuth::new(jwt_secret); let metrics = Metrics::new(); gitforge_db::set_metrics(metrics.clone()); @@ -158,7 +186,7 @@ impl ApiServer { // Protected routes (auth required) let pool_arc = Arc::new(pool); let protected_routes = Router::new() - .merge(repo_routes()) + .merge(repo_routes(repo_storage_root)) .merge(ci_routes()) .merge(runner_routes()) .merge(user_routes()) diff --git a/crates/gitforge-api/tests/integration.rs b/crates/gitforge-api/tests/integration.rs index d3f53c07..ff283613 100644 --- a/crates/gitforge-api/tests/integration.rs +++ b/crates/gitforge-api/tests/integration.rs @@ -26,12 +26,37 @@ use gitforge_storage::{Artifact, ArtifactStore}; use std::sync::Arc; use tower::ServiceExt; +/// Build an API server whose bare-repository storage is a scratch +/// directory the test owns. +/// +/// The server resolves its storage root once at startup, so the test +/// injects the root rather than relying on a `GIT_ROOT` environment +/// variable or a CWD-relative fallback. The returned guard keeps the +/// directory alive for as long as the server is used. +fn api_with_scratch_storage( + pool: gitforge_db::Pool, +) -> (gitforge_api::ApiServer, tempfile::TempDir) { + let dir = tempfile::tempdir().unwrap(); + let server = + gitforge_api::ApiServer::with_storage("test-secret", pool, Some(dir.path().to_path_buf())); + (server, dir) +} + /// Serializes the tests that mutate GITFORGE_PUBLIC_TELEMETRY. The /// decision is read at router-build time from the process env, which is /// shared by every test thread in this binary — without the lock the /// default-posture tests race the opt-in test and randomly invert. static TELEMETRY_ENV_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); +/// Serializes the tests that mutate GIT_ROOT. +/// +/// The repository storage root is read from the environment at +/// server-build time, and the environment is shared by every test thread +/// in this binary. Only the tests that create a repository touch it, and +/// they inject their own scratch root, so a concurrent `GIT_ROOT` write +/// cannot redirect a repository another test provisions. +static REPO_STORAGE_ENV_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); + async fn webhook_fixture( config: serde_json::Value, ) -> ( @@ -349,7 +374,7 @@ async fn test_create_repo_with_valid_auth() { .await .unwrap(); - let server = ApiServer::new("test-secret", pool); + let (server, _storage) = api_with_scratch_storage(pool); let app = server.into_router(); // Generate valid token @@ -399,7 +424,7 @@ async fn test_repository_access_is_owner_scoped() { .await .unwrap(); - let server = ApiServer::new("test-secret", pool); + let (server, _storage) = api_with_scratch_storage(pool); let app = server.into_router(); let auth = ApiAuth::new("test-secret"); let owner_token = auth.generate_token(owner.id, "owner", "developer").unwrap(); @@ -460,6 +485,85 @@ async fn test_repository_access_is_owner_scoped() { ); } +/// The repository storage root is resolved once, at the startup edge, and +/// carried into the request path as data. +/// +/// A handler that re-read the environment on every request would let a +/// sibling thread move where repositories are provisioned underneath a +/// live request. This pins that contract: the root injected when the +/// server is built wins over the environment, and the environment is not +/// consulted again once a request is in flight. +#[tokio::test] +async fn test_repo_storage_root_is_resolved_at_startup_not_per_request() { + let from_env = tempfile::tempdir().unwrap(); + let injected = tempfile::tempdir().unwrap(); + + // The environment names one root... + { + let _guard = REPO_STORAGE_ENV_LOCK.lock().unwrap(); + // Safety: the lock serialises every GIT_ROOT mutation in this + // binary, and the previous value is restored below. + unsafe { std::env::set_var("GIT_ROOT", from_env.path()) }; + } + + let pool = Pool::memory().await.unwrap(); + pool.migrate().await.unwrap(); + + let user = gitforge_db::models::User::new( + "storage-root".to_string(), + "storage-root@example.com".to_string(), + "hash".to_string(), + ); + gitforge_db::queries::UserQueries::create(&pool, &user) + .await + .unwrap(); + + // ...and the server is built with a different one injected. + let server = ApiServer::with_storage("test-secret", pool, Some(injected.path().to_path_buf())); + let app = server.into_router(); + let auth = ApiAuth::new("test-secret"); + let token = auth + .generate_token(user.id, "storage-root", "developer") + .unwrap(); + + { + let _guard = REPO_STORAGE_ENV_LOCK.lock().unwrap(); + // Safety: as above. + unsafe { std::env::remove_var("GIT_ROOT") }; + } + + let response = app + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri("/api/repos") + .header("Authorization", format!("Bearer {token}")) + .header("Content-Type", "application/json") + .body(Body::from(r#"{"name":"root-pinning"}"#)) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::CREATED); + + // The bare repository landed under the injected root, and the + // environment root was never written to. + let provisioned = std::fs::read_dir(injected.path()) + .unwrap() + .filter_map(Result::ok) + .filter(|entry| entry.path().join("HEAD").exists()) + .count(); + assert_eq!( + provisioned, 1, + "expected exactly one bare repository under the injected root" + ); + assert!( + std::fs::read_dir(from_env.path()).unwrap().next().is_none(), + "the environment root must not have been written to" + ); +} + #[tokio::test] async fn test_admin_role_management_is_authorized_and_persisted() { let pool = Pool::memory().await.unwrap(); @@ -1107,6 +1211,7 @@ async fn test_api_artifacts_are_scoped_to_job_repository_owner() { created_at: chrono::Utc::now(), error: None, ref_name: None, + push_base_sha: None, }, ) .await @@ -2293,6 +2398,7 @@ async fn test_api_job_control_enforces_repository_ownership_and_persists_cancel( created_at: chrono::Utc::now(), error: None, ref_name: None, + push_base_sha: None, }, ) .await @@ -2441,6 +2547,7 @@ async fn test_api_pipeline_runs_with_jobs() { created_at: chrono::Utc::now(), error: None, ref_name: None, + push_base_sha: None, }; gitforge_db::queries::PipelineRunQueries::create(&pool, &run) .await diff --git a/crates/gitforge-api/tests/ref_policy_routes.rs b/crates/gitforge-api/tests/ref_policy_routes.rs index 7a5a5ae4..d0ec1ece 100644 --- a/crates/gitforge-api/tests/ref_policy_routes.rs +++ b/crates/gitforge-api/tests/ref_policy_routes.rs @@ -31,9 +31,10 @@ use serde_json::{json, Value}; use std::sync::OnceLock; use tower::ServiceExt; -/// One shared GIT_ROOT for the whole test binary: `update_ref_policy` reads -/// the environment variable on every call, and the tests in one binary share -/// a process. Repositories provisioned through the storage backend land in +/// One shared storage root for the whole test binary. The tests in one +/// binary share a process, so the root is created once and injected into +/// every server rather than read from the process environment on each +/// request. Repositories provisioned through the storage backend land in /// here as real bare repositories so the git config write can succeed. fn git_root() -> &'static str { static GIT_ROOT: OnceLock = OnceLock::new(); @@ -41,8 +42,6 @@ fn git_root() -> &'static str { let dir = std::env::temp_dir().join(format!("gitforge-ref-policy-tests-{}", std::process::id())); std::fs::create_dir_all(&dir).unwrap(); - // Safety: single-threaded process-wide initialization via OnceLock. - std::env::set_var("GIT_ROOT", &dir); dir.to_string_lossy().into_owned() }) } @@ -93,7 +92,12 @@ async fn seed(owner_username: &str, repo_name: &str) -> Fixture { let auth = ApiAuth::new("test-secret"); let token_for = |user: &User, role: &str| auth.generate_token(user.id, &user.username, role).unwrap(); - let app = ApiServer::new("test-secret", pool.clone()).into_router(); + let app = ApiServer::with_storage( + "test-secret", + pool.clone(), + Some(std::path::PathBuf::from(git_root())), + ) + .into_router(); Fixture { app, pool, diff --git a/crates/gitforge-ci/src/dag.rs b/crates/gitforge-ci/src/dag.rs index 0fb29164..662c1d6f 100644 --- a/crates/gitforge-ci/src/dag.rs +++ b/crates/gitforge-ci/src/dag.rs @@ -354,6 +354,37 @@ mod tests { ); } + #[test] + fn test_push_base_reaches_every_job() { + // The push base is published through the pipeline `environment` + // map, so the DAG builder is what carries it into every job — a + // job that never receives it cannot diff against the push. + let pipeline = PipelineDefinition { + name: "test".to_string(), + version: "1.0".to_string(), + trigger_on: vec![], + environment: HashMap::new(), + jobs: vec![ + make_job("detect-changes", vec![]), + make_job("clippy", vec![]), + ], + }; + + let augmented = crate::pipeline::with_push_base(&pipeline, Some("abc123")); + let run_id = gitforge_common::PipelineRunId::new(); + let graph = DagBuilder::build(&augmented, run_id).unwrap(); + + assert_eq!(graph.nodes.len(), 2); + for node in &graph.nodes { + assert_eq!( + node.definition.env.get(crate::pipeline::PUSH_BASE_ENV), + Some(&"abc123".to_string()), + "job {} must carry the push base", + node.name + ); + } + } + #[test] fn test_detect_cycle() { let jobs = vec![ diff --git a/crates/gitforge-ci/src/engine.rs b/crates/gitforge-ci/src/engine.rs index 060cc62d..8dc8ec29 100644 --- a/crates/gitforge-ci/src/engine.rs +++ b/crates/gitforge-ci/src/engine.rs @@ -1694,4 +1694,51 @@ mod tests { assert_eq!(state.status, PipelineStatus::Succeeded); assert_eq!(engine.skipped_jobs().await, vec![clippy]); } + + /// GF-12: a dispatched job must not be released again, and the + /// completion that replays its assignment must land. + /// + /// `ready_jobs()` releases every job whose mirror still says + /// `Queued`. The scheduler announces a dispatch with `JobAssigned`, + /// and until that event was consumed the mirror kept saying + /// `Queued`: every later completion of the run then re-enqueued the + /// running job, and that duplicate enqueue demoted the job's durable + /// row, refused the runner's heartbeat and rotated its lease. + #[tokio::test] + async fn a_dispatched_job_is_not_released_again() { + let event = PipelineTriggerEvent::new( + PipelineId::new(), + RepoId::new(), + "abc123".to_string(), + TriggerType::Push, + ); + let engine = CiEngine::new(event, make_parallel_pipeline()) + .await + .unwrap(); + engine.start().await.unwrap(); + + let ready = engine.ready_jobs().await; + assert_eq!(ready.len(), 2); + let dispatched = ready[0]; + let runner_id = gitforge_common::RunnerId::new(); + engine.assign_job(dispatched, runner_id).await.unwrap(); + + assert!( + !engine.ready_jobs().await.contains(&dispatched), + "a dispatched job must not be released again" + ); + + // The completion consumer replays the assignment when the runner + // reports back: the replay is idempotent, and a job that fails + // while it is still assigned is a failure, not an invalid + // transition. + engine.assign_job(dispatched, runner_id).await.unwrap(); + engine.start_job(dispatched).await.unwrap(); + engine + .fail_job(dispatched, -1, "runner reported failure".to_string()) + .await + .unwrap(); + let state = engine.state().await; + assert_eq!(state.jobs[&dispatched].status(), JobStatus::Failed); + } } diff --git a/crates/gitforge-ci/src/lib.rs b/crates/gitforge-ci/src/lib.rs index 82afdb31..c4112f76 100644 --- a/crates/gitforge-ci/src/lib.rs +++ b/crates/gitforge-ci/src/lib.rs @@ -12,7 +12,7 @@ pub use dag::{DagBuilder, JobGraph, JobNode}; pub use engine::CiEngine; pub use executor::PipelineExecutor; pub use pipeline::{ - parse_timeout_secs, JobDefinition, PipelineDefinition, PipelineTriggerEvent, StepDefinition, - TriggerType, + parse_timeout_secs, with_push_base, JobDefinition, PipelineDefinition, PipelineTriggerEvent, + StepDefinition, TriggerType, PUSH_BASE_ENV, }; pub use state::JobStateMachine; diff --git a/crates/gitforge-ci/src/pipeline.rs b/crates/gitforge-ci/src/pipeline.rs index f7f9ea65..b32b2c05 100644 --- a/crates/gitforge-ci/src/pipeline.rs +++ b/crates/gitforge-ci/src/pipeline.rs @@ -8,6 +8,15 @@ pub const DEFAULT_TIMEOUT_SECS: u64 = 300; pub const MIN_TIMEOUT_SECS: u64 = 5; pub const MAX_TIMEOUT_SECS: u64 = 24 * 60 * 60; +/// Environment variable carrying the commit the pushed ref pointed at +/// *before* the push. +/// +/// This is the only base that describes the whole push: a push may carry +/// several commits, and the tip commit alone is not the push. A consumer +/// that diffs against the tip misses every commit but the last one, which +/// can skip clippy/test/coverage on a code change — a false green. +pub const PUSH_BASE_ENV: &str = "CI_PUSH_BASE_SHA"; + /// Parse a human-readable job timeout into bounded seconds. /// /// Supported forms are plain seconds (`900`) and a single suffix of `s`, @@ -145,6 +154,31 @@ impl PipelineDefinition { } } +/// Return a copy of `pipeline` whose shared environment carries the push +/// base commit. +/// +/// The clone is deliberate: the committed definition stays pristine for the +/// persisted pipeline config and the pipeline cache, while the augmented +/// copy is what the engine turns into job definitions. `None` (or an empty +/// string) means the push had no pre-push base — a branch creation — and +/// the clone is returned unchanged so consumers see no variable at all and +/// fail safe. +pub fn with_push_base( + pipeline: &PipelineDefinition, + push_base: Option<&str>, +) -> PipelineDefinition { + let mut augmented = pipeline.clone(); + if let Some(sha) = push_base { + let sha = sha.trim(); + if !sha.is_empty() { + augmented + .environment + .insert(PUSH_BASE_ENV.to_string(), sha.to_string()); + } + } + augmented +} + /// Job definition within a pipeline #[derive(Debug, Clone, Serialize, Deserialize)] pub struct JobDefinition { @@ -673,6 +707,68 @@ jobs: assert!(off.jobs[0].fault.is_none()); } + #[test] + fn test_with_push_base_publishes_the_base_in_the_clone_only() { + let original = PipelineDefinition { + name: "test".to_string(), + version: "1.0".to_string(), + trigger_on: vec![TriggerType::Push], + environment: HashMap::new(), + jobs: vec![], + }; + + let augmented = with_push_base(&original, Some("abc123")); + + assert_eq!( + augmented.environment.get(PUSH_BASE_ENV), + Some(&"abc123".to_string()) + ); + // The committed definition must stay pristine: it is what the + // pipeline cache and the persisted pipeline config hold. + assert!(!original.environment.contains_key(PUSH_BASE_ENV)); + } + + #[test] + fn test_with_push_base_without_a_base_publishes_nothing() { + let original = PipelineDefinition { + name: "test".to_string(), + version: "1.0".to_string(), + trigger_on: vec![TriggerType::Push], + environment: HashMap::new(), + jobs: vec![], + }; + + for push_base in [None, Some(""), Some(" ")] { + let augmented = with_push_base(&original, push_base); + assert!( + !augmented.environment.contains_key(PUSH_BASE_ENV), + "no push base must publish no variable (push_base={push_base:?})" + ); + } + } + + #[test] + fn test_with_push_base_keeps_the_existing_environment() { + let original = PipelineDefinition { + name: "test".to_string(), + version: "1.0".to_string(), + trigger_on: vec![TriggerType::Push], + environment: HashMap::from([("EXISTING".to_string(), "value".to_string())]), + jobs: vec![], + }; + + let augmented = with_push_base(&original, Some("abc123")); + + assert_eq!( + augmented.environment.get("EXISTING"), + Some(&"value".to_string()) + ); + assert_eq!( + augmented.environment.get(PUSH_BASE_ENV), + Some(&"abc123".to_string()) + ); + } + #[test] fn test_pipeline_trigger_event_with_actor() { let user_id = gitforge_common::UserId::new(); @@ -686,4 +782,45 @@ jobs: assert_eq!(event.actor_id, Some(user_id)); } + + /// The repo's own pipeline is a product artifact: the CI service parses + /// it at push time and a syntax or key error there turns every push + /// into a red pipeline with no useful diagnostic. Parsing it here + /// turns that into a local test failure. + #[test] + fn test_committed_pipeline_parses_and_keeps_its_docker_socket_mount() { + let yaml = include_str!("../../../.gitforge.yml"); + let pipeline = PipelineDefinition::parse(yaml).expect("committed pipeline must parse"); + + let job = pipeline + .jobs + .iter() + .find(|job| job.name == "docker-integration") + .expect("docker-integration job"); + let socket = job + .volumes + .iter() + .find(|volume| volume.destination == "/var/run/docker.sock") + .expect("docker socket mount"); + // The placeholder, not a fixed path: the socket of the daemon the + // runner actually uses is deployment-specific (a rootless podman + // runner's socket is not /var/run/docker.sock). + assert_eq!(socket.source.as_deref(), Some("{docker_socket}")); + assert!(socket.read_only); + assert_eq!( + job.env.get("DOCKER_HOST").map(String::as_str), + Some("unix:///var/run/docker.sock") + ); + + // The job runs the daemon-dependent tests that `cargo test` skips; + // without the `--ignored` filter the job would pass while running + // nothing. + let step = job + .steps + .iter() + .find(|step| step.run.contains("--ignored")) + .expect("a step running the ignored tests"); + assert!(step.run.contains("gitforge-sandbox")); + assert!(step.run.contains("gitforge-runner")); + } } diff --git a/crates/gitforge-ci/src/state.rs b/crates/gitforge-ci/src/state.rs index bc5d766c..f30c335a 100644 --- a/crates/gitforge-ci/src/state.rs +++ b/crates/gitforge-ci/src/state.rs @@ -83,6 +83,18 @@ impl JobStateMachine { /// Transition to assigned state pub fn assign(&mut self, runner_id: RunnerId) -> Result<()> { + // Re-asserting an assignment is idempotent: the scheduler emits + // `JobAssigned` when it dispatches a job, and the completion + // consumer replays the same transition when the runner reports + // back. The replay must not read as an invalid transition. + if self.status == JobStatus::Assigned { + if self.runner_id.is_some_and(|current| current != runner_id) { + return Err(gitforge_common::Error::invalid_input( + "job is already assigned to another runner", + )); + } + return Ok(()); + } self.ensure_valid_transition(JobStatus::Assigned)?; self.status = JobStatus::Assigned; self.runner_id = Some(runner_id); @@ -187,6 +199,11 @@ impl JobStateMachine { // From assigned (JobStatus::Assigned, JobStatus::Running) => true, + // A job that fails while it is assigned to a runner never + // reaches `Running` in this mirror: the runner reports the + // failure, and the mirror is still at `Assigned` (the + // assignment is asserted when the job is dispatched). + (JobStatus::Assigned, JobStatus::Failed) => true, (JobStatus::Assigned, JobStatus::Cancelled) => true, // Same watchdog rationale as the Queued case above. (JobStatus::Assigned, JobStatus::TimedOut) => true, diff --git a/crates/gitforge-common/Cargo.toml b/crates/gitforge-common/Cargo.toml index 1cb04796..bf1e0257 100644 --- a/crates/gitforge-common/Cargo.toml +++ b/crates/gitforge-common/Cargo.toml @@ -16,6 +16,7 @@ async-trait = { workspace = true } bcrypt = { workspace = true } [dev-dependencies] +serde_json = { workspace = true } serde_yaml = { workspace = true } [lints] diff --git a/crates/gitforge-common/build.rs b/crates/gitforge-common/build.rs new file mode 100644 index 00000000..e1f09b97 --- /dev/null +++ b/crates/gitforge-common/build.rs @@ -0,0 +1,200 @@ +//! Build-time provenance stamp. +//! +//! The release chain has to be able to prove that the binaries it ships were +//! built from the commit the bundle claims. The release binaries live in +//! gitignored paths, so a bundle assembled from stale leftovers looks +//! exactly like a fresh one to every other gate: the only honest link +//! between "this bundle claims commit X" and "these bytes were built from +//! commit X" has to be carried by the binaries themselves. +//! +//! The stamp is resolved in this order: +//! +//! 1. `GITFORGE_BUILD_COMMIT` — an explicit override, for builds that happen +//! outside a git checkout (a source copy on a build host, for example). +//! 2. `git rev-parse HEAD` — the checkout this crate is built from. +//! 3. `unknown` — no provenance is available; the release tooling refuses +//! to ship a binary that cannot name its commit. +//! +//! The state of the tree is stamped next to the commit: a binary built from +//! a modified working tree does not correspond to the commit it claims, so +//! `dirty` records whether the tree was clean at build time. +//! +//! The HOW is stamped last: the toolchain that compiled the binary and the +//! moment it was compiled. A bundle's metadata used to describe the cutting +//! host instead of the artifact it ships (GF-15, 2026-11-06): it recorded the +//! `rustc --version` of whichever machine ran the cut, and the time of the +//! cut, so the metadata of a Fedora-built binary claimed the toolchain of the +//! instance that assembled it. The only honest description of a build is the +//! one the build itself records, so the toolchain and the build time are +//! stamped into the binary and read back out of it. + +use std::path::PathBuf; +use std::process::Command; + +/// The value stamped when no commit could be determined. +const UNKNOWN: &str = "unknown"; + +fn main() { + println!("cargo:rerun-if-env-changed=GITFORGE_BUILD_COMMIT"); + println!("cargo:rerun-if-env-changed=GITFORGE_BUILD_DIRTY"); + println!("cargo:rerun-if-env-changed=GITFORGE_BUILD_TOOLCHAIN"); + println!("cargo:rerun-if-env-changed=GITFORGE_BUILD_EPOCH"); + watch_checkout(); + let commit = resolve_commit(); + let dirty = resolve_dirty(); + let toolchain = resolve_toolchain(); + let epoch = resolve_epoch(); + println!("cargo:rustc-env=GITFORGE_BUILD_COMMIT={commit}"); + println!("cargo:rustc-env=GITFORGE_BUILD_DIRTY={dirty}"); + println!("cargo:rustc-env=GITFORGE_BUILD_TOOLCHAIN={toolchain}"); + println!("cargo:rustc-env=GITFORGE_BUILD_EPOCH={epoch}"); +} + +/// Run `git` and return its trimmed stdout, or `None` when git is missing, +/// the command fails, or its output is not valid UTF-8. +fn git(args: &[&str]) -> Option { + let output = Command::new("git").args(args).output().ok()?; + if !output.status.success() { + return None; + } + let stdout = String::from_utf8(output.stdout).ok()?; + Some(stdout.trim().to_string()) +} + +/// The commit the binaries are being built from. +fn resolve_commit() -> String { + if let Ok(value) = std::env::var("GITFORGE_BUILD_COMMIT") { + let trimmed = value.trim(); + if !trimmed.is_empty() { + return trimmed.to_string(); + } + } + git(&["rev-parse", "HEAD"]) + .filter(|commit| !commit.is_empty()) + .unwrap_or_else(|| UNKNOWN.to_string()) +} + +/// Whether the working tree carried uncommitted changes at build time: +/// `false`, `true`, or `unknown` when git could not answer. +/// +/// An explicit `GITFORGE_BUILD_DIRTY` overrides the answer. A build from a +/// source copy has no checkout to ask, and the operator asserting the state +/// of the copy is better than the release tooling refusing every such build +/// without a way forward. +fn resolve_dirty() -> String { + if let Ok(value) = std::env::var("GITFORGE_BUILD_DIRTY") { + let trimmed = value.trim(); + if !trimmed.is_empty() { + return trimmed.to_string(); + } + } + match git(&["status", "--porcelain", "--untracked-files=no"]) { + Some(output) if output.is_empty() => "false".to_string(), + Some(_) => "true".to_string(), + None => UNKNOWN.to_string(), + } +} + +/// The toolchain that is compiling this binary: the `rustc --version` line +/// of the compiler in use, or `unknown` when there is none to ask. +/// +/// An explicit `GITFORGE_BUILD_TOOLCHAIN` overrides the answer, the same way +/// `GITFORGE_BUILD_COMMIT` does: a build host that cannot be asked (a +/// container with a pinned toolchain, for example) can state the toolchain +/// it used instead. +fn resolve_toolchain() -> String { + if let Ok(value) = std::env::var("GITFORGE_BUILD_TOOLCHAIN") { + let trimmed = value.trim(); + if !trimmed.is_empty() { + return trimmed.to_string(); + } + } + let rustc = std::env::var("RUSTC").unwrap_or_else(|_| "rustc".to_string()); + Command::new(rustc) + .arg("--version") + .output() + .ok() + .filter(|output| output.status.success()) + .and_then(|output| String::from_utf8(output.stdout).ok()) + .map(|stdout| stdout.trim().to_string()) + .filter(|stdout| !stdout.is_empty()) + .unwrap_or_else(|| UNKNOWN.to_string()) +} + +/// The moment this binary was compiled, as seconds since the Unix epoch. +/// +/// The epoch is stamped rather than a formatted timestamp so that the +/// formatting (and the calendar arithmetic behind it) stays in the library, +/// where it is unit-tested, instead of in this build script. +/// +/// An explicit `GITFORGE_BUILD_EPOCH` overrides the answer, so a build whose +/// clock cannot be trusted can state when it happened. +fn resolve_epoch() -> String { + if let Ok(value) = std::env::var("GITFORGE_BUILD_EPOCH") { + let trimmed = value.trim(); + if !trimmed.is_empty() { + return trimmed.to_string(); + } + } + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or_else( + |_| UNKNOWN.to_string(), + |elapsed| elapsed.as_secs().to_string(), + ) +} + +/// Tell cargo to re-run this build script when the checkout moves. +/// +/// Without this, a build script that resolves the commit from git keeps its +/// first answer: switching branches or advancing the current branch does not +/// touch any file in the package, so cargo would happily relink binaries +/// that still claim the previous commit. That is the exact failure this +/// stamp exists to prevent. +fn watch_checkout() { + let Some(git_dir) = git(&["rev-parse", "--absolute-git-dir"]) else { + return; + }; + let git_dir = PathBuf::from(git_dir); + let head_path = git_dir.join("HEAD"); + if !head_path.exists() { + return; + } + println!("cargo:rerun-if-changed={}", head_path.display()); + + // A branch that advances does not change HEAD's own bytes, so the ref + // HEAD points at has to be watched as well. + let Some(common_dir) = git(&["rev-parse", "--git-common-dir"]) else { + return; + }; + let common_dir = absolute_path(&common_dir); + let Ok(head_contents) = std::fs::read_to_string(&head_path) else { + return; + }; + let Some(reference) = head_contents.trim().strip_prefix("ref: ") else { + // Detached HEAD: the commit lives in HEAD itself, which is watched. + return; + }; + let ref_path = common_dir.join(reference.trim()); + if ref_path.exists() { + println!("cargo:rerun-if-changed={}", ref_path.display()); + return; + } + // The ref is packed rather than loose; packed-refs carries it. + let packed_refs = common_dir.join("packed-refs"); + if packed_refs.exists() { + println!("cargo:rerun-if-changed={}", packed_refs.display()); + } +} + +/// Resolve a possibly relative path against the current directory. +fn absolute_path(path: &str) -> PathBuf { + let candidate = PathBuf::from(path); + if candidate.is_absolute() { + return candidate; + } + match std::env::current_dir() { + Ok(current) => current.join(candidate), + Err(_) => candidate, + } +} diff --git a/crates/gitforge-common/src/build.rs b/crates/gitforge-common/src/build.rs new file mode 100644 index 00000000..4c429eac --- /dev/null +++ b/crates/gitforge-common/src/build.rs @@ -0,0 +1,161 @@ +//! Build provenance carried by the released binaries. +//! +//! Every release binary names the commit it was built from and whether the +//! tree it was built from was clean. The release chain asks a binary for +//! that stamp and refuses to ship one that does not match the commit the +//! bundle claims — the tool, not an operator procedure, is what makes the +//! provenance claim true. +//! +//! The stamp is produced by this crate's build script (see `build.rs`) and +//! reaches every released binary because all four services depend on +//! `gitforge-common`. + +/// The commit this binary was built from, or `unknown` when the build had +/// neither a git checkout nor an explicit `GITFORGE_BUILD_COMMIT`. +pub const BUILD_COMMIT: &str = env!("GITFORGE_BUILD_COMMIT"); + +/// The state of the tree this binary was built from: `false` (clean), +/// `true` (modified), or `unknown` (git could not answer). +pub const BUILD_DIRTY: &str = env!("GITFORGE_BUILD_DIRTY"); + +/// The toolchain that compiled this binary, as `rustc --version` reports it, +/// or `unknown` when there was no compiler to ask. +pub const BUILD_TOOLCHAIN: &str = env!("GITFORGE_BUILD_TOOLCHAIN"); + +/// The moment this binary was compiled, in seconds since the Unix epoch, or +/// `unknown` when the clock could not be read. +pub const BUILD_EPOCH: &str = env!("GITFORGE_BUILD_EPOCH"); + +/// The value used when a build fact could not be established. +const UNKNOWN: &str = "unknown"; + +/// When this binary was compiled, as an RFC 3339 UTC timestamp. +/// +/// The epoch is stamped at build time and formatted here, so the calendar +/// arithmetic lives in a place that is unit-tested rather than in a build +/// script that no test ever runs. +pub fn build_timestamp_utc() -> String { + format_epoch(BUILD_EPOCH) +} + +/// Format a stamped epoch as an RFC 3339 UTC timestamp. +/// +/// A value that is not a moment in time degrades to `unknown` rather than +/// panicking: a binary whose clock could not be read still has to be able +/// to answer `--version`, and the release tooling refuses an `unknown` +/// build time on its own. +fn format_epoch(epoch: &str) -> String { + let Ok(seconds) = epoch.trim().parse::() else { + return UNKNOWN.to_string(); + }; + match chrono::DateTime::from_timestamp(seconds, 0) { + Some(timestamp) => timestamp.format("%Y-%m-%dT%H:%M:%SZ").to_string(), + None => UNKNOWN.to_string(), + } +} + +/// The single line a release binary prints for `--version`. +/// +/// The format is part of the release contract: `gitforge-release-stamps` +/// parses `commit=`, `dirty=`, `toolchain=` and `built=` out of it to decide +/// whether a bundle may be cut, and to describe the bundle from the binaries +/// it ships, so the field names are not free-form. +/// +/// The toolchain and the build time are the LAST two fields on purpose: the +/// toolchain line carries spaces (`rustc 1.99.1 (58cf8f9c3 2026-10-13)`), +/// so a parser has to anchor on the field that follows it. +pub fn version_line(component: &str) -> String { + let version = env!("CARGO_PKG_VERSION"); + format!( + "{component} {version} commit={BUILD_COMMIT} dirty={BUILD_DIRTY} toolchain={BUILD_TOOLCHAIN} built={}", + build_timestamp_utc() + ) +} + +/// The version line when the caller asked for the version, `None` otherwise. +/// +/// Services call this before any other work so that asking a binary for its +/// provenance never starts a scheduler, opens a database, or binds a port. +pub fn version_line_if_requested( + component: &str, + args: impl IntoIterator, +) -> Option { + let requested = args + .into_iter() + .any(|arg| arg == "--version" || arg == "-V"); + if requested { + Some(version_line(component)) + } else { + None + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn version_line_carries_the_release_fields() { + let line = version_line("api"); + assert!(line.starts_with("api "), "unexpected version line: {line}"); + assert!( + line.contains(&format!("commit={BUILD_COMMIT}")), + "version line must name its commit: {line}" + ); + assert!( + line.contains(&format!("dirty={BUILD_DIRTY}")), + "version line must name its tree state: {line}" + ); + assert!( + line.contains(&format!("toolchain={BUILD_TOOLCHAIN}")), + "version line must name its toolchain: {line}" + ); + assert!( + line.contains(&format!("built={}", build_timestamp_utc())), + "version line must name when it was built: {line}" + ); + } + + #[test] + fn the_toolchain_and_build_time_are_present_and_last() { + let line = version_line("api"); + let toolchain = format!("toolchain={BUILD_TOOLCHAIN}"); + let built = format!("built={}", build_timestamp_utc()); + let toolchain_at = line.find(&toolchain).expect("toolchain field"); + let built_at = line.find(&built).expect("built field"); + assert!( + toolchain_at < built_at, + "the toolchain field must be anchored by the built field: {line}" + ); + } + + #[test] + fn an_epoch_is_formatted_as_utc() { + assert_eq!(format_epoch("0"), "1970-01-01T00:00:00Z"); + assert_eq!(format_epoch("1762992000"), "2025-11-13T00:00:00Z"); + } + + #[test] + fn an_unreadable_clock_is_reported_as_unknown() { + assert_eq!(format_epoch("not-a-number"), UNKNOWN); + assert_eq!(format_epoch(""), UNKNOWN); + // Far beyond what the calendar can represent. + assert_eq!(format_epoch("999999999999999"), UNKNOWN); + } + + #[test] + fn version_flag_is_recognised_and_other_arguments_are_not() { + assert!(version_line_if_requested("ci", vec!["--version".to_string()]).is_some()); + assert!(version_line_if_requested("ci", vec!["-V".to_string()]).is_some()); + assert!(version_line_if_requested("ci", Vec::::new()).is_none()); + assert!( + version_line_if_requested("ci", vec!["--port".to_string(), "42781".to_string()]) + .is_none() + ); + } + + #[test] + fn version_line_is_component_specific() { + assert_ne!(version_line("api"), version_line("ci")); + } +} diff --git a/crates/gitforge-common/src/job_row.rs b/crates/gitforge-common/src/job_row.rs new file mode 100644 index 00000000..6c17fdd5 --- /dev/null +++ b/crates/gitforge-common/src/job_row.rs @@ -0,0 +1,95 @@ +//! The authoritative store's view of a job id, carried as data. +//! +//! A container's `com.gitforce.job_id` label only means something if the +//! store has a row for that id. The runner's warm pool minted job-shaped +//! ids for containers that were never given to a job, and the difference +//! between such a placeholder and a real job is exactly the difference +//! between a row and no row. +//! +//! The store's answer is a value, not a lookup a decision performs: a +//! decision that reads the store itself inherits every concurrent +//! writer's timing, so the answer is resolved once at the edge and +//! handed to the decision as data. + +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; + +/// What the authoritative store says about a job id. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum JobRowState { + /// The store has a row for this job id: it is a real job. + Present, + /// The store has no row for this job id: it was never a job. + Absent, + /// The store could not be consulted. Never a licence to remove. + Unknown, +} + +impl JobRowState { + /// The wire form of this state, for logs and receipts. + pub fn as_str(&self) -> &'static str { + match self { + JobRowState::Present => "present", + JobRowState::Absent => "absent", + JobRowState::Unknown => "unknown", + } + } +} + +/// Bulk job-existence request. +/// +/// The runner asks about the job ids it stamped on the containers it +/// found, so the store can say which of them were ever jobs. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct JobExistenceRequest { + /// The job ids to answer for. + pub job_ids: Vec, +} + +/// Bulk job-existence response: one answer per requested id. +/// +/// A requested id that the store cannot speak for is answered `unknown` +/// rather than omitted, so an incomplete answer is visible as such and +/// never reads as "absent". +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct JobExistenceResponse { + /// The store's view of each requested job id. + pub states: HashMap, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_wire_form_round_trips() { + for state in [ + JobRowState::Present, + JobRowState::Absent, + JobRowState::Unknown, + ] { + let json = serde_json::to_string(&state).expect("serialize state"); + let parsed: JobRowState = serde_json::from_str(&json).expect("parse state"); + assert_eq!(parsed, state); + assert_eq!(json, format!("\"{}\"", state.as_str())); + } + } + + #[test] + fn a_response_round_trips() { + let response = JobExistenceResponse { + states: HashMap::from([ + ("a".to_string(), JobRowState::Present), + ("b".to_string(), JobRowState::Absent), + ("c".to_string(), JobRowState::Unknown), + ]), + }; + let json = serde_json::to_string(&response).expect("serialize response"); + let parsed: JobExistenceResponse = serde_json::from_str(&json).expect("parse response"); + assert_eq!(parsed.states.len(), 3); + assert_eq!(parsed.states["a"], JobRowState::Present); + assert_eq!(parsed.states["b"], JobRowState::Absent); + assert_eq!(parsed.states["c"], JobRowState::Unknown); + } +} diff --git a/crates/gitforge-common/src/lib.rs b/crates/gitforge-common/src/lib.rs index 3df96f69..5afc4cea 100644 --- a/crates/gitforge-common/src/lib.rs +++ b/crates/gitforge-common/src/lib.rs @@ -6,9 +6,11 @@ //! - Result type aliases //! - Time utilities +pub mod build; pub mod error; pub mod fault; pub mod ids; +pub mod job_row; pub mod lane; pub mod password; pub mod result; @@ -22,10 +24,14 @@ pub use ids::{ JobId, JobStatus, PipelineId, PipelineRunId, PipelineStatus, RepoId, RunnerId, SshKeyId, StepId, UserId, }; +pub use job_row::{JobExistenceRequest, JobExistenceResponse, JobRowState}; pub use lane::Lane; pub use rollout::RolloutFlags; pub use time::DateTime; -pub use volume::{image_cache_key, JobCache, VolumeSpec, VolumeType}; +pub use volume::{ + cache_root, docker_socket, docker_socket_with, image_cache_key, is_socket_source, JobCache, + VolumeSpec, VolumeType, DEFAULT_CACHE_ROOT, DEFAULT_DOCKER_SOCKET, +}; /// True when `hash` is git's "ref does not exist" sentinel: empty or all /// zero digits. `git receive-pack` sends an all-zero new hash on branch diff --git a/crates/gitforge-common/src/volume.rs b/crates/gitforge-common/src/volume.rs index cd319cbe..10d63bb5 100644 --- a/crates/gitforge-common/src/volume.rs +++ b/crates/gitforge-common/src/volume.rs @@ -54,13 +54,20 @@ impl VolumeSpec { } } - /// The host source with the `{image}` and `{cache_root}` placeholders - /// expanded against the job's image and the runner's cache root. - pub fn expanded_source(&self, image: &str, cache_root: &str) -> Option { + /// The host source with the `{image}`, `{cache_root}` and + /// `{docker_socket}` placeholders expanded against the job's image, the + /// runner's cache root, and the runner's daemon socket. + pub fn expanded_source( + &self, + image: &str, + cache_root: &str, + docker_socket: &str, + ) -> Option { self.source.as_ref().map(|source| { source .replace("{image}", &image_cache_key(image)) .replace("{cache_root}", cache_root) + .replace("{docker_socket}", docker_socket) }) } @@ -155,6 +162,106 @@ impl JobCache { } } +/// Where the runner keeps job caches when neither an override nor a home +/// directory is available. Overridable with `GITFORGE_CACHE_ROOT`; the +/// fallback matches the usual system cache path. +pub const DEFAULT_CACHE_ROOT: &str = "/var/cache/gitforge"; + +/// The cache root for the given environment. +/// +/// The override wins. Failing that, the home directory's conventional +/// cache location. Only when neither is available does the system path +/// apply. +/// +/// The home directory is part of the contract, not a convenience: +/// `docs/RUNBOOK.md` documents the default as `~/.cache/gitforge`, and a +/// runner that resolved a path it cannot write lost every job cache +/// without saying so (2026-11-05, ledger row GF-09: the runner resolved +/// `/var/cache/gitforge`, which its own unit cannot write, so the +/// platform's own pipeline cold-built on every run). +pub fn cache_root_with(override_root: Option<&str>, home: Option<&str>) -> String { + let override_root = override_root.map(str::trim).unwrap_or_default(); + if !override_root.is_empty() { + return override_root.to_string(); + } + let home = home.map(str::trim).unwrap_or_default(); + if !home.is_empty() { + return std::path::Path::new(home) + .join(".cache") + .join("gitforge") + .to_string_lossy() + .into_owned(); + } + DEFAULT_CACHE_ROOT.to_string() +} + +/// The runner's cache root, taken from the environment. +/// +/// Call this once at the edge - where a job starts, and where a container +/// is created - and pass the value down. A decision function that reads +/// process-global state internally inherits whatever any other thread is +/// doing to it at that instant. +/// +/// One definition for the whole platform: the runner (which locks the +/// directory a job mounts) and the sandbox (which mounts it) must agree +/// on which host directory a cache points at, or the lock would guard a +/// path nobody writes to. +pub fn cache_root() -> String { + cache_root_with( + std::env::var("GITFORGE_CACHE_ROOT").ok().as_deref(), + std::env::var("HOME").ok().as_deref(), + ) +} + +/// The well-known system-wide Docker daemon socket. +pub const DEFAULT_DOCKER_SOCKET: &str = "/var/run/docker.sock"; + +/// The Docker daemon socket named by a `DOCKER_HOST` value. +/// +/// `DOCKER_HOST` wins when it names a unix socket: a rootless podman's +/// socket is not the system-wide path. The scheme is stripped, because the +/// value is used as a host bind source, not as a URL. +pub fn docker_socket_with(docker_host: Option<&str>) -> String { + docker_host + .and_then(|host| host.strip_prefix("unix://")) + .filter(|path| !path.is_empty()) + .unwrap_or(DEFAULT_DOCKER_SOCKET) + .to_string() +} + +/// The Docker daemon socket this runner talks to. +/// +/// One definition for the whole platform: the sandbox connects to this +/// socket, the reconciler reconciles this daemon's containers, and a +/// `{docker_socket}` mount binds this socket into the job's container. A +/// runner that bound one daemon's socket while talking to another would run +/// the job against a daemon it does not supervise. +/// +/// Call this once at the edge - where a job starts, and where a container +/// is created - and pass the value down. +pub fn docker_socket() -> String { + docker_socket_with(std::env::var("DOCKER_HOST").ok().as_deref()) +} + +/// Whether a bind source names a unix socket rather than a directory. +/// +/// A socket is not a directory: the runner must not create one in its +/// place, must not relabel it (`:z` rewrites the host socket's SELinux +/// label - a host-state change the runner does not own), and must not lock +/// it like a shared build directory. +#[cfg(unix)] +pub fn is_socket_source(source: &str) -> bool { + use std::os::unix::fs::FileTypeExt; + + std::fs::metadata(source).is_ok_and(|metadata| metadata.file_type().is_socket()) +} + +/// Whether a bind source names a unix socket rather than a directory. +#[cfg(not(unix))] +pub fn is_socket_source(_source: &str) -> bool { + false +} + /// A filesystem-safe key for an image reference. /// /// Caches are keyed by this value so a rebuilt image (new tag) starts from @@ -236,8 +343,12 @@ mod tests { size: None, }; assert_eq!( - spec.expanded_source("dsc-ci-rust:10", "/var/cache/gitforge") - .as_deref(), + spec.expanded_source( + "dsc-ci-rust:10", + "/var/cache/gitforge", + "/var/run/docker.sock" + ) + .as_deref(), Some("/var/cache/gitforge-cargo/dsc-ci-rust-10") ); } @@ -251,6 +362,103 @@ mod tests { ); assert_eq!(image_cache_key("a_b.c"), "a_b.c"); } + + #[test] + fn the_cache_override_wins_over_the_home_directory() { + assert_eq!( + cache_root_with(Some("/nas/cache"), Some("/home/runner")), + "/nas/cache" + ); + } + + #[test] + fn the_home_directory_carries_the_documented_default() { + assert_eq!( + cache_root_with(None, Some("/home/runner")), + "/home/runner/.cache/gitforge" + ); + } + + #[test] + fn a_blank_override_or_home_falls_through() { + assert_eq!( + cache_root_with(Some(" "), Some("/home/runner")), + "/home/runner/.cache/gitforge" + ); + assert_eq!(cache_root_with(None, Some("")), DEFAULT_CACHE_ROOT); + } + + #[test] + fn without_an_override_or_a_home_the_system_path_applies() { + assert_eq!(cache_root_with(None, None), DEFAULT_CACHE_ROOT); + } +} + +#[cfg(test)] +mod socket_tests { + use super::*; + + #[test] + fn the_daemon_socket_defaults_to_the_well_known_path() { + assert_eq!(docker_socket_with(None), DEFAULT_DOCKER_SOCKET); + } + + #[test] + fn a_unix_docker_host_names_the_socket() { + assert_eq!( + docker_socket_with(Some("unix:///run/user/1000/podman/podman.sock")), + "/run/user/1000/podman/podman.sock" + ); + } + + #[test] + fn a_tcp_docker_host_is_not_a_socket_path() { + assert_eq!( + docker_socket_with(Some("tcp://127.0.0.1:2375")), + DEFAULT_DOCKER_SOCKET + ); + } + + #[test] + fn a_blank_docker_host_falls_through() { + assert_eq!(docker_socket_with(Some("unix://")), DEFAULT_DOCKER_SOCKET); + assert_eq!(docker_socket_with(Some("")), DEFAULT_DOCKER_SOCKET); + } + + #[test] + fn the_socket_placeholder_expands_to_the_daemon_socket() { + let spec = VolumeSpec { + volume_type: Some(VolumeType::Bind), + source: Some("{docker_socket}".to_string()), + destination: "/var/run/docker.sock".to_string(), + read_only: true, + size: None, + }; + assert_eq!( + spec.expanded_source( + "dsc-ci-rust:16", + "/var/cache/gitforge", + "/run/user/1000/podman/podman.sock" + ) + .as_deref(), + Some("/run/user/1000/podman/podman.sock") + ); + } + + #[cfg(unix)] + #[test] + fn a_socket_source_is_not_a_directory() { + let root = + std::env::temp_dir().join(format!("gitforge-socket-test-{}", uuid::Uuid::new_v4())); + std::fs::create_dir_all(&root).expect("a scratch directory"); + let socket = root.join("docker.sock"); + let _listener = std::os::unix::net::UnixListener::bind(&socket).expect("a unix socket"); + + assert!(is_socket_source(&socket.to_string_lossy())); + assert!(!is_socket_source(&root.to_string_lossy())); + + let _ = std::fs::remove_dir_all(&root); + } } #[cfg(test)] @@ -267,8 +475,12 @@ mod cache_tests { size: None, }; assert_eq!( - spec.expanded_source("dsc-ci-rust:10", "/var/cache/gitforge") - .as_deref(), + spec.expanded_source( + "dsc-ci-rust:10", + "/var/cache/gitforge", + "/var/run/docker.sock" + ) + .as_deref(), Some("/var/cache/gitforge/cargo/dsc-ci-rust-10") ); } diff --git a/crates/gitforge-db/src/connection.rs b/crates/gitforge-db/src/connection.rs index fdf7b054..c30d875d 100644 --- a/crates/gitforge-db/src/connection.rs +++ b/crates/gitforge-db/src/connection.rs @@ -232,6 +232,30 @@ pub struct Pool { pool: SqlitePool, } +/// Execute a migration statement, tolerating a concurrent process that +/// applied it first. Returns true when this call applied the change. +/// +/// Every service migrates at boot, and a release cutover starts the +/// stateless pair together: two services can probe the schema at the same +/// moment and race on the same ADD COLUMN. The loser must not fail its +/// boot over a change the winner already made, so the duplicate-column +/// error is tolerated; every other error still fails the migration. +async fn execute_tolerating_duplicate(pool: &SqlitePool, statement: &'static str) -> Result { + match sqlx::query(statement).execute(pool).await { + Ok(_) => Ok(true), + Err(error) => { + let message = error.to_string(); + if message.contains("duplicate column name") { + Ok(false) + } else { + Err(Error::database(format!( + "failed to migrate schema with '{statement}': {error}" + ))) + } + } + } +} + impl Pool { /// Create a new connection pool from a database URL or file path pub async fn new(database_url: &str) -> Result { @@ -318,18 +342,11 @@ impl Pool { // Add the role column for databases created before role persistence // existed. Existing accounts receive the least-privileged developer // role and can be promoted explicitly by an administrative workflow. - if let Err(error) = - sqlx::query("ALTER TABLE users ADD COLUMN role TEXT NOT NULL DEFAULT 'developer'") - .execute(&self.pool) - .await - { - let message = error.to_string(); - if !message.contains("duplicate column name") { - return Err(Error::database(format!( - "failed to migrate users table: {error}" - ))); - } - } + execute_tolerating_duplicate( + &self.pool, + "ALTER TABLE users ADD COLUMN role TEXT NOT NULL DEFAULT 'developer'", + ) + .await?; // Create repositories table sqlx::query( @@ -357,14 +374,7 @@ impl Pool { "ALTER TABLE repositories ADD COLUMN required_checks TEXT NOT NULL DEFAULT '[]'", "ALTER TABLE repositories ADD COLUMN deny_non_fast_forward INTEGER NOT NULL DEFAULT 0", ] { - if let Err(error) = sqlx::query(statement).execute(&self.pool).await { - let message = error.to_string(); - if !message.contains("duplicate column name") { - return Err(Error::database(format!( - "failed to migrate repositories table: {error}" - ))); - } - } + execute_tolerating_duplicate(&self.pool, statement).await?; } // Create ssh_keys table. A key's fingerprint is globally unique: @@ -433,6 +443,7 @@ impl Pool { created_at TEXT NOT NULL, error TEXT, ref_name TEXT, + push_base_sha TEXT, FOREIGN KEY (pipeline_id) REFERENCES pipelines(id), FOREIGN KEY (repo_id) REFERENCES repositories(id) ) @@ -451,14 +462,9 @@ impl Pool { for statement in [ "ALTER TABLE pipeline_runs ADD COLUMN error TEXT", "ALTER TABLE pipeline_runs ADD COLUMN ref_name TEXT", + "ALTER TABLE pipeline_runs ADD COLUMN push_base_sha TEXT", ] { - if let Err(error) = sqlx::query(statement).execute(&self.pool).await { - if !error.to_string().contains("duplicate column name") { - return Err(Error::database(format!( - "failed to migrate pipeline_runs table: {error}" - ))); - } - } + execute_tolerating_duplicate(&self.pool, statement).await?; } // Create pipeline_trigger_requests table: the durable hand-off @@ -509,16 +515,21 @@ impl Pool { )) })?; if has_pipeline_id.is_none() { - sqlx::query("ALTER TABLE pipeline_trigger_requests ADD COLUMN pipeline_id TEXT") - .execute(&self.pool) - .await - .map_err(|e| { - Error::database(format!( - "failed to add pipeline_id to pipeline_trigger_requests: {e}" - )) - })?; + // A concurrent boot of another service can win the race between + // this probe and the ADD COLUMN. The loser must not fail its + // boot over a column the winner already added, so the + // duplicate is tolerated the way every other ADD COLUMN + // migration here is. + execute_tolerating_duplicate( + &self.pool, + "ALTER TABLE pipeline_trigger_requests ADD COLUMN pipeline_id TEXT", + ) + .await?; // The old active-triggers index keyed on (repo_id, new_hash) - // only; drop it so the widened replacement can be created. + // only; drop it so the widened replacement can be created. The + // process that lost the race above reaches this line too, and + // the drop is idempotent: whichever process creates the + // widened index last leaves it in place. sqlx::query("DROP INDEX IF EXISTS idx_trigger_requests_active") .execute(&self.pool) .await @@ -726,14 +737,7 @@ impl Pool { // explains its wait instead of just reading 'queued'. "ALTER TABLE jobs ADD COLUMN deferred_reason TEXT", ] { - if let Err(error) = sqlx::query(statement).execute(&self.pool).await { - let message = error.to_string(); - if !message.contains("duplicate column name") { - return Err(Error::database(format!( - "failed to migrate jobs table: {error}" - ))); - } - } + execute_tolerating_duplicate(&self.pool, statement).await?; } // Create artifacts table @@ -800,13 +804,7 @@ impl Pool { "ALTER TABLE events ADD COLUMN delivery_until TEXT", "ALTER TABLE events ADD COLUMN delivery_attempts INTEGER NOT NULL DEFAULT 0", ] { - if let Err(error) = sqlx::query(statement).execute(&self.pool).await { - if !error.to_string().contains("duplicate column name") { - return Err(Error::database(format!( - "failed to migrate events table: {error}" - ))); - } - } + execute_tolerating_duplicate(&self.pool, statement).await?; } // Operator submissions use a durable idempotency record so retries @@ -1182,6 +1180,77 @@ mod tests { let _ = std::fs::remove_file(db_path.with_extension("db-shm")); } + /// A boot must not fail because a concurrent boot applied the same + /// schema change first. The release cutover starts the stateless pair + /// together, so two services really do race on the same ADD COLUMN. + #[tokio::test] + async fn test_add_column_tolerates_a_concurrent_winner() { + let pool = Pool::memory().await.unwrap(); + sqlx::query("CREATE TABLE race (id TEXT PRIMARY KEY)") + .execute(pool.pool()) + .await + .unwrap(); + assert!( + execute_tolerating_duplicate(pool.pool(), "ALTER TABLE race ADD COLUMN c TEXT") + .await + .unwrap(), + "the first caller adds the column" + ); + assert!( + !execute_tolerating_duplicate(pool.pool(), "ALTER TABLE race ADD COLUMN c TEXT") + .await + .unwrap(), + "a caller that lost the race must not fail its boot" + ); + } + + /// Whole-schema concurrency: three services migrating the same file at + /// once must all succeed. This is the release cutover's actual shape, + /// and the widened trigger-dedup index must survive the race. + #[tokio::test] + async fn test_concurrent_migrations_all_succeed() { + let db_path = + std::env::temp_dir().join(format!("gitforge-migrate-race-{}.db", uuid::Uuid::new_v4())); + let url = db_path.to_string_lossy().to_string(); + // Create the file first: this test is about the migration race, + // not about two processes racing to create the database file + // itself (which is a different, connect-time lock). + { + let pool = Pool::new(&url).await.unwrap(); + sqlx::query("CREATE TABLE IF NOT EXISTS marker (id TEXT PRIMARY KEY)") + .execute(pool.pool()) + .await + .unwrap(); + } + let mut tasks = Vec::new(); + for _ in 0..3 { + let url = url.clone(); + tasks.push(tokio::spawn(async move { + let pool = Pool::new(&url).await.unwrap(); + pool.migrate().await + })); + } + for task in tasks { + task.await + .unwrap() + .expect("a concurrent migration must not fail"); + } + let pool = Pool::new(&url).await.unwrap(); + let columns = sqlx::query("PRAGMA table_info(pipeline_trigger_requests)") + .fetch_all(pool.pool()) + .await + .unwrap(); + let names = columns + .iter() + .map(|row| row.try_get::("name").unwrap()) + .collect::>(); + assert!(names.iter().any(|name| name == "pipeline_id")); + drop(pool); + let _ = std::fs::remove_file(&db_path); + let _ = std::fs::remove_file(db_path.with_extension("db-wal")); + let _ = std::fs::remove_file(db_path.with_extension("db-shm")); + } + #[test] fn test_pool_clone() { // Pool is Clone, verify it can be cloned diff --git a/crates/gitforge-db/src/models/job.rs b/crates/gitforge-db/src/models/job.rs index 4564d336..20f5746c 100644 --- a/crates/gitforge-db/src/models/job.rs +++ b/crates/gitforge-db/src/models/job.rs @@ -69,6 +69,40 @@ impl JobStatus { ) } + /// Classify a recorded completion receipt into a terminal status. + /// + /// Receipts reach the durable row in more than one shape. The + /// scheduler's runner-completion receipt carries `success` (a boolean) + /// and an `outcome` class; the reconciler's own timeout receipt carries + /// a `status` string. Reading only `status` graded every + /// runner-completion receipt as `failed`, so a job that really + /// succeeded was recorded as failed (GF-01). + /// + /// Precedence is strict and per-field: `status`, then `outcome`, then + /// the `success` boolean. The first field that is present decides the + /// verdict, and a present-but-unusable value yields `None` rather than + /// falling through to a weaker field — bad evidence must not be + /// laundered into a different verdict. The caller fails closed. + pub fn from_receipt(receipt: &serde_json::Value) -> Option { + if let Some(status) = receipt.get("status") { + return status + .as_str() + .and_then(Self::from_str) + .filter(Self::is_terminal); + } + if let Some(outcome) = receipt.get("outcome") { + return outcome + .as_str() + .and_then(Self::from_str) + .filter(Self::is_terminal); + } + match receipt.get("success").and_then(serde_json::Value::as_bool) { + Some(true) => Some(JobStatus::Succeeded), + Some(false) => Some(JobStatus::Failed), + None => None, + } + } + /// Map the durable status onto the engine's copy of the enum. /// /// Restart recovery grafts durable rows onto a rebuilt CI engine, whose diff --git a/crates/gitforge-db/src/models/pipeline.rs b/crates/gitforge-db/src/models/pipeline.rs index a403ee35..8538080d 100644 --- a/crates/gitforge-db/src/models/pipeline.rs +++ b/crates/gitforge-db/src/models/pipeline.rs @@ -80,6 +80,12 @@ pub struct PipelineRun { /// Git ref the run was triggered for. `None` for rows that predate the /// column; supersede filtering skips such runs rather than guessing. pub ref_name: Option, + /// The commit the pushed ref pointed at *before* the push. `None` for + /// runs that predate the column or that were not triggered by a push. + /// + /// This is the only base that describes the whole push: a push may + /// carry several commits, and the tip commit alone is not the push. + pub push_base_sha: Option, } impl PipelineRun { @@ -102,6 +108,7 @@ impl PipelineRun { created_at: Utc::now(), error: None, ref_name: None, + push_base_sha: None, } } diff --git a/crates/gitforge-db/src/queries.rs b/crates/gitforge-db/src/queries.rs index 8213b9ba..655a98ad 100644 --- a/crates/gitforge-db/src/queries.rs +++ b/crates/gitforge-db/src/queries.rs @@ -246,6 +246,9 @@ fn hydrate_pipeline_run(row: sqlx::sqlite::SqliteRow) -> Result Result { let mut transaction = begin_immediate(pool.pool(), "run supersede").await?; let stale: Vec = sqlx::query_scalar( - "SELECT id FROM pipeline_runs \ - WHERE repo_id = ? AND pipeline_id = ? AND id != ? \ - AND ref_name = ? \ - AND status IN ('pending', 'running', 'queued') \ - AND created_at < (SELECT created_at FROM pipeline_runs WHERE id = ?)", + "SELECT pr.id FROM pipeline_runs pr \ + JOIN pipelines p ON p.id = pr.pipeline_id \ + JOIN pipelines target ON target.id = ? AND target.repo_id = pr.repo_id \ + WHERE pr.repo_id = ? \ + AND p.name = target.name AND p.trigger_type = target.trigger_type \ + AND pr.id != ? \ + AND pr.ref_name = ? \ + AND pr.status IN ('pending', 'running', 'queued') \ + AND pr.created_at < (SELECT created_at FROM pipeline_runs WHERE id = ?)", ) - .bind(repo_id.to_string()) .bind(pipeline_id.to_string()) + .bind(repo_id.to_string()) .bind(keep_run_id.to_string()) .bind(ref_name) .bind(keep_run_id.to_string()) @@ -1038,8 +1055,8 @@ impl PipelineRunQueries { pub async fn create(pool: &Pool, run: &crate::models::PipelineRun) -> Result<()> { sqlx::query( r#" - INSERT INTO pipeline_runs (id, pipeline_id, repo_id, status, triggered_by, commit_hash, started_at, finished_at, created_at, error, ref_name) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + INSERT INTO pipeline_runs (id, pipeline_id, repo_id, status, triggered_by, commit_hash, started_at, finished_at, created_at, error, ref_name, push_base_sha) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) "#, ) .bind(run.id.to_string()) @@ -1053,6 +1070,7 @@ impl PipelineRunQueries { .bind(run.created_at.to_rfc3339()) .bind(&run.error) .bind(&run.ref_name) + .bind(&run.push_base_sha) .execute(pool.pool()) .await .map_err(|e| Error::database(format!("failed to create pipeline run: {e}")))?; @@ -1847,6 +1865,33 @@ impl JobQueries { Ok(()) } + /// Move a job to `queued`, but only while it is not live. + /// + /// A job a runner holds is `assigned` or `running`, and that durable + /// status is the invariant every later write is fenced by: the + /// heartbeat, the start, and the completion are all gated on it. An + /// enqueue that demotes such a row to `queued` takes the lease away + /// from the runner that holds it - the heartbeat is refused, the row + /// becomes dispatchable again, the scheduler rotates the lease token, + /// and the completion is rejected (GF-12, measured 2026-11-05 on the + /// GitForge CI service: a running job was demoted by a duplicate + /// enqueue, and three executions of that job were then rejected with + /// `completion_persistence_failed` before the job was fenced as + /// `runner_lost_while_running`). + /// + /// Returns whether a row changed. `false` means the job is live (or + /// terminal) and must not be queued again. + pub async fn mark_queued_if_dispatchable(pool: &Pool, id: JobId) -> Result { + let result = sqlx::query( + "UPDATE jobs SET status = 'queued' WHERE id = ? AND status IN ('pending', 'queued')", + ) + .bind(id.to_string()) + .execute(pool.pool()) + .await + .map_err(|e| Error::database(format!("failed to mark job queued: {e}")))?; + Ok(result.rows_affected() == 1) + } + /// Cancel every never-dispatched (`pending`/`queued`) job of one run and /// return how many rows changed. /// @@ -2492,13 +2537,7 @@ impl JobQueries { let verdict = result_json .as_deref() .and_then(|json| serde_json::from_str::(json).ok()) - .and_then(|receipt| { - receipt - .get("status") - .and_then(|status| status.as_str()) - .and_then(JobStatus::from_str) - }) - .filter(JobStatus::is_terminal) + .and_then(|receipt| JobStatus::from_receipt(&receipt)) .unwrap_or(JobStatus::Failed); let result = sqlx::query( "UPDATE jobs SET status = ?, runner_id = NULL, lease_token = NULL WHERE id = ? AND status IN ('pending', 'queued', 'assigned', 'running') AND finished_at IS NOT NULL", @@ -4993,6 +5032,264 @@ mod tests { ); } + // The push base and the ref name must survive the durable write. The + // `jobs` table has no env column, so the run row is the only durable + // carrier of the base: without it a restarted control plane cannot + // re-publish the base to the run's jobs. + #[tokio::test] + async fn test_activate_pipeline_and_create_run_persists_ref_name_and_push_base() { + let pool = Pool::memory().await.unwrap(); + pool.migrate().await.unwrap(); + + let owner = crate::models::User::new( + "push-base-owner".to_string(), + "push-base-owner@example.com".to_string(), + "hash".to_string(), + ); + crate::queries::UserQueries::create(&pool, &owner) + .await + .unwrap(); + let repo = crate::models::Repository::new( + "push-base-repo".to_string(), + owner.id, + "/git/push-base-repo".to_string(), + ); + RepoQueries::create(&pool, &repo).await.unwrap(); + + let pipeline = crate::models::Pipeline { + id: PipelineId::new(), + repo_id: repo.id, + name: "ci".to_string(), + trigger_type: "push".to_string(), + config: serde_json::json!({}), + created_at: chrono::Utc::now(), + active: true, + }; + let mut run = crate::models::PipelineRun::new( + pipeline.id, + repo.id, + "push".to_string(), + "b".repeat(40), + ); + run.ref_name = Some("refs/heads/main".to_string()); + let base = "a".repeat(40); + run.push_base_sha = Some(base.clone()); + + PipelineQueries::activate_pipeline_and_create_run(&pool, &pipeline, &run) + .await + .unwrap(); + + let stored = PipelineRunQueries::get(&pool, run.id) + .await + .unwrap() + .expect("run row"); + assert_eq!(stored.ref_name.as_deref(), Some("refs/heads/main")); + assert_eq!(stored.push_base_sha.as_deref(), Some(base.as_str())); + } + + // Regression: the push path writes its run through + // `activate_pipeline_and_create_run`, and that INSERT used to drop + // `ref_name`. The supersede filter is `AND ref_name = ?`, so a NULL + // ref_name made the documented "newest-SHA-wins" pass a no-op for every + // push run: an older queued run on the same branch was never cancelled. + // + // The shape here is the production one: every push mints a fresh + // pipeline version row, so the two runs carry different `pipeline_id`s + // even though they are the same pipeline. Matching on the version row + // id could never reach the previous push's run. + #[tokio::test] + async fn test_supersede_stale_runs_cancels_an_older_run_created_by_the_push_path() { + let pool = Pool::memory().await.unwrap(); + pool.migrate().await.unwrap(); + + let owner = crate::models::User::new( + "supersede-push-owner".to_string(), + "supersede-push-owner@example.com".to_string(), + "hash".to_string(), + ); + crate::queries::UserQueries::create(&pool, &owner) + .await + .unwrap(); + let repo = crate::models::Repository::new( + "supersede-push-repo".to_string(), + owner.id, + "/git/supersede-push-repo".to_string(), + ); + RepoQueries::create(&pool, &repo).await.unwrap(); + + let pipeline = crate::models::Pipeline { + id: PipelineId::new(), + repo_id: repo.id, + name: "ci".to_string(), + trigger_type: "push".to_string(), + config: serde_json::json!({}), + created_at: chrono::Utc::now(), + active: true, + }; + + // The older run is created exactly the way a push creates it: its + // own pipeline version row, carrying the pre-push base. + let mut older = crate::models::PipelineRun::new( + pipeline.id, + repo.id, + "push".to_string(), + "a".repeat(40), + ); + older.created_at = Utc::now() - chrono::Duration::hours(1); + older.ref_name = Some("refs/heads/main".to_string()); + older.push_base_sha = Some("c".repeat(40)); + PipelineQueries::activate_pipeline_and_create_run(&pool, &pipeline, &older) + .await + .unwrap(); + + // The newer push mints its own pipeline version row for the same + // (repo, name) — the version-row id differs, the pipeline does not. + let newer_pipeline = crate::models::Pipeline { + id: PipelineId::new(), + repo_id: repo.id, + name: "ci".to_string(), + trigger_type: "push".to_string(), + config: serde_json::json!({}), + created_at: chrono::Utc::now(), + active: true, + }; + let mut newer = crate::models::PipelineRun::new( + newer_pipeline.id, + repo.id, + "push".to_string(), + "b".repeat(40), + ); + newer.created_at = Utc::now(); + newer.ref_name = Some("refs/heads/main".to_string()); + PipelineQueries::activate_pipeline_and_create_run(&pool, &newer_pipeline, &newer) + .await + .unwrap(); + assert_ne!( + newer_pipeline.id, pipeline.id, + "consecutive pushes must carry different pipeline version row ids" + ); + + let superseded = PipelineQueries::supersede_stale_runs( + &pool, + repo.id, + "refs/heads/main", + newer_pipeline.id, + newer.id, + ) + .await + .unwrap(); + assert_eq!( + superseded, 1, + "an older queued push run on the same ref must be superseded" + ); + + let older_row = PipelineRunQueries::get(&pool, older.id) + .await + .unwrap() + .expect("older run row"); + assert_eq!(older_row.status, "cancelled"); + assert_eq!( + older_row.error.as_deref(), + Some("superseded by a newer push") + ); + assert_eq!(older_row.ref_name.as_deref(), Some("refs/heads/main")); + } + + // A repo can hold a manual-triggered pipeline and a push-triggered + // pipeline under the same name (they alternate which version is + // active). A push must not cancel the manual pipeline's queued runs: + // the supersede scope is the pipeline's identity — its name AND + // trigger type — not the name alone. + #[tokio::test] + async fn test_supersede_stale_runs_leaves_a_same_named_manual_pipeline_alone() { + let pool = Pool::memory().await.unwrap(); + pool.migrate().await.unwrap(); + + let owner = crate::models::User::new( + "supersede-manual-owner".to_string(), + "supersede-manual-owner@example.com".to_string(), + "hash".to_string(), + ); + crate::queries::UserQueries::create(&pool, &owner) + .await + .unwrap(); + let repo = crate::models::Repository::new( + "supersede-manual-repo".to_string(), + owner.id, + "/git/supersede-manual-repo".to_string(), + ); + RepoQueries::create(&pool, &repo).await.unwrap(); + + let manual_pipeline = crate::models::Pipeline { + id: PipelineId::new(), + repo_id: repo.id, + name: "ci".to_string(), + trigger_type: "manual".to_string(), + config: serde_json::json!({}), + created_at: chrono::Utc::now(), + active: true, + }; + PipelineQueries::create(&pool, &manual_pipeline) + .await + .unwrap(); + + let mut manual = crate::models::PipelineRun::new( + manual_pipeline.id, + repo.id, + "webhook".to_string(), + "a".repeat(40), + ); + manual.created_at = Utc::now() - chrono::Duration::hours(1); + manual.ref_name = Some("refs/heads/main".to_string()); + PipelineRunQueries::create(&pool, &manual).await.unwrap(); + + // The push activates its own version row for the same (repo, name) + // and retires the manual one — exactly what the push path does. + let push_pipeline = crate::models::Pipeline { + id: PipelineId::new(), + repo_id: repo.id, + name: "ci".to_string(), + trigger_type: "push".to_string(), + config: serde_json::json!({}), + created_at: chrono::Utc::now(), + active: true, + }; + let mut pushed = crate::models::PipelineRun::new( + push_pipeline.id, + repo.id, + "push".to_string(), + "b".repeat(40), + ); + pushed.created_at = Utc::now(); + pushed.ref_name = Some("refs/heads/main".to_string()); + PipelineQueries::activate_pipeline_and_create_run(&pool, &push_pipeline, &pushed) + .await + .unwrap(); + + let superseded = PipelineQueries::supersede_stale_runs( + &pool, + repo.id, + "refs/heads/main", + push_pipeline.id, + pushed.id, + ) + .await + .unwrap(); + assert_eq!( + superseded, 0, + "a push must not cancel a manual pipeline's run" + ); + + let manual_row = PipelineRunQueries::get(&pool, manual.id) + .await + .unwrap() + .expect("manual run row"); + assert_eq!( + manual_row.status, "pending", + "the manual pipeline's run is untouched" + ); + } + #[tokio::test] async fn test_trigger_request_requeue_stale_and_terminal_failure() { let pool = Pool::memory().await.unwrap(); @@ -6202,11 +6499,24 @@ mod tests { let mut unknown_receipt = crate::models::Job::new(run.id, "receipt-unknown".to_string()); let mut garbage_receipt = crate::models::Job::new(run.id, "receipt-garbage".to_string()); let mut live_queued = crate::models::Job::new(run.id, "still-queued".to_string()); + // GF-01: the scheduler's runner-completion receipt carries + // `success`, not `status`. Grading only `status` read this row's + // `success: true` as no verdict at all and failed it closed, so a + // green job was recorded as failed. + let mut runner_completion = + crate::models::Job::new(run.id, "receipt-runner-completion".to_string()); + // A backend failure keeps its class through the receipt: the + // runner records `outcome`, and the row must not come back as a + // plain code failure. + let mut runner_infrastructure = + crate::models::Job::new(run.id, "receipt-runner-infrastructure".to_string()); for job in [ &mut succeeded_receipt, &mut unknown_receipt, &mut garbage_receipt, &mut live_queued, + &mut runner_completion, + &mut runner_infrastructure, ] { JobQueries::create(&pool, job).await.unwrap(); } @@ -6244,8 +6554,18 @@ mod tests { tear(unknown_receipt.id, r#"{"status":"exploded"}"#).await; tear(garbage_receipt.id, "runner died mid-write").await; tear(assigned_receipt.id, r#"{"status":"failed"}"#).await; + tear( + runner_completion.id, + r#"{"job_id":"x","success":true,"exit_code":0,"step_results":[],"artifacts":[]}"#, + ) + .await; + tear( + runner_infrastructure.id, + r#"{"job_id":"y","success":false,"outcome":"infrastructure_failure","exit_code":-1}"#, + ) + .await; - assert_eq!(JobQueries::reconcile_evidence_rows(&pool).await.unwrap(), 4); + assert_eq!(JobQueries::reconcile_evidence_rows(&pool).await.unwrap(), 6); let graded = |job_id: gitforge_common::JobId| { let pool = &pool; @@ -6256,6 +6576,14 @@ mod tests { assert_eq!(graded(unknown_receipt.id).await.status, "failed"); assert_eq!(graded(garbage_receipt.id).await.status, "failed"); assert_eq!(graded(assigned_receipt.id).await.status, "failed"); + // A runner-completion receipt grades from its `success` boolean. + assert_eq!(graded(runner_completion.id).await.status, "succeeded"); + // A backend failure keeps `infrastructure_failure`, not `failed`: + // the two mean different things to an operator (R6.3). + assert_eq!( + graded(runner_infrastructure.id).await.status, + "infrastructure_failure" + ); // The repair grades from evidence without rewriting it: the receipt // columns keep their values and no start time is invented. diff --git a/crates/gitforge-runner/Cargo.toml b/crates/gitforge-runner/Cargo.toml index 7d18a259..d7a12778 100644 --- a/crates/gitforge-runner/Cargo.toml +++ b/crates/gitforge-runner/Cargo.toml @@ -23,6 +23,7 @@ async-trait = { workspace = true } reqwest = { workspace = true } sha2 = { workspace = true } hex = { workspace = true } +libc = "0.2" [dev-dependencies] tempfile = { workspace = true } diff --git a/crates/gitforge-runner/src/agent.rs b/crates/gitforge-runner/src/agent.rs index 1e615f25..8205a427 100644 --- a/crates/gitforge-runner/src/agent.rs +++ b/crates/gitforge-runner/src/agent.rs @@ -1,11 +1,15 @@ //! Runner agent use crate::executor::{ExecutableJob, JobExecutor, JobStep}; -use gitforge_common::{Error, JobId, PipelineRunId, Result, RunnerId, VolumeSpec}; +use gitforge_common::{ + Error, JobExistenceRequest, JobExistenceResponse, JobId, JobRowState, PipelineRunId, Result, + RunnerId, VolumeSpec, +}; use gitforge_db::models::Runner; use gitforge_sandbox::{ - ActiveJobRegistry, DockerContainerSource, DockerSandbox, OutputSink, OutputStream, - ReconcileReport, Reconciler, ReconcilerPolicy, StepResult, + reclaim_orphaned_pool_containers, ActiveJobRegistry, DockerContainerSource, DockerSandbox, + JobExistenceResolver, OutputSink, OutputStream, PoolIdentity, PoolPolicy, ReconcileReport, + Reconciler, ReconcilerPolicy, StepResult, }; use gitforge_storage::ArtifactReceipt; use reqwest::Client; @@ -886,7 +890,7 @@ impl RunnerAgent { .map_err(|e| Error::internal(format!("failed to create HTTP client: {e}")))?; let sandbox = DockerSandbox::connect_required().await?; - let executor = JobExecutor::new().await?; + let executor = JobExecutor::new(pool_identity(&config)).await?; Ok(Self { config, @@ -1104,6 +1108,13 @@ impl RunnerAgent { // service explicitly enables deletion via GITFORGE_RECONCILE_DELETE. self.spawn_reconciler(active_jobs.clone()).await; + // Spawn the warm-pool reaper: it reclaims the pooled containers a + // previous runner process left behind when it died or restarted. + // The pool lives only in memory, so its containers outlive the + // process that made them and nothing else in the tree can reach + // them (GF-17). + self.spawn_pool_reaper().await; + // Start job fetch loop let fetch_interval = self.config.fetch_interval_secs; let fetch_client = self.client.clone(); @@ -1230,6 +1241,34 @@ impl RunnerAgent { Ok(()) } + /// Spawn the background warm-pool reaper loop. + /// + /// The reaper owns the pool's orphans: containers that carry this + /// pool's owner label but a generation from a process that is gone. + /// It never touches the current process's own pool, and it never + /// touches a container whose creating process is still alive. + async fn spawn_pool_reaper(&self) { + let identity = self.executor.pool_identity().clone(); + let policy = PoolPolicy { + ttl: crate::executor::pool_ttl(), + ..PoolPolicy::default() + }; + let interval_secs = match std::env::var("GITFORGE_POOL_REAP_INTERVAL_SECS") { + Ok(raw) => match raw.trim().parse::() { + Ok(secs) if secs > 0 => secs, + _ => { + tracing::warn!( + "GITFORGE_POOL_REAP_INTERVAL_SECS is not a positive number of seconds; using the default" + ); + 300 + } + }, + Err(_) => 300, + }; + let handle = tokio::spawn(run_pool_reaper_loop(identity, policy, interval_secs)); + self.background_tasks.lock().await.push(handle); + } + /// Spawn the background abandoned-container reconciler loop. /// /// Env policy (documented defaults keep startup safe): @@ -1246,17 +1285,36 @@ impl RunnerAgent { /// always retained. A policy failure is logged and the loop is not /// started: the reconciler must never guess. async fn spawn_reconciler(&self, active_jobs: ActiveJobRegistry) { - let config = match load_reconciler_config_from_env() { + let mut config = match load_reconciler_config_from_env() { Ok(config) => config, Err(error) => { tracing::error!(%error, "abandoned-container reconciler NOT started"); return; } }; + config.store = self.store_lookup(); let handle = tokio::spawn(run_reconciler_loop(active_jobs, config)); *self.reconciler_task.lock().await = Some(handle); } + /// The store lookup the reconciler uses: the scheduler's bulk + /// job-existence endpoint, reached with the credential the runner + /// already presents for every other scheduler call. + /// + /// Absent when the runner has no scheduler credential: the + /// store-backed rule then has no store to consult and retains every + /// container it could have reached. + fn store_lookup(&self) -> Option { + let token = self.config.scheduler_token.as_deref()?; + if token.is_empty() { + return None; + } + Some(StoreLookup { + scheduler_url: self.config.scheduler_url.clone(), + scheduler_token: Arc::from(token), + }) + } + /// Stop the runner agent /// If force is true, cancel all running jobs immediately. /// Otherwise, wait for jobs to complete gracefully. @@ -1804,6 +1862,32 @@ pub(crate) struct ReconcilerLoopConfig { policy: ReconcilerPolicy, interval: Duration, receipt_path: Option, + /// Where to ask the store about job ids, and with what credential. + /// Absent means the store-backed rule has no store to consult and + /// retains every container it could have reached. + store: Option, +} + +/// The store lookup the reconciler uses: the scheduler's bulk +/// job-existence endpoint. +/// +/// The scheduler owns the authoritative store, so the runner asks it +/// rather than reading the database itself. +#[derive(Clone)] +pub(crate) struct StoreLookup { + scheduler_url: String, + scheduler_token: Arc, +} + +/// The credential is redacted: a derived `Debug` would put the bearer +/// token into every log line that formats a loop config. +impl fmt::Debug for StoreLookup { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("StoreLookup") + .field("scheduler_url", &self.scheduler_url) + .field("scheduler_token", &"") + .finish() + } } /// Outcome of a single receipt write. Used by tests and the production loop @@ -1828,6 +1912,15 @@ fn load_reconciler_config_from_env() -> std::result::Result match validate_reconciler_receipt_path(&raw) { Ok(path) => Some(path), @@ -1845,6 +1938,8 @@ fn load_reconciler_config_from_env() -> std::result::Result std::result::Result source, + Err(error) => { + tracing::warn!("warm-pool reaper unavailable: {}", error); + return; + } + }; + // The owner's liveness is injected, not read from process-global state + // inside the decision: the reaper passes the resolver in, so the + // decision is testable without touching the process table. + let owner_state: Box Option + Send + Sync> = + Box::new(|pid: u32| gitforge_sandbox::process_alive(pid)); + let mut ticker = interval(Duration::from_secs(interval_secs)); + loop { + // Startup pass runs immediately, then bounded periodic. + ticker.tick().await; + let now_ms = chrono::Utc::now().timestamp_millis(); + match reclaim_orphaned_pool_containers(&source, &identity, &policy, now_ms, &owner_state) + .await + { + Ok(report) => { + if !report.reclaimed.is_empty() || !report.failed.is_empty() { + tracing::info!( + reclaimed = report.reclaimed.len(), + failed = report.failed.len(), + retained = report.retained.len(), + bounded = report.bounded, + "warm-pool reclaim pass complete" + ); + } + for (id, reason) in &report.failed { + tracing::warn!("failed to reclaim pooled container {}: {}", id, reason); + } + } + Err(error) => tracing::warn!("warm-pool reclaim pass failed: {}", error), + } + } +} + +/// Bounded timeout for every Docker call the reaper makes. +const POOL_REAPER_CALL_TIMEOUT: Duration = Duration::from_secs(10); + +/// The identity that owns this runner's warm pool. +/// +/// The owner separates pools that belong to different runners on the same +/// host; the generation separates the containers this process created from +/// the ones a previous process left behind. +fn pool_identity(config: &RunnerConfig) -> PoolIdentity { + let host = std::env::var("HOSTNAME") + .ok() + .or_else(|| { + std::fs::read_to_string("/proc/sys/kernel/hostname") + .ok() + .map(|name| name.trim().to_string()) + }) + .filter(|name| !name.is_empty()) + .unwrap_or_else(|| "unknown-host".to_string()); + let owner = format!("{}@{}", config.name, host); + // A fresh UUID per process: two processes of the same runner never + // share a generation, so neither can reclaim the other's pool. + let generation = JobId::new().to_string(); + PoolIdentity::new(owner, generation) +} + +/// Resolve job existence through the scheduler's bulk endpoint. +/// +/// The scheduler owns the authoritative store, so the runner asks it +/// rather than reading the database itself. Every failure — an +/// unreachable scheduler, a rejected credential, an unparsable answer — +/// leaves every id `unknown`, which retains the containers. +struct SchedulerJobExistence { + client: Client, + url: String, + token: Arc, +} + +impl SchedulerJobExistence { + fn new(store: &StoreLookup) -> Result { + let client = Client::builder() + .timeout(STORE_LOOKUP_TIMEOUT) + .build() + .map_err(|e| Error::internal(format!("failed to create store-lookup client: {e}")))?; + let url = format!( + "{}/jobs/existence", + store.scheduler_url.trim_end_matches('/') + ); + Ok(Self { + client, + url, + token: store.scheduler_token.clone(), + }) + } +} + +/// Bounded timeout for one store lookup. The lookup is on the reconcile +/// pass's critical path, so a hung scheduler delays the pass instead of +/// stalling it forever. +const STORE_LOOKUP_TIMEOUT: Duration = Duration::from_secs(10); + +#[async_trait::async_trait] +impl JobExistenceResolver for SchedulerJobExistence { + async fn resolve(&self, job_ids: &[String]) -> HashMap { + let mut states: HashMap = job_ids + .iter() + .map(|job_id| (job_id.clone(), JobRowState::Unknown)) + .collect(); + if job_ids.is_empty() { + return states; + } + let response = self + .client + .post(&self.url) + .bearer_auth(self.token.as_ref()) + .json(&JobExistenceRequest { + job_ids: job_ids.to_vec(), + }) + .send() + .await; + let Ok(response) = response else { + tracing::warn!("store lookup failed; every job id stays unknown"); + return states; + }; + if !response.status().is_success() { + tracing::warn!( + status = %response.status(), + "store lookup was refused; every job id stays unknown" + ); + return states; + } + let Ok(body) = response.json::().await else { + tracing::warn!("store lookup answer was unparsable; every job id stays unknown"); + return states; + }; + for (job_id, state) in body.states { + states.insert(job_id, state); + } + states + } +} + pub(crate) async fn run_reconciler_loop( active_jobs: ActiveJobRegistry, config: ReconcilerLoopConfig, @@ -1893,7 +2138,22 @@ pub(crate) async fn run_reconciler_loop( return; } }; - let reconciler = Reconciler::new(source, config.policy); + let orphan_rule_armed = config.policy.orphan_reclaim_enabled; + let mut reconciler = Reconciler::new(source, config.policy); + if orphan_rule_armed { + match config.store.as_ref().map(SchedulerJobExistence::new) { + Some(Ok(resolver)) => { + reconciler = reconciler.with_resolver(Arc::new(resolver)); + } + Some(Err(error)) => tracing::warn!( + %error, + "store lookup unavailable; the legacy-orphan rule retains every container" + ), + None => tracing::warn!( + "the legacy-orphan rule is armed but no store lookup is configured; every container is retained" + ), + } + } let mut ticker = interval(config.interval); loop { // Startup pass runs immediately, then bounded periodic. @@ -1910,6 +2170,9 @@ pub(crate) async fn run_reconciler_loop( already_gone = report.already_gone, failed = report.failed, deletion_enabled = report.policy_deletion_enabled, + legacy_orphans = report.legacy_orphans, + store_requested = report.store_requested, + store_answered = report.store_answered, "abandoned-container reconcile pass complete" ); if let Some(path) = &config.receipt_path { @@ -2060,6 +2323,9 @@ mod reconciler_loop_tests { removed: 1, already_gone: 0, failed: 0, + store_requested: 0, + store_answered: 0, + legacy_orphans: 0, entries: Vec::new(), }; @@ -2129,6 +2395,7 @@ mod reconciler_loop_tests { }, interval: Duration::from_secs(60), receipt_path: None, + store: None, }; let registry_one = registry.clone(); let first = tokio::spawn(run_reconciler_loop(registry_one, config.clone())); @@ -2499,6 +2766,32 @@ mod tests { assert!(agent.runner.is_none()); } + #[test] + fn pool_identity_separates_runners_and_generations() { + let config = RunnerConfig { + name: "runner-a".to_string(), + ..RunnerConfig::default() + }; + let first = pool_identity(&config); + let second = pool_identity(&config); + // Same runner, same owner: the owner is what lets one runner + // reclaim its own orphans. + assert_eq!(first.owner, second.owner); + assert!(first.owner.starts_with("runner-a@")); + // Different generation: a container from one process is never + // mistaken for the current process's own container. + assert_ne!(first.generation, second.generation); + // The creating process is the current one. + assert_eq!(first.pid, std::process::id()); + + let other_config = RunnerConfig { + name: "runner-b".to_string(), + ..RunnerConfig::default() + }; + let other = pool_identity(&other_config); + assert_ne!(other.owner, first.owner); + } + #[tokio::test] async fn test_runner_config_default() { let config = RunnerConfig::default(); @@ -4133,3 +4426,139 @@ mod tests { assert_eq!(guard.identity_rejections, 1); } } + +/// The runner half of the store-backed legacy-orphan rule. +/// +/// These tests pin both ends of the wire: the request the runner sends +/// (method, path, credential, body) and the way it reads the answer. The +/// fail-closed contract is the point: every way the lookup can fail must +/// leave every id `unknown`, because `unknown` is the only answer that +/// never authorises a container removal. +#[cfg(test)] +mod store_lookup_tests { + use super::*; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + /// Serve exactly one HTTP request with `response` and hand back the + /// raw request text the client sent. + async fn spawn_stub(response: String) -> (String, tokio::task::JoinHandle) { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let handle = tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.unwrap(); + let mut request = Vec::new(); + let mut buffer = [0_u8; 1024]; + loop { + let read = socket.read(&mut buffer).await.unwrap(); + if read == 0 { + break; + } + request.extend_from_slice(&buffer[..read]); + let text = String::from_utf8_lossy(&request).to_string(); + let Some(header_end) = text.find("\r\n\r\n") else { + continue; + }; + let content_length = text[..header_end] + .lines() + .find_map(|line| { + let (name, value) = line.split_once(':')?; + if name.eq_ignore_ascii_case("content-length") { + value.trim().parse::().ok() + } else { + None + } + }) + .unwrap_or(0); + if text.len() >= header_end + 4 + content_length { + break; + } + } + socket.write_all(response.as_bytes()).await.unwrap(); + socket.flush().await.unwrap(); + String::from_utf8_lossy(&request).to_string() + }); + (format!("http://{address}"), handle) + } + + fn http_response(status: &str, body: &str) -> String { + format!( + "HTTP/1.1 {status}\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}", + body.len() + ) + } + + fn resolver_for(url: &str) -> SchedulerJobExistence { + SchedulerJobExistence::new(&StoreLookup { + scheduler_url: url.to_string(), + scheduler_token: Arc::from("runner-secret"), + }) + .expect("resolver") + } + + /// The runner asks the scheduler's bulk endpoint with its own bearer + /// credential, and reads the answer back per id. + #[tokio::test] + async fn the_resolver_asks_the_scheduler_with_the_runner_credential() { + let body = serde_json::json!({ + "states": { "id-a": "absent", "id-b": "present" } + }) + .to_string(); + let (url, handle) = spawn_stub(http_response("200 OK", &body)).await; + let resolver = resolver_for(&url); + let ids = vec!["id-a".to_string(), "id-b".to_string()]; + let states = resolver.resolve(&ids).await; + let request = handle.await.unwrap(); + let lowered = request.to_lowercase(); + assert!( + lowered.starts_with("post /jobs/existence "), + "unexpected request line: {request}" + ); + assert!( + lowered.contains("authorization: bearer runner-secret"), + "the runner credential must authorize the lookup: {request}" + ); + assert!( + lowered.contains("\"id-a\""), + "the requested ids must travel in the body: {request}" + ); + assert_eq!(states.get("id-a"), Some(&JobRowState::Absent)); + assert_eq!(states.get("id-b"), Some(&JobRowState::Present)); + } + + /// A refused lookup is not evidence of absence: every id stays + /// `unknown`. + #[tokio::test] + async fn a_refused_lookup_leaves_every_id_unknown() { + let (url, handle) = spawn_stub(http_response( + "500 Internal Server Error", + &serde_json::json!({ "error": "boom" }).to_string(), + )) + .await; + let resolver = resolver_for(&url); + let ids = vec!["id-a".to_string(), "id-b".to_string()]; + let states = resolver.resolve(&ids).await; + let _ = handle.await.unwrap(); + assert!(states.values().all(|state| *state == JobRowState::Unknown)); + } + + /// An unparsable answer is not evidence of absence either. + #[tokio::test] + async fn an_unparsable_answer_leaves_every_id_unknown() { + let (url, handle) = spawn_stub(http_response("200 OK", "not json at all")).await; + let resolver = resolver_for(&url); + let ids = vec!["id-a".to_string()]; + let states = resolver.resolve(&ids).await; + let _ = handle.await.unwrap(); + assert!(states.values().all(|state| *state == JobRowState::Unknown)); + } + + /// An unreachable scheduler is the same contract: the lookup fails, so + /// every id stays `unknown` and every container is retained. + #[tokio::test] + async fn an_unreachable_scheduler_leaves_every_id_unknown() { + let resolver = resolver_for("http://127.0.0.1:1"); + let ids = vec!["id-a".to_string(), "id-b".to_string()]; + let states = resolver.resolve(&ids).await; + assert!(states.values().all(|state| *state == JobRowState::Unknown)); + } +} diff --git a/crates/gitforge-runner/src/executor.rs b/crates/gitforge-runner/src/executor.rs index 9a52e50d..56e5e9ac 100644 --- a/crates/gitforge-runner/src/executor.rs +++ b/crates/gitforge-runner/src/executor.rs @@ -1,9 +1,10 @@ //! Job executor with container pooling use crate::fault::{fault_error, fault_outcome}; +use crate::mount_lock::lock_job_mounts; use gitforge_common::{Fault, JobId, PipelineRunId, RepoId, Result, VolumeSpec}; use gitforge_sandbox::{ - DockerSandbox, OutputSink, Sandbox, SandboxInstance, SandboxLimits, StepResult, + DockerSandbox, OutputSink, PoolIdentity, Sandbox, SandboxInstance, SandboxLimits, StepResult, }; use gitforge_storage::{ Artifact, ArtifactReceipt, ArtifactStore, FileJobLogStore, FileStorage, JobReceipt, LogReceipt, @@ -20,6 +21,27 @@ use tokio::time::{timeout, Duration, Instant}; /// Default number of pre-warmed containers per image const POOL_SIZE: usize = 2; +/// How long a pooled container may sit in the pool before it is +/// destroyed. A pooled container is a spare, not a reservation: keeping +/// one past its TTL only holds resources for a job that may never come. +const POOL_TTL: Duration = Duration::from_secs(1800); + +/// The warm pool's time-to-live. +/// +/// One definition, shared by the in-process pool (which expires a spare +/// before handing it out) and the reaper's policy (which decides when an +/// orphan may be reclaimed). Two different numbers would let the two +/// mechanisms disagree about the same container. +pub fn pool_ttl() -> Duration { + POOL_TTL +} + +/// A pooled container and the moment it entered the pool. +struct PooledInstance { + instance: SandboxInstance, + created_at: Instant, +} + /// Sandbox acquisition ceiling in seconds. Container creation under load /// (whole layer-stack copies on the daemon's old vfs storage) could /// legitimately take minutes, so operators can raise the 60-second default @@ -163,31 +185,51 @@ fn runtime_toolchain_fetch(steps: &[JobStep]) -> Option { /// A pool of pre-warmed container instances pub struct ContainerPool { - pools: Arc>>>, // image -> instances + pools: Arc>>>, // image -> instances sandbox: Arc, + /// Ownership identity of this pool. Every pooled container carries it + /// as labels, so a later runner process can recognise the containers + /// this pool left behind and reclaim them (GF-17). + identity: PoolIdentity, + /// How long a pooled container may sit before it is destroyed. + ttl: Duration, } impl ContainerPool { - /// Create a new container pool - pub async fn new() -> Result { + /// Create a new container pool owned by `identity`. + pub async fn new(identity: PoolIdentity) -> Result { let sandbox = DockerSandbox::connect_required().await?; Ok(Self { pools: Arc::new(RwLock::new(HashMap::new())), sandbox: Arc::new(sandbox), + identity, + ttl: POOL_TTL, }) } + /// The pool's ownership identity. The agent uses it to reclaim the + /// containers this pool orphaned when a previous runner process died. + pub fn identity(&self) -> &PoolIdentity { + &self.identity + } + /// Pre-warm containers for an image pub async fn prewarm(&self, image: &str, count: usize) -> Result<()> { let mut pools = self.pools.write().await; let instances = pools.entry(image.to_string()).or_insert_with(Vec::new); while instances.len() < count { - let id = JobId::new(); - match self.sandbox.create(id, image, sandbox_limits(), &[]).await { + match self + .sandbox + .create_pooled(&self.identity, image, sandbox_limits()) + .await + { Ok(instance) => { tracing::info!("pre-warmed container for image {}", image); - instances.push(instance); + instances.push(PooledInstance { + instance, + created_at: Instant::now(), + }); } Err(e) => { tracing::warn!("failed to pre-warm container: {}", e); @@ -225,15 +267,39 @@ impl ContainerPool { .create(*job_id, image, sandbox_limits(), volumes) .await; } - let mut pools = self.pools.write().await; - - // Try to get from pool - if let Some(instances) = pools.get_mut(image) { - if let Some(instance) = instances.pop() { - tracing::debug!("reusing pooled container for job {}", job_id); - return Ok(instance); + // Try to get from pool. A container that outlived the pool's TTL + // is destroyed rather than handed out: it is a spare, not a + // reservation, and a stale spare costs the same resources as a + // fresh one without the warmth. + let ttl = self.ttl; + let mut expired: Vec = Vec::new(); + let mut fresh: Option = None; + { + let mut pools = self.pools.write().await; + if let Some(instances) = pools.get_mut(image) { + while let Some(entry) = instances.pop() { + if entry.created_at.elapsed() >= ttl { + expired.push(entry.instance); + } else { + fresh = Some(entry.instance); + break; + } + } } } + for stale in expired { + tracing::info!( + "destroying pooled container {} that outlived its TTL", + stale.container_id + ); + if let Err(error) = self.sandbox.destroy(stale).await { + tracing::warn!("failed to destroy expired pooled container: {}", error); + } + } + if let Some(instance) = fresh { + tracing::debug!("reusing pooled container for job {}", job_id); + return Ok(instance); + } // Pool empty or no pool for this image, create new tracing::debug!("creating new container for job {} (pool empty)", job_id); @@ -263,11 +329,19 @@ impl ContainerPool { tracing::warn!("failed to reset pooled container: {}", e); return; } - // Create fresh instance for the pool - let id = JobId::new(); - match self.sandbox.create(id, image, sandbox_limits(), &[]).await { + // Create fresh instance for the pool. It carries the pool's + // ownership labels, so it is reclaimable if this process dies + // with it still warm (GF-17). + match self + .sandbox + .create_pooled(&self.identity, image, sandbox_limits()) + .await + { Ok(new_instance) => { - instances.push(new_instance); + instances.push(PooledInstance { + instance: new_instance, + created_at: Instant::now(), + }); tracing::debug!("returned container to pool"); } Err(e) => { @@ -281,6 +355,62 @@ impl ContainerPool { } } } + + /// Destroy every container this pool holds. + /// + /// The pool exists only in this process's memory, so a runner that + /// stops without draining leaves its containers running with nobody + /// left to own them: they outlive the process, no job row refers to + /// them, and no other cleanup path in the tree can reach them + /// (GF-17). Draining on shutdown is what keeps a pooled container + /// from outliving the pool that made it. + pub async fn drain(&self) { + let drained: Vec = { + let mut pools = self.pools.write().await; + let mut drained = Vec::new(); + for instances in pools.values_mut() { + for entry in instances.drain(..) { + drained.push(entry.instance); + } + } + pools.clear(); + drained + }; + for instance in drained { + if let Err(error) = self.sandbox.destroy(instance).await { + tracing::warn!("failed to destroy pooled container on drain: {}", error); + } + } + } + + /// Destroy the pooled containers that have outlived their TTL. + /// Returns how many were destroyed. + pub async fn sweep_expired(&self) -> usize { + let ttl = self.ttl; + let expired: Vec = { + let mut pools = self.pools.write().await; + let mut expired = Vec::new(); + for instances in pools.values_mut() { + let mut kept = Vec::with_capacity(instances.len()); + for entry in instances.drain(..) { + if entry.created_at.elapsed() >= ttl { + expired.push(entry.instance); + } else { + kept.push(entry); + } + } + *instances = kept; + } + expired + }; + let count = expired.len(); + for instance in expired { + if let Err(error) = self.sandbox.destroy(instance).await { + tracing::warn!("failed to destroy expired pooled container: {}", error); + } + } + count + } } /// Job executor @@ -294,9 +424,15 @@ pub struct JobExecutor { } impl JobExecutor { - /// Create a new job executor - pub async fn new() -> Result { - let pool = ContainerPool::new().await?; + /// The identity of the pool this executor owns. + pub fn pool_identity(&self) -> &PoolIdentity { + self.pool.identity() + } + + /// Create a new job executor whose container pool is owned by + /// `identity`. + pub async fn new(identity: PoolIdentity) -> Result { + let pool = ContainerPool::new(identity).await?; let storage_root = std::env::var("GITFORGE_ARTIFACT_ROOT") .unwrap_or_else(|_| "target/gitforge-artifacts".to_string()); let artifact_storage = FileStorage::new(storage_root.clone()).await?; @@ -427,7 +563,7 @@ impl JobExecutor { /// original accumulated-result behavior. pub async fn execute_with_output( &self, - job: ExecutableJob, + mut job: ExecutableJob, output_sink: Option>, ) -> JobResult { let job_id = job.job_id; // Copy type @@ -491,6 +627,52 @@ impl JobExecutor { }; } + // GF-11: two jobs that mount the same host directory must not run + // at the same time. A shared build cache is exactly that: a + // rebuild in one run replaces the binaries another run is about to + // execute, and that run dies with ENOENT on the missing binary + // (measured 2026-11-05 on the DIL repository, where two coverage + // jobs lost their test binaries to a concurrent rebuild). The + // runner holds an exclusive advisory lock on every writable host + // directory the job mounts for the whole job. The wait is charged + // to the job's own budget: a job that cannot get the directory is + // refused instead of running against a directory another job is + // writing to. + let lock_wait = deadline.saturating_duration_since(Instant::now()); + let cache_root = gitforge_common::cache_root(); + let docker_socket = gitforge_common::docker_socket(); + let (mount_locks, locked_volumes) = match lock_job_mounts( + &job.image, + &job.volumes, + &cache_root, + &docker_socket, + lock_wait, + ) + .await + { + Ok(locked) => locked, + Err(error) => { + let completed_at = chrono::Utc::now(); + tracing::warn!(%job_id, %error, "could not lock the job's shared host mounts"); + return JobResult { + job_id, + success: false, + exit_code: -1, + step_results: Vec::new(), + artifacts: Vec::new(), + logs: None, + started_at, + completed_at, + timed_out: false, + error: Some(format!("shared mount unavailable: {error}")), + workspace_path: job.working_dir.clone(), + infrastructure_failure: true, + }; + } + }; + let _mount_locks = mount_locks; + job.volumes = locked_volumes; + // Acquire container from pool. Both failure shapes here are the // container backend failing, not the commit (F40's wedged podman // surfaced exactly as the Err arm), so they carry the backend @@ -730,14 +912,70 @@ impl JobExecutor { instances.remove(&job_id) }; if let (Some((image, workspace, inst)), false) = (released, timed_out) { + // The teardown has to COMPLETE. A cancelled teardown leaves the + // job's container running forever (measured 2026-11-05: nine + // leaked containers, one per DIL job whose workspace walk + // exceeded the budget - and a leaked container blocks every + // release cutover, whose drain gate counts live job + // containers). The workspace ownership cleanup is bounded + // inside the sandbox, so what remains here is a stop and a + // remove; the budget covers them with room to spare. if timeout( - Duration::from_secs(30), + Duration::from_secs(60), self.pool.release(&image, inst, workspace.as_deref()), ) .await .is_err() { - tracing::warn!("timed out cleaning up sandbox for job {}", job_id); + // The teardown did not finish. Leaving the container behind + // is not an option: a container nobody stops runs forever, + // and the release drain gate counts live job containers, so + // one abandoned container blocks every future cutover + // (measured 2026-11-05: eight leaked containers from + // succeeded jobs, and a release that could not drain for + // four hours). Remove it by job label, forcefully, without + // the workspace walk that timed out. + tracing::error!( + "timed out cleaning up sandbox for job {}; forcing container removal", + job_id + ); + match uuid::Uuid::parse_str(&job_id.to_string()) { + Ok(uuid) => { + let removed = timeout( + Duration::from_secs(30), + self.pool.sandbox.remove_job_containers(JobId::from(uuid)), + ) + .await; + match removed { + Ok(Ok(())) => { + tracing::info!( + "forced container removal finished for job {}", + job_id + ); + } + Ok(Err(error)) => { + tracing::error!( + "forced container removal failed for job {}: {}", + job_id, + error + ); + } + Err(_) => { + tracing::error!( + "forced container removal did not finish for job {}", + job_id + ); + } + } + } + Err(error) => { + tracing::error!( + "job {} has no parseable id, so its container cannot be removed by label: {}", + job_id, + error + ); + } + } } } @@ -846,6 +1084,12 @@ impl JobExecutor { } } + // The warm pool is drained too. It is not reachable from + // `active_instances` (a pooled container never ran a job), and a + // pool that outlives its process is a leak with no owner + // (GF-17). + self.pool.drain().await; + let mut count = self.active_job_count.write().await; *count = 0; } @@ -1149,7 +1393,10 @@ mod tests { #[tokio::test] #[ignore] async fn test_real_docker_job_timeout_reaps_sandbox() { - let executor = JobExecutor::new().await.expect("Docker must be available"); + let identity = PoolIdentity::new("test@localhost", "test-generation"); + let executor = JobExecutor::new(identity) + .await + .expect("Docker must be available"); let job_id = JobId::new(); let job = ExecutableJob::new(job_id, PipelineRunId::new(), "alpine:latest".to_string()) .with_steps(vec![JobStep::new("hang", "sleep 30")]) diff --git a/crates/gitforge-runner/src/lib.rs b/crates/gitforge-runner/src/lib.rs index b9669f0b..0c7d1120 100644 --- a/crates/gitforge-runner/src/lib.rs +++ b/crates/gitforge-runner/src/lib.rs @@ -5,6 +5,7 @@ pub mod agent; pub mod executor; pub mod fault; +pub mod mount_lock; pub use agent::{RunnerAgent, RunnerConfig}; pub use executor::JobExecutor; diff --git a/crates/gitforge-runner/src/mount_lock.rs b/crates/gitforge-runner/src/mount_lock.rs new file mode 100644 index 00000000..d944bd32 --- /dev/null +++ b/crates/gitforge-runner/src/mount_lock.rs @@ -0,0 +1,357 @@ +//! Serializing the host directories a job mounts. +//! +//! Two pipeline runs that mount the same host directory compile into one +//! build tree. A rebuild in one run replaces the binaries another run is +//! about to execute, and that run dies with ENOENT on the missing binary +//! (GF-11, measured 2026-11-05 on the DIL repository, where two coverage +//! jobs lost their test binaries to a concurrent rebuild). +//! +//! The runner takes an exclusive advisory lock on the host directory +//! itself and holds it for the job's lifetime. Locking the directory +//! rather than a lock file beside it keeps one source of truth: the runner +//! locks exactly the path it hands to the sandbox, there is no file for the +//! job's container to delete (the container mounts the directory, not a +//! sibling of it), and no second path convention to keep in step with the +//! mount. +//! +//! The lock is advisory and lives on the host. Only the runner creates +//! containers, so every writer of a shared host directory passes through +//! it. +//! +//! A mount the runner cannot lock is refused: the job does not run, +//! rather than running against a directory with a job that is writing to +//! it. The sandbox refuses a mount it cannot create the same way, so the +//! runner's behaviour matches the mount contract, and neither of them can +//! leave a job running with a contract the pipeline never declared (GF-10, +//! 2026-11-05). + +use std::fmt; +use std::fs::{self, File}; +use std::io; +use std::os::fd::AsRawFd; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use gitforge_common::{is_socket_source, VolumeSpec, VolumeType}; +use tokio::time::Instant; + +/// How long to wait between attempts to take a busy directory. +const POLL_INTERVAL: Duration = Duration::from_millis(100); + +/// Why a shared host directory could not be locked. +#[derive(Debug)] +pub enum MountLockError { + /// The directory could not be created, opened, or locked. + Unavailable { path: PathBuf, error: io::Error }, + /// Another job holds the directory and did not release it in time. + Busy { path: PathBuf, waited: Duration }, +} + +impl fmt::Display for MountLockError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Unavailable { path, error } => { + write!(formatter, "{} is not lockable: {error}", path.display()) + } + Self::Busy { path, waited } => write!( + formatter, + "{} stayed busy for {}s", + path.display(), + waited.as_secs() + ), + } + } +} + +impl std::error::Error for MountLockError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + Self::Unavailable { error, .. } => Some(error), + Self::Busy { .. } => None, + } + } +} + +/// An exclusive advisory lock on a host directory. +/// +/// The lock is released when the value is dropped. +#[derive(Debug)] +pub struct MountLock { + directory: File, + path: PathBuf, +} + +impl MountLock { + /// Take the lock, waiting up to `wait` for a holder to release it. + pub async fn acquire(path: &Path, wait: Duration) -> Result { + if let Err(error) = fs::create_dir_all(path) { + return Err(MountLockError::Unavailable { + path: path.to_path_buf(), + error, + }); + } + let directory = File::open(path).map_err(|error| MountLockError::Unavailable { + path: path.to_path_buf(), + error, + })?; + let deadline = Instant::now() + wait; + loop { + match try_lock(&directory) { + Ok(true) => { + return Ok(Self { + directory, + path: path.to_path_buf(), + }) + } + Ok(false) => {} + Err(error) => { + return Err(MountLockError::Unavailable { + path: path.to_path_buf(), + error, + }) + } + } + if Instant::now() >= deadline { + return Err(MountLockError::Busy { + path: path.to_path_buf(), + waited: wait, + }); + } + tokio::time::sleep(POLL_INTERVAL).await; + } + } + + /// The host directory this lock guards. + pub fn path(&self) -> &Path { + &self.path + } +} + +impl Drop for MountLock { + fn drop(&mut self) { + // Closing the descriptor releases the lock too; unlocking states + // the intent and releases it a little earlier. + unsafe { + libc::flock(self.directory.as_raw_fd(), libc::LOCK_UN); + } + } +} + +/// Try to take the lock without waiting. +/// +/// A second open of the same directory is a different open file +/// description, so the lock conflicts with a holder in the same process +/// too - which is what makes the runner's own jobs exclude each other. +fn try_lock(directory: &File) -> io::Result { + // SAFETY: the descriptor is open and owned for the duration of the call. + let result = unsafe { libc::flock(directory.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) }; + if result == 0 { + return Ok(true); + } + let error = io::Error::last_os_error(); + if error.raw_os_error() == Some(libc::EWOULDBLOCK) { + return Ok(false); + } + Err(error) +} + +/// The host directories a job shares with other jobs. +/// +/// Only writable bind mounts of a host path are shared: a read-only mount +/// cannot be written to, a tmpfs lives inside the container, a named +/// volume is managed by the container daemon rather than by a host path +/// the runner can lock, and a unix socket is not a directory at all. +pub fn shared_sources( + image: &str, + volumes: &[VolumeSpec], + cache_root: &str, + docker_socket: &str, +) -> Vec { + let mut sources: Vec = volumes + .iter() + .filter(|volume| !volume.read_only && volume.resolved_type() == VolumeType::Bind) + .filter_map(|volume| volume.expanded_source(image, cache_root, docker_socket)) + // The daemon socket is not a lockable directory: locking it would + // try to create it, and it is shared by every job on the runner. + .filter(|source| !is_socket_source(source)) + .map(PathBuf::from) + .filter(|path| path.is_absolute()) + .collect(); + sources.sort(); + sources.dedup(); + sources +} + +/// Lock every shared host directory the job mounts. +/// +/// Returns the locks to hold for the job's lifetime and the volumes the +/// job's container may mount. +/// +/// A directory that stays busy for the whole wait is a conflict the runner +/// cannot resolve: the job is refused rather than run against a directory +/// another job is writing to. A directory the runner cannot lock at all is +/// refused the same way - a mount the runner cannot protect is a mount it +/// cannot promise, and mounting it unshared is the corruption this lock +/// exists to prevent (GF-10, 2026-11-05: the mount used to be dropped, so +/// the job quietly ran with a contract the pipeline never declared). +pub async fn lock_job_mounts( + image: &str, + volumes: &[VolumeSpec], + cache_root: &str, + docker_socket: &str, + wait: Duration, +) -> Result<(Vec, Vec), MountLockError> { + let deadline = Instant::now() + wait; + let mut locks = Vec::new(); + for source in shared_sources(image, volumes, cache_root, docker_socket) { + let remaining = deadline.saturating_duration_since(Instant::now()); + match MountLock::acquire(&source, remaining).await { + Ok(lock) => locks.push(lock), + Err(error @ MountLockError::Unavailable { .. }) => { + tracing::warn!( + path = %source.display(), + %error, + "shared mount is not lockable; the job is refused" + ); + return Err(error); + } + Err(error @ MountLockError::Busy { .. }) => return Err(error), + } + } + Ok((locks, volumes.to_vec())) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::time::Duration; + + fn bind(source: &str) -> VolumeSpec { + VolumeSpec { + volume_type: Some(VolumeType::Bind), + source: Some(source.to_string()), + destination: "/cargo-target".to_string(), + read_only: false, + size: None, + } + } + + #[tokio::test] + async fn a_second_holder_waits_for_the_first() { + let root = tempfile::tempdir().expect("a scratch directory"); + let shared = root.path().join("shared"); + let first = MountLock::acquire(&shared, Duration::from_secs(1)) + .await + .expect("the first holder takes the lock"); + assert_eq!(first.path(), shared); + + let second = MountLock::acquire(&shared, Duration::from_millis(200)).await; + assert!( + matches!(second, Err(MountLockError::Busy { .. })), + "a second holder must be refused, got {second:?}" + ); + } + + #[tokio::test] + async fn the_lock_is_released_when_the_holder_drops_it() { + let root = tempfile::tempdir().expect("a scratch directory"); + let shared = root.path().join("shared"); + let first = MountLock::acquire(&shared, Duration::from_secs(1)) + .await + .expect("the first holder takes the lock"); + drop(first); + let second = MountLock::acquire(&shared, Duration::from_secs(1)) + .await + .expect("the lock is free once the holder drops it"); + assert_eq!(second.path(), shared); + } + + #[tokio::test] + async fn different_directories_do_not_block_each_other() { + let root = tempfile::tempdir().expect("a scratch directory"); + let one = root.path().join("one"); + let two = root.path().join("two"); + let held = MountLock::acquire(&one, Duration::from_secs(1)) + .await + .expect("the first holder takes the lock"); + let other = MountLock::acquire(&two, Duration::from_millis(200)).await; + assert!(other.is_ok(), "a different directory must not be refused"); + drop(held); + } + + #[test] + fn only_writable_bind_mounts_are_shared() { + let root = tempfile::tempdir().expect("a scratch directory"); + let writable = root.path().join("writable"); + let read_only = root.path().join("read-only"); + let mut read_only_spec = bind(read_only.to_str().expect("a utf-8 path")); + read_only_spec.read_only = true; + let tmpfs = VolumeSpec { + volume_type: Some(VolumeType::Tmpfs), + source: None, + destination: "/scratch".to_string(), + read_only: false, + size: None, + }; + let relative = bind("relative/path"); + + let sources = shared_sources( + "backend-ci-rust:1", + &[ + bind(writable.to_str().expect("a utf-8 path")), + read_only_spec, + tmpfs, + relative, + ], + "/var/cache/gitforge", + "/var/run/docker.sock", + ); + assert_eq!(sources, vec![writable]); + } + + #[tokio::test] + async fn a_busy_mount_refuses_the_job() { + let root = tempfile::tempdir().expect("a scratch directory"); + let shared = root.path().join("shared"); + let held = MountLock::acquire(&shared, Duration::from_secs(1)) + .await + .expect("the first holder takes the lock"); + let outcome = lock_job_mounts( + "backend-ci-rust:1", + &[bind(shared.to_str().expect("a utf-8 path"))], + "/var/cache/gitforge", + "/var/run/docker.sock", + Duration::from_millis(200), + ) + .await; + assert!( + matches!(outcome, Err(MountLockError::Busy { .. })), + "a busy mount must refuse the job, got {outcome:?}" + ); + drop(held); + } + + #[tokio::test] + async fn a_mount_that_cannot_be_locked_refuses_the_job() { + let root = tempfile::tempdir().expect("a scratch directory"); + // A regular file cannot be a parent directory, so the mount source + // can neither be created nor opened. + let blocker = root.path().join("blocker"); + fs::write(&blocker, b"not a directory").expect("a scratch file"); + let unusable = blocker.join("cache"); + let volume = bind(unusable.to_str().expect("a utf-8 path")); + + let outcome = lock_job_mounts( + "backend-ci-rust:1", + &[volume], + "/var/cache/gitforge", + "/var/run/docker.sock", + Duration::from_millis(200), + ) + .await; + assert!( + matches!(outcome, Err(MountLockError::Unavailable { .. })), + "a mount the runner cannot lock must refuse the job, got {outcome:?}" + ); + } +} diff --git a/crates/gitforge-sandbox/src/docker.rs b/crates/gitforge-sandbox/src/docker.rs index 6b06b292..1d3bb5ad 100644 --- a/crates/gitforge-sandbox/src/docker.rs +++ b/crates/gitforge-sandbox/src/docker.rs @@ -12,7 +12,10 @@ use bollard::query_parameters::{ use bollard::Docker; use bollard::API_DEFAULT_VERSION; use futures_util::StreamExt; -use gitforge_common::{Error, JobId, Result, VolumeSpec, VolumeType}; +use gitforge_common::{ + cache_root, is_socket_source, Error, JobId, Result, VolumeSpec, VolumeType, + DEFAULT_DOCKER_SOCKET, +}; use std::collections::HashMap; use std::path::Path; use std::sync::Arc; @@ -21,6 +24,19 @@ use tokio::time::{sleep, timeout, Duration}; /// Docker's CFS bandwidth period, in microseconds. const CPU_PERIOD_MICROS: i64 = 100_000; +/// The Docker client timeout, in seconds. +/// +/// bollard's default is 120 seconds. An operator can raise it for a daemon +/// on slow storage, where a client timeout failed jobs before any +/// acquisition window the runner granted could elapse. +pub(crate) fn docker_client_timeout() -> u64 { + std::env::var("GITFORGE_DOCKER_TIMEOUT_SECS") + .ok() + .and_then(|value| value.parse::().ok()) + .filter(|secs| *secs > 0) + .unwrap_or(120) // bollard's DEFAULT_TIMEOUT +} + /// Convert a requested core count to a safe CFS quota, bounded by the /// host's available parallelism. `cpu_ms` is a runtime budget and must not /// be used as a per-period quota. @@ -142,6 +158,12 @@ pub struct DockerSandbox { default_limits: SandboxLimits, /// If true, this is a stub sandbox for testing and should not be used in production is_stub: bool, + /// The daemon socket this sandbox talks to, as a host path. + /// + /// A `{docker_socket}` mount binds this exact path, so the socket a + /// job mounts is the socket the sandbox talks to - not whatever the + /// environment says at container-creation time. + docker_socket: String, } impl DockerSandbox { @@ -237,18 +259,12 @@ impl DockerSandbox { // granted could elapse. (Storage has since moved to overlay2, but // the rule stands: keep bollard's default unless the operator // raises it.) - let client_timeout = std::env::var("GITFORGE_DOCKER_TIMEOUT_SECS") - .ok() - .and_then(|value| value.parse::().ok()) - .filter(|secs| *secs > 0) - .unwrap_or(120); // bollard's DEFAULT_TIMEOUT - // Same socket selection as `connect_with_local_defaults`: an explicit - // unix:// DOCKER_HOST wins, otherwise the well-known socket path - // (bollard's DEFAULT_SOCKET; the constant is not re-exported). - let socket = std::env::var("DOCKER_HOST") - .ok() - .filter(|host| host.starts_with("unix://")) - .unwrap_or_else(|| "unix:///var/run/docker.sock".to_owned()); + let client_timeout = docker_client_timeout(); + // One definition for the whole platform: the sandbox connects to + // this socket, the reconciler reconciles this daemon's + // containers, and a `{docker_socket}` mount binds this socket + // into the job's container. + let socket = gitforge_common::docker_socket(); let docker = Docker::connect_with_unix(&socket, client_timeout, API_DEFAULT_VERSION) .map_err(|e| Error::sandbox(format!("failed to connect to Docker: {e}")))?; @@ -264,6 +280,7 @@ impl DockerSandbox { docker: Some(docker), default_limits: SandboxLimits::default(), is_stub: false, + docker_socket: socket, }) } @@ -331,6 +348,7 @@ impl DockerSandbox { docker: None, default_limits: SandboxLimits::default(), is_stub: true, + docker_socket: DEFAULT_DOCKER_SOCKET.to_owned(), } } @@ -345,6 +363,7 @@ impl DockerSandbox { docker: None, default_limits: limits, is_stub: true, + docker_socket: DEFAULT_DOCKER_SOCKET.to_owned(), } } @@ -426,29 +445,150 @@ impl DockerSandbox { } Ok(()) } + /// Create a container, stamping `extra_labels` onto it in addition to + /// the managed and job-id labels every container carries. + async fn create_with_extra_labels( + &self, + job_id: JobId, + image: &str, + limits: SandboxLimits, + volumes: &[VolumeSpec], + extra_labels: &[(String, String)], + ) -> Result { + if let Some(ref docker) = self.docker { + // Ensure image is available + self.ensure_image(image).await?; + self.remove_job_containers(job_id).await?; + + let container_name = Self::container_name(job_id); + let job_label = job_id.to_string(); + let mut labels = HashMap::from([ + ("com.gitforce.managed".to_owned(), "true".to_owned()), + ("com.gitforce.job_id".to_owned(), job_label.clone()), + ]); + // Pool containers carry the pool's ownership labels on top of + // the managed/job-id pair, so the pool can tell its own + // containers from job containers without guessing. + for (key, value) in extra_labels { + labels.insert(key.clone(), value.clone()); + } + + // A mount the runner cannot prepare is a contract it cannot + // keep: the pipeline declared this directory and the job + // needs it. Failing here is what makes the reason visible - + // the runner reports it as an infrastructure failure, so it + // reaches the job's own result instead of a log line (GF-10, + // 2026-11-05: the whole mount set used to be dropped, and a + // job whose cache had vanished just ran cold and timed out). + let volume_mounts = + build_volume_mounts(volumes, image, &cache_root(), &self.docker_socket)?; + + // Build host config with resource limits + let host_config = HostConfig { + memory: Some((limits.memory_mb * 1024 * 1024) as i64), + cpu_period: Some(CPU_PERIOD_MICROS), + cpu_quota: Some(host_cpu_quota_micros(limits.cpu_cores)), + network_mode: if limits.network { + None + } else { + Some("none".to_string()) + }, + binds: if volume_mounts.binds.is_empty() { + None + } else { + Some(volume_mounts.binds) + }, + mounts: if volume_mounts.mounts.is_empty() { + None + } else { + Some(volume_mounts.mounts) + }, + // A container that mounts the daemon socket cannot keep + // SELinux labeling: the socket's label does not match the + // container's, and an enforcing host refuses the + // connection. + security_opt: if volume_mounts.disable_selinux { + Some(vec!["label=disable".to_string()]) + } else { + None + }, + ..Default::default() + }; + + // Create container + let config = ContainerCreateBody { + image: Some(image.to_owned()), + // Keep the container alive for the whole job. A 3600s keepalive + // capped every job at one hour: when sleep exited, Docker + // stopped the container and SIGKILLed any running exec + // (exit 137 at exactly 3600s, OOMKilled=false). The container + // is removed per-job in destroy(), so this only bounds a + // leaked container's lifetime. + cmd: Some(vec![ + "sleep".to_owned(), + "86400".to_owned(), // 24h; job deadlines clamp to 24h + ]), + host_config: Some(host_config), + labels: Some(labels), + ..Default::default() + }; + + let options = CreateContainerOptions { + name: Some(container_name.clone()), + platform: String::new(), + }; + + let response = docker + .create_container(Some(options), config) + .await + .map_err(|e| Error::sandbox(format!("failed to create container: {e}")))?; + + // Start container + docker + .start_container(&response.id, None) + .await + .map_err(|e| Error::sandbox(format!("failed to start container: {e}")))?; + + tracing::info!("Created container {} for job {}", response.id, job_id); + + Ok(SandboxInstance { + container_id: response.id, + job_id, + workspace_path: None, + }) + } else { + // Stub mode - no Docker available + Ok(SandboxInstance { + container_id: format!("gitforce-job-{job_id}"), + job_id, + workspace_path: None, + }) + } + } + + /// Create a container that belongs to a warm pool. + /// + /// A pooled container is created with a placeholder job id (it is not + /// a job) and with the pool's ownership labels, so a later runner + /// process can recognise the containers this pool left behind and + /// reclaim them (GF-17). + pub async fn create_pooled( + &self, + identity: &crate::pool::PoolIdentity, + image: &str, + limits: SandboxLimits, + ) -> Result { + let job_id = JobId::new(); + let extra: Vec<(String, String)> = crate::pool::pool_labels(identity).into_iter().collect(); + self.create_with_extra_labels(job_id, image, limits, &[], &extra) + .await + } } // --------------------------------------------------------------------------- // Volume mounts // --------------------------------------------------------------------------- -/// Where the runner keeps job caches when the pipeline does not name an -/// absolute location. Overridable with `GITFORGE_CACHE_ROOT`; the default -/// matches the usual system cache path. -const DEFAULT_CACHE_ROOT: &str = "/var/cache/gitforge"; - -/// The runner's cache root, read from the environment ONCE at the edge -/// (where a container is created) and passed down as a value. A decision -/// function that reads process-global state internally inherits whatever -/// any other thread is doing to it at that instant. -fn cache_root() -> String { - std::env::var("GITFORGE_CACHE_ROOT") - .ok() - .map(|value| value.trim().to_string()) - .filter(|value| !value.is_empty()) - .unwrap_or_else(|| DEFAULT_CACHE_ROOT.to_string()) -} - /// Parse a human-readable size (`4G`, `512M`, `1024K`) into bytes. fn parse_size_bytes(raw: &str) -> Option { let trimmed = raw.trim(); @@ -469,18 +609,44 @@ fn parse_size_bytes(raw: &str) -> Option { value.checked_mul(multiplier) } +/// The container mounts a job asked for. +#[derive(Debug)] +struct VolumeMounts { + /// Bind sources in `HostConfig.binds` form. + binds: Vec, + /// Structured mounts (tmpfs, named volumes). + mounts: Vec, + /// The container mounts a unix socket, so SELinux labeling has to be + /// disabled for it. + /// + /// The daemon socket's own label does not match the label a container + /// gets by default, so an enforcing host refuses the connection + /// (measured on the Fedora runner, 2026-11-06: the connection is + /// refused without the opt). Relabeling the socket instead - `:z` - + /// rewrites the label of a host file the runner does not own, so the + /// bind goes in unlabeled and the container runs with labeling + /// disabled. + disable_selinux: bool, +} + /// Turn the requested volumes into docker mounts. /// /// Binds become `HostConfig.binds` entries so they inherit the daemon's /// SELinux relabel behaviour (the workspace mount already relies on `:z`); /// tmpfs and named volumes become structured mounts. +/// +/// A bind whose source is a unix socket is the one exception to the +/// relabel: the source is the daemon's own socket, not a host directory +/// the runner owns. fn build_volume_mounts( volumes: &[VolumeSpec], image: &str, cache_root: &str, -) -> Result<(Vec, Vec)> { + docker_socket: &str, +) -> Result { let mut binds = Vec::new(); let mut mounts = Vec::new(); + let mut disable_selinux = false; for volume in volumes { volume @@ -488,27 +654,39 @@ fn build_volume_mounts( .map_err(|reason| Error::sandbox(format!("invalid volume mount: {reason}")))?; match volume.resolved_type() { VolumeType::Bind => { - let source = volume.expanded_source(image, cache_root).ok_or_else(|| { - Error::sandbox(format!( - "volume mount {} requires a source", - volume.destination - )) - })?; + let source = volume + .expanded_source(image, cache_root, docker_socket) + .ok_or_else(|| { + Error::sandbox(format!( + "volume mount {} requires a source", + volume.destination + )) + })?; if !source.starts_with('/') { return Err(Error::sandbox(format!( "volume mount source must be an absolute path: {source}" ))); } - // The daemon creates a missing bind source as root, which - // the job's user could not write to. Create it here, as the - // runner's own user, so a cache mount is writable. - if let Err(error) = std::fs::create_dir_all(&source) { - return Err(Error::sandbox(format!( - "could not prepare volume mount source {source}: {error}" - ))); - } let access = if volume.read_only { "ro" } else { "rw" }; - binds.push(format!("{source}:{}:{access},z", volume.destination)); + if is_socket_source(&source) { + // A socket is not a directory: the daemon created it, + // and creating it here would fail. The bind goes in + // without a relabel - `:z` would rewrite the label of + // the host's own socket file - and the container runs + // with SELinux labeling disabled instead. + binds.push(format!("{source}:{}:{access}", volume.destination)); + disable_selinux = true; + } else { + // The daemon creates a missing bind source as root, which + // the job's user could not write to. Create it here, as the + // runner's own user, so a cache mount is writable. + if let Err(error) = std::fs::create_dir_all(&source) { + return Err(Error::sandbox(format!( + "could not prepare volume mount source {source}: {error}" + ))); + } + binds.push(format!("{source}:{}:{access},z", volume.destination)); + } } VolumeType::Tmpfs => { let tmpfs_options = @@ -552,7 +730,11 @@ fn build_volume_mounts( } } - Ok((binds, mounts)) + Ok(VolumeMounts { + binds, + mounts, + disable_selinux, + }) } /// Sandbox trait @@ -660,93 +842,8 @@ impl Sandbox for DockerSandbox { limits: SandboxLimits, volumes: &[VolumeSpec], ) -> Result { - if let Some(ref docker) = self.docker { - // Ensure image is available - self.ensure_image(image).await?; - self.remove_job_containers(job_id).await?; - - let container_name = Self::container_name(job_id); - let job_label = job_id.to_string(); - let labels = HashMap::from([ - ("com.gitforce.managed".to_owned(), "true".to_owned()), - ("com.gitforce.job_id".to_owned(), job_label.clone()), - ]); - - let (binds, mounts) = match build_volume_mounts(volumes, image, &cache_root()) { - Ok(mounts) => mounts, - Err(error) => { - // A cache mount that cannot be prepared degrades to a - // cold build: the job still runs, it just does not - // reuse the previous run's artifacts. Failing the job - // here would turn a missing cache directory into a red - // pipeline for a change that is only an optimisation. - tracing::warn!( - %error, - "volume mounts unavailable; continuing without them" - ); - (Vec::new(), Vec::new()) - } - }; - - // Build host config with resource limits - let host_config = HostConfig { - memory: Some((limits.memory_mb * 1024 * 1024) as i64), - cpu_period: Some(CPU_PERIOD_MICROS), - cpu_quota: Some(host_cpu_quota_micros(limits.cpu_cores)), - network_mode: if limits.network { - None - } else { - Some("none".to_string()) - }, - binds: if binds.is_empty() { None } else { Some(binds) }, - mounts: if mounts.is_empty() { - None - } else { - Some(mounts) - }, - ..Default::default() - }; - - // Create container - let config = ContainerCreateBody { - image: Some(image.to_owned()), - cmd: Some(vec!["sleep".to_owned(), "3600".to_owned()]), // Keep container alive - host_config: Some(host_config), - labels: Some(labels), - ..Default::default() - }; - - let options = CreateContainerOptions { - name: Some(container_name.clone()), - platform: String::new(), - }; - - let response = docker - .create_container(Some(options), config) - .await - .map_err(|e| Error::sandbox(format!("failed to create container: {e}")))?; - - // Start container - docker - .start_container(&response.id, None) - .await - .map_err(|e| Error::sandbox(format!("failed to start container: {e}")))?; - - tracing::info!("Created container {} for job {}", response.id, job_id); - - Ok(SandboxInstance { - container_id: response.id, - job_id, - workspace_path: None, - }) - } else { - // Stub mode - no Docker available - Ok(SandboxInstance { - container_id: format!("gitforce-job-{job_id}"), - job_id, - workspace_path: None, - }) - } + self.create_with_extra_labels(job_id, image, limits, volumes, &[]) + .await } async fn create_with_workspace( @@ -776,19 +873,10 @@ impl Sandbox for DockerSandbox { ("com.gitforce.managed".to_owned(), "true".to_owned()), ("com.gitforce.job_id".to_owned(), job_label), ]); - let (volume_binds, volume_mounts) = - match build_volume_mounts(volumes, image, &cache_root()) { - Ok(mounts) => mounts, - Err(error) => { - // See `create`: an unusable cache mount degrades to a - // cold build instead of failing the job. - tracing::warn!( - %error, - "volume mounts unavailable; continuing without them" - ); - (Vec::new(), Vec::new()) - } - }; + // See `create`: a mount the runner cannot prepare fails the + // job rather than vanishing from the contract. + let volume_mounts = + build_volume_mounts(volumes, image, &cache_root(), &self.docker_socket)?; let host_config = HostConfig { memory: Some((limits.memory_mb * 1024 * 1024) as i64), cpu_period: Some(CPU_PERIOD_MICROS), @@ -805,19 +893,33 @@ impl Sandbox for DockerSandbox { // checkout and can leave one container unable to see files. binds: Some({ let mut binds = vec![format!("{}:/workspace:z", workspace_path)]; - binds.extend(volume_binds); + binds.extend(volume_mounts.binds); binds }), - mounts: if volume_mounts.is_empty() { + mounts: if volume_mounts.mounts.is_empty() { None } else { - Some(volume_mounts) + Some(volume_mounts.mounts) + }, + // A container that mounts the daemon socket cannot keep + // SELinux labeling: the socket's label does not match the + // container's, and an enforcing host refuses the + // connection. + security_opt: if volume_mounts.disable_selinux { + Some(vec!["label=disable".to_string()]) + } else { + None }, ..Default::default() }; let config = ContainerCreateBody { image: Some(image.to_owned()), - cmd: Some(vec!["sleep".to_owned(), "3600".to_owned()]), + // Same keepalive-lifetime rationale as the plain sandbox: + // a 3600s sleep SIGKILLed running execs at exactly 1h. + cmd: Some(vec![ + "sleep".to_owned(), + "86400".to_owned(), // 24h; job deadlines clamp to 24h + ]), working_dir: Some("/workspace".to_owned()), // The workspace bind-mount keeps the host checkout's uid (the // job user inside the image is usually root), and git's @@ -876,11 +978,17 @@ impl Sandbox for DockerSandbox { // Create exec instance. The workspace git trust vars are always // layered in for workspace execs; caller env (`KEY=VALUE` pairs) // rides alongside them and never displaces them. - let exec_env = compose_exec_env(instance.workspace_path.is_some(), env); + let has_workspace = instance.workspace_path.is_some(); + let exec_env = compose_exec_env(has_workspace, env); + let exec_command: Vec<&str> = if has_workspace { + wrap_with_git_config_bootstrap(command) + } else { + command.to_vec() + }; let config = CreateExecOptions { attach_stdout: Some(true), attach_stderr: Some(true), - cmd: Some(command.to_vec()), + cmd: Some(exec_command), // `create_with_workspace` mounts the host checkout at this // fixed container path. Docker exec does not inherit the // container's configured working directory, so set it here @@ -1024,14 +1132,44 @@ impl Sandbox for DockerSandbox { // the runner user on the host can delete the artifact files without // requiring privileged escalation. if let Some(ref workspace) = instance.workspace_path { - if let Err(e) = cleanup_workspace(docker, &instance.container_id, workspace).await { - tracing::warn!( - "workspace ownership cleanup failed for {}: {} \ - (artifact files may require privileged deletion)", - workspace, - e - ); - // Proceed to container teardown even if chown fails + // The ownership cleanup is a convenience, not a precondition + // of the teardown: the CI service removes a workspace it + // cannot delete with a container-assisted cleanup. The + // teardown, on the other hand, MUST happen - a cancelled + // teardown leaves the container running forever. + // + // The recursive chown walks the whole workspace tree. On a + // network filesystem, over a vendored source tree, that walk + // takes minutes: the runner's teardown budget expired first, + // and the container was never stopped or removed (measured + // 2026-11-05: nine leaked job containers, each one a "timed + // out cleaning up sandbox" warning - and the leak blocks + // every release cutover, whose drain gate counts live job + // containers). So the cleanup runs under its own budget and + // the container is always torn down. + match bounded_cleanup( + cleanup_workspace(docker, &instance.container_id, workspace), + WORKSPACE_CLEANUP_BUDGET, + ) + .await + { + WorkspaceCleanup::Completed => {} + WorkspaceCleanup::Failed(error) => { + tracing::warn!( + "workspace ownership cleanup failed for {}: {} \ + (artifact files may require privileged deletion)", + workspace, + error + ); + } + WorkspaceCleanup::TimedOut => { + tracing::warn!( + "workspace ownership cleanup exceeded {}s for {} \ + (artifact files may require privileged deletion)", + WORKSPACE_CLEANUP_BUDGET.as_secs(), + workspace + ); + } } } @@ -1113,17 +1251,43 @@ fn resolve_runner_uid_gid() -> Result<(u32, u32)> { /// Git environment for workspace execs. The checkout is created on the host /// by the runner user but execs run as the container's root, so git refuses -/// every operation with "detected dubious ownership in repository at -/// '/workspace'" — which silently turned any pipeline step guarding with -/// `|| true` (tag probes, version detection) into a no-op. Tell git this -/// one mount is trusted; scoped to `/workspace` rather than `*` so it -/// cannot leak meaning into repositories steps create themselves. +/// every operation with "detected dubious ownership" — which silently turned +/// any pipeline step guarding with `|| true` (tag probes, version detection) +/// into a no-op. Trust every repository the exec can touch. +/// +/// This must stay `*`, not `/workspace`: repositories are not only mounted +/// at the mount root, they are also created inside the workspace by steps +/// and test fixtures (e.g. `git init seed && git config user.email ...` +/// under `/workspace/target/...`). `safe.directory` matches the exact path, +/// never a prefix, so a `/workspace` entry leaves every such sub-repository +/// rejected — observed on fedora-docker where the workspace rides an sshfs +/// mount mapping all files to uid 1000 while the container runs as root: +/// `git config` in a freshly-created fixture repo failed with "not in a git +/// directory" and dropped 10 ci unit tests (run f2a1a2d9, 2026-11-04). +/// Within a job sandbox whose only host surface is the workspace mount, +/// blanket trust costs nothing real: the mount is per-run scratch data. fn workspace_exec_git_env() -> Vec<&'static str> { - vec![ - "GIT_CONFIG_COUNT=1", - "GIT_CONFIG_KEY_0=safe.directory", - "GIT_CONFIG_VALUE_0=/workspace", - ] + vec!["GIT_CONFIG_GLOBAL=/tmp/.gitforge-gitconfig"] +} + +/// Git protects the blanket `safe.directory = *` wildcard: it is honored +/// only from system or global configuration, never from command-scoped +/// `GIT_CONFIG_*` env entries (an exact path works there, but fixtures and +/// steps create repositories at unpredictable paths under the mount). +/// `GIT_CONFIG_GLOBAL` however can point anywhere via env, so each exec +/// materializes a tiny global config granting the wildcard before it runs. +const GIT_GLOBAL_CONFIG_BOOTSTRAP: &str = "mkdir -p /tmp && \ +printf '[safe]\n\tdirectory = *\n' > /tmp/.gitforge-gitconfig; exec \"$@\""; + +/// Wrap `command` so the exec first writes the global git config that the +/// `GIT_CONFIG_GLOBAL` env entry (see `workspace_exec_git_env`) points at, +/// then execs the original command. `sh` is present in every job image this +/// runner serves; the write is idempotent and costs one fork per step. +fn wrap_with_git_config_bootstrap<'a>(command: &[&'a str]) -> Vec<&'a str> { + let mut wrapped: Vec<&str> = Vec::with_capacity(command.len() + 4); + wrapped.extend_from_slice(&["sh", "-ec", GIT_GLOBAL_CONFIG_BOOTSTRAP, "--"]); + wrapped.extend_from_slice(command); + wrapped } /// Container-level variant of `workspace_exec_git_env` (owned strings for @@ -1152,6 +1316,38 @@ fn workspace_git_env() -> Vec { /// call; if the stream is still running the exit code may not yet be visible. /// - Failures are warned but never block container teardown (caller handles /// fail-open semantics). +/// - The cleanup runs under a budget of its own (`WORKSPACE_CLEANUP_BUDGET`) +/// so a tree that takes minutes to walk can never cancel the teardown. +const WORKSPACE_CLEANUP_BUDGET: Duration = Duration::from_secs(15); + +/// Outcome of a bounded workspace ownership cleanup. +#[derive(Debug, PartialEq, Eq)] +enum WorkspaceCleanup { + /// The cleanup finished: the tree belongs to the runner user. + Completed, + /// The cleanup failed; the reason is reported. + Failed(String), + /// The cleanup exceeded its budget and was abandoned. + TimedOut, +} + +/// Run the workspace ownership cleanup under its own budget. +/// +/// The cleanup is best-effort: whatever it does or does not achieve, the +/// caller's container teardown continues. A cleanup that never returns must +/// not be able to swallow the teardown that follows it - that is what left +/// nine job containers running on the NAS instance (2026-11-05). +async fn bounded_cleanup(cleanup: F, budget: Duration) -> WorkspaceCleanup +where + F: std::future::Future>, +{ + match timeout(budget, cleanup).await { + Ok(Ok(())) => WorkspaceCleanup::Completed, + Ok(Err(error)) => WorkspaceCleanup::Failed(error.to_string()), + Err(_) => WorkspaceCleanup::TimedOut, + } +} + async fn cleanup_workspace(docker: &Docker, container_id: &str, workspace: &str) -> Result<()> { let (uid, gid) = resolve_runner_uid_gid()?; @@ -1239,15 +1435,10 @@ mod tests { #[test] fn test_workspace_git_env_grants_safe_directory_via_env_config() { let env = workspace_git_env(); - // git's env-based config protocol: COUNT declares the entries, each - // KEY_n/VALUE_n pair supplies one. The key must be the safe.directory - // setting and the value must be the mounted workspace path — this trio - // keeps host-owned workspace mounts usable without trusting unrelated - // repositories. - assert_eq!(env.len(), 3); - assert_eq!(env[0], "GIT_CONFIG_COUNT=1"); - assert_eq!(env[1], "GIT_CONFIG_KEY_0=safe.directory"); - assert_eq!(env[2], "GIT_CONFIG_VALUE_0=/workspace"); + // The blanket `safe.directory = *` wildcard is honored only from + // global/system config, so the exec exports GIT_CONFIG_GLOBAL aimed + // at the file the bootstrap wrapper materializes. + assert_eq!(env, &["GIT_CONFIG_GLOBAL=/tmp/.gitforge-gitconfig"]); } /// Pipeline env reaches the exec as `KEY=VALUE` pairs layered after the @@ -1257,10 +1448,10 @@ mod tests { let caller = vec!["PYTHONPATH=src".to_string(), "DI_ENV=test".to_string()]; let with_workspace = compose_exec_env(true, &caller); - assert_eq!(with_workspace.len(), 5); - assert_eq!(with_workspace[0], "GIT_CONFIG_COUNT=1"); - assert_eq!(with_workspace[3], "PYTHONPATH=src"); - assert_eq!(with_workspace[4], "DI_ENV=test"); + assert_eq!(with_workspace.len(), 3); + assert!(with_workspace[0].starts_with("GIT_CONFIG_GLOBAL=")); + assert_eq!(with_workspace[1], "PYTHONPATH=src"); + assert_eq!(with_workspace[2], "DI_ENV=test"); let without_workspace = compose_exec_env(false, &caller); assert_eq!(without_workspace, caller); @@ -1347,20 +1538,70 @@ mod tests { size: None, }]; - let (binds, mounts) = - build_volume_mounts(&volumes, "dsc-ci-rust:10", &root.display().to_string()).unwrap(); - assert!(mounts.is_empty()); - assert_eq!(binds.len(), 1); + let volume_mounts = build_volume_mounts( + &volumes, + "dsc-ci-rust:10", + &root.display().to_string(), + "/var/run/docker.sock", + ) + .unwrap(); + assert!(volume_mounts.mounts.is_empty()); + assert_eq!(volume_mounts.binds.len(), 1); assert_eq!( - binds[0], + volume_mounts.binds[0], format!("{}/cargo/dsc-ci-rust-10:/cargo-target:rw,z", root.display()) ); + assert!( + !volume_mounts.disable_selinux, + "a directory mount keeps the container's own labeling" + ); // The mount source is created by the runner, as the runner's own // user, so the job's user can write into it. assert!(root.join("cargo/dsc-ci-rust-10").is_dir()); let _ = std::fs::remove_dir_all(&root); } + #[cfg(unix)] + #[test] + fn test_build_volume_mounts_binds_the_daemon_socket_without_a_relabel() { + // The docker-integration job declares `source: "{docker_socket}"`. + // Before the placeholder was expanded, the runner tried to bind the + // literal string and the job died with "volume mount source must be + // an absolute path: {docker_socket}" (CI run f2292b0e, 2026-11-06). + let root = + std::env::temp_dir().join(format!("gitforge-socket-test-{}", uuid::Uuid::new_v4())); + std::fs::create_dir_all(&root).expect("a scratch directory"); + let socket = root.join("docker.sock"); + let _listener = std::os::unix::net::UnixListener::bind(&socket).expect("a unix socket"); + let volumes = vec![VolumeSpec { + volume_type: Some(VolumeType::Bind), + source: Some("{docker_socket}".to_string()), + destination: "/var/run/docker.sock".to_string(), + read_only: true, + size: None, + }]; + + let volume_mounts = build_volume_mounts( + &volumes, + "dsc-ci-rust:16", + "/var/cache/gitforge", + &socket.to_string_lossy(), + ) + .expect("a socket mount is a mount the runner can honour"); + + assert_eq!( + volume_mounts.binds, + vec![format!("{}:/var/run/docker.sock:ro", socket.display())], + "the socket bind carries no relabel: `:z` would rewrite the host socket's label" + ); + assert!( + volume_mounts.disable_selinux, + "a container that mounts the daemon socket cannot keep SELinux labeling" + ); + + let _ = std::fs::remove_dir_all(&root); + } + #[test] fn test_build_volume_mounts_read_only_bind() { let root = @@ -1373,9 +1614,17 @@ mod tests { size: None, }]; - let (binds, _) = - build_volume_mounts(&volumes, "dsc-ci-rust:10", &root.display().to_string()).unwrap(); - assert_eq!(binds[0], format!("{}:/cache:ro,z", root.display())); + let volume_mounts = build_volume_mounts( + &volumes, + "dsc-ci-rust:10", + &root.display().to_string(), + "/var/run/docker.sock", + ) + .unwrap(); + assert_eq!( + volume_mounts.binds[0], + format!("{}:/cache:ro,z", root.display()) + ); let _ = std::fs::remove_dir_all(&root); } @@ -1389,13 +1638,18 @@ mod tests { size: Some("4G".to_string()), }]; - let (binds, mounts) = - build_volume_mounts(&volumes, "dsc-ci-rust:10", "/var/cache/gitforge").unwrap(); - assert!(binds.is_empty()); - assert_eq!(mounts.len(), 1); - assert_eq!(mounts[0].target.as_deref(), Some("/scratch")); + let volume_mounts = build_volume_mounts( + &volumes, + "dsc-ci-rust:10", + "/var/cache/gitforge", + "/var/run/docker.sock", + ) + .unwrap(); + assert!(volume_mounts.binds.is_empty()); + assert_eq!(volume_mounts.mounts.len(), 1); + assert_eq!(volume_mounts.mounts[0].target.as_deref(), Some("/scratch")); assert_eq!( - mounts[0] + volume_mounts.mounts[0] .tmpfs_options .as_ref() .and_then(|options| options.size_bytes), @@ -1413,7 +1667,43 @@ mod tests { size: None, }]; - assert!(build_volume_mounts(&volumes, "dsc-ci-rust:10", "/var/cache/gitforge").is_err()); + assert!(build_volume_mounts( + &volumes, + "dsc-ci-rust:10", + "/var/cache/gitforge", + "/var/run/docker.sock" + ) + .is_err()); + } + + #[test] + fn test_build_volume_mounts_reports_a_source_it_cannot_prepare() { + // A bind source under a regular file can never become a directory, + // so the runner cannot prepare it. That is an error the job has to + // report, not a mount to drop quietly: the job would otherwise run + // with a contract the pipeline never declared (GF-10, 2026-11-05). + let binary = std::env::current_exe().expect("the test binary's own path"); + let volumes = vec![VolumeSpec { + volume_type: Some(VolumeType::Bind), + source: Some(binary.join("cache").to_string_lossy().into_owned()), + destination: "/cargo-target".to_string(), + read_only: false, + size: None, + }]; + + let error = build_volume_mounts( + &volumes, + "dsc-ci-rust:10", + "/var/cache/gitforge", + "/var/run/docker.sock", + ) + .expect_err("a source the runner cannot prepare must be an error"); + assert!( + error + .to_string() + .contains("could not prepare volume mount source"), + "the error must name the mount it could not prepare, got {error}" + ); } #[test] @@ -2074,10 +2364,8 @@ mod tests { #[test] fn workspace_git_env_marks_the_mount_trusted() { let env = super::workspace_exec_git_env(); - assert!(env.contains(&"GIT_CONFIG_COUNT=1")); - assert!(env.contains(&"GIT_CONFIG_KEY_0=safe.directory")); - assert!(env.contains(&"GIT_CONFIG_VALUE_0=/workspace")); - assert_eq!(env.len(), 3); + assert_eq!(env.len(), 1); + assert!(env[0].starts_with("GIT_CONFIG_GLOBAL=")); } /// Verify that `create_with_workspace` called with no path also produces @@ -2359,4 +2647,105 @@ mod tests { "stub destroy must not fail even with workspace_path set" ); } + + /// The ownership cleanup must never be able to swallow the teardown that + /// follows it: a cleanup that never returns is abandoned and the caller + /// proceeds. The regression is the leaked job container (2026-11-05). + #[tokio::test] + async fn test_a_cleanup_that_never_returns_is_abandoned() { + let outcome = bounded_cleanup( + async { + sleep(Duration::from_secs(30)).await; + Ok(()) + }, + Duration::from_millis(50), + ) + .await; + assert_eq!(outcome, WorkspaceCleanup::TimedOut); + } + + /// A failed cleanup is reported, not propagated: the teardown continues. + #[tokio::test] + async fn test_a_failed_cleanup_is_reported() { + let outcome = bounded_cleanup( + async { Err(Error::sandbox("chown failed")) }, + Duration::from_secs(5), + ) + .await; + assert!(matches!(outcome, WorkspaceCleanup::Failed(_))); + } + + /// A cleanup that finishes in time is reported as completed. + #[tokio::test] + async fn test_a_completed_cleanup_is_reported_as_completed() { + let outcome = bounded_cleanup(async { Ok(()) }, Duration::from_secs(5)).await; + assert_eq!(outcome, WorkspaceCleanup::Completed); + } + + /// The backstop the runner uses when a teardown overruns its budget: + /// the container is removed by its job label. The label is the only + /// ownership evidence, and another job's container is never touched. + /// + /// This is the primitive that makes the leak class closeable. The + /// background reconciler refuses to remove a RUNNING container - its + /// policy keeps running containers eligible for nobody - so a teardown + /// that overran its budget could only be cleaned up by an explicit, + /// label-scoped removal (2026-11-05: eight leaked containers that no + /// other mechanism would ever reap). + #[tokio::test] + #[ignore] // requires a Docker daemon + async fn test_remove_job_containers_removes_only_that_jobs_containers() { + let sandbox = match DockerSandbox::connect_required().await { + Ok(sandbox) => sandbox, + Err(_) => return, + }; + let owner = JobId::new(); + let other = JobId::new(); + let image = "backend-ci-rust:1"; + let owned = match sandbox + .create(owner, image, SandboxLimits::default(), &[]) + .await + { + Ok(instance) => instance, + Err(_) => return, + }; + let foreign = match sandbox + .create(other, image, SandboxLimits::default(), &[]) + .await + { + Ok(instance) => instance, + Err(_) => { + let _ = sandbox.remove_job_containers(owner).await; + return; + } + }; + let docker = match sandbox.docker.as_ref() { + Some(docker) => docker, + None => return, + }; + assert!(docker + .inspect_container(&owned.container_id, None) + .await + .is_ok()); + assert!(docker + .inspect_container(&foreign.container_id, None) + .await + .is_ok()); + + sandbox + .remove_job_containers(owner) + .await + .expect("the label-scoped removal must succeed"); + + assert!(docker + .inspect_container(&owned.container_id, None) + .await + .is_err()); + assert!(docker + .inspect_container(&foreign.container_id, None) + .await + .is_ok()); + + let _ = sandbox.remove_job_containers(other).await; + } } diff --git a/crates/gitforge-sandbox/src/lib.rs b/crates/gitforge-sandbox/src/lib.rs index 23e5ac47..1f19c34f 100644 --- a/crates/gitforge-sandbox/src/lib.rs +++ b/crates/gitforge-sandbox/src/lib.rs @@ -5,6 +5,7 @@ pub mod active_jobs; pub mod docker; pub mod limits; +pub mod pool; pub mod reconciler; pub use active_jobs::{ActiveJobGuard, ActiveJobRegistry, ActiveJobSnapshot}; @@ -12,7 +13,12 @@ pub use docker::{ BackendHealth, DockerSandbox, OutputSink, OutputStream, Sandbox, SandboxInstance, StepResult, }; pub use limits::SandboxLimits; +pub use pool::{ + classify_pool_container, pool_labels, process_alive, reclaim_orphaned_pool_containers, + OwnerState, PoolDecision, PoolIdentity, PoolPolicy, PoolReclaimReport, +}; pub use reconciler::{ - classify, ContainerRecord, ContainerSource, Decision, DockerContainerSource, ReconcileReport, - Reconciler, ReconcilerPolicy, RemovalOutcome, + classify, ContainerRecord, ContainerSource, Decision, DockerContainerSource, + JobExistenceResolver, ReconcileReport, Reconciler, ReconcilerPolicy, RemovalOutcome, + UnknownJobExistence, }; diff --git a/crates/gitforge-sandbox/src/pool.rs b/crates/gitforge-sandbox/src/pool.rs new file mode 100644 index 00000000..28f0a2eb --- /dev/null +++ b/crates/gitforge-sandbox/src/pool.rs @@ -0,0 +1,657 @@ +//! Ownership and reclamation rules for the runner's warm container pool. +//! +//! The pool is a pre-warm cache of containers: the runner keeps a small +//! number of spare containers per image so a job does not pay container +//! creation latency. Those containers are created with a placeholder job id +//! that has no row in the store, and they are never handed a real job id +//! afterwards — Docker labels are immutable, so a container that leaves the +//! pool keeps the placeholder label for the rest of its life. +//! +//! That leaves the pool invisible to every other cleanup path in the tree: +//! `remove_job_containers` scopes by exact label equality with a real job +//! id, and the general reconciler retains every running container. When a +//! runner process dies or restarts, its in-memory pool map is lost while the +//! containers it created keep running — they leak, and nothing in the tree +//! can reach them (GF-17). +//! +//! This module is the pool's own reclamation contract: a pooled container +//! is owned by the pool, it is identified by the labels set at creation +//! time, and it is reclaimable only when every ownership fact is +//! established. The decision is a pure function so it can be tested +//! without a Docker daemon; the removal loop is a thin shell around it. + +use crate::reconciler::{ContainerRecord, ContainerSource, RemovalOutcome}; +use gitforge_common::{Error, Result}; +use std::collections::HashMap; +use std::time::Duration; + +/// Label marking a container as belonging to a warm pool. +pub const POOL_LABEL: &str = "com.gitforce.pool"; + +/// Label naming the pool owner that created the container. +pub const POOL_OWNER_LABEL: &str = "com.gitforce.pool.owner"; + +/// Label naming the pool generation. One generation per runner process, +/// so a container from a previous process is distinguishable from one the +/// current process owns. +pub const POOL_GENERATION_LABEL: &str = "com.gitforce.pool.generation"; + +/// Label naming the PID of the runner process that created the container. +pub const POOL_PID_LABEL: &str = "com.gitforce.pool.pid"; + +/// The identity of the pool that owns a set of containers. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PoolIdentity { + /// Stable owner of the pool (runner identity). Two pools with + /// different owners never reclaim each other's containers. + pub owner: String, + /// One generation per runner process. Containers from another + /// generation belong to a process that is gone. + pub generation: String, + /// PID of the owning process, used to tell "the process that made this + /// is gone" from "another live process made this". + pub pid: u32, +} + +impl PoolIdentity { + /// Build an identity for the current process. + pub fn new(owner: impl Into, generation: impl Into) -> Self { + Self { + owner: owner.into(), + generation: generation.into(), + pid: std::process::id(), + } + } +} + +/// Policy for reclaiming pooled containers. +#[derive(Debug, Clone)] +pub struct PoolPolicy { + /// How long a pooled container may sit before it is eligible for + /// reclamation. Default: 30 minutes. + pub ttl: Duration, + /// Upper bound on containers removed per pass. Bounded blast radius. + /// Default: 16. + pub max_removals: usize, +} + +impl Default for PoolPolicy { + fn default() -> Self { + Self { + ttl: Duration::from_secs(1800), + max_removals: 16, + } + } +} + +impl PoolPolicy { + /// Fail-closed validation: the TTL and the blast radius must be + /// positive. + pub fn validate(&self) -> Result<()> { + if self.ttl.is_zero() { + return Err(Error::sandbox("pool policy rejected: ttl must be positive")); + } + if self.max_removals == 0 { + return Err(Error::sandbox( + "pool policy rejected: max removals must be positive", + )); + } + Ok(()) + } +} + +/// The decision for one container against the pool's reclamation rules. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PoolDecision { + /// The container is not a pooled container at all. + NotPool, + /// The container belongs to a different pool owner. + RetainOtherOwner, + /// The container belongs to this process's own pool. + RetainCurrentGeneration, + /// The creating process is still alive, so the container may be + /// serving a job right now. + RetainOwnerAlive, + /// The creating process's PID cannot be checked, so ownership cannot + /// be established. + RetainUnknownOwner, + /// The container's age is unknown, so its TTL cannot be applied. + RetainUnknownAge, + /// The container is still inside its TTL. + RetainWithinTtl, + /// The container is an orphan of a dead pool generation and may be + /// removed. + Reclaim, +} + +/// What is known about the process that created a container. +pub type OwnerState = dyn Fn(u32) -> Option + Send + Sync; + +/// Look up whether a PID is alive. Returns `None` when the answer cannot +/// be established (for example when the runner cannot see the PID +/// namespace its container was created in), which keeps the caller +/// fail-safe: an unknown owner is never reclaimed. +pub fn process_alive(pid: u32) -> Option { + // Our own PID must be visible for the namespace to be shared with the + // processes we are asked about. + if !std::path::Path::new("/proc/self").exists() { + return None; + } + let path = format!("/proc/{pid}"); + if std::path::Path::new(&path).exists() { + Some(true) + } else { + Some(false) + } +} + +/// Pure decision function: classify one container against the pool's +/// reclamation rules. No I/O; the same inputs always produce the same +/// decision. +pub fn classify_pool_container( + record: &ContainerRecord, + identity: &PoolIdentity, + policy: &PoolPolicy, + now_ms: i64, + owner_state: &OwnerState, +) -> PoolDecision { + // 1. Is it a pooled container at all? + if record.labels.get(POOL_LABEL).map(String::as_str) != Some("true") { + return PoolDecision::NotPool; + } + // 2. Ownership: only our own pool's containers are ours to reclaim. + match record.labels.get(POOL_OWNER_LABEL) { + Some(owner) if owner == &identity.owner => {} + _ => return PoolDecision::RetainOtherOwner, + } + // 3. Generation: our own generation is in use by definition. + match record.labels.get(POOL_GENERATION_LABEL) { + Some(generation) if generation == &identity.generation => { + return PoolDecision::RetainCurrentGeneration; + } + Some(_) => {} + // A container with no generation cannot be attributed to a dead + // process, so it is retained. + None => return PoolDecision::RetainUnknownOwner, + } + // 4. The creating process: a live PID may be serving a job right now. + let Some(raw_pid) = record.labels.get(POOL_PID_LABEL) else { + return PoolDecision::RetainUnknownOwner; + }; + let Ok(pid) = raw_pid.parse::() else { + return PoolDecision::RetainUnknownOwner; + }; + match owner_state(pid) { + Some(true) => return PoolDecision::RetainOwnerAlive, + None => return PoolDecision::RetainUnknownOwner, + Some(false) => {} + } + // 5. Age: an unknown age cannot be TTL-checked. + let Some(created_at_ms) = record.created_at_ms else { + return PoolDecision::RetainUnknownAge; + }; + let age_ms = now_ms.saturating_sub(created_at_ms); + if age_ms < policy.ttl.as_millis() as i64 { + return PoolDecision::RetainWithinTtl; + } + PoolDecision::Reclaim +} + +/// Outcome of one reclamation pass. +#[derive(Debug, Clone, Default)] +pub struct PoolReclaimReport { + /// Containers removed, by container id. + pub reclaimed: Vec, + /// Containers that were considered and kept, by decision. + pub retained: Vec<(String, PoolDecision)>, + /// Containers whose removal failed. + pub failed: Vec<(String, String)>, + /// True when the pass stopped early because the blast radius bound + /// was reached. + pub bounded: bool, +} + +/// Reclaim the pooled containers orphaned by a previous runner process. +/// +/// The pass is idempotent: a container that is already gone is reported as +/// reclaimed without an error, and a container whose removal fails is +/// recorded rather than retried forever in the same pass. +pub async fn reclaim_orphaned_pool_containers( + source: &S, + identity: &PoolIdentity, + policy: &PoolPolicy, + now_ms: i64, + owner_state: &OwnerState, +) -> Result { + policy.validate()?; + let mut report = PoolReclaimReport::default(); + let containers = source.list_containers().await?; + + for record in containers { + let decision = classify_pool_container(&record, identity, policy, now_ms, owner_state); + if decision != PoolDecision::Reclaim { + if decision != PoolDecision::NotPool { + report.retained.push((record.id.clone(), decision)); + } + continue; + } + if report.reclaimed.len() >= policy.max_removals { + report.bounded = true; + report + .retained + .push((record.id.clone(), PoolDecision::Reclaim)); + continue; + } + match source.remove_container(&record.id).await { + Ok(RemovalOutcome::Removed) | Ok(RemovalOutcome::AlreadyGone) => { + report.reclaimed.push(record.id.clone()); + } + Ok(RemovalOutcome::Failed) => { + report + .failed + .push((record.id.clone(), "removal refused".to_string())); + } + Err(error) => { + report.failed.push((record.id.clone(), error.to_string())); + } + } + } + Ok(report) +} + +/// The pool labels for a container, as a map. Used by the sandbox to +/// stamp pooled containers and by tests to build records. +pub fn pool_labels(identity: &PoolIdentity) -> HashMap { + HashMap::from([ + (POOL_LABEL.to_string(), "true".to_string()), + (POOL_OWNER_LABEL.to_string(), identity.owner.clone()), + ( + POOL_GENERATION_LABEL.to_string(), + identity.generation.clone(), + ), + (POOL_PID_LABEL.to_string(), identity.pid.to_string()), + ]) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn identity() -> PoolIdentity { + PoolIdentity { + owner: "runner@host".to_string(), + generation: "gen-current".to_string(), + pid: 4242, + } + } + + fn record( + id: &str, + labels: HashMap, + created_at_ms: Option, + ) -> ContainerRecord { + ContainerRecord { + id: id.to_string(), + labels, + running: true, + exited_at_ms: None, + created_at_ms, + } + } + + fn pooled(labels: Vec<(&str, &str)>) -> HashMap { + labels + .into_iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect() + } + + /// A container source that records removals instead of talking to a + /// Docker daemon. + struct FakeSource { + containers: Vec, + removed: std::sync::Mutex>, + } + + impl FakeSource { + fn new(containers: Vec) -> Self { + Self { + containers, + removed: std::sync::Mutex::new(Vec::new()), + } + } + } + + #[async_trait::async_trait] + impl ContainerSource for FakeSource { + async fn list_containers(&self) -> Result> { + Ok(self.containers.clone()) + } + + async fn remove_container(&self, id: &str) -> Result { + self.removed + .lock() + .expect("fake source lock") + .push(id.to_string()); + Ok(RemovalOutcome::Removed) + } + } + + /// A pooled container from a dead generation, older than the TTL. + fn orphan(id: &str, age_ms: i64) -> ContainerRecord { + record( + id, + pooled(vec![ + (POOL_LABEL, "true"), + (POOL_OWNER_LABEL, "runner@host"), + (POOL_GENERATION_LABEL, "gen-old"), + (POOL_PID_LABEL, "1"), + ]), + Some(NOW - age_ms), + ) + } + + fn dead_owner(_pid: u32) -> Option { + Some(false) + } + + fn live_owner(_pid: u32) -> Option { + Some(true) + } + + fn unknown_owner(_pid: u32) -> Option { + None + } + + const NOW: i64 = 10_000_000; + + #[test] + fn a_non_pooled_container_is_not_ours_to_judge() { + let record = record( + "c1", + pooled(vec![("com.gitforce.managed", "true")]), + Some(NOW - 10_000), + ); + let decision = classify_pool_container( + &record, + &identity(), + &PoolPolicy::default(), + NOW, + &dead_owner, + ); + assert_eq!(decision, PoolDecision::NotPool); + } + + #[test] + fn another_owners_pool_is_never_reclaimed() { + let record = record( + "c1", + pooled(vec![ + (POOL_LABEL, "true"), + (POOL_OWNER_LABEL, "other@host"), + (POOL_GENERATION_LABEL, "gen-old"), + (POOL_PID_LABEL, "1"), + ]), + Some(NOW - 10_000_000), + ); + let decision = classify_pool_container( + &record, + &identity(), + &PoolPolicy::default(), + NOW, + &dead_owner, + ); + assert_eq!(decision, PoolDecision::RetainOtherOwner); + } + + #[test] + fn our_own_generation_is_retained_even_when_old() { + let record = record( + "c1", + pooled(vec![ + (POOL_LABEL, "true"), + (POOL_OWNER_LABEL, "runner@host"), + (POOL_GENERATION_LABEL, "gen-current"), + (POOL_PID_LABEL, "4242"), + ]), + Some(NOW - 100_000_000), + ); + let decision = classify_pool_container( + &record, + &identity(), + &PoolPolicy::default(), + NOW, + &live_owner, + ); + assert_eq!(decision, PoolDecision::RetainCurrentGeneration); + } + + #[test] + fn a_live_creating_process_keeps_its_container() { + let record = record( + "c1", + pooled(vec![ + (POOL_LABEL, "true"), + (POOL_OWNER_LABEL, "runner@host"), + (POOL_GENERATION_LABEL, "gen-old"), + (POOL_PID_LABEL, "99"), + ]), + Some(NOW - 100_000_000), + ); + let decision = classify_pool_container( + &record, + &identity(), + &PoolPolicy::default(), + NOW, + &live_owner, + ); + assert_eq!(decision, PoolDecision::RetainOwnerAlive); + } + + #[test] + fn an_unknown_owner_is_retained() { + let record = record( + "c1", + pooled(vec![ + (POOL_LABEL, "true"), + (POOL_OWNER_LABEL, "runner@host"), + (POOL_GENERATION_LABEL, "gen-old"), + (POOL_PID_LABEL, "99"), + ]), + Some(NOW - 100_000_000), + ); + let decision = classify_pool_container( + &record, + &identity(), + &PoolPolicy::default(), + NOW, + &unknown_owner, + ); + assert_eq!(decision, PoolDecision::RetainUnknownOwner); + } + + #[test] + fn a_container_without_a_pid_is_retained() { + let record = record( + "c1", + pooled(vec![ + (POOL_LABEL, "true"), + (POOL_OWNER_LABEL, "runner@host"), + (POOL_GENERATION_LABEL, "gen-old"), + ]), + Some(NOW - 100_000_000), + ); + let decision = classify_pool_container( + &record, + &identity(), + &PoolPolicy::default(), + NOW, + &dead_owner, + ); + assert_eq!(decision, PoolDecision::RetainUnknownOwner); + } + + #[test] + fn a_container_inside_its_ttl_is_retained() { + let record = record( + "c1", + pooled(vec![ + (POOL_LABEL, "true"), + (POOL_OWNER_LABEL, "runner@host"), + (POOL_GENERATION_LABEL, "gen-old"), + (POOL_PID_LABEL, "99"), + ]), + Some(NOW - 60_000), + ); + let decision = classify_pool_container( + &record, + &identity(), + &PoolPolicy::default(), + NOW, + &dead_owner, + ); + assert_eq!(decision, PoolDecision::RetainWithinTtl); + } + + #[test] + fn an_orphan_past_its_ttl_is_reclaimed() { + let record = record( + "c1", + pooled(vec![ + (POOL_LABEL, "true"), + (POOL_OWNER_LABEL, "runner@host"), + (POOL_GENERATION_LABEL, "gen-old"), + (POOL_PID_LABEL, "99"), + ]), + Some(NOW - 100_000_000), + ); + let decision = classify_pool_container( + &record, + &identity(), + &PoolPolicy::default(), + NOW, + &dead_owner, + ); + assert_eq!(decision, PoolDecision::Reclaim); + } + + #[test] + fn an_unknown_age_is_retained() { + let record = record( + "c1", + pooled(vec![ + (POOL_LABEL, "true"), + (POOL_OWNER_LABEL, "runner@host"), + (POOL_GENERATION_LABEL, "gen-old"), + (POOL_PID_LABEL, "99"), + ]), + None, + ); + let decision = classify_pool_container( + &record, + &identity(), + &PoolPolicy::default(), + NOW, + &dead_owner, + ); + assert_eq!(decision, PoolDecision::RetainUnknownAge); + } + + #[test] + fn a_zero_ttl_policy_is_refused() { + let policy = PoolPolicy { + ttl: Duration::ZERO, + max_removals: 4, + }; + assert!(policy.validate().is_err()); + } + + #[tokio::test] + async fn a_pass_reclaims_only_orphans_of_a_dead_generation() { + let source = FakeSource::new(vec![ + orphan("orphan-old", 10_000_000), + record( + "live-owner", + pooled(vec![ + (POOL_LABEL, "true"), + (POOL_OWNER_LABEL, "runner@host"), + (POOL_GENERATION_LABEL, "gen-old"), + (POOL_PID_LABEL, "2"), + ]), + Some(NOW - 10_000_000), + ), + record( + "other-owner", + pooled(vec![ + (POOL_LABEL, "true"), + (POOL_OWNER_LABEL, "other@host"), + (POOL_GENERATION_LABEL, "gen-old"), + (POOL_PID_LABEL, "1"), + ]), + Some(NOW - 10_000_000), + ), + ]); + let owner_state: &OwnerState = &|pid: u32| { + if pid == 1 { + Some(false) + } else { + Some(true) + } + }; + let report = reclaim_orphaned_pool_containers( + &source, + &identity(), + &PoolPolicy::default(), + NOW, + &owner_state, + ) + .await + .expect("reclaim pass"); + assert_eq!(report.reclaimed, vec!["orphan-old".to_string()]); + assert_eq!( + source.removed.lock().expect("fake source lock").as_slice(), + &["orphan-old".to_string()] + ); + } + + #[tokio::test] + async fn a_pass_is_bounded_by_the_blast_radius() { + let containers: Vec = (0..5) + .map(|index| orphan(&format!("orphan-{index}"), 10_000_000)) + .collect(); + let source = FakeSource::new(containers); + let policy = PoolPolicy { + ttl: Duration::from_secs(1), + max_removals: 2, + }; + let owner_state: &OwnerState = &dead_owner; + let report = + reclaim_orphaned_pool_containers(&source, &identity(), &policy, NOW, &owner_state) + .await + .expect("reclaim pass"); + assert_eq!(report.reclaimed.len(), 2); + assert!(report.bounded); + assert_eq!( + source.removed.lock().expect("fake source lock").len(), + 2, + "the pass must not remove more than the bound allows" + ); + } + + #[tokio::test] + async fn an_unknown_owner_state_reclaims_nothing() { + let source = FakeSource::new(vec![orphan("orphan", 10_000_000)]); + let owner_state: &OwnerState = &unknown_owner; + let report = reclaim_orphaned_pool_containers( + &source, + &identity(), + &PoolPolicy::default(), + NOW, + &owner_state, + ) + .await + .expect("reclaim pass"); + assert!(report.reclaimed.is_empty()); + assert!(source.removed.lock().expect("fake source lock").is_empty()); + } +} diff --git a/crates/gitforge-sandbox/src/reconciler.rs b/crates/gitforge-sandbox/src/reconciler.rs index 1997494c..6cdbc897 100644 --- a/crates/gitforge-sandbox/src/reconciler.rs +++ b/crates/gitforge-sandbox/src/reconciler.rs @@ -16,7 +16,9 @@ //! abandonment; the grace period and deletion policy decide the rest. //! 3. **Grace period** — a container must have been in a non-running state //! for at least `grace` before it is eligible. Running containers are -//! never removed regardless of metadata age. +//! never removed regardless of metadata age — with one exception, the +//! store-backed legacy-orphan rule below, which is the only way to +//! reach the warm pool's pre-fix leftovers. //! 4. **Concurrency** — removal is idempotent; Docker 404 (already gone) //! and 409 (concurrent teardown in progress) are treated as successful //! no-ops so a race with the normal teardown never fails. @@ -28,11 +30,21 @@ //! 7. **Receipt** — the resulting [`ReconcileReport`] serializes counts, //! container IDs, decisions, and outcomes. It never contains logs, //! tokens, or environment values. +//! 8. **Store-backed legacy orphans** — a container whose +//! `com.gitforce.job_id` the authoritative store has no row for was +//! never a job container: the warm pool minted a job-shaped id for a +//! container it never gave to a job, and named the container after it. +//! Such a container is unreachable by every other rule — it carries no +//! pool label, so the pool's own rules do not own it, and it is +//! running, so the general rule retains it. The store's answer is +//! injected as data ([`JobRowState`]), the rule is armed explicitly, +//! and it fails closed at every unknown: an unreadable store, an +//! unknown age, and a container inside the TTL all retain. use crate::active_jobs::ActiveJobRegistry; use bollard::query_parameters::{ListContainersOptions, RemoveContainerOptions}; use chrono::DateTime; -use gitforge_common::{Error, Result}; +use gitforge_common::{Error, JobRowState, Result}; use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::sync::Arc; @@ -53,6 +65,11 @@ pub enum Decision { RetainMalformedLabel, /// Container is still running; age is irrelevant. RetainRunning, + /// Container belongs to a warm pool. The pool owns its own + /// containers and reclaims them under the pool's own rules + /// (`crate::pool`); the general reconciler never removes them, so + /// the two mechanisms cannot disagree about a live container. + RetainPool, /// Owning job is currently active/claimed per authoritative state. RetainActiveJob, /// Non-running for less than the configured grace period. @@ -62,11 +79,23 @@ pub enum Decision { /// Abandoned per policy: unknown job, past grace. Subject to removal /// only when deletion is enabled. Eligible, + /// The store has no row for this container's job id, the container is + /// past the orphan TTL, and the policy allows removal: the warm + /// pool's placeholder, which no other rule can reach. + EligibleLegacyOrphan, + /// The store could not be consulted for this container's job id. + /// Fail-closed: an unknown answer never authorises a removal. + RetainStoreUnknown, + /// The store has no row for this container's job id, but the + /// container is too young to be an orphan. + RetainOrphanWithinTtl, + /// The container's age is unknown, so its TTL cannot be applied. + RetainUnknownAge, } impl Decision { pub fn is_eligible(&self) -> bool { - matches!(self, Decision::Eligible) + matches!(self, Decision::Eligible | Decision::EligibleLegacyOrphan) } } @@ -112,6 +141,13 @@ pub struct ReconcilerPolicy { /// Per-Docker-call timeout so one hung API call cannot stall job /// admission. Default: 10 seconds. pub call_timeout: Duration, + /// Arm the store-backed legacy-orphan rule. When false (the default) + /// the reconciler never removes a container on the strength of a + /// store lookup alone, and never asks the store at all. + pub orphan_reclaim_enabled: bool, + /// Minimum age a container must have before the store-backed rule may + /// call it an orphan. Default: 1 hour. Must be positive. + pub orphan_ttl: Duration, } impl Default for ReconcilerPolicy { @@ -121,6 +157,8 @@ impl Default for ReconcilerPolicy { grace: Duration::from_secs(3600), max_removals: 64, call_timeout: Duration::from_secs(10), + orphan_reclaim_enabled: false, + orphan_ttl: Duration::from_secs(3600), } } } @@ -144,16 +182,27 @@ impl ReconcilerPolicy { "reconciler policy rejected: call timeout must be positive", )); } + if self.orphan_ttl.is_zero() { + return Err(Error::sandbox( + "reconciler policy rejected: orphan TTL must be positive", + )); + } Ok(()) } } -/// Pure decision function: classify one container against policy and the -/// authoritative active-job set. No I/O; the same inputs always produce -/// the same decision. +/// Pure decision function: classify one container against policy, the +/// authoritative active-job set, and the authoritative store's view of the +/// container's job id. No I/O; the same inputs always produce the same +/// decision. +/// +/// `store_state` is the store's answer as data — it is resolved once at +/// the edge and handed in, so this function never reads the store (or any +/// other process-global state) itself. pub fn classify( record: &ContainerRecord, active_jobs: &std::collections::HashSet, + store_state: JobRowState, policy: &ReconcilerPolicy, now: Now, ) -> Decision { @@ -177,11 +226,57 @@ pub fn classify( if active_jobs.contains(raw_job) { return Decision::RetainActiveJob; } - // 4. Never remove a running container. + // 4. Pooled containers belong to the warm pool, which reclaims them + // under its own rules (owner, generation, TTL, liveness of the + // creating process). The general reconciler does not remove them: + // a pooled container that is serving a job looks exactly like an + // orphan to any rule that only sees "running" and "not an active + // job id" - and the pool's label for a job container is a + // placeholder, not the job id. + if record.labels.contains_key(crate::pool::POOL_LABEL) { + return Decision::RetainPool; + } + // 5. Store-backed legacy-orphan rule. + // + // The warm pool minted a job-shaped id for a container it never + // gave to a job, and named the container after it. The store is + // the only authority that can tell such a container apart from a + // real one, because a real job always has a durable row: the row + // is written when the job is enqueued, which is before the job + // can be dispatched to a runner. + // + // The rule is armed explicitly and fails closed at every unknown: + // an unreadable store, an unknown age, and a container inside the + // TTL all retain. + if policy.orphan_reclaim_enabled { + match store_state { + // The store could not be consulted. Never a licence to remove. + JobRowState::Unknown => return Decision::RetainStoreUnknown, + // The store knows this id: it is a real job, and the ordinary + // rules below decide. + JobRowState::Present => {} + // The store has never heard of this id: this container was + // never a job container. + JobRowState::Absent => { + let Some(created_at_ms) = record.created_at_ms else { + return Decision::RetainUnknownAge; + }; + let age_ms = now.now_ms.saturating_sub(created_at_ms); + if age_ms < policy.orphan_ttl.as_millis() as i64 { + return Decision::RetainOrphanWithinTtl; + } + if !policy.deletion_enabled { + return Decision::RetainPolicyDisabled; + } + return Decision::EligibleLegacyOrphan; + } + } + } + // 6. Never remove a running container. if record.running { return Decision::RetainRunning; } - // 5. Grace period: must be non-running for at least `grace`. + // 7. Grace period: must be non-running for at least `grace`. // Fail-closed: an unknown exit time never qualifies. let Some(exited_at) = record.exited_at_ms else { return Decision::RetainGracePeriod; @@ -190,13 +285,53 @@ pub fn classify( if non_running_for < policy.grace.as_millis() as i64 { return Decision::RetainGracePeriod; } - // 6. Deletion authority. + // 8. Deletion authority. if !policy.deletion_enabled { return Decision::RetainPolicyDisabled; } Decision::Eligible } +/// Resolves the authoritative store's view of a job id. +/// +/// The reconciler is the edge that talks to the store; the decision core +/// takes the answer as data. A resolver that cannot answer must answer +/// `Unknown`, which never authorises a removal. +#[async_trait::async_trait] +pub trait JobExistenceResolver: Send + Sync { + /// Resolve the store's view of the given job ids. + /// + /// Ids the resolver does not mention are `Unknown`: an omitted id is + /// not an absent id. + async fn resolve(&self, job_ids: &[String]) -> HashMap; +} + +/// A resolver that knows nothing: every id is `Unknown`. +/// +/// This is the default, so a reconciler with no store connection can never +/// remove a container on the strength of a store lookup. +pub struct UnknownJobExistence; + +#[async_trait::async_trait] +impl JobExistenceResolver for UnknownJobExistence { + async fn resolve(&self, job_ids: &[String]) -> HashMap { + job_ids + .iter() + .map(|job_id| (job_id.clone(), JobRowState::Unknown)) + .collect() + } +} + +/// The store's answer for one reconcile pass. +struct StoreAnswers { + /// The answer per job id the candidates carry. + states: HashMap, + /// How many distinct job ids the pass asked about. + requested: usize, + /// How many of them the store answered (present or absent). + answered: usize, +} + /// Outcome of one removal attempt. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] @@ -234,6 +369,12 @@ pub struct ReconcileReport { pub active_jobs_hash: String, pub candidates_seen: usize, pub eligible: usize, + /// Distinct job ids the pass asked the store about. + pub store_requested: usize, + /// How many of them the store answered (present or absent). + pub store_answered: usize, + /// Containers the store-backed legacy-orphan rule called eligible. + pub legacy_orphans: usize, pub removed: usize, pub already_gone: usize, pub failed: usize, @@ -266,11 +407,64 @@ fn fnv1a_hex(input: &str) -> String { pub struct Reconciler { source: Arc, policy: ReconcilerPolicy, + resolver: Arc, } impl Reconciler { pub fn new(source: Arc, policy: ReconcilerPolicy) -> Self { - Self { source, policy } + Self { + source, + policy, + resolver: Arc::new(UnknownJobExistence), + } + } + + /// Attach the store resolver the reconciler asks about job ids. + /// + /// Without one the reconciler answers `unknown` for every id, which + /// retains every container the store-backed rule could have reached. + pub fn with_resolver(mut self, resolver: Arc) -> Self { + self.resolver = resolver; + self + } + + /// Ask the store about the job ids the candidates carry. + /// + /// One batched lookup per pass. The lookup is skipped entirely when + /// the rule is not armed, so an unarmed reconciler never talks to the + /// store at all. + async fn resolve_store_states(&self, containers: &[ContainerRecord]) -> StoreAnswers { + let mut ids: Vec = Vec::new(); + let mut seen: std::collections::HashSet = std::collections::HashSet::new(); + for record in containers { + if let Some(job_id) = record.labels.get(JOB_ID_LABEL) { + if seen.insert(job_id.clone()) { + ids.push(job_id.clone()); + } + } + } + if !self.policy.orphan_reclaim_enabled || ids.is_empty() { + return StoreAnswers { + states: HashMap::new(), + requested: ids.len(), + answered: 0, + }; + } + let states = self.resolver.resolve(&ids).await; + let answered = ids + .iter() + .filter(|id| { + matches!( + states.get(*id), + Some(JobRowState::Present) | Some(JobRowState::Absent) + ) + }) + .count(); + StoreAnswers { + states, + requested: ids.len(), + answered, + } } pub fn policy(&self) -> &ReconcilerPolicy { @@ -294,15 +488,25 @@ impl Reconciler { report.policy_deletion_enabled = self.policy.deletion_enabled; report.policy_grace_secs = self.policy.grace.as_secs(); report.runtime = "docker".to_string(); + let store = self.resolve_store_states(&containers).await; + report.store_requested = store.requested; + report.store_answered = store.answered; for record in containers { - let decision = classify(&record, &active_jobs, &self.policy, now); let job_id = record.labels.get(JOB_ID_LABEL).cloned(); + let store_state = job_id + .as_deref() + .and_then(|id| store.states.get(id).copied()) + .unwrap_or(JobRowState::Unknown); + let decision = classify(&record, &active_jobs, store_state, &self.policy, now); let exited_age_ms = record.exited_at_ms.map(|t| now.now_ms.saturating_sub(t)); report.candidates_seen += 1; if decision.is_eligible() { report.eligible += 1; } + if decision == Decision::EligibleLegacyOrphan { + report.legacy_orphans += 1; + } report.entries.push(ReportEntry { container_id: record.id, job_id, @@ -337,15 +541,25 @@ impl Reconciler { report.policy_deletion_enabled = self.policy.deletion_enabled; report.policy_grace_secs = self.policy.grace.as_secs(); report.runtime = "docker".to_string(); + let store = self.resolve_store_states(&containers).await; + report.store_requested = store.requested; + report.store_answered = store.answered; for record in containers { - let decision = classify(&record, &active_jobs, &self.policy, now); let job_id = record.labels.get(JOB_ID_LABEL).cloned(); + let store_state = job_id + .as_deref() + .and_then(|id| store.states.get(id).copied()) + .unwrap_or(JobRowState::Unknown); + let decision = classify(&record, &active_jobs, store_state, &self.policy, now); let exited_age_ms = record.exited_at_ms.map(|t| now.now_ms.saturating_sub(t)); report.candidates_seen += 1; if decision.is_eligible() { report.eligible += 1; } + if decision == Decision::EligibleLegacyOrphan { + report.legacy_orphans += 1; + } report.entries.push(ReportEntry { container_id: record.id, job_id, @@ -404,6 +618,9 @@ impl Reconciler { ), candidates_seen: 0, eligible: 0, + store_requested: 0, + store_answered: 0, + legacy_orphans: 0, removed: 0, already_gone: 0, failed: 0, @@ -434,8 +651,16 @@ impl DockerContainerSource { "reconciler Docker call timeout must be positive", )); } - let docker = bollard::Docker::connect_with_local_defaults() - .map_err(|e| Error::sandbox(format!("reconciler docker connect failed: {e}")))?; + // The same daemon the sandbox talks to: the reconciler removes the + // containers the runner leaked, and a reconciler pointed at another + // daemon would never see them. + let socket = gitforge_common::docker_socket(); + let docker = bollard::Docker::connect_with_unix( + &socket, + crate::docker::docker_client_timeout(), + bollard::API_DEFAULT_VERSION, + ) + .map_err(|e| Error::sandbox(format!("reconciler docker connect failed: {e}")))?; Ok(Self { docker, call_timeout, @@ -583,7 +808,13 @@ mod tests { created_at_ms: Some(0), }; assert_eq!( - classify(&plain, &HashSet::new(), &policy(), now()), + classify( + &plain, + &HashSet::new(), + JobRowState::Unknown, + &policy(), + now() + ), Decision::RetainNotOwned ); @@ -596,7 +827,13 @@ mod tests { ..plain.clone() }; assert_eq!( - classify(&wrong_value, &HashSet::new(), &policy(), now()), + classify( + &wrong_value, + &HashSet::new(), + JobRowState::Unknown, + &policy(), + now() + ), Decision::RetainNotOwned ); @@ -607,7 +844,13 @@ mod tests { ..plain.clone() }; assert_eq!( - classify(&name_only, &HashSet::new(), &policy(), now()), + classify( + &name_only, + &HashSet::new(), + JobRowState::Unknown, + &policy(), + now() + ), Decision::RetainNotOwned ); } @@ -622,7 +865,13 @@ mod tests { created_at_ms: Some(0), }; assert_eq!( - classify(&plain, &HashSet::new(), &policy(), now()), + classify( + &plain, + &HashSet::new(), + JobRowState::Unknown, + &policy(), + now() + ), Decision::RetainMalformedLabel ); let bad = ContainerRecord { @@ -636,11 +885,246 @@ mod tests { ..plain.clone() }; assert_eq!( - classify(&bad, &HashSet::new(), &policy(), now()), + classify( + &bad, + &HashSet::new(), + JobRowState::Unknown, + &policy(), + now() + ), Decision::RetainMalformedLabel ); } + #[test] + fn pooled_container_is_retained_by_the_general_reconciler() { + // A pooled container carries a placeholder job id, so it looks + // exactly like an abandoned job container to any rule that only + // sees "managed" and "not an active job id". The general + // reconciler must not touch it: the pool's own reclamation rules + // (owner, generation, TTL, liveness) decide, and the two + // mechanisms must not disagree about the same container. + let rec = ContainerRecord { + id: "pooled-1".into(), + labels: HashMap::from([ + (MANAGED_LABEL.to_string(), "true".to_string()), + (JOB_ID_LABEL.to_string(), uuid::Uuid::new_v4().to_string()), + (crate::pool::POOL_LABEL.to_string(), "true".to_string()), + ]), + running: true, + exited_at_ms: None, + created_at_ms: Some(9_000_000), + }; + assert_eq!( + classify( + &rec, + &HashSet::new(), + JobRowState::Unknown, + &policy(), + now() + ), + Decision::RetainPool + ); + } + + /// The warm pool's pre-fix leftovers: managed, a job-shaped id that + /// was never a job, no pool label, running, and old. + fn legacy_orphan(job: &str) -> ContainerRecord { + ContainerRecord { + id: format!("cid-{job}"), + labels: HashMap::from([ + (MANAGED_LABEL.to_string(), "true".to_string()), + (JOB_ID_LABEL.to_string(), job.to_string()), + ]), + running: true, + exited_at_ms: None, + created_at_ms: Some(NOW_MS - 10_000_000), + } + } + + /// A policy with the store-backed legacy-orphan rule armed. + fn orphan_policy() -> ReconcilerPolicy { + ReconcilerPolicy { + deletion_enabled: true, + orphan_reclaim_enabled: true, + ..policy() + } + } + + /// A resolver that answers from a fixed table. Ids it does not know + /// are `Unknown`, exactly as a store that cannot be read would answer. + struct FixedResolver { + states: HashMap, + } + + #[async_trait::async_trait] + impl JobExistenceResolver for FixedResolver { + async fn resolve(&self, job_ids: &[String]) -> HashMap { + job_ids + .iter() + .map(|id| { + ( + id.clone(), + self.states.get(id).copied().unwrap_or(JobRowState::Unknown), + ) + }) + .collect() + } + } + + #[test] + fn the_store_backed_rule_is_off_by_default() { + let job = uuid::Uuid::new_v4().to_string(); + let rec = legacy_orphan(&job); + assert_eq!( + classify(&rec, &HashSet::new(), JobRowState::Absent, &policy(), now()), + Decision::RetainRunning, + "an unarmed reconciler never removes a container on a store lookup alone" + ); + } + + #[test] + fn a_legacy_orphan_the_store_has_never_heard_of_is_eligible() { + let job = uuid::Uuid::new_v4().to_string(); + let rec = legacy_orphan(&job); + assert_eq!( + classify( + &rec, + &HashSet::new(), + JobRowState::Absent, + &orphan_policy(), + now() + ), + Decision::EligibleLegacyOrphan + ); + } + + #[test] + fn an_unknown_store_answer_retains_the_container() { + let job = uuid::Uuid::new_v4().to_string(); + let rec = legacy_orphan(&job); + assert_eq!( + classify( + &rec, + &HashSet::new(), + JobRowState::Unknown, + &orphan_policy(), + now() + ), + Decision::RetainStoreUnknown + ); + } + + #[test] + fn a_present_store_answer_keeps_the_ordinary_rules() { + let job = uuid::Uuid::new_v4().to_string(); + let rec = legacy_orphan(&job); + assert_eq!( + classify( + &rec, + &HashSet::new(), + JobRowState::Present, + &orphan_policy(), + now() + ), + Decision::RetainRunning, + "a job the store knows is decided by the ordinary rules" + ); + } + + #[test] + fn an_orphan_inside_its_ttl_is_retained() { + let job = uuid::Uuid::new_v4().to_string(); + let mut rec = legacy_orphan(&job); + rec.created_at_ms = Some(NOW_MS - 60_000); + assert_eq!( + classify( + &rec, + &HashSet::new(), + JobRowState::Absent, + &orphan_policy(), + now() + ), + Decision::RetainOrphanWithinTtl + ); + } + + #[test] + fn an_orphan_with_an_unknown_age_is_retained() { + let job = uuid::Uuid::new_v4().to_string(); + let mut rec = legacy_orphan(&job); + rec.created_at_ms = None; + assert_eq!( + classify( + &rec, + &HashSet::new(), + JobRowState::Absent, + &orphan_policy(), + now() + ), + Decision::RetainUnknownAge + ); + } + + #[test] + fn the_pool_keeps_its_containers_even_with_the_rule_armed() { + let job = uuid::Uuid::new_v4().to_string(); + let mut rec = legacy_orphan(&job); + rec.labels + .insert(crate::pool::POOL_LABEL.to_string(), "true".to_string()); + assert_eq!( + classify( + &rec, + &HashSet::new(), + JobRowState::Absent, + &orphan_policy(), + now() + ), + Decision::RetainPool, + "the pool's own contract owns pooled containers; the two rules must not disagree" + ); + } + + #[test] + fn an_active_job_is_retained_even_with_the_rule_armed() { + let job = uuid::Uuid::new_v4().to_string(); + let rec = legacy_orphan(&job); + let active = HashSet::from([job.clone()]); + assert_eq!( + classify(&rec, &active, JobRowState::Absent, &orphan_policy(), now()), + Decision::RetainActiveJob + ); + } + + #[tokio::test] + async fn a_reconcile_pass_removes_a_legacy_orphan_the_store_denies() { + let job = uuid::Uuid::new_v4().to_string(); + let source = Arc::new(MockSource::new(vec![legacy_orphan(&job)])); + let resolver = Arc::new(FixedResolver { + states: HashMap::from([(job.clone(), JobRowState::Absent)]), + }); + let reconciler = Reconciler::new(source.clone(), orphan_policy()).with_resolver(resolver); + let registry = ActiveJobRegistry::new(); + let report = reconciler.reconcile(®istry, now()).await.unwrap(); + + assert_eq!(report.legacy_orphans, 1); + assert_eq!(report.removed, 1); + assert_eq!(source.removed_ids(), vec![format!("cid-{job}")]); + } + + #[tokio::test] + async fn a_reconcile_pass_without_a_resolver_retains_every_container() { + let job = uuid::Uuid::new_v4().to_string(); + let source = Arc::new(MockSource::new(vec![legacy_orphan(&job)])); + let reconciler = Reconciler::new(source.clone(), orphan_policy()); + let registry = ActiveJobRegistry::new(); + let report = reconciler.reconcile(®istry, now()).await.unwrap(); + + assert_eq!(report.removed, 0); + assert_eq!(report.legacy_orphans, 0); + assert!(source.removed_ids().is_empty()); + } + #[test] fn running_container_never_removed_even_when_ancient() { let rec = owned( @@ -649,7 +1133,13 @@ mod tests { Some(10 * 365 * 24 * 3600 * 1000), ); assert_eq!( - classify(&rec, &HashSet::new(), &policy(), now()), + classify( + &rec, + &HashSet::new(), + JobRowState::Unknown, + &policy(), + now() + ), Decision::RetainRunning ); } @@ -660,7 +1150,7 @@ mod tests { let rec = owned(&job, false, Some(10 * 365 * 24 * 3600 * 1000)); let active = HashSet::from([job.clone()]); assert_eq!( - classify(&rec, &active, &policy(), now()), + classify(&rec, &active, JobRowState::Unknown, &policy(), now()), Decision::RetainActiveJob ); } @@ -671,19 +1161,37 @@ mod tests { // Exited 59 minutes ago: inside grace. let fresh = owned(&job, false, Some(59 * 60 * 1000)); assert_eq!( - classify(&fresh, &HashSet::new(), &policy(), now()), + classify( + &fresh, + &HashSet::new(), + JobRowState::Unknown, + &policy(), + now() + ), Decision::RetainGracePeriod ); // Exited exactly at grace: eligible. let boundary = owned(&job, false, Some(3600 * 1000)); assert_eq!( - classify(&boundary, &HashSet::new(), &policy(), now()), + classify( + &boundary, + &HashSet::new(), + JobRowState::Unknown, + &policy(), + now() + ), Decision::Eligible ); // Unknown exit time: fail closed. let unknown = owned(&job, false, None); assert_eq!( - classify(&unknown, &HashSet::new(), &policy(), now()), + classify( + &unknown, + &HashSet::new(), + JobRowState::Unknown, + &policy(), + now() + ), Decision::RetainGracePeriod ); } @@ -697,7 +1205,13 @@ mod tests { ..policy() }; assert_eq!( - classify(&rec, &HashSet::new(), &census_only, now()), + classify( + &rec, + &HashSet::new(), + JobRowState::Unknown, + &census_only, + now() + ), Decision::RetainPolicyDisabled ); } diff --git a/crates/gitforge-scheduler/src/assigner.rs b/crates/gitforge-scheduler/src/assigner.rs index e33e35e0..d1c4ca25 100644 --- a/crates/gitforge-scheduler/src/assigner.rs +++ b/crates/gitforge-scheduler/src/assigner.rs @@ -2,7 +2,8 @@ use crate::policy::{SchedulingPolicy, SimplePolicy}; use crate::queue::{JobQueue, Priority, QueuedJob}; -use gitforge_common::{JobId, Lane, PipelineRunId, RepoId, RunnerId, VolumeSpec}; +use crate::resource_class::workspace_cargo_test; +use gitforge_common::{JobId, JobRowState, Lane, PipelineRunId, RepoId, RunnerId, VolumeSpec}; use gitforge_db::models::{ Job as DbJob, JobStatus, PipelineRun as DbPipelineRun, Runner, RUNNER_HEARTBEAT_OFFLINE_AFTER_SECS, @@ -65,6 +66,11 @@ pub enum DeferredReason { LaneLimit, FastReservedCapacity, ScratchHeadroom, + /// The job's exclusive resource class was already in flight on every + /// runner that had capacity. The class is an admission rule of the + /// scheduler, so the exclusion has to be named: a job blocked by it used + /// to sit queued with no reason at all (GF-14). + ResourceClassConflict, } impl DeferredReason { @@ -73,6 +79,7 @@ impl DeferredReason { DeferredReason::LaneLimit => "lane_limit", DeferredReason::FastReservedCapacity => "fast_reserved_capacity", DeferredReason::ScratchHeadroom => "scratch_headroom", + DeferredReason::ResourceClassConflict => "resource_class_conflict", } } } @@ -212,8 +219,18 @@ pub enum SchedulerCommand { /// Scheduler event #[derive(Debug, Clone)] pub enum SchedulerEvent { - /// Job assigned to runner - JobAssigned { job_id: JobId, runner_id: RunnerId }, + /// Job assigned to runner. + /// + /// The run id travels with the event: the CI service keys its engines + /// by run, and the engine's mirror has to leave `Queued` when the job + /// is dispatched. A dispatched job that stays `Queued` in the mirror + /// is re-released by every later completion of the run, and that + /// duplicate enqueue demoted the running job's durable row (GF-12). + JobAssigned { + job_id: JobId, + runner_id: RunnerId, + pipeline_run_id: PipelineRunId, + }, /// No runner available for job NoRunnerAvailable { job_id: JobId }, /// Runner submitted a terminal job receipt. @@ -283,21 +300,18 @@ pub const DEFAULT_JOB_TIMEOUT_SECS: u64 = 300; /// Return the scheduler resource class for jobs that must not overlap on one /// runner. Workspace-wide Cargo tests contend for the same checkout and build /// artifacts, so a runner admits at most one of them at a time. +/// +/// The class is decided by the cargo SUBCOMMAND the command runs, not by the +/// presence of the words `cargo`, `test` and `--workspace` anywhere in the +/// command text (GF-14). The word scan classified a command that merely +/// MENTIONED them — in a comment, or in another cargo subcommand such as +/// `cargo llvm-cov` — as a workspace cargo test, and such a job was then +/// excluded from runners that were free to take it. fn resource_class(definition: &JobExecutionDefinition) -> Option<&'static str> { definition .commands .iter() - .any(|command| { - command - .split_whitespace() - .any(|argument| argument == "cargo") - && command - .split_whitespace() - .any(|argument| argument == "test") - && command - .split_whitespace() - .any(|argument| argument == "--workspace") - }) + .any(|command| workspace_cargo_test(command)) .then_some("workspace-cargo-test") } @@ -479,6 +493,47 @@ impl Scheduler { || state.queue.contains(job_id) } + /// Resolve the authoritative store's view of a batch of job ids. + /// + /// The answer comes from the durable store only. A scheduler without a + /// store cannot speak for jobs another process planned, so it answers + /// `Unknown` for every id — the fail-closed answer, because `Unknown` + /// is the one that never turns into a removal. + /// + /// The distinction matters because the runner stamps a job-shaped id on + /// every container it creates, including the warm-pool containers that + /// were never given to a job. A real job always has a durable row: the + /// row is written when the job is enqueued, which is before the job can + /// be dispatched to a runner. + pub async fn job_row_states(&self, job_ids: &[JobId]) -> HashMap { + let mut states: HashMap = job_ids + .iter() + .map(|job_id| (*job_id, JobRowState::Unknown)) + .collect(); + let Some(pool) = &self.db_pool else { + return states; + }; + let loaded = match gitforge_db::queries::JobQueries::get_many(pool, job_ids).await { + Ok(loaded) => loaded, + Err(error) => { + tracing::warn!( + %error, + "job existence lookup failed; every id stays unknown" + ); + return states; + } + }; + for job_id in job_ids { + let state = if loaded.contains_key(job_id) { + JobRowState::Present + } else { + JobRowState::Absent + }; + states.insert(*job_id, state); + } + states + } + /// Read a durable pipeline run for the CI status adapter. pub async fn get_pipeline_run( &self, @@ -588,6 +643,21 @@ impl Scheduler { definition: JobExecutionDefinition, ) -> anyhow::Result<()> { let timeout_secs = definition.timeout_secs.clamp(5, 24 * 60 * 60); + // A job this scheduler already dispatched is live: its durable row + // is `assigned`/`running` and its runner holds the lease. Queueing + // it again demotes that row, which refuses the runner's heartbeat + // and rotates its lease token out from under it (GF-12). The + // enqueue of a live job is a no-op. + { + let state = self.state.read().await; + if state.assigned_jobs.contains_key(&job_id) { + tracing::debug!( + %job_id, + "job is already dispatched; the duplicate enqueue is a no-op" + ); + return Ok(()); + } + } if let Some(pool) = &self.db_pool { let mut db_job = DbJob::new(pipeline_run_id, format!("job-{job_id}")); db_job.id = job_id; @@ -608,10 +678,20 @@ impl Scheduler { }) .await?; } - persist_with_retry("status update", job_id, || { - gitforge_db::queries::JobQueries::update_status(pool, job_id, "queued") - }) - .await?; + // The durable status of a job a runner holds is the invariant + // every later write is fenced by. Moving such a row back to + // `queued` takes the lease away from the runner that holds it + // (GF-12), so the transition is guarded: a row that is not + // dispatchable stays as it is and is not queued again. + let dispatchable = + gitforge_db::queries::JobQueries::mark_queued_if_dispatchable(pool, job_id).await?; + if !dispatchable { + tracing::debug!( + %job_id, + "job row is live; the enqueue leaves it to the runner that holds it" + ); + return Ok(()); + } let commands = definition.commands.clone(); let image = definition.image.clone(); let working_dir = definition.working_dir.clone(); @@ -1544,6 +1624,11 @@ impl Scheduler { // Select runner using policy let mut available = runners.clone(); + // A runner whose free slot is taken away by the exclusive resource + // class is a different block from a runner that is simply busy: + // only the class is an admission rule of the scheduler, so + // remember when the class is what removed the capacity. + let mut class_conflict_blocked = false; for runner in &mut available { let in_flight = state .assigned_jobs @@ -1553,6 +1638,9 @@ impl Scheduler { runner.capacity = (runner.capacity - in_flight).max(0); if let Some(definition) = &definition { if runner_has_resource_class_conflict(&state, runner.id, definition) { + if runner.capacity > 0 { + class_conflict_blocked = true; + } runner.capacity = 0; } } @@ -1581,7 +1669,18 @@ impl Scheduler { Ok(true) => {} Ok(false) => { state.queue.remove(job_id); - state.job_leases.remove(&job_id); + // The durable row refused the assignment + // because it is already held. If this + // scheduler still mirrors the job as + // assigned, the runner holds the lease this + // scheduler gave it: dropping the mirror's + // lease here would make every completion it + // sends fail the in-memory lease check, and + // the job would be fenced as lost while it + // is running (GF-12). + if !state.assigned_jobs.contains_key(&job_id) { + state.job_leases.remove(&job_id); + } tracing::debug!(%job_id, "job assignment won by another scheduler"); continue; } @@ -1619,6 +1718,7 @@ impl Scheduler { let event = SchedulerEvent::JobAssigned { job_id, runner_id: r_id, + pipeline_run_id, }; let _ = self.event_tx.send(event); } @@ -1626,10 +1726,26 @@ impl Scheduler { // No runner can take this job right now. Leave it queued // and consider the next one so a single blocked job cannot // idle the runner's remaining capacity. - tracing::debug!( - "no runner available for job {}, skipping to next queued job", - job_id - ); + if class_conflict_blocked { + // The exclusive resource class is what removed the + // capacity of every runner that had any. Name it + // durably: a job blocked by the class used to sit + // queued with no reason at all (GF-14). + tracing::debug!( + "job {} deferred: resource class already in flight", + job_id + ); + self.record_deferral( + &mut state, + job_id, + DeferredReason::ResourceClassConflict, + ); + } else { + tracing::debug!( + "no runner available for job {}, skipping to next queued job", + job_id + ); + } skipped.insert(job_id); } } @@ -1947,16 +2063,40 @@ impl Scheduler { /// Return the current lease for a job, creating one for an existing /// assignment when needed. Repeated calls are idempotent. pub async fn ensure_job_lease(&self, job_id: JobId) -> Option { - let lease = { - let mut state = self.state.write().await; + let mirrored = { + let state = self.state.read().await; if !state.assigned_jobs.contains_key(&job_id) { return None; } - state - .job_leases - .entry(job_id) - .or_insert_with(|| Uuid::new_v4().to_string()) - .clone() + state.job_leases.get(&job_id).cloned() + }; + let lease = match mirrored { + Some(lease) => lease, + None => { + // The mirror has no lease for a job it is assigned. The + // durable row may still carry the token the runner holds: + // adopting it keeps that lease valid, where minting a new + // one would rotate the token under the runner that is + // executing the job and every completion it sends would + // then be rejected (GF-12). + let adopted = match &self.db_pool { + Some(pool) => gitforge_db::queries::JobQueries::get(pool, job_id) + .await + .ok() + .flatten() + .and_then(|job| job.lease_token), + None => None, + }; + let mut state = self.state.write().await; + if !state.assigned_jobs.contains_key(&job_id) { + return None; + } + state + .job_leases + .entry(job_id) + .or_insert_with(|| adopted.unwrap_or_else(|| Uuid::new_v4().to_string())) + .clone() + } }; // Keep the durable row in sync with the lease handed to the runner. // Lease validation reads the database, and a stale or missing row @@ -2520,6 +2660,122 @@ mod tests { ); } + /// The DIL coverage job's command verbatim. It mentions "the test binary" + /// in a comment and runs `cargo llvm-cov --workspace` — the shape the + /// word scan misfiled into the workspace-cargo-test class (GF-14). + fn coverage_definition() -> JobExecutionDefinition { + JobExecutionDefinition { + commands: vec![ + "command -v cargo-llvm-cov >/dev/null 2>&1 || { echo \"cargo-llvm-cov not in image; skipping\"; exit 0; }\nexport CARGO_BUILD_JOBS=4\n# This is the job that died on 2026-11-05 with ENOENT on the\n# test binary; the runner's mount lock is what keeps the tree\n# it executes from being replaced under it.\ncargo llvm-cov --workspace --summary-only --fail-under-lines 95".to_string(), + ], + image: "rust:1".to_string(), + working_dir: None, + timeout_secs: DEFAULT_JOB_TIMEOUT_SECS, + env: HashMap::new(), + volumes: Vec::new(), + lane: gitforge_common::Lane::Standard, + fault: None, + } + } + + #[tokio::test] + async fn test_a_job_that_only_mentions_a_cargo_test_is_not_excluded_by_the_class() { + // GF-14: the DIL coverage job runs `cargo llvm-cov --workspace` and its + // comment mentions "the test binary". The word scan put it into the + // workspace-cargo-test class, so a platform `cargo test --workspace` + // job on the only runner excluded it from a runner that was free to + // take it. + let scheduler = Scheduler::new(); + let repo_id = RepoId::new(); + let run_id = PipelineRunId::new(); + + let coverage_job = JobId::new(); + scheduler + .enqueue_with_definition_and_image_and_timeout( + coverage_job, + run_id, + repo_id, + coverage_definition(), + ) + .await + .unwrap(); + + let runner = make_runner(RunnerId::new(), "test-runner", "online", 2); + let runner_id = runner.id; + scheduler.register_runner(runner).await; + + // One workspace-cargo-test job already in flight on that runner. + let running_test_job = JobId::new(); + { + let mut state = scheduler.state.write().await; + state + .job_definitions + .insert(running_test_job, workspace_test_definition()); + state + .assigned_jobs + .insert(running_test_job, (runner_id, run_id, repo_id)); + } + + scheduler.process_queue().await; + + assert_eq!( + scheduler.is_assigned(coverage_job).await, + Some(runner_id), + "a job that only mentions a cargo test must not be excluded by the class" + ); + } + + #[tokio::test] + async fn test_a_class_excluded_job_names_its_blocker() { + // GF-14: the exclusion is invisible unless it is named. A job held + // back by the exclusive class has to carry the reason, the same way + // a lane-limited job does. + let scheduler = Scheduler::new(); + let repo_id = RepoId::new(); + let run_id = PipelineRunId::new(); + + let blocked_job = JobId::new(); + scheduler + .enqueue_with_definition_and_image_and_timeout( + blocked_job, + run_id, + repo_id, + workspace_test_definition(), + ) + .await + .unwrap(); + + let runner = make_runner(RunnerId::new(), "test-runner", "online", 2); + let runner_id = runner.id; + scheduler.register_runner(runner).await; + + let running_test_job = JobId::new(); + { + let mut state = scheduler.state.write().await; + state + .job_definitions + .insert(running_test_job, workspace_test_definition()); + state + .assigned_jobs + .insert(running_test_job, (runner_id, run_id, repo_id)); + } + + scheduler.process_queue().await; + + assert!(scheduler.is_assigned(blocked_job).await.is_none()); + let status = scheduler.queue_status().await.unwrap(); + let deferred = status + .queued + .iter() + .find(|job| job.job_id == blocked_job) + .expect("the excluded job is still queued"); + assert_eq!( + deferred.deferred_reason.as_deref(), + Some("resource_class_conflict"), + "the exclusion must name its blocker" + ); + } + #[tokio::test] async fn test_fairness_counts_running_jobs_not_visible_backlog() { let scheduler = Scheduler::new(); @@ -2728,6 +2984,7 @@ mod tests { created_at: chrono::Utc::now(), error: None, ref_name: None, + push_base_sha: None, }, ) .await @@ -4432,4 +4689,180 @@ mod tests { let status = scheduler.queue_status().await.unwrap(); assert!(status.queued.iter().all(|job| job.job_id != second)); } + + /// GF-12: a duplicate enqueue must not break the lease of a job that + /// is already running. + /// + /// The CI service re-enqueues every job its engine still holds as + /// `Queued`, and the engine's mirror stayed `Queued` for a dispatched + /// job (nothing consumed the scheduler's `JobAssigned` event). The + /// duplicate enqueue demoted the live job's durable row to `queued` + /// with an unguarded UPDATE: the runner's heartbeat was refused, the + /// row became dispatchable again, the lease token was rotated under + /// the runner that was executing the job, and every completion it sent + /// was rejected - three executions of the job, then a + /// `runner_lost_while_running` fence. + /// + /// The contract pinned here is the runner's: while the job runs, the + /// duplicate enqueue leaves the row, the lease and the heartbeat + /// alone, and the completion lands. + #[tokio::test] + async fn a_duplicate_enqueue_does_not_break_a_running_jobs_lease() { + let pool = gitforge_db::Pool::memory().await.unwrap(); + pool.migrate().await.unwrap(); + let (run_id, repo_id) = seed_durable_run(&pool).await; + let scheduler = Scheduler::with_db(pool.clone()); + let job_id = JobId::new(); + // The durable row's runner is a foreign key: the runner has to + // exist before the job can be leased to it. + let runner = + gitforge_db::models::Runner::new("gf12-runner".to_string(), RunnerType::Docker, 1); + gitforge_db::queries::RunnerQueries::create(&pool, &runner) + .await + .unwrap(); + let runner_id = runner.id; + let lease = "lease-token-of-the-running-job"; + let definition = JobExecutionDefinition { + commands: vec!["cargo test".to_string()], + image: "backend-ci-rust:1".to_string(), + working_dir: Some("/workspace".to_string()), + timeout_secs: 900, + env: HashMap::new(), + volumes: Vec::new(), + lane: gitforge_common::Lane::Standard, + fault: None, + }; + + scheduler + .enqueue_with_definition_and_image_and_timeout( + job_id, + run_id, + repo_id, + definition.clone(), + ) + .await + .unwrap(); + + // The runner claims the job and starts it. + assert!( + gitforge_db::queries::JobQueries::assign_with_lease(&pool, job_id, runner_id, lease) + .await + .unwrap(), + "the runner claims the queued job" + ); + assert!( + gitforge_db::queries::JobQueries::start_with_lease(&pool, job_id, runner_id, lease) + .await + .unwrap(), + "the runner starts the claimed job" + ); + + // The duplicate enqueue the CI service issues when another job of + // the same run completes. + scheduler + .enqueue_with_definition_and_image_and_timeout(job_id, run_id, repo_id, definition) + .await + .unwrap(); + + let row = gitforge_db::queries::JobQueries::get(&pool, job_id) + .await + .unwrap() + .expect("the job row"); + assert_eq!( + row.status, "running", + "the live job's row must not be demoted by a duplicate enqueue" + ); + assert_eq!( + row.lease_token.as_deref(), + Some(lease), + "the lease must not rotate under the running runner" + ); + + // The runner's heartbeat is accepted and its completion lands. + assert!( + gitforge_db::queries::JobQueries::heartbeat(&pool, job_id, runner_id, lease) + .await + .unwrap(), + "the running job's heartbeat must still be accepted" + ); + assert!( + gitforge_db::queries::JobQueries::complete_with_lease( + &pool, + job_id, + runner_id, + lease, + "succeeded", + "{}", + ) + .await + .unwrap(), + "the running job's completion must land" + ); + } + + /// GF-12: the lease handed to a runner is the durable row's token. + /// + /// The mirror dropped its lease when an assignment was refused, and + /// the next lease for that job was minted fresh - which rotated the + /// token the runner was holding and made every completion it sent + /// fail the lease check. The durable token is the lease: it is adopted + /// instead of replaced. + #[tokio::test] + async fn the_lease_handed_to_a_runner_is_the_durable_token() { + let pool = gitforge_db::Pool::memory().await.unwrap(); + pool.migrate().await.unwrap(); + let (run_id, repo_id) = seed_durable_run(&pool).await; + let scheduler = Scheduler::with_db(pool.clone()); + let job_id = JobId::new(); + let runner = gitforge_db::models::Runner::new( + "gf12-adopting-runner".to_string(), + RunnerType::Docker, + 1, + ); + gitforge_db::queries::RunnerQueries::create(&pool, &runner) + .await + .unwrap(); + let runner_id = runner.id; + let lease = "durable-lease-token"; + + scheduler + .enqueue_with_definition_and_image_and_timeout( + job_id, + run_id, + repo_id, + JobExecutionDefinition { + commands: vec!["cargo test".to_string()], + image: "backend-ci-rust:1".to_string(), + working_dir: Some("/workspace".to_string()), + timeout_secs: 900, + env: HashMap::new(), + volumes: Vec::new(), + lane: gitforge_common::Lane::Standard, + fault: None, + }, + ) + .await + .unwrap(); + assert!(gitforge_db::queries::JobQueries::assign_with_lease( + &pool, job_id, runner_id, lease + ) + .await + .unwrap()); + + // The scheduler mirrors the assignment, then loses its lease (the + // state a refused assignment produced). + { + let mut state = scheduler.state.write().await; + state + .assigned_jobs + .insert(job_id, (runner_id, run_id, repo_id)); + state.job_leases.remove(&job_id); + } + + let handed_out = scheduler.ensure_job_lease(job_id).await.unwrap(); + assert_eq!( + handed_out, lease, + "the runner must be handed the token the durable row carries" + ); + } } diff --git a/crates/gitforge-scheduler/src/lib.rs b/crates/gitforge-scheduler/src/lib.rs index 0f473fd7..c7022ce9 100644 --- a/crates/gitforge-scheduler/src/lib.rs +++ b/crates/gitforge-scheduler/src/lib.rs @@ -5,6 +5,7 @@ pub mod assigner; pub mod policy; pub mod queue; +pub mod resource_class; pub mod server; pub use assigner::{Scheduler, SchedulerEvent}; diff --git a/crates/gitforge-scheduler/src/resource_class.rs b/crates/gitforge-scheduler/src/resource_class.rs new file mode 100644 index 00000000..decb7d7c --- /dev/null +++ b/crates/gitforge-scheduler/src/resource_class.rs @@ -0,0 +1,376 @@ +//! Classification of the scheduler's exclusive resource classes. +//! +//! A resource class marks work that must not overlap on one runner. The only +//! class today is `workspace-cargo-test`: a workspace-wide Cargo test builds +//! the whole workspace into one target tree, so two of them on one runner +//! fight over the same build artifacts. +//! +//! The class is decided by the cargo SUBCOMMAND the command actually runs. +//! The earlier classifier scanned the command TEXT for the words `cargo`, +//! `test` and `--workspace` independently, so any command that merely +//! MENTIONED those words landed in the class — a comment line, or another +//! cargo subcommand such as `cargo llvm-cov`. A job that was not a workspace +//! cargo test was then excluded from runners that were free to take it, and +//! the exclusion was invisible in the job's own evidence. +//! +//! The scan below is lexical and deliberately conservative: it reads the +//! subcommand of a cargo invocation and requires `--workspace` (or its +//! `--all` alias) among that invocation's arguments. +//! +//! TWO DIRECTIONS, CHOSEN DELIBERATELY. +//! +//! * A false POSITIVE (a job that is not a workspace cargo test, classified as +//! one) costs a scheduling deferral. The deferral is recorded in the job's +//! durable `deferred_reason`, so the cost is visible and bounded. +//! * A false NEGATIVE (a workspace cargo test that is not classified) lets +//! two workspace cargo tests build the same target tree on one runner. That +//! is the failure that killed the coverage jobs with ENOENT on the test +//! binary on 2026-11-05, and it is silent. +//! +//! So the scan looks for a cargo invocation ANYWHERE in a segment, not only at +//! the segment's first program word. A wrapper program in front of the +//! invocation — `flock cargo test --workspace` — still runs that +//! invocation, and the wrapper is exactly what the older, looser classifier +//! used to catch. The residual imprecision is an unquoted textual mention of +//! a cargo test inside a segment; it costs a visible deferral, never a +//! corrupted build. + +/// Cargo's global options that take a separate value argument. The value is +/// never the subcommand, so the subcommand scan steps over both. +const CARGO_VALUE_OPTIONS: &[&str] = &[ + "--color", + "--config", + "--manifest-path", + "--target", + "--target-dir", + "-Z", +]; + +/// Whether the command runs `cargo test` over the whole workspace. +/// +/// The command text is a shell script, so the scan looks for a segment that +/// runs a cargo invocation whose SUBCOMMAND is `test` and whose arguments +/// include `--workspace` (or its `--all` alias). Comment lines are skipped, +/// and the workspace flag must belong to the cargo test invocation itself, so +/// a mention of "cargo test --workspace" inside a comment or an echoed +/// string does not create the class. +pub fn workspace_cargo_test(command: &str) -> bool { + logical_lines(command).iter().any(|line| { + shell_segments(line) + .iter() + .any(|segment| segment_runs_workspace_cargo_test(segment)) + }) +} + +/// Join backslash-continued lines into the logical lines a shell would run. +fn logical_lines(command: &str) -> Vec { + let mut lines: Vec = Vec::new(); + let mut pending = String::new(); + for raw in command.lines() { + let trimmed = raw.trim_end(); + let continued = trimmed.ends_with('\\'); + let piece = if continued { + &trimmed[..trimmed.len() - 1] + } else { + trimmed + }; + pending.push_str(piece); + pending.push(' '); + if !continued { + lines.push(std::mem::take(&mut pending)); + } + } + if !pending.trim().is_empty() { + lines.push(pending); + } + lines +} + +/// Split a logical line into the shell segments separated by `&&`, `||`, `;` +/// and `|`. A segment is one command's worth of words. Separators inside +/// quotes are not separators, and a `#` comment runs to the end of the line +/// without contributing a segment. +fn shell_segments(line: &str) -> Vec { + let mut segments: Vec = Vec::new(); + let mut current = String::new(); + let mut quote: Option = None; + let mut characters = line.chars().peekable(); + while let Some(character) = characters.next() { + if let Some(open) = quote { + current.push(character); + if character == open { + quote = None; + } + continue; + } + if character == '\'' || character == '"' { + quote = Some(character); + current.push(character); + continue; + } + if character == '#' && at_word_boundary(¤t) { + // A comment runs to the end of the line; its text is not a command. + break; + } + let separator = match character { + '&' | '|' if characters.peek() == Some(&character) => { + characters.next(); + true + } + '|' | ';' => true, + _ => false, + }; + if separator { + segments.push(std::mem::take(&mut current)); + continue; + } + current.push(character); + } + segments.push(current); + segments +} + +/// Whether the segment so far ends at a word boundary, where a `#` starts a +/// comment rather than continuing a word. +fn at_word_boundary(current: &str) -> bool { + current.trim_end().is_empty() +} + +/// Split a shell segment into words, honouring single and double quotes. +fn shell_words(segment: &str) -> Vec { + let mut words: Vec = Vec::new(); + let mut current = String::new(); + let mut has_word = false; + let mut quote: Option = None; + for character in segment.chars() { + if let Some(open) = quote { + if character == open { + quote = None; + } else { + current.push(character); + } + continue; + } + if character == '\'' || character == '"' { + quote = Some(character); + has_word = true; + continue; + } + if character.is_whitespace() { + if has_word { + words.push(std::mem::take(&mut current)); + has_word = false; + } + continue; + } + current.push(character); + has_word = true; + } + if has_word { + words.push(current); + } + words +} + +/// Whether one shell segment runs a workspace-wide cargo test. +/// +/// Every word position is considered as the start of a cargo invocation, so a +/// wrapper program in front of the invocation does not hide it. +fn segment_runs_workspace_cargo_test(segment: &str) -> bool { + let words = shell_words(segment); + words.iter().enumerate().any(|(start, word)| { + is_cargo_program(word) && cargo_invocation_is_workspace_test(&words, start) + }) +} + +/// Whether the cargo invocation that starts at `start` is a workspace-wide +/// cargo test. +fn cargo_invocation_is_workspace_test(words: &[String], start: usize) -> bool { + let mut index = start + 1; + // Step over cargo's global options to reach the subcommand. + while let Some(word) = words.get(index) { + if word.starts_with('+') { + // A toolchain override (`+nightly`) is not a subcommand. + index += 1; + continue; + } + if word.starts_with('-') { + let takes_value = CARGO_VALUE_OPTIONS.contains(&word.as_str()) && !word.contains('='); + index += if takes_value { 2 } else { 1 }; + continue; + } + break; + } + // The first bare word after cargo is the subcommand. + let Some(subcommand) = words.get(index) else { + return false; + }; + if subcommand != "test" { + return false; + } + // `--workspace` (or its `--all` alias) must be an argument of THIS + // invocation, not of another command in the same script. + words[index + 1..] + .iter() + .any(|argument| argument == "--workspace" || argument == "--all") +} + +/// Whether a word is a `NAME=value` environment assignment. +fn is_env_assignment(word: &str) -> bool { + let Some((name, _)) = word.split_once('=') else { + return false; + }; + let mut characters = name.chars(); + match characters.next() { + Some(first) if first.is_ascii_alphabetic() || first == '_' => {} + _ => return false, + } + characters.all(|character| character.is_ascii_alphanumeric() || character == '_') +} + +/// Whether a word invokes cargo. +fn is_cargo_program(word: &str) -> bool { + // An environment assignment is a value, not a program word, even when its + // value happens to end in `/cargo`. + if is_env_assignment(word) { + return false; + } + word == "cargo" || word.ends_with("/cargo") +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The DIL coverage job's command verbatim. The comment mentions "the + /// test binary" and the invocation is `cargo llvm-cov --workspace`; the + /// old word scan classified it as a workspace cargo test. + const DIL_COVERAGE_COMMAND: &str = "command -v cargo-llvm-cov >/dev/null 2>&1 || { echo \"cargo-llvm-cov not in image; skipping\"; exit 0; }\nexport CARGO_BUILD_JOBS=4\n# GF-11 (no job-level lock any more - the runner serializes the\n# jobs that share this host cache directory; see the note below).\n# The runner holds that lock for the job's lifetime, so a job\n# that finds the directory busy waits for it instead of\n# compiling into a tree another job is rewriting.\n# The procedure that used to compensate for the defect - `flock\n# /cargo-target/.dil-cargo.lock` - was removed with the fix.\n# This is the job that died on 2026-11-05 with ENOENT on the\n# test binary; the runner's mount lock is what keeps the tree\n# it executes from being replaced under it.\ncargo llvm-cov --workspace --summary-only --fail-under-lines 95"; + + #[test] + fn a_real_workspace_cargo_test_is_classified() { + assert!(workspace_cargo_test("cargo test --workspace")); + assert!(workspace_cargo_test( + "cargo test --workspace -- --test-threads=2" + )); + assert!(workspace_cargo_test( + "export CARGO_BUILD_JOBS=4\ncargo test --workspace" + )); + } + + #[test] + fn the_dil_coverage_command_is_not_a_workspace_cargo_test() { + assert!( + !workspace_cargo_test(DIL_COVERAGE_COMMAND), + "cargo llvm-cov with a comment mentioning the test binary is not a cargo test" + ); + } + + #[test] + fn another_cargo_subcommand_is_not_a_workspace_cargo_test() { + assert!(!workspace_cargo_test( + "cargo llvm-cov --workspace --summary-only" + )); + assert!(!workspace_cargo_test( + "cargo clippy --workspace --all-targets -- -D warnings" + )); + assert!(!workspace_cargo_test("cargo build --workspace")); + } + + #[test] + fn a_comment_mentioning_the_whole_invocation_is_not_classified() { + assert!(!workspace_cargo_test( + "# run cargo test --workspace later\ncargo fmt --all -- --check" + )); + } + + #[test] + fn a_cargo_test_without_the_workspace_flag_is_not_classified() { + assert!(!workspace_cargo_test("cargo test -p gitforge-scheduler")); + assert!(!workspace_cargo_test("cargo test")); + } + + #[test] + fn an_echoed_invocation_is_not_classified() { + assert!(!workspace_cargo_test("echo 'cargo test --workspace'")); + } + + #[test] + fn the_workspace_flag_must_belong_to_the_cargo_test_invocation() { + assert!(!workspace_cargo_test( + "cargo test -p foo && cargo build --workspace" + )); + assert!(workspace_cargo_test( + "cargo build --workspace && cargo test --workspace" + )); + } + + #[test] + fn a_per_package_test_is_not_a_workspace_cargo_test() { + // The clippy invocation carries `--workspace`; the test invocation + // does not. The old word scan classified this script because the two + // words appeared anywhere in it. + assert!(!workspace_cargo_test( + "cargo clippy --locked --workspace --all-targets -- -D warnings\ncargo test --locked -p aegis-core --all-targets -- --test-threads=1" + )); + } + + #[test] + fn cargo_global_options_do_not_hide_the_subcommand() { + assert!(workspace_cargo_test( + "cargo --manifest-path /workspace/Cargo.toml test --workspace" + )); + assert!(workspace_cargo_test( + "cargo --color always test --workspace" + )); + assert!(workspace_cargo_test("cargo +nightly test --workspace")); + } + + #[test] + fn the_all_alias_is_the_workspace_flag() { + assert!(workspace_cargo_test("cargo test --all")); + assert!(!workspace_cargo_test("cargo test --all-features")); + } + + #[test] + fn an_env_prefixed_invocation_is_classified() { + assert!(workspace_cargo_test( + "CARGO_BUILD_JOBS=4 cargo test --workspace" + )); + assert!(workspace_cargo_test("env FOO=bar cargo test --workspace")); + } + + #[test] + fn a_backslash_continued_invocation_is_classified() { + assert!(workspace_cargo_test("cargo test \\\n --workspace")); + } + + #[test] + fn a_path_to_cargo_is_classified() { + assert!(workspace_cargo_test( + "/usr/local/bin/cargo test --workspace" + )); + } + + #[test] + fn a_wrapped_invocation_is_classified() { + // A wrapper program runs the cargo invocation it is given, so the + // invocation is still a workspace cargo test. Missing it would let two + // workspace cargo tests build the same target tree on one runner. + assert!(workspace_cargo_test( + "flock /cargo-target/.dil-cargo.lock \\\n cargo test --workspace" + )); + assert!(workspace_cargo_test("timeout 60 cargo test --workspace")); + assert!(workspace_cargo_test("nohup cargo test --workspace")); + } + + #[test] + fn a_wrapped_non_workspace_invocation_is_not_classified() { + assert!(!workspace_cargo_test( + "flock /cargo-target/.dil-cargo.lock \\\n cargo llvm-cov --workspace --summary-only" + )); + assert!(!workspace_cargo_test( + "flock /cargo-target/.dil-cargo.lock \\\n cargo clippy --workspace --all-targets" + )); + } +} diff --git a/crates/gitforge-scheduler/src/server.rs b/crates/gitforge-scheduler/src/server.rs index 3b9be420..e8c641d9 100644 --- a/crates/gitforge-scheduler/src/server.rs +++ b/crates/gitforge-scheduler/src/server.rs @@ -13,7 +13,9 @@ use axum::{ routing::{get, post}, Json, Router, }; -use gitforge_common::{JobId, RunnerId, VolumeSpec}; +use gitforge_common::{ + JobExistenceRequest, JobExistenceResponse, JobId, JobRowState, RunnerId, VolumeSpec, +}; use gitforge_db::models::{Runner, RunnerType}; use gitforge_storage::{Artifact, ArtifactId, ArtifactStore, FileStorage, MAX_ARTIFACT_BYTES}; use serde::{Deserialize, Serialize}; @@ -169,6 +171,7 @@ pub fn scheduler_routes_with_tokens( let runner_routes = Router::new() .route("/runners/{id}/heartbeat", post(runner_heartbeat)) .route("/jobs/pending", get(get_pending_jobs)) + .route("/jobs/existence", post(post_job_existence)) .route("/queue/status", get(get_queue_status)) .route("/jobs/{id}/claim", post(claim_job)) .route("/jobs/{id}/started", post(start_job)) @@ -207,6 +210,44 @@ pub fn scheduler_routes_with_tokens( .with_state(state) } +/// Answer, per job id, whether the authoritative store has a row for it. +/// +/// The runner stamps a job-shaped id on every container it creates. For a +/// real job the store has a row, because the row is written when the job +/// is enqueued — before the job can be dispatched to a runner. A container +/// whose job id the store has never heard of was therefore never a job +/// container, which is the only way to tell the warm pool's leftovers +/// apart from an abandoned job. +/// +/// The answer is per id and never global: an id the store cannot speak +/// for is answered `unknown`, and `unknown` never authorises a removal. +async fn post_job_existence( + State(state): State, + Json(request): Json, +) -> impl IntoResponse { + let mut parsed: Vec = Vec::with_capacity(request.job_ids.len()); + let mut states: HashMap = HashMap::new(); + for raw in request.job_ids { + match Uuid::parse_str(raw.trim()) { + Ok(uuid) => parsed.push(JobId::from(uuid)), + // A malformed id is not evidence of anything. Answer `unknown` + // so a parse failure can never become a removal. + Err(_) => { + states.insert(raw, JobRowState::Unknown); + } + } + } + let resolved = state.scheduler.job_row_states(&parsed).await; + for job_id in parsed { + let answer = resolved + .get(&job_id) + .copied() + .unwrap_or(JobRowState::Unknown); + states.insert(job_id.to_string(), answer); + } + (StatusCode::OK, Json(JobExistenceResponse { states })) +} + async fn require_scheduler_auth( request: Request, next: Next, @@ -1045,9 +1086,18 @@ async fn complete_job( .map(RunnerId::from); let lease_token = request["lease_token"].as_str(); + // An explicit, known `outcome` classifies the terminal row; anything + // else keeps the historical boolean mapping (older runners). + let outcome = JobOutcome::from_request(success, request["outcome"].as_str()); + // The receipt records the classification the durable row was graded + // with, not just the runner's boolean. A receipt that kept only + // `success` could not say "the backend failed, not the commit", so a + // stranded infrastructure failure was indistinguishable from a code + // failure when the durable status write was lost (GF-01). let receipt = serde_json::json!({ "job_id": job_id.to_string(), "success": success, + "outcome": outcome.as_str(), "exit_code": exit_code, "error": error, "step_results": request["step_results"].clone(), @@ -1063,9 +1113,6 @@ async fn complete_job( } let assigned_runner = state.scheduler.is_assigned(job_id).await; - // An explicit, known `outcome` classifies the terminal row; anything - // else keeps the historical boolean mapping (older runners). - let outcome = JobOutcome::from_request(success, request["outcome"].as_str()); let completion = match (runner_id, lease_token, assigned_runner) { (Some(runner_id), Some(lease_token), Some(_)) => { state @@ -2199,4 +2246,148 @@ mod tests { .unwrap(); assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); } + + /// The bulk existence endpoint answers per id against the authoritative + /// store: a durable job row reads `present`, an id the store has never + /// heard of reads `absent`, and a malformed id reads `unknown`. + /// + /// The distinction matters because the store row is written at enqueue + /// time, before dispatch: "no row" means "never was a job", which is + /// the only thing that separates a warm-pool leftover from an + /// abandoned job container. + #[tokio::test] + async fn test_job_existence_answers_present_absent_and_unknown() { + let pool = gitforge_db::Pool::memory().await.unwrap(); + pool.migrate().await.unwrap(); + let (known, _other) = seed_restart_scenario(&pool, "job-existence").await; + let absent = JobId::new(); + let app: Router = scheduler_routes_with_tokens( + create_state(crate::Scheduler::with_db(pool)), + Some(Arc::from("runner-secret")), + Some(Arc::from("operator-secret")), + ); + let response = app + .oneshot( + Request::builder() + .method("POST") + .uri("/jobs/existence") + .header("authorization", "Bearer runner-secret") + .header("content-type", "application/json") + .body(axum::body::Body::from( + serde_json::json!({ + "job_ids": [ + known.to_string(), + absent.to_string(), + "not-a-uuid", + ], + }) + .to_string(), + )) + .unwrap(), + ) + .await + .unwrap(); + let status = response.status(); + assert_eq!(status, StatusCode::OK); + let bytes = axum::body::to_bytes(response.into_body(), 64 * 1024) + .await + .unwrap(); + let payload: JobExistenceResponse = serde_json::from_slice(&bytes).unwrap(); + assert_eq!( + payload.states.get(&known.to_string()), + Some(&JobRowState::Present), + "a seeded job row must read present" + ); + assert_eq!( + payload.states.get(&absent.to_string()), + Some(&JobRowState::Absent), + "an id the store has never heard of must read absent" + ); + assert_eq!( + payload.states.get("not-a-uuid"), + Some(&JobRowState::Unknown), + "a malformed id must read unknown, never absent" + ); + } + + /// A scheduler that owns no store cannot speak for jobs another process + /// planned, so every id reads `unknown` — the answer that never + /// authorises a removal. + #[tokio::test] + async fn test_job_existence_without_a_store_answers_unknown() { + let job_id = JobId::new(); + let app: Router = scheduler_routes_with_tokens( + create_state(crate::Scheduler::new()), + Some(Arc::from("runner-secret")), + Some(Arc::from("operator-secret")), + ); + let response = app + .oneshot( + Request::builder() + .method("POST") + .uri("/jobs/existence") + .header("authorization", "Bearer runner-secret") + .header("content-type", "application/json") + .body(axum::body::Body::from( + serde_json::json!({ "job_ids": [job_id.to_string()] }).to_string(), + )) + .unwrap(), + ) + .await + .unwrap(); + let status = response.status(); + assert_eq!(status, StatusCode::OK); + let bytes = axum::body::to_bytes(response.into_body(), 64 * 1024) + .await + .unwrap(); + let payload: JobExistenceResponse = serde_json::from_slice(&bytes).unwrap(); + assert_eq!( + payload.states.get(&job_id.to_string()), + Some(&JobRowState::Unknown), + "a store-less scheduler must answer unknown, never absent" + ); + } + + /// The endpoint sits in the runner's credential scope, because the + /// runner is the only component that needs to ask. A wrong or absent + /// credential is refused before the store is consulted. + #[tokio::test] + async fn test_job_existence_requires_the_runner_credential() { + let app: Router = scheduler_routes_with_tokens( + create_state(crate::Scheduler::new()), + Some(Arc::from("runner-secret")), + Some(Arc::from("operator-secret")), + ); + let refused = app + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri("/jobs/existence") + .header("authorization", "Bearer wrong-secret") + .header("content-type", "application/json") + .body(axum::body::Body::from( + serde_json::json!({ "job_ids": [JobId::new().to_string()] }).to_string(), + )) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(refused.status(), StatusCode::UNAUTHORIZED); + + let anonymous = app + .oneshot( + Request::builder() + .method("POST") + .uri("/jobs/existence") + .header("content-type", "application/json") + .body(axum::body::Body::from( + serde_json::json!({ "job_ids": [JobId::new().to_string()] }).to_string(), + )) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(anonymous.status(), StatusCode::UNAUTHORIZED); + } } diff --git a/docs/CHANGELOG_RECENT.md b/docs/CHANGELOG_RECENT.md index f207a3c2..52e15712 100644 --- a/docs/CHANGELOG_RECENT.md +++ b/docs/CHANGELOG_RECENT.md @@ -4,6 +4,27 @@ All notable changes to GitForge will be documented in this file. ## [Unreleased] +### Fixed + +- **The release channel's busy gate counts the store, not the container + names** (GF-16): `scripts/gitforge-release-auto` decided whether the fleet + was busy by counting the runner's `gitforce-job-*` containers. The runner + keeps a warm pool of containers that have no job row at all, so the count + was permanently non-zero and every cutover was deferred forever — the + release channel could never promote anything, and it reported the + condition as an ordinary "deferred", which reads like normal operation. + The gate now reads the authoritative store through the new + `scripts/gitforge-release-busy`: a job is in flight when its row is + `assigned` or `running` AND its proof of life is still inside the fence + grace window — the same liveness term the scheduler's `requeue_inflight` + uses, so a row left behind by a dead runner cannot wedge the gate either. + Queued and pending rows are queued work and do not block. An unreadable + store fails closed: the pass refuses the cutover instead of starting on + an unknown busy state. Regression: + `tests/release-auto-busy-gate-selftest.sh`, wired into the `lint` job, + which pins the store-backed count, the stale-row case, the fail-closed + behaviour, and the absence of the container filter. + ## [0.6.16] - 2026-10-10 ### Added diff --git a/docs/RUNBOOK.md b/docs/RUNBOOK.md index a7e14dc2..8026ea1f 100644 --- a/docs/RUNBOOK.md +++ b/docs/RUNBOOK.md @@ -168,7 +168,7 @@ GITFORGE_SCHEDULER_TOKEN= \ | `GITFORGE_RUNNER_REGISTRATION_TOKEN` | No | _(none)_ | Bearer token presented to the scheduler's runner-registration surface (`POST /runners` on the CI/scheduler port, and `POST /api/runners` on the gateway). Falls back to `GITFORGE_SCHEDULER_TOKEN` when unset. Registration is fail-closed: with no token configured on the scheduler, every registration is refused (503) | | `GITFORGE_REGISTER_ATTEMPTS` | No | `6` | Registration attempts before giving up when the scheduler is unreachable | | `GITFORGE_REGISTER_BACKOFF_SECS` | No | `1` | Initial registration retry delay; doubles per attempt up to 30s | -| `GITFORGE_CACHE_ROOT` | No | `~/.cache/gitforge` | Host root for job cache mounts (compiled cargo artifacts). Used when a job declares a `cache:` block: the runner bind-mounts `/cargo/` at the job's `mount_path` inside the container. Falls back to `/var/cache/gitforge` when `$HOME` is unset. A cache that cannot be prepared is skipped with a warning — jobs still run cold. | +| `GITFORGE_CACHE_ROOT` | No | `~/.cache/gitforge` | Host root for job cache mounts (compiled cargo artifacts). Used when a job declares a `cache:` block: the runner bind-mounts `/cargo/` at the job's `mount_path` inside the container. Falls back to `/var/cache/gitforge` when `$HOME` is unset. A mount that cannot be prepared fails the job: the runner reports it as an infrastructure failure, so the reason reaches the job's own result instead of a log line. | > **Startup behavior**: If `GITFORGE_SCHEDULER_URL` is missing or empty, the runner exits immediately > with a clear error message. Invalid values for numeric variables (non-integer) also cause a fast @@ -196,31 +196,74 @@ pins. The full cycle is mechanical: # the persisted definition, name for name). scripts/gitforge-release-gate <40-hex-source-commit> -# 2. Bundle: preflight + build release binaries + MANIFEST.sha256 + READY. +# 2. Build the release binaries from the checkout and install them into the +# source root. Every binary is stamped with the commit it was built from; +# one that does not report that commit is never installed. +scripts/gitforge-release-build /nas/Temp/repos/GitForge + +# 3. Bundle: preflight + provenance check + MANIFEST.sha256 + READY. GITFORGE_SOURCE_COMMIT= \ scripts/gitforge-release-bundle /nas/Temp/repos/GitForge \ /nas/Temp/repos/GitForge/releases gitforge-- -# 3. Promote + restart, drain-gated and rollback-ready: +# 4. Promote + restart, drain-gated and rollback-ready: ./scripts/gitforge-release-auto # dry-run: prints the plan ./scripts/gitforge-release-auto --apply # or AUTO_UPDATE=1 ``` `gitforge-release-auto` refuses to run twice concurrently (flock on -`releases/.auto-update.lock`), defers while any `gitforge-job-*` container is -still running (drain gate — pending-only runs are just queued and do not -block), flips `gitforge-current` atomically, restarts the `gitforge@*` units, -and leaves the previous bundle in place for instant rollback (re-point the -symlink and restart). Verify after cutover: `systemctl` unit states, `/health` -on :42780/:42781/:42782, and `scripts/gitforge-status` with +`releases/.auto-update.lock`), defers while the authoritative store reports a +job in flight, flips `gitforge-current` atomically, restarts the `gitforge@*` +units, and leaves the previous bundle in place for instant rollback +(re-point the symlink and restart). Verify after cutover: `systemctl` unit +states, `/health` on :42780/:42781/:42782, and `scripts/gitforge-status` with `GITFORGE_RELEASE_ROOT=/nas/Temp/repos/GitForge/releases/gitforge-current` (the default path reports false drift on this instance). +The busy signal is read from the authoritative store by +`scripts/gitforge-release-busy` — never from the runner's container names. A +job counts as in flight when its row is `assigned` or `running` AND its proof +of life is still inside the fence grace window +(`GITFORGE_JOB_FENCE_GRACE_SECS`, default 300s — the scheduler's own +default, read from the same knob). Queued and pending rows are queued work +and do not block. Two properties follow from that definition: + + - the gate cannot be wedged by a stale row. A `running` row whose runner + died is fenced by the scheduler itself once its proof of life ages past + the grace window; counting it as busy would defer every cutover until + someone intervened by hand. + - the container names cannot be the signal. The runner keeps a warm pool of + containers that have no job row at all, so counting them is permanently + non-zero (GF-16, 2026-11-06): every cutover was deferred forever, and + the channel reported that as an ordinary "deferred", which reads like + normal operation. + +If the store cannot be read, the pass refuses the cutover (exit 1) rather +than guessing. An unreadable store is a failure to surface, not a clean +deferral — a silently unreadable store stalls the channel with no signal. + Known boot behavior: the ci unit runs migrations, then the startup workspace sweep, then spawns the event consumer. Triggers published while the consumer is down or busy are durable and land once it drains — a `202` with a null `pipeline_run_id` means queued, not lost; re-query before re-triggering. +Every release binary carries the commit it was built from, and the release +chain proves it. The build stamps the commit into the binaries +(`crates/gitforge-common/build.rs`, surfaced as `--version` on each service); +`gitforge-release-build` refuses to install a binary that does not report the +checkout's commit; `gitforge-release-preflight` and `gitforge-release-verify` +refuse a bundle whose binaries disagree with the `source_commit` the bundle +claims. The claim is therefore carried by the binaries themselves rather than +by whoever assembled the bundle. + +This check exists because the release binaries live in gitignored paths at the +root of the source root: a bundle assembled from stale leftovers passed every +other gate — they were present, they were executable, and the checksum +manifest was computed from whatever was copied — while stamping a commit they +were never built from (GF-14, 2026-11-05). The build must therefore run in a +git checkout; a build from a source copy sets `GITFORGE_BUILD_COMMIT` (and +`GITFORGE_BUILD_DIRTY`) explicitly, and the same check applies. + Bundle cuts are atomic as of 2026-10-10: `gitforge-release-bundle` assembles under `releases/.incomplete-/` and renames into place only once `READY` exists, so a half-cut bundle can never sit at the final name (and a @@ -246,7 +289,10 @@ scripts/gitforge-remote-cutover --units user \ ``` The script polls the job table until nothing is `assigned`/`running` (a -pending-only backlog does not block), stops the scheduler units first +pending-only backlog does not block), using the same reader as the NAS +promote — so a row left behind by a dead runner does not hold the drain +either, and the sqlite3 CLI is no longer required on the target host +(the python3 sqlite3 fallback serves it), stops the scheduler units first (`ci`, then `runner`, then `api`/`git-server`), flips the symlink with `mv -T`, restarts stateless before schedulers, and prints the boot-adoption journal lines. Units that are not installed — or masked (retired) — on the diff --git a/docs/audits/OPENCODIFIER_QUALITY_2026-11-03.md b/docs/audits/OPENCODIFIER_QUALITY_2026-11-03.md new file mode 100644 index 00000000..30e6d363 --- /dev/null +++ b/docs/audits/OPENCODIFIER_QUALITY_2026-11-03.md @@ -0,0 +1,113 @@ +# OpenCodifier Quality Audit — GitForge — 2026-11-03 + +Scope: procedural workarounds (docs compensating for tool defects), code smells, test quality, CLAUDE.md anti-patterns. Read-only audit. + +## 1. Procedural workarounds + +Discriminator applied: does skipping the procedure make the tool misbehave? + +1. **HIGH — sqlx stale prepared-statement panic; procedure = "stop everything, then start together."** + `docs/RUNBOOK.md:264-276`: a live migration (ALTER TABLE) on the shared SQLite DB while peers hold prepared `SELECT *` statements causes `SqliteRow::current index out of bounds` panics (ledger F40, 2026-10-10). The runbook prescribes a drain-gated stop-everything-then-start-together sequence. Underlying defect (peers don't handle schema-change reprojection) is fixable in code: version-check handshake at connection setup (fail fast with a clear error), or pool recycling on schema-version mismatch. Disposition: fix now — bounded change; keep the cutover procedure until shipped and mark it as debt with the defect reference in RUNBOOK. +2. **HIGH — masked-unit auto-update rollback (tool fixed; needs regression test).** + `docs/RUNBOOK.md:283-288`: NAS runner unit is masked; pre-fix `gitforge-release-auto` made every release roll back forever. Scripts now skip masked units. Disposition: fix now — add a regression test for the masked-unit skip in scripts/gitforge-release-auto so the failure can't silently return. +3. **MED — GitHub auto-merge from CI fails; documented manual-merge workaround.** + `docs/planning/PHASE2_PLAN.md:133`: third-party GITHUB_TOKEN can't set auto-merge (needs a GitHub App). Workaround: manual merge per PR. Fixable by minting an App token or merging host-side. Disposition: fix now (small script); record as debt if no App is wanted. +4. **MED — GitHub ruleset temporary-bypass dance, documented 4+ times.** + `docs/planning/CAMPAIGN_PLAN_2026-09-28.md:72,208`, `CAMPAIGN_PLAN_2026-10-03.md:208`, `docs/BRANCH_STRATEGY.md:75`, `docs/MIRROR_POLICY.md:49`: back up rulesets → bypass grant → merge → restore. Branch protection itself is legitimate policy, but the manual grant/restore cycle is a tool gap. Disposition: record as debt with owner — a `scripts/` helper wrapping bypass+merge+restore atomically converts the procedure into a tool. +5. **MED — SELinux labeling gotcha for ad-hoc docker probes.** + `docs/RUNBOOK.md:260-263`: ad-hoc bind-mount probes need `--security-opt label=disable` or writes fail despite correct modes; the runner's own job workspaces already handle labeling. The fix exists in the tool but ad-hoc scripts don't reuse it. Disposition: fix now — bake labeling handling into scripts that bind-mount host dirs. +6. **LOW — systemd CLI quirks (`XDG_RUNTIME_DIR`, `.service` suffix).** + `docs/RUNBOOK.md:255-259`. Environment quirk, not a GitForge defect. Disposition: no action. +7. **LOW — `GITFORGE_SCHEDULER_TOKEN` "temporary compatibility fallback" with no owner or removal condition.** + `docs/HARNESS_JOB_CONTRACT_V1.md:9`. Disposition: record as debt — add owner + removal milestone per Fix-the-Tool rule 2. +8. **HIGH (stale-doc risk) — runner registration fail-open.** + `docs/planning/HANDOFF.md:138-146` lists it as a known issue, but `crates/gitforge-runner/src/agent.rs:57-60,124,329` shows `allow_standalone` now defaults to **false** (fail-closed, opt-in legacy mode). HANDOFF is a stale false record — the P0 was fixed but the doc still says fail-open. Disposition: fix now — update HANDOFF (or archive it with a status header); verify the config path that sets `allow_standalone` is denied in the trust policy. + + +1. **HIGH — `crates/gitforge-db/src/queries.rs` is 7,253 lines.** Confirmed. Single file holds Repo/Pipeline/Run/User queries with `#[cfg(test)]` modules inline. Disposition: record as debt; split by aggregate into `queries/{repo,pipeline,run,user}.rs` in a dedicated refactor milestone. +2. **MED — `services/ci/src/main.rs` is 6,493 lines.** A main.rs of this size means routing + handlers + state live in the binary crate rather than the `gitforge-ci` library (engine.rs is a separate 1,697). Disposition: record as debt; move handlers into `gitforge-ci` so tests exercise the library, not a spawned binary. +3. **MED — triplicated `error_response` helpers in gitforge-api.** `crates/gitforge-api/src/server.rs:334`, `src/routes/ssh_keys.rs:60`, `src/routes/review.rs:105` — three near-identical JSON error constructors with slightly different signatures (`code: &str` vs `error: &str`, `String` vs `&str` message). Classic copy-paste error handling. Disposition: fix now — small mechanical consolidation into one helper. +4. **LOW — unwrap/expect production sites: 0** per `scripts/unwrap-production-count.py` (2,144 raw grep hits are test contexts + `#[cfg(test)]` modules; spot check of queries.rs:4013 confirmed test code). CLAUDE.md rule honored. Disposition: none — the wrapper script is doing its job. +5. **LOW — no TODO/FIXME/HACK markers in crates/ or services/ non-test code; no versioned files (`*_v1`, `*.bak`); no hardcoded credential literals found.** Disposition: none. +6. **MED — `crates/gitforge-scheduler/src/assigner.rs` (4,435 lines) and `crates/gitforge-runner/src/agent.rs` (4,135 lines)** oversized single files. Disposition: record as debt alongside #1. +7. **Info — `services/ci` vs `crates/gitforge-api/routes/ci.rs` (1,583 lines)**: API crate re-implements CI route handlers adjacent to the service binary; boundary between thin gateway and owning service is blurred. Same disposition as #2. + +## 3. Test quality + +1. **MED — status-code-only assertions outnumber body checks in several route suites.** Counts of `assert_eq!(status, ...)` vs body-content asserts: `artifact_routes.rs` 18 vs 10, `runners_routes.rs` 23 vs 19, `ssh_key_routes.rs` 12 vs 7. Several artifact tests assert `StatusCode::OK` without reading the artifact payload (e.g. artifact_routes.rs:248, :295). A route that returns 200 with the wrong/empty body passes. Disposition: fix now — add payload assertions to the OK paths in these three files (bounded, ~15 sites). +2. **LOW — good counterexamples exist**: ci_routes.rs:190-202 checks ownership enforcement with body content AND forbidden body text; webhook_routes.rs has 35 body asserts vs 15 status. The pattern is established; suites in #1 just lag it. +3. **HIGH — `#[ignore]`-gated Docker integration tests are the ONLY coverage of runner executor timeout reaping and sandbox behavior.** `crates/gitforge-runner/src/executor.rs`, `crates/gitforge-sandbox/src/docker.rs` carry `#[ignore]` tests that pass only `cargo test -- --ignored` against a live daemon (docs/planning/IMPROVEMENTS.md:53-57). Nothing in the pre-push gate runs them; a release gate that never sets `--ignored` gives a false pass on the isolation/timeout path. Disposition: fix now — wire `--ignored` into the CI job that runs on the docker-capable fedora runner (one .gitforge.yml line), and keep the ignore for local runs. +4. **HIGH — coverage gate silently skips: 11 `return 0` escape hatches, zero Rust support.** `.githooks/pre-push.d/30-coverage-gate` only understands Go and Python coverage profiles; for this Rust workspace it always exits 0 without running anything — a false-pass gate wearing a gate's name. Same story for `10-complexity-gate` (6 skips, no Rust). This is the purest OpenCodifier class: the *procedure* (coverage gate) exists; the *tool* (the gate script) does not measure this repo's language. Disposition: fix now — replace both with the repo's actual Rust routes (cargo-tarpaulin/lcov via `scripts/qualityctl`, which exists) or delete the Dark Factory Go/Python scripts from this repo. +5. **LOW — pre-push smoke test is 2-thread bounded to dodge a known load race** (`.githooks/pre-push:9-12`: git_http_edges / ci_trigger_flow flake class). This is a documented workaround for test isolation (spawned-binary protocol tests racing under load). Underlying fixable defect: the tests share ports/dirs or lack backpressure. Disposition: record as debt with owner — fix the race, then unbound the parallelism. + +## 4. CLAUDE.md anti-patterns (Rust standards) + +Covered in §2 items 4-5: production unwrap/expect = 0 (verified via scripts/unwrap-production-count.py + spot checks), no TODO/FIXME/HACK, no versioned/backup files, no hardcoded credential literals in crates/ or services/. One caveat: `services/*/src/main.rs` binaries were included in the scan; repo-wide exclusions (fuzz/, target/) are not production code. Disposition: clean. + +## Summary + +GitForge's discipline is unusually good for the OpenCodifier failure classes: zero production unwraps, no TODO debt, fail-closed runner (though HANDOFF.md still claims otherwise). The four fix-now items: (1) pre-push coverage/complexity gates are Go/Python-only scripts that always exit 0 on this Rust workspace — false-pass gates, replace or delete; (2) #[ignore] Docker tests (executor timeout reaping, sandbox) never run in any gate; (3) HANDOFF.md stale fail-open claim contradicts agent.rs:124 fail-closed default; (4) ~15 status-only assertions in artifact/runners/ssh_key route suites. Highest-value debt: sqlx schema-change panic masked by the stop-everything cutover procedure (RUNBOOK F40), and the 7k-line queries.rs / 6.5k-line ci main.rs monoliths. + +## 5. Phased plan derived from this audit (2026-11-03) + +Item 3 of the Summary was fixed during this session: HANDOFF.md:140 now +points at the RESOLVED verdict under "Next Steps". The rest, ordered by +phase. + +### Phase P0 — false-pass gates (fix first; they mask everything else) +1. DONE 2026-11-03 — both Dark Factory gate scripts removed from + `.githooks/pre-push.d/`. The surviving `pre-push.d` member + (`20-clean-branch`, advisory) is now invoked from `.githooks/pre-push` + with a real commit range (upstream..HEAD) instead of git's + ref-vs-itself arguments, so its commit-message check actually runs. + The Rust routes stay where they were: `.gitforge.yml` coverage job + and `scripts/gitforge-quick-check.sh`. +2. DONE 2026-11-03 — new `docker-integration` job in `.gitforge.yml` + (test → docker-integration → coverage chain) runs + `cargo test -p gitforge-sandbox -p gitforge-runner --lib --tests + -- --ignored --test-threads=2`. The job mounts the daemon socket via + the `{docker_socket}` volume placeholder (resolves to the socket of + the daemon THIS runner drives — rootful docker on the NAS, rootless + podman on fedora), and the sandbox runs socket-mounting containers + with SELinux labeling disabled after live probes on both hosts + confirmed the refusal without it. `#[ignore]` is kept for local + runs. A `gitforge-ci` test pins the job's presence, socket mount, + and the `--ignored` filter in its step. +3. DONE 2026-11-03 — `scripts/gitforge-release-auto` gained + `--print-restart-plan`, and + `tests/release-auto-restart-plan-selftest.sh` asserts with a fake + `systemctl` that a masked unit is excluded from the restart plan + (both directions pinned; the test fails when the skip is reverted). + Wired into the `.gitforge.yml` lint job and the shellcheck glob. + +### Phase P1 — correctness consolidation (bounded, mechanical) +4. Consolidate the three `error_response` helpers (server.rs:334, + routes/ssh_keys.rs:60, routes/review.rs:105) into one. +5. Add payload assertions to ~15 status-only OK paths in + artifact_routes.rs, runners_routes.rs, ssh_key_routes.rs. +6. Land the sandbox-timeout fix already staged in executor.rs (job + timeout inherited, not a hardcoded 3600s ceiling), with a pinned test + citing run 0d4b44c4. +7. Container hardening: `cap_drop`, `no-new-privileges:true`, explicit + non-root `user` in `gitforge-sandbox/src/docker.rs` HostConfig. + +### Phase P2 — debt with owners (one per defect class) +8. sqlx schema-change handshake: version-check at connection setup, + fail-fast with a clear error (ledger F40); then shrink the RUNBOOK + stop-everything cutover to the residual case, referencing the defect. +9. `GITFORGE_SCHEDULER_TOKEN` fallback: add owner + removal milestone in + HARNESS_JOB_CONTRACT_V1.md. +10. Ruleset bypass helper: `scripts/` wrapper for grant→merge→restore. +11. Pre-push 2-thread bound: fix the git_http_edges/ci_trigger_flow + port/dir race, then unbind. + +### Phase P3 — structural (dedicated milestones, no behavior change) +12. Split `gitforge-db/src/queries.rs` (7,253 lines) into + `queries/{repo,pipeline,run,user}.rs`. +13. Move `services/ci` handlers into the `gitforge-ci` library crate. +14. Split `scheduler/assigner.rs` (4,435) and `runner/agent.rs` (4,135). +15. Docs scaffolding: add `docs/PROJECT_STRUCTURE.md`/`CONVENTIONS.md`. + +### Deferred / not planned +- GitHub App for auto-merge (only worth it if PR volume justifies it). +- SELinux labeling in ad-hoc probe scripts (operational, not product). diff --git a/docs/knowledge/CONSOLIDATED_OPERATIONS.md b/docs/knowledge/CONSOLIDATED_OPERATIONS.md index 659ee71c..25db846f 100644 --- a/docs/knowledge/CONSOLIDATED_OPERATIONS.md +++ b/docs/knowledge/CONSOLIDATED_OPERATIONS.md @@ -140,9 +140,10 @@ Source: docs/RUNBOOK.md line 168 (`GITFORGE_CACHE_ROOT`); the - The runner bind-mounts `/cargo/` at the job's `mount_path`; default `~/.cache/gitforge`, fallback `/var/cache/gitforge` when `$HOME` is unset - (crates/gitforge-runner/src/agent.rs:2756). -- A cache that cannot be prepared is skipped with a warning; jobs run - cold (RUNBOOK line 168). + (crates/gitforge-common/src/volume.rs, `cache_root_with`). +- A mount that cannot be prepared fails the job; the runner reports it + as an infrastructure failure, so the reason lands in the job's own + result (RUNBOOK line 171). - **Rollout gate**: a runner binary older than the feature silently ignores the `volumes` field (serde drops unknown fields, crates/gitforge-common/src/volume.rs:35-40), so a green run proves diff --git a/docs/planning/CI_STORAGE_EFFICIENCY_ARCHITECTURE_2026-10-31.md b/docs/planning/CI_STORAGE_EFFICIENCY_ARCHITECTURE_2026-10-31.md deleted file mode 100644 index 4fedb4c6..00000000 --- a/docs/planning/CI_STORAGE_EFFICIENCY_ARCHITECTURE_2026-10-31.md +++ /dev/null @@ -1,403 +0,0 @@ -# GitForge Execution Efficiency Architecture - -**Date:** 2026-10-31 -**Status:** Proposed implementation architecture; no runtime behavior changes in this documentation PR -**Umbrella:** #342 -**Workstreams:** #338, #339, #340, #341 - -## 1. Executive recommendation - -GitForge should evolve into a resource-aware execution platform with three coordinated foundations: - -1. **Admission-controlled execution:** separate control-plane traffic from compute, and schedule fast, standard, heavy and background work with explicit resource reservations and fairness. -2. **Immutable shared storage:** use a content-addressed store for reusable non-Git artifacts, with explicit manifests/references, per-trust-domain namespaces, managed workspaces and lease-aware garbage collection. -3. **Sound incremental validation:** map code changes to tests/features, reuse prior test results only when all relevant inputs are fingerprint-identical, and fall back to wider validation whenever the graph or cache cannot prove safety. - -Do not start by replacing GitForge's scheduler, Git storage, Cargo, Docker, or every language toolchain. Preserve the current durable queue/lease/receipt model, build the missing contracts around it, benchmark in controlled stages, and land independently deployable changes. - -The highest leverage is likely to come from queue admission, avoiding duplicate heavy validation, shared immutable dependency/compiler artifacts, and isolation of writable workspaces. Changing implementation languages or adopting an experimental compiler is a secondary, evidence-driven optimization. - -## 2. Current-state audit - -Repository baseline inspected: main at 5ceb106d60f3eb528392daac6ad258cc317edc18 (latest main commit visible during this audit). - -### 2.1 Existing strengths to preserve - -- Rust workspace with separate API, CI engine, scheduler, runner, sandbox, process, build, storage, database, Git protocol and CLI crates. -- Durable scheduler/database job state, leases, cancellation, runner heartbeat handling and restart recovery. -- Pipeline DAG with dependencies, environment propagation, skip conditions, and a cache block that expands into a host volume plus tool environment. -- Queue has Normal/High/Low priorities and repository-aware fairness based on currently running jobs. It also exposes fair ordering for queue-status reporting. -- A narrow resource class already prevents two workspace-wide Cargo test jobs from overlapping on one runner. -- Runner Docker integration supports host binds, tmpfs and named volumes, timeout/resource limits, sandbox cleanup and backend health checks. -- Storage traits exist for artifacts, caches, logs and receipts. The open #336 proposal adds atomic publication of individual file-backed records. -- The self-hosted pipeline already uses image-keyed Cargo target mounts, and a mount failure is designed to degrade to a cold build rather than change correctness. - -These are valuable primitives. The proposed work should extend them rather than create duplicate job identity, object identity, lease, or receipt systems. - -### 2.2 Confirmed gaps - -| Area | Current state | Consequence | -|---|---|---| -| Queueing | Priority heap and repository fairness, but no general resource-vector admission or explicit fast/heavy lanes | Large jobs can compete with small validation and consume scarce host resources | -| Queue fairness | Priority is ordered before repo load; aging/weighted service guarantees are not a complete contract | Continuous higher-priority traffic can starve lower classes | -| Resource classes | One special workspace-Cargo-test class; resource use is not generalized across jobs | CPU, RAM, pids, scratch and I/O are not admitted as one budget | -| Cache identity | CacheKey components are concatenated and filesystem name uses 16 hex characters | Ambiguous component boundaries and truncated identifiers should be removed | -| Cache storage | Key-addressed data plus JSON metadata; no shared immutable blob layer or explicit global reference graph | Identical content is duplicated; retention and GC are difficult | -| CI cache mount | Cargo target path keyed by a sanitized image string | Different image names can normalize to the same path; cache correctness should use canonical digests | -| Workspaces | Native Git worktrees/manual lifecycle and runner workspaces | Ownership/cleanup and per-job I/O costs remain operator burdens | -| Concurrent builds | Current self-pipeline is intentionally a linear chain because the jobs share a mutable workspace/target | Parallelism is currently limited; simply fanning out jobs would be unsafe | -| Test selection | Main has changed-file classification and job-level skip_if filtering. A prior, unmerged #322 branch adds affected-crate graph selection; it is not integrated into main. Per-test change-impact and trusted result reuse are not on main | Full test/coverage work can still be repeated unnecessarily | -| Observability | Queue/job status, receipts, logs and metrics exist | Missing unified queue wait, host pressure, physical I/O, cache effectiveness and workspace-cost signals | - -The October 10 worktree census documented 44 live Git worktrees: 25 clean, 17 with small dirty WIP, one additional dirty main worktree, and one 3,962-file build-artifact spill. The important finding is not the exact historical count; it is that Git cleanup must be owner/lease aware and distinguish build output from source changes. - -### 2.3 Existing work to review rather than duplicate - -- Issue #322 and branch docs/gitforge-smart-ci-dependency-graph-20261019 contain a gitforge-graph crate, cargo-metadata graph ingestion, affected-crate and feature queries, fast CLI checks, a changed-crate script, feature/known-issue data, and an impact-run migration. At audit time that branch was six commits ahead but 70 commits behind main. Rebase and independently review those commits; do not merge the stale branch wholesale. -- Issue #277 has actual evidence of extreme host pressure, D-state waits, high storage reads/writes and low free disk during heavy CI. This is the rationale for admission and resource telemetry. -- Issue #284 documents overlapping Rust CI, Integration Tests, GitForge CI and other Actions lanes competing for the same self-hosted pool. -- Issue #318 documents public fork PR code executing on persistent self-hosted infrastructure and required-check gaps. This is a security gate for any shared cache or runner design. -- PR #336 proposes atomic same-directory temporary publication for individual artifact/cache/receipt files. It does not provide a transaction spanning data and metadata, and does not yet supply the CAS/refcount/GC architecture. -- PR #337 and PR #258 are separate release-contract and SSH-trigger work. Do not mix their ownership with the storage/queue changes. - -### 2.4 Current validation blockers observed - -At main commit 5ceb106d, the October 30 workflows showed: - -- Rust Build/Test/Clippy failed compiling services/git-server due to an incompatible russh auth_publickey callback signature in services/git-server/src/ssh_server.rs. -- Rust Security failed compiling services/api/tests/api_gateway_flow.rs because the test referenced an undefined harness_passphrase. -- Supply Chain failed cargo-vet because 15 dependency versions lacked safe-to-deploy audit attestations. -- CodeQL Rust analysis was blocked by the same git-server compilation error. -- The GitForge self-hosted poll step received HTTP 401 when querying the pipeline-runs API, causing the status adapter to time out/fail rather than read the durable run outcome. -- PR #336's inspected head had Build, Test, Clippy, Coverage, ordinary Security Audit and Integration green, but Supply Chain still failed on the 15 missing attestations and GitForge polling still returned 401. Its full required gate was not green. - -These are release blockers. Cache optimization cannot turn an uncompiled or unauthenticated gate into valid evidence. Resolve them in Gate 0 and do not report a green release based on partial CI. - -## 3. Target system architecture - -### 3.1 Separate control-plane and compute-plane concerns - -Keep API, auth, Git smart HTTP/SSH, webhooks, trigger/outbox delivery, runner heartbeats, cancellation, queue inspection and receipt writes responsive under compile/test load. - -No expensive compilation, package installation, test selection scan, archive extraction, or large filesystem traversal should run synchronously on a request path. Persist the intent/outbox record, acknowledge it within a bounded durability budget, and let workers perform the compute asynchronously. - -Measure the request path separately from job throughput. An idle API must not be penalized by a large number of queued jobs that have not been admitted to execute. - -### 3.2 Work lanes - -| Lane | Intended work | Scheduling objective | -|---|---|---| -| Control | Auth, Git refs/receive-pack response, webhook persistence, heartbeats, cancel/status, lightweight metadata | Predictable low tail latency | -| Fast | Format, manifests, route/schema checks, changed-file static checks, targeted small tests | Short feedback time; reserved capacity | -| Standard | Normal build and affected test suite | Fair throughput | -| Heavy | Full workspace tests, coverage, fuzz, mutation, release build, large integration suites | Bounded resource consumption and fair progress | -| Background | Cache prewarm, graph rebuild, GC, log compaction, scheduled full validation | Run only with spare capacity or explicit schedule | - -These are scheduling classes, not five independent schedulers. Keep one durable job identity and one lease contract. Add lane and resource request fields to that contract. - -Fast-lane capacity must be reserved but bounded. Use weighted service/aging so a constant stream of fast work cannot starve heavy validation. Heavy work must be deferred fairly, not silently lost. Queue status should state exactly why an item is waiting. - -### 3.3 Resource-vector admission - -Each job declares or receives a resource class with a request/limit vector: - -- CPU quota / shares and maximum child process count. -- Memory reservation and hard limit. -- Scratch/workspace/cache capacity. -- I/O weight or concurrency budget where supported. -- Network policy and optional egress class. -- Expected wall time and timeout. -- Runner/image/platform requirements. - -The scheduler admits work only when both runner capacity and host-level headroom allow it. Host metrics should include cgroup CPU/memory, PSI CPU/memory/I/O where available, disk free bytes/inodes, I/O throughput/latency, Docker storage headroom, active container count, process count and queue wait. - -Use measured runner profiles, not hard-coded guesses. A 2-core/2-GB host should normally admit far fewer concurrent heavy compiles than a larger runner. A small fast lane may still run while one heavy task is executing, provided measured budgets allow it. - -When there is not enough capacity, leave the job durably queued with a reason such as runner_unavailable, cpu_budget, memory_headroom, scratch_headroom, io_pressure, per_repo_limit, lane_limit or dependency_wait. Do not spawn an unbounded number of tasks that only wait for semaphores. - -### 3.4 Fairness and overload behavior - -- Weighted fair dispatch across tenant and repository, with per-repo in-flight limits. -- Priority aging so older normal/heavy jobs eventually advance. -- Per-lane concurrency and reserved fast-path slots. -- Admission caps on total pending work, with explicit policy response/backpressure to the trigger source when safe. -- Idempotent trigger coalescing only when semantics match (same repository, ref/commit, pipeline definition digest, actor/trust context and trigger class). Never merge a manual run or privileged run into an untrusted PR run. -- Queue reasons and age visible through CLI/API/metrics. -- Circuit breakers for degraded storage/runner state; keep health/status endpoints light. - -### 3.5 Workspace and storage model - -Keep Git's own object database canonical for commits, trees and blobs. Git worktrees already share Git object storage; the extra bytes to address are checked-out working files, Cargo target/build outputs, package/dependency stores, container layers, test outputs, logs and generated artifacts. - -Introduce a content-addressed store (CAS) for immutable non-Git objects: - -- Full digest (BLAKE3 or SHA-256) and size, verified on write/read. -- Canonical, length-delimited encoding for cache keys; do not concatenate ambiguous fields or use truncated digest prefixes as identity. -- Immutable blob paths and manifest/reference records. -- A durable ref/index that points from tenant/repository/job/commit/toolchain manifests to object digests. -- Namespace and trust policy on every reference. -- Per-object temporary write, file sync and atomic rename; recovery reconciles partial file/index publication. -- Optional chunked storage for large archives/logs only after measurement shows dedup savings exceed hashing, indexing and metadata overhead. - -Cache identity must cover the complete reproducibility boundary: source tree digest, lockfile(s), toolchain version, target triple, OS/arch/ABI, runner image digest, features/profile/build flags, pipeline definition, relevant environment contract, test binary/args and graph schema. Never put secret material in the key; use a secret-version identifier only when a secret affects the output and the execution semantics can prove it. - -The current sanitized image string used for Cargo cache paths should be replaced by a canonical image-reference digest. Sanitization can collide (different punctuation can become the same hyphen). - -#### Workspace providers - -Use a portable provider API with layered backends: - -1. Native Git worktree and separate target/cache directory: portable baseline. -2. Reflink copy or OverlayFS lower/upper/work directories on supported Linux filesystems. -3. Btrfs read-only base subvolume plus per-job writable snapshot when the runner filesystem is Btrfs and all relevant paths reside on the same filesystem. -4. Safe ordinary copy fallback. - -A workspace record owns its path, repository, commit, branch/ref, creator/agent/job, lease, state, filesystem provider, base digest, created/last-used time and cleanup policy. - -Do not hard-link mutable files. A change in a hard-linked file can mutate the shared inode. Do not symlink writable target output to a shared location. Shared package/cache store entries should be immutable and read-only to consumers; only use same-filesystem hard links where the provider, trust model and mutation guarantees are proven. - -Do not share a writable Cargo target directory between concurrent unrelated builds. Parallel CI lanes need separate workspaces and target outputs, or a cache layer that deduplicates compiler invocations without concurrent mutation. A shared Cargo registry/git source download cache is a different risk class from sharing target/. - -Btrfs snapshots share copy-on-write extents and are useful for quick workspace branching, but they are not backups. Keep backup/restore independent. Check snapshot support and quota behavior on the host before relying on it; retain a non-Btrfs fallback. - -#### CAS references and garbage collection - -Use explicit references and leases. GC must: -1. Mark live objects from durable manifests, active jobs/leases, worktrees, pinned releases, retention rules and recovery records. -2. Produce a dry-run report with object IDs, logical/physical bytes, reason and owner. -3. Move eligible unreferenced objects through a quarantine/grace period. -4. Recheck references/leases and only then delete. -5. Reconcile orphan metadata and orphan blob files without guessing ownership. - -Protect against the publish-then-reference race: a just-published object must be protected by an upload/transaction lease until the durable reference is committed. GC must never remove an object merely because its reference registration has not completed yet. - -Expose logical bytes, physical bytes, dedup ratio, cache hit/miss, verified reads, corrupt objects, active pins, leased bytes, garbage candidates and bytes actually reclaimed. - -### 3.6 Session, log and debug knowledge compaction - -This is a distinct data lifecycle built on the immutable store, not a lossy replacement for raw logs. - -Keep the raw source immutable and recoverable. Add derived layers: - -- Normalize logs and conversation traces into structured events (timestamp, source, tool, command, result, file/path references, error, test outcome, token/resource usage). -- Compress repetitive streams and deduplicate byte-identical chunks. -- Store extracted decisions, constraints, fixes, test evidence and reusable knowledge as a versioned summary/knowledge artifact. -- Include source digest and exact byte/line/turn offsets for provenance. -- Provide search/retrieval back to the raw object. -- Make retention policy explicit per raw logs, traces, summaries and artifacts. - -A 2% retained-size target is a benchmark hypothesis, not a promise. A summarizer must not discard critical decisions, error messages, security findings, commands, test evidence or provenance. Build a golden recall corpus; compare extracted knowledge against source and require exact retrieval of critical facts before raw retention can be reduced. - -## 4. Sound incremental validation and test-result reuse - -### 4.1 Reuse the existing graph work - -Rebase issue #322's graph branch onto current main and inspect each commit. Retain the graph/node/edge model and CLI fast-check pieces where correct, but update storage, pipeline and scheduler integration contracts. Do not assume that crate-level reverse dependencies are sufficient for test-level correctness. - -### 4.2 Build a test impact index - -Collect: -- Static dependencies from Cargo metadata and language build/project manifests. -- Test-to-code dynamic coverage data, ideally file/function/symbol granularity. -- Test binary identity, filter/arguments, fixtures, environment and runner image. -- Feature-to-code and feature-to-test links. -- Known failing, flaky, ignored, global-state, integration and network-dependent tests. - -For a change, compute the affected closure. A changed implementation file selects tests that executed that code and tests in dependent crates/modules. New/changed tests always execute. Failed/flaky tests are never reused from a prior pass. Global integration tests and tests with unknown dependencies remain included. - -### 4.3 Safe test cache - -A prior pass is reusable only if the complete test fingerprint matches: - -- test identity/binary and test args/filter; -- source and transitive dependency digests; -- dependency lockfiles and generated inputs; -- compiler/runtime, target, OS/arch/ABI and runner image; -- build profile, features and relevant flags; -- fixtures, configuration and environment contract; -- graph/index schema and execution adapter version. - -Tests that observe time, random sources, network, mutable external services, global state, host files or non-hermetic secrets are non-cacheable until those inputs are captured or isolated. - -Record an explicit result class: executed_pass, executed_fail, flaky, timeout, skipped_by_selection, cache_reused, or not_eligible. Cache reuse must not be represented as a test that just ran. Receipts should show the cached result's source commit/fingerprint and age. - -If graph construction, coverage, cache lookup, object verification or environment matching fails, fail open to the conservative test set (up to the full suite). Require periodic full-suite and full coverage/fuzz jobs, plus release full validation. During rollout, run selected and full suites on the same commit and compare; any missed failure blocks enabling skip decisions. - -### 4.4 Shared test-runner interface - -Use the existing OpenTestSuite prototype as an external test adapter project rather than creating a parallel test framework inside the scheduler. The adapter contract should support discovery, selection, execution, retries, per-test resource labels, test groups, output/trace/coverage collection, structured result, artifact publication and deterministic environment identity. - -For Rust, evaluate cargo-nextest as the execution runner, preserving Cargo as build/metadata manager and cargo test as fallback. For Python, pytest-testmon is a useful reference for dynamic test dependencies based on Coverage.py; its own docs require an initial full test run with collection enabled. It is not a direct implementation for Rust. For JS/TS, normalize Biome/Oxlint diagnostics and test runner outputs behind language adapters. - -## 5. Toolchain and runtime optimization research - -The linked projects are valuable, but each should be treated according to maturity and benchmark scope. - -| Project/tool | What it contributes | Proposed use | Limitation / guardrail | -|---|---|---|---| -| t3dotgg/lust-lang | Rust-subset compiler with daemon and incremental edit loop; prototype README reports very fast checks/builds for its benchmark | Experimental agent fast-check pilot for code within supported subset | Current README says no generics, traits or closures. Do not replace rustc, Clippy, Cargo or release tests | -| pingdotgg/ts-rust | Rust port of TypeScript 7 compiler/checker/LSP with conformance and performance work | Optional TypeScript checker pilot on real project fixtures | Early release pinned to a particular TypeScript revision; require exact diagnostics/build-mode/LSP comparison before it can gate | -| cargo-nextest | Process-per-test runner, test concurrency controls, groups, retries and structured reporting | Primary Rust test runner pilot | Cargo still builds; tune parallelism to allocated CPU/memory, not host core count | -| sccache | Local/remote compiler invocation cache | CI compiler cache for cacheable rustc invocations | Rust incremental compilation and some linked crate types are not cacheable; measure cache hit rate by crate | -| cargo-chef | Docker dependency-layer cache | Improve prebuilt CI image/dependency layer construction | Not a replacement for incremental builds inside a live workspace | -| mold / LLD | Faster native linkers | Controlled linker benchmark for Rust/C/C++ projects | Verify platform/toolchain compatibility and binary correctness | -| Biome / Oxlint | Fast JS/TS lint/format implementations | Use for rules that match the project's required ESLint/Prettier behavior | Do not silently drop plugins or rule coverage | -| uv / Ruff / pytest-xdist | Fast Python env/package operations, lint/format, parallel test execution | Python project adapter defaults after compatibility test | pytest remains the authority; cap concurrency to resource requests | -| pytest-testmon | Per-test coverage dependency tracking and changed-code test selection | Reference implementation for dynamic test impact | Initial full collection required; non-deterministic test inputs require conservative invalidation | - -### Rust-specific strategy - -- Keep Cargo as workspace/build orchestrator. Avoid describing nextest or sccache as replacements for Cargo. -- Benchmark cold/warm Cargo registry and Git source caches, target artifacts, sccache, cargo-chef image layers, cargo-nextest, and mold/LLD independently. -- Separate compilation concurrency from test execution concurrency. -- On dedicated self-hosted runner profiles, tune Cargo build jobs and nextest threads to the actual CPU/memory reservation. Do not set each to all host cores when multiple jobs can overlap. -- Consider building dependency layers once per toolchain/image/lockfile digest, then reuse through immutable artifacts. - -### Service/API strategy - -The server challenge repositories are useful because they make workload shape, correctness, database configuration and latency objectives explicit. Apply those principles to GitForge endpoints and queue paths, not their user counts: - -- 1-core/2-GB/SQLite profile: control-plane behavior under severe memory and CPU constraints, with durable writes. -- 8-core/Postgres multicore profile: concurrency, database contention, scheduler throughput and p95/p99 under mixed load. -- 16-core service profile: connection count, memory, accept-loop/runtime behavior and database separation. -- Database configs: reproduce each database with same schema/workload before changing production config. Do not copy parameter values across hardware/DB versions. - -Profile before replacing async frameworks, rewriting services, switching languages or hand-tuning database settings. The likely first wins are queue admission, prepared statements/pool bounds, request-path isolation, fewer repeated builds, cache correctness and avoiding I/O storms. - -## 6. Benchmark and validation programme - -### 6.1 Baseline workload - -Create reproducible repositories representing: -- Rust workspace with several crates, tests, proc macros and build scripts. -- TypeScript monorepo with project references, test packages and lint plugins. -- Python workspace with local packages, pytest fixtures and integration tests. -- Large repository with generated assets, logs and artifacts. -- Multi-project burst with many small pushes plus a few heavy full builds. -- Failure cases: storage near full, cache corrupt/missing, runner outage, DB slow/locked, stale lease, duplicate webhook, scheduler restart and cancellation. - -### 6.2 Profiles - -Measure on 2-core/2-GB, 4-core, 8-core/16-GB and 16-core/32-GB classes where available. Include local NVMe and slower disk/storage profiles where possible. Use a separate load generator for request benchmarks. - -### 6.3 Metrics - -Capture: -- API endpoint p50/p95/p99 latency, throughput and error rate. -- Trigger-to-ack and trigger-to-run latency. -- Queue age and wait by lane, repo, reason and runner. -- Job runtime and resource usage (CPU-seconds, peak RSS, pids, scratch, I/O bytes and pressure). -- Workspace create/restore/cleanup duration and physical bytes allocated. -- Cache hit/miss, bytes reused/downloaded, checksum failures and compiler cache effectiveness. -- CAS logical/physical bytes, dedup ratio, object count, GC delay/reclaimed bytes. -- Selected-test count vs full count, selection reason, cached outcomes, and false-negative differential results. -- DB pool wait, transactions, lock/wal checkpoint time and write latency. - -Run cold and warm cases, repeated trials and realistic mixed workloads. Save raw benchmark artifacts and exact software/input versions. - -### 6.4 Rollout gates - -1. Feature disabled: collect baseline only. -2. Shadow mode: calculate lanes, resource decisions and test impact but execute the existing complete suite. -3. Opt-in projects/runners: enable bounded queues and immutable cache reads; keep full test execution. -4. Differential test selection: run selected plus full validation; compare outcomes. -5. Per-project opt-in cache reuse after no missed failures over an agreed evidence window. -6. Default-on only after exact-head CI/GitForge gates, security review, recovery tests, performance evidence and rollback procedure. - -## 7. Phased execution - -### Gate 0 — correctness and trust (must precede optimization rollout) - -- Fix the current russh callback signature compile error and API test fixture compile error on main (the open #336 branch contains fixes for both; still require exact-head validation). -- Complete cargo-vet review of all 15 unattested dependency versions using evidence; do not add blanket exemptions (#345). -- Fix and test GitForge API-token vs trigger-token contracts for both enqueue and status/poll operations; distinguish 401 auth failures from missing run rows/timeouts (#344). -- Close #318's public-fork isolation and required-check semantics before running untrusted code against persistent caches/runners. -- Validate PR #336 atomic publication semantics at exact head; resolve the Supply Chain and GitForge poll failures; do not call it fully green until both GitHub and GitForge evidence succeeds. -- Close or coordinate #337 and #258 independently; avoid overlapping stale branches on Git server files. - -### Gate 1 — observability and bounded admission - -- Add lane/resource/queue-reason fields and metrics without changing scheduling decisions. -- Measure host and runner pressure and produce queue deferral recommendations in shadow mode. -- Introduce global/per-runner/per-repo admission limits and priority aging with fault tests. -- Protect control-plane capacity and keep trigger persistence/ack independent of job execution. - -### Gate 2 — storage/workspace prototype - -- Define CAS/reference/namespace schema and object API. -- Implement atomic verified object publication and reference/lease registration. -- Build a workspace provider interface with native Git worktree baseline. -- Prototype OverlayFS/reflink and Btrfs only on appropriate filesystems. -- Add dry-run mark/sweep and recovery tests before destructive GC is enabled. - -### Gate 3 — incremental tests - -- Rebase and review #322 graph code. -- Add per-test dynamic mapping and exact execution fingerprints. -- Integrate OpenTestSuite adapter and cargo-nextest pilot. -- Start in shadow/differential mode; keep full suite as required check. - -### Gate 4 — toolchain bake-off and measured rollout - -- Benchmark sccache/nextest/cargo-chef/mold, JS and Python alternatives. -- Run end-to-end mixed-project load tests. -- Enable only the optimizations that produce verified improvements without reducing correctness/security. - -## 8. Required failure-injection tests - -- Two workers concurrently publish the same CAS key/object. -- Worker crashes between blob publication and reference commit. -- GC overlaps active publication, workspace lease renewal and job completion. -- Dirty worktree belongs to another live agent; GC must preserve it. -- Stale lock is old but still owned; GC must preserve it until fencing succeeds. -- Cache data is corrupted, truncated or metadata is missing. -- Cache namespace is accessed by a different tenant/trust class. -- Runner is out of scratch space or cgroup memory pressure is high. -- Fast lane is flooded while a heavy job waits; both must make progress. -- One repository floods the scheduler; other repositories still dispatch. -- Scheduler/CI restart while jobs are queued, assigned, running and terminal. -- Test graph misses an edge; selection falls back to full scope. -- Changed fixture, environment, compiler or lockfile invalidates cached test result. -- Flaky/failed/new tests are not reused from prior successful receipts. -- Poll API returns 401/403/404/5xx; the Actions adapter reports the actual cause and never reports false success. - -## 9. Exit criteria - -This initiative is complete only when: - -- A mixed-project workload has bounded queues, explicit deferral reasons, fair progress and responsive control-plane p95/p99. -- Resource limits are enforced and verified at the runner boundary. -- Shared content is immutable, tenant-scoped, integrity-checked and recoverable. -- Workspace lifecycle and GC cannot delete live/dirty work. -- Repeated dependency/build/test inputs are demonstrably reused with measured I/O/storage savings. -- Incremental test selection has zero missed failures under differential full-suite validation and still has scheduled full/release gates. -- The system exposes enough telemetry to explain queue delay, storage use, cache behavior and per-job resource cost. -- Rollback remains possible by disabling lanes/cache/test selection and returning to current full validation behavior. - -## References - -### GitForge -- #342 umbrella epic; #338 resource scheduling; #339 CAS/workspaces/GC; #340 incremental tests; #341 toolchain benchmarks; #344 CI status-poll authentication; #345 cargo-vet attestations. -- #277 resource/storage headroom; #284 duplicate lanes; #318 runner trust; #322 smart CI; #288 push/outbox behavior; #335 Amortyx-Stack registration/evidence; #336 atomic publication; #337 release bundle tests; #258 SSH push trigger. - -### Storage/workspace -- pnpm store design: https://github.com/pnpm/pnpm/blob/main/pnpm/docs/settings/store.md -- Btrfs subvolumes/snapshots: https://btrfs.readthedocs.io/en/latest/Subvolumes.html - -### Build/test tooling -- lust-lang: https://github.com/t3dotgg/lust-lang -- ts-rust: https://github.com/pingdotgg/ts-rust -- sccache Rust caveats: https://github.com/mozilla/sccache/blob/main/docs/Rust.md -- cargo-chef: https://github.com/LukeMathWalker/cargo-chef -- cargo-nextest: https://nexte.st/docs/running/ -- pytest-testmon: https://www.testmon.org/ - -### Service and database benchmarks -- 1-vCPU/2-GB server challenge: https://github.com/arjaythedev/twelve-dollar-server-challenge -- Multicore server challenge: https://github.com/arjaythedev/multicore-server-challenge -- Million-users challenge: https://github.com/arjaythedev/million-users-challenge -- Database benchmark configs: https://github.com/arjaythedev/database-benchmark-configs -- Software engineering resources: https://github.com/arjaythedev/software-engineering-resources - ---- -This document defines the intended architecture and execution order. It is not evidence that the runtime changes are implemented or that current CI failures are fixed. diff --git a/docs/planning/EXECUTION_PLAN_20261103.md b/docs/planning/EXECUTION_PLAN_20261103.md index b1440a2e..4be8aebd 100644 --- a/docs/planning/EXECUTION_PLAN_20261103.md +++ b/docs/planning/EXECUTION_PLAN_20261103.md @@ -41,6 +41,83 @@ authoritative stores (gitforge.db, the API, the filesystem) at claim time. | F-16: image registry allowlist + digest pinning | `GITFORGE_ALLOWED_IMAGE_REGISTRIES` env-gated check in `gitforge-ci::engine` (core takes allowlist as data; 5 unit tests incl. port-prefix boundary). Unset = unrestricted | | RUNBOOK drift | metrics section corrected; one-runner fleet pause policy documented with the unpause path | +## Runner image identity across a multi-host fleet (2026-11-05) + +F-16 shipped the *gate* (allowlist + digest pinning required), but the +fleet cannot currently satisfy it: there is no shared registry the runners +can reach, so every repository still distributes its job images by hand. +This section records the measurements that define the durable fix. + +**The failure.** A DSC pipeline pinned each job's image by Docker image +ID. The same content resolves to different IDs per storage driver — +swarmone's classic overlay2 reports the image's *config* digest +(`sha256:0402417a…`), fedora's overlayfs + containerd snapshotter reports +the *manifest-list* digest (`sha256:fede6b3c…`). `SimplePolicy::select_runner` +filters `status=="online" && capacity>0` and takes `max_by_key(capacity)`, +so `fedora-docker` (5) outranks `swarmone-docker` (4) and every dsc job +landed on fedora, where the pinned IDs did not exist — "No such image" at +sandbox creation, which cancels every job chained behind it. The pipeline's +correctness depended on which runner won a capacity race. + +**What is and is not portable** (measured 2026-11-05, same image content +on both hosts): + +| Identity | Portable across drivers? | Value | +| ----------------------------------------- | ------------------------ | ----- | +| `.Id` (local image ID) | **no** | `0402417a…` (swarmone) vs `fede6b3c…` (fedora) | +| image **config** digest | yes | `sha256:0402417a…` on both | +| **diffID** fingerprint (sorted layer set) | yes | `sha256:393acaf2…` on both | +| registry **manifest** digest, pushed independently per host | **no** | `6e8aeb89…` (swarmone) vs `71f39742…` (fedora) — each host re-compresses layers differently | + +The last row matters: a registry manifest digest identifies *a particular +publication*, not the content. A shared registry only yields a portable +digest if it is the **single publisher** — otherwise two hosts that each +push their own copy mint different digests for identical content. + +**The pull path already supports it.** `ensure_image` +(`crates/gitforge-sandbox/src/docker.rs:361`) passes the whole reference to +Docker's `create_image` `fromImage`. A combined `repo@sha256:…` reference +is accepted there — verified live against a temporary registry +(`POST /v1.44/images/create?fromImage=127.0.0.1:5002/dsc-ci-node@sha256:71f3974…` +→ HTTP 200, `Digest: sha256:71f3974…`). So F-16's contract is achievable +without further sandbox changes. + +**Why the repo-side workaround is debt, not a fix.** The DSC repository +(pinned by the tag contract in `.gitforce.yml` + the baked lockfile stamp +check) makes a tag behave like a fixed reference: the image bakes the +sha256 of the lockfile it was warmed from, and `lint-node`/`lint-rust` fail +fast when the checkout's lockfile diverges from it; the diffID fingerprint +proves the image is byte-identical on every runner host. That is a +procedure compensating for the missing capability. It is recorded as +explicit debt, owned by the operator, with the following shape: + +1. **One shared registry, reachable from every runner**, published from a + single build host so the manifest digest is canonical. Today + `gitforge-local-registry.service` publishes `127.0.0.1:5000` only — + loopback-only and unreachable from fedora — and both daemons'` + `insecure-registries` cover only `127.0.0.0/8`, so a LAN-bound registry + additionally needs `insecure-registries` on both daemons (a + `/etc/docker/daemon.json` edit plus a docker restart on the CI host). + A temporary registry bound to `192.168.0.201:5002` was verified + reachable from both hosts (HTTP 200) — the network path is not the + blocker; daemon configuration is. +2. **`GITFORGE_ALLOWED_IMAGE_REGISTRIES`** set to that registry's prefix + on the scheduler, so the F-16 gate becomes active and every job image + must be a digest-pinned reference under it. +3. **Job images pinned as `registry/repo@sha256:`** in + each repository's pipeline definition. +4. **Runner affinity** (or capacity tuning) so a job's image is present + on whichever runner the scheduler selects. There is no affinity + mechanism today: the `runners` table has no `repo_id` and `labels` is + hardcoded `"[]"`. `select_runner` has no notion of "can this runner + serve this image". + +Item 4 is the substantive scheduler change: with a shared registry, a +runner can *pull* an image it does not have, so the cold-pull cost and the +60-second sandbox acquisition cap (`GITFORGE_SANDBOX_ACQUIRE_SECS`) become +the binding constraint instead of image presence. An affinity preference +plus a warm-pull strategy on the runner is the natural follow-on. + ## Known remaining work (ordered) 1. **Gate the three code changes** (gauge refresh, supersede, image diff --git a/docs/planning/GITFORGE-CI-AUDIT-AND-IMPROVEMENT-PLAN.md b/docs/planning/GITFORGE-CI-AUDIT-AND-IMPROVEMENT-PLAN.md index 66ba4694..40272eb8 100644 --- a/docs/planning/GITFORGE-CI-AUDIT-AND-IMPROVEMENT-PLAN.md +++ b/docs/planning/GITFORGE-CI-AUDIT-AND-IMPROVEMENT-PLAN.md @@ -249,10 +249,11 @@ the container. `~/.cache/gitforge`, else `/var/cache/gitforge`). The runner creates the host directory itself, as its own user, because the daemon would create a missing bind source as root — unwritable for the job's user. -- A cache mount that cannot be prepared degrades to a cold build with a - runner-side warning. It is an optimisation, never a correctness - requirement, and a missing cache directory must not turn into a red - pipeline. +- A mount that cannot be prepared fails the job. The runner reports it + as an infrastructure failure, so the reason reaches the job's own + result instead of a log line (GF-10, 2026-11-05: a job whose cache had + vanished used to run cold, time out, and show nothing about the mount + that was missing). **Measured on the Fedora instance (2026-10-22), clippy/test/coverage job durations:** diff --git a/scripts/gitforge-release-auto b/scripts/gitforge-release-auto index 904ccd6f..c2382739 100755 --- a/scripts/gitforge-release-auto +++ b/scripts/gitforge-release-auto @@ -7,33 +7,45 @@ # same chain an operator runs by hand, executed mechanically. # # Default mode is a dry-run report: what is promoted now, what would change, -# whether the runners are busy. Acting requires --apply (or AUTO_UPDATE=1, +# whether any job is in flight. Acting requires --apply (or AUTO_UPDATE=1, # which is how the systemd timer opt-in works). Two safety gates sit in # front of any action: # - a bundle must pass gitforge-release-verify (binaries, manifest, READY) -# - ci/runner are only restarted when no `gitforce-job-*` container is -# running; a busy fleet defers the whole cutover to the next pass -# (pending-only runs are just queued work — restarting ci mid-flight is +# - ci/runner are only restarted when the authoritative store reports no +# job in flight; a busy fleet defers the whole cutover to the next pass +# (queued-only work is just queued work — restarting ci mid-flight is # the heartbeat-loss incident class, not a maintenance no-op) # -# Usage: gitforge-release-auto [--apply] +# The busy signal is the store's job rows, read by gitforge-release-busy — +# NOT the runner's container names. The runner keeps a warm pool of +# containers that have no job row at all, so counting containers is +# permanently non-zero and defers every cutover forever (GF-16). +# +# Usage: gitforge-release-auto [--apply|--print-restart-plan] +# --print-restart-plan print the units this pass would restart, as this +# host's unit state sees them, and exit # Environment: # GITFORGE_AUTO_REPO checkout root (default: derived from this script) # GITFORGE_AUTO_API api health URL (default http://localhost:42780) # GITFORGE_AUTO_WAIT seconds to wait for post-restart health (default 60) +# GITFORGE_RELEASE_DB database the busy gate reads (default: the live +# database, resolved the way the release gate +# resolves it) # Exit codes: 0 nothing to do / updated / deferred cleanly, 1 real failure. set -u -o pipefail apply=0 +plan_only=0 case "${1:-}" in "") ;; --apply) apply=1 ;; + --print-restart-plan) plan_only=1 ;; -h|--help) - sed -n '2,28p' "$0" + awk 'NR>1 && /^#/ {print} NR>1 && !/^#/ {exit}' "$0" exit 0 ;; *) - printf 'usage: gitforge-release-auto [--apply]\n' >&2 + printf 'usage: gitforge-release-auto [--apply|--print-restart-plan]\n' >&2 exit 64 ;; esac @@ -56,8 +68,12 @@ die() { printf 'release-auto: %s\n' "$1" >&2; exit 1; } # restart set is what the host actually runs. Failing the whole cutover on # a masked unit made every promote roll back forever (2026-10-10: the NAS # runner was retired via mask when CI moved to the fedora host). +# A masked TEMPLATE INSTANCE is invisible to list-unit-files (it enumerates +# unit files, not instances), so the retired-component check fell through and +# the cutover tried to restart a masked unit. LoadState is the unit's own +# state and reports "masked" for concrete units and instances alike. unit_masked() { - [[ "$(systemctl list-unit-files "$1" --no-legend 2>/dev/null | awk '{print $2}')" == "masked" ]] + [[ "$(systemctl show "$1" -p LoadState --value 2>/dev/null)" == "masked" ]] } RESTART_STATELESS=() for u in gitforge@api.service gitforge@git-server.service; do @@ -77,6 +93,14 @@ for u in gitforge@ci.service gitforge@runner.service; do done restart_list() { [[ "${#@}" -gt 0 ]] && printf '%s ' "$@" || printf '(none)'; } +# The restart plan as this host sees it. A masked unit is a retired +# component, so it is excluded; the plan is what the operator (or the +# regression test) can check against the host's own unit state. +if [[ "$plan_only" -eq 1 ]]; then + say "restart plan: stateless=$(restart_list "${RESTART_STATELESS[@]}") scheduled=$(restart_list "${RESTART_SCHED[@]}")" + exit 0 +fi + # One cutover at a time, across operators and timer ticks. mkdir -p "$releases_dir" exec 9>"$lock_file" @@ -135,13 +159,27 @@ if [[ -z "$candidate" ]]; then exit 0 fi -# Runners busy? Any live `gitforce-job-*` container defers the cutover. -job_containers=$(docker ps -q --filter "name=gitforce-job-" 2>/dev/null | wc -l) +# Runners busy? The authoritative store decides, not the container names: +# the runner keeps a warm pool of containers that have no job row, so a +# container count is permanently non-zero and defers every cutover forever +# (GF-16). gitforge-release-busy counts job rows that are `assigned` or +# `running` AND still inside the fence grace window, which is the same +# liveness term the scheduler uses to decide whether a row is alive — so a +# row left behind by a dead runner cannot wedge this gate. +# +# An unreadable store fails closed: the cutover is refused, never started +# blind on an unknown busy state. That is a real failure, not a clean +# deferral — a silently unreadable store would stall the channel forever +# with no signal. +if ! inflight=$("$script_dir/gitforge-release-busy"); then + die "cannot read the in-flight job count from the store; the cutover is deferred until the store is readable" +fi +[[ "$inflight" =~ ^[0-9]+$ ]] || die "unexpected in-flight job count: '$inflight'" -say "promoted=$promoted_id candidate=$candidate_id apply=$apply job_containers=$job_containers" +say "promoted=$promoted_id candidate=$candidate_id apply=$apply inflight=$inflight" -if [[ "$job_containers" -gt 0 ]]; then - say "deferred: $job_containers job container(s) running; retry on the next pass" +if [[ "$inflight" -gt 0 ]]; then + say "deferred: $inflight job(s) in flight; retry on the next pass" exit 0 fi diff --git a/scripts/gitforge-release-build b/scripts/gitforge-release-build new file mode 100755 index 00000000..18f90215 --- /dev/null +++ b/scripts/gitforge-release-build @@ -0,0 +1,75 @@ +#!/usr/bin/env bash +# Build the four release binaries from a checkout and install them into the +# release source root. +# +# Usage: gitforge-release-build [target-dir] +# +# The bundler copies the release binaries from the root of the source root, +# where they live in gitignored paths. This script is how they get there: it +# builds them from the checkout, refuses any binary that does not report the +# commit the checkout is on, and only then installs it. A binary built from +# another commit never reaches the source root. +# +# The build must run in a git checkout: the binaries are stamped with their +# commit at build time (crates/gitforge-common/build.rs). For a build from a +# source copy without a checkout, set GITFORGE_BUILD_COMMIT to the commit the +# copy was made from, and GITFORGE_BUILD_DIRTY to the state of that copy; the +# same stamp check then applies. +# +# Builds belong on the build host with the spare NVMe, not on the instance +# that runs the services. +set -euo pipefail + +source_root="${1:-}" +target_dir="${2:-}" + +if [[ -z "$source_root" ]]; then + echo 'usage: gitforge-release-build [target-dir]' >&2 + exit 64 +fi + +if [[ ! -d "$source_root" ]]; then + printf 'release build: source root does not exist: %s\n' "$source_root" >&2 + exit 1 +fi + +expected_commit="${GITFORGE_BUILD_COMMIT:-}" +if [[ -z "$expected_commit" ]]; then + expected_commit=$(git -C "$source_root" rev-parse HEAD 2>/dev/null || true) +fi +if [[ ! "$expected_commit" =~ ^[0-9a-f]{40}$ ]]; then + printf 'release build: the source root must be a git checkout, or GITFORGE_BUILD_COMMIT must name the commit\n' >&2 + exit 1 +fi + +if [[ -z "$target_dir" ]]; then + target_dir="$source_root/target" +fi + +# Pin the commit for the build so the binaries cannot drift from it: the +# build script stamps this value into every binary it compiles. +export GITFORGE_BUILD_COMMIT="$expected_commit" +export CARGO_TARGET_DIR="$target_dir" + +# Pin how the build happened as well. The bundle's metadata describes the +# artifact, so the toolchain and the build time have to be the ones the +# BUILD used - not the ones of whichever host assembles the bundle +# (GF-15, 2026-11-06). The build script stamps both into every binary. +export GITFORGE_BUILD_TOOLCHAIN="${GITFORGE_BUILD_TOOLCHAIN:-$(rustc --version 2>/dev/null || true)}" +export GITFORGE_BUILD_EPOCH="${GITFORGE_BUILD_EPOCH:-$(date -u +%s)}" + +printf 'release build: building %s from %s\n' "$expected_commit" "$source_root" +cargo build --release --manifest-path "$source_root/Cargo.toml" \ + -p api -p ci -p git-server -p runner + +built_dir="$target_dir/release" + +# The stamp check is the point of this script: a binary that does not report +# the commit it was built from is never installed. +"$(dirname "$0")/gitforge-release-stamps" "$built_dir" "$expected_commit" + +for binary in api ci git-server runner; do + install -m 0755 "$built_dir/$binary" "$source_root/$binary" +done + +printf 'release build: installed %d binaries into %s\n' 4 "$source_root" diff --git a/scripts/gitforge-release-bundle b/scripts/gitforge-release-bundle index eb760ed7..827c90a0 100755 --- a/scripts/gitforge-release-bundle +++ b/scripts/gitforge-release-bundle @@ -20,8 +20,6 @@ if [[ ! "$release_id" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]]; then exit 64 fi -"$(dirname "$0")/gitforge-release-preflight" "$source_root" - source_commit="${GITFORGE_SOURCE_COMMIT:-}" if [[ -z "$source_commit" ]]; then source_commit=$(git -C "$source_root" rev-parse HEAD 2>/dev/null || true) @@ -31,19 +29,14 @@ if [[ ! "$source_commit" =~ ^[0-9a-f]{40}$ ]]; then exit 1 fi +# The preflight now carries the commit, so it can also prove that the +# candidate binaries are the ones this bundle claims to ship. +"$(dirname "$0")/gitforge-release-preflight" "$source_root" "$source_commit" + # A cut may only assemble on top of a full, honest green for this exact # commit (see scripts/gitforge-release-gate). "$(dirname "$0")/gitforge-release-gate" "$source_commit" -toolchain="${GITFORGE_TOOLCHAIN:-}" -if [[ -z "$toolchain" ]]; then - toolchain=$(rustc --version 2>/dev/null || true) -fi -if [[ -z "$toolchain" ]]; then - printf 'release bundle: build toolchain is required (set GITFORGE_TOOLCHAIN or install rustc)\n' >&2 - exit 1 -fi - release_dir="$bundle_root/$release_id" if [[ -e "$release_dir" ]]; then printf 'release bundle: refusing to overwrite existing release: %s\n' "$release_dir" >&2 @@ -65,6 +58,18 @@ if [[ -e "$staging_dir" ]]; then find "$staging_dir" -depth -delete fi +# A failed cut must not leave a staging tree behind. The next attempt +# clears it, but a tree that outlives its cut is a directory in the bundle +# root that looks like a release to anything listing it. +cleanup_staging() { + local status=$? + if [[ -d "$staging_dir" ]]; then + find "$staging_dir" -depth -delete 2>/dev/null || true + fi + exit "$status" +} +trap cleanup_staging ERR + mkdir -p "$staging_dir/bin" "$staging_dir/scripts" "$staging_dir/systemd/user" # The services run as the unprivileged `gitforge` user, which reaches the @@ -84,6 +89,12 @@ for binary in api ci git-server runner; do install -m 0755 "$source_root/$binary" "$staging_dir/bin/$binary" done +# Prove the staged bytes carry the claimed commit BEFORE the bundle is +# published. The verify step below runs after the atomic rename, and a +# bundle that fails there would sit at the final name, blocking every retry +# behind the overwrite refusal (GF-14, 2026-11-05). +"$(dirname "$0")/gitforge-release-stamps" "$staging_dir/bin" "$source_commit" + install -m 0755 "$source_root/scripts/gitforge-status" "$staging_dir/scripts/gitforge-status" for unit in gitforge-ci.service.example gitforge-runner.service.example; do @@ -94,14 +105,21 @@ for policy in gitforge-resource-limits.conf gitforge-ci-resource-limits.conf git install -m 0644 "$(dirname "$0")/../systemd/user/$policy" "$staging_dir/systemd/user/$policy" done -build_timestamp=$(date -u +%Y-%m-%dT%H:%M:%SZ) +# The metadata has to describe the ARTIFACT, so it is read out of the +# binaries the bundle ships. Taking it from the cutting host instead made +# the bundle claim the toolchain of the machine that assembled it while the +# binaries were compiled elsewhere (GF-15, 2026-11-06). +build_toolchain=$("$(dirname "$0")/gitforge-release-stamps" --field toolchain "$staging_dir/bin") +build_timestamp=$("$(dirname "$0")/gitforge-release-stamps" --field built "$staging_dir/bin") +cut_timestamp=$(date -u +%Y-%m-%dT%H:%M:%SZ) ( cd "$staging_dir" cat > RELEASE_METADATA.txt < READY <&2; exit 1; } + +# The live database lives wherever the services find it. The path is read +# from the same environment the services run with, so a deployment that +# moves the database does not leave this script reading a stale copy. +gitforge_database_path() { + local root="$1" + if [[ -n "${GITFORGE_RELEASE_DB:-}" ]]; then + printf '%s\n' "$GITFORGE_RELEASE_DB" + return 0 + fi + local url="${GITFORGE_DATABASE_URL:-}" + if [[ -z "$url" && -r /etc/gitforge/database.env ]]; then + url="$(sed -n 's/^GITFORGE_DATABASE_URL=//p' /etc/gitforge/database.env | tail -n 1)" + url="${url%\"}" + url="${url#\"}" + fi + if [[ "$url" == sqlite:* ]]; then + url="${url#sqlite://}" + url="${url#sqlite:}" + url="${url%%\?*}" + if [[ -n "$url" ]]; then + printf '%s\n' "$url" + return 0 + fi + fi + printf '%s\n' "$root/data/gitforge.db" +} + +# The fence grace window. Only a positive integer is honoured; anything +# else falls back to the default, mirroring the scheduler's own parser. +fence_grace_secs() { + local raw="${GITFORGE_JOB_FENCE_GRACE_SECS:-}" + if [[ "$raw" =~ ^[0-9]+$ ]] && [[ "$raw" -gt 0 ]]; then + printf '%s\n' "$raw" + return 0 + fi + printf '%s\n' "$DEFAULT_FENCE_GRACE_SECS" +} + +repo_root="$(cd "$(dirname "$0")/.." && pwd)" +db_path="$(gitforge_database_path "$repo_root")" +[[ -f "$db_path" ]] || die "live database not found: $db_path (set GITFORGE_RELEASE_DB)" + +if command -v sqlite3 >/dev/null 2>&1; then + db_reader=cli +else + if ! command -v python3 >/dev/null 2>&1; then + die "sqlite3 CLI not found and no python3 for the fallback; one of them is required to read the store" + fi + if ! python3 -c 'import sqlite3' >/dev/null 2>&1; then + die "sqlite3 CLI not found and python3 lacks the sqlite3 module; install either to read the store" + fi + db_reader=python + say "sqlite3 CLI not found; reading the store via the python3 sqlite3 fallback" >&2 +fi + +grace="$(fence_grace_secs)" +# Every value interpolated into the SQL is strictly validated first — the +# grace window is a positive integer or the default — and both readers +# execute the identical SQL text. +sql="SELECT count(*) FROM jobs j LEFT JOIN runners r ON r.id = j.runner_id WHERE j.status IN ('assigned','running') AND datetime(COALESCE(j.heartbeat_at, r.last_heartbeat, j.started_at), '+' || ${grace} || ' seconds') > datetime('now');" + +# Read-only access with a busy timeout: the live services hold write locks +# and a bare reader default is to fail instantly instead of waiting. Both +# readers emit one value: the count. +db_count() { + if [[ "$db_reader" == "cli" ]]; then + sqlite3 -batch -readonly "$db_path" ".timeout 15000" "$sql" + else + python3 - "$db_path" "$sql" <<'PY' +import os +import sqlite3 +import sys +from urllib.parse import quote + +db_path, sql = sys.argv[1], sys.argv[2] +# mode=ro never writes and never creates a database; timeout=15.0 is the +# Python equivalent of `.timeout 15000`. The path is percent-escaped so +# filesystem characters that are URI metacharacters survive. +try: + connection = sqlite3.connect( + "file:" + quote(os.path.abspath(db_path)) + "?mode=ro", + uri=True, + timeout=15.0, + ) + try: + rows = connection.execute(sql).fetchall() + finally: + connection.close() +except sqlite3.Error as error: + print(f"release-busy: python sqlite3 fallback: query failed: {error}", file=sys.stderr) + sys.exit(1) +for row in rows: + print("|".join("" if value is None else str(value) for value in row)) +PY + fi +} + +count="$(db_count)" || die "query failed: cannot read the in-flight job count from $db_path" +count="${count//$'\n'/}" +[[ "$count" =~ ^[0-9]+$ ]] || die "unexpected in-flight job count from $db_path: '$count'" +printf '%s\n' "$count" diff --git a/scripts/gitforge-release-preflight b/scripts/gitforge-release-preflight index a2e8e8c4..58e050c3 100755 --- a/scripts/gitforge-release-preflight +++ b/scripts/gitforge-release-preflight @@ -2,8 +2,9 @@ set -euo pipefail source_root="${1:-}" -if [[ -z "$source_root" ]]; then - echo 'usage: gitforge-release-preflight ' >&2 +expected_commit="${2:-}" +if [[ -z "$source_root" || -z "$expected_commit" ]]; then + echo 'usage: gitforge-release-preflight ' >&2 exit 64 fi @@ -26,3 +27,10 @@ if ((${#missing[@]})); then fi printf 'release preflight: all candidate binaries present (%d); CI owns scheduler\n' "${#required[@]}" + +# Presence and executability are not provenance. The release binaries live in +# gitignored paths, so stale leftovers are indistinguishable from fresh ones +# by looking at the checkout: only the binaries themselves can say which +# commit they were built from (GF-14, 2026-11-05). The expected commit is +# required here so that no caller can skip the check. +"$(dirname "$0")/gitforge-release-stamps" "$source_root" "$expected_commit" diff --git a/scripts/gitforge-release-stamps b/scripts/gitforge-release-stamps new file mode 100755 index 00000000..326bf8e4 --- /dev/null +++ b/scripts/gitforge-release-stamps @@ -0,0 +1,149 @@ +#!/usr/bin/env bash +# Verify that the release binaries in a directory were built from the commit +# the bundle claims, and describe how they were built. +# +# Usage: +# gitforge-release-stamps +# gitforge-release-stamps --field +# +# Every release binary answers `--version` with a line carrying +# `commit=<40-hex>`, `dirty=false`, `toolchain=` and +# `built=`. A binary that cannot be executed, that names a +# different commit, that was built from a modified tree, or that cannot say +# how it was built, is refused. +# +# The `--field` mode prints the value the binaries agree on, and is how the +# bundler and the verifier describe a bundle FROM the binaries it ships +# instead of from the machine that assembled it (GF-15, 2026-11-06). +# +# This check is what makes the bundle's provenance claim true. The release +# binaries live in gitignored paths, so a bundle assembled from stale +# leftovers used to pass every other gate while stamping a commit it was +# never built from (GF-14, 2026-11-05). The claim now has to be carried by +# the binaries themselves. +set -euo pipefail + +required=(api ci git-server runner) + +usage() { + echo 'usage: gitforge-release-stamps ' >&2 + echo ' gitforge-release-stamps --field ' >&2 + exit 64 +} + +# The version line a binary prints, or a refusal. +version_line_of() { + local path="$1" output + if [[ ! -x "$path" ]]; then + printf 'release stamps: missing or non-executable binary: %s\n' "$path" >&2 + exit 1 + fi + if ! output=$(timeout 15 "$path" --version 2>/dev/null); then + printf 'release stamps: %s did not report a version (non-zero exit or timeout)\n' "$path" >&2 + exit 1 + fi + printf '%s\n' "$output" +} + +# One field of a version line. +# +# `toolchain` carries spaces (a `rustc --version` line does), so it is +# anchored on the field that follows it rather than on a word boundary. +field_of() { + local output="$1" field="$2" value + case "$field" in + commit) value=$(printf '%s\n' "$output" | sed -n 's/.*commit=\([0-9a-f]\{40\}\).*/\1/p') ;; + dirty) value=$(printf '%s\n' "$output" | sed -n 's/.*dirty=\([a-z]*\).*/\1/p') ;; + toolchain) value=$(printf '%s\n' "$output" | sed -n 's/.* toolchain=\(.*\) built=.*/\1/p') ;; + built) value=$(printf '%s\n' "$output" | sed -n 's/.* built=\(.*\)/\1/p') ;; + *) + printf 'release stamps: unknown field: %s\n' "$field" >&2 + exit 64 + ;; + esac + printf '%s' "${value%%$'\n'*}" +} + +if [[ "${1:-}" == "--field" ]]; then + field="${2:-}" + binary_dir="${3:-}" + if [[ -z "$field" || -z "$binary_dir" ]]; then + usage + fi + case "$field" in + toolchain|built) ;; + *) + printf 'release stamps: unknown field: %s\n' "$field" >&2 + exit 64 + ;; + esac + + agreed="" + for binary in "${required[@]}"; do + line=$(version_line_of "$binary_dir/$binary") + value=$(field_of "$line" "$field") + if [[ -z "$value" ]]; then + printf 'release stamps: %s does not report %s\n' "$binary_dir/$binary" "$field" >&2 + exit 1 + fi + if [[ -z "$agreed" ]]; then + agreed="$value" + elif [[ "$value" != "$agreed" ]]; then + printf 'release stamps: the binaries disagree about %s (%s vs %s)\n' \ + "$field" "$agreed" "$value" >&2 + exit 1 + fi + done + printf '%s\n' "$agreed" + exit 0 +fi + +binary_dir="${1:-}" +expected_commit="${2:-}" + +if [[ -z "$binary_dir" || -z "$expected_commit" ]]; then + usage +fi + +if [[ ! "$expected_commit" =~ ^[0-9a-f]{40}$ ]]; then + printf 'release stamps: expected commit must be 40 lowercase hex characters: %s\n' "$expected_commit" >&2 + exit 64 +fi + +for binary in "${required[@]}"; do + path="$binary_dir/$binary" + output=$(version_line_of "$path") + + # sed keeps its exit status at 0 when it matches nothing, so an unstamped + # binary lands here as an empty value rather than a failed pipeline. + commit=$(field_of "$output" commit) + dirty=$(field_of "$output" dirty) + toolchain=$(field_of "$output" toolchain) + built=$(field_of "$output" built) + + if [[ -z "$commit" ]]; then + printf 'release stamps: %s does not report a build commit\n' "$path" >&2 + exit 1 + fi + if [[ "$commit" != "$expected_commit" ]]; then + printf 'release stamps: %s was built from %s but the bundle claims %s\n' \ + "$path" "$commit" "$expected_commit" >&2 + exit 1 + fi + if [[ "$dirty" != "false" ]]; then + printf 'release stamps: %s was built from a tree that was not clean (dirty=%s)\n' \ + "$path" "${dirty:-unknown}" >&2 + exit 1 + fi + if [[ -z "$toolchain" || "$toolchain" == "unknown" ]]; then + printf 'release stamps: %s does not report the toolchain it was built with\n' "$path" >&2 + exit 1 + fi + if [[ -z "$built" || "$built" == "unknown" ]]; then + printf 'release stamps: %s does not report when it was built\n' "$path" >&2 + exit 1 + fi + + printf 'release stamps: %s commit=%s dirty=%s built=%s toolchain=%s\n' \ + "$binary" "$commit" "$dirty" "$built" "$toolchain" +done diff --git a/scripts/gitforge-release-verify b/scripts/gitforge-release-verify index 708ca31d..bf744a4e 100755 --- a/scripts/gitforge-release-verify +++ b/scripts/gitforge-release-verify @@ -55,6 +55,29 @@ if [[ -z "$metadata_release" || "$metadata_release" != "$ready_release" || -z "$ exit 1 fi +# The bundle's provenance claim has to be carried by the binaries it ships. +# The manifest only proves the bundle has not been edited since the cut; it +# says nothing about which commit the binaries were built from (GF-14, +# 2026-11-05). +"$(dirname "$0")/gitforge-release-stamps" "$release_dir/bin" "$metadata_commit" + +# The metadata has to describe the binaries, not the machine that assembled +# them. A bundle whose metadata names a different toolchain or a different +# build time than its own binaries carry is describing something else +# (GF-15, 2026-11-06). +binary_toolchain=$("$(dirname "$0")/gitforge-release-stamps" --field toolchain "$release_dir/bin") +binary_built=$("$(dirname "$0")/gitforge-release-stamps" --field built "$release_dir/bin") +if [[ "$metadata_toolchain" != "$binary_toolchain" ]]; then + printf 'release verify: metadata toolchain (%s) does not describe the shipped binaries (%s)\n' \ + "$metadata_toolchain" "$binary_toolchain" >&2 + exit 1 +fi +if [[ "$metadata_timestamp" != "$binary_built" ]]; then + printf 'release verify: metadata build time (%s) does not describe the shipped binaries (%s)\n' \ + "$metadata_timestamp" "$binary_built" >&2 + exit 1 +fi + ci_unit="$release_dir/systemd/user/gitforge-ci.service.example" runner_unit="$release_dir/systemd/user/gitforge-runner.service.example" generic_policy="$release_dir/systemd/user/gitforge-resource-limits.conf" diff --git a/scripts/gitforge-remote-cutover b/scripts/gitforge-remote-cutover index 455b769b..73ce922a 100755 --- a/scripts/gitforge-remote-cutover +++ b/scripts/gitforge-remote-cutover @@ -3,9 +3,13 @@ # # Generalizes the 2026-10-10 v0.6.16 fedora cutover, which ran as a one-off # hand-edited script. Order of operations is the proven one: -# 1. Wait for zero in-flight jobs (assigned/running) so no job container -# is orphaned mid-flight. A pending-only backlog is fine — that is just -# queued work, and waiting for it would deadlock a busy instance. +# 1. Wait for zero in-flight jobs so no job container is orphaned +# mid-flight. The count comes from scripts/gitforge-release-busy: a +# job row that is `assigned` or `running` AND still inside the fence +# grace window. A pending-only backlog is fine — that is just queued +# work, and waiting for it would deadlock a busy instance. A row left +# behind by a dead runner does not block either: the scheduler fences +# it once its proof of life ages past the grace window (GF-16). # 2. Stop the scheduler units FIRST (ci is the dispatcher; stopping the # api first would let push triggers pile onto a dead consumer). # 3. Flip the release symlink atomically (symlink-to-temp + mv -T). @@ -95,22 +99,41 @@ esac # A masked unit is retired on this host; restarting it fails the whole # cutover for no operational reason (2026-10-10: the NAS runner mask made -# every promote roll back). list-unit-files state 'masked' is the only -# reliable discriminator — `systemctl cat` exits 0 even for masked units. -# list-unit-files is strict about names: a bare "gitforge-ci" matches -# nothing while "gitforge-ci.service" does (only globs are lenient). +# every promote roll back). `systemctl cat` exits 0 even for masked units, +# so the retired state comes from LoadState below. Names are normalised to +# the full unit name because a bare "gitforge-ci" matches nothing while +# "gitforge-ci.service" does (only globs are lenient). unit_file_name() { printf '%s.service' "${1%.service}"; } +# `systemctl list-unit-files` enumerates unit FILES, not template INSTANCES. +# On the NAS profile every unit is an instance of the gitforge@.service +# template, so every one of them resolved to "not installed": the cutover +# flipped the release symlink, stopped nothing and started nothing, and the +# old processes kept serving the old binaries - a silent no-op release that +# reports success. Detection is therefore instance-aware: +# - LoadState is the unit's own state. A retired unit reports "masked", +# which list-unit-files cannot see for an instance. +# - list-units --all reports whether the manager actually knows the +# instance. +# A concrete unit file (the fedora user profile: gitforge-api.service) is +# still detected through list-unit-files. +unit_load_state() { + systemctl "${scope[@]}" show "$(unit_file_name "$1")" -p LoadState --value 2>/dev/null +} unit_masked() { - [[ "$(systemctl "${scope[@]}" list-unit-files "$(unit_file_name "$1")" --no-legend 2>/dev/null | awk '{print $2}')" == "masked" ]] + [[ "$(unit_load_state "$1")" == "masked" ]] } unit_present() { - systemctl "${scope[@]}" list-unit-files "$(unit_file_name "$1")" --no-legend 2>/dev/null | grep -q . + local name + name="$(unit_file_name "$1")" + systemctl "${scope[@]}" list-unit-files "$name" --no-legend 2>/dev/null \ + | awk -v u="$name" '$1 == u { found = 1 } END { exit !found }' && return 0 + systemctl "${scope[@]}" list-units --all --no-legend --plain "$name" 2>/dev/null | grep -q . } for u in "$api_unit" "$ci_unit" "$git_unit" "$runner_unit"; do - if ! unit_present "$u"; then - say "skipping $u: not installed on this host" - elif unit_masked "$u"; then + if unit_masked "$u"; then say "skipping $u: masked (retired on this host)" + elif ! unit_present "$u"; then + say "skipping $u: not installed on this host" fi done @@ -148,17 +171,25 @@ if ((dry_run == 1)); then fi # --- 1. Drain --------------------------------------------------------------- -command -v sqlite3 >/dev/null 2>&1 \ - || die "sqlite3 CLI not found; the drain gate cannot read the DB. Install sqlite3 (drain can be hand-verified: SELECT count(*) FROM jobs WHERE status IN ('assigned','running'))" +# The busy signal comes from the same reader the release channel uses, so +# both agree on what "in flight" means. The reader prefers the sqlite3 CLI +# and falls back to the python3 sqlite3 module, and it never writes to the +# database. +busy_script="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)/gitforge-release-busy" +[[ -x "$busy_script" ]] || die "the busy-gate helper is missing: $busy_script" +drain_stderr="$(mktemp)" +trap 'rm -f "$drain_stderr"' EXIT while true; do - inflight=$(sqlite3 -readonly "$db_path" ".timeout 15000" \ - "SELECT count(*) FROM jobs WHERE status IN ('assigned','running');" 2>/dev/null) + if ! inflight=$(GITFORGE_RELEASE_DB="$db_path" "$busy_script" 2>"$drain_stderr"); then + # An unreadable DB must not spin forever — fail loudly instead. + die "drain query failed: cannot verify in-flight jobs on $db_path ($(cat "$drain_stderr"))" + fi + if [[ ! "$inflight" =~ ^[0-9]+$ ]]; then + die "drain query failed (unexpected output '$inflight'): cannot verify in-flight jobs on $db_path" + fi if [[ "$inflight" == "0" ]]; then say "$(date +%H:%M:%S) drained; cutting over" break - elif [[ -z "$inflight" ]]; then - # An unreadable DB must not spin forever — fail loudly instead. - die "drain query failed (empty output): cannot verify in-flight jobs on $db_path" fi say "$(date +%H:%M:%S) in-flight=${inflight}; waiting" sleep 5 diff --git a/services/api/src/main.rs b/services/api/src/main.rs index 0e07072a..5fa04f2e 100644 --- a/services/api/src/main.rs +++ b/services/api/src/main.rs @@ -14,6 +14,17 @@ use tokio::time::timeout; #[tokio::main] async fn main() -> anyhow::Result<()> { + // The release chain asks a binary for the commit it was built from + // (scripts/gitforge-release-stamps). Answer before any other work so + // that asking for provenance never starts a service, opens a database + // or binds a port. + if let Some(line) = + gitforge_common::build::version_line_if_requested("api", std::env::args().skip(1)) + { + println!("{line}"); + return Ok(()); + } + // This service is configured through environment variables. Rejecting // unsupported arguments prevents an operator from believing that a custom // port/host was applied when the process would otherwise silently ignore diff --git a/services/api/tests/api_gateway_flow.rs b/services/api/tests/api_gateway_flow.rs index ed5fc1c7..c0ba5f1c 100644 --- a/services/api/tests/api_gateway_flow.rs +++ b/services/api/tests/api_gateway_flow.rs @@ -34,7 +34,7 @@ struct ApiService { child: tokio::process::Child, port: u16, db_path: PathBuf, - harness_passphrase: String, + passphrase: String, } impl Drop for ApiService { @@ -83,6 +83,7 @@ async fn spawn_api() -> ApiService { .env("DATABASE_URL", format!("sqlite:{}", db_path.display())) .env("JWT_SECRET", "harness-jwt-secret") .env("GITFORGE_ARTIFACT_ROOT", &artifacts) + .env("GIT_ROOT", base.join("git-root")) .env("RUST_LOG", "warn") .stdout(Stdio::null()) .stderr(Stdio::null()) @@ -111,7 +112,7 @@ async fn spawn_api() -> ApiService { child, port, db_path, - harness_passphrase, + passphrase: harness_passphrase, } } @@ -145,7 +146,7 @@ async fn test_login_and_authenticated_repository_and_ssh_key_flow() { .expect("bad login request"); assert_eq!(bad.status(), reqwest::StatusCode::UNAUTHORIZED); - let good = login(&service.harness_passphrase) + let good = login(&service.passphrase) .send() .await .expect("login request"); @@ -286,7 +287,7 @@ async fn test_pipeline_run_status_rejects_invalid_credentials() { .post(format!("{base}/auth/login")) .json(&serde_json::json!({ "username": "gatewayowner", - "password": &service.harness_passphrase, + "password": &service.passphrase, })) .send() .await diff --git a/services/ci/src/main.rs b/services/ci/src/main.rs index 85b022de..17964eaa 100644 --- a/services/ci/src/main.rs +++ b/services/ci/src/main.rs @@ -168,6 +168,16 @@ impl TriggerFailure { #[tokio::main] async fn main() -> anyhow::Result<()> { + // The release chain asks a binary for the commit it was built from + // (scripts/gitforge-release-stamps). Answer before any other work so + // that asking for provenance never starts a service, opens a database + // or binds a port. + if let Some(line) = + gitforge_common::build::version_line_if_requested("ci", std::env::args().skip(1)) + { + println!("{line}"); + return Ok(()); + } // Initialize logging tracing_subscriber::fmt() .with_env_filter( @@ -2527,9 +2537,27 @@ async fn handle_push_event( let trigger_event = create_trigger_event(repo_id, &payload.new_hash, ref_name); let pipeline_id = trigger_event.pipeline_id; + // The commit the ref pointed at BEFORE this push. A branch creation + // carries an all-zero old hash: there is no base, so nothing is + // published and the consumer fails safe. The base must travel with the + // run because the CI workspace clones AFTER the push landed — inside + // the workspace `origin/main` already IS the pushed tip, so a merge-base + // range diff is empty by construction and the only surviving base is + // the one the platform captured at push time. + let push_base = if gitforge_common::is_zero_hash(&payload.old_hash) { + None + } else { + Some(payload.old_hash.clone()) + }; + + // The engine runs the augmented definition; the committed definition + // stays pristine for the pipeline cache and the persisted pipeline + // config, so a later run does not inherit a stale push base. + let engine_pipeline = gitforge_ci::pipeline::with_push_base(&pipeline, push_base.as_deref()); + // Create and start the CI engine let engine = Arc::new( - CiEngine::new(trigger_event, pipeline.clone()) + CiEngine::new(trigger_event, engine_pipeline) .await .map_err(|e| TriggerFailure::terminal(e.into()))?, ); @@ -2574,6 +2602,7 @@ async fn handle_push_event( ); db_run.id = state.run_id; db_run.ref_name = Some(payload.ref_name.clone()); + db_run.push_base_sha = push_base.clone(); db_run.start(); gitforge_db::queries::PipelineQueries::activate_pipeline_and_create_run( pool, @@ -2621,6 +2650,25 @@ async fn handle_push_event( state.run_id )) })?; + // Durable-plan BEFORE the workspace clone. prepare_run_workspace + // clones the whole bare repository, and on slow storage it can + // take minutes or hang (GF-19: the 2026-11-06 runs stranded as + // `running` with ZERO job rows because planning ran only after + // the clone returned). The clone does not affect the job SET or + // any definition field except the workspace fallback in + // `working_dir`, and enqueue_ready_jobs' upsert re-persists the + // full execution plan with the real workspace path, so a row + // written now is corrected before any runner sees it. A hung + // clone now strands a PLANNED run: rows exist, reconciliation + // and the store see the run's true shape, and a clone failure + // grades the run failed with traceable rows instead of a hollow + // `running`. + if let Err(error) = persist_planned_jobs(pool, &engine, state.run_id, None).await { + tracing::error!(run = %state.run_id, %error, "failed to persist planned jobs before workspace prep"); + fail_run(pool, state.run_id, &error).await; + pipeline_registry.write().await.remove(&state.run_id); + return Err(TriggerFailure::terminal_after_run(error, state.run_id)); + } match prepare_run_workspace(pool, repo_id, state.run_id, &payload.new_hash).await { Ok(path) => Some(path), Err(error) => { @@ -2821,14 +2869,43 @@ async fn run_scheduler_event_consumer( Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => continue, Err(_) => break, }; - let SchedulerEvent::JobCompleted { - job_id, - pipeline_run_id, - runner_id, - success, - } = event - else { - continue; + let (job_id, pipeline_run_id, runner_id, success) = match event { + SchedulerEvent::JobAssigned { + job_id, + runner_id, + pipeline_run_id, + } => { + // The engine's mirror must leave `Queued` when the job is + // dispatched: `ready_jobs()` releases every job that is + // still `Queued`, so a dispatched job whose mirror stayed + // `Queued` was re-enqueued by every later completion of + // the run. That duplicate enqueue demoted the running + // job's durable row, which refused the runner's heartbeat + // and rotated its lease token (GF-12). + let engine = pipeline_registry + .read() + .await + .get(&pipeline_run_id) + .cloned(); + let Some(engine) = engine else { + tracing::warn!( + "assignment received for unknown pipeline run {}", + pipeline_run_id + ); + continue; + }; + if let Err(error) = engine.assign_job(job_id, runner_id).await { + tracing::error!(%job_id, %error, "failed to mark the dispatched job assigned"); + } + continue; + } + SchedulerEvent::JobCompleted { + job_id, + pipeline_run_id, + runner_id, + success, + } => (job_id, pipeline_run_id, runner_id, success), + _ => continue, }; let engine = pipeline_registry .read() @@ -3141,6 +3218,18 @@ async fn persist_planned_jobs( // Durable-write discipline (F21/F23): losing one row to a transient // busy aborts the whole loop and strands a half-planned run (the // caller fails it), so each row fights contention instead. + // Skip rows the pre-clone planning pass already wrote (GF-19: + // planning runs before AND after the workspace clone so a hung + // clone cannot strand a jobless `running` row). A plain re-create + // would hit the unique id and fail the run; the working_dir the + // second pass would correct is fixed at dispatch time anyway, when + // enqueue_ready_jobs' upsert rewrites the full definition. + if gitforge_db::queries::JobQueries::get(pool, db_job.id) + .await + .is_ok_and(|row| row.is_some()) + { + continue; + } persist_with_retry(|| gitforge_db::queries::JobQueries::create(pool, &db_job)).await?; } tracing::info!(run = %run_id, planned = planned.len(), "persisted planned job rows"); @@ -3276,6 +3365,13 @@ async fn rebuild_live_engines( // the engine absorbed it — all leave no running row behind, and the // stalled chains observed live (tail rows pending for ~21h across a // control-plane restart) are exactly of that shape. + // The persisted pipeline config holds the committed definition + // (no push base); the run row holds the base the push carried. + // Re-injecting it here is what makes the push base survive a + // control-plane restart — the `jobs` table has no env column, so + // this is the only durable carrier. + let definition = + gitforge_ci::pipeline::with_push_base(&definition, run.push_base_sha.as_deref()); let engine = match CiEngine::rebuild( run.id, run.pipeline_id, @@ -5990,6 +6086,248 @@ jobs: ); } + // --- Push base publication (CI_PUSH_BASE_SHA) --- + // + // The CI workspace clones AFTER the push landed, so inside the + // workspace `origin/main` already IS the pushed tip: a merge-base range + // diff is empty by construction, and the tip commit alone describes + // only the last commit of a multi-commit push. The pre-push tip is the + // only base that describes the whole push, and it reaches the consumer + // solely through the job environment. + + /// A two-job definition whose pipeline-level environment is empty, so + /// the push base is the only variable a job can carry. + fn push_base_pipeline() -> PipelineDefinition { + let job = |name: &str, needs: &[&str]| JobDefinition { + name: name.to_string(), + image: "rust:latest".to_string(), + needs: needs.iter().map(ToString::to_string).collect(), + env: HashMap::new(), + steps: vec![StepDefinition { + name: format!("{name}-step"), + run: "true".to_string(), + env: None, + working_directory: None, + condition: None, + }], + timeout: None, + retry: None, + skip_if: None, + volumes: Vec::new(), + cache: None, + lane: Default::default(), + fault: None, + }; + PipelineDefinition { + name: "push-base-pipeline".to_string(), + version: "1.0".to_string(), + trigger_on: vec![TriggerType::Push], + environment: HashMap::new(), + jobs: vec![ + job("detect-changes", &[]), + job("clippy", &["detect-changes"]), + ], + } + } + + /// A push envelope for `repo_id` carrying `old_hash` as the pre-push + /// tip of the ref. + fn push_envelope(repo_id: gitforge_common::RepoId, old_hash: &str) -> EventEnvelope { + EventEnvelope::new( + EventType::PushReceived, + EventPayload::PushReceived(PushReceivedPayload { + repo_id, + ref_name: "refs/heads/main".to_string(), + old_hash: old_hash.to_string(), + new_hash: "b".repeat(40), + pusher_id: None, + pipeline_id: None, + }), + Some(repo_id), + None, + ) + } + + /// What one push produced: the durable run row, the environment of every + /// planned job, and the definition the pipeline cache kept. + struct PushOutcome { + run: gitforge_db::models::PipelineRun, + job_envs: Vec<(String, HashMap)>, + cached_environment: HashMap, + persisted_environment: HashMap, + } + + /// Drive one push through `handle_push_event` and collect what it + /// produced. + async fn plan_push(old_hash: &str) -> PushOutcome { + let pool = gitforge_db::Pool::memory().await.expect("memory pool"); + pool.migrate().await.expect("migrate"); + let user = gitforge_db::models::User::new( + "push-base-owner".to_string(), + "push-base-owner@example.test".to_string(), + "hash".to_string(), + ); + gitforge_db::queries::UserQueries::create(&pool, &user) + .await + .expect("create user"); + let repo_id = gitforge_common::RepoId::new(); + gitforge_db::queries::RepoQueries::create( + &pool, + &gitforge_db::models::Repository::new_with_id( + repo_id, + "push-base-repo".to_string(), + user.id, + std::env::temp_dir() + .join(format!("gitforge-push-base-{repo_id}")) + .display() + .to_string(), + ), + ) + .await + .expect("create repository"); + + let scheduler = Arc::new(Scheduler::new()); + let pipeline_cache: Arc> = + Arc::new(std::sync::Mutex::new(HashMap::new())); + pipeline_cache + .lock() + .unwrap_or_else(PoisonError::into_inner) + .insert(repo_id, push_base_pipeline()); + // The workspace is pre-seeded so the push plans without cloning: the + // subject here is the environment the jobs carry, not the checkout. + let workspace_paths: Arc< + std::sync::Mutex>>, + > = Arc::new(std::sync::Mutex::new(HashMap::new())); + workspace_paths + .lock() + .unwrap_or_else(PoisonError::into_inner) + .insert( + repo_id, + Some("/tmp/gitforge-push-base-workspace".to_string()), + ); + let run_workspace_paths = run_workspace_paths_cache(); + let pipeline_registry: Arc> = + Arc::new(tokio::sync::RwLock::new(HashMap::new())); + + let outcome = handle_push_event( + &push_envelope(repo_id, old_hash), + &scheduler, + &pipeline_cache, + Some(&pool), + &workspace_paths, + &run_workspace_paths, + &pipeline_registry, + ) + .await + .expect("a push plans a run"); + let TriggerOutcome::Started(run_id) = outcome else { + panic!("a push must start a run"); + }; + + let run = gitforge_db::queries::PipelineRunQueries::get(&pool, run_id) + .await + .expect("read run row") + .expect("durable run row"); + let engine = pipeline_registry + .read() + .await + .get(&run_id) + .cloned() + .expect("the planned engine is registered"); + let job_envs = engine + .planned_jobs() + .into_iter() + .map(|(job_id, name)| { + let env = engine + .job_definition(job_id) + .expect("planned job definition") + .env; + (name, env) + }) + .collect(); + + let cached_environment = pipeline_cache + .lock() + .unwrap_or_else(PoisonError::into_inner) + .get(&repo_id) + .expect("cached definition") + .environment + .clone(); + let persisted_environment = + gitforge_db::queries::PipelineQueries::get(&pool, run.pipeline_id) + .await + .expect("read pipeline row") + .expect("durable pipeline row") + .config + .get("environment") + .and_then(|value| serde_json::from_value(value.clone()).ok()) + .unwrap_or_default(); + + PushOutcome { + run, + job_envs, + cached_environment, + persisted_environment, + } + } + + // The pre-push tip must reach EVERY planned job: a classifier that + // reads only its own job's environment sees an empty range otherwise. + #[tokio::test] + async fn test_push_event_publishes_the_push_base_to_every_job() { + let old_hash = "a".repeat(40); + let outcome = plan_push(&old_hash).await; + + assert_eq!( + outcome.run.push_base_sha.as_deref(), + Some(old_hash.as_str()), + "the durable run row carries the pre-push tip" + ); + assert_eq!(outcome.run.ref_name.as_deref(), Some("refs/heads/main")); + assert_eq!(outcome.job_envs.len(), 2, "the pipeline has two jobs"); + for (name, env) in &outcome.job_envs { + assert_eq!( + env.get(gitforge_ci::pipeline::PUSH_BASE_ENV), + Some(&old_hash), + "job {name} must carry the pre-push tip" + ); + } + + // The push base belongs to the RUN, not to the definition: the + // cached definition and the persisted pipeline config stay pristine + // so a later run cannot inherit a stale base. + assert!( + !outcome + .cached_environment + .contains_key(gitforge_ci::pipeline::PUSH_BASE_ENV), + "the cached definition must not carry a push base" + ); + assert!( + !outcome + .persisted_environment + .contains_key(gitforge_ci::pipeline::PUSH_BASE_ENV), + "the persisted pipeline config must not carry a push base" + ); + } + + // A branch creation has no pre-push tip: nothing is published, so the + // consumer sees no variable and fails safe. + #[tokio::test] + async fn test_push_event_with_a_zero_old_hash_publishes_no_push_base() { + let outcome = plan_push(&"0".repeat(40)).await; + + assert_eq!( + outcome.run.push_base_sha, None, + "a branch creation has no pre-push tip" + ); + for (name, env) in &outcome.job_envs { + assert!( + !env.contains_key(gitforge_ci::pipeline::PUSH_BASE_ENV), + "job {name} must not carry a push base for a branch creation" + ); + } + } + // --- Trigger settlement classification --- // // A committed definition that can never parse must fail terminally and diff --git a/services/git-server/src/main.rs b/services/git-server/src/main.rs index 53e311f8..e2e66986 100644 --- a/services/git-server/src/main.rs +++ b/services/git-server/src/main.rs @@ -48,6 +48,16 @@ struct InfoRefsQuery { #[tokio::main] async fn main() -> anyhow::Result<()> { + // The release chain asks a binary for the commit it was built from + // (scripts/gitforge-release-stamps). Answer before any other work so + // that asking for provenance never starts a service, opens a database + // or binds a port. + if let Some(line) = + gitforge_common::build::version_line_if_requested("git-server", std::env::args().skip(1)) + { + println!("{line}"); + return Ok(()); + } // Initialize logging tracing_subscriber::fmt() .with_env_filter( diff --git a/services/runner/src/main.rs b/services/runner/src/main.rs index 7d2d2684..cfba0721 100644 --- a/services/runner/src/main.rs +++ b/services/runner/src/main.rs @@ -12,6 +12,16 @@ use tokio::time::timeout; #[tokio::main] async fn main() -> anyhow::Result<()> { + // The release chain asks a binary for the commit it was built from + // (scripts/gitforge-release-stamps). Answer before any other work so + // that asking for provenance never starts a service, opens a database + // or binds a port. + if let Some(line) = + gitforge_common::build::version_line_if_requested("runner", std::env::args().skip(1)) + { + println!("{line}"); + return Ok(()); + } // Initialize logging tracing_subscriber::fmt() .with_env_filter( diff --git a/systemd/user/README.md b/systemd/user/README.md index 96894ae5..3a55591a 100644 --- a/systemd/user/README.md +++ b/systemd/user/README.md @@ -58,8 +58,11 @@ event causes `status` to become `attention` until it is completed or returned to pending. Before assembling a release bundle, run -`scripts/gitforge-release-preflight `. It fails closed -unless `api`, `ci`, `git-server`, and `runner` are all present and executable. +`scripts/gitforge-release-preflight `. It +fails closed unless `api`, `ci`, `git-server`, and `runner` are all present +and executable, and unless each of them reports that it was built from +`` — the release binaries live in gitignored paths, so their +own stamp is the only evidence that they are not stale leftovers. The candidate CI binary owns the scheduler HTTP API, so the legacy standalone `gitforge-scheduler-service` must not be mixed into the bundle. diff --git a/tests/release-auto-busy-gate-selftest.sh b/tests/release-auto-busy-gate-selftest.sh new file mode 100755 index 00000000..2e68d336 --- /dev/null +++ b/tests/release-auto-busy-gate-selftest.sh @@ -0,0 +1,345 @@ +#!/usr/bin/env bash +# Contract test for the release channel's busy gate (GF-16). +# +# Why this needs a test: the gate used to count the runner's +# `gitforce-job-*` containers. The runner keeps a warm pool of containers +# that have NO job row at all, so the count was permanently non-zero and +# every cutover was deferred forever - the release channel could never +# promote anything, and it reported the condition as a normal "deferred", +# which looks like ordinary operation. +# +# The gate now reads the authoritative store: a job is in flight when its +# row is `assigned` or `running` AND its proof of life is still inside the +# fence grace window (the same liveness term the scheduler uses to decide +# whether a row is alive). The cases below pin the three properties that +# matter: +# - the busy signal is the store, not the container names: a `docker` +# stub on PATH fails the test if it is ever called, and the script's +# source must not mention the container-name filter at all, +# - a stale row cannot wedge the gate: a `running` row whose heartbeat is +# older than the grace window does not count as in flight, +# - an unreadable store fails closed: the cutover is refused, never +# started on an unknown busy state. +# +# Both readers of the store are exercised: the sqlite3 CLI when the host +# has one, and the python3 fallback that the CI image (no sqlite3 CLI) +# forces. +# +# Usage: bash tests/release-auto-busy-gate-selftest.sh +set -u -o pipefail + +repo_root=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd) +busy_script="$repo_root/scripts/gitforge-release-busy" +auto_script="$repo_root/scripts/gitforge-release-auto" + +[[ -f "$busy_script" ]] || { printf 'selftest: missing %s\n' "$busy_script" >&2; exit 1; } +[[ -f "$auto_script" ]] || { printf 'selftest: missing %s\n' "$auto_script" >&2; exit 1; } + +tmp=$(mktemp -d) +trap 'find "$tmp" -depth -delete 2>/dev/null || true' EXIT + +fail=0 +check() { + if [[ "$3" == "$2" ]]; then + printf 'ok %s\n' "$1" + else + printf 'FAIL %s: expected [%s], got [%s]\n' "$1" "$2" "$3" >&2 + fail=1 + fi +} +check_contains() { + if printf '%s\n' "$3" | grep -qF -- "$2"; then + printf 'ok %s\n' "$1" + else + printf 'FAIL %s: expected to find: %s\n' "$1" "$2" >&2 + printf '%s\n' "$3" >&2 + fail=1 + fi +} +check_absent() { + if printf '%s\n' "$3" | grep -qF -- "$2"; then + printf 'FAIL %s: did not expect to find: %s\n' "$1" "$2" >&2 + printf '%s\n' "$3" >&2 + fail=1 + else + printf 'ok %s\n' "$1" + fi +} + +# ── The fixture builder ─────────────────────────────────────────────────────── +# One database per case, with the schema the live store uses (jobs and +# runners, the columns the gate reads) and timestamps in the store's own +# RFC3339-with-offset format. +cat >"$tmp/mkdb.py" <<'PY' +import datetime +import os +import sqlite3 +import sys + +SCHEMA = """ +CREATE TABLE runners ( + id TEXT PRIMARY KEY, name TEXT NOT NULL, runner_type TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'offline', capacity INTEGER NOT NULL DEFAULT 1, + labels TEXT NOT NULL DEFAULT '[]', last_heartbeat TEXT, + created_at TEXT NOT NULL, updated_at TEXT NOT NULL); +CREATE TABLE jobs ( + id TEXT PRIMARY KEY, pipeline_run_id TEXT NOT NULL, name TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'pending', runner_id TEXT, started_at TEXT, + finished_at TEXT, retry_count INTEGER NOT NULL DEFAULT 0, + created_at TEXT NOT NULL, commands TEXT NOT NULL DEFAULT '[]', + image TEXT NOT NULL DEFAULT 'rust:latest', working_dir TEXT, + result_json TEXT, lease_token TEXT, lease_generation INTEGER NOT NULL DEFAULT 0, + timeout_secs INTEGER NOT NULL DEFAULT 300, heartbeat_at TEXT, + lane TEXT NOT NULL DEFAULT 'standard', deferred_reason TEXT); +""" + +# job_hb / runner_hb: seconds ago, or None for "no such timestamp". +# runner: "live" (a runner whose heartbeat is 10s old), "dead" (a runner +# whose heartbeat is an hour old), or "missing" (no runner row at all). +CASES = { + "empty": [], + "fresh-running": [{"id": "j1", "status": "running", "job_hb": 10, "runner": "live"}], + "stale-running": [{"id": "j1", "status": "running", "job_hb": 3600, "runner": "dead"}], + "assigned-live-runner": [{"id": "j1", "status": "assigned", "job_hb": None, "runner": "live"}], + "assigned-dead-runner": [{"id": "j1", "status": "assigned", "job_hb": None, "runner": "dead"}], + "queued": [{"id": "j1", "status": "queued", "job_hb": 10, "runner": "live"}], + "pending": [{"id": "j1", "status": "pending", "job_hb": 10, "runner": "live"}], + "succeeded": [{"id": "j1", "status": "succeeded", "job_hb": 10, "runner": "live"}], + "mixed": [ + {"id": "j1", "status": "running", "job_hb": 10, "runner": "live"}, + {"id": "j2", "status": "running", "job_hb": 3600, "runner": "dead"}, + {"id": "j3", "status": "queued", "job_hb": 10, "runner": "live"}, + {"id": "j4", "status": "assigned", "job_hb": None, "runner": "live"}, + ], + "runner-heartbeat-only": [{"id": "j1", "status": "running", "job_hb": None, "runner": "live"}], + "orphan-running": [{"id": "j1", "status": "running", "job_hb": 10, "runner": "missing"}], + "orphan-no-proof": [{"id": "j1", "status": "running", "job_hb": None, "runner": "missing", + "started": None}], +} + + +def stamp(moment): + if moment is None: + return None + return moment.strftime("%Y-%m-%dT%H:%M:%S.%f") + "+00:00" + + +def main(): + db_path, case = sys.argv[1], sys.argv[2] + if os.path.exists(db_path): + os.unlink(db_path) + now = datetime.datetime.now(datetime.timezone.utc) + ago = lambda seconds: now - datetime.timedelta(seconds=seconds) # noqa: E731 + connection = sqlite3.connect(db_path) + connection.executescript(SCHEMA) + for spec in CASES[case]: + job_id = spec["id"] + started = spec.get("started", 600) + runner_id = None + if spec["runner"] == "live": + runner_id = "runner-" + job_id + connection.execute( + "INSERT INTO runners VALUES (?,?,?,?,?,?,?,?,?)", + (runner_id, "runner-" + job_id, "docker", "online", 1, "[]", + stamp(ago(10)), stamp(ago(600)), stamp(ago(600)))) + elif spec["runner"] == "dead": + runner_id = "runner-" + job_id + connection.execute( + "INSERT INTO runners VALUES (?,?,?,?,?,?,?,?,?)", + (runner_id, "runner-" + job_id, "docker", "offline", 1, "[]", + stamp(ago(3600)), stamp(ago(600)), stamp(ago(600)))) + connection.execute( + "INSERT INTO jobs (id,pipeline_run_id,name,status,runner_id,started_at," + "created_at,heartbeat_at) VALUES (?,?,?,?,?,?,?,?)", + (job_id, "run-1", job_id, spec["status"], runner_id, + stamp(ago(started)) if started is not None else None, + stamp(ago(600)), + stamp(ago(spec["job_hb"])) if spec["job_hb"] is not None else None)) + connection.commit() + connection.close() + + +main() +PY + +# ── A PATH with no sqlite3 CLI, to force the python3 fallback ──────────────── +nosql="$tmp/nosql" +mkdir -p "$nosql" +for tool in bash env sed dirname tail python3 cat; do + resolved="$(command -v "$tool" 2>/dev/null || true)" + [[ -n "$resolved" ]] && ln -sf "$resolved" "$nosql/$tool" +done + +digest() { + python3 - "$1" <<'PY' +import hashlib +import sys +print(hashlib.sha256(open(sys.argv[1], "rb").read()).hexdigest()) +PY +} + +# ── Case set 1: the store reader ───────────────────────────────────────────── +declare -A EXPECTED=( + [empty]=0 + [fresh-running]=1 + [stale-running]=0 + [assigned-live-runner]=1 + [assigned-dead-runner]=0 + [queued]=0 + [pending]=0 + [succeeded]=0 + [mixed]=2 + [runner-heartbeat-only]=1 + [orphan-running]=1 + [orphan-no-proof]=0 +) + +for case in empty fresh-running stale-running assigned-live-runner assigned-dead-runner \ + queued pending succeeded mixed runner-heartbeat-only orphan-running orphan-no-proof; do + db="$tmp/db-$case.sqlite" + python3 "$tmp/mkdb.py" "$db" "$case" || { fail=1; continue; } + before="$(digest "$db")" + + # The reader this host would use (sqlite3 CLI when it has one). + out="$(GITFORGE_RELEASE_DB="$db" bash "$busy_script" 2>/dev/null)" + check "reader/$case" "${EXPECTED[$case]}" "$out" + + # The reader the CI image is forced onto (no sqlite3 CLI on PATH). + out="$(PATH="$nosql" GITFORGE_RELEASE_DB="$db" bash "$busy_script" 2>/dev/null)" + check "reader-fallback/$case" "${EXPECTED[$case]}" "$out" + + # The reader must not modify the store it reads. + check "read-only/$case" "$before" "$(digest "$db")" +done + +# A store that cannot be read fails closed, with a message that names the +# problem - a silently unreadable store would stall the channel forever. +out="$(GITFORGE_RELEASE_DB="$tmp/does-not-exist.sqlite" bash "$busy_script" 2>&1)" +status=$? +check "missing-store/exit" "1" "$status" +check_contains "missing-store/message" "live database not found" "$out" + +# ── Case set 2: the release channel's wiring ───────────────────────────────── +# The scripts under test live in a directory of their own, because the +# release script calls its siblings by path. +wire="$tmp/wire" +mkdir -p "$wire/scripts" "$wire/repo/releases" +cp "$auto_script" "$busy_script" "$wire/scripts/" +chmod +x "$wire/scripts/gitforge-release-auto" "$wire/scripts/gitforge-release-busy" +# The verification step is not what this test is about. +printf '#!/usr/bin/env bash\nexit 0\n' >"$wire/scripts/gitforge-release-verify" +chmod +x "$wire/scripts/gitforge-release-verify" + +promoted="$wire/repo/releases/gitforge-aaaaaaa-20260101" +candidate="$wire/repo/releases/gitforge-bbbbbbb-20260102" +mkdir -p "$promoted" "$candidate" +printf 'ready\n' >"$promoted/READY" +printf 'ready\n' >"$candidate/READY" +ln -sfn "$promoted" "$wire/repo/releases/gitforge-current" + +stubs="$wire/bin" +mkdir -p "$stubs" +cat >"$stubs/systemctl" <<'STUB' +#!/usr/bin/env bash +# Every unit this host runs is enabled; the retired-unit cases are covered +# by tests/release-auto-restart-plan-selftest.sh. +for arg in "$@"; do + if [[ "$arg" == "LoadState" ]]; then + printf 'enabled\n' + exit 0 + fi +done +exit 0 +STUB +cat >"$stubs/docker" <<'STUB' +#!/usr/bin/env bash +# The busy gate must never consult the container names. +printf 'docker was called\n' >>"${GF_DOCKER_CALLED:?}" +exit 1 +STUB +chmod +x "$stubs/systemctl" "$stubs/docker" + +run_auto() { + local db="$1" + PATH="$stubs:$PATH" \ + GF_DOCKER_CALLED="$wire/docker-called" \ + GITFORGE_AUTO_REPO="$wire/repo" \ + GITFORGE_RELEASE_DB="$db" \ + bash "$wire/scripts/gitforge-release-auto" 2>&1 +} + +db_busy="$tmp/db-busy.sqlite" +python3 "$tmp/mkdb.py" "$db_busy" fresh-running +db_idle="$tmp/db-idle.sqlite" +python3 "$tmp/mkdb.py" "$db_idle" empty + +# A job in flight defers the cutover, and the channel says so. +out="$(run_auto "$db_busy")" +status=$? +check "busy/exit" "0" "$status" +check_contains "busy/deferred" "deferred: 1 job(s) in flight" "$out" +check_contains "busy/inflight-reported" "inflight=1" "$out" +check "busy/no-docker" "" "$(cat "$wire/docker-called" 2>/dev/null || true)" + +# An idle store lets the pass continue to the dry-run report. +out="$(run_auto "$db_idle")" +status=$? +check "idle/exit" "0" "$status" +check_contains "idle/dry-run" "dry-run: would promote gitforge-bbbbbbb-20260102" "$out" +check "idle/no-docker" "" "$(cat "$wire/docker-called" 2>/dev/null || true)" + +# An unreadable store refuses the cutover instead of guessing. +out="$(run_auto "$tmp/does-not-exist.sqlite")" +status=$? +check "unreadable/exit" "1" "$status" +check_contains "unreadable/message" "cannot read the in-flight job count" "$out" +check_absent "unreadable/no-promotion" "would promote" "$out" +check "unreadable/link-untouched" "gitforge-aaaaaaa-20260101" \ + "$(basename "$(readlink -- "$wire/repo/releases/gitforge-current")")" + +# The source must not go back to counting containers. +check_absent "source/no-container-filter" "gitforce-job-" "$(cat "$auto_script")" + +# - Case set 3: the operator cutover's drain gate ---------------------------- +# The sibling cutover script drains on the same signal. It must not count a +# row left behind by a dead runner as work in flight (that would wait for +# ever), and it must wait for a job that really is running. +cutover="$repo_root/scripts/gitforge-remote-cutover" +if [[ -f "$cutover" ]]; then + cp "$cutover" "$wire/scripts/" + chmod +x "$wire/scripts/gitforge-remote-cutover" + bundle="$wire/bundle" + mkdir -p "$bundle/bin" + install -m 0755 /dev/null "$bundle/bin/ci" + ln -sfn "$promoted" "$wire/link" + + db_stale="$tmp/db-stale.sqlite" + python3 "$tmp/mkdb.py" "$db_stale" stale-running + + run_cutover() { + PATH="$stubs:$PATH" timeout 30 bash "$wire/scripts/gitforge-remote-cutover" \ + --units user --db "$1" --bundle "$bundle" --link "$wire/link" 2>&1 + } + + # An idle store drains. + out="$(run_cutover "$db_idle")" + check_contains "cutover/idle-drains" "drained; cutting over" "$out" + + # A store whose only running row is stale drains too - the row is dead + # work, and the scheduler fences it on its own. + out="$(run_cutover "$db_stale")" + check_contains "cutover/stale-drains" "drained; cutting over" "$out" + + # A job that really is running makes the cutover wait. + out="$(PATH="$stubs:$PATH" timeout 8 bash "$wire/scripts/gitforge-remote-cutover" \ + --units user --db "$db_busy" --bundle "$bundle" --link "$wire/link" 2>&1)" + check_contains "cutover/busy-waits" "in-flight=1; waiting" "$out" +else + printf 'ok cutover drain cases (script absent)\n' +fi + +if ((fail)); then + printf 'selftest: FAILED\n' >&2 + exit 1 +fi +printf 'selftest: OK\n' diff --git a/tests/release-auto-restart-plan-selftest.sh b/tests/release-auto-restart-plan-selftest.sh new file mode 100755 index 00000000..da9dbd21 --- /dev/null +++ b/tests/release-auto-restart-plan-selftest.sh @@ -0,0 +1,112 @@ +#!/usr/bin/env bash +# Regression test: a unit masked on this host must be excluded from the +# release-auto restart set. +# +# Why this needs a test (2026-10-10 incident): the NAS runner was retired +# by masking its unit when CI moved to the fedora host. The pre-fix script +# restarted a hardcoded unit list, so every promote tried to restart the +# masked unit, failed, and rolled back - forever. The fix skips masked +# units; without a test, a future edit that restores the hardcoded list +# fails only in production, on the release channel. +# +# The script classifies units by asking systemd, so the test puts a fake +# `systemctl` on PATH and asserts both directions: a masked unit is +# skipped, and a unit that is not masked is still restarted. +# +# The fake answers the query the script actually makes - +# `systemctl show -p LoadState --value` - which reports "masked" +# for a template instance as well as for a concrete unit file. The older +# `systemctl list-unit-files --no-legend` form cannot see an +# instance at all, which is how a masked instance reached the restart set +# in the first place. +# +# Usage: bash tests/release-auto-restart-plan-selftest.sh +set -u -o pipefail + +repo_root=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd) +script="$repo_root/scripts/gitforge-release-auto" + +[[ -f "$script" ]] || { printf 'selftest: missing %s\n' "$script" >&2; exit 1; } + +tmp=$(mktemp -d) +trap 'find "$tmp" -depth -delete 2>/dev/null || true' EXIT + +bin="$tmp/bin" +mkdir -p "$bin" "$tmp/repo" + +# The fake systemctl reports the units named in FAKE_MASKED as masked and +# everything else as enabled. +cat >"$bin/systemctl" <<'FAKE' +#!/usr/bin/env bash +set -u +# `systemctl show -p LoadState --value`: the unit is the second +# argument, and the answer is the bare state. +unit="${2:-}" +case ",${FAKE_MASKED:-}," in + *",$unit,"*) printf 'masked\n' ;; + *) printf 'enabled\n' ;; +esac +FAKE +chmod +x "$bin/systemctl" + +run_plan() { + PATH="$bin:$PATH" \ + FAKE_MASKED="$1" \ + GITFORGE_AUTO_REPO="$tmp/repo" \ + bash "$script" --print-restart-plan 2>&1 +} + +fail=0 +check_contains() { + if [[ "$1" == *"$2"* ]]; then + printf 'ok %s\n' "$3" + else + printf 'FAIL %s: expected to find: %s\n' "$3" "$2" >&2 + printf '%s\n' "$1" >&2 + fail=1 + fi +} +check_absent() { + if [[ "$1" == *"$2"* ]]; then + printf 'FAIL %s: did not expect to find: %s\n' "$3" "$2" >&2 + printf '%s\n' "$1" >&2 + fail=1 + else + printf 'ok %s\n' "$3" + fi +} + +# - Case 1: a masked runner unit is skipped --------------------------------- +out="$(run_plan "gitforge@runner.service")" +check_contains "$out" "skipping gitforge@runner.service: masked on this host" \ + "masked unit must be reported as skipped" +check_contains "$out" "stateless=gitforge@api.service gitforge@git-server.service" \ + "unmasked stateless units must stay in the plan" +check_contains "$out" "scheduled=gitforge@ci.service" \ + "unmasked scheduled units must stay in the plan" +check_absent "$out" "scheduled=gitforge@runner.service" \ + "masked unit must not appear in the restart plan" + +# - Case 2: with nothing masked, every unit is restarted --------------------- +out="$(run_plan "")" +check_absent "$out" "masked on this host" \ + "no unit is masked, so nothing may be reported as skipped" +check_contains "$out" "stateless=gitforge@api.service gitforge@git-server.service" \ + "stateless units must stay in the plan" +check_contains "$out" "scheduled=gitforge@ci.service gitforge@runner.service" \ + "scheduled units must stay in the plan" + +# - Case 3: a masked stateless unit is skipped too --------------------------- +out="$(run_plan "gitforge@git-server.service")" +check_contains "$out" "skipping gitforge@git-server.service: masked on this host" \ + "masked stateless unit must be reported as skipped" +check_contains "$out" "stateless=gitforge@api.service" \ + "remaining stateless units must stay in the plan" +check_absent "$out" "stateless=gitforge@api.service gitforge@git-server.service" \ + "masked stateless unit must not appear in the restart plan" + +if ((fail)); then + printf 'selftest: FAILED\n' >&2 + exit 1 +fi +printf 'selftest: OK\n' diff --git a/tests/remote-cutover-unit-detection-selftest.sh b/tests/remote-cutover-unit-detection-selftest.sh new file mode 100755 index 00000000..8d98f6ca --- /dev/null +++ b/tests/remote-cutover-unit-detection-selftest.sh @@ -0,0 +1,165 @@ +#!/usr/bin/env bash +# Regression selftest for the cutover's unit detection. +# +# The cutover decides which units to stop and start from what systemctl +# reports. A detection gap there is a SILENT NO-OP RELEASE: the symlink +# flips, nothing restarts, and the old binaries keep serving while the +# cutover reports success. That is exactly what happened on the NAS +# profile, where every unit is an INSTANCE of the gitforge@.service +# template and `systemctl list-unit-files` enumerates no instance at all. +# +# The cases below drive the real script with a stubbed systemctl that +# reproduces the two topologies the cutover meets: +# - the NAS profile: template instances known to the manager, absent from +# list-unit-files, one of them retired (masked), +# - the fedora profile: concrete unit files that were never started, so +# the manager knows nothing about them at runtime. +set -u -o pipefail + +repo_root=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd) +script="$repo_root/scripts/gitforge-remote-cutover" + +[[ -f "$script" ]] || { printf 'selftest: missing %s\n' "$script" >&2; exit 1; } + +tmp=$(mktemp -d) +trap 'find "$tmp" -depth -delete 2>/dev/null || true' EXIT + +mkdir -p "$tmp/bin" "$tmp/bundle/bin" +: >"$tmp/db" +install -m 0755 /dev/null "$tmp/bundle/bin/ci" +ln -sfn "$tmp/bundle" "$tmp/link" + +cat >"$tmp/bin/systemctl" <<'STUB' +#!/usr/bin/env bash +# Stub systemctl. Fixture files, one name [state] per line: +# GF_FIXTURE_UNIT_FILES concrete unit files (what list-unit-files sees) +# GF_FIXTURE_UNITS instances the manager knows (what list-units sees) +# GF_FIXTURE_MASKED retired instances (LoadState=masked) +set -u +verb="" +unit="" +for arg in "$@"; do + case "$arg" in + --user|--system|--all|--no-legend|--plain) continue ;; + esac + if [[ -z "$verb" ]]; then + case "$arg" in + list-unit-files|list-units|show) verb="$arg" ;; + esac + continue + fi + case "$arg" in + -p|--value|LoadState) continue ;; + esac + unit="$arg" +done + +case "$verb" in + list-unit-files) + [[ -s "${GF_FIXTURE_UNIT_FILES:-}" ]] || exit 0 + while read -r name state; do + [[ -n "$name" ]] || continue + [[ "$name" == "$unit" ]] && printf '%s %s\n' "$name" "$state" + done <"$GF_FIXTURE_UNIT_FILES" + ;; + list-units) + [[ -s "${GF_FIXTURE_UNITS:-}" ]] || exit 0 + while read -r name state; do + [[ -n "$name" ]] || continue + [[ "$name" == "$unit" ]] && printf '%s %s active running Stub Unit\n' "$name" "$state" + done <"$GF_FIXTURE_UNITS" + ;; + show) + if [[ -s "${GF_FIXTURE_MASKED:-}" ]]; then + while read -r name; do + [[ -n "$name" ]] || continue + [[ "$name" == "$unit" ]] && { printf 'masked\n'; exit 0; } + done <"$GF_FIXTURE_MASKED" + fi + if [[ -s "${GF_FIXTURE_UNITS:-}" ]]; then + while read -r name state; do + [[ -n "$name" ]] || continue + [[ "$name" == "$unit" ]] && { printf '%s\n' "$state"; exit 0; } + done <"$GF_FIXTURE_UNITS" + fi + printf 'not-found\n' + ;; +esac +exit 0 +STUB +chmod +x "$tmp/bin/systemctl" + +: >"$tmp/unit_files" +: >"$tmp/units" +: >"$tmp/masked" +export GF_FIXTURE_UNIT_FILES="$tmp/unit_files" +export GF_FIXTURE_UNITS="$tmp/units" +export GF_FIXTURE_MASKED="$tmp/masked" + +run_cutover() { + PATH="$tmp/bin:$PATH" bash "$script" \ + --units "$1" --db "$tmp/db" --bundle "$tmp/bundle" --link "$tmp/link" --dry-run 2>&1 +} + +fail=0 +check() { + if printf '%s\n' "$3" | grep -qF -- "$2"; then + printf 'ok %s\n' "$1" + else + printf 'FAIL %s: expected to find: %s\n' "$1" "$2" >&2 + printf '%s\n' "$3" >&2 + fail=1 + fi +} +check_absent() { + if printf '%s\n' "$3" | grep -qF -- "$2"; then + printf 'FAIL %s: expected NOT to find: %s\n' "$1" "$2" >&2 + printf '%s\n' "$3" >&2 + fail=1 + else + printf 'ok %s\n' "$1" + fi +} + +# --- Case A: NAS profile, template instances known to the manager ----------- +printf 'gitforge@api.service loaded\ngitforge@ci.service loaded\ngitforge@git-server.service loaded\ngitforge@runner.service loaded\n' >"$tmp/units" +out=$(run_cutover system) +check "template instance schedulers detected" \ + "drain gate: scheduler units gitforge@ci.service gitforge@runner.service" "$out" +check "template instance stateless detected" \ + "stateless units: gitforge@api.service gitforge@git-server.service" "$out" + +# --- Case B: one retired instance (masked) --------------------------------- +printf 'gitforge@api.service loaded\ngitforge@ci.service loaded\ngitforge@git-server.service loaded\n' >"$tmp/units" +printf 'gitforge@runner.service\n' >"$tmp/masked" +out=$(run_cutover system) +check "masked instance reported as retired" \ + "skipping gitforge@runner.service: masked (retired on this host)" "$out" +check "masked instance excluded from the scheduler set" \ + "drain gate: scheduler units gitforge@ci.service" "$out" +check_absent "masked instance not restarted" \ + "scheduler units gitforge@ci.service gitforge@runner.service" "$out" + +# --- Case C: fedora profile, concrete unit files never started ------------- +: >"$tmp/masked" +: >"$tmp/units" +printf 'gitforge-api.service enabled\ngitforge-ci.service enabled\ngitforge-git-server.service enabled\ngitforge-runner.service enabled\n' >"$tmp/unit_files" +out=$(run_cutover user) +check "concrete unit file schedulers detected" \ + "drain gate: scheduler units gitforge-ci gitforge-runner" "$out" +check "concrete unit file stateless detected" \ + "stateless units: gitforge-api gitforge-git-server" "$out" + +# --- Case D: nothing deployed on this host --------------------------------- +: >"$tmp/unit_files" +out=$(run_cutover system) +check "no units known means no scheduler restart" \ + "drain gate: scheduler units (none)" "$out" +check "no units known means no stateless restart" \ + "stateless units: (none)" "$out" + +if ((fail)); then + printf 'selftest: FAILED\n' >&2 + exit 1 +fi +printf 'selftest: OK\n' diff --git a/tests/test_release_bundle.py b/tests/test_release_bundle.py index 4a490734..402a109f 100644 --- a/tests/test_release_bundle.py +++ b/tests/test_release_bundle.py @@ -8,7 +8,9 @@ checksum manifest, unit examples, and resource policies; 2. the verifier accepts the valid bundle and its manifest; 3. tampering with a bundled binary fails verification; -4. malformed or incomplete bundles fail closed with useful diagnostics. +4. malformed or incomplete bundles fail closed with useful diagnostics; +5. the bundle's provenance claim is enforced: binaries that were not built + from the claimed commit are refused, by the bundler and by the verifier. The tests are deterministic: no network, no real build artifacts, and every test assembles its own bundle from a fresh temporary fixture. Run with either @@ -28,6 +30,7 @@ REPO_ROOT = Path(__file__).resolve().parents[1] BUNDLE_SCRIPT = REPO_ROOT / "scripts" / "gitforge-release-bundle" VERIFY_SCRIPT = REPO_ROOT / "scripts" / "gitforge-release-verify" +STAMPS_SCRIPT = REPO_ROOT / "scripts" / "gitforge-release-stamps" REQUIRED_BINARIES = ("api", "ci", "git-server", "runner") UNIT_EXAMPLES = ( @@ -43,7 +46,18 @@ # The bundler only requires a 40-hex source commit; this synthetic value keeps # the test independent of whatever the checkout HEAD happens to be. SOURCE_COMMIT = "4a1f2b3c4d5e6f708192a3b4c5d6e7f8091a2b3c" -TOOLCHAIN = "test-rustc 1.0.0 (canary)" +OTHER_COMMIT = "0f38a46345ac82efec9493310aa5c556c31c1ac5" +# The toolchain and the build time the BINARIES carry. A toolchain line +# carries spaces, exactly as `rustc --version` reports it. +BUILD_TOOLCHAIN = "rustc 1.99.1 (58cf8f9c3 2026-10-13)" +BUILD_TIME = "2026-11-06T08:15:00Z" + +# The toolchain of the host that runs the CUT. It is deliberately different +# from the one the binaries were built with: a bundle whose metadata names +# this toolchain describes the cutting host instead of the artifact it +# ships (GF-15, 2026-11-06). +CUTTING_HOST_TOOLCHAIN = "rustc 1.99.0 (b940084d7 2026-09-28)" + RELEASE_ID = "20260909-canary" MANIFEST_FILES = frozenset( @@ -68,9 +82,7 @@ def setUp(self): self.source_root.mkdir() self.bundle_root.mkdir() for name in REQUIRED_BINARIES: - self._write_executable( - self.source_root / name, f"#!/bin/sh\n# test stub: {name}\nexit 0\n" - ) + self._write_executable(self.source_root / name, self._stub(name)) # The bundler copies the status helper from the candidate source root, # not from this repository, so the fixture must provide one. self._write_executable( @@ -87,10 +99,45 @@ def _write_executable(path, content): path.write_text(content) path.chmod(0o755) + @staticmethod + def _stub( + name, + commit=SOURCE_COMMIT, + dirty="false", + stamped=True, + toolchain=BUILD_TOOLCHAIN, + built=BUILD_TIME, + ): + """A stand-in for a release binary. + + Real release binaries answer `--version` with the commit they were + built from, the toolchain that compiled them and the moment they + were compiled; the release chain refuses a bundle whose binaries do + not agree with the commit the bundle claims, and describes the + bundle from those same fields. The stub reproduces that contract so + the contract can be tested without a real build. + """ + if not stamped: + return f"#!/bin/sh\n# test stub: {name}\nexit 0\n" + return ( + f"#!/bin/sh\n" + f"# test stub: {name}\n" + f'case "$1" in\n' + f" --version|-V)\n" + f' echo "{name} 0.0.0-test commit={commit} dirty={dirty} ' + f"toolchain={toolchain} built={built}\"\n" + f" exit 0\n" + f" ;;\n" + f"esac\n" + f"exit 0\n" + ) + def _env(self): env = os.environ.copy() env["GITFORGE_SOURCE_COMMIT"] = SOURCE_COMMIT - env["GITFORGE_TOOLCHAIN"] = TOOLCHAIN + # The host that runs the cut states its own toolchain. The bundler + # must not use it: the bundle describes the binaries it ships. + env["GITFORGE_TOOLCHAIN"] = CUTTING_HOST_TOOLCHAIN # The bundler consults the live-instance release gate, which no # test fixture can satisfy; the gate itself is covered by # tests/test_release_gate.py against a purpose-built database. @@ -166,9 +213,11 @@ def test_valid_bundle_contains_required_payload(self): metadata = self._read_kv(self.release_dir / "RELEASE_METADATA.txt") self.assertEqual(metadata.get("release_id"), RELEASE_ID) self.assertEqual(metadata.get("source_commit"), SOURCE_COMMIT) - self.assertEqual(metadata.get("toolchain"), TOOLCHAIN) + # The metadata describes the binaries, not the cutting host. + self.assertEqual(metadata.get("toolchain"), BUILD_TOOLCHAIN) + self.assertEqual(metadata.get("build_timestamp_utc"), BUILD_TIME) self.assertRegex( - metadata.get("build_timestamp_utc", ""), + metadata.get("bundle_cut_utc", ""), r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$", ) @@ -314,6 +363,205 @@ def test_verify_rejects_missing_directory(self): self.assertEqual(result.returncode, 1) self.assertIn("release verify: directory does not exist:", result.stderr) + # ─── 5. the bundle's provenance claim is enforced ──────────────────────── + # + # GF-14 (2026-11-05): the bundler copied whatever binaries sat in the + # candidate source root and stamped them with the commit the root was + # on. The release binaries live in gitignored paths, so a bundle of + # stale leftovers passed every other gate while claiming a commit it was + # never built from. These cases pin the fix: the claim now has to be + # carried by the binaries, and the tooling refuses when it is not. + + def _refused_cut(self): + """Run the bundler against a release id that was never used. + + The fixture already published one valid bundle, so a refusal has to + be judged on its own release id: the bundler refuses to overwrite an + existing release, and that refusal would otherwise mask the + provenance diagnostic. + """ + release_id = RELEASE_ID + "-refused" + result = self._run( + BUNDLE_SCRIPT, str(self.source_root), str(self.bundle_root), release_id + ) + self.assertNotEqual(result.returncode, 0, "the bundler accepted the bundle") + self.assertFalse( + (self.bundle_root / release_id).exists(), + "a refused cut published a bundle anyway", + ) + self.assertEqual( + sorted(p.name for p in self.bundle_root.iterdir()), + [RELEASE_ID], + "a refused cut left a staging tree behind", + ) + return result + + def _rewrite_manifest(self): + """Recompute the checksum manifest. + + Used by the cases below so that the checksum gate cannot be the + reason a verification fails: they target the provenance gate. + """ + entries = ["READY", "RELEASE_METADATA.txt"] + entries += [f"bin/{name}" for name in REQUIRED_BINARIES] + entries += ["scripts/gitforge-status"] + entries += [f"systemd/user/{name}" for name in UNIT_EXAMPLES] + entries += [f"systemd/user/{name}" for name in RESOURCE_POLICIES] + lines = [] + for rel_path in entries: + digest = hashlib.sha256((self.release_dir / rel_path).read_bytes()).hexdigest() + lines.append(f"{digest} {rel_path}\n") + lines.sort() + (self.release_dir / "MANIFEST.sha256").write_text("".join(lines)) + + def test_bundler_refuses_binaries_built_from_another_commit(self): + self._write_executable( + self.source_root / "ci", self._stub("ci", commit=OTHER_COMMIT) + ) + result = self._refused_cut() + self.assertIn("was built from", result.stderr) + self.assertIn(OTHER_COMMIT, result.stderr) + self.assertIn(SOURCE_COMMIT, result.stderr) + + def test_bundler_refuses_binaries_without_a_build_stamp(self): + self._write_executable( + self.source_root / "runner", self._stub("runner", stamped=False) + ) + result = self._refused_cut() + self.assertIn("does not report a build commit", result.stderr) + + def test_bundler_refuses_binaries_built_from_a_modified_tree(self): + self._write_executable(self.source_root / "api", self._stub("api", dirty="true")) + result = self._refused_cut() + self.assertIn("was built from a tree that was not clean", result.stderr) + + def test_verifier_refuses_a_bundle_whose_binary_disagrees_with_metadata(self): + # A bundle cut by the pre-fix bundler: the manifest is internally + # consistent, the metadata claims SOURCE_COMMIT, and the binary says + # otherwise. Only the provenance gate can catch this. + self._write_executable( + self.release_dir / "bin" / "ci", self._stub("ci", commit=OTHER_COMMIT) + ) + self._rewrite_manifest() + + result = self.verify() + self.assertNotEqual( + result.returncode, 0, "verifier accepted a bundle whose binary lies about its commit" + ) + self.assertIn("was built from", result.stderr) + self.assertIn(OTHER_COMMIT, result.stderr) + self.assertNotIn("valid bundle=", result.stdout) + + def test_verifier_refuses_a_bundle_built_from_a_modified_tree(self): + self._write_executable( + self.release_dir / "bin" / "api", self._stub("api", dirty="true") + ) + self._rewrite_manifest() + + result = self.verify() + self.assertNotEqual( + result.returncode, 0, "verifier accepted a bundle from a modified tree" + ) + self.assertIn("was built from a tree that was not clean", result.stderr) + self.assertNotIn("valid bundle=", result.stdout) + + def test_stamps_check_accepts_matching_binaries_and_rejects_others(self): + accepted = self._run(STAMPS_SCRIPT, str(self.source_root), SOURCE_COMMIT) + self.assertEqual( + accepted.returncode, 0, f"stamps refused matching binaries:\n{accepted.stderr}" + ) + self.assertIn(SOURCE_COMMIT, accepted.stdout) + + rejected = self._run(STAMPS_SCRIPT, str(self.source_root), OTHER_COMMIT) + self.assertEqual(rejected.returncode, 1) + self.assertIn("was built from", rejected.stderr) + + + # ─── 6. the metadata describes the artifact it ships (GF-15) ──────────── + # + # GF-15 (2026-11-06): the bundle's metadata named the CUTTING host - its + # `rustc --version` and the time of the cut - so a bundle of binaries + # compiled on one host claimed the toolchain of the host that assembled + # it. The metadata has to be read out of the binaries instead. + + def test_metadata_describes_the_binaries_not_the_cutting_host(self): + metadata = self._read_kv(self.release_dir / "RELEASE_METADATA.txt") + self.assertEqual(metadata.get("toolchain"), BUILD_TOOLCHAIN) + self.assertNotEqual(metadata.get("toolchain"), CUTTING_HOST_TOOLCHAIN) + self.assertEqual(metadata.get("build_timestamp_utc"), BUILD_TIME) + # The cut itself is still recorded - as its own field. + self.assertIn("bundle_cut_utc", metadata) + + def test_verifier_refuses_metadata_that_names_another_toolchain(self): + metadata_path = self.release_dir / "RELEASE_METADATA.txt" + lines = metadata_path.read_text().splitlines() + metadata_path.write_text( + "\n".join( + f"toolchain={CUTTING_HOST_TOOLCHAIN}" + if line.startswith("toolchain=") + else line + for line in lines + ) + + "\n" + ) + self._rewrite_manifest() + + result = self.verify() + self.assertNotEqual( + result.returncode, 0, "verifier accepted metadata that describes another build" + ) + self.assertIn("does not describe the shipped binaries", result.stderr) + self.assertNotIn("valid bundle=", result.stdout) + + def test_verifier_refuses_a_metadata_build_time_the_binaries_do_not_carry(self): + metadata_path = self.release_dir / "RELEASE_METADATA.txt" + lines = metadata_path.read_text().splitlines() + metadata_path.write_text( + "\n".join( + "build_timestamp_utc=2020-01-01T00:00:00Z" + if line.startswith("build_timestamp_utc=") + else line + for line in lines + ) + + "\n" + ) + self._rewrite_manifest() + + result = self.verify() + self.assertNotEqual( + result.returncode, 0, "verifier accepted a metadata build time the binaries do not carry" + ) + self.assertIn("does not describe the shipped binaries", result.stderr) + self.assertNotIn("valid bundle=", result.stdout) + + def test_bundler_refuses_binaries_that_disagree_about_the_toolchain(self): + self._write_executable( + self.source_root / "runner", self._stub("runner", toolchain="rustc 0.0.1 (other)") + ) + result = self._refused_cut() + self.assertIn("disagree about toolchain", result.stderr) + + def test_bundler_refuses_binaries_that_cannot_say_when_they_were_built(self): + self._write_executable(self.source_root / "api", self._stub("api", built="unknown")) + result = self._refused_cut() + self.assertIn("does not report when it was built", result.stderr) + + def test_stamps_field_mode_reports_the_value_the_binaries_agree_on(self): + for field, expected in (("toolchain", BUILD_TOOLCHAIN), ("built", BUILD_TIME)): + result = self._run(STAMPS_SCRIPT, "--field", field, str(self.source_root)) + self.assertEqual( + result.returncode, 0, f"field mode refused:\n{result.stdout}\n{result.stderr}" + ) + self.assertEqual(result.stdout.strip(), expected) + + def test_stamps_field_mode_refuses_binaries_that_disagree(self): + self._write_executable( + self.source_root / "ci", self._stub("ci", toolchain="rustc 0.0.1 (other)") + ) + result = self._run(STAMPS_SCRIPT, "--field", "toolchain", str(self.source_root)) + self.assertNotEqual(result.returncode, 0, "field mode accepted disagreeing binaries") + self.assertIn("disagree about toolchain", result.stderr) + if __name__ == "__main__": unittest.main()