diff --git a/controller/main.go b/controller/main.go index 8919eb18..c369fb3f 100644 --- a/controller/main.go +++ b/controller/main.go @@ -76,7 +76,9 @@ func run(ctx context.Context) error { scaler := &Scaler{runners: newRunnerState(), dockerClient: docker, scalesetClient: client, logger: logger, config: cfg, scaleSetID: set.ID} if err := scaler.recoverStale(ctx); err != nil { return err } scaler.writeStatus() - go scaler.publishStatus(ctx, time.Minute) + statusTicker := time.NewTicker(time.Minute) + defer statusTicker.Stop() + go scaler.publishStatus(ctx, statusTicker.C) defer scaler.shutdown(context.WithoutCancel(ctx)) hostname, err := os.Hostname() if err != nil { return fmt.Errorf("get hostname: %w", err) } diff --git a/controller/status.go b/controller/status.go index d63e7f16..bbc268b6 100644 --- a/controller/status.go +++ b/controller/status.go @@ -53,14 +53,12 @@ func (s *Scaler) writeStatus() { if err != nil { s.logger.Warn("write controller status", "error", err) } } -func (s *Scaler) publishStatus(ctx context.Context, interval time.Duration) { - ticker := time.NewTicker(interval) - defer ticker.Stop() +func (s *Scaler) publishStatus(ctx context.Context, ticks <-chan time.Time) { for { select { case <-ctx.Done(): return - case <-ticker.C: + case <-ticks: s.writeStatus() } } diff --git a/controller/status_test.go b/controller/status_test.go index 8ed3bd59..2f304658 100644 --- a/controller/status_test.go +++ b/controller/status_test.go @@ -113,26 +113,19 @@ func TestStatusPublisherRefreshesIdleSnapshot(t *testing.T) { } ctx, cancel := context.WithCancel(context.Background()) defer cancel() - done := make(chan struct{}) - go func() { - scaler.publishStatus(ctx, time.Millisecond) - close(done) - }() - deadline := time.After(time.Second) - for { - if _, err := os.Stat(path); err == nil { - cancel() - select { - case <-done: - case <-time.After(time.Second): - t.Fatal("idle status publisher did not stop") - } - return - } - select { - case <-deadline: - t.Fatal("idle status publisher did not refresh snapshot") - case <-time.After(time.Millisecond): - } + original := encodeControllerStatus + encodeControllerStatus = func(file *os.File, value controllerStatus) error { + err := original(file, value) + cancel() + return err } + defer func() { encodeControllerStatus = original }() + ticks := make(chan time.Time, 1) + ticks <- time.Now() + scaler.publishStatus(ctx, ticks) + body, err := os.ReadFile(path) + if err != nil { t.Fatal("idle status publisher did not refresh snapshot") } + var got controllerStatus + if err := json.Unmarshal(body, &got); err != nil { t.Fatal(err) } + if got.Current != 0 || got.Busy != 0 { t.Fatalf("published non-idle status: %+v", got) } } diff --git a/docs/ADDING-A-HOST.md b/docs/ADDING-A-HOST.md index 03325b5c..f1ff9ab4 100644 --- a/docs/ADDING-A-HOST.md +++ b/docs/ADDING-A-HOST.md @@ -102,7 +102,7 @@ sudo ./scripts/install-worker-controller.sh \ --controller example-ci-01 ``` -Use `--adopt` when converting an existing manual controller. The installer validates the complete configuration, renders host-local runtime state, installs the pinned engine, creates a controller checkpoint, builds images, installs maintenance timers, and verifies health. +Use `--adopt` when converting an existing manual controller. The installer validates the complete configuration, renders host-local runtime state, installs the pinned engine, and validates the candidate Compose configuration. It may build images with a distinct inert runner tag before creating a checkpoint and draining. A build that would retag the runner image used by the active controller waits until after checkpoint and drain. Managed preflight still runs after drain, before activation. A failed pre-transaction build may leave candidate image or layer artifacts, but it does not change installed state or stop the active controller. The configuration repository credential, when required, must be read-only and host-side. Credentials are never accepted in command arguments. diff --git a/docs/DESIRED-STATE.md b/docs/DESIRED-STATE.md index 5d262b7f..422f4ae4 100644 --- a/docs/DESIRED-STATE.md +++ b/docs/DESIRED-STATE.md @@ -136,12 +136,13 @@ The installer: 3. selects exactly one logical controller; 4. renders `/etc/ci-fleet/ci-fleet.env` without secret values; 5. fetches and verifies the pinned public engine commit; -6. creates a root-only controller checkpoint; -7. drains the current controller and waits for every managed runner to finish, including orphaned runners left after a stopped or crashed controller; -8. runs managed preflight and builds the pinned runner and controller images; -9. installs health, cleanup, and pinned-state drift unit definitions; -10. starts the controller only when its desired state is active and verifies runtime health; -11. atomically records redacted installation state, then enables the maintenance timers. +6. validates the candidate Compose configuration and builds images before the transaction when no active installed controller uses the candidate runner tag; +7. creates a root-only controller checkpoint; +8. drains the current controller and waits for every managed runner to finish, including orphaned runners left after a stopped or crashed controller; +9. runs managed preflight and, when the candidate would retag the active controller's runner image, builds the images after drain; +10. installs health, cleanup, and pinned-state drift unit definitions; +11. starts the controller only when its desired state is active and verifies runtime health; +12. atomically records redacted installation state, then enables the maintenance timers. A successful second `--install` run reports `NO_CHANGE` and performs no unnecessary replacement. A successful engine upgrade advances both the runtime release and the maintenance installer-manager to the same pinned commit; rollback restores both. @@ -240,7 +241,7 @@ Legacy project-specific hosts remain until CI, promotion, and deployment no long ## Failure and recovery behavior -Before mutation, the installer records the prior rendered environment, installation metadata, runtime release, installer-manager release, and maintenance unit/timer state under `/var/lib/ci-fleet/checkpoints`. Each checkpoint is staged and atomically renamed with a completion marker; rollback ignores partial staging directories. Build and validation happen before the active release changes. A failed activation or health check drains the candidate, restores those artifacts, restarts the prior controller only when no managed runner is active, and verifies prior-release health before reporting rollback success. A host-local installer lock serializes every check and mutation. Runtime and installer-manager releases are staged on their respective target filesystems and renamed atomically so a failed copy cannot masquerade as an installed immutable release. +Before mutation, the installer records the prior rendered environment, installation metadata, runtime release, installer-manager release, and maintenance unit/timer state under `/var/lib/ci-fleet/checkpoints`. Each checkpoint is staged and atomically renamed with a completion marker; rollback ignores partial staging directories. Compose validation happens before the checkpoint. A build using a distinct inert runner tag may also happen before the checkpoint; failure may leave candidate image or layer artifacts, but installed state and the active controller remain unchanged. A build that would retag the active controller's runner image happens only after checkpoint and drain. A failed activation or health check drains the candidate, restores those artifacts, restarts the prior controller only when no managed runner is active, and verifies prior-release health before reporting rollback success. A host-local installer lock serializes every check and mutation. Runtime and installer-manager releases are staged on their respective target filesystems and renamed atomically so a failed copy cannot masquerade as an installed immutable release. Installer checkpoints and machine backups serve different failure classes. A checkpoint rolls back a single failed reconciliation. Recoverability of the machine itself is governed by each host's declared failure boundary (see [Adding a host](ADDING-A-HOST.md)): a disposable controller needs no machine backup at all — recovery is rebuilding from reviewed Git-authored desired state — while a non-disposable host follows its own documented local backup policy. diff --git a/host/systemd/ci-fleet-reconcile.service b/host/systemd/ci-fleet-reconcile.service index d8f7d875..639e2cf9 100644 --- a/host/systemd/ci-fleet-reconcile.service +++ b/host/systemd/ci-fleet-reconcile.service @@ -9,6 +9,10 @@ Type=oneshot User=root WorkingDirectory=/opt/ci-fleet/manager/current ExecStart=/opt/ci-fleet/manager/current/scripts/remote-reconcile.sh +# One hour for the installer lock, plus one hour for a cold build and transaction. +TimeoutStartSec=2h +# Allow checkpoint restoration after TimeoutStartSec sends SIGTERM. +TimeoutStopSec=15min Restart=no # 0=noop, 3=drift/invalid — timer retries SuccessExitStatus=0 3 diff --git a/scripts/install-worker-controller.sh b/scripts/install-worker-controller.sh index 639f36d3..4e3104cb 100755 --- a/scripts/install-worker-controller.sh +++ b/scripts/install-worker-controller.sh @@ -36,10 +36,13 @@ EOF note() { printf '%s\n' "$*"; } die() { printf 'ERROR: %s\n' "$*" >&2 + trap - ERR + trap '' TERM if [[ ${transaction_active:-false} == true ]] && declare -F restore_checkpoint >/dev/null; then restore_checkpoint || true transaction_active=false fi + trap - ERR exit 2 } @@ -693,13 +696,46 @@ run_candidate_preflight() { } build_candidate() { - run_candidate_preflight - compose "$release_dir" "$candidate_env" config --quiet compose "$release_dir" "$candidate_env" build runner-image controller } +load_checkpoint_images() { + local environment=$1 image_ids=${2:-} output + output=$(python3 - "$environment" "$image_ids" "$repo_root/scripts" <<'PY' +import re +import sys +from pathlib import Path + +sys.path.insert(0, sys.argv[3]) +from desired_state import parse_env + +image_keys = ("CI_FLEET_RUNNER_IMAGE", "CI_FLEET_CONTROLLER_IMAGE") +values = parse_env(Path(sys.argv[1]), allow_unknown=True) +if any(not values.get(key) for key in image_keys): + raise SystemExit(1) +for key in image_keys: + print(values[key]) + +if sys.argv[2]: + id_keys = tuple(f"{key}_ID" for key in image_keys) + values = parse_env(Path(sys.argv[2]), allow_unknown=True) + live_key = "CI_FLEET_CONTROLLER_LIVE_IMAGE_ID" + if not set(id_keys).issubset(values) or not set(values) <= {*id_keys, live_key} or any(values[key] != "absent" and not re.fullmatch(r"sha256:[0-9a-f]{64}", values[key]) for key in id_keys): + raise SystemExit(1) + if live_key in values and (not re.fullmatch(r"sha256:[0-9a-f]{64}", values[live_key]) or values[live_key] == values[id_keys[1]]): + raise SystemExit(1) + for key in id_keys: + print(values[key]) + if live_key in values: + print(values[live_key]) +PY + ) || return 1 + mapfile -t checkpoint_images <<<"$output" + [[ ${#checkpoint_images[@]} == 2 || -n "$image_ids" && ( ${#checkpoint_images[@]} == 4 || ${#checkpoint_images[@]} == 5 ) ]] +} + make_checkpoint() { - local timestamp target unit timer final_checkpoint staged_checkpoint + local timestamp target unit timer final_checkpoint staged_checkpoint expected_owner=0 runner_id controller_id controller_live_id= timestamp=$(date -u +%Y%m%dT%H%M%SZ) final_checkpoint=$checkpoints_dir/${timestamp}-$$ install -d -m 0700 "$checkpoints_dir" @@ -707,7 +743,34 @@ make_checkpoint() { staging_paths+=("$staged_checkpoint") checkpoint_dir=$staged_checkpoint install -d -m 0700 "$checkpoint_dir/systemd" - [[ ! -f "$rendered_env" ]] || install -m 0600 "$rendered_env" "$checkpoint_dir/ci-fleet.env" + printf '2\n' >"$checkpoint_dir/format-version" + chmod 0600 "$checkpoint_dir/format-version" + if [[ -f "$rendered_env" ]]; then + [[ "$testing" != 1 ]] || expected_owner=$(id -u) + [[ $(stat -c %u "$rendered_env") == "$expected_owner" && $(stat -c %a "$rendered_env") == 600 ]] || die "rendered environment must be owned by root with mode 0600: $rendered_env" + install -m 0600 "$rendered_env" "$checkpoint_dir/ci-fleet.env" + load_checkpoint_images "$checkpoint_dir/ci-fleet.env" || die 'installed image tags are invalid' + if ! runner_id=$(docker image inspect --format '{{.Id}}' "${checkpoint_images[0]}" 2>/dev/null); then + docker info >/dev/null 2>&1 || die 'Docker daemon is unavailable' + runner_id=absent + fi + if ! controller_id=$(docker image inspect --format '{{.Id}}' "${checkpoint_images[1]}" 2>/dev/null); then + docker info >/dev/null 2>&1 || die 'Docker daemon is unavailable' + controller_id=absent + fi + if controller_live_id=$(docker inspect --format '{{.Image}}' "$controller_container" 2>/dev/null); then + [[ "$controller_live_id" != "$controller_id" ]] || controller_live_id= + else + docker info >/dev/null 2>&1 || die 'Docker daemon is unavailable' + controller_live_id= + fi + [[ "$runner_id" == absent || "$runner_id" =~ ^sha256:[0-9a-f]{64}$ ]] || die 'installed runner image ID is invalid' + [[ "$controller_id" == absent || "$controller_id" =~ ^sha256:[0-9a-f]{64}$ ]] || die 'installed controller image ID is invalid' + [[ -z "$controller_live_id" || "$controller_live_id" =~ ^sha256:[0-9a-f]{64}$ ]] || die 'live controller image ID is invalid' + printf 'CI_FLEET_RUNNER_IMAGE_ID=%s\nCI_FLEET_CONTROLLER_IMAGE_ID=%s\n' "$runner_id" "$controller_id" >"$checkpoint_dir/image-ids.env" + [[ -z "$controller_live_id" ]] || printf 'CI_FLEET_CONTROLLER_LIVE_IMAGE_ID=%s\n' "$controller_live_id" >>"$checkpoint_dir/image-ids.env" + chmod 0600 "$checkpoint_dir/image-ids.env" + fi [[ ! -f "$state_file" ]] || install -m 0600 "$state_file" "$checkpoint_dir/install-state.json" target=$(current_runtime_release) if [[ -n "$target" ]]; then @@ -744,12 +807,20 @@ make_checkpoint() { } try_drain_current() { - local deadline count old_release status paused=false force_nonterminal=${1:-false} + local deadline count old_release='' status paused=false force_nonterminal=${1:-false} local drain_env=${2:-$rendered_env} fallback_release=${3:-} shutdown_timeout=${CI_FLEET_DRAIN_TIMEOUT_SECONDS:-300} drain_error= status=$(controller_status) case "$status" in - running|''|exited|created|dead) ;; + running|'') ;; + exited|created|dead) + [[ -f "$drain_env" ]] || { drain_error="cannot stop restartable controller state without its rendered environment: $status"; return 1; } + old_release=$(current_runtime_release) + [[ -n "$old_release" ]] || old_release=$fallback_release + [[ -n "$old_release" ]] || { drain_error="cannot stop restartable controller state without its runtime release: $status"; return 1; } + compose "$old_release" "$drain_env" stop --timeout "$shutdown_timeout" controller >/dev/null 2>&1 || { drain_error="failed to stop restartable controller state: $status"; return 1; } + status= + ;; *) if [[ "$force_nonterminal" != true ]]; then drain_error="cannot safely drain controller in non-terminal state: $status" @@ -759,6 +830,9 @@ try_drain_current() { old_release=$(current_runtime_release) [[ -n "$old_release" ]] || old_release=$fallback_release [[ -n "$old_release" ]] || { drain_error='cannot stop a non-terminal candidate without its runtime release'; return 1; } + if [[ "$status" == paused ]]; then + compose "$old_release" "$drain_env" unpause controller >/dev/null 2>&1 || { drain_error="failed to unpause non-terminal candidate state: $status"; return 1; } + fi compose "$old_release" "$drain_env" stop --timeout "$shutdown_timeout" controller >/dev/null 2>&1 || { drain_error="failed to stop non-terminal candidate state: $status"; return 1; } status= ;; @@ -768,8 +842,14 @@ try_drain_current() { old_release=$(current_runtime_release) [[ -n "$old_release" ]] || old_release=$fallback_release if [[ -z "$old_release" || ! -f "$old_release/deploy/compose.yaml" ]]; then drain_error='cannot locate the running controller Compose release for safe adoption'; return 1; fi - if ! compose "$old_release" "$drain_env" pause controller >/dev/null; then drain_error='could not pause the controller for drain'; return 1; fi - paused=true + if [[ $(docker inspect --format '{{.State.Paused}}' "$controller_container" 2>/dev/null || true) == true ]]; then + paused=true + elif compose "$old_release" "$drain_env" pause controller >/dev/null; then + paused=true + else + drain_error='could not pause the controller for drain' + return 1 + fi fi deadline=$((SECONDS + ${CI_FLEET_DRAIN_TIMEOUT_SECONDS:-300})) while :; do @@ -783,30 +863,34 @@ try_drain_current() { sleep 2 done note 'DRAIN_READY managed_runners=0' - if [[ "$status" != running ]]; then - note 'DRAIN_OK managed_runners=0' - return 0 + if [[ "$status" == running ]]; then + compose "$old_release" "$drain_env" kill --signal SIGTERM controller >/dev/null || { + compose "$old_release" "$drain_env" unpause controller >/dev/null 2>&1 || true + drain_error='failed to signal the paused controller for graceful scale-set cleanup' + return 1 + } + if [[ $(docker inspect --format '{{.State.Paused}}' "$controller_container" 2>/dev/null || true) == true ]]; then + compose "$old_release" "$drain_env" unpause controller >/dev/null || { + drain_error='failed to unpause the signaled controller for graceful shutdown' + return 1 + } + fi + compose "$old_release" "$drain_env" stop --timeout "$shutdown_timeout" controller >/dev/null || { + drain_error='could not stop the drained controller' + return 1 + } fi - compose "$old_release" "$drain_env" kill --signal SIGTERM controller >/dev/null || { - compose "$old_release" "$drain_env" unpause controller >/dev/null 2>&1 || true - drain_error='failed to signal the paused controller for graceful scale-set cleanup' - return 1 - } - if [[ $(docker inspect --format '{{.State.Paused}}' "$controller_container" 2>/dev/null || true) == true ]]; then - compose "$old_release" "$drain_env" unpause controller >/dev/null || { - drain_error='failed to unpause the signaled controller for graceful shutdown' + if [[ "$force_nonterminal" == true && -n "$old_release" ]]; then + compose "$old_release" "$drain_env" rm -f controller >/dev/null || { + drain_error='could not remove the stopped candidate controller' return 1 } fi - compose "$old_release" "$drain_env" stop --timeout "$shutdown_timeout" controller >/dev/null || { - drain_error='could not stop the drained controller' - return 1 - } note 'DRAIN_OK managed_runners=0' } drain_current() { - try_drain_current || die "$drain_error" + try_drain_current "$@" || die "$drain_error" } install_systemd_units() { @@ -951,9 +1035,36 @@ restore_systemd_snapshot() { } restore_checkpoint() { - local target restored_state failed=0 checkpoint_release='' drain_env=$rendered_env drain_release='' + local target restored_state actual index expected_owner=0 failed=0 checkpoint_release='' drain_env=$rendered_env drain_release='' restore_images=false new_format=false + local restore_controller_tag_after_start=false + local format_marker=$checkpoint_dir/format-version image_ids=$checkpoint_dir/image-ids.env [[ -n "$checkpoint_dir" && -d "$checkpoint_dir" ]] || return 1 + [[ "$testing" != 1 ]] || expected_owner=$(id -u) + if [[ -e "$format_marker" || -L "$format_marker" ]]; then + if [[ -L "$format_marker" || ! -f "$format_marker" || $(stat -c %u "$format_marker") != "$expected_owner" \ + || $(stat -c %a "$format_marker") != 600 || $(stat -c %s "$format_marker") != 2 || $(<"$format_marker") != 2 ]]; then + note 'ROLLBACK_FAILED reason=checkpoint format marker is invalid' + return 1 + fi + new_format=true + elif [[ -e "$image_ids" || -L "$image_ids" ]]; then + note 'ROLLBACK_FAILED reason=checkpoint image mappings are invalid' + return 1 + elif [[ -f "$checkpoint_dir/ci-fleet.env" ]]; then + note 'ROLLBACK_LEGACY_IMAGE_STATE_UNVERIFIED' + fi [[ ! -f "$checkpoint_dir/release-target" ]] || checkpoint_release=$(<"$checkpoint_dir/release-target") + if $new_format && [[ -f "$checkpoint_dir/ci-fleet.env" ]]; then + if [[ ! -f "$image_ids" || -L "$image_ids" || $(stat -c %u "$image_ids") != "$expected_owner" || $(stat -c %a "$image_ids") != 600 ]] \ + || ! load_checkpoint_images "$checkpoint_dir/ci-fleet.env" "$image_ids"; then + note 'ROLLBACK_FAILED reason=checkpoint image mappings are invalid' + return 1 + fi + restore_images=true + elif $new_format && [[ -e "$image_ids" || -L "$image_ids" ]]; then + note 'ROLLBACK_FAILED reason=checkpoint image mappings are invalid' + return 1 + fi drain_release=$(current_runtime_release) if [[ -f "$rendered_env" ]]; then load_installed_controller_identity "$temporary/no-install-state" "$rendered_env" @@ -966,6 +1077,10 @@ restore_checkpoint() { note "ROLLBACK_FAILED reason=$drain_error" return 1 fi + if ! remove_inactive_managed_runners; then + note 'ROLLBACK_FAILED reason=could not remove inactive managed runners' + return 1 + fi if [[ -f "$checkpoint_dir/install-state.json" || -f "$checkpoint_dir/ci-fleet.env" ]]; then load_installed_controller_identity "$checkpoint_dir/install-state.json" "$checkpoint_dir/ci-fleet.env" fi @@ -1003,6 +1118,31 @@ restore_checkpoint() { else rm -f "$manager_current" || failed=1 fi + if $restore_images; then + for index in 0 1; do + if [[ "$index" == 1 && ${#checkpoint_images[@]} == 5 ]]; then + docker image tag "${checkpoint_images[4]}" "${checkpoint_images[index]}" || failed=1 + actual=$(docker image inspect --format '{{.Id}}' "${checkpoint_images[index]}" 2>/dev/null) || failed=1 + [[ "$actual" == "${checkpoint_images[4]}" ]] || failed=1 + restore_controller_tag_after_start=true + elif [[ ${checkpoint_images[index + 2]} == absent ]]; then + if docker image inspect --format '{{.Id}}' "${checkpoint_images[index]}" >/dev/null 2>&1; then + docker image rm "${checkpoint_images[index]}" >/dev/null || failed=1 + else + docker info >/dev/null 2>&1 || failed=1 + fi + if docker image inspect --format '{{.Id}}' "${checkpoint_images[index]}" >/dev/null 2>&1; then + failed=1 + else + docker info >/dev/null 2>&1 || failed=1 + fi + else + docker image tag "${checkpoint_images[index + 2]}" "${checkpoint_images[index]}" || failed=1 + actual=$(docker image inspect --format '{{.Id}}' "${checkpoint_images[index]}" 2>/dev/null) || failed=1 + [[ "$actual" == "${checkpoint_images[index + 2]}" ]] || failed=1 + fi + done + fi restore_systemd_snapshot || failed=1 if [[ -n "$release_dir" && -f "$rendered_env" ]]; then restored_state=$(awk -F= '$1 == "CI_FLEET_CONTROLLER_STATE" {print $2}' "$rendered_env") || failed=1 @@ -1016,6 +1156,20 @@ restore_checkpoint() { fi fi fi + if $restore_controller_tag_after_start && ((failed == 0)); then + if [[ ${checkpoint_images[3]} == absent ]]; then + docker image rm "${checkpoint_images[1]}" >/dev/null || failed=1 + if docker image inspect --format '{{.Id}}' "${checkpoint_images[1]}" >/dev/null 2>&1; then + failed=1 + else + docker info >/dev/null 2>&1 || failed=1 + fi + else + docker image tag "${checkpoint_images[3]}" "${checkpoint_images[1]}" || failed=1 + actual=$(docker image inspect --format '{{.Id}}' "${checkpoint_images[1]}" 2>/dev/null) || failed=1 + [[ "$actual" == "${checkpoint_images[3]}" ]] || failed=1 + fi + fi set -e trap on_error ERR if ((failed != 0)); then @@ -1025,14 +1179,24 @@ restore_checkpoint() { note "ROLLBACK_RESTORED checkpoint=$checkpoint_dir" } -on_error() { - local status=$? +rollback_and_exit() { + local status=$1 + trap - ERR + trap '' TERM if $transaction_active; then restore_checkpoint || true + transaction_active=false fi + trap - ERR exit "$status" } +on_error() { + local status=$? + rollback_and_exit "$status" +} +on_term() { rollback_and_exit 143; } trap on_error ERR +trap on_term TERM perform_check() { local count @@ -1047,7 +1211,8 @@ perform_check() { } perform_converge() { - local count existing_status desired_controller_id=$controller_id + local count existing_status candidate_runner_image candidate_controller_image installed_runner_image installed_controller_image live_runner_image expected_owner=0 + local desired_controller_id=$controller_id build_before_drain=false if [[ "$mode" == upgrade && ! -f "$state_file" ]]; then die '--upgrade requires an existing managed installation; use --install or --adopt' fi @@ -1065,6 +1230,26 @@ perform_converge() { return fi install_release + compose "$release_dir" "$candidate_env" config --quiet + candidate_runner_image=$(awk -F= '$1 == "CI_FLEET_RUNNER_IMAGE" {count++; value=substr($0, index($0, "=") + 1)} END {if (count != 1) exit 1; print value}' "$candidate_env") || die 'rendered candidate runner image is invalid' + candidate_controller_image=$(awk -F= '$1 == "CI_FLEET_CONTROLLER_IMAGE" {count++; value=substr($0, index($0, "=") + 1)} END {if (count != 1) exit 1; print value}' "$candidate_env") || die 'rendered candidate controller image is invalid' + [[ "$testing" != 1 ]] || expected_owner=$(id -u) + case "$existing_status" in + '') + [[ "$mode" == install && ! -f "$rendered_env" && ! -f "$state_file" ]] && build_before_drain=true + ;; + running|exited|created|dead) + if [[ -f "$rendered_env" && $(stat -c %u "$rendered_env") == "$expected_owner" && $(stat -c %a "$rendered_env") == 600 ]] \ + && installed_runner_image=$(awk -F= '$1 == "CI_FLEET_RUNNER_IMAGE" {count++; value=substr($0, index($0, "=") + 1)} END {if (count != 1) exit 1; print value}' "$rendered_env") \ + && installed_controller_image=$(awk -F= '$1 == "CI_FLEET_CONTROLLER_IMAGE" {count++; value=substr($0, index($0, "=") + 1)} END {if (count != 1) exit 1; print value}' "$rendered_env") \ + && live_runner_image=$(docker inspect --format '{{range .Config.Env}}{{println .}}{{end}}' "$controller_container" 2>/dev/null | awk -F= '$1 == "CI_FLEET_RUNNER_IMAGE" {count++; value=substr($0, index($0, "=") + 1)} END {if (count != 1) exit 1; print value}') \ + && [[ -n "$installed_runner_image" && -n "$installed_controller_image" && -n "$live_runner_image" ]]; then + if [[ "$candidate_runner_image" != "$installed_runner_image" && "$candidate_runner_image" != "$live_runner_image" \ + && "$candidate_controller_image" != "$installed_controller_image" ]]; then build_before_drain=true; fi + fi + ;; + esac + if $build_before_drain; then build_candidate; require_commands; fi make_checkpoint transaction_active=true if [[ -f "$state_file" || -f "$rendered_env" ]]; then @@ -1072,10 +1257,15 @@ perform_converge() { elif [[ "$mode" == adopt ]]; then die '--adopt requires a trusted installed controller identity' fi - drain_current + drain_current false "$rendered_env" "$release_dir" + if [[ "$testing" == 1 && -n ${CI_FLEET_TEST_PAUSE_AFTER_DRAIN_FILE:-} ]]; then + : >"$CI_FLEET_TEST_PAUSE_AFTER_DRAIN_FILE" + while [[ ! -f "$CI_FLEET_TEST_PAUSE_AFTER_DRAIN_FILE.continue" ]]; do sleep 0.05; done + fi controller_id=$desired_controller_id + run_candidate_preflight + if ! $build_before_drain; then build_candidate; fi [[ "$target_state" == active ]] || remove_inactive_managed_runners - build_candidate activate_candidate transaction_active=false note "CONVERGED mode=$mode controller=$controller_id config_ref=$config_ref engine_ref=$engine_ref state=$target_state" @@ -1095,12 +1285,17 @@ perform_rollback() { } perform_uninstall() { - local old_release= + local candidate old_release='' old_ref='' load_installed_controller_identity - old_release=$(current_runtime_release) + for candidate in "$(current_runtime_release)" "$(readlink -f "$manager_current" 2>/dev/null || true)" "$repo_root"; do + [[ -n "$candidate" && -f "$candidate/.ci-fleet-engine-ref" ]] || continue + old_ref=$(<"$candidate/.ci-fleet-engine-ref") + if [[ "$old_ref" =~ ^[0-9a-f]{40}$ ]] && runtime_release_complete "$candidate" "$old_ref"; then old_release=$candidate; break; fi + done + [[ -n "$old_release" ]] || die 'a trusted complete release is required to uninstall the controller' make_checkpoint transaction_active=true - drain_current + drain_current false "$rendered_env" "$old_release" remove_inactive_managed_runners if [[ -n "$old_release" && -f "$rendered_env" ]]; then compose "$old_release" "$rendered_env" down --remove-orphans || true diff --git a/scripts/test-install-worker-controller.sh b/scripts/test-install-worker-controller.sh index cce7fd89..e8498dcb 100755 --- a/scripts/test-install-worker-controller.sh +++ b/scripts/test-install-worker-controller.sh @@ -25,6 +25,7 @@ set -u state=${FAKE_DOCKER_STATE:?} status_file=${FAKE_CONTROLLER_STATUS_FILE:-} paused_state=${FAKE_PAUSED_STATE:-} +stopped_state=${FAKE_STOPPED_CONTROLLER_STATE:-} if [[ -n ${FAKE_REQUIRE_LOCAL_DOCKER_ENDPOINT:-} ]]; then expected_socket=${CI_FLEET_ROOT_PREFIX:-}/var/run/docker.sock [[ ${DOCKER_HOST:-} == "unix://$expected_socket" ]] || { @@ -34,11 +35,12 @@ if [[ -n ${FAKE_REQUIRE_LOCAL_DOCKER_ENDPOINT:-} ]]; then fi case "${1:-}" in info) + if [[ -n ${FAKE_DOCKER_INFO_FAIL_AFTER_IMAGE_INSPECT:-} && -f $FAKE_DOCKER_INFO_FAIL_AFTER_IMAGE_INSPECT ]]; then exit 1; fi [[ "$*" != *DockerRootDir* ]] || printf '%s\n' "${CI_FLEET_DOCKER_ROOT:?}" exit 0 ;; inspect) - [[ -f "$state" ]] || exit 1 + [[ -f "$state" || -n "$stopped_state" && -f "$stopped_state" ]] || exit 1 if [[ "$*" == *'.Config.Env'* ]]; then [[ -n "${FAKE_CONTROLLER_ENV_FILE:-}" && -f "$FAKE_CONTROLLER_ENV_FILE" ]] || exit 1 cat "$FAKE_CONTROLLER_ENV_FILE" @@ -48,7 +50,7 @@ case "${1:-}" in [[ -n "${FAKE_CONTROLLER_IMAGE_ID_FILE:-}" && -f "$FAKE_CONTROLLER_IMAGE_ID_FILE" ]] || exit 1 cat "$FAKE_CONTROLLER_IMAGE_ID_FILE" elif [[ "$*" == *'.State.Status'* ]]; then - if [[ -n "$status_file" && -f "$status_file" ]]; then cat "$status_file"; else printf '%s\n' "${FAKE_CONTROLLER_STATUS:-running}"; fi + if [[ -n "$paused_state" && -f "$paused_state" ]]; then printf 'paused\n'; elif [[ -n "$status_file" && -f "$status_file" ]]; then cat "$status_file"; elif [[ -n "$stopped_state" && -f "$stopped_state" ]]; then printf 'exited\n'; else printf '%s\n' "${FAKE_CONTROLLER_STATUS:-running}"; fi elif [[ "$*" == *'.State.Paused'* ]]; then if [[ -n "$paused_state" && -f "$paused_state" ]]; then printf 'true\n'; else printf 'false\n'; fi else @@ -64,25 +66,81 @@ case "${1:-}" in printf 'managed-runner\n' elif [[ -n "${FAKE_RUNNER_STATE:-}" && -f "$FAKE_RUNNER_STATE" ]]; then printf 'managed-runner\n' + elif [[ "$*" != *'io.randomdevelopment.ci-fleet.kind=runner'* && -n "${FAKE_ACTIVE_MANAGED_STATE:-}" && -f "$FAKE_ACTIVE_MANAGED_STATE" ]]; then + printf 'active-managed-container\n' fi exit 0 ;; image) - [[ "${2:-}" == inspect ]] || exit 1 - image=${!#} - [[ -z "${FAKE_IMAGE_INSPECT_LOG:-}" ]] || printf '%s\n' "$image" >>"$FAKE_IMAGE_INSPECT_LOG" - if [[ "$image" == "${FAKE_RUNNER_IMAGE:-}" ]]; then - image_state=${FAKE_RUNNER_IMAGE_STATE:-} - elif [[ "$image" == "${FAKE_CONTROLLER_IMAGE:-}" ]]; then - image_state=${FAKE_CONTROLLER_IMAGE_STATE:-} - else - exit 1 - fi - [[ -f "$image_state" ]] || exit 1 - if [[ "$*" == *'{{.Id}}'* ]]; then printf 'sha256:%s\n' "$(<"$image_state")"; else cat "$image_state"; fi + case "${2:-}" in + inspect) + image=${!#} + [[ -z "${FAKE_IMAGE_INSPECT_LOG:-}" ]] || printf '%s\n' "$image" >>"$FAKE_IMAGE_INSPECT_LOG" + if [[ "$image" == "${FAKE_RUNNER_IMAGE:-}" || "$image" == "${FAKE_PRIOR_RUNNER_IMAGE:-}" || "$image" == "${FAKE_PREVIOUS_RUNNER_IMAGE:-}" ]]; then + image_state=${FAKE_RUNNER_IMAGE_STATE:-} + image_id_state=${FAKE_RUNNER_IMAGE_ID_STATE:-} + elif [[ "$image" == "${FAKE_CONTROLLER_IMAGE:-}" || "$image" == "${FAKE_PRIOR_CONTROLLER_IMAGE:-}" || "$image" == "${FAKE_PREVIOUS_CONTROLLER_IMAGE:-}" ]]; then + image_state=${FAKE_CONTROLLER_IMAGE_STATE:-} + image_id_state=${FAKE_CONTROLLER_IMAGE_ID_STATE:-} + else + exit 1 + fi + if [[ "$*" == *'{{.Id}}'* ]]; then + if [[ ! -f "$image_id_state" ]]; then + [[ -z ${FAKE_DOCKER_INFO_FAIL_AFTER_IMAGE_INSPECT:-} ]] || : >"$FAKE_DOCKER_INFO_FAIL_AFTER_IMAGE_INSPECT" + exit 1 + fi + cat "$image_id_state" + else + [[ -f "$image_state" ]] && cat "$image_state" + fi + ;; + tag) + image_id=${3:-} + image=${4:-} + [[ -n "${FAKE_AVAILABLE_IMAGE_IDS:-}" && -f "$FAKE_AVAILABLE_IMAGE_IDS" ]] || exit 1 + grep -Fxq "$image_id" "$FAKE_AVAILABLE_IMAGE_IDS" || exit 1 + if [[ "$image" == "${FAKE_RUNNER_IMAGE:-}" || "$image" == "${FAKE_PRIOR_RUNNER_IMAGE:-}" || "$image" == "${FAKE_PREVIOUS_RUNNER_IMAGE:-}" ]]; then + image_id_state=${FAKE_RUNNER_IMAGE_ID_STATE:-} + elif [[ "$image" == "${FAKE_CONTROLLER_IMAGE:-}" || "$image" == "${FAKE_PRIOR_CONTROLLER_IMAGE:-}" || "$image" == "${FAKE_PREVIOUS_CONTROLLER_IMAGE:-}" ]]; then + image_id_state=${FAKE_CONTROLLER_IMAGE_ID_STATE:-} + else + exit 1 + fi + printf '%s\n' "$image_id" >"$image_id_state" + [[ -z "${FAKE_COMPOSE_LOG:-}" ]] || printf 'image-tag|%s|%s\n' "$image_id" "$image" >>"$FAKE_COMPOSE_LOG" + ;; + rm) + image=${3:-} + if [[ "$image" == "${FAKE_RUNNER_IMAGE:-}" || "$image" == "${FAKE_PRIOR_RUNNER_IMAGE:-}" || "$image" == "${FAKE_PREVIOUS_RUNNER_IMAGE:-}" ]]; then + image_kind=runner + image_state=${FAKE_RUNNER_IMAGE_STATE:-} + image_id_state=${FAKE_RUNNER_IMAGE_ID_STATE:-} + elif [[ "$image" == "${FAKE_CONTROLLER_IMAGE:-}" || "$image" == "${FAKE_PRIOR_CONTROLLER_IMAGE:-}" || "$image" == "${FAKE_PREVIOUS_CONTROLLER_IMAGE:-}" ]]; then + image_kind=controller + image_state=${FAKE_CONTROLLER_IMAGE_STATE:-} + image_id_state=${FAKE_CONTROLLER_IMAGE_ID_STATE:-} + else + exit 1 + fi + if [[ "$image_kind" == runner && -n "${FAKE_ALL_RUNNER_STATE:-}" && -f "$FAKE_ALL_RUNNER_STATE" ]]; then + [[ -z "${FAKE_COMPOSE_LOG:-}" ]] || printf 'image-rm-blocked|%s\n' "$image" >>"$FAKE_COMPOSE_LOG" + exit 48 + fi + if [[ -n "$stopped_state" && -f "$stopped_state" && -n "${FAKE_CONTROLLER_IMAGE_ID_FILE:-}" && -f "$FAKE_CONTROLLER_IMAGE_ID_FILE" && -f "$image_id_state" ]] \ + && cmp -s "$FAKE_CONTROLLER_IMAGE_ID_FILE" "$image_id_state"; then + [[ -z "${FAKE_COMPOSE_LOG:-}" ]] || printf 'image-rm-blocked|%s\n' "$image" >>"$FAKE_COMPOSE_LOG" + exit 48 + fi + rm -f "$image_state" "$image_id_state" + [[ -z "${FAKE_COMPOSE_LOG:-}" ]] || printf 'image-rm|%s\n' "$image" >>"$FAKE_COMPOSE_LOG" + ;; + *) exit 1 ;; + esac ;; rm) (($# >= 2)) || exit 1 + [[ -z "${FAKE_COMPOSE_LOG:-}" ]] || printf 'container-rm|%s\n' "$*" >>"$FAKE_COMPOSE_LOG" [[ -z "${FAKE_ALL_RUNNER_STATE:-}" ]] || rm -f "$FAKE_ALL_RUNNER_STATE" ;; volume|network) @@ -106,13 +164,25 @@ case "${1:-}" in fi case "$command" in up) + controller_image=$(awk -F= '$1 == "CI_FLEET_CONTROLLER_IMAGE" {print substr($0, index($0, "=") + 1)}' "$env_file") + "$0" image inspect --format '{{.Id}}' "$controller_image" >/dev/null 2>&1 || { + printf 'configured controller image is unavailable: %s\n' "$controller_image" >&2 + exit 49 + } + if [[ -n "${FAKE_DELAY_UP_ONCE:-}" && -f "$FAKE_DELAY_UP_ONCE" ]]; then + delay_marker=$FAKE_DELAY_UP_ONCE + rm -f "$delay_marker" + : >"${delay_marker}.entered" + sleep 2 + fi if [[ -n "${FAKE_FAIL_UP_ONCE:-}" && -f "$FAKE_FAIL_UP_ONCE" ]]; then rm -f "$FAKE_FAIL_UP_ONCE" exit 42 fi : >"$state" + [[ -z "$stopped_state" ]] || rm -f "$stopped_state" [[ -z "${FAKE_CONTROLLER_PROVENANCE_FILE:-}" ]] || printf '%s\n' "${FAKE_ENGINE_REF:?}" >"$FAKE_CONTROLLER_PROVENANCE_FILE" - [[ -z "${FAKE_CONTROLLER_IMAGE_ID_FILE:-}" ]] || printf 'sha256:%s\n' "${FAKE_ENGINE_REF:?}" >"$FAKE_CONTROLLER_IMAGE_ID_FILE" + [[ -z "${FAKE_CONTROLLER_IMAGE_ID_FILE:-}" || -z "${FAKE_CONTROLLER_IMAGE_ID_STATE:-}" ]] || cp "$FAKE_CONTROLLER_IMAGE_ID_STATE" "$FAKE_CONTROLLER_IMAGE_ID_FILE" [[ -z "${FAKE_CONTROLLER_ENV_FILE:-}" ]] || cp "$env_file" "$FAKE_CONTROLLER_ENV_FILE" if [[ -n "${FAKE_RESTART_AFTER_UP:-}" && -f "$FAKE_RESTART_AFTER_UP" ]]; then rm -f "$FAKE_RESTART_AFTER_UP" @@ -122,13 +192,24 @@ case "${1:-}" in fi ;; stop) + if [[ -n "$paused_state" && -f "$paused_state" ]]; then exit 42; fi if [[ -n "${FAKE_STOP_FAIL:-}" && -f "$FAKE_STOP_FAIL" ]]; then exit 42; fi - rm -f "$state"; [[ -z "$status_file" ]] || rm -f "$status_file"; [[ -z "$paused_state" ]] || rm -f "$paused_state"; [[ -z "${FAKE_CONTROLLER_PROVENANCE_FILE:-}" ]] || rm -f "$FAKE_CONTROLLER_PROVENANCE_FILE"; [[ -z "${FAKE_CONTROLLER_IMAGE_ID_FILE:-}" ]] || rm -f "$FAKE_CONTROLLER_IMAGE_ID_FILE"; [[ -z "${FAKE_CONTROLLER_ENV_FILE:-}" ]] || rm -f "$FAKE_CONTROLLER_ENV_FILE" + rm -f "$state"; [[ -z "$status_file" ]] || rm -f "$status_file"; [[ -z "$paused_state" ]] || rm -f "$paused_state" + if [[ -n "$stopped_state" ]]; then + : >"$stopped_state" + else + [[ -z "${FAKE_CONTROLLER_PROVENANCE_FILE:-}" ]] || rm -f "$FAKE_CONTROLLER_PROVENANCE_FILE"; [[ -z "${FAKE_CONTROLLER_IMAGE_ID_FILE:-}" ]] || rm -f "$FAKE_CONTROLLER_IMAGE_ID_FILE"; [[ -z "${FAKE_CONTROLLER_ENV_FILE:-}" ]] || rm -f "$FAKE_CONTROLLER_ENV_FILE" + fi + [[ -z "${FAKE_ACTIVE_MANAGED_AFTER_STOP:-}" ]] || : >"$FAKE_ACTIVE_MANAGED_AFTER_STOP" ;; - down|rm) rm -f "$state"; [[ -z "$status_file" ]] || rm -f "$status_file"; [[ -z "$paused_state" ]] || rm -f "$paused_state"; [[ -z "${FAKE_CONTROLLER_PROVENANCE_FILE:-}" ]] || rm -f "$FAKE_CONTROLLER_PROVENANCE_FILE"; [[ -z "${FAKE_CONTROLLER_IMAGE_ID_FILE:-}" ]] || rm -f "$FAKE_CONTROLLER_IMAGE_ID_FILE"; [[ -z "${FAKE_CONTROLLER_ENV_FILE:-}" ]] || rm -f "$FAKE_CONTROLLER_ENV_FILE" ;; + down|rm) rm -f "$state"; [[ -z "$status_file" ]] || rm -f "$status_file"; [[ -z "$paused_state" ]] || rm -f "$paused_state"; [[ -z "$stopped_state" ]] || rm -f "$stopped_state"; [[ -z "${FAKE_CONTROLLER_PROVENANCE_FILE:-}" ]] || rm -f "$FAKE_CONTROLLER_PROVENANCE_FILE"; [[ -z "${FAKE_CONTROLLER_IMAGE_ID_FILE:-}" ]] || rm -f "$FAKE_CONTROLLER_IMAGE_ID_FILE"; [[ -z "${FAKE_CONTROLLER_ENV_FILE:-}" ]] || rm -f "$FAKE_CONTROLLER_ENV_FILE" ;; pause) + if [[ -n "$paused_state" && -f "$paused_state" ]]; then + printf 'Error response from daemon: container is already paused\n' >&2 + exit 42 + fi [[ -z "$paused_state" ]] || : >"$paused_state" - [[ -z "${FAKE_RUNNER_STATE:-}" ]] || rm -f "$FAKE_RUNNER_STATE" + [[ -n "${FAKE_KEEP_RUNNER_ON_PAUSE:-}" || -z "${FAKE_RUNNER_STATE:-}" ]] || rm -f "$FAKE_RUNNER_STATE" ;; unpause) [[ -z "$paused_state" ]] || rm -f "$paused_state" ;; kill) @@ -139,10 +220,21 @@ case "${1:-}" in [[ -z "$paused_state" ]] || rm -f "$paused_state" ;; build) + [[ -z "${FAKE_FAIL_BUILD:-}" ]] || exit 46 + if [[ -n "${FAKE_PARTIAL_BUILD_FAIL:-}" ]]; then + printf '%s\n' "${FAKE_PARTIAL_RUNNER_IMAGE_ID:?}" >>"${FAKE_AVAILABLE_IMAGE_IDS:?}" + printf '%s\n' "$FAKE_PARTIAL_RUNNER_IMAGE_ID" >"${FAKE_RUNNER_IMAGE_ID_STATE:?}" + exit 46 + fi [[ -z "${FAKE_RUNNER_IMAGE_STATE:-}" ]] || printf '%s\n' "${FAKE_ENGINE_REF:?}" >"$FAKE_RUNNER_IMAGE_STATE" [[ -z "${FAKE_CONTROLLER_IMAGE_STATE:-}" ]] || printf '%s\n' "${FAKE_ENGINE_REF:?}" >"$FAKE_CONTROLLER_IMAGE_STATE" + [[ -z "${FAKE_RUNNER_IMAGE_ID_STATE:-}" ]] || printf '%s\n' "${FAKE_CANDIDATE_RUNNER_IMAGE_ID:?}" >"$FAKE_RUNNER_IMAGE_ID_STATE" + [[ -z "${FAKE_CONTROLLER_IMAGE_ID_STATE:-}" ]] || printf '%s\n' "${FAKE_CANDIDATE_CONTROLLER_IMAGE_ID:?}" >"$FAKE_CONTROLLER_IMAGE_ID_STATE" + [[ -z "${FAKE_AVAILABLE_IMAGE_IDS:-}" ]] || printf '%s\n%s\n' "$FAKE_CANDIDATE_RUNNER_IMAGE_ID" "$FAKE_CANDIDATE_CONTROLLER_IMAGE_ID" >>"$FAKE_AVAILABLE_IMAGE_IDS" + [[ -z "${FAKE_DISK_USED_PERCENT_AFTER_BUILD:-}" || -z "${FAKE_DISK_USED_PERCENT_FILE:-}" ]] || printf '%s\n' "$FAKE_DISK_USED_PERCENT_AFTER_BUILD" >"$FAKE_DISK_USED_PERCENT_FILE" ;; - config|logs) ;; + config) [[ -z "${FAKE_FAIL_CONFIG:-}" ]] || exit 47 ;; + logs) ;; *) exit 1 ;; esac ;; @@ -156,6 +248,7 @@ if [[ "${1:-}" == enable && "${2:-}" == --now && ! -f "${CI_FLEET_ROOT_PREFIX:-} exit 98 fi if [[ -n "${FAKE_FAIL_TIMER_ENABLE:-}" && "${1:-}" == enable && "${2:-}" == --now && "$*" == *ci-fleet-reconcile.timer* ]]; then + [[ -z "${FAKE_CREATE_STOPPED_RUNNER_ON_TIMER_FAILURE:-}" || -z "${FAKE_ALL_RUNNER_STATE:-}" ]] || : >"$FAKE_ALL_RUNNER_STATE" exit 97 fi if [[ -n "${FAKE_DISABLED_TIMER:-}" && ( "${1:-}" == is-enabled || "${1:-}" == is-active ) && $# == 3 && "${3:-}" == "$FAKE_DISABLED_TIMER" ]]; then @@ -199,12 +292,18 @@ chmod 700 "$fake_bin/git" cat >"$fake_bin/df" <<'EOF' #!/usr/bin/env bash -if [[ -n ${FAKE_DISK_USED_PERCENT:-} ]]; then +if [[ -n ${FAKE_DISK_USED_PERCENT_FILE:-} && -f $FAKE_DISK_USED_PERCENT_FILE ]]; then + used=$(<"$FAKE_DISK_USED_PERCENT_FILE") +elif [[ -n ${FAKE_DISK_USED_PERCENT:-} ]]; then + used=$FAKE_DISK_USED_PERCENT +else + exec "$REAL_DF" "$@" +fi +if [[ -n ${used:-} ]]; then printf 'Filesystem 1024-blocks Used Available Capacity Mounted on\n' - printf 'fixture 100 90 10 %s%% /fixture\n' "$FAKE_DISK_USED_PERCENT" + printf 'fixture 100 90 10 %s%% /fixture\n' "$used" exit 0 fi -exec "$REAL_DF" "$@" EOF chmod 700 "$fake_bin/df" @@ -264,6 +363,12 @@ export FAKE_RUNNER_IMAGE=$runner_image export FAKE_CONTROLLER_IMAGE=ci-fleet-controller:${engine_ref:0:12} export FAKE_RUNNER_IMAGE_STATE=$tmp/runner-image-present export FAKE_CONTROLLER_IMAGE_STATE=$tmp/controller-image-present +export FAKE_RUNNER_IMAGE_ID_STATE=$tmp/runner-image-id +export FAKE_CONTROLLER_IMAGE_ID_STATE=$tmp/controller-image-id +export FAKE_AVAILABLE_IMAGE_IDS=$tmp/available-image-ids +export FAKE_CANDIDATE_RUNNER_IMAGE_ID=sha256:1111111111111111111111111111111111111111111111111111111111111111 +export FAKE_CANDIDATE_CONTROLLER_IMAGE_ID=sha256:2222222222222222222222222222222222222222222222222222222222222222 +export FAKE_PARTIAL_RUNNER_IMAGE_ID=sha256:3333333333333333333333333333333333333333333333333333333333333333 export FAKE_IMAGE_INSPECT_LOG=$tmp/image-inspects for dockerfile in "$repo_root/controller/Dockerfile" "$repo_root/runner/Dockerfile"; do grep -Fq "LABEL org.opencontainers.image.revision=\"\${CI_FLEET_COMMIT}\"" "$dockerfile" || fail "managed image lacks engine provenance label: $dockerfile" @@ -398,6 +503,11 @@ grep -Fq 'CONVERGED mode=install' <<<"$first" || fail 'fresh install did not con [[ -L "$root/opt/ci-fleet/current" && -f "$root/var/lib/ci-fleet/install-state.json" ]] || fail 'fresh install state is incomplete' [[ $(readlink -f "$root/opt/ci-fleet/manager/current") == "$root/opt/ci-fleet/manager/releases/$engine_ref" ]] || fail 'installer manager did not activate the desired engine release' [[ -f "$FAKE_DOCKER_STATE" ]] || fail 'active controller was not started' +fresh_checkpoint=$(find "$root/var/lib/ci-fleet/checkpoints" -mindepth 1 -maxdepth 1 -type d ! -name '.checkpoint.staging.*' -print -quit) +fresh_format_marker=$fresh_checkpoint/format-version +[[ -f "$fresh_format_marker" && ! -L "$fresh_format_marker" && $(stat -c '%u:%a:%s' "$fresh_format_marker") == "$(id -u):600:2" && $(<"$fresh_format_marker") == 2 ]] || fail 'fresh checkpoint lacks the exact root-owned mode-0600 format marker' +[[ ! -e "$fresh_checkpoint/ci-fleet.env" && ! -e "$fresh_checkpoint/image-ids.env" ]] || fail 'fresh checkpoint stored prior managed image state' +if find "$root/var/lib/ci-fleet/checkpoints" -name image-ids.env -print -quit | grep -q .; then fail 'fresh install checkpoint stored a prior image map'; fi install_state=$root/var/lib/ci-fleet/install-state.json chmod 644 "$install_state" expect_failure 'install state must be owned by root with mode 0600' env CI_FLEET_INSTALL_STATE_FILE="$install_state" CI_FLEET_INSTALLER="$installer" "$repo_root/scripts/check-installed-state.sh" @@ -407,8 +517,9 @@ expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one" >/dev/n rendered_env=$root/etc/ci-fleet/ci-fleet.env chmod 644 "$rendered_env" expect_failure 'DRIFT rendered_environment' "$installer" --check "${base_args[@]}" --ref "$ref_one" -expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one" >/dev/null -[[ $(stat -c %a "$rendered_env") == 600 ]] || fail 'convergence did not repair rendered-environment mode' +expect_failure 'rendered environment must be owned by root with mode 0600' "$installer" --install "${base_args[@]}" --ref "$ref_one" +[[ $(stat -c %a "$rendered_env") == 644 ]] || fail 'untrusted rendered environment was changed before checkpoint validation' +chmod 600 "$rendered_env" export FAKE_REQUIRE_LOCAL_DOCKER_ENDPOINT=1 manual_health_result=0 "$repo_root/scripts/healthcheck.sh" >/dev/null || manual_health_result=$? @@ -480,10 +591,10 @@ printf '%040d\n' 0 >"$FAKE_CONTROLLER_PROVENANCE_FILE" expect_failure 'DRIFT controller_runtime' "$installer" --check "${base_args[@]}" --ref "$ref_one" expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one" >/dev/null [[ $(<"$FAKE_CONTROLLER_PROVENANCE_FILE") == "$engine_ref" ]] || fail 'controller convergence did not restore running image provenance' -printf 'sha256:%040d\n' 0 >"$FAKE_CONTROLLER_IMAGE_ID_FILE" +printf 'sha256:%064d\n' 0 >"$FAKE_CONTROLLER_IMAGE_ID_FILE" expect_failure 'DRIFT controller_runtime' "$installer" --check "${base_args[@]}" --ref "$ref_one" expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one" >/dev/null -[[ $(<"$FAKE_CONTROLLER_IMAGE_ID_FILE") == "sha256:$engine_ref" ]] || fail 'controller convergence did not restore live image identity' +[[ $(<"$FAKE_CONTROLLER_IMAGE_ID_FILE") == "$FAKE_CANDIDATE_CONTROLLER_IMAGE_ID" ]] || fail 'controller convergence did not restore live image identity' python3 -c 'from pathlib import Path; import sys; path = Path(sys.argv[1]); path.write_text(path.read_text().replace("CI_FLEET_MAX_RUNNERS=1", "CI_FLEET_MAX_RUNNERS=9"))' "$FAKE_CONTROLLER_ENV_FILE" grep -Fxq 'CI_FLEET_MAX_RUNNERS=9' "$FAKE_CONTROLLER_ENV_FILE" || fail 'live-environment fixture did not mutate' expect_failure 'DRIFT controller_runtime' "$installer" --check "${base_args[@]}" --ref "$ref_one" @@ -618,6 +729,409 @@ ln -sfn "$prior_manager" "$root/opt/ci-fleet/manager/current" expect_failure 'DRIFT maintenance_timers' "$installer" --check "${base_args[@]}" --ref "$ref_one" ref_two=$(write_config active 2 2) +prior_runner_image=ci-fleet-runner:prior +prior_controller_image=ci-fleet-controller:prior +export FAKE_PRIOR_RUNNER_IMAGE=$prior_runner_image +export FAKE_PRIOR_CONTROLLER_IMAGE=$prior_controller_image +for environment in "$rendered_env" "$FAKE_CONTROLLER_ENV_FILE"; do + python3 -c 'from pathlib import Path; import sys; path = Path(sys.argv[1]); path.write_text(path.read_text().replace(sys.argv[2], sys.argv[3]))' "$environment" "$FAKE_RUNNER_IMAGE" "$prior_runner_image" +done +python3 -c 'from pathlib import Path; import sys; path = Path(sys.argv[1]); path.write_text(path.read_text().replace(sys.argv[2], sys.argv[3]))' "$rendered_env" "$FAKE_CONTROLLER_IMAGE" "$prior_controller_image" +build_failure_output=$tmp/build-failure.out +build_failure_root=$tmp/build-failure-root +cp -a "$root" "$build_failure_root" +export FAKE_COMPOSE_LOG=$tmp/stopped-distinct-tag-build-compose.log +: >"$FAKE_COMPOSE_LOG" +printf 'exited\n' >"$FAKE_CONTROLLER_STATUS_FILE" +export FAKE_FAIL_BUILD=1 +if "$installer" --upgrade "${base_args[@]}" --ref "$ref_two" >"$build_failure_output" 2>&1; then + fail 'candidate build failure unexpectedly succeeded' +fi +unset FAKE_FAIL_BUILD +if grep -Eq 'CHECKPOINT_CREATED|DRAIN_READY|ROLLBACK_' "$build_failure_output"; then fail 'candidate build failure entered the transaction'; fi +grep -q '^build|' "$FAKE_COMPOSE_LOG" || fail 'stopped controller with distinct trusted tags did not prebuild' +if grep -q '^stop|' "$FAKE_COMPOSE_LOG"; then fail 'stopped controller with distinct trusted tags was stopped before prebuild'; fi +diff -r "$build_failure_root" "$root" >/dev/null || fail 'candidate build failure changed host state' +[[ -f "$FAKE_DOCKER_STATE" ]] || fail 'candidate build failure stopped the installed controller' +unset FAKE_COMPOSE_LOG +python3 -c 'from pathlib import Path; import sys; path = Path(sys.argv[1]); path.write_text(path.read_text().replace(sys.argv[2], sys.argv[3]))' "$rendered_env" "$prior_controller_image" "$FAKE_CONTROLLER_IMAGE" +same_controller_output=$tmp/same-controller-build.out +export FAKE_COMPOSE_LOG=$tmp/same-controller-build-compose.log +: >"$FAKE_COMPOSE_LOG" +export FAKE_FAIL_BUILD=1 +if "$installer" --upgrade "${base_args[@]}" --ref "$ref_two" >"$same_controller_output" 2>&1; then + fail 'same-controller-tag build failure unexpectedly succeeded' +fi +unset FAKE_FAIL_BUILD +stop_line=$(grep -n -m1 '^stop|' "$FAKE_COMPOSE_LOG" | cut -d: -f1 || true) +build_line=$(grep -n -m1 '^build|' "$FAKE_COMPOSE_LOG" | cut -d: -f1 || true) +[[ -n "$stop_line" && -n "$build_line" && "$stop_line" -lt "$build_line" ]] || fail 'installed controller tag was rebuilt before checkpoint and stop' +grep -Fq 'CHECKPOINT_CREATED' "$same_controller_output" || fail 'same controller tag was rebuilt without a checkpoint' +grep -Fq 'ROLLBACK_RESTORED' "$same_controller_output" || fail 'same-controller-tag build failure did not restore the checkpoint' +grep -Fxq "CI_FLEET_RUNNER_IMAGE=$prior_runner_image" "$rendered_env" || fail 'same-controller-tag build failure changed installed runner state' +grep -Fxq "CI_FLEET_CONTROLLER_IMAGE=$FAKE_CONTROLLER_IMAGE" "$rendered_env" || fail 'same-controller-tag build failure changed installed controller state' +[[ $(readlink -f "$root/opt/ci-fleet/manager/current") == "$prior_manager" ]] || fail 'same-controller-tag build failure changed the installed manager' +unset FAKE_COMPOSE_LOG +python3 -c 'from pathlib import Path; import sys; path = Path(sys.argv[1]); path.write_text(path.read_text().replace(sys.argv[2], "CI_FLEET_RUNNER_IMAGE="))' "$FAKE_CONTROLLER_ENV_FILE" "CI_FLEET_RUNNER_IMAGE=$prior_runner_image" +empty_live_image_output=$tmp/empty-live-image-build.out +export FAKE_COMPOSE_LOG=$tmp/empty-live-image-build-compose.log +: >"$FAKE_COMPOSE_LOG" +export FAKE_FAIL_BUILD=1 +if "$installer" --upgrade "${base_args[@]}" --ref "$ref_two" >"$empty_live_image_output" 2>&1; then + fail 'empty-live-image candidate build failure unexpectedly succeeded' +fi +unset FAKE_FAIL_BUILD +stop_line=$(grep -n -m1 '^stop|' "$FAKE_COMPOSE_LOG" | cut -d: -f1 || true) +build_line=$(grep -n -m1 '^build|' "$FAKE_COMPOSE_LOG" | cut -d: -f1 || true) +[[ -n "$stop_line" && -n "$build_line" && "$stop_line" -lt "$build_line" ]] || fail 'empty live runner image was accepted before drain' +grep -Fq 'DRAIN_OK managed_runners=0' "$empty_live_image_output" || fail 'empty live runner image build did not wait for drain' +grep -Fq 'ROLLBACK_RESTORED' "$empty_live_image_output" || fail 'empty live runner image build failure did not restore the checkpoint' +[[ -f "$FAKE_DOCKER_STATE" ]] || fail 'empty live runner image build failure stopped the installed controller' +grep -Fxq "CI_FLEET_RUNNER_IMAGE=$prior_runner_image" "$rendered_env" || fail 'empty live runner image build failure changed installed environment' +grep -Fxq "CI_FLEET_RUNNER_IMAGE=$prior_runner_image" "$FAKE_CONTROLLER_ENV_FILE" || fail 'empty live runner image build failure did not restore the installed controller' +grep -Fq 'CI_FLEET_MAX_RUNNERS=1' "$rendered_env" || fail 'empty live runner image build failure changed installed state' +[[ $(readlink -f "$root/opt/ci-fleet/manager/current") == "$prior_manager" ]] || fail 'empty live runner image build failure changed the installed manager' +unset FAKE_COMPOSE_LOG +[[ ${CI_FLEET_TEST_STOP_AFTER_EMPTY_LIVE_IMAGE_BUILD:-0} != 1 ]] || { printf 'EMPTY_LIVE_IMAGE_BUILD_REGRESSION_OK\n'; exit 0; } +python3 -c 'from pathlib import Path; import sys; path = Path(sys.argv[1]); path.write_text(path.read_text().replace(sys.argv[2], sys.argv[3]))' "$FAKE_CONTROLLER_ENV_FILE" "$prior_runner_image" "$FAKE_RUNNER_IMAGE" +live_drift_build_output=$tmp/live-drift-build.out +export FAKE_COMPOSE_LOG=$tmp/live-drift-build-compose.log +: >"$FAKE_COMPOSE_LOG" +export FAKE_FAIL_BUILD=1 +if "$installer" --upgrade "${base_args[@]}" --ref "$ref_two" >"$live_drift_build_output" 2>&1; then + fail 'live-drift candidate build failure unexpectedly succeeded' +fi +unset FAKE_FAIL_BUILD +stop_line=$(grep -n -m1 '^stop|' "$FAKE_COMPOSE_LOG" | cut -d: -f1 || true) +build_line=$(grep -n -m1 '^build|' "$FAKE_COMPOSE_LOG" | cut -d: -f1 || true) +[[ -n "$stop_line" && -n "$build_line" && "$stop_line" -lt "$build_line" ]] || fail 'live drift runner tag was built before drain' +grep -Fq 'DRAIN_OK managed_runners=0' "$live_drift_build_output" || fail 'live drift runner tag build did not wait for drain' +grep -Fq 'ROLLBACK_RESTORED' "$live_drift_build_output" || fail 'live drift runner tag build failure did not restore the checkpoint' +unset FAKE_COMPOSE_LOG +[[ ${CI_FLEET_TEST_STOP_AFTER_LIVE_DRIFT_BUILD:-0} != 1 ]] || { printf 'LIVE_DRIFT_BUILD_REGRESSION_OK\n'; exit 0; } +python3 -c 'from pathlib import Path; import sys; path = Path(sys.argv[1]); path.write_text(path.read_text().replace(sys.argv[2], sys.argv[3]))' "$rendered_env" "$FAKE_CONTROLLER_IMAGE" "$prior_controller_image" +postbuild_capacity_output=$tmp/postbuild-capacity.out +export FAKE_DISK_USED_PERCENT_FILE=$tmp/docker-disk-used-percent +export FAKE_DISK_USED_PERCENT_AFTER_BUILD=80 +printf '79\n' >"$FAKE_DISK_USED_PERCENT_FILE" +if "$installer" --upgrade "${base_args[@]}" --ref "$ref_two" >"$postbuild_capacity_output" 2>&1; then + fail 'post-build Docker capacity failure unexpectedly succeeded' +fi +unset FAKE_DISK_USED_PERCENT_AFTER_BUILD FAKE_DISK_USED_PERCENT_FILE +grep -Fq 'Docker filesystem' "$postbuild_capacity_output" || fail 'post-build Docker capacity failure was not reported' +if grep -Eq 'CHECKPOINT_CREATED|DRAIN_READY|ROLLBACK_' "$postbuild_capacity_output"; then fail 'post-build Docker capacity failure entered the transaction'; fi +[[ -f "$FAKE_DOCKER_STATE" ]] || fail 'post-build Docker capacity failure stopped the installed controller' +grep -Fq 'CI_FLEET_MAX_RUNNERS=1' "$rendered_env" || fail 'post-build Docker capacity failure changed installed state' +[[ ${CI_FLEET_TEST_STOP_AFTER_POSTBUILD_CAPACITY:-0} != 1 ]] || { printf 'POSTBUILD_CAPACITY_REGRESSION_OK\n'; exit 0; } +python3 -c 'from pathlib import Path; import sys; path = Path(sys.argv[1]); path.write_text(path.read_text().replace(sys.argv[2], sys.argv[3]))' "$rendered_env" "$prior_controller_image" "$FAKE_CONTROLLER_IMAGE" +for environment in "$rendered_env" "$FAKE_CONTROLLER_ENV_FILE"; do + python3 -c 'from pathlib import Path; import sys; path = Path(sys.argv[1]); path.write_text(path.read_text().replace(sys.argv[2], sys.argv[3]))' "$environment" "$prior_runner_image" "$FAKE_RUNNER_IMAGE" +done +config_failure_output=$tmp/config-failure.out +export FAKE_FAIL_CONFIG=1 +if "$installer" --upgrade "${base_args[@]}" --ref "$ref_two" >"$config_failure_output" 2>&1; then + fail 'candidate Compose validation failure unexpectedly succeeded' +fi +unset FAKE_FAIL_CONFIG +if grep -Eq 'CHECKPOINT_CREATED|DRAIN_READY|ROLLBACK_' "$config_failure_output"; then fail 'candidate Compose validation failure entered the transaction'; fi +[[ -f "$FAKE_DOCKER_STATE" ]] || fail 'candidate Compose validation failure stopped the installed controller' +invalid_image_output=$tmp/invalid-image-id.out +export FAKE_COMPOSE_LOG=$tmp/invalid-image-id-compose.log +: >"$FAKE_COMPOSE_LOG" +prior_runner_image_id=$(<"$FAKE_RUNNER_IMAGE_ID_STATE") +printf 'not-a-sha256-image-id\n' >"$FAKE_RUNNER_IMAGE_ID_STATE" +if "$installer" --upgrade "${base_args[@]}" --ref "$ref_two" >"$invalid_image_output" 2>&1; then + fail 'invalid installed image ID unexpectedly reached the transaction' +fi +printf '%s\n' "$prior_runner_image_id" >"$FAKE_RUNNER_IMAGE_ID_STATE" +grep -Fq 'installed runner image ID is invalid' "$invalid_image_output" || fail 'invalid installed image ID was not rejected' +if grep -Eq 'CHECKPOINT_CREATED|DRAIN_READY|ROLLBACK_' "$invalid_image_output" || grep -Eq '^(stop|build)\|' "$FAKE_COMPOSE_LOG"; then fail 'invalid installed image ID caused a transaction side effect'; fi +[[ -f "$FAKE_DOCKER_STATE" ]] || fail 'invalid installed image ID stopped the installed controller' +grep -Fq 'CI_FLEET_MAX_RUNNERS=1' "$rendered_env" || fail 'invalid installed image ID changed installed state' +[[ $(readlink -f "$root/opt/ci-fleet/manager/current") == "$prior_manager" ]] || fail 'invalid installed image ID changed the installed manager' +missing_tag_output=$tmp/missing-tag-build.out +export FAKE_COMPOSE_LOG=$tmp/missing-tag-build-compose.log +: >"$FAKE_COMPOSE_LOG" +prior_runner_image_id=$(<"$FAKE_RUNNER_IMAGE_ID_STATE") +prior_controller_image_id=$(<"$FAKE_CONTROLLER_IMAGE_ID_STATE") +rm -f "$FAKE_RUNNER_IMAGE_STATE" "$FAKE_RUNNER_IMAGE_ID_STATE" +export FAKE_PARTIAL_BUILD_FAIL=1 +if "$installer" --upgrade "${base_args[@]}" --ref "$ref_two" >"$missing_tag_output" 2>&1; then + fail 'missing-tag candidate build failure unexpectedly succeeded' +fi +unset FAKE_PARTIAL_BUILD_FAIL +grep -Fq 'DRAIN_OK managed_runners=0' "$missing_tag_output" || fail "missing managed tag did not reach the deferred build failure: $(<"$missing_tag_output")" +grep -Fq 'ROLLBACK_RESTORED' "$missing_tag_output" || fail 'missing-tag build failure did not restore the checkpoint' +checkpoint_path=$(awk '$1 == "CHECKPOINT_CREATED" {sub(/^path=/, "", $2); value=$2} END {print value}' "$missing_tag_output") +grep -Fxq 'CI_FLEET_RUNNER_IMAGE_ID=absent' "$checkpoint_path/image-ids.env" || fail 'checkpoint did not record the missing runner tag' +[[ ! -e "$FAKE_RUNNER_IMAGE_STATE" && ! -e "$FAKE_RUNNER_IMAGE_ID_STATE" ]] || fail 'rollback retained a runner tag that was absent before repair' +[[ $(<"$FAKE_CONTROLLER_IMAGE_ID_STATE") == "$prior_controller_image_id" ]] || fail 'missing-tag rollback did not restore the prior controller image ID' +grep -Fxq "CI_FLEET_RUNNER_IMAGE=$FAKE_RUNNER_IMAGE" "$rendered_env" || fail 'missing-tag rollback changed the installed environment' +grep -Fq '"configured_max_runners": 1' "$install_state" || fail 'missing-tag rollback changed installed state' +[[ $(readlink -f "$root/opt/ci-fleet/manager/current") == "$prior_manager" ]] || fail 'missing-tag rollback changed the installed manager' +runner_rm_line=$(grep -n -m1 "^image-rm|$FAKE_RUNNER_IMAGE$" "$FAKE_COMPOSE_LOG" | cut -d: -f1 || true) +controller_tag_line=$(grep -n -m1 "^image-tag|$prior_controller_image_id|$FAKE_CONTROLLER_IMAGE$" "$FAKE_COMPOSE_LOG" | cut -d: -f1 || true) +rollback_up_line=$(grep -n '^up|' "$FAKE_COMPOSE_LOG" | tail -n1 | cut -d: -f1 || true) +[[ -n "$runner_rm_line" && -n "$controller_tag_line" && -n "$rollback_up_line" && "$runner_rm_line" -lt "$rollback_up_line" && "$controller_tag_line" -lt "$rollback_up_line" ]] || fail 'missing-tag rollback restarted the controller before restoring both image states' +daemon_failure_output=$tmp/missing-tag-daemon-failure.out +export FAKE_DOCKER_INFO_FAIL_AFTER_IMAGE_INSPECT=$tmp/docker-info-fail-after-image-inspect +: >"$FAKE_COMPOSE_LOG" +if "$installer" --upgrade "${base_args[@]}" --ref "$ref_two" >"$daemon_failure_output" 2>&1; then + fail 'missing-tag daemon failure unexpectedly succeeded' +fi +rm -f "$FAKE_DOCKER_INFO_FAIL_AFTER_IMAGE_INSPECT" +unset FAKE_DOCKER_INFO_FAIL_AFTER_IMAGE_INSPECT +grep -Fq 'Docker daemon is unavailable' "$daemon_failure_output" || fail 'image absence was accepted without confirming Docker availability' +if grep -Eq 'CHECKPOINT_CREATED|DRAIN_READY|ROLLBACK_' "$daemon_failure_output" || grep -Eq '^(stop|build)\|' "$FAKE_COMPOSE_LOG"; then fail 'Docker daemon failure entered the transaction'; fi +uninstall_output=$(expect_success "$installer" --uninstall) +grep -Fq 'UNINSTALL_OK' <<<"$uninstall_output" || fail 'uninstall rejected a missing managed image tag' +expect_success "$installer" --rollback >/dev/null +[[ ! -e "$FAKE_RUNNER_IMAGE_STATE" && ! -e "$FAKE_RUNNER_IMAGE_ID_STATE" && -f "$FAKE_DOCKER_STATE" ]] || fail 'uninstall rollback did not restore the missing-tag installation' +printf '%s\n' "$engine_ref" >"$FAKE_RUNNER_IMAGE_STATE" +printf '%s\n' "$prior_runner_image_id" >"$FAKE_RUNNER_IMAGE_ID_STATE" +unset FAKE_COMPOSE_LOG +absent_controller_output=$tmp/absent-controller-timer-rollback.out +export FAKE_COMPOSE_LOG=$tmp/absent-controller-timer-rollback-compose.log +export FAKE_STOPPED_CONTROLLER_STATE=$tmp/stopped-controller +: >"$FAKE_COMPOSE_LOG" +prior_controller_image_id=sha256:4444444444444444444444444444444444444444444444444444444444444444 +printf '%s\n' "$prior_controller_image_id" >>"$FAKE_AVAILABLE_IMAGE_IDS" +printf '%s\n' "$prior_controller_image_id" >"$FAKE_CONTROLLER_IMAGE_ID_STATE" +printf '%s\n' "$prior_controller_image_id" >"$FAKE_CONTROLLER_IMAGE_ID_FILE" +rm -f "$FAKE_CONTROLLER_IMAGE_STATE" "$FAKE_CONTROLLER_IMAGE_ID_STATE" +[[ $(<"$FAKE_CONTROLLER_IMAGE_ID_FILE") == "$prior_controller_image_id" ]] || fail 'untagged running controller lost its exact image ID' +export FAKE_FAIL_TIMER_ENABLE=1 +if "$installer" --upgrade --config-repo "$config_repo" --config-identity fixture-org/fleet-config --controller example-ci-01 --ref "$ref_two" >"$absent_controller_output" 2>&1; then + fail 'absent-controller timer activation failure unexpectedly succeeded' +fi +unset FAKE_FAIL_TIMER_ENABLE +grep -Fq 'ROLLBACK_RESTORED' "$absent_controller_output" || fail "absent-controller timer failure did not restore the checkpoint: $(<"$absent_controller_output"); compose log: $(<"$FAKE_COMPOSE_LOG")" +checkpoint_path=$(awk '$1 == "CHECKPOINT_CREATED" {sub(/^path=/, "", $2); value=$2} END {print value}' "$absent_controller_output") +grep -Fxq 'CI_FLEET_CONTROLLER_IMAGE_ID=absent' "$checkpoint_path/image-ids.env" || fail 'checkpoint did not preserve the absent controller tag' +grep -Fxq "CI_FLEET_CONTROLLER_LIVE_IMAGE_ID=$prior_controller_image_id" "$checkpoint_path/image-ids.env" || fail 'checkpoint omitted the untagged live controller image ID' +[[ ! -e "$FAKE_CONTROLLER_IMAGE_STATE" && ! -e "$FAKE_CONTROLLER_IMAGE_ID_STATE" ]] || fail 'timer-failure rollback retained a controller tag that was previously absent' +[[ -f "$FAKE_DOCKER_STATE" && ! -f "$FAKE_STOPPED_CONTROLLER_STATE" ]] || fail 'timer-failure rollback did not restart the prior controller' +[[ $(<"$FAKE_CONTROLLER_IMAGE_ID_FILE") == "$prior_controller_image_id" ]] || fail 'timer-failure rollback did not recreate the exact prior controller image' +candidate_up_line=$(grep -n -m1 '^up|' "$FAKE_COMPOSE_LOG" | cut -d: -f1 || true) +rollback_stop_line=$(grep -n '^stop|' "$FAKE_COMPOSE_LOG" | tail -n1 | cut -d: -f1 || true) +candidate_rm_line=$(grep -n -m1 '^rm|' "$FAKE_COMPOSE_LOG" | cut -d: -f1 || true) +controller_tag_line=$(grep -n -m1 "^image-tag|$prior_controller_image_id|$FAKE_CONTROLLER_IMAGE$" "$FAKE_COMPOSE_LOG" | cut -d: -f1 || true) +rollback_up_line=$(grep -n '^up|' "$FAKE_COMPOSE_LOG" | tail -n1 | cut -d: -f1 || true) +controller_rm_line=$(grep -n "^image-rm|$FAKE_CONTROLLER_IMAGE$" "$FAKE_COMPOSE_LOG" | tail -n1 | cut -d: -f1 || true) +[[ -n "$candidate_up_line" && -n "$rollback_stop_line" && -n "$candidate_rm_line" && -n "$controller_tag_line" && -n "$rollback_up_line" && -n "$controller_rm_line" \ + && "$candidate_up_line" -lt "$rollback_stop_line" && "$rollback_stop_line" -lt "$candidate_rm_line" && "$candidate_rm_line" -lt "$controller_tag_line" \ + && "$controller_tag_line" -lt "$rollback_up_line" && "$rollback_up_line" -lt "$controller_rm_line" ]] \ + || fail 'timer-failure rollback did not remove the candidate, recreate the prior controller, and remove its temporary tag' +printf '%s\n' "$engine_ref" >"$FAKE_CONTROLLER_IMAGE_STATE" +printf '%s\n' "$prior_controller_image_id" >"$FAKE_CONTROLLER_IMAGE_ID_STATE" +printf '%s\n' "$prior_controller_image_id" >"$FAKE_CONTROLLER_IMAGE_ID_FILE" +printf '%s\n' "$engine_ref" >"$FAKE_CONTROLLER_PROVENANCE_FILE" +unset FAKE_STOPPED_CONTROLLER_STATE FAKE_COMPOSE_LOG +[[ ${CI_FLEET_TEST_STOP_AFTER_ABSENT_CONTROLLER_ROLLBACK:-0} != 1 ]] || { printf 'ABSENT_CONTROLLER_ROLLBACK_REGRESSION_OK\n'; exit 0; } +drifted_controller_output=$tmp/drifted-controller-timer-rollback.out +export FAKE_COMPOSE_LOG=$tmp/drifted-controller-timer-rollback-compose.log +: >"$FAKE_COMPOSE_LOG" +drifted_live_controller_image_id=sha256:5555555555555555555555555555555555555555555555555555555555555555 +drifted_tag_controller_image_id=sha256:6666666666666666666666666666666666666666666666666666666666666666 +printf '%s\n%s\n' "$drifted_live_controller_image_id" "$drifted_tag_controller_image_id" >>"$FAKE_AVAILABLE_IMAGE_IDS" +printf '%s\n' "$engine_ref" >"$FAKE_CONTROLLER_IMAGE_STATE" +printf '%s\n' "$drifted_tag_controller_image_id" >"$FAKE_CONTROLLER_IMAGE_ID_STATE" +printf '%s\n' "$drifted_live_controller_image_id" >"$FAKE_CONTROLLER_IMAGE_ID_FILE" +export FAKE_FAIL_TIMER_ENABLE=1 +if "$installer" --upgrade --config-repo "$config_repo" --config-identity fixture-org/fleet-config --controller example-ci-01 --ref "$ref_two" >"$drifted_controller_output" 2>&1; then + fail 'drifted-controller timer activation failure unexpectedly succeeded' +fi +unset FAKE_FAIL_TIMER_ENABLE +grep -Fq 'CHECKPOINT_CREATED' "$drifted_controller_output" || fail 'drifted-controller timer failure changed state before checkpointing' +grep -Fq 'DRAIN_OK managed_runners=0' "$drifted_controller_output" || fail 'drifted-controller timer failure did not drain before activation' +grep -Fq 'ROLLBACK_RESTORED' "$drifted_controller_output" || fail "drifted-controller timer failure did not restore the checkpoint: $(<"$drifted_controller_output"); compose log: $(<"$FAKE_COMPOSE_LOG")" +[[ $(<"$FAKE_CONTROLLER_IMAGE_ID_STATE") == "$drifted_tag_controller_image_id" ]] || fail 'drifted-controller rollback did not preserve the prior tag mapping' +[[ $(<"$FAKE_CONTROLLER_IMAGE_ID_FILE") == "$drifted_live_controller_image_id" ]] || fail 'drifted-controller rollback did not recreate the exact prior controller image' +checkpoint_path=$(awk '$1 == "CHECKPOINT_CREATED" {sub(/^path=/, "", $2); value=$2} END {print value}' "$drifted_controller_output") +grep -Fxq "CI_FLEET_CONTROLLER_IMAGE_ID=$drifted_tag_controller_image_id" "$checkpoint_path/image-ids.env" || fail 'checkpoint omitted the drifted controller tag image ID' +grep -Fxq "CI_FLEET_CONTROLLER_LIVE_IMAGE_ID=$drifted_live_controller_image_id" "$checkpoint_path/image-ids.env" || fail 'checkpoint omitted the drifted live controller image ID' +unset FAKE_COMPOSE_LOG +[[ ${CI_FLEET_TEST_STOP_AFTER_DRIFTED_CONTROLLER_ROLLBACK:-0} != 1 ]] || { printf 'DRIFTED_CONTROLLER_ROLLBACK_REGRESSION_OK\n'; exit 0; } +printf '%s\n' "$drifted_tag_controller_image_id" >"$FAKE_CONTROLLER_IMAGE_ID_FILE" +stopped_runner_output=$tmp/stopped-runner-timer-rollback.out +export FAKE_COMPOSE_LOG=$tmp/stopped-runner-timer-rollback-compose.log +export FAKE_ALL_RUNNER_STATE=$tmp/stopped-candidate-runner +: >"$FAKE_COMPOSE_LOG" +rm -f "$FAKE_RUNNER_IMAGE_STATE" "$FAKE_RUNNER_IMAGE_ID_STATE" "$FAKE_ALL_RUNNER_STATE" +export FAKE_CREATE_STOPPED_RUNNER_ON_TIMER_FAILURE=1 +export FAKE_FAIL_TIMER_ENABLE=1 +if "$installer" --upgrade --config-repo "$config_repo" --config-identity fixture-org/fleet-config --controller example-ci-01 --ref "$ref_two" >"$stopped_runner_output" 2>&1; then + fail 'stopped-runner timer activation failure unexpectedly succeeded' +fi +unset FAKE_CREATE_STOPPED_RUNNER_ON_TIMER_FAILURE FAKE_FAIL_TIMER_ENABLE +grep -Fq 'CHECKPOINT_CREATED' "$stopped_runner_output" || fail 'stopped-runner timer failure changed state before checkpointing' +grep -Fq 'DRAIN_OK managed_runners=0' "$stopped_runner_output" || fail 'stopped-runner timer failure did not drain before activation' +grep -Fq 'ROLLBACK_RESTORED' "$stopped_runner_output" || fail "stopped candidate runner blocked absent-tag rollback: $(<"$stopped_runner_output"); compose log: $(<"$FAKE_COMPOSE_LOG")" +checkpoint_path=$(awk '$1 == "CHECKPOINT_CREATED" {sub(/^path=/, "", $2); value=$2} END {print value}' "$stopped_runner_output") +grep -Fxq 'CI_FLEET_RUNNER_IMAGE_ID=absent' "$checkpoint_path/image-ids.env" || fail 'stopped-runner checkpoint did not preserve the absent runner tag' +[[ ! -e "$FAKE_ALL_RUNNER_STATE" ]] || fail 'rollback retained the stopped candidate runner' +[[ ! -e "$FAKE_RUNNER_IMAGE_STATE" && ! -e "$FAKE_RUNNER_IMAGE_ID_STATE" ]] || fail 'stopped-runner rollback retained a runner tag that was previously absent' +container_rm_line=$(grep -n -m1 '^container-rm|' "$FAKE_COMPOSE_LOG" | cut -d: -f1 || true) +runner_rm_line=$(grep -n -m1 "^image-rm|$FAKE_RUNNER_IMAGE$" "$FAKE_COMPOSE_LOG" | cut -d: -f1 || true) +rollback_up_line=$(grep -n '^up|' "$FAKE_COMPOSE_LOG" | tail -n1 | cut -d: -f1 || true) +[[ -n "$container_rm_line" && -n "$runner_rm_line" && -n "$rollback_up_line" && "$container_rm_line" -lt "$runner_rm_line" && "$runner_rm_line" -lt "$rollback_up_line" ]] || fail 'rollback did not remove the stopped candidate runner before restoring images and restarting the controller' +printf '%s\n' "$engine_ref" >"$FAKE_RUNNER_IMAGE_STATE" +printf '%s\n' "$prior_runner_image_id" >"$FAKE_RUNNER_IMAGE_ID_STATE" +unset FAKE_ALL_RUNNER_STATE FAKE_COMPOSE_LOG +[[ ${CI_FLEET_TEST_STOP_AFTER_STOPPED_RUNNER_ROLLBACK:-0} != 1 ]] || { printf 'STOPPED_RUNNER_ROLLBACK_REGRESSION_OK\n'; exit 0; } +restartable_tag_build_output=$tmp/restartable-tag-build.out +export FAKE_COMPOSE_LOG=$tmp/restartable-tag-build-compose.log +: >"$FAKE_COMPOSE_LOG" +printf 'exited\n' >"$FAKE_CONTROLLER_STATUS_FILE" +prior_runner_image_id=$(<"$FAKE_RUNNER_IMAGE_ID_STATE") +prior_controller_image_id=$(<"$FAKE_CONTROLLER_IMAGE_ID_STATE") +export FAKE_PARTIAL_BUILD_FAIL=1 +if "$installer" --upgrade "${base_args[@]}" --ref "$ref_two" >"$restartable_tag_build_output" 2>&1; then + fail 'restartable-tag candidate build failure unexpectedly succeeded' +fi +unset FAKE_PARTIAL_BUILD_FAIL +stop_line=$(grep -n -m1 '^stop|' "$FAKE_COMPOSE_LOG" | cut -d: -f1 || true) +build_line=$(grep -n -m1 '^build|' "$FAKE_COMPOSE_LOG" | cut -d: -f1 || true) +[[ -n "$stop_line" && -n "$build_line" && "$stop_line" -lt "$build_line" ]] || fail 'restartable controller runner tag was built before stop' +grep -Fq 'DRAIN_OK managed_runners=0' "$restartable_tag_build_output" || fail 'restartable controller runner tag build did not wait for drain' +grep -Fq 'ROLLBACK_RESTORED' "$restartable_tag_build_output" || fail 'restartable controller runner tag build failure did not restore the checkpoint' +[[ -f "$FAKE_DOCKER_STATE" && ! -f "$FAKE_CONTROLLER_STATUS_FILE" ]] || fail 'restartable controller runner tag build failure did not restore the installed controller' +grep -Fxq "CI_FLEET_RUNNER_IMAGE=$FAKE_RUNNER_IMAGE" "$rendered_env" || fail 'restartable controller runner tag build failure changed installed environment' +grep -Fxq "CI_FLEET_RUNNER_IMAGE=$FAKE_RUNNER_IMAGE" "$FAKE_CONTROLLER_ENV_FILE" || fail 'restartable controller runner tag build failure did not restore the installed controller environment' +grep -Fq 'CI_FLEET_MAX_RUNNERS=1' "$rendered_env" || fail 'restartable controller runner tag build failure changed installed state' +[[ $(readlink -f "$root/opt/ci-fleet/manager/current") == "$prior_manager" ]] || fail 'restartable controller runner tag build failure changed the installed manager' +[[ $(<"$FAKE_RUNNER_IMAGE_ID_STATE") == "$prior_runner_image_id" ]] || fail 'rollback reported restored but left the runner tag on the partial build image' +[[ $(<"$FAKE_CONTROLLER_IMAGE_ID_STATE") == "$prior_controller_image_id" ]] || fail 'rollback did not restore the prior controller tag image ID' +checkpoint_path=$(awk '$1 == "CHECKPOINT_CREATED" {sub(/^path=/, "", $2); value=$2} END {print value}' "$restartable_tag_build_output") +image_ids_file=$checkpoint_path/image-ids.env +format_marker=$checkpoint_path/format-version +[[ -d "$checkpoint_path" && $(stat -c %a "$checkpoint_path") == 700 ]] || fail 'image rollback checkpoint is not a mode-0700 directory' +[[ -f "$format_marker" && ! -L "$format_marker" && $(stat -c '%u:%a:%s' "$format_marker") == "$(id -u):600:2" && $(<"$format_marker") == 2 ]] || fail 'managed checkpoint lacks the exact root-owned mode-0600 format marker' +[[ -f "$image_ids_file" && $(stat -c %a "$image_ids_file") == 600 && $(wc -l <"$image_ids_file") == 2 ]] || fail 'checkpoint image map is not exactly one mode-0600 two-ID file' +grep -Fxq "CI_FLEET_RUNNER_IMAGE_ID=$prior_runner_image_id" "$image_ids_file" || fail 'checkpoint omitted the prior runner image ID' +grep -Fxq "CI_FLEET_CONTROLLER_IMAGE_ID=$prior_controller_image_id" "$image_ids_file" || fail 'checkpoint omitted the prior controller image ID' +runner_tag_line=$(grep -n -m1 "^image-tag|$prior_runner_image_id|$FAKE_RUNNER_IMAGE$" "$FAKE_COMPOSE_LOG" | cut -d: -f1 || true) +controller_tag_line=$(grep -n -m1 "^image-tag|$prior_controller_image_id|$FAKE_CONTROLLER_IMAGE$" "$FAKE_COMPOSE_LOG" | cut -d: -f1 || true) +rollback_up_line=$(grep -n '^up|' "$FAKE_COMPOSE_LOG" | tail -n1 | cut -d: -f1 || true) +[[ -n "$runner_tag_line" && -n "$controller_tag_line" && -n "$rollback_up_line" && "$runner_tag_line" -lt "$rollback_up_line" && "$controller_tag_line" -lt "$rollback_up_line" ]] || fail 'rollback started the prior controller before restoring both image tags' +[[ ${CI_FLEET_TEST_STOP_AFTER_RESTARTABLE_TAG_BUILD:-0} != 1 ]] || { printf 'RESTARTABLE_TAG_BUILD_REGRESSION_OK\n'; exit 0; } +printf 'CI_FLEET_RUNNER_IMAGE_ID=invalid\nCI_FLEET_CONTROLLER_IMAGE_ID=%s\n' "$prior_controller_image_id" >"$image_ids_file" +: >"$FAKE_COMPOSE_LOG" +rm -f "$FAKE_DOCKER_STATE" "$FAKE_CONTROLLER_STATUS_FILE" "$FAKE_CONTROLLER_PROVENANCE_FILE" "$FAKE_CONTROLLER_IMAGE_ID_FILE" "$FAKE_CONTROLLER_ENV_FILE" +expect_failure 'checkpoint image mappings are invalid' "$installer" --rollback +if grep -q '^up|' "$FAKE_COMPOSE_LOG"; then fail 'malformed checkpoint image ID restarted the prior controller'; fi +[[ ! -f "$FAKE_DOCKER_STATE" ]] || fail 'malformed checkpoint image ID restored the prior controller' +printf 'CI_FLEET_RUNNER_IMAGE_ID=%s\nCI_FLEET_CONTROLLER_IMAGE_ID=%s\n' "$prior_runner_image_id" "$prior_controller_image_id" >"$image_ids_file" +expect_success "$installer" --rollback >/dev/null +[[ -f "$FAKE_DOCKER_STATE" ]] || fail 'repaired checkpoint did not restore the prior controller' +legacy_checkpoint=$root/var/lib/ci-fleet/checkpoints/legacy-checkpoint +cp -a "$checkpoint_path" "$legacy_checkpoint" +rm -f "$legacy_checkpoint/format-version" "$legacy_checkpoint/image-ids.env" +touch "$legacy_checkpoint/.complete" +: >"$FAKE_COMPOSE_LOG" +legacy_output=$tmp/legacy-checkpoint.out +if ! "$installer" --rollback >"$legacy_output" 2>&1; then + fail "legacy checkpoint was rejected: $(<"$legacy_output")" +fi +grep -Fq 'ROLLBACK_LEGACY_IMAGE_STATE_UNVERIFIED' "$legacy_output" || fail 'legacy rollback omitted its image-state warning' +if grep -Eq '^image-(tag|rm)\|' "$FAKE_COMPOSE_LOG"; then fail 'legacy rollback claimed or changed image identity'; fi +printf '3\n' >"$legacy_checkpoint/format-version" +chmod 0600 "$legacy_checkpoint/format-version" +: >"$FAKE_COMPOSE_LOG" +expect_failure 'checkpoint format marker is invalid' "$installer" --rollback +if [[ ! -f "$FAKE_DOCKER_STATE" ]] || grep -Eq '^(stop|up|image-(tag|rm))\|' "$FAKE_COMPOSE_LOG"; then fail 'unknown checkpoint version caused rollback side effects'; fi +printf '2\n' >"$legacy_checkpoint/format-version" +: >"$FAKE_COMPOSE_LOG" +expect_failure 'checkpoint image mappings are invalid' "$installer" --rollback +if [[ ! -f "$FAKE_DOCKER_STATE" ]] || grep -Eq '^(stop|up|image-(tag|rm))\|' "$FAKE_COMPOSE_LOG"; then fail 'version-2 checkpoint without image state caused rollback side effects'; fi +rm -rf "$legacy_checkpoint" +unset FAKE_COMPOSE_LOG +[[ ${CI_FLEET_TEST_STOP_AFTER_CHECKPOINT_COMPAT:-0} != 1 ]] || { printf 'CHECKPOINT_COMPAT_REGRESSIONS_OK\n'; exit 0; } +[[ ${CI_FLEET_TEST_STOP_AFTER_IMAGE_ROLLBACK:-0} != 1 ]] || { printf 'IMAGE_ROLLBACK_REGRESSION_OK\n'; exit 0; } +terminate_upgrade() { + local output=$1 marker=$2 second_term_marker=${3:-} pid status=0 attempt + export CI_FLEET_TEST_PAUSE_AFTER_DRAIN_FILE=$marker + if [[ -n "$second_term_marker" ]]; then + : >"$second_term_marker" + export FAKE_DELAY_UP_ONCE=$second_term_marker + fi + "$installer" --upgrade "${base_args[@]}" --ref "$ref_two" >"$output" 2>&1 & + pid=$! + for ((attempt = 0; attempt < 200; attempt++)); do + [[ ! -f "$marker" ]] || break + sleep 0.05 + done + if [[ ! -f "$marker" ]]; then + kill -TERM "$pid" 2>/dev/null || true + wait "$pid" 2>/dev/null || true + fail 'TERM regression did not reach the drained transaction' + fi + kill -TERM "$pid" + if [[ -n "$second_term_marker" ]]; then + for ((attempt = 0; attempt < 200; attempt++)); do + [[ ! -f "${second_term_marker}.entered" ]] || break + sleep 0.05 + done + [[ -f "${second_term_marker}.entered" ]] || fail 'TERM regression did not enter checkpoint restoration' + kill -TERM "$pid" + fi + if wait "$pid"; then status=0; else status=$?; fi + unset CI_FLEET_TEST_PAUSE_AFTER_DRAIN_FILE FAKE_DELAY_UP_ONCE + [[ "$status" == 143 ]] || fail "TERM regression changed the signal exit status: $status" +} +paused_term_output=$tmp/paused-term-rollback.out +export FAKE_RUNNER_STATE=$tmp/paused-term-managed-runner +export FAKE_KEEP_RUNNER_ON_PAUSE=1 +: >"$FAKE_RUNNER_STATE" +rm -f "$FAKE_PAUSED_STATE" +"$installer" --upgrade "${base_args[@]}" --ref "$ref_two" >"$paused_term_output" 2>&1 & +paused_term_pid=$! +for ((attempt = 0; attempt < 200; attempt++)); do + [[ ! -f "$FAKE_PAUSED_STATE" ]] || break + sleep 0.05 +done +if [[ ! -f "$FAKE_PAUSED_STATE" ]]; then + kill -TERM "$paused_term_pid" 2>/dev/null || true + wait "$paused_term_pid" 2>/dev/null || true + fail 'TERM regression did not reach the paused drain phase' +fi +if grep -Fq 'DRAIN_OK managed_runners=0' "$paused_term_output"; then fail 'TERM regression passed the drain phase before signaling'; fi +kill -TERM "$paused_term_pid" +rm -f "$FAKE_RUNNER_STATE" +if wait "$paused_term_pid"; then paused_term_status=0; else paused_term_status=$?; fi +unset FAKE_KEEP_RUNNER_ON_PAUSE FAKE_RUNNER_STATE +[[ "$paused_term_status" == 143 ]] || fail "paused TERM regression changed the signal exit status: $paused_term_status" +grep -Fq 'ROLLBACK_RESTORED' "$paused_term_output" || fail "paused TERM did not report checkpoint restoration: $(<"$paused_term_output")" +[[ ! -f "$FAKE_PAUSED_STATE" && -f "$FAKE_DOCKER_STATE" ]] || fail 'TERM before DRAIN_OK did not restore the active controller' +grep -Fq 'CI_FLEET_MAX_RUNNERS=1' "$rendered_env" || fail 'TERM before DRAIN_OK did not restore installed state' +[[ $(readlink -f "$root/opt/ci-fleet/manager/current") == "$prior_manager" ]] || fail 'TERM before DRAIN_OK did not restore the prior manager release' +[[ ${CI_FLEET_TEST_STOP_AFTER_PAUSED_TERM:-0} != 1 ]] || { printf 'PAUSED_TERM_REGRESSION_OK\n'; exit 0; } +term_output=$tmp/term-rollback.out +terminate_upgrade "$term_output" "$tmp/term-pause" "$tmp/term-rollback-up" +grep -Fq 'ROLLBACK_RESTORED' "$term_output" || fail "TERM did not report checkpoint restoration: $(<"$term_output")" +[[ ! -f "$FAKE_PAUSED_STATE" && -f "$FAKE_DOCKER_STATE" ]] || fail 'TERM did not restore the active controller' +grep -Fq 'CI_FLEET_MAX_RUNNERS=1' "$rendered_env" || fail 'TERM did not restore installed state' +term_failure_output=$tmp/term-rollback-failure.out +export FAKE_FAIL_UP_ONCE=$tmp/term-rollback-fail-up +: >"$FAKE_FAIL_UP_ONCE" +terminate_upgrade "$term_failure_output" "$tmp/term-failure-pause" +unset FAKE_FAIL_UP_ONCE +grep -Fq 'ROLLBACK_FAILED' "$term_failure_output" || fail 'TERM rollback failure was not reported' +expect_success "$installer" --rollback >/dev/null +[[ -f "$FAKE_DOCKER_STATE" ]] || fail 'explicit rollback did not recover after TERM rollback failure' +[[ ${CI_FLEET_TEST_STOP_AFTER_TERM_ROLLBACK:-0} != 1 ]] || { printf 'TERM_ROLLBACK_REGRESSION_OK\n'; exit 0; } +managed_preflight_output=$tmp/managed-preflight.out +export FAKE_ACTIVE_MANAGED_STATE=$tmp/active-managed-after-drain +export FAKE_ACTIVE_MANAGED_AFTER_STOP=$FAKE_ACTIVE_MANAGED_STATE +if "$installer" --upgrade "${base_args[@]}" --ref "$ref_two" >"$managed_preflight_output" 2>&1; then + fail 'managed candidate preflight skipped an active managed container after drain' +fi +unset FAKE_ACTIVE_MANAGED_AFTER_STOP FAKE_ACTIVE_MANAGED_STATE +grep -Fq 'DRAIN_OK managed_runners=0' "$managed_preflight_output" || fail 'managed candidate preflight ran before drain completed' +grep -Fq 'managed containers already active for this instance' "$managed_preflight_output" || fail 'managed candidate preflight did not check active managed containers' +grep -Fq 'ROLLBACK_RESTORED' "$managed_preflight_output" || fail 'managed candidate preflight failure did not restore the checkpoint' +rm -f "$tmp/active-managed-after-drain" +[[ ${CI_FLEET_TEST_STOP_AFTER_MANAGED_PREFLIGHT:-0} != 1 ]] || { printf 'MANAGED_PREFLIGHT_REGRESSION_OK\n'; exit 0; } export FAKE_FAIL_KILL_ONCE=$tmp/fail-kill-once : >"$FAKE_FAIL_KILL_ONCE" expect_failure 'failed to signal the paused controller' "$installer" --upgrade "${base_args[@]}" --ref "$ref_two" @@ -628,7 +1142,7 @@ export FAKE_RUNNER_STATE=$tmp/managed-runner-active export FAKE_FAIL_UP_ONCE=$tmp/fail-up-once : >"$FAKE_FAIL_UP_ONCE" expect_failure 'ROLLBACK_RESTORED' "$installer" --upgrade "${base_args[@]}" --ref "$ref_two" -[[ ! -f "$FAKE_RUNNER_STATE" ]] || fail 'upgrade preflight ran before the active runner was drained' +[[ ! -f "$FAKE_RUNNER_STATE" ]] || fail 'upgrade did not drain the active runner before activation' unset FAKE_RUNNER_STATE unset FAKE_FAIL_UP_ONCE grep -Fq 'CI_FLEET_MAX_RUNNERS=1' "$root/etc/ci-fleet/ci-fleet.env" || fail 'failed activation did not restore capacity one' @@ -655,12 +1169,19 @@ expect_success "$installer" --rollback >/dev/null grep -Fq 'CI_FLEET_MAX_RUNNERS=1' "$root/etc/ci-fleet/ci-fleet.env" || fail 'rollback did not restore capacity one' ref_three=$(write_config drained 2 2) +missing_current_output=$tmp/missing-current-restartable.out +export FAKE_COMPOSE_LOG=$tmp/missing-current-restartable-compose.log +: >"$FAKE_COMPOSE_LOG" +rm -f "$root/opt/ci-fleet/current" printf 'dead\n' >"$FAKE_CONTROLLER_STATUS_FILE" -export FAKE_STOP_FAIL=$tmp/stop-dead-fails -: >"$FAKE_STOP_FAIL" -expect_success "$installer" --upgrade "${base_args[@]}" --ref "$ref_three" >/dev/null -unset FAKE_STOP_FAIL +if ! "$installer" --upgrade "${base_args[@]}" --ref "$ref_three" >"$missing_current_output" 2>&1; then + grep -Fq 'cannot stop restartable controller state without its runtime release: dead' "$missing_current_output" || fail "missing-current convergence failed unexpectedly: $(<"$missing_current_output")" + if grep -Eq '^(stop|build|up|down|rm)\|' "$FAKE_COMPOSE_LOG"; then fail 'missing-current convergence mutated the controller before validating a drain release'; fi + fail "missing-current convergence did not use the validated candidate release: $(<"$missing_current_output")" +fi +unset FAKE_COMPOSE_LOG [[ ! -f "$FAKE_DOCKER_STATE" && ! -f "$FAKE_CONTROLLER_STATUS_FILE" ]] || fail 'non-active convergence retained a dead controller' +[[ $(readlink -f "$root/opt/ci-fleet/current") == "$root/opt/ci-fleet/releases/$engine_ref" ]] || fail 'missing-current convergence did not repair the current release link' grep -Fq 'CI_FLEET_CONTROLLER_STATE=drained' "$root/etc/ci-fleet/ci-fleet.env" || fail 'drained state was not rendered' grep -Fq 'CI_FLEET_MAX_RUNNERS=0' "$root/etc/ci-fleet/ci-fleet.env" || fail 'drained controller retained effective capacity' [[ ! -f "$FAKE_DOCKER_STATE" ]] || fail 'drained controller remained running' @@ -695,16 +1216,30 @@ export FAKE_ALL_RUNNER_STATE=$tmp/uninstall-stopped-managed-runner mkdir -p "$root/var/lib/ci-fleet/health" printf '{"status":"healthy"}\n' >"$root/var/lib/ci-fleet/health/latest.json" : >"$FAKE_DOCKER_PS_LOG" -expect_success "$installer" --uninstall >/dev/null +uninstall_output=$tmp/dangling-current-uninstall.out +export FAKE_COMPOSE_LOG=$tmp/dangling-current-uninstall-compose.log +: >"$FAKE_COMPOSE_LOG" +export FAKE_STOPPED_CONTROLLER_STATE=$tmp/uninstall-created-controller +: >"$FAKE_STOPPED_CONTROLLER_STATE" +printf 'created\n' >"$FAKE_CONTROLLER_STATUS_FILE" +ln -sfn "$root/opt/ci-fleet/releases/missing/release" "$root/opt/ci-fleet/current" +if ! "$installer" --uninstall >"$uninstall_output" 2>&1; then + grep -Fq 'cannot stop restartable controller state without its runtime release: created' "$uninstall_output" || fail "dangling-current uninstall failed unexpectedly: $(<"$uninstall_output")" + if grep -Eq '^(stop|build|up|down|rm)\|' "$FAKE_COMPOSE_LOG"; then fail 'dangling-current uninstall mutated the controller before validating a drain release'; fi + fail "dangling-current uninstall did not use a validated installed release: $(<"$uninstall_output")" +fi +unset FAKE_STOPPED_CONTROLLER_STATE FAKE_COMPOSE_LOG +grep -Fq 'UNINSTALL_OK' "$uninstall_output" || fail 'dangling-current uninstall did not complete' [[ ! -f "$FAKE_RUNNER_STATE_ONCE" ]] || fail 'uninstall did not wait for an orphaned managed runner' [[ ! -f "$FAKE_ALL_RUNNER_STATE" ]] || fail 'uninstall retained stopped managed runners' grep -Fq 'label=io.randomdevelopment.ci-fleet.instance=example-ci-01' "$FAKE_DOCKER_PS_LOG" || fail 'uninstall runner cleanup was not scoped to the installed instance' if grep -Eq 'label=io.randomdevelopment.ci-fleet.instance=$' "$FAKE_DOCKER_PS_LOG"; then fail 'uninstall runner cleanup used an empty instance filter'; fi unset FAKE_RUNNER_STATE_ONCE FAKE_ALL_RUNNER_STATE -[[ ! -e "$root/opt/ci-fleet/current" && ! -e "$root/var/lib/ci-fleet/install-state.json" ]] || fail 'uninstall left active installation state' +[[ ! -e "$root/opt/ci-fleet/current" && ! -L "$root/opt/ci-fleet/current" && ! -e "$root/var/lib/ci-fleet/install-state.json" ]] || fail 'uninstall left active installation state' [[ -f "$host_config" && -f "$pem" ]] || fail 'uninstall removed preserved host credentials' [[ -f "$root/etc/ci-fleet/monitoring.env" ]] || fail 'uninstall removed host-local monitoring configuration' [[ ! -e "$root/var/lib/ci-fleet/health" ]] || fail 'uninstall retained fleet-owned health state' +[[ ${CI_FLEET_TEST_STOP_AFTER_CURRENT_LINK_FALLBACK:-0} != 1 ]] || { printf 'CURRENT_LINK_FALLBACK_REGRESSIONS_OK\n'; exit 0; } adopt_root=$tmp/adopt-host export CI_FLEET_ROOT_PREFIX=$adopt_root @@ -728,6 +1263,8 @@ printf '%s\n' \ "CI_FLEET_GITHUB_APP_PRIVATE_KEY_FILE=$adopt_pem" \ 'CI_FLEET_RUNNER_TTL=6h' \ 'CI_FLEET_CONTROLLER_STATE=active' \ + "CI_FLEET_RUNNER_IMAGE=$FAKE_RUNNER_IMAGE" \ + "CI_FLEET_CONTROLLER_IMAGE=$FAKE_CONTROLLER_IMAGE" \ 'CI_FLEET_INSTANCE=legacy-ci-01' >"$adopt_root/etc/ci-fleet/ci-fleet.env" chmod 600 "$adopt_root/etc/ci-fleet/ci-fleet.env" printf 'CI_FLEET_HEALTH_DISK_WARN_PERCENT=75\n' >"$adopt_root/etc/ci-fleet/monitoring.env" @@ -763,6 +1300,8 @@ expect_failure 'selected engine does not support status reporting configuration' legacy_required_ref=$(write_config active 1 1 "$legacy_engine_ref" true omit) expect_failure 'selected engine does not advertise required status reporting' "$installer" --upgrade "${base_args[@]}" --ref "$legacy_required_ref" legacy_ref=$(write_config active 1 1 "$legacy_engine_ref" omit omit) +export FAKE_PREVIOUS_RUNNER_IMAGE=$FAKE_RUNNER_IMAGE +export FAKE_PREVIOUS_CONTROLLER_IMAGE=$FAKE_CONTROLLER_IMAGE export FAKE_ENGINE_REF=$legacy_engine_ref export FAKE_RUNNER_IMAGE=ci-fleet-runner:${legacy_engine_ref:0:12} export FAKE_CONTROLLER_IMAGE=ci-fleet-controller:${legacy_engine_ref:0:12} diff --git a/scripts/test_remote_reconcile.py b/scripts/test_remote_reconcile.py index 75a1b12c..d92e206f 100644 --- a/scripts/test_remote_reconcile.py +++ b/scripts/test_remote_reconcile.py @@ -283,6 +283,15 @@ def test_service_file_exists(self): self.assertIn("ExecStart", content) self.assertIn("remote-reconcile.sh", content) + def test_service_timeouts_cover_lock_build_and_rollback(self): + svc = REPO_ROOT / "host" / "systemd" / "ci-fleet-reconcile.service" + content = svc.read_text() + self.assertIn("lock_wait_seconds=3600", RECONCILE_SCRIPT.read_text()) + self.assertIn("TimeoutStartSec=2h", content) + self.assertIn("One hour for the installer lock, plus one hour for a cold build and transaction.", content) + self.assertIn("TimeoutStopSec=15min", content) + self.assertIn("Allow checkpoint restoration after TimeoutStartSec sends SIGTERM.", content) + def test_timer_file_exists(self): timer = REPO_ROOT / "host" / "systemd" / "ci-fleet-reconcile.timer" self.assertTrue(timer.exists())