From 6a12516ac9d1c6a1c67af78f094b6687662cf4ca Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Sat, 15 Aug 2026 19:45:35 -0500 Subject: [PATCH 1/4] fix: complete worker host preflight --- docs/DESIRED-STATE.md | 2 ++ scripts/install-worker-controller.sh | 19 +++++++++++- scripts/test-install-worker-controller.sh | 37 +++++++++++++++++++++-- 3 files changed, 54 insertions(+), 4 deletions(-) diff --git a/docs/DESIRED-STATE.md b/docs/DESIRED-STATE.md index dca40f5c..dbc6fa1e 100644 --- a/docs/DESIRED-STATE.md +++ b/docs/DESIRED-STATE.md @@ -62,6 +62,8 @@ GitHub App and runner-group creation remain the bootstrap responsibility tracked ## Install a fresh controller +The managed installer supports Debian 12 or newer. Before reading configuration or changing the host, it verifies Docker Engine and Compose v2, Git, curl, jq, the system CA bundle, direct Docker-socket access, and that the Docker filesystem is below the documented 80% warning threshold. + Run the command from a reviewed checkout of ci-fleet on the target Linux Docker machine: ```bash diff --git a/scripts/install-worker-controller.sh b/scripts/install-worker-controller.sh index ad156566..e064d82a 100755 --- a/scripts/install-worker-controller.sh +++ b/scripts/install-worker-controller.sh @@ -135,12 +135,29 @@ cleanup_temporary() { trap cleanup_temporary EXIT require_commands() { - local command + local command docker_root disk_used os_id os_release os_version socket for command in git python3 docker tar install cmp readlink systemctl stat awk grep date flock mktemp; do command -v "$command" >/dev/null || die "$command is required" done docker info >/dev/null 2>&1 || die 'Docker daemon is unavailable' docker compose version >/dev/null 2>&1 || die 'Docker Compose v2 is unavailable' + [[ "$mode" == rollback || "$mode" == uninstall ]] && return + + for command in curl jq df; do command -v "$command" >/dev/null || die "$command is required"; done + os_release=$(root_path /etc/os-release) + [[ -r "$os_release" ]] || die 'supported Linux release metadata is unavailable' + os_id=$(awk -F= '$1 == "ID" {gsub(/"/, "", $2); print $2}' "$os_release") + os_version=$(awk -F= '$1 == "VERSION_ID" {gsub(/"/, "", $2); print $2}' "$os_release") + [[ "$os_id" == debian && "$os_version" =~ ^[0-9]+$ ]] || die 'supported Linux is Debian 12 or newer' + ((10#$os_version >= 12)) || die 'supported Linux is Debian 12 or newer' + [[ -r $(root_path /etc/ssl/certs/ca-certificates.crt) ]] || die 'CA certificate bundle is unavailable' + socket=$(root_path /var/run/docker.sock) + [[ -S "$socket" && -r "$socket" && -w "$socket" || "$testing" == 1 && -e "$socket" ]] || die 'Docker socket is unavailable or inaccessible' + docker_root=$(docker info --format '{{.DockerRootDir}}' 2>/dev/null) || die 'Docker root directory is unavailable' + [[ "$docker_root" == /* ]] || die 'Docker root directory is invalid' + disk_used=$(df -P "$docker_root" 2>/dev/null | awk 'NR == 2 {gsub(/%/, "", $5); print $5}') + [[ "$disk_used" =~ ^[0-9]{1,3}$ ]] || die 'Docker disk capacity could not be determined' + ((disk_used < 80)) || die 'Docker filesystem must remain below 80% utilization' } validate_common_arguments() { diff --git a/scripts/test-install-worker-controller.sh b/scripts/test-install-worker-controller.sh index 445a77d0..3b484b6b 100755 --- a/scripts/test-install-worker-controller.sh +++ b/scripts/test-install-worker-controller.sh @@ -16,6 +16,8 @@ export REAL_TAR REAL_TAR=$(command -v tar) export REAL_GIT REAL_GIT=$(command -v git) +export REAL_DF +REAL_DF=$(command -v df) cat >"$fake_bin/docker" <<'EOF' #!/usr/bin/env bash @@ -24,7 +26,10 @@ state=${FAKE_DOCKER_STATE:?} status_file=${FAKE_CONTROLLER_STATUS_FILE:-} paused_state=${FAKE_PAUSED_STATE:-} case "${1:-}" in - info) exit 0 ;; + info) + [[ "$*" != *DockerRootDir* ]] || printf '%s\n' "${CI_FLEET_DOCKER_ROOT:?}" + exit 0 + ;; inspect) [[ -f "$state" ]] || exit 1 if [[ "$*" == *'.Config.Env'* ]]; then @@ -185,6 +190,17 @@ exec "$REAL_GIT" "$@" EOF chmod 700 "$fake_bin/git" +cat >"$fake_bin/df" <<'EOF' +#!/usr/bin/env bash +if [[ -n ${FAKE_DISK_USED_PERCENT:-} ]]; then + printf 'Filesystem 1024-blocks Used Available Capacity Mounted on\n' + printf 'fixture 100 90 10 %s%% /fixture\n' "$FAKE_DISK_USED_PERCENT" + exit 0 +fi +exec "$REAL_DF" "$@" +EOF +chmod 700 "$fake_bin/df" + export PATH="$fake_bin:$PATH" export FAKE_DOCKER_STATE=$tmp/docker-controller-running export FAKE_CONTROLLER_STATUS_FILE=$tmp/docker-controller-status @@ -285,7 +301,10 @@ PY root=$tmp/host export CI_FLEET_ROOT_PREFIX=$root export CI_FLEET_DOCKER_ROOT=$root/var/lib/docker -mkdir -p "$root/etc/ci-fleet/secrets" "$CI_FLEET_DOCKER_ROOT" +mkdir -p "$root/etc/ci-fleet/secrets" "$root/etc/ssl/certs" "$root/var/run" "$CI_FLEET_DOCKER_ROOT" +printf 'ID=debian\nVERSION_ID="12"\n' >"$root/etc/os-release" +printf 'fixture CA bundle\n' >"$root/etc/ssl/certs/ca-certificates.crt" +: >"$root/var/run/docker.sock" pem=$root/etc/ci-fleet/secrets/github-app.pem printf 'fixture only\n' >"$pem" chmod 600 "$pem" @@ -311,10 +330,19 @@ git -C "$config_repo" reset -q --hard "$ref_one" installer=$repo_root/scripts/install-worker-controller.sh base_args=(--config-repo "$config_repo" --controller example-ci-01) +printf 'ID=example\nVERSION_ID="1"\n' >"$root/etc/os-release" +expect_failure 'supported Linux is Debian 12 or newer' "$installer" --check "${base_args[@]}" --ref "$ref_one" +printf 'ID=debian\nVERSION_ID="12"\n' >"$root/etc/os-release" +export FAKE_DISK_USED_PERCENT=80 +expect_failure 'Docker filesystem must remain below 80% utilization' "$installer" --check "${base_args[@]}" --ref "$ref_one" +unset FAKE_DISK_USED_PERCENT + staged_checkpoint="$root/var/lib/ci-fleet/checkpoints/.checkpoint.staging.interrupted" mkdir -p "$staged_checkpoint" : >"$staged_checkpoint/.complete" +mv "$root/etc/os-release" "$root/etc/os-release.missing" expect_failure 'no controller checkpoint is available' "$installer" --rollback +mv "$root/etc/os-release.missing" "$root/etc/os-release" rm -rf "$staged_checkpoint" expect_failure 'secret-bearing files are forbidden' "$installer" --check "${base_args[@]}" --ref "$forbidden_ref" expect_failure 'possible committed secret detected' "$installer" --check "${base_args[@]}" --ref "$secret_ref" @@ -636,7 +664,10 @@ unset FAKE_RUNNER_STATE_ONCE FAKE_ALL_RUNNER_STATE adopt_root=$tmp/adopt-host export CI_FLEET_ROOT_PREFIX=$adopt_root export FAKE_DOCKER_STATE=$tmp/adopt-controller-running -mkdir -p "$adopt_root/etc/ci-fleet/secrets" "$adopt_root/opt/ci-fleet/deploy" "$adopt_root/opt/ci-fleet/scripts" +mkdir -p "$adopt_root/etc/ci-fleet/secrets" "$adopt_root/etc/ssl/certs" "$adopt_root/var/run" "$adopt_root/opt/ci-fleet/deploy" "$adopt_root/opt/ci-fleet/scripts" +printf 'ID=debian\nVERSION_ID="12"\n' >"$adopt_root/etc/os-release" +printf 'fixture CA bundle\n' >"$adopt_root/etc/ssl/certs/ca-certificates.crt" +: >"$adopt_root/var/run/docker.sock" adopt_pem=$adopt_root/etc/ci-fleet/secrets/github-app.pem printf 'fixture only\n' >"$adopt_pem" chmod 600 "$adopt_pem" From 181efe0e899d318d23c970903ff954cfb6f33d35 Mon Sep 17 00:00:00 2001 From: Nicks Hermes Date: Sat, 15 Aug 2026 20:25:31 -0500 Subject: [PATCH 2/4] fix: bind worker recovery to local Docker --- docs/DESIRED-STATE.md | 2 +- scripts/install-worker-controller.sh | 11 +++++++++-- scripts/test-install-worker-controller.sh | 2 ++ 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/docs/DESIRED-STATE.md b/docs/DESIRED-STATE.md index dbc6fa1e..4779af4b 100644 --- a/docs/DESIRED-STATE.md +++ b/docs/DESIRED-STATE.md @@ -62,7 +62,7 @@ GitHub App and runner-group creation remain the bootstrap responsibility tracked ## Install a fresh controller -The managed installer supports Debian 12 or newer. Before reading configuration or changing the host, it verifies Docker Engine and Compose v2, Git, curl, jq, the system CA bundle, direct Docker-socket access, and that the Docker filesystem is below the documented 80% warning threshold. +The managed installer supports Debian 12 or newer. Before reading configuration or changing the host, it verifies Docker Engine and Compose v2, Git, curl, jq, the system CA bundle, direct Docker-socket access, and that the Docker filesystem is below the documented 80% warning threshold. It rejects alternate Docker endpoints and contexts, then pins every lifecycle command to the verified local Unix socket. Rollback and uninstall require only their recovery tools, not Git, tar, cmp, host-release metadata, CA bootstrap, or capacity checks. Run the command from a reviewed checkout of ci-fleet on the target Linux Docker machine: diff --git a/scripts/install-worker-controller.sh b/scripts/install-worker-controller.sh index e064d82a..9f4455e9 100755 --- a/scripts/install-worker-controller.sh +++ b/scripts/install-worker-controller.sh @@ -136,9 +136,17 @@ trap cleanup_temporary EXIT require_commands() { local command docker_root disk_used os_id os_release os_version socket - for command in git python3 docker tar install cmp readlink systemctl stat awk grep date flock mktemp; do + local -a required=(python3 docker install readlink systemctl stat awk grep date flock mktemp) + if [[ "$mode" != rollback && "$mode" != uninstall ]]; then required+=(git tar cmp); fi + for command in "${required[@]}"; do command -v "$command" >/dev/null || die "$command is required" done + socket=$(root_path /var/run/docker.sock) + [[ -z ${DOCKER_HOST:-} || ${DOCKER_HOST} == "unix://$socket" ]] || die 'alternate Docker endpoints are not supported; use the local Docker socket' + [[ -z ${DOCKER_CONTEXT:-} ]] || die 'alternate Docker contexts are not supported; use the local Docker socket' + DOCKER_HOST=unix://$socket + export DOCKER_HOST + unset DOCKER_CONTEXT DOCKER_TLS_VERIFY DOCKER_CERT_PATH docker info >/dev/null 2>&1 || die 'Docker daemon is unavailable' docker compose version >/dev/null 2>&1 || die 'Docker Compose v2 is unavailable' [[ "$mode" == rollback || "$mode" == uninstall ]] && return @@ -151,7 +159,6 @@ require_commands() { [[ "$os_id" == debian && "$os_version" =~ ^[0-9]+$ ]] || die 'supported Linux is Debian 12 or newer' ((10#$os_version >= 12)) || die 'supported Linux is Debian 12 or newer' [[ -r $(root_path /etc/ssl/certs/ca-certificates.crt) ]] || die 'CA certificate bundle is unavailable' - socket=$(root_path /var/run/docker.sock) [[ -S "$socket" && -r "$socket" && -w "$socket" || "$testing" == 1 && -e "$socket" ]] || die 'Docker socket is unavailable or inaccessible' docker_root=$(docker info --format '{{.DockerRootDir}}' 2>/dev/null) || die 'Docker root directory is unavailable' [[ "$docker_root" == /* ]] || die 'Docker root directory is invalid' diff --git a/scripts/test-install-worker-controller.sh b/scripts/test-install-worker-controller.sh index 3b484b6b..51d6899b 100755 --- a/scripts/test-install-worker-controller.sh +++ b/scripts/test-install-worker-controller.sh @@ -330,6 +330,8 @@ git -C "$config_repo" reset -q --hard "$ref_one" installer=$repo_root/scripts/install-worker-controller.sh base_args=(--config-repo "$config_repo" --controller example-ci-01) +expect_failure 'alternate Docker endpoints are not supported' env DOCKER_HOST=tcp://example.invalid:2376 "$installer" --check "${base_args[@]}" --ref "$ref_one" +expect_failure 'alternate Docker contexts are not supported' env DOCKER_CONTEXT=remote "$installer" --check "${base_args[@]}" --ref "$ref_one" printf 'ID=example\nVERSION_ID="1"\n' >"$root/etc/os-release" expect_failure 'supported Linux is Debian 12 or newer' "$installer" --check "${base_args[@]}" --ref "$ref_one" printf 'ID=debian\nVERSION_ID="12"\n' >"$root/etc/os-release" From 04c3f38a1ac750e4871ce745ed89be6df63a2388 Mon Sep 17 00:00:00 2001 From: Nick's Hermes <1572453+Nickfost@users.noreply.github.com> Date: Thu, 27 Aug 2026 22:39:43 -0500 Subject: [PATCH 3/4] fix: require openssl before enabling remote reconciliation The installer enables ci-fleet-reconcile.timer during install, and remote reconciliation signs the GitHub App JWT with openssl (github-app-token.sh). Require openssl in the install/check preflight so the enabled timer cannot fail at runtime on a host without it. Co-Authored-By: Hermes Agent --- scripts/install-worker-controller.sh | 2 +- scripts/test-install-worker-controller.sh | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/scripts/install-worker-controller.sh b/scripts/install-worker-controller.sh index 9f4455e9..2a2b2dda 100755 --- a/scripts/install-worker-controller.sh +++ b/scripts/install-worker-controller.sh @@ -151,7 +151,7 @@ require_commands() { docker compose version >/dev/null 2>&1 || die 'Docker Compose v2 is unavailable' [[ "$mode" == rollback || "$mode" == uninstall ]] && return - for command in curl jq df; do command -v "$command" >/dev/null || die "$command is required"; done + for command in curl jq df openssl; do command -v "$command" >/dev/null || die "$command is required"; done os_release=$(root_path /etc/os-release) [[ -r "$os_release" ]] || die 'supported Linux release metadata is unavailable' os_id=$(awk -F= '$1 == "ID" {gsub(/"/, "", $2); print $2}' "$os_release") diff --git a/scripts/test-install-worker-controller.sh b/scripts/test-install-worker-controller.sh index 51d6899b..10c8e59e 100755 --- a/scripts/test-install-worker-controller.sh +++ b/scripts/test-install-worker-controller.sh @@ -202,6 +202,23 @@ EOF chmod 700 "$fake_bin/df" export PATH="$fake_bin:$PATH" + +# Build a PATH that mirrors the real one but omits openssl, so the +# installer's command-presence preflight can be exercised for the +# remote-reconciliation dependency without disturbing the rest of the test. +no_ssl_dir=$tmp/bin-no-ssl +mkdir -p "$no_ssl_dir" +IFS=: read -ra path_dirs <<< "$PATH" +for dir in "${path_dirs[@]}"; do + [[ -d "$dir" ]] || continue + for entry in "$dir"/*; do + base=$(basename "$entry") + [[ "$base" == openssl ]] && continue + [[ -e "$no_ssl_dir/$base" ]] || ln -sf "$entry" "$no_ssl_dir/$base" 2>/dev/null + done +done +export NO_SSL_PATH="$no_ssl_dir" + export FAKE_DOCKER_STATE=$tmp/docker-controller-running export FAKE_CONTROLLER_STATUS_FILE=$tmp/docker-controller-status export FAKE_PAUSED_STATE=$tmp/docker-controller-paused @@ -339,6 +356,11 @@ export FAKE_DISK_USED_PERCENT=80 expect_failure 'Docker filesystem must remain below 80% utilization' "$installer" --check "${base_args[@]}" --ref "$ref_one" unset FAKE_DISK_USED_PERCENT +# Remote reconciliation enables the ci-fleet-reconcile timer during install, +# and reconciliation signs the GitHub App JWT with openssl. Require openssl +# before install/check so the enabled timer cannot fail at runtime. +expect_failure 'openssl is required' env PATH="$NO_SSL_PATH" "$installer" --check "${base_args[@]}" --ref "$ref_one" + staged_checkpoint="$root/var/lib/ci-fleet/checkpoints/.checkpoint.staging.interrupted" mkdir -p "$staged_checkpoint" : >"$staged_checkpoint/.complete" From 70dbba48ae7e43848af8f8059b6daca14540dadc Mon Sep 17 00:00:00 2001 From: Nickfost <1572453+Nickfost@users.noreply.github.com> Date: Fri, 28 Aug 2026 01:42:27 -0500 Subject: [PATCH 4/4] fix: pin maintenance docker context and accept default --- scripts/cleanup.sh | 7 +++++++ scripts/docker-local-env.sh | 17 +++++++++++++++++ scripts/healthcheck.sh | 3 +++ scripts/install-worker-controller.sh | 2 +- scripts/test-install-worker-controller.sh | 16 ++++++++++++++++ 5 files changed, 44 insertions(+), 1 deletion(-) create mode 100644 scripts/docker-local-env.sh diff --git a/scripts/cleanup.sh b/scripts/cleanup.sh index 6ddcd4b9..b1f6682a 100755 --- a/scripts/cleanup.sh +++ b/scripts/cleanup.sh @@ -1,6 +1,11 @@ #!/usr/bin/env bash set -Eeuo pipefail +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +# shellcheck disable=SC1091 +source "$repo_root/scripts/docker-local-env.sh" +use_local_docker + apply=false instance="${CI_FLEET_INSTANCE:-}" @@ -18,6 +23,8 @@ while (($#)); do shift done +use_local_docker + command -v docker >/dev/null || { echo "ERROR docker is unavailable" >&2; exit 1; } docker info >/dev/null diff --git a/scripts/docker-local-env.sh b/scripts/docker-local-env.sh new file mode 100644 index 00000000..96da247e --- /dev/null +++ b/scripts/docker-local-env.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +use_local_docker() { + local socket=${CI_FLEET_DOCKER_SOCKET:-${CI_FLEET_ROOT_PREFIX:-}/var/run/docker.sock} + [[ -n "$socket" ]] || socket=/var/run/docker.sock + [[ -z ${DOCKER_HOST:-} || ${DOCKER_HOST} == "unix://$socket" ]] || { + printf 'ERROR: alternate Docker endpoints are not supported; use the local Docker socket\n' >&2 + return 1 + } + [[ -z ${DOCKER_CONTEXT:-} || ${DOCKER_CONTEXT} == default ]] || { + printf 'ERROR: alternate Docker contexts are not supported; use the local Docker socket\n' >&2 + return 1 + } + export DOCKER_HOST="unix://$socket" + export DOCKER_CONTEXT=default + unset DOCKER_TLS_VERIFY DOCKER_CERT_PATH DOCKER_CONFIG XDG_RUNTIME_DIR +} diff --git a/scripts/healthcheck.sh b/scripts/healthcheck.sh index c06bf63f..e02c40cf 100755 --- a/scripts/healthcheck.sh +++ b/scripts/healthcheck.sh @@ -1,6 +1,8 @@ #!/usr/bin/env bash set -Eeuo pipefail repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +# shellcheck disable=SC1091 +source "$repo_root/scripts/docker-local-env.sh" environment=/etc/ci-fleet/ci-fleet.env args=(local) if [[ ${CI_FLEET_TESTING:-0} == 1 && -n ${CI_FLEET_ROOT_PREFIX:-} ]]; then @@ -13,4 +15,5 @@ if [[ -r $environment ]]; then . "$environment" set +a fi +use_local_docker exec python3 "$repo_root/scripts/health.py" "${args[@]}" "$@" diff --git a/scripts/install-worker-controller.sh b/scripts/install-worker-controller.sh index 2a2b2dda..639f36d3 100755 --- a/scripts/install-worker-controller.sh +++ b/scripts/install-worker-controller.sh @@ -142,8 +142,8 @@ require_commands() { command -v "$command" >/dev/null || die "$command is required" done socket=$(root_path /var/run/docker.sock) + [[ -z ${DOCKER_CONTEXT:-} || ${DOCKER_CONTEXT} == default ]] || die 'alternate Docker contexts are not supported; use the local Docker socket' [[ -z ${DOCKER_HOST:-} || ${DOCKER_HOST} == "unix://$socket" ]] || die 'alternate Docker endpoints are not supported; use the local Docker socket' - [[ -z ${DOCKER_CONTEXT:-} ]] || die 'alternate Docker contexts are not supported; use the local Docker socket' DOCKER_HOST=unix://$socket export DOCKER_HOST unset DOCKER_CONTEXT DOCKER_TLS_VERIFY DOCKER_CERT_PATH diff --git a/scripts/test-install-worker-controller.sh b/scripts/test-install-worker-controller.sh index 0d4917eb..11d96935 100755 --- a/scripts/test-install-worker-controller.sh +++ b/scripts/test-install-worker-controller.sh @@ -25,6 +25,13 @@ set -u state=${FAKE_DOCKER_STATE:?} status_file=${FAKE_CONTROLLER_STATUS_FILE:-} paused_state=${FAKE_PAUSED_STATE:-} +if [[ -n ${FAKE_REQUIRE_LOCAL_DOCKER_ENDPOINT:-} ]]; then + expected_socket=${CI_FLEET_ROOT_PREFIX:-}/var/run/docker.sock + [[ ${DOCKER_HOST:-} == "unix://$expected_socket" ]] || { + printf 'expected local Docker socket %s, got %s\n' "unix://$expected_socket" "${DOCKER_HOST:-}" >&2 + exit 91 + } +fi case "${1:-}" in info) [[ "$*" != *DockerRootDir* ]] || printf '%s\n' "${CI_FLEET_DOCKER_ROOT:?}" @@ -360,6 +367,12 @@ export FAKE_DISK_USED_PERCENT=80 expect_failure 'Docker filesystem must remain below 80% utilization' "$installer" --check "${base_args[@]}" --ref "$ref_one" unset FAKE_DISK_USED_PERCENT +# The installed maintenance scripts must pin the local Docker daemon themselves, +# not just inherit it from the installer. +export FAKE_REQUIRE_LOCAL_DOCKER_ENDPOINT=1 +expect_success "$repo_root/scripts/cleanup.sh" --apply +unset FAKE_REQUIRE_LOCAL_DOCKER_ENDPOINT + # Remote reconciliation enables the ci-fleet-reconcile timer during install, # and reconciliation signs the GitHub App JWT with openssl. Require openssl # before install/check so the enabled timer cannot fail at runtime. @@ -380,6 +393,7 @@ unset FAKE_WRONG_HOST_CONFIG_OWNER expect_failure 'managed installs require the default' "$installer" --check "${base_args[@]}" --ref "$ref_one" --host-config "$tmp/custom-host.env" first=$(expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one") +expect_success env DOCKER_CONTEXT=default "$installer" --check "${base_args[@]}" --ref "$ref_one" grep -Fq 'CONVERGED mode=install' <<<"$first" || fail 'fresh install did not converge' [[ -L "$root/opt/ci-fleet/current" && -f "$root/var/lib/ci-fleet/install-state.json" ]] || fail 'fresh install state is incomplete' [[ $(readlink -f "$root/opt/ci-fleet/manager/current") == "$root/opt/ci-fleet/manager/releases/$engine_ref" ]] || fail 'installer manager did not activate the desired engine release' @@ -395,9 +409,11 @@ chmod 644 "$rendered_env" expect_failure 'DRIFT rendered_environment' "$installer" --check "${base_args[@]}" --ref "$ref_one" expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one" >/dev/null [[ $(stat -c %a "$rendered_env") == 600 ]] || fail 'convergence did not repair rendered-environment mode' +export FAKE_REQUIRE_LOCAL_DOCKER_ENDPOINT=1 manual_health_result=0 "$repo_root/scripts/healthcheck.sh" >/dev/null || manual_health_result=$? ((manual_health_result < 2)) || fail 'manual healthcheck did not source rendered capacity' +unset FAKE_REQUIRE_LOCAL_DOCKER_ENDPOINT export FAKE_WRONG_INSTALL_STATE_OWNER=$install_state expect_failure 'install state must be owned by root with mode 0600' env CI_FLEET_INSTALL_STATE_FILE="$install_state" CI_FLEET_INSTALLER="$installer" "$repo_root/scripts/check-installed-state.sh" unset FAKE_WRONG_INSTALL_STATE_OWNER