From f95940cc6795c3fd3859dee72ee5449bbb11c090 Mon Sep 17 00:00:00 2001 From: Eli Bosley Date: Thu, 27 Aug 2026 11:29:34 -0400 Subject: [PATCH 01/21] chore(ai-review): record shared cache design forecast --- .../feat-shared-build-cache-8c24ca91be08.json | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 .limetech/ai-review-markers/feat-shared-build-cache-8c24ca91be08.json diff --git a/.limetech/ai-review-markers/feat-shared-build-cache-8c24ca91be08.json b/.limetech/ai-review-markers/feat-shared-build-cache-8c24ca91be08.json new file mode 100644 index 00000000..3e3b4bc3 --- /dev/null +++ b/.limetech/ai-review-markers/feat-shared-build-cache-8c24ca91be08.json @@ -0,0 +1,25 @@ +{ + "disposition": "PATCH", + "expected_structural_delta": "Four existing production modules plus defaults reference; approximately 120-180 production lines, two nonsecret generated representations, no new service/runtime owner/public operation. Three existing roots and three semantic hops. One focused test around150lines, one real-Docker test around100lines, documentation around100lines.", + "minimum_design": "Extend the current settings/parser and drain fingerprint. Render immutable nonsecret profile.env and buildkitd.toml snapshots, mount them read-only through both provider adapters, and document explicit workflow cache import/export and job-owned credentials.", + "outcome": "Opt-in existing-registry build-cache profiles with smaller local BuildKit GC budgets, preserving per-slot Docker ownership and active jobs.", + "reuse_decisions": [ + { + "decision": "extend", + "name": "Existing settings, allowlisted parser and confgen/drain", + "rationale": "One configuration owner; preserve disabled-mode fingerprint and drain busy jobs before profile changes." + }, + { + "decision": "extend", + "name": "GitHub and GitLab provider mount adapters", + "rationale": "Reuse their existing direct and nested DinD bind paths." + }, + { + "decision": "not applicable", + "name": "Shared pull-through mirror", + "rationale": "Pull-only image cache cannot accept exported build caches; existing registry remains external and job-authenticated." + } + ], + "schema": "limetech.ai-review-marker.v2", + "stage": "forecast" +} From 923de11a42c82af991da87408a634e0b0f273b31 Mon Sep 17 00:00:00 2001 From: Eli Bosley Date: Thu, 27 Aug 2026 11:41:40 -0400 Subject: [PATCH 02/21] feat: add opt-in registry build-cache profiles --- .github/workflows/lint.yml | 3 + README.md | 1 + docs/build-cache.md | 113 ++++++++++++++++ .../emhttp/plugins/ci-runner-farm/README.md | 6 + .../ci-runner-farm/RunnerFarmSettings.page | 34 ++++- .../emhttp/plugins/ci-runner-farm/default.cfg | 3 + .../include/providers/github.sh | 5 +- .../include/providers/gitlab.sh | 9 ++ .../ci-runner-farm/include/runner-farm.sh | 88 +++++++++++- tests/build-cache-integration.sh | 93 +++++++++++++ tests/build-cache.sh | 127 ++++++++++++++++++ tests/check.sh | 1 + tests/run-linux-checks.sh | 1 + 13 files changed, 479 insertions(+), 5 deletions(-) create mode 100644 docs/build-cache.md create mode 100644 tests/build-cache-integration.sh create mode 100644 tests/build-cache.sh diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 73312467..baa44805 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -33,3 +33,6 @@ jobs: - name: Lint generated GitLab Runner configuration run: bash tests/gitlab-runner-lint.sh + + - name: Verify shared build cache with isolated builders + run: bash tests/build-cache-integration.sh diff --git a/README.md b/README.md index 5274d7c1..a28c53d2 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,7 @@ provider's credentials and runtime. | Concurrent runner slots | Each slot accepts one job at a time and can have CPU and memory limits, keeping CI from starving the rest of the host. | | GitHub and GitLab providers | Keep the existing GitHub Actions integration or select GitLab.com/self-managed GitLab. | | Warm shared caches | Reuse npm, yarn, pnpm, Playwright, Cargo, sccache, or custom cache directories across jobs. | +| Optional registry build-cache profile | Supply explicit CI workflows with an existing registry cache location and a smaller per-builder GC budget. No new server or shared Docker data root. See the [setup guide](docs/build-cache.md). | | Slot-scoped Docker-in-Docker | Give each runner slot a private privileged Docker daemon without exposing Unraid's existing Docker socket by default; privileged DinD is still capable of host compromise. | | Bring your own job image | Pull a remote image or edit and build a provider-specific starter image in the plugin. | | Named runner pools | Route jobs to purpose-built pools with independent fixed capacity, labels/tags, CPU, memory, and images. | diff --git a/docs/build-cache.md b/docs/build-cache.md new file mode 100644 index 00000000..c8f21637 --- /dev/null +++ b/docs/build-cache.md @@ -0,0 +1,113 @@ +# Share build caches through an existing registry + +The optional **Build cache profile** lets CI jobs reuse exported Docker build +layers across runners. Each slot keeps its own Docker daemon and writable data. +The plugin supplies configuration, not a registry server or registry credentials. +The existing pull-through image mirror remains separate and cannot accept cache +exports. + +## Configure the farm + +1. Open **Settings → CI Runner Farm → Settings → Build cache profile**. +2. Select **Existing registry (workflow opt-in)**. +3. Enter a tagless repository, such as `registry.example.com/team/build-cache`. +4. Set **Local cache budget per builder (GiB)**. The default is 20 GiB. +5. Apply the settings. Busy jobs keep their current profile until they finish. +6. Update the build workflows as described below. + +The configuration keys are `BUILD_CACHE_MODE` (`off` or `registry`), +`BUILD_CACHE_REPOSITORY`, and `BUILD_CACHE_LOCAL_GIB` (1–1024). +Off preserves existing behavior, including runner configuration fingerprints. +Registry mode requires Docker-in-Docker. Host-socket mode is not supported. + +Each GitHub runner and GitLab Docker-executor job receives two read-only files: + +- `/etc/ci-runner-farm/build-cache/profile.env`: the repository and config path. +- `/etc/ci-runner-farm/build-cache/buildkitd.toml`: the OCI worker's GC budget. + +Profiles are immutable snapshots. Changes do not rewrite files mounted by busy +jobs. Old snapshots remain in the plugin's runtime directory until reboot. +The current snapshot is generated again during runner provisioning. + +## Use the profile in a workflow + +Use a current Buildx client and BuildKit version that supports `maxUsedSpace`. +This profile targets the OCI worker in a `docker-container` builder. It does not +configure arbitrary existing builders, remote builders, or `docker build` calls. +GitLab job images must include the Docker CLI and Buildx plugin. + +Authenticate to the cache registry inside the job, using a credential with only +the required repository permissions. The plugin does not expose its host PAT, +runner token, or image-pull credentials through this profile. Configure private +CA trust in the job/builder when required. The profile does not disable TLS or +relax the farm's network firewall. A registry blocked by strict isolation remains +blocked. + +This shell example works in a GitHub shell job or GitLab job. Set +`CACHE_SCOPE` from a stable hash of the full project, build target, platform, and +branch identity. Use the same scope on different runners to reuse their cache. +Keep output image references separate from cache references. + +```bash +set -euo pipefail +. /etc/ci-runner-farm/build-cache/profile.env +: "${CACHE_SCOPE:?Set a project/target/platform/branch-specific cache scope}" +[[ "$CACHE_SCOPE" =~ ^[a-z0-9][a-z0-9_.-]{0,100}$ ]] || exit 1 +cache_ref="${CRF_BUILD_CACHE_REPOSITORY}:${CACHE_SCOPE}" +builder="crf-$(cat /proc/sys/kernel/random/uuid)" +docker buildx create --name "$builder" --driver docker-container \ + --buildkitd-config "$CRF_BUILDKIT_CONFIG" --bootstrap +trap 'docker buildx rm "$builder"' EXIT +docker buildx build --builder "$builder" \ + --cache-from "type=registry,ref=$cache_ref" \ + --cache-to "type=registry,ref=$cache_ref,mode=max,image-manifest=true" \ + --load -t example/app:ci . +``` + +The example removes only its own builder after the job. Registry cache survives +that cleanup. A workflow may retain its own builder instead, but it must recreate +that builder when the profile changes so the new GC budget takes effect. +GitHub jobs using `container:` must also mount the profile directory read-only +into their job container. Shell jobs receive it directly. + +For GitHub's Docker actions, load `profile.env` in a shell step and write the +needed values to `$GITHUB_OUTPUT`. Pass the config path to +`docker/setup-buildx-action` as `buildkitd-config`. Pass explicitly scoped refs to +`docker/build-push-action` as `cache-from` and `cache-to`. Do not assume that the +runner's environment automatically appears in GitHub's expression `env` context. +See Docker's [builder configuration](https://docs.docker.com/build/ci/github-actions/configure-builder/) +and [registry cache examples](https://docs.docker.com/build/ci/github-actions/cache/). + +## Scope, failure, and storage limits + +Cache names are not an access-control boundary. Only trusted jobs should share +a writable cache repository. Separate projects and trust levels with registry +permissions and separate repositories. Never give untrusted pull requests a +credential that can overwrite trusted caches. Branch-specific tags avoid normal +collisions but do not stop a credential holder from choosing another tag. +Serialize exports to the same ref, or use separate refs for concurrent writers. +Use BuildKit secret mounts for secrets; never bake credentials into layers. + +The profile provides no implicit `latest` cache tag. A missing import cache is a +normal cold build under Docker's behavior. Export failures remain job failures +unless the workflow deliberately changes Docker's default error handling. Do not +add `ignore-error=true` or fall back to an unconfigured builder to hide failures. + +The local budget is a garbage-collection target, not a hard quota. Active build +data can exceed it. It does not cover Docker images, other builders, workspaces, +package caches, or the image mirror. Exported caches also need registry retention +and garbage collection, controlled by the registry owner. Reducing local budgets +trades disk use for cache downloads and depends on registry availability. + +Enabling this option does not prune old builders or immediately recover their +disk space. Migrate workflows first, then retire unused builders through their +normal owner. Disabling the option removes the profile from newly provisioned +runners after drain; remove workflow references to the profile before disabling. + +## Verification + +`bash tests/build-cache.sh` checks profile validation, snapshots, fingerprints, +and both providers' mount contracts without external services. +`bash tests/build-cache-integration.sh` uses disposable local Docker builders and +a test registry to prove cross-builder cache reuse and effective GC settings. +It does not contact a farm host or modify existing runners. diff --git a/src/usr/local/emhttp/plugins/ci-runner-farm/README.md b/src/usr/local/emhttp/plugins/ci-runner-farm/README.md index 686c9df5..9e019edf 100644 --- a/src/usr/local/emhttp/plugins/ci-runner-farm/README.md +++ b/src/usr/local/emhttp/plugins/ci-runner-farm/README.md @@ -5,6 +5,12 @@ resource-capped slots, warm package/image caches, Docker-in-Docker, and optional autoscaling. One provider owns the farm at a time; GitHub remains the default so existing installations keep their behavior. +The optional **Build cache profile** supplies an existing registry cache location +and a per-builder garbage-collection budget (20 GiB by default). It is off by +default and requires Docker-in-Docker plus explicit workflow integration. +It creates no registry server, shares no writable Docker root, and passes no +host credentials to jobs. See the [build-cache setup guide](https://github.com/unraid/ci-runner-farm/blob/main/docs/build-cache.md). + Named pools work with both providers. Each pool can use its own fixed capacity, labels or tags, CPU, memory, and runner/job image. GitHub pools require organization scope. GitLab pools require a pool-specific `glrt-` runner token diff --git a/src/usr/local/emhttp/plugins/ci-runner-farm/RunnerFarmSettings.page b/src/usr/local/emhttp/plugins/ci-runner-farm/RunnerFarmSettings.page index 47b8678e..18d169c0 100644 --- a/src/usr/local/emhttp/plugins/ci-runner-farm/RunnerFarmSettings.page +++ b/src/usr/local/emhttp/plugins/ci-runner-farm/RunnerFarmSettings.page @@ -35,6 +35,7 @@ $defaults = [ 'CACHE_ROOT'=>'/mnt/cache/github-runner', 'WORK_TMPFS_SIZE'=>'8g', 'CACHE_MOUNTS'=>'pnpm-store:/home/runner/.local/share/pnpm/store npm:/home/runner/.npm yarn:/home/runner/.cache/yarn ms-playwright:/home/runner/.cache/ms-playwright', 'DIND'=>'true', 'SHARE_DOCKER_SOCK'=>'false', 'SHARED_IMAGE_CACHE'=>'true', 'NETWORK_ISOLATION'=>'off', + 'BUILD_CACHE_MODE'=>'off', 'BUILD_CACHE_REPOSITORY'=>'', 'BUILD_CACHE_LOCAL_GIB'=>'20', 'IMAGE_AUTOUPDATE'=>'false', 'IMAGE_AUTOUPDATE_INTERVAL'=>'1800', 'IMAGE_DRAIN_TIMEOUT'=>'3600', 'DASHBOARD_WIDGET_ENABLE'=>'true', 'AUTOSCALE'=>'false', 'AUTOSCALE_MIN'=>'2', 'AUTOSCALE_MAX'=>'16', 'AUTOSCALE_MIN_IDLE'=>'2', @@ -429,6 +430,30 @@ _(Shared image cache)_: > Runs a shared `registry:2` pull-through cache (Docker-in-Docker only) so images used across the fleet are pulled from Docker Hub once, not once per runner. Bound to the Docker bridge gateway, not the LAN — so it is not exposed off the host, though (being on the bridge gateway) it is reachable unauthenticated by any container on the default Docker bridge, not only this plugin's runners. It only caches public Docker Hub images. Turn off if you don't want the extra container. The host port defaults to 5000; if that clashes with another service, set `MIRROR_PORT` in `/boot/config/plugins/ci-runner-farm/ci-runner-farm.cfg` and Restart the fleet. :end +### Build cache profile + +_(Shared build cache)_: +: + +_(Cache repository)_: +: + +_(Local cache budget per builder (GiB))_: +: + +:crf_build_cache_plug: +> Requires Docker-in-Docker and an existing writable registry. Enter a repository without a URL scheme, tag, digest, or credentials. No registry server is created. +> +> Workflows must load `/etc/ci-runner-farm/build-cache/profile.env`, configure a `docker-container` builder with its BuildKit configuration, and explicitly import/export scoped cache references. Jobs supply their own registry credentials. See the [build-cache setup guide](https://github.com/unraid/ci-runner-farm/blob/main/docs/build-cache.md). +> +> The budget controls BuildKit garbage collection, not total disk use or a hard quota. Other builders and active builds can exceed it. Apply updates each runner after its active job finishes. Existing caches are not pruned. +:end + +### Network isolation + _(Network isolation)_: : +:crf_build_cache_mode_plug: +> Requires Docker-in-Docker and an existing writable registry. No registry server is created. +:end + _(Cache repository)_: : +:crf_build_cache_repository_plug: +> Enter a repository without a URL scheme, tag, digest, or credentials. Workflows must load `/etc/ci-runner-farm/build-cache/profile.env`, configure a `docker-container` builder with its BuildKit configuration, and explicitly import/export scoped cache references. Jobs supply their own registry credentials. See the [build-cache setup guide](https://github.com/unraid/ci-runner-farm/blob/main/docs/build-cache.md). +:end + _(Local cache budget per builder (GiB))_: : :crf_build_cache_plug: -> Requires Docker-in-Docker and an existing writable registry. Enter a repository without a URL scheme, tag, digest, or credentials. No registry server is created. -> -> Workflows must load `/etc/ci-runner-farm/build-cache/profile.env`, configure a `docker-container` builder with its BuildKit configuration, and explicitly import/export scoped cache references. Jobs supply their own registry credentials. See the [build-cache setup guide](https://github.com/unraid/ci-runner-farm/blob/main/docs/build-cache.md). -> -> The budget controls BuildKit garbage collection, not total disk use or a hard quota. Other builders and active builds can exceed it. Apply updates each runner after its active job finishes. Existing caches are not pruned. +> The budget controls BuildKit garbage collection, not total disk use or a hard quota. Other builders and active builds can exceed it. In classic mode, Apply updates runners after active jobs finish. Named pools require an explicit Fleet Restart. Wait for the new runners before using the profile in workflows. Existing caches are not pruned. :end ### Network isolation diff --git a/tests/build-cache-integration.sh b/tests/build-cache-integration.sh index e09f138b..2db61c1b 100644 --- a/tests/build-cache-integration.sh +++ b/tests/build-cache-integration.sh @@ -2,6 +2,18 @@ # Real registry reuse across two isolated builders. Never prunes a Docker daemon. set -euo pipefail cd "$(dirname "$0")/.." +if [ "$(uname -s)" = Darwin ]; then + command -v brew >/dev/null 2>&1 || { + echo 'macOS cache integration requires Homebrew Coreutils (brew install coreutils).' >&2 + exit 1 + } + coreutils_bin="$(brew --prefix coreutils)/libexec/gnubin" + [ -x "$coreutils_bin/mv" ] || { + echo 'Install GNU tools for macOS cache integration: brew install coreutils' >&2 + exit 1 + } + export PATH="$coreutils_bin:$PATH" +fi tmp="$(mktemp -d)" suffix="$(basename "$tmp" | tr '[:upper:].' '[:lower:]-')" network="crf-cache-$suffix" From ff41baa59945a60071ab500d21786653813705b1 Mon Sep 17 00:00:00 2001 From: Eli Bosley Date: Thu, 27 Aug 2026 11:49:25 -0400 Subject: [PATCH 04/21] chore(ai-review): record final review receipt --- .../feat-shared-build-cache-8c24ca91be08.json | 25 +++---------------- 1 file changed, 4 insertions(+), 21 deletions(-) diff --git a/.limetech/ai-review-markers/feat-shared-build-cache-8c24ca91be08.json b/.limetech/ai-review-markers/feat-shared-build-cache-8c24ca91be08.json index 3e3b4bc3..75b79753 100644 --- a/.limetech/ai-review-markers/feat-shared-build-cache-8c24ca91be08.json +++ b/.limetech/ai-review-markers/feat-shared-build-cache-8c24ca91be08.json @@ -1,25 +1,8 @@ { "disposition": "PATCH", - "expected_structural_delta": "Four existing production modules plus defaults reference; approximately 120-180 production lines, two nonsecret generated representations, no new service/runtime owner/public operation. Three existing roots and three semantic hops. One focused test around150lines, one real-Docker test around100lines, documentation around100lines.", - "minimum_design": "Extend the current settings/parser and drain fingerprint. Render immutable nonsecret profile.env and buildkitd.toml snapshots, mount them read-only through both provider adapters, and document explicit workflow cache import/export and job-owned credentials.", - "outcome": "Opt-in existing-registry build-cache profiles with smaller local BuildKit GC budgets, preserving per-slot Docker ownership and active jobs.", - "reuse_decisions": [ - { - "decision": "extend", - "name": "Existing settings, allowlisted parser and confgen/drain", - "rationale": "One configuration owner; preserve disabled-mode fingerprint and drain busy jobs before profile changes." - }, - { - "decision": "extend", - "name": "GitHub and GitLab provider mount adapters", - "rationale": "Reuse their existing direct and nested DinD bind paths." - }, - { - "decision": "not applicable", - "name": "Shared pull-through mirror", - "rationale": "Pull-only image cache cannot accept exported build caches; existing registry remains external and job-authenticated." - } - ], + "forecast_commit": "c214cf26f7868ce28f4f7db38bb9993264e3f764", + "reviewed_sha": "25758df09c724822f9e0da1d9011495f54a45e40", "schema": "limetech.ai-review-marker.v2", - "stage": "forecast" + "stage": "final", + "unresolved_proportionality_findings": [] } From f9adf7aa067fb0467f3b2df779acdcca5ae69cc8 Mon Sep 17 00:00:00 2001 From: Eli Bosley Date: Thu, 27 Aug 2026 11:53:39 -0400 Subject: [PATCH 05/21] test: expose nested Docker integration failures --- tests/build-cache-integration.sh | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/build-cache-integration.sh b/tests/build-cache-integration.sh index 2db61c1b..c9fb10b6 100644 --- a/tests/build-cache-integration.sh +++ b/tests/build-cache-integration.sh @@ -97,9 +97,16 @@ for attempt in $(seq 1 30); do sleep 1 done docker exec "$sidecar_id" docker info >/dev/null 2>&1 -docker exec "$sidecar_id" docker run --rm \ +if docker exec "$sidecar_id" docker run --rm \ --mount "type=bind,src=$profile,dst=/etc/ci-runner-farm/build-cache,readonly" \ busybox:1.37.0 sh -c '. /etc/ci-runner-farm/build-cache/profile.env && test "$CRF_BUILD_CACHE_MODE" = registry && - ! touch /etc/ci-runner-farm/build-cache/forbidden' > "$tmp/nested.log" 2>&1 + ! touch /etc/ci-runner-farm/build-cache/forbidden' > "$tmp/nested.log" 2>&1; then + : +else + result=$? + echo 'Nested read-only profile test failed:' >&2 + tail -20 "$tmp/nested.log" >&2 + exit "$result" +fi echo 'build-cache-integration: independent cache reuse, effective GC budgets, and direct/nested read-only profiles: OK' From b9d5e748a1754c2408be2d8b07f94a3104f5e59b Mon Sep 17 00:00:00 2001 From: Eli Bosley Date: Thu, 27 Aug 2026 11:58:16 -0400 Subject: [PATCH 06/21] test: keep cache binds outside the DinD tmpfs --- tests/build-cache-integration.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) mode change 100644 => 100755 tests/build-cache-integration.sh diff --git a/tests/build-cache-integration.sh b/tests/build-cache-integration.sh old mode 100644 new mode 100755 index c9fb10b6..7fc681a3 --- a/tests/build-cache-integration.sh +++ b/tests/build-cache-integration.sh @@ -14,7 +14,9 @@ if [ "$(uname -s)" = Darwin ]; then } export PATH="$coreutils_bin:$PATH" fi -tmp="$(mktemp -d)" +# DinD mounts its own tmpfs over /tmp, hiding nested bind sources there. Keep +# the fixture outside that overlay and preserve identical host/sidecar paths. +tmp="$(mktemp -d /var/tmp/crf-build-cache.XXXXXX)" suffix="$(basename "$tmp" | tr '[:upper:].' '[:lower:]-')" network="crf-cache-$suffix" registry="crf-registry-$suffix" @@ -97,6 +99,8 @@ for attempt in $(seq 1 30); do sleep 1 done docker exec "$sidecar_id" docker info >/dev/null 2>&1 +docker exec "$sidecar_id" test -s "$profile/profile.env" +docker exec "$sidecar_id" test -s "$profile/buildkitd.toml" if docker exec "$sidecar_id" docker run --rm \ --mount "type=bind,src=$profile,dst=/etc/ci-runner-farm/build-cache,readonly" \ busybox:1.37.0 sh -c '. /etc/ci-runner-farm/build-cache/profile.env && From acad205d0f8f16bfe82e66f4fcf304c2c85681e6 Mon Sep 17 00:00:00 2001 From: Eli Bosley Date: Thu, 27 Aug 2026 11:58:16 -0400 Subject: [PATCH 07/21] chore: mark shell scripts executable --- .../emhttp/plugins/ci-runner-farm/include/providers/github.sh | 0 .../emhttp/plugins/ci-runner-farm/include/providers/gitlab.sh | 0 .../local/emhttp/plugins/ci-runner-farm/include/runner-farm.sh | 0 .../local/emhttp/plugins/ci-runner-farm/include/runner-pools.sh | 0 tests/build-cache.sh | 0 tests/exec-csrf.sh | 0 tests/firewall-transition.sh | 0 tests/gitlab-runner-lint.sh | 0 tests/lease-selfheal.sh | 0 tests/log-redaction.sh | 0 tests/numeric-config.sh | 0 tests/ownership-safety.sh | 0 tests/pool-runtime.sh | 0 tests/provider-mocks.sh | 0 tests/resource-ownership.sh | 0 tests/runner-pools.sh | 0 16 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 src/usr/local/emhttp/plugins/ci-runner-farm/include/providers/github.sh mode change 100644 => 100755 src/usr/local/emhttp/plugins/ci-runner-farm/include/providers/gitlab.sh mode change 100644 => 100755 src/usr/local/emhttp/plugins/ci-runner-farm/include/runner-farm.sh mode change 100644 => 100755 src/usr/local/emhttp/plugins/ci-runner-farm/include/runner-pools.sh mode change 100644 => 100755 tests/build-cache.sh mode change 100644 => 100755 tests/exec-csrf.sh mode change 100644 => 100755 tests/firewall-transition.sh mode change 100644 => 100755 tests/gitlab-runner-lint.sh mode change 100644 => 100755 tests/lease-selfheal.sh mode change 100644 => 100755 tests/log-redaction.sh mode change 100644 => 100755 tests/numeric-config.sh mode change 100644 => 100755 tests/ownership-safety.sh mode change 100644 => 100755 tests/pool-runtime.sh mode change 100644 => 100755 tests/provider-mocks.sh mode change 100644 => 100755 tests/resource-ownership.sh mode change 100644 => 100755 tests/runner-pools.sh diff --git a/src/usr/local/emhttp/plugins/ci-runner-farm/include/providers/github.sh b/src/usr/local/emhttp/plugins/ci-runner-farm/include/providers/github.sh old mode 100644 new mode 100755 diff --git a/src/usr/local/emhttp/plugins/ci-runner-farm/include/providers/gitlab.sh b/src/usr/local/emhttp/plugins/ci-runner-farm/include/providers/gitlab.sh old mode 100644 new mode 100755 diff --git a/src/usr/local/emhttp/plugins/ci-runner-farm/include/runner-farm.sh b/src/usr/local/emhttp/plugins/ci-runner-farm/include/runner-farm.sh old mode 100644 new mode 100755 diff --git a/src/usr/local/emhttp/plugins/ci-runner-farm/include/runner-pools.sh b/src/usr/local/emhttp/plugins/ci-runner-farm/include/runner-pools.sh old mode 100644 new mode 100755 diff --git a/tests/build-cache.sh b/tests/build-cache.sh old mode 100644 new mode 100755 diff --git a/tests/exec-csrf.sh b/tests/exec-csrf.sh old mode 100644 new mode 100755 diff --git a/tests/firewall-transition.sh b/tests/firewall-transition.sh old mode 100644 new mode 100755 diff --git a/tests/gitlab-runner-lint.sh b/tests/gitlab-runner-lint.sh old mode 100644 new mode 100755 diff --git a/tests/lease-selfheal.sh b/tests/lease-selfheal.sh old mode 100644 new mode 100755 diff --git a/tests/log-redaction.sh b/tests/log-redaction.sh old mode 100644 new mode 100755 diff --git a/tests/numeric-config.sh b/tests/numeric-config.sh old mode 100644 new mode 100755 diff --git a/tests/ownership-safety.sh b/tests/ownership-safety.sh old mode 100644 new mode 100755 diff --git a/tests/pool-runtime.sh b/tests/pool-runtime.sh old mode 100644 new mode 100755 diff --git a/tests/provider-mocks.sh b/tests/provider-mocks.sh old mode 100644 new mode 100755 diff --git a/tests/resource-ownership.sh b/tests/resource-ownership.sh old mode 100644 new mode 100755 diff --git a/tests/runner-pools.sh b/tests/runner-pools.sh old mode 100644 new mode 100755 From a39fbeafdb61da55e50f462f1a941e8f3f57366f Mon Sep 17 00:00:00 2001 From: Eli Bosley Date: Thu, 27 Aug 2026 11:58:16 -0400 Subject: [PATCH 08/21] fix(ci): validate documentation links against the candidate tree --- .mega-linter.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.mega-linter.yml b/.mega-linter.yml index 390f156e..37879c63 100644 --- a/.mega-linter.yml +++ b/.mega-linter.yml @@ -70,3 +70,10 @@ FILTER_REGEX_EXCLUDE: (^LICENSE$|^CHANGELOG\.md$|\.git/|megalinter-reports/) # uses .json with comments rather than .jsonc), so allow comments/trailing # commas for all JSON rather than relying on file extension. JSON_JSONLINT_ARGUMENTS: "--comments --trailing-newline --trailing-commas" + +# The shipped README needs public URLs, but new docs do not exist on main +# before a PR merges. Validate this repository's main-file links against the +# candidate checkout inside MegaLinter. Missing local targets still fail. +SPELL_LYCHEE_ARGUMENTS: + - --remap + - '^https://github\.com/unraid/ci-runner-farm/blob/main/ file:///github/workspace/' From 08c160a8a02f13526b5ba9d320687f9b46468fff Mon Sep 17 00:00:00 2001 From: Eli Bosley Date: Thu, 27 Aug 2026 11:58:16 -0400 Subject: [PATCH 09/21] chore(ai-review): refresh lint repair receipt --- .../ai-review-markers/feat-shared-build-cache-8c24ca91be08.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.limetech/ai-review-markers/feat-shared-build-cache-8c24ca91be08.json b/.limetech/ai-review-markers/feat-shared-build-cache-8c24ca91be08.json index 75b79753..2268687c 100644 --- a/.limetech/ai-review-markers/feat-shared-build-cache-8c24ca91be08.json +++ b/.limetech/ai-review-markers/feat-shared-build-cache-8c24ca91be08.json @@ -1,7 +1,7 @@ { "disposition": "PATCH", "forecast_commit": "c214cf26f7868ce28f4f7db38bb9993264e3f764", - "reviewed_sha": "25758df09c724822f9e0da1d9011495f54a45e40", + "reviewed_sha": "2ce1f8b4388535d97fde531429ec243387ceec55", "schema": "limetech.ai-review-marker.v2", "stage": "final", "unresolved_proportionality_findings": [] From 6ab6336dbc771762ebf69284523feabd090bbb61 Mon Sep 17 00:00:00 2001 From: Eli Bosley Date: Thu, 27 Aug 2026 12:05:56 -0400 Subject: [PATCH 10/21] fix(ci): avoid autolinking the URL match expression --- .mega-linter.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.mega-linter.yml b/.mega-linter.yml index 37879c63..fa450c18 100644 --- a/.mega-linter.yml +++ b/.mega-linter.yml @@ -76,4 +76,4 @@ JSON_JSONLINT_ARGUMENTS: "--comments --trailing-newline --trailing-commas" # candidate checkout inside MegaLinter. Missing local targets still fail. SPELL_LYCHEE_ARGUMENTS: - --remap - - '^https://github\.com/unraid/ci-runner-farm/blob/main/ file:///github/workspace/' + - '^https[:]//github[.]com/unraid/ci-runner-farm/blob/main/ file:///github/workspace/' From b078063cc8d47ed64fb0a328caf294ca035cc7d8 Mon Sep 17 00:00:00 2001 From: Eli Bosley Date: Thu, 27 Aug 2026 12:05:57 -0400 Subject: [PATCH 11/21] chore(ai-review): refresh lint repair receipt --- .../ai-review-markers/feat-shared-build-cache-8c24ca91be08.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.limetech/ai-review-markers/feat-shared-build-cache-8c24ca91be08.json b/.limetech/ai-review-markers/feat-shared-build-cache-8c24ca91be08.json index 2268687c..dd79fa04 100644 --- a/.limetech/ai-review-markers/feat-shared-build-cache-8c24ca91be08.json +++ b/.limetech/ai-review-markers/feat-shared-build-cache-8c24ca91be08.json @@ -1,7 +1,7 @@ { "disposition": "PATCH", "forecast_commit": "c214cf26f7868ce28f4f7db38bb9993264e3f764", - "reviewed_sha": "2ce1f8b4388535d97fde531429ec243387ceec55", + "reviewed_sha": "36e45683dda6c2c118fec496d3349f58a6521346", "schema": "limetech.ai-review-marker.v2", "stage": "final", "unresolved_proportionality_findings": [] From d7b431a8a62d3ed6ef23d1d88b209b00fa4bb072 Mon Sep 17 00:00:00 2001 From: Eli Bosley Date: Thu, 27 Aug 2026 12:10:15 -0400 Subject: [PATCH 12/21] test: fail explicitly when builder volumes are shared --- tests/build-cache-integration.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/build-cache-integration.sh b/tests/build-cache-integration.sh index 7fc681a3..9104c169 100755 --- a/tests/build-cache-integration.sh +++ b/tests/build-cache-integration.sh @@ -80,8 +80,12 @@ for n in 1 2; do exit 1 fi done -[ -s "$tmp/volume-1" ] && [ -s "$tmp/volume-2" ] -! cmp -s "$tmp/volume-1" "$tmp/volume-2" +[ -s "$tmp/volume-1" ] +[ -s "$tmp/volume-2" ] +if cmp -s "$tmp/volume-1" "$tmp/volume-2"; then + echo 'BuildKit builders must use distinct local volumes.' >&2 + exit 1 +fi # The RUN operation, not just base-image metadata, must come from registry cache. awk '/RUN echo registry-cache-proof/ {step=$1} $1 == step && $2 == "CACHED" {found=1} END {exit !found}' "$tmp/build-2.log" grep -q 'importing cache manifest from cache.test:5000/team/cache:project-target-platform-main' "$tmp/build-2.log" From 2f7f3a9da4bc3623681fa9f9444106a617abca2d Mon Sep 17 00:00:00 2001 From: Eli Bosley Date: Thu, 27 Aug 2026 12:10:15 -0400 Subject: [PATCH 13/21] chore(ai-review): record volume assertion review --- .../ai-review-markers/feat-shared-build-cache-8c24ca91be08.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.limetech/ai-review-markers/feat-shared-build-cache-8c24ca91be08.json b/.limetech/ai-review-markers/feat-shared-build-cache-8c24ca91be08.json index dd79fa04..358ec32e 100644 --- a/.limetech/ai-review-markers/feat-shared-build-cache-8c24ca91be08.json +++ b/.limetech/ai-review-markers/feat-shared-build-cache-8c24ca91be08.json @@ -1,7 +1,7 @@ { "disposition": "PATCH", "forecast_commit": "c214cf26f7868ce28f4f7db38bb9993264e3f764", - "reviewed_sha": "36e45683dda6c2c118fec496d3349f58a6521346", + "reviewed_sha": "1347b90876b239ab5b0935d266b81cf3c0ed6c41", "schema": "limetech.ai-review-marker.v2", "stage": "final", "unresolved_proportionality_findings": [] From 106b47bd11752f9a8c8e71f9b88f527370d4ba1b Mon Sep 17 00:00:00 2001 From: Eli Bosley Date: Thu, 27 Aug 2026 12:34:28 -0400 Subject: [PATCH 14/21] fix(cache): retain runner data across stop and restart --- README.md | 1 + docs/build-cache.md | 25 ++++ .../ci-runner-farm/include/runner-farm.sh | 9 +- tests/cache-retention.sh | 141 ++++++++++++++++++ tests/check.sh | 1 + 5 files changed, 172 insertions(+), 5 deletions(-) create mode 100755 tests/cache-retention.sh diff --git a/README.md b/README.md index a28c53d2..1130975e 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ provider's credentials and runtime. | GitHub and GitLab providers | Keep the existing GitHub Actions integration or select GitLab.com/self-managed GitLab. | | Warm shared caches | Reuse npm, yarn, pnpm, Playwright, Cargo, sccache, or custom cache directories across jobs. | | Optional registry build-cache profile | Supply explicit CI workflows with an existing registry cache location and a smaller per-builder GC budget. No new server or shared Docker data root. See the [setup guide](docs/build-cache.md). | +| Cache-preserving Stop and Restart | Keep per-slot Docker and GitLab job caches across maintenance. Explicit `prune-cache` deletes retained caches. Permanent slot retirement still deletes that slot's data. See [cache retention](docs/build-cache.md#keep-caches-across-stop-and-restart). | | Slot-scoped Docker-in-Docker | Give each runner slot a private privileged Docker daemon without exposing Unraid's existing Docker socket by default; privileged DinD is still capable of host compromise. | | Bring your own job image | Pull a remote image or edit and build a provider-specific starter image in the plugin. | | Named runner pools | Route jobs to purpose-built pools with independent fixed capacity, labels/tags, CPU, memory, and images. | diff --git a/docs/build-cache.md b/docs/build-cache.md index 8663ee1a..b6d1a01c 100644 --- a/docs/build-cache.md +++ b/docs/build-cache.md @@ -106,10 +106,35 @@ normal owner. Remove workflow references to the profile before disabling the option. Classic fleets remove the mount as runners drain and are replaced. Named pools require the same scheduled Fleet Restart described above. +## Keep caches across Stop and Restart + +**Stop** and **Restart** retain per-slot Docker data, GitLab job caches, and the +shared image mirror cache. A later Start reuses the retained data for the same +slot and cache root. This applies with the build cache profile on or off. +The plugin still removes runner containers and performs provider credential +cleanup. Plugin uninstall and credential removal also retain caches because +they use Stop. + +Stop does not guarantee that active jobs finish. Schedule maintenance before +using Stop or Restart. Autoscale-down and permanent slot retirement still delete +that slot's Docker data and GitLab job cache. Registry exports are unaffected. + +To delete retained local caches, stop the fleet, then run the explicit command: + +```bash +/usr/local/emhttp/plugins/ci-runner-farm/include/runner-farm.sh prune-cache +``` + +This command deletes plugin-owned cache directories under the configured cache +root. It refuses to proceed while managed runners, sidecars, or job containers +remain. Cache retention does not reduce existing disk use. + ## Verification `bash tests/build-cache.sh` checks profile validation, snapshots, fingerprints, and both providers' mount contracts without external services. +`bash tests/cache-retention.sh` checks Stop/Restart retention and explicit cache +deletion for both providers using disposable on-disk fixtures. `bash tests/build-cache-integration.sh` uses disposable local Docker builders and a test registry to prove cross-builder cache reuse and effective GC settings. It does not contact a farm host or modify existing runners. diff --git a/src/usr/local/emhttp/plugins/ci-runner-farm/include/runner-farm.sh b/src/usr/local/emhttp/plugins/ci-runner-farm/include/runner-farm.sh index b28b157d..8272b080 100755 --- a/src/usr/local/emhttp/plugins/ci-runner-farm/include/runner-farm.sh +++ b/src/usr/local/emhttp/plugins/ci-runner-farm/include/runner-farm.sh @@ -2462,10 +2462,9 @@ cleanup_orphan_github_validations() { # Full teardown: daemons, runner containers, and the shared pull-through mirror. # Reached from the UI Stop button AND from plugin uninstall (the .plg remove step -# calls 'stop'), so it must leave nothing running. The mirror's on-pool cache dir -# ($CACHE_ROOT/registry-mirror) is intentionally left behind — like the config and -# token — so a later Start rebuilds the container with its cache warm; only the -# container is removed here, not the cached layers. +# calls 'stop'), so it must leave nothing running. Retain on-pool runner and +# mirror caches so a later Start can reuse them. Explicit prune-cache deletes +# retained caches. Permanent slot retirement still deletes that slot's data. cmd_stop() { # Cancel every process that can create fleet resources before examining the # current Docker state. In particular, a sleeping boot worker must not wake @@ -2486,7 +2485,7 @@ cmd_stop() { while IFS= read -r c; do [ -n "$c" ] || continue log "stopping $c (graceful deregister)" - remove_runner "$c" || stop_failed=1 + remove_runner "$c" false || stop_failed=1 done <<< "$names" fi # A provider removal that failed closed can intentionally leave a manager and diff --git a/tests/cache-retention.sh b/tests/cache-retention.sh new file mode 100755 index 00000000..7a0e26fd --- /dev/null +++ b/tests/cache-retention.sh @@ -0,0 +1,141 @@ +#!/usr/bin/env bash +# Run the real Stop/removal policy against disposable on-disk cache fixtures. +set -euo pipefail +cd "$(dirname "$0")/.." +tmp="$(mktemp -d)" +trap 'rm -rf "$tmp"' EXIT +fail() { echo "CACHE RETENTION FAIL: $*" >&2; exit 1; } + +for provider in github gitlab; do + ( + fixture_provider="$provider" + export CRF_SOURCE_ONLY=1 CRF_CFGDIR="$tmp/$provider/config" CRF_RUNDIR="$tmp/$provider/run" + mkdir -p "$CRF_CFGDIR" "$CRF_RUNDIR" + # shellcheck source=/dev/null + source src/usr/local/emhttp/plugins/ci-runner-farm/include/runner-farm.sh + CI_PROVIDER="$provider" + CACHE_ROOT="$tmp/$provider/cache" + NETWORK_ISOLATION=off + CACHE_MOUNTS='' + NO_REGISTER=1 + slot=ci-runner-1 + fixture_id=$(printf '%064d' 1) + present="$tmp/$provider/present" + effects="$tmp/$provider/effects" + touch "$present" "$effects" + for dir in docker gitlab-cache registry-mirror; do + mkdir -p "$CACHE_ROOT/$dir/$slot" + printf 'retained\n' > "$CACHE_ROOT/$dir/$slot/proof" + done + + # Replace external Docker/API/process boundaries, not Stop, remove_runner, + # the provider removal adapters, or their cache retention decisions. + crf_safe_cache_root() { printf '%s\n' "$CACHE_ROOT"; } + boot_autostart_stop() { return 0; } + autoscale_stop() { return 0; } + imageupdate_stop() { return 0; } + reconcile_stop() { return 0; } + quiesce_gitlab_managers_for_stop() { return 0; } + managed_names() { if [ -e "$present" ]; then echo "$slot"; fi; } + managed_runner_snapshot() { printf '%s|%s|runner|1|generation\n' "$fixture_id" "$fixture_provider"; } + provider_stop_container() { printf 'stopped %s\n' "$CRF_REMOVE_ID" >> "$effects"; } + provider_remove_container() { rm "$present"; } + provider_stop_remove_container() { provider_stop_container "$1" && provider_remove_container "$1"; } + github_deregister_runner_api() { printf 'unregistered\n' >> "$effects"; } + gitlab_unregister_manager() { printf 'unregistered\n' >> "$effects"; } + gitlab_running_executor_containers() { return 0; } + gitlab_remove_host_jobs() { return 0; } + gitlab_scrub_retired_slot_credentials() { printf 'credentials scrubbed\n' >> "$effects"; } + gitlab_assert_no_orphan_manager_configs() { return 0; } + gitlab_cleanup_orphan_sidecars() { return 0; } + cleanup_orphan_github_validations() { return 0; } + firewall_clear() { printf 'firewall cleared\n' >> "$effects"; } + docker() { + case "$1" in + inspect) return 1 ;; + ps) + if [ -e "$present" ] && [[ "$*" = *'label=net.unraid.ci-runner-farm.managed=true'* ]]; then + echo "$slot" + fi ;; + *) return 1 ;; + esac + } + + cmd_stop >/dev/null || fail "$provider Stop failed" + [ ! -e "$present" ] || fail "$provider container survived Stop" + grep -qx 'unregistered' "$effects" || fail "$provider registration was not removed" + [ -f "$CACHE_ROOT/docker/$slot/proof" ] || fail "$provider Stop deleted Docker data" + [ -f "$CACHE_ROOT/gitlab-cache/$slot/proof" ] || fail "$provider Stop deleted job cache" + [ -f "$CACHE_ROOT/registry-mirror/$slot/proof" ] || fail "$provider Stop deleted mirror data" + if [ "$provider" = gitlab ]; then + grep -qx 'credentials scrubbed' "$effects" || fail 'retention skipped credential cleanup' + fi + + # Restart uses the real Stop path before reloading and starting the fleet. + touch "$present" + reload_locked_snapshot() { return 0; } + cmd_start() { + [ ! -e "$present" ] || fail 'Restart did not stop the old container' + [ -f "$CACHE_ROOT/docker/$slot/proof" ] || fail 'Restart lost Docker data before Start' + [ -f "$CACHE_ROOT/gitlab-cache/$slot/proof" ] || fail 'Restart lost job cache before Start' + touch "$present" + } + cmd_restart >/dev/null || fail "$provider Restart failed" + [ -e "$present" ] || fail "$provider Restart did not start a replacement" + + # Creation maps the retained slot directory, not a fresh per-container path. + ( + DIND=true + BUILD_CACHE_MODE=off + host() { echo fixture; } + runner_host_service_ipv4() { echo 192.0.2.10; } + if [ "$fixture_provider" = github ]; then + github_build_args 1 + printf '%s\n' "${ARGS[@]}" | grep -qxF "$CACHE_ROOT/docker/$slot:/var/lib/docker" + else + GITLAB_RUNNER_TOKEN=glrt-fixture-only + gitlab_write_config 1 "$slot" + grep -qF "$CACHE_ROOT/gitlab-cache/$slot:/cache" "$CFGDIR/gitlab-runners/$slot/config.toml" + docker() { + case "$1" in + inspect) return 1 ;; + run) printf '%s\n' "$@" > "$tmp/sidecar.args" ;; + --host) return 0 ;; + *) return 1 ;; + esac + } + gitlab_start_sidecar 1 "$slot" + grep -qxF "$CACHE_ROOT/docker/$slot:/var/lib/docker" "$tmp/sidecar.args" + fi + [ -f "$CACHE_ROOT/docker/$slot/proof" ] || fail 'creation overwrote retained data' + ) + + # Explicit prune must still refuse a live/retained manager. + if cmd_prune_cache >/dev/null 2>&1; then fail 'prune accepted an existing manager'; fi + [ -f "$CACHE_ROOT/docker/$slot/proof" ] || fail 'refused prune deleted Docker data' + + # Failed removal cannot clear shared isolation or allow Restart to start. + ( + provider_remove_container() { return 1; } + firewall_clear() { fail 'failed Stop cleared the firewall'; } + cmd_start() { fail 'failed Stop allowed Restart to start'; } + if cmd_restart >/dev/null 2>&1; then fail 'Restart accepted failed removal'; fi + [ -f "$CACHE_ROOT/docker/$slot/proof" ] || fail 'failed Stop deleted Docker data' + ) + + # Permanent slot retirement still opts into deletion, independent of Stop. + touch "$present" + remove_runner "$slot" true >/dev/null || fail "$provider permanent removal failed" + [ ! -e "$CACHE_ROOT/docker/$slot" ] || fail "$provider permanent removal retained Docker data" + [ -f "$CACHE_ROOT/registry-mirror/$slot/proof" ] || fail 'slot retirement deleted shared mirror data' + + # The existing explicit prune removes retained data only after Stop. + mkdir -p "$CACHE_ROOT/docker/$slot" + touch "$CACHE_ROOT/docker/$slot/proof" + cmd_prune_cache >/dev/null || fail 'explicit prune failed' + [ ! -e "$CACHE_ROOT/docker" ] || fail 'explicit prune did not remove retained Docker data' + [ ! -e "$CACHE_ROOT/registry-mirror" ] || fail 'explicit prune did not remove mirror data' + ) +done + +echo 'cache-retention: Stop/Restart preserve both providers; explicit retirement/prune delete caches' diff --git a/tests/check.sh b/tests/check.sh index e153dcd2..d25befc1 100755 --- a/tests/check.sh +++ b/tests/check.sh @@ -30,6 +30,7 @@ done < <(find "$RUNTIME" -type f \( -name '*.php' -o -name '*.page' \) -print | echo "== Contracts and package ==" bash tests/config-parity.sh bash tests/build-cache.sh +bash tests/cache-retention.sh bash tests/image-validation.sh bash tests/boot-log.sh bash tests/editorconfig.sh From e4400e53eb16fdd09758a4123ae5dd52ec72c9d6 Mon Sep 17 00:00:00 2001 From: Eli Bosley Date: Thu, 27 Aug 2026 12:38:11 -0400 Subject: [PATCH 15/21] docs(cache): clarify retained and retired cache lifecycles --- docs/build-cache.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/build-cache.md b/docs/build-cache.md index b6d1a01c..29b3f265 100644 --- a/docs/build-cache.md +++ b/docs/build-cache.md @@ -112,12 +112,13 @@ Named pools require the same scheduled Fleet Restart described above. shared image mirror cache. A later Start reuses the retained data for the same slot and cache root. This applies with the build cache profile on or off. The plugin still removes runner containers and performs provider credential -cleanup. Plugin uninstall and credential removal also retain caches because -they use Stop. +cleanup. Plugin uninstall and active GitLab runner-token removal also retain +caches because they use Stop. Stop does not guarantee that active jobs finish. Schedule maintenance before -using Stop or Restart. Autoscale-down and permanent slot retirement still delete -that slot's Docker data and GitLab job cache. Registry exports are unaffected. +using Stop or Restart. Manual scale-down, autoscale-down, and permanent slot +retirement still delete that slot's Docker data and GitLab job cache. Registry +exports are unaffected. To delete retained local caches, stop the fleet, then run the explicit command: From 52c98039bf38adb262bda9e4fffe1167aa6be5c4 Mon Sep 17 00:00:00 2001 From: Eli Bosley Date: Thu, 27 Aug 2026 12:38:40 -0400 Subject: [PATCH 16/21] chore(ai-review): record cache retention review --- .../ai-review-markers/feat-shared-build-cache-8c24ca91be08.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.limetech/ai-review-markers/feat-shared-build-cache-8c24ca91be08.json b/.limetech/ai-review-markers/feat-shared-build-cache-8c24ca91be08.json index 358ec32e..bb66c4df 100644 --- a/.limetech/ai-review-markers/feat-shared-build-cache-8c24ca91be08.json +++ b/.limetech/ai-review-markers/feat-shared-build-cache-8c24ca91be08.json @@ -1,7 +1,7 @@ { "disposition": "PATCH", "forecast_commit": "c214cf26f7868ce28f4f7db38bb9993264e3f764", - "reviewed_sha": "1347b90876b239ab5b0935d266b81cf3c0ed6c41", + "reviewed_sha": "2a7b0c4d2000a7440007f4b886af1597cb68fabd", "schema": "limetech.ai-review-marker.v2", "stage": "final", "unresolved_proportionality_findings": [] From 3082e96fc22d32f98841cb5cd2e22612063ea8fd Mon Sep 17 00:00:00 2001 From: Eli Bosley Date: Thu, 27 Aug 2026 15:05:14 -0400 Subject: [PATCH 17/21] test(cache): verify authenticated registry sharing on dispatch --- .github/workflows/lint.yml | 34 ++++++++++++++++++++++++++++++++ docs/build-cache.md | 31 +++++++++++++++++++++++++++++ tests/build-cache-integration.sh | 31 ++++++++++++++++++++++------- 3 files changed, 89 insertions(+), 7 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index baa44805..01d24d5e 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -10,6 +10,11 @@ on: branches: - main workflow_dispatch: + inputs: + registry_cache_test: + description: 'Also verify authenticated GHCR cache reuse (synthetic data only)' + type: boolean + default: false permissions: contents: read @@ -36,3 +41,32 @@ jobs: - name: Verify shared build cache with isolated builders run: bash tests/build-cache-integration.sh + + registry-cache: + name: Authenticated registry cache proof + if: github.event_name == 'workflow_dispatch' && inputs.registry_cache_test + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + packages: write + steps: + - name: Checkout trusted dispatch ref + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Verify cache export and import with job-owned credentials + shell: bash + env: + REGISTRY_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -euo pipefail + export DOCKER_CONFIG + DOCKER_CONFIG="$(mktemp -d "$RUNNER_TEMP/crf-registry-auth.XXXXXX")" + trap 'rm -rf -- "$DOCKER_CONFIG"' EXIT + printf '%s' "$REGISTRY_TOKEN" | docker login ghcr.io -u "$GITHUB_ACTOR" --password-stdin + unset REGISTRY_TOKEN + export CRF_CACHE_TEST_REPOSITORY="ghcr.io/${GITHUB_REPOSITORY,,}-build-cache" + export CRF_CACHE_TEST_TAG="proof-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + bash tests/build-cache-integration.sh diff --git a/docs/build-cache.md b/docs/build-cache.md index 29b3f265..b58ed569 100644 --- a/docs/build-cache.md +++ b/docs/build-cache.md @@ -140,6 +140,37 @@ deletion for both providers using disposable on-disk fixtures. a test registry to prove cross-builder cache reuse and effective GC settings. It does not contact a farm host or modify existing runners. +### Verify an authenticated registry + +The **Lint** workflow has a manual **Also verify authenticated GHCR cache reuse** +option. Run it only from a trusted, reviewed ref. Pull-request events cannot +start this package-writing job. GitHub assigns its runner. + +The job uses its short-lived `GITHUB_TOKEN` with `packages: write`, not a farm +credential. It exports synthetic Busybox layers to +`ghcr.io//-build-cache:proof--`. +Two fresh builders must have different local volumes, and the second must reuse +the exported `RUN` layer. Authentication or export errors fail the test. +The test never substitutes a local registry when the external registry fails. + +New GHCR packages are private by default. After the first run, verify that the +package remains private and is linked to the workflow repository. Keep package +access limited to authorized workflows. The job removes its temporary Docker +credentials and local test resources. It leaves the small proof tag for +inspection; the package owner controls retention. + +For another registry, authenticate through a temporary, job-owned Docker config. +Set `CRF_CACHE_TEST_REPOSITORY` to its tagless repository and +`CRF_CACHE_TEST_TAG` to a unique tag starting with `proof-`. Then run +`bash tests/build-cache-integration.sh`. Both inputs are required together. +Do not use the farm's host PAT, runner token, or image-pull credential. + +This check uses the plugin's generated profile in an isolated fixture. It does +not prove that a deployed runner has adopted its profile or that a production +workflow uses it. Verify the deployed read-only mount separately, then configure +each authorized build workflow as described above. Enabling the farm option +alone does not change existing build commands or reduce their old cache data. + On macOS, use `bash tests/run-linux-checks.sh` with Docker running and Homebrew Coreutils installed (`brew install coreutils`). The wrapper runs the shell suite in Linux. Its real-Docker integration stage uses the installed GNU tools on the diff --git a/tests/build-cache-integration.sh b/tests/build-cache-integration.sh index 9104c169..6771c392 100755 --- a/tests/build-cache-integration.sh +++ b/tests/build-cache-integration.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash # Real registry reuse across two isolated builders. Never prunes a Docker daemon. +# External mode uses caller-owned Docker auth and uploads only this dummy context. set -euo pipefail cd "$(dirname "$0")/.." if [ "$(uname -s)" = Darwin ]; then @@ -14,6 +15,16 @@ if [ "$(uname -s)" = Darwin ]; then } export PATH="$coreutils_bin:$PATH" fi +external_registry=false +if [ -n "${CRF_CACHE_TEST_REPOSITORY+x}${CRF_CACHE_TEST_TAG+x}" ]; then + : "${CRF_CACHE_TEST_REPOSITORY:?External cache test requires a repository}" + : "${CRF_CACHE_TEST_TAG:?External cache test requires a unique proof tag}" + [[ "$CRF_CACHE_TEST_TAG" =~ ^proof-[a-z0-9][a-z0-9_.-]{0,100}$ ]] || { + echo 'External cache test tag must start with proof- and contain a safe, unique suffix.' >&2 + exit 1 + } + external_registry=true +fi # DinD mounts its own tmpfs over /tmp, hiding nested bind sources there. Keep # the fixture outside that overlay and preserve identical host/sidecar paths. tmp="$(mktemp -d /var/tmp/crf-build-cache.XXXXXX)" @@ -37,16 +48,19 @@ mkdir -p "$CRF_CFGDIR" "$CRF_RUNDIR" # shellcheck source=/dev/null source src/usr/local/emhttp/plugins/ci-runner-farm/include/runner-farm.sh BUILD_CACHE_MODE=registry -BUILD_CACHE_REPOSITORY=cache.test:5000/team/cache +BUILD_CACHE_REPOSITORY="${CRF_CACHE_TEST_REPOSITORY:-cache.test:5000/team/cache}" BUILD_CACHE_LOCAL_GIB=20 profile="$(build_cache_profile)" +cache_ref="$BUILD_CACHE_REPOSITORY:${CRF_CACHE_TEST_TAG:-project-target-platform-main}" cp "$profile/buildkitd.toml" "$tmp/buildkitd.toml" -# Transport override belongs only to this disposable test, not the farm profile. -printf '\n[registry."cache.test:5000"]\n http = true\n' >> "$tmp/buildkitd.toml" registry_image='registry@sha256:a3d8aaa63ed8681a604f1dea0aa03f100d5895b6a58ace528858a7b332415373' buildkit_image='moby/buildkit@sha256:28a898719c18a33f4e8000685287fa36fd0dd9560c6440227d3a732d79bb41d8' network_id="$(docker network create "$network")" -registry_id="$(docker run -d --name "$registry" --network "$network" --network-alias cache.test "$registry_image")" +if [ "$external_registry" = false ]; then + # Plain HTTP belongs only to the disposable registry, never an external one. + printf '\n[registry."cache.test:5000"]\n http = true\n' >> "$tmp/buildkitd.toml" + registry_id="$(docker run -d --name "$registry" --network "$network" --network-alias cache.test "$registry_image")" +fi mkdir "$tmp/context" printf 'FROM busybox:1.37.0\nRUN echo registry-cache-proof > /proof\n' > "$tmp/context/Dockerfile" for n in 1 2; do @@ -71,9 +85,9 @@ for n in 1 2; do docker inspect -f '{{range .Mounts}}{{if eq .Destination "/var/lib/buildkit"}}{{.Name}}{{end}}{{end}}' \ "$container" > "$tmp/volume-$n" args=(--builder "$builder" --progress=plain - --cache-to "type=registry,ref=$BUILD_CACHE_REPOSITORY:project-target-platform-main,mode=max,image-manifest=true") + --cache-to "type=registry,ref=$cache_ref,mode=max,image-manifest=true") if [ "$n" = 2 ]; then - args+=(--cache-from "type=registry,ref=$BUILD_CACHE_REPOSITORY:project-target-platform-main") + args+=(--cache-from "type=registry,ref=$cache_ref") fi if ! docker buildx build "${args[@]}" "$tmp/context" > "$tmp/build-$n.log" 2>&1; then tail -35 "$tmp/build-$n.log" >&2 @@ -88,7 +102,7 @@ if cmp -s "$tmp/volume-1" "$tmp/volume-2"; then fi # The RUN operation, not just base-image metadata, must come from registry cache. awk '/RUN echo registry-cache-proof/ {step=$1} $1 == step && $2 == "CACHED" {found=1} END {exit !found}' "$tmp/build-2.log" -grep -q 'importing cache manifest from cache.test:5000/team/cache:project-target-platform-main' "$tmp/build-2.log" +grep -Fq "importing cache manifest from $cache_ref" "$tmp/build-2.log" docker run --rm --mount "type=bind,src=$profile,dst=/etc/ci-runner-farm/build-cache,readonly" \ busybox:1.37.0 sh -c '. /etc/ci-runner-farm/build-cache/profile.env && test "$CRF_BUILD_CACHE_MODE" = registry && @@ -118,3 +132,6 @@ else exit "$result" fi echo 'build-cache-integration: independent cache reuse, effective GC budgets, and direct/nested read-only profiles: OK' +if [ "$external_registry" = true ]; then + printf 'External registry proof retained at %s (synthetic data only).\n' "$cache_ref" +fi From 3541a35f4b27c72158b397cfc692726dfedb6b38 Mon Sep 17 00:00:00 2001 From: Eli Bosley Date: Thu, 27 Aug 2026 15:08:23 -0400 Subject: [PATCH 18/21] chore(ai-review): record registry proof review --- .../ai-review-markers/feat-shared-build-cache-8c24ca91be08.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.limetech/ai-review-markers/feat-shared-build-cache-8c24ca91be08.json b/.limetech/ai-review-markers/feat-shared-build-cache-8c24ca91be08.json index bb66c4df..21104f86 100644 --- a/.limetech/ai-review-markers/feat-shared-build-cache-8c24ca91be08.json +++ b/.limetech/ai-review-markers/feat-shared-build-cache-8c24ca91be08.json @@ -1,7 +1,7 @@ { "disposition": "PATCH", "forecast_commit": "c214cf26f7868ce28f4f7db38bb9993264e3f764", - "reviewed_sha": "2a7b0c4d2000a7440007f4b886af1597cb68fabd", + "reviewed_sha": "2a37be66c02936475a6d3fe3f823c8ec849f3d87", "schema": "limetech.ai-review-marker.v2", "stage": "final", "unresolved_proportionality_findings": [] From ec0653ecf661eb7bab8dbd5a3a989d9cb568bbaf Mon Sep 17 00:00:00 2001 From: Eli Bosley Date: Thu, 27 Aug 2026 15:10:39 -0400 Subject: [PATCH 19/21] docs(cache): require observed registry visibility before activation --- docs/build-cache.md | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/docs/build-cache.md b/docs/build-cache.md index b58ed569..85c4f343 100644 --- a/docs/build-cache.md +++ b/docs/build-cache.md @@ -153,11 +153,16 @@ Two fresh builders must have different local volumes, and the second must reuse the exported `RUN` layer. Authentication or export errors fail the test. The test never substitutes a local registry when the external registry fails. -New GHCR packages are private by default. After the first run, verify that the -package remains private and is linked to the workflow repository. Keep package -access limited to authorized workflows. The job removes its temporary Docker -credentials and local test resources. It leaves the small proof tag for -inspection; the package owner controls retention. +Inspect the package's actual visibility, repository link, and workflow access +after the run. Do not assume that it is private. This repository's initial +synthetic proof produced a public package. Never use a public proof repository +for private build caches. A private production cache needs its own authorized +repository and access policy before the farm points to it. + +The job removes its temporary Docker credentials and local test resources. It +leaves the small proof tag for inspection; the package owner controls retention. +A successful proof against a public package establishes authenticated exports +and cache reuse, but does not establish private-package read authorization. For another registry, authenticate through a temporary, job-owned Docker config. Set `CRF_CACHE_TEST_REPOSITORY` to its tagless repository and From 72619fb8519755d0146ee0d087bb2080a76a79b2 Mon Sep 17 00:00:00 2001 From: Eli Bosley Date: Thu, 27 Aug 2026 15:11:41 -0400 Subject: [PATCH 20/21] chore(ai-review): record registry visibility review --- .../ai-review-markers/feat-shared-build-cache-8c24ca91be08.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.limetech/ai-review-markers/feat-shared-build-cache-8c24ca91be08.json b/.limetech/ai-review-markers/feat-shared-build-cache-8c24ca91be08.json index 21104f86..c8672c06 100644 --- a/.limetech/ai-review-markers/feat-shared-build-cache-8c24ca91be08.json +++ b/.limetech/ai-review-markers/feat-shared-build-cache-8c24ca91be08.json @@ -1,7 +1,7 @@ { "disposition": "PATCH", "forecast_commit": "c214cf26f7868ce28f4f7db38bb9993264e3f764", - "reviewed_sha": "2a37be66c02936475a6d3fe3f823c8ec849f3d87", + "reviewed_sha": "8b5ad92f18df6276a0dc4562f6cfbd033259f5cb", "schema": "limetech.ai-review-marker.v2", "stage": "final", "unresolved_proportionality_findings": [] From 0ac920cbc81640c935762b9758bcb395d70fa2b1 Mon Sep 17 00:00:00 2001 From: Eli Bosley Date: Mon, 21 Sep 2026 17:41:26 -0400 Subject: [PATCH 21/21] fix(security): bound retained cache and registry proof trust - Purpose: prevent retained runner state and package-writing proof jobs from crossing trust boundaries.\n- Before: stopped slots reused per-slot state without an identity check, and manual registry proof dispatches could run from any ref.\n- Problem: provider, project, registry, or workflow-code changes could inherit private data or package-write authority.\n- New behavior: record provider/config identity, purge per-slot state on mismatch, and permit registry proof only from main.\n- How: validate cache paths, atomically store identities, purge on replacement, and cover the lifecycle with regression tests. --- .github/workflows/lint.yml | 2 +- docs/build-cache.md | 10 +- .../ci-runner-farm/include/runner-farm.sh | 120 ++++++++++++++++-- tests/cache-retention.sh | 29 ++++- tests/provider-mocks.sh | 7 + 5 files changed, 153 insertions(+), 15 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 01d24d5e..54a6763c 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -44,7 +44,7 @@ jobs: registry-cache: name: Authenticated registry cache proof - if: github.event_name == 'workflow_dispatch' && inputs.registry_cache_test + if: github.event_name == 'workflow_dispatch' && inputs.registry_cache_test && github.ref == 'refs/heads/main' runs-on: ubuntu-latest timeout-minutes: 10 permissions: diff --git a/docs/build-cache.md b/docs/build-cache.md index 85c4f343..7a7c5d06 100644 --- a/docs/build-cache.md +++ b/docs/build-cache.md @@ -111,6 +111,11 @@ Named pools require the same scheduled Fleet Restart described above. **Stop** and **Restart** retain per-slot Docker data, GitLab job caches, and the shared image mirror cache. A later Start reuses the retained data for the same slot and cache root. This applies with the build cache profile on or off. +The plugin records the provider and baked runner configuration identity for each +retained slot. If that identity changes, including a provider, owner, project, +registry, or trust-scope change, the plugin purges that slot's retained Docker, +workspace, job-cache, socket, and log data before starting it again. The shared +image mirror remains separate and is not treated as per-slot job state. The plugin still removes runner containers and performs provider credential cleanup. Plugin uninstall and active GitLab runner-token removal also retain caches because they use Stop. @@ -143,8 +148,9 @@ It does not contact a farm host or modify existing runners. ### Verify an authenticated registry The **Lint** workflow has a manual **Also verify authenticated GHCR cache reuse** -option. Run it only from a trusted, reviewed ref. Pull-request events cannot -start this package-writing job. GitHub assigns its runner. +option. The package-writing job accepts dispatches only from the repository's +`main` branch. Pull-request events and other refs cannot start it. Review the +exact `main` commit before dispatching. GitHub assigns its runner. The job uses its short-lived `GITHUB_TOKEN` with `packages: write`, not a farm credential. It exports synthetic Busybox layers to diff --git a/src/usr/local/emhttp/plugins/ci-runner-farm/include/runner-farm.sh b/src/usr/local/emhttp/plugins/ci-runner-farm/include/runner-farm.sh index 8272b080..e92bf5c9 100755 --- a/src/usr/local/emhttp/plugins/ci-runner-farm/include/runner-farm.sh +++ b/src/usr/local/emhttp/plugins/ci-runner-farm/include/runner-farm.sh @@ -1970,14 +1970,18 @@ clear_args_tmpdir() { } start_one() { - local idx="$1" name="${NAME_PREFIX}-$1" snapshot + local idx="$1" name="${NAME_PREFIX}-$1" snapshot current_gen if docker inspect "$name" >/dev/null 2>&1; then snapshot="$(managed_runner_snapshot "$name")" \ || { err "refusing fixed-name collision while starting $name"; return 1; } log "owned runner $name already exists; skipping" return 0 fi - provider_call start_one "$idx" "$name" + current_gen="$(crf_confgen)" || return 1 + crf_prepare_slot_cache "$name" "$CI_PROVIDER" "$current_gen" || return 1 + provider_call start_one "$idx" "$name" || return 1 + crf_record_cache_identity "$name" "$CI_PROVIDER" "$current_gen" \ + || { err "runner $name started, but its cache identity could not be recorded; future starts will purge its retained data"; return 1; } } start_configured_capacity() { @@ -2006,12 +2010,18 @@ start_configured_capacity() { # needs a fresh short-lived registration token; a stale/provider-switched GitLab # manager must unregister its old persisted identity before replacement. recreate_stopped_runner() { - local c="$1" supplied_snapshot="${2:-}" snapshot id provider role idx gen pool + local c="$1" supplied_snapshot="${2:-}" snapshot id provider role idx gen pool current_gen purge=true snapshot="$supplied_snapshot" [ -n "$snapshot" ] || snapshot="$(managed_runner_snapshot "$c")" || return 1 IFS='|' read -r id provider role idx gen <<< "$snapshot" pool="$(runner_pool "$c")" || return 1 - remove_runner "$c" false "$id" "$provider" || return 1 + current_gen="$(expected_runner_confgen "$c")" || return 1 + if [ "$provider" = "$CI_PROVIDER" ] \ + && [ "$gen" = "$current_gen" ] \ + && crf_cache_identity_matches "$c" "$provider" "$gen"; then + purge=false + fi + remove_runner "$c" "$purge" "$id" "$provider" || return 1 pool_activate "$pool" || { err "runner $c belongs to unknown pool $pool"; return 1; } start_one "$idx" } @@ -2022,7 +2032,7 @@ recreate_stopped_runner() { # routine outage would create needless remote churn and make recovery depend on # GitLab availability. start_stopped_managed() { - local c st provider idx names snapshot id role gen + local c st provider idx names snapshot id role gen current_gen names="$(managed_names)" || return 1 for c in $names; do [ -n "$c" ] || continue @@ -2031,8 +2041,10 @@ start_stopped_managed() { st="$(docker inspect -f '{{.State.Running}}' "$id" 2>/dev/null)" \ || { err "could not inspect stopped/running state for owned runner $c"; return 1; } [ "$st" = "true" ] && continue + current_gen="$(expected_runner_confgen "$c")" || return 1 if [ "$provider" = gitlab ] && [ "$CI_PROVIDER" = gitlab ] \ - && [ "$gen" = "$(crf_confgen)" ]; then + && [ "$gen" = "$current_gen" ] \ + && crf_cache_identity_matches "$c" "$provider" "$gen"; then log "restarting stopped GitLab manager $c with its persisted system ID" gitlab_start_stopped "$c" "$idx" "$id" || return 1 else @@ -2405,7 +2417,10 @@ remove_runner() { CRF_REMOVE_SLOT="$c" CRF_REMOVE_ID="$immutable_id" CRF_REMOVE_PROVIDER="$provider" - "${provider}_remove_runner" "$c" "$purge" + "${provider}_remove_runner" "$c" "$purge" || return 1 + if [ "$purge" = true ]; then + crf_purge_slot_cache "$c" + fi } # Stop every running GitLab manager concurrently before a full fleet teardown. @@ -2476,7 +2491,7 @@ cmd_stop() { lifecycle_stop || return 1 reconcile_stop || return 1 quiesce_gitlab_managers_for_stop || return 1 - local names c remaining remaining_managers stop_failed=0 + local names c remaining remaining_managers stop_failed=0 snapshot id provider role index gen names="$(managed_names)" \ || { err "could not enumerate managed runners before stop"; return 1; } if [ -z "$names" ]; then @@ -2485,7 +2500,14 @@ cmd_stop() { while IFS= read -r c; do [ -n "$c" ] || continue log "stopping $c (graceful deregister)" - remove_runner "$c" false || stop_failed=1 + snapshot="$(managed_runner_snapshot "$c")" || { stop_failed=1; continue; } + IFS='|' read -r id provider role index gen <<< "$snapshot" + if remove_runner "$c" false; then + crf_record_cache_identity "$c" "$provider" "$gen" \ + || { err "could not record retained cache identity for $c; future starts will purge its per-slot data"; stop_failed=1; } + else + stop_failed=1 + fi done <<< "$names" fi # A provider removal that failed closed can intentionally leave a manager and @@ -3027,6 +3049,84 @@ crf_safe_cache_root() { esac } +# Retained per-slot data is reusable only when provider and baked runner +# configuration still match. Missing identity is treated as unsafe legacy data. +crf_cache_slot_path() { + local kind="$1" name="$2" root path real + case "$kind" in + docker|work|dind-logs|gitlab-cache|gitlab-sockets) ;; + *) return 1 ;; + esac + case "$name" in ''|*[!A-Za-z0-9_.-]*) return 1 ;; esac + root="$(crf_safe_cache_root)" || return 1 + path="$root/$kind/$name" + real="$(realpath -m -- "$path" 2>/dev/null)" || return 1 + [ "$real" = "$path" ] || return 1 + case "$real" in "$root"/*) printf '%s\n' "$real" ;; *) return 1 ;; esac +} + +crf_cache_identity_path() { + local name="$1" root + case "$name" in ''|*[!A-Za-z0-9_.-]*) return 1 ;; esac + root="$(crf_safe_cache_root)" || return 1 + printf '%s/.crf-cache-identities/%s\n' "$root" "$name" +} + +crf_cache_identity_matches() { + local name="$1" provider="$2" gen="$3" path recorded_provider recorded_gen extra kind + path="$(crf_cache_identity_path "$name")" || return 1 + [ -f "$path" ] && [ ! -L "$path" ] || return 1 + IFS='|' read -r recorded_provider recorded_gen extra < "$path" || return 1 + [ -n "$recorded_provider" ] && [ -n "$recorded_gen" ] && [ -z "$extra" ] \ + || return 1 + [ "$recorded_provider" = "$provider" ] && [ "$recorded_gen" = "$gen" ] || return 1 + for kind in docker work dind-logs gitlab-cache gitlab-sockets; do + crf_cache_slot_path "$kind" "$name" >/dev/null || return 1 + done +} + +crf_record_cache_identity() { + local name="$1" provider="$2" gen="$3" path dir tmp + path="$(crf_cache_identity_path "$name")" || return 1 + dir="${path%/*}" + [ ! -L "$dir" ] || return 1 + mkdir -p "$dir" && chmod 700 "$dir" 2>/dev/null || return 1 + [ ! -L "$path" ] || return 1 + tmp="$(mktemp "$dir/.identity.XXXXXX")" || return 1 + if ! ( umask 077; printf '%s|%s\n' "$provider" "$gen" > "$tmp" ); then + rm -f -- "$tmp" + return 1 + fi + chmod 600 "$tmp" 2>/dev/null || { rm -f -- "$tmp"; return 1; } + mv -f -- "$tmp" "$path" || { rm -f -- "$tmp"; return 1; } +} + +crf_forget_cache_identity() { + local path + path="$(crf_cache_identity_path "$1")" || return 1 + rm -f -- "$path" +} + +crf_purge_slot_cache() { + local name="$1" kind path failed=0 + for kind in docker work dind-logs gitlab-cache gitlab-sockets; do + path="$(crf_cache_slot_path "$kind" "$name")" \ + || { err "refusing cache cleanup for unsafe slot '$name'"; failed=1; continue; } + if [ -e "$path" ] || [ -L "$path" ]; then + rm -rf -- "$path" || failed=1 + fi + done + crf_forget_cache_identity "$name" || failed=1 + return "$failed" +} + +crf_prepare_slot_cache() { + local name="$1" provider="$2" gen="$3" + crf_cache_identity_matches "$name" "$provider" "$gen" && return 0 + log "cache identity for $name is absent or changed; purging retained per-slot data" + crf_purge_slot_cache "$name" +} + # Resolve a CACHE_MOUNTS host subdir against the (canonical) cache root and confirm # it stays UNDER that root — rejecting `../` traversal or absolute paths in the # space-separated, web-settable CACHE_MOUNTS list before they reach mkdir/chown -R @@ -3703,7 +3803,7 @@ cmd_prune_cache() { [ -z "$active" ] \ || { err "refusing to prune cache while GitLab executor containers exist"; return 1; } root="$(crf_safe_cache_root)" || { err "refusing to prune-cache: CACHE_ROOT='$CACHE_ROOT' is unsafe (system dir, share/pool root, or unresolvable — point it at /mnt//)"; return 1; } - dirs="docker work dind-logs gitlab-cache gitlab-sockets registry-mirror $CACHE_PKG_DIRS" + dirs="docker work dind-logs gitlab-cache gitlab-sockets registry-mirror .crf-cache-identities $CACHE_PKG_DIRS" for m in $CACHE_MOUNTS; do dirs="$dirs ${m%%:*}"; done for d in $dirs; do case "$d" in ''|.|..|*/*) continue ;; esac # simple child names only — never a path/traversal diff --git a/tests/cache-retention.sh b/tests/cache-retention.sh index 7a0e26fd..b89ca45e 100755 --- a/tests/cache-retention.sh +++ b/tests/cache-retention.sh @@ -19,18 +19,43 @@ for provider in github gitlab; do CACHE_MOUNTS='' NO_REGISTER=1 slot=ci-runner-1 + crf_safe_cache_root() { printf '%s\n' "$CACHE_ROOT"; } fixture_id=$(printf '%064d' 1) present="$tmp/$provider/present" effects="$tmp/$provider/effects" touch "$present" "$effects" - for dir in docker gitlab-cache registry-mirror; do + for dir in docker work dind-logs gitlab-cache gitlab-sockets registry-mirror; do + mkdir -p "$CACHE_ROOT/$dir/$slot" + printf 'retained\n' > "$CACHE_ROOT/$dir/$slot/proof" + done + + # Unknown or changed identity must never reuse retained per-slot state. + crf_prepare_slot_cache "$slot" "$fixture_provider" old-generation || fail 'legacy cache identity was reused' + for dir in docker work dind-logs gitlab-cache gitlab-sockets; do + [ ! -e "$CACHE_ROOT/$dir/$slot" ] || fail "identity purge retained $dir data" + done + [ -f "$CACHE_ROOT/registry-mirror/$slot/proof" ] || fail 'identity purge deleted shared mirror data' + for dir in docker work dind-logs gitlab-cache gitlab-sockets; do mkdir -p "$CACHE_ROOT/$dir/$slot" printf 'retained\n' > "$CACHE_ROOT/$dir/$slot/proof" done + crf_record_cache_identity "$slot" "$fixture_provider" old-generation \ + || fail 'could not record cache identity' + crf_prepare_slot_cache "$slot" "$fixture_provider" old-generation \ + || fail 'matching cache identity was not reusable' + crf_prepare_slot_cache "$slot" "$fixture_provider" new-generation \ + || fail 'changed cache identity could not be purged' + [ ! -e "$CACHE_ROOT/docker/$slot" ] || fail 'changed identity retained Docker data' + [ -f "$CACHE_ROOT/registry-mirror/$slot/proof" ] || fail 'changed identity deleted shared mirror data' + for dir in docker work dind-logs gitlab-cache gitlab-sockets; do + mkdir -p "$CACHE_ROOT/$dir/$slot" + printf 'retained\n' > "$CACHE_ROOT/$dir/$slot/proof" + done + crf_record_cache_identity "$slot" "$fixture_provider" generation \ + || fail 'could not restore current cache identity' # Replace external Docker/API/process boundaries, not Stop, remove_runner, # the provider removal adapters, or their cache retention decisions. - crf_safe_cache_root() { printf '%s\n' "$CACHE_ROOT"; } boot_autostart_stop() { return 0; } autoscale_stop() { return 0; } imageupdate_stop() { return 0; } diff --git a/tests/provider-mocks.sh b/tests/provider-mocks.sh index 965aa242..fa623ead 100755 --- a/tests/provider-mocks.sh +++ b/tests/provider-mocks.sh @@ -1166,6 +1166,12 @@ fi DISPATCH_LOG="$tmp/start-stopped-dispatch.log" DISPATCH_MANAGER_ID=bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb CI_PROVIDER=gitlab + crf_safe_cache_root() { printf '%s\n' "$CACHE_ROOT"; } + for cache_dir in docker work dind-logs gitlab-cache gitlab-sockets; do + mkdir -p "$CACHE_ROOT/$cache_dir/ci-runner-1" + done + crf_record_cache_identity ci-runner-1 gitlab dispatch-generation \ + || fail "could not record stopped-manager cache identity" managed_names() { printf '%s\n' ci-runner-1; } managed_runner_snapshot() { printf '%s|gitlab|manager|1|dispatch-generation\n' "$DISPATCH_MANAGER_ID" @@ -1608,6 +1614,7 @@ grep -qx 'recycle ci-runner-1' "$ACTION_LOG" || fail "image update bypassed cmd_ # than weakening isolation underneath the survivor. : > "$ACTION_LOG" managed_names() { printf '%s\n' ci-runner-1; } +managed_runner_snapshot() { printf '%s|gitlab|manager|1|stop-generation\n' manager-id; } remove_runner() { printf 'remove %s\n' "$1" >> "$ACTION_LOG"; return 1; } autoscale_stop() { :; }; imageupdate_stop() { :; } quiesce_gitlab_managers_for_stop() { :; }