diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1d16c99..93c9f22 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,24 +4,27 @@ on: pull_request: push: branches: [main] + workflow_dispatch: permissions: contents: read jobs: - # Each test suite runs only when its action changed, or when this workflow did. + # Each test suite runs only when its action changed, or when this workflow did. A manual run runs them all. changes: runs-on: ubuntu-latest permissions: contents: read pull-requests: read outputs: - review: ${{ steps.filter.outputs.review }} - ci_metrics: ${{ steps.filter.outputs.ci_metrics }} - plan_ci_tier: ${{ steps.filter.outputs.plan_ci_tier }} + review: ${{ github.event_name == 'workflow_dispatch' && 'true' || steps.filter.outputs.review }} + ci_metrics: ${{ github.event_name == 'workflow_dispatch' && 'true' || steps.filter.outputs.ci_metrics }} + plan_ci_tier: ${{ github.event_name == 'workflow_dispatch' && 'true' || steps.filter.outputs.plan_ci_tier }} + setup_ccache: ${{ github.event_name == 'workflow_dispatch' && 'true' || steps.filter.outputs.setup_ccache }} steps: - uses: actions/checkout@v4.1.2 - uses: dorny/paths-filter@v3 + if: github.event_name != 'workflow_dispatch' id: filter with: filters: | @@ -36,6 +39,9 @@ jobs: plan_ci_tier: - .github/workflows/ci.yml - PlanCITier/** + setup_ccache: + - .github/workflows/ci.yml + - SetupCcache/** review-tests: name: Review action tests @@ -71,11 +77,22 @@ jobs: node-version: 20 - run: node --test PlanCITier/ + # The end-to-end smoke test needs the org's private ccache stores and the ubuntu-iar runners, + # which this public repository can't use. It runs in StaflSystems/ci-smoke-tests instead. + setup-ccache-lint: + name: SetupCcache lint + needs: changes + if: needs.changes.outputs.setup_ccache == 'true' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4.1.2 + - run: shellcheck SetupCcache/*.sh + # The required check. It runs on every PR and passes only if each suite that should run passed # and the rest were skipped. ci-status: name: CI Status - needs: [changes, review-tests, ci-metrics-tests, plan-ci-tier-tests] + needs: [changes, review-tests, ci-metrics-tests, plan-ci-tier-tests, setup-ccache-lint] if: always() && !cancelled() runs-on: ubuntu-latest steps: @@ -85,9 +102,11 @@ jobs: REVIEW: ${{ needs.review-tests.result }} CI_METRICS: ${{ needs.ci-metrics-tests.result }} PLAN_CI_TIER: ${{ needs.plan-ci-tier-tests.result }} + SETUP_CCACHE: ${{ needs.setup-ccache-lint.result }} RUN_REVIEW: ${{ needs.changes.outputs.review }} RUN_CI_METRICS: ${{ needs.changes.outputs.ci_metrics }} RUN_PLAN_CI_TIER: ${{ needs.changes.outputs.plan_ci_tier }} + RUN_SETUP_CCACHE: ${{ needs.changes.outputs.setup_ccache }} run: | failed=0 expect() { # job, its result, whether it should run @@ -102,4 +121,5 @@ jobs: expect "Review action tests" "$REVIEW" "$RUN_REVIEW" expect "CIMetrics tests" "$CI_METRICS" "$RUN_CI_METRICS" expect "PlanCITier tests" "$PLAN_CI_TIER" "$RUN_PLAN_CI_TIER" + expect "SetupCcache lint" "$SETUP_CCACHE" "$RUN_SETUP_CCACHE" exit $failed diff --git a/SetupCcache/README.md b/SetupCcache/README.md new file mode 100644 index 0000000..aa326f0 --- /dev/null +++ b/SetupCcache/README.md @@ -0,0 +1,88 @@ +# Setup ccache + +Sets up ccache for a C/C++ build job. This implements Proposal 2 of the CI spend RFC: one shared +ccache store per kind of runner, which every branch reads and writes, instead of the GitHub Actions +cache. Mid-stack PRs then reuse objects built by the PRs below them, and the Actions cache quota +stops mattering. + +| Runner | Store | How the job reaches it | +| --- | --- | --- | +| Self-hosted (`ubuntu-iar`) | Stafl-EMB03 on the office LAN (`LAN_CCACHE_URL`) | Plain HTTP with basic auth; the server's firewall only lets Stafl-EMB02 in | +| GitHub-hosted | `ccache1.ci.emb.staflsystems.com` on AWS (`AWS_CCACHE_URL`) | A ghostunnel client on `localhost:8080`, which forwards over TLS. ccache's HTTP backend can't speak HTTPS | + +The two kinds of runner build with different images and compilers, so they never share an object +and splitting the store costs no hits. Server setup: [CI Servers](https://staflsystems.atlassian.net/wiki/spaces/EM/pages/193363969/CI+Servers) +(LAN) and [Embedded AWS Ccache](https://staflsystems.atlassian.net/wiki/spaces/EM/pages/2279080022/Embedded+AWS+Ccache). + +## Modes + +`mode` is usually the repository variable `CCACHE_MODE`, so switching a repo needs no PR, and +switching back is the rollback. + +- **`actions`** (default): today's behavior. ccache caches in the GitHub Actions cache through + `hendrikmuhs/ccache-action`, under `key`. +- **`remote`**: ccache uses the shared store for this runner, and the Actions cache isn't touched. + The job zeroes ccache's stats at the start and prints `ccache -s` at the end, which + `CIMetrics --ccache` reads. + +`write-only: 'true'` compiles every file and writes the result, without reading the store +(ccache's `recache`). Set it on pushes to main: only main tags releases, so tagged binaries never +reuse an object a PR wrote, and main still seeds the store for the next PRs. In `actions` mode it +skips the restore, as the BMS2000 repos' `restore: ${{ github.ref != 'refs/heads/main' }}` does today. + +If the store can't be reached, the job warns and builds without it. ccache also treats a store +error during the build as a miss, so an outage only makes builds slower. + +## Usage + +```yaml + steps: + - uses: actions/checkout@v4 + + - uses: StaflSystems/CustomGithubActions/SetupCcache@main + with: + mode: ${{ vars.CCACHE_MODE || 'actions' }} + key: ${{ github.job }}-${{ matrix.build-preset }} + write-only: ${{ github.ref == 'refs/heads/main' }} + fork: ${{ endsWith(matrix.configure-preset, '-iar') }} + aws-url: ${{ vars.AWS_CCACHE_URL }} + aws-password: ${{ secrets.AWS_CCACHE_HTTP_PASSWORD }} + lan-url: ${{ vars.LAN_CCACHE_URL }} + lan-password: ${{ secrets.LAN_CCACHE_HTTP_PASSWORD }} + + - run: cmake --preset ${{ matrix.configure-preset }} -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache +``` + +It replaces both the copied "setup ccache" step that downloads the IAR-capable fork (`fork: 'true'`) +and the `hendrikmuhs/ccache-action` step. It sets `CCACHE_CONFIGPATH` to `ccache_config.conf` in +the workspace when that file exists. Set `max_size` there only for `actions` mode; the shared stores +have their own limits. + +## ccache versions + +The jobs run three versions today: Ubuntu's 4.5.1 in the CI image, 4.9 in newer images, and the +IAR-capable fork (4.12). So the store is set with environment variables that all of them accept: + +- `CCACHE_SECONDARY_STORAGE`: the setting's name before 4.7, which renamed it `remote_storage`. + 4.7 and later still accept the old name. +- `CCACHE_REMOTE_ONLY=1`: skips the local cache, which starts empty in every job anyway. It needs + 4.7; older versions ignore unknown `CCACHE_*` variables. Don't put `remote_only` in + `ccache_config.conf`: ccache stops with an error on a config file setting it doesn't know. + +## Testing + +This repository's CI shellchecks the scripts on every PR that changes this action. + +The end-to-end smoke test runs in the private repository +[StaflSystems/ci-smoke-tests](https://github.com/StaflSystems/ci-smoke-tests), because this one is +public: the org's ccache variables and secrets are for private repositories only, and the +self-hosted runners don't take jobs from public repositories. Run it by hand against a branch +before merging a change here: + +```bash +gh workflow run setup-ccache-smoke.yml -R StaflSystems/ci-smoke-tests -f ref= +``` + +For each store and ccache version, it builds 20 files three times with `smoke.sh`: the first build +must miss and write to the store, the second must hit in the store after the local cache is +cleared, and a third, with the store unreachable, must still pass. diff --git a/SetupCcache/action.yml b/SetupCcache/action.yml new file mode 100644 index 0000000..e84629f --- /dev/null +++ b/SetupCcache/action.yml @@ -0,0 +1,86 @@ +name: Setup ccache +description: > + Sets up ccache for a C/C++ build job, caching either in the GitHub Actions cache or in the shared + ccache stores: the office LAN store for self-hosted runners, the AWS store for GitHub-hosted ones. + +inputs: + mode: + description: > + 'actions' caches in the GitHub Actions cache, as before. 'remote' uses the shared store and + skips the Actions cache. Usually the repository variable CCACHE_MODE. + default: actions + key: + description: Actions-cache key, as for hendrikmuhs/ccache-action. Only used in 'actions' mode. + default: '' + write-only: + description: > + 'true' compiles every file and writes the results without reading any. Set it on pushes to + main, so tagged builds never reuse an object a PR wrote. In 'actions' mode it skips the restore. + default: 'false' + fork: + description: "'true' installs StaflSystems' IAR-capable ccache fork instead of the system ccache." + default: 'false' + config-path: + description: ccache config file, relative to the workspace. Skipped if it doesn't exist. + default: ccache_config.conf + aws-url: + description: The AWS store, for GitHub-hosted runners. Pass vars.AWS_CCACHE_URL. + default: '' + aws-password: + description: Password of the AWS store's ci user. Pass secrets.AWS_CCACHE_HTTP_PASSWORD. + default: '' + lan-url: + description: The office LAN store, for self-hosted runners. Pass vars.LAN_CCACHE_URL. + default: '' + lan-password: + description: Password of the LAN store's ci user. Pass secrets.LAN_CCACHE_HTTP_PASSWORD. + default: '' + +outputs: + store: + description: The store this job uses ('aws', 'lan'), or 'none' in 'actions' mode or if it couldn't reach one. + value: ${{ steps.remote.outputs.store || 'none' }} + +runs: + using: composite + steps: + - name: Install the IAR-capable ccache + if: inputs.fork == 'true' + shell: bash + run: | + dir="$RUNNER_TEMP/ccache-fork" + rm -rf "$dir" && mkdir -p "$dir" && cd "$dir" + curl -fsSL -o ccache.tar.xz \ + https://github.com/StaflSystems/ccache/releases/download/v4.12.2/ccache-iarsupport.26383df4-linux-x86_64.tar.xz + tar xf ccache.tar.xz + mv ccache-iarsupport.26383df4-linux-x86_64/ccache . + echo "$dir" >> "$GITHUB_PATH" + + - name: Use the repository's ccache config + shell: bash + env: + CONFIG_PATH: ${{ inputs.config-path }} + run: | + if [ -f "$GITHUB_WORKSPACE/$CONFIG_PATH" ]; then + echo "CCACHE_CONFIGPATH=$GITHUB_WORKSPACE/$CONFIG_PATH" >> "$GITHUB_ENV" + fi + + - name: Connect to the shared ccache store + id: remote + if: inputs.mode == 'remote' + shell: bash + env: + RUNNER_ENVIRONMENT: ${{ runner.environment }} + AWS_URL: ${{ inputs.aws-url }} + AWS_PASSWORD: ${{ inputs.aws-password }} + LAN_URL: ${{ inputs.lan-url }} + LAN_PASSWORD: ${{ inputs.lan-password }} + WRITE_ONLY: ${{ inputs.write-only }} + run: bash "$GITHUB_ACTION_PATH/connect.sh" + + - name: ccache + uses: hendrikmuhs/ccache-action@v1.2 + with: + key: ${{ inputs.key }} + restore: ${{ inputs.mode != 'remote' && inputs.write-only != 'true' }} + save: ${{ inputs.mode != 'remote' }} diff --git a/SetupCcache/connect.sh b/SetupCcache/connect.sh new file mode 100755 index 0000000..8cbcd5f --- /dev/null +++ b/SetupCcache/connect.sh @@ -0,0 +1,107 @@ +#!/usr/bin/env bash +# Points ccache at the shared store for this runner: the LAN store on self-hosted runners, the AWS +# store (through a local TLS tunnel) on GitHub-hosted ones. +# +# ccache's HTTP backend can't speak HTTPS, so on GitHub-hosted runners a ghostunnel client listens +# on localhost:8080 and forwards to the AWS store over TLS. +# +# Fails open: if the store can't be reached, the job warns and builds without it. +set -uo pipefail + +GHOSTUNNEL_VERSION=v1.11.3 +declare -A GHOSTUNNEL_SHA256=( + [amd64]=70766f99d6fd439cdf7912161f08a7d66a587a7fc7a79c1bce7fa332aec6fc89 + [arm64]=04d5c1855eeab6b9ae5fbdda3158604db82f3978ebb5958e601c70817c453576 +) +TUNNEL_PORT=8080 + +warn_and_skip() { + echo "::warning title=ccache::$1 Building without the shared ccache store." + echo "store=none" >> "$GITHUB_OUTPUT" + exit 0 +} + +# Percent-encode for the user-info part of a URL. +urlencode() { + local LC_ALL=C s="$1" out="" c i + for ((i = 0; i < ${#s}; i++)); do + c="${s:i:1}" + case "$c" in + [a-zA-Z0-9.~_-]) out+="$c" ;; + *) printf -v c '%%%02X' "'$c"; out+="$c" ;; + esac + done + printf '%s' "$out" +} + +# Succeeds once something accepts TCP connections on host:port, within about 15 seconds. +wait_for_port() { + for _ in $(seq 10); do + # timeout: a firewall that drops packets would otherwise hang the connect for minutes. + # shellcheck disable=SC2016 # $1 and $2 expand in the inner shell + timeout 1 bash -c 'exec 3<>"/dev/tcp/$1/$2"' _ "$1" "$2" 2>/dev/null && return 0 + sleep 0.5 + done + return 1 +} + +start_tunnel() { + local target="$1" arch bin + case "$(uname -m)" in + x86_64) arch=amd64 ;; + aarch64 | arm64) arch=arm64 ;; + *) return 1 ;; + esac + command -v curl > /dev/null || return 1 + bin="$RUNNER_TEMP/ghostunnel" + curl -fsSL -o "$bin" \ + "https://github.com/ghostunnel/ghostunnel/releases/download/$GHOSTUNNEL_VERSION/ghostunnel-linux-$arch" || return 1 + echo "${GHOSTUNNEL_SHA256[$arch]} $bin" | sha256sum -c --quiet - || return 1 + chmod +x "$bin" + # setsid and nohup keep the tunnel running after this step's shell exits; it ends with the job. + # The server is checked against the system trust store; the store has no client certificate. + nohup setsid "$bin" client --listen "localhost:$TUNNEL_PORT" --target "$target" \ + --disable-authentication > "$RUNNER_TEMP/ghostunnel.log" 2>&1 < /dev/null & + wait_for_port localhost "$TUNNEL_PORT" +} + +if [ "$RUNNER_ENVIRONMENT" = self-hosted ]; then + store=lan url="$LAN_URL" password="$LAN_PASSWORD" +else + store=aws url="$AWS_URL" password="$AWS_PASSWORD" +fi +if [ -z "$url" ] || [ -z "$password" ]; then + warn_and_skip "No URL or password for the $store store." +fi + +scheme="${url%%://*}" +hostport="${url#*://}" +hostport="${hostport%%/*}" + +if [ "$scheme" = https ]; then + [[ "$hostport" == *:* ]] || hostport="$hostport:443" + start_tunnel "$hostport" || warn_and_skip "Couldn't start the TLS tunnel to $hostport." + endpoint="localhost:$TUNNEL_PORT" +else + [[ "$hostport" == *:* ]] || hostport="$hostport:80" + wait_for_port "${hostport%:*}" "${hostport##*:}" || warn_and_skip "Couldn't reach $hostport." + endpoint="$hostport" +fi + +encoded="$(urlencode "$password")" +echo "::add-mask::$encoded" +{ + # SECONDARY_STORAGE is ccache 4.4–4.6's name for remote_storage; 4.7 and later still accept it. + echo "CCACHE_SECONDARY_STORAGE=http://ci:$encoded@$endpoint|layout=bazel|connect-timeout=1000" + # ccache 4.7 and later skip the local cache, which starts empty in every job anyway. + # Older versions ignore it. + echo "CCACHE_REMOTE_ONLY=1" + if [ "$WRITE_ONLY" = true ]; then + echo "CCACHE_RECACHE=1" + fi +} >> "$GITHUB_ENV" +echo "store=$store" >> "$GITHUB_OUTPUT" + +mode="reads and writes" +[ "$WRITE_ONLY" = true ] && mode="writes only" +echo "::notice title=ccache::Using the $store store ($hostport), $mode." diff --git a/SetupCcache/smoke.sh b/SetupCcache/smoke.sh new file mode 100755 index 0000000..3a76ca0 --- /dev/null +++ b/SetupCcache/smoke.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +# Smoke test for SetupCcache, run by .github/workflows/setup-ccache-smoke.yml in the private +# StaflSystems/ci-smoke-tests repository. +# +# smoke.sh miss first build: every file misses, and nothing errors writing to the store +# smoke.sh hit after clearing the local cache: every file hits in the store +# smoke.sh down store unreachable: the build still passes, with every file missing +# +# The sources include run_id.h, which holds the run ID, so a run never hits an earlier run's objects. +set -euo pipefail + +expect="$1" +cd "$GITHUB_WORKSPACE/smoke" + +if [ "$expect" = down ]; then + sed -i 's/"$/-down"/' run_id.h + # Nothing listens on port 9. + export CCACHE_SECONDARY_STORAGE="http://ci:x@127.0.0.1:9|layout=bazel|connect-timeout=200" +fi + +ccache -C > /dev/null +ccache -z > /dev/null +for f in f*.c; do + ccache gcc -c "$f" -o "${f%.c}.o" +done + +stat() { + # --print-stats keys: secondary_* in ccache 4.4–4.6, remote_* from 4.7. + local v + v="$(ccache --print-stats | awk -v a="$1" -v b="$2" '$1 == a || $1 == b { s += $2 } END { print s + 0 }')" + echo "${v:-0}" +} +sources=(f*.c) +files=${#sources[@]} +hits=$(($(stat direct_cache_hit _) + $(stat preprocessed_cache_hit _))) +misses=$(stat cache_miss _) +remote_hits=$(stat secondary_storage_hit remote_storage_hit) +errors=$(($(stat secondary_storage_error remote_storage_error) + $(stat secondary_storage_timeout remote_storage_timeout))) + +echo "$(ccache --version | head -1): $files files, $hits hits ($remote_hits from the store), $misses misses, $errors store errors or timeouts" + +fail() { + echo "::error title=SetupCcache smoke test::$expect: $1" + exit 1 +} +case "$expect" in + miss) + [ "$misses" -eq "$files" ] || fail "expected $files misses, got $misses" + [ "$errors" -eq 0 ] || fail "$errors store errors or timeouts" + ;; + hit) + # The local cache was just cleared, so every hit came from the store. Don't compare the store's + # own counter with the file count: ccache 4.5 counts a manifest and an object per file. + [ "$hits" -eq "$files" ] || fail "expected $files hits, got $hits" + [ "$remote_hits" -ge "$files" ] || fail "expected at least $files store hits, got $remote_hits" + [ "$errors" -eq 0 ] || fail "$errors store errors or timeouts" + ;; + down) + [ "$misses" -eq "$files" ] || fail "expected $files misses, got $misses" + ;; +esac