Add SetupCcache: use the shared ccache stores from CI jobs

.github/workflows/ci.yml

@@ -4,24 +4,27 @@ on:
   pull_request:
   push:
     branches: [main]
+  workflow_dispatch:
 
 permissions:
   contents: read
 
 jobs:
-  # Each test suite runs only when its action changed, or when this workflow did.
+  # Each test suite runs only when its action changed, or when this workflow did. A manual run runs them all.
   changes:
     runs-on: ubuntu-latest
     permissions:
       contents: read
       pull-requests: read
     outputs:
-      review: ${{ steps.filter.outputs.review }}
-      ci_metrics: ${{ steps.filter.outputs.ci_metrics }}
-      plan_ci_tier: ${{ steps.filter.outputs.plan_ci_tier }}
+      review: ${{ github.event_name == 'workflow_dispatch' && 'true' || steps.filter.outputs.review }}
+      ci_metrics: ${{ github.event_name == 'workflow_dispatch' && 'true' || steps.filter.outputs.ci_metrics }}
+      plan_ci_tier: ${{ github.event_name == 'workflow_dispatch' && 'true' || steps.filter.outputs.plan_ci_tier }}
+      setup_ccache: ${{ github.event_name == 'workflow_dispatch' && 'true' || steps.filter.outputs.setup_ccache }}
     steps:
       - uses: actions/checkout@v4.1.2
       - uses: dorny/paths-filter@v3
+        if: github.event_name != 'workflow_dispatch'
         id: filter
         with:
           filters: |
@@ -36,6 +39,9 @@ jobs:
             plan_ci_tier:
               - .github/workflows/ci.yml
               - PlanCITier/**
+            setup_ccache:
+              - .github/workflows/ci.yml
+              - SetupCcache/**
 
   review-tests:
     name: Review action tests
@@ -71,11 +77,22 @@ jobs:
           node-version: 20
       - run: node --test PlanCITier/
 
+  # The end-to-end smoke test needs the org's private ccache stores and the ubuntu-iar runners,
+  # which this public repository can't use. It runs in StaflSystems/ci-smoke-tests instead.
+  setup-ccache-lint:
+    name: SetupCcache lint
+    needs: changes
+    if: needs.changes.outputs.setup_ccache == 'true'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4.1.2
+      - run: shellcheck SetupCcache/*.sh
+
   # The required check. It runs on every PR and passes only if each suite that should run passed
   # and the rest were skipped.
   ci-status:
     name: CI Status
-    needs: [changes, review-tests, ci-metrics-tests, plan-ci-tier-tests]
+    needs: [changes, review-tests, ci-metrics-tests, plan-ci-tier-tests, setup-ccache-lint]
     if: always() && !cancelled()
     runs-on: ubuntu-latest
     steps:
@@ -85,9 +102,11 @@ jobs:
           REVIEW: ${{ needs.review-tests.result }}
           CI_METRICS: ${{ needs.ci-metrics-tests.result }}
           PLAN_CI_TIER: ${{ needs.plan-ci-tier-tests.result }}
+          SETUP_CCACHE: ${{ needs.setup-ccache-lint.result }}
           RUN_REVIEW: ${{ needs.changes.outputs.review }}
           RUN_CI_METRICS: ${{ needs.changes.outputs.ci_metrics }}
           RUN_PLAN_CI_TIER: ${{ needs.changes.outputs.plan_ci_tier }}
+          RUN_SETUP_CCACHE: ${{ needs.changes.outputs.setup_ccache }}
         run: |
           failed=0
           expect() { # job, its result, whether it should run
@@ -102,4 +121,5 @@ jobs:
           expect "Review action tests" "$REVIEW" "$RUN_REVIEW"
           expect "CIMetrics tests" "$CI_METRICS" "$RUN_CI_METRICS"
           expect "PlanCITier tests" "$PLAN_CI_TIER" "$RUN_PLAN_CI_TIER"
+          expect "SetupCcache lint" "$SETUP_CCACHE" "$RUN_SETUP_CCACHE"
           exit $failed

SetupCcache/README.md

@@ -0,0 +1,88 @@
+# Setup ccache
+
+Sets up ccache for a C/C++ build job. This implements Proposal 2 of the CI spend RFC: one shared
+ccache store per kind of runner, which every branch reads and writes, instead of the GitHub Actions
+cache. Mid-stack PRs then reuse objects built by the PRs below them, and the Actions cache quota
+stops mattering.
+
+| Runner | Store | How the job reaches it |
+| --- | --- | --- |
+| Self-hosted (`ubuntu-iar`) | Stafl-EMB03 on the office LAN (`LAN_CCACHE_URL`) | Plain HTTP with basic auth; the server's firewall only lets Stafl-EMB02 in |
+| GitHub-hosted | `ccache1.ci.emb.staflsystems.com` on AWS (`AWS_CCACHE_URL`) | A ghostunnel client on `localhost:8080`, which forwards over TLS. ccache's HTTP backend can't speak HTTPS |
+
+The two kinds of runner build with different images and compilers, so they never share an object
+and splitting the store costs no hits. Server setup: [CI Servers](https://staflsystems.atlassian.net/wiki/spaces/EM/pages/193363969/CI+Servers)
+(LAN) and [Embedded AWS Ccache](https://staflsystems.atlassian.net/wiki/spaces/EM/pages/2279080022/Embedded+AWS+Ccache).
+
+## Modes
+
+`mode` is usually the repository variable `CCACHE_MODE`, so switching a repo needs no PR, and
+switching back is the rollback.
+
+- **`actions`** (default): today's behavior. ccache caches in the GitHub Actions cache through
+  `hendrikmuhs/ccache-action`, under `key`.
+- **`remote`**: ccache uses the shared store for this runner, and the Actions cache isn't touched.
+  The job zeroes ccache's stats at the start and prints `ccache -s` at the end, which
+  `CIMetrics --ccache` reads.
+
+`write-only: 'true'` compiles every file and writes the result, without reading the store
+(ccache's `recache`). Set it on pushes to main: only main tags releases, so tagged binaries never
+reuse an object a PR wrote, and main still seeds the store for the next PRs. In `actions` mode it
+skips the restore, as the BMS2000 repos' `restore: ${{ github.ref != 'refs/heads/main' }}` does today.
+
+If the store can't be reached, the job warns and builds without it. ccache also treats a store
+error during the build as a miss, so an outage only makes builds slower.
+
+## Usage
+
+```yaml
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: StaflSystems/CustomGithubActions/SetupCcache@main
+        with:
+          mode: ${{ vars.CCACHE_MODE || 'actions' }}
+          key: ${{ github.job }}-${{ matrix.build-preset }}
+          write-only: ${{ github.ref == 'refs/heads/main' }}
+          fork: ${{ endsWith(matrix.configure-preset, '-iar') }}
+          aws-url: ${{ vars.AWS_CCACHE_URL }}
+          aws-password: ${{ secrets.AWS_CCACHE_HTTP_PASSWORD }}
+          lan-url: ${{ vars.LAN_CCACHE_URL }}
+          lan-password: ${{ secrets.LAN_CCACHE_HTTP_PASSWORD }}
+
+      - run: cmake --preset ${{ matrix.configure-preset }} -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
+```
+
+It replaces both the copied "setup ccache" step that downloads the IAR-capable fork (`fork: 'true'`)
+and the `hendrikmuhs/ccache-action` step. It sets `CCACHE_CONFIGPATH` to `ccache_config.conf` in
+the workspace when that file exists. Set `max_size` there only for `actions` mode; the shared stores
+have their own limits.
+
+## ccache versions
+
+The jobs run three versions today: Ubuntu's 4.5.1 in the CI image, 4.9 in newer images, and the
+IAR-capable fork (4.12). So the store is set with environment variables that all of them accept:
+
+- `CCACHE_SECONDARY_STORAGE`: the setting's name before 4.7, which renamed it `remote_storage`.
+  4.7 and later still accept the old name.
+- `CCACHE_REMOTE_ONLY=1`: skips the local cache, which starts empty in every job anyway. It needs
+  4.7; older versions ignore unknown `CCACHE_*` variables. Don't put `remote_only` in
+  `ccache_config.conf`: ccache stops with an error on a config file setting it doesn't know.
+
+## Testing
+
+This repository's CI shellchecks the scripts on every PR that changes this action.
+
+The end-to-end smoke test runs in the private repository
+[StaflSystems/ci-smoke-tests](https://github.com/StaflSystems/ci-smoke-tests), because this one is
+public: the org's ccache variables and secrets are for private repositories only, and the
+self-hosted runners don't take jobs from public repositories. Run it by hand against a branch
+before merging a change here:
+
+```bash
+gh workflow run setup-ccache-smoke.yml -R StaflSystems/ci-smoke-tests -f ref=<branch>
+```
+
+For each store and ccache version, it builds 20 files three times with `smoke.sh`: the first build
+must miss and write to the store, the second must hit in the store after the local cache is
+cleared, and a third, with the store unreachable, must still pass.

SetupCcache/action.yml

@@ -0,0 +1,86 @@
+name: Setup ccache
+description: >
+  Sets up ccache for a C/C++ build job, caching either in the GitHub Actions cache or in the shared
+  ccache stores: the office LAN store for self-hosted runners, the AWS store for GitHub-hosted ones.
+
+inputs:
+  mode:
+    description: >
+      'actions' caches in the GitHub Actions cache, as before. 'remote' uses the shared store and
+      skips the Actions cache. Usually the repository variable CCACHE_MODE.
+    default: actions
+  key:
+    description: Actions-cache key, as for hendrikmuhs/ccache-action. Only used in 'actions' mode.
+    default: ''
+  write-only:
+    description: >
+      'true' compiles every file and writes the results without reading any. Set it on pushes to
+      main, so tagged builds never reuse an object a PR wrote. In 'actions' mode it skips the restore.
+    default: 'false'
+  fork:
+    description: "'true' installs StaflSystems' IAR-capable ccache fork instead of the system ccache."
+    default: 'false'
+  config-path:
+    description: ccache config file, relative to the workspace. Skipped if it doesn't exist.
+    default: ccache_config.conf
+  aws-url:
+    description: The AWS store, for GitHub-hosted runners. Pass vars.AWS_CCACHE_URL.
+    default: ''
+  aws-password:
+    description: Password of the AWS store's ci user. Pass secrets.AWS_CCACHE_HTTP_PASSWORD.
+    default: ''
+  lan-url:
+    description: The office LAN store, for self-hosted runners. Pass vars.LAN_CCACHE_URL.
+    default: ''
+  lan-password:
+    description: Password of the LAN store's ci user. Pass secrets.LAN_CCACHE_HTTP_PASSWORD.
+    default: ''
+
+outputs:
+  store:
+    description: The store this job uses ('aws', 'lan'), or 'none' in 'actions' mode or if it couldn't reach one.
+    value: ${{ steps.remote.outputs.store || 'none' }}
+
+runs:
+  using: composite
+  steps:
+    - name: Install the IAR-capable ccache
+      if: inputs.fork == 'true'
+      shell: bash
+      run: |
+        dir="$RUNNER_TEMP/ccache-fork"
+        rm -rf "$dir" && mkdir -p "$dir" && cd "$dir"
+        curl -fsSL -o ccache.tar.xz \
+          https://github.com/StaflSystems/ccache/releases/download/v4.12.2/ccache-iarsupport.26383df4-linux-x86_64.tar.xz
+        tar xf ccache.tar.xz
+        mv ccache-iarsupport.26383df4-linux-x86_64/ccache .
+        echo "$dir" >> "$GITHUB_PATH"
+
+    - name: Use the repository's ccache config
+      shell: bash
+      env:
+        CONFIG_PATH: ${{ inputs.config-path }}
+      run: |
+        if [ -f "$GITHUB_WORKSPACE/$CONFIG_PATH" ]; then
+          echo "CCACHE_CONFIGPATH=$GITHUB_WORKSPACE/$CONFIG_PATH" >> "$GITHUB_ENV"
+        fi
+
+    - name: Connect to the shared ccache store
+      id: remote
+      if: inputs.mode == 'remote'
+      shell: bash
+      env:
+        RUNNER_ENVIRONMENT: ${{ runner.environment }}
+        AWS_URL: ${{ inputs.aws-url }}
+        AWS_PASSWORD: ${{ inputs.aws-password }}
+        LAN_URL: ${{ inputs.lan-url }}
+        LAN_PASSWORD: ${{ inputs.lan-password }}
+        WRITE_ONLY: ${{ inputs.write-only }}
+      run: bash "$GITHUB_ACTION_PATH/connect.sh"
+
+    - name: ccache
+      uses: hendrikmuhs/ccache-action@v1.2
+      with:
+        key: ${{ inputs.key }}
+        restore: ${{ inputs.mode != 'remote' && inputs.write-only != 'true' }}
+        save: ${{ inputs.mode != 'remote' }}

SetupCcache/connect.sh

@@ -0,0 +1,107 @@
+#!/usr/bin/env bash
+# Points ccache at the shared store for this runner: the LAN store on self-hosted runners, the AWS
+# store (through a local TLS tunnel) on GitHub-hosted ones.
+#
+# ccache's HTTP backend can't speak HTTPS, so on GitHub-hosted runners a ghostunnel client listens
+# on localhost:8080 and forwards to the AWS store over TLS.
+#
+# Fails open: if the store can't be reached, the job warns and builds without it.
+set -uo pipefail
+
+GHOSTUNNEL_VERSION=v1.11.3
+declare -A GHOSTUNNEL_SHA256=(
+  [amd64]=70766f99d6fd439cdf7912161f08a7d66a587a7fc7a79c1bce7fa332aec6fc89
+  [arm64]=04d5c1855eeab6b9ae5fbdda3158604db82f3978ebb5958e601c70817c453576
+)
+TUNNEL_PORT=8080
+
+warn_and_skip() {
+  echo "::warning title=ccache::$1 Building without the shared ccache store."
+  echo "store=none" >> "$GITHUB_OUTPUT"
+  exit 0
+}
+
+# Percent-encode for the user-info part of a URL.
+urlencode() {
+  local LC_ALL=C s="$1" out="" c i
+  for ((i = 0; i < ${#s}; i++)); do
+    c="${s:i:1}"
+    case "$c" in
+      [a-zA-Z0-9.~_-]) out+="$c" ;;
+      *) printf -v c '%%%02X' "'$c"; out+="$c" ;;
+    esac
+  done
+  printf '%s' "$out"
+}
+
+# Succeeds once something accepts TCP connections on host:port, within about 15 seconds.
+wait_for_port() {
+  for _ in $(seq 10); do
+    # timeout: a firewall that drops packets would otherwise hang the connect for minutes.
+    # shellcheck disable=SC2016 # $1 and $2 expand in the inner shell
+    timeout 1 bash -c 'exec 3<>"/dev/tcp/$1/$2"' _ "$1" "$2" 2>/dev/null && return 0
+    sleep 0.5
+  done
+  return 1
+}
+
+start_tunnel() {
+  local target="$1" arch bin
+  case "$(uname -m)" in
+    x86_64) arch=amd64 ;;
+    aarch64 | arm64) arch=arm64 ;;
+    *) return 1 ;;
+  esac
+  command -v curl > /dev/null || return 1
+  bin="$RUNNER_TEMP/ghostunnel"
+  curl -fsSL -o "$bin" \
+    "https://github.com/ghostunnel/ghostunnel/releases/download/$GHOSTUNNEL_VERSION/ghostunnel-linux-$arch" || return 1
+  echo "${GHOSTUNNEL_SHA256[$arch]}  $bin" | sha256sum -c --quiet - || return 1
+  chmod +x "$bin"
+  # setsid and nohup keep the tunnel running after this step's shell exits; it ends with the job.
+  # The server is checked against the system trust store; the store has no client certificate.
+  nohup setsid "$bin" client --listen "localhost:$TUNNEL_PORT" --target "$target" \
+    --disable-authentication > "$RUNNER_TEMP/ghostunnel.log" 2>&1 < /dev/null &
+  wait_for_port localhost "$TUNNEL_PORT"
+}
+
+if [ "$RUNNER_ENVIRONMENT" = self-hosted ]; then
+  store=lan url="$LAN_URL" password="$LAN_PASSWORD"
+else
+  store=aws url="$AWS_URL" password="$AWS_PASSWORD"
+fi
+if [ -z "$url" ] || [ -z "$password" ]; then
+  warn_and_skip "No URL or password for the $store store."
+fi
+
+scheme="${url%%://*}"
+hostport="${url#*://}"
+hostport="${hostport%%/*}"
+
+if [ "$scheme" = https ]; then
+  [[ "$hostport" == *:* ]] || hostport="$hostport:443"
+  start_tunnel "$hostport" || warn_and_skip "Couldn't start the TLS tunnel to $hostport."
+  endpoint="localhost:$TUNNEL_PORT"
+else
+  [[ "$hostport" == *:* ]] || hostport="$hostport:80"
+  wait_for_port "${hostport%:*}" "${hostport##*:}" || warn_and_skip "Couldn't reach $hostport."
+  endpoint="$hostport"
+fi
+
+encoded="$(urlencode "$password")"
+echo "::add-mask::$encoded"
+{
+  # SECONDARY_STORAGE is ccache 4.4–4.6's name for remote_storage; 4.7 and later still accept it.
+  echo "CCACHE_SECONDARY_STORAGE=http://ci:$encoded@$endpoint|layout=bazel|connect-timeout=1000"
+  # ccache 4.7 and later skip the local cache, which starts empty in every job anyway.
+  # Older versions ignore it.
+  echo "CCACHE_REMOTE_ONLY=1"
+  if [ "$WRITE_ONLY" = true ]; then
+    echo "CCACHE_RECACHE=1"
+  fi
+} >> "$GITHUB_ENV"
+echo "store=$store" >> "$GITHUB_OUTPUT"
+
+mode="reads and writes"
+[ "$WRITE_ONLY" = true ] && mode="writes only"
+echo "::notice title=ccache::Using the $store store ($hostport), $mode."