Scan every proposed configuration commit

.github/workflows/validate.yml

@@ -23,6 +23,30 @@ jobs:
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
         with:
           persist-credentials: false
+          fetch-depth: 0
+
+      - name: Scan every proposed commit for secrets
+        env:
+          EVENT_NAME: ${{ github.event_name }}
+          BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.before }}
+          HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
+        run: |
+          scanner=scripts/scan_committed_secrets.py
+          if [[ "$EVENT_NAME" == pull_request ]] && git cat-file -e "$BASE_SHA:$scanner" 2>/dev/null; then
+            git show "$BASE_SHA:$scanner" >"$RUNNER_TEMP/trusted-secret-scanner.py"
+            scanner="$RUNNER_TEMP/trusted-secret-scanner.py"
+          fi
+          if [[ -z "$BASE_SHA" || "$BASE_SHA" =~ ^0+$ ]]; then
+            mapfile -t commits < <(git rev-list --reverse "$HEAD_SHA")
+          else
+            mapfile -t commits < <(git rev-list --reverse "$BASE_SHA..$HEAD_SHA")
+          fi
+          if [[ ! " ${commits[*]} " == *" $HEAD_SHA "* ]]; then
+            commits+=("$HEAD_SHA")
+          fi
+          for commit in "${commits[@]}"; do
+            python3 "$scanner" --repository "$GITHUB_WORKSPACE" --commit "$commit"
+          done
 
       - name: Install static analysis tools
         run: sudo apt-get update && sudo apt-get install -y shellcheck

controller/Dockerfile

@@ -3,6 +3,7 @@ FROM golang:1.26.5-bookworm AS build
 WORKDIR /src
 COPY go.mod *.go ./
 RUN go mod tidy && go mod verify
+RUN go test ./...
 ARG CI_FLEET_VERSION=dev
 ARG CI_FLEET_COMMIT=unknown
 RUN CGO_ENABLED=0 go build -trimpath -ldflags="-s -w -X main.version=${CI_FLEET_VERSION} -X main.commitSHA=${CI_FLEET_COMMIT}" -o /out/ci-fleet-controller .

controller/admin.go

@@ -0,0 +1,76 @@
+package main
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/actions/scaleset"
+)
+
+type scaleSetAdmin interface {
+	GetRunnerGroupByName(context.Context, string) (*scaleset.RunnerGroup, error)
+	GetRunnerScaleSet(context.Context, int, string) (*scaleset.RunnerScaleSet, error)
+	GetRunnerScaleSetByID(context.Context, int) (*scaleset.RunnerScaleSet, error)
+	DeleteRunnerScaleSet(context.Context, int) error
+}
+
+func removeIdleScaleSet(ctx context.Context, cfg Config, client scaleSetAdmin) (bool, error) {
+	runnerGroupID := 1
+	if cfg.RunnerGroup != scaleset.DefaultRunnerGroup {
+		group, err := client.GetRunnerGroupByName(ctx, cfg.RunnerGroup)
+		if err != nil {
+			return false, fmt.Errorf("find runner group: %w", err)
+		}
+		runnerGroupID = group.ID
+	}
+	set, err := client.GetRunnerScaleSet(ctx, runnerGroupID, cfg.ScaleSetName)
+	if err != nil {
+		return false, fmt.Errorf("find runner scale set: %w", err)
+	}
+	if set == nil {
+		return false, nil
+	}
+	if set.Name != cfg.ScaleSetName || set.RunnerGroupID != runnerGroupID {
+		return false, fmt.Errorf("runner scale set identity does not match the selected configuration")
+	}
+	if set.Statistics == nil {
+		set, err = client.GetRunnerScaleSetByID(ctx, set.ID)
+		if err != nil {
+			return false, fmt.Errorf("read runner scale set statistics: %w", err)
+		}
+		if set == nil || set.Name != cfg.ScaleSetName || set.RunnerGroupID != runnerGroupID {
+			return false, fmt.Errorf("runner scale set identity changed while checking statistics")
+		}
+	}
+	if set.Statistics == nil {
+		return false, fmt.Errorf("runner scale set has no idle-state statistics")
+	}
+	if *set.Statistics != (scaleset.RunnerScaleSetStatistic{}) {
+		return false, fmt.Errorf("runner scale set is not idle")
+	}
+	if err := client.DeleteRunnerScaleSet(ctx, set.ID); err != nil {
+		return false, fmt.Errorf("delete idle runner scale set: %w", err)
+	}
+	return true, nil
+}
+
+func deleteIdleScaleSet(ctx context.Context) error {
+	cfg, err := configFromEnv()
+	if err != nil {
+		return fmt.Errorf("configuration: %w", err)
+	}
+	client, err := cfg.scaleSetClient()
+	if err != nil {
+		return fmt.Errorf("create scale-set client: %w", err)
+	}
+	deleted, err := removeIdleScaleSet(ctx, cfg, client)
+	if err != nil {
+		return err
+	}
+	if deleted {
+		fmt.Printf("deleted idle runner scale set %q\n", cfg.ScaleSetName)
+	} else {
+		fmt.Printf("runner scale set %q is already absent\n", cfg.ScaleSetName)
+	}
+	return nil
+}

controller/admin_test.go

@@ -0,0 +1,82 @@
+package main
+
+import (
+	"context"
+	"testing"
+
+	"github.com/actions/scaleset"
+)
+
+type fakeScaleSetAdmin struct {
+	group   *scaleset.RunnerGroup
+	set     *scaleset.RunnerScaleSet
+	detail  *scaleset.RunnerScaleSet
+	deleted bool
+}
+
+func (f *fakeScaleSetAdmin) GetRunnerGroupByName(context.Context, string) (*scaleset.RunnerGroup, error) {
+	return f.group, nil
+}
+
+func (f *fakeScaleSetAdmin) GetRunnerScaleSet(context.Context, int, string) (*scaleset.RunnerScaleSet, error) {
+	return f.set, nil
+}
+
+func (f *fakeScaleSetAdmin) GetRunnerScaleSetByID(context.Context, int) (*scaleset.RunnerScaleSet, error) {
+	return f.detail, nil
+}
+
+func (f *fakeScaleSetAdmin) DeleteRunnerScaleSet(context.Context, int) error {
+	f.deleted = true
+	return nil
+}
+
+func TestRemoveIdleScaleSet(t *testing.T) {
+	cfg := Config{RunnerGroup: "trusted-private-ci", ScaleSetName: "docker-ci-example"}
+	idle := scaleset.RunnerScaleSetStatistic{}
+	newAdmin := func(statistics *scaleset.RunnerScaleSetStatistic) *fakeScaleSetAdmin {
+		return &fakeScaleSetAdmin{
+			group: &scaleset.RunnerGroup{ID: 7, Name: cfg.RunnerGroup},
+			set: &scaleset.RunnerScaleSet{ID: 42, Name: cfg.ScaleSetName, RunnerGroupID: 7, Statistics: statistics},
+		}
+	}
+
+	admin := newAdmin(&idle)
+	deleted, err := removeIdleScaleSet(context.Background(), cfg, admin)
+	if err != nil || !deleted || !admin.deleted {
+		t.Fatalf("idle scale set was not deleted: deleted=%v called=%v err=%v", deleted, admin.deleted, err)
+	}
+
+	absent := newAdmin(&idle)
+	absent.set = nil
+	deleted, err = removeIdleScaleSet(context.Background(), cfg, absent)
+	if err != nil || deleted || absent.deleted {
+		t.Fatalf("absent scale set should be a no-op: deleted=%v called=%v err=%v", deleted, absent.deleted, err)
+	}
+
+	withoutStatistics := newAdmin(nil)
+	withoutStatistics.detail = &scaleset.RunnerScaleSet{ID: 42, Name: cfg.ScaleSetName, RunnerGroupID: 7, Statistics: &idle}
+	deleted, err = removeIdleScaleSet(context.Background(), cfg, withoutStatistics)
+	if err != nil || !deleted || !withoutStatistics.deleted {
+		t.Fatalf("detailed idle statistics were not used: deleted=%v called=%v err=%v", deleted, withoutStatistics.deleted, err)
+	}
+
+	busy := map[string]scaleset.RunnerScaleSetStatistic{
+		"available_jobs":     {TotalAvailableJobs: 1},
+		"acquired_jobs":      {TotalAcquiredJobs: 1},
+		"assigned_jobs":      {TotalAssignedJobs: 1},
+		"running_jobs":       {TotalRunningJobs: 1},
+		"registered_runners": {TotalRegisteredRunners: 1},
+		"busy_runners":       {TotalBusyRunners: 1},
+		"idle_runners":       {TotalIdleRunners: 1},
+	}
+	for name, statistics := range busy {
+		t.Run(name, func(t *testing.T) {
+			admin := newAdmin(&statistics)
+			deleted, err := removeIdleScaleSet(context.Background(), cfg, admin)
+			if err == nil || deleted || admin.deleted {
+				t.Fatalf("non-idle scale set was deletable: deleted=%v called=%v err=%v", deleted, admin.deleted, err)
+			}
+		})
+	}
+}

controller/main.go

@@ -24,7 +24,16 @@ var (
 func main() {
 	ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
 	defer cancel()
-	if err := run(ctx); err != nil {
+	var err error
+	switch {
+	case len(os.Args) == 1:
+		err = run(ctx)
+	case len(os.Args) == 2 && os.Args[1] == "--delete-idle-scale-set":
+		err = deleteIdleScaleSet(ctx)
+	default:
+		err = fmt.Errorf("unsupported arguments")
+	}
+	if err != nil {
 		fmt.Fprintln(os.Stderr, err)
 		os.Exit(1)
 	}

docs/DEPLOYMENT-PROTOTYPE.md

@@ -62,3 +62,14 @@ CI_FLEET_INSTANCE=HOST-ID scripts/cleanup.sh
 ```
 
 Inspect the dry-run. After confirming there is no active job, rerun with `--apply`. Do not delete unrelated Docker resources. Removal of the GitHub-side scale set is normally performed by the controller during graceful shutdown; verify it in GitHub before deleting the App installation.
+
+### Recover an orphaned idle scale set
+
+An unclean controller exit can leave its exact GitHub-side scale-set name behind and make the replacement fail with `already exists`. Use the controller's `--delete-idle-scale-set` administrative mode only after all of these are true:
+
+- the selected scale-set name and runner group come from trusted installed configuration;
+- no repository workflow targeting its label is queued or running;
+- local managed runner count and project-container residue are both zero;
+- the administrative controller image was built from a reviewed public commit.
+
+The command looks up only the configured runner group and exact scale-set name. It refuses deletion unless GitHub reports zero available, acquired, assigned, and running jobs and zero registered, busy, and idle runners. It prints no scale-set ID or credential data. After deletion, restart the previous controller first; recreating the same empty scale set and passing healthcheck is the rollback/recovery proof before desired-state adoption continues.

scripts/scan_committed_secrets.py

@@ -4,7 +4,9 @@
 from __future__ import annotations
 
 import argparse
+import os
 import re
+import stat
 import subprocess
 import sys
 from pathlib import Path
@@ -26,42 +28,79 @@
 def parse_args() -> argparse.Namespace:
     parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument("--repository", type=Path, default=ROOT)
-    parser.add_argument("--commit", help="immutable commit whose blobs should be scanned")
+    revision = parser.add_mutually_exclusive_group()
+    revision.add_argument("--commit", help="immutable commit whose blobs should be scanned")
+    revision.add_argument("--commit-range", help="immutable base..head range whose commit trees should be scanned")
     return parser.parse_args()
 
 
 def main() -> int:
     args = parse_args()
     repository = args.repository.resolve()
+    git_root = Path(
+        subprocess.run(
+            ["git", "-C", str(repository), "rev-parse", "--show-toplevel"],
+            check=True,
+            stdout=subprocess.PIPE,
+            text=True,
+        ).stdout.strip()
+    ).resolve()
+    git_prefix = repository.relative_to(git_root).as_posix()
     if args.commit is not None and not re.fullmatch(r"[0-9a-f]{40}", args.commit):
         print("--commit must be a full lowercase commit SHA", file=sys.stderr)
         return 2
-    listing = ["git", "-C", str(repository)]
-    listing += ["ls-tree", "-rz", "--name-only", args.commit] if args.commit else ["ls-files", "-z"]
-    tracked = subprocess.run(
-        listing,
-        check=True,
-        stdout=subprocess.PIPE,
-    ).stdout.split(b"\0")
-    findings: list[str] = []
-    for raw in tracked:
-        if not raw:
-            continue
-        relative = raw.decode("utf-8")
-        if args.commit is None:
-            data = (repository / relative).read_bytes()
-        else:
+    if args.commit_range is not None and not re.fullmatch(r"[0-9a-f]{40}(?:\.\.[0-9a-f]{40})?", args.commit_range):
+        print("--commit-range must be a full lowercase head SHA or two full SHAs separated by ..", file=sys.stderr)
+        return 2
+    commits: list[str | None] = [args.commit]
+    if args.commit_range:
+        commits = subprocess.run(
+            ["git", "-C", str(repository), "rev-list", "--reverse", args.commit_range],
+            check=True,
+            stdout=subprocess.PIPE,
+            text=True,
+        ).stdout.splitlines()
+        selected_head = args.commit_range.rsplit("..", 1)[-1]
+        if selected_head not in commits:
+            commits.append(selected_head)
+    findings: set[str] = set()
+    for commit in commits:
+        listing = ["git", "-C", str(repository)]
+        listing += ["ls-tree", "-rz", "--name-only", commit] if commit else ["ls-files", "-z"]
+        tracked = subprocess.run(listing, check=True, stdout=subprocess.PIPE).stdout.split(b"\0")
+        for raw in tracked:
+            if not raw:
+                continue
+            relative = raw.decode("utf-8")
+            object_path = relative if git_prefix == "." else f"{git_prefix}/{relative}"
+            revision = f"{commit}:{object_path}" if commit else f":{object_path}"
             data = subprocess.run(
-                ["git", "-C", str(repository), "cat-file", "blob", f"{args.commit}:{relative}"],
+                ["git", "-C", str(repository), "cat-file", "blob", revision],
                 check=True,
                 stdout=subprocess.PIPE,
             ).stdout
-        for match in PATTERN.finditer(data):
-            line = data.count(b"\n", 0, match.start()) + 1
-            findings.append(f"{relative}:{line}")
+            for match in PATTERN.finditer(data):
+                line = data.count(b"\n", 0, match.start()) + 1
+                prefix = f"{commit}:" if args.commit_range else ""
+                findings.add(f"{prefix}{relative}:{line}")
+            if commit is None:
+                working_path = repository / relative
+                try:
+                    metadata = working_path.lstat()
+                except FileNotFoundError:
+                    continue
+                if stat.S_ISLNK(metadata.st_mode):
+                    working_data = os.readlink(working_path).encode("utf-8")
+                elif stat.S_ISREG(metadata.st_mode):
+                    working_data = working_path.read_bytes()
+                else:
+                    continue
+                for match in PATTERN.finditer(working_data):
+                    line = working_data.count(b"\n", 0, match.start()) + 1
+                    findings.add(f"{relative}:{line}")
     if findings:
         print("possible committed secret detected:", file=sys.stderr)
-        print("\n".join(findings), file=sys.stderr)
+        print("\n".join(sorted(findings)), file=sys.stderr)
         return 1
     print("OK: no high-confidence secret material in tracked files")
     return 0

templates/config-repository/.github/workflows/validate.yml

@@ -22,6 +22,7 @@ jobs:
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
         with:
           persist-credentials: false
+          fetch-depth: 0
 
       - name: Validate shell and Python syntax
         run: |
@@ -32,7 +33,27 @@ jobs:
         run: python3 scripts/test_policy.py
 
       - name: Scan committed file contents for secrets
-        run: python3 scripts/scan_committed_secrets.py
+        env:
+          EVENT_NAME: ${{ github.event_name }}
+          BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.before }}
+          HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
+        run: |
+          scanner=scripts/scan_committed_secrets.py
+          if [[ "$EVENT_NAME" == pull_request ]] && git cat-file -e "$BASE_SHA:$scanner" 2>/dev/null; then
+            git show "$BASE_SHA:$scanner" >"$RUNNER_TEMP/trusted-secret-scanner.py"
+            scanner="$RUNNER_TEMP/trusted-secret-scanner.py"
+          fi
+          if [[ -z "$BASE_SHA" || "$BASE_SHA" =~ ^0+$ ]]; then
+            mapfile -t commits < <(git rev-list --reverse "$HEAD_SHA")
+          else
+            mapfile -t commits < <(git rev-list --reverse "$BASE_SHA..$HEAD_SHA")
+          fi
+          if [[ ! " ${commits[*]} " == *" $HEAD_SHA "* ]]; then
+            commits+=("$HEAD_SHA")
+          fi
+          for commit in "${commits[@]}"; do
+            python3 "$scanner" --repository "$GITHUB_WORKSPACE" --commit "$commit"
+          done
 
       - name: Validate reference configurations
         run: |

templates/config-repository/AGENTS.md

@@ -18,6 +18,7 @@ Before committing configuration changes, run:
 - Never add addresses, VM IDs, storage identifiers, backup identifiers, SSH details, or rendered runtime configuration.
 - Do not weaken `public_repositories: false` for Docker-socket runner pools.
 - Infrastructure configuration owns capacity. Application workflows submit all independent jobs and do not use `max-parallel` to model fleet size.
+- Each GitHub runner group belongs to exactly one runner pool; do not create ambiguous cross-pool assignments.
 - The sum of active and drained controller maxima must not exceed the pool capacity budget.
 - Controller engine revisions and reusable workflows must be pinned to full reviewed commit SHAs.
 - Production environments must require approval and must not deploy automatically.