diff --git a/.github/workflows/devcontainers.yaml b/.github/workflows/devcontainers.yaml index 2d9a26a0a..c7775fb1d 100644 --- a/.github/workflows/devcontainers.yaml +++ b/.github/workflows/devcontainers.yaml @@ -2,11 +2,12 @@ name: dev containers # Verifies the modular dev containers (spec 026). # -# drift: regenerated files must match their inputs, and the generator's fixture -# tests must pass. -# build: builds the full profile on amd64 and arm64 and runs its checks inside -# the container. PR-A: full profile only; PR-B (T021) adds the -# affected-profile matrix. +# drift: regenerated files must match their inputs, and the generator's fixture +# tests must pass. +# profiles: lists the profiles affected by the change (PR diff or push diff) and +# emits the build matrix: affected profile x {amd64, arm64}. +# build: builds each affected profile on its arch and runs that profile's +# checks (compile checks) inside the container. on: pull_request: paths: @@ -65,17 +66,87 @@ jobs: - name: Run generator fixture tests run: make test-gen-devcontainers + profiles: + name: profiles + permissions: + contents: read + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + any: ${{ steps.affected.outputs.any }} + profiles: ${{ steps.affected.outputs.profiles }} + matrix: ${{ steps.affected.outputs.matrix }} + steps: + - name: Check out + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + # The generator reads each submodule's system-deps.toml (FR-004a), so + # the submodules must be present for --affected to run. + submodules: true + persist-credentials: false + + - name: Compute affected profiles + id: affected + env: + EVENT_NAME: ${{ github.event_name }} + BASE_SHA: ${{ github.event.pull_request.base.sha }} + PUSH_BEFORE: ${{ github.event.before }} + HEAD_SHA: ${{ github.sha }} + run: | + set -euo pipefail + changed="${RUNNER_TEMP}/changed-files.txt" + if [ "$EVENT_NAME" = "pull_request" ]; then + git diff --name-only "${BASE_SHA}...HEAD" > "$changed" + elif [ -z "$PUSH_BEFORE" ] || [[ "$PUSH_BEFORE" =~ ^0+$ ]]; then + # New branch or unknown base: treat every profile as affected. + # profiles.toml is a global trigger, so --affected returns all ids. + echo ".devcontainer/profiles.toml" > "$changed" + elif ! git diff --name-only "$PUSH_BEFORE" "$HEAD_SHA" > "$changed" 2>/dev/null; then + # Base commit unreachable (force-push): treat every profile as affected. + echo ".devcontainer/profiles.toml" > "$changed" + fi + + mapfile -t files < "$changed" + ids="" + # --affected requires at least one path, so an empty diff is handled here. + if [ "${#files[@]}" -gt 0 ]; then + ids="$(python3 hack/gen-devcontainers.py --affected "${files[@]}")" + fi + echo "Affected profiles: ${ids:-none}" + + IDS="$ids" python3 - >> "$GITHUB_OUTPUT" <<'PY' + import json + import os + + ids = [line for line in os.environ.get("IDS", "").splitlines() if line] + rows = [] + for pid in ids: + if pid == "full": + config = ".devcontainer/devcontainer.json" + else: + config = f".devcontainer/{pid}/devcontainer.json" + for arch in ("amd64", "arm64"): + rows.append({"profile": pid, "config": config, "arch": arch}) + if not rows: + # Placeholder so the matrix is never empty; the build job's `if` skips it. + rows = [{"profile": "none", "config": "", "arch": "amd64"}] + print(f"any={'true' if ids else 'false'}") + print("profiles=" + json.dumps(ids, separators=(",", ":"))) + print("matrix=" + json.dumps({"include": rows}, separators=(",", ":"))) + PY + build: - name: build (${{ matrix.arch }}) - needs: drift + name: build (${{ matrix.profile }}, ${{ matrix.arch }}) + needs: [drift, profiles] + if: needs.profiles.outputs.any == 'true' permissions: contents: read runs-on: ${{ matrix.arch == 'arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }} timeout-minutes: 60 strategy: fail-fast: false - matrix: - arch: [amd64, arm64] + matrix: ${{ fromJSON(needs.profiles.outputs.matrix) }} steps: - name: Check out uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -87,30 +158,33 @@ jobs: id: timer run: echo "started=$(date +%s)" >> "$GITHUB_OUTPUT" - - name: Expand full profile checks + - name: Expand profile checks id: checks + env: + PROFILE: ${{ matrix.profile }} run: | set -euo pipefail - cmd="$(python3 hack/gen-devcontainers.py --checks full)" + cmd="$(python3 hack/gen-devcontainers.py --checks "$PROFILE")" echo "cmd=${cmd}" >> "$GITHUB_OUTPUT" - - name: Build full profile and run its checks + - name: Build profile and run its checks uses: devcontainers/ci@513af61f4de4f75d37e4438f184ba4358f0fc1ca # v0.3.1900000450 with: push: never - configFile: .devcontainer/devcontainer.json + configFile: ${{ matrix.config }} runCmd: ${{ steps.checks.outputs.cmd }} - name: Write build summary if: always() && steps.timer.outcome == 'success' env: STARTED: ${{ steps.timer.outputs.started }} + PROFILE: ${{ matrix.profile }} MATRIX_ARCH: ${{ matrix.arch }} run: | set -euo pipefail elapsed=$(( $(date +%s) - STARTED )) { - echo "### Dev containers: full (${MATRIX_ARCH})" + echo "### Dev containers: ${PROFILE} (${MATRIX_ARCH})" echo "" echo "- Setup seconds: ${elapsed}" echo "- Workspace size (du -sx): $(du -sx "$GITHUB_WORKSPACE" | cut -f1) KiB" diff --git a/Makefile b/Makefile index f1df15cf7..7b0b00ea1 100644 --- a/Makefile +++ b/Makefile @@ -77,6 +77,7 @@ build-go: ## Build all Go binaries .PHONY: build-web build-web: ## Build web dashboard + @hack/need.sh npm cd web && npm ci && npm run build # -------- test -------- @@ -192,6 +193,7 @@ e2e-images: ## Build operator/api/agent/fakeoidc images tagged for e2e .PHONY: test-e2e test-e2e: ## Run E2E tests (CLUSTER=kind spins an ephemeral cluster; CLUSTER=remote reuses REMOTE_KUBECONFIG) + @hack/need.sh $(if $(filter remote,$(CLUSTER)),go kubectl helm,go docker kind kubectl helm) ifeq ($(CLUSTER),remote) cd test/e2e && GAMEPLANE_E2E_REUSE_CLUSTER=1 \ GAMEPLANE_E2E_CONTEXT=$(REMOTE_CONTEXT) KUBECONFIG=$(REMOTE_KUBECONFIG) \ @@ -209,6 +211,7 @@ test-e2e-keep: ## Re-run E2E tests against an already-up cluster (skip create/de .PHONY: test-e2e-bucket test-e2e-bucket: ## Run one CI e2e bucket against an already-up cluster (BUCKET=operator|api-auth|api-roles|api-rbac|api-agent|api-mods|ratelimit|bot-fast|bot-heavy|multicluster|upgrade). BUCKET=multicluster additionally needs a second kind cluster up and GAMEPLANE_E2E_CLUSTER_B set to its name (default gameplane-e2e-b). + @hack/need.sh go kubectl helm cd test/e2e && GAMEPLANE_E2E_REUSE_CLUSTER=1 GAMEPLANE_E2E_CLUSTER=$(KIND_E2E_CLUSTER) \ go test -tags=e2e -timeout 50m -v -run "$$(./buckets.sh regex $(BUCKET))" ./... @@ -312,6 +315,7 @@ lint: check-doc-versions check-links check-features-sync check-specs check-telem .PHONY: lint-go lint-go: ## Run golangci-lint across all modules + @hack/need.sh golangci-lint @for m in $(GO_MODULES); do \ (cd $$m && golangci-lint run) || exit $$?; \ done @@ -339,19 +343,24 @@ images: $(addprefix image-,$(IMAGES)) image-audit-syslog image-tunnel-frp image- # Pattern rule for standard-shaped images image-%: ## Build container image + @hack/need.sh docker docker build -t $(REGISTRY)/$*:$(TAG) -f $*/Dockerfile . # Non-standard images (target name ≠ dir, or special Dockerfile paths) image-audit-syslog: ## Build audit-syslog-bridge image + @hack/need.sh docker docker build -t $(REGISTRY)/audit-syslog-bridge:$(TAG) -f audit-syslog-bridge/Dockerfile . image-tunnel-frp: ## Build tunnel-frp image + @hack/need.sh docker docker build -t $(REGISTRY)/tunnel-frp:$(TAG) -f tunnel/Dockerfile.frp . image-tunnel-tailscale: ## Build tunnel-tailscale image + @hack/need.sh docker docker build -t $(REGISTRY)/tunnel-tailscale:$(TAG) -f tunnel/Dockerfile.tailscale . image-tunnel-playit: ## Build tunnel-playit image + @hack/need.sh docker docker build -t $(REGISTRY)/tunnel-playit:$(TAG) -f tunnel/Dockerfile.playit . # -------- codegen -------- @@ -427,6 +436,7 @@ ADDRESS_MANAGER ?= metallb endif dev-up: ## Create/prepare cluster + install Gameplane (CLUSTER=kind|remote) + @hack/need.sh $(if $(filter remote,$(CLUSTER)),docker kubectl helm oras,docker kind kubectl helm oras) ifeq ($(CLUSTER),remote) $(MAKE) images TAG=$(TAG) $(MAKE) dev-push TAG=$(TAG) @@ -441,6 +451,7 @@ else endif dev-load: ## Load every image `make images` builds into the kind cluster (kind only) + @hack/need.sh docker kind @for img in $(ALL_IMAGE_NAMES); do \ kind load docker-image $(REGISTRY)/$$img:$(TAG) --name $(KIND_CLUSTER) || exit $$?; \ done @@ -452,6 +463,7 @@ dev-push: ## Push operator/api/web/agent images to REGISTRY (remote clusters) docker push $(REGISTRY)/agent:$(TAG) dev-install: ## Install Gameplane Helm chart into the selected cluster + @hack/need.sh helm # CRD schema changes land in-place on upgrade via the chart's pre-upgrade # hook (crds.autoApply), which runs `kubectl apply --server-side` — no # manual apply needed here anymore, and it works for any `helm upgrade`, @@ -482,11 +494,13 @@ endif .PHONY: modules-push MODULE_REGISTRY ?= localhost:5001 modules-push: ## Push every modules/ as an OCI bundle to MODULE_REGISTRY + @hack/need.sh oras ./modules/build.sh push --registry $(MODULE_REGISTRY) --plain-http # -------- web dev -------- .PHONY: web-dev web-dev: ## Run web dashboard against in-cluster API + @hack/need.sh npm cd web && npm run dev # -------- tidy -------- diff --git a/hack/need.sh b/hack/need.sh new file mode 100755 index 000000000..aa4cd5773 --- /dev/null +++ b/hack/need.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +# need.sh: missing-tool guard for Makefile recipes (spec 026, FR-010, research R6). +# +# Usage: hack/need.sh ... +# +# Exits 0 when every tool is on PATH. Otherwise, for each missing tool, prints +# which dev container profile installs it and exits 127 ("command not found"). +# +# Lookup: .devcontainer/tool-index.tsv (generated by hack/gen-devcontainers.py) +# maps a dependency name to its owning profiles and declaring folder. Index rows +# use dependency names, so the binaries npm and npx are mapped to "node" below. +# When the index is absent, or has no row for the tool, only the plain +# "not installed" line is printed. +set -euo pipefail + +if [ "$#" -eq 0 ]; then + echo "usage: hack/need.sh ..." >&2 + exit 2 +fi + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +INDEX="$ROOT/.devcontainer/tool-index.tsv" + +# Binary name -> dependency name used in tool-index.tsv. +# npm and npx are both provided by the node dependency. +dep_name() { + case "$1" in + npm|npx) echo "node" ;; + *) echo "$1" ;; + esac +} + +# Print the index row for dependency $1 as "nameprofilesdir", or nothing. +lookup() { + local want="$1" name="" profiles="" dir="" + [ -f "$INDEX" ] || return 0 + while IFS=$'\t' read -r name profiles dir || [ -n "$name" ]; do + if [ "$name" = "$want" ]; then + printf '%s\t%s\t%s\n' "$name" "$profiles" "$dir" + return 0 + fi + done < "$INDEX" + return 0 +} + +missing=0 +for tool in "$@"; do + if command -v "$tool" >/dev/null 2>&1; then + continue + fi + missing=1 + row="$(lookup "$(dep_name "$tool")")" + if [ -z "$row" ]; then + echo "need: $tool is not installed." >&2 + continue + fi + IFS=$'\t' read -r _ profiles dir <<< "$row" + # At most the first two profile ids, smallest profile first. + p1="" p2="" + IFS=',' read -r p1 p2 _ <<< "$profiles" + if [ -n "$p2" ]; then + open="\"$p1\" or \"$p2\"" + else + open="\"$p1\"" + fi + case "$dir" in + baseline|extras:*) source_note=".devcontainer/profiles.toml" ;; + *) source_note="$dir/system-deps.toml" ;; + esac + echo "need: $tool is not installed in this dev container." >&2 + echo " Open the $open profile (declared in $source_note)." >&2 +done + +if [ "$missing" -ne 0 ]; then + exit 127 +fi +exit 0 diff --git a/specs/026-modular-dev-containers/tasks.md b/specs/026-modular-dev-containers/tasks.md index 881842a1b..77f51b7dd 100644 --- a/specs/026-modular-dev-containers/tasks.md +++ b/specs/026-modular-dev-containers/tasks.md @@ -87,8 +87,8 @@ description: "Task list for spec 026 modular dev containers" **Independent Test**: quickstart §2. Each component profile builds in CI and passes its checks, and tools outside its scope are absent. - [X] T015 [US1] Run `make devcontainers` and commit `.devcontainer/{web,api,operator,agent,go-services,website,modules}/*` (generated). -- [ ] T016 [P] [US1] Create `hack/need.sh` per [contracts/setup-and-need.md](contracts/setup-and-need.md) (reads `.devcontainer/tool-index.tsv`, exits 127, falls back to a plain message when the index is absent). -- [ ] T017 [US1] Call `hack/need.sh` as the first recipe line of the Makefile targets from research R6 (`dev-up`, `dev-load`, `dev-install`, `modules-push`, `test-e2e`, `test-e2e-bucket`, `lint-go`, `images`, `web-dev`, `build-web`), naming the tools each one runs. Recipe bodies stay otherwise unchanged. +- [X] T016 [P] [US1] Create `hack/need.sh` per [contracts/setup-and-need.md](contracts/setup-and-need.md) (reads `.devcontainer/tool-index.tsv`, exits 127, falls back to a plain message when the index is absent). +- [X] T017 [US1] Call `hack/need.sh` as the first recipe line of the Makefile targets from research R6 (`dev-up`, `dev-load`, `dev-install`, `modules-push`, `test-e2e`, `test-e2e-bucket`, `lint-go`, `images`, `web-dev`, `build-web`), naming the tools each one runs. Recipe bodies stay otherwise unchanged. - [X] T018 [US1] In the `website` and `modules` component profiles, set submodule init to only that submodule (`SUBMODULES` in `setup.env`), and confirm the generated `setup.env` has no `go_mod_download` or `envtest` step (spec US1 scenarios 2 and 3). --- @@ -110,8 +110,8 @@ description: "Task list for spec 026 modular dev containers" **Independent Test**: quickstart §1 and §5. -- [ ] T021 [US5] Extend `.github/workflows/devcontainers.yaml` per research R7: path triggers, a `profiles` job that runs `python3 hack/gen-devcontainers.py --affected` on the PR's changed files and emits a JSON matrix, and a `build` matrix of affected profile × {amd64 `ubuntu-latest`, arm64 `ubuntu-24.04-arm`} running each profile's `checks` with `devcontainers/ci` (SHA-pinned, `push: never`). Write setup seconds and `du -sx` of the workspace and image size to `$GITHUB_STEP_SUMMARY`. Every action is SHA-pinned with a version comment (spec 008 rules). Set `permissions: contents: read`. -- [ ] T022 [US5] Add a `drift` step that runs `make test-gen-devcontainers` (fixture tests) next to `--check` in `.github/workflows/devcontainers.yaml`. +- [X] T021 [US5] Extend `.github/workflows/devcontainers.yaml` per research R7: path triggers, a `profiles` job that runs `python3 hack/gen-devcontainers.py --affected` on the PR's changed files and emits a JSON matrix, and a `build` matrix of affected profile × {amd64 `ubuntu-latest`, arm64 `ubuntu-24.04-arm`} running each profile's `checks` with `devcontainers/ci` (SHA-pinned, `push: never`). Write setup seconds and `du -sx` of the workspace and image size to `$GITHUB_STEP_SUMMARY`. Every action is SHA-pinned with a version comment (spec 008 rules). Set `permissions: contents: read`. +- [X] T022 [US5] Add a `drift` step that runs `make test-gen-devcontainers` (fixture tests) next to `--check` in `.github/workflows/devcontainers.yaml`. - [X] T023 [US5] Add `.github/workflows/devcontainers.yaml` to whatever lists hardened workflows today (the spec 008 action-pin check), if such a list exists. The T001 scout confirms where it is. Done: nothing to register; zizmor and actionlint discover every workflow (ci.yaml:1592, :1639), see inventory.md Answers (c). **Checkpoint (PR-B)**: every profile green on both arches. Job summary shows SC-001 and SC-002 numbers. Open PR-B.