feat(supply-chain): install the Pulumi CLI and provider plugins from our own mirror

.github/workflows/ci.yml

@@ -51,7 +51,7 @@ jobs:
       # input: the action then only installs the CLI (no login, no credentials).
       - uses: pulumi/actions@8e5e406f4007fca908480587cb9893c07090f58d # v7.0.0
         with:
-          pulumi-version: "3.251.0"
+          pulumi-version: "3.253.0"
 
       - name: go build
         run: go build ./...

.github/workflows/release.yml

@@ -25,7 +25,7 @@ jobs:
       # that quietly skipped them.
       - uses: pulumi/actions@8e5e406f4007fca908480587cb9893c07090f58d # v7.0.0
         with:
-          pulumi-version: "3.251.0"
+          pulumi-version: "3.253.0"
 
       - name: go build
         run: go build ./...

AGENTS.md

@@ -706,14 +706,36 @@ realized in two halves:
   with `auto.NewLocalWorkspace` / `auto.*StackInlineSource` — the **Automation API**, which is a Go
   SDK that *drives a `pulumi` binary*: it shells out to whatever `pulumi` is first on `PATH`. There
   is no in-process engine. Treat the CLI as a real dependency with a pinned version:
-  `action.yml` installs it (`pulumi-version` input, currently **3.253.0**) so every consumer gets the
-  engine we validated rather than whatever its runner image ships, and
-  `.github/workflows/{ci,release}.yml` pin their own via `pulumi/actions`.
+  `action.yml` installs it so every consumer gets the engine we validated rather than whatever its
+  runner image ships (see the four-way pin table below).
   Leaving it unpinned is not a theoretical risk — a GitHub runner-image roll moved the preinstalled
   CLI 3.253.0 → 3.256.0 and broke every prd deploy with `InvalidDigest` on the R2 checkpoint write,
   with no diff in inforge itself. When bumping, bump it deliberately and verify a real deploy against
-  the R2 backend; a newer CLI may additionally need
-  `AWS_REQUEST_CHECKSUM_CALCULATION=when_required` for S3-compatible stores.
+  the R2 backend. **CI cannot catch a regression here**: our own workflows install the CLI via
+  `pulumi/actions`, not via `action.yml`, so the consumer install path this repo ships is exercised
+  only by a real consumer deploy.
+- **The Pulumi version is pinned in FOUR places and they must move together.** All four are
+  currently **3.253.0**:
+
+  | Where | What it pins |
+  |---|---|
+  | `go.mod` → `github.com/pulumi/pulumi/sdk/v3` | the SDK compiled into the binary |
+  | `action.yml` → `pulumi-version` input | the CLI every **consumer** deploys with |
+  | `.github/workflows/ci.yml` → `pulumi/actions` | the CLI our tests run against |
+  | `.github/workflows/release.yml` → `pulumi/actions` | the CLI the release build runs against |
+
+  The SDK and the CLI are the same product on two sides of a process boundary, and **nothing in the
+  toolchain links any of them** — Dependabot bumps `go.mod` and cannot see the other three, so
+  accepting an SDK PR on its own silently reopens the skew. Move all four in one PR, or none.
+  Grep `3\.25` before claiming they agree; a partial bump is the failure mode, and it is invisible
+  until a deploy behaves differently from CI.
+- **Third-party binaries install from `wardnet/toolchain-mirror`, verified.** The Pulumi CLI
+  (`action.yml`) and every provider plugin (`cmd/inforge/plugins.go`) are downloaded from our mirror
+  and checked against the `SHA256SUMS` it publishes. **Mirror a version before pinning it here** — an
+  unmirrored version fails the install with a message naming the mirror. Fetching from a mirror
+  without verifying the digest would only relocate the trust, so the check is not optional: the plugin
+  path previously ran unverified bytes as part of a production deploy. The mirror's SHA-256 is
+  stronger than upstream's own guarantee for plugins — the provider repos publish only **SHA-1**.
 - **A third-party apt repo must be PROBED before its sources file is written.** `apt-get update`
   fails hard on an unreachable source and our installers wrap it in retry-then-exit-1, so a
   sources file naming a suite the vendor does not publish breaks **every later apt-using step on

action.yml

@@ -26,12 +26,63 @@ runs:
     # preinstalled CLI 3.253.0 -> 3.256.0, and 3.256.0 fails to write the R2
     # checkpoint with `InvalidDigest: The checksum or Content-MD5 you specified is
     # not valid` (S3 PutObject 400). The same inforge v6.1.3 binary succeeded on
-    # the older image the day before. Pinning here makes the CLI a version we
-    # control and bump deliberately, like any other dependency.
+    # the older image the day before.
+    #
+    # We install from wardnet/toolchain-mirror, not from Pulumi, and verify the
+    # download against the SHA256SUMS that mirror published. Pinning alone stops
+    # drift but still leaves a third-party host in the deploy path; mirroring means
+    # the bytes are ours and a pinned version cannot change or vanish. The digest
+    # check is the point of the exercise — fetching from a mirror without verifying
+    # moves the trust rather than establishing it.
+    #
+    # $GITHUB_PATH PREPENDS, so this CLI wins over any preinstalled on the image.
     - name: Install Pulumi CLI
-      uses: pulumi/actions@8e5e406f4007fca908480587cb9893c07090f58d # v7.0.0
-      with:
-        pulumi-version: ${{ inputs.pulumi-version }}
+      shell: bash
+      env:
+        PULUMI_VERSION: ${{ inputs.pulumi-version }}
+      run: |
+        set -euo pipefail
+
+        case "$(uname -s)/$(uname -m)" in
+          Linux/x86_64)   PLATFORM=linux-x64 ;;
+          Linux/aarch64)  PLATFORM=linux-arm64 ;;
+          Darwin/arm64)   PLATFORM=darwin-arm64 ;;
+          *) echo "::error::no mirrored Pulumi CLI for $(uname -s)/$(uname -m). The mirror carries only the platforms we run; add it in wardnet/toolchain-mirror if this is a real target."; exit 1 ;;
+        esac
+
+        MIRROR="https://github.com/wardnet/toolchain-mirror/releases/download/pulumi-v${PULUMI_VERSION}"
+        ARCHIVE="pulumi-v${PULUMI_VERSION}-${PLATFORM}.tar.gz"
+        DEST="${RUNNER_TEMP}/pulumi-cli"
+        mkdir -p "${DEST}"
+
+        echo "installing Pulumi CLI ${PULUMI_VERSION} (${PLATFORM}) from the mirror..."
+        curl -fsSL -o "${DEST}/${ARCHIVE}" "${MIRROR}/${ARCHIVE}"
+        curl -fsSL -o "${DEST}/SHA256SUMS" "${MIRROR}/SHA256SUMS"
+
+        # Verify ONLY the archive we fetched: SHA256SUMS covers every mirrored
+        # platform, and `sha256sum -c` fails on entries whose files are absent.
+        ( cd "${DEST}" && grep -F " ${ARCHIVE}" SHA256SUMS | sha256sum -c - )
+
+        tar -xzf "${DEST}/${ARCHIVE}" -C "${DEST}"
+        # The archive unpacks to a pulumi/ directory holding the CLI and its
+        # language hosts; they must stay together, so PATH points at that dir.
+        echo "${DEST}/pulumi" >> "$GITHUB_PATH"
+
+    - name: Verify the Pulumi CLI on PATH is the pinned one
+      shell: bash
+      env:
+        PULUMI_VERSION: ${{ inputs.pulumi-version }}
+      run: |
+        set -euo pipefail
+        # Guards the failure this whole change exists to prevent: if PATH ordering
+        # ever resolves to the runner image's CLI instead of ours, fail here with a
+        # clear message rather than silently deploying on an unpinned engine.
+        got="$(pulumi version)"
+        if [ "${got#v}" != "${PULUMI_VERSION}" ]; then
+          echo "::error::pulumi on PATH is ${got}, expected v${PULUMI_VERSION} — the mirrored CLI is not winning PATH resolution"
+          exit 1
+        fi
+        echo "pulumi ${got} (mirrored, verified)"
 
     - name: Install inforge
       shell: bash

cmd/inforge/plugins.go

@@ -4,12 +4,15 @@ import (
 	"archive/tar"
 	"compress/gzip"
 	"context"
+	"crypto/sha256"
+	"encoding/hex"
 	"fmt"
 	"io"
 	"net/http"
 	"os"
 	"path/filepath"
 	"runtime"
+	"strings"
 
 	"github.com/spf13/cobra"
 )
@@ -32,55 +35,142 @@ func newPluginsCmd() *cobra.Command {
 		SilenceUsage:  true,
 		SilenceErrors: true,
 		RunE: func(cmd *cobra.Command, _ []string) error {
-			ctx := cmd.Context()
-
-			// Standard Pulumi providers: download tar.gz from their GitHub releases.
-			// Versions are pinned to match the SDK modules in go.mod.
-			type stdPlugin struct{ name, version, repo string }
-			for _, p := range []stdPlugin{
-				{"hcloud", "1.38.0", "pulumi/pulumi-hcloud"},
-				{"cloudflare", "6.17.0", "pulumi/pulumi-cloudflare"},
-				// pulumi-random backs stable per-service database passwords (ADR-0036).
-				{"random", "4.16.8", "pulumi/pulumi-random"},
-				// pulumiverse/grafana pushes dashboards + alerts (ADR-0038). Note the
-				// pulumiverse org publishes the same asset layout as pulumi/*.
-				{"grafana", "1.0.0", "pulumiverse/pulumi-grafana"},
-			} {
-				fmt.Printf("installing pulumi-resource-%s v%s...\n", p.name, p.version)
-				if err := installPulumiPlugin(ctx, p.name, p.version, p.repo); err != nil {
-					return fmt.Errorf("install %s: %w", p.name, err)
-				}
-				fmt.Printf("  installed pulumi-resource-%s\n", p.name)
-			}
-
-			// No custom (raw-binary) providers ship today: ADR-0036 retired the Neon
-			// plugin and self-hosted Postgres needs none. The seam remains if one returns.
-
-			fmt.Println("all plugins installed")
-			return nil
+			return installAllPlugins(cmd.Context(), mirrorPluginBase)
 		},
 	}
 
 	plugins.AddCommand(install)
 	return plugins
 }
 
-// installPulumiPlugin downloads a published Pulumi provider archive from GitHub
-// and extracts the binary into the Pulumi plugins directory.
-func installPulumiPlugin(ctx context.Context, name, ver, repo string) error {
-	goos := runtime.GOOS
-	goarch := runtime.GOARCH
+// stdPlugin is one pinned provider plugin.
+type stdPlugin struct{ name, version string }
+
+// stdPlugins is the set of provider plugins every deploy needs. It is DATA, kept
+// at package scope rather than buried in the command closure so it can be
+// asserted directly — a typo in a version here surfaces as a mirror 404 at deploy
+// time, which is a slow and confusing way to find it.
+//
+// Versions are pinned to match the SDK modules in go.mod, and each must be
+// mirrored (see mirrorRepo) before it can be installed.
+var stdPlugins = []stdPlugin{
+	{"hcloud", "1.38.0"},
+	{"cloudflare", "6.17.0"},
+	// pulumi-random backs stable per-service database passwords (ADR-0036).
+	{"random", "4.16.8"},
+	// pulumiverse/grafana pushes dashboards + alerts (ADR-0038).
+	{"grafana", "1.0.0"},
+}
+
+// installAllPlugins installs every pinned plugin, resolving each one's source
+// through baseFor. Production passes mirrorPluginBase; a test can pass a stub so
+// the loop is exercisable without reaching the network.
+//
+// No custom (raw-binary) providers ship today: ADR-0036 retired the Neon plugin
+// and self-hosted Postgres needs none. The seam remains if one returns.
+func installAllPlugins(ctx context.Context, baseFor func(name, ver string) string) error {
+	for _, p := range stdPlugins {
+		fmt.Printf("installing pulumi-resource-%s v%s...\n", p.name, p.version)
+		if err := installPulumiPlugin(ctx, p.name, p.version, baseFor(p.name, p.version)); err != nil {
+			return fmt.Errorf("install %s: %w", p.name, err)
+		}
+		fmt.Printf("  installed pulumi-resource-%s\n", p.name)
+	}
+	fmt.Println("all plugins installed")
+	return nil
+}
+
+// mirrorRepo is the release host for every third-party binary the toolchain
+// installs. Mirroring is what makes a pinned version actually pinned: a pin stops
+// us from silently moving, but the artifact still lived on someone else's host and
+// could change or disappear underneath it. Add a version there before pinning it
+// here — see that repo's README.
+const mirrorRepo = "wardnet/toolchain-mirror"
+
+// installPulumiPlugin downloads a Pulumi provider archive from our mirror,
+// verifies it against the mirror's SHA256SUMS, and extracts the binary into the
+// Pulumi plugins directory.
+//
+// Previously this fetched straight from each provider's GitHub releases with no
+// verification at all — whatever bytes arrived were extracted and executed as part
+// of a production deploy. Two things changed:
+//
+//   - the source is our mirror, so a pinned version cannot change or vanish;
+//   - the download is verified, because fetching from a mirror without checking
+//     the digest just relocates the trust instead of establishing it.
+//
+// Note the digest is SHA-256 even though the upstream provider repos publish only
+// SHA-1: the mirror computes its own over the bytes it stored, so what we verify
+// here is stronger than anything upstream offers for these artifacts.
+// base is the release URL to fetch from — supplied by the caller rather than
+// derived here so the whole install path (fetch digest → verify → extract) is
+// exercisable against a local server in tests. Production callers pass
+// mirrorPluginBase; nothing else is a supported source.
+func installPulumiPlugin(ctx context.Context, name, ver, base string) error {
 	binary := "pulumi-resource-" + name
+	archive := pluginArchiveName(name, ver, runtime.GOOS, runtime.GOARCH)
 
-	// Pulumi provider archives use hyphen-separated os-arch (e.g. linux-amd64).
-	archive := fmt.Sprintf("%s-v%s-%s-%s.tar.gz", binary, ver, goos, goarch)
-	url := fmt.Sprintf("https://github.com/%s/releases/download/v%s/%s", repo, ver, archive)
+	want, err := mirrorDigest(ctx, base+"/SHA256SUMS", archive)
+	if err != nil {
+		return err
+	}
 
 	pluginDir, err := pulumiPluginDir(name, ver)
 	if err != nil {
 		return err
 	}
-	return downloadAndExtractTarGz(ctx, url, pluginDir, binary)
+	return downloadAndExtractTarGzVerified(ctx, base+"/"+archive, pluginDir, binary, want)
+}
+
+// pluginArchiveName builds the provider archive filename for an os/arch pair.
+//
+// Provider archives use the Go GOARCH spelling (linux-amd64), which is NOT the
+// spelling the Pulumi CLI's own archives use for the same machine (linux-x64).
+// Getting this wrong produces a 404 that reads like "this version was never
+// mirrored", so the convention is isolated here and tested rather than inlined.
+func pluginArchiveName(name, ver, goos, goarch string) string {
+	return fmt.Sprintf("pulumi-resource-%s-v%s-%s-%s.tar.gz", name, ver, goos, goarch)
+}
+
+// mirrorPluginBase is the mirror release URL for one plugin version. The tag
+// scheme (plugin-<name>-v<version>) is the mirror's contract — see that repo's
+// README — so producer and consumer must agree on it exactly.
+func mirrorPluginBase(name, ver string) string {
+	return fmt.Sprintf("https://github.com/%s/releases/download/plugin-%s-v%s", mirrorRepo, name, ver)
+}
+
+// mirrorDigest fetches a mirror release's SHA256SUMS and returns the expected
+// digest for one file. A missing entry is an error, not a skip: the whole point is
+// that nothing is installed unverified, so "no digest published" must fail loudly
+// rather than quietly degrade to the old unverified behaviour.
+func mirrorDigest(ctx context.Context, sumsURL, file string) (string, error) {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, sumsURL, nil)
+	if err != nil {
+		return "", err
+	}
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return "", err
+	}
+	defer func() { _ = resp.Body.Close() }()
+	if resp.StatusCode != http.StatusOK {
+		return "", fmt.Errorf("HTTP %d fetching %s — is this version mirrored? see %s", resp.StatusCode, sumsURL, mirrorRepo)
+	}
+
+	// SHA256SUMS is small and fully trusted input from our own release; cap the
+	// read anyway so a wrong URL can't stream unbounded into memory.
+	body, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
+	if err != nil {
+		return "", err
+	}
+	for line := range strings.SplitSeq(string(body), "\n") {
+		fields := strings.Fields(line)
+		if len(fields) == 2 && fields[1] == file {
+			return fields[0], nil
+		}
+	}
+	return "", fmt.Errorf("%s has no entry for %s — the mirrored release is incomplete for %s/%s",
+		sumsURL, file, runtime.GOOS, runtime.GOARCH)
 }
 
 func pulumiPluginDir(name, ver string) (string, error) {
@@ -125,7 +215,15 @@ func downloadBinary(ctx context.Context, url, dst string, mode os.FileMode) erro
 	return closeErr
 }
 
-func downloadAndExtractTarGz(ctx context.Context, url, dir, binaryName string) error {
+// downloadAndExtractTarGzVerified downloads an archive, checks its SHA-256
+// against wantDigest, and only then extracts binaryName from it.
+//
+// The archive is staged to a temp file and verified BEFORE a single byte is
+// extracted. Hashing while streaming straight into the extractor would be less
+// code, but it would write an executable to the plugin directory and only
+// afterwards discover the bytes were wrong — and that executable is run as part
+// of a production deploy. Verify first, then extract.
+func downloadAndExtractTarGzVerified(ctx context.Context, url, dir, binaryName, wantDigest string) error {
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
 	if err != nil {
 		return err
@@ -136,11 +234,40 @@ func downloadAndExtractTarGz(ctx context.Context, url, dir, binaryName string) e
 	}
 	defer func() { _ = resp.Body.Close() }()
 	if resp.StatusCode != http.StatusOK {
-		return fmt.Errorf("HTTP %d fetching %s — asset not found for %s/%s",
-			resp.StatusCode, url, runtime.GOOS, runtime.GOARCH)
+		return fmt.Errorf("HTTP %d fetching %s — asset not found for %s/%s (is this version mirrored? see %s)",
+			resp.StatusCode, url, runtime.GOOS, runtime.GOARCH, mirrorRepo)
+	}
+
+	tmp, err := os.CreateTemp("", "inforge-plugin-*.tar.gz")
+	if err != nil {
+		return err
+	}
+	defer func() { _ = os.Remove(tmp.Name()) }()
+
+	sum := sha256.New()
+	n, copyErr := io.CopyN(io.MultiWriter(tmp, sum), resp.Body, maxPluginBinarySize+1)
+	if closeErr := tmp.Close(); closeErr != nil && copyErr == nil {
+		return closeErr
+	}
+	if copyErr != nil && copyErr != io.EOF {
+		return copyErr
+	}
+	if n > maxPluginBinarySize {
+		return fmt.Errorf("archive %s exceeds %d bytes, refusing to extract", url, maxPluginBinarySize)
+	}
+
+	if got := hex.EncodeToString(sum.Sum(nil)); got != wantDigest {
+		return fmt.Errorf("checksum mismatch for %s:\n  want %s\n  got  %s\nrefusing to install — the mirrored artifact does not match its published SHA256SUMS",
+			url, wantDigest, got)
+	}
+
+	f, err := os.Open(tmp.Name()) // #nosec G304 -- tmp.Name() is our own os.CreateTemp path, not external input
+	if err != nil {
+		return err
 	}
+	defer func() { _ = f.Close() }()
 
-	gz, err := gzip.NewReader(resp.Body)
+	gz, err := gzip.NewReader(f)
 	if err != nil {
 		return fmt.Errorf("gzip: %w", err)
 	}