From 704ad85cbc9ed77b419644cff59133dffa8841f8 Mon Sep 17 00:00:00 2001 From: Charlie Wyse Date: Wed, 29 Jul 2026 15:13:08 -0700 Subject: [PATCH 1/3] fix: added api and orch --- .github/workflows/publish.yml | 97 +++++++++++++++++++++++++--- .github/workflows/release-please.yml | 4 ++ .release-please-manifest.json | 6 +- packages/api/Makefile | 40 ++++++++++++ packages/dashboard-api/Makefile | 35 ++++++++++ packages/nomad-nodepool-apm/Makefile | 38 ++++++++++- packages/orchestrator/Makefile | 47 +++++++++++++- release-please-config.json | 24 +++++++ 8 files changed, 278 insertions(+), 13 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 2325a48c10..2e2141bea5 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -51,6 +51,18 @@ jobs: # tags. kind=image case "${TAG}" in + api-v*) + # api releases as one unit with the infra db-migrator: its + # docker-bake.hcl builds both images (the api bakes in the + # migration timestamp its startup guard expects, the migrator + # bakes in the migrations, so one shared version makes schema + # skew impossible). Both images land in the single `api` + # registry repo (e2b-artifacts/api/api and + # e2b-artifacts/api/db-migrator), which also avoids colliding + # with belt's separate db-migrator repo. + component=api + kind=bake + ;; docker-reverse-proxy-v*) component=docker-reverse-proxy # Build context is packages/ (the Dockerfile needs shared/ and db/). @@ -75,6 +87,13 @@ jobs: context=packages/clickhouse build_args="" ;; + dashboard-api-v*) + component=dashboard-api + # Build context is packages/ (the Dockerfile needs shared/). + dockerfile=packages/dashboard-api/Dockerfile + context=packages + build_args="COMMIT_SHA EXPECTED_MIGRATION_TIMESTAMP" + ;; envd-v*) # envd is a raw static binary (CGO_ENABLED=0), not an image: # published as a versioned object in the public @@ -84,6 +103,28 @@ jobs: component=envd kind=binary ;; + orchestrator-v*) + # One release covers all three fc-env-pipeline binaries built + # from packages/orchestrator: `make build` produces bin/ + # orchestrator and bin/clean-nfs-cache, and template-manager IS + # the orchestrator binary uploaded under a second object name + # (see the package Makefile's upload/template-manager), so no + # separate template-manager object is published. The Makefile's + # ORCHESTRATOR_VERSION flow downloads these and uploads them to + # the client's fc-env-pipeline bucket under the usual names. + component=orchestrator + kind=binary + binary_files="orchestrator clean-nfs-cache" + ;; + nomad-nodepool-apm-v*) + # Two Nomad plugin binaries built and released together from + # one package; the Makefile's NOMAD_NODEPOOL_APM_VERSION flow + # downloads both and uploads them to the client's + # fc-env-pipeline bucket under the usual names. + component=nomad-nodepool-apm + kind=binary + binary_files="nomad-nodepool-apm nomad-deployment-aware-target" + ;; *) # A stray tag caught by the loose glob is a no-op; an explicit # dispatch for one is a mistake and should fail loudly. @@ -117,10 +158,15 @@ jobs: echo "dockerfile=${dockerfile}" echo "context=${context}" echo "build_args=${build_args}" - else - # /// - echo "binary_uri=gs://e2b-artifact-binaries/${component}/v${version}/${component}" + elif [ "${kind}" = "binary" ]; then + # Versioned directory in the public binaries bucket; most + # binary components ship one file named after the component, + # multi-binary packages (orchestrator) list theirs explicitly. + echo "binary_dir=gs://e2b-artifact-binaries/${component}/v${version}" + echo "binary_files=${binary_files:-${component}}" fi + # kind=bake needs no extra outputs: the bake step derives its + # registry paths from the component and version. } >> "${GITHUB_OUTPUT}" - name: Checkout repository @@ -147,11 +193,11 @@ jobs: uses: google-github-actions/setup-gcloud@v3 - name: Configure Docker auth - if: ${{ steps.pkg.outputs.kind == 'image' }} + if: ${{ steps.pkg.outputs.kind == 'image' || steps.pkg.outputs.kind == 'bake' }} run: gcloud auth configure-docker us-docker.pkg.dev --quiet - name: Set up Docker Buildx - if: ${{ steps.pkg.outputs.kind == 'image' }} + if: ${{ steps.pkg.outputs.kind == 'image' || steps.pkg.outputs.kind == 'bake' }} uses: docker/setup-buildx-action@v3 - name: Build and push released image @@ -177,6 +223,11 @@ jobs: case "${arg}" in COMMIT_SHA) args+=( --build-arg COMMIT_SHA="$(git rev-parse --short HEAD)" ) ;; VERSION) args+=( --build-arg VERSION="${IMAGE_TAG}" ) ;; + # The newest DB migration version in the tagged source — the + # startup migration guard (api, dashboard-api). Derived from + # the checkout, so it matches a from-source build; without it + # the guard is silently disabled (empty parses to 0). + EXPECTED_MIGRATION_TIMESTAMP) args+=( --build-arg EXPECTED_MIGRATION_TIMESTAMP="$(scripts/get-latest-migration.sh)" ) ;; esac done docker buildx build \ @@ -189,6 +240,32 @@ jobs: -f "${DOCKERFILE}" \ "${CONTEXT}" + # Bake components (kind=bake): api's docker-bake.hcl builds the api and + # db-migrator images together; both are pushed into the `api` registry + # repo under the shared version tag. Tag overrides replace the bake + # file's REGISTRY_PREFIX-derived tags; the migration timestamp is + # computed from the tag's checkout, matching a from-source build. + - name: Build and push released images (bake) + if: ${{ steps.pkg.outputs.kind == 'bake' }} + env: + COMPONENT: ${{ steps.pkg.outputs.component }} + VERSION: ${{ steps.pkg.outputs.version }} + run: | + IMAGE_TAG="v${VERSION}" + REGISTRY="us-docker.pkg.dev/e2b-artifacts/${COMPONENT}" + EXPECTED_MIGRATION_TIMESTAMP="$(scripts/get-latest-migration.sh)" + COMMIT_SHA="$(git rev-parse --short HEAD)" + cd packages + REGISTRY_PREFIX="${REGISTRY}" \ + COMMIT_SHA="${COMMIT_SHA}" \ + EXPECTED_MIGRATION_TIMESTAMP="${EXPECTED_MIGRATION_TIMESTAMP}" \ + docker buildx bake -f "${COMPONENT}/docker-bake.hcl" \ + --provenance=false \ + --sbom=false \ + --set "api.tags=${REGISTRY}/api:${IMAGE_TAG}" \ + --set "db-migrator.tags=${REGISTRY}/db-migrator:${IMAGE_TAG}" \ + --push + # Binary components (kind=binary): build with the repo Go toolchain and # upload the versioned object to the public e2b-artifact-binaries # bucket. The package Makefile's `build` target derives the commit sha @@ -212,9 +289,13 @@ jobs: # consumers fetch via # https://storage.googleapis.com/e2b-artifact-binaries/... — the bucket # is public-read. - - name: Upload released binary + - name: Upload released binaries if: ${{ steps.pkg.outputs.kind == 'binary' }} env: COMPONENT: ${{ steps.pkg.outputs.component }} - BINARY_URI: ${{ steps.pkg.outputs.binary_uri }} - run: gcloud storage cp "packages/${COMPONENT}/bin/${COMPONENT}" "${BINARY_URI}" + BINARY_DIR: ${{ steps.pkg.outputs.binary_dir }} + BINARY_FILES: ${{ steps.pkg.outputs.binary_files }} + run: | + for f in ${BINARY_FILES}; do + gcloud storage cp "packages/${COMPONENT}/bin/${f}" "${BINARY_DIR}/${f}" + done diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml index 3650e60c1b..8f0f8e771e 100644 --- a/.github/workflows/release-please.yml +++ b/.github/workflows/release-please.yml @@ -117,6 +117,10 @@ jobs: client-proxy) path=packages/client-proxy ;; clickhouse-migrator) path=packages/clickhouse ;; envd) path=packages/envd ;; + orchestrator) path=packages/orchestrator ;; + api) path=packages/api ;; + dashboard-api) path=packages/dashboard-api ;; + nomad-nodepool-apm) path=packages/nomad-nodepool-apm ;; *) echo "::error::release merge for unknown component '${component}' — add it to this workflow's package list" exit 1 diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 96d15c7876..30d1e4e5ef 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -2,5 +2,9 @@ "packages/docker-reverse-proxy": "0.0.1", "packages/client-proxy": "0.0.1", "packages/clickhouse": "0.0.1", - "packages/envd": "0.0.1" + "packages/envd": "0.0.1", + "packages/orchestrator": "0.0.0", + "packages/api": "0.0.0", + "packages/dashboard-api": "0.0.0", + "packages/nomad-nodepool-apm": "0.0.0" } diff --git a/packages/api/Makefile b/packages/api/Makefile index 64fbbd91e4..a8d72dbfc8 100644 --- a/packages/api/Makefile +++ b/packages/api/Makefile @@ -14,6 +14,27 @@ else REGISTRY_PREFIX := $(GCP_REGION)-docker.pkg.dev/$(GCP_PROJECT_ID)/$(PREFIX)core endif +# E2B-hosted registry repo holding released, prebuilt api AND db-migrator +# images — they release as one unit from this package's docker-bake.hcl. +# Published to by the release-please workflow on tagging a release. +E2B_ARTIFACTS_REGISTRY ?= us-docker.pkg.dev/e2b-artifacts/api + +# Optional version "hook" for the released-image flow. +# When empty (default), build-and-upload bakes both images from source and +# pushes to the client's own core repo (the existing flow, unchanged). +# When set (e.g. API_VERSION=v0.1.0), skip building: pull the released api +# and db-migrator images from $(E2B_ARTIFACTS_REGISTRY) and retag/push them +# into the client's core repo. One API_VERSION covers BOTH images on purpose: +# the api bakes in the migration timestamp its startup guard expects and the +# migrator bakes in the migrations, so a shared version makes schema skew +# between them impossible. +# +# Set it either on the command line (make ... API_VERSION=v0.1.0) or in the +# active .env.${ENV} file, which is included above. Quotes are stripped so +# both `=v0.1.0` and `="v0.1.0"` work; a command-line value always wins over +# the env file. +API_VERSION := $(strip $(subst ",,$(API_VERSION))) + .PHONY: generate generate: go generate ./... @@ -73,12 +94,31 @@ profiler: .PHONY: build-and-upload build-and-upload: +ifeq ($(strip $(API_VERSION)),) + # Existing flow: bake both images from source and push to the client's + # own core repo. $(eval COMMIT_SHA := $(shell git rev-parse --short HEAD)) $(eval EXPECTED_MIGRATION_TIMESTAMP := $(expectedMigration)) cd .. && REGISTRY_PREFIX=$(REGISTRY_PREFIX) \ COMMIT_SHA=$(COMMIT_SHA) \ EXPECTED_MIGRATION_TIMESTAMP=$(EXPECTED_MIGRATION_TIMESTAMP) \ docker buildx bake -f api/docker-bake.hcl --push +else + # Released-image flow: pull the prebuilt, versioned api + db-migrator + # images from the E2B artifacts registry and retag/push them into the + # client's own core repo. + @echo "Using released api + db-migrator $(API_VERSION) from $(E2B_ARTIFACTS_REGISTRY)" + docker pull --platform linux/amd64 $(E2B_ARTIFACTS_REGISTRY)/api:$(API_VERSION) + docker pull --platform linux/amd64 $(E2B_ARTIFACTS_REGISTRY)/db-migrator:$(API_VERSION) + docker tag $(E2B_ARTIFACTS_REGISTRY)/api:$(API_VERSION) $(REGISTRY_PREFIX)/api:latest + docker tag $(E2B_ARTIFACTS_REGISTRY)/api:$(API_VERSION) $(REGISTRY_PREFIX)/api:$(API_VERSION) + docker tag $(E2B_ARTIFACTS_REGISTRY)/db-migrator:$(API_VERSION) $(REGISTRY_PREFIX)/db-migrator:latest + docker tag $(E2B_ARTIFACTS_REGISTRY)/db-migrator:$(API_VERSION) $(REGISTRY_PREFIX)/db-migrator:$(API_VERSION) + docker push $(REGISTRY_PREFIX)/api:latest + docker push $(REGISTRY_PREFIX)/api:$(API_VERSION) + docker push $(REGISTRY_PREFIX)/db-migrator:latest + docker push $(REGISTRY_PREFIX)/db-migrator:$(API_VERSION) +endif .PHONY: test test: diff --git a/packages/dashboard-api/Makefile b/packages/dashboard-api/Makefile index 8aea7bfb63..471eb52c29 100644 --- a/packages/dashboard-api/Makefile +++ b/packages/dashboard-api/Makefile @@ -10,6 +10,29 @@ else IMAGE_REGISTRY := $(GCP_REGION)-docker.pkg.dev/$(GCP_PROJECT_ID)/$(PREFIX)core/dashboard-api endif +# E2B-hosted registry that holds released, prebuilt dashboard-api images. +# Published to by the release-please workflow on tagging a release. +E2B_ARTIFACTS_REGISTRY ?= us-docker.pkg.dev/e2b-artifacts/dashboard-api/dashboard-api + +# Optional version "hook" for the released-image flow. +# When empty (default), build-and-upload builds from source and pushes to the +# client's own core repo (the existing flow, unchanged). +# When set (e.g. DASHBOARD_API_VERSION=v0.1.0), skip building: pull the +# released image from $(E2B_ARTIFACTS_REGISTRY) and retag/push it into the +# client's core repo, so the `dashboard-api:latest` lookup keeps working. +# +# Set it either on the command line (make ... DASHBOARD_API_VERSION=v0.1.0) +# or in the active .env.${ENV} file, which is included above. Quotes are +# stripped so both `=v0.1.0` and `="v0.1.0"` work; a command-line value +# always wins over the env file. +# +# Bump in lockstep with API_VERSION: dashboard-api guards startup on the +# same DB's migration timestamp, and the migrations come from the db-migrator +# image that releases WITH the api — a dashboard-api pinned ahead of +# API_VERSION expects migrations the pinned migrator doesn't carry and fails +# its startup check. +DASHBOARD_API_VERSION := $(strip $(subst ",,$(DASHBOARD_API_VERSION))) + HOSTNAME := $(shell hostname 2> /dev/null || hostnamectl hostname 2> /dev/null) $(if $(HOSTNAME),,$(error Failed to determine hostname: both 'hostname' and 'hostnamectl' failed)) @@ -31,9 +54,21 @@ build-debug: .PHONY: build-and-upload build-and-upload: +ifeq ($(strip $(DASHBOARD_API_VERSION)),) + # Existing flow: build from source and push to the client's own core repo. $(eval COMMIT_SHA := $(shell git rev-parse --short HEAD)) $(eval EXPECTED_MIGRATION_TIMESTAMP := $(expectedMigration)) @docker build --platform linux/amd64 --tag $(IMAGE_REGISTRY) --tag $(IMAGE_REGISTRY):$(COMMIT_SHA) --push --build-arg COMMIT_SHA="$(COMMIT_SHA)" --build-arg EXPECTED_MIGRATION_TIMESTAMP="$(EXPECTED_MIGRATION_TIMESTAMP)" -f ./Dockerfile .. +else + # Released-image flow: pull the prebuilt, versioned image from the E2B + # artifacts registry and retag/push it into the client's own core repo. + @echo "Using released dashboard-api $(DASHBOARD_API_VERSION) from $(E2B_ARTIFACTS_REGISTRY)" + docker pull --platform linux/amd64 $(E2B_ARTIFACTS_REGISTRY):$(DASHBOARD_API_VERSION) + docker tag $(E2B_ARTIFACTS_REGISTRY):$(DASHBOARD_API_VERSION) $(IMAGE_REGISTRY):latest + docker tag $(E2B_ARTIFACTS_REGISTRY):$(DASHBOARD_API_VERSION) $(IMAGE_REGISTRY):$(DASHBOARD_API_VERSION) + docker push $(IMAGE_REGISTRY):latest + docker push $(IMAGE_REGISTRY):$(DASHBOARD_API_VERSION) +endif .PHONY: run run: diff --git a/packages/nomad-nodepool-apm/Makefile b/packages/nomad-nodepool-apm/Makefile index 5c04248062..68a1acc799 100644 --- a/packages/nomad-nodepool-apm/Makefile +++ b/packages/nomad-nodepool-apm/Makefile @@ -4,6 +4,24 @@ ENV := $(shell cat ../../.last_used_env || echo "not-set") AWS_BUCKET_PREFIX ?= $(PREFIX)$(AWS_ACCOUNT_ID)- GCP_BUCKET_PREFIX ?= $(GCP_PROJECT_ID)- +# Public bucket holding released, prebuilt plugin binaries (objects are +# /nomad-nodepool-apm and /nomad-deployment-aware-target). +# Published to by the release-please workflow on tagging a release; +# public-read, so downloads need no credentials. +E2B_ARTIFACT_BINARIES_URL ?= https://storage.googleapis.com/e2b-artifact-binaries/nomad-nodepool-apm + +# Optional version "hook" for the released-artifact flow. +# When empty (default), build-and-upload builds from source and uploads the +# binaries to the client's fc-env-pipeline bucket (the existing flow, +# unchanged). When set (e.g. NOMAD_NODEPOOL_APM_VERSION=v0.1.0), skip +# building: download the released binaries and upload those instead. Both +# plugins release together under one version. +# +# Set it either on the command line or in the active .env.${ENV} file, which +# is included above. Quotes are stripped so both `=v0.1.0` and `="v0.1.0"` +# work; a command-line value always wins over the env file. +NOMAD_NODEPOOL_APM_VERSION := $(strip $(subst ",,$(NOMAD_NODEPOOL_APM_VERSION))) + PLUGIN_NAME := nomad-nodepool-apm TARGET_PLUGIN_NAME := nomad-deployment-aware-target BIN_DIR := bin @@ -41,5 +59,23 @@ else gcloud storage cp --cache-control="no-cache, max-age=0" $(TARGET_BINARY) "gs://${GCP_BUCKET_PREFIX}fc-env-pipeline/nomad-deployment-aware-target.$(COMMIT_SHA)" endif +# Released-artifact flow: download the prebuilt, versioned binaries from the +# public E2B artifact-binaries bucket. +.PHONY: pull-released +pull-released: + @echo "Using released nomad-nodepool-apm $(NOMAD_NODEPOOL_APM_VERSION) from $(E2B_ARTIFACT_BINARIES_URL)" + @mkdir -p $(BIN_DIR) + curl -fL -o $(BINARY) "$(E2B_ARTIFACT_BINARIES_URL)/$(NOMAD_NODEPOOL_APM_VERSION)/$(PLUGIN_NAME)" + curl -fL -o $(TARGET_BINARY) "$(E2B_ARTIFACT_BINARIES_URL)/$(NOMAD_NODEPOOL_APM_VERSION)/$(TARGET_PLUGIN_NAME)" + +# The binaries come from source (default) or from released objects when +# NOMAD_NODEPOOL_APM_VERSION is set. Either way they land in ./bin and the +# upload target is identical. +ifeq ($(strip $(NOMAD_NODEPOOL_APM_VERSION)),) +BIN_SOURCE := build +else +BIN_SOURCE := pull-released +endif + .PHONY: build-and-upload -build-and-upload: build upload +build-and-upload: $(BIN_SOURCE) upload diff --git a/packages/orchestrator/Makefile b/packages/orchestrator/Makefile index 473a72aec5..a2ef8d6735 100644 --- a/packages/orchestrator/Makefile +++ b/packages/orchestrator/Makefile @@ -4,6 +4,29 @@ ENV := $(shell cat ../../.last_used_env || echo "not-set") AWS_BUCKET_PREFIX ?= $(PREFIX)$(AWS_ACCOUNT_ID)- GCP_BUCKET_PREFIX ?= $(GCP_PROJECT_ID)- +# Public bucket holding released, prebuilt orchestrator-package binaries +# (objects are /orchestrator and /clean-nfs-cache; there is +# no separate template-manager object — it is the orchestrator binary +# uploaded under a second name, matching upload/template-manager below). +# Published to by the release-please workflow on tagging a release; +# public-read, so downloads need no credentials. +E2B_ARTIFACT_BINARIES_URL ?= https://storage.googleapis.com/e2b-artifact-binaries/orchestrator + +# Optional version "hook" for the released-artifact flow. +# When empty (default), the build-and-upload/* targets build from source and +# upload the binaries to the client's fc-env-pipeline bucket (the existing +# flow, unchanged). When set (e.g. ORCHESTRATOR_VERSION=v0.1.0), skip +# building: download the released binaries and upload those instead. The env +# pipeline keeps fetching from the same bucket objects either way. One +# version covers orchestrator, template-manager, and clean-nfs-cache — they +# release together. +# +# Set it either on the command line (make ... ORCHESTRATOR_VERSION=v0.1.0) or +# in the active .env.${ENV} file, which is included above. Quotes are +# stripped so both `=v0.1.0` and `="v0.1.0"` work; a command-line value +# always wins over the env file. +ORCHESTRATOR_VERSION := $(strip $(subst ",,$(ORCHESTRATOR_VERSION))) + HOSTNAME := $(shell hostname 2> /dev/null || hostnamectl hostname 2> /dev/null) $(if $(HOSTNAME),,$(error Failed to determine hostname: both 'hostname' and 'hostnamectl' failed)) @@ -120,14 +143,32 @@ else gcloud storage cp --cache-control="no-cache, max-age=0" ./bin/orchestrator "gs://${GCP_BUCKET_PREFIX}fc-env-pipeline/template-manager.$(COMMIT_SHA)" endif +# Released-artifact flow: download the prebuilt, versioned binaries from the +# public E2B artifact-binaries bucket. +.PHONY: pull-released +pull-released: + @echo "Using released orchestrator $(ORCHESTRATOR_VERSION) from $(E2B_ARTIFACT_BINARIES_URL)" + mkdir -p bin + curl -fL -o bin/orchestrator "$(E2B_ARTIFACT_BINARIES_URL)/$(ORCHESTRATOR_VERSION)/orchestrator" + curl -fL -o bin/clean-nfs-cache "$(E2B_ARTIFACT_BINARIES_URL)/$(ORCHESTRATOR_VERSION)/clean-nfs-cache" + +# The binaries come from source (default) or from released objects when +# ORCHESTRATOR_VERSION is set. Either way they land in ./bin and the +# upload/* targets are identical. +ifeq ($(strip $(ORCHESTRATOR_VERSION)),) +BIN_SOURCE := build +else +BIN_SOURCE := pull-released +endif + .PHONY: build-and-upload/clean-nfs-cache -build-and-upload/clean-nfs-cache: build upload/clean-nfs-cache +build-and-upload/clean-nfs-cache: $(BIN_SOURCE) upload/clean-nfs-cache .PHONY: build-and-upload/orchestrator -build-and-upload/orchestrator: build upload/orchestrator +build-and-upload/orchestrator: $(BIN_SOURCE) upload/orchestrator .PHONY: build-and-upload/template-manager -build-and-upload/template-manager: build upload/template-manager +build-and-upload/template-manager: $(BIN_SOURCE) upload/template-manager .PHONY: test diff --git a/release-please-config.json b/release-please-config.json index 3f67e903a8..e6c8e6ed59 100644 --- a/release-please-config.json +++ b/release-please-config.json @@ -29,6 +29,30 @@ "component": "envd", "include-v-in-tag": true, "changelog-path": "CHANGELOG.md" + }, + "packages/orchestrator": { + "release-type": "go", + "component": "orchestrator", + "include-v-in-tag": true, + "changelog-path": "CHANGELOG.md" + }, + "packages/api": { + "release-type": "go", + "component": "api", + "include-v-in-tag": true, + "changelog-path": "CHANGELOG.md" + }, + "packages/dashboard-api": { + "release-type": "go", + "component": "dashboard-api", + "include-v-in-tag": true, + "changelog-path": "CHANGELOG.md" + }, + "packages/nomad-nodepool-apm": { + "release-type": "go", + "component": "nomad-nodepool-apm", + "include-v-in-tag": true, + "changelog-path": "CHANGELOG.md" } } } From c99e59144181a556312b0b836617f79323a8b001 Mon Sep 17 00:00:00 2001 From: Charlie Wyse Date: Thu, 30 Jul 2026 11:13:09 -0700 Subject: [PATCH 2/3] added checksum for downloads --- .github/workflows/publish.yml | 34 +++++++++++++++++++++++++++- packages/nomad-nodepool-apm/Makefile | 12 +++++++++- packages/orchestrator/Makefile | 12 +++++++++- 3 files changed, 55 insertions(+), 3 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 2e2141bea5..cbc0481320 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -21,7 +21,10 @@ on: required: true permissions: - contents: read + # contents: write (not read): binary releases attach a SHA256SUMS asset to + # the tag's GitHub release so consumers can verify bucket downloads against + # a trust root outside the bucket (see the checksums step below). + contents: write id-token: write # One publish per tag at a time. The e2b-artifacts registry has @@ -299,3 +302,32 @@ jobs: for f in ${BINARY_FILES}; do gcloud storage cp "packages/${COMPONENT}/bin/${f}" "${BINARY_DIR}/${f}" done + + # Checksums go to the tag's GitHub release, NOT to the bucket: a + # SHA256SUMS object next to the binaries would share their trust domain + # (whoever could replace a binary could replace it too), so the package + # Makefiles' pull-released flows verify bucket downloads against this + # release asset instead — tampering would have to compromise both GCS + # and GitHub. + - name: Attach checksums to the GitHub release + if: ${{ steps.pkg.outputs.kind == 'binary' }} + env: + COMPONENT: ${{ steps.pkg.outputs.component }} + BINARY_FILES: ${{ steps.pkg.outputs.binary_files }} + TAG: ${{ inputs.tag || github.ref_name }} + GH_TOKEN: ${{ github.token }} + run: | + cd "packages/${COMPONENT}/bin" + sha256sum ${BINARY_FILES} > SHA256SUMS + cat SHA256SUMS + if gh release view "${TAG}" --repo "${GITHUB_REPOSITORY}" >/dev/null 2>&1; then + # --clobber keeps a workflow_dispatch re-publish idempotent (the + # binaries themselves can't change: the bucket rejects overwrites). + gh release upload "${TAG}" SHA256SUMS --clobber --repo "${GITHUB_REPOSITORY}" + else + # Hand-pushed RC tags have no Release Please release; create a + # prerelease to carry the checksums. + gh release create "${TAG}" SHA256SUMS --prerelease --verify-tag \ + --title "${TAG}" --notes "Checksums for the ${TAG} binaries." \ + --repo "${GITHUB_REPOSITORY}" + fi diff --git a/packages/nomad-nodepool-apm/Makefile b/packages/nomad-nodepool-apm/Makefile index 68a1acc799..43ba8f4b79 100644 --- a/packages/nomad-nodepool-apm/Makefile +++ b/packages/nomad-nodepool-apm/Makefile @@ -10,6 +10,12 @@ GCP_BUCKET_PREFIX ?= $(GCP_PROJECT_ID)- # public-read, so downloads need no credentials. E2B_ARTIFACT_BINARIES_URL ?= https://storage.googleapis.com/e2b-artifact-binaries/nomad-nodepool-apm +# Where the release's SHA256SUMS asset lives. Deliberately a different trust +# domain than the binaries bucket: a checksum file sitting next to the +# binaries could be replaced together with them, so pull-released verifies +# the downloads against the GitHub release instead. +E2B_RELEASES_URL ?= https://github.com/e2b-dev/infra/releases/download + # Optional version "hook" for the released-artifact flow. # When empty (default), build-and-upload builds from source and uploads the # binaries to the client's fc-env-pipeline bucket (the existing flow, @@ -60,13 +66,17 @@ else endif # Released-artifact flow: download the prebuilt, versioned binaries from the -# public E2B artifact-binaries bucket. +# public E2B artifact-binaries bucket and verify them against the SHA256SUMS +# asset on the release's GitHub release before anything is uploaded. A failed +# check stops make here, so tampered binaries never reach fc-env-pipeline. .PHONY: pull-released pull-released: @echo "Using released nomad-nodepool-apm $(NOMAD_NODEPOOL_APM_VERSION) from $(E2B_ARTIFACT_BINARIES_URL)" @mkdir -p $(BIN_DIR) curl -fL -o $(BINARY) "$(E2B_ARTIFACT_BINARIES_URL)/$(NOMAD_NODEPOOL_APM_VERSION)/$(PLUGIN_NAME)" curl -fL -o $(TARGET_BINARY) "$(E2B_ARTIFACT_BINARIES_URL)/$(NOMAD_NODEPOOL_APM_VERSION)/$(TARGET_PLUGIN_NAME)" + curl -fL -o $(BIN_DIR)/SHA256SUMS "$(E2B_RELEASES_URL)/$(PLUGIN_NAME)-$(NOMAD_NODEPOOL_APM_VERSION)/SHA256SUMS" + cd $(BIN_DIR) && if command -v sha256sum > /dev/null 2>&1; then sha256sum -c SHA256SUMS; else shasum -a 256 -c SHA256SUMS; fi # The binaries come from source (default) or from released objects when # NOMAD_NODEPOOL_APM_VERSION is set. Either way they land in ./bin and the diff --git a/packages/orchestrator/Makefile b/packages/orchestrator/Makefile index a2ef8d6735..df968a2128 100644 --- a/packages/orchestrator/Makefile +++ b/packages/orchestrator/Makefile @@ -12,6 +12,12 @@ GCP_BUCKET_PREFIX ?= $(GCP_PROJECT_ID)- # public-read, so downloads need no credentials. E2B_ARTIFACT_BINARIES_URL ?= https://storage.googleapis.com/e2b-artifact-binaries/orchestrator +# Where the release's SHA256SUMS asset lives. Deliberately a different trust +# domain than the binaries bucket: a checksum file sitting next to the +# binaries could be replaced together with them, so pull-released verifies +# the downloads against the GitHub release instead. +E2B_RELEASES_URL ?= https://github.com/e2b-dev/infra/releases/download + # Optional version "hook" for the released-artifact flow. # When empty (default), the build-and-upload/* targets build from source and # upload the binaries to the client's fc-env-pipeline bucket (the existing @@ -144,13 +150,17 @@ else endif # Released-artifact flow: download the prebuilt, versioned binaries from the -# public E2B artifact-binaries bucket. +# public E2B artifact-binaries bucket and verify them against the SHA256SUMS +# asset on the release's GitHub release before anything is uploaded. A failed +# check stops make here, so tampered binaries never reach fc-env-pipeline. .PHONY: pull-released pull-released: @echo "Using released orchestrator $(ORCHESTRATOR_VERSION) from $(E2B_ARTIFACT_BINARIES_URL)" mkdir -p bin curl -fL -o bin/orchestrator "$(E2B_ARTIFACT_BINARIES_URL)/$(ORCHESTRATOR_VERSION)/orchestrator" curl -fL -o bin/clean-nfs-cache "$(E2B_ARTIFACT_BINARIES_URL)/$(ORCHESTRATOR_VERSION)/clean-nfs-cache" + curl -fL -o bin/SHA256SUMS "$(E2B_RELEASES_URL)/orchestrator-$(ORCHESTRATOR_VERSION)/SHA256SUMS" + cd bin && if command -v sha256sum > /dev/null 2>&1; then sha256sum -c SHA256SUMS; else shasum -a 256 -c SHA256SUMS; fi # The binaries come from source (default) or from released objects when # ORCHESTRATOR_VERSION is set. Either way they land in ./bin and the From 424e210a8d1cb752f5f83a1f2413a37b67d99f12 Mon Sep 17 00:00:00 2001 From: Charlie Wyse Date: Thu, 30 Jul 2026 11:20:40 -0700 Subject: [PATCH 3/3] Revert "added checksum for downloads" This reverts commit c99e59144181a556312b0b836617f79323a8b001. --- .github/workflows/publish.yml | 34 +--------------------------- packages/nomad-nodepool-apm/Makefile | 12 +--------- packages/orchestrator/Makefile | 12 +--------- 3 files changed, 3 insertions(+), 55 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index cbc0481320..2e2141bea5 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -21,10 +21,7 @@ on: required: true permissions: - # contents: write (not read): binary releases attach a SHA256SUMS asset to - # the tag's GitHub release so consumers can verify bucket downloads against - # a trust root outside the bucket (see the checksums step below). - contents: write + contents: read id-token: write # One publish per tag at a time. The e2b-artifacts registry has @@ -302,32 +299,3 @@ jobs: for f in ${BINARY_FILES}; do gcloud storage cp "packages/${COMPONENT}/bin/${f}" "${BINARY_DIR}/${f}" done - - # Checksums go to the tag's GitHub release, NOT to the bucket: a - # SHA256SUMS object next to the binaries would share their trust domain - # (whoever could replace a binary could replace it too), so the package - # Makefiles' pull-released flows verify bucket downloads against this - # release asset instead — tampering would have to compromise both GCS - # and GitHub. - - name: Attach checksums to the GitHub release - if: ${{ steps.pkg.outputs.kind == 'binary' }} - env: - COMPONENT: ${{ steps.pkg.outputs.component }} - BINARY_FILES: ${{ steps.pkg.outputs.binary_files }} - TAG: ${{ inputs.tag || github.ref_name }} - GH_TOKEN: ${{ github.token }} - run: | - cd "packages/${COMPONENT}/bin" - sha256sum ${BINARY_FILES} > SHA256SUMS - cat SHA256SUMS - if gh release view "${TAG}" --repo "${GITHUB_REPOSITORY}" >/dev/null 2>&1; then - # --clobber keeps a workflow_dispatch re-publish idempotent (the - # binaries themselves can't change: the bucket rejects overwrites). - gh release upload "${TAG}" SHA256SUMS --clobber --repo "${GITHUB_REPOSITORY}" - else - # Hand-pushed RC tags have no Release Please release; create a - # prerelease to carry the checksums. - gh release create "${TAG}" SHA256SUMS --prerelease --verify-tag \ - --title "${TAG}" --notes "Checksums for the ${TAG} binaries." \ - --repo "${GITHUB_REPOSITORY}" - fi diff --git a/packages/nomad-nodepool-apm/Makefile b/packages/nomad-nodepool-apm/Makefile index 43ba8f4b79..68a1acc799 100644 --- a/packages/nomad-nodepool-apm/Makefile +++ b/packages/nomad-nodepool-apm/Makefile @@ -10,12 +10,6 @@ GCP_BUCKET_PREFIX ?= $(GCP_PROJECT_ID)- # public-read, so downloads need no credentials. E2B_ARTIFACT_BINARIES_URL ?= https://storage.googleapis.com/e2b-artifact-binaries/nomad-nodepool-apm -# Where the release's SHA256SUMS asset lives. Deliberately a different trust -# domain than the binaries bucket: a checksum file sitting next to the -# binaries could be replaced together with them, so pull-released verifies -# the downloads against the GitHub release instead. -E2B_RELEASES_URL ?= https://github.com/e2b-dev/infra/releases/download - # Optional version "hook" for the released-artifact flow. # When empty (default), build-and-upload builds from source and uploads the # binaries to the client's fc-env-pipeline bucket (the existing flow, @@ -66,17 +60,13 @@ else endif # Released-artifact flow: download the prebuilt, versioned binaries from the -# public E2B artifact-binaries bucket and verify them against the SHA256SUMS -# asset on the release's GitHub release before anything is uploaded. A failed -# check stops make here, so tampered binaries never reach fc-env-pipeline. +# public E2B artifact-binaries bucket. .PHONY: pull-released pull-released: @echo "Using released nomad-nodepool-apm $(NOMAD_NODEPOOL_APM_VERSION) from $(E2B_ARTIFACT_BINARIES_URL)" @mkdir -p $(BIN_DIR) curl -fL -o $(BINARY) "$(E2B_ARTIFACT_BINARIES_URL)/$(NOMAD_NODEPOOL_APM_VERSION)/$(PLUGIN_NAME)" curl -fL -o $(TARGET_BINARY) "$(E2B_ARTIFACT_BINARIES_URL)/$(NOMAD_NODEPOOL_APM_VERSION)/$(TARGET_PLUGIN_NAME)" - curl -fL -o $(BIN_DIR)/SHA256SUMS "$(E2B_RELEASES_URL)/$(PLUGIN_NAME)-$(NOMAD_NODEPOOL_APM_VERSION)/SHA256SUMS" - cd $(BIN_DIR) && if command -v sha256sum > /dev/null 2>&1; then sha256sum -c SHA256SUMS; else shasum -a 256 -c SHA256SUMS; fi # The binaries come from source (default) or from released objects when # NOMAD_NODEPOOL_APM_VERSION is set. Either way they land in ./bin and the diff --git a/packages/orchestrator/Makefile b/packages/orchestrator/Makefile index df968a2128..a2ef8d6735 100644 --- a/packages/orchestrator/Makefile +++ b/packages/orchestrator/Makefile @@ -12,12 +12,6 @@ GCP_BUCKET_PREFIX ?= $(GCP_PROJECT_ID)- # public-read, so downloads need no credentials. E2B_ARTIFACT_BINARIES_URL ?= https://storage.googleapis.com/e2b-artifact-binaries/orchestrator -# Where the release's SHA256SUMS asset lives. Deliberately a different trust -# domain than the binaries bucket: a checksum file sitting next to the -# binaries could be replaced together with them, so pull-released verifies -# the downloads against the GitHub release instead. -E2B_RELEASES_URL ?= https://github.com/e2b-dev/infra/releases/download - # Optional version "hook" for the released-artifact flow. # When empty (default), the build-and-upload/* targets build from source and # upload the binaries to the client's fc-env-pipeline bucket (the existing @@ -150,17 +144,13 @@ else endif # Released-artifact flow: download the prebuilt, versioned binaries from the -# public E2B artifact-binaries bucket and verify them against the SHA256SUMS -# asset on the release's GitHub release before anything is uploaded. A failed -# check stops make here, so tampered binaries never reach fc-env-pipeline. +# public E2B artifact-binaries bucket. .PHONY: pull-released pull-released: @echo "Using released orchestrator $(ORCHESTRATOR_VERSION) from $(E2B_ARTIFACT_BINARIES_URL)" mkdir -p bin curl -fL -o bin/orchestrator "$(E2B_ARTIFACT_BINARIES_URL)/$(ORCHESTRATOR_VERSION)/orchestrator" curl -fL -o bin/clean-nfs-cache "$(E2B_ARTIFACT_BINARIES_URL)/$(ORCHESTRATOR_VERSION)/clean-nfs-cache" - curl -fL -o bin/SHA256SUMS "$(E2B_RELEASES_URL)/orchestrator-$(ORCHESTRATOR_VERSION)/SHA256SUMS" - cd bin && if command -v sha256sum > /dev/null 2>&1; then sha256sum -c SHA256SUMS; else shasum -a 256 -c SHA256SUMS; fi # The binaries come from source (default) or from released objects when # ORCHESTRATOR_VERSION is set. Either way they land in ./bin and the