diff --git a/.github/TRINO_IMAGE_RELEASES.md b/.github/TRINO_IMAGE_RELEASES.md new file mode 100644 index 000000000000..998349dccdd9 --- /dev/null +++ b/.github/TRINO_IMAGE_RELEASES.md @@ -0,0 +1,66 @@ +# PostHog image releases + +The publisher runs only for pushes to `master` and manual dispatches on `master`. +Tag pushes no longer publish images. Use a manual dispatch on `master` to add a +readable release alias. Pull requests run contract tests without package-write +permission. + +The ordered tag is `r<12-digit first-parent commit count>-<6-character revision>`. +The checkout must contain complete history and match the workflow revision. +This orders releases by source history, independently of build completion order. + +The canonical image repository is `posthog-trino` in `us-east-1`. +The workflow derives its registry address from the authenticated ECR login. +The publisher mirrors the same image to `ghcr.io/posthog/trino` for existing +consumers. It builds once, then copies the image and its layers between +registries without rebuilding. + +The image is wrapped in an OCI index carrying manifest-level +`org.opencontainers.image.source` and `org.opencontainers.image.revision` +annotations. The ordered tag, full revision alias, and optional readable alias +resolve to this same index digest. ECR receives only the ordered and full +revision aliases. Readable aliases remain GHCR-only. The existing charts state +dispatch receives the digest only after both registries pass read-back checks +and continues to run only after a push to `master`. Merging this workflow can +therefore roll the existing dev Trino deployment independently of new cells. + +The publisher uses a Buildx `docker-container` builder and the registry exporter +with `oci-mediatypes=true`. The default local `core/docker/build.sh` behavior +stays unchanged. The publisher rejects non-OCI image manifests before it adds +annotations: Buildx does not add index annotations to Docker manifest lists. +See the [Buildx index creation implementation](https://github.com/docker/buildx/blob/master/util/imagetools/create.go) +and [BuildKit OCI exporter option](https://github.com/moby/buildkit/blob/master/exporter/containerimage/exptypes/keys.go). + +Retries and manual runs reuse a verified ECR release and skip the build. If the +workflow stopped before it created the ordered index, it reuses the immutable +`build-` staging image instead. That image carries source and +revision annotations on its OCI manifest. Staging tags are not eligible +releases. If GHCR failed after ECR succeeded, a retry repairs only missing +aliases. Existing immutable aliases must contain exactly the expected digest; +the publisher never overwrites them. Only a GHCR readable alias can move. +Registry read failures or incorrect provenance stop publication. Do not delete +an ordered tag to force a rebuild; publish a new source commit instead. + +Before merging, apply the separate infrastructure change that creates the +fully immutable ECR repository and the master-only publisher role. Configure the +organization Actions secret `AWS_ECR_PUBLISH_IAM_ROLE` with the publisher role ARN +and grant this repository access: +`arn:aws:iam:::role/github-trino-publish-role`. This is an operator +setup step, not a change performed by this workflow. The workflow masks the AWS +account ID in logs and does not store its registry address in source. OIDC trust must +allow only `repo:PostHog/trino:ref:refs/heads/master`. The role must allow image +push/read operations but no image deletion or repository-policy changes. + +The workflow serializes its publishers. This prevents races within this +workflow, but does not itself establish registry immutability or exclusive +writers. Before enabling ECR release discovery, verify the applied repository +immutability, protected source history, repository access, and effective writer +permissions. GHCR remains a compatibility mirror and is not the new cells' +trusted release source. This change does not modify repository rules, package +access, Actions secrets, or registry settings. + +Run the local contract tests with `python3 .github/bin/test_trino_release.py`. +These tests mock the registry commands; they do not publish images. After the +first authorized real publication, verify its ordered tag, index annotations, +and revision-alias digest through the registry before enabling deployment +discovery. diff --git a/.github/bin/test_trino_release.py b/.github/bin/test_trino_release.py new file mode 100644 index 000000000000..4ce3c54f1ccc --- /dev/null +++ b/.github/bin/test_trino_release.py @@ -0,0 +1,346 @@ +#!/usr/bin/env python3 + +import json +import os +from pathlib import Path +import subprocess +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[2] +SCRIPT = ROOT / ".github/bin/trino-release.sh" +REPOSITORY = "111122223333.dkr.ecr.us-east-1.amazonaws.com/posthog-trino" +MIRROR = "ghcr.io/posthog/trino" +RAW_DIGEST = "sha256:" + "1" * 64 +RELEASE_DIGEST = "sha256:" + "2" * 64 + +DOCKER = r'''#!/usr/bin/env python3 +import json +import os +from pathlib import Path +import sys + +path = Path(os.environ["MOCK_REGISTRY"]) +state = json.loads(path.read_text()) +args = sys.argv[1:] +if args[:3] == ["buildx", "imagetools", "inspect"]: + reference = args[-1] + if reference in state.get("unavailable_references", {}): + print(state["unavailable_references"][reference], file=sys.stderr) + sys.exit(1) + if state.get("unavailable"): + print(state["unavailable"], file=sys.stderr) + sys.exit(1) + digest = reference.split("@", 1)[1] if "@" in reference else state["tags"].get(reference) + if not digest: + print("manifest unknown", file=sys.stderr) + sys.exit(1) + if reference in state.get("readback_mismatch", []): + digest = "sha256:" + "3" * 64 + print(json.dumps(state["manifests"][digest] if "--raw" in args else digest)) +elif args[:3] == ["buildx", "imagetools", "create"]: + target = args[args.index("--tag") + 1] + if target in state.get("fail_targets", []): + print("registry temporarily unavailable", file=sys.stderr) + sys.exit(1) + if target.startswith("111122223333.") and target in state["tags"]: + print("ImageTagAlreadyExistsException", file=sys.stderr) + sys.exit(1) + digest = args[-1].split("@", 1)[1] + if "--annotation" in args: + digest = "sha256:" + "2" * 64 + annotations = dict(args[i + 1].removeprefix("index:").split("=", 1) + for i, item in enumerate(args) if item == "--annotation") + state["manifests"][digest] = {"annotations": annotations, + "mediaType": "application/vnd.oci.image.index.v1+json"} + state["tags"][target] = digest + state["writes"].append(target) + path.write_text(json.dumps(state)) +else: + raise RuntimeError(args) +''' + + +class ReleaseContractTest(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.directory = Path(self.temporary.name) + self.registry = self.directory / "registry.json" + self.registry.write_text(json.dumps({ + "tags": {}, "manifests": {RAW_DIGEST: { + "mediaType": "application/vnd.oci.image.manifest.v1+json"}}, "writes": []})) + self.output = self.directory / "output" + self.output.touch() + for name, content in { + "docker": DOCKER, + "timeout": '#!/bin/sh\nshift\nshift\nexec "$@"\n', + }.items(): + command = self.directory / name + command.write_text(content) + command.chmod(0o755) + self.sha = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ROOT, text=True).strip() + count = int(subprocess.check_output(["git", "rev-list", "--first-parent", "--count", "HEAD"], cwd=ROOT, text=True)) + self.tag = f"r{count:012d}-{self.sha[:6]}" + self.environment = dict(os.environ, PATH=f"{self.directory}:{os.environ['PATH']}", + ECR_REPOSITORY=REPOSITORY, + MOCK_REGISTRY=str(self.registry), GITHUB_OUTPUT=str(self.output), + GITHUB_REPOSITORY="PostHog/trino", GITHUB_REF="refs/heads/master", + GITHUB_EVENT_NAME="push", GITHUB_SHA=self.sha, + GITHUB_RUN_ID="12345", GITHUB_RUN_ATTEMPT="1", BUILD_DIGEST=RAW_DIGEST) + state = self.state() + state["manifests"][RAW_DIGEST]["annotations"] = { + "org.opencontainers.image.source": "https://github.com/PostHog/trino", + "org.opencontainers.image.revision": self.sha} + self.registry.write_text(json.dumps(state)) + + def run_phase(self, phase, success=True): + result = subprocess.run(["bash", str(SCRIPT), phase], cwd=ROOT, + env=self.environment, capture_output=True, text=True) + self.assertEqual(result.returncode == 0, success, result.stderr) + return result + + def state(self): + return json.loads(self.registry.read_text()) + + def test_new_release_and_rerun_preserve_digest_and_provenance(self): + self.run_phase("prepare") + self.assertIn(f"ordered-tag={self.tag}\ndigest=\n", self.output.read_text()) + self.run_phase("publish") + state = self.state() + self.assertEqual(state["tags"][f"{REPOSITORY}:{self.tag}"], RELEASE_DIGEST) + self.assertEqual(state["tags"][f"{REPOSITORY}:{self.sha}"], RELEASE_DIGEST) + self.assertEqual(state["tags"][f"{MIRROR}:{self.tag}"], RELEASE_DIGEST) + self.assertEqual(state["tags"][f"{MIRROR}:{self.sha}"], RELEASE_DIGEST) + self.assertEqual(state["manifests"][RELEASE_DIGEST]["annotations"], { + "org.opencontainers.image.source": "https://github.com/PostHog/trino", + "org.opencontainers.image.revision": self.sha}) + self.environment["BUILD_DIGEST"] = "" + self.environment["GITHUB_RUN_ATTEMPT"] = "2" + self.environment["READABLE_TAG"] = "test-release" + self.run_phase("prepare") + self.assertIn(f"digest={RELEASE_DIGEST}", self.output.read_text()) + self.run_phase("publish") + state = self.state() + self.assertEqual(state["writes"].count(f"{REPOSITORY}:{self.tag}"), 1) + self.assertEqual(state["tags"][f"{MIRROR}:test-release"], RELEASE_DIGEST) + self.assertNotIn(f"{REPOSITORY}:test-release", state["tags"]) + self.assertEqual(state["writes"].count(f"{REPOSITORY}:{self.sha}"), 1) + + def test_rejects_untrusted_refs_events_and_source_mismatch(self): + for key, value in [("GITHUB_REF", "refs/heads/feature"), + ("GITHUB_EVENT_NAME", "pull_request"), + ("GITHUB_REPOSITORY", "someone/trino"), + ("GITHUB_SHA", "a" * 40)]: + with self.subTest(key=key): + original = self.environment[key] + self.environment[key] = value + self.run_phase("publish", success=False) + self.environment[key] = original + self.assertEqual(self.state()["writes"], []) + + def test_canonical_ecr_repository_is_required(self): + for repository in ["", "ghcr.io/posthog/trino", REPOSITORY.replace("us-east-1", "eu-west-1"), + REPOSITORY.replace("posthog-trino", "other"), "https://" + REPOSITORY]: + with self.subTest(repository=repository): + self.environment["ECR_REPOSITORY"] = repository + self.run_phase("prepare", success=False) + self.run_phase("publish", success=False) + self.assertEqual(self.output.read_text(), "") + self.assertEqual(self.state()["writes"], []) + + def test_registry_error_is_not_treated_as_absence(self): + state = self.state() + state["unavailable"] = "unauthorized: access denied" + self.registry.write_text(json.dumps(state)) + self.run_phase("prepare", success=False) + self.run_phase("publish", success=False) + self.assertEqual(self.state()["writes"], []) + + def test_rejects_existing_release_with_wrong_provenance(self): + state = self.state() + state["tags"][f"{REPOSITORY}:{self.tag}"] = RAW_DIGEST + self.registry.write_text(json.dumps(state)) + self.run_phase("prepare", success=False) + self.run_phase("publish", success=False) + self.assertEqual(self.state()["writes"], []) + + def test_readable_alias_cannot_overwrite_an_ordered_release(self): + for tag in ["r000000000001-abcdef", "a" * 40, "--invalid"]: + with self.subTest(tag=tag): + self.environment["READABLE_TAG"] = tag + self.run_phase("publish", success=False) + self.assertEqual(self.state()["writes"], []) + + def test_generic_not_found_is_not_release_absence(self): + state = self.state() + state["unavailable"] = "docker: command not found" + self.registry.write_text(json.dumps(state)) + self.run_phase("prepare", success=False) + self.run_phase("publish", success=False) + self.assertEqual(self.state()["writes"], []) + + def test_docker_manifest_cannot_silently_drop_index_annotations(self): + state = self.state() + state["manifests"][RAW_DIGEST]["mediaType"] = "application/vnd.docker.distribution.manifest.v2+json" + self.registry.write_text(json.dumps(state)) + self.run_phase("publish", success=False) + self.assertEqual(self.state()["writes"], []) + + def test_partial_ecr_publication_recovers_ghcr_without_building(self): + state = self.state() + state["fail_targets"] = [f"{MIRROR}:{self.tag}"] + self.registry.write_text(json.dumps(state)) + self.run_phase("publish", success=False) + self.assertNotIn("digest=", self.output.read_text()) + state = self.state() + self.assertEqual(state["tags"][f"{REPOSITORY}:{self.tag}"], RELEASE_DIGEST) + state["fail_targets"] = [] + self.registry.write_text(json.dumps(state)) + self.environment["BUILD_DIGEST"] = "" + self.run_phase("prepare") + self.assertIn(f"digest={RELEASE_DIGEST}\n", self.output.read_text()) + self.run_phase("publish") + state = self.state() + self.assertEqual(state["tags"][f"{MIRROR}:{self.sha}"], RELEASE_DIGEST) + self.assertEqual(state["writes"].count(f"{REPOSITORY}:{self.tag}"), 1) + self.assertEqual(state["writes"].count(f"{REPOSITORY}:{self.sha}"), 1) + + def test_ordered_lookup_error_cannot_fall_back_to_valid_staging(self): + state = self.state() + state["tags"][f"{REPOSITORY}:build-{self.sha}"] = RAW_DIGEST + state["unavailable_references"] = {f"{REPOSITORY}:{self.tag}": "unauthorized: access denied"} + self.registry.write_text(json.dumps(state)) + self.run_phase("prepare", success=False) + self.run_phase("publish", success=False) + self.assertEqual(self.output.read_text(), "") + self.assertEqual(self.state()["writes"], []) + + def test_invalid_ordered_provenance_cannot_fall_back_to_valid_staging(self): + state = self.state() + state["tags"][f"{REPOSITORY}:build-{self.sha}"] = RAW_DIGEST + state["tags"][f"{REPOSITORY}:{self.tag}"] = RELEASE_DIGEST + state["manifests"][RELEASE_DIGEST] = { + "mediaType": "application/vnd.oci.image.index.v1+json", + "annotations": {"org.opencontainers.image.source": "https://github.com/someone/trino", + "org.opencontainers.image.revision": self.sha}} + self.registry.write_text(json.dumps(state)) + self.run_phase("prepare", success=False) + self.run_phase("publish", success=False) + self.assertEqual(self.output.read_text(), "") + self.assertEqual(self.state()["writes"], []) + + def test_partial_raw_build_recovers_before_ordered_release(self): + state = self.state() + state["tags"][f"{REPOSITORY}:build-{self.sha}"] = RAW_DIGEST + state["fail_targets"] = [f"{REPOSITORY}:{self.tag}"] + self.registry.write_text(json.dumps(state)) + self.run_phase("publish", success=False) + self.run_phase("prepare") + self.assertIn(f"digest=\nbuild-digest={RAW_DIGEST}\n", self.output.read_text()) + state = self.state() + state["fail_targets"] = [] + self.registry.write_text(json.dumps(state)) + self.run_phase("publish") + self.assertEqual(self.state()["tags"][f"{MIRROR}:{self.tag}"], RELEASE_DIGEST) + + def test_staged_build_requires_matching_source_provenance(self): + state = self.state() + state["tags"][f"{REPOSITORY}:build-{self.sha}"] = RAW_DIGEST + state["manifests"][RAW_DIGEST]["annotations"]["org.opencontainers.image.revision"] = "a" * 40 + self.registry.write_text(json.dumps(state)) + self.run_phase("prepare", success=False) + self.run_phase("publish", success=False) + self.assertEqual(self.state()["writes"], []) + + def test_immutable_alias_conflict_does_not_overwrite(self): + for repository in [REPOSITORY, MIRROR]: + with self.subTest(repository=repository): + state = self.state() + target = f"{repository}:{self.sha}" + state["tags"][target] = RAW_DIGEST + self.registry.write_text(json.dumps(state)) + self.run_phase("publish", success=False) + state = self.state() + self.assertEqual(state["tags"][target], RAW_DIGEST) + self.assertNotIn(target, state["writes"]) + del state["tags"][target] + self.registry.write_text(json.dumps(state)) + + def test_ghcr_readback_mismatch_prevents_state_output(self): + state = self.state() + state["readback_mismatch"] = [f"{MIRROR}:{self.tag}"] + self.registry.write_text(json.dumps(state)) + self.run_phase("publish", success=False) + self.assertEqual(self.output.read_text(), "") + + def test_readable_ghcr_alias_can_move_but_ecr_stays_immutable(self): + state = self.state() + state["tags"][f"{MIRROR}:latest"] = RAW_DIGEST + self.registry.write_text(json.dumps(state)) + self.environment["READABLE_TAG"] = "latest" + self.run_phase("publish") + state = self.state() + self.assertEqual(state["tags"][f"{MIRROR}:latest"], RELEASE_DIGEST) + self.assertNotIn(f"{REPOSITORY}:latest", state["tags"]) + + +class ImageBuildContractTest(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.directory = Path(self.temporary.name) + self.script = self.directory / "core/docker/build.sh" + self.script.parent.mkdir(parents=True) + self.script.write_text((ROOT / "core/docker/build.sh").read_text()) + self.calls = self.directory / "docker-arguments.json" + self.bin = self.directory / "bin" + self.bin.mkdir() + commands = { + self.directory / "mvnw": '#!/bin/sh\nprintf "test-version\\n"\n', + self.bin / "docker": '#!/usr/bin/env python3\nimport json, os, sys\nfrom pathlib import Path\nPath(os.environ["BUILD_ARGUMENTS"]).write_text(json.dumps(sys.argv[1:]))\n', + } + for name in ["cp", "tar", "mv", "rm"]: + commands[self.bin / name] = '#!/bin/sh\nexit 0\n' + for command, content in commands.items(): + command.write_text(content) + command.chmod(0o755) + self.environment = dict(os.environ, PATH=f"{self.bin}:{os.environ['PATH']}", + TMPDIR=str(self.directory), BUILD_ARGUMENTS=str(self.calls), + OCI_SOURCE="https://github.com/PostHog/trino", OCI_REVISION="a" * 40) + + def build(self, *arguments): + return subprocess.run(["bash", str(self.script), *arguments], env=self.environment, + capture_output=True, text=True) + + def test_oci_publication_uses_registry_exporter(self): + reference = f"{REPOSITORY}:build-12345-1" + result = self.build("-a", "arm64", "-x", "-o", reference) + self.assertEqual(result.returncode, 0, result.stderr) + arguments = json.loads(self.calls.read_text()) + self.assertEqual(arguments[:2], ["buildx", "build"]) + self.assertEqual(arguments[arguments.index("--output") + 1], "type=registry,oci-mediatypes=true") + self.assertIn("--provenance=false", arguments) + self.assertEqual(arguments[arguments.index("--tag") + 1], reference) + self.assertEqual(arguments[arguments.index("--platform") + 1], "linux/arm64") + self.assertIn("manifest:org.opencontainers.image.source=https://github.com/PostHog/trino", arguments) + self.assertIn("manifest:org.opencontainers.image.revision=" + "a" * 40, arguments) + + def test_default_local_build_stays_local(self): + result = self.build("-a", "arm64", "-x") + self.assertEqual(result.returncode, 0, result.stderr) + arguments = json.loads(self.calls.read_text()) + self.assertEqual(arguments[0], "build") + self.assertNotIn("--output", arguments) + self.assertEqual(arguments[arguments.index("-t") + 1], "trino:test-version-arm64") + + def test_oci_publication_rejects_multiple_architectures_and_local_tests(self): + for arguments in [("-x", "-o", "test"), ("-a", "arm64", "-o", "test")]: + with self.subTest(arguments=arguments): + self.assertNotEqual(self.build(*arguments).returncode, 0) + self.assertFalse(self.calls.exists()) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/bin/trino-release.sh b/.github/bin/trino-release.sh new file mode 100644 index 000000000000..427cebce5cd7 --- /dev/null +++ b/.github/bin/trino-release.sh @@ -0,0 +1,114 @@ +#!/usr/bin/env bash + +set -euo pipefail + +fail() { printf '%s\n' "$*" >&2; exit 1; } + +[[ "${GITHUB_REPOSITORY:-}" == PostHog/trino ]] || fail 'Only PostHog/trino can publish releases' +[[ "${GITHUB_REF:-}" == refs/heads/master ]] || fail 'Only master can publish releases' +[[ "${GITHUB_EVENT_NAME:-}" == push || "${GITHUB_EVENT_NAME:-}" == workflow_dispatch ]] || fail 'Unsupported release event' +[[ "${GITHUB_SHA:-}" =~ ^[0-9a-f]{40}$ ]] || fail 'A full source revision is required' +[[ "$(git rev-parse HEAD)" == "$GITHUB_SHA" ]] || fail 'Source revision does not match checkout' +[[ "$(git rev-parse --is-shallow-repository)" == false ]] || fail 'Complete source history is required' + +position="$(git rev-list --first-parent --count HEAD)" +[[ "$position" =~ ^[1-9][0-9]{0,11}$ ]] || fail 'Source position is outside the release tag range' +printf -v ordered_tag 'r%012d-%.6s' "$position" "$GITHUB_SHA" +repository="${ECR_REPOSITORY:-}" +[[ "$repository" =~ ^[0-9]{12}\.dkr\.ecr\.us-east-1\.amazonaws\.com/posthog-trino$ ]] || fail 'A canonical us-east-1 ECR repository is required' +mirror=ghcr.io/posthog/trino +source_url=https://github.com/PostHog/trino +build_tag="build-$GITHUB_SHA" + +scratch="$(mktemp -d)" +trap 'rm -r "$scratch"' EXIT + +inspect_digest() { + timeout --kill-after=10s 30s docker buildx imagetools inspect \ + --format '{{json .Manifest.Digest}}' "$1" | jq -er 'select(test("^sha256:[0-9a-f]{64}$"))' +} + +existing_digest() { + local reference=$1 + local digest + if digest="$(inspect_digest "$reference" 2> "$scratch/inspect-error")"; then + printf '%s\n' "$digest" + elif grep -Eqi 'manifest unknown' "$scratch/inspect-error" || + grep -Fqx "ERROR: $reference: not found" "$scratch/inspect-error"; then + return 0 + else + fail "Cannot determine whether $reference exists" + fi +} + +verify_manifest() { + timeout --kill-after=10s 30s docker buildx imagetools inspect --raw "$1" | + jq -e --arg revision "$GITHUB_SHA" --arg source "$source_url" --arg media_type "$2" ' + .mediaType == $media_type and + .annotations["org.opencontainers.image.revision"] == $revision and + .annotations["org.opencontainers.image.source"] == $source + ' >/dev/null || fail 'Image has invalid media type or source provenance' +} + +existing_release() { + local digest + digest="$(existing_digest "$repository:$ordered_tag")" || return 1 + if [[ -n "$digest" ]]; then + verify_manifest "$repository@$digest" application/vnd.oci.image.index.v1+json || return 1 + printf '%s\n' "$digest" + fi +} + +ensure_alias() { + local target=$1 digest=$2 replace=${3:-false} + local existing + existing="$(existing_digest "$target")" || return 1 + [[ "$existing" != "$digest" ]] || return 0 + [[ -z "$existing" || "$replace" == true ]] || fail "Immutable alias already contains a different image: $target" + timeout --kill-after=10s 600s docker buildx imagetools create --prefer-index=false \ + --tag "$target" "$repository@$digest" + [[ "$(inspect_digest "$target")" == "$digest" ]] || fail 'Alias read-back does not match release' +} + +case "${1:-}" in + prepare) + digest="$(existing_release)" + build_digest= + if [[ -z "$digest" ]]; then + build_digest="$(existing_digest "$repository:$build_tag")" + if [[ -n "$build_digest" ]]; then + verify_manifest "$repository@$build_digest" application/vnd.oci.image.manifest.v1+json + fi + fi + printf 'ordered-tag=%s\ndigest=%s\nbuild-digest=%s\n' "$ordered_tag" "$digest" "$build_digest" >> "${GITHUB_OUTPUT:?}" + ;; + publish) + readable_tag="${READABLE_TAG:-$GITHUB_SHA}" + [[ "$readable_tag" =~ ^[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}$ ]] || fail 'Invalid image alias' + [[ ! "$readable_tag" =~ ^r[0-9]{12}-[0-9a-f]{6}$ ]] || fail 'Readable tags cannot replace ordered releases' + [[ ! "$readable_tag" =~ ^[0-9a-f]{40}$ || "$readable_tag" == "$GITHUB_SHA" ]] || fail 'Readable tags cannot replace another source revision' + digest="$(existing_release)" + if [[ -z "$digest" ]]; then + [[ "${BUILD_DIGEST:-}" =~ ^sha256:[0-9a-f]{64}$ ]] || fail 'Build digest is required for a new release' + verify_manifest "$repository@$BUILD_DIGEST" application/vnd.oci.image.manifest.v1+json + timeout --kill-after=10s 60s docker buildx imagetools create \ + --annotation "index:org.opencontainers.image.source=$source_url" \ + --annotation "index:org.opencontainers.image.revision=$GITHUB_SHA" \ + --tag "$repository:$ordered_tag" "$repository@$BUILD_DIGEST" + digest="$(existing_release)" + [[ -n "$digest" ]] || fail 'Ordered release was not published' + fi + + ensure_alias "$repository:$GITHUB_SHA" "$digest" + ensure_alias "$mirror:$ordered_tag" "$digest" + ensure_alias "$mirror:$GITHUB_SHA" "$digest" + if [[ "$readable_tag" != "$GITHUB_SHA" ]]; then + ensure_alias "$mirror:$readable_tag" "$digest" true + fi + verify_manifest "$mirror@$digest" application/vnd.oci.image.index.v1+json + [[ "$(inspect_digest "$repository:$ordered_tag")" == "$digest" ]] || fail 'ECR release changed during publication' + [[ "$(inspect_digest "$mirror:$ordered_tag")" == "$digest" ]] || fail 'GHCR release does not match ECR' + printf 'digest=%s\n' "$digest" >> "${GITHUB_OUTPUT:?}" + ;; + *) fail 'Expected prepare or publish' ;; +esac diff --git a/.github/workflows/docker-publish-posthog.yml b/.github/workflows/docker-publish-posthog.yml index a2de89cf9e0a..a8175c6e1590 100644 --- a/.github/workflows/docker-publish-posthog.yml +++ b/.github/workflows/docker-publish-posthog.yml @@ -1,7 +1,8 @@ name: docker-publish-posthog # Builds the PostHog fork's Trino server image (with the ducklake connector) -# for linux/arm64 and pushes it to GHCR. The managed-warehouse trino Karpenter +# for linux/arm64 and publishes the same digest to ECR and GHCR. +# The managed-warehouse trino Karpenter # node pool is arm64-only, so amd64 is skipped to keep CI fast. # # Every run also tells PostHog/charts what it built. Charts records the image @@ -10,14 +11,16 @@ name: docker-publish-posthog # environment gates. See the deploy job at the end of this file. on: - # A merge to the deployment branch builds and reaches the dev cell on its - # own. Named releases and one-off rebuilds keep working through the other - # two triggers. + # Only the deployment branch publishes. Manual runs can add readable aliases. push: branches: - master - tags: - - "posthog-*" + pull_request: + paths: + - .github/workflows/docker-publish-posthog.yml + - .github/bin/trino-release.sh + - .github/bin/test_trino_release.py + - core/docker/build.sh workflow_dispatch: inputs: tag: @@ -29,32 +32,81 @@ on: permissions: contents: read -env: - IMAGE: ghcr.io/posthog/trino +concurrency: + group: trino-image-publisher-${{ github.event_name == 'pull_request' && github.event.pull_request.number || 'master' }} + cancel-in-progress: false jobs: + test-release-contract: + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + persist-credentials: false + fetch-depth: 0 + - run: python3 .github/bin/test_trino_release.py + build-and-push: + needs: test-release-contract + if: >- + github.repository == 'PostHog/trino' + && github.ref == 'refs/heads/master' + && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') runs-on: ubuntu-24.04-arm timeout-minutes: 150 permissions: contents: read + id-token: write packages: write outputs: - digest: ${{ steps.push.outputs.digest }} + digest: ${{ steps.release.outputs.digest }} steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 with: persist-credentials: false + fetch-depth: 0 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@61815dcd50bd041e203e49132bacad1fd04d2708 # v5.1.1 + with: + role-to-assume: ${{ secrets.AWS_ECR_PUBLISH_IAM_ROLE }} + aws-region: us-east-1 + mask-aws-account-id: true + + - name: Log in to ECR + id: ecr + uses: aws-actions/amazon-ecr-login@062b18b96a7aff071d4dc91bc00c4c1a7945b076 # v2.0.1 + + - name: Log in to GHCR + env: + GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GHCR_USER: ${{ github.actor }} + run: echo "${GHCR_TOKEN}" | docker login ghcr.io -u "${GHCR_USER}" --password-stdin + + - name: Resolve source-ordered release + id: metadata + env: + ECR_REPOSITORY: ${{ steps.ecr.outputs.registry }}/posthog-trino + run: bash .github/bin/trino-release.sh prepare + + - name: Set up OCI registry exporter + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 + if: steps.metadata.outputs.digest == '' && steps.metadata.outputs.build-digest == '' + with: + driver: docker-container # No Maven cache here on purpose. This job publishes an image and any # branch build can write that cache, so reading it would let a branch # decide what goes into a published artifact. - uses: actions/setup-java@cf277c60eb25467037889841efdb72551f06f6c3 # v4 + if: steps.metadata.outputs.digest == '' && steps.metadata.outputs.build-digest == '' with: distribution: temurin java-version: 25 - name: Build trino-server tarball (skip tests and checks) + if: steps.metadata.outputs.digest == '' && steps.metadata.outputs.build-digest == '' # Full build: the trino-server provisio assembly resolves every plugin # zip plus trino-server-core from the local repo, so -pl/-am is not # sufficient. @@ -62,69 +114,40 @@ jobs: ./mvnw clean install \ -T 1C -DskipTests -Dmaven.javadoc.skip=true -Dair.check.skip-all=true - - name: Build arm64 image - run: core/docker/build.sh -a arm64 -x + - name: Build and push arm64 OCI image + id: push + if: steps.metadata.outputs.digest == '' && steps.metadata.outputs.build-digest == '' + env: + ECR_REPOSITORY: ${{ steps.ecr.outputs.registry }}/posthog-trino + OCI_SOURCE: https://github.com/PostHog/trino + OCI_REVISION: ${{ github.sha }} + run: | + set -euo pipefail + staging_image="${ECR_REPOSITORY}:build-${GITHUB_SHA}" + core/docker/build.sh -a arm64 -x -o "$staging_image" + digest="$(docker buildx imagetools inspect --format '{{json .Manifest.Digest}}' "$staging_image" | jq -er 'select(test("^sha256:[0-9a-f]{64}$"))')" + echo "digest=${digest}" >> "$GITHUB_OUTPUT" - name: Resolve image tag id: tag - # A branch push has no name to take a tag from, so it is published - # under its commit alone. The other two triggers keep their readable - # tag, and every run publishes the commit tag as well, because that is - # what the deploy job records. + # Legacy consumers use the commit alias. Manual runs can add another alias. env: EVENT_NAME: ${{ github.event_name }} INPUT_TAG: ${{ inputs.tag }} run: | case "${EVENT_NAME}" in workflow_dispatch) tag="${INPUT_TAG}" ;; - *) if [ "${GITHUB_REF_TYPE}" = "tag" ]; then - tag="${GITHUB_REF_NAME#posthog-}" - else - tag="${GITHUB_SHA}" - fi ;; + *) tag="${GITHUB_SHA}" ;; esac echo "tag=${tag}" >> "$GITHUB_OUTPUT" - - name: Log in to GHCR - env: - GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GHCR_USER: ${{ github.actor }} - run: echo "${GHCR_TOKEN}" | docker login ghcr.io -u "${GHCR_USER}" --password-stdin - - - name: Push image - id: push - # Two tags for one image: the readable one a person asked for, and the - # commit, which the deploy job pins by digest. A run triggered by a - # branch push resolves both to the commit, and the second tag and push - # are then a no-op on the same reference. + - name: Publish verified release aliases + id: release env: - TAG: ${{ steps.tag.outputs.tag }} - run: | - set -euo pipefail - TRINO_VERSION="$(./mvnw --quiet help:evaluate -Dexpression=project.version -DforceStdout --raw-streams)" - docker tag "trino:${TRINO_VERSION}-arm64" "${IMAGE}:${TAG}" - docker push "${IMAGE}:${TAG}" - docker tag "trino:${TRINO_VERSION}-arm64" "${IMAGE}:${GITHUB_SHA}" - docker push "${IMAGE}:${GITHUB_SHA}" - - # RepoDigests is populated by the push above. Read the entry for this - # repository, so a digest another tag left behind cannot be picked. - # - # No pipeline here. `grep | head` under `set -o pipefail` fails the - # step when head closes the pipe first, which is a real failure mode - # in this org (see the same warning in charts state-update.yml). - digests="$(docker inspect --format '{{range .RepoDigests}}{{println .}}{{end}}' "${IMAGE}:${GITHUB_SHA}")" - digest="" - while IFS= read -r entry; do - case "${entry}" in - "${IMAGE}@"*) digest="${entry#*@}"; break ;; - esac - done <<< "${digests}" - if [ -z "${digest}" ]; then - echo "::error::could not read the digest of ${IMAGE}:${GITHUB_SHA}" - exit 1 - fi - echo "digest=${digest}" >> "$GITHUB_OUTPUT" + ECR_REPOSITORY: ${{ steps.ecr.outputs.registry }}/posthog-trino + BUILD_DIGEST: ${{ steps.push.outputs.digest || steps.metadata.outputs.build-digest }} + READABLE_TAG: ${{ steps.tag.outputs.tag }} + run: bash .github/bin/trino-release.sh publish deploy: # Tells charts what to run. Charts writes state/trino.yaml and rolls the diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 000000000000..0a7554a10f0e --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,38 @@ +# Agent guidance + +These instructions apply to all automated coding agents working in this repository. + +## Public repository and data handling + +**This repository is public. Never publish customer data or internal information here.** +This applies to code, comments, documentation, test fixtures, commit messages, +PR titles and descriptions, review comments, issues, screenshots, and CI logs or artifacts. + +- Do not include customer names, customer or organization identifiers, production data, + credentials, tokens, or other secrets. +- Do not include real AWS account IDs, role ARNs, internal hostnames, cluster names, + private endpoints, or other internal-only resource identifiers. +- Use clearly synthetic examples or placeholders such as ``, ``, + and ``. Do not copy diagnostic output into a public artifact + without checking and redacting it first. +- Keep necessary internal diagnostic details in an approved private channel, not in + this repository or its GitHub discussions. +- Obtain deployment-specific values through approved runtime configuration or GitHub + Actions secrets. Never print secret values, and mask internal identifiers in CI logs. +- Review the diff and all text intended for publication before committing or posting. + If information might be internal, redact it or ask before publishing it. +- If internal information was already published, stop further disclosure and report + it privately. Removing it from the latest file does not remove it from Git history, + logs, or previously published artifacts. Do not rewrite shared history without approval. + +## Repository conventions + +- Follow [CLAUDE.md](CLAUDE.md) and applicable directory-specific guidance. +- Before writing Java code, read [.github/DEVELOPMENT.md](.github/DEVELOPMENT.md) in full. + Use its build, test, and style instructions rather than copying commands from other projects. +- Keep changes focused and preserve unrelated work. Prefer correctness, maintainability, + and explicit configuration over shortcuts. +- For behavior changes, add or update realistic regression tests and run the relevant + checks. Report failures accurately; do not weaken tests to make CI pass. +- Document configuration defaults and operational changes where they belong. +- Report what was tested and distinguish local validation from a real publication or deployment. diff --git a/CLAUDE.md b/CLAUDE.md index bbfad92eafd6..045d7f3daf3d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,5 +1,7 @@ # Trino โ€” Claude guidance +All agents must follow [AGENTS.md](AGENTS.md), including its public-repository data-handling rules. + **Before writing Java code, you must first read [`.github/DEVELOPMENT.md`](.github/DEVELOPMENT.md) in full** โ€” it's the authoritative source for code-style rules (mocks, `var`, switch statements, method naming, `format()`, `TrinoException` error codes, AssertJ, Guava immutables, and more). diff --git a/core/docker/build.sh b/core/docker/build.sh index 695b28867249..7c3228218914 100755 --- a/core/docker/build.sh +++ b/core/docker/build.sh @@ -14,6 +14,8 @@ Builds the Trino Docker image -r Build the specified Trino release version, downloads all required artifacts -j Build the Trino release with specified JDK distribution -x Skip image tests +-o Publish a single-platform OCI image to this full image reference (requires -x) + Set OCI_SOURCE and OCI_REVISION to identify the source of the published image EOF } @@ -27,13 +29,14 @@ ARCHITECTURES=(amd64 arm64) TRINO_VERSION= TAG_PREFIX=trino SERVER_ARTIFACT=trino-server +OCI_IMAGE= TEMURIN_RELEASE=$("${SOURCE_DIR}/mvnw" -f "${SOURCE_DIR}/pom.xml" --quiet help:evaluate -Dexpression=temurin.release -DforceStdout --raw-streams) TEMURIN_DOWNLOAD_URL="https://api.adoptium.net/v3/binary/version/{release_name}/linux/{arch}/jdk/hotspot/normal/eclipse?project=jdk" SKIP_TESTS=false -while getopts ":a:h:r:p:t:j:x" o; do +while getopts ":a:h:r:p:t:j:xo:" o; do case "${o}" in a) IFS=, read -ra ARCH_ARG <<< "$OPTARG" @@ -64,6 +67,9 @@ while getopts ":a:h:r:p:t:j:x" o; do x) SKIP_TESTS=true ;; + o) + OCI_IMAGE=${OPTARG} + ;; *) usage exit 1 @@ -72,6 +78,11 @@ while getopts ":a:h:r:p:t:j:x" o; do done shift $((OPTIND - 1)) +if [[ -n "$OCI_IMAGE" && ( "${#ARCHITECTURES[@]}" != 1 || "$SKIP_TESTS" != true ) ]]; then + echo >&2 "OCI publication requires one architecture and skipped local image tests" + exit 1 +fi + function check_environment() { if ! command -v jq &> /dev/null; then echo >&2 "Please install jq" @@ -131,7 +142,15 @@ TAG="${TAG_PREFIX}:${TRINO_VERSION}" for arch in "${ARCHITECTURES[@]}"; do JDK_DOWNLOAD_LINK="$(temurin_download_uri "${TEMURIN_RELEASE}" "${arch}")" echo "๐Ÿซ™ Building the image for $arch with JDK ${JDK_DOWNLOAD_LINK}" - docker build \ + build_command=(docker build) + output_arguments=(-t "${TAG}-$arch") + if [[ -n "$OCI_IMAGE" ]]; then + build_command=(docker buildx build) + output_arguments=(--output type=registry,oci-mediatypes=true --provenance=false --tag "$OCI_IMAGE" + --annotation "manifest:org.opencontainers.image.source=${OCI_SOURCE:?}" + --annotation "manifest:org.opencontainers.image.revision=${OCI_REVISION:?}") + fi + "${build_command[@]}" \ "${WORK_DIR}" \ --progress=plain \ --pull \ @@ -140,7 +159,7 @@ for arch in "${ARCHITECTURES[@]}"; do --build-arg JDK_DOWNLOAD_LINK="${JDK_DOWNLOAD_LINK}" \ --platform "linux/$arch" \ -f Dockerfile \ - -t "${TAG}-$arch" + "${output_arguments[@]}" done echo "๐Ÿงน Cleaning up the build context directory" @@ -157,4 +176,3 @@ else docker image inspect -f '๐Ÿš€ Built {{.RepoTags}} {{.Id}}' "${TAG}-$arch" done fi -