From 946eb5d60cc2fae485f1a1cef7273fce1ba12fa6 Mon Sep 17 00:00:00 2001 From: rdark <260691+rdark@users.noreply.github.com> Date: Fri, 30 Oct 2026 12:33:52 +0000 Subject: [PATCH 1/2] Fetch cached OCI charts without contacting the registry (#307) --- helm/private/import.bzl | 116 +++++++++++++++++++++++----------------- 1 file changed, 66 insertions(+), 50 deletions(-) diff --git a/helm/private/import.bzl b/helm/private/import.bzl index 4eb1b42..5da03ea 100644 --- a/helm/private/import.bzl +++ b/helm/private/import.bzl @@ -130,55 +130,71 @@ def _helm_import_repository_impl(repository_ctx): url, _, chart_version = chart_url.rpartition(":") hostname, _, chart_path = url.removeprefix("oci://").partition("/") - au = authn.new(repository_ctx) - token = au.get_token(hostname, chart_path) - - # Find the digest for the layer with the chart package in the image manifest: - manifest_json = "manifest.json" - manifest_url = "https://{}/v2/{}/manifests/{}".format( - hostname, - chart_path, - chart_version, - ) - repository_ctx.download( - output = manifest_json, - url = manifest_url, - auth = { - manifest_url: token, - }, - # Copied from `helm pull --debug` - headers = { - "Accept": [ - "application/vnd.docker.distribution.manifest.v2+json", - "application/vnd.docker.distribution.manifest.list.v2+json", - "application/vnd.oci.image.manifest.v1+json", - "application/vnd.oci.image.index.v1+json", - "*/*", - ], - }, - ) - manifest = json.decode(repository_ctx.read(manifest_json)) - - # https://helm.sh/docs/topics/registries/#helm-chart-manifest - if manifest["config"]["mediaType"] != "application/vnd.cncf.helm.config.v1+json": - fail("{} is not a Helm chart package".format(chart_url)) - - chart_digest = _find_chart_digest(manifest) - chart_blob_url = "https://{}/v2/{}/blobs/{}".format( - hostname, - chart_path, - chart_digest, - ) - - # Download the chart package: - result = repository_ctx.download( - output = repository_ctx.path(chart_file), - url = chart_blob_url, - sha256 = repository_ctx.attr.sha256, - auth = { - chart_blob_url: token, - }, - ) + # A Helm chart's OCI layer is the chart .tgz, so `sha256` names its + # blob, and a cached blob needs neither a token nor the manifest. + result = None + if repository_ctx.attr.sha256: + result = repository_ctx.download( + output = repository_ctx.path(chart_file), + url = "https://{}/v2/{}/blobs/sha256:{}".format( + hostname, + chart_path, + repository_ctx.attr.sha256, + ), + sha256 = repository_ctx.attr.sha256, + allow_fail = True, + ) + + if not result or not result.success: + au = authn.new(repository_ctx) + token = au.get_token(hostname, chart_path) + + # Find the digest for the layer with the chart package in the image manifest: + manifest_json = "manifest.json" + manifest_url = "https://{}/v2/{}/manifests/{}".format( + hostname, + chart_path, + chart_version, + ) + repository_ctx.download( + output = manifest_json, + url = manifest_url, + auth = { + manifest_url: token, + }, + # Copied from `helm pull --debug` + headers = { + "Accept": [ + "application/vnd.docker.distribution.manifest.v2+json", + "application/vnd.docker.distribution.manifest.list.v2+json", + "application/vnd.oci.image.manifest.v1+json", + "application/vnd.oci.image.index.v1+json", + "*/*", + ], + }, + ) + manifest = json.decode(repository_ctx.read(manifest_json)) + + # https://helm.sh/docs/topics/registries/#helm-chart-manifest + if manifest.get("config", {}).get("mediaType") != "application/vnd.cncf.helm.config.v1+json": + fail("{} is not a Helm chart package".format(chart_url)) + + chart_digest = _find_chart_digest(manifest) + chart_blob_url = "https://{}/v2/{}/blobs/{}".format( + hostname, + chart_path, + chart_digest, + ) + + # Download the chart package: + result = repository_ctx.download( + output = repository_ctx.path(chart_file), + url = chart_blob_url, + sha256 = repository_ctx.attr.sha256, + auth = { + chart_blob_url: token, + }, + ) else: fail("cannot download {} from {}, unsupported scheme".format(repository_ctx.attr.chart_name, chart_url)) @@ -227,7 +243,7 @@ helm_import_repository = repository_rule( doc = "Chart repository url where to locate the requested chart. Mutually exclusive with `url`.", ), "sha256": attr.string( - doc = "The expected SHA-256 hash of the chart imported.", + doc = "The expected SHA-256 hash of the chart imported. For an `oci://` url, a chart already in the repository cache is then fetched without any request to the registry.", ), "url": attr.string( doc = "The url where a chart can be directly downloaded. Mutually exclusive with `chart_name`, `repository`, and `version`", From f8d296e5b28b00c606ee3054723373c3065de0d0 Mon Sep 17 00:00:00 2001 From: Richard Clark Date: Fri, 30 Oct 2026 13:13:38 +0000 Subject: [PATCH 2/2] Add `helm_charts` extension that stores OCI chart digests as facts * Add the `helm_charts` module extension with a `chart(name, url, sha256)` tag. It takes the digest from the tag, the stored facts or the registry manifest, then declares `helm_import_repository` with it. On Bazel 8.5+ a looked-up digest is stored as a fact in `MODULE.bazel.lock`. Older Bazel records the repository and its digest in the lockfile instead. * Move the registry code from `import.bzl` to `oci_chart.bzl` so it runs with `repository_ctx` or `module_ctx`. `import_authn.bzl` blanks a file where `module_ctx` has no `delete`. * Take the chart name and version from an `oci://` url, and split an http file name at the first dash followed by a version number. A version such as `1.21.0-beta.0` no longer gives the target `cert-manager-1.21.0`. * Add unit tests for the helpers, and chart tests for a looked-up, a pinned and a pre-release chart. --- MODULE.bazel | 31 +++++++ helm/extensions.bzl | 106 +++++++++++++++++++++ helm/private/chart_utils.bzl | 93 +++++++++++++++++++ helm/private/import.bzl | 71 ++++---------- helm/private/import_authn.bzl | 16 +++- helm/private/oci_chart.bzl | 109 ++++++++++++++++++++++ tests/helm_charts/BUILD.bazel | 45 +++++++++ tests/helm_charts/chart_sha256_test.go | 42 +++++++++ tests/helm_charts/chart_utils_test.bzl | 123 +++++++++++++++++++++++++ 9 files changed, 577 insertions(+), 59 deletions(-) create mode 100644 helm/private/chart_utils.bzl create mode 100644 helm/private/oci_chart.bzl create mode 100644 tests/helm_charts/BUILD.bazel create mode 100644 tests/helm_charts/chart_sha256_test.go create mode 100644 tests/helm_charts/chart_utils_test.bzl diff --git a/MODULE.bazel b/MODULE.bazel index d6c1df1..75e2f19 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -128,3 +128,34 @@ use_repo( "rules_helm_test_img_container_base", "rules_helm_test_oci_container_base", ) + +helm_charts = use_extension( + "@rules_helm//helm:extensions.bzl", + "helm_charts", + dev_dependency = True, +) +helm_charts.chart( + name = "helm_charts_test_cert_manager", + url = "oci://quay.io/jetstack/charts/cert-manager:1.19.4", +) +helm_charts.chart( + name = "helm_charts_test_cert_manager_pinned", + sha256 = "d0c202f57b8653eb957fb5b2f9472c1b11cf6b51acc5c777e4b824f7b5a9ea87", + url = "oci://quay.io/jetstack/charts/cert-manager:1.19.4", +) +helm_charts.chart( + name = "helm_charts_test_pre_release", + sha256 = "0041fc0111fab7d62beed6d46d283f5ef4d851ad8c77055466bb4c82e36b6a6d", + url = "oci://quay.io/jetstack/charts/cert-manager:1.21.0-beta.0", +) +helm_charts.chart( + name = "helm_charts_test_vault_operator", + url = "oci://ghcr.io/bank-vaults/helm-charts/vault-operator:1.23.0", +) +use_repo( + helm_charts, + "helm_charts_test_cert_manager", + "helm_charts_test_cert_manager_pinned", + "helm_charts_test_pre_release", + "helm_charts_test_vault_operator", +) diff --git a/helm/extensions.bzl b/helm/extensions.bzl index ffe3f6e..f127d21 100644 --- a/helm/extensions.bzl +++ b/helm/extensions.bzl @@ -1,6 +1,10 @@ """Bzlmod extensions""" +load("//helm/private:chart_utils.bzl", "chart_utils") load("//helm/private:extension_utils.bzl", "find_modules") +load("//helm/private:import.bzl", "helm_import_repository") +load("//helm/private:import_authn.bzl", "authn") +load("//helm/private:oci_chart.bzl", "oci_chart") load( "//helm/private:repositories.bzl", "helm_host_alias_repository", @@ -186,6 +190,45 @@ def _helm_impl(module_ctx): reproducible = True, ) +def _lookup_sha256(module_ctx, state, url): + if url not in state["sha256"]: + if "au" not in state: + state["au"] = authn.new(module_ctx) + state["sha256"][url] = oci_chart.resolve_sha256(module_ctx, state["au"], url) + return state["sha256"][url] + +def _helm_charts_impl(module_ctx): + # Digests persisted in the lockfile by a previous evaluation (Bazel 8.5+). + facts = getattr(module_ctx, "facts", {}) + used_facts = {} + state = {"sha256": {}} + lookup = lambda url: _lookup_sha256(module_ctx, state, url) + + for mod in module_ctx.modules: + for tag in mod.tags.chart: + if not chart_utils.is_chart_url(tag.url): + fail("helm_charts.chart(name = \"{}\") needs a url of the form `oci:///:`, got `{}`".format(tag.name, tag.url)) + if tag.sha256 and not chart_utils.is_sha256(tag.sha256): + fail("helm_charts.chart(name = \"{}\") needs a `sha256` of 64 lower case hexadecimal digits, got `{}`".format(tag.name, tag.sha256)) + + chart = chart_utils.chart_sha256(tag.sha256, tag.url, facts, lookup) + if chart.is_fact: + used_facts[tag.url] = chart.sha256 + + helm_import_repository( + name = tag.name, + url = tag.url, + sha256 = chart.sha256, + ) + + # Without facts, a looked-up digest is kept only if the extension is not reproducible: + # the lockfile then records the repositories, and with them the digests. + kwargs = {"reproducible": hasattr(module_ctx, "facts") or not used_facts} + if hasattr(module_ctx, "facts"): + kwargs["facts"] = used_facts + + return module_ctx.extension_metadata(**kwargs) + _host_tools = tag_class( doc = """\ An extension for creating a host alias repository that provides a shorter name for the host platform's helm binary. @@ -307,6 +350,53 @@ register_toolchains("@with_diff_toolchains//:all") }, ) +_chart = tag_class( + doc = """\ +Fetch a Helm chart from an OCI registry with `helm_import_repository`. + +The repository holds a `helm_import` target named after the chart, which is the last \ +path segment of `url` without the version. Refer to it as `@//:`. The \ +shorthand `@` does not resolve to the chart. + +If `sha256` is unset, the extension reads the digest of the chart package from the \ +registry manifest, and trusts the registry for it on first use. On Bazel 8.5 and newer \ +the digest is stored as a fact in `MODULE.bazel.lock`, keyed by `url`, so later \ +evaluations make no registry request for it. Older Bazel versions record the digest in \ +the lockfile as part of the repository. Without a lockfile, the digest is looked up \ +whenever the extension is evaluated, and a version tag that the registry moves gives a \ +different chart. Set `sha256` to pin the digest in `MODULE.bazel`. + +A stored digest is not checked against the registry. If the registry replaces the \ +chart behind a version tag, the fetch fails with a checksum mismatch. Delete the \ +entry for `url` from the `facts` section of `MODULE.bazel.lock` to look the digest \ +up again. + +```python +helm_charts = use_extension("@rules_helm//helm:extensions.bzl", "helm_charts") +helm_charts.chart( + name = "karpenter", + url = "oci://public.ecr.aws/karpenter/karpenter:1.9.0", +) +use_repo(helm_charts, "karpenter") + +# Then you can use @karpenter//:karpenter as a dependency in your BUILD files +``` +""", + attrs = { + "name": attr.string( + doc = "The name of the repository.", + mandatory = True, + ), + "sha256": attr.string( + doc = "The expected SHA-256 hash of the chart package. If unset, the extension looks it up in the registry.", + ), + "url": attr.string( + doc = "The chart url, in the form `oci:///:`.", + mandatory = True, + ), + }, +) + helm = module_extension( implementation = _helm_impl, tag_classes = { @@ -315,3 +405,19 @@ helm = module_extension( "toolchain": _toolchain, }, ) + +helm_charts = module_extension( + doc = """\ +Module extension for fetching Helm charts from OCI registries. + +Charts live in their own extension, so an unreachable registry does not stop the `helm` \ +extension from resolving toolchains. All tags are resolved in one evaluation, so a \ +lookup that fails (an unknown version or an unreachable registry) fails every \ +repository from this extension. Set `sha256` to skip the lookup. Tags from every module \ +are used, so repository names must be unique across modules. +""", + implementation = _helm_charts_impl, + tag_classes = { + "chart": _chart, + }, +) diff --git a/helm/private/chart_utils.bzl b/helm/private/chart_utils.bzl new file mode 100644 index 0000000..7e309b1 --- /dev/null +++ b/helm/private/chart_utils.bzl @@ -0,0 +1,93 @@ +"""Helpers for importing Helm charts that need no repository or module context.""" + +def _is_sha256(value): + """Check that a value is a hex-encoded sha256 digest in lower case. + + Args: + value (object): The value to check. + + Returns: + bool: True if the value is a string of 64 lower case hex digits. + """ + return type(value) == "string" and len(value) == 64 and not value.strip("0123456789abcdef") + +def _is_chart_url(url): + """Check that a url has the form `oci:///:`. + + Args: + url (str): The url to check. + + Returns: + bool: True if the url has that form. + """ + for char in url.elems(): + if char.isspace(): + return False + + url_without_version, _, version = url.rpartition(":") + hostname, _, path = url_without_version.removeprefix("oci://").partition("/") + return bool( + url.startswith("oci://") and + hostname and + path and + "" not in path.split("/") and + version and + "/" not in version and + ":" not in path and + "@" not in path, + ) + +def _starts_with_version(text): + """Check that text starts like a version: an optional `v`, then `.`.""" + major, dot, rest = text.removeprefix("v").partition(".") + return bool(dot and major.isdigit() and rest[:1].isdigit()) + +def _split_chart_file(chart_file): + """Split a chart file name into the chart name and the version. + + A chart file is named `-.tgz`. Both parts can contain a dash, + so the version starts at the first dash that is followed by a version number. + + Args: + chart_file (str): The chart file name. + + Returns: + tuple: The chart name and the chart version. + """ + stem = chart_file.removesuffix(".tgz") + parts = stem.split("-") + for i in range(1, len(parts)): + if _starts_with_version(parts[i]): + return "-".join(parts[:i]), "-".join(parts[i:]) + + # No version number: split at the last dash. + name, _, version = stem.rpartition("-") + return name, version + +def _chart_sha256(pinned, url, facts, lookup): + """Choose the sha256 for a chart from a tag, the stored facts or the registry. + + Args: + pinned (str): The sha256 set in the tag, or an empty string. + url (str): The chart url. + facts (dict or Facts): The digests stored by a previous evaluation, by chart url. Only `get` is called on it. + lookup (callable): Called with the chart url, returns the sha256 from the registry. + + Returns: + struct: The `sha256` to use, and `is_fact`, which is True when the digest + did not come from the tag and so belongs in the stored facts. + """ + if pinned: + return struct(sha256 = pinned, is_fact = False) + + sha256 = facts.get(url) + if not _is_sha256(sha256): + sha256 = lookup(url) + return struct(sha256 = sha256, is_fact = True) + +chart_utils = struct( + chart_sha256 = _chart_sha256, + is_chart_url = _is_chart_url, + is_sha256 = _is_sha256, + split_chart_file = _split_chart_file, +) diff --git a/helm/private/import.bzl b/helm/private/import.bzl index 5da03ea..cc8fbf4 100644 --- a/helm/private/import.bzl +++ b/helm/private/import.bzl @@ -1,6 +1,8 @@ """Helm rules for managing external dependencies""" +load(":chart_utils.bzl", "chart_utils") load(":import_authn.bzl", "authn") +load(":oci_chart.bzl", "oci_chart") load(":providers.bzl", "HelmPackageInfo") def _helm_import_impl(ctx): @@ -60,6 +62,13 @@ def _find_chart_url(repository_ctx, repo_file, chart_name, chart_version): return "{}/{}".format(repository_ctx.attr.repository, url) fail("cannot find {} (version {}) in {}".format(chart_name, chart_version, repository_ctx.attr.repository)) +def _get_chart_name_and_version(chart_url, chart_file): + if chart_url.startswith("oci"): + # The url names the chart and the version exactly. + chart_name, chart_version = chart_url.split("/")[-1].split(":") + return chart_name, chart_version + return chart_utils.split_chart_file(chart_file) + def _get_chart_file_name(chart_url): if chart_url.startswith("http") and chart_url.endswith(".tgz"): return chart_url.split("/")[-1] @@ -68,13 +77,6 @@ def _get_chart_file_name(chart_url): return "{}-{}.tgz".format(chart_name, chart_version) fail("cannot determine chart file name from {}".format(chart_url)) -def _find_chart_digest(manifest): - for layer in manifest["layers"]: - # https://helm.sh/docs/topics/registries/#helm-chart-manifest - if layer["mediaType"] == "application/vnd.cncf.helm.chart.content.v1.tar+gzip": - return layer["digest"] - fail("could not find chart content layer") - _HELM_DEP_BUILD_FILE = """\ load("@rules_helm//helm:defs.bzl", "helm_import") @@ -127,8 +129,7 @@ def _helm_import_repository_impl(repository_ctx): sha256 = repository_ctx.attr.sha256, ) elif chart_url.startswith("oci"): - url, _, chart_version = chart_url.rpartition(":") - hostname, _, chart_path = url.removeprefix("oci://").partition("/") + chart = oci_chart.parse_url(chart_url) # A Helm chart's OCI layer is the chart .tgz, so `sha256` names its # blob, and a cached blob needs neither a token nor the manifest. @@ -136,55 +137,15 @@ def _helm_import_repository_impl(repository_ctx): if repository_ctx.attr.sha256: result = repository_ctx.download( output = repository_ctx.path(chart_file), - url = "https://{}/v2/{}/blobs/sha256:{}".format( - hostname, - chart_path, - repository_ctx.attr.sha256, - ), + url = oci_chart.blob_url(chart, "sha256:" + repository_ctx.attr.sha256), sha256 = repository_ctx.attr.sha256, allow_fail = True, ) if not result or not result.success: au = authn.new(repository_ctx) - token = au.get_token(hostname, chart_path) - - # Find the digest for the layer with the chart package in the image manifest: - manifest_json = "manifest.json" - manifest_url = "https://{}/v2/{}/manifests/{}".format( - hostname, - chart_path, - chart_version, - ) - repository_ctx.download( - output = manifest_json, - url = manifest_url, - auth = { - manifest_url: token, - }, - # Copied from `helm pull --debug` - headers = { - "Accept": [ - "application/vnd.docker.distribution.manifest.v2+json", - "application/vnd.docker.distribution.manifest.list.v2+json", - "application/vnd.oci.image.manifest.v1+json", - "application/vnd.oci.image.index.v1+json", - "*/*", - ], - }, - ) - manifest = json.decode(repository_ctx.read(manifest_json)) - - # https://helm.sh/docs/topics/registries/#helm-chart-manifest - if manifest.get("config", {}).get("mediaType") != "application/vnd.cncf.helm.config.v1+json": - fail("{} is not a Helm chart package".format(chart_url)) - - chart_digest = _find_chart_digest(manifest) - chart_blob_url = "https://{}/v2/{}/blobs/{}".format( - hostname, - chart_path, - chart_digest, - ) + layer = oci_chart.find_chart_layer(repository_ctx, au, chart_url) + chart_blob_url = oci_chart.blob_url(chart, layer.digest) # Download the chart package: result = repository_ctx.download( @@ -192,14 +153,14 @@ def _helm_import_repository_impl(repository_ctx): url = chart_blob_url, sha256 = repository_ctx.attr.sha256, auth = { - chart_blob_url: token, + chart_blob_url: layer.token, }, ) else: fail("cannot download {} from {}, unsupported scheme".format(repository_ctx.attr.chart_name, chart_url)) - chart_name, _, chart_version = chart_file.removesuffix(".tgz").rpartition("-") + chart_name, chart_version = _get_chart_name_and_version(chart_url, chart_file) repository_ctx.file("BUILD.bazel", content = _HELM_DEP_BUILD_FILE.format( chart_name = chart_name, @@ -234,7 +195,7 @@ def _helm_import_repository_impl(repository_ctx): helm_import_repository = repository_rule( implementation = _helm_import_repository_impl, - doc = "A rule for fetching external Helm charts from an arbitrary URL or repository.", + doc = "A rule for fetching external Helm charts from an arbitrary URL or repository. With bzlmod, the `helm_charts` extension in `@rules_helm//helm:extensions.bzl` fetches `oci://` charts with this rule and stores the digest it looks up in `MODULE.bazel.lock`.", attrs = { "chart_name": attr.string( doc = "Chart name to import. Must be set if `repository` is specified", diff --git a/helm/private/import_authn.bzl b/helm/private/import_authn.bzl index e7605cb..f4d37f9 100644 --- a/helm/private/import_authn.bzl +++ b/helm/private/import_authn.bzl @@ -1,4 +1,4 @@ -"""Repository rule for OCI repo authentication.""" +"""OCI registry authentication for repository rules and module extensions.""" load("@bazel_lib//lib:base64.bzl", "base64") load("@bazel_lib//lib:repo_utils.bzl", "repo_utils") @@ -73,6 +73,14 @@ def _get_auth_file_path(repository_ctx): return None +def _remove(ctx, path): + """Delete a file, or blank it where the context has no `delete` (`module_ctx`).""" + if hasattr(ctx, "delete"): + ctx.delete(path) + else: + # Blank the file so a token does not stay on disk. + ctx.file(path, content = "") + def _fetch_auth_via_creds_helper(repository_ctx, raw_host, helper_name, allow_fail = False): credential_helper_name = "docker-credential-{}".format(helper_name) credential_helper = repository_ctx.which(credential_helper_name) @@ -103,7 +111,7 @@ exec "{}" get <<< "$1" """.format(credential_helper), else: return {} - repository_ctx.delete(executable) + _remove(repository_ctx, executable) response = json.decode(result.stdout) # If the username and secret are empty, the user does not have a login. @@ -205,10 +213,10 @@ def _get_token(repository_ctx, state, hostname, chart_path): ) if allow_fail and not result.success: repository_ctx.file("www-authenticate.json", content = "") - repository_ctx.delete("www-authenticate.json") + _remove(repository_ctx, "www-authenticate.json") return {} auth_raw = repository_ctx.read("www-authenticate.json") - repository_ctx.delete("www-authenticate.json") + _remove(repository_ctx, "www-authenticate.json") auth = json.decode(auth_raw) diff --git a/helm/private/oci_chart.bzl b/helm/private/oci_chart.bzl new file mode 100644 index 0000000..faa72e1 --- /dev/null +++ b/helm/private/oci_chart.bzl @@ -0,0 +1,109 @@ +"""Helpers for locating a Helm chart in an OCI registry. + +These work with both `repository_ctx` and `module_ctx`. +""" + +# https://helm.sh/docs/topics/registries/#helm-chart-manifest +_CONFIG_MEDIA_TYPE = "application/vnd.cncf.helm.config.v1+json" +_CHART_LAYER_MEDIA_TYPE = "application/vnd.cncf.helm.chart.content.v1.tar+gzip" + +def _parse_url(chart_url): + """Split an `oci:///:` chart url. + + Args: + chart_url (str): The chart url. + + Returns: + struct: The `hostname`, `path` and `version` of the chart. + """ + url, _, version = chart_url.rpartition(":") + hostname, _, path = url.removeprefix("oci://").partition("/") + return struct( + hostname = hostname, + path = path, + version = version, + ) + +def _blob_url(chart, digest): + """Return the url of a blob in the chart's repository. + + Args: + chart (struct): A chart from `parse_url`. + digest (str): The blob digest, in the form `sha256:`. + + Returns: + str: The blob url. + """ + return "https://{}/v2/{}/blobs/{}".format(chart.hostname, chart.path, digest) + +def _find_chart_layer(ctx, au, chart_url): + """Find the layer that holds the chart package in the image manifest. + + Args: + ctx (ctx): The `repository_ctx` or `module_ctx` used to download the manifest. + au (struct): An authenticator from `authn.new`. + chart_url (str): The chart url. + + Returns: + struct: The `digest` of the layer, in the form `sha256:`, and the + `token` that authenticates requests to the registry. + """ + chart = _parse_url(chart_url) + token = au.get_token(chart.hostname, chart.path) + + manifest_json = "manifest.json" + manifest_url = "https://{}/v2/{}/manifests/{}".format( + chart.hostname, + chart.path, + chart.version, + ) + ctx.download( + output = manifest_json, + url = manifest_url, + auth = { + manifest_url: token, + }, + # Copied from `helm pull --debug` + headers = { + "Accept": [ + "application/vnd.docker.distribution.manifest.v2+json", + "application/vnd.docker.distribution.manifest.list.v2+json", + "application/vnd.oci.image.manifest.v1+json", + "application/vnd.oci.image.index.v1+json", + "*/*", + ], + }, + ) + manifest = json.decode(ctx.read(manifest_json)) + + if manifest.get("config", {}).get("mediaType") != _CONFIG_MEDIA_TYPE: + fail("{} is not a Helm chart package".format(chart_url)) + + for layer in manifest["layers"]: + if layer["mediaType"] == _CHART_LAYER_MEDIA_TYPE: + return struct(digest = layer["digest"], token = token) + fail("could not find chart content layer") + +def _resolve_sha256(ctx, au, chart_url): + """Look up the sha256 of a chart package in the registry. + + Args: + ctx (ctx): The `repository_ctx` or `module_ctx` used to download the manifest. + au (struct): An authenticator from `authn.new`. + chart_url (str): The chart url. + + Returns: + str: The hex-encoded sha256 of the chart package. + """ + digest = _find_chart_layer(ctx, au, chart_url).digest + algorithm, _, sha256 = digest.partition(":") + if algorithm != "sha256": + fail("{} has a chart layer with the unsupported digest {}".format(chart_url, digest)) + return sha256 + +oci_chart = struct( + blob_url = _blob_url, + find_chart_layer = _find_chart_layer, + parse_url = _parse_url, + resolve_sha256 = _resolve_sha256, +) diff --git a/tests/helm_charts/BUILD.bazel b/tests/helm_charts/BUILD.bazel new file mode 100644 index 0000000..355a1cd --- /dev/null +++ b/tests/helm_charts/BUILD.bazel @@ -0,0 +1,45 @@ +load("@rules_go//go:def.bzl", "go_test") +load(":chart_utils_test.bzl", "chart_utils_test_suite") + +chart_utils_test_suite(name = "chart_utils") + +# `looked_up` and `second_looked_up` have no `sha256` in their `MODULE.bazel` tags, so the +# extension looks the digest up (and, on Bazel 8.5+, stores it as a lockfile fact). The two +# charts are on different registries, to cover reuse of the authenticator and of the manifest +# file. `pinned` and `pre_release` set `sha256` and make no lookup. The `pre_release` version +# contains a dash, so the label `//:cert-manager` checks that the target is named after the +# chart and not after part of the version. The expected digests match `tests/test_deps.bzl`, +# except for `pre_release`, which is not in that file. +_CHARTS = { + "looked_up": ( + "@helm_charts_test_cert_manager//:cert-manager", + "d0c202f57b8653eb957fb5b2f9472c1b11cf6b51acc5c777e4b824f7b5a9ea87", + ), + "pinned": ( + "@helm_charts_test_cert_manager_pinned//:cert-manager", + "d0c202f57b8653eb957fb5b2f9472c1b11cf6b51acc5c777e4b824f7b5a9ea87", + ), + "pre_release": ( + "@helm_charts_test_pre_release//:cert-manager", + "0041fc0111fab7d62beed6d46d283f5ef4d851ad8c77055466bb4c82e36b6a6d", + ), + "second_looked_up": ( + "@helm_charts_test_vault_operator//:vault-operator", + "9abd372c9d843b974d918442cd0e68c9669f57d8630566ce74dfaf63447c282a", + ), +} + +[ + go_test( + name = "chart_sha256_{}_test".format(name), + size = "small", + srcs = ["chart_sha256_test.go"], + data = [label], + env = { + "CHART": "$(rlocationpath {})".format(label), + "EXPECTED_SHA256": sha256, + }, + deps = ["@rules_go//go/runfiles"], + ) + for name, (label, sha256) in _CHARTS.items() +] diff --git a/tests/helm_charts/chart_sha256_test.go b/tests/helm_charts/chart_sha256_test.go new file mode 100644 index 0000000..4f626a4 --- /dev/null +++ b/tests/helm_charts/chart_sha256_test.go @@ -0,0 +1,42 @@ +package main + +import ( + "crypto/sha256" + "encoding/hex" + "io" + "os" + "testing" + + "github.com/bazelbuild/rules_go/go/runfiles" +) + +func mustEnv(t *testing.T, key string) string { + t.Helper() + value := os.Getenv(key) + if value == "" { + t.Fatalf("%s is not set", key) + } + return value +} + +func TestChartSha256(t *testing.T) { + path, err := runfiles.Rlocation(mustEnv(t, "CHART")) + if err != nil { + t.Fatalf("Failed to locate the chart: %v", err) + } + + file, err := os.Open(path) + if err != nil { + t.Fatalf("Failed to open the chart: %v", err) + } + defer file.Close() + + hash := sha256.New() + if _, err := io.Copy(hash, file); err != nil { + t.Fatalf("Failed to read the chart: %v", err) + } + + if got, want := hex.EncodeToString(hash.Sum(nil)), mustEnv(t, "EXPECTED_SHA256"); got != want { + t.Errorf("chart sha256 = %s, want %s", got, want) + } +} diff --git a/tests/helm_charts/chart_utils_test.bzl b/tests/helm_charts/chart_utils_test.bzl new file mode 100644 index 0000000..7284ad7 --- /dev/null +++ b/tests/helm_charts/chart_utils_test.bzl @@ -0,0 +1,123 @@ +"""Unit tests for the chart import helpers.""" + +load("@bazel_skylib//lib:unittest.bzl", "asserts", "unittest") + +# buildifier: disable=bzl-visibility +load("//helm/private:chart_utils.bzl", "chart_utils") + +_SHA = "d0c202f57b8653eb957fb5b2f9472c1b11cf6b51acc5c777e4b824f7b5a9ea87" +_OTHER_SHA = "9abd372c9d843b974d918442cd0e68c9669f57d8630566ce74dfaf63447c282a" +_URL = "oci://quay.io/jetstack/charts/cert-manager:1.19.4" + +def _split_chart_file_test_impl(ctx): + env = unittest.begin(ctx) + for chart_file, expected in { + "app-2-1.0.0.tgz": ("app-2", "1.0.0"), + "cert-manager-1.21.0-beta.0.tgz": ("cert-manager", "1.21.0-beta.0"), + "foo-1.0.0-rc.1.tgz": ("foo", "1.0.0-rc.1"), + "foo-1.tgz": ("foo", "1"), + "redis-21.2.5.tgz": ("redis", "21.2.5"), + "trust-manager-v0.24.0.tgz": ("trust-manager", "v0.24.0"), + "vault-operator-1.23.0.tgz": ("vault-operator", "1.23.0"), + }.items(): + asserts.equals(env, expected, chart_utils.split_chart_file(chart_file), chart_file) + return unittest.end(env) + +split_chart_file_test = unittest.make(_split_chart_file_test_impl) + +def _is_sha256_test_impl(ctx): + env = unittest.begin(ctx) + asserts.true(env, chart_utils.is_sha256(_SHA)) + for value in [ + None, + "", + "abc", + _SHA.upper(), + "sha256:" + _SHA, + _SHA + "0", + _SHA[:-1] + "g", + ]: + asserts.false(env, chart_utils.is_sha256(value), str(value)) + return unittest.end(env) + +is_sha256_test = unittest.make(_is_sha256_test_impl) + +def _is_chart_url_test_impl(ctx): + env = unittest.begin(ctx) + for url in [ + _URL, + "oci://public.ecr.aws/karpenter/karpenter:1.9.0", + "oci://localhost:5000/charts/app:1.0.0-rc.1", + ]: + asserts.true(env, chart_utils.is_chart_url(url), url) + for url in [ + "", + "oci://quay.io/jetstack/charts/cert-manager", + "oci://quay.io:443/cert-manager", + "oci://quay.io/jetstack/charts/cert-manager:", + "oci://quay.io/jetstack/charts/cert-manager@sha256:abc", + "oci://quay.io/a:1:2", + "oci:///x/y:1", + "oci://quay.io//jetstack/charts/cert-manager:1.19.4", + "oci://quay.io/jetstack/charts/cert-manager/:1.19.4", + "oci://quay.io/jetstack/charts/cert-manager:1.19.4\n", + "oci://quay.io/jetstack/charts/cert-manager:1.19.4 ", + "oci://quay.io/jetstack/charts/cert manager:1.19.4", + "oci://quay.io/jetstack/charts/cert-manager:1.19.4\t", + "oci://quay.io:1", + "https://quay.io/jetstack/charts/cert-manager:1.19.4", + ]: + asserts.false(env, chart_utils.is_chart_url(url), url) + return unittest.end(env) + +is_chart_url_test = unittest.make(_is_chart_url_test_impl) + +def _chart_sha256_test_impl(ctx): + env = unittest.begin(ctx) + + # A digest in the tag is used as it is, with no lookup and no fact. + calls = [] + lookup = lambda url: calls.append(url) or _OTHER_SHA + chart = chart_utils.chart_sha256(_SHA, _URL, {_URL: _OTHER_SHA}, lookup) + asserts.equals(env, _SHA, chart.sha256) + asserts.false(env, chart.is_fact) + asserts.equals(env, [], calls) + + # A stored digest is used with no lookup, and stays in the facts. + calls = [] + chart = chart_utils.chart_sha256("", _URL, {_URL: _SHA}, lookup) + asserts.equals(env, _SHA, chart.sha256) + asserts.true(env, chart.is_fact) + asserts.equals(env, [], calls) + + # Without a stored digest, the registry result is used and becomes a fact. + calls = [] + chart = chart_utils.chart_sha256("", _URL, {}, lookup) + asserts.equals(env, _OTHER_SHA, chart.sha256) + asserts.true(env, chart.is_fact) + asserts.equals(env, [_URL], calls) + + # A stored value that is not a digest is looked up again. + calls = [] + chart = chart_utils.chart_sha256("", _URL, {_URL: "not-a-digest"}, lookup) + asserts.equals(env, _OTHER_SHA, chart.sha256) + asserts.equals(env, [_URL], calls) + + # A digest stored for another chart is not used. + calls = [] + chart = chart_utils.chart_sha256("", _URL, {"oci://quay.io/other/chart:1.0.0": _SHA}, lookup) + asserts.equals(env, _OTHER_SHA, chart.sha256) + asserts.equals(env, [_URL], calls) + + return unittest.end(env) + +chart_sha256_test = unittest.make(_chart_sha256_test_impl) + +def chart_utils_test_suite(name): + unittest.suite( + name, + split_chart_file_test, + is_sha256_test, + is_chart_url_test, + chart_sha256_test, + )