diff --git a/.github/scripts/track-binaries.py b/.github/scripts/track-binaries.py new file mode 100644 index 00000000..bcd2ed01 --- /dev/null +++ b/.github/scripts/track-binaries.py @@ -0,0 +1,593 @@ +#!/usr/bin/env python3 +"""Track patch releases of the upstream binaries pinned by version + sha256. + +A component is one upstream version plus every sha256 pin derived from it. The +two move in one change: a new version with an old checksum is a broken +BuildStream source ref. + + component version sha256 pins + kubernetes include/kubeadm.yml kubernetes-version kubelet, kubeadm, kubectl ref: in + cri-tools include/kubeadm.yml crictl-version crictl tarball elements/kubeadm/ + containerd include/kubeadm.yml containerd-version containerd static tarball kubeadm-bin.bst + runc include/kubeadm.yml runc-version runc.amd64 + cni-plugins include/kubeadm.yml cni-plugins-version CNI plugins tarball + k0s include/k0s.yml k0s-k8s-version and k0s binary ref: in + k0s-patch elements/k0s/k0s-bin.bst + oras Justfile oras_image tag setup-oras url + checksum in .github/workflows/*.yml + +check Newest release of each component inside its pinned MAJOR.MINOR series: + Kubernetes from dl.k8s.io/release/stable-X.Y.txt, the rest from the + project's GitHub releases. Drafts, prereleases and releases that lack + the pinned assets do not count. +apply Moves one component to a version (default: the newest in its series) + and rewrites every pin derived from it. Each sha256 is read from the + checksum file the project publishes next to the asset, then confirmed + by downloading the asset and hashing it. Nothing is written unless + every pin verifies. + +Only patch releases are proposed automatically. A minor bump is a decision: the +kubeadm payload follows the minor of the cluster it joins +(docs/skills/kubeadm-sysext.md). Make it by hand with +`apply COMPONENT --version X.Y.Z`, which runs the same verification. + +Why not Renovate: a regex manager needs the version and its digest in one +match, but here the version lives in include/*.yml or the Justfile and the +digests in elements/*.bst or a workflow. One Kubernetes version feeds three +dl.k8s.io checksums, which no Renovate datasource computes, and the k0s version +is split into two atoms. postUpgradeTasks, which could fetch checksums, are +blocked on the hosted Renovate app. + +Why not `bst source track`: it re-downloads every source of the element instead +of the bumped component's, takes whatever bytes it gets as the new ref instead +of checking the upstream checksum, and writes the element back in its own YAML +layout (it re-indents every list in elements/k0s/k0s-bin.bst). BuildStream +still checks each ref against the bytes when the pull request's build fetches +the sources. + +Stdlib only. GITHUB_TOKEN or GH_TOKEN, if set, is sent to api.github.com only. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import sys +import urllib.error +import urllib.parse +import urllib.request +from dataclasses import dataclass +from pathlib import Path +from typing import NamedTuple + +ROOT = Path(__file__).resolve().parents[2] +TIMEOUT = 60 +MAX_RELEASE_PAGES = 20 + +URL_LINE_RE = re.compile(r"^\s*(?:-\s+)?url:\s*(?P\S+)\s*$") +VARIABLE_RE = re.compile(r'^\s+(?P[A-Za-z][\w-]*):\s*"(?P[^"]*)"', re.MULTILINE) +ALIAS_RE = re.compile(r"^\s+(?P[A-Za-z][\w-]*):\s*(?Phttps://\S+)\s*$", re.MULTILINE) +SUMS_LINE_RE = re.compile(r"^(?P[0-9a-fA-F]{64})(?:\s+\*?(?P\S+))?$") + + +class TrackError(Exception): + pass + + +def _open(url: str): + headers = {"User-Agent": "projectbluefin-server-track-binaries"} + if url.startswith("https://api.github.com/"): + headers["Accept"] = "application/vnd.github+json" + token = os.environ.get("GITHUB_TOKEN") or os.environ.get("GH_TOKEN") + if token: + # Never on downloads: release assets redirect to object storage, + # which must not see the token. + headers["Authorization"] = f"Bearer {token}" + try: + return urllib.request.urlopen(urllib.request.Request(url, headers=headers), timeout=TIMEOUT) + except urllib.error.HTTPError as err: + raise TrackError(f"GET {url}: HTTP {err.code}") from None + except urllib.error.URLError as err: + raise TrackError(f"GET {url}: {err.reason}") from None + + +def _get(url: str) -> bytes: + with _open(url) as response: + return response.read() + + +def _download_sha256(url: str) -> str: + digest = hashlib.sha256() + with _open(url) as response: + while chunk := response.read(1 << 20): + digest.update(chunk) + return digest.hexdigest() + + +class Tree: + """Repository files, read on first use and written back only by save().""" + + def __init__(self, root: Path): + self.root = Path(root) + self._text: dict[str, str] = {} + self._original: dict[str, str] = {} + + def __getitem__(self, path: str) -> str: + if path not in self._text: + try: + text = (self.root / path).read_text(encoding="utf-8") + except FileNotFoundError: + raise TrackError(f"{path}: file not found") from None + self._text[path] = self._original[path] = text + return self._text[path] + + def __setitem__(self, path: str, text: str) -> None: + self[path] + self._text[path] = text + + def copy(self) -> Tree: + other = Tree(self.root) + other._text = dict(self._text) + other._original = dict(self._original) + return other + + def glob(self, *patterns: str) -> list[str]: + return sorted({p.relative_to(self.root).as_posix() for pattern in patterns for p in self.root.glob(pattern)}) + + def save(self) -> list[str]: + changed = sorted(path for path, text in self._text.items() if text != self._original[path]) + for path in changed: + (self.root / path).write_text(self._text[path], encoding="utf-8") + return changed + + +@dataclass(frozen=True) +class Pin: + """A sha256 on the `key:` line in the same mapping as `url: anchor` in `path`.""" + + path: str + anchor: str + key: str + url: str + sums: str + + @property + def asset(self) -> str: + return urllib.parse.unquote(self.url.rsplit("/", 1)[1]) + + +def _key_in_mapping(lines: list[str], start: int, column: int, key: str) -> int | None: + for j in range(start + 1, len(lines)): + text = lines[j].strip() + if not text or text.startswith("#"): + continue + indent = len(lines[j]) - len(lines[j].lstrip()) + if indent < column: + return None + if indent == column and re.match(rf"{re.escape(key)}:\s*\S+\s*$", text): + return j + return None + + +def _checksum_lines(lines: list[str], pin: Pin) -> list[int]: + """Line numbers of `pin.key:` for every `url: pin.anchor` line; each must have one.""" + found = [] + for i, line in enumerate(lines): + match = URL_LINE_RE.match(line) + if not match or match["url"] != pin.anchor: + continue + j = _key_in_mapping(lines, i, line.index("url:"), pin.key) + if j is None: + raise TrackError(f"{pin.path}: no `{pin.key}:` in the mapping of `url: {pin.anchor}`") + found.append(j) + if not found: + raise TrackError(f"{pin.path}: no `url: {pin.anchor}`") + return found + + +def read_pin(tree: Tree, pin: Pin) -> str: + lines = tree[pin.path].splitlines(keepends=True) + values = {lines[i].split(":", 1)[1].strip() for i in _checksum_lines(lines, pin)} + if len(values) != 1: + raise TrackError(f"{pin.path}: `url: {pin.anchor}` is pinned to different checksums: {sorted(values)}") + return values.pop() + + +def write_pin(tree: Tree, pin: Pin, sha256: str) -> None: + lines = tree[pin.path].splitlines(keepends=True) + value = re.compile(rf"^(\s*{re.escape(pin.key)}:\s*)\S+") + for i in _checksum_lines(lines, pin): + lines[i] = value.sub(lambda m: m[1] + sha256, lines[i], count=1) + tree[pin.path] = "".join(lines) + + +def _variable(tree: Tree, path: str, name: str) -> re.Match: + matches = [m for m in VARIABLE_RE.finditer(tree[path]) if m["name"] == name] + if len(matches) != 1: + raise TrackError(f'{path}: expected exactly one `{name}: "..."`, found {len(matches)}') + return matches[0] + + +def _expand(template: str, variables: dict[str, str], aliases: dict[str, str], where: str) -> str: + """Resolve a source url the way BuildStream does: `%{...}` variables, then the alias.""" + for _ in range(len(variables) + 1): + refs = re.findall(r"%\{([A-Za-z][\w-]*)\}", template) + if not refs: + break + for ref in refs: + if ref not in variables: + raise TrackError(f"{where}: `{template}` uses undefined variable %{{{ref}}}") + template = template.replace("%{" + ref + "}", variables[ref]) + else: + raise TrackError(f"{where}: circular variable reference in `{template}`") + alias, _, rest = template.partition(":") + if alias in aliases: + template = aliases[alias] + rest + if not template.startswith("https://"): + raise TrackError(f"{where}: `{template}` does not resolve to an https:// URL") + return template + + +def _numbers(version: str) -> tuple[int, ...]: + return tuple(int(n) for n in re.findall(r"\d+", version)) + + +def _series(version: str) -> str: + return ".".join(str(n) for n in _numbers(version)[:2]) + + +class Component: + name: str + repo: str + # Checksum file URL; {url} is the asset URL, {dir} its directory. + sums: str + stable_channel = "" + version_re = r"(\d+\.\d+\.\d+)" + + def current(self, tree: Tree) -> str: + raise NotImplementedError + + def set_version(self, tree: Tree, version: str) -> None: + raise NotImplementedError + + def pins(self, tree: Tree) -> list[Pin]: + raise NotImplementedError + + def parse(self, version: str) -> tuple[str, ...]: + match = re.fullmatch(self.version_re, version) + if not match: + raise TrackError(f"{self.name}: `{version}` does not match {self.version_re}") + return match.groups() + + def _sums(self, url: str, version: str) -> str: + return self.sums.format(url=url, dir=url.rsplit("/", 1)[0], version=version) + + def notes(self, version: str) -> str: + return f"https://github.com/{self.repo}/releases/tag/{urllib.parse.quote('v' + version)}" + + +class BstComponent(Component): + """Version atoms in a BuildStream include; sha256 refs on one element's sources.""" + + def __init__(self, name, repo, sums, include, variables, element, marker, + stable_channel="", version_re=None, version_fmt="{0}"): + self.name, self.repo, self.sums, self.stable_channel = name, repo, sums, stable_channel + self.include, self.variables, self.element, self.marker = include, variables, element, marker + self.version_re = version_re or Component.version_re + self.version_fmt = version_fmt + + def current(self, tree): + return self.version_fmt.format(*(_variable(tree, self.include, v)["value"] for v in self.variables)) + + def set_version(self, tree, version): + for name, value in zip(self.variables, self.parse(version)): + match = _variable(tree, self.include, name) + text = tree[self.include] + tree[self.include] = text[: match.start("value")] + value + text[match.end("value") :] + + def pins(self, tree): + variables = {m["name"]: m["value"] for m in VARIABLE_RE.finditer(tree[self.include])} + aliases = {m["name"]: m["url"] for m in ALIAS_RE.finditer(tree["include/aliases.yml"])} + version = self.current(tree) + pins = [] + for line in tree[self.element].splitlines(): + match = URL_LINE_RE.match(line) + if match and self.marker in match["url"]: + url = _expand(match["url"], variables, aliases, self.element) + pins.append(Pin(self.element, match["url"], "ref", url, self._sums(url, version))) + if not pins: + raise TrackError(f"{self.element}: no source url contains {self.marker}") + return pins + + +class OrasComponent(Component): + """The Justfile's ORAS image tag plus every setup-oras url and checksum.""" + + name = "oras" + repo = "oras-project/oras" + sums = "{dir}/oras_{version}_checksums.txt" + WORKFLOWS = (".github/workflows/*.yml", ".github/workflows/*.yaml") + IMAGE_RE = re.compile(r"(?Pghcr\.io/oras-project/oras:v)(?P\d+\.\d+\.\d+)(?![\w.])") + URL_RE = re.compile( + r"https://github\.com/oras-project/oras/releases/download/" + r"v(?P\d+\.\d+\.\d+)/oras_(?P\d+\.\d+\.\d+)_linux_amd64\.tar\.gz" + ) + + def _sites(self, tree): + sites = [("Justfile", m["version"]) for m in self.IMAGE_RE.finditer(tree["Justfile"])] + if not sites: + raise TrackError("Justfile: no ghcr.io/oras-project/oras:vX.Y.Z image") + urls = [(path, m) for path in tree.glob(*self.WORKFLOWS) for m in self.URL_RE.finditer(tree[path])] + if not urls: + raise TrackError(".github/workflows: no setup-oras release url") + for path, match in urls: + sites += [(path, match["tag"]), (path, match["version"])] + return sites, urls + + def current(self, tree): + sites, _ = self._sites(tree) + versions = {version for _, version in sites} + if len(versions) != 1: + pinned = ", ".join(f"{path}: {version}" for path, version in sorted(set(sites))) + raise TrackError(f"ORAS pins disagree ({pinned}); make them one version first") + return versions.pop() + + def set_version(self, tree, version): + self.parse(version) + _, urls = self._sites(tree) + tree["Justfile"] = self.IMAGE_RE.sub(lambda m: m["head"] + version, tree["Justfile"]) + new = f"https://github.com/oras-project/oras/releases/download/v{version}/oras_{version}_linux_amd64.tar.gz" + for path in sorted({path for path, _ in urls}): + tree[path] = self.URL_RE.sub(new, tree[path]) + + def pins(self, tree): + version = self.current(tree) + _, urls = self._sites(tree) + return [ + Pin(path, url, "checksum", url, self._sums(url, version)) + for path, url in sorted({(path, m[0]) for path, m in urls}) + ] + + +KUBEADM = dict(include="include/kubeadm.yml", element="elements/kubeadm/kubeadm-bin.bst") +COMPONENTS: dict[str, Component] = { + c.name: c + for c in ( + BstComponent("kubernetes", "kubernetes/kubernetes", "{url}.sha256", + variables=("kubernetes-version",), marker="%{kubernetes-version}", + stable_channel="https://dl.k8s.io/release/stable-{series}.txt", **KUBEADM), + BstComponent("cri-tools", "kubernetes-sigs/cri-tools", "{url}.sha256", + variables=("crictl-version",), marker="%{crictl-version}", **KUBEADM), + BstComponent("containerd", "containerd/containerd", "{url}.sha256sum", + variables=("containerd-version",), marker="%{containerd-version}", **KUBEADM), + BstComponent("runc", "opencontainers/runc", "{dir}/runc.sha256sum", + variables=("runc-version",), marker="%{runc-version}", **KUBEADM), + BstComponent("cni-plugins", "containernetworking/plugins", "{url}.sha256", + variables=("cni-plugins-version",), marker="%{cni-plugins-version}", **KUBEADM), + BstComponent("k0s", "k0sproject/k0s", "{dir}/sha256sums.txt", + include="include/k0s.yml", variables=("k0s-k8s-version", "k0s-patch"), + element="elements/k0s/k0s-bin.bst", marker="%{k0s-upstream-tag}", + version_re=r"(\d+\.\d+\.\d+)\+k0s\.(\d+)", version_fmt="{0}+k0s.{1}"), + OrasComponent(), + ) +} + + +def _pins_at(component: Component, tree: Tree, version: str) -> list[Pin]: + scratch = tree.copy() + component.set_version(scratch, version) + return component.pins(scratch) + + +def _github_releases(repo: str) -> list[dict]: + releases = [] + for page in range(1, MAX_RELEASE_PAGES + 1): + batch = json.loads(_get(f"https://api.github.com/repos/{repo}/releases?per_page=100&page={page}")) + releases += batch + if len(batch) < 100: + return releases + raise TrackError(f"{repo}: more than {MAX_RELEASE_PAGES * 100} releases") + + +def _candidates(component: Component, tree: Tree, series: str) -> list[str]: + if component.stable_channel: + url = component.stable_channel.format(series=series) + text = _get(url).decode().strip() + if not re.fullmatch(rf"v{re.escape(series)}\.\d+", text): + raise TrackError(f"{url} says `{text}`, not a {series}.x release") + return [text[1:]] + found = [] + prefix = f"https://github.com/{component.repo}/releases/download/" + for release in _github_releases(component.repo): + tag = release.get("tag_name", "") + if release.get("draft") or release.get("prerelease") or not tag.startswith("v"): + continue + version = tag[1:] + if not re.fullmatch(component.version_re, version) or _series(version) != series: + continue + # The pinned assets must be attached already: a release can be + # published while its assets are still uploading. + names = {asset["name"] for asset in release.get("assets", [])} + wanted = { + urllib.parse.unquote(url.rsplit("/", 1)[1]) + for pin in _pins_at(component, tree, version) + for url in (pin.url, pin.sums) + if url.startswith(prefix) + } + if wanted <= names: + found.append(version) + return found + + +def newest(component: Component, tree: Tree) -> str: + """Newest release in the pinned series, or the current version if none is newer.""" + current = component.current(tree) + return max([current, *_candidates(component, tree, _series(current))], key=_numbers) + + +def published_sha256(text: str, asset: str, url: str) -> str: + """The sha256 a checksum file gives for `asset`. + + Accepts a bare digest (dl.k8s.io, cri-tools) or `digest [*]name` lines + (sha256sum output, also inside runc's PGP-signed file). + """ + named: dict[str, set[str]] = {} + bare = [] + for line in text.splitlines(): + match = SUMS_LINE_RE.match(line.strip()) + if not match: + continue + if match["name"]: + named.setdefault(match["name"].rsplit("/", 1)[-1], set()).add(match["sha"].lower()) + else: + bare.append(match["sha"].lower()) + if named: + digests = named.get(asset, set()) + if len(digests) != 1: + raise TrackError(f"{url} lists {len(digests)} sha256 for {asset}") + return digests.pop() + if len(bare) == 1: + return bare[0] + raise TrackError(f"{url} is not a sha256 checksum file for {asset}") + + +class Change(NamedTuple): + pin: Pin + before: str + after: str + + +@dataclass +class Result: + component: Component + old: str + new: str + changes: list[Change] + files: list[str] + mentions: list[str] + + +def apply(root: Path, name: str, version: str | None = None) -> Result | None: + """Move `name` to `version` (default: newest in series) and rewrite its pins. + + Returns None when no version is given and the pin is already the newest. + """ + component = COMPONENTS[name] + tree = Tree(root) + old = component.current(tree) + new = version or newest(component, tree) + if version is None and new == old: + return None + component.set_version(tree, new) + sums: dict[str, str] = {} + hashed: dict[str, str] = {} + changes = [] + for pin in component.pins(tree): + if pin.sums not in sums: + sums[pin.sums] = _get(pin.sums).decode("utf-8", "replace") + sha = published_sha256(sums[pin.sums], pin.asset, pin.sums) + if pin.url not in hashed: + hashed[pin.url] = _download_sha256(pin.url) + if hashed[pin.url] != sha: + raise TrackError(f"{pin.url} hashes to {hashed[pin.url]}, but {pin.sums} says {sha}") + changes.append(Change(pin, read_pin(tree, pin), sha)) + write_pin(tree, pin, sha) + files = tree.save() + stale = re.compile(rf"(? str: + component, old, new = result.component, result.old, result.new + if _series(old) == _series(new): + head = f"Patch release of **{component.name}** in the pinned `{_series(new)}` series: `{old}` → `{new}`." + else: + head = f"Moves **{component.name}** from `{old}` to `{new}`, a series change." + lines = [head, "", f"Release notes: {component.notes(new)}", "", "| File | Asset | sha256 |", "| --- | --- | --- |"] + for pin, before, after in result.changes: + was = "unchanged" if before == after else f"was `{before}`" + lines.append(f"| `{pin.path}` | [`{pin.asset}`]({pin.url}) | `{after}` ({was}) |") + lines += [ + "", + "Every sha256 comes from the checksum file upstream publishes next to the", + "asset, and matched the sha256 of the downloaded asset:", + "", + *[f"- {url}" for url in dict.fromkeys(change.pin.sums for change in result.changes)], + ] + if result.mentions: + lines += ["", f"These lines of the changed files still mention `{old}`:", ""] + lines += [f"- `{mention}`" for mention in result.mentions] + lines += [ + "", + "Only patch releases inside the pinned series are proposed; a minor bump is a", + f"manual `.github/scripts/track-binaries.py apply {component.name} --version ...`.", + "The tracker regenerates this branch from `main`, so push fixes to a branch", + "of your own.", + ] + return "\n".join(lines) + "\n" + + +def check(root: Path) -> tuple[list[dict], list[str]]: + tree = Tree(root) + rows, errors = [], [] + for name, component in COMPONENTS.items(): + try: + current = component.current(tree) + latest = newest(component, tree) + except TrackError as err: + errors.append(f"{name}: {err}") + continue + rows.append({"component": name, "series": _series(current), "current": current, "latest": latest}) + return rows, errors + + +def main(argv: list[str] | None = None, root: Path = ROOT) -> int: + parser = argparse.ArgumentParser(description="Track patch releases of the pinned upstream binaries.") + commands = parser.add_subparsers(dest="command", required=True) + check_cmd = commands.add_parser("check", help="report the newest release in each pinned series") + check_cmd.add_argument("--json", action="store_true", help="print the pending updates as a JSON list on stdout") + apply_cmd = commands.add_parser("apply", help="move one component and rewrite its sha256 pins") + apply_cmd.add_argument("component", choices=COMPONENTS) + apply_cmd.add_argument("--version", help="target version (default: newest in the pinned series)") + apply_cmd.add_argument("--summary", type=Path, help="write a Markdown pull request body to this file") + args = parser.parse_args(argv) + + if args.command == "check": + rows, errors = check(root) + table = sys.stderr if args.json else sys.stdout + print(f"{'component':<12} {'series':<6} {'current':<14} {'latest':<14} state", file=table) + for row in rows: + state = "update" if row["latest"] != row["current"] else "current" + print(f"{row['component']:<12} {row['series']:<6} {row['current']:<14} {row['latest']:<14} {state}", file=table) + for error in errors: + print(f"ERROR: {error}", file=sys.stderr) + if args.json: + print(json.dumps([row for row in rows if row["latest"] != row["current"]])) + return 1 if errors else 0 + + try: + result = apply(root, args.component, args.version) + except TrackError as err: + print(f"ERROR: {err}", file=sys.stderr) + return 1 + if result is None: + print(f"{args.component}: already the newest release of its series", file=sys.stderr) + return 0 + if args.summary: + args.summary.write_text(summary(result), encoding="utf-8") + print(f"{args.component}: {result.old} -> {result.new}", file=sys.stderr) + for path in result.files: + print(path) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/workflows/track-binaries.yml b/.github/workflows/track-binaries.yml new file mode 100644 index 00000000..ce5f4bda --- /dev/null +++ b/.github/workflows/track-binaries.yml @@ -0,0 +1,145 @@ +name: Track upstream binaries + +# Proposes patch releases of the upstream binaries pinned by version and +# sha256 (the kubeadm sysext payload, k0s, ORAS): one pull request per +# component, on its own auto/track-binaries/ branch. What is +# tracked, how each sha256 is verified, and why this is not a Renovate rule: +# .github/scripts/track-binaries.py. +# +# Same shape as track-junctions.yml: schedule and manual dispatch only, never +# pull_request, and pull requests opened with the mergeraptor app token so +# that they get CI. +on: + schedule: + # After track-junctions (08:00 UTC). + - cron: '30 8 * * *' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: track-binaries + cancel-in-progress: false + +jobs: + check: + runs-on: ubuntu-26.04 + timeout-minutes: 10 + outputs: + updates: ${{ steps.check.outputs.updates }} + steps: + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: main + persist-credentials: false + + - name: Find newer patch releases + id: check + env: + # Read-only; only raises the API rate limit for the release listings. + GITHUB_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + # A component whose upstream cannot be read fails this job, but only + # after the other components' updates are recorded: one outage must + # neither pass as "nothing to track" nor hold back the rest. + status=0 + python3 .github/scripts/track-binaries.py check --json > "$RUNNER_TEMP/updates.json" || status=$? + echo "updates=$(cat "$RUNNER_TEMP/updates.json")" >> "$GITHUB_OUTPUT" + exit "$status" + + propose: + needs: check + if: ${{ !cancelled() && needs.check.outputs.updates != '' && needs.check.outputs.updates != '[]' }} + runs-on: ubuntu-26.04 + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.check.outputs.updates) }} + steps: + # GITHUB_TOKEN stays read-only. The push and the pull request use the + # mergeraptor app token (pushes made with GITHUB_TOKEN never trigger CI), + # narrowed to what this job does. + - name: Get mergeraptor token + id: app-token + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3 + with: + app-id: ${{ secrets.MERGERAPTOR_APP_ID }} + private-key: ${{ secrets.MERGERAPTOR_PRIVATE_KEY }} + permission-contents: write + permission-pull-requests: write + # The ORAS pins live in .github/workflows/build.yml, and pushing a + # change to a workflow file needs the Workflows permission. An empty + # value requests nothing. + permission-workflows: ${{ matrix.component == 'oras' && 'write' || '' }} + + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: main + fetch-depth: 0 + token: ${{ steps.app-token.outputs.token }} + + - name: Apply the release and verify its checksums + env: + COMPONENT: ${{ matrix.component }} + VERSION: ${{ matrix.latest }} + run: | + set -euo pipefail + python3 .github/scripts/track-binaries.py apply "$COMPONENT" --version "$VERSION" \ + --summary "$RUNNER_TEMP/pr-body.md" > "$RUNNER_TEMP/files.txt" + + - name: Create or update the pull request + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + COMPONENT: ${{ matrix.component }} + FROM: ${{ matrix.current }} + TO: ${{ matrix.latest }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + set -euo pipefail + branch="auto/track-binaries/${COMPONENT}" + title="chore(deps): update ${COMPONENT} ${FROM} -> ${TO}" + if [ ! -s "$RUNNER_TEMP/files.txt" ]; then + echo "${COMPONENT} needs no change." + exit 0 + fi + + # One pull request per component. An open one with this exact title + # already proposes this release, so leave it and its CI run alone. A + # closed one with this title was turned down (or reverted); do not + # reopen it every day, the next release brings a new title. No + # pipes into grep and no `|| true`: a failed query must stop here, + # not pass for "no pull request". + open=$(gh pr list --head "$branch" --base main --state open --json number,title --jq '.[0] | select(.) | "\(.number)\t\(.title)"') + number="${open%%$'\t'*}" + if [ -n "$open" ] && [ "${open#*$'\t'}" = "$title" ]; then + echo "#${number} already proposes ${COMPONENT} ${TO}." + exit 0 + fi + closed=$(gh pr list --head "$branch" --base main --state closed --json title --jq '.[].title') + if [ -z "$open" ] && grep -Fxq -- "$title" <<< "$closed"; then + echo "A closed pull request already proposed '${title}'; not reopening it." + exit 0 + fi + + # No attribution trailers, as in track-junctions.yml: this commit is + # the deterministic output of track-binaries.py. + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git checkout -B "$branch" + mapfile -t files < "$RUNNER_TEMP/files.txt" + git add -- "${files[@]}" + git commit -m "$title" + git push --force-with-lease origin "$branch" + + printf '\n---\n*Generated by [Track upstream binaries](%s)*\n' "$RUN_URL" >> "$RUNNER_TEMP/pr-body.md" + if [ -n "$open" ]; then + gh pr edit "$number" --title "$title" --body-file "$RUNNER_TEMP/pr-body.md" + echo "Updated PR: $(gh pr view "$number" --json url --jq '.url')" + else + echo "Created PR: $(gh pr create --base main --head "$branch" --title "$title" --body-file "$RUNNER_TEMP/pr-body.md")" + fi diff --git a/docs/skills/ci-tooling.md b/docs/skills/ci-tooling.md index 4eb28c2e..b4c6d063 100644 --- a/docs/skills/ci-tooling.md +++ b/docs/skills/ci-tooling.md @@ -4,7 +4,7 @@ description: CI workflow conventions for Bluefin Server. Use when writing or edi metadata: type: reference status: stable - last_updated: "2026-09-27" + last_updated: "2026-09-28" context7-sources: - /websites/github_en_actions - /websites/cli_github_manual @@ -121,6 +121,7 @@ sudo_cmd := if `podman info >/dev/null 2>&1 && echo 1 || echo 0` == "1" { "" } e | `release` | `build.yml` | `push/main`, `workflow_dispatch` | Publishes `dist/diskless/` as-is: an immutable GitHub Release tagged `v` plus an ORAS OCI artifact at `ghcr.io//bluefin-server:,latest` (one layer per file, artifact type `application/vnd.projectbluefin.server.release.v1`) (`if: ${{ !failure() && !cancelled() && github.ref == 'refs/heads/main' }}`). `contents: write` + `packages: write`. | | `docs` | `docs-checks.yml` | `pull_request`, `push/main` | Runs markdown and skill metadata checks via `docs-checks.py`. Read-only token. | | `unit` | `unit-tests.yml` | `pull_request`, `push/main` | Runs pytest and BATS unit test suites. Read-only token. | +| `check`, `propose` | `track-binaries.yml` | `schedule` (08:30 UTC), `workflow_dispatch` | `check` finds the newest patch release in each pinned series of the upstream binaries pinned by version + sha256 (Kubernetes, cri-tools, containerd, runc, CNI plugins, k0s, ORAS) with `.github/scripts/track-binaries.py`; `propose` moves each version together with its sha256 pins, verified against upstream's checksum files and the downloaded assets, and opens or updates one PR per component on `auto/track-binaries/`. Minor bumps stay manual (`kubeadm-sysext.md`, `k0s-sysext.md`). Read-only `GITHUB_TOKEN`; writes use the mergeraptor app token narrowed to `contents` + `pull-requests` (+ `workflows` for ORAS, pinned in `build.yml`). Never on `pull_request`. | GitHub Actions runs the **complete BuildStream compilation pipeline** using `/mnt` SSD storage on the runner for podman and BuildStream caches. Release assets are diff --git a/docs/skills/k0s-sysext.md b/docs/skills/k0s-sysext.md index 2893f30a..60a15ea5 100644 --- a/docs/skills/k0s-sysext.md +++ b/docs/skills/k0s-sysext.md @@ -4,7 +4,7 @@ description: Build and ship the k0s systemd-sysext for Bluefin Server. KubeStell metadata: type: how-to status: stable - last_updated: "2026-09-27" + last_updated: "2026-09-28" context7-sources: - /systemd/systemd --- @@ -85,6 +85,7 @@ Design choices: | `Justfile` | `build-sysext` / `export-sysext` targets. | | `.github/workflows/build.yml` | Builds, signs, and publishes sysext assets. | | `.github/scripts/check-k0s-version.py` | Fails closed if any consumer restates the k0s version instead of deriving it. | +| `.github/scripts/track-binaries.py` | Moves both atoms and the SHA256 together: `track-binaries.yml` proposes patch releases of the pinned minor, `apply k0s --version X.Y.Z+k0s.N` makes a minor bump by hand (then move the series in `tests/unit/test_k0s_version.py`). | ## Build Outputs diff --git a/docs/skills/kubeadm-sysext.md b/docs/skills/kubeadm-sysext.md index 4a70f2a4..af9615f3 100644 --- a/docs/skills/kubeadm-sysext.md +++ b/docs/skills/kubeadm-sysext.md @@ -4,7 +4,7 @@ description: Build, ship and operate the opt-in kubeadm worker systemd-sysext (k metadata: type: how-to status: stable - last_updated: "2026-09-27" + last_updated: "2026-09-28" --- # kubeadm worker sysext @@ -43,6 +43,32 @@ Nothing ships under `/opt`: `/opt/cni/bin` stays a writable host directory for the cluster CNI (Cilium's `cilium-cni`). containerd searches `/opt/cni/bin` then `/usr/libexec/cni`. +## Versions + +The sysext runs the Kubernetes minor of the cluster it joins: the series of +`kubernetes-version` in `include/kubeadm.yml`. It does not follow k0s, which +bundles its own Kubernetes for a separate cluster +([k0s-sysext.md](k0s-sysext.md)); the two minors move independently. +`kubeadm join` must use the minor that last ran `kubeadm init` or +`kubeadm upgrade` on that cluster, and kubelet may be older than the API server +but never newer. Patch releases change neither constraint. + +- **Patches** are automatic. `.github/workflows/track-binaries.yml` opens one + pull request per component (Kubernetes, cri-tools, containerd, runc, CNI + plugins) when a newer patch of its pinned series is out, with the version + and its sha256 refs changed together and checked against the checksum files + upstream publishes. +- **Minors** are manual, once the cluster's control plane runs the new minor: + 1. `python3 .github/scripts/track-binaries.py apply kubernetes --version X.Y.Z`, + and the same for `cri-tools` (its minor follows Kubernetes). `apply` + verifies the checksums exactly as the tracker does. + 2. Set `pause-image` to the tag that + `kubeadm config images list --kubernetes-version vX.Y.Z` prints. + 3. Check containerd's `RELEASES.md` support table for the new minor; move + `containerd` and `runc` with `apply` if it asks for newer ones. + 4. Move the series in `tests/unit/test_kubeadm_sysext.py`, then boot-verify + as in [Verify](#verify). + ## Runtime contract - No preset enables anything; `80-kubeadm.preset` disables both units against diff --git a/include/k0s.yml b/include/k0s.yml index acf0707f..66c981f1 100644 --- a/include/k0s.yml +++ b/include/k0s.yml @@ -5,8 +5,9 @@ # project.conf (`installer-version`) or include/image.yml (`image-version`) — those # axes name the FSDK point release and the OS build and change for unrelated reasons. # -# Only the two atoms below are edited when k0s is bumped. Every other spelling -# of the k0s version in this repository is derived from them: +# Only the two atoms below are edited when k0s is bumped, by hand or by +# .github/workflows/track-binaries.yml. Every other spelling of the k0s version +# in this repository is derived from them; for release v1.36.4+k0s.0: # # %{k0s-k8s-version} upstream Kubernetes version 1.36.4 # %{k0s-patch} upstream k0s patch suffix number 0 diff --git a/include/kubeadm.yml b/include/kubeadm.yml index dd9a6c14..a5478922 100644 --- a/include/kubeadm.yml +++ b/include/kubeadm.yml @@ -1,21 +1,25 @@ # Single source of truth for the kubeadm sysext payload versions. # -# Pinned to what the target kubeadm cluster's existing nodes run, so a -# Bluefin Server worker is indistinguishable from them at the CRI/kubelet -# level. The sha256 of every download sits next to its source in +# Each version stays on the release series the target kubeadm cluster's nodes +# run, so a Bluefin Server worker matches them at the CRI/kubelet level. +# .github/workflows/track-binaries.yml proposes patch releases inside those +# series; moving to another series is manual (docs/skills/kubeadm-sysext.md). +# The sha256 of every download sits next to its source in # elements/kubeadm/kubeadm-bin.bst; the sysext itself is locked to the image # version (kubeadm_.raw.zst). # -# %{kubernetes-version} kubelet, kubeadm, kubectl (dl.k8s.io) -# %{crictl-version} cri-tools -# %{containerd-version} containerd static build -# %{runc-version} runc static build -# %{cni-plugins-version} containernetworking/plugins -# %{pause-image} CRI sandbox image (matches kubeadm-flags.env) +# A comment line separates each version from the next, so the tracker's +# per-component pull requests never edit adjacent lines and merge in any order. variables: + # kubelet, kubeadm, kubectl (dl.k8s.io) kubernetes-version: "1.34.3" + # crictl (cri-tools) crictl-version: "1.34.0" + # containerd static build containerd-version: "2.1.5" + # runc static build runc-version: "1.3.3" + # containernetworking/plugins cni-plugins-version: "1.1.1" + # CRI sandbox image (matches kubeadm-flags.env) pause-image: "registry.k8s.io/pause:3.10.1" diff --git a/tests/unit/test_k0s_version.py b/tests/unit/test_k0s_version.py index ddf83633..99760fac 100644 --- a/tests/unit/test_k0s_version.py +++ b/tests/unit/test_k0s_version.py @@ -11,8 +11,10 @@ def test_k0s_version_ssot(): assert k0s_yml.is_file(), "include/k0s.yml missing" data = yaml.safe_load(k0s_yml.read_text()) vars_ = data.get("variables", {}) - assert vars_.get("k0s-k8s-version") == "1.36.4" - assert vars_.get("k0s-patch") == "0" + # Patch releases arrive through .github/workflows/track-binaries.yml; the + # 1.36 series only moves by hand. + assert re.fullmatch(r"1\.36\.\d+", vars_.get("k0s-k8s-version", "")) + assert re.fullmatch(r"\d+", vars_.get("k0s-patch", "")) assert vars_.get("k0s-version") == "%{k0s-k8s-version}-k0s.%{k0s-patch}" @@ -22,7 +24,7 @@ def test_k0s_bin_element(): content = bin_bst.read_text() assert "base/base-stack.bst" in content assert "github:k0sproject/k0s/releases/download/" in content - assert "ca1e9e68107335846e8296777fce2ccd654284e6265b4b5d32c34ead872af98f" in content + assert re.search(r"^\s+ref: [0-9a-f]{64}$", content, re.MULTILINE) def test_k0s_version_checker_reads_the_component_transfer(): diff --git a/tests/unit/test_kubeadm_sysext.py b/tests/unit/test_kubeadm_sysext.py index c92f393d..52effa27 100644 --- a/tests/unit/test_kubeadm_sysext.py +++ b/tests/unit/test_kubeadm_sysext.py @@ -30,15 +30,23 @@ def versions() -> dict[str, str]: return yaml.safe_load(VERSIONS.read_text(encoding="utf-8"))["variables"] -def test_versions_match_the_existing_cluster_nodes() -> None: - assert versions() == { - "kubernetes-version": "1.34.3", - "crictl-version": "1.34.0", - "containerd-version": "2.1.5", - "runc-version": "1.3.3", - "cni-plugins-version": "1.1.1", - "pause-image": "registry.k8s.io/pause:3.10.1", +def test_versions_stay_on_the_cluster_nodes_series() -> None: + # Patch releases arrive through .github/workflows/track-binaries.yml; a + # series change is a deliberate edit here (docs/skills/kubeadm-sysext.md). + pinned = versions() + series = {} + for name, version in pinned.items(): + if name != "pause-image": + assert re.fullmatch(r"\d+\.\d+\.\d+", version), name + series[name] = version.rsplit(".", 1)[0] + assert series == { + "kubernetes-version": "1.34", + "crictl-version": "1.34", + "containerd-version": "2.1", + "runc-version": "1.3", + "cni-plugins-version": "1.1", } + assert pinned["pause-image"] == "registry.k8s.io/pause:3.10.1" def test_every_download_is_pinned_by_sha256() -> None: diff --git a/tests/unit/test_track_binaries.py b/tests/unit/test_track_binaries.py new file mode 100644 index 00000000..c75b6b34 --- /dev/null +++ b/tests/unit/test_track_binaries.py @@ -0,0 +1,446 @@ +"""Behaviour of .github/scripts/track-binaries.py. + +No network: urllib.request.urlopen is replaced by a fake upstream serving the +release listings, checksum files and assets each test registers. The fixture +repository mirrors the layout of the real include/, elements/, Justfile and +workflow files; the last tests read the real files, without network, so a +reformat that the tracker cannot follow fails here instead of in the scheduled +run. +""" + +import hashlib +import importlib.util +import io +import json +import sys +import urllib.error +import urllib.request +from email.message import Message +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[2] +SCRIPT = ROOT / ".github" / "scripts" / "track-binaries.py" + +_spec = importlib.util.spec_from_file_location("track_binaries", SCRIPT) +assert _spec is not None and _spec.loader is not None +track = importlib.util.module_from_spec(_spec) +sys.modules["track_binaries"] = track +_spec.loader.exec_module(track) + + +def sha(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +OLD = {name: sha(name.encode()) for name in ("kubelet", "kubeadm", "kubectl", "crictl", "containerd", "runc", "cni", "k0s", "oras")} + +FILES = { + "include/aliases.yml": "aliases:\n github: https://github.com/\n k8s_dl: https://dl.k8s.io/\n", + "include/kubeadm.yml": """variables: + # kubelet, kubeadm, kubectl + kubernetes-version: "1.34.3" + # crictl + crictl-version: "1.34.0" + # containerd + containerd-version: "2.1.5" + # runc + runc-version: "1.3.3" + # CNI plugins + cni-plugins-version: "1.1.1" + # sandbox + pause-image: "registry.k8s.io/pause:3.10.1" +""", + "elements/kubeadm/kubeadm-bin.bst": f"""kind: manual + +(@): +- include/kubeadm.yml + +sources: +- kind: remote + url: k8s_dl:release/v%{{kubernetes-version}}/bin/linux/amd64/kubelet + ref: {OLD['kubelet']} + directory: k8s +- kind: remote + url: k8s_dl:release/v%{{kubernetes-version}}/bin/linux/amd64/kubeadm + ref: {OLD['kubeadm']} + directory: k8s +- kind: remote + url: k8s_dl:release/v%{{kubernetes-version}}/bin/linux/amd64/kubectl + ref: {OLD['kubectl']} + directory: k8s +- kind: tar + url: github:kubernetes-sigs/cri-tools/releases/download/v%{{crictl-version}}/crictl-v%{{crictl-version}}-linux-amd64.tar.gz + ref: {OLD['crictl']} + base-dir: '' +- kind: tar + url: github:containerd/containerd/releases/download/v%{{containerd-version}}/containerd-static-%{{containerd-version}}-linux-amd64.tar.gz + ref: {OLD['containerd']} + base-dir: bin +- kind: remote + url: github:opencontainers/runc/releases/download/v%{{runc-version}}/runc.amd64 + ref: {OLD['runc']} + directory: runc +- kind: tar + url: github:containernetworking/plugins/releases/download/v%{{cni-plugins-version}}/cni-plugins-linux-amd64-v%{{cni-plugins-version}}.tgz + ref: {OLD['cni']} + base-dir: '' + +config: + install-commands: + - install -m 0755 runc/runc.amd64 "%{{install-root}}%{{bindir}}/runc" +""", + "include/k0s.yml": """variables: + k0s-k8s-version: "1.36.4" + k0s-patch: "0" + + k0s-upstream-tag: "v%{k0s-k8s-version}%2Bk0s.%{k0s-patch}" + k0s-version: "%{k0s-k8s-version}-k0s.%{k0s-patch}" +""", + "elements/k0s/k0s-bin.bst": f"""kind: manual +(@): include/k0s.yml + +sources: + - kind: remote + url: github:k0sproject/k0s/releases/download/%{{k0s-upstream-tag}}/k0s-%{{k0s-upstream-tag}}-amd64 + ref: {OLD['k0s']} + +config: + install-commands: + - install -D -m 0755 k0s-%{{k0s-upstream-tag}}-amd64 "%{{install-root}}/usr/bin/k0s" +""", + "Justfile": 'export oras_image := env("ORAS_IMAGE", "ghcr.io/oras-project/oras:v1.3.4")\n', + ".github/workflows/build.yml": f"""jobs: + release: + steps: + # pin 1.3.4 (matching the Justfile's oras_image) by URL and checksum. + - name: Set up oras + uses: oras-project/setup-oras@1d808f7d7f6995cc68b7bf507bfe5c5446e1dc9d # v2.0.1 + with: + url: https://github.com/oras-project/oras/releases/download/v1.3.4/oras_1.3.4_linux_amd64.tar.gz + checksum: {OLD['oras']} + + - name: Push + run: oras push + release-dry-run: + steps: + - name: Set up oras + uses: oras-project/setup-oras@1d808f7d7f6995cc68b7bf507bfe5c5446e1dc9d # v2.0.1 + with: + url: https://github.com/oras-project/oras/releases/download/v1.3.4/oras_1.3.4_linux_amd64.tar.gz + checksum: {OLD['oras']} +""", +} + + +class Upstream: + """Serves registered URLs; everything else is a 404.""" + + def __init__(self): + self.files: dict[str, bytes] = {} + self.requests: list[urllib.request.Request] = [] + + def urlopen(self, request, timeout=None): + self.requests.append(request) + if request.full_url not in self.files: + raise urllib.error.HTTPError(request.full_url, 404, "Not Found", Message(), None) + return io.BytesIO(self.files[request.full_url]) + + def asset(self, url: str, sums_url: str | None = None, name: str | None = None) -> str: + data = f"payload of {url}".encode() + self.files[url] = data + if sums_url: + self.files[sums_url] = f"{sha(data)} {name or url.rsplit('/', 1)[1]}\n".encode() + return sha(data) + + def releases(self, repo: str, *pages: list[dict]) -> None: + for number, page in enumerate(pages, 1): + url = f"https://api.github.com/repos/{repo}/releases?per_page=100&page={number}" + self.files[url] = json.dumps(page).encode() + + +def release(tag: str, *assets: str, prerelease: bool = False, draft: bool = False) -> dict: + return {"tag_name": tag, "prerelease": prerelease, "draft": draft, "assets": [{"name": a} for a in assets]} + + +def containerd(version: str) -> dict: + name = f"containerd-static-{version}-linux-amd64.tar.gz" + return release(f"v{version}", name, f"{name}.sha256sum") + + +CONTAINERD = "https://github.com/containerd/containerd/releases/download/v2.1.9/containerd-static-2.1.9-linux-amd64.tar.gz" + + +@pytest.fixture +def upstream(monkeypatch): + fake = Upstream() + monkeypatch.setattr(urllib.request, "urlopen", fake.urlopen) + monkeypatch.delenv("GITHUB_TOKEN", raising=False) + monkeypatch.delenv("GH_TOKEN", raising=False) + return fake + + +@pytest.fixture +def repo(tmp_path): + for path, text in FILES.items(): + (tmp_path / path).parent.mkdir(parents=True, exist_ok=True) + (tmp_path / path).write_text(text, encoding="utf-8") + return tmp_path + + +def snapshot(root: Path) -> dict[str, str]: + return {path: (root / path).read_text(encoding="utf-8") for path in FILES} + + +def changed_lines(before: dict[str, str], root: Path) -> dict[str, list[tuple[str, str]]]: + diff = {} + for path, text in before.items(): + pairs = [(a, b) for a, b in zip(text.splitlines(), (root / path).read_text(encoding="utf-8").splitlines()) if a != b] + if pairs: + diff[path] = pairs + return diff + + +def newest(name: str, repo: Path) -> str: + return track.newest(track.COMPONENTS[name], track.Tree(repo)) + + +def test_newest_stays_inside_the_pinned_minor_and_skips_unreleased(repo, upstream): + upstream.releases("containerd/containerd", [ + containerd("2.2.0"), + {**containerd("2.1.11"), "prerelease": True}, + containerd("2.1.10-rc.1"), + {**containerd("2.1.9"), "draft": True}, + containerd("2.1.8"), + containerd("2.1.5"), + containerd("1.7.30"), + release("api/v1.9.0"), + ]) + assert newest("containerd", repo) == "2.1.8" + + +def test_release_still_missing_its_assets_is_not_a_candidate(repo, upstream): + upstream.releases("containerd/containerd", [ + release("v2.1.9", "containerd-static-2.1.9-linux-amd64.tar.gz"), + containerd("2.1.8"), + ]) + assert newest("containerd", repo) == "2.1.8" + + +def test_release_listing_is_paginated(repo, upstream): + upstream.releases("containerd/containerd", [containerd(f"1.7.{n}") for n in range(100)], [containerd("2.1.9")]) + assert newest("containerd", repo) == "2.1.9" + + +def test_kubernetes_follows_the_dl_k8s_io_stable_channel(repo, upstream): + upstream.files["https://dl.k8s.io/release/stable-1.34.txt"] = b"v1.34.12\n" + assert newest("kubernetes", repo) == "1.34.12" + upstream.files["https://dl.k8s.io/release/stable-1.34.txt"] = b"v1.35.0\n" + with pytest.raises(track.TrackError, match="not a 1.34.x release"): + newest("kubernetes", repo) + + +def test_k0s_orders_by_kubernetes_patch_then_k0s_suffix(repo, upstream): + def k0s(tag: str) -> dict: + return release(tag, f"k0s-{tag}-amd64", "sha256sums.txt") + + upstream.releases("k0sproject/k0s", [k0s("v1.37.0+k0s.0"), k0s("v1.36.4+k0s.1"), k0s("v1.36.3+k0s.2"), k0s("v1.36.4+k0s.0")]) + assert newest("k0s", repo) == "1.36.4+k0s.1" + upstream.releases("k0sproject/k0s", [k0s("v1.36.5+k0s.0"), k0s("v1.36.4+k0s.1")]) + assert newest("k0s", repo) == "1.36.5+k0s.0" + + +def test_check_json_lists_pending_updates_and_still_reports_errors(repo, upstream, monkeypatch, capsys): + monkeypatch.setattr(track, "COMPONENTS", {n: track.COMPONENTS[n] for n in ("containerd", "runc", "cni-plugins")}) + upstream.releases("containerd/containerd", [containerd("2.1.9"), containerd("2.1.5")]) + upstream.releases("opencontainers/runc", [release("v1.3.3", "runc.amd64", "runc.sha256sum")]) + assert track.main(["check", "--json"], root=repo) == 1 + out, err = capsys.readouterr() + assert json.loads(out) == [{"component": "containerd", "series": "2.1", "current": "2.1.5", "latest": "2.1.9"}] + assert "runc 1.3 1.3.3 1.3.3 current" in err + assert "ERROR: cni-plugins: GET https://api.github.com/repos/containernetworking/plugins/releases" in err + + +def test_apply_moves_the_version_and_only_its_own_ref(repo, upstream, tmp_path, capsys): + upstream.releases("containerd/containerd", [containerd("2.1.9"), containerd("2.1.5")]) + new = upstream.asset(CONTAINERD, CONTAINERD + ".sha256sum") + before = snapshot(repo) + body = tmp_path / "body.md" + + assert track.main(["apply", "containerd", "--summary", str(body)], root=repo) == 0 + + assert changed_lines(before, repo) == { + "include/kubeadm.yml": [(' containerd-version: "2.1.5"', ' containerd-version: "2.1.9"')], + "elements/kubeadm/kubeadm-bin.bst": [(f" ref: {OLD['containerd']}", f" ref: {new}")], + } + assert capsys.readouterr().out.split() == ["elements/kubeadm/kubeadm-bin.bst", "include/kubeadm.yml"] + text = body.read_text() + assert "Patch release of **containerd** in the pinned `2.1` series: `2.1.5` → `2.1.9`." in text + assert f"`{new}` (was `{OLD['containerd']}`)" in text + assert f"- {CONTAINERD}.sha256sum" in text + assert "https://github.com/containerd/containerd/releases/tag/v2.1.9" in text + + +def test_apply_kubernetes_rewrites_all_three_binaries_from_bare_digests(repo, upstream): + upstream.files["https://dl.k8s.io/release/stable-1.34.txt"] = b"v1.34.12" + new = {} + for binary in ("kubelet", "kubeadm", "kubectl"): + url = f"https://dl.k8s.io/release/v1.34.12/bin/linux/amd64/{binary}" + upstream.files[url] = binary.encode() * 3 + upstream.files[url + ".sha256"] = sha(binary.encode() * 3).encode() + new[binary] = sha(binary.encode() * 3) + before = snapshot(repo) + + assert track.main(["apply", "kubernetes"], root=repo) == 0 + + assert changed_lines(before, repo) == { + "include/kubeadm.yml": [(' kubernetes-version: "1.34.3"', ' kubernetes-version: "1.34.12"')], + "elements/kubeadm/kubeadm-bin.bst": [(f" ref: {OLD[b]}", f" ref: {new[b]}") for b in ("kubelet", "kubeadm", "kubectl")], + } + + +def test_apply_runc_reads_its_pgp_signed_sums_file(repo, upstream): + url = "https://github.com/opencontainers/runc/releases/download/v1.3.6/runc.amd64" + new = upstream.asset(url) + upstream.files[url.rsplit("/", 1)[0] + "/runc.sha256sum"] = ( + "-----BEGIN PGP SIGNED MESSAGE-----\nHash: SHA512\n\n" + f"{sha(b'arm64')} runc.arm64\n{new} runc.amd64\n{sha(b'tar')} runc.tar.xz\n" + "-----BEGIN PGP SIGNATURE-----\n\niQIzBAEBCgAdFiEE\n-----END PGP SIGNATURE-----\n" + ).encode() + + assert track.main(["apply", "runc", "--version", "1.3.6"], root=repo) == 0 + + assert f"ref: {new}" in (repo / "elements/kubeadm/kubeadm-bin.bst").read_text() + + +def test_apply_k0s_moves_both_atoms_and_downloads_the_escaped_tag(repo, upstream): + url = "https://github.com/k0sproject/k0s/releases/download/v1.36.4%2Bk0s.1/k0s-v1.36.4%2Bk0s.1-amd64" + new = upstream.asset(url) + upstream.files[url.rsplit("/", 1)[0] + "/sha256sums.txt"] = ( + f"{sha(b'airgap')} *k0s-airgap-bundle-v1.36.4+k0s.1-linux-amd64.tar\n{new} *k0s-v1.36.4+k0s.1-amd64\n" + ).encode() + before = snapshot(repo) + + assert track.main(["apply", "k0s", "--version", "1.36.4+k0s.1"], root=repo) == 0 + + assert changed_lines(before, repo) == { + "include/k0s.yml": [(' k0s-patch: "0"', ' k0s-patch: "1"')], + "elements/k0s/k0s-bin.bst": [(f" ref: {OLD['k0s']}", f" ref: {new}")], + } + assert url in [r.full_url for r in upstream.requests] + + +def test_apply_oras_moves_the_justfile_tag_and_every_setup_oras_pin(repo, upstream, tmp_path): + url = "https://github.com/oras-project/oras/releases/download/v1.3.5/oras_1.3.5_linux_amd64.tar.gz" + new = upstream.asset(url, url.rsplit("/", 1)[0] + "/oras_1.3.5_checksums.txt") + body = tmp_path / "body.md" + + assert track.main(["apply", "oras", "--version", "1.3.5", "--summary", str(body)], root=repo) == 0 + + assert "ghcr.io/oras-project/oras:v1.3.5" in (repo / "Justfile").read_text() + workflow = (repo / ".github/workflows/build.yml").read_text() + assert workflow.count(f"url: {url}") == 2 + assert workflow.count(f"checksum: {new}") == 2 + assert "1.3.4" in workflow.split("\n")[3] + assert "`.github/workflows/build.yml:4: # pin 1.3.4 (matching the Justfile's oras_image) by URL and checksum.`" in body.read_text() + + +def test_oras_pins_that_disagree_are_refused(repo, upstream): + (repo / "Justfile").write_text('export oras_image := env("ORAS_IMAGE", "ghcr.io/oras-project/oras:v1.3.3")\n') + with pytest.raises(track.TrackError, match=r"ORAS pins disagree \(\.github/workflows/build\.yml: 1\.3\.4, Justfile: 1\.3\.3\)"): + track.apply(repo, "oras", "1.3.5") + + +def test_asset_that_does_not_match_its_published_checksum_writes_nothing(repo, upstream, capsys): + upstream.releases("containerd/containerd", [containerd("2.1.9")]) + upstream.asset(CONTAINERD, CONTAINERD + ".sha256sum") + upstream.files[CONTAINERD] = b"tampered" + before = snapshot(repo) + + assert track.main(["apply", "containerd"], root=repo) == 1 + + assert snapshot(repo) == before + assert f"{CONTAINERD} hashes to {sha(b'tampered')}" in capsys.readouterr().err + + +def test_checksum_file_that_omits_the_asset_is_refused(repo, upstream): + upstream.asset(CONTAINERD) + upstream.files[CONTAINERD + ".sha256sum"] = f"{sha(b'x')} containerd-2.1.9-linux-amd64.tar.gz\n".encode() + with pytest.raises(track.TrackError, match="lists 0 sha256 for containerd-static-2.1.9-linux-amd64.tar.gz"): + track.apply(repo, "containerd", "2.1.9") + + +def test_apply_is_a_no_op_when_the_pin_is_already_newest(repo, upstream, capsys): + upstream.releases("containerd/containerd", [containerd("2.1.5"), containerd("2.1.4")]) + before = snapshot(repo) + assert track.main(["apply", "containerd"], root=repo) == 0 + assert snapshot(repo) == before + assert capsys.readouterr().out == "" + + +def test_manual_minor_bump_uses_the_same_verification(repo, upstream, tmp_path): + for binary in ("kubelet", "kubeadm", "kubectl"): + url = f"https://dl.k8s.io/release/v1.35.2/bin/linux/amd64/{binary}" + upstream.files[url] = binary.encode() + upstream.files[url + ".sha256"] = sha(binary.encode()).encode() + body = tmp_path / "body.md" + + assert track.main(["apply", "kubernetes", "--version", "1.35.2", "--summary", str(body)], root=repo) == 0 + + assert 'kubernetes-version: "1.35.2"' in (repo / "include/kubeadm.yml").read_text() + assert "a series change" in body.read_text() + assert not any("stable-" in r.full_url for r in upstream.requests) + + +def test_malformed_version_is_refused_before_any_download(repo, upstream): + with pytest.raises(track.TrackError, match="does not match"): + track.apply(repo, "k0s", "1.36.5") + assert upstream.requests == [] + + +def test_source_without_its_own_ref_does_not_borrow_the_next_one(repo, upstream): + bst = repo / "elements/kubeadm/kubeadm-bin.bst" + bst.write_text(bst.read_text().replace(f" ref: {OLD['containerd']}\n", "")) + upstream.asset(CONTAINERD, CONTAINERD + ".sha256sum") + with pytest.raises(track.TrackError, match="no `ref:` in the mapping of `url: github:containerd/"): + track.apply(repo, "containerd", "2.1.9") + assert f"ref: {OLD['runc']}" in bst.read_text() + + +def test_github_token_is_sent_to_the_api_and_never_to_downloads(repo, upstream, monkeypatch): + monkeypatch.setenv("GITHUB_TOKEN", "secret-token") + upstream.releases("containerd/containerd", [containerd("2.1.9")]) + upstream.asset(CONTAINERD, CONTAINERD + ".sha256sum") + + track.apply(repo, "containerd") + + auth = {r.full_url.split("?")[0]: r.get_header("Authorization") for r in upstream.requests} + assert auth == { + "https://api.github.com/repos/containerd/containerd/releases": "Bearer secret-token", + CONTAINERD + ".sha256sum": None, + CONTAINERD: None, + } + + +@pytest.mark.parametrize("name", list(track.COMPONENTS)) +def test_real_pins_are_readable(name): + tree = track.Tree(ROOT) + component = track.COMPONENTS[name] + component.parse(component.current(tree)) + pins = component.pins(tree) + assert pins + for pin in pins: + assert pin.url.startswith("https://") and "%{" not in pin.url + assert track.SUMS_LINE_RE.match(track.read_pin(tree, pin)), pin + + +def test_every_pinned_source_belongs_to_exactly_one_component(): + tree = track.Tree(ROOT) + elements = {c.element for c in track.COMPONENTS.values() if isinstance(c, track.BstComponent)} + for element in sorted(elements): + for line in tree[element].splitlines(): + match = track.URL_LINE_RE.match(line) + if match: + owners = [c.name for c in track.COMPONENTS.values() if getattr(c, "marker", None) and c.marker in match["url"]] + assert len(owners) == 1, f"{element}: {match['url']} is tracked by {owners}"