From 58e010bdcdad5a358ea94210de2cb3a82ac80c22 Mon Sep 17 00:00:00 2001 From: Jeffrey Sica Date: Mon, 28 Sep 2026 22:28:20 -0500 Subject: [PATCH 1/5] feat(ci): add track-binaries.py to move pinned binaries with verified sha256 Upstream binaries are pinned by a version in include/*.yml (or the Justfile) and a sha256 in elements/*.bst (or a workflow). A version bump without its checksums is a broken BuildStream source ref, so the two have to move in one change. `check` reports the newest release inside each pinned MAJOR.MINOR series: Kubernetes from dl.k8s.io stable-X.Y.txt, cri-tools, containerd, runc, CNI plugins, k0s and ORAS from GitHub releases, skipping drafts, prereleases and releases whose pinned assets are not attached yet. `apply` moves one component and rewrites every pin derived from it, taking each sha256 from the checksum file upstream publishes and confirming it against the downloaded asset before writing anything. `apply --version` makes a manual minor bump with the same verification. Refs are written in place rather than with `bst source track`, which re-downloads every source of the element, trusts whatever bytes it gets, and re-indents every list in elements/k0s/k0s-bin.bst. Against live upstream both produce identical refs for kubelet, kubeadm, kubectl 1.34.12, containerd 2.1.9, runc 1.3.6 and k0s 1.36.4+k0s.1. Tests fake urllib.request.urlopen (no network) and include a contract test that reads the real repository files. Assisted-by: Claude Opus 5.5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/scripts/track-binaries.py | 593 ++++++++++++++++++++++++++++++ tests/unit/test_track_binaries.py | 446 ++++++++++++++++++++++ 2 files changed, 1039 insertions(+) create mode 100644 .github/scripts/track-binaries.py create mode 100644 tests/unit/test_track_binaries.py diff --git a/.github/scripts/track-binaries.py b/.github/scripts/track-binaries.py new file mode 100644 index 00000000..bcd2ed01 --- /dev/null +++ b/.github/scripts/track-binaries.py @@ -0,0 +1,593 @@ +#!/usr/bin/env python3 +"""Track patch releases of the upstream binaries pinned by version + sha256. + +A component is one upstream version plus every sha256 pin derived from it. The +two move in one change: a new version with an old checksum is a broken +BuildStream source ref. + + component version sha256 pins + kubernetes include/kubeadm.yml kubernetes-version kubelet, kubeadm, kubectl ref: in + cri-tools include/kubeadm.yml crictl-version crictl tarball elements/kubeadm/ + containerd include/kubeadm.yml containerd-version containerd static tarball kubeadm-bin.bst + runc include/kubeadm.yml runc-version runc.amd64 + cni-plugins include/kubeadm.yml cni-plugins-version CNI plugins tarball + k0s include/k0s.yml k0s-k8s-version and k0s binary ref: in + k0s-patch elements/k0s/k0s-bin.bst + oras Justfile oras_image tag setup-oras url + checksum in .github/workflows/*.yml + +check Newest release of each component inside its pinned MAJOR.MINOR series: + Kubernetes from dl.k8s.io/release/stable-X.Y.txt, the rest from the + project's GitHub releases. Drafts, prereleases and releases that lack + the pinned assets do not count. +apply Moves one component to a version (default: the newest in its series) + and rewrites every pin derived from it. Each sha256 is read from the + checksum file the project publishes next to the asset, then confirmed + by downloading the asset and hashing it. Nothing is written unless + every pin verifies. + +Only patch releases are proposed automatically. A minor bump is a decision: the +kubeadm payload follows the minor of the cluster it joins +(docs/skills/kubeadm-sysext.md). Make it by hand with +`apply COMPONENT --version X.Y.Z`, which runs the same verification. + +Why not Renovate: a regex manager needs the version and its digest in one +match, but here the version lives in include/*.yml or the Justfile and the +digests in elements/*.bst or a workflow. One Kubernetes version feeds three +dl.k8s.io checksums, which no Renovate datasource computes, and the k0s version +is split into two atoms. postUpgradeTasks, which could fetch checksums, are +blocked on the hosted Renovate app. + +Why not `bst source track`: it re-downloads every source of the element instead +of the bumped component's, takes whatever bytes it gets as the new ref instead +of checking the upstream checksum, and writes the element back in its own YAML +layout (it re-indents every list in elements/k0s/k0s-bin.bst). BuildStream +still checks each ref against the bytes when the pull request's build fetches +the sources. + +Stdlib only. GITHUB_TOKEN or GH_TOKEN, if set, is sent to api.github.com only. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import sys +import urllib.error +import urllib.parse +import urllib.request +from dataclasses import dataclass +from pathlib import Path +from typing import NamedTuple + +ROOT = Path(__file__).resolve().parents[2] +TIMEOUT = 60 +MAX_RELEASE_PAGES = 20 + +URL_LINE_RE = re.compile(r"^\s*(?:-\s+)?url:\s*(?P\S+)\s*$") +VARIABLE_RE = re.compile(r'^\s+(?P[A-Za-z][\w-]*):\s*"(?P[^"]*)"', re.MULTILINE) +ALIAS_RE = re.compile(r"^\s+(?P[A-Za-z][\w-]*):\s*(?Phttps://\S+)\s*$", re.MULTILINE) +SUMS_LINE_RE = re.compile(r"^(?P[0-9a-fA-F]{64})(?:\s+\*?(?P\S+))?$") + + +class TrackError(Exception): + pass + + +def _open(url: str): + headers = {"User-Agent": "projectbluefin-server-track-binaries"} + if url.startswith("https://api.github.com/"): + headers["Accept"] = "application/vnd.github+json" + token = os.environ.get("GITHUB_TOKEN") or os.environ.get("GH_TOKEN") + if token: + # Never on downloads: release assets redirect to object storage, + # which must not see the token. + headers["Authorization"] = f"Bearer {token}" + try: + return urllib.request.urlopen(urllib.request.Request(url, headers=headers), timeout=TIMEOUT) + except urllib.error.HTTPError as err: + raise TrackError(f"GET {url}: HTTP {err.code}") from None + except urllib.error.URLError as err: + raise TrackError(f"GET {url}: {err.reason}") from None + + +def _get(url: str) -> bytes: + with _open(url) as response: + return response.read() + + +def _download_sha256(url: str) -> str: + digest = hashlib.sha256() + with _open(url) as response: + while chunk := response.read(1 << 20): + digest.update(chunk) + return digest.hexdigest() + + +class Tree: + """Repository files, read on first use and written back only by save().""" + + def __init__(self, root: Path): + self.root = Path(root) + self._text: dict[str, str] = {} + self._original: dict[str, str] = {} + + def __getitem__(self, path: str) -> str: + if path not in self._text: + try: + text = (self.root / path).read_text(encoding="utf-8") + except FileNotFoundError: + raise TrackError(f"{path}: file not found") from None + self._text[path] = self._original[path] = text + return self._text[path] + + def __setitem__(self, path: str, text: str) -> None: + self[path] + self._text[path] = text + + def copy(self) -> Tree: + other = Tree(self.root) + other._text = dict(self._text) + other._original = dict(self._original) + return other + + def glob(self, *patterns: str) -> list[str]: + return sorted({p.relative_to(self.root).as_posix() for pattern in patterns for p in self.root.glob(pattern)}) + + def save(self) -> list[str]: + changed = sorted(path for path, text in self._text.items() if text != self._original[path]) + for path in changed: + (self.root / path).write_text(self._text[path], encoding="utf-8") + return changed + + +@dataclass(frozen=True) +class Pin: + """A sha256 on the `key:` line in the same mapping as `url: anchor` in `path`.""" + + path: str + anchor: str + key: str + url: str + sums: str + + @property + def asset(self) -> str: + return urllib.parse.unquote(self.url.rsplit("/", 1)[1]) + + +def _key_in_mapping(lines: list[str], start: int, column: int, key: str) -> int | None: + for j in range(start + 1, len(lines)): + text = lines[j].strip() + if not text or text.startswith("#"): + continue + indent = len(lines[j]) - len(lines[j].lstrip()) + if indent < column: + return None + if indent == column and re.match(rf"{re.escape(key)}:\s*\S+\s*$", text): + return j + return None + + +def _checksum_lines(lines: list[str], pin: Pin) -> list[int]: + """Line numbers of `pin.key:` for every `url: pin.anchor` line; each must have one.""" + found = [] + for i, line in enumerate(lines): + match = URL_LINE_RE.match(line) + if not match or match["url"] != pin.anchor: + continue + j = _key_in_mapping(lines, i, line.index("url:"), pin.key) + if j is None: + raise TrackError(f"{pin.path}: no `{pin.key}:` in the mapping of `url: {pin.anchor}`") + found.append(j) + if not found: + raise TrackError(f"{pin.path}: no `url: {pin.anchor}`") + return found + + +def read_pin(tree: Tree, pin: Pin) -> str: + lines = tree[pin.path].splitlines(keepends=True) + values = {lines[i].split(":", 1)[1].strip() for i in _checksum_lines(lines, pin)} + if len(values) != 1: + raise TrackError(f"{pin.path}: `url: {pin.anchor}` is pinned to different checksums: {sorted(values)}") + return values.pop() + + +def write_pin(tree: Tree, pin: Pin, sha256: str) -> None: + lines = tree[pin.path].splitlines(keepends=True) + value = re.compile(rf"^(\s*{re.escape(pin.key)}:\s*)\S+") + for i in _checksum_lines(lines, pin): + lines[i] = value.sub(lambda m: m[1] + sha256, lines[i], count=1) + tree[pin.path] = "".join(lines) + + +def _variable(tree: Tree, path: str, name: str) -> re.Match: + matches = [m for m in VARIABLE_RE.finditer(tree[path]) if m["name"] == name] + if len(matches) != 1: + raise TrackError(f'{path}: expected exactly one `{name}: "..."`, found {len(matches)}') + return matches[0] + + +def _expand(template: str, variables: dict[str, str], aliases: dict[str, str], where: str) -> str: + """Resolve a source url the way BuildStream does: `%{...}` variables, then the alias.""" + for _ in range(len(variables) + 1): + refs = re.findall(r"%\{([A-Za-z][\w-]*)\}", template) + if not refs: + break + for ref in refs: + if ref not in variables: + raise TrackError(f"{where}: `{template}` uses undefined variable %{{{ref}}}") + template = template.replace("%{" + ref + "}", variables[ref]) + else: + raise TrackError(f"{where}: circular variable reference in `{template}`") + alias, _, rest = template.partition(":") + if alias in aliases: + template = aliases[alias] + rest + if not template.startswith("https://"): + raise TrackError(f"{where}: `{template}` does not resolve to an https:// URL") + return template + + +def _numbers(version: str) -> tuple[int, ...]: + return tuple(int(n) for n in re.findall(r"\d+", version)) + + +def _series(version: str) -> str: + return ".".join(str(n) for n in _numbers(version)[:2]) + + +class Component: + name: str + repo: str + # Checksum file URL; {url} is the asset URL, {dir} its directory. + sums: str + stable_channel = "" + version_re = r"(\d+\.\d+\.\d+)" + + def current(self, tree: Tree) -> str: + raise NotImplementedError + + def set_version(self, tree: Tree, version: str) -> None: + raise NotImplementedError + + def pins(self, tree: Tree) -> list[Pin]: + raise NotImplementedError + + def parse(self, version: str) -> tuple[str, ...]: + match = re.fullmatch(self.version_re, version) + if not match: + raise TrackError(f"{self.name}: `{version}` does not match {self.version_re}") + return match.groups() + + def _sums(self, url: str, version: str) -> str: + return self.sums.format(url=url, dir=url.rsplit("/", 1)[0], version=version) + + def notes(self, version: str) -> str: + return f"https://github.com/{self.repo}/releases/tag/{urllib.parse.quote('v' + version)}" + + +class BstComponent(Component): + """Version atoms in a BuildStream include; sha256 refs on one element's sources.""" + + def __init__(self, name, repo, sums, include, variables, element, marker, + stable_channel="", version_re=None, version_fmt="{0}"): + self.name, self.repo, self.sums, self.stable_channel = name, repo, sums, stable_channel + self.include, self.variables, self.element, self.marker = include, variables, element, marker + self.version_re = version_re or Component.version_re + self.version_fmt = version_fmt + + def current(self, tree): + return self.version_fmt.format(*(_variable(tree, self.include, v)["value"] for v in self.variables)) + + def set_version(self, tree, version): + for name, value in zip(self.variables, self.parse(version)): + match = _variable(tree, self.include, name) + text = tree[self.include] + tree[self.include] = text[: match.start("value")] + value + text[match.end("value") :] + + def pins(self, tree): + variables = {m["name"]: m["value"] for m in VARIABLE_RE.finditer(tree[self.include])} + aliases = {m["name"]: m["url"] for m in ALIAS_RE.finditer(tree["include/aliases.yml"])} + version = self.current(tree) + pins = [] + for line in tree[self.element].splitlines(): + match = URL_LINE_RE.match(line) + if match and self.marker in match["url"]: + url = _expand(match["url"], variables, aliases, self.element) + pins.append(Pin(self.element, match["url"], "ref", url, self._sums(url, version))) + if not pins: + raise TrackError(f"{self.element}: no source url contains {self.marker}") + return pins + + +class OrasComponent(Component): + """The Justfile's ORAS image tag plus every setup-oras url and checksum.""" + + name = "oras" + repo = "oras-project/oras" + sums = "{dir}/oras_{version}_checksums.txt" + WORKFLOWS = (".github/workflows/*.yml", ".github/workflows/*.yaml") + IMAGE_RE = re.compile(r"(?Pghcr\.io/oras-project/oras:v)(?P\d+\.\d+\.\d+)(?![\w.])") + URL_RE = re.compile( + r"https://github\.com/oras-project/oras/releases/download/" + r"v(?P\d+\.\d+\.\d+)/oras_(?P\d+\.\d+\.\d+)_linux_amd64\.tar\.gz" + ) + + def _sites(self, tree): + sites = [("Justfile", m["version"]) for m in self.IMAGE_RE.finditer(tree["Justfile"])] + if not sites: + raise TrackError("Justfile: no ghcr.io/oras-project/oras:vX.Y.Z image") + urls = [(path, m) for path in tree.glob(*self.WORKFLOWS) for m in self.URL_RE.finditer(tree[path])] + if not urls: + raise TrackError(".github/workflows: no setup-oras release url") + for path, match in urls: + sites += [(path, match["tag"]), (path, match["version"])] + return sites, urls + + def current(self, tree): + sites, _ = self._sites(tree) + versions = {version for _, version in sites} + if len(versions) != 1: + pinned = ", ".join(f"{path}: {version}" for path, version in sorted(set(sites))) + raise TrackError(f"ORAS pins disagree ({pinned}); make them one version first") + return versions.pop() + + def set_version(self, tree, version): + self.parse(version) + _, urls = self._sites(tree) + tree["Justfile"] = self.IMAGE_RE.sub(lambda m: m["head"] + version, tree["Justfile"]) + new = f"https://github.com/oras-project/oras/releases/download/v{version}/oras_{version}_linux_amd64.tar.gz" + for path in sorted({path for path, _ in urls}): + tree[path] = self.URL_RE.sub(new, tree[path]) + + def pins(self, tree): + version = self.current(tree) + _, urls = self._sites(tree) + return [ + Pin(path, url, "checksum", url, self._sums(url, version)) + for path, url in sorted({(path, m[0]) for path, m in urls}) + ] + + +KUBEADM = dict(include="include/kubeadm.yml", element="elements/kubeadm/kubeadm-bin.bst") +COMPONENTS: dict[str, Component] = { + c.name: c + for c in ( + BstComponent("kubernetes", "kubernetes/kubernetes", "{url}.sha256", + variables=("kubernetes-version",), marker="%{kubernetes-version}", + stable_channel="https://dl.k8s.io/release/stable-{series}.txt", **KUBEADM), + BstComponent("cri-tools", "kubernetes-sigs/cri-tools", "{url}.sha256", + variables=("crictl-version",), marker="%{crictl-version}", **KUBEADM), + BstComponent("containerd", "containerd/containerd", "{url}.sha256sum", + variables=("containerd-version",), marker="%{containerd-version}", **KUBEADM), + BstComponent("runc", "opencontainers/runc", "{dir}/runc.sha256sum", + variables=("runc-version",), marker="%{runc-version}", **KUBEADM), + BstComponent("cni-plugins", "containernetworking/plugins", "{url}.sha256", + variables=("cni-plugins-version",), marker="%{cni-plugins-version}", **KUBEADM), + BstComponent("k0s", "k0sproject/k0s", "{dir}/sha256sums.txt", + include="include/k0s.yml", variables=("k0s-k8s-version", "k0s-patch"), + element="elements/k0s/k0s-bin.bst", marker="%{k0s-upstream-tag}", + version_re=r"(\d+\.\d+\.\d+)\+k0s\.(\d+)", version_fmt="{0}+k0s.{1}"), + OrasComponent(), + ) +} + + +def _pins_at(component: Component, tree: Tree, version: str) -> list[Pin]: + scratch = tree.copy() + component.set_version(scratch, version) + return component.pins(scratch) + + +def _github_releases(repo: str) -> list[dict]: + releases = [] + for page in range(1, MAX_RELEASE_PAGES + 1): + batch = json.loads(_get(f"https://api.github.com/repos/{repo}/releases?per_page=100&page={page}")) + releases += batch + if len(batch) < 100: + return releases + raise TrackError(f"{repo}: more than {MAX_RELEASE_PAGES * 100} releases") + + +def _candidates(component: Component, tree: Tree, series: str) -> list[str]: + if component.stable_channel: + url = component.stable_channel.format(series=series) + text = _get(url).decode().strip() + if not re.fullmatch(rf"v{re.escape(series)}\.\d+", text): + raise TrackError(f"{url} says `{text}`, not a {series}.x release") + return [text[1:]] + found = [] + prefix = f"https://github.com/{component.repo}/releases/download/" + for release in _github_releases(component.repo): + tag = release.get("tag_name", "") + if release.get("draft") or release.get("prerelease") or not tag.startswith("v"): + continue + version = tag[1:] + if not re.fullmatch(component.version_re, version) or _series(version) != series: + continue + # The pinned assets must be attached already: a release can be + # published while its assets are still uploading. + names = {asset["name"] for asset in release.get("assets", [])} + wanted = { + urllib.parse.unquote(url.rsplit("/", 1)[1]) + for pin in _pins_at(component, tree, version) + for url in (pin.url, pin.sums) + if url.startswith(prefix) + } + if wanted <= names: + found.append(version) + return found + + +def newest(component: Component, tree: Tree) -> str: + """Newest release in the pinned series, or the current version if none is newer.""" + current = component.current(tree) + return max([current, *_candidates(component, tree, _series(current))], key=_numbers) + + +def published_sha256(text: str, asset: str, url: str) -> str: + """The sha256 a checksum file gives for `asset`. + + Accepts a bare digest (dl.k8s.io, cri-tools) or `digest [*]name` lines + (sha256sum output, also inside runc's PGP-signed file). + """ + named: dict[str, set[str]] = {} + bare = [] + for line in text.splitlines(): + match = SUMS_LINE_RE.match(line.strip()) + if not match: + continue + if match["name"]: + named.setdefault(match["name"].rsplit("/", 1)[-1], set()).add(match["sha"].lower()) + else: + bare.append(match["sha"].lower()) + if named: + digests = named.get(asset, set()) + if len(digests) != 1: + raise TrackError(f"{url} lists {len(digests)} sha256 for {asset}") + return digests.pop() + if len(bare) == 1: + return bare[0] + raise TrackError(f"{url} is not a sha256 checksum file for {asset}") + + +class Change(NamedTuple): + pin: Pin + before: str + after: str + + +@dataclass +class Result: + component: Component + old: str + new: str + changes: list[Change] + files: list[str] + mentions: list[str] + + +def apply(root: Path, name: str, version: str | None = None) -> Result | None: + """Move `name` to `version` (default: newest in series) and rewrite its pins. + + Returns None when no version is given and the pin is already the newest. + """ + component = COMPONENTS[name] + tree = Tree(root) + old = component.current(tree) + new = version or newest(component, tree) + if version is None and new == old: + return None + component.set_version(tree, new) + sums: dict[str, str] = {} + hashed: dict[str, str] = {} + changes = [] + for pin in component.pins(tree): + if pin.sums not in sums: + sums[pin.sums] = _get(pin.sums).decode("utf-8", "replace") + sha = published_sha256(sums[pin.sums], pin.asset, pin.sums) + if pin.url not in hashed: + hashed[pin.url] = _download_sha256(pin.url) + if hashed[pin.url] != sha: + raise TrackError(f"{pin.url} hashes to {hashed[pin.url]}, but {pin.sums} says {sha}") + changes.append(Change(pin, read_pin(tree, pin), sha)) + write_pin(tree, pin, sha) + files = tree.save() + stale = re.compile(rf"(? str: + component, old, new = result.component, result.old, result.new + if _series(old) == _series(new): + head = f"Patch release of **{component.name}** in the pinned `{_series(new)}` series: `{old}` → `{new}`." + else: + head = f"Moves **{component.name}** from `{old}` to `{new}`, a series change." + lines = [head, "", f"Release notes: {component.notes(new)}", "", "| File | Asset | sha256 |", "| --- | --- | --- |"] + for pin, before, after in result.changes: + was = "unchanged" if before == after else f"was `{before}`" + lines.append(f"| `{pin.path}` | [`{pin.asset}`]({pin.url}) | `{after}` ({was}) |") + lines += [ + "", + "Every sha256 comes from the checksum file upstream publishes next to the", + "asset, and matched the sha256 of the downloaded asset:", + "", + *[f"- {url}" for url in dict.fromkeys(change.pin.sums for change in result.changes)], + ] + if result.mentions: + lines += ["", f"These lines of the changed files still mention `{old}`:", ""] + lines += [f"- `{mention}`" for mention in result.mentions] + lines += [ + "", + "Only patch releases inside the pinned series are proposed; a minor bump is a", + f"manual `.github/scripts/track-binaries.py apply {component.name} --version ...`.", + "The tracker regenerates this branch from `main`, so push fixes to a branch", + "of your own.", + ] + return "\n".join(lines) + "\n" + + +def check(root: Path) -> tuple[list[dict], list[str]]: + tree = Tree(root) + rows, errors = [], [] + for name, component in COMPONENTS.items(): + try: + current = component.current(tree) + latest = newest(component, tree) + except TrackError as err: + errors.append(f"{name}: {err}") + continue + rows.append({"component": name, "series": _series(current), "current": current, "latest": latest}) + return rows, errors + + +def main(argv: list[str] | None = None, root: Path = ROOT) -> int: + parser = argparse.ArgumentParser(description="Track patch releases of the pinned upstream binaries.") + commands = parser.add_subparsers(dest="command", required=True) + check_cmd = commands.add_parser("check", help="report the newest release in each pinned series") + check_cmd.add_argument("--json", action="store_true", help="print the pending updates as a JSON list on stdout") + apply_cmd = commands.add_parser("apply", help="move one component and rewrite its sha256 pins") + apply_cmd.add_argument("component", choices=COMPONENTS) + apply_cmd.add_argument("--version", help="target version (default: newest in the pinned series)") + apply_cmd.add_argument("--summary", type=Path, help="write a Markdown pull request body to this file") + args = parser.parse_args(argv) + + if args.command == "check": + rows, errors = check(root) + table = sys.stderr if args.json else sys.stdout + print(f"{'component':<12} {'series':<6} {'current':<14} {'latest':<14} state", file=table) + for row in rows: + state = "update" if row["latest"] != row["current"] else "current" + print(f"{row['component']:<12} {row['series']:<6} {row['current']:<14} {row['latest']:<14} {state}", file=table) + for error in errors: + print(f"ERROR: {error}", file=sys.stderr) + if args.json: + print(json.dumps([row for row in rows if row["latest"] != row["current"]])) + return 1 if errors else 0 + + try: + result = apply(root, args.component, args.version) + except TrackError as err: + print(f"ERROR: {err}", file=sys.stderr) + return 1 + if result is None: + print(f"{args.component}: already the newest release of its series", file=sys.stderr) + return 0 + if args.summary: + args.summary.write_text(summary(result), encoding="utf-8") + print(f"{args.component}: {result.old} -> {result.new}", file=sys.stderr) + for path in result.files: + print(path) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/unit/test_track_binaries.py b/tests/unit/test_track_binaries.py new file mode 100644 index 00000000..c75b6b34 --- /dev/null +++ b/tests/unit/test_track_binaries.py @@ -0,0 +1,446 @@ +"""Behaviour of .github/scripts/track-binaries.py. + +No network: urllib.request.urlopen is replaced by a fake upstream serving the +release listings, checksum files and assets each test registers. The fixture +repository mirrors the layout of the real include/, elements/, Justfile and +workflow files; the last tests read the real files, without network, so a +reformat that the tracker cannot follow fails here instead of in the scheduled +run. +""" + +import hashlib +import importlib.util +import io +import json +import sys +import urllib.error +import urllib.request +from email.message import Message +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[2] +SCRIPT = ROOT / ".github" / "scripts" / "track-binaries.py" + +_spec = importlib.util.spec_from_file_location("track_binaries", SCRIPT) +assert _spec is not None and _spec.loader is not None +track = importlib.util.module_from_spec(_spec) +sys.modules["track_binaries"] = track +_spec.loader.exec_module(track) + + +def sha(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +OLD = {name: sha(name.encode()) for name in ("kubelet", "kubeadm", "kubectl", "crictl", "containerd", "runc", "cni", "k0s", "oras")} + +FILES = { + "include/aliases.yml": "aliases:\n github: https://github.com/\n k8s_dl: https://dl.k8s.io/\n", + "include/kubeadm.yml": """variables: + # kubelet, kubeadm, kubectl + kubernetes-version: "1.34.3" + # crictl + crictl-version: "1.34.0" + # containerd + containerd-version: "2.1.5" + # runc + runc-version: "1.3.3" + # CNI plugins + cni-plugins-version: "1.1.1" + # sandbox + pause-image: "registry.k8s.io/pause:3.10.1" +""", + "elements/kubeadm/kubeadm-bin.bst": f"""kind: manual + +(@): +- include/kubeadm.yml + +sources: +- kind: remote + url: k8s_dl:release/v%{{kubernetes-version}}/bin/linux/amd64/kubelet + ref: {OLD['kubelet']} + directory: k8s +- kind: remote + url: k8s_dl:release/v%{{kubernetes-version}}/bin/linux/amd64/kubeadm + ref: {OLD['kubeadm']} + directory: k8s +- kind: remote + url: k8s_dl:release/v%{{kubernetes-version}}/bin/linux/amd64/kubectl + ref: {OLD['kubectl']} + directory: k8s +- kind: tar + url: github:kubernetes-sigs/cri-tools/releases/download/v%{{crictl-version}}/crictl-v%{{crictl-version}}-linux-amd64.tar.gz + ref: {OLD['crictl']} + base-dir: '' +- kind: tar + url: github:containerd/containerd/releases/download/v%{{containerd-version}}/containerd-static-%{{containerd-version}}-linux-amd64.tar.gz + ref: {OLD['containerd']} + base-dir: bin +- kind: remote + url: github:opencontainers/runc/releases/download/v%{{runc-version}}/runc.amd64 + ref: {OLD['runc']} + directory: runc +- kind: tar + url: github:containernetworking/plugins/releases/download/v%{{cni-plugins-version}}/cni-plugins-linux-amd64-v%{{cni-plugins-version}}.tgz + ref: {OLD['cni']} + base-dir: '' + +config: + install-commands: + - install -m 0755 runc/runc.amd64 "%{{install-root}}%{{bindir}}/runc" +""", + "include/k0s.yml": """variables: + k0s-k8s-version: "1.36.4" + k0s-patch: "0" + + k0s-upstream-tag: "v%{k0s-k8s-version}%2Bk0s.%{k0s-patch}" + k0s-version: "%{k0s-k8s-version}-k0s.%{k0s-patch}" +""", + "elements/k0s/k0s-bin.bst": f"""kind: manual +(@): include/k0s.yml + +sources: + - kind: remote + url: github:k0sproject/k0s/releases/download/%{{k0s-upstream-tag}}/k0s-%{{k0s-upstream-tag}}-amd64 + ref: {OLD['k0s']} + +config: + install-commands: + - install -D -m 0755 k0s-%{{k0s-upstream-tag}}-amd64 "%{{install-root}}/usr/bin/k0s" +""", + "Justfile": 'export oras_image := env("ORAS_IMAGE", "ghcr.io/oras-project/oras:v1.3.4")\n', + ".github/workflows/build.yml": f"""jobs: + release: + steps: + # pin 1.3.4 (matching the Justfile's oras_image) by URL and checksum. + - name: Set up oras + uses: oras-project/setup-oras@1d808f7d7f6995cc68b7bf507bfe5c5446e1dc9d # v2.0.1 + with: + url: https://github.com/oras-project/oras/releases/download/v1.3.4/oras_1.3.4_linux_amd64.tar.gz + checksum: {OLD['oras']} + + - name: Push + run: oras push + release-dry-run: + steps: + - name: Set up oras + uses: oras-project/setup-oras@1d808f7d7f6995cc68b7bf507bfe5c5446e1dc9d # v2.0.1 + with: + url: https://github.com/oras-project/oras/releases/download/v1.3.4/oras_1.3.4_linux_amd64.tar.gz + checksum: {OLD['oras']} +""", +} + + +class Upstream: + """Serves registered URLs; everything else is a 404.""" + + def __init__(self): + self.files: dict[str, bytes] = {} + self.requests: list[urllib.request.Request] = [] + + def urlopen(self, request, timeout=None): + self.requests.append(request) + if request.full_url not in self.files: + raise urllib.error.HTTPError(request.full_url, 404, "Not Found", Message(), None) + return io.BytesIO(self.files[request.full_url]) + + def asset(self, url: str, sums_url: str | None = None, name: str | None = None) -> str: + data = f"payload of {url}".encode() + self.files[url] = data + if sums_url: + self.files[sums_url] = f"{sha(data)} {name or url.rsplit('/', 1)[1]}\n".encode() + return sha(data) + + def releases(self, repo: str, *pages: list[dict]) -> None: + for number, page in enumerate(pages, 1): + url = f"https://api.github.com/repos/{repo}/releases?per_page=100&page={number}" + self.files[url] = json.dumps(page).encode() + + +def release(tag: str, *assets: str, prerelease: bool = False, draft: bool = False) -> dict: + return {"tag_name": tag, "prerelease": prerelease, "draft": draft, "assets": [{"name": a} for a in assets]} + + +def containerd(version: str) -> dict: + name = f"containerd-static-{version}-linux-amd64.tar.gz" + return release(f"v{version}", name, f"{name}.sha256sum") + + +CONTAINERD = "https://github.com/containerd/containerd/releases/download/v2.1.9/containerd-static-2.1.9-linux-amd64.tar.gz" + + +@pytest.fixture +def upstream(monkeypatch): + fake = Upstream() + monkeypatch.setattr(urllib.request, "urlopen", fake.urlopen) + monkeypatch.delenv("GITHUB_TOKEN", raising=False) + monkeypatch.delenv("GH_TOKEN", raising=False) + return fake + + +@pytest.fixture +def repo(tmp_path): + for path, text in FILES.items(): + (tmp_path / path).parent.mkdir(parents=True, exist_ok=True) + (tmp_path / path).write_text(text, encoding="utf-8") + return tmp_path + + +def snapshot(root: Path) -> dict[str, str]: + return {path: (root / path).read_text(encoding="utf-8") for path in FILES} + + +def changed_lines(before: dict[str, str], root: Path) -> dict[str, list[tuple[str, str]]]: + diff = {} + for path, text in before.items(): + pairs = [(a, b) for a, b in zip(text.splitlines(), (root / path).read_text(encoding="utf-8").splitlines()) if a != b] + if pairs: + diff[path] = pairs + return diff + + +def newest(name: str, repo: Path) -> str: + return track.newest(track.COMPONENTS[name], track.Tree(repo)) + + +def test_newest_stays_inside_the_pinned_minor_and_skips_unreleased(repo, upstream): + upstream.releases("containerd/containerd", [ + containerd("2.2.0"), + {**containerd("2.1.11"), "prerelease": True}, + containerd("2.1.10-rc.1"), + {**containerd("2.1.9"), "draft": True}, + containerd("2.1.8"), + containerd("2.1.5"), + containerd("1.7.30"), + release("api/v1.9.0"), + ]) + assert newest("containerd", repo) == "2.1.8" + + +def test_release_still_missing_its_assets_is_not_a_candidate(repo, upstream): + upstream.releases("containerd/containerd", [ + release("v2.1.9", "containerd-static-2.1.9-linux-amd64.tar.gz"), + containerd("2.1.8"), + ]) + assert newest("containerd", repo) == "2.1.8" + + +def test_release_listing_is_paginated(repo, upstream): + upstream.releases("containerd/containerd", [containerd(f"1.7.{n}") for n in range(100)], [containerd("2.1.9")]) + assert newest("containerd", repo) == "2.1.9" + + +def test_kubernetes_follows_the_dl_k8s_io_stable_channel(repo, upstream): + upstream.files["https://dl.k8s.io/release/stable-1.34.txt"] = b"v1.34.12\n" + assert newest("kubernetes", repo) == "1.34.12" + upstream.files["https://dl.k8s.io/release/stable-1.34.txt"] = b"v1.35.0\n" + with pytest.raises(track.TrackError, match="not a 1.34.x release"): + newest("kubernetes", repo) + + +def test_k0s_orders_by_kubernetes_patch_then_k0s_suffix(repo, upstream): + def k0s(tag: str) -> dict: + return release(tag, f"k0s-{tag}-amd64", "sha256sums.txt") + + upstream.releases("k0sproject/k0s", [k0s("v1.37.0+k0s.0"), k0s("v1.36.4+k0s.1"), k0s("v1.36.3+k0s.2"), k0s("v1.36.4+k0s.0")]) + assert newest("k0s", repo) == "1.36.4+k0s.1" + upstream.releases("k0sproject/k0s", [k0s("v1.36.5+k0s.0"), k0s("v1.36.4+k0s.1")]) + assert newest("k0s", repo) == "1.36.5+k0s.0" + + +def test_check_json_lists_pending_updates_and_still_reports_errors(repo, upstream, monkeypatch, capsys): + monkeypatch.setattr(track, "COMPONENTS", {n: track.COMPONENTS[n] for n in ("containerd", "runc", "cni-plugins")}) + upstream.releases("containerd/containerd", [containerd("2.1.9"), containerd("2.1.5")]) + upstream.releases("opencontainers/runc", [release("v1.3.3", "runc.amd64", "runc.sha256sum")]) + assert track.main(["check", "--json"], root=repo) == 1 + out, err = capsys.readouterr() + assert json.loads(out) == [{"component": "containerd", "series": "2.1", "current": "2.1.5", "latest": "2.1.9"}] + assert "runc 1.3 1.3.3 1.3.3 current" in err + assert "ERROR: cni-plugins: GET https://api.github.com/repos/containernetworking/plugins/releases" in err + + +def test_apply_moves_the_version_and_only_its_own_ref(repo, upstream, tmp_path, capsys): + upstream.releases("containerd/containerd", [containerd("2.1.9"), containerd("2.1.5")]) + new = upstream.asset(CONTAINERD, CONTAINERD + ".sha256sum") + before = snapshot(repo) + body = tmp_path / "body.md" + + assert track.main(["apply", "containerd", "--summary", str(body)], root=repo) == 0 + + assert changed_lines(before, repo) == { + "include/kubeadm.yml": [(' containerd-version: "2.1.5"', ' containerd-version: "2.1.9"')], + "elements/kubeadm/kubeadm-bin.bst": [(f" ref: {OLD['containerd']}", f" ref: {new}")], + } + assert capsys.readouterr().out.split() == ["elements/kubeadm/kubeadm-bin.bst", "include/kubeadm.yml"] + text = body.read_text() + assert "Patch release of **containerd** in the pinned `2.1` series: `2.1.5` → `2.1.9`." in text + assert f"`{new}` (was `{OLD['containerd']}`)" in text + assert f"- {CONTAINERD}.sha256sum" in text + assert "https://github.com/containerd/containerd/releases/tag/v2.1.9" in text + + +def test_apply_kubernetes_rewrites_all_three_binaries_from_bare_digests(repo, upstream): + upstream.files["https://dl.k8s.io/release/stable-1.34.txt"] = b"v1.34.12" + new = {} + for binary in ("kubelet", "kubeadm", "kubectl"): + url = f"https://dl.k8s.io/release/v1.34.12/bin/linux/amd64/{binary}" + upstream.files[url] = binary.encode() * 3 + upstream.files[url + ".sha256"] = sha(binary.encode() * 3).encode() + new[binary] = sha(binary.encode() * 3) + before = snapshot(repo) + + assert track.main(["apply", "kubernetes"], root=repo) == 0 + + assert changed_lines(before, repo) == { + "include/kubeadm.yml": [(' kubernetes-version: "1.34.3"', ' kubernetes-version: "1.34.12"')], + "elements/kubeadm/kubeadm-bin.bst": [(f" ref: {OLD[b]}", f" ref: {new[b]}") for b in ("kubelet", "kubeadm", "kubectl")], + } + + +def test_apply_runc_reads_its_pgp_signed_sums_file(repo, upstream): + url = "https://github.com/opencontainers/runc/releases/download/v1.3.6/runc.amd64" + new = upstream.asset(url) + upstream.files[url.rsplit("/", 1)[0] + "/runc.sha256sum"] = ( + "-----BEGIN PGP SIGNED MESSAGE-----\nHash: SHA512\n\n" + f"{sha(b'arm64')} runc.arm64\n{new} runc.amd64\n{sha(b'tar')} runc.tar.xz\n" + "-----BEGIN PGP SIGNATURE-----\n\niQIzBAEBCgAdFiEE\n-----END PGP SIGNATURE-----\n" + ).encode() + + assert track.main(["apply", "runc", "--version", "1.3.6"], root=repo) == 0 + + assert f"ref: {new}" in (repo / "elements/kubeadm/kubeadm-bin.bst").read_text() + + +def test_apply_k0s_moves_both_atoms_and_downloads_the_escaped_tag(repo, upstream): + url = "https://github.com/k0sproject/k0s/releases/download/v1.36.4%2Bk0s.1/k0s-v1.36.4%2Bk0s.1-amd64" + new = upstream.asset(url) + upstream.files[url.rsplit("/", 1)[0] + "/sha256sums.txt"] = ( + f"{sha(b'airgap')} *k0s-airgap-bundle-v1.36.4+k0s.1-linux-amd64.tar\n{new} *k0s-v1.36.4+k0s.1-amd64\n" + ).encode() + before = snapshot(repo) + + assert track.main(["apply", "k0s", "--version", "1.36.4+k0s.1"], root=repo) == 0 + + assert changed_lines(before, repo) == { + "include/k0s.yml": [(' k0s-patch: "0"', ' k0s-patch: "1"')], + "elements/k0s/k0s-bin.bst": [(f" ref: {OLD['k0s']}", f" ref: {new}")], + } + assert url in [r.full_url for r in upstream.requests] + + +def test_apply_oras_moves_the_justfile_tag_and_every_setup_oras_pin(repo, upstream, tmp_path): + url = "https://github.com/oras-project/oras/releases/download/v1.3.5/oras_1.3.5_linux_amd64.tar.gz" + new = upstream.asset(url, url.rsplit("/", 1)[0] + "/oras_1.3.5_checksums.txt") + body = tmp_path / "body.md" + + assert track.main(["apply", "oras", "--version", "1.3.5", "--summary", str(body)], root=repo) == 0 + + assert "ghcr.io/oras-project/oras:v1.3.5" in (repo / "Justfile").read_text() + workflow = (repo / ".github/workflows/build.yml").read_text() + assert workflow.count(f"url: {url}") == 2 + assert workflow.count(f"checksum: {new}") == 2 + assert "1.3.4" in workflow.split("\n")[3] + assert "`.github/workflows/build.yml:4: # pin 1.3.4 (matching the Justfile's oras_image) by URL and checksum.`" in body.read_text() + + +def test_oras_pins_that_disagree_are_refused(repo, upstream): + (repo / "Justfile").write_text('export oras_image := env("ORAS_IMAGE", "ghcr.io/oras-project/oras:v1.3.3")\n') + with pytest.raises(track.TrackError, match=r"ORAS pins disagree \(\.github/workflows/build\.yml: 1\.3\.4, Justfile: 1\.3\.3\)"): + track.apply(repo, "oras", "1.3.5") + + +def test_asset_that_does_not_match_its_published_checksum_writes_nothing(repo, upstream, capsys): + upstream.releases("containerd/containerd", [containerd("2.1.9")]) + upstream.asset(CONTAINERD, CONTAINERD + ".sha256sum") + upstream.files[CONTAINERD] = b"tampered" + before = snapshot(repo) + + assert track.main(["apply", "containerd"], root=repo) == 1 + + assert snapshot(repo) == before + assert f"{CONTAINERD} hashes to {sha(b'tampered')}" in capsys.readouterr().err + + +def test_checksum_file_that_omits_the_asset_is_refused(repo, upstream): + upstream.asset(CONTAINERD) + upstream.files[CONTAINERD + ".sha256sum"] = f"{sha(b'x')} containerd-2.1.9-linux-amd64.tar.gz\n".encode() + with pytest.raises(track.TrackError, match="lists 0 sha256 for containerd-static-2.1.9-linux-amd64.tar.gz"): + track.apply(repo, "containerd", "2.1.9") + + +def test_apply_is_a_no_op_when_the_pin_is_already_newest(repo, upstream, capsys): + upstream.releases("containerd/containerd", [containerd("2.1.5"), containerd("2.1.4")]) + before = snapshot(repo) + assert track.main(["apply", "containerd"], root=repo) == 0 + assert snapshot(repo) == before + assert capsys.readouterr().out == "" + + +def test_manual_minor_bump_uses_the_same_verification(repo, upstream, tmp_path): + for binary in ("kubelet", "kubeadm", "kubectl"): + url = f"https://dl.k8s.io/release/v1.35.2/bin/linux/amd64/{binary}" + upstream.files[url] = binary.encode() + upstream.files[url + ".sha256"] = sha(binary.encode()).encode() + body = tmp_path / "body.md" + + assert track.main(["apply", "kubernetes", "--version", "1.35.2", "--summary", str(body)], root=repo) == 0 + + assert 'kubernetes-version: "1.35.2"' in (repo / "include/kubeadm.yml").read_text() + assert "a series change" in body.read_text() + assert not any("stable-" in r.full_url for r in upstream.requests) + + +def test_malformed_version_is_refused_before_any_download(repo, upstream): + with pytest.raises(track.TrackError, match="does not match"): + track.apply(repo, "k0s", "1.36.5") + assert upstream.requests == [] + + +def test_source_without_its_own_ref_does_not_borrow_the_next_one(repo, upstream): + bst = repo / "elements/kubeadm/kubeadm-bin.bst" + bst.write_text(bst.read_text().replace(f" ref: {OLD['containerd']}\n", "")) + upstream.asset(CONTAINERD, CONTAINERD + ".sha256sum") + with pytest.raises(track.TrackError, match="no `ref:` in the mapping of `url: github:containerd/"): + track.apply(repo, "containerd", "2.1.9") + assert f"ref: {OLD['runc']}" in bst.read_text() + + +def test_github_token_is_sent_to_the_api_and_never_to_downloads(repo, upstream, monkeypatch): + monkeypatch.setenv("GITHUB_TOKEN", "secret-token") + upstream.releases("containerd/containerd", [containerd("2.1.9")]) + upstream.asset(CONTAINERD, CONTAINERD + ".sha256sum") + + track.apply(repo, "containerd") + + auth = {r.full_url.split("?")[0]: r.get_header("Authorization") for r in upstream.requests} + assert auth == { + "https://api.github.com/repos/containerd/containerd/releases": "Bearer secret-token", + CONTAINERD + ".sha256sum": None, + CONTAINERD: None, + } + + +@pytest.mark.parametrize("name", list(track.COMPONENTS)) +def test_real_pins_are_readable(name): + tree = track.Tree(ROOT) + component = track.COMPONENTS[name] + component.parse(component.current(tree)) + pins = component.pins(tree) + assert pins + for pin in pins: + assert pin.url.startswith("https://") and "%{" not in pin.url + assert track.SUMS_LINE_RE.match(track.read_pin(tree, pin)), pin + + +def test_every_pinned_source_belongs_to_exactly_one_component(): + tree = track.Tree(ROOT) + elements = {c.element for c in track.COMPONENTS.values() if isinstance(c, track.BstComponent)} + for element in sorted(elements): + for line in tree[element].splitlines(): + match = track.URL_LINE_RE.match(line) + if match: + owners = [c.name for c in track.COMPONENTS.values() if getattr(c, "marker", None) and c.marker in match["url"]] + assert len(owners) == 1, f"{element}: {match['url']} is tracked by {owners}" From 0c097f9dcc90e8d4f1a4abe7540739307d8e6812 Mon Sep 17 00:00:00 2001 From: Jeffrey Sica Date: Mon, 28 Sep 2026 22:28:31 -0500 Subject: [PATCH 2/5] refactor(kubeadm): pin release series and keep version lines apart The kubeadm payload now follows the release series of the cluster's nodes rather than their exact patch, so security patches can land without a hand-written PR. test_versions_stay_on_the_cluster_nodes_series asserts the series (Kubernetes and cri-tools 1.34, containerd 2.1, runc 1.3, CNI plugins 1.1) instead of one release each; moving a series stays a deliberate edit. Each version in include/kubeadm.yml now sits under its own comment line. The tracker opens one pull request per component, and git reports a conflict when two changes touch adjacent lines, so with the old layout merging the containerd bump would have conflicted the runc one. Assisted-by: Claude Opus 5.5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- include/kubeadm.yml | 22 +++++++++++++--------- tests/unit/test_kubeadm_sysext.py | 24 ++++++++++++++++-------- 2 files changed, 29 insertions(+), 17 deletions(-) diff --git a/include/kubeadm.yml b/include/kubeadm.yml index dd9a6c14..a5478922 100644 --- a/include/kubeadm.yml +++ b/include/kubeadm.yml @@ -1,21 +1,25 @@ # Single source of truth for the kubeadm sysext payload versions. # -# Pinned to what the target kubeadm cluster's existing nodes run, so a -# Bluefin Server worker is indistinguishable from them at the CRI/kubelet -# level. The sha256 of every download sits next to its source in +# Each version stays on the release series the target kubeadm cluster's nodes +# run, so a Bluefin Server worker matches them at the CRI/kubelet level. +# .github/workflows/track-binaries.yml proposes patch releases inside those +# series; moving to another series is manual (docs/skills/kubeadm-sysext.md). +# The sha256 of every download sits next to its source in # elements/kubeadm/kubeadm-bin.bst; the sysext itself is locked to the image # version (kubeadm_.raw.zst). # -# %{kubernetes-version} kubelet, kubeadm, kubectl (dl.k8s.io) -# %{crictl-version} cri-tools -# %{containerd-version} containerd static build -# %{runc-version} runc static build -# %{cni-plugins-version} containernetworking/plugins -# %{pause-image} CRI sandbox image (matches kubeadm-flags.env) +# A comment line separates each version from the next, so the tracker's +# per-component pull requests never edit adjacent lines and merge in any order. variables: + # kubelet, kubeadm, kubectl (dl.k8s.io) kubernetes-version: "1.34.3" + # crictl (cri-tools) crictl-version: "1.34.0" + # containerd static build containerd-version: "2.1.5" + # runc static build runc-version: "1.3.3" + # containernetworking/plugins cni-plugins-version: "1.1.1" + # CRI sandbox image (matches kubeadm-flags.env) pause-image: "registry.k8s.io/pause:3.10.1" diff --git a/tests/unit/test_kubeadm_sysext.py b/tests/unit/test_kubeadm_sysext.py index c92f393d..52effa27 100644 --- a/tests/unit/test_kubeadm_sysext.py +++ b/tests/unit/test_kubeadm_sysext.py @@ -30,15 +30,23 @@ def versions() -> dict[str, str]: return yaml.safe_load(VERSIONS.read_text(encoding="utf-8"))["variables"] -def test_versions_match_the_existing_cluster_nodes() -> None: - assert versions() == { - "kubernetes-version": "1.34.3", - "crictl-version": "1.34.0", - "containerd-version": "2.1.5", - "runc-version": "1.3.3", - "cni-plugins-version": "1.1.1", - "pause-image": "registry.k8s.io/pause:3.10.1", +def test_versions_stay_on_the_cluster_nodes_series() -> None: + # Patch releases arrive through .github/workflows/track-binaries.yml; a + # series change is a deliberate edit here (docs/skills/kubeadm-sysext.md). + pinned = versions() + series = {} + for name, version in pinned.items(): + if name != "pause-image": + assert re.fullmatch(r"\d+\.\d+\.\d+", version), name + series[name] = version.rsplit(".", 1)[0] + assert series == { + "kubernetes-version": "1.34", + "crictl-version": "1.34", + "containerd-version": "2.1", + "runc-version": "1.3", + "cni-plugins-version": "1.1", } + assert pinned["pause-image"] == "registry.k8s.io/pause:3.10.1" def test_every_download_is_pinned_by_sha256() -> None: From bfae6d7eb3e71aba1f66f79f95bd7ce272c0a42d Mon Sep 17 00:00:00 2001 From: Jeffrey Sica Date: Mon, 28 Sep 2026 22:28:31 -0500 Subject: [PATCH 3/5] refactor(k0s): pin the 1.36 series instead of one release test_k0s_version.py asserted k0s 1.36.4+k0s.0 and its sha256, so any k0s patch release failed the unit tests. It now asserts the 1.36 series and a sha256-shaped ref; check-k0s-version.py still guards how the version is derived. include/k0s.yml's derivation table is labelled as an example for v1.36.4+k0s.0, so it stays true after the atoms move. Assisted-by: Claude Opus 5.5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- include/k0s.yml | 5 +++-- tests/unit/test_k0s_version.py | 8 +++++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/include/k0s.yml b/include/k0s.yml index acf0707f..66c981f1 100644 --- a/include/k0s.yml +++ b/include/k0s.yml @@ -5,8 +5,9 @@ # project.conf (`installer-version`) or include/image.yml (`image-version`) — those # axes name the FSDK point release and the OS build and change for unrelated reasons. # -# Only the two atoms below are edited when k0s is bumped. Every other spelling -# of the k0s version in this repository is derived from them: +# Only the two atoms below are edited when k0s is bumped, by hand or by +# .github/workflows/track-binaries.yml. Every other spelling of the k0s version +# in this repository is derived from them; for release v1.36.4+k0s.0: # # %{k0s-k8s-version} upstream Kubernetes version 1.36.4 # %{k0s-patch} upstream k0s patch suffix number 0 diff --git a/tests/unit/test_k0s_version.py b/tests/unit/test_k0s_version.py index ddf83633..99760fac 100644 --- a/tests/unit/test_k0s_version.py +++ b/tests/unit/test_k0s_version.py @@ -11,8 +11,10 @@ def test_k0s_version_ssot(): assert k0s_yml.is_file(), "include/k0s.yml missing" data = yaml.safe_load(k0s_yml.read_text()) vars_ = data.get("variables", {}) - assert vars_.get("k0s-k8s-version") == "1.36.4" - assert vars_.get("k0s-patch") == "0" + # Patch releases arrive through .github/workflows/track-binaries.yml; the + # 1.36 series only moves by hand. + assert re.fullmatch(r"1\.36\.\d+", vars_.get("k0s-k8s-version", "")) + assert re.fullmatch(r"\d+", vars_.get("k0s-patch", "")) assert vars_.get("k0s-version") == "%{k0s-k8s-version}-k0s.%{k0s-patch}" @@ -22,7 +24,7 @@ def test_k0s_bin_element(): content = bin_bst.read_text() assert "base/base-stack.bst" in content assert "github:k0sproject/k0s/releases/download/" in content - assert "ca1e9e68107335846e8296777fce2ccd654284e6265b4b5d32c34ead872af98f" in content + assert re.search(r"^\s+ref: [0-9a-f]{64}$", content, re.MULTILINE) def test_k0s_version_checker_reads_the_component_transfer(): From 9d26664e9703646e55215733fec9de21c17c0351 Mon Sep 17 00:00:00 2001 From: Jeffrey Sica Date: Mon, 28 Sep 2026 22:28:42 -0500 Subject: [PATCH 4/5] ci: track patch releases of upstream binaries Kubernetes, cri-tools, containerd, runc, CNI plugins, k0s and ORAS are pinned by version and sha256 in files Renovate cannot update together, so security patch releases needed hand-written PRs. track-binaries.yml runs daily after track-junctions and on dispatch, never on pull_request. `check` lists the components with a newer patch in their pinned series; `propose` applies each one in its own matrix leg and opens or updates one PR per component on auto/track-binaries/. An open PR with the same title is left alone; a closed or reverted one with the same title is not reopened. GITHUB_TOKEN stays read-only. Pushes and PRs use the mergeraptor app token, narrowed to contents and pull-requests, plus workflows for the ORAS leg because its pins live in build.yml. Assisted-by: Claude Opus 5.5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/track-binaries.yml | 145 +++++++++++++++++++++++++++ docs/skills/ci-tooling.md | 3 +- 2 files changed, 147 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/track-binaries.yml diff --git a/.github/workflows/track-binaries.yml b/.github/workflows/track-binaries.yml new file mode 100644 index 00000000..ce5f4bda --- /dev/null +++ b/.github/workflows/track-binaries.yml @@ -0,0 +1,145 @@ +name: Track upstream binaries + +# Proposes patch releases of the upstream binaries pinned by version and +# sha256 (the kubeadm sysext payload, k0s, ORAS): one pull request per +# component, on its own auto/track-binaries/ branch. What is +# tracked, how each sha256 is verified, and why this is not a Renovate rule: +# .github/scripts/track-binaries.py. +# +# Same shape as track-junctions.yml: schedule and manual dispatch only, never +# pull_request, and pull requests opened with the mergeraptor app token so +# that they get CI. +on: + schedule: + # After track-junctions (08:00 UTC). + - cron: '30 8 * * *' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: track-binaries + cancel-in-progress: false + +jobs: + check: + runs-on: ubuntu-26.04 + timeout-minutes: 10 + outputs: + updates: ${{ steps.check.outputs.updates }} + steps: + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: main + persist-credentials: false + + - name: Find newer patch releases + id: check + env: + # Read-only; only raises the API rate limit for the release listings. + GITHUB_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + # A component whose upstream cannot be read fails this job, but only + # after the other components' updates are recorded: one outage must + # neither pass as "nothing to track" nor hold back the rest. + status=0 + python3 .github/scripts/track-binaries.py check --json > "$RUNNER_TEMP/updates.json" || status=$? + echo "updates=$(cat "$RUNNER_TEMP/updates.json")" >> "$GITHUB_OUTPUT" + exit "$status" + + propose: + needs: check + if: ${{ !cancelled() && needs.check.outputs.updates != '' && needs.check.outputs.updates != '[]' }} + runs-on: ubuntu-26.04 + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.check.outputs.updates) }} + steps: + # GITHUB_TOKEN stays read-only. The push and the pull request use the + # mergeraptor app token (pushes made with GITHUB_TOKEN never trigger CI), + # narrowed to what this job does. + - name: Get mergeraptor token + id: app-token + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3 + with: + app-id: ${{ secrets.MERGERAPTOR_APP_ID }} + private-key: ${{ secrets.MERGERAPTOR_PRIVATE_KEY }} + permission-contents: write + permission-pull-requests: write + # The ORAS pins live in .github/workflows/build.yml, and pushing a + # change to a workflow file needs the Workflows permission. An empty + # value requests nothing. + permission-workflows: ${{ matrix.component == 'oras' && 'write' || '' }} + + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: main + fetch-depth: 0 + token: ${{ steps.app-token.outputs.token }} + + - name: Apply the release and verify its checksums + env: + COMPONENT: ${{ matrix.component }} + VERSION: ${{ matrix.latest }} + run: | + set -euo pipefail + python3 .github/scripts/track-binaries.py apply "$COMPONENT" --version "$VERSION" \ + --summary "$RUNNER_TEMP/pr-body.md" > "$RUNNER_TEMP/files.txt" + + - name: Create or update the pull request + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + COMPONENT: ${{ matrix.component }} + FROM: ${{ matrix.current }} + TO: ${{ matrix.latest }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + set -euo pipefail + branch="auto/track-binaries/${COMPONENT}" + title="chore(deps): update ${COMPONENT} ${FROM} -> ${TO}" + if [ ! -s "$RUNNER_TEMP/files.txt" ]; then + echo "${COMPONENT} needs no change." + exit 0 + fi + + # One pull request per component. An open one with this exact title + # already proposes this release, so leave it and its CI run alone. A + # closed one with this title was turned down (or reverted); do not + # reopen it every day, the next release brings a new title. No + # pipes into grep and no `|| true`: a failed query must stop here, + # not pass for "no pull request". + open=$(gh pr list --head "$branch" --base main --state open --json number,title --jq '.[0] | select(.) | "\(.number)\t\(.title)"') + number="${open%%$'\t'*}" + if [ -n "$open" ] && [ "${open#*$'\t'}" = "$title" ]; then + echo "#${number} already proposes ${COMPONENT} ${TO}." + exit 0 + fi + closed=$(gh pr list --head "$branch" --base main --state closed --json title --jq '.[].title') + if [ -z "$open" ] && grep -Fxq -- "$title" <<< "$closed"; then + echo "A closed pull request already proposed '${title}'; not reopening it." + exit 0 + fi + + # No attribution trailers, as in track-junctions.yml: this commit is + # the deterministic output of track-binaries.py. + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git checkout -B "$branch" + mapfile -t files < "$RUNNER_TEMP/files.txt" + git add -- "${files[@]}" + git commit -m "$title" + git push --force-with-lease origin "$branch" + + printf '\n---\n*Generated by [Track upstream binaries](%s)*\n' "$RUN_URL" >> "$RUNNER_TEMP/pr-body.md" + if [ -n "$open" ]; then + gh pr edit "$number" --title "$title" --body-file "$RUNNER_TEMP/pr-body.md" + echo "Updated PR: $(gh pr view "$number" --json url --jq '.url')" + else + echo "Created PR: $(gh pr create --base main --head "$branch" --title "$title" --body-file "$RUNNER_TEMP/pr-body.md")" + fi diff --git a/docs/skills/ci-tooling.md b/docs/skills/ci-tooling.md index 4eb28c2e..b4c6d063 100644 --- a/docs/skills/ci-tooling.md +++ b/docs/skills/ci-tooling.md @@ -4,7 +4,7 @@ description: CI workflow conventions for Bluefin Server. Use when writing or edi metadata: type: reference status: stable - last_updated: "2026-09-27" + last_updated: "2026-09-28" context7-sources: - /websites/github_en_actions - /websites/cli_github_manual @@ -121,6 +121,7 @@ sudo_cmd := if `podman info >/dev/null 2>&1 && echo 1 || echo 0` == "1" { "" } e | `release` | `build.yml` | `push/main`, `workflow_dispatch` | Publishes `dist/diskless/` as-is: an immutable GitHub Release tagged `v` plus an ORAS OCI artifact at `ghcr.io//bluefin-server:,latest` (one layer per file, artifact type `application/vnd.projectbluefin.server.release.v1`) (`if: ${{ !failure() && !cancelled() && github.ref == 'refs/heads/main' }}`). `contents: write` + `packages: write`. | | `docs` | `docs-checks.yml` | `pull_request`, `push/main` | Runs markdown and skill metadata checks via `docs-checks.py`. Read-only token. | | `unit` | `unit-tests.yml` | `pull_request`, `push/main` | Runs pytest and BATS unit test suites. Read-only token. | +| `check`, `propose` | `track-binaries.yml` | `schedule` (08:30 UTC), `workflow_dispatch` | `check` finds the newest patch release in each pinned series of the upstream binaries pinned by version + sha256 (Kubernetes, cri-tools, containerd, runc, CNI plugins, k0s, ORAS) with `.github/scripts/track-binaries.py`; `propose` moves each version together with its sha256 pins, verified against upstream's checksum files and the downloaded assets, and opens or updates one PR per component on `auto/track-binaries/`. Minor bumps stay manual (`kubeadm-sysext.md`, `k0s-sysext.md`). Read-only `GITHUB_TOKEN`; writes use the mergeraptor app token narrowed to `contents` + `pull-requests` (+ `workflows` for ORAS, pinned in `build.yml`). Never on `pull_request`. | GitHub Actions runs the **complete BuildStream compilation pipeline** using `/mnt` SSD storage on the runner for podman and BuildStream caches. Release assets are From 29045d2df830c631b47af45bea7640358802fab8 Mon Sep 17 00:00:00 2001 From: Jeffrey Sica Date: Mon, 28 Sep 2026 22:28:42 -0500 Subject: [PATCH 5/5] docs: document Kubernetes version skew and manual minor bumps kubeadm-sysext.md: the sysext runs the Kubernetes minor of the cluster it joins, independently of k0s's bundled Kubernetes; kubeadm join must match the control plane's minor and kubelet must not be newer than the API server. Patches arrive from track-binaries.yml; a minor bump is track-binaries.py apply for kubernetes and cri-tools, the pause image, containerd's support table and the series in the unit test. k0s-sysext.md: where the k0s patch and minor bumps now come from. Assisted-by: Claude Opus 5.5 via GitHub Copilot Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/skills/k0s-sysext.md | 3 ++- docs/skills/kubeadm-sysext.md | 28 +++++++++++++++++++++++++++- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/docs/skills/k0s-sysext.md b/docs/skills/k0s-sysext.md index 2893f30a..60a15ea5 100644 --- a/docs/skills/k0s-sysext.md +++ b/docs/skills/k0s-sysext.md @@ -4,7 +4,7 @@ description: Build and ship the k0s systemd-sysext for Bluefin Server. KubeStell metadata: type: how-to status: stable - last_updated: "2026-09-27" + last_updated: "2026-09-28" context7-sources: - /systemd/systemd --- @@ -85,6 +85,7 @@ Design choices: | `Justfile` | `build-sysext` / `export-sysext` targets. | | `.github/workflows/build.yml` | Builds, signs, and publishes sysext assets. | | `.github/scripts/check-k0s-version.py` | Fails closed if any consumer restates the k0s version instead of deriving it. | +| `.github/scripts/track-binaries.py` | Moves both atoms and the SHA256 together: `track-binaries.yml` proposes patch releases of the pinned minor, `apply k0s --version X.Y.Z+k0s.N` makes a minor bump by hand (then move the series in `tests/unit/test_k0s_version.py`). | ## Build Outputs diff --git a/docs/skills/kubeadm-sysext.md b/docs/skills/kubeadm-sysext.md index 4a70f2a4..af9615f3 100644 --- a/docs/skills/kubeadm-sysext.md +++ b/docs/skills/kubeadm-sysext.md @@ -4,7 +4,7 @@ description: Build, ship and operate the opt-in kubeadm worker systemd-sysext (k metadata: type: how-to status: stable - last_updated: "2026-09-27" + last_updated: "2026-09-28" --- # kubeadm worker sysext @@ -43,6 +43,32 @@ Nothing ships under `/opt`: `/opt/cni/bin` stays a writable host directory for the cluster CNI (Cilium's `cilium-cni`). containerd searches `/opt/cni/bin` then `/usr/libexec/cni`. +## Versions + +The sysext runs the Kubernetes minor of the cluster it joins: the series of +`kubernetes-version` in `include/kubeadm.yml`. It does not follow k0s, which +bundles its own Kubernetes for a separate cluster +([k0s-sysext.md](k0s-sysext.md)); the two minors move independently. +`kubeadm join` must use the minor that last ran `kubeadm init` or +`kubeadm upgrade` on that cluster, and kubelet may be older than the API server +but never newer. Patch releases change neither constraint. + +- **Patches** are automatic. `.github/workflows/track-binaries.yml` opens one + pull request per component (Kubernetes, cri-tools, containerd, runc, CNI + plugins) when a newer patch of its pinned series is out, with the version + and its sha256 refs changed together and checked against the checksum files + upstream publishes. +- **Minors** are manual, once the cluster's control plane runs the new minor: + 1. `python3 .github/scripts/track-binaries.py apply kubernetes --version X.Y.Z`, + and the same for `cri-tools` (its minor follows Kubernetes). `apply` + verifies the checksums exactly as the tracker does. + 2. Set `pause-image` to the tag that + `kubeadm config images list --kubernetes-version vX.Y.Z` prints. + 3. Check containerd's `RELEASES.md` support table for the new minor; move + `containerd` and `runc` with `apply` if it asks for newer ones. + 4. Move the series in `tests/unit/test_kubeadm_sysext.py`, then boot-verify + as in [Verify](#verify). + ## Runtime contract - No preset enables anything; `80-kubeadm.preset` disables both units against