From 722d1aee735dfc42d6b692372544537a5c5e544d Mon Sep 17 00:00:00 2001 From: Zach Vorhies Date: Wed, 28 Oct 2026 22:03:18 -0700 Subject: [PATCH 1/2] ci(glibc): gate PRs on the manylinux_2_17 symbol floor (#1629) The manylinux_2_17 tag is assigned from a filename map in ci/publish.py, and `cargo check` never links, so a too-new glibc symbol survives every PR gate and only fails in the release cross-link lane (libc::memfd_create, 2.27+). Add ci/check_glibc_floor.py, which reads a linked ELF's dynamic symbol table and fails on any GLIBC_ version above the floor, naming the symbols that need it. Wire it into check-ubuntu.yml as a new glibc-floor job that cross-links release fbuild-cli + fbuild-daemon for x86_64 and aarch64 under soldr's glibc-2.17 sysroot, with its own build-cache generation. Tests cover numeric version ordering, per-version symbol attribution, the at-floor/above-floor boundary, and the CLI's 0/1/2 exit codes. --- .github/workflows/check-ubuntu.yml | 57 +++++++++ agents/docs/cross-compilation.md | 14 +++ ci/check_glibc_floor.py | 187 +++++++++++++++++++++++++++++ ci/test_check_glibc_floor.py | 133 ++++++++++++++++++++ 4 files changed, 391 insertions(+) create mode 100644 ci/check_glibc_floor.py create mode 100644 ci/test_check_glibc_floor.py diff --git a/.github/workflows/check-ubuntu.yml b/.github/workflows/check-ubuntu.yml index 0c59dc642..ee9b3c2bb 100644 --- a/.github/workflows/check-ubuntu.yml +++ b/.github/workflows/check-ubuntu.yml @@ -112,6 +112,63 @@ jobs: - name: Test large library archive run: soldr cargo test -p fbuild-library large_archive_handles_windows_command_limit_and_quoted_paths -- --ignored --nocapture + glibc-floor: + # FastLED/fbuild#1629. The manylinux_2_17 tag is assigned from a filename + # map in ci/publish.py, not measured, and `cargo check` never links -- so a + # too-new glibc symbol (libc::memfd_create, glibc 2.27+, was the real case) + # survives every PR gate and only fails in the release cross-link lane. + # Cross-link the two shipped binaries for both Linux arches under soldr's + # glibc-2.17 sysroot, then read the linked symbol tables. No new workflow + # file: this is another job in the existing Ubuntu lane. + name: glibc 2.17 floor (linux x86_64 + aarch64) + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - name: Checkout source + uses: actions/checkout@v6 + with: + fetch-depth: 0 + ref: ${{ inputs.ref }} + - name: Verify candidate SHA + if: inputs.ref != '' + env: + CANDIDATE_SHA: ${{ inputs.ref }} + run: | + [[ "$CANDIDATE_SHA" =~ ^[0-9a-f]{40}$ ]] + test "$(git rev-parse HEAD)" = "$CANDIDATE_SHA" + - name: Set up uv + uses: astral-sh/setup-uv@v3 + with: + python-version: "3.12" + # Own build-cache store: this job cross-links two release targets and + # must not compete with (or poison) the test job's cache generation. + - name: Setup soldr + uses: zackees/setup-soldr@6bf04d4196ee26fe9d51550187013a7f32955e20 # setup-soldr#582 exact-main promotion canary + with: + cache-key-suffix: glibc-floor + job-status: ${{ job.status }} + cache: true + build-cache: true + target-cache: true + prebuild-deps: none + linker: platform-default + - name: Self-test the floor guard + run: uv run --no-project python -m unittest ci.test_check_glibc_floor + - name: Cross-link release binaries + timeout-minutes: 35 + run: | + for target in x86_64-unknown-linux-gnu aarch64-unknown-linux-gnu; do + soldr prepare --target "$target" + soldr build --release --target "$target" -p fbuild-cli -p fbuild-daemon + done + - name: Enforce the manylinux_2_17 floor + run: | + uv run --no-project python ci/check_glibc_floor.py --floor 2.17 \ + target/x86_64-unknown-linux-gnu/release/fbuild \ + target/x86_64-unknown-linux-gnu/release/fbuild-daemon \ + target/aarch64-unknown-linux-gnu/release/fbuild \ + target/aarch64-unknown-linux-gnu/release/fbuild-daemon + python-facade-tests: # FastLED/fbuild#1485 §5.2/§6: the embedded-CPython AT-P integration # tests need libpython at link and run time, so they're `#[ignore]`d diff --git a/agents/docs/cross-compilation.md b/agents/docs/cross-compilation.md index 927ca72e7..ce2dcd570 100644 --- a/agents/docs/cross-compilation.md +++ b/agents/docs/cross-compilation.md @@ -94,6 +94,20 @@ objdump -T .so | grep -o 'GLIBC_[0-9.]*' | sort -Vu | tail -4 python3 -c "d=open('','rb').read(8); print(d[:4].hex(), hex(int.from_bytes(d[4:8],'little')))" ``` +The same check is a PR gate now (`ci/check_glibc_floor.py`, job `glibc-floor` +in `check-ubuntu.yml`): it cross-links `fbuild-cli` + `fbuild-daemon` for +`x86_64-unknown-linux-gnu` and `aarch64-unknown-linux-gnu` under soldr's +glibc-2.17 sysroot and fails on any `GLIBC_` symbol above the floor, naming the +symbols that need it. Run it locally against any linked binary: + +```bash +uv run --no-project python ci/check_glibc_floor.py --floor 2.17 +``` + +A *host-native* debug binary is not a valid input: it links against the host +glibc (2.39+ on recent distros) and will always fail. The guard is meaningful +only on artifacts built through soldr's catalogue sysroot. + The published wheel is downloadable, so the shipped floor is always checkable after the fact: diff --git a/ci/check_glibc_floor.py b/ci/check_glibc_floor.py new file mode 100644 index 000000000..ccbf02c9a --- /dev/null +++ b/ci/check_glibc_floor.py @@ -0,0 +1,187 @@ +"""Enforce the manylinux glibc floor on linked Linux binaries (#1629). + +`ci/publish.py` assigns the `manylinux_2_17` wheel tag from a filename map, not +from the artifact, and `cargo check` never links -- so a wheel can promise +glibc >= 2.17 while needing a symbol from 2.27. That is exactly how +`libc::memfd_create` (glibc 2.27+) slipped past every PR gate and only failed in +the release cross-link lane. + +The floor is a property of the *dynamic symbol table*, so this guard reads an +already-linked binary with `objdump -T` and fails when any `GLIBC_x.y` version +requirement is newer than the floor. It never builds anything; CI links the +release binaries first, then runs this over each artifact. + +Usage:: + + uv run --no-project python ci/check_glibc_floor.py BINARY [BINARY ...] + uv run --no-project python ci/check_glibc_floor.py --floor 2.17 BINARY + +Exit codes: 0 = at or below the floor, 1 = above the floor, 2 = the artifact +could not be inspected (missing file, no toolchain, not a glibc binary). +""" + +from __future__ import annotations + +import argparse +import re +import subprocess +import sys +from collections import defaultdict +from dataclasses import dataclass +from pathlib import Path + +DEFAULT_FLOOR = "2.17" +DEFAULT_OBJDUMP = "objdump" + +_VERSION_RE = re.compile(r"^(\d+)\.(\d+)$") +# `objdump -T` prints the requirement in the version column: +# 0000000000000000 DF *UND* 0000000000000000 GLIBC_2.27 memfd_create +_SYMBOL_RE = re.compile(r"\bGLIBC_(\d+)\.(\d+)(?:\.(\d+))?\b") + + +@dataclass(frozen=True, order=True) +class Version: + """A glibc symbol-version number, comparable field by field.""" + + major: int + minor: int + patch: int + + @classmethod + def parse(cls, text: str) -> Version: + match = _VERSION_RE.match(text) + if match is None: + raise ValueError(f"expected MAJOR.MINOR, got {text!r}") + return cls(int(match.group(1)), int(match.group(2)), 0) + + @classmethod + def from_match(cls, match: re.Match[str]) -> Version: + return cls(int(match.group(1)), int(match.group(2)), int(match.group(3) or 0)) + + def __str__(self) -> str: + base = f"{self.major}.{self.minor}" + return base if self.patch == 0 else f"{base}.{self.patch}" + + +@dataclass(frozen=True) +class Finding: + """One binary that needs a glibc symbol newer than the floor.""" + + binary: Path + version: Version + symbols: tuple[str, ...] + + +def parse_requirements(symbol_table: str) -> dict[Version, set[str]]: + """Map each required `GLIBC_x.y` version to the symbols that need it.""" + requirements: dict[Version, set[str]] = defaultdict(set) + for line in symbol_table.splitlines(): + match = _SYMBOL_RE.search(line) + if match is None: + continue + fields = line.split() + symbol = fields[-1] if fields else "" + requirements[Version.from_match(match)].add(symbol) + return dict(requirements) + + +def max_version(requirements: dict[Version, set[str]]) -> Version | None: + return max(requirements) if requirements else None + + +def violations( + binary: Path, requirements: dict[Version, set[str]], floor: Version +) -> list[Finding]: + """Return one finding per too-new version, oldest first.""" + return [ + Finding(binary, version, tuple(sorted(requirements[version]))) + for version in sorted(requirements) + if version > floor + ] + + +def read_symbol_table(binary: Path, objdump: str) -> str: + result = subprocess.run( + [objdump, "-T", str(binary)], + capture_output=True, + text=True, + check=False, + ) + if result.returncode != 0: + raise RuntimeError( + f"{objdump} -T {binary} failed ({result.returncode}): " + f"{result.stderr.strip()}" + ) + return result.stdout + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--floor", + default=DEFAULT_FLOOR, + help=f"highest permitted glibc symbol version (default {DEFAULT_FLOOR})", + ) + parser.add_argument( + "--objdump", + default=DEFAULT_OBJDUMP, + help=f"objdump executable to use (default {DEFAULT_OBJDUMP})", + ) + parser.add_argument("binaries", nargs="+", help="linked ELF binaries to inspect") + args = parser.parse_args(argv) + + try: + floor = Version.parse(str(args.floor)) + except ValueError as error: + parser.error(str(error)) + return 2 # unreachable; parser.error exits + + findings: list[Finding] = [] + failures: list[str] = [] + for name in args.binaries: + binary = Path(str(name)) + if not binary.is_file(): + failures.append(f"{binary}: no such file") + continue + try: + table = read_symbol_table(binary, str(args.objdump)) + except (OSError, RuntimeError) as error: + failures.append(f"{error}") + continue + requirements = parse_requirements(table) + newest = max_version(requirements) + if newest is None: + failures.append(f"{binary}: no GLIBC_ symbol versions found") + continue + status = "OK" if newest <= floor else "TOO NEW" + print( + f"glibc floor {floor}: {binary} (max GLIBC_{newest}) {status}", flush=True + ) + findings.extend(violations(binary, requirements, floor)) + + for finding in findings: + print( + f"error: {finding.binary} requires GLIBC_{finding.version} (floor {floor})", + file=sys.stderr, + ) + print( + f" GLIBC_{finding.version}: {', '.join(finding.symbols)}", file=sys.stderr + ) + for failure in failures: + print(f"error: {failure}", file=sys.stderr) + + if findings: + print( + "The manylinux_2_17 promise is broken by the symbols above; call the " + "raw syscall (libc::syscall(libc::SYS_...)) or keep a fallback for " + "glibc older than the floor.", + file=sys.stderr, + ) + return 1 + if failures: + return 2 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/ci/test_check_glibc_floor.py b/ci/test_check_glibc_floor.py new file mode 100644 index 000000000..cff9de8a2 --- /dev/null +++ b/ci/test_check_glibc_floor.py @@ -0,0 +1,133 @@ +"""Tests for ci/check_glibc_floor.py (FastLED/fbuild#1629).""" + +from __future__ import annotations + +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from check_glibc_floor import ( # noqa: E402 + Version, + max_version, + parse_requirements, + violations, +) + +SCRIPT = Path(__file__).resolve().parent / "check_glibc_floor.py" + +SYMBOL_TABLE = """\ +0000000000000000 DF *UND*\t0000000000000000 GLIBC_2.2.5 __libc_start_main +0000000000000000 DF *UND*\t0000000000000000 GLIBC_2.2.5 fopen64 +0000000000000000 DF *UND*\t0000000000000000 GLIBC_2.16 memfd_create +0000000000000000 DF *UND*\t0000000000000000 GLIBC_2.17 clock_gettime +""" + +TOO_NEW_TABLE = SYMBOL_TABLE + ( + "0000000000000000 DF *UND*\t0000000000000000 GLIBC_2.27 memfd_create\n" + "0000000000000000 DF *UND*\t0000000000000000 GLIBC_2.28 statx\n" +) + + +class VersionTests(unittest.TestCase): + def test_patch_defaults_to_zero(self) -> None: + self.assertEqual(Version(2, 17, 0), Version.parse("2.17")) + + def test_numeric_not_lexical_ordering(self) -> None: + self.assertLess(Version.parse("2.2"), Version.parse("2.9")) + self.assertLess(Version.parse("2.16"), Version.parse("2.17")) + + def test_rejects_malformed_floor(self) -> None: + with self.assertRaises(ValueError): + Version.parse("2") + + +class ParseTests(unittest.TestCase): + def test_collects_symbols_per_version(self) -> None: + requirements = parse_requirements(SYMBOL_TABLE) + self.assertEqual(3, len(requirements)) + self.assertEqual( + {"__libc_start_main", "fopen64"}, requirements[Version(2, 2, 5)] + ) + self.assertEqual({"memfd_create"}, requirements[Version(2, 16, 0)]) + + def test_empty_table_has_no_max(self) -> None: + self.assertEqual({}, parse_requirements("")) + self.assertIsNone(max_version(parse_requirements(""))) + + +class ViolationTests(unittest.TestCase): + def test_at_or_below_floor_is_clean(self) -> None: + binary = Path("fbuild") + requirements = parse_requirements(SYMBOL_TABLE) + self.assertEqual([], violations(binary, requirements, Version(2, 17, 0))) + self.assertEqual([], violations(binary, requirements, Version(2, 40, 0))) + + def test_above_floor_reports_symbols(self) -> None: + findings = violations( + Path("fbuild"), parse_requirements(TOO_NEW_TABLE), Version(2, 17, 0) + ) + self.assertEqual( + [Version(2, 27, 0), Version(2, 28, 0)], [f.version for f in findings] + ) + self.assertEqual(("memfd_create",), findings[0].symbols) + self.assertEqual(("statx",), findings[1].symbols) + + +class CommandLineTests(unittest.TestCase): + def run_guard( + self, directory: Path, *args: str + ) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, str(SCRIPT), *args], + capture_output=True, + text=True, + check=False, + cwd=directory, + ) + + def fake_objdump(self, directory: Path, table: str) -> Path: + table_path = directory / "table.txt" + table_path.write_text(table, encoding="utf-8") + tool = directory / "objdump" + tool.write_text(f'#!/bin/sh\ncat "{table_path}"\n', encoding="utf-8") + tool.chmod(0o755) + return tool + + def test_passes_on_green_binary(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + directory = Path(tmp) + tool = self.fake_objdump(directory, SYMBOL_TABLE) + (directory / "fbuild").write_text("", encoding="utf-8") + result = self.run_guard( + directory, "--objdump", str(tool), "--floor", "2.17", "fbuild" + ) + self.assertEqual(0, result.returncode, result.stderr) + self.assertIn("max GLIBC_2.17", result.stdout) + + def test_fails_on_too_new_binary(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + directory = Path(tmp) + tool = self.fake_objdump(directory, TOO_NEW_TABLE) + (directory / "fbuild").write_text("", encoding="utf-8") + result = self.run_guard( + directory, "--objdump", str(tool), "--floor", "2.17", "fbuild" + ) + self.assertEqual(1, result.returncode, result.stderr) + self.assertIn("requires GLIBC_2.28 (floor 2.17)", result.stderr) + self.assertIn("statx", result.stderr) + + def test_missing_binary_is_inspection_error(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + directory = Path(tmp) + tool = self.fake_objdump(directory, SYMBOL_TABLE) + result = self.run_guard(directory, "--objdump", str(tool), "absent") + self.assertEqual(2, result.returncode, result.stderr) + self.assertIn("no such file", result.stderr) + + +if __name__ == "__main__": + unittest.main() From 01cb22b643e853e679b78e8db51ff1e1c4593971 Mon Sep 17 00:00:00 2001 From: Zach Vorhies Date: Wed, 28 Oct 2026 22:21:27 -0700 Subject: [PATCH 2/2] ci(glibc): give the cold-cache floor job headroom --- .github/workflows/check-ubuntu.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/check-ubuntu.yml b/.github/workflows/check-ubuntu.yml index ee9b3c2bb..8fe1e1028 100644 --- a/.github/workflows/check-ubuntu.yml +++ b/.github/workflows/check-ubuntu.yml @@ -122,7 +122,10 @@ jobs: # file: this is another job in the existing Ubuntu lane. name: glibc 2.17 floor (linux x86_64 + aarch64) runs-on: ubuntu-latest - timeout-minutes: 45 + # Cold cache: two release cross-links of fbuild-cli + fbuild-daemon. + # Measured ~16 min for the daemon alone on a loaded 16-core host with an + # empty cache; 60 leaves room for the first (cache-miss) run. + timeout-minutes: 60 steps: - name: Checkout source uses: actions/checkout@v6 @@ -155,7 +158,7 @@ jobs: - name: Self-test the floor guard run: uv run --no-project python -m unittest ci.test_check_glibc_floor - name: Cross-link release binaries - timeout-minutes: 35 + timeout-minutes: 45 run: | for target in x86_64-unknown-linux-gnu aarch64-unknown-linux-gnu; do soldr prepare --target "$target"