diff --git a/tests/test_hummingbird_gaps.py b/tests/test_hummingbird_gaps.py index ace38ad5e..b5e8b9c14 100644 --- a/tests/test_hummingbird_gaps.py +++ b/tests/test_hummingbird_gaps.py @@ -6,7 +6,9 @@ rules carry the meaning and were unpinned: ``lines`` must drop comments and blanks so a commented-out package is not counted as shipped, and ``main`` must include every manifest section except ``excluded`` so a version-specific list -is still part of the contract. +is still part of the contract. A third rule joins them: the runtime contract's +``[unavailable]`` packages are policy exclusions, so they must leave the +contract instead of being reported as parity Hummingbird still owes. These tests build manifests and package lists on disk; nothing pulls an image. """ @@ -50,17 +52,20 @@ class MainTests(unittest.TestCase): def setUp(self): self.root = Path(tempfile.mkdtemp(prefix="gaps-main-")) - def run_main(self, manifest: str, image: str, repo: str, extra=()): + def run_main(self, manifest: str, image: str, repo: str, extra=(), policy="[unavailable]\npackages = []\n"): manifest_path = self.root / "manifest.toml" + policy_path = self.root / "policy.toml" image_path = self.root / "image.txt" repo_path = self.root / "repo.txt" output = self.root / "reports" / "hummingbird-gap.json" manifest_path.write_text(manifest) + policy_path.write_text(policy) image_path.write_text(image) repo_path.write_text(repo) argv = [ "recalculate_hummingbird_gaps.py", "--manifest", str(manifest_path), + "--policy", str(policy_path), "--image-packages", str(image_path), "--repo-packages", str(repo_path), "--output", str(output), @@ -82,7 +87,8 @@ def test_partitions_contract_across_image_repo_and_missing(self): self.assertEqual(report["available_from_repo_only"], ["grub2"]) self.assertEqual(report["missing_from_hummingbird"], ["mozjs140"]) self.assertEqual(report["counts"], - {"contract": 3, "image": 1, "repo_only": 1, "missing": 1}) + {"contract": 3, "excluded_as_unavailable": 0, + "image": 1, "repo_only": 1, "missing": 1}) def test_image_wins_over_repo_so_repo_only_excludes_it(self): _, report = self.run_main( @@ -135,7 +141,8 @@ def test_image_packages_outside_the_contract_are_ignored(self): "fish\nsomething-else\n", "another-thing\n", ) - self.assertEqual(report["counts"], {"contract": 1, "image": 1, "repo_only": 0, "missing": 0}) + self.assertEqual(report["counts"], {"contract": 1, "excluded_as_unavailable": 0, + "image": 1, "repo_only": 0, "missing": 0}) def test_output_lists_are_sorted_and_image_ref_is_recorded(self): _, report = self.run_main( @@ -147,6 +154,75 @@ def test_output_lists_are_sorted_and_image_ref_is_recorded(self): self.assertEqual(report["missing_from_hummingbird"], ["fish", "grub2", "zsh"]) self.assertEqual(report["image"], "ghcr.io/example/hummingbird:latest") + def test_unavailable_packages_leave_the_contract(self): + # The runtime contract's [unavailable] list is debt Utah decided not to + # carry. Counting it as missing would report parity Hummingbird is not + # expected to close — see issue #112 for firefox specifically. + _, report = self.run_main( + "[base]\npackages = ['fish', 'firefox', 'grub2']\n", + "", + "", + policy="[unavailable]\npackages = ['firefox']\n", + ) + self.assertEqual(report["contract_binary_packages"], ["fish", "grub2"]) + self.assertNotIn("firefox", report["missing_from_hummingbird"]) + self.assertEqual(report["counts"]["contract"], 2) + self.assertEqual(report["counts"]["missing"], 2) + + def test_dropped_packages_are_reported_so_the_exclusion_is_auditable(self): + _, report = self.run_main( + "[base]\npackages = ['fish', 'firefox', 'zsh']\n", + "", + "", + policy="[unavailable]\npackages = ['zsh', 'firefox']\n", + ) + self.assertEqual(report["excluded_as_unavailable"], ["firefox", "zsh"]) + self.assertEqual(report["counts"]["excluded_as_unavailable"], 2) + + def test_unavailable_entry_outside_the_contract_is_not_reported_as_dropped(self): + # Only names the manifest actually requested can be dropped from it, so + # an exception for a package this manifest never named stays invisible + # rather than inflating the audit list. + _, report = self.run_main( + "[base]\npackages = ['fish']\n", + "", + "", + policy="[unavailable]\npackages = ['not-in-this-manifest']\n", + ) + self.assertEqual(report["excluded_as_unavailable"], []) + self.assertEqual(report["contract_binary_packages"], ["fish"]) + + def test_unavailable_package_is_dropped_even_when_hummingbird_ships_it(self): + # Availability does not revive an exception: the package is excluded by + # policy, not by absence, so it must not reappear as satisfied parity. + _, report = self.run_main( + "[base]\npackages = ['fish', 'firefox']\n", + "firefox\n", + "", + policy="[unavailable]\npackages = ['firefox']\n", + ) + self.assertEqual(report["available_from_image"], []) + self.assertEqual(report["counts"]["image"], 0) + + def test_policy_without_an_unavailable_section_drops_nothing(self): + _, report = self.run_main( + "[base]\npackages = ['fish']\n", + "", + "", + policy="[base]\nimage = 'example'\n", + ) + self.assertEqual(report["excluded_as_unavailable"], []) + self.assertEqual(report["contract_binary_packages"], ["fish"]) + + def test_malformed_unavailable_section_is_rejected(self): + with self.assertRaises(ValueError): + self.run_main( + "[base]\npackages = ['fish']\n", + "", + "", + policy="[unavailable]\npackages = 'firefox'\n", + ) + def test_measured_at_is_utc_iso8601(self): _, report = self.run_main("[base]\npackages = ['fish']\n", "", "") stamp = datetime.fromisoformat(report["measured_at"]) @@ -157,11 +233,14 @@ def test_output_parent_directory_is_created(self): output = self.root / "deep" / "nested" / "gap.json" manifest = self.root / "m.toml" manifest.write_text("[base]\npackages = ['fish']\n") + policy = self.root / "p.toml" + policy.write_text("[unavailable]\npackages = []\n") empty = self.root / "empty.txt" empty.write_text("") argv = [ "recalculate_hummingbird_gaps.py", "--manifest", str(manifest), + "--policy", str(policy), "--image-packages", str(empty), "--repo-packages", str(empty), "--output", str(output), @@ -191,6 +270,31 @@ def test_default_manifest_yields_a_non_empty_contract(self): excluded = set(manifest.get("excluded", {}).get("packages", [])) self.assertFalse(contract & excluded, "excluded packages leaked into the contract") + def test_real_policy_removes_firefox_from_the_real_contract(self): + # End-to-end pin for issue #112: Utah ships Firefox as the + # org.mozilla.firefox Flatpak, so the gap report must not carry it as + # parity Hummingbird still owes. + root = Path(__file__).resolve().parent.parent + directory = Path(tempfile.mkdtemp(prefix="gaps-real-")) + empty = directory / "empty.txt" + empty.write_text("") + output = directory / "gap.json" + argv = [ + "recalculate_hummingbird_gaps.py", + "--manifest", str(root / "config" / "bluefin-packages.toml"), + "--policy", str(root / "config" / "runtime-contract.toml"), + "--image-packages", str(empty), + "--repo-packages", str(empty), + "--output", str(output), + "--image", "ghcr.io/example/hummingbird:latest", + ] + with mock.patch("sys.argv", argv): + self.assertEqual(gaps.main(), 0) + report = json.loads(output.read_text()) + self.assertNotIn("firefox", report["contract_binary_packages"]) + self.assertNotIn("firefox", report["missing_from_hummingbird"]) + self.assertIn("firefox", report["excluded_as_unavailable"]) + if __name__ == "__main__": unittest.main() diff --git a/tools/recalculate_hummingbird_gaps.py b/tools/recalculate_hummingbird_gaps.py index 53b7b77e5..d222857aa 100644 --- a/tools/recalculate_hummingbird_gaps.py +++ b/tools/recalculate_hummingbird_gaps.py @@ -5,10 +5,15 @@ import argparse import json +import sys import tomllib from datetime import UTC, datetime from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +from tools.runtime_contract import load_policy, package_section + def lines(path: Path) -> set[str]: return {line.strip() for line in path.read_text().splitlines() if line.strip() and not line.startswith("#")} @@ -17,6 +22,7 @@ def lines(path: Path) -> set[str]: def main() -> int: parser = argparse.ArgumentParser() parser.add_argument("--manifest", type=Path, default=Path("config/bluefin-packages.toml")) + parser.add_argument("--policy", type=Path, default=Path("config/runtime-contract.toml")) parser.add_argument("--image-packages", type=Path, required=True) parser.add_argument("--repo-packages", type=Path, required=True) parser.add_argument("--output", type=Path, default=Path("reports/hummingbird-gap.json")) @@ -30,17 +36,24 @@ def main() -> int: for section, values in manifest.items(): if section != "excluded" and isinstance(values, dict): contract.update(values.get("packages", [])) + # Packages the runtime contract declares unavailable are debt Utah has + # already decided not to carry, each with an issue-backed reason. Counting + # them as missing would report parity Hummingbird is not expected to close. + dropped = set(package_section(load_policy(args.policy), "unavailable")) & contract + contract -= dropped image, repo = lines(args.image_packages), lines(args.repo_packages) available = image | repo report = { "measured_at": datetime.now(UTC).isoformat(), "image": args.image, "contract_binary_packages": sorted(contract), + "excluded_as_unavailable": sorted(dropped), "available_from_image": sorted(contract & image), "available_from_repo_only": sorted((contract & repo) - image), "missing_from_hummingbird": sorted(contract - available), "counts": { "contract": len(contract), + "excluded_as_unavailable": len(dropped), "image": len(contract & image), "repo_only": len((contract & repo) - image), "missing": len(contract - available),