diff --git a/e2e/ci_bootstrap_suite.sh b/e2e/ci_bootstrap_suite.sh index e2cc21a5..bc7527a5 100755 --- a/e2e/ci_bootstrap_suite.sh +++ b/e2e/ci_bootstrap_suite.sh @@ -26,6 +26,7 @@ run_test "bootstrap_prerelease" run_test "bootstrap_cache" run_test "bootstrap_sdist_only" run_test "bootstrap_multiple_versions" +run_test "bootstrap_max_release_age" test_section "bootstrap test-mode tests" run_test "mode_resolution" diff --git a/e2e/test_bootstrap_max_release_age.sh b/e2e/test_bootstrap_max_release_age.sh new file mode 100755 index 00000000..93da5b2b --- /dev/null +++ b/e2e/test_bootstrap_max_release_age.sh @@ -0,0 +1,98 @@ +#!/bin/bash +# -*- indent-tabs-mode: nil; tab-width: 2; sh-indentation: 2; -*- + +# Test bootstrap with --max-release-age flag +# Tests that old versions are filtered out by the max release age window +# and that the filter also applies to build dependencies + +SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +source "$SCRIPTDIR/common.sh" + +# certifi PyPI upload timeline (actual upload_time from PyPI JSON API): +# +# certifi 2025.11.12 2025-11-12 (should be filtered — too old) +# certifi 2026.1.4 2026-01-04 (should be filtered — too old) +# certifi 2026.2.25 2026-02-25 (should be included — recent enough) +# certifi 2026.4.22 2026-04-22 (should be included — recent enough) +# +# Compute --max-release-age so certifi 2026.2.25 is inside the window +# but certifi 2026.1.4 is outside. We anchor on certifi 2026.2.25's +# upload date and add a buffer. +MAX_AGE=$(python3 -c " +from datetime import date +# Age of certifi 2026.2.25 (uploaded 2026-02-25) + 10 day buffer +age = (date.today() - date(2026, 2, 25)).days + 10 +print(age) +") + +echo "Using --max-release-age=$MAX_AGE" + +fromager \ + --log-file="$OUTDIR/bootstrap.log" \ + --error-log-file="$OUTDIR/fromager-errors.log" \ + --sdists-repo="$OUTDIR/sdists-repo" \ + --wheels-repo="$OUTDIR/wheels-repo" \ + --work-dir="$OUTDIR/work-dir" \ + bootstrap \ + --multiple-versions \ + --max-release-age="$MAX_AGE" \ + 'certifi>=2025.11,<=2026.5' + +# Verify that recent versions were built (within age window) +echo "" +echo "Checking for expected versions..." +for version in 2026.2.25 2026.4.22; do + if find "$OUTDIR/wheels-repo/downloads/" -name "certifi-$version-*.whl" | grep -q .; then + echo "✓ Found wheel for certifi $version (within max-release-age window)" + else + echo "✗ Missing wheel for certifi $version" + echo "ERROR: certifi $version should be within the max-release-age window" + echo "" + echo "Found wheels:" + find "$OUTDIR/wheels-repo/downloads/" -name 'certifi-*.whl' + exit 1 + fi +done + +# Verify that old versions were filtered out +echo "" +echo "Checking that old versions were filtered..." +UNEXPECTED="" +for version in 2025.11.12 2026.1.4; do + if find "$OUTDIR/wheels-repo/downloads/" -name "certifi-$version-*.whl" | grep -q .; then + echo "✗ Found wheel for certifi $version — should have been filtered by max-release-age" + UNEXPECTED="$UNEXPECTED $version" + else + echo "✓ certifi $version correctly filtered out by max-release-age" + fi +done + +if [ -n "$UNEXPECTED" ]; then + echo "" + echo "ERROR: --max-release-age should have excluded:$UNEXPECTED" + exit 1 +fi + +# Verify that max-release-age filtering was applied (check log) +echo "" +echo "Checking log for max-release-age filtering..." +if grep -q "published within.*days" "$OUTDIR/bootstrap.log"; then + echo "✓ Log confirms max-release-age filtering was applied" +else + echo "✗ No max-release-age filtering found in log" + exit 1 +fi + +# Verify that build dependencies were also resolved within the window +# setuptools is the build dependency for certifi +echo "" +echo "Checking that build dependencies were resolved..." +if find "$OUTDIR/wheels-repo/downloads/" -name "setuptools-*.whl" | grep -q .; then + echo "✓ setuptools was built (build dependency of certifi)" +else + echo "✗ setuptools was not built — build dependency resolution may have failed" + exit 1 +fi + +echo "" +echo "SUCCESS: --max-release-age correctly filtered old versions and resolved build dependencies" diff --git a/e2e/test_bootstrap_multiple_versions.sh b/e2e/test_bootstrap_multiple_versions.sh index 7588adb6..53c3794d 100755 --- a/e2e/test_bootstrap_multiple_versions.sh +++ b/e2e/test_bootstrap_multiple_versions.sh @@ -16,6 +16,14 @@ cat > "$constraints_file" <=3.9,<3.12 EOF +# Compute --max-release-age dynamically: days since tomli 2.0.0 was uploaded +# to PyPI (2021-12-13) plus a buffer, so the oldest version is always included. +MAX_AGE=$(python3 -c " +from datetime import date +age = (date.today() - date(2021, 12, 13)).days +print(age + 30) +") + # Use tomli with a version range that matches exactly 3 versions (2.0.0, 2.0.1, 2.0.2) # tomli has no runtime dependencies, making it fast to bootstrap # It uses flit-core as build backend, and we allow multiple flit-core versions @@ -31,6 +39,7 @@ fromager \ --constraints-file="$constraints_file" \ bootstrap \ --multiple-versions \ + --max-release-age="$MAX_AGE" \ 'tomli>=2.0,<=2.0.2' # Check that wheels were built diff --git a/src/fromager/bootstrap_requirement_resolver.py b/src/fromager/bootstrap_requirement_resolver.py index 9e1fefa3..4f0752e7 100644 --- a/src/fromager/bootstrap_requirement_resolver.py +++ b/src/fromager/bootstrap_requirement_resolver.py @@ -173,7 +173,10 @@ def _resolve( sdist_server_url=resolver.PYPI_SERVER_URL, req_type=req_type, ) - return resolver.find_all_matching_from_provider(provider, req) + max_age_cutoff = resolver._compute_max_age_cutoff(self.ctx) + return resolver.find_all_matching_from_provider( + provider, req, max_age_cutoff=max_age_cutoff + ) def get_cached_resolution( self, diff --git a/src/fromager/commands/bootstrap.py b/src/fromager/commands/bootstrap.py index a6268e73..ab8abb02 100644 --- a/src/fromager/commands/bootstrap.py +++ b/src/fromager/commands/bootstrap.py @@ -110,6 +110,13 @@ def _get_requirements_from_args( default=False, help="Bootstrap all matching versions instead of only the highest version", ) +@click.option( + "--max-release-age", + "max_release_age", + type=click.IntRange(min=0), + default=0, + help="Reject package versions published more than this many days ago (0 disables the check). Required with --multiple-versions.", +) @click.argument("toplevel", nargs=-1) @click.pass_obj def bootstrap( @@ -121,6 +128,7 @@ def bootstrap( skip_constraints: bool, test_mode: bool, multiple_versions: bool, + max_release_age: int, toplevel: list[str], ) -> None: """Compute and build the dependencies of a set of requirements recursively @@ -156,6 +164,10 @@ def bootstrap( ) if multiple_versions: + if max_release_age <= 0: + raise click.UsageError( + "--max-release-age is required when using --multiple-versions" + ) logger.info( "multiple versions mode enabled: will bootstrap all matching versions" ) @@ -168,6 +180,13 @@ def bootstrap( ) skip_constraints = True + if max_release_age > 0: + wkctx.set_max_release_age(max_release_age) + logger.info( + "max release age: rejecting versions older than %d days", + max_release_age, + ) + pre_built = wkctx.settings.list_pre_built() if pre_built: logger.info("treating %s as pre-built wheels", sorted(pre_built)) @@ -492,6 +511,13 @@ def write_constraints_file( default=False, help="Bootstrap all matching versions instead of only the highest version", ) +@click.option( + "--max-release-age", + "max_release_age", + type=click.IntRange(min=0), + default=0, + help="Reject package versions published more than this many days ago (0 disables the check). Required with --multiple-versions.", +) @click.argument("toplevel", nargs=-1) @click.pass_obj @click.pass_context @@ -506,6 +532,7 @@ def bootstrap_parallel( force: bool, max_workers: int | None, multiple_versions: bool, + max_release_age: int, toplevel: list[str], ) -> None: """Bootstrap and build-parallel @@ -533,6 +560,7 @@ def bootstrap_parallel( sdist_only=True, skip_constraints=skip_constraints, multiple_versions=multiple_versions, + max_release_age=max_release_age, toplevel=toplevel, ) diff --git a/src/fromager/context.py b/src/fromager/context.py index 18cb8b63..f1efb3bf 100644 --- a/src/fromager/context.py +++ b/src/fromager/context.py @@ -63,6 +63,7 @@ def __init__( settings_dir: pathlib.Path | None = None, wheel_server_url: str = "", cooldown: Cooldown | None = None, + max_release_age: datetime.timedelta | None = None, ): if active_settings is None: active_settings = packagesettings.Settings( @@ -113,6 +114,15 @@ def __init__( self._parallel_builds = False self.cooldown: Cooldown | None = cooldown + self._max_release_age: datetime.timedelta | None = max_release_age + + @property + def max_release_age(self) -> datetime.timedelta | None: + return self._max_release_age + + def set_max_release_age(self, days: int) -> None: + """Set the maximum release age in days.""" + self._max_release_age = datetime.timedelta(days=days) def enable_parallel_builds(self) -> None: self._parallel_builds = True diff --git a/src/fromager/resolver.py b/src/fromager/resolver.py index a192fde3..adaef98f 100644 --- a/src/fromager/resolver.py +++ b/src/fromager/resolver.py @@ -105,7 +105,10 @@ def resolve( ignore_platform=ignore_platform, ) provider.cooldown = resolve_package_cooldown(ctx, req) - results = find_all_matching_from_provider(provider, req) + max_age_cutoff = _compute_max_age_cutoff(ctx) + results = find_all_matching_from_provider( + provider, req, max_age_cutoff=max_age_cutoff + ) return results[0] @@ -167,6 +170,24 @@ def resolve_package_cooldown( ) +def _compute_max_age_cutoff( + ctx: context.WorkContext, +) -> datetime.datetime | None: + """Compute the cutoff time for max release age filtering. + + Returns the oldest acceptable upload time, or None if disabled. + Uses the cooldown's bootstrap_time for consistency across a single run. + """ + if ctx.max_release_age is None: + return None + bootstrap_time = ( + ctx.cooldown.bootstrap_time + if ctx.cooldown is not None + else datetime.datetime.now(datetime.UTC) + ) + return bootstrap_time - ctx.max_release_age + + def extract_filename_from_url(url: str) -> str: """Extract filename from URL and decode it.""" path = urlparse(url).path @@ -203,13 +224,20 @@ def ending(self, state: typing.Any) -> None: def find_all_matching_from_provider( - provider: BaseProvider, req: Requirement + provider: BaseProvider, + req: Requirement, + max_age_cutoff: datetime.datetime | None = None, ) -> list[tuple[str, Version]]: """Find all matching candidates from provider without full dependency resolution. This function collects ALL candidates that match the requirement, rather than performing full dependency resolution to find a single best candidate. + Args: + provider: The provider to query for candidates. + req: The requirement to match. + max_age_cutoff: If set, reject candidates published before this time. + Returns list of (url, version) tuples sorted by version (highest first). IMPORTANT: This bypasses resolvelib's full resolver to collect all matching @@ -242,10 +270,45 @@ def find_all_matching_from_provider( f"Unable to resolve requirement specifier {req} with constraint {constraint} using {provider_desc}: {original_msg}" ) from err + # Materialize candidates so we can iterate more than once if filtering + candidates_list = list(candidates) + logger.info( + "%s: found %d candidate(s) matching %s", + req.name, + len(candidates_list), + req, + ) + + if max_age_cutoff is not None: + max_age_days = (datetime.datetime.now(datetime.UTC) - max_age_cutoff).days + filtered = [ + c + for c in candidates_list + if c.upload_time is None or c.upload_time >= max_age_cutoff + ] + if filtered: + logger.info( + "%s: have %d candidate(s) of %s published within %d days", + req.name, + len(filtered), + req, + max_age_days, + ) + candidates_list = filtered + else: + logger.warning( + "%s: all %d candidate(s) of %s are older than %d days, " + "keeping all to avoid empty resolution", + req.name, + len(candidates_list), + req, + max_age_days, + ) + # Convert candidates to list of (url, version) tuples # Candidates are sorted by version (highest first) by BaseProvider.find_matches() # which calls sorted(candidates, key=attrgetter("version", "build_tag"), reverse=True) - return [(candidate.url, candidate.version) for candidate in candidates] + return [(c.url, c.version) for c in candidates_list] def get_project_from_pypi( diff --git a/src/fromager/sources.py b/src/fromager/sources.py index e2a75b9b..f5f60963 100644 --- a/src/fromager/sources.py +++ b/src/fromager/sources.py @@ -181,7 +181,10 @@ def resolve_source( ) # Get all matching candidates from provider - results = resolver.find_all_matching_from_provider(provider, req) + max_age_cutoff = resolver._compute_max_age_cutoff(ctx) + results = resolver.find_all_matching_from_provider( + provider, req, max_age_cutoff=max_age_cutoff + ) # Return highest version (first in sorted list) url, version = results[0] diff --git a/tests/test_bootstrap.py b/tests/test_bootstrap.py index fa990ec4..23b77366 100644 --- a/tests/test_bootstrap.py +++ b/tests/test_bootstrap.py @@ -2,6 +2,7 @@ import logging import pathlib import textwrap +from datetime import timedelta from unittest.mock import Mock, patch import pytest @@ -576,6 +577,8 @@ def test_multiple_versions_auto_disables_constraints( "-r", "req.txt", "--multiple-versions", + "--max-release-age", + "45", ], obj=tmp_context, ) @@ -623,6 +626,8 @@ def test_multiple_versions_with_skip_constraints_no_duplicate_log( "req.txt", "--multiple-versions", "--skip-constraints", + "--max-release-age", + "45", ], obj=tmp_context, ) @@ -679,6 +684,67 @@ def test_without_multiple_versions_constraints_not_disabled( assert mock_write_constraints.called +def test_multiple_versions_requires_max_release_age( + tmp_context: context.WorkContext, +) -> None: + """Test that --multiple-versions without --max-release-age fails.""" + runner = CliRunner() + with runner.isolated_filesystem(): + pathlib.Path("req.txt").write_text("setuptools>=60\n") + result = runner.invoke( + bootstrap.bootstrap, + [ + "-r", + "req.txt", + "--multiple-versions", + ], + obj=tmp_context, + ) + assert result.exit_code == 2 + assert "--max-release-age is required" in result.output + + +@patch("fromager.commands.bootstrap.bootstrapper.Bootstrapper") +@patch("fromager.commands.bootstrap.server.start_wheel_server") +@patch("fromager.commands.bootstrap.progress.progress_context") +@patch("fromager.commands.bootstrap.metrics.summarize") +def test_max_release_age_sets_context( + mock_metrics: Mock, + mock_progress: Mock, + mock_server: Mock, + mock_bootstrapper: Mock, + tmp_context: context.WorkContext, + caplog: pytest.LogCaptureFixture, +) -> None: + """Test that --max-release-age stores the value on WorkContext.""" + mock_progress.return_value.__enter__.return_value = Mock() + mock_progress.return_value.__exit__.return_value = None + mock_bt_instance = Mock() + mock_bt_instance.resolve_and_add_top_level.return_value = ("url", Version("1.0")) + mock_bt_instance.finalize.return_value = 0 + mock_bootstrapper.return_value = mock_bt_instance + + runner = CliRunner() + with runner.isolated_filesystem(): + pathlib.Path("req.txt").write_text("setuptools>=60\n") + with caplog.at_level(logging.INFO): + result = runner.invoke( + bootstrap.bootstrap, + [ + "-r", + "req.txt", + "--multiple-versions", + "--max-release-age", + "45", + ], + obj=tmp_context, + ) + + assert result.exit_code == 0 + assert tmp_context.max_release_age == timedelta(days=45) + assert "rejecting versions older than 45 days" in caplog.text + + @patch("fromager.gitutils.git_clone") def test_resolve_version_from_git_url_with_submodules_enabled( mock_git_clone: Mock, diff --git a/tests/test_cooldown.py b/tests/test_cooldown.py index dc35a1e3..f0ff32f8 100644 --- a/tests/test_cooldown.py +++ b/tests/test_cooldown.py @@ -673,3 +673,190 @@ def test_local_wheel_server_allows_without_upload_time( _, version = results[0] assert str(version) == "1.3.2" assert "cooldown check skipped" in caplog.text + + +# --------------------------------------------------------------------------- +# max-release-age tests +# --------------------------------------------------------------------------- + +# Uses the same _cooldown_json_response fixture: +# 2.0.0 uploaded 2026-03-24 → 2 days old +# 1.3.2 uploaded 2026-03-15 → 11 days old +# 1.2.2 uploaded 2026-01-01 → 84 days old +# _BOOTSTRAP_TIME = 2026-03-26 + +# max_age_cutoff = bootstrap_time - max_release_age +# With 30 days: cutoff = 2026-02-24 → keeps 2.0.0 and 1.3.2, filters 1.2.2 +# With 5 days: cutoff = 2026-03-21 → keeps only 2.0.0 + + +def test_max_release_age_filters_old_versions( + caplog: pytest.LogCaptureFixture, +) -> None: + """Versions older than max-release-age are filtered out.""" + max_age_cutoff = _BOOTSTRAP_TIME - datetime.timedelta(days=30) + with requests_mock.Mocker() as r: + r.get( + "https://pypi.org/simple/test-pkg/", + json=_cooldown_json_response, + headers={"Content-Type": _PYPI_SIMPLE_JSON_CONTENT_TYPE}, + ) + provider = resolver.PyPIProvider(include_sdists=True) + with caplog.at_level(logging.INFO, logger="fromager.resolver"): + results = resolver.find_all_matching_from_provider( + provider, Requirement("test-pkg"), max_age_cutoff=max_age_cutoff + ) + + versions = [str(v) for _, v in results] + assert "2.0.0" in versions + assert "1.3.2" in versions + assert "1.2.2" not in versions + assert "found 3 candidate(s)" in caplog.text + assert "have 2 candidate(s)" in caplog.text + assert "published within" in caplog.text + + +def test_max_release_age_keeps_only_recent( + caplog: pytest.LogCaptureFixture, +) -> None: + """With a tight max-release-age, only very recent versions survive.""" + max_age_cutoff = _BOOTSTRAP_TIME - datetime.timedelta(days=5) + with requests_mock.Mocker() as r: + r.get( + "https://pypi.org/simple/test-pkg/", + json=_cooldown_json_response, + headers={"Content-Type": _PYPI_SIMPLE_JSON_CONTENT_TYPE}, + ) + provider = resolver.PyPIProvider(include_sdists=True) + with caplog.at_level(logging.INFO, logger="fromager.resolver"): + results = resolver.find_all_matching_from_provider( + provider, Requirement("test-pkg"), max_age_cutoff=max_age_cutoff + ) + + versions = [str(v) for _, v in results] + assert versions == ["2.0.0"] + assert "found 3 candidate(s)" in caplog.text + assert "have 1 candidate(s)" in caplog.text + assert "published within" in caplog.text + + +def test_max_release_age_disabled_returns_all() -> None: + """When max_age_cutoff is None, all versions are returned.""" + with requests_mock.Mocker() as r: + r.get( + "https://pypi.org/simple/test-pkg/", + json=_cooldown_json_response, + headers={"Content-Type": _PYPI_SIMPLE_JSON_CONTENT_TYPE}, + ) + provider = resolver.PyPIProvider(include_sdists=True) + results = resolver.find_all_matching_from_provider( + provider, Requirement("test-pkg"), max_age_cutoff=None + ) + + versions = [str(v) for _, v in results] + assert "2.0.0" in versions + assert "1.3.2" in versions + assert "1.2.2" in versions + + +def test_max_release_age_all_too_old_keeps_all( + caplog: pytest.LogCaptureFixture, +) -> None: + """When all versions are older than cutoff, keep all to avoid empty resolution.""" + max_age_cutoff = _BOOTSTRAP_TIME + datetime.timedelta(days=1) + with requests_mock.Mocker() as r: + r.get( + "https://pypi.org/simple/test-pkg/", + json=_cooldown_json_response, + headers={"Content-Type": _PYPI_SIMPLE_JSON_CONTENT_TYPE}, + ) + provider = resolver.PyPIProvider(include_sdists=True) + with caplog.at_level(logging.WARNING, logger="fromager.resolver"): + results = resolver.find_all_matching_from_provider( + provider, Requirement("test-pkg"), max_age_cutoff=max_age_cutoff + ) + + versions = [str(v) for _, v in results] + assert len(versions) == 3 + assert "keeping all to avoid empty resolution" in caplog.text + + +def test_max_release_age_candidates_without_upload_time_pass_through() -> None: + """Candidates without upload_time are not filtered out by max-release-age.""" + no_timestamp_response = { + "meta": {"api-version": "1.1"}, + "name": "test-pkg", + "files": [ + { + "filename": "test_pkg-1.0.0-py3-none-any.whl", + "url": "https://files.pythonhosted.org/packages/test_pkg-1.0.0-py3-none-any.whl", + "hashes": {"sha256": "aaa"}, + }, + ], + } + max_age_cutoff = _BOOTSTRAP_TIME - datetime.timedelta(days=5) + with requests_mock.Mocker() as r: + r.get( + "https://pypi.org/simple/test-pkg/", + json=no_timestamp_response, + headers={"Content-Type": _PYPI_SIMPLE_JSON_CONTENT_TYPE}, + ) + provider = resolver.PyPIProvider(include_sdists=True) + results = resolver.find_all_matching_from_provider( + provider, Requirement("test-pkg"), max_age_cutoff=max_age_cutoff + ) + + assert len(results) == 1 + assert str(results[0][1]) == "1.0.0" + + +def test_max_release_age_combined_with_cooldown() -> None: + """Both cooldown (min-release-age) and max-release-age work together as a window.""" + max_age_cutoff = _BOOTSTRAP_TIME - datetime.timedelta(days=30) + with requests_mock.Mocker() as r: + r.get( + "https://pypi.org/simple/test-pkg/", + json=_cooldown_json_response, + headers={"Content-Type": _PYPI_SIMPLE_JSON_CONTENT_TYPE}, + ) + # Cooldown blocks 2.0.0 (too new), max-release-age blocks 1.2.2 (too old) + provider = resolver.PyPIProvider(include_sdists=True, cooldown=_COOLDOWN) + results = resolver.find_all_matching_from_provider( + provider, Requirement("test-pkg"), max_age_cutoff=max_age_cutoff + ) + + versions = [str(v) for _, v in results] + assert versions == ["1.3.2"] + + +def test_compute_max_age_cutoff_with_cooldown( + tmp_context: context.WorkContext, +) -> None: + """_compute_max_age_cutoff uses cooldown's bootstrap_time when available.""" + tmp_context.cooldown = Cooldown( + min_age=datetime.timedelta(days=7), + bootstrap_time=_BOOTSTRAP_TIME, + ) + tmp_context.set_max_release_age(30) + cutoff = resolver._compute_max_age_cutoff(tmp_context) + assert cutoff == _BOOTSTRAP_TIME - datetime.timedelta(days=30) + + +def test_compute_max_age_cutoff_without_cooldown( + tmp_context: context.WorkContext, +) -> None: + """_compute_max_age_cutoff uses current time when no cooldown is set.""" + tmp_context.cooldown = None + tmp_context.set_max_release_age(30) + cutoff = resolver._compute_max_age_cutoff(tmp_context) + assert cutoff is not None + expected = datetime.datetime.now(datetime.UTC) - datetime.timedelta(days=30) + assert abs((cutoff - expected).total_seconds()) < 2 + + +def test_compute_max_age_cutoff_disabled( + tmp_context: context.WorkContext, +) -> None: + """_compute_max_age_cutoff returns None when max_release_age is not set.""" + cutoff = resolver._compute_max_age_cutoff(tmp_context) + assert cutoff is None