test(e2e): add bootstrap --test-mode failure tests

e2e/ci_bootstrap_suite.sh

@@ -27,6 +27,12 @@ run_test "bootstrap_cache"
 run_test "bootstrap_sdist_only"
 run_test "bootstrap_multiple_versions"
 
+test_section "bootstrap test-mode tests"
+run_test "mode_resolution"
+run_test "mode_deps"
+run_test "mode_build"
+run_test "mode_fallback"
+
 test_section "bootstrap git URL tests"
 run_test "bootstrap_git_url"
 run_test "bootstrap_git_url_tag"

e2e/test_build_failure/pyproject.toml

@@ -0,0 +1,8 @@
+[build-system]
+requires = ["setuptools"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "test_build_failure"
+version = "1.0.0"
+description = "Test fixture that intentionally fails to build"

e2e/test_build_failure/setup.py

@@ -0,0 +1,23 @@
+# mypy: ignore-errors
+"""Setup script that intentionally fails during wheel build.
+
+This fixture is designed to pass metadata extraction but fail during
+actual wheel building, producing a 'bootstrap' failure in test-mode.
+The failure is triggered by a custom build_ext command that always fails.
+"""
+
+from setuptools import Extension, setup
+from setuptools.command.build_ext import build_ext
+
+
+class FailingBuildExt(build_ext):
+    """Custom build_ext that always fails."""
+
+    def run(self) -> None:
+        raise RuntimeError("Intentional build failure for e2e testing")
+
+
+setup(
+    ext_modules=[Extension("test_build_failure._dummy", sources=["missing.c"])],
+    cmdclass={"build_ext": FailingBuildExt},
+)

e2e/test_build_failure/test_build_failure/__init__.py

@@ -0,0 +1 @@
+"""Test fixture package for e2e build failure tests."""

e2e/test_mode_build.sh

@@ -0,0 +1,72 @@
+#!/bin/bash
+# -*- indent-tabs-mode: nil; tab-width: 2; sh-indentation: 2; -*-
+
+# Test --test-mode with a package that fails to build (no prebuilt fallback)
+# Uses a local fixture that fails during wheel build; since it's not on PyPI,
+# prebuilt fallback also fails and the failure is recorded.
+# See: https://github.com/python-wheel-build/fromager/issues/895
+
+SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+source "$SCRIPTDIR/common.sh"
+pass=true
+
+DIST="test_build_failure"
+FIXTURE_DIR="$SCRIPTDIR/test_build_failure"
+
+# Initialize the fixture as a git repo (files are committed without .git)
+created_git=false
+if [ ! -d "$FIXTURE_DIR/.git" ]; then
+  created_git=true
+  (cd "$FIXTURE_DIR" && \
+   git init -q && \
+   git config user.email "test@example.com" && \
+   git config user.name "Test User" && \
+   git add -A && \
+   git commit -q -m "init")
+fi
+
+# Clean up .git on exit if we created it
+trap '[ "$created_git" = true ] && rm -rf "$FIXTURE_DIR/.git"' EXIT
+
+echo "$DIST @ git+file://${FIXTURE_DIR}" > "$OUTDIR/requirements.txt"
+
+set +e
+fromager \
+  --log-file="$OUTDIR/bootstrap.log" \
+  --sdists-repo="$OUTDIR/sdists-repo" \
+  --wheels-repo="$OUTDIR/wheels-repo" \
+  --work-dir="$OUTDIR/work-dir" \
+  bootstrap --test-mode -r "$OUTDIR/requirements.txt"
+exit_code=$?
+set -e
+
+if [ "$exit_code" -ne 1 ]; then
+  echo "FAIL: expected exit code 1, got $exit_code" 1>&2
+  pass=false
+fi
+
+failures_file=$(find "$OUTDIR/work-dir" -name "test-mode-failures-*.json" 2>/dev/null | head -1)
+
+if [ -z "$failures_file" ]; then
+  echo "FAIL: no test-mode-failures JSON file found" 1>&2
+  pass=false
+else
+  if ! jq -e ".failures[] | select(.package == \"$DIST\")" "$failures_file" >/dev/null 2>&1; then
+    echo "FAIL: $DIST not found in failures" 1>&2
+    pass=false
+  fi
+
+  # Must be 'bootstrap' failure (actual build failure, not resolution)
+  failure_type=$(jq -r "[.failures[] | select(.package == \"$DIST\")][0].failure_type" "$failures_file")
+  if [ "$failure_type" != "bootstrap" ]; then
+    echo "FAIL: expected failure_type 'bootstrap', got '$failure_type'" 1>&2
+    pass=false
+  fi
+fi
+
+if ! grep -q "test mode enabled" "$OUTDIR/bootstrap.log"; then
+  echo "FAIL: 'test mode enabled' not in log" 1>&2
+  pass=false
+fi
+
+$pass

e2e/test_mode_deps.sh

@@ -0,0 +1,58 @@
+#!/bin/bash
+# -*- indent-tabs-mode: nil; tab-width: 2; sh-indentation: 2; -*-
+
+# Test --test-mode with a secondary dependency that fails to resolve
+# stevedore depends on pbr; we constrain pbr to a non-existent version
+# See: https://github.com/python-wheel-build/fromager/issues/895
+
+SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+source "$SCRIPTDIR/common.sh"
+pass=true
+
+DIST="stevedore"
+VER="5.2.0"
+
+# Constrain pbr to a version that doesn't exist
+echo "pbr==99999.0.0" > "$OUTDIR/constraints.txt"
+
+set +e
+fromager \
+  --log-file="$OUTDIR/bootstrap.log" \
+  --sdists-repo="$OUTDIR/sdists-repo" \
+  --wheels-repo="$OUTDIR/wheels-repo" \
+  --work-dir="$OUTDIR/work-dir" \
+  --constraints-file="$OUTDIR/constraints.txt" \
+  bootstrap --test-mode "${DIST}==${VER}"
+exit_code=$?
+set -e
+
+if [ "$exit_code" -ne 1 ]; then
+  echo "FAIL: expected exit code 1, got $exit_code" 1>&2
+  pass=false
+fi
+
+failures_file=$(find "$OUTDIR/work-dir" -name "test-mode-failures-*.json" 2>/dev/null | head -1)
+
+if [ -z "$failures_file" ]; then
+  echo "FAIL: no test-mode-failures JSON file found" 1>&2
+  pass=false
+else
+  if ! jq -e '.failures[] | select(.package == "pbr")' "$failures_file" >/dev/null 2>&1; then
+    echo "FAIL: pbr not found in failures" 1>&2
+    pass=false
+  fi
+
+  failure_type=$(jq -r '[.failures[] | select(.package == "pbr")][0].failure_type' "$failures_file")
+  if [ "$failure_type" != "resolution" ]; then
+    echo "FAIL: expected failure_type 'resolution' for pbr, got '$failure_type'" 1>&2
+    pass=false
+  fi
+fi
+
+# stevedore should have resolved successfully
+if ! grep -q "stevedore.*resolves to" "$OUTDIR/bootstrap.log"; then
+  echo "FAIL: stevedore did not resolve" 1>&2
+  pass=false
+fi
+
+$pass

e2e/test_mode_fallback.sh

@@ -0,0 +1,77 @@
+#!/bin/bash
+# -*- indent-tabs-mode: nil; tab-width: 2; sh-indentation: 2; -*-
+
+# Test --test-mode with a build failure that falls back to prebuilt wheel
+# Uses a broken patch to fail the source build, then falls back to PyPI wheel.
+# See: https://github.com/python-wheel-build/fromager/issues/895
+
+SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+source "$SCRIPTDIR/common.sh"
+pass=true
+
+DIST="setuptools"
+VER="75.8.0"
+
+# Force source build (not prebuilt)
+mkdir -p "$OUTDIR/settings"
+cat > "$OUTDIR/settings/${DIST}.yaml" << EOF
+variants:
+  cpu:
+    pre_built: false
+EOF
+
+# Create a patch that will fail (wrong content for setup.py)
+mkdir -p "$OUTDIR/patches/${DIST}"
+cat > "$OUTDIR/patches/${DIST}/break-build.patch" << 'EOF'
+--- a/setup.py
++++ b/setup.py
+@@ -1,3 +1,3 @@
+-wrong content
+-that does not match
+-the actual file
++will not
++be applied
++ever
+EOF
+
+set +e
+fromager \
+  --log-file="$OUTDIR/bootstrap.log" \
+  --sdists-repo="$OUTDIR/sdists-repo" \
+  --wheels-repo="$OUTDIR/wheels-repo" \
+  --work-dir="$OUTDIR/work-dir" \
+  --settings-dir="$OUTDIR/settings" \
+  --patches-dir="$OUTDIR/patches" \
+  bootstrap --test-mode "${DIST}==${VER}"
+exit_code=$?
+set -e
+
+# Exit code should be 0 (fallback succeeded)
+if [ "$exit_code" -ne 0 ]; then
+  echo "FAIL: expected exit code 0, got $exit_code" 1>&2
+  pass=false
+fi
+
+# Patch should have been attempted
+if ! grep -q "applying patch file.*break-build.patch" "$OUTDIR/bootstrap.log"; then
+  echo "FAIL: patch was not applied" 1>&2
+  pass=false
+fi
+
+# Fallback should have succeeded
+if ! grep -q "successfully used pre-built wheel" "$OUTDIR/bootstrap.log"; then
+  echo "FAIL: prebuilt fallback did not succeed" 1>&2
+  pass=false
+fi
+
+# No failures should be recorded
+failures_file=$(find "$OUTDIR/work-dir" -name "test-mode-failures-*.json" 2>/dev/null | head -1)
+if [ -n "$failures_file" ]; then
+  failure_count=$(jq '.failures | length' "$failures_file")
+  if [ "$failure_count" -gt 0 ]; then
+    echo "FAIL: expected 0 failures, got $failure_count" 1>&2
+    pass=false
+  fi
+fi
+
+$pass

e2e/test_mode_resolution.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+# -*- indent-tabs-mode: nil; tab-width: 2; sh-indentation: 2; -*-
+
+# Test --test-mode with a non-existent package (top-level resolution failure)
+# See: https://github.com/python-wheel-build/fromager/issues/895
+
+SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+source "$SCRIPTDIR/common.sh"
+pass=true
+
+DIST="nonexistent-package-xyz-12345-does-not-exist"
+
+set +e
+fromager \
+  --log-file="$OUTDIR/bootstrap.log" \
+  --sdists-repo="$OUTDIR/sdists-repo" \
+  --wheels-repo="$OUTDIR/wheels-repo" \
+  --work-dir="$OUTDIR/work-dir" \
+  bootstrap --test-mode "${DIST}==1.0.0"
+exit_code=$?
+set -e
+
+if [ "$exit_code" -ne 1 ]; then
+  echo "FAIL: expected exit code 1, got $exit_code" 1>&2
+  pass=false
+fi
+
+failures_file=$(find "$OUTDIR/work-dir" -name "test-mode-failures-*.json" 2>/dev/null | head -1)
+
+if [ -z "$failures_file" ]; then
+  echo "FAIL: no test-mode-failures JSON file found" 1>&2
+  pass=false
+else
+  if ! jq -e ".failures[] | select(.package == \"$DIST\")" "$failures_file" >/dev/null 2>&1; then
+    echo "FAIL: $DIST not found in failures" 1>&2
+    pass=false
+  fi
+
+  failure_type=$(jq -r '.failures[0].failure_type' "$failures_file")
+  if [ "$failure_type" != "resolution" ]; then
+    echo "FAIL: expected failure_type 'resolution', got '$failure_type'" 1>&2
+    pass=false
+  fi
+fi
+
+if ! grep -q "test mode enabled" "$OUTDIR/bootstrap.log"; then
+  echo "FAIL: 'test mode enabled' not in log" 1>&2
+  pass=false
+fi
+
+$pass