diff --git a/benchmarks/history.py b/benchmarks/history.py index 474a9d7b7..fac24577f 100644 --- a/benchmarks/history.py +++ b/benchmarks/history.py @@ -6,7 +6,8 @@ Each hardware set has one CSV under ``benchmarks/history/``, named by its slug, with one row per run, backend, case, stepper and width. Rows compare only within one -hardware set, backend, width, case revision and resolved integrator. +hardware set, backend, native build profile, width, case revision and resolved +integrator. """ from __future__ import annotations @@ -111,25 +112,33 @@ def read(path: Path) -> list[Record]: def comparable(row: Record) -> tuple[str, ...]: """Key the rows that measure the same thing on one hardware set. - Two rows compare only with the same backend, memory domains spanned, width, - threading, case definition, stepper, resolved integrator, tolerances and noise - model: a changed library default is a new configuration, not a speed-up. + Two rows compare only with the same backend, native build profile, memory + domains spanned, width, threading, case definition, stepper, resolved + integrator, tolerances and noise model: a changed library default is a new + configuration, not a speed-up, and so is a local build beside the distributed + wheel. The profile is the first field of ``native_build``; its compiler and + libraries are not keyed, since a toolchain update inside the wheel is part of + what a release ships. """ - return tuple( - str(row.get(key) or "") - for key in ( - "backend", - "numa_domains", - "threads", - "threading", - "case", - "revision", - "stepper", - "integrator", - "rtol", - "atol", - "noise", - ) + profile = str(row.get("native_build") or "").split(";", 1)[0].strip() + return ( + str(row.get("backend") or ""), + profile, + *( + str(row.get(key) or "") + for key in ( + "numa_domains", + "threads", + "threading", + "case", + "revision", + "stepper", + "integrator", + "rtol", + "atol", + "noise", + ) + ), ) diff --git a/benchmarks/track.py b/benchmarks/track.py index 0431de223..e3178aa1d 100644 --- a/benchmarks/track.py +++ b/benchmarks/track.py @@ -42,15 +42,20 @@ BACKENDS = ("numpy_ref", "native") """The backends a benchmark can name.""" +DIRTY_STAMP = "+dirty" +"""The suffix the native build adds to its commit when built from a modified tree.""" + REPEAT_TOLERANCE = 1e-9 """The relative L2 difference two repeats' final fields may have: rounding only.""" __all__ = [ "BACKENDS", + "DIRTY_STAMP", "REPEAT_TOLERANCE", "admissible", "history_rows", "imported_elsewhere", + "native_build_commit", "refuse_partial", "selection_options", "show_results", @@ -269,16 +274,17 @@ def history_rows( ) -> list[Record]: """Shape a complete run's results as history rows.""" build = provenance.get("build_info") or {} + native_commit, native_dirty = native_build_commit(provenance) date = datetime.now(UTC).strftime("%Y-%m-%dT%H:%MZ") return [ { "date": date, "solprop": provenance["packages"]["solprop"], "commit": (source.get("revision") or "")[:12], - "dirty": int(bool(source.get("dirty"))), + "dirty": int(bool(source.get("dirty")) or native_dirty), "backend": backend, "native": build.get("package_version"), - "native_commit": (build.get("git_commit") or "")[:12] or None, + "native_commit": (native_commit or "")[:12] or None, "native_build": "; ".join( str(build[key]) for key in ("build_profile", "compiler", "fft_backend", "blas_backend") @@ -354,11 +360,37 @@ def admissible(results: list[Record]) -> str | None: return None +def native_build_commit(provenance: Record) -> tuple[str | None, bool]: + """Split the native build's commit from the ``+dirty`` its build stamps. + + CMake appends ``+dirty`` when the tree it built from had uncommitted changes; + Git cannot resolve the stamped name, so callers compare the bare commit. + """ + commit = (provenance.get("build_info") or {}).get("git_commit") + if commit and commit.endswith(DIRTY_STAMP): + return commit.removesuffix(DIRTY_STAMP), True + return commit, False + + def imported_elsewhere( - results: list[Record], provenance: Record, source_state: Record + results: list[Record], + provenance: Record, + source_state: Record, + *, + allow_dirty: bool = False, ) -> str | None: - """Say if the workers timed code other than this checkout's commit.""" - commit = (provenance.get("build_info") or {}).get("git_commit") + """Say if the workers timed code other than this checkout's commit. + + A native build from a modified tree is admitted only with ``allow_dirty``, and + its rows are then marked dirty by `history_rows`. + """ + commit, dirty = native_build_commit(provenance) + if dirty and not allow_dirty: + return ( + f"the native backend was built from a modified tree ({(commit or '')[:12]}" + f"{DIRTY_STAMP}): commit the changes and rebuild it, or pass " + "--allow-dirty to record this run anyway (its rows are marked dirty)" + ) revision = source_state.get("revision") # A build from an earlier commit is current if native/ has not changed since. if ( @@ -472,7 +504,9 @@ def track(args: argparse.Namespace) -> int: f"Results: {output.resolve()} ({(time.perf_counter() - started) / 60:.0f} min)" ) if args.append_history: - refusal = refusal or imported_elsewhere(results, provenance, source) + refusal = refusal or imported_elsewhere( + results, provenance, source, allow_dirty=args.allow_dirty + ) if refusal is not None: print(f"Not added to the history: {refusal}.", file=sys.stderr) return 1 diff --git a/changelog.d/138-bench-history-build.fixed.md b/changelog.d/138-bench-history-build.fixed.md new file mode 100644 index 000000000..7cbd01766 --- /dev/null +++ b/changelog.d/138-bench-history-build.fixed.md @@ -0,0 +1 @@ +The benchmark history no longer reports a change of native build profile (a local build after the release's wheel) as a speed change: rows compare only within one build profile. `python -m benchmarks track --append-history --allow-dirty` now admits a native build stamped `+dirty`, marking its rows dirty, and without the flag refuses it by naming the modified tree instead of asking for a rebuild; a modified `benchmarks/history` file no longer stamps the native build `+dirty` ([#138](https://github.com/LupoLab/solprop/issues/138)). diff --git a/docs/dev/benchmarking.md b/docs/dev/benchmarking.md index 921ce5e8b..5ef0776a1 100644 --- a/docs/dev/benchmarking.md +++ b/docs/dev/benchmarking.md @@ -158,8 +158,9 @@ with at least three repeats, and only when: native run, a native build from a commit after which nothing under `native/` changed but its documentation and tests; - nothing under `src/`, `native/` or `benchmarks/` is uncommitted (the history files - aside, so one checkout can record several runs), unless `--allow-dirty`, whose rows - are marked dirty; + aside, so one checkout can record several runs), and the native build was not made + from a modified tree (its commit stamped `+dirty`; the history files do not count + here either), unless `--allow-dirty`, which admits both and marks the rows dirty; - `SOLPROP_FFTW_PLAN` is unset: a row does not record how it planned, and measured planning changes its construction time and its median. @@ -174,10 +175,13 @@ run: its date, solprop version, the configurations it ran and how many started o busy machine, and its change against the first and the previous recorded runs. A change is the geometric mean of median-time ratios (below one is faster) over the configurations both runs share, counted in brackets. Two rows are the same -configuration only with the same number of memory (NUMA) domains spanned, threading -policy, case revision, stepper, resolved integrator, tolerances and noise model. A -changed library default therefore starts a new series rather than reading as a -speed-up, and so does an allocation spread over more memory domains than usual: on a +configuration only with the same native build profile (the first field of +`native_build`: `distribution` for a wheel, `local` for a build from source), number +of memory (NUMA) domains spanned, threading policy, case revision, stepper, resolved +integrator, tolerances and noise model. A changed library default therefore starts a +new series rather than reading as a speed-up, and so does a local build after wheel +rows (a compiler or library update inside the wheel still compares: it is part of +what a release ships), or an allocation spread over more memory domains than usual: on a cluster, keep the cores together (`--distribution=block:block` under Slurm). ## Recorded hardware diff --git a/native/CMakeLists.txt b/native/CMakeLists.txt index 0d23fc6da..e78ee5504 100644 --- a/native/CMakeLists.txt +++ b/native/CMakeLists.txt @@ -177,8 +177,10 @@ execute_process( OUTPUT_STRIP_TRAILING_WHITESPACE COMMAND_ERROR_IS_FATAL ANY ) +# The benchmark history is what a tracking run writes, not what it times: one checkout +# may record several runs, and a modified history must not stamp the build dirty. execute_process( - COMMAND git diff --quiet --ignore-submodules HEAD + COMMAND git diff --quiet --ignore-submodules HEAD -- ":/" ":(top,exclude)benchmarks/history" WORKING_DIRECTORY "${PROJECT_SOURCE_DIR}" RESULT_VARIABLE SOLPROP_NATIVE_TREE_DIRTY ) diff --git a/tests/test_benchmark_track.py b/tests/test_benchmark_track.py index e5bab87be..6ca3d8acc 100644 --- a/tests/test_benchmark_track.py +++ b/tests/test_benchmark_track.py @@ -178,6 +178,70 @@ def test_history_rows_append_read_and_summarise(tmp_path): history.append(tmp_path / "new.csv", [{"date": "x"}]) +def test_a_build_profile_change_starts_a_new_series_and_a_toolchain_change_does_not( + tmp_path, +): + # Issue #138: a local build after wheel rows read as the release's speed-up. + base: dict[str, object] = dict.fromkeys(history.COLUMNS, "") + base.update( + date="2026-11-01T08:00Z", solprop="0.1.0", commit="a" * 12, dirty=0, + backend="native", slug="x", threads=1, threading="passive", numa_domains=1, + case="rdw", revision=1, stepper="default", median_s="10.0", + native_build="distribution; GCC 14; FFTW_OPENMP; OpenBLAS", + ) # fmt: skip + toolchain = { + **base, "date": "2026-11-02T08:00Z", "commit": "b" * 12, "median_s": "9.0", + "native_build": "distribution; GCC 15; FFTW_OPENMP; OpenBLAS", + } # fmt: skip + local = { + **base, "date": "2026-11-03T08:00Z", "commit": "c" * 12, "median_s": "7.0", + "native_build": "local; GCC 15; FFTW_OPENMP; MKL", + } # fmt: skip + path = tmp_path / "build.csv" + history.append(path, [base, toolchain, local]) + shared = [row["shared_previous"] for row in history.summary(history.read(path))] + assert shared == [1, 1, 0] + assert history.comparable(base) == history.comparable(toolchain) + assert history.comparable(base) != history.comparable(local) + + +def test_a_dirty_native_build_is_admitted_only_with_allow_dirty(): + # Issue #138: `+dirty` went to `git diff`, which cannot resolve it, so the + # run was refused as stale whatever --allow-dirty said. + good = [result_row()] + revision = subprocess.check_output( + ["git", "-C", str(runner.ROOT), "rev-parse", "HEAD"], text=True + ).strip() + source = {"revision": revision} + dirty = {"build_info": {"git_commit": revision + track.DIRTY_STAMP}} + assert track.native_build_commit(dirty) == (revision, True) + assert track.native_build_commit({"build_info": {"git_commit": revision}}) == ( + revision, + False, + ) + assert track.native_build_commit({}) == (None, False) + refusal = str(track.imported_elsewhere(good, dirty, source)) + assert "modified tree" in refusal + assert "rebuild it (uv sync" not in refusal + assert track.imported_elsewhere(good, dirty, source, allow_dirty=True) is None + # A dirty build of a stale commit is still stale. + stale = {"build_info": {"git_commit": "0" * 40 + track.DIRTY_STAMP}} + assert "rebuild" in str( + track.imported_elsewhere(good, stale, source, allow_dirty=True) + ) + machine = { + "slug": "test-cpu-2c", "cpu_model": "Test CPU", "instance_type": None, + "system": "Linux", "machine": "x86_64", "numa_domains": 1, + } # fmt: skip + provenance = {"packages": {"solprop": "1.0"}, **dirty} + rows = track.history_rows( + good, machine, {**source, "dirty": False}, provenance, "native" + ) + assert [(row["dirty"], row["native_commit"]) for row in rows] == [ + (1, revision[:12]) + ] + + def test_older_history_files_gain_new_columns_and_newer_ones_are_refused(tmp_path): path = tmp_path / "old.csv" with path.open("w", encoding="utf-8", newline="") as stream: