Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 28 additions & 19 deletions benchmarks/history.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,8 @@

Each hardware set has one CSV under ``benchmarks/history/``, named by its slug, with
one row per run, backend, case, stepper and width. Rows compare only within one
hardware set, backend, width, case revision and resolved integrator.
hardware set, backend, native build profile, width, case revision and resolved
integrator.
"""

from __future__ import annotations
Expand Down Expand Up @@ -111,25 +112,33 @@ def read(path: Path) -> list[Record]:
def comparable(row: Record) -> tuple[str, ...]:
"""Key the rows that measure the same thing on one hardware set.

Two rows compare only with the same backend, memory domains spanned, width,
threading, case definition, stepper, resolved integrator, tolerances and noise
model: a changed library default is a new configuration, not a speed-up.
Two rows compare only with the same backend, native build profile, memory
domains spanned, width, threading, case definition, stepper, resolved
integrator, tolerances and noise model: a changed library default is a new
configuration, not a speed-up, and so is a local build beside the distributed
wheel. The profile is the first field of ``native_build``; its compiler and
libraries are not keyed, since a toolchain update inside the wheel is part of
what a release ships.
"""
return tuple(
str(row.get(key) or "")
for key in (
"backend",
"numa_domains",
"threads",
"threading",
"case",
"revision",
"stepper",
"integrator",
"rtol",
"atol",
"noise",
)
profile = str(row.get("native_build") or "").split(";", 1)[0].strip()
return (
str(row.get("backend") or ""),
profile,
*(
str(row.get(key) or "")
for key in (
"numa_domains",
"threads",
"threading",
"case",
"revision",
"stepper",
"integrator",
"rtol",
"atol",
"noise",
)
),
)


Expand Down
46 changes: 40 additions & 6 deletions benchmarks/track.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,15 +42,20 @@
BACKENDS = ("numpy_ref", "native")
"""The backends a benchmark can name."""

DIRTY_STAMP = "+dirty"
"""The suffix the native build adds to its commit when built from a modified tree."""

REPEAT_TOLERANCE = 1e-9
"""The relative L2 difference two repeats' final fields may have: rounding only."""

__all__ = [
"BACKENDS",
"DIRTY_STAMP",
"REPEAT_TOLERANCE",
"admissible",
"history_rows",
"imported_elsewhere",
"native_build_commit",
"refuse_partial",
"selection_options",
"show_results",
Expand Down Expand Up @@ -269,16 +274,17 @@ def history_rows(
) -> list[Record]:
"""Shape a complete run's results as history rows."""
build = provenance.get("build_info") or {}
native_commit, native_dirty = native_build_commit(provenance)
date = datetime.now(UTC).strftime("%Y-%m-%dT%H:%MZ")
return [
{
"date": date,
"solprop": provenance["packages"]["solprop"],
"commit": (source.get("revision") or "")[:12],
"dirty": int(bool(source.get("dirty"))),
"dirty": int(bool(source.get("dirty")) or native_dirty),
"backend": backend,
"native": build.get("package_version"),
"native_commit": (build.get("git_commit") or "")[:12] or None,
"native_commit": (native_commit or "")[:12] or None,
"native_build": "; ".join(
str(build[key])
for key in ("build_profile", "compiler", "fft_backend", "blas_backend")
Expand Down Expand Up @@ -354,11 +360,37 @@ def admissible(results: list[Record]) -> str | None:
return None


def native_build_commit(provenance: Record) -> tuple[str | None, bool]:
"""Split the native build's commit from the ``+dirty`` its build stamps.

CMake appends ``+dirty`` when the tree it built from had uncommitted changes;
Git cannot resolve the stamped name, so callers compare the bare commit.
"""
commit = (provenance.get("build_info") or {}).get("git_commit")
if commit and commit.endswith(DIRTY_STAMP):
return commit.removesuffix(DIRTY_STAMP), True
return commit, False


def imported_elsewhere(
results: list[Record], provenance: Record, source_state: Record
results: list[Record],
provenance: Record,
source_state: Record,
*,
allow_dirty: bool = False,
) -> str | None:
"""Say if the workers timed code other than this checkout's commit."""
commit = (provenance.get("build_info") or {}).get("git_commit")
"""Say if the workers timed code other than this checkout's commit.

A native build from a modified tree is admitted only with ``allow_dirty``, and
its rows are then marked dirty by `history_rows`.
"""
commit, dirty = native_build_commit(provenance)
if dirty and not allow_dirty:
return (
f"the native backend was built from a modified tree ({(commit or '')[:12]}"
f"{DIRTY_STAMP}): commit the changes and rebuild it, or pass "
"--allow-dirty to record this run anyway (its rows are marked dirty)"
)
revision = source_state.get("revision")
# A build from an earlier commit is current if native/ has not changed since.
if (
Expand Down Expand Up @@ -472,7 +504,9 @@ def track(args: argparse.Namespace) -> int:
f"Results: {output.resolve()} ({(time.perf_counter() - started) / 60:.0f} min)"
)
if args.append_history:
refusal = refusal or imported_elsewhere(results, provenance, source)
refusal = refusal or imported_elsewhere(
results, provenance, source, allow_dirty=args.allow_dirty
)
if refusal is not None:
print(f"Not added to the history: {refusal}.", file=sys.stderr)
return 1
Expand Down
1 change: 1 addition & 0 deletions changelog.d/138-bench-history-build.fixed.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
The benchmark history no longer reports a change of native build profile (a local build after the release's wheel) as a speed change: rows compare only within one build profile. `python -m benchmarks track --append-history --allow-dirty` now admits a native build stamped `+dirty`, marking its rows dirty, and without the flag refuses it by naming the modified tree instead of asking for a rebuild; a modified `benchmarks/history` file no longer stamps the native build `+dirty` ([#138](https://github.com/LupoLab/solprop/issues/138)).
16 changes: 10 additions & 6 deletions docs/dev/benchmarking.md
Original file line number Diff line number Diff line change
Expand Up @@ -158,8 +158,9 @@ with at least three repeats, and only when:
native run, a native build from a commit after which nothing under `native/`
changed but its documentation and tests;
- nothing under `src/`, `native/` or `benchmarks/` is uncommitted (the history files
aside, so one checkout can record several runs), unless `--allow-dirty`, whose rows
are marked dirty;
aside, so one checkout can record several runs), and the native build was not made
from a modified tree (its commit stamped `+dirty`; the history files do not count
here either), unless `--allow-dirty`, which admits both and marks the rows dirty;
- `SOLPROP_FFTW_PLAN` is unset: a row does not record how it planned, and measured
planning changes its construction time and its median.

Expand All @@ -174,10 +175,13 @@ run: its date, solprop version, the configurations it ran and how many started o
busy machine, and its change against the first and the previous recorded runs. A
change is the geometric mean of median-time ratios (below one is faster) over the
configurations both runs share, counted in brackets. Two rows are the same
configuration only with the same number of memory (NUMA) domains spanned, threading
policy, case revision, stepper, resolved integrator, tolerances and noise model. A
changed library default therefore starts a new series rather than reading as a
speed-up, and so does an allocation spread over more memory domains than usual: on a
configuration only with the same native build profile (the first field of
`native_build`: `distribution` for a wheel, `local` for a build from source), number
of memory (NUMA) domains spanned, threading policy, case revision, stepper, resolved
integrator, tolerances and noise model. A changed library default therefore starts a
new series rather than reading as a speed-up, and so does a local build after wheel
rows (a compiler or library update inside the wheel still compares: it is part of
what a release ships), or an allocation spread over more memory domains than usual: on a
cluster, keep the cores together (`--distribution=block:block` under Slurm).

## Recorded hardware
Expand Down
4 changes: 3 additions & 1 deletion native/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -177,8 +177,10 @@ execute_process(
OUTPUT_STRIP_TRAILING_WHITESPACE
COMMAND_ERROR_IS_FATAL ANY
)
# The benchmark history is what a tracking run writes, not what it times: one checkout
# may record several runs, and a modified history must not stamp the build dirty.
execute_process(
COMMAND git diff --quiet --ignore-submodules HEAD
COMMAND git diff --quiet --ignore-submodules HEAD -- ":/" ":(top,exclude)benchmarks/history"
WORKING_DIRECTORY "${PROJECT_SOURCE_DIR}"
RESULT_VARIABLE SOLPROP_NATIVE_TREE_DIRTY
)
Expand Down
64 changes: 64 additions & 0 deletions tests/test_benchmark_track.py
Original file line number Diff line number Diff line change
Expand Up @@ -178,6 +178,70 @@ def test_history_rows_append_read_and_summarise(tmp_path):
history.append(tmp_path / "new.csv", [{"date": "x"}])


def test_a_build_profile_change_starts_a_new_series_and_a_toolchain_change_does_not(
tmp_path,
):
# Issue #138: a local build after wheel rows read as the release's speed-up.
base: dict[str, object] = dict.fromkeys(history.COLUMNS, "")
base.update(
date="2026-11-01T08:00Z", solprop="0.1.0", commit="a" * 12, dirty=0,
backend="native", slug="x", threads=1, threading="passive", numa_domains=1,
case="rdw", revision=1, stepper="default", median_s="10.0",
native_build="distribution; GCC 14; FFTW_OPENMP; OpenBLAS",
) # fmt: skip
toolchain = {
**base, "date": "2026-11-02T08:00Z", "commit": "b" * 12, "median_s": "9.0",
"native_build": "distribution; GCC 15; FFTW_OPENMP; OpenBLAS",
} # fmt: skip
local = {
**base, "date": "2026-11-03T08:00Z", "commit": "c" * 12, "median_s": "7.0",
"native_build": "local; GCC 15; FFTW_OPENMP; MKL",
} # fmt: skip
path = tmp_path / "build.csv"
history.append(path, [base, toolchain, local])
shared = [row["shared_previous"] for row in history.summary(history.read(path))]
assert shared == [1, 1, 0]
assert history.comparable(base) == history.comparable(toolchain)
assert history.comparable(base) != history.comparable(local)


def test_a_dirty_native_build_is_admitted_only_with_allow_dirty():
# Issue #138: `<commit>+dirty` went to `git diff`, which cannot resolve it, so the
# run was refused as stale whatever --allow-dirty said.
good = [result_row()]
revision = subprocess.check_output(
["git", "-C", str(runner.ROOT), "rev-parse", "HEAD"], text=True
).strip()
source = {"revision": revision}
dirty = {"build_info": {"git_commit": revision + track.DIRTY_STAMP}}
assert track.native_build_commit(dirty) == (revision, True)
assert track.native_build_commit({"build_info": {"git_commit": revision}}) == (
revision,
False,
)
assert track.native_build_commit({}) == (None, False)
refusal = str(track.imported_elsewhere(good, dirty, source))
assert "modified tree" in refusal
assert "rebuild it (uv sync" not in refusal
assert track.imported_elsewhere(good, dirty, source, allow_dirty=True) is None
# A dirty build of a stale commit is still stale.
stale = {"build_info": {"git_commit": "0" * 40 + track.DIRTY_STAMP}}
assert "rebuild" in str(
track.imported_elsewhere(good, stale, source, allow_dirty=True)
)
machine = {
"slug": "test-cpu-2c", "cpu_model": "Test CPU", "instance_type": None,
"system": "Linux", "machine": "x86_64", "numa_domains": 1,
} # fmt: skip
provenance = {"packages": {"solprop": "1.0"}, **dirty}
rows = track.history_rows(
good, machine, {**source, "dirty": False}, provenance, "native"
)
assert [(row["dirty"], row["native_commit"]) for row in rows] == [
(1, revision[:12])
]


def test_older_history_files_gain_new_columns_and_newer_ones_are_refused(tmp_path):
path = tmp_path / "old.csv"
with path.open("w", encoding="utf-8", newline="") as stream:
Expand Down
Loading